diff --git a/.gitignore b/.gitignore index ee44a9639..67dc51000 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ .idea target +spoon.classpath-app.tmp diff --git a/pom.xml b/pom.xml index d391e3f36..1c0295e88 100644 --- a/pom.xml +++ b/pom.xml @@ -15,6 +15,12 @@ 3.14.16 + + fr.inria.gforge.spoon + spoon-core + 7.5.0 + + com.google.code.gson gson diff --git a/src/main/java/DocumentIndexer.java b/src/main/java/DocumentIndexer.java index 0a1118cdb..550250a0d 100644 --- a/src/main/java/DocumentIndexer.java +++ b/src/main/java/DocumentIndexer.java @@ -1,29 +1,12 @@ import com.github.javaparser.Position; import com.github.javaparser.Range; -import com.github.javaparser.StaticJavaParser; -import com.github.javaparser.ast.CompilationUnit; -import com.github.javaparser.ast.Node; -import com.github.javaparser.ast.body.FieldDeclaration; -import com.github.javaparser.ast.body.MethodDeclaration; -import com.github.javaparser.ast.body.Parameter; -import com.github.javaparser.ast.expr.FieldAccessExpr; -import com.github.javaparser.ast.expr.MethodCallExpr; -import com.github.javaparser.ast.expr.NameExpr; -import com.github.javaparser.ast.expr.VariableDeclarationExpr; -import com.github.javaparser.ast.nodeTypes.NodeWithSimpleName; -import com.github.javaparser.ast.visitor.VoidVisitorAdapter; -import com.github.javaparser.resolution.declarations.ResolvedClassDeclaration; -import com.github.javaparser.resolution.declarations.ResolvedDeclaration; -import com.github.javaparser.resolution.types.ResolvedType; -import com.github.javaparser.symbolsolver.JavaSymbolSolver; -import com.github.javaparser.symbolsolver.javaparsermodel.declarations.*; -import com.github.javaparser.symbolsolver.model.resolution.TypeSolver; -import com.github.javaparser.symbolsolver.resolution.typesolvers.CombinedTypeSolver; -import com.github.javaparser.symbolsolver.resolution.typesolvers.JavaParserTypeSolver; -import com.github.javaparser.symbolsolver.resolution.typesolvers.ReflectionTypeSolver; - -import java.io.File; -import java.io.FileNotFoundException; +import spoon.reflect.code.*; +import spoon.reflect.cu.SourcePosition; +import spoon.reflect.cu.position.NoSourcePosition; +import spoon.reflect.declaration.*; +import spoon.reflect.reference.CtTypeReference; +import spoon.reflect.visitor.CtScanner; + import java.io.IOException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; @@ -34,6 +17,7 @@ public class DocumentIndexer { private String projectRoot; private boolean noContents; private String pathname; + private CtElement spoonElement; private String projectId; private Emitter emitter; private Map indexers; @@ -46,6 +30,7 @@ public DocumentIndexer( String projectRoot, boolean noContents, String pathname, + CtElement spoonElement, String projectId, Emitter emitter, Map indexers @@ -53,6 +38,7 @@ public DocumentIndexer( this.projectRoot = projectRoot; this.noContents = noContents; this.pathname = pathname; + this.spoonElement = spoonElement; this.projectId = projectId; this.emitter = emitter; this.indexers = indexers; @@ -72,10 +58,6 @@ public void index() { } private void doIndex() { - JavaSymbolSolver symbolSolver = getSymbolSolver(); - StaticJavaParser.getConfiguration().setSymbolResolver(symbolSolver); - CompilationUnit cu = parse(); - Map args = Map.of( "languageId", "java", "uri", String.format("file://%s", Paths.get(pathname).toAbsolutePath().toString()) @@ -91,7 +73,7 @@ private void doIndex() { } this.documentId = emitter.emitVertex("document", args); - cu.accept(new LSIFVisitor(symbolSolver), null); + this.spoonElement.accept(new LSIFVisitor()); } public void postIndex() { @@ -103,25 +85,6 @@ public void postIndex() { emitter.emitEdge("contains", Map.of("outV", documentId, "inVs", rangeIds.stream().sorted().toArray())); } - private JavaSymbolSolver getSymbolSolver() { - return new JavaSymbolSolver(getTypeSolver()); - } - - private TypeSolver getTypeSolver() { - CombinedTypeSolver typeSolver = new CombinedTypeSolver(); - typeSolver.add(new JavaParserTypeSolver(projectRoot)); - typeSolver.add(new ReflectionTypeSolver()); - return typeSolver; - } - - private CompilationUnit parse() { - try { - return StaticJavaParser.parse(new File(pathname)); - } catch (FileNotFoundException ex) { - throw new RuntimeException(String.format("Failed to parse %s", pathname)); - } - } - private void linkUses(DefinitionMeta meta, String documentId) { String resultId = emitter.emitVertex("referenceResult", Map.of()); @@ -143,55 +106,99 @@ private void linkUses(DefinitionMeta meta, String documentId) { } } - private class LSIFVisitor extends VoidVisitorAdapter { - private JavaSymbolSolver symbolSolver; + private class LSIFVisitor extends CtScanner { + // TODO add support for more language constructs: + // https://github.com/INRIA/spoon/blob/master/src/main/java/spoon/reflect/visitor/CtScanner.java - public LSIFVisitor(JavaSymbolSolver symbolSolver) { - this.symbolSolver = symbolSolver; + @Override + public void visitCtParameter(CtParameter el) { + emitDefinition(mkRange(el.getPosition()), mkDoc(el.getType(), el.getDocComment())); } - // TODO - field access - // TODO - types to classes - // TODO - enums - // TODO - inner classes? + @Override + public void visitCtVariableRead(CtVariableRead el) { + emitUse( + mkRange(el.getPosition()), + mkRange(el.getVariable().getDeclaration().getPosition()), + el.getVariable().getDeclaration().getDocComment(), + el.getVariable().getDeclaration().getPosition().getFile().getPath() + ); + } @Override - public void visit(final MethodCallExpr n, final Void arg) { - super.visit(n, arg); - emitUse(n); + public void visitCtMethod(CtMethod el) { + super.visitCtMethod(el); + emitDefinition(nameRange(el), mkDoc(el.getType(), el.getDocComment())); } @Override - public void visit(final VariableDeclarationExpr n, final Void arg) { - super.visit(n, arg); - emitDefinition(n); + public void visitCtInvocation(CtInvocation el) { + super.visitCtInvocation(el); + if (el.getPosition() instanceof NoSourcePosition) { + return; + } + if (el.getExecutable() == null || el.getExecutable().getDeclaration() == null) { + return; + } + Range useRange = identifierRange(el, el.getExecutable().getSimpleName()); + emitUse( + useRange, + nameRange(el.getExecutable().getDeclaration()), + el.getExecutable().getDeclaration().getDocComment(), + el.getExecutable().getDeclaration().getPosition().getFile().getPath() + ); } @Override - public void visit(final MethodDeclaration n, final Void arg) { - super.visit(n, arg); - emitDefinition(n); + public void visitCtField(CtField el) { + super.visitCtField(el); + emitDefinition(nameRange(el), mkDoc(el.getType(), el.getDocComment())); } @Override - public void visit(final NameExpr n, final Void arg) { - super.visit(n, arg); - emitUse(n); + public void visitCtFieldRead(CtFieldRead el) { + super.visitCtFieldRead(el); + if (el.getPosition() instanceof NoSourcePosition) { + return; + } + Range useRange = identifierRange(el, el.getVariable().getSimpleName()); + CtField decl = el.getVariable().getDeclaration(); + if (decl == null) { + return; + } + emitUse( + useRange, + nameRange(decl), + decl.getDocComment(), + decl.getPosition().getFile().getPath() + ); } @Override - public void visit(final Parameter n, final Void arg) { - super.visit(n, arg); - emitDefinition(n); + public void visitCtClass(CtClass el) { + super.visitCtClass(el); + emitDefinition(nameRange(el), el.getDocComment()); } - private void emitDefinition(Node n) { - Optional rangeContainer = getRange(n); - if (!rangeContainer.isPresent()) { - throw new RuntimeException("Unexpected range-less AST node: " + n.toString()); + @Override + public void visitCtTypeReference(CtTypeReference el) { + super.visitCtTypeReference(el); + if (el.getPosition() instanceof NoSourcePosition) { + return; + } + CtType decl = el.getDeclaration(); + if (decl == null) { + return; } + emitUse( + mkRange(el.getPosition()), + nameRange(decl), + decl.getDocComment(), + decl.getPosition().getFile().getPath() + ); + } - Range range = rangeContainer.get(); + private void emitDefinition(Range range, String doc) { if (definitions.containsKey(range)) { return; } @@ -199,8 +206,8 @@ private void emitDefinition(Node n) { String hoverId = emitter.emitVertex("hoverResult", Map.of( "result", Map.of( "contents", Map.of( - "language", "java", - "value", n.toString() + "kind", "markdown", + "value", doc ) ) )); @@ -214,68 +221,23 @@ private void emitDefinition(Node n) { definitions.put(range, new DefinitionMeta(rangeId, resultSetId)); // + contents? } - private void emitUse(Node n) { - ResolvedDeclaration definition; - try { - definition = symbolSolver.resolveDeclaration(n, ResolvedDeclaration.class); - } catch (RuntimeException ex) { - // ignore - return; - } - - Node definitionNode = getNode(definition); - if (definitionNode == null) { - return; - } + private void emitUse(Range use, Range def, String doc, String defPath) { + DocumentIndexer indexer = indexers.get(defPath); - Optional definitionPathContainer = definitionNode.findCompilationUnit() - .flatMap(cu -> cu.getStorage()) - .map(s -> s.getPath().toString()); - - if (definitionPathContainer.isPresent() && definitionPathContainer.get().equals(pathname)) { - emitDefinition(definitionNode); + if (defPath.equals(pathname)) { + emitDefinition(def, doc); } else { - indexers.get(definitionPathContainer.get()).index(); + indexer.index(); } - emitUse(n, definitionNode, definitionPathContainer.orElse(pathname)); - } + System.out.println(pathname + ":" + humanRange(use) + " -> " + defPath + ":" + humanRange(def)); - private Node getNode(ResolvedDeclaration def) { - return - JavaParserClassDeclaration.class.isInstance(def) ? JavaParserClassDeclaration.class.cast(def).getWrappedNode() - : JavaParserConstructorDeclaration.class.isInstance(def) ? JavaParserConstructorDeclaration.class.cast(def).getWrappedNode() -// : JavaParserEnumConstantDeclaration.class.isInstance(def) ? JavaParserEnumConstantDeclaration.class.cast(def).getWrappedNode() -// : JavaParserEnumDeclaration.class.isInstance(def) ? JavaParserEnumDeclaration.class.cast(def).getWrappedNode() - : JavaParserFieldDeclaration.class.isInstance(def) ? JavaParserFieldDeclaration.class.cast(def).getWrappedNode() -// : JavaParserInterfaceDeclaration.class.isInstance(def) ? JavaParserInterfaceDeclaration.class.cast(def).getWrappedNode() - : JavaParserMethodDeclaration.class.isInstance(def) ? JavaParserMethodDeclaration.class.cast(def).getWrappedNode() - : JavaParserParameterDeclaration.class.isInstance(def) ? JavaParserParameterDeclaration.class.cast(def).getWrappedNode() - : JavaParserSymbolDeclaration.class.isInstance(def) ? JavaParserSymbolDeclaration.class.cast(def).getWrappedNode() -// : JavaParserTypeVariableDeclaration.class.isInstance(def) ? JavaParserTypeVariableDeclaration.class.cast(def).getWrappedNode() - : JavaParserVariableDeclaration.class.isInstance(def) ? JavaParserVariableDeclaration.class.cast(def).getWrappedNode() - : null; - } - - private void emitUse(Node n, Node definition, String definitionPath) { - DocumentIndexer indexer = indexers.get(definitionPath); - - Optional rangeContainer = getRange(n); - Optional definitionRangeContainer = getRange(definition); - - if (!rangeContainer.isPresent()) { - throw new RuntimeException("Unexpected range-less AST node: " + n.toString()); + DefinitionMeta meta = indexer.definitions.get(def); + if (meta == null) { + throw new RuntimeException("BUG definition doesn't exist, usePath " + pathname + ", use " + humanRange(use) + ", defPath " + defPath + ", def " + humanRange(def)); } - if (!definitionRangeContainer.isPresent()) { - throw new RuntimeException("Unexpected range-less AST node: " + definition.toString()); - } - - Range range = rangeContainer.get(); - Range definitionRange = definitionRangeContainer.get(); - DefinitionMeta meta = indexer.definitions.get(definitionRange); - - String rangeId = emitter.emitVertex("range", createRange(range)); + String rangeId = emitter.emitVertex("range", createRange(use)); emitter.emitEdge("next", Map.of("outV", rangeId, "inV", meta.resultSetId)); if (meta.definitionResultId == null) { @@ -297,27 +259,39 @@ private void emitUse(Node n, Node definition, String definitionPath) { meta.referenceRangeIds.put(documentId, set); } - private Optional getRange(Node decl) { - if (decl.getClass().isAssignableFrom(VariableDeclarationExpr.class)) { - // Unwrap variable declarations - decl = ((VariableDeclarationExpr) decl).getVariables().get(0); // TODO - how to support multiple? - } + private Range nameRange(CtNamedElement a) { + return nameRange(a.getPosition(), a.getSimpleName()); + } - if (decl.getClass().isAssignableFrom(FieldDeclaration.class)) { - // Unwrap variable declarations - decl = ((FieldDeclaration) decl).getVariables().get(0); // TODO - how to support multiple? - } + private Range nameRange(SourcePosition a, String name) { + return Range.range( + a.getLine(), + a.getColumn(), + a.getLine(), + a.getColumn() + name.length() + ); + } - try { - // Extract only the name portion of the AST node - return ((NodeWithSimpleName) decl).getName().getRange(); - } catch (ClassCastException ex) { - // ignore - } + private Range nameRange(Range a, String name) { + return Range.range( + a.begin.line, + a.begin.column, + a.end.line, + a.end.column + name.length() + ); + } + + private String mkDoc(CtTypeReference t, String docComment) { + return "```java\n" + t + "\n```" + (docComment.equals("") ? "" : "\n---\n" + docComment); + } - return decl.getRange(); + private String humanRange(Range r) { + return r.begin.line + ":" + r.begin.column + "-" + r.end.line + ":" + r.end.column; } + private Range mkRange(SourcePosition pos) { + return Range.range(pos.getLine(), pos.getColumn(), pos.getEndLine(), pos.getEndColumn()); + } private Map createRange(Range range) { return Map.of("start", createPosition(range.begin), "end", createPosition(range.end)); @@ -326,5 +300,15 @@ private Map createRange(Range range) { private Map createPosition(Position position) { return Map.of("line", position.line - 1, "character", position.column - 1); } + + private Range identifierRange(CtTargetedExpression a, String b) { + // Ugh, we only want the range for the identifier, not the whole expression. + // This will probably break on fields/methods that are spread across lines. + // +1 for `.`, +1 because (I'm guessing) end is exclusive + SourcePosition p = a.getTarget().getPosition(); + return p instanceof NoSourcePosition ? + mkRange(a.getPosition()) : + Range.range(p.getEndLine(), p.getEndColumn() + 1 + 1, p.getEndLine(), p.getEndColumn() + 1 + 1 + b.length()); + } } } diff --git a/src/main/java/ProjectIndexer.java b/src/main/java/ProjectIndexer.java index c6f9f6567..3fa318513 100644 --- a/src/main/java/ProjectIndexer.java +++ b/src/main/java/ProjectIndexer.java @@ -1,10 +1,15 @@ +import spoon.MavenLauncher; +import spoon.reflect.declaration.CtElement; +import spoon.reflect.declaration.CtModule; +import spoon.reflect.declaration.CtPackage; +import spoon.reflect.declaration.CtType; +import spoon.reflect.visitor.CtIterator; + import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -31,30 +36,33 @@ public void index() { "kind", "java" )); - List files = createFileStream() - .map(x -> x.toString()) - .filter(f -> f.endsWith(".java")) - .sorted() - .collect(Collectors.toList()); + MavenLauncher spoon = new MavenLauncher(arguments.projectRoot, MavenLauncher.SOURCE_TYPE.ALL_SOURCE); + spoon.buildModel(); + Map typeByFile = new HashMap(); + for (CtElement el : spoon.getModel().getAllTypes()) { + System.out.println("Found " + el.getPosition().getFile().getPath()); + typeByFile.put(el.getPosition().getFile().getPath(), el); + } Map indexers = new HashMap<>(); - for (String pathname : files) { - indexers.put(pathname, new DocumentIndexer( + for (Map.Entry pathname : typeByFile.entrySet()) { + indexers.put(pathname.getKey(), new DocumentIndexer( arguments.projectRoot, arguments.noContents, - pathname, + pathname.getKey(), + pathname.getValue(), projectId, emitter, indexers )); } - for (String pathname : files) { - indexers.get(pathname).index(); + for (DocumentIndexer indexer : indexers.values()) { + indexer.index(); } - for (String pathname : files) { - indexers.get(pathname).postIndex(); + for (DocumentIndexer indexer : indexers.values()) { + indexer.postIndex(); } for (DocumentIndexer indexer : indexers.values()) {