Parser parity test (#984)

- Introduce `ParserParityTestHelper` which takes a dir and parses the
given project with `RewriteMavenProjectParser` (comparing) and
`RewritePropjectParser` (system under test). The parsing result is then
compared using AssertJ's `usingRecursiveComparison` which verifies that
all markers and their fields of both parsing results are equal proving
that both parsers yield the same result.
- Introduce ´TestProjectHelper` which helps set up test projects.
- Fixes multiple bugs, e.g. calculation of the classpath. 
- Cleanups and refactorings

Author: fabapp2

.github/workflows/check-license-headers.yml

@@ -129,8 +129,11 @@ public String getArtifactId() {
         return pomModel.getArtifactId();
     }
 
+    /**
+     * FIXME: when the version of parent pom is null (inherited by it's parent) the version will be null.
+     */
     public String getVersion() {
-        return pomModel.getVersion();
+        return pomModel.getVersion() == null ? pomModel.getParent().getVersion() : pomModel.getVersion();
     }
 
     @Override
@@ -223,4 +226,8 @@ public Path getModulePath() {
     public ProjectId getProjectId() {
         return projectId;
     }
+
+    public Object getProjectEncoding() {
+        return getPomModel().getProperties().get("project.build.sourceEncoding");
+    }
 }

sbm-support-rewrite/src/main/java/org/springframework/sbm/parsers/MavenProject.java

@@ -18,26 +18,29 @@
 import lombok.extern.slf4j.Slf4j;
 import org.jetbrains.annotations.NotNull;
 import org.openrewrite.ExecutionContext;
+import org.openrewrite.FileAttributes;
 import org.openrewrite.Parser;
 import org.openrewrite.SourceFile;
 import org.openrewrite.internal.lang.Nullable;
 import org.openrewrite.java.JavaParser;
 import org.openrewrite.java.internal.JavaTypeCache;
 import org.openrewrite.java.marker.JavaSourceSet;
+import org.openrewrite.java.tree.J;
+import org.openrewrite.java.tree.JavaType;
 import org.openrewrite.marker.Generated;
 import org.openrewrite.marker.Marker;
 import org.openrewrite.marker.Markers;
 import org.openrewrite.xml.tree.Xml;
 import org.springframework.core.io.Resource;
 import org.springframework.sbm.utils.ResourceUtil;
 
+import java.io.InputStream;
+import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-import java.util.Set;
+import java.util.*;
 import java.util.function.Predicate;
+import java.util.function.Supplier;
 import java.util.function.UnaryOperator;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
@@ -73,7 +76,7 @@ public <T extends SourceFile> UnaryOperator<T> addProvenance(
     /**
      * Parse Java sources and resources under {@code src/main} of current module.
      */
-    public List<SourceFile> processMainSources(
+    public SourceSetParsingResult processMainSources(
             Path baseDir,
             List<Resource> resources,
             Xml.Document moduleBuildFile,
@@ -116,20 +119,40 @@ public List<SourceFile> processMainSources(
         javaParserBuilder.typeCache(typeCache);
 
         Iterable<Parser.Input> inputs = mainJavaSources.stream()
-                .map(r -> new Parser.Input(ResourceUtil.getPath(r), () -> ResourceUtil.getInputStream(r)))
+                .map(r -> {
+                    FileAttributes fileAttributes = null;
+                    Path path = ResourceUtil.getPath(r);
+                    boolean isSynthetic = Files.exists(path);
+                    Supplier<InputStream> inputStreamSupplier = () -> ResourceUtil.getInputStream(r);
+                    Parser.Input input = new Parser.Input(path, fileAttributes, inputStreamSupplier, isSynthetic);
+                    return input;
+                })
                 .toList();
 
-        Stream<? extends SourceFile> cus = Stream.of(javaParserBuilder)
-                .map(JavaParser.Builder::build)
-                .flatMap(parser -> parser.parseInputs(inputs, baseDir, executionContext))
-                .peek(s -> alreadyParsed.add(baseDir.resolve(s.getSourcePath())));
+        Set<JavaType.FullyQualified> localClassesCp = new HashSet<>();
+        JavaSourceSet javaSourceSet = sourceSet("main", dependencies, typeCache);
+        List<? extends SourceFile> cus = javaParserBuilder.build()
+                .parseInputs(inputs, baseDir, executionContext)
+                .peek(s -> {
+                    ((J.CompilationUnit)s).getClasses()
+                            .stream()
+                            .map(J.ClassDeclaration::getType)
+                            .forEach(localClassesCp::add);
+
+                    alreadyParsed.add(baseDir.resolve(s.getSourcePath()));
+                })
+                .toList();
 
+        // TODO: This is a hack:
+        // Parsed java sources are not themselves on the classpath (here).
+        // The actual parsing happens when the stream is terminated (toList),
+        // therefore the toList() must be called before the parsed compilation units can be added to the classpath
         List<Marker> mainProjectProvenance = new ArrayList<>(provenanceMarkers);
-        mainProjectProvenance.add(sourceSet("main", dependencies, typeCache));
+        javaSourceSet = appendToClasspath(localClassesCp, javaSourceSet);
+        mainProjectProvenance.add(javaSourceSet);
 
-        // FIXME: 945 Why target and not all main?
         List<Path> parsedJavaPaths = javaSourcesInTarget.stream().map(ResourceUtil::getPath).toList();
-        Stream<SourceFile> parsedJava = cus.map(addProvenance(baseDir, mainProjectProvenance, parsedJavaPaths));
+        Stream<SourceFile> parsedJava = cus.stream().map(addProvenance(baseDir, mainProjectProvenance, parsedJavaPaths));
         log.debug("[%s] Scanned %d java source files in main scope.".formatted(currentProject, mainJavaSources.size()));
 
         //Filter out any generated source files from the returned list, as we do not want to apply the recipe to the
@@ -148,7 +171,22 @@ public List<SourceFile> processMainSources(
         log.debug("[%s] Scanned %d resource files in main scope.".formatted(currentProject, (alreadyParsed.size() - sourcesParsedBefore)));
         // Any resources parsed from "main/resources" should also have the main source set added to them.
         sourceFiles.addAll(parsedResourceFiles);
-        return sourceFiles;
+        return new SourceSetParsingResult(sourceFiles, javaSourceSet.getClasspath());
+    }
+
+    /**
+     * Add entries that don't exist in the classpath of {@code javaSourceSet} from {@code appendingClasspath}.
+     */
+    @NotNull
+    private static JavaSourceSet appendToClasspath(Set<JavaType.FullyQualified> appendingClasspath, JavaSourceSet javaSourceSet) {
+        List<JavaType.FullyQualified> curCp = javaSourceSet.getClasspath();
+        appendingClasspath.forEach(f -> {
+            if(!curCp.contains(f)) {
+                curCp.add(f);
+            }
+        });
+        javaSourceSet = javaSourceSet.withClasspath(new ArrayList<>(curCp));
+        return javaSourceSet;
     }
 
     @NotNull
@@ -160,7 +198,7 @@ private static JavaSourceSet sourceSet(String name, List<Path> dependencies, Jav
     /**
      * Parse Java sources and resource files under {@code src/test}.
      */
-    public List<SourceFile> processTestSources(
+    public SourceSetParsingResult processTestSources(
             Path baseDir,
             Xml.Document moduleBuildFile,
             JavaParser.Builder<? extends JavaParser, ?> javaParserBuilder,
@@ -169,8 +207,8 @@ public List<SourceFile> processTestSources(
             Set<Path> alreadyParsed,
             ExecutionContext executionContext,
             MavenProject currentProject,
-            List<Resource> resources
-    ) {
+            List<Resource> resources,
+            List<JavaType.FullyQualified> classpath) {
         log.info("Processing test sources in module '%s'".formatted(currentProject.getProjectId()));
 
         List<Path> testDependencies = currentProject.getTestClasspathElements();
@@ -186,14 +224,25 @@ public List<SourceFile> processTestSources(
                 .map(r -> new Parser.Input(ResourceUtil.getPath(r), () -> ResourceUtil.getInputStream(r)))
                 .toList();
 
-        Stream<? extends SourceFile> cus = Stream.of(javaParserBuilder)
-                .map(JavaParser.Builder::build)
-                .flatMap(parser -> parser.parseInputs(inputs, baseDir, executionContext));
+        final List<JavaType.FullyQualified> localClassesCp = new ArrayList<>();
+        List<? extends SourceFile> cus = javaParserBuilder.build()
+                .parseInputs(inputs, baseDir, executionContext)
+                .peek(s -> {
+                    ((J.CompilationUnit) s).getClasses()
+                            .stream()
+                            .map(J.ClassDeclaration::getType)
+                            .forEach(localClassesCp::add);
+                    alreadyParsed.add(baseDir.resolve(s.getSourcePath()));
+                })
+                .toList();
 
         List<Marker> markers = new ArrayList<>(provenanceMarkers);
-        markers.add(sourceSet("test", testDependencies, typeCache));
 
-        Stream<SourceFile> parsedJava = cus.map(addProvenance(baseDir, markers, null));
+        JavaSourceSet javaSourceSet = sourceSet("test", testDependencies, typeCache);
+        Set<JavaType.FullyQualified> curClasspath = Stream.concat(classpath.stream(), localClassesCp.stream()).collect(Collectors.toSet());
+        javaSourceSet = appendToClasspath(curClasspath, javaSourceSet);
+        markers.add(javaSourceSet);
+        Stream<SourceFile> parsedJava = cus.stream().map(addProvenance(baseDir, markers, null));
 
         log.debug("[%s] Scanned %d java source files in test scope.".formatted(currentProject, testJavaSources.size()));
         Stream<SourceFile> sourceFiles = parsedJava;
@@ -205,7 +254,7 @@ public List<SourceFile> processTestSources(
         log.debug("[%s] Scanned %d resource files in test scope.".formatted(currentProject, (alreadyParsed.size() - sourcesParsedBefore)));
         sourceFiles = Stream.concat(sourceFiles, parsedResourceFiles);
         List<SourceFile> result = sourceFiles.toList();
-        return result;
+        return new SourceSetParsingResult(result, javaSourceSet.getClasspath());
     }
 
 

sbm-support-rewrite/src/main/java/org/springframework/sbm/parsers/ModuleParser.java

@@ -79,4 +79,5 @@ private void addSourceFileToModel(Path baseDir, List<MavenProject> sortedProject
                 .filter(p -> ResourceUtil.getPath(p.getPomFile()).toString().equals(baseDir.resolve(s.getSourcePath()).toString()))
                 .forEach(p -> p.setSourceFile(s));
     }
+
 }

sbm-support-rewrite/src/main/java/org/springframework/sbm/parsers/ParserContext.java

@@ -19,6 +19,7 @@
 import lombok.Setter;
 import org.springframework.boot.context.properties.ConfigurationProperties;
 
+import java.nio.file.Path;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
@@ -47,7 +48,7 @@ public class ParserProperties {
     /**
      * Directory used by RocksdbMavenPomCache when pomCacheEnabled is true
      */
-    private String pomCacheDirectory;
+    private String pomCacheDirectory = Path.of(System.getProperty("user.home")).resolve(".rewrite-cache").toAbsolutePath().normalize().toString();
 
     /**
      * Comma-separated list of patterns used to create PathMatcher

sbm-support-rewrite/src/main/java/org/springframework/sbm/parsers/ParserProperties.java

@@ -85,6 +85,11 @@ private List<Resource> filterIgnoredResources(Path baseDir, Resource[] resources
         List<Resource> resultingResources = Stream.of(resources)
                 .filter(r -> isAccepted(baseDir, r, pathMatchers))
                 .toList();
+
+        if(resultingResources.isEmpty()) {
+            throw new IllegalArgumentException("No resources were scanned. Check directory and ignore patterns.");
+        }
+
         return resultingResources;
     }
 
@@ -101,7 +106,8 @@ private boolean isAccepted(Path baseDir, Resource r, List<PathMatcher> pathMatch
                 })
                 .findFirst();
         if(isIgnored.isPresent() && log.isInfoEnabled()) {
-            log.info("Ignoring scanned resource '%s'.".formatted(baseDir.relativize(ResourceUtil.getPath(r))));
+            Set<String> ignoredPathPatterns = parserProperties.getIgnoredPathPatterns();
+            log.info("Ignoring scanned resource '%s' given these path matchers: %s.".formatted(baseDir.relativize(ResourceUtil.getPath(r)), ignoredPathPatterns));
         }
         return isIgnored.isEmpty();
     }

sbm-support-rewrite/src/main/java/org/springframework/sbm/parsers/ProjectScanner.java

@@ -17,6 +17,7 @@
 
 import lombok.extern.slf4j.Slf4j;
 import org.openrewrite.ExecutionContext;
+import org.openrewrite.maven.MavenExecutionContextView;
 import org.openrewrite.maven.cache.*;
 import org.openrewrite.maven.utilities.MavenArtifactDownloader;
 import org.openrewrite.tree.ParsingEventListener;
@@ -54,14 +55,6 @@
 @Import({ScanScope.class, ScopeConfiguration.class})
 public class RewriteParserConfiguration {
 
-    @Autowired
-    private ParserProperties parserProperties;
-
-//    @Bean
-//    ProvenanceMarkerFactory provenanceMarkerFactory(MavenMojoProjectParserFactory projectParserFactory) {
-//        return new ProvenanceMarkerFactory(projectParserFactory);
-//    }
-
     @Bean
     MavenPasswordDecrypter mavenPasswordDecrypter() {
         return new MavenPasswordDecrypter();

sbm-support-rewrite/src/main/java/org/springframework/sbm/parsers/RewriteParserConfiguration.java

@@ -21,6 +21,8 @@
 import org.openrewrite.ExecutionContext;
 import org.openrewrite.SourceFile;
 import org.openrewrite.marker.Marker;
+import org.openrewrite.maven.MavenExecutionContextView;
+import org.openrewrite.maven.MavenSettings;
 import org.openrewrite.style.NamedStyles;
 import org.openrewrite.tree.ParsingEventListener;
 import org.openrewrite.tree.ParsingExecutionContextView;
@@ -35,6 +37,8 @@
 import org.springframework.sbm.parsers.maven.ProvenanceMarkerFactory;
 import org.springframework.sbm.scopes.ScanScope;
 
+import java.net.URI;
+import java.nio.charset.Charset;
 import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.List;
@@ -117,7 +121,6 @@ public RewriteProjectParsingResult parse(Path givenBaseDir, List<Resource> resou
         List<Xml.Document> parsedBuildFiles = buildFileParser.parseBuildFiles(baseDir, parserContext.getBuildFileResources(), parserContext.getActiveProfiles(), executionContext, parserProperties.isSkipMavenParsing(), provenanceMarkers);
         parserContext.setParsedBuildFiles(parsedBuildFiles);
 
-
         log.trace("Start to parse %d source files in %d modules".formatted(resources.size() + parsedBuildFiles.size(), parsedBuildFiles.size()));
         List<SourceFile> otherSourceFiles = sourceFileParser.parseOtherSourceFiles(baseDir, parserContext, resources, provenanceMarkers, styles, executionContext);
 

sbm-support-rewrite/src/main/java/org/springframework/sbm/parsers/RewriteProjectParser.java

@@ -13,22 +13,15 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.springframework.sbm.parsers.maven;
+package org.springframework.sbm.parsers;
 
-import lombok.RequiredArgsConstructor;
-import lombok.extern.slf4j.Slf4j;
-import org.openrewrite.maven.utilities.MavenArtifactDownloader;
+import org.openrewrite.SourceFile;
+import org.openrewrite.java.tree.JavaType;
+
+import java.util.List;
 
 /**
  * @author Fabian Krüger
  */
-@Slf4j
-@RequiredArgsConstructor
-class MavenMojoProjectParserPrivateMethods {
-
-    private final MavenMojoProjectParserFactory mavenMojoProjectParserFactory;
-    private final MavenArtifactDownloader artifactDownloader;
-
-
-
+public record SourceSetParsingResult(List<SourceFile> sourceFiles, List<JavaType.FullyQualified> classpath) {
 }

sbm-support-rewrite/src/test/java/org/springframework/sbm/parsers/maven/MavenMojoProjectParserPrivateMethods.java renamed to sbm-support-rewrite/src/main/java/org/springframework/sbm/parsers/SourceSetParsingResult.java

@@ -21,7 +21,9 @@
 import org.openrewrite.Parser;
 import org.openrewrite.SourceFile;
 import org.openrewrite.marker.Marker;
+import org.openrewrite.maven.MavenExecutionContextView;
 import org.openrewrite.maven.MavenParser;
+import org.openrewrite.maven.MavenSettings;
 import org.openrewrite.xml.tree.Xml;
 import org.springframework.core.io.Resource;
 import org.springframework.sbm.utils.ResourceUtil;
@@ -46,14 +48,14 @@
 public class BuildFileParser {
 
     /**
-     * Parse a list of Maven Pom files to a Map of {@code Path} and their parsed {@link Xml.Document}s.
-     * The {@link Xml.Document}s are marked with {@link org.openrewrite.maven.tree.MavenResolutionResult} and the provided provenance markers.
+     * Parse a list of Maven Pom files to a {@code List} of {@link Xml.Document}s.
+     * The {@link Xml.Document}s get marked with {@link org.openrewrite.maven.tree.MavenResolutionResult} and the provided provenance markers.
      *
      * @param baseDir the {@link Path} to the root of the scanned project
      * @param buildFiles the list of resources for relevant pom files.
-     * @param activeProfiles teh active Maven profiles
+     * @param activeProfiles the active Maven profiles
      * @param executionContext the ExecutionContext to use
-*    * @param skipMavenParsing skip parsing Maven files
+     * @param skipMavenParsing skip parsing Maven files
      * @param provenanceMarkers the map of markers to be added
      */
     public List<Xml.Document> parseBuildFiles(
@@ -138,7 +140,10 @@ private Stream<Xml.Document> parsePoms(Path baseDir, List<Resource> pomFiles, Ma
     }
 
     private void initializeMavenSettings(ExecutionContext executionContext) {
-
+        // FIXME: https://github.com/spring-projects-experimental/spring-boot-migrator/issues/880
+        String repo = "file://" + Path.of(System.getProperty("user.home")).resolve(".m2/repository") + "/";
+        MavenSettings mavenSettings = new MavenSettings(repo, null, null, null, null);
+        MavenExecutionContextView.view(executionContext).setMavenSettings(mavenSettings);
     }
 
     public List<Resource> filterAndSortBuildFiles(List<Resource> resources) {