Merge pull request scala#10472 from lrytz/sbt-bridge-2023

Author: SethTisue

build.sbt

@@ -45,6 +45,7 @@ val jnaDep            = "net.java.dev.jna"               % "jna"
 val jlineDeps         = Seq(jlineDep, jnaDep)
 val testInterfaceDep  = "org.scala-sbt"                  % "test-interface"                   % "1.0"
 val diffUtilsDep      = "io.github.java-diff-utils"      % "java-diff-utils"                  % "4.12"
+val compilerInterfaceDep = "org.scala-sbt"               % "compiler-interface"               % "1.9.3"
 
 val projectFolder = settingKey[String]("subfolder in src when using configureAsSubproject, else the project name")
 
@@ -275,8 +276,26 @@ def fixPom(extra: (String, scala.xml.Node)*): Setting[_] = {
   ) ++ extra) }
 }
 
+def ivyDependencyFilter(deps: Seq[(String, String)], scalaBinaryVersion: String) = {
+  import scala.xml._
+  import scala.xml.transform._
+  new RuleTransformer(new RewriteRule {
+    override def transform(node: Node) = node match {
+      case e: Elem if e.label == "dependency" && {
+        val org = e.attribute("org").getOrElse("").toString
+        val name = e.attribute("name").getOrElse("").toString
+        deps.exists { case (g, a) =>
+          org == g && (name == a || name == (a + "_" + scalaBinaryVersion))
+        }
+      } => Seq.empty
+      case n => n
+    }
+  })
+}
+
 val pomDependencyExclusions =
   settingKey[Seq[(String, String)]]("List of (groupId, artifactId) pairs to exclude from the POM and ivy.xml")
+lazy val fixCsrIvy = taskKey[Unit]("Apply pomDependencyExclusions to coursier ivy")
 
 Global / pomDependencyExclusions := Nil
 
@@ -294,27 +313,47 @@ lazy val removePomDependencies: Seq[Setting[_]] = Seq(
               e.child.contains(<groupId>{g}</groupId>) &&
                 (e.child.contains(<artifactId>{a}</artifactId>) || e.child.contains(<artifactId>{a + "_" + scalaBinaryVersion.value}</artifactId>))
             } => Seq.empty
-        case n => Seq(n)
+        case n => n
       }
     }).transform(Seq(n2)).head
   },
+  fixCsrIvy := {
+    //  - coursier makes target/sbt-bridge/resolution-cache/org.scala-lang/scala2-sbt-bridge/2.13.12-bin-SNAPSHOT/resolved.xml.xml
+    //  - copied to target/sbt-bridge//ivy-2.13.12-bin-SNAPSHOT.xml
+    //  - copied to ~/.ivy2/local/org.scala-lang/scala2-sbt-bridge/2.13.12-bin-SNAPSHOT/ivys/ivy.xml
+    import scala.jdk.CollectionConverters._
+    import scala.xml._
+    val currentProject = csrProject.value
+    val ivyModule = org.apache.ivy.core.module.id.ModuleRevisionId.newInstance(
+      currentProject.module.organization.value,
+      currentProject.module.name.value,
+      currentProject.version,
+      currentProject.module.attributes.asJava)
+    val ivyFile = ivySbt.value.withIvy(streams.value.log)(_.getResolutionCacheManager).getResolvedIvyFileInCache(ivyModule)
+    val e = ivyDependencyFilter(pomDependencyExclusions.value, scalaBinaryVersion.value)
+      .transform(Seq(XML.loadFile(ivyFile))).head
+    XML.save(ivyFile.getAbsolutePath, e, xmlDecl = true)
+  },
+  publishConfiguration := Def.taskDyn {
+    val pc = publishConfiguration.value
+    Def.task {
+      fixCsrIvy.value
+      pc
+    }
+  }.value,
+  publishLocalConfiguration := Def.taskDyn {
+    val pc = publishLocalConfiguration.value
+    Def.task {
+      fixCsrIvy.value
+      pc
+    }
+  }.value,
   deliverLocal := {
+    // this doesn't seem to do anything currently, it probably worked before sbt used coursier
     import scala.xml._
-    import scala.xml.transform._
     val f = deliverLocal.value
-    val deps = pomDependencyExclusions.value
-    val e = new RuleTransformer(new RewriteRule {
-      override def transform(node: Node) = node match {
-        case e: Elem if e.label == "dependency" && {
-          val org = e.attribute("org").getOrElse("").toString
-          val name = e.attribute("name").getOrElse("").toString
-          deps.exists { case (g, a) =>
-             org == g && (name == a || name == (a + "_" + scalaBinaryVersion.value))
-          }
-        } => Seq.empty
-        case n => Seq(n)
-      }
-    }).transform(Seq(XML.loadFile(f))).head
+    val e = ivyDependencyFilter(pomDependencyExclusions.value, scalaBinaryVersion.value)
+      .transform(Seq(XML.loadFile(f))).head
     XML.save(f.getAbsolutePath, e, xmlDecl = true)
     f
   }
@@ -579,6 +618,42 @@ lazy val scaladoc = configureAsSubproject(project)
   )
   .dependsOn(compiler)
 
+// dependencies on compiler and compiler-interface are "provided" to align with scala3-sbt-bridge
+lazy val sbtBridge = configureAsSubproject(project, srcdir = Some("sbt-bridge"))
+  .settings(Osgi.settings)
+  .settings(AutomaticModuleName.settings("scala.sbtbridge"))
+  //.settings(fatalWarningsSettings)
+  .settings(
+    name := "scala2-sbt-bridge",
+    description := "sbt compiler bridge for Scala 2",
+    libraryDependencies += compilerInterfaceDep % Provided,
+    generateServiceProviderResources("xsbti.compile.CompilerInterface2" -> "scala.tools.xsbt.CompilerBridge"),
+    generateServiceProviderResources("xsbti.compile.ConsoleInterface1"  -> "scala.tools.xsbt.ConsoleBridge"),
+    generateServiceProviderResources("xsbti.compile.ScaladocInterface2" -> "scala.tools.xsbt.ScaladocBridge"),
+    generateServiceProviderResources("xsbti.InteractiveConsoleFactory"  -> "scala.tools.xsbt.InteractiveConsoleBridgeFactory"),
+    Compile / managedResourceDirectories := Seq((Compile / resourceManaged).value),
+    pomDependencyExclusions ++= List((organization.value, "scala-repl-frontend"), (organization.value, "scala-compiler-doc")),
+    fixPom(
+      "/project/name" -> <name>Scala 2 sbt Bridge</name>,
+      "/project/description" -> <description>sbt compiler bridge for Scala 2</description>,
+      "/project/packaging" -> <packaging>jar</packaging>
+    ),
+    headerLicense := Some(HeaderLicense.Custom(
+      s"""Zinc - The incremental compiler for Scala.
+         |Copyright Scala Center, Lightbend, and Mark Harrah
+         |
+         |Scala (${(ThisBuild/homepage).value.get})
+         |Copyright EPFL and Lightbend, Inc.
+         |
+         |Licensed under Apache License 2.0
+         |(http://www.apache.org/licenses/LICENSE-2.0).
+         |
+         |See the NOTICE file distributed with this work for
+         |additional information regarding copyright ownership.
+         |""".stripMargin)),
+  )
+  .dependsOn(compiler % Provided, replFrontend, scaladoc)
+
 lazy val scalap = configureAsSubproject(project)
   .settings(fatalWarningsSettings)
   .settings(
@@ -727,7 +802,7 @@ val addOpensForTesting = "-XX:+IgnoreUnrecognizedVMOptions" +: "--add-exports=jd
   Seq("java.util.concurrent.atomic", "java.lang", "java.lang.reflect", "java.net").map(p => s"--add-opens=java.base/$p=ALL-UNNAMED")
 
 lazy val junit = project.in(file("test") / "junit")
-  .dependsOn(testkit, compiler, replFrontend, scaladoc)
+  .dependsOn(testkit, compiler, replFrontend, scaladoc, sbtBridge)
   .settings(commonSettings)
   .settings(disableDocs)
   .settings(fatalWarningsSettings)
@@ -745,7 +820,7 @@ lazy val junit = project.in(file("test") / "junit")
       "-Ypatmat-exhaust-depth", "40", // despite not caring about patmat exhaustiveness, we still get warnings for this
     ),
     Compile / javacOptions ++= Seq("-Xlint"),
-    libraryDependencies ++= Seq(junitInterfaceDep, jolDep, diffUtilsDep),
+    libraryDependencies ++= Seq(junitInterfaceDep, jolDep, diffUtilsDep, compilerInterfaceDep),
     testOptions += Tests.Argument(TestFrameworks.JUnit, "-a", "-v", "-s"),
     Compile / unmanagedSourceDirectories := Nil,
     Test / unmanagedSourceDirectories := List(baseDirectory.value),
@@ -1048,7 +1123,7 @@ lazy val root: Project = (project in file("."))
 
     setIncOptions
   )
-  .aggregate(library, reflect, compiler, interactive, repl, replFrontend,
+  .aggregate(library, reflect, compiler, interactive, repl, replFrontend, sbtBridge,
     scaladoc, scalap, testkit, partest, junit, scalacheck, tasty, tastytest, scalaDist).settings(
     Compile / sources := Seq.empty,
     onLoadMessage := s"""|*** Welcome to the sbt build definition for Scala! ***

src/sbt-bridge/scala/tools/xsbt/API.scala

@@ -0,0 +1,222 @@
+/*
+ * Zinc - The incremental compiler for Scala.
+ * Copyright Scala Center, Lightbend, and Mark Harrah
+ *
+ * Scala (https://www.scala-lang.org)
+ * Copyright EPFL and Lightbend, Inc.
+ *
+ * Licensed under Apache License 2.0
+ * (http://www.apache.org/licenses/LICENSE-2.0).
+ *
+ * See the NOTICE file distributed with this work for
+ * additional information regarding copyright ownership.
+ */
+
+package scala.tools
+package xsbt
+
+import scala.tools.nsc.Phase
+import scala.tools.nsc.symtab.Flags
+import xsbti.api._
+import xsbti.VirtualFile
+
+object API {
+  val name = "xsbt-api"
+}
+
+final class API(val global: CallbackGlobal) extends Compat with GlobalHelpers with ClassName {
+  import global._
+
+  import scala.collection.mutable
+  private val nonLocalClassSymbolsInCurrentUnits = new mutable.HashSet[Symbol]()
+
+  def newPhase(prev: Phase) = new ApiPhase(prev)
+  class ApiPhase(prev: Phase) extends GlobalPhase(prev) {
+    override def description = "Extracts the public API from source files."
+    def name = API.name
+    override def run(): Unit = {
+      val start = System.currentTimeMillis
+      super.run()
+
+      // After processing all units, register generated classes
+      registerGeneratedClasses(nonLocalClassSymbolsInCurrentUnits.iterator)
+      nonLocalClassSymbolsInCurrentUnits.clear()
+
+      callback.apiPhaseCompleted()
+      val stop = System.currentTimeMillis
+      debuglog("API phase took : " + ((stop - start) / 1000.0) + " s")
+    }
+
+    // TODO In 2.13, shouldSkipThisPhaseForJava should be overridden instead of cancelled
+    // override def shouldSkipThisPhaseForJava = !global.callback.isPickleJava
+    override def cancelled(unit: CompilationUnit) = {
+      if (Thread.interrupted()) reporter.cancelled = true
+      reporter.cancelled || unit.isJava && !global.callback.isPickleJava
+    }
+
+    def apply(unit: global.CompilationUnit): Unit = processUnit(unit)
+
+    private def processUnit(unit: CompilationUnit): Unit = {
+      if (!unit.isJava || global.callback.isPickleJava) {
+        processScalaUnit(unit)
+      }
+    }
+
+    private def processScalaUnit(unit: CompilationUnit): Unit = {
+      val sourceFile: VirtualFile = unit.source.file match { case AbstractZincFile(vf) => vf }
+      debuglog("Traversing " + sourceFile)
+      callback.startSource(sourceFile)
+      val extractApi = new ExtractAPI[global.type](global, sourceFile)
+      val traverser = new TopLevelHandler(extractApi)
+      traverser.apply(unit.body)
+
+      val extractUsedNames = new ExtractUsedNames[global.type](global)
+      extractUsedNames.extractAndReport(unit)
+
+      val classApis = traverser.allNonLocalClasses
+      val mainClasses = traverser.mainClasses
+
+      // Use of iterators make this code easier to profile
+
+      val classApisIt = classApis.iterator
+      while (classApisIt.hasNext) {
+        callback.api(sourceFile, classApisIt.next())
+      }
+
+      val mainClassesIt = mainClasses.iterator
+      while (mainClassesIt.hasNext) {
+        callback.mainClass(sourceFile, mainClassesIt.next())
+      }
+
+      extractApi.allExtractedNonLocalSymbols.foreach { cs =>
+        // Only add the class symbols defined in this compilation unit
+        if (cs.sourceFile != null) nonLocalClassSymbolsInCurrentUnits.+=(cs)
+      }
+    }
+  }
+
+  private case class FlattenedNames(binaryName: String, className: String)
+
+  /**
+   * Registers only non-local generated classes in the callback by extracting
+   * information about its names and using the names to generate class file paths.
+   *
+   * Mimics the previous logic that was present in `Analyzer`, despite the fact
+   * that now we construct the names that the compiler will give to every non-local
+   * class independently of genbcode.
+   *
+   * Why do we do this? The motivation is that we want to run the incremental algorithm
+   * independently of the compiler pipeline. This independence enables us to:
+   *
+   * 1. Offload the incremental compiler logic out of the primary pipeline and
+   *    run the incremental phases concurrently.
+   * 2. Know before the compilation is completed whether another compilation will or
+   *    will not be required. This is important to make incremental compilation work
+   *    with pipelining and enables further optimizations; for example, we can start
+   *    subsequent incremental compilations before (!) the initial compilation is done.
+   *    This can buy us ~30-40% faster incremental compiler iterations.
+   *
+   * This method only takes care of non-local classes because local classes have no
+   * relevance in the correctness of the algorithm and can be registered after genbcode.
+   * Local classes are only used to construct the relations of products and to produce
+   * the list of generated files + stamps, but names referring to local classes **never**
+   * show up in the name hashes of classes' APIs, hence never considered for name hashing.
+   *
+   * As local class files are owned by other classes that change whenever they change,
+   * we could most likely live without adding their class files to the products relation
+   * and registering their stamps. However, to be on the safe side, we will continue to
+   * register the local products in `Analyzer`.
+   *
+   * @param allClassSymbols The class symbols found in all the compilation units.
+   */
+  def registerGeneratedClasses(classSymbols: Iterator[Symbol]): Unit = {
+    classSymbols.foreach { symbol =>
+      val sourceFile = symbol.sourceFile
+      val sourceVF0 =
+        if (sourceFile == null) symbol.enclosingTopLevelClass.sourceFile
+        else sourceFile
+      val sourceVF: Option[VirtualFile] = sourceVF0 match {
+        case AbstractZincFile(vf) => Some(vf)
+        // This could be scala.reflect.io.FileZipArchive$LeakyEntry
+        case _ => None
+      }
+
+      def registerProductNames(names: FlattenedNames): Unit = {
+        // Guard against a local class in case it surreptitiously leaks here
+        if (!symbol.isLocalClass) {
+          val pathToClassFile = s"${names.binaryName}.class"
+          val classFile = {
+            JarUtils.outputJar match {
+              case Some(outputJar) =>
+                new java.io.File(JarUtils.classNameInJar(outputJar, pathToClassFile))
+              case None =>
+                val outputDir = global.settings.outputDirs.outputDirFor(sourceFile).file
+                new java.io.File(outputDir, pathToClassFile)
+            }
+          }
+          val zincClassName = names.className
+          val srcClassName = classNameAsString(symbol)
+          sourceVF foreach { source =>
+            callback.generatedNonLocalClass(
+              source,
+              classFile.toPath,
+              zincClassName,
+              srcClassName
+            )
+          }
+        } else ()
+      }
+
+      val names = FlattenedNames(
+        fullName(symbol, java.io.File.separatorChar, symbol.moduleSuffix, true),
+        fullName(symbol, '.', symbol.moduleSuffix, false)
+      )
+
+      registerProductNames(names)
+
+      // Register the names of top-level module symbols that emit two class files
+      val isTopLevelUniqueModule =
+        symbol.owner.isPackageClass && symbol.isModuleClass && symbol.companionClass == NoSymbol
+      if (isTopLevelUniqueModule || symbol.isPackageObject) {
+        val names = FlattenedNames(
+          fullName(symbol, java.io.File.separatorChar, "", true),
+          fullName(symbol, '.', "", false)
+        )
+        registerProductNames(names)
+      }
+    }
+  }
+
+  private final class TopLevelHandler(extractApi: ExtractAPI[global.type])
+      extends TopLevelTraverser {
+    def allNonLocalClasses: Set[ClassLike] = {
+      extractApi.allExtractedNonLocalClasses
+    }
+
+    def mainClasses: Set[String] = extractApi.mainClasses
+
+    def `class`(c: Symbol): Unit = {
+      extractApi.extractAllClassesOf(c.owner, c)
+    }
+  }
+
+  private abstract class TopLevelTraverser extends Traverser {
+    def `class`(s: Symbol): Unit
+    override def traverse(tree: Tree): Unit = {
+      tree match {
+        case (_: ClassDef | _: ModuleDef) if isTopLevel(tree.symbol) => `class`(tree.symbol)
+        case _: PackageDef =>
+          super.traverse(tree)
+        case _ =>
+      }
+    }
+    def isTopLevel(sym: Symbol): Boolean = {
+      !ignoredSymbol(sym) &&
+      sym.isStatic &&
+      !sym.isImplClass &&
+      (!sym.hasFlag(Flags.JAVA) || global.callback.isPickleJava) &&
+      !sym.isNestedClass
+    }
+  }
+
+}