diff --git a/build.sbt b/build.sbt index c308b42e..e5803471 100644 --- a/build.sbt +++ b/build.sbt @@ -71,7 +71,9 @@ lazy val runtime = CrossProject("scalac-scoverage-runtime", file("scalac-scovera .settings(appSettings: _*) .jvmSettings( fork in Test := true, - libraryDependencies += "org.scalatest" %% "scalatest" % ScalatestVersion % "test" + libraryDependencies ++= Seq( + "org.scalatest" %% "scalatest" % ScalatestVersion % "test" + ) ) .jsSettings( libraryDependencies += "org.scalatest" %%% "scalatest" % ScalatestVersion % "test", diff --git a/scalac-scoverage-plugin/src/main/scala/scoverage/Constants.scala b/scalac-scoverage-plugin/src/main/scala/scoverage/Constants.scala index e1b1e57e..cd25571b 100644 --- a/scalac-scoverage-plugin/src/main/scala/scoverage/Constants.scala +++ b/scalac-scoverage-plugin/src/main/scala/scoverage/Constants.scala @@ -1,13 +1,19 @@ package scoverage object Constants { - // the file that contains the statement mappings - val CoverageFileName = "scoverage.coverage" // the final scoverage report val XMLReportFilename = "scoverage.xml" val XMLReportFilenameWithDebug = "scoverage-debug.xml" // directory that contains all the measurement data but not reports val DataDir = "scoverage-data" + /********************************************************************* + * If updating any of the file names below, an update is also required to * + * the corresponding file name in Invoker.scala * + *********************************************************************/ + // the file that contains the statement mappings + val CoverageFileName = "scoverage.coverage" // the prefix the measurement files have val MeasurementsPrefix = "scoverage.measurements." + //subdir to suffix to classpath when [writeToClasspath] option is in use" + val ClasspathSubdir = "META-INF/scoverage" } diff --git a/scalac-scoverage-plugin/src/main/scala/scoverage/plugin.scala b/scalac-scoverage-plugin/src/main/scala/scoverage/plugin.scala index 774cc164..726d62c2 100644 --- a/scalac-scoverage-plugin/src/main/scala/scoverage/plugin.scala +++ b/scalac-scoverage-plugin/src/main/scala/scoverage/plugin.scala @@ -11,7 +11,6 @@ import scala.tools.nsc.transform.{Transform, TypingTransformers} /** @author Stephen Samuel */ class ScoveragePlugin(val global: Global) extends Plugin { - override val name: String = "scoverage" override val description: String = "scoverage code coverage compiler plugin" private val (extraAfterPhase, extraBeforePhase) = processPhaseOptions(pluginOptions) @@ -35,17 +34,31 @@ class ScoveragePlugin(val global: Global) extends Plugin { options.dataDir = opt.substring("dataDir:".length) } else if (opt.startsWith("extraAfterPhase:") || opt.startsWith("extraBeforePhase:")) { // skip here, these flags are processed elsewhere - } else { + } else if (opt.startsWith("writeToClasspath:")) { + try { + options.writeToClasspath = opt.substring("writeToClasspath:".length).toBoolean + } catch { + case _: IllegalArgumentException => throw new RuntimeException("writeToClasspath can only be a boolean value: true or false") + case e: Exception => throw e + } + } else{ error("Unknown option: " + opt) } } + if (!opts.exists(_.startsWith("dataDir:"))) throw new RuntimeException("Cannot invoke plugin without specifying ") + instrumentationComponent.setOptions(options) } override val optionsHelp: Option[String] = Some(Seq( - "-P:scoverage:dataDir: where the coverage files should be written\n", + "-P:scoverage:writeToClasspath: if true, use the system property variable [scoverage_measurement_path] " + + "to store measurements and store instruments file to the output classpath " + + "where compiler emits classfiles. " + + "Uses dataDir to create a targetId and then overrides the datadir with the output classpath.", + + "-P:scoverage:dataDir: where the coverage files should be written\n Acts as a targteId in case [writeToClasspath] is true", "-P:scoverage:excludedPackages:; semicolon separated list of regexs for packages to exclude", "-P:scoverage:excludedFiles:; semicolon separated list of regexs for paths to exclude", "-P:scoverage:excludedSymbols:; semicolon separated list of regexs for symbols to exclude", @@ -82,6 +95,16 @@ class ScoverageOptions { var excludedFiles: Seq[String] = Nil var excludedSymbols: Seq[String] = Seq("scala.reflect.api.Exprs.Expr", "scala.reflect.api.Trees.Tree", "scala.reflect.macros.Universe.Tree") var dataDir: String = IOUtils.getTempPath + + // Adding additional option for https://github.com/scoverage/scalac-scoverage-plugin/issues/265 + // If the option is false, scoverage works the usual way. + // If the option is true, the instrument files [scoverage.coverage] are stored on to the classpath and + // the measurement files are written to the directory specified by the system property + // `scoverage_measurement_path`. Setting [writeToClasspath] to true creates a `targetId` + // out of the given value of [dataDir] and overrides the [dataDir] to point + // to the output classpath of the compiler. + // By default, the option is set to false. + var writeToClasspath: Boolean = false } class ScoverageInstrumentationComponent(val global: Global, extraAfterPhase: Option[String], extraBeforePhase: Option[String]) @@ -107,10 +130,49 @@ class ScoverageInstrumentationComponent(val global: Global, extraAfterPhase: Opt private var options: ScoverageOptions = new ScoverageOptions() private var coverageFilter: CoverageFilter = AllCoverageFilter + // This [targetId] is used below in case [writeToClasspath] option + // is true. It is created using the option given in [dataDir]. It is used for: + // 1. Creating a subdir under the classpath to store the instrument files, i.e + // path to instrument files will be `< classpath >/META-INF/scoverage/< targetId >/scoverage.coverage` + // 2. Instrumenting the binaries with the [targetId], i.e binaries will be instrumented with + // `Invoker.invokedWriteToClasspath(< statement id >, < targetId >)` + // Consequently, at runtime, we can get the "correct" instrument file located at `.../< targetId >/scoverage.coverage` + // from the classpath and store it with its appropriate measurements file. + // NOTE: It is on user to provide unique value of targetIds by setting the [dataDir] option + // uniquely for every target in case [writeToClasspath] is true. + private var targetId: String = "" + + // Checks if [path] is a relative path. + def validateRelativePath(path: String): String = { + val tmp: File = new File(path) + if (tmp.isAbsolute){ + throw new RuntimeException("dataDir must be a relative path in case writeToClasspath is true") + }else { + path + } + } + def setOptions(options: ScoverageOptions): Unit = { + this.options = options + + // If [writeToClasspath] is true: + // 1. set targetId to relative path specified by options.dataDir + // 2. override options.dataDir to point to output classpath. + if(options.writeToClasspath){ + settings.outputDirs.getSingleOutput match { + case Some(dest) => + //creating targetId out of [dataDir] + targetId = validateRelativePath(options.dataDir) + // Overriding [dataDir] to `< classpath >/META-INF/scoverage/< targetId >`. + // This is where the instrument file [scoverage.coverage] for this target will now be stored. + options.dataDir = s"${dest.toString()}/${Constants.ClasspathSubdir}/$targetId" + case None => throw new RuntimeException("No output classpath specified.") + } + } coverageFilter = new RegexCoverageFilter(options.excludedPackages, options.excludedFiles, options.excludedSymbols) new File(options.dataDir).mkdirs() // ensure data directory is created + } override def newPhase(prev: scala.tools.nsc.Phase): Phase = new Phase(prev) { @@ -128,7 +190,16 @@ class ScoverageInstrumentationComponent(val global: Global, extraAfterPhase: Opt Serializer.serialize(coverage, Serializer.coverageFile(options.dataDir)) reporter.echo(s"Wrote instrumentation file [${Serializer.coverageFile(options.dataDir)}]") - reporter.echo(s"Will write measurement data to [${options.dataDir}]") + + // Measurements are not necessarily written to + // [options.dataDir] in case [writeToClasspath] is set to true. + if(!options.writeToClasspath) { + reporter.echo(s"Will write measurement data to [${ options.dataDir }]") + } + else { + reporter.echo(s"Will write measurements data to the directory specified by system" + + s" variable scoverage_measurement_path at runtime.") + } } } @@ -152,6 +223,24 @@ class ScoverageInstrumentationComponent(val global: Global, extraAfterPhase: Opt def safeLine(tree: Tree): Int = if (tree.pos.isDefined) tree.pos.line else -1 def safeSource(tree: Tree): Option[SourceFile] = if (tree.pos.isDefined) Some(tree.pos.source) else None + var termName: String = "" + var secondArg: String = "" + if (options.writeToClasspath){ + /** + * We pass in [targetId] as one of the instruments with + * [invokedUseEnvironment] as it helps in + * 1. grabbing the "correct" instruments file from the classpath at runtime. + * 2. creating a unique subdir for each target at runtime. + * iff targetIds are unique per target. + */ + termName = "invokedWriteToClasspath" + secondArg = targetId + } + else{ + termName = "invoked" + secondArg = options.dataDir + } + def invokeCall(id: Int): Tree = { Apply( Select( @@ -159,14 +248,14 @@ class ScoverageInstrumentationComponent(val global: Global, extraAfterPhase: Opt Ident("scoverage"), newTermName("Invoker") ), - newTermName("invoked") + newTermName(termName) ), List( Literal( Constant(id) ), Literal( - Constant(options.dataDir) + Constant(secondArg) ) ) ) @@ -288,6 +377,7 @@ class ScoverageInstrumentationComponent(val global: Global, extraAfterPhase: Opt def allConstArgs(args: List[Tree]) = args.forall(arg => arg.isInstanceOf[Literal] || arg.isInstanceOf[Ident]) def process(tree: Tree): Tree = { + tree match { // // non ranged inside ranged will break validation after typer, which only kicks in for yrangepos. diff --git a/scalac-scoverage-runtime/jvm/src/test/scala/scoverage/InvokerUseEnvironmentTest.scala b/scalac-scoverage-runtime/jvm/src/test/scala/scoverage/InvokerUseEnvironmentTest.scala new file mode 100644 index 00000000..0ced1440 --- /dev/null +++ b/scalac-scoverage-runtime/jvm/src/test/scala/scoverage/InvokerUseEnvironmentTest.scala @@ -0,0 +1,64 @@ +package scoverage + +import org.scalatest.{BeforeAndAfter, FunSuite} +import scoverage.Platform.File + +/** + * Verify that [[Invoker.invokedWriteToClasspath()]] works as expected. + */ +class InvokerUseEnvironmentTest extends FunSuite with BeforeAndAfter { + + + val targetIds = Array( + new File("target_id_0"), + new File("target_id_1") + ) + + before { + System.setProperty("scoverage_measurement_path","scoverage_java_prop") + } + + test("calling Invoker.invokedUseEnvironments puts the data on sys property scoverage_measurement_path") { + + val testIds: Set[Int] = (1 to 10).toSet + + testIds.map { i: Int => Invoker.invokedWriteToClasspath(i, targetIds(i % 2).toString)} + + // Verify that [Invoker.dataDir] has been set correctly. + assert(Invoker.dataDir == System.getProperty("scoverage_measurement_path")) + + // Verify measurements went to correct directory under the system property. + val dir0 = s"${Invoker.dataDir}/${targetIds(0).toString}" + val measurementFiles3 = Invoker.findMeasurementFiles(dir0) + val idsFromFile3 = Invoker.invoked(measurementFiles3.toIndexedSeq) + assert (idsFromFile3 == testIds.filter { i: Int => i % 2 == 0}) + + + val dir1 = s"${Invoker.dataDir}/${targetIds(1).toString}" + val measurementFiles4 = Invoker.findMeasurementFiles(dir1) + val idsFromFile4 = Invoker.invoked(measurementFiles4.toIndexedSeq) + assert (idsFromFile4 == testIds.filter { i: Int => i % 2 == 1}) + + } + + after { + deleteMeasurementFolders() + } + + + private def deleteMeasurementFolders(): Unit = { + val d = s"${Invoker.dataDir}" + targetIds.foreach (i => { + val f = new File(s"$d/${i.toString}") + if (f.isDirectory) + f.listFiles().foreach(_.delete()) + }) + + val f2 = new File(d) + if(f2.isDirectory) + f2.listFiles().foreach(_.delete()) + f2.delete() + } + + +} diff --git a/scalac-scoverage-runtime/shared/src/main/scala/scoverage/Invoker.scala b/scalac-scoverage-runtime/shared/src/main/scala/scoverage/Invoker.scala index f7d222bf..c933a7e3 100644 --- a/scalac-scoverage-runtime/shared/src/main/scala/scoverage/Invoker.scala +++ b/scalac-scoverage-runtime/shared/src/main/scala/scoverage/Invoker.scala @@ -1,18 +1,30 @@ package scoverage -import scala.collection.{mutable, Set} +import scala.collection.{Set, mutable} +import java.nio.file.{Files, Paths, StandardCopyOption} import scoverage.Platform._ /** @author Stephen Samuel */ object Invoker { private val MeasurementsPrefix = "scoverage.measurements." - private val threadFiles = new ThreadLocal[mutable.HashMap[String, FileWriter]] + private val CoverageFileName = "scoverage.coverage" + private val ClasspathSubdir = "META-INF/scoverage" + private val threadFiles = new ThreadLocal[mutable.HashMap[String, FileWriter]] // For each data directory we maintain a thread-safe set tracking the ids that we've already // seen and recorded. We're using a map as a set, so we only care about its keys and can ignore // its values. private val dataDirToIds = ThreadSafeMap.empty[String, ThreadSafeMap[Int, Any]] + // System property from where we get the path to output dir for measurements + private val measurementFileEnv = "scoverage_measurement_path" + // Used to store measurements files in case system property is not set + private val subdir = "scoverageMeasurements" + + // Get the output data directory from the System property [scoverage_measurement_path] + // variable or write to a subdir. + lazy val dataDir: String = System.getProperty(measurementFileEnv, subdir) + /** * We record that the given id has been invoked by appending its id to the coverage @@ -30,6 +42,7 @@ object Invoker { * @param dataDir the directory where the measurement data is held */ def invoked(id: Int, dataDir: String): Unit = { + // [sam] we can do this simple check to save writing out to a file. // This won't work across JVMs but since there's no harm in writing out the same id multiple // times since for coverage we only care about 1 or more, (it just slows things down to @@ -59,6 +72,43 @@ object Invoker { } } + /** + * Invokes above method after adding additional features to make it work with + * https://github.com/scoverage/scalac-scoverage-plugin/issues/265. Typically, we + * add coverage instrument files [scoverage.coverage] along with the measurement files at runtime + * rather than compile time. This is needed as the runtime process may be running + * on a different host than the compile time one. + * However, since the instrument files are on the classpath, we can retrieve the instruments from different hosts. + * + * @param id the id of the statement that was invoked + * @param targetId targetId helps in grabbing the "correct" instrument file from cp that corresponds + * to the measurement file written during this invocation. + */ + def invokedWriteToClasspath(id: Int, targetId: String): Unit = { + // Adding subdir to the basedir. This is because every + // unique target has a instrument file [scoverage.coverage] for which corresponding + // measurements [scoverage.measurements] will be generated and it is possible that a + // test file tests multiple targets. And thus, multiple measurement files will be generated + // which needs to be stored SEPARATELY from each other (but along with THEIR instrument files). + // Thus, to segregate (instruments + measurements) for each target, we need this subdir. + // This was not an issue with previous version as then we explicitly + // specified a different [dataDir] each time we compiled a target. + val newDataDir = dataDir + s"/$targetId" + + if (!dataDirToIds.contains(newDataDir)) { + // Guard against SI-7943: "TrieMap method getOrElseUpdate is not thread-safe". + dataDirToIds.synchronized { + if (!dataDirToIds.contains(newDataDir)) { + // copy instruments. + new File(newDataDir).mkdirs() + copyCoverageFile(targetId, newDataDir) + dataDirToIds(newDataDir) = ThreadSafeMap.empty[Int, Any] + } + } + } + invoked(id, newDataDir) + } + def measurementFile(dataDir: File): File = measurementFile(dataDir.getAbsolutePath) def measurementFile(dataDir: String): File = new File(dataDir, MeasurementsPrefix + Thread.currentThread.getId) @@ -67,6 +117,26 @@ object Invoker { override def accept(pathname: File): Boolean = pathname.getName.startsWith(MeasurementsPrefix) }) + + /** + * Copy instruments file from classpath. + * + * @param targetId targetId helps in grabbing the "correct" instrument file from cp that corresponds + * to the measurement file written during this invocation. + * @param dest directory path for destination of coverage file + */ + def copyCoverageFile(targetId: String, dest: String): Unit = { + + // Getting the instrument file `META-INF/scoverage/< targetId >/scoverage.coverage` from classpath + val r = Thread.currentThread.getContextClassLoader.getResourceAsStream(s"$ClasspathSubdir/$targetId/$CoverageFileName") + + // Copying the instruments file to the directory that will contain measurements. + if (r != null) { + Files.copy(r, Paths.get(s"$dest/$CoverageFileName"), StandardCopyOption.REPLACE_EXISTING) + r.close() + } + } + // loads all the invoked statement ids from the given files def invoked(files: Seq[File]): Set[Int] = { val acc = mutable.Set[Int]() @@ -81,4 +151,5 @@ object Invoker { } acc } + }