diff --git a/compiler/src/dotty/tools/dotc/core/tasty/TastyPickler.scala b/compiler/src/dotty/tools/dotc/core/tasty/TastyPickler.scala index cc2e4dd58a24..9806470cb926 100644 --- a/compiler/src/dotty/tools/dotc/core/tasty/TastyPickler.scala +++ b/compiler/src/dotty/tools/dotc/core/tasty/TastyPickler.scala @@ -6,7 +6,6 @@ package tasty import TastyFormat._ import collection.mutable import TastyBuffer._ -import java.util.UUID import core.Symbols.Symbol import ast.tpd import Decorators._ @@ -14,17 +13,6 @@ import Decorators._ class TastyPickler { private val sections = new mutable.ArrayBuffer[(NameRef, TastyBuffer)] - val uuid = UUID.randomUUID() - - private val headerBuffer = { - val buf = new TastyBuffer(24) - for (ch <- header) buf.writeByte(ch.toByte) - buf.writeNat(MajorVersion) - buf.writeNat(MinorVersion) - buf.writeUncompressedLong(uuid.getMostSignificantBits) - buf.writeUncompressedLong(uuid.getLeastSignificantBits) - buf - } val nameBuffer = new NameBuffer @@ -36,6 +24,20 @@ class TastyPickler { buf.assemble() buf.length + natSize(buf.length) } + + val uuidLow: Long = pjwHash64(nameBuffer.bytes) + val uuidHi: Long = sections.iterator.map(x => pjwHash64(x._2.bytes)).fold(0L)(_ ^ _) + + val headerBuffer = { + val buf = new TastyBuffer(header.length + 24) + for (ch <- header) buf.writeByte(ch.toByte) + buf.writeNat(MajorVersion) + buf.writeNat(MinorVersion) + buf.writeUncompressedLong(uuidLow) + buf.writeUncompressedLong(uuidHi) + buf + } + val totalSize = headerBuffer.length + lengthWithLength(nameBuffer) + { @@ -69,4 +71,23 @@ class TastyPickler { var addrOfSym: Symbol => Option[Addr] = (_ => None) val treePkl = new TreePickler(this) + + /** Returns a non-cryptographic 64-bit hash of the array. + * + * from https://en.wikipedia.org/wiki/PJW_hash_function#Implementation + */ + private def pjwHash64(data: Array[Byte]): Long = { + var h = 0L + var high = 0L + var i = 0 + while (i < data.length) { + h = (h << 4) + data(i) + high = h & 0xF0000000L + if (high != 0) + h ^= high >> 24 + h &= ~high + i += 1 + } + h + } } diff --git a/compiler/test/dotty/tools/dotc/CompilationTests.scala b/compiler/test/dotty/tools/dotc/CompilationTests.scala index df7a1141cec9..162b93d87067 100644 --- a/compiler/test/dotty/tools/dotc/CompilationTests.scala +++ b/compiler/test/dotty/tools/dotc/CompilationTests.scala @@ -207,30 +207,6 @@ class CompilationTests extends ParallelTesting { compileDir("../library/src", allowDeepSubtypes.and("-Ycheck-reentrant", "-strict", "-priorityclasspath", defaultOutputDir)) - def sources(paths: JStream[Path], excludedFiles: List[String] = Nil): List[String] = - paths.iterator().asScala - .filter(path => - (path.toString.endsWith(".scala") || path.toString.endsWith(".java")) - && !excludedFiles.contains(path.getFileName.toString)) - .map(_.toString).toList - - val compilerDir = Paths.get("../compiler/src") - val compilerSources = sources(Files.walk(compilerDir)) - - val backendDir = Paths.get("../scala-backend/src/compiler/scala/tools/nsc/backend") - val backendJvmDir = Paths.get("../scala-backend/src/compiler/scala/tools/nsc/backend/jvm") - - // NOTE: Keep these exclusions synchronized with the ones in the sbt build (Build.scala) - val backendExcluded = - List("JavaPlatform.scala", "Platform.scala", "ScalaPrimitives.scala") - val backendJvmExcluded = - List("BCodeICodeCommon.scala", "GenASM.scala", "GenBCode.scala", "ScalacBackendInterface.scala") - - val backendSources = - sources(Files.list(backendDir), excludedFiles = backendExcluded) - val backendJvmSources = - sources(Files.list(backendJvmDir), excludedFiles = backendJvmExcluded) - def dotty1 = { compileList( "dotty1", @@ -261,6 +237,117 @@ class CompilationTests extends ParallelTesting { } :: Nil }.map(_.checkCompile()).foreach(_.delete()) } + + @Test def bytecodeIdemporency: Unit = { + var failed = 0 + var total = 0 + val blacklisted = Set( + // Bridges on collections in different order. Second one in scala2 order. + "pos/Map/scala/collection/immutable/Map", + "pos/Map/scala/collection/immutable/AbstractMap", + "pos/t1203a/NodeSeq", + "pos/i2345/Whatever" + ) + def checkIdempotency(): Unit = { + val groupedBytecodeFiles: List[(Path, Path, Path, Path)] = { + val bytecodeFiles = { + def bytecodeFiles(paths: JStream[Path]): List[Path] = { + def isBytecode(file: String) = file.endsWith(".class") || file.endsWith(".tasty") + paths.iterator.asScala.filter(path => isBytecode(path.toString)).toList + } + val compilerDir1 = Paths.get("../out/idempotency1") + val compilerDir2 = Paths.get("../out/idempotency2") + bytecodeFiles(Files.walk(compilerDir1)) ++ bytecodeFiles(Files.walk(compilerDir2)) + } + val groups = bytecodeFiles.groupBy(f => f.toString.substring("../out/idempotencyN/".length, f.toString.length - 6)) + groups.filterNot(x => blacklisted(x._1)).valuesIterator.flatMap { g => + def pred(f: Path, i: Int, isTasty: Boolean) = + f.toString.contains("idempotency" + i) && f.toString.endsWith(if (isTasty) ".tasty" else ".class") + val class1 = g.find(f => pred(f, 1, isTasty = false)) + val class2 = g.find(f => pred(f, 2, isTasty = false)) + val tasty1 = g.find(f => pred(f, 1, isTasty = true)) + val tasty2 = g.find(f => pred(f, 2, isTasty = true)) + assert(class1.isDefined, "Could not find class in idempotency1 for " + class2) + assert(class2.isDefined, "Could not find class in idempotency2 for " + class1) + if (tasty1.isEmpty || tasty2.isEmpty) Nil + else List(Tuple4(class1.get, tasty1.get, class2.get, tasty2.get)) + }.toList + } + + for ((class1, tasty1, class2, tasty2) <- groupedBytecodeFiles) { + total += 1 + val bytes1 = Files.readAllBytes(class1) + val bytes2 = Files.readAllBytes(class2) + if (!java.util.Arrays.equals(bytes1, bytes2)) { + failed += 1 + val tastyBytes1 = Files.readAllBytes(tasty1) + val tastyBytes2 = Files.readAllBytes(tasty2) + if (java.util.Arrays.equals(tastyBytes1, tastyBytes2)) + println(s"Idempotency test failed between $class1 and $class1 (same tasty)") + else + println(s"Idempotency test failed between $tasty1 and $tasty2") + /* Dump bytes to console, could be useful if issue only appears in CI. + * Create the .class locally with Files.write(path, Array[Byte](...)) with the printed array + */ + // println(bytes1.mkString("Array[Byte](", ",", ")")) + // println(bytes2.mkString("Array[Byte](", ",", ")")) + } + } + } + + val opt = defaultOptions.and("-YemitTasty") + + def idempotency1() = { + compileList("dotty1", compilerSources ++ backendSources ++ backendJvmSources, opt) + + compileFilesInDir("../tests/pos", opt) + } + def idempotency2() = { + compileList("dotty1", compilerSources ++ backendSources ++ backendJvmSources, opt) + + compileFilesInDir("../tests/pos", opt) + } + + val tests = (idempotency1() + idempotency2()).keepOutput.checkCompile() + + assert(new java.io.File("../out/idempotency1/").exists) + assert(new java.io.File("../out/idempotency2/").exists) + + val t0 = System.currentTimeMillis() + checkIdempotency() + println(s"checked bytecode idempotency (${(System.currentTimeMillis() - t0) / 1000.0} sec)") + + tests.delete() + + assert(failed == 0, s"Failed $failed idempotency checks (out of $total)") + } + + + private val (compilerSources, backendSources, backendJvmSources) = { + def sources(paths: JStream[Path], excludedFiles: List[String] = Nil): List[String] = + paths.iterator().asScala + .filter(path => + (path.toString.endsWith(".scala") || path.toString.endsWith(".java")) + && !excludedFiles.contains(path.getFileName.toString)) + .map(_.toString).toList + + val compilerDir = Paths.get("../compiler/src") + val compilerSources0 = sources(Files.walk(compilerDir)) + + val backendDir = Paths.get("../scala-backend/src/compiler/scala/tools/nsc/backend") + val backendJvmDir = Paths.get("../scala-backend/src/compiler/scala/tools/nsc/backend/jvm") + + // NOTE: Keep these exclusions synchronized with the ones in the sbt build (Build.scala) + val backendExcluded = + List("JavaPlatform.scala", "Platform.scala", "ScalaPrimitives.scala") + val backendJvmExcluded = + List("BCodeICodeCommon.scala", "GenASM.scala", "GenBCode.scala", "ScalacBackendInterface.scala") + + val backendSources0 = + sources(Files.list(backendDir), excludedFiles = backendExcluded) + val backendJvmSources0 = + sources(Files.list(backendJvmDir), excludedFiles = backendJvmExcluded) + + (compilerSources0, backendSources0, backendJvmSources0) + } } object CompilationTests {