Skip to content

Fix #2185: Add bytecode idempotency checks. #2272

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 8 commits into from
May 5, 2017
Merged

Fix #2185: Add bytecode idempotency checks. #2272

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
5d29e7e
Fix #2185: Add bytecode idempotency checks.
nicolasstucki Apr 17, 2017
1dd46f2
Compute tasty UUID based on nameBuffer and sections hash.
nicolasstucki Apr 19, 2017
3caa001
Use mumur hashing for tasty uuid.
nicolasstucki Apr 19, 2017
54560a6
Port and use 64-bit CityHash for tasty uuid.
nicolasstucki Apr 27, 2017
5cbd788
Add copyright
nicolasstucki Apr 28, 2017
c73fd4b
Fix byte to long casts.
nicolasstucki May 3, 2017
eed4b9b
Add test to idempotency blacklist.
nicolasstucki May 3, 2017
000b432
Implement TastyHash based on PJW hash.
nicolasstucki May 3, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 33 additions & 12 deletions compiler/src/dotty/tools/dotc/core/tasty/TastyPickler.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,25 +6,13 @@ package tasty
import TastyFormat._
import collection.mutable
import TastyBuffer._
import java.util.UUID
import core.Symbols.Symbol
import ast.tpd
import Decorators._

class TastyPickler {

private val sections = new mutable.ArrayBuffer[(NameRef, TastyBuffer)]
val uuid = UUID.randomUUID()

private val headerBuffer = {
val buf = new TastyBuffer(24)
for (ch <- header) buf.writeByte(ch.toByte)
buf.writeNat(MajorVersion)
buf.writeNat(MinorVersion)
buf.writeUncompressedLong(uuid.getMostSignificantBits)
buf.writeUncompressedLong(uuid.getLeastSignificantBits)
buf
}

val nameBuffer = new NameBuffer

Expand All @@ -36,6 +24,20 @@ class TastyPickler {
buf.assemble()
buf.length + natSize(buf.length)
}

val uuidLow: Long = pjwHash64(nameBuffer.bytes)
val uuidHi: Long = sections.iterator.map(x => pjwHash64(x._2.bytes)).fold(0L)(_ ^ _)

val headerBuffer = {
val buf = new TastyBuffer(header.length + 24)
for (ch <- header) buf.writeByte(ch.toByte)
buf.writeNat(MajorVersion)
buf.writeNat(MinorVersion)
buf.writeUncompressedLong(uuidLow)
buf.writeUncompressedLong(uuidHi)
buf
}

val totalSize =
headerBuffer.length +
lengthWithLength(nameBuffer) + {
Expand Down Expand Up @@ -69,4 +71,23 @@ class TastyPickler {
var addrOfSym: Symbol => Option[Addr] = (_ => None)

val treePkl = new TreePickler(this)

/** Returns a non-cryptographic 64-bit hash of the array.
*
* from https://en.wikipedia.org/wiki/PJW_hash_function#Implementation
*/
private def pjwHash64(data: Array[Byte]): Long = {
var h = 0L
var high = 0L
var i = 0
while (i < data.length) {
h = (h << 4) + data(i)
high = h & 0xF0000000L
if (high != 0)
h ^= high >> 24
h &= ~high
i += 1
}
h
}
}
135 changes: 111 additions & 24 deletions compiler/test/dotty/tools/dotc/CompilationTests.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -207,30 +207,6 @@ class CompilationTests extends ParallelTesting {
compileDir("../library/src",
allowDeepSubtypes.and("-Ycheck-reentrant", "-strict", "-priorityclasspath", defaultOutputDir))

def sources(paths: JStream[Path], excludedFiles: List[String] = Nil): List[String] =
paths.iterator().asScala
.filter(path =>
(path.toString.endsWith(".scala") || path.toString.endsWith(".java"))
&& !excludedFiles.contains(path.getFileName.toString))
.map(_.toString).toList

val compilerDir = Paths.get("../compiler/src")
val compilerSources = sources(Files.walk(compilerDir))

val backendDir = Paths.get("../scala-backend/src/compiler/scala/tools/nsc/backend")
val backendJvmDir = Paths.get("../scala-backend/src/compiler/scala/tools/nsc/backend/jvm")

// NOTE: Keep these exclusions synchronized with the ones in the sbt build (Build.scala)
val backendExcluded =
List("JavaPlatform.scala", "Platform.scala", "ScalaPrimitives.scala")
val backendJvmExcluded =
List("BCodeICodeCommon.scala", "GenASM.scala", "GenBCode.scala", "ScalacBackendInterface.scala")

val backendSources =
sources(Files.list(backendDir), excludedFiles = backendExcluded)
val backendJvmSources =
sources(Files.list(backendJvmDir), excludedFiles = backendJvmExcluded)

def dotty1 = {
compileList(
"dotty1",
Expand Down Expand Up @@ -261,6 +237,117 @@ class CompilationTests extends ParallelTesting {
} :: Nil
}.map(_.checkCompile()).foreach(_.delete())
}

@Test def bytecodeIdemporency: Unit = {
var failed = 0
var total = 0
val blacklisted = Set(
// Bridges on collections in different order. Second one in scala2 order.
"pos/Map/scala/collection/immutable/Map",
"pos/Map/scala/collection/immutable/AbstractMap",
"pos/t1203a/NodeSeq",
"pos/i2345/Whatever"
)
def checkIdempotency(): Unit = {
val groupedBytecodeFiles: List[(Path, Path, Path, Path)] = {
val bytecodeFiles = {
def bytecodeFiles(paths: JStream[Path]): List[Path] = {
def isBytecode(file: String) = file.endsWith(".class") || file.endsWith(".tasty")
paths.iterator.asScala.filter(path => isBytecode(path.toString)).toList
}
val compilerDir1 = Paths.get("../out/idempotency1")
val compilerDir2 = Paths.get("../out/idempotency2")
bytecodeFiles(Files.walk(compilerDir1)) ++ bytecodeFiles(Files.walk(compilerDir2))
}
val groups = bytecodeFiles.groupBy(f => f.toString.substring("../out/idempotencyN/".length, f.toString.length - 6))
groups.filterNot(x => blacklisted(x._1)).valuesIterator.flatMap { g =>
def pred(f: Path, i: Int, isTasty: Boolean) =
f.toString.contains("idempotency" + i) && f.toString.endsWith(if (isTasty) ".tasty" else ".class")
val class1 = g.find(f => pred(f, 1, isTasty = false))
val class2 = g.find(f => pred(f, 2, isTasty = false))
val tasty1 = g.find(f => pred(f, 1, isTasty = true))
val tasty2 = g.find(f => pred(f, 2, isTasty = true))
assert(class1.isDefined, "Could not find class in idempotency1 for " + class2)
assert(class2.isDefined, "Could not find class in idempotency2 for " + class1)
if (tasty1.isEmpty || tasty2.isEmpty) Nil
else List(Tuple4(class1.get, tasty1.get, class2.get, tasty2.get))
}.toList
}

for ((class1, tasty1, class2, tasty2) <- groupedBytecodeFiles) {
total += 1
val bytes1 = Files.readAllBytes(class1)
val bytes2 = Files.readAllBytes(class2)
if (!java.util.Arrays.equals(bytes1, bytes2)) {
failed += 1
val tastyBytes1 = Files.readAllBytes(tasty1)
val tastyBytes2 = Files.readAllBytes(tasty2)
if (java.util.Arrays.equals(tastyBytes1, tastyBytes2))
println(s"Idempotency test failed between $class1 and $class1 (same tasty)")
else
println(s"Idempotency test failed between $tasty1 and $tasty2")
/* Dump bytes to console, could be useful if issue only appears in CI.
* Create the .class locally with Files.write(path, Array[Byte](...)) with the printed array
*/
// println(bytes1.mkString("Array[Byte](", ",", ")"))
// println(bytes2.mkString("Array[Byte](", ",", ")"))
}
}
}

val opt = defaultOptions.and("-YemitTasty")

def idempotency1() = {
compileList("dotty1", compilerSources ++ backendSources ++ backendJvmSources, opt) +
compileFilesInDir("../tests/pos", opt)
}
def idempotency2() = {
compileList("dotty1", compilerSources ++ backendSources ++ backendJvmSources, opt) +
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It would be interesting to combine this with a bootstrap test: Compile the compiler, then compile the compiler with the compiled compiler, and check equality.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That will be interesting, but I would add it as a second step in another PR.

compileFilesInDir("../tests/pos", opt)
}

val tests = (idempotency1() + idempotency2()).keepOutput.checkCompile()

assert(new java.io.File("../out/idempotency1/").exists)
assert(new java.io.File("../out/idempotency2/").exists)

val t0 = System.currentTimeMillis()
checkIdempotency()
println(s"checked bytecode idempotency (${(System.currentTimeMillis() - t0) / 1000.0} sec)")

tests.delete()

assert(failed == 0, s"Failed $failed idempotency checks (out of $total)")
}


private val (compilerSources, backendSources, backendJvmSources) = {
def sources(paths: JStream[Path], excludedFiles: List[String] = Nil): List[String] =
paths.iterator().asScala
.filter(path =>
(path.toString.endsWith(".scala") || path.toString.endsWith(".java"))
&& !excludedFiles.contains(path.getFileName.toString))
.map(_.toString).toList

val compilerDir = Paths.get("../compiler/src")
val compilerSources0 = sources(Files.walk(compilerDir))

val backendDir = Paths.get("../scala-backend/src/compiler/scala/tools/nsc/backend")
val backendJvmDir = Paths.get("../scala-backend/src/compiler/scala/tools/nsc/backend/jvm")

// NOTE: Keep these exclusions synchronized with the ones in the sbt build (Build.scala)
val backendExcluded =
List("JavaPlatform.scala", "Platform.scala", "ScalaPrimitives.scala")
val backendJvmExcluded =
List("BCodeICodeCommon.scala", "GenASM.scala", "GenBCode.scala", "ScalacBackendInterface.scala")

val backendSources0 =
sources(Files.list(backendDir), excludedFiles = backendExcluded)
val backendJvmSources0 =
sources(Files.list(backendJvmDir), excludedFiles = backendJvmExcluded)

(compilerSources0, backendSources0, backendJvmSources0)
}
}

object CompilationTests {
Expand Down