new TASTy header format - add experimental version and tooling version fields

compiler/src/dotty/tools/dotc/core/tasty/TastyPickler.scala

@@ -12,6 +12,15 @@ import core.Symbols.{Symbol, ClassSymbol}
 import ast.tpd
 import Decorators._
 
+object TastyPickler {
+
+  private val versionStringBytes = {
+    val compilerString = s"Scala compiler ${config.Properties.versionString}"
+    compilerString.getBytes(java.nio.charset.StandardCharsets.UTF_8)
+  }
+
+}
+
 class TastyPickler(val rootCls: ClassSymbol) {
 
   private val sections = new mutable.ArrayBuffer[(NameRef, TastyBuffer)]
@@ -37,10 +46,13 @@ class TastyPickler(val rootCls: ClassSymbol) {
     val uuidHi: Long = otherSectionHashes.fold(0L)(_ ^ _)
 
     val headerBuffer = {
-      val buf = new TastyBuffer(header.length + 24)
+      val buf = new TastyBuffer(header.length + TastyPickler.versionStringBytes.length + 32)
       for (ch <- header) buf.writeByte(ch.toByte)
       buf.writeNat(MajorVersion)
       buf.writeNat(MinorVersion)
+      buf.writeNat(ExperimentalVersion)
+      buf.writeNat(TastyPickler.versionStringBytes.length)
+      buf.writeBytes(TastyPickler.versionStringBytes, TastyPickler.versionStringBytes.length)
       buf.writeUncompressedLong(uuidLow)
       buf.writeUncompressedLong(uuidHi)
       buf

compiler/test/dotty/tools/dotc/CompilationTests.scala

@@ -247,7 +247,7 @@ class CompilationTests {
           //  "-source", "3.1",  // TODO: re-enable once we allow : @unchecked in pattern definitions. Right now, lots of narrowing pattern definitions fail.
           ))(libGroup)
 
-    val tastyCoreSources = sources(Paths.get("tasty/src")) ++ sources(Paths.get("tasty/src-bootstrapped"))
+    val tastyCoreSources = sources(Paths.get("tasty/src"))
     val tastyCore = compileList("tastyCore", tastyCoreSources, opt)(tastyCoreGroup)
 
     val compilerSources = sources(Paths.get("compiler/src")) ++ sources(Paths.get("compiler/src-bootstrapped"))

tasty/src/dotty/tools/tasty/TastyFormat.scala

@@ -19,10 +19,11 @@ Micro-syntax:
 
 Macro-format:
 
-  File          = Header majorVersion_Nat minorVersion_Nat UUID
+  File          = Header majorVersion_Nat minorVersion_Nat experimentalVersion_Nat VersionString UUID
                   nameTable_Length Name* Section*
   Header        = 0x5CA1AB1F
   UUID          = Byte*16                 -- random UUID
+  VersionString = Length UTF8-CodePoint*  -- string that represents the compiler that produced the TASTy
 
   Section       = NameRef Length Bytes
   Length        = Nat                     -- length of rest of entry in bytes
@@ -263,8 +264,25 @@ Standard Section: "Comments" Comment*
 object TastyFormat {
 
   final val header: Array[Int] = Array(0x5C, 0xA1, 0xAB, 0x1F)
-  val MajorVersion: Int = 27
-  val MinorVersion: Int = 0
+
+  /** Each increment of the MajorVersion begins a new series of backward compatible TASTy versions
+   *  - The MajorVersion is and will always be the first value in a TASTy document after the header bytes.
+   *  - A TASTy document can then be further parsed if and only if the MajorVersion read
+   *    from a TASTy file is equal to or less than this value.
+   *  - i.e. `MajorVersion` and `header` are the only stable parts of TASTy that can be used to decide how to parse
+   *    the rest of the document.
+   */
+  final val MajorVersion: Int = 28
+
+  /** The last minor version to break forward compatability */
+  final val MinorVersion: Int = 0
+
+  /** A transitionary marker:
+   *  - 0 means that `MinorVersion` is from a final release of TASTy
+   *  - A positive number means TASTy was produced by an experimental compiler, released in between
+   *    increasing `MinorVersion`.
+   */
+  final val ExperimentalVersion: Int = 1
 
   final val ASTsSection = "ASTs"
   final val PositionsSection = "Positions"

tasty/src/dotty/tools/tasty/TastyHeaderUnpickler.scala

@@ -0,0 +1,136 @@
+package dotty.tools.tasty
+
+import java.util.UUID
+
+import TastyFormat.{MajorVersion, MinorVersion, ExperimentalVersion, header}
+
+/**
+ * The Tasty Header consists of four fields:
+ * - uuid
+ *   - contains a hash of the sections of the TASTy file
+ * - majorVersion
+ *   - matching the TASTy format version that last broke backwards compatibility
+ * - minorVersion
+ *   - matching the TASTy format version that last broke forward compatibility
+ * - experimentalVersion
+ *   - 0 for final compiler version
+ *   - positive for between minor versions and forward compatibility
+ *     is broken since the previous stable version.
+ * - toolingVersion
+ *   - arbitrary string representing the tooling that produced the TASTy
+ */
+sealed abstract case class TastyHeader(
+  uuid: UUID,
+  majorVersion: Int,
+  minorVersion: Int,
+  experimentalVersion: Int,
+  toolingVersion: String // String could lead to a lot of duplication, perhaps cache headers?
+)
+
+class TastyHeaderUnpickler(reader: TastyReader) {
+  import TastyHeaderUnpickler._
+  import reader._
+
+  def this(bytes: Array[Byte]) = this(new TastyReader(bytes))
+
+  /** reads and verifies the TASTy version, extracting the UUID */
+  def readHeader(): UUID =
+    readFullHeader().uuid
+
+  /** reads and verifies the TASTy version, extracting the whole header */
+  def readFullHeader(): TastyHeader = {
+
+    for (i <- 0 until header.length)
+      check(readByte() == header(i), "not a TASTy file")
+    val fileMajor = readNat()
+    if (fileMajor <= 27) { // old behavior before `tasty-core` 3.0.0-M4
+      val fileMinor = readNat()
+      val signature = signatureString(expectedMinor, fileMajor, fileMinor.toString)
+      throw new UnpickleException(signature + backIncompatAddendum + myAddendum)
+    }
+    else {
+      val fileMinor = readNat()
+      val fileExperimental = readNat()
+      val toolingVersion = {
+        val length = readNat()
+        val start = currentAddr
+        val end = start + length
+        goto(end)
+        new String(bytes, start.index, length) // NOTE: new string object here (should probably cache as it leaks)
+      }
+
+      val validVersion = ( // inlined from `dotty.tools.tasty.TastyVersionFormatTest.TastyVersion.<:<`
+        fileMajor == MajorVersion && (
+          if (fileExperimental == ExperimentalVersion) {
+            if (ExperimentalVersion == 0) {
+              fileMinor <= MinorVersion
+            }
+            else {
+              fileMinor == MinorVersion
+            }
+          }
+          else {
+            fileExperimental == 0 && fileMinor < MinorVersion
+          }
+        )
+      )
+      check(validVersion, {
+        val foundMinor = showMinorVersion(fileMinor, fileExperimental)
+        val producedByAddendum = s"\nThe TASTy file was produced by $toolingVersion.$myAddendum"
+        val signature = signatureString(expectedMinor, fileMajor, foundMinor)
+        if (fileExperimental != 0)
+          signature + unstableAddendum + producedByAddendum
+        else if (fileMajor < MajorVersion)
+          signature + backIncompatAddendum + producedByAddendum
+        else
+          signature + forwardIncompatAddendum + producedByAddendum
+      }
+      )
+      val uuid = new UUID(readUncompressedLong(), readUncompressedLong())
+      new TastyHeader(uuid, fileMajor, fileMinor, fileExperimental, toolingVersion) {}
+    }
+  }
+
+  def isAtEnd: Boolean = reader.isAtEnd
+
+  private def check(cond: Boolean, msg: => String): Unit = {
+    if (!cond) throw new UnpickleException(msg)
+  }
+}
+
+object TastyHeaderUnpickler {
+
+  private def showMinorVersion(min: Int, exp: Int) = {
+    val expStr = if (exp == 0) "" else s" [unstable release: $exp]"
+    s"$min$expStr"
+  }
+
+  private def expectedMinor = showMinorVersion(MinorVersion, ExperimentalVersion)
+
+  private def myAddendum = {
+    if (ExperimentalVersion > 0)
+      "\nNote that your tooling is currently using an unstable TASTy version."
+    else
+      ""
+  }
+
+  private def signatureString(minorVersion: String, fileMajor: Int, fileMinor: String) = (
+    s"""TASTy signature has wrong version.
+      | expected: {majorVersion: $MajorVersion, minorVersion: $minorVersion}
+      | found   : {majorVersion: $fileMajor, minorVersion: $fileMinor}
+      |
+      |""".stripMargin
+  )
+
+  private def unstableAddendum =
+    """This TASTy file was produced by an unstable release.
+      |To read this TASTy file, your tooling must be at the same version.""".stripMargin
+
+  private def backIncompatAddendum =
+    """This TASTy file was produced by a backwards incompatible release.
+      |Please recompile this TASTy with a later version.""".stripMargin
+
+  private def forwardIncompatAddendum =
+    """This TASTy file was produced by a forwards incompatible release.
+      |To read this TASTy file, please upgrade your tooling.""".stripMargin
+}

tasty/test/dotty/tools/tasty/TastyHeaderUnpicklerTest.scala

@@ -0,0 +1,84 @@
+package dotty.tools.tasty
+
+import org.junit.Assert._
+import org.junit.{Test, Ignore}
+
+import TastyFormat._
+import TastyBuffer._
+
+@Ignore // comment if you want to experiment with error messages
+class TastyHeaderUnpicklerTest {
+
+  import TastyHeaderUnpicklerTest._
+
+  @Test def vanilla: Unit = {
+    runTest(MajorVersion, MinorVersion, ExperimentalVersion, "Scala compiler 3.0.0-M4-bin-SNAPSHOT-git-12345")
+  }
+
+  @Test def failBumpExperimental: Unit = {
+    (runTest(MajorVersion, MinorVersion, ExperimentalVersion + 1, "Scala compiler 3.0.0-M4-bin-SNAPSHOT-git-12345"))
+  }
+
+  @Test def failBumpMinor: Unit = {
+    (runTest(MajorVersion, MinorVersion + 1, ExperimentalVersion, "Scala compiler 3.1.0-RC1"))
+  }
+
+  @Test def failBumpMajor: Unit = {
+    (runTest(MajorVersion + 1, MinorVersion, ExperimentalVersion, "Scala compiler 4.0.0-M1"))
+  }
+
+  @Test def failBumpMajorFinal: Unit = {
+    (runTest(MajorVersion + 1, MinorVersion, 0, "Scala compiler 4.0.0"))
+  }
+
+  @Test def okSubtractExperimental: Unit = {
+    (runTest(MajorVersion, MinorVersion, ExperimentalVersion - 1, "Scala compiler 3.0.0"))
+  }
+
+  @Test def okSubtractMinor: Unit = {
+    (runTest(MajorVersion, MinorVersion - 1, ExperimentalVersion, "Scala compiler 3.0.0-M4-bin-SNAPSHOT-git-12345"))
+  }
+
+  @Test def failSubtractMajor: Unit = {
+    (runTest(MajorVersion - 1, MinorVersion, ExperimentalVersion, "Scala compiler 3.0.0-M4-bin-SNAPSHOT-git-12345"))
+  }
+
+}
+
+object TastyHeaderUnpicklerTest {
+
+
+  def fillHeader(maj: Int, min: Int, exp: Int, compiler: String): TastyBuffer = {
+    val compilerBytes = compiler.getBytes(java.nio.charset.StandardCharsets.UTF_8)
+    val buf = new TastyBuffer(header.length + 32 + compilerBytes.length)
+    for (ch <- header) buf.writeByte(ch.toByte)
+    buf.writeNat(maj)
+    buf.writeNat(min)
+    buf.writeNat(exp)
+    buf.writeNat(compilerBytes.length)
+    buf.writeBytes(compilerBytes, compilerBytes.length)
+    buf.writeUncompressedLong(237478l)
+    buf.writeUncompressedLong(324789l)
+    buf
+  }
+
+  def runTest(maj: Int, min: Int, exp: Int, compiler: String): Unit = {
+    val headerBuffer = fillHeader(maj, min, exp, compiler)
+    val bs = headerBuffer.bytes.clone
+
+    val hr = new TastyHeaderUnpickler(bs)
+
+    hr.readFullHeader()
+  }
+
+  def expectUnpickleError(op: => Unit) = {
+    try {
+      op
+      fail()
+    }
+    catch {
+      case err: UnpickleException => ()
+    }
+  }
+
+}