diff --git a/compiler/src/dotty/tools/dotc/config/Properties.scala b/compiler/src/dotty/tools/dotc/config/Properties.scala index ec22bad2f788..4e084e050cf6 100644 --- a/compiler/src/dotty/tools/dotc/config/Properties.scala +++ b/compiler/src/dotty/tools/dotc/config/Properties.scala @@ -64,18 +64,24 @@ trait PropertiesTrait { */ def versionNumberString: String = scalaPropOrEmpty("version.number") - /** The version number of the jar this was loaded from plus "version " prefix, - * or "version (unknown)" if it cannot be determined. + /** The version number of the jar this was loaded from, + * or `"(unknown)"` if it cannot be determined. */ - val versionString: String = { + val simpleVersionString: String = { val v = scalaPropOrElse("version.number", "(unknown)") - "version " + scalaPropOrElse("version.number", "(unknown)") + { + v + ( if (v.contains("SNAPSHOT") || v.contains("NIGHTLY")) "-git-" + scalaPropOrElse("git.hash", "(unknown)") - else "" - } + else + "" + ) } + /** The version number of the jar this was loaded from plus `"version "` prefix, + * or `"version (unknown)"` if it cannot be determined. + */ + val versionString: String = "version " + simpleVersionString + /** Whether the current version of compiler is experimental * * 1. Snapshot and nightly releases are experimental. diff --git a/compiler/src/dotty/tools/dotc/core/tasty/TastyPickler.scala b/compiler/src/dotty/tools/dotc/core/tasty/TastyPickler.scala index 247d8c8c10cf..3aeb7e6f35c9 100644 --- a/compiler/src/dotty/tools/dotc/core/tasty/TastyPickler.scala +++ b/compiler/src/dotty/tools/dotc/core/tasty/TastyPickler.scala @@ -12,6 +12,15 @@ import core.Symbols.{Symbol, ClassSymbol} import ast.tpd import Decorators._ +object TastyPickler { + + private val versionStringBytes = { + val compilerString = s"Scala ${config.Properties.simpleVersionString}" + compilerString.getBytes(java.nio.charset.StandardCharsets.UTF_8) + } + +} + class TastyPickler(val rootCls: ClassSymbol) { private val sections = new mutable.ArrayBuffer[(NameRef, TastyBuffer)] @@ -37,10 +46,13 @@ class TastyPickler(val rootCls: ClassSymbol) { val uuidHi: Long = otherSectionHashes.fold(0L)(_ ^ _) val headerBuffer = { - val buf = new TastyBuffer(header.length + 24) + val buf = new TastyBuffer(header.length + TastyPickler.versionStringBytes.length + 32) for (ch <- header) buf.writeByte(ch.toByte) buf.writeNat(MajorVersion) buf.writeNat(MinorVersion) + buf.writeNat(ExperimentalVersion) + buf.writeNat(TastyPickler.versionStringBytes.length) + buf.writeBytes(TastyPickler.versionStringBytes, TastyPickler.versionStringBytes.length) buf.writeUncompressedLong(uuidLow) buf.writeUncompressedLong(uuidHi) buf diff --git a/compiler/test/dotty/tools/dotc/CompilationTests.scala b/compiler/test/dotty/tools/dotc/CompilationTests.scala index f2df0f534b48..6ebac535557a 100644 --- a/compiler/test/dotty/tools/dotc/CompilationTests.scala +++ b/compiler/test/dotty/tools/dotc/CompilationTests.scala @@ -247,7 +247,7 @@ class CompilationTests { // "-source", "3.1", // TODO: re-enable once we allow : @unchecked in pattern definitions. Right now, lots of narrowing pattern definitions fail. ))(libGroup) - val tastyCoreSources = sources(Paths.get("tasty/src")) ++ sources(Paths.get("tasty/src-bootstrapped")) + val tastyCoreSources = sources(Paths.get("tasty/src")) val tastyCore = compileList("tastyCore", tastyCoreSources, opt)(tastyCoreGroup) val compilerSources = sources(Paths.get("compiler/src")) ++ sources(Paths.get("compiler/src-bootstrapped")) diff --git a/tasty/src-bootstrapped/dotty/tools/tasty/TastyHeaderUnpickler.scala b/tasty/src-bootstrapped/dotty/tools/tasty/TastyHeaderUnpickler.scala deleted file mode 100644 index 766df348becf..000000000000 --- a/tasty/src-bootstrapped/dotty/tools/tasty/TastyHeaderUnpickler.scala +++ /dev/null @@ -1,46 +0,0 @@ -package dotty.tools.tasty - -import java.util.UUID - -import TastyFormat.{MajorVersion, MinorVersion, header} - -sealed abstract case class TastyHeader(uuid: UUID, majorVersion: Int, minorVersion: Int) - -class TastyHeaderUnpickler(reader: TastyReader) { - import reader._ - - def this(bytes: Array[Byte]) = this(new TastyReader(bytes)) - - /** delegates to `readFullHeader`, extracting the UUID */ - def readHeader(): UUID = - readFullHeader().uuid - - /**Reads the header of a Tasty File, the returned header, `h`, - * has the following properties: - * - `h.majorVersion == TastyFormat.MajorVersion` - * - `0 <= h.minorVersion <= TastyFormat.MinorVersion` - */ - def readFullHeader(): TastyHeader = { - for (i <- 0 until header.length) - check(readByte() == header(i), "not a TASTy file") - val majorVersion = readNat() - val minorVersion = readNat() - val validVersion = ( - majorVersion == MajorVersion - && minorVersion >= 0 && minorVersion <= MinorVersion - ) - check(validVersion, - s"""TASTy signature has wrong version. - | expected: $MajorVersion.$MinorVersion - | found : ${majorVersion}.${minorVersion}""".stripMargin - ) - val uuid = new UUID(readUncompressedLong(), readUncompressedLong()) - new TastyHeader(uuid, majorVersion, minorVersion) {} - } - - def isAtEnd: Boolean = reader.isAtEnd - - private def check(cond: Boolean, msg: => String): Unit = { - if (!cond) throw new UnpickleException(msg) - } -} diff --git a/tasty/src-non-bootstrapped/dotty/tools/tasty/TastyHeaderUnpickler.scala b/tasty/src-non-bootstrapped/dotty/tools/tasty/TastyHeaderUnpickler.scala deleted file mode 100644 index 05e421750411..000000000000 --- a/tasty/src-non-bootstrapped/dotty/tools/tasty/TastyHeaderUnpickler.scala +++ /dev/null @@ -1,29 +0,0 @@ -package dotty.tools.tasty - -import java.util.UUID - -import TastyFormat.{MajorVersion, MinorVersion, header} - -class TastyHeaderUnpickler(reader: TastyReader) { - import reader._ - - def this(bytes: Array[Byte]) = this(new TastyReader(bytes)) - - def readHeader(): UUID = { - for (i <- 0 until header.length) - check(readByte() == header(i), "not a TASTy file") - val major = readNat() - val minor = readNat() - check(major == MajorVersion && minor <= MinorVersion, - s"""TASTy signature has wrong version. - | expected: $MajorVersion.$MinorVersion - | found : $major.$minor""".stripMargin) - new UUID(readUncompressedLong(), readUncompressedLong()) - } - - def isAtEnd: Boolean = reader.isAtEnd - - private def check(cond: Boolean, msg: => String): Unit = { - if (!cond) throw new UnpickleException(msg) - } -} diff --git a/tasty/src/dotty/tools/tasty/TastyFormat.scala b/tasty/src/dotty/tools/tasty/TastyFormat.scala index 4e22b7519d12..f5969169353a 100644 --- a/tasty/src/dotty/tools/tasty/TastyFormat.scala +++ b/tasty/src/dotty/tools/tasty/TastyFormat.scala @@ -19,10 +19,11 @@ Micro-syntax: Macro-format: - File = Header majorVersion_Nat minorVersion_Nat UUID + File = Header majorVersion_Nat minorVersion_Nat experimentalVersion_Nat VersionString UUID nameTable_Length Name* Section* Header = 0x5CA1AB1F UUID = Byte*16 -- random UUID + VersionString = Length UTF8-CodePoint* -- string that represents the compiler that produced the TASTy Section = NameRef Length Bytes Length = Nat -- length of rest of entry in bytes @@ -262,9 +263,100 @@ Standard Section: "Comments" Comment* object TastyFormat { + /** The first four bytes of a TASTy file, followed by four values: + * - `MajorVersion: Int` - see definition in `TastyFormat` + * - `MinorVersion: Int` - see definition in `TastyFormat` + * - `ExperimentalVersion: Int` - see definition in `TastyFormat` + * - `ToolingVersion: String` - arbitrary length string representing the tool that produced the TASTy. + */ final val header: Array[Int] = Array(0x5C, 0xA1, 0xAB, 0x1F) - val MajorVersion: Int = 27 - val MinorVersion: Int = 0 + + /**Natural number. Each increment of the `MajorVersion` begins a + * new series of backward compatible TASTy versions. + * + * A TASTy file in either the preceeding or succeeding series is + * incompatible with the current value. + */ + final val MajorVersion: Int = 28 + + /**Natural number. Each increment of the `MinorVersion`, within + * a series declared by the `MajorVersion`, breaks forward + * compatibility, but remains backwards compatible, with all + * preceeding `MinorVersion`. + */ + final val MinorVersion: Int = 0 + + /**Natural Number. The `ExperimentalVersion` allows for + * experimentation with changes to TASTy without committing + * to any guarantees of compatibility. + * + * A zero value indicates that the TASTy version is from a + * stable, final release. + * + * A strictly positive value indicates that the TASTy + * version is experimental. An experimental TASTy file + * can only be read by a tool with the same version. + * However, tooling with an experimental TASTy version + * is able to read final TASTy documents if the file's + * `MinorVersion` is strictly less than the current value. + */ + final val ExperimentalVersion: Int = 1 + + /**This method implements a binary relation (`<:<`) between two TASTy versions. + * We label the lhs `file` and rhs `compiler`. + * if `file <:< compiler` then the TASTy file is valid to be read. + * + * TASTy versions have a partial order, + * for example `a <:< b` and `b <:< a` are both false if `a` and `b` have different major versions. + * + * We follow the given algorithm: + * ``` + * if file.major != compiler.major then + * return incompatible + * if compiler.experimental == 0 then + * if file.experimental != 0 then + * return incompatible + * if file.minor > compiler.minor then + * return incompatible + * else + * return compatible + * else invariant[compiler.experimental != 0] + * if file.experimental == compiler.experimental then + * if file.minor == compiler.minor then + * return compatible (all fields equal) + * else + * return incompatible + * else if file.experimental == 0, + * if file.minor < compiler.minor then + * return compatible (an experimental version can read a previous released version) + * else + * return incompatible (an experimental version cannot read its own minor version or any later version) + * else invariant[file.experimental is non-0 and different than compiler.experimental] + * return incompatible + * ``` + */ + def isVersionCompatible( + fileMajor: Int, + fileMinor: Int, + fileExperimental: Int, + compilerMajor: Int, + compilerMinor: Int, + compilerExperimental: Int + ): Boolean = ( + fileMajor == compilerMajor && ( + if (fileExperimental == compilerExperimental) { + if (compilerExperimental == 0) { + fileMinor <= compilerMinor + } + else { + fileMinor == compilerMinor + } + } + else { + fileExperimental == 0 && fileMinor < compilerMinor + } + ) + ) final val ASTsSection = "ASTs" final val PositionsSection = "Positions" diff --git a/tasty/src/dotty/tools/tasty/TastyHeaderUnpickler.scala b/tasty/src/dotty/tools/tasty/TastyHeaderUnpickler.scala new file mode 100644 index 000000000000..c0ed5dbd58fa --- /dev/null +++ b/tasty/src/dotty/tools/tasty/TastyHeaderUnpickler.scala @@ -0,0 +1,128 @@ +package dotty.tools.tasty + +import java.util.UUID + +import TastyFormat.{MajorVersion, MinorVersion, ExperimentalVersion, header} + +/** + * The Tasty Header consists of four fields: + * - uuid + * - contains a hash of the sections of the TASTy file + * - majorVersion + * - matching the TASTy format version that last broke backwards compatibility + * - minorVersion + * - matching the TASTy format version that last broke forward compatibility + * - experimentalVersion + * - 0 for final compiler version + * - positive for between minor versions and forward compatibility + * is broken since the previous stable version. + * - toolingVersion + * - arbitrary string representing the tooling that produced the TASTy + */ +sealed abstract case class TastyHeader( + uuid: UUID, + majorVersion: Int, + minorVersion: Int, + experimentalVersion: Int, + toolingVersion: String +) + +class TastyHeaderUnpickler(reader: TastyReader) { + import TastyHeaderUnpickler._ + import reader._ + + def this(bytes: Array[Byte]) = this(new TastyReader(bytes)) + + /** reads and verifies the TASTy version, extracting the UUID */ + def readHeader(): UUID = + readFullHeader().uuid + + /** reads and verifies the TASTy version, extracting the whole header */ + def readFullHeader(): TastyHeader = { + + for (i <- 0 until header.length) + check(readByte() == header(i), "not a TASTy file") + val fileMajor = readNat() + if (fileMajor <= 27) { // old behavior before `tasty-core` 3.0.0-M4 + val fileMinor = readNat() + val signature = signatureString(fileMajor, fileMinor, 0) + throw new UnpickleException(signature + backIncompatAddendum + toolingAddendum) + } + else { + val fileMinor = readNat() + val fileExperimental = readNat() + val toolingVersion = { + val length = readNat() + val start = currentAddr + val end = start + length + goto(end) + new String(bytes, start.index, length) + } + + val validVersion = TastyFormat.isVersionCompatible( + fileMajor = fileMajor, + fileMinor = fileMinor, + fileExperimental = fileExperimental, + compilerMajor = MajorVersion, + compilerMinor = MinorVersion, + compilerExperimental = ExperimentalVersion + ) + + check(validVersion, { + val signature = signatureString(fileMajor, fileMinor, fileExperimental) + val producedByAddendum = s"\nThe TASTy file was produced by $toolingVersion.$toolingAddendum" + val msg = ( + if (fileExperimental != 0) unstableAddendum + else if (fileMajor < MajorVersion) backIncompatAddendum + else forwardIncompatAddendum + ) + signature + msg + producedByAddendum + }) + + val uuid = new UUID(readUncompressedLong(), readUncompressedLong()) + new TastyHeader(uuid, fileMajor, fileMinor, fileExperimental, toolingVersion) {} + } + } + + def isAtEnd: Boolean = reader.isAtEnd + + private def check(cond: Boolean, msg: => String): Unit = { + if (!cond) throw new UnpickleException(msg) + } +} + +object TastyHeaderUnpickler { + + private def toolingAddendum = ( + if (ExperimentalVersion > 0) + "\nNote that your tooling is currently using an unstable TASTy version." + else + "" + ) + + private def signatureString(fileMajor: Int, fileMinor: Int, fileExperimental: Int) = { + def showMinorVersion(min: Int, exp: Int) = { + val expStr = if (exp == 0) "" else s" [unstable release: $exp]" + s"$min$expStr" + } + val minorVersion = showMinorVersion(MinorVersion, ExperimentalVersion) + val fileMinorVersion = showMinorVersion(fileMinor, fileExperimental) + s"""TASTy signature has wrong version. + | expected: {majorVersion: $MajorVersion, minorVersion: $minorVersion} + | found : {majorVersion: $fileMajor, minorVersion: $fileMinorVersion} + | + |""".stripMargin + } + + private def unstableAddendum = + """This TASTy file was produced by an unstable release. + |To read this TASTy file, your tooling must be at the same version.""".stripMargin + + private def backIncompatAddendum = + """This TASTy file was produced by an earlier release that is not supported anymore. + |Please recompile this TASTy with a later version.""".stripMargin + + private def forwardIncompatAddendum = + """This TASTy file was produced by a more recent, forwards incompatible release. + |To read this TASTy file, please upgrade your tooling.""".stripMargin +} diff --git a/tasty/test/dotty/tools/tasty/TastyHeaderUnpicklerTest.scala b/tasty/test/dotty/tools/tasty/TastyHeaderUnpicklerTest.scala new file mode 100644 index 000000000000..b9f2aff3f564 --- /dev/null +++ b/tasty/test/dotty/tools/tasty/TastyHeaderUnpicklerTest.scala @@ -0,0 +1,84 @@ +package dotty.tools.tasty + +import org.junit.Assert._ +import org.junit.{Test, Ignore} + +import TastyFormat._ +import TastyBuffer._ + +@Ignore // comment if you want to experiment with error messages +class TastyHeaderUnpicklerTest { + + import TastyHeaderUnpicklerTest._ + + @Test def vanilla: Unit = { + runTest(MajorVersion, MinorVersion, ExperimentalVersion, "Scala 3.0.0-M4-bin-SNAPSHOT-git-12345") + } + + @Test def failBumpExperimental: Unit = { + (runTest(MajorVersion, MinorVersion, ExperimentalVersion + 1, "Scala 3.0.0-M4-bin-SNAPSHOT-git-12345")) + } + + @Test def failBumpMinor: Unit = { + (runTest(MajorVersion, MinorVersion + 1, ExperimentalVersion, "Scala 3.1.0-RC1")) + } + + @Test def failBumpMajor: Unit = { + (runTest(MajorVersion + 1, MinorVersion, ExperimentalVersion, "Scala 4.0.0-M1")) + } + + @Test def failBumpMajorFinal: Unit = { + (runTest(MajorVersion + 1, MinorVersion, 0, "Scala 4.0.0")) + } + + @Test def okSubtractExperimental: Unit = { + (runTest(MajorVersion, MinorVersion, ExperimentalVersion - 1, "Scala 3.0.0")) + } + + @Test def okSubtractMinor: Unit = { + (runTest(MajorVersion, MinorVersion - 1, ExperimentalVersion, "Scala 3.0.0-M4-bin-SNAPSHOT-git-12345")) + } + + @Test def failSubtractMajor: Unit = { + (runTest(MajorVersion - 1, MinorVersion, ExperimentalVersion, "Scala 3.0.0-M4-bin-SNAPSHOT-git-12345")) + } + +} + +object TastyHeaderUnpicklerTest { + + + def fillHeader(maj: Int, min: Int, exp: Int, compiler: String): TastyBuffer = { + val compilerBytes = compiler.getBytes(java.nio.charset.StandardCharsets.UTF_8) + val buf = new TastyBuffer(header.length + 32 + compilerBytes.length) + for (ch <- header) buf.writeByte(ch.toByte) + buf.writeNat(maj) + buf.writeNat(min) + buf.writeNat(exp) + buf.writeNat(compilerBytes.length) + buf.writeBytes(compilerBytes, compilerBytes.length) + buf.writeUncompressedLong(237478l) + buf.writeUncompressedLong(324789l) + buf + } + + def runTest(maj: Int, min: Int, exp: Int, compiler: String): Unit = { + val headerBuffer = fillHeader(maj, min, exp, compiler) + val bs = headerBuffer.bytes.clone + + val hr = new TastyHeaderUnpickler(bs) + + hr.readFullHeader() + } + + def expectUnpickleError(op: => Unit) = { + try { + op + fail() + } + catch { + case err: UnpickleException => () + } + } + +} diff --git a/tasty/test/dotty/tools/tasty/TastyVersionFormatTest.scala b/tasty/test/dotty/tools/tasty/TastyVersionFormatTest.scala new file mode 100644 index 000000000000..b5cb58910e36 --- /dev/null +++ b/tasty/test/dotty/tools/tasty/TastyVersionFormatTest.scala @@ -0,0 +1,82 @@ +package dotty.tools.tasty + +import org.junit.Assert._ +import org.junit.{Test, Ignore} + +import TastyFormat._ +import TastyBuffer._ + +class TastyVersionFormatTest { + + import TastyVersionFormatTest._ + + /** aliases `TastyVersion.apply` */ + def compiler(major: Int, minor: Int, experimental: Int) = TastyVersion(major, minor, experimental) + + /** aliases `TastyVersion.apply` */ + def file(major: Int, minor: Int, experimental: Int) = TastyVersion(major, minor, experimental) + + @Test def accept_ExperimentalReadEQExperimental_EQMinor: Unit = { + assert(file(28,1,1) <:< compiler(28,1,1)) // same minor, same experimental + } + + @Test def accept_ExperimentalReadFinal_LTMinor: Unit = { + assert(file(28,0,0) <:< compiler(28,1,1)) // preceding minor + } + + @Test def accept_FinalReadFinal_LTEqualMinor: Unit = { + assert(file(28,0,0) <:< compiler(28,1,0)) // preceding minor + assert(file(28,0,0) <:< compiler(28,0,0)) // same minor + } + + /** these cases are unrelated because a final compiler can only read final tasty of <= minor version */ + @Test def reject_FinalReadFinal_GTMinor: Unit = { + assert(file(28,2,0) unrelatedTo compiler(28,1,0)) // succeeding minor + } + + /** these cases are unrelated because a final compiler can not read experimental tasty */ + @Test def reject_FinalReadExperimental: Unit = { + assert(file(28,0,1) unrelatedTo compiler(28,1,0)) // preceding minor + assert(file(28,1,1) unrelatedTo compiler(28,1,0)) // same minor + assert(file(28,2,1) unrelatedTo compiler(28,1,0)) // succeeding minor + } + + /** These cases are unrelated because an experimental compiler can only read final tasty of < minor version */ + @Test def reject_ExperimentalReadFinal_GTEqualMinor: Unit = { + assert(file(28,2,0) unrelatedTo compiler(28,1,1)) // succeeding minor + assert(file(28,1,0) unrelatedTo compiler(28,1,1)) // equal minor + } + + /**These cases are unrelated because both compiler and file are experimental, + * and with unequal experimental part. + */ + @Test def reject_ExperimentalReadNEExperimental: Unit = { + assert(file(28,1,2) unrelatedTo compiler(28,1,1)) // same minor version, succeeding experimental + assert(file(28,1,1) unrelatedTo compiler(28,1,2)) // same minor version, preceding experimental + } + + /** these cases are unrelated because the major version must be identical */ + @Test def reject_NEMajor: Unit = { + assert(file(27,0,0) unrelatedTo compiler(28,0,0)) // less than + assert(file(29,0,0) unrelatedTo compiler(28,0,0)) // greater than + } + +} + +object TastyVersionFormatTest { + + case class TastyVersion(major: Int, minor: Int, experimental: Int) { file => + def <:<(compiler: TastyVersion): Boolean = TastyFormat.isVersionCompatible( + fileMajor = file.major, + fileMinor = file.minor, + fileExperimental = file.experimental, + compilerMajor = compiler.major, + compilerMinor = compiler.minor, + compilerExperimental = compiler.experimental + ) + + /**if `file unrelated compiler` then tasty file must be rejected.*/ + def unrelatedTo(compiler: TastyVersion): Boolean = !(file <:< compiler) + } + +}