New IC: Detect changes to class annotations

Both the new and old incremental compilation (IC) analysis rely on
Kotlin class metadata to detect a change.

However, Kotlin metadata currently doesn't contain info about
annotations (KT-57919), so the IC will not be able to detect a change
to them.

With this commit, we'll fix the new IC such that it can detect a change
to class annotations by not relying only on metadata.

We currently scope this fix to the new IC (cross-module analysis) first.
We'll fix this issue for within-module analysis later.

Performance: There seems to be no performance impact from this change.
Snapshotting the 400MB ideaIC-2022.1.4/app.jar takes 4.1s before and
after this change.

Test: Added ClasspathChangesComputerTest.testChangedAnnotations
^KT-58289: Fixed

Author: hungvietnguyen

build-common/src/org/jetbrains/kotlin/incremental/IncrementalJvmCache.kt

@@ -441,7 +441,7 @@ open class IncrementalJvmCache(
         storageFile: File,
         icContext: IncrementalCompilationContext,
     ) :
-        BasicStringMap<Map<String, Any>>(storageFile, MapExternalizer(StringExternalizer, ConstantValueExternalizer), icContext) {
+        BasicStringMap<Map<String, Long>>(storageFile, MapExternalizer(StringExternalizer, LongExternalizer), icContext) {
 
         operator fun contains(className: JvmClassName): Boolean =
             className.internalName in storage
@@ -451,7 +451,7 @@ open class IncrementalJvmCache(
             val key = kotlinClassInfo.className.internalName
             val oldMap = storage[key] ?: emptyMap()
 
-            val newMap = kotlinClassInfo.constantsMap
+            val newMap = kotlinClassInfo.extraInfo.constantSnapshots
             if (newMap.isNotEmpty()) {
                 storage[key] = newMap
             } else {
@@ -489,8 +489,8 @@ open class IncrementalJvmCache(
             storage.remove(className.internalName)
         }
 
-        override fun dumpValue(value: Map<String, Any>): String =
-            value.dumpMap(Any::toString)
+        override fun dumpValue(value: Map<String, Long>): String =
+            value.dumpMap(Long::toString)
     }
 
     private inner class PackagePartMap(
@@ -592,7 +592,7 @@ open class IncrementalJvmCache(
             val key = kotlinClassInfo.className.internalName
             val oldMap = storage[key] ?: emptyMap()
 
-            val newMap = kotlinClassInfo.inlineFunctionsAndAccessorsMap
+            val newMap = kotlinClassInfo.extraInfo.inlineFunctionOrAccessorSnapshots
             if (newMap.isNotEmpty()) {
                 storage[key] = newMap
             } else {

build-common/src/org/jetbrains/kotlin/incremental/KotlinClassInfo.kt

@@ -274,8 +274,7 @@ class DelegateDataExternalizer<T>(
 
     override fun read(input: DataInput): T {
         val typeIndex = input.readByte().toInt()
-        @Suppress("UNCHECKED_CAST")
-        return typesExternalizers[typeIndex].read(input) as T
+        return typesExternalizers[typeIndex].read(input)
     }
 }
 

build-common/src/org/jetbrains/kotlin/incremental/storage/externalizers.kt

@@ -34,17 +34,13 @@ class ClassFileWithContentsProvider(
     val classFile: ClassFile,
     val contentsProvider: () -> ByteArray
 ) {
-
-    fun loadContents(): ClassFileWithContents {
-        val classContents = contentsProvider.invoke()
-        val classInfo = BasicClassInfo.compute(classContents)
-        return ClassFileWithContents(classFile, classContents, classInfo)
-    }
+    fun loadContents() = ClassFileWithContents(classFile, contentsProvider.invoke())
 }
 
 /** Information about the location of a .class file ([ClassFile]) and its contents. */
 class ClassFileWithContents(
     @Suppress("unused") val classFile: ClassFile,
-    val contents: ByteArray,
-    val classInfo: BasicClassInfo
-)
+    val contents: ByteArray
+) {
+    val classInfo: BasicClassInfo = BasicClassInfo.compute(contents)
+}

compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/classpathDiff/ClassFile.kt

@@ -246,6 +246,22 @@ object ClasspathChangesComputer {
         //     classes, and symbols in removed classes.
         incrementalJvmCache.clearCacheForRemovedClasses(changesCollector)
 
+        // IncrementalJvmCache currently doesn't use the `KotlinClassInfo.extraInfo.classSnapshotExcludingMembers` info when comparing
+        // classes, so we need to do it here.
+        // TODO(KT-58289): Ensure IncrementalJvmCache uses that info when comparing classes.
+        val currentClassSnapshotsExcludingMembers = currentClassSnapshots
+            .associate { it.classId to it.classMemberLevelSnapshot!!.extraInfo.classSnapshotExcludingMembers }
+            .filter { it.value != null }
+        val previousClassSnapshotsExcludingMembers = previousClassSnapshots
+            .associate { it.classId to it.classMemberLevelSnapshot!!.extraInfo.classSnapshotExcludingMembers }
+            .filter { it.value != null }
+        previousClassSnapshotsExcludingMembers.keys.intersect(currentClassSnapshotsExcludingMembers.keys).forEach {
+            if (previousClassSnapshotsExcludingMembers[it]!! != currentClassSnapshotsExcludingMembers[it]!!) {
+                // `areSubclassesAffected = false` as we don't need to compute impacted symbols at this step
+                changesCollector.collectSignature(fqName = it.asSingleFqName(), areSubclassesAffected = false)
+            }
+        }
+
         // Get the changes and clean up
         val dirtyData = changesCollector.getChangedSymbols(DoNothingICReporter)
         workingDir.deleteRecursively()

compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathChangesComputer.kt

@@ -8,7 +8,6 @@ package org.jetbrains.kotlin.incremental.classpathDiff
 import com.intellij.util.containers.Interner
 import com.intellij.util.io.DataExternalizer
 import org.jetbrains.kotlin.build.report.metrics.BuildPerformanceMetric
-import org.jetbrains.kotlin.incremental.ConstantValueExternalizer
 import org.jetbrains.kotlin.incremental.KotlinClassInfo
 import org.jetbrains.kotlin.incremental.storage.*
 import org.jetbrains.kotlin.load.kotlin.header.KotlinClassHeader
@@ -162,8 +161,7 @@ internal object KotlinClassInfoExternalizer : DataExternalizer<KotlinClassInfo>
         ListExternalizer(StringExternalizer).save(output, info.classHeaderData.toList())
         ListExternalizer(StringExternalizer).save(output, info.classHeaderStrings.toList())
         NullableValueExternalizer(StringExternalizer).save(output, info.multifileClassName)
-        MapExternalizer(StringExternalizer, ConstantValueExternalizer).save(output, info.constantsMap)
-        MapExternalizer(InlineFunctionOrAccessorExternalizer, LongExternalizer).save(output, info.inlineFunctionsAndAccessorsMap)
+        ExtraInfoExternalizer.save(output, info.extraInfo)
     }
 
     override fun read(input: DataInput): KotlinClassInfo {
@@ -174,8 +172,24 @@ internal object KotlinClassInfoExternalizer : DataExternalizer<KotlinClassInfo>
             classHeaderData = ListExternalizer(StringExternalizer).read(input).toTypedArray(),
             classHeaderStrings = ListExternalizer(StringExternalizer).read(input).toTypedArray(),
             multifileClassName = NullableValueExternalizer(StringExternalizer).read(input),
-            constantsMap = MapExternalizer(StringExternalizer, ConstantValueExternalizer).read(input),
-            inlineFunctionsAndAccessorsMap = MapExternalizer(InlineFunctionOrAccessorExternalizer, LongExternalizer).read(input)
+            extraInfo = ExtraInfoExternalizer.read(input)
+        )
+    }
+}
+
+internal object ExtraInfoExternalizer : DataExternalizer<KotlinClassInfo.ExtraInfo> {
+
+    override fun save(output: DataOutput, info: KotlinClassInfo.ExtraInfo) {
+        NullableValueExternalizer(LongExternalizer).save(output, info.classSnapshotExcludingMembers)
+        MapExternalizer(StringExternalizer, LongExternalizer).save(output, info.constantSnapshots)
+        MapExternalizer(InlineFunctionOrAccessorExternalizer, LongExternalizer).save(output, info.inlineFunctionOrAccessorSnapshots)
+    }
+
+    override fun read(input: DataInput): KotlinClassInfo.ExtraInfo {
+        return KotlinClassInfo.ExtraInfo(
+            classSnapshotExcludingMembers = NullableValueExternalizer(LongExternalizer).read(input),
+            constantSnapshots = MapExternalizer(StringExternalizer, LongExternalizer).read(input),
+            inlineFunctionOrAccessorSnapshots = MapExternalizer(InlineFunctionOrAccessorExternalizer, LongExternalizer).read(input)
         )
     }
 }
@@ -200,7 +214,7 @@ private object JavaClassSnapshotExternalizer : DataExternalizer<JavaClassSnapsho
     }
 }
 
-private object JavaClassMemberLevelSnapshotExternalizer : DataExternalizer<JavaClassMemberLevelSnapshot> {
+internal object JavaClassMemberLevelSnapshotExternalizer : DataExternalizer<JavaClassMemberLevelSnapshot> {
 
     override fun save(output: DataOutput, snapshot: JavaClassMemberLevelSnapshot) {
         JavaElementSnapshotExternalizer.save(output, snapshot.classAbiExcludingMembers)

compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathSnapshotSerializer.kt

@@ -9,15 +9,23 @@ import org.jetbrains.kotlin.build.report.metrics.BuildMetricsReporter
 import org.jetbrains.kotlin.build.report.metrics.BuildTime
 import org.jetbrains.kotlin.build.report.metrics.DoNothingBuildMetricsReporter
 import org.jetbrains.kotlin.build.report.metrics.measure
+import org.jetbrains.kotlin.incremental.ClassNodeSnapshotter.snapshotClass
+import org.jetbrains.kotlin.incremental.ClassNodeSnapshotter.snapshotClassExcludingMembers
+import org.jetbrains.kotlin.incremental.ClassNodeSnapshotter.snapshotField
+import org.jetbrains.kotlin.incremental.ClassNodeSnapshotter.snapshotMethod
+import org.jetbrains.kotlin.incremental.ClassNodeSnapshotter.sortClassMembers
 import org.jetbrains.kotlin.incremental.DifferenceCalculatorForPackageFacade.Companion.getNonPrivateMembers
 import org.jetbrains.kotlin.incremental.KotlinClassInfo
 import org.jetbrains.kotlin.incremental.PackagePartProtoData
 import org.jetbrains.kotlin.incremental.classpathDiff.ClassSnapshotGranularity.CLASS_MEMBER_LEVEL
-import org.jetbrains.kotlin.incremental.md5
+import org.jetbrains.kotlin.incremental.hashToLong
 import org.jetbrains.kotlin.incremental.storage.toByteArray
 import org.jetbrains.kotlin.konan.file.use
 import org.jetbrains.kotlin.load.kotlin.header.KotlinClassHeader.Kind.*
 import org.jetbrains.kotlin.resolve.jvm.JvmClassName
+import org.jetbrains.org.objectweb.asm.ClassReader
+import org.jetbrains.org.objectweb.asm.Opcodes
+import org.jetbrains.org.objectweb.asm.tree.ClassNode
 import java.io.Closeable
 import java.io.File
 import java.util.zip.ZipEntry
@@ -94,7 +102,7 @@ object ClassSnapshotter {
                         snapshotKotlinClass(clazz, granularity)
                     }
                     else -> metrics.measure(BuildTime.SNAPSHOT_JAVA_CLASSES) {
-                        JavaClassSnapshotter.snapshot(clazz, granularity)
+                        snapshotJavaClass(clazz, granularity)
                     }
                 }
             }
@@ -120,11 +128,7 @@ object ClassSnapshotter {
         }
     }
 
-    /**
-     * Computes a [KotlinClassSnapshot] of the given Kotlin class.
-     *
-     * (The caller must ensure that the given class is a Kotlin class.)
-     */
+    /** Computes a [KotlinClassSnapshot] of the given Kotlin class. */
     private fun snapshotKotlinClass(classFile: ClassFileWithContents, granularity: ClassSnapshotGranularity): KotlinClassSnapshot {
         val kotlinClassInfo =
             KotlinClassInfo.createFrom(classFile.classInfo.classId, classFile.classInfo.kotlinClassHeader!!, classFile.contents)
@@ -145,13 +149,60 @@ object ClassSnapshotter {
             )
             MULTIFILE_CLASS -> MultifileClassKotlinClassSnapshot(
                 classId, classAbiHash, classMemberLevelSnapshot,
-                constantNames = kotlinClassInfo.constantsMap.keys
+                constantNames = kotlinClassInfo.extraInfo.constantSnapshots.keys
             )
             SYNTHETIC_CLASS -> error("Unexpected class $classId with class kind ${SYNTHETIC_CLASS.name} (synthetic classes should have been removed earlier)")
             UNKNOWN -> error("Can't handle class $classId with class kind ${UNKNOWN.name}")
         }
     }
 
+    /** Computes a [JavaClassSnapshot] of the given Java class. */
+    private fun snapshotJavaClass(classFile: ClassFileWithContents, granularity: ClassSnapshotGranularity): JavaClassSnapshot {
+        // For incremental compilation, we only care about the ABI info of a class. There are 2 approaches:
+        //   1. Collect ABI info directly
+        //   2. Remove non-ABI info from the full class
+        // Note that for incremental compilation to be correct, all ABI info must be collected exhaustively (now and in the future when
+        // there are updates to Java/ASM), whereas it is acceptable if non-ABI info is not removed completely.
+        // In the following, we will use the second approach as it is safer and easier.
+
+        val classNode = ClassNode()
+        val classReader = ClassReader(classFile.contents)
+
+        // Note the `parsingOptions` passed to `classReader`:
+        //   - Pass SKIP_CODE as method bodies are not important
+        //   - Do not pass SKIP_DEBUG as debug info (e.g., method parameter names) may be important
+        classReader.accept(classNode, ClassReader.SKIP_CODE)
+        sortClassMembers(classNode)
+
+        // Remove private fields and methods
+        fun Int.isPrivate() = (this and Opcodes.ACC_PRIVATE) != 0
+        classNode.fields.removeIf { it.access.isPrivate() }
+        classNode.methods.removeIf { it.access.isPrivate() }
+
+        // Snapshot the class
+        val classMemberLevelSnapshot = if (granularity == CLASS_MEMBER_LEVEL) {
+            JavaClassMemberLevelSnapshot(
+                classAbiExcludingMembers = JavaElementSnapshot(classNode.name, snapshotClassExcludingMembers(classNode)),
+                fieldsAbi = classNode.fields.map { JavaElementSnapshot(it.name, snapshotField(it)) },
+                methodsAbi = classNode.methods.map { JavaElementSnapshot(it.name, snapshotMethod(it, classNode.version)) }
+            )
+        } else {
+            null
+        }
+        val classAbiHash = if (granularity == CLASS_MEMBER_LEVEL) {
+            JavaClassMemberLevelSnapshotExternalizer.toByteArray(classMemberLevelSnapshot!!).hashToLong()
+        } else {
+            snapshotClass(classNode)
+        }
+
+        return JavaClassSnapshot(
+            classId = classFile.classInfo.classId,
+            classAbiHash = classAbiHash,
+            classMemberLevelSnapshot = classMemberLevelSnapshot,
+            supertypes = classFile.classInfo.supertypes
+        )
+    }
+
 }
 
 private sealed interface DirectoryOrJarReader : Closeable {
@@ -227,9 +278,3 @@ private class JarReader(jar: File) : DirectoryOrJarReader {
         zipFile.close()
     }
 }
-
-internal fun ByteArray.hashToLong(): Long {
-    // Note: The returned type `Long` is 64-bit, but we currently don't have a good 64-bit hash function.
-    // The method below uses `md5` which is 128-bit and converts it to `Long`.
-    return md5()
-}

compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathSnapshotter.kt

@@ -259,6 +259,9 @@ class KotlinOnlyClasspathChangesComputerTest : ClasspathChangesComputerTest() {
     /** Regression test for KT-55021. */
     @Test
     fun testRenameFileFacade() {
+        // Check that classpath changes computation doesn't fail.
+        // Ideally, the returned changes should be empty (renaming a file facade alone shouldn't cause any `LookupSymbol`s to change), but
+        // it is currently not the case. However, this is just a small efficiency, not a serious bug.
         val changes = computeClasspathChanges(File(testDataDir, "KotlinOnly/testRenameFileFacade/src"), tmpDir)
         Changes(
             lookupSymbols = setOf(
@@ -269,6 +272,20 @@ class KotlinOnlyClasspathChangesComputerTest : ClasspathChangesComputerTest() {
         ).assertEquals(changes)
     }
 
+    /** Regression test for KT-58289.*/
+    @Test
+    fun testChangedAnnotations() {
+        val changes = computeClasspathChanges(File(testDataDir, "KotlinOnly/testChangedAnnotations/src"), tmpDir)
+        Changes(
+            lookupSymbols = setOf(
+                LookupSymbol(name = "SomeClassWithChangedAnnotation", scope = "com.example"),
+            ),
+            fqNames = setOf(
+                "com.example.SomeClassWithChangedAnnotation",
+            )
+        ).assertEquals(changes)
+    }
+
     /** Tests [SupertypesInheritorsImpact]. */
     @Test
     override fun testImpactComputation_SupertypesInheritors() {