[K/Wasm] Don't add mappings into source-maps for unavailable sources

IDEA JavaScript debugger gets stuck if many files are listed inside source maps (^FL-27262).
To help the debugger, we filter sources from modules not in the user project.
It's a temporary solution until the third-party libraries debugging is supported (it depends on the issue with file absolute path in KLIB by default ^KT-58406)

Author: JSMonk

compiler/cli/cli-common/gen/org/jetbrains/kotlin/cli/common/arguments/K2JSCompilerArgumentsCopyGenerated.kt

@@ -19,6 +19,7 @@ fun copyK2JSCompilerArguments(from: K2JSCompilerArguments, to: K2JSCompilerArgum
     to.friendModulesDisabled = from.friendModulesDisabled
     to.generateDts = from.generateDts
     to.generatePolyfills = from.generatePolyfills
+    to.includeUnavailableSourcesIntoSourceMap = from.includeUnavailableSourcesIntoSourceMap
     to.includes = from.includes
     to.irBaseClassInMetadata = from.irBaseClassInMetadata
     to.irBuildCache = from.irBuildCache

compiler/cli/cli-common/src/org/jetbrains/kotlin/cli/common/arguments/K2JSCompilerArguments.kt

@@ -274,6 +274,16 @@ In combination with '-meta-info', this generates both IR and pre-IR versions of
             field = value
         }
 
+    @Argument(
+        value = "-Xwasm-source-map-include-mappings-from-unavailable-sources",
+        description = "Insert source mappings from libraries even if their sources are unavailable on the end-user machine."
+    )
+    var includeUnavailableSourcesIntoSourceMap = false
+        set(value) {
+            checkFrozen()
+            field = value
+        }
+
     @Argument(
         value = "-Xir-dce-dump-reachability-info-to-file",
         valueDescription = "<path>",

compiler/cli/cli-js/src/org/jetbrains/kotlin/cli/js/K2JsIrCompiler.kt

@@ -209,6 +209,7 @@ class K2JsIrCompiler : CLICompiler<K2JSCompilerArguments>() {
         configuration.put(WasmConfigurationKeys.WASM_GENERATE_WAT, arguments.wasmGenerateWat)
         configuration.put(WasmConfigurationKeys.WASM_USE_TRAPS_INSTEAD_OF_EXCEPTIONS, arguments.wasmUseTrapsInsteadOfExceptions)
         configuration.put(WasmConfigurationKeys.WASM_USE_NEW_EXCEPTION_PROPOSAL, arguments.wasmUseNewExceptionProposal)
+
         configuration.putIfNotNull(WasmConfigurationKeys.WASM_TARGET, arguments.wasmTarget?.let(WasmTarget::fromName))
 
         configuration.put(JSConfigurationKeys.OPTIMIZE_GENERATED_JS, arguments.optimizeGeneratedJs)
@@ -839,6 +840,7 @@ class K2JsIrCompiler : CLICompiler<K2JSCompilerArguments>() {
             sourceMapContentEmbedding = SourceMapSourceEmbedding.INLINING
         }
         configuration.put(JSConfigurationKeys.SOURCE_MAP_EMBED_SOURCES, sourceMapContentEmbedding)
+        configuration.put(JSConfigurationKeys.SOURCE_MAP_INCLUDE_MAPPINGS_FROM_UNAVAILABLE_FILES, arguments.includeUnavailableSourcesIntoSourceMap)
 
         if (!arguments.sourceMap && sourceMapEmbedContentString != null) {
             messageCollector.report(WARNING, "source-map-embed-sources argument has no effect without source map", null)

compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/SourceMapsInfo.kt

@@ -17,6 +17,7 @@ data class SourceMapsInfo(
     val outputDir: File?,
     val sourceMapContentEmbedding: SourceMapSourceEmbedding,
     val namesPolicy: SourceMapNamesPolicy,
+    val includeUnavailableSourcesIntoSourceMap: Boolean = false
 ) {
     companion object {
         fun from(configuration: CompilerConfiguration): SourceMapsInfo? =
@@ -26,7 +27,8 @@ data class SourceMapsInfo(
                     configuration.get(JSConfigurationKeys.SOURCE_MAP_SOURCE_ROOTS, emptyList()),
                     configuration.get(JSConfigurationKeys.OUTPUT_DIR),
                     configuration.get(JSConfigurationKeys.SOURCE_MAP_EMBED_SOURCES, SourceMapSourceEmbedding.INLINING),
-                    configuration.get(JSConfigurationKeys.SOURCEMAP_NAMES_POLICY, SourceMapNamesPolicy.SIMPLE_NAMES)
+                    configuration.get(JSConfigurationKeys.SOURCEMAP_NAMES_POLICY, SourceMapNamesPolicy.SIMPLE_NAMES),
+                    configuration.getBoolean(JSConfigurationKeys.SOURCE_MAP_INCLUDE_MAPPINGS_FROM_UNAVAILABLE_FILES),
                 )
             } else {
                 null

compiler/ir/backend.wasm/src/org/jetbrains/kotlin/backend/wasm/ir2wasm/locationUtils.kt

@@ -46,12 +46,23 @@ fun IrElement.getSourceLocation(file: IrFile?, type: LocationType = LocationType
 
     if (line < 0 || column < 0) return SourceLocation.NoLocation("startLine or startColumn < 0")
 
-    if (file.isIgnoredFile) {
-        return SourceLocation.IgnoredLocation(path, line, column)
-    }
-
+    val module = file.module.name.asString()
 
-    return SourceLocation.Location(path, line, column)
+    return if (file.isIgnoredFile) {
+        SourceLocation.IgnoredLocation(
+            module,
+            path,
+            line,
+            column
+        )
+    } else {
+        SourceLocation.Location(
+            module,
+            path,
+            line,
+            column
+        )
+    }
 }
 
 fun WasmExpressionBuilder.buildUnreachableForVerifier() {

compiler/ir/backend.wasm/src/org/jetbrains/kotlin/backend/wasm/utils/SourceMapGenerator.kt

@@ -39,8 +39,12 @@ class SourceMapGenerator(
         val sourceMapBuilder =
             SourceMap3Builder(null, { error("This should not be called for Kotlin/Wasm") }, sourceMapsInfo.sourceMapPrefix)
 
-        val pathResolver =
-            SourceFilePathResolver.create(sourceMapsInfo.sourceRoots, sourceMapsInfo.sourceMapPrefix, sourceMapsInfo.outputDir)
+        val pathResolver = SourceFilePathResolver.create(
+            sourceMapsInfo.sourceRoots,
+            sourceMapsInfo.sourceMapPrefix,
+            sourceMapsInfo.outputDir,
+            sourceMapsInfo.includeUnavailableSourcesIntoSourceMap
+        )
 
         var prev: SourceLocation.Location? = null
         var prevGeneratedLine = 0
@@ -62,12 +66,21 @@ class SourceMapGenerator(
                 // TODO: add the ignored location into "ignoreList" in future
                 is SourceLocation.NoLocation, is SourceLocation.IgnoredLocation -> sourceMapBuilder.addEmptyMapping(generatedLocation.column)
                 is SourceLocation.Location -> {
-                    sourceLocation.apply {
-                        // TODO resulting path goes too deep since temporary directory we compiled first is deeper than final destination.
-                        val relativePath = pathResolver.getPathRelativeToSourceRoots(File(file)).replace(Regex("^\\.\\./"), "")
-                        sourceMapBuilder.addMapping(relativePath, null, { null }, line, column, null, generatedLocation.column)
-                        prev = this
-                    }
+                    // TODO resulting path goes too deep since temporary directory we compiled first is deeper than final destination.
+                    val relativePath = pathResolver
+                        .getPathRelativeToSourceRootsIfExists(sourceLocation.module, File(sourceLocation.file))
+                        ?.replace(Regex("^\\.\\./"), "") ?: continue
+
+                    sourceMapBuilder.addMapping(
+                        relativePath,
+                        null,
+                        { null },
+                        sourceLocation.line,
+                        sourceLocation.column,
+                        null,
+                        generatedLocation.column
+                    )
+                    prev = sourceLocation
                 }
             }
 

compiler/testData/cli/js/jsExtraHelp.out

@@ -10,6 +10,8 @@ where advanced options include:
   -Xfriend-modules-disabled  Disable internal declaration export.
   -Xgenerate-dts             Generate a TypeScript declaration .d.ts file alongside the JS file. This is available only in the IR backend.
   -Xgenerate-polyfills       Generate polyfills for features from the ES6+ standards.
+  -Xwasm-source-map-include-mappings-from-unavailable-sources
+                             Insert source mappings from libraries even if their sources are unavailable on the end-user machine.
   -Xinclude=<path>           Path to an intermediate library that should be processed in the same manner as source files.
   -Xir-base-class-in-metadata Write base classes into metadata.
   -Xir-build-cache           Use the compiler to build the cache.

compiler/tests-common-new/tests/org/jetbrains/kotlin/test/directives/WasmEnvironmentConfigurationDirectives.kt

@@ -44,6 +44,11 @@ object WasmEnvironmentConfigurationDirectives : SimpleDirectivesContainer() {
         applicability = DirectiveApplicability.Global
     )
 
+    val SOURCE_MAP_INCLUDE_MAPPINGS_FROM_UNAVAILABLE_FILES by directive(
+        description = "Insert source mappings from libraries even if their sources are unavailable on the end-user machine",
+        applicability = DirectiveApplicability.Global
+    )
+
     val RUN_THIRD_PARTY_OPTIMIZER by directive(
         description = "Also run third-party optimizer (for now, only binaryen is supported) after the main compilation",
     )

compiler/tests-common-new/tests/org/jetbrains/kotlin/test/services/configuration/WasmEnvironmentConfigurator.kt

@@ -19,6 +19,7 @@ import org.jetbrains.kotlin.test.directives.JsEnvironmentConfigurationDirectives
 import org.jetbrains.kotlin.test.directives.JsEnvironmentConfigurationDirectives.SOURCE_MAP_EMBED_SOURCES
 import org.jetbrains.kotlin.test.directives.WasmEnvironmentConfigurationDirectives
 import org.jetbrains.kotlin.test.directives.WasmEnvironmentConfigurationDirectives.DISABLE_WASM_EXCEPTION_HANDLING
+import org.jetbrains.kotlin.test.directives.WasmEnvironmentConfigurationDirectives.SOURCE_MAP_INCLUDE_MAPPINGS_FROM_UNAVAILABLE_FILES
 import org.jetbrains.kotlin.test.directives.WasmEnvironmentConfigurationDirectives.USE_NEW_EXCEPTION_HANDLING_PROPOSAL
 import org.jetbrains.kotlin.test.directives.model.DirectivesContainer
 import org.jetbrains.kotlin.test.directives.model.RegisteredDirectives
@@ -93,11 +94,13 @@ abstract class WasmEnvironmentConfigurator(testServices: TestServices) : Environ
         val sourceDirs = module.files.map { it.originalFile.parent }.distinct()
         configuration.put(JSConfigurationKeys.SOURCE_MAP_SOURCE_ROOTS, sourceDirs)
         configuration.put(JSConfigurationKeys.SOURCE_MAP, true)
+        configuration.put(JSConfigurationKeys.SOURCE_MAP_INCLUDE_MAPPINGS_FROM_UNAVAILABLE_FILES, SOURCE_MAP_INCLUDE_MAPPINGS_FROM_UNAVAILABLE_FILES in registeredDirectives)
 
         val sourceMapSourceEmbedding = registeredDirectives[SOURCE_MAP_EMBED_SOURCES].singleOrNull() ?: SourceMapSourceEmbedding.NEVER
         configuration.put(JSConfigurationKeys.SOURCE_MAP_EMBED_SOURCES, sourceMapSourceEmbedding)
 
         configuration.put(WasmConfigurationKeys.WASM_USE_TRAPS_INSTEAD_OF_EXCEPTIONS, DISABLE_WASM_EXCEPTION_HANDLING in registeredDirectives)
         configuration.put(WasmConfigurationKeys.WASM_USE_NEW_EXCEPTION_PROPOSAL, USE_NEW_EXCEPTION_HANDLING_PROPOSAL in registeredDirectives)
+
     }
 }

js/js.frontend/src/org/jetbrains/kotlin/js/config/JSConfigurationKeys.java

@@ -41,6 +41,9 @@ public class JSConfigurationKeys {
     public static final CompilerConfigurationKey<SourceMapNamesPolicy> SOURCEMAP_NAMES_POLICY = CompilerConfigurationKey.create(
             "a policy to generate a mapping from generated identifiers to their corresponding original names");
 
+    public static final CompilerConfigurationKey<Boolean> SOURCE_MAP_INCLUDE_MAPPINGS_FROM_UNAVAILABLE_FILES = CompilerConfigurationKey.create(
+            "insert source mappings from libraries even if their sources are unavailable on the end-user machine");
+
     public static final CompilerConfigurationKey<Boolean> META_INFO =
             CompilerConfigurationKey.create("generate .meta.js and .kjsm files");
 

js/js.sourcemap/src/org/jetbrains/kotlin/js/sourceMap/SourceFilePathResolver.kt

@@ -7,13 +7,19 @@ package org.jetbrains.kotlin.js.sourceMap
 import org.jetbrains.kotlin.config.CompilerConfiguration
 import org.jetbrains.kotlin.js.config.JSConfigurationKeys
 import org.jetbrains.kotlin.js.config.JsConfig
+import org.jetbrains.kotlin.utils.addToStdlib.runIf
 import java.io.File
 import java.io.IOException
 
-class SourceFilePathResolver(sourceRoots: List<File>, outputDir: File? = null) {
+class SourceFilePathResolver(
+    sourceRoots: List<File>,
+    outputDir: File? = null,
+    private val includeUnavailableSourcesIntoSourceMap: Boolean = false
+) {
     private val sourceRoots = sourceRoots.mapTo(mutableSetOf<File>()) { it.absoluteFile }
     private val outputDirPathResolver = outputDir?.let(::RelativePathCalculator)
     private val cache = mutableMapOf<File, String>()
+    private val modulesAndTheirSourcesStatus = hashMapOf<String, Boolean>()
 
     @Throws(IOException::class)
     fun getPathRelativeToSourceRoots(file: File): String {
@@ -25,6 +31,12 @@ class SourceFilePathResolver(sourceRoots: List<File>, outputDir: File? = null) {
         return path
     }
 
+    @Throws(IOException::class)
+    fun getPathRelativeToSourceRootsIfExists(moduleId: String, file: File): String? {
+        val moduleSourcesShouldBeAdded = includeUnavailableSourcesIntoSourceMap || modulesAndTheirSourcesStatus.getOrPut(moduleId) { file.exists() }
+        return runIf(moduleSourcesShouldBeAdded) { getPathRelativeToSourceRoots(file) }
+    }
+
     @Throws(IOException::class)
     private fun calculatePathRelativeToSourceRoots(file: File): String {
         val pathRelativeToOutput = calculatePathRelativeToOutput(file)
@@ -57,15 +69,22 @@ class SourceFilePathResolver(sourceRoots: List<File>, outputDir: File? = null) {
         fun create(configuration: CompilerConfiguration) = create(
             sourceRoots = configuration.get(JSConfigurationKeys.SOURCE_MAP_SOURCE_ROOTS, emptyList()),
             sourceMapPrefix = configuration.get(JSConfigurationKeys.SOURCE_MAP_PREFIX, ""),
-            outputDir = configuration.get(JSConfigurationKeys.OUTPUT_DIR)
+            outputDir = configuration.get(JSConfigurationKeys.OUTPUT_DIR),
+            includeUnavailableSourcesIntoSourceMap = configuration.getBoolean(JSConfigurationKeys.SOURCE_MAP_INCLUDE_MAPPINGS_FROM_UNAVAILABLE_FILES)
         )
 
         @JvmStatic
-        fun create(sourceRoots: List<String>, sourceMapPrefix: String, outputDir: File?): SourceFilePathResolver {
+        fun create(
+            sourceRoots: List<String>,
+            sourceMapPrefix: String,
+            outputDir: File?,
+            includeUnavailableSourcesIntoSourceMap: Boolean = false,
+        ): SourceFilePathResolver {
             val generateRelativePathsInSourceMap = sourceMapPrefix.isEmpty() && sourceRoots.isEmpty()
             return SourceFilePathResolver(
                 sourceRoots.map(::File),
-                outputDir.takeIf { generateRelativePathsInSourceMap }
+                outputDir.takeIf { generateRelativePathsInSourceMap },
+                includeUnavailableSourcesIntoSourceMap
             )
         }
     }

wasm/wasm.ir/src/org/jetbrains/kotlin/wasm/ir/convertors/WasmIrToBinary.kt

@@ -737,6 +737,7 @@ private class SourceLocationMappingToBinary(
 ) : SourceLocationMapping() {
     override val generatedLocation: SourceLocation.Location by lazy {
         SourceLocation.Location(
+            module = "",
             file = "",
             line = 0,
             column = offsets.sumOf {

wasm/wasm.ir/src/org/jetbrains/kotlin/wasm/ir/convertors/WasmIrToText.kt

@@ -83,7 +83,7 @@ class WasmIrToText(
             debugInformationGenerator?.addSourceLocation(
                 SourceLocationMappingToText(
                     it,
-                    SourceLocation.Location("", stringBuilder.lineNumber, stringBuilder.columnNumber),
+                    SourceLocation.Location("", "", stringBuilder.lineNumber, stringBuilder.columnNumber),
                 )
             )
         }

wasm/wasm.ir/src/org/jetbrains/kotlin/wasm/ir/source/location/SourceLocation.kt

@@ -9,9 +9,19 @@ sealed class SourceLocation {
     object NoLocation : SourceLocation()
 
     // Both line and column are zero-based
-    data class Location(val file: String, val line: Int, val column: Int) : SourceLocation()
+    data class Location(
+        val module: String,
+        val file: String,
+        val line: Int,
+        val column: Int
+    ) : SourceLocation()
 
-    data class IgnoredLocation(val file: String, val line: Int, val column: Int) : SourceLocation()
+    data class IgnoredLocation(
+        val module: String,
+        val file: String,
+        val line: Int,
+        val column: Int
+    ) : SourceLocation()
 
     companion object {
         @Suppress("FunctionName", "UNUSED_PARAMETER")

wasm/wasm.tests/test/org/jetbrains/kotlin/wasm/test/AbstractFirWasmTest.kt

@@ -132,6 +132,7 @@ open class AbstractFirWasmJsSteppingTest : AbstractFirWasmJsTest(
         commonConfigurationForWasmBlackBoxCodegenTest()
         defaultDirectives {
             +WasmEnvironmentConfigurationDirectives.GENERATE_SOURCE_MAP
+            +WasmEnvironmentConfigurationDirectives.SOURCE_MAP_INCLUDE_MAPPINGS_FROM_UNAVAILABLE_FILES
         }
     }
 }

wasm/wasm.tests/test/org/jetbrains/kotlin/wasm/test/AbstractK1WasmTest.kt

@@ -80,6 +80,7 @@ open class AbstractK1WasmSteppingTest : AbstractK1WasmTest(
         commonConfigurationForWasmBlackBoxCodegenTest()
         defaultDirectives {
             +WasmEnvironmentConfigurationDirectives.GENERATE_SOURCE_MAP
+            +WasmEnvironmentConfigurationDirectives.SOURCE_MAP_INCLUDE_MAPPINGS_FROM_UNAVAILABLE_FILES
         }
     }
 }