Fix some issues with preprocess source maps

src/compiler/preprocess/index.ts

@@ -39,14 +39,18 @@ function parse_attributes(str: string) {
 	return attrs;
 }
 
+function get_file_basename(filename: string) {
+	return filename.split(/[/\\]/).pop();
+}
+
 interface Replacement {
 	offset: number;
 	length: number;
 	replacement: StringWithSourcemap;
 }
 
 async function replace_async(
-	filename: string,
+	file_basename: string,
 	source: string,
 	get_location: ReturnType<typeof getLocator>,
 	re: RegExp,
@@ -73,13 +77,13 @@ async function replace_async(
 	)) {
 		// content = unchanged source characters before the replaced segment
 		const content = StringWithSourcemap.from_source(
-			filename, source.slice(last_end, offset), get_location(last_end));
+			file_basename, source.slice(last_end, offset), get_location(last_end));
 		out.concat(content).concat(replacement);
 		last_end = offset + length;
 	}
 	// final_content = unchanged source characters after last replaced segment
 	const final_content = StringWithSourcemap.from_source(
-		filename, source.slice(last_end), get_location(last_end));
+		file_basename, source.slice(last_end), get_location(last_end));
 	return out.concat(final_content);
 }
 
@@ -156,11 +160,34 @@ function decoded_sourcemap_from_generator(generator: any) {
 	return map;
 }
 
+/**
+ * Heuristic used to find index of component source inside source map sources.
+ */
+function guess_source_index(
+	file_basename: string,
+	decoded_map: DecodedSourceMap
+): number {
+	if (!file_basename) {
+		return decoded_map.sources.findIndex(source => !source);
+	}
+	// different tools produce different sources
+	// (file name, relative path, absolute path)
+	const index = decoded_map.sources.findIndex(source => {
+		const source_basename = source && get_file_basename(source);
+		return source_basename === file_basename;
+	});
+	if (index !== -1) {
+		// also normalize it in on source map
+		decoded_map.sources[index] = file_basename;
+	}
+	return index;
+}
+
 /**
  * Convert a preprocessor output and its leading prefix and trailing suffix into StringWithSourceMap
  */
 function get_replacement(
-	filename: string,
+	file_basename: string,
 	offset: number,
 	get_location: ReturnType<typeof getLocator>,
 	original: string,
@@ -171,9 +198,9 @@ function get_replacement(
 
 	// Convert the unchanged prefix and suffix to StringWithSourcemap
 	const prefix_with_map = StringWithSourcemap.from_source(
-		filename, prefix, get_location(offset));
+		file_basename, prefix, get_location(offset));
 	const suffix_with_map = StringWithSourcemap.from_source(
-		filename, suffix, get_location(offset + prefix.length + original.length));
+		file_basename, suffix, get_location(offset + prefix.length + original.length));
 
 	// Convert the preprocessed code and its sourcemap to a StringWithSourcemap
 	let decoded_map: DecodedSourceMap;
@@ -186,7 +213,9 @@ function get_replacement(
 			// import decoded sourcemap from mozilla/source-map/SourceMapGenerator
 			decoded_map = decoded_sourcemap_from_generator(decoded_map);
 		}
-		sourcemap_add_offset(decoded_map, get_location(offset + prefix.length));
+		// offset only segments pointing at original component source
+		const source_index = guess_source_index(file_basename, decoded_map);
+		sourcemap_add_offset(decoded_map, get_location(offset + prefix.length), source_index);
 	}
 	const processed_with_map = StringWithSourcemap.from_processed(processed.code, decoded_map);
 
@@ -203,6 +232,9 @@ export default async function preprocess(
 	const filename = (options && options.filename) || preprocessor.filename; // legacy
 	const dependencies = [];
 
+	// preprocess source must be relative to itself
+	const file_basename = filename && get_file_basename(filename);
+
 	const preprocessors = preprocessor
 		? Array.isArray(preprocessor) ? preprocessor : [preprocessor]
 		: [];
@@ -246,13 +278,13 @@ export default async function preprocess(
 			: /<!--[^]*?-->|<script(\s[^]*?)?(?:>([^]*?)<\/script>|\/>)/gi;
 
 		const res = await replace_async(
-			filename,
+			file_basename,
 			source,
 			get_location,
 			tag_regex,
 			async (match, attributes = '', content = '', offset) => {
 				const no_change = () => StringWithSourcemap.from_source(
-					filename, match, get_location(offset));
+					file_basename, match, get_location(offset));
 				if (!attributes && !content) {
 					return no_change();
 				}
@@ -268,7 +300,7 @@ export default async function preprocess(
 
 				if (!processed) return no_change();
 				if (processed.dependencies) dependencies.push(...processed.dependencies);
-				return get_replacement(filename, offset, get_location, content, processed, `<${tag_name}${attributes}>`, `</${tag_name}>`);
+				return get_replacement(file_basename, offset, get_location, content, processed, `<${tag_name}${attributes}>`, `</${tag_name}>`);
 			}
 		);
 		source = res.string;
@@ -285,7 +317,7 @@ export default async function preprocess(
 
 	// Combine all the source maps for each preprocessor function into one
 	const map: RawSourceMap = combine_sourcemaps(
-		filename,
+		file_basename,
 		sourcemap_list
 	);
 

src/compiler/utils/string_with_sourcemap.ts

@@ -13,21 +13,22 @@ function last_line_length(s: string) {
 
 // mutate map in-place
 export function sourcemap_add_offset(
-	map: DecodedSourceMap, offset: SourceLocation
+	map: DecodedSourceMap, offset: SourceLocation, source_index: number
 ) {
-	if (map.mappings.length == 0) return map;
-	// shift columns in first line
-	const segment_list = map.mappings[0];
-	for (let segment = 0; segment < segment_list.length; segment++) {
-		const seg = segment_list[segment];
-		if (seg[3]) seg[3] += offset.column;
-	}
+	if (map.mappings.length == 0 || source_index < 0) return;
 	// shift lines
 	for (let line = 0; line < map.mappings.length; line++) {
 		const segment_list = map.mappings[line];
 		for (let segment = 0; segment < segment_list.length; segment++) {
 			const seg = segment_list[segment];
-			if (seg[2]) seg[2] += offset.line;
+			// shift only segments that belong to component source file
+			if (seg.length >= 4 && seg[1] === source_index) {
+				// shift columns pointing at the first line
+				if (seg[2] === 0) {
+					seg[3] += offset.column;
+				}
+				seg[2] += offset.line;
+			}
 		}
 	}
 }
@@ -97,6 +98,9 @@ export class StringWithSourcemap {
 			return this;
 		}
 
+		// compute last line length before mutating
+		const column_offset = last_line_length(this.string);
+
 		this.string += other.string;
 
 		const m1 = this.map;
@@ -117,24 +121,24 @@ export class StringWithSourcemap {
 				const segment_list = m2.mappings[line];
 				for (let segment = 0; segment < segment_list.length; segment++) {
 					const seg = segment_list[segment];
-					if (seg[1]) seg[1] = new_source_idx[seg[1]];
-					if (seg[4]) seg[4] = new_name_idx[seg[4]];
+					if (seg.length > 1) seg[1] = new_source_idx[seg[1]];
+					if (seg.length > 4) seg[4] = new_name_idx[seg[4]];
 				}
 			}
 		} else if (sources_idx_changed) {
 			for (let line = 0; line < m2.mappings.length; line++) {
 				const segment_list = m2.mappings[line];
 				for (let segment = 0; segment < segment_list.length; segment++) {
 					const seg = segment_list[segment];
-					if (seg[1]) seg[1] = new_source_idx[seg[1]];
+					if (seg.length > 1) seg[1] = new_source_idx[seg[1]];
 				}
 			}
 		} else if (names_idx_changed) {
 			for (let line = 0; line < m2.mappings.length; line++) {
 				const segment_list = m2.mappings[line];
 				for (let segment = 0; segment < segment_list.length; segment++) {
 					const seg = segment_list[segment];
-					if (seg[4]) seg[4] = new_name_idx[seg[4]];
+					if (seg.length > 4) seg[4] = new_name_idx[seg[4]];
 				}
 			}
 		}
@@ -146,7 +150,6 @@ export class StringWithSourcemap {
 		// 2. first line of second map
 		// columns of 2 must be shifted
 
-		const column_offset = last_line_length(this.string);
 		if (m2.mappings.length > 0 && column_offset > 0) {
 			const first_line = m2.mappings[0];
 			for (let i = 0; i < first_line.length; i++) {
@@ -164,7 +167,17 @@ export class StringWithSourcemap {
 	}
 
 	static from_processed(string: string, map?: DecodedSourceMap): StringWithSourcemap {
-		if (map) return new StringWithSourcemap(string, map);
+		if (map) {
+			// ensure that count of source map mappings lines 
+			// is equal to count of generated code lines
+			// (some tools may produce less)
+			const missing_lines = string.split('\n').length - map.mappings.length;
+			for (let i = 0; i < missing_lines; i++) {
+				map.mappings.push([]);
+			}
+			return new StringWithSourcemap(string, map);
+		}
+
 		if (string == '') return new StringWithSourcemap();
 		map = { version: 3, names: [], sources: [], mappings: [] };
 

test/sourcemaps/helpers.ts

@@ -1,4 +1,26 @@
-import MagicString from 'magic-string';
+import MagicString, { Bundle } from 'magic-string';
+
+export function magic_string_bundle(
+	inputs: Array<{ code: string | MagicString, filename: string }>,
+	filename = 'bundle.js',
+	separator = '\n'
+) {
+	const bundle = new Bundle({ separator });
+	inputs.forEach(({ code, filename }) => {
+		bundle.addSource({
+			filename,
+			content: typeof code === 'string' ? new MagicString(code) : code
+		});
+	});
+	return {
+		code: bundle.toString(),
+		map: bundle.generateMap({
+			source: filename,
+			hires: true,
+			includeContent: false
+		})
+	};
+}
 
 export function magic_string_preprocessor_result(filename: string, src: MagicString) {
 	return {

test/sourcemaps/samples/external/_config.js

@@ -0,0 +1,23 @@
+import { magic_string_bundle } from '../../helpers';
+
+export const COMMON = ':global(html) { height: 100%; }\n';
+
+// TODO: removing '\n' breaks test
+// - _actual.svelte.map looks correct
+// - _actual.css.map adds reference to </style> on input.svelte
+export const STYLES = '.awesome { color: orange; }\n';
+
+export default {
+	css_map_sources: ['common.scss', 'styles.scss'],
+	js_map_sources: [],
+	preprocess: [
+		{
+			style: () => {
+				return magic_string_bundle([
+					{ filename: 'common.scss', code: COMMON },
+					{ filename: 'styles.scss', code: STYLES }
+				]);
+			}
+		}
+	]
+};

test/sourcemaps/samples/external/input.svelte

@@ -0,0 +1,3 @@
+<style lang="scss" src="./styles.scss"></style>
+
+<div class="awesome">Divs ftw!</div>

test/sourcemaps/samples/external/test.js

@@ -0,0 +1,27 @@
+import { getLocator } from 'locate-character';
+import { COMMON, STYLES } from './_config';
+
+export function test({ assert, input, preprocessed }) {
+
+	const assertMapped = (locateInput, code, filename) => {
+		const sourceLoc = locateInput(code);
+		const transformedLoc = preprocessed.locate_1(code);
+		assert.deepEqual(
+			preprocessed.mapConsumer.originalPositionFor(transformedLoc),
+			{
+				source: filename,
+				name: null,
+				line: sourceLoc.line + 1,
+				column: sourceLoc.column
+			},
+			`failed to locate "${code}"`
+		);
+	};
+
+	// Transformed script, main file
+	assertMapped(input.locate, 'Divs ftw!', 'input.svelte');
+
+	// External files
+	assertMapped(getLocator(COMMON), 'height: 100%;', 'common.scss');
+	assertMapped(getLocator(STYLES), 'color: orange;', 'styles.scss');
+}

test/sourcemaps/samples/guess-source/_config.js

@@ -0,0 +1,19 @@
+import { magic_string_bundle } from '../../helpers';
+
+export const PREPEND = 'console.log("COUNTER_START")';
+export const APPEND = 'console.log("COUNTER_END")';
+
+export default {
+	js_map_sources: ['input.svelte', 'src/prepend.js', 'src/append.js'],
+	preprocess: [
+		{
+			script: ({ content }) => {
+				return magic_string_bundle([
+					{ filename: 'src/prepend.js', code: PREPEND },
+					{ filename: 'src/input.svelte', code: content },
+					{ filename: 'src/append.js', code: APPEND }
+				]);
+			}
+		}
+	]
+};

test/sourcemaps/samples/guess-source/input.svelte

@@ -0,0 +1,6 @@
+<script lang="ts">
+	let count = 3;
+</script>
+
+<h1>Hello world!</h1>
+<div>Counter value: {count}</div>

test/sourcemaps/samples/guess-source/test.js

@@ -0,0 +1,30 @@
+import { getLocator } from 'locate-character';
+import { APPEND, PREPEND } from './_config';
+
+export function test({ assert, input, preprocessed }) {
+
+	const assertMapped = (locateInput, code, filename) => {
+		const sourceLoc = locateInput(code);
+		const transformedLoc = preprocessed.locate_1(code);
+		assert.deepEqual(
+			preprocessed.mapConsumer.originalPositionFor(transformedLoc),
+			{
+				source: filename,
+				name: null,
+				line: sourceLoc.line + 1,
+				column: sourceLoc.column
+			},
+			`failed to locate "${code}"`
+		);
+	};
+
+	// Transformed script, main file
+	assertMapped(input.locate, 'let count = 3;', 'input.svelte');
+
+	// Untouched markup, main file
+	assertMapped(input.locate, '<h1>Hello world!</h1>', 'input.svelte');
+
+	// External files
+	assertMapped(getLocator(PREPEND), '"COUNTER_START"', 'src/prepend.js');
+	assertMapped(getLocator(APPEND), '"COUNTER_END"', 'src/append.js');
+}