Add sourcemap support to preprocessors

Co-authored-by: Milan Hauth <[email protected]>

Author: halfnelson

package-lock.json

@@ -56,6 +56,7 @@
   },
   "homepage": "https://github.com/sveltejs/svelte#README",
   "devDependencies": {
+    "@ampproject/remapping": "^0.3.0",
     "@rollup/plugin-commonjs": "^11.0.0",
     "@rollup/plugin-json": "^4.0.1",
     "@rollup/plugin-node-resolve": "^6.0.0",
@@ -89,6 +90,7 @@
     "rollup": "^1.27.14",
     "source-map": "^0.7.3",
     "source-map-support": "^0.5.13",
+    "sourcemap-codec": "^1.4.8",
     "tiny-glob": "^0.2.6",
     "tslib": "^1.10.0",
     "typescript": "^3.5.3"

package.json

@@ -29,6 +29,7 @@ import add_to_set from './utils/add_to_set';
 import check_graph_for_cycles from './utils/check_graph_for_cycles';
 import { print, x, b } from 'code-red';
 import { is_reserved_keyword } from './utils/reserved_keywords';
+import { combine_sourcemaps, sourcemap_define_tostring_tourl } from '../utils/string_with_sourcemap';
 import Element from './nodes/Element';
 
 interface ComponentOptions {
@@ -330,6 +331,29 @@ export default class Component {
 			js.map.sourcesContent = [
 				this.source
 			];
+
+			if (compile_options.sourcemap) {
+				if (js.map) {
+					js.map = combine_sourcemaps(
+						this.file,
+						[
+							js.map, // idx 1: internal
+							compile_options.sourcemap // idx 0: external: svelte.preprocess, etc
+						]
+					);
+					sourcemap_define_tostring_tourl(js.map);
+				}
+				if (css.map) {
+					css.map = combine_sourcemaps(
+						this.file,
+						[
+							css.map, // idx 1: internal
+							compile_options.sourcemap // idx 0: external: svelte.preprocess, etc
+						]
+					);
+					sourcemap_define_tostring_tourl(css.map);
+				}
+			}
 		}
 
 		return {

src/compiler/compile/Component.ts

@@ -11,6 +11,7 @@ const valid_options = [
 	'format',
 	'name',
 	'filename',
+	'sourcemap',
 	'generate',
 	'outputFilename',
 	'cssOutputFilename',

src/compiler/compile/index.ts

@@ -110,6 +110,7 @@ export interface CompileOptions {
 	filename?: string;
 	generate?: 'dom' | 'ssr' | false;
 
+	sourcemap?: object | string;
 	outputFilename?: string;
 	cssOutputFilename?: string;
 	sveltePath?: string;

src/compiler/interfaces.ts

@@ -1,6 +1,11 @@
+import { RawSourceMap, DecodedSourceMap } from '@ampproject/remapping/dist/types/types';
+import { decode as decode_mappings } from 'sourcemap-codec';
+import { getLocator } from 'locate-character';
+import { StringWithSourcemap, sourcemap_add_offset, combine_sourcemaps } from '../utils/string_with_sourcemap';
+
 export interface Processed {
 	code: string;
-	map?: object | string;
+	map?: string | object;  // we be opaque with the type here to avoid dependency on the remapping module for our public types.
 	dependencies?: string[];
 }
 
@@ -37,12 +42,18 @@ function parse_attributes(str: string) {
 interface Replacement {
 	offset: number;
 	length: number;
-	replacement: string;
+	replacement: StringWithSourcemap;
 }
 
-async function replace_async(str: string, re: RegExp, func: (...any) => Promise<string>) {
+async function replace_async(
+	filename: string,
+	source: string,
+	get_location: ReturnType<typeof getLocator>,
+	re: RegExp,
+	func: (...any) => Promise<StringWithSourcemap>
+): Promise<StringWithSourcemap> {
 	const replacements: Array<Promise<Replacement>> = [];
-	str.replace(re, (...args) => {
+	source.replace(re, (...args) => {
 		replacements.push(
 			func(...args).then(
 				res =>
@@ -55,16 +66,52 @@ async function replace_async(str: string, re: RegExp, func: (...any) => Promise<
 		);
 		return '';
 	});
-	let out = '';
+	const out = new StringWithSourcemap();
 	let last_end = 0;
 	for (const { offset, length, replacement } of await Promise.all(
 		replacements
 	)) {
-		out += str.slice(last_end, offset) + replacement;
+		// content = unchanged source characters before the replaced segment
+		const content = StringWithSourcemap.from_source(
+			filename, source.slice(last_end, offset), get_location(last_end));
+		out.concat(content).concat(replacement);
 		last_end = offset + length;
 	}
-	out += str.slice(last_end);
-	return out;
+	// final_content = unchanged source characters after last replaced segment
+	const final_content = StringWithSourcemap.from_source(
+		filename, source.slice(last_end), get_location(last_end));
+	return out.concat(final_content);
+}
+
+// Convert a preprocessor output and its leading prefix and trailing suffix into StringWithSourceMap
+function get_replacement(
+	filename: string,
+	offset: number,
+	get_location: ReturnType<typeof getLocator>,
+	original: string,
+	processed: Processed,
+	prefix: string,
+	suffix: string
+): StringWithSourcemap {
+
+	// Convert the unchanged prefix and suffix to StringWithSourcemap
+	const prefix_with_map = StringWithSourcemap.from_source(
+		filename, prefix, get_location(offset));
+	const suffix_with_map = StringWithSourcemap.from_source(
+		filename, suffix, get_location(offset + prefix.length + original.length));
+
+	// Convert the preprocessed code and its sourcemap to a StringWithSourcemap
+	let decoded_map: DecodedSourceMap;
+	if (processed.map) {
+		decoded_map = typeof processed.map === 'string' ? JSON.parse(processed.map) : processed.map;
+		if (typeof(decoded_map.mappings) === 'string')
+			decoded_map.mappings = decode_mappings(decoded_map.mappings);
+		sourcemap_add_offset(decoded_map, get_location(offset + prefix.length));
+	}
+	const processed_with_map = StringWithSourcemap.from_processed(processed.code, decoded_map);
+
+	// Surround the processed code with the prefix and suffix, retaining valid sourcemappings
+	return prefix_with_map.concat(processed_with_map).concat(suffix_with_map);
 }
 
 export default async function preprocess(
@@ -76,60 +123,107 @@ export default async function preprocess(
 	const filename = (options && options.filename) || preprocessor.filename; // legacy
 	const dependencies = [];
 
-	const preprocessors = Array.isArray(preprocessor) ? preprocessor : [preprocessor];
+	const preprocessors = Array.isArray(preprocessor) ? preprocessor : [preprocessor || {}];
 
 	const markup = preprocessors.map(p => p.markup).filter(Boolean);
 	const script = preprocessors.map(p => p.script).filter(Boolean);
 	const style = preprocessors.map(p => p.style).filter(Boolean);
 
+	// sourcemap_list is sorted in reverse order from last map (index 0) to first map (index -1)
+	// so we use sourcemap_list.unshift() to add new maps
+	// https://github.com/ampproject/remapping#multiple-transformations-of-a-file
+	const sourcemap_list: Array<DecodedSourceMap | RawSourceMap> = [];
+
+	// TODO keep track: what preprocessor generated what sourcemap? to make debugging easier = detect low-resolution sourcemaps in fn combine_mappings
+
 	for (const fn of markup) {
+
+		// run markup preprocessor
 		const processed = await fn({
 			content: source,
 			filename
 		});
+
 		if (processed && processed.dependencies) dependencies.push(...processed.dependencies);
 		source = processed ? processed.code : source;
+		if (processed && processed.map)
+			sourcemap_list.unshift(
+				typeof(processed.map) === 'string'
+					? JSON.parse(processed.map)
+					: processed.map
+			);
 	}
 
 	for (const fn of script) {
-		source = await replace_async(
+		const get_location = getLocator(source);
+		const res = await replace_async(
+			filename,
 			source,
+			get_location,
 			/<!--[^]*?-->|<script(\s[^]*?)?(?:>([^]*?)<\/script>|\/>)/gi,
-			async (match, attributes = '', content = '') => {
+			async (match, attributes = '', content = '', offset) => {
+				const no_change = () => StringWithSourcemap.from_source(
+					filename, match, get_location(offset));
 				if (!attributes && !content) {
-					return match;
+					return no_change();
 				}
 				attributes = attributes || '';
+				content = content || '';
+
+				// run script preprocessor
 				const processed = await fn({
 					content,
 					attributes: parse_attributes(attributes),
 					filename
 				});
 				if (processed && processed.dependencies) dependencies.push(...processed.dependencies);
-				return processed ? `<script${attributes}>${processed.code}</script>` : match;
+				return processed
+					? get_replacement(filename, offset, get_location, content, processed, `<script${attributes}>`, '</script>')
+					: no_change();
 			}
 		);
+		source = res.string;
+		sourcemap_list.unshift(res.map);
 	}
 
 	for (const fn of style) {
-		source = await replace_async(
+		const get_location = getLocator(source);
+		const res = await replace_async(
+			filename,
 			source,
+			get_location,
 			/<!--[^]*?-->|<style(\s[^]*?)?(?:>([^]*?)<\/style>|\/>)/gi,
-			async (match, attributes = '', content = '') => {
+			async (match, attributes = '', content = '', offset) => {
+				const no_change = () => StringWithSourcemap.from_source(
+					filename, match, get_location(offset));
 				if (!attributes && !content) {
-					return match;
+					return no_change();
 				}
+				attributes = attributes || '';
+				content = content || '';
+
+				// run style preprocessor
 				const processed: Processed = await fn({
 					content,
 					attributes: parse_attributes(attributes),
 					filename
 				});
 				if (processed && processed.dependencies) dependencies.push(...processed.dependencies);
-				return processed ? `<style${attributes}>${processed.code}</style>` : match;
+				return processed
+					? get_replacement(filename, offset, get_location, content, processed, `<style${attributes}>`, '</style>')
+					: no_change();
 			}
 		);
+		source = res.string;
+		sourcemap_list.unshift(res.map);
 	}
 
+	// Combine all the source maps for each preprocessor function into one
+	const map: RawSourceMap = combine_sourcemaps(
+		filename,
+		sourcemap_list
+	);
+
 	return {
 		// TODO return separated output, in future version where svelte.compile supports it:
 		// style: { code: styleCode, map: styleMap },
@@ -138,7 +232,7 @@ export default async function preprocess(
 
 		code: source,
 		dependencies: [...new Set(dependencies)],
-
+		map: (map as object),
 		toString() {
 			return source;
 		}