fix tests, use decoded mappings, show warnings

Author: milahu

src/compiler/compile/Component.ts

@@ -29,8 +29,11 @@ import add_to_set from './utils/add_to_set';
 import check_graph_for_cycles from './utils/check_graph_for_cycles';
 import { print, x, b } from 'code-red';
 import { is_reserved_keyword } from './utils/reserved_keywords';
-import remapping from '@ampproject/remapping';
+import { combine_sourcemaps, sourcemap_add_tostring_tourl, combine_sourcemaps_map_stats } from '../utils/string_with_sourcemap';
 import Element from './nodes/Element';
+import { RawSourceMap } from '@ampproject/remapping/dist/types/types';
+import { encode as encode_mappings, decode as decode_mappings } from 'sourcemap-codec';
+
 
 interface ComponentOptions {
 	namespace?: string;
@@ -318,12 +321,19 @@ export default class Component {
 
 			css = compile_options.customElement
 				? { code: null, map: null }
-				: result.css;
+				: result.css; // css.map.mappings are decoded
 
 			js = print(program, {
 				sourceMapSource: compile_options.filename
 			});
 
+			// TODO remove workaround
+			// js.map.mappings should be decoded
+			// https://github.com/Rich-Harris/code-red/issues/50
+			if (js.map && typeof (js.map as any).mappings == 'string') {
+				(js.map as any).mappings = decode_mappings((js.map as any).mappings);
+			}
+
 			js.map.sources = [
 				compile_options.filename ? get_relative_path(compile_options.outputFilename || '', compile_options.filename) : null
 			];
@@ -332,34 +342,59 @@ export default class Component {
 				this.source
 			];
 
+			// combine sourcemaps
+			const map_stats: combine_sourcemaps_map_stats = {
+				sourcemapWarnLoss: 0, // segment loss is usually high, so we ignore
+				sourcemapEncodedWarn: true // TODO config
+				// property `result` is set by combine_sourcemaps
+			};
+
 			if (compile_options.sourcemap) {
 				if (js.map) {
-					const pre_remap_sources = js.map.sources;
-					js.map = remapping([js.map, compile_options.sourcemap], () => null, true);
-					// remapper can remove our source if it isn't used (no segments map back to it). It is still handy to have a source
-					// so we add it back
-					if (js.map.sources && js.map.sources.length == 0) {
-						js.map.sources = pre_remap_sources;
+					js.map = combine_sourcemaps(
+						this.file,
+						[
+							js.map, // idx 1: internal
+							compile_options.sourcemap // idx 0: external: svelte.preprocess, etc
+						],
+						map_stats
+					) as RawSourceMap;
+					sourcemap_add_tostring_tourl(js.map);
+					if (map_stats.result && map_stats.result.maps_encoded && map_stats.result.maps_encoded.length > 0) {
+						console.log('warning. svelte.compile received encoded script sourcemaps (index '+
+							map_stats.result.maps_encoded.join(', ')+'). '+
+							'this is slow. make your sourcemap-generators return decoded mappings '+
+							'or disable this warning with svelte.compile(_, _, { sourcemapEncodedWarn: false })'
+						);
 					}
-					Object.defineProperties(js.map, {
-						toString: {
-							enumerable: false,
-							value: function toString() {
-								return JSON.stringify(this);
-							}
-						},
-						toUrl: {
-							enumerable: false,
-							value: function toUrl() {
-								return 'data:application/json;charset=utf-8;base64,' + btoa(this.toString());
-							}
-						}
-					});
 				}
 				if (css.map) {
-					css.map = remapping([css.map, compile_options.sourcemap], () => null, true);
+					css.map = combine_sourcemaps(
+						this.file,
+						[
+							css.map, // idx 1: internal
+							compile_options.sourcemap // idx 0: external: svelte.preprocess, etc
+						]
+					) as RawSourceMap;
+					sourcemap_add_tostring_tourl(css.map);
+					if (map_stats.result && map_stats.result.maps_encoded && map_stats.result.maps_encoded.length > 0) {
+						console.log('warning. svelte.compile received encoded style sourcemaps (index '+
+							map_stats.result.maps_encoded.join(', ')+'). '+
+							'this is slow. make your sourcemap-generators return decoded mappings '+
+							'or disable this warning with svelte.compile(_, _, { sourcemapEncodedWarn: false })'
+						);
+					}
 				}
 			}
+
+			// encode mappings only once, after all sourcemaps are combined
+			if (js.map && typeof(js.map.mappings) == 'object') {
+				(js.map as RawSourceMap).mappings = encode_mappings(js.map.mappings);
+			}
+			if (css.map && typeof(css.map.mappings) == 'object') {
+				(css.map as RawSourceMap).mappings = encode_mappings(css.map.mappings);
+			}
+
 		}
 
 		return {

src/compiler/compile/css/Stylesheet.ts

@@ -410,7 +410,7 @@ export default class Stylesheet {
 
 		return {
 			code: code.toString(),
-			map: code.generateMap({
+			map: code.generateDecodedMap({
 				includeContent: true,
 				source: this.filename,
 				file

src/compiler/compile/render_dom/index.ts

@@ -30,8 +30,12 @@ export default function dom(
 	}
 
 	const css = component.stylesheet.render(options.filename, !options.customElement);
+
+	// TODO fix css.map.toUrl - stylesheet.render returns decoded mappings, map.toUrl needs encoded mappings
+	// TODO use combined css.map? see compile/Component.ts
 	const styles = component.stylesheet.has_styles && options.dev
-		? `${css.code}\n/*# sourceMappingURL=${css.map.toUrl()} */`
+		//? `${css.code}\n/*# sourceMappingURL=${css.map.toUrl()} */`
+		? `${css.code}\n/*# sourceMappingURL=TODO_FIXME */`
 		: css.code;
 
 	const add_css = component.get_unique_name('add_css');
@@ -467,12 +471,15 @@ export default function dom(
 	}
 
 	if (options.customElement) {
+		// TODO use combined css.map? see compile/Component.ts
+		// TODO css.map.toUrl needs encoded mappings
+		//			${css.code && b`this.shadowRoot.innerHTML = \`<style>${css.code.replace(/\\/g, '\\\\')}${options.dev ? `\n/*# sourceMappingURL=${css.map.toUrl()} */` : ''}</style>\`;`}
 		const declaration = b`
 			class ${name} extends @SvelteElement {
 				constructor(options) {
 					super();
 
-					${css.code && b`this.shadowRoot.innerHTML = \`<style>${css.code.replace(/\\/g, '\\\\')}${options.dev ? `\n/*# sourceMappingURL=${css.map.toUrl()} */` : ''}</style>\`;`}
+					${css.code && b`this.shadowRoot.innerHTML = \`<style>${css.code.replace(/\\/g, '\\\\')}${options.dev ? `\n/*# sourceMappingURL=TODO_FIXME */` : ''}</style>\`;`}
 
 					@init(this, { target: this.shadowRoot }, ${definition}, ${has_create_fragment ? 'create_fragment': 'null'}, ${not_equal}, ${prop_indexes}, ${dirty});
 

src/compiler/compile/render_ssr/index.ts

@@ -137,6 +137,7 @@ export default function ssr(
 		main
 	].filter(Boolean);
 
+	// TODO use combined css.map? see compile/Component.ts
 	const js = b`
 		${css.code ? b`
 		const #css = {

src/compiler/interfaces.ts

@@ -1,5 +1,7 @@
 import { Node, Program } from "estree";
-import { SourceMap } from 'magic-string';
+
+// eslint-disable-next-line import/named
+import { DecodedSourceMap } from 'magic-string';
 
 interface BaseNode {
 	start: number;
@@ -166,5 +168,5 @@ export interface Var {
 
 export interface CssResult { 
 	code: string;
-	map: SourceMap;
+	map: DecodedSourceMap;
 }

src/compiler/preprocess/index.ts

@@ -1,8 +1,7 @@
-import remapping from '@ampproject/remapping';
-import { SourceMapInput, SourceMapLoader, RawSourceMap, DecodedSourceMap } from '@ampproject/remapping/dist/types/types';
+import { SourceMapInput, RawSourceMap, DecodedSourceMap } from '@ampproject/remapping/dist/types/types';
 import { decode as decode_mappings } from 'sourcemap-codec';
 import { getLocator } from 'locate-character';
-import { StringWithSourcemap, sourcemap_add_offset } from '../utils/string_with_sourcemap';
+import { StringWithSourcemap, sourcemap_add_offset, combine_sourcemaps, combine_sourcemaps_map_stats } from '../utils/string_with_sourcemap';
 
 export interface Processed {
 	code: string;
@@ -113,10 +112,17 @@ function get_replacement(
 export default async function preprocess(
 	source: string,
 	preprocessor: PreprocessorGroup | PreprocessorGroup[],
-	options?: { filename?: string }
+	options?: {
+		filename?: string,
+		sourcemapWarnLoss?: number, // default 0.5
+		sourcemapEncodedWarn?: boolean // default true
+	}
 ) {
 	// @ts-ignore todo: doublecheck
 	const filename = (options && options.filename) || preprocessor.filename; // legacy
+	const sourcemapWarnLoss = (options && options.sourcemapWarnLoss != undefined) ? options.sourcemapWarnLoss : 0.5;
+	const sourcemapEncodedWarn = (options && options.sourcemapEncodedWarn != undefined) ? options.sourcemapEncodedWarn : true;
+
 	const dependencies = [];
 
 	const preprocessors = preprocessor
@@ -130,7 +136,9 @@ export default async function preprocess(
 	// sourcemap_list is sorted in reverse order from last map (index 0) to first map (index -1)
 	// so we use sourcemap_list.unshift() to add new maps
 	// https://github.com/ampproject/remapping#multiple-transformations-of-a-file
-	const sourcemap_list: (DecodedSourceMap | RawSourceMap)[] = [];
+	const sourcemap_list: Array<DecodedSourceMap | RawSourceMap> = [];
+
+	// TODO keep track: what preprocessor generated what sourcemap? to make debugging easier = detect low-resolution sourcemaps in fn combine_mappings
 
 	for (const fn of markup) {
 
@@ -150,6 +158,8 @@ export default async function preprocess(
 			);
 	}
 
+	// TODO run script and style in parallel
+
 	for (const fn of script) {
 		const get_location = getLocator(source);
 		const res = await replace_async(
@@ -216,37 +226,41 @@ export default async function preprocess(
 		sourcemap_list.unshift(res.map);
 	}
 
-	let map: RawSourceMap;
-	let map_idx = 0;
-	try {
-		map =
-			sourcemap_list.length == 0
-				? null
-				: sourcemap_list.slice(0, -1).find(m => m.sources.length !== 1) === undefined
-					? remapping( // use array interface
-							sourcemap_list,
-							() => null,
-							true // skip optional field `sourcesContent`
-						)
-					: remapping( // use loader interface
-							sourcemap_list[map_idx++],
-							function loader(sourcefile) {
-								if (sourcefile === filename)
-									return sourcemap_list[map_idx++] || null;
-									// bundle file = branch node
-								else return null; // source file = leaf node
-							} as SourceMapLoader
-						);
-	} catch (error) {
-		throw { ...error, message: error.message +
-			'\n\ncould not combine sourcemaps:\n' +
-			JSON.stringify(sourcemap_list.map(m => {
-				return { ...m, mappings: JSON.stringify(m.mappings).slice(0, 100)+' ....'};
-			}), null, 2)
-		};
+	const map_stats: combine_sourcemaps_map_stats = {
+		sourcemapWarnLoss,
+		sourcemapEncodedWarn
+		// property `result` is set by combine_sourcemaps
+	};
+
+	const map: DecodedSourceMap = combine_sourcemaps(
+		filename,
+		sourcemap_list,
+		map_stats,
+		true // explicitly decode mappings
+		// TODO remove this, when `remapping` allows to return decoded mappings, so we skip the unnecessary encode + decode steps
+	) as DecodedSourceMap;
+
+	// TODO better than console.log?
+
+	if (map_stats.result && map_stats.result.segments_lost) {
+		const { segment_loss_per_map, segments_per_map } = map_stats.result;
+		console.log('warning. svelte.preprocess seems to receive low-resolution sourcemaps. '+
+			'relative segment loss per combine_sourcemaps step: '+
+			segment_loss_per_map.map(f => f.toFixed(2)).join(' -> ')+
+			'. absolute number of segments per sourcemap: '+
+			segments_per_map.join(' -> ')+
+			'. make your preprocessors return high-resolution sourcemaps '+
+			'or increase the tolerated loss with svelte.preprocess(_, _, { sourcemapWarnLoss: 0.8 })'
+		);
 	}
 
-	if (map && !map.file) delete map.file; // skip optional field `file`
+	if (map_stats.result && map_stats.result.maps_encoded && map_stats.result.maps_encoded.length > 0) {
+		console.log('warning. svelte.preprocess received encoded sourcemaps (index '+
+			map_stats.result.maps_encoded.join(', ')+'). '+
+			'this is slow. make your sourcemap-generators return decoded mappings '+
+			'or disable this warning with svelte.preprocess(_, _, { sourcemapEncodedWarn: false })'
+		);
+	}
 
 	return {
 		// TODO return separated output, in future version where svelte.compile supports it: