Basic support for markup, style and script preprocessors

Suggestion for sveltejs#181 and sveltejs#876

Author: esarbanis

package.json

@@ -46,6 +46,7 @@
     "acorn": "^5.1.1",
     "chalk": "^2.0.1",
     "codecov": "^2.2.0",
+    "coffeescript": "^2.0.2",
     "console-group": "^0.3.2",
     "css-tree": "1.0.0-alpha22",
     "eslint": "^4.3.0",
@@ -54,13 +55,16 @@
     "estree-walker": "^0.5.1",
     "glob": "^7.1.1",
     "jsdom": "^11.1.0",
+    "less": "^2.7.3",
     "locate-character": "^2.0.0",
     "magic-string": "^0.22.3",
     "mocha": "^3.2.0",
     "nightmare": "^2.10.0",
     "node-resolve": "^1.3.3",
+    "node-sass": "^4.7.1",
     "nyc": "^11.1.0",
     "prettier": "^1.7.0",
+    "pug": "^2.0.0-rc.4",
     "reify": "^0.12.3",
     "rollup": "^0.48.2",
     "rollup-plugin-buble": "^0.15.0",
@@ -73,6 +77,7 @@
     "rollup-watch": "^4.3.1",
     "source-map": "^0.5.6",
     "source-map-support": "^0.4.8",
+    "stylus": "^0.54.5",
     "ts-node": "^3.3.0",
     "tslib": "^1.8.0",
     "typescript": "^2.6.1"

src/index.ts

@@ -4,71 +4,140 @@ import generate from './generators/dom/index';
 import generateSSR from './generators/server-side-rendering/index';
 import { assign } from './shared/index.js';
 import Stylesheet from './css/Stylesheet';
-import { Parsed, CompileOptions, Warning } from './interfaces';
+import { Parsed, CompileOptions, Warning, PreprocessOptions, Preprocessor } from './interfaces';
+import { SourceMap } from 'magic-string';
 
 const version = '__VERSION__';
 
 function normalizeOptions(options: CompileOptions): CompileOptions {
-	let normalizedOptions = assign({ generate: 'dom' }, options);
+	let normalizedOptions = assign( { generate: 'dom', preprocessor: false }, options );
 	const { onwarn, onerror } = normalizedOptions;
 	normalizedOptions.onwarn = onwarn
-		? (warning: Warning) => onwarn(warning, defaultOnwarn)
+		? (warning: Warning) => onwarn( warning, defaultOnwarn )
 		: defaultOnwarn;
 	normalizedOptions.onerror = onerror
-		? (error: Error) => onerror(error, defaultOnerror)
+		? (error: Error) => onerror( error, defaultOnerror )
 		: defaultOnerror;
 	return normalizedOptions;
 }
 
 function defaultOnwarn(warning: Warning) {
-	if (warning.loc) {
+	if ( warning.loc ) {
 		console.warn(
 			`(${warning.loc.line}:${warning.loc.column}) – ${warning.message}`
 		); // eslint-disable-line no-console
 	} else {
-		console.warn(warning.message); // eslint-disable-line no-console
+		console.warn( warning.message ); // eslint-disable-line no-console
 	}
 }
 
 function defaultOnerror(error: Error) {
 	throw error;
 }
 
-export function compile(source: string, _options: CompileOptions) {
-	const options = normalizeOptions(_options);
+function _parseAttributeValue(value: string | boolean) {
+	if ( value === 'true' || value === 'false' ) {
+		return value === 'true';
+	}
+	return (<string>value).replace( /"/ig, '' );
+}
+
+function _parseStyleAttributes(str: string) {
+	const attrs = {};
+	str.split( /\s+/ ).filter( Boolean ).forEach( attr => {
+		const [name, value] = attr.split( '=' );
+		attrs[name] = _parseAttributeValue( value );
+	} );
+	return attrs;
+}
+
+async function _doPreprocess(source, type: 'script' | 'style', preprocessor: Preprocessor) {
+	const exp = new RegExp( `<${type}([\\S\\s]*?)>([\\S\\s]*?)<\\/${type}>`, 'ig' );
+	const match = exp.exec( source );
+	if ( match ) {
+		const attributes: Record<string, string | boolean> = _parseStyleAttributes( match[1] );
+		const content: string = match[2];
+		const processed: { code: string, map?: SourceMap | string } = await preprocessor( {
+			content,
+			attributes
+		} );
+		return source.replace( content, processed.code || content );
+	}
+}
+
+export async function preprocess(source: string, options: PreprocessOptions) {
+	const { markup, style, script } = options;
+	if ( !!markup ) {
+		try {
+			const processed: { code: string, map?: SourceMap | string } = await markup( { content: source } );
+			source = processed.code;
+		} catch (error) {
+			defaultOnerror( error );
+		}
+	}
 
+	if ( !!style ) {
+		try {
+			source = await _doPreprocess( source, 'style', style );
+		} catch (error) {
+			defaultOnerror( error );
+		}
+	}
+
+	if ( !!script ) {
+		try {
+			source = await _doPreprocess( source, 'script', script );
+		} catch (error) {
+			defaultOnerror( error );
+		}
+	}
+
+	return {
+		// TODO return separated output, in future version where svelte.compile supports it:
+		// style: { code: styleCode, map: styleMap },
+		// script { code: scriptCode, map: scriptMap },
+		// markup { code: markupCode, map: markupMap },
+
+		toString() {
+			return source;
+		}
+	};
+}
+
+export function compile(source: string, _options: CompileOptions) {
+	const options = normalizeOptions( _options );
 	let parsed: Parsed;
 
 	try {
-		parsed = parse(source, options);
+		parsed = parse( source, options );
 	} catch (err) {
-		options.onerror(err);
+		options.onerror( err );
 		return;
 	}
 
-	const stylesheet = new Stylesheet(source, parsed, options.filename, options.cascade !== false);
+	const stylesheet = new Stylesheet( source, parsed, options.filename, options.cascade !== false );
 
-	validate(parsed, source, stylesheet, options);
+	validate( parsed, source, stylesheet, options );
 
 	const compiler = options.generate === 'ssr' ? generateSSR : generate;
 
-	return compiler(parsed, source, stylesheet, options);
-}
+	return compiler( parsed, source, stylesheet, options );
+};
 
 export function create(source: string, _options: CompileOptions = {}) {
 	_options.format = 'eval';
 
-	const compiled = compile(source, _options);
+	const compiled = compile( source, _options );
 
-	if (!compiled || !compiled.code) {
+	if ( !compiled || !compiled.code ) {
 		return;
 	}
 
 	try {
-		return (0,eval)(compiled.code);
+		return (0, eval)( compiled.code );
 	} catch (err) {
-		if (_options.onerror) {
-			_options.onerror(err);
+		if ( _options.onerror ) {
+			_options.onerror( err );
 			return;
 		} else {
 			throw err;

src/interfaces.ts

@@ -1,3 +1,5 @@
+import {SourceMap} from 'magic-string';
+
 export interface Node {
 	start: number;
 	end: number;
@@ -59,6 +61,7 @@ export interface CompileOptions {
 
 	onerror?: (error: Error) => void;
 	onwarn?: (warning: Warning) => void;
+	preprocessor?: ((raw: string) => string) | false ;
 }
 
 export interface GenerateOptions {
@@ -77,4 +80,12 @@ export interface Visitor {
 export interface CustomElementOptions {
 	tag?: string;
 	props?: string[];
-}
+}
+
+export interface PreprocessOptions {
+	markup?: (options: {content: string}) => { code: string, map?: SourceMap | string };
+	style?: Preprocessor;
+	script?: Preprocessor;
+}
+
+export type Preprocessor = (options: {content: string, attributes: Record<string, string | boolean>}) => { code: string, map?: SourceMap | string };

test/preprocess/index.js

@@ -0,0 +1,169 @@
+import assert from 'assert';
+import * as fs from 'fs';
+import {parse} from 'acorn';
+import {addLineNumbers, env, normalizeHtml, svelte} from '../helpers.js';
+
+function tryRequire(file) {
+	try {
+		const mod = require(file);
+		return mod.default || mod;
+	} catch (err) {
+		if (err.code !== 'MODULE_NOT_FOUND') throw err;
+		return null;
+	}
+}
+
+function normalizeWarning(warning) {
+	warning.frame = warning.frame.replace(/^\n/, '').
+		replace(/^\t+/gm, '').
+		replace(/\s+$/gm, '');
+	delete warning.filename;
+	delete warning.toString;
+	return warning;
+}
+
+function checkCodeIsValid(code) {
+	try {
+		parse(code);
+	} catch (err) {
+		console.error(addLineNumbers(code));
+		throw new Error(err.message);
+	}
+}
+
+describe('preprocess', () => {
+	fs.readdirSync('test/preprocess/samples').forEach(dir => {
+		if (dir[0] === '.') return;
+
+		// add .solo to a sample directory name to only run that test
+		const solo = /\.solo/.test(dir);
+		const skip = /\.skip/.test(dir);
+
+		if (solo && process.env.CI) {
+			throw new Error('Forgot to remove `solo: true` from test');
+		}
+
+		(solo ? it.only : skip ? it.skip : it)(dir, () => {
+			const config = tryRequire(`./samples/${dir}/_config.js`) || {};
+			const input = fs.existsSync(`test/preprocess/samples/${dir}/input.pug`) ?
+				fs.readFileSync(`test/preprocess/samples/${dir}/input.pug`,
+					'utf-8').replace(/\s+$/, '') :
+				fs.readFileSync(`test/preprocess/samples/${dir}/input.html`,
+					'utf-8').replace(/\s+$/, '');
+
+			svelte.preprocess(input, config).
+				then(processed => processed.toString()).
+				then(processed => {
+
+					const expectedWarnings = (config.warnings || []).map(
+						normalizeWarning);
+					const domWarnings = [];
+					const ssrWarnings = [];
+
+					const dom = svelte.compile(
+						processed,
+						Object.assign(config, {
+							format: 'iife',
+							name: 'SvelteComponent',
+							onwarn: warning => {
+								domWarnings.push(warning);
+							},
+						}),
+					);
+
+					const ssr = svelte.compile(
+						processed,
+						Object.assign(config, {
+							format: 'iife',
+							generate: 'ssr',
+							name: 'SvelteComponent',
+							onwarn: warning => {
+								ssrWarnings.push(warning);
+							},
+						}),
+					);
+
+					// check the code is valid
+					checkCodeIsValid(dom.code);
+					checkCodeIsValid(ssr.code);
+
+					assert.equal(dom.css, ssr.css);
+
+					assert.deepEqual(
+						domWarnings.map(normalizeWarning),
+						ssrWarnings.map(normalizeWarning),
+					);
+					assert.deepEqual(domWarnings.map(normalizeWarning), expectedWarnings);
+
+					const expected = {
+						html: read(`test/preprocess/samples/${dir}/expected.html`),
+						css: read(`test/preprocess/samples/${dir}/expected.css`),
+					};
+
+					if (expected.css !== null) {
+						fs.writeFileSync(`test/preprocess/samples/${dir}/_actual.css`,
+							dom.css);
+						assert.equal(dom.css.replace(/svelte-\d+/g, 'svelte-xyz'),
+							expected.css);
+					}
+
+					// verify that the right elements have scoping selectors
+					if (expected.html !== null) {
+						const window = env();
+
+						// dom
+						try {
+							const Component = eval(
+								`(function () { ${dom.code}; return SvelteComponent; }())`,
+							);
+							const target = window.document.querySelector('main');
+
+							new Component({target, data: config.data});
+							const html = target.innerHTML;
+
+							fs.writeFileSync(`test/preprocess/samples/${dir}/_actual.html`,
+								html);
+
+							assert.equal(
+								normalizeHtml(window,
+									html.replace(/svelte-\d+/g, 'svelte-xyz')),
+								normalizeHtml(window, expected.html),
+							);
+						} catch (err) {
+							console.log(dom.code);
+							throw err;
+						}
+
+						// ssr
+						try {
+							const component = eval(
+								`(function () { ${ssr.code}; return SvelteComponent; }())`,
+							);
+
+							assert.equal(
+								normalizeHtml(
+									window,
+									component.render(config.data).
+										replace(/svelte-\d+/g, 'svelte-xyz'),
+								),
+								normalizeHtml(window, expected.html),
+							);
+						} catch (err) {
+							console.log(ssr.code);
+							throw err;
+						}
+					}
+				}).catch(error => {
+				throw error;
+			});
+		});
+	});
+});
+
+function read(file) {
+	try {
+		return fs.readFileSync(file, 'utf-8');
+	} catch (err) {
+		return null;
+	}
+}