From 6861d22e37d49b23b96f7f76ced0f652e8738515 Mon Sep 17 00:00:00 2001 From: DimitarTachev Date: Wed, 18 Sep 2019 11:13:34 +0300 Subject: [PATCH 1/6] feat: snapshot in docker container when the local tools are not available --- plugins/NativeScriptSnapshotPlugin/index.js | 1 + .../NativeScriptSnapshotPlugin/options.json | 9 +- ...oject-snapshot-generator-cli-ags-parser.js | 6 +- .../android/project-snapshot-generator.js | 11 +- snapshot/android/snapshot-generator.js | 297 +++++++++++++----- snapshot/android/utils.js | 49 ++- templates/webpack.angular.js | 2 + templates/webpack.javascript.js | 2 + templates/webpack.typescript.js | 4 +- templates/webpack.vue.js | 2 + 10 files changed, 301 insertions(+), 82 deletions(-) diff --git a/plugins/NativeScriptSnapshotPlugin/index.js b/plugins/NativeScriptSnapshotPlugin/index.js index 97cebd7e..3c583ac3 100644 --- a/plugins/NativeScriptSnapshotPlugin/index.js +++ b/plugins/NativeScriptSnapshotPlugin/index.js @@ -113,6 +113,7 @@ exports.NativeScriptSnapshotPlugin = (function () { useLibs: options.useLibs, androidNdkPath: options.androidNdkPath, v8Version: options.v8Version, + snapshotInDocker: options.snapshotInDocker }).then(() => { // Make the original files empty inputFiles.forEach(inputFile => diff --git a/plugins/NativeScriptSnapshotPlugin/options.json b/plugins/NativeScriptSnapshotPlugin/options.json index d20b74ac..c82eb401 100644 --- a/plugins/NativeScriptSnapshotPlugin/options.json +++ b/plugins/NativeScriptSnapshotPlugin/options.json @@ -5,8 +5,8 @@ "type": "string" }, "angular": { - "type": "boolean", - "default": false + "type": "boolean", + "default": false }, "chunk": { "type": "string" @@ -39,6 +39,9 @@ "type": "boolean", "default": false }, + "snapshotInDocker": { + "type": "boolean" + }, "v8Version": { "type": "string" }, @@ -56,4 +59,4 @@ "webpackConfig" ], "additionalProperties": false -} +} \ No newline at end of file diff --git a/snapshot/android/project-snapshot-generator-cli-ags-parser.js b/snapshot/android/project-snapshot-generator-cli-ags-parser.js index 76af75b1..97cea86d 100644 --- a/snapshot/android/project-snapshot-generator-cli-ags-parser.js +++ b/snapshot/android/project-snapshot-generator-cli-ags-parser.js @@ -9,6 +9,10 @@ module.exports = function parseProjectSnapshotGeneratorArgs() { result.useLibs = parseBool(result.useLibs); } + if (result.snapshotInDocker !== undefined) { + result.snapshotInDocker = parseBool(result.snapshotInDocker); + } + if (result.install !== undefined) { result.install = parseBool(result.install); } @@ -22,7 +26,7 @@ function parseJsonFromProcessArgs() { var currentKey = ""; var currentValue = ""; - args.forEach(function(value, index, array) { + args.forEach(function (value, index, array) { if (value.startsWith("--")) { // if is key addKeyAndValueToResult(currentKey, currentValue, result); currentKey = value.slice(2); diff --git a/snapshot/android/project-snapshot-generator.js b/snapshot/android/project-snapshot-generator.js index a208024f..00053708 100644 --- a/snapshot/android/project-snapshot-generator.js +++ b/snapshot/android/project-snapshot-generator.js @@ -114,7 +114,9 @@ ProjectSnapshotGenerator.installSnapshotArtefacts = function (projectRoot) { shelljs.cp("-R", blobsSrcPath + "/", resolve(appPath, "../snapshots")); /* - Rename TNSSnapshot.blob files to snapshot.blob files. The xxd tool uses the file name for the name of the static array. This is why the *.blob files are initially named TNSSnapshot.blob. After the xxd step, they must be renamed to snapshot.blob, because this is the filename that the Android runtime is looking for. + Rename TNSSnapshot.blob files to snapshot.blob files. The xxd tool uses the file name for the name of the static array. + This is why the *.blob files are initially named TNSSnapshot.blob. + After the xxd step, they must be renamed to snapshot.blob, because this is the filename that the Android runtime is looking for. */ shelljs.exec("find " + blobsDestinationPath + " -name '*.blob' -execdir mv {} snapshot.blob ';'"); @@ -170,7 +172,7 @@ ProjectSnapshotGenerator.prototype.getV8Version = function (generationOptions) { // try to get the V8 Version from the settings.json file in android runtime folder const runtimeV8Version = getAndroidV8Version(this.options.projectRoot); - if(runtimeV8Version) { + if (runtimeV8Version) { return resolve(runtimeV8Version); } @@ -184,7 +186,7 @@ ProjectSnapshotGenerator.prototype.getV8Version = function (generationOptions) { const version = findV8Version(runtimeVersion, latestVersionsMap) return resolve(version); }) - .catch(reject); + .catch(reject); } else { return resolve(v8Version); } @@ -254,7 +256,8 @@ ProjectSnapshotGenerator.prototype.generate = function (generationOptions) { useLibs: generationOptions.useLibs || false, inputFiles: generationOptions.inputFiles || [join(this.options.projectRoot, "__snapshot.js")], androidNdkPath, - mksnapshotParams: mksnapshotParams + mksnapshotParams: mksnapshotParams, + snapshotInDocker: generationOptions.snapshotInDocker }; return generator.generate(options).then(() => { diff --git a/snapshot/android/snapshot-generator.js b/snapshot/android/snapshot-generator.js index 4de65fa6..85821f77 100644 --- a/snapshot/android/snapshot-generator.js +++ b/snapshot/android/snapshot-generator.js @@ -1,12 +1,13 @@ const fs = require("fs"); -const { dirname, join, EOL } = require("path"); -const os = require("os"); +const { dirname, relative, join, EOL } = require("path"); const child_process = require("child_process"); const shelljs = require("shelljs"); -const { createDirectory, downloadFile } = require("./utils"); +const { createDirectory, downloadFile, getHostOS, getHostOSArch, CONSTANTS, has32BitArch, isMacOSCatalinaOrHigher } = require("./utils"); +const SNAPSHOTS_DOCKER_IMAGE = "nativescript/v8-snapshot:latest"; +const SNAPSHOT_TOOLS_DIR_NAME = "mksnapshot-tools"; const NDK_BUILD_SEED_PATH = join(__dirname, "snapshot-generator-tools/ndk-build"); const BUNDLE_PREAMBLE_PATH = join(__dirname, "snapshot-generator-tools/bundle-preamble.js"); const BUNDLE_ENDING_PATH = join(__dirname, "snapshot-generator-tools/bundle-ending.js"); @@ -14,6 +15,8 @@ const INCLUDE_GRADLE_PATH = join(__dirname, "snapshot-generator-tools/include.gr const MKSNAPSHOT_TOOLS_DOWNLOAD_ROOT_URL = "https://raw.githubusercontent.com/NativeScript/mksnapshot-tools/production/"; const MKSNAPSHOT_TOOLS_DOWNLOAD_TIMEOUT = 60000; const SNAPSHOT_BLOB_NAME = "TNSSnapshot"; +const DOCKER_IMAGE_OS = "linux"; +const DOCKER_IMAGE_ARCH = "x64"; function shellJsExecuteInDir(dir, action) { const currentDir = shelljs.pwd(); @@ -25,17 +28,6 @@ function shellJsExecuteInDir(dir, action) { } } -function getHostOS() { - const hostOS = os.type().toLowerCase(); - if (hostOS.startsWith("darwin")) - return "darwin"; - if (hostOS.startsWith("linux")) - return "linux"; - if (hostOS.startsWith("win")) - return "win"; - return hostOS; -} - function SnapshotGenerator(options) { this.buildPath = options.buildPath || join(__dirname, "build"); } @@ -43,8 +35,24 @@ module.exports = SnapshotGenerator; SnapshotGenerator.SNAPSHOT_PACKAGE_NANE = "nativescript-android-snapshot"; +SnapshotGenerator.prototype.shouldSnapshotInDocker = function (hostOS) { + let shouldSnapshotInDocker = false; + const generateInDockerMessage = "The snapshots will be generated in a docker container."; + if (hostOS == CONSTANTS.WIN_OS_NAME) { + console.log(`The V8 snapshot tools are not supported on Windows. ${generateInDockerMessage}`); + shouldSnapshotInDocker = true; + } else if (isMacOSCatalinaOrHigher() && has32BitArch(options.targetArchs)) { + console.log(`Starting from macOS Catalina, the 32-bit processes are no longer supported. ${generateInDockerMessage}`); + shouldSnapshotInDocker = true; + } + + return shouldSnapshotInDocker; +} + SnapshotGenerator.prototype.preprocessInputFiles = function (inputFiles, outputFile) { // Make some modifcations on the original bundle and save it on the specified path + + const bundlePreambleContent = fs.readFileSync(BUNDLE_PREAMBLE_PATH, "utf8"); const bundleEndingContent = fs.readFileSync(BUNDLE_ENDING_PATH, "utf8"); @@ -67,9 +75,34 @@ SnapshotGenerator.prototype.preprocessInputFiles = function (inputFiles, outputF const snapshotToolsDownloads = {}; -SnapshotGenerator.prototype.downloadMksnapshotTool = function (snapshotToolsPath, v8Version, targetArch) { - const hostOS = getHostOS(); - const mksnapshotToolRelativePath = join("mksnapshot-tools", "v8-v" + v8Version, hostOS + "-" + os.arch(), "mksnapshot-" + targetArch); +SnapshotGenerator.prototype.downloadMkSnapshotTools = function (snapshotToolsPath, v8Version, targetArchs, useDocker) { + var toolsOS = ""; + var toolsArch = ""; + if (typeof useDocker === "boolean") { + if (useDocker) { + toolsOS = DOCKER_IMAGE_OS; + toolsArch = DOCKER_IMAGE_ARCH; + } else { + toolsOS = getHostOS(); + toolsArch = getHostOSArch(); + } + } else { + toolsOS = getHostOS(); + toolsArch = getHostOSArch(); + fallbackToDocker = true; + } + + + + return Promise.all(targetArchs.map((arch) => { + return this.downloadMkSnapshotTool(snapshotToolsPath, v8Version, arch, toolsOS, toolsArch).then(path => { + return { path, arch }; + }); + })); +} + +SnapshotGenerator.prototype.downloadMkSnapshotTool = function (snapshotToolsPath, v8Version, targetArch, hostOS, hostArch) { + const mksnapshotToolRelativePath = join(SNAPSHOT_TOOLS_DIR_NAME, "v8-v" + v8Version, hostOS + "-" + hostArch, "mksnapshot-" + targetArch); const mksnapshotToolPath = join(snapshotToolsPath, mksnapshotToolRelativePath); if (fs.existsSync(mksnapshotToolPath)) return Promise.resolve(mksnapshotToolPath); @@ -106,70 +139,73 @@ SnapshotGenerator.prototype.convertToAndroidArchName = function (archName) { } } -SnapshotGenerator.prototype.runMksnapshotTool = function (snapshotToolsPath, inputFile, v8Version, targetArchs, buildCSource, mksnapshotParams) { +SnapshotGenerator.prototype.generateSnapshots = function (snapshotToolsPath, inputFile, v8Version, targetArchs, buildCSource, mksnapshotParams, useDocker) { // Cleans the snapshot build folder + debugger; shelljs.rm("-rf", join(this.buildPath, "snapshots")); + return this.downloadMkSnapshotTools(snapshotToolsPath, v8Version, targetArchs, useDocker).then((localTools) => { + var snapshotInDocker = !!useDocker; + var shouldDownloadDockerTools = false; + if (!snapshotInDocker) { + snapshotInDocker = localTools.some(tool => !this.canUseSnapshotTool(tool.path)); + shouldDownloadDockerTools = snapshotInDocker; + } - const mksnapshotStdErrPath = join(this.buildPath, "mksnapshot-stderr.txt"); - - return Promise.all(targetArchs.map((arch) => { - return this.downloadMksnapshotTool(snapshotToolsPath, v8Version, arch).then((currentArchMksnapshotToolPath) => { - if (!fs.existsSync(currentArchMksnapshotToolPath)) { - throw new Error("Can't find mksnapshot tool for " + arch + " at path " + currentArchMksnapshotToolPath); - } - - const androidArch = this.convertToAndroidArchName(arch); - console.log("***** Generating snapshot for " + androidArch + " *****"); + if (shouldDownloadDockerTools) { + return this.downloadMkSnapshotTools(snapshotToolsPath, v8Version, targetArchs, true).then((dockerTools) => { + console.log(`Executing '${snapshotToolPath}' in a docker container.`); + return this.runMKSnapshotTools(snapshotToolsPath, dockerTools, inputFile, mksnapshotParams, buildCSource, snapshotInDocker); + }); + } else { + return this.runMKSnapshotTools(snapshotToolsPath, localTools, inputFile, mksnapshotParams, buildCSource, snapshotInDocker); + } + }); +} - // Generate .blob file - const currentArchBlobOutputPath = join(this.buildPath, "snapshots/blobs", androidArch); - shelljs.mkdir("-p", currentArchBlobOutputPath); - var params = "--profile_deserialization"; - if (mksnapshotParams) { - // Starting from android runtime 5.3.0, the parameters passed to mksnapshot are read from the settings.json file - params = mksnapshotParams; - } - const command = `${currentArchMksnapshotToolPath} ${inputFile} --startup_blob ${join(currentArchBlobOutputPath, `${SNAPSHOT_BLOB_NAME}.blob`)} ${params}`; - - return new Promise((resolve, reject) => { - const child = child_process.exec(command, { encoding: "utf8" }, (error, stdout, stderr) => { - const errorHeader = `Target architecture: ${androidArch}\n`; - let errorFooter = ``; - if (stderr.length || error) { - try { - require(inputFile); - } catch (e) { - errorFooter = `\nJavaScript execution error: ${e.stack}$`; - } - } - if (stderr.length) { - const message = `${errorHeader}${stderr}${errorFooter}`; - reject(new Error(message)); - } else if (error) { - error.message = `${errorHeader}${error.message}${errorFooter}`; - reject(error); - } else { - console.log(stdout); - resolve(); - } - }) - }).then(() => { - // Generate .c file - if (buildCSource) { - const currentArchSrcOutputPath = join(this.buildPath, "snapshots/src", androidArch); - shelljs.mkdir("-p", currentArchSrcOutputPath); - shellJsExecuteInDir(currentArchBlobOutputPath, function () { - shelljs.exec(`xxd -i ${SNAPSHOT_BLOB_NAME}.blob > ${join(currentArchSrcOutputPath, `${SNAPSHOT_BLOB_NAME}.c`)}`); - }); - } +SnapshotGenerator.prototype.runMKSnapshotTools = function (snapshotToolsBasePath, snapshotTools, inputFile, mksnapshotParams, buildCSource, snapshotInDocker) { + let currentSnapshotOperation = Promise.resolve(); + const canRunInParallel = snapshotTools.length <= 1 || !snapshotInDocker; + return Promise.all(snapshotTools.map((tool) => { + if (canRunInParallel) { + return this.runMKSnapshotTool(tool, mksnapshotParams, inputFile, snapshotInDocker, snapshotToolsBasePath, buildCSource); + } else { + currentSnapshotOperation = currentSnapshotOperation.then(() => { + return this.runMKSnapshotTool(tool, mksnapshotParams, inputFile, snapshotInDocker, snapshotToolsBasePath, buildCSource); }); - }); + + return currentSnapshotOperation; + } })).then(() => { console.log("***** Finished generating snapshots. *****"); }); } +SnapshotGenerator.prototype.canUseSnapshotTool = function (snapshotToolPath) { + try { + child_process.execSync(`${snapshotToolPath} --help`); + return true; + } + catch (error) { + console.log(`Unable to execute '${snapshotToolPath}' locally.Error message: '${error.message}'`); + return false; + } +} + +SnapshotGenerator.prototype.setupDocker = function () { + const installMessage = "Install Docker and add it to your PATH in order to build snapshots."; + try { + // e.g. Docker version 19.03.2, build 6a30dfc + child_process.execSync(`docker --version`); + // TODO: require a minimum version? + } + catch (error) { + throw new Error(`Docker installation cannot be found. ${installMessage}`); + } + + child_process.execSync(`docker pull ${SNAPSHOTS_DOCKER_IMAGE}`); +} + SnapshotGenerator.prototype.buildSnapshotLibs = function (androidNdkBuildPath, targetArchs) { // Compile *.c files to produce *.so libraries with ndk-build tool const ndkBuildPath = join(this.buildPath, "ndk-build"); @@ -199,15 +235,18 @@ SnapshotGenerator.prototype.generate = function (options) { console.log("***** Starting snapshot generation using V8 version: ", options.v8Version); this.preprocessInputFiles(options.inputFiles, preprocessedInputFile); + const hostOS = getHostOS(); + const snapshotInDocker = options.snapshotInDocker || this.shouldSnapshotInDocker(hostOS); // generates the actual .blob and .c files - return this.runMksnapshotTool( + return this.generateSnapshots( options.snapshotToolsPath, preprocessedInputFile, options.v8Version, options.targetArchs, options.useLibs, - options.mksnapshotParams + options.mksnapshotParams, + snapshotInDocker ).then(() => { this.buildIncludeGradle(); if (options.useLibs) { @@ -217,3 +256,117 @@ SnapshotGenerator.prototype.generate = function (options) { return this.buildPath; }); } + +SnapshotGenerator.prototype.runMKSnapshotTool = function (tool, mksnapshotParams, inputFile, snapshotInDocker, snapshotToolsBasePath, buildCSource) { + const currentArchMksnapshotToolPath = tool.path; + const arch = tool.arch; + if (!fs.existsSync(currentArchMksnapshotToolPath)) { + throw new Error("Can't find mksnapshot tool for " + arch + " at path " + currentArchMksnapshotToolPath); + } + + const androidArch = this.convertToAndroidArchName(arch); + console.log("***** Generating snapshot for " + androidArch + " *****"); + // Generate .blob file + const currentArchBlobOutputPath = join(this.buildPath, "snapshots/blobs", androidArch); + shelljs.mkdir("-p", currentArchBlobOutputPath); + let dockerCurrentArchBlobOutputPath = ""; + var params = "--profile_deserialization"; + if (mksnapshotParams) { + // Starting from android runtime 5.3.0, the parameters passed to mksnapshot are read from the settings.json file + params = mksnapshotParams; + } + return new Promise((resolve, reject) => { + let snapshotToolPath = currentArchMksnapshotToolPath; + const inputFileDir = dirname(inputFile); + let inputFilePath = inputFile; + const appDirInDocker = "/app"; + let outputPath = currentArchBlobOutputPath; + if (snapshotInDocker) { + this.setupDocker(); + // create snapshots dir in docker + dockerCurrentArchBlobOutputPath = join(inputFileDir, "snapshots", androidArch); + shelljs.mkdir("-p", dockerCurrentArchBlobOutputPath); + outputPath = join(appDirInDocker, relative(inputFileDir, dockerCurrentArchBlobOutputPath)); + // calculate input in docker + inputFilePath = join(appDirInDocker, relative(inputFileDir, inputFile)); + // calculate snapshotTool path + snapshotToolPath = join(appDirInDocker, relative(snapshotToolsBasePath, currentArchMksnapshotToolPath)); + } + let command = `${snapshotToolPath} ${inputFilePath} --startup_blob ${join(outputPath, `${SNAPSHOT_BLOB_NAME}.blob`)} ${params}`; + if (snapshotInDocker) { + // we cannot mount the source tools folder as its not shared by default: + // docker: Error response from daemon: Mounts denied: + // The path /var/folders/h2/1yck52fx2mg7c790vhcw90s8087sk8/T/snapshot-tools/mksnapshot-tools + // is not shared from OS X and is not known to Docker. + const currentToolRelativeToSnapshotTools = relative(snapshotToolsBasePath, currentArchMksnapshotToolPath); + const currentToolDestination = join(inputFileDir, currentToolRelativeToSnapshotTools) + debugger; + createDirectory(dirname(currentToolDestination)); + shelljs.cp(currentArchMksnapshotToolPath, join(inputFileDir, currentToolRelativeToSnapshotTools)); + command = `docker run -v "${inputFileDir}:${appDirInDocker}" ${SNAPSHOTS_DOCKER_IMAGE} /bin/sh -c "${command}"`; + } + child_process.exec(command, { encoding: "utf8" }, (error, stdout, stderr) => { + const errorHeader = `Target architecture: ${androidArch}\n`; + let errorFooter = ``; + if (stderr.length || error) { + try { + require(inputFile); + } + catch (e) { + errorFooter = `\nJavaScript execution error: ${e.stack}$`; + } + } + if (stderr.length) { + const message = `${errorHeader}${stderr}${errorFooter}`; + reject(new Error(message)); + } + else if (error) { + error.message = `${errorHeader}${error.message}${errorFooter}`; + reject(error); + } + else { + console.log(stdout); + // Generate .c file + /// TODO: test in docker + if (buildCSource) { + let srcOutputPath = ""; + let dockerCurrentArchSrcOutputPath = ""; + if (snapshotInDocker) { + dockerCurrentArchSrcOutputPath = join(inputFileDir, "snapshots/src", androidArch); + shelljs.mkdir("-p", dockerCurrentArchSrcOutputPath); + srcOutputPath = join(appDirInDocker, relative(inputFileDir, dockerCurrentArchSrcOutputPath)); + } else { + srcOutputPath = join(this.buildPath, "snapshots/src", androidArch); + shelljs.mkdir("-p", srcOutputPath); + } + + const buildCSourceCommand = `xxd -i ${SNAPSHOT_BLOB_NAME}.blob > ${join(srcOutputPath, `${SNAPSHOT_BLOB_NAME}.c`)}`; + if (snapshotInDocker) { + // add vim in order to get xxd + const commandInDocker = + `docker run -v "${inputFileDir}:${appDirInDocker}" ${SNAPSHOTS_DOCKER_IMAGE} /bin/sh -c "cd ${outputPath} && apk add vim && ${buildCSourceCommand}"`; + child_process.execSync(commandInDocker); + } else { + shellJsExecuteInDir(currentArchBlobOutputPath, function () { + shelljs.exec(buildCSourceCommand); + }); + } + + if (snapshotInDocker) { + createDirectory(join(this.buildPath, "snapshots/src", androidArch)); + shelljs.cp("-R", dockerCurrentArchSrcOutputPath + "/", join(this.buildPath, "snapshots/src", androidArch)); + shelljs.rm("-rf", dockerCurrentArchSrcOutputPath); + } + } + + // TODO: move cleanup to afterPrepare? + if (snapshotInDocker) { + shelljs.cp("-R", dockerCurrentArchBlobOutputPath + "/", currentArchBlobOutputPath); + shelljs.rm("-rf", dockerCurrentArchBlobOutputPath); + } + resolve(); + } + }); + }); +} + diff --git a/snapshot/android/utils.js b/snapshot/android/utils.js index 2b63ae28..8d822e53 100644 --- a/snapshot/android/utils.js +++ b/snapshot/android/utils.js @@ -1,17 +1,59 @@ const { chmodSync, createWriteStream, existsSync } = require("fs"); const { tmpdir, EOL } = require("os"); -const { dirname, join } = require("path"); +const { join } = require("path"); +const os = require("os"); const { mkdir } = require("shelljs"); const { get } = require("request"); const { getProxySettings } = require("proxy-lib"); +const semver = require("semver"); const CONSTANTS = { SNAPSHOT_TMP_DIR: join(tmpdir(), "snapshot-tools"), + MAC_OS_NAME: "darwin", + WIN_OS_NAME: "win", + LINUX_OS_NAME: "linux" }; const createDirectory = dir => mkdir('-p', dir); +function getHostOS() { + const hostOS = os.type().toLowerCase(); + if (hostOS.startsWith(CONSTANTS.MAC_OS_NAME)) + return CONSTANTS.MAC_OS_NAME; + if (hostOS.startsWith(CONSTANTS.LINUX_OS_NAME)) + return CONSTANTS.LINUX_OS_NAME; + if (hostOS.startsWith(CONSTANTS.WIN_OS_NAME)) + return CONSTANTS.WIN_OS_NAME; + return hostOS; +} + +function getHostOSVersion() { + return os.release(); +} + +function getHostOSArch() { + return os.arch(); +} + +function has32BitArch(targetArchs) { + return Array.isArray(targetArchs) && targetArchs.some(arch => arch === "arm" || arch === "ia32") +} + +function isMacOSCatalinaOrHigher() { + const isCatalinaOrHigher = false; + const catalinaVersion = "19.0.0"; + const hostOS = getHostOS(); + if (hostOS === CONSTANTS.MAC_OS_NAME) { + const hostOSVersion = getHostOSVersion(); + if (semver.gte(hostOSVersion, catalinaVersion)) { + isCatalinaOrHigher = true; + } + } + + return isCatalinaOrHigher; +} + const downloadFile = (url, destinationFilePath, timeout) => new Promise((resolve, reject) => { getRequestOptions(url, timeout) @@ -64,6 +106,11 @@ const getRequestOptions = (url, timeout) => module.exports = { CONSTANTS, createDirectory, + has32BitArch, + getHostOS, + getHostOSVersion, + getHostOSArch, + isMacOSCatalinaOrHigher, downloadFile, getJsonFile, }; diff --git a/templates/webpack.angular.js b/templates/webpack.angular.js index e6110ff0..f15d8bea 100644 --- a/templates/webpack.angular.js +++ b/templates/webpack.angular.js @@ -50,6 +50,7 @@ module.exports = env => { hmr, // --env.hmr, unitTesting, // --env.unitTesting verbose, // --env.verbose + snapshotInDocker // --env.snapshotInDocker } = env; const isAnySourceMapEnabled = !!sourceMap || !!hiddenSourceMap; @@ -308,6 +309,7 @@ module.exports = env => { ], projectRoot, webpackConfig: config, + snapshotInDocker })); } diff --git a/templates/webpack.javascript.js b/templates/webpack.javascript.js index 7460dd2a..05142a36 100644 --- a/templates/webpack.javascript.js +++ b/templates/webpack.javascript.js @@ -44,6 +44,7 @@ module.exports = env => { hmr, // --env.hmr, unitTesting, // --env.unitTesting, verbose, // --env.verbose + snapshotInDocker // --env.snapshotInDocker } = env; const isAnySourceMapEnabled = !!sourceMap || !!hiddenSourceMap; @@ -249,6 +250,7 @@ module.exports = env => { ], projectRoot, webpackConfig: config, + snapshotInDocker })); } diff --git a/templates/webpack.typescript.js b/templates/webpack.typescript.js index 44a60a10..d124be1a 100644 --- a/templates/webpack.typescript.js +++ b/templates/webpack.typescript.js @@ -45,6 +45,7 @@ module.exports = env => { hmr, // --env.hmr, unitTesting, // --env.unitTesting, verbose, // --env.verbose + snapshotInDocker // --env.snapshotInDocker } = env; const isAnySourceMapEnabled = !!sourceMap || !!hiddenSourceMap; const externals = nsWebpack.getConvertedExternals(env.externals); @@ -183,7 +184,7 @@ module.exports = env => { }, ].filter(loader => !!loader) }, - + { test: /\.(ts|css|scss|html|xml)$/, use: "nativescript-dev-webpack/hmr/hot-loader" @@ -277,6 +278,7 @@ module.exports = env => { ], projectRoot, webpackConfig: config, + snapshotInDocker })); } diff --git a/templates/webpack.vue.js b/templates/webpack.vue.js index 28bbfe9f..ba273189 100644 --- a/templates/webpack.vue.js +++ b/templates/webpack.vue.js @@ -47,6 +47,7 @@ module.exports = env => { hiddenSourceMap, // --env.hiddenSourceMap unitTesting, // --env.unitTesting verbose, // --env.verbose + snapshotInDocker // --env.snapshotInDocker } = env; const isAnySourceMapEnabled = !!sourceMap || !!hiddenSourceMap; @@ -296,6 +297,7 @@ module.exports = env => { ], projectRoot, webpackConfig: config, + snapshotInDocker })); } From a8138d266d8d5651bf96328a64264dcba53999c1 Mon Sep 17 00:00:00 2001 From: DimitarTachev Date: Wed, 18 Sep 2019 16:25:17 +0300 Subject: [PATCH 2/6] refactor: make the snapshot generation more readable --- snapshot/android/snapshot-generator.js | 261 +++++++++++++------------ snapshot/android/utils.js | 10 +- 2 files changed, 142 insertions(+), 129 deletions(-) diff --git a/snapshot/android/snapshot-generator.js b/snapshot/android/snapshot-generator.js index 85821f77..cc1c8e62 100644 --- a/snapshot/android/snapshot-generator.js +++ b/snapshot/android/snapshot-generator.js @@ -1,10 +1,11 @@ const fs = require("fs"); const { dirname, relative, join, EOL } = require("path"); const child_process = require("child_process"); +const { convertToUnixPath } = require("../../lib/utils"); const shelljs = require("shelljs"); -const { createDirectory, downloadFile, getHostOS, getHostOSArch, CONSTANTS, has32BitArch, isMacOSCatalinaOrHigher } = require("./utils"); +const { createDirectory, downloadFile, getHostOS, getHostOSArch, CONSTANTS, has32BitArch, isMacOSCatalinaOrHigher, isSubPath } = require("./utils"); const SNAPSHOTS_DOCKER_IMAGE = "nativescript/v8-snapshot:latest"; const SNAPSHOT_TOOLS_DIR_NAME = "mksnapshot-tools"; @@ -75,33 +76,25 @@ SnapshotGenerator.prototype.preprocessInputFiles = function (inputFiles, outputF const snapshotToolsDownloads = {}; -SnapshotGenerator.prototype.downloadMkSnapshotTools = function (snapshotToolsPath, v8Version, targetArchs, useDocker) { +SnapshotGenerator.prototype.downloadMksnapshotTools = function (snapshotToolsPath, v8Version, targetArchs, snapshotInDocker) { var toolsOS = ""; var toolsArch = ""; - if (typeof useDocker === "boolean") { - if (useDocker) { - toolsOS = DOCKER_IMAGE_OS; - toolsArch = DOCKER_IMAGE_ARCH; - } else { - toolsOS = getHostOS(); - toolsArch = getHostOSArch(); - } + if (snapshotInDocker) { + toolsOS = DOCKER_IMAGE_OS; + toolsArch = DOCKER_IMAGE_ARCH; } else { toolsOS = getHostOS(); toolsArch = getHostOSArch(); - fallbackToDocker = true; } - - return Promise.all(targetArchs.map((arch) => { - return this.downloadMkSnapshotTool(snapshotToolsPath, v8Version, arch, toolsOS, toolsArch).then(path => { + return this.downloadMksnapshotTool(snapshotToolsPath, v8Version, arch, toolsOS, toolsArch).then(path => { return { path, arch }; }); })); } -SnapshotGenerator.prototype.downloadMkSnapshotTool = function (snapshotToolsPath, v8Version, targetArch, hostOS, hostArch) { +SnapshotGenerator.prototype.downloadMksnapshotTool = function (snapshotToolsPath, v8Version, targetArch, hostOS, hostArch) { const mksnapshotToolRelativePath = join(SNAPSHOT_TOOLS_DIR_NAME, "v8-v" + v8Version, hostOS + "-" + hostArch, "mksnapshot-" + targetArch); const mksnapshotToolPath = join(snapshotToolsPath, mksnapshotToolRelativePath); if (fs.existsSync(mksnapshotToolPath)) @@ -139,12 +132,10 @@ SnapshotGenerator.prototype.convertToAndroidArchName = function (archName) { } } -SnapshotGenerator.prototype.generateSnapshots = function (snapshotToolsPath, inputFile, v8Version, targetArchs, buildCSource, mksnapshotParams, useDocker) { +SnapshotGenerator.prototype.generateSnapshots = function (snapshotToolsPath, inputFile, v8Version, targetArchs, buildCSource, mksnapshotParams, snapshotInDocker) { // Cleans the snapshot build folder - debugger; shelljs.rm("-rf", join(this.buildPath, "snapshots")); - return this.downloadMkSnapshotTools(snapshotToolsPath, v8Version, targetArchs, useDocker).then((localTools) => { - var snapshotInDocker = !!useDocker; + return this.downloadMksnapshotTools(snapshotToolsPath, v8Version, targetArchs, snapshotInDocker).then((localTools) => { var shouldDownloadDockerTools = false; if (!snapshotInDocker) { snapshotInDocker = localTools.some(tool => !this.canUseSnapshotTool(tool.path)); @@ -152,26 +143,26 @@ SnapshotGenerator.prototype.generateSnapshots = function (snapshotToolsPath, inp } if (shouldDownloadDockerTools) { - return this.downloadMkSnapshotTools(snapshotToolsPath, v8Version, targetArchs, true).then((dockerTools) => { + return this.downloadMksnapshotTools(snapshotToolsPath, v8Version, targetArchs, true).then((dockerTools) => { console.log(`Executing '${snapshotToolPath}' in a docker container.`); - return this.runMKSnapshotTools(snapshotToolsPath, dockerTools, inputFile, mksnapshotParams, buildCSource, snapshotInDocker); + return this.runMksnapshotTools(snapshotToolsPath, dockerTools, inputFile, mksnapshotParams, buildCSource, snapshotInDocker); }); } else { - return this.runMKSnapshotTools(snapshotToolsPath, localTools, inputFile, mksnapshotParams, buildCSource, snapshotInDocker); + return this.runMksnapshotTools(snapshotToolsPath, localTools, inputFile, mksnapshotParams, buildCSource, snapshotInDocker); } }); } -SnapshotGenerator.prototype.runMKSnapshotTools = function (snapshotToolsBasePath, snapshotTools, inputFile, mksnapshotParams, buildCSource, snapshotInDocker) { +SnapshotGenerator.prototype.runMksnapshotTools = function (snapshotToolsBasePath, snapshotTools, inputFile, mksnapshotParams, buildCSource, snapshotInDocker) { let currentSnapshotOperation = Promise.resolve(); const canRunInParallel = snapshotTools.length <= 1 || !snapshotInDocker; return Promise.all(snapshotTools.map((tool) => { if (canRunInParallel) { - return this.runMKSnapshotTool(tool, mksnapshotParams, inputFile, snapshotInDocker, snapshotToolsBasePath, buildCSource); + return this.runMksnapshotTool(tool, mksnapshotParams, inputFile, snapshotInDocker, snapshotToolsBasePath, buildCSource); } else { currentSnapshotOperation = currentSnapshotOperation.then(() => { - return this.runMKSnapshotTool(tool, mksnapshotParams, inputFile, snapshotInDocker, snapshotToolsBasePath, buildCSource); + return this.runMksnapshotTool(tool, mksnapshotParams, inputFile, snapshotInDocker, snapshotToolsBasePath, buildCSource); }); return currentSnapshotOperation; @@ -193,14 +184,11 @@ SnapshotGenerator.prototype.canUseSnapshotTool = function (snapshotToolPath) { } SnapshotGenerator.prototype.setupDocker = function () { - const installMessage = "Install Docker and add it to your PATH in order to build snapshots."; try { - // e.g. Docker version 19.03.2, build 6a30dfc child_process.execSync(`docker --version`); - // TODO: require a minimum version? } catch (error) { - throw new Error(`Docker installation cannot be found. ${installMessage}`); + throw new Error(`Docker installation cannot be found. Install Docker and add it to your PATH in order to build snapshots.`); } child_process.execSync(`docker pull ${SNAPSHOTS_DOCKER_IMAGE}`); @@ -257,116 +245,133 @@ SnapshotGenerator.prototype.generate = function (options) { }); } -SnapshotGenerator.prototype.runMKSnapshotTool = function (tool, mksnapshotParams, inputFile, snapshotInDocker, snapshotToolsBasePath, buildCSource) { - const currentArchMksnapshotToolPath = tool.path; - const arch = tool.arch; - if (!fs.existsSync(currentArchMksnapshotToolPath)) { - throw new Error("Can't find mksnapshot tool for " + arch + " at path " + currentArchMksnapshotToolPath); +SnapshotGenerator.prototype.getSnapshotToolCommand = function (snapshotToolPath, inputFilePath, outputPath, toolParams) { + return `${snapshotToolPath} ${inputFilePath} --startup_blob ${join(outputPath, `${SNAPSHOT_BLOB_NAME}.blob`)} ${toolParams}`; +} + +SnapshotGenerator.prototype.getXxdCommand = function (srcOutputDir) { + return `xxd -i ${SNAPSHOT_BLOB_NAME}.blob > ${join(srcOutputDir, `${SNAPSHOT_BLOB_NAME}.c`)}`; +} + +SnapshotGenerator.prototype.getPathInDocker = function (mappedLocalDir, mappedDockerDir, targetPath) { + if (!isSubPath(mappedLocalDir, targetPath)) { + throw new Error(`Cannot determine a docker path. '${targetPath}' should be inside '${mappedLocalDir}'`) + } + + const pathInDocker = join(mappedDockerDir, relative(mappedLocalDir, targetPath)); + + return convertToUnixPath(pathInDocker); +} + +SnapshotGenerator.prototype.handleSnapshotToolResult = function (error, stdout, stderr, androidArch) { + let toolError = null; + const errorHeader = `Target architecture: ${androidArch}\n`; + let errorFooter = ``; + if (stderr.length || error) { + try { + require(inputFile); + } + catch (e) { + errorFooter = `\nJavaScript execution error: ${e.stack}$`; + } + } + + if (stderr.length) { + const message = `${errorHeader}${stderr}${errorFooter}`; + toolError = new Error(message); + } + else if (error) { + error.message = `${errorHeader}${error.message}${errorFooter}`; + toolError = error; + } else { + console.log(stdout); + } + + return toolError; +} + +SnapshotGenerator.prototype.copySnapshotTool = function (allToolsDir, targetTool, destinationDir) { + // we cannot mount the source tools folder as its not shared by default: + // docker: Error response from daemon: Mounts denied: + // The path /var/folders/h2/1yck52fx2mg7c790vhcw90s8087sk8/T/snapshot-tools/mksnapshot-tools + // is not shared from OS X and is not known to Docker. + const toolPathRelativeToAllToolsDir = relative(allToolsDir, targetTool); + const toolDestinationPath = join(destinationDir, toolPathRelativeToAllToolsDir) + createDirectory(dirname(toolDestinationPath)); + shelljs.cp(targetTool, toolDestinationPath); + + return toolDestinationPath; +} + +SnapshotGenerator.prototype.buildCSource = function (androidArch, blobInputDir, snapshotInDocker) { + const srcOutputDir = join(this.buildPath, "snapshots/src", androidArch); + createDirectory(srcOutputDir); + let command = ""; + if (snapshotInDocker) { + const blobsInputInDocker = `/blobs/${androidArch}` + const srcOutputDirInDocker = `/dist/src/${androidArch}`; + const buildCSourceCommand = this.getXxdCommand(srcOutputDirInDocker); + // add vim in order to get xxd + command = `docker run -v "${blobInputDir}:${blobsInputInDocker}" -v "${srcOutputDir}:${srcOutputDirInDocker}" ${SNAPSHOTS_DOCKER_IMAGE} /bin/sh -c "cd ${blobsInputInDocker} && apk add vim && ${buildCSourceCommand}"`; + } + else { + command = this.getXxdCommand(srcOutputDir); } + shellJsExecuteInDir(blobInputDir, function () { + shelljs.exec(command); + }); +} - const androidArch = this.convertToAndroidArchName(arch); - console.log("***** Generating snapshot for " + androidArch + " *****"); - // Generate .blob file - const currentArchBlobOutputPath = join(this.buildPath, "snapshots/blobs", androidArch); - shelljs.mkdir("-p", currentArchBlobOutputPath); - let dockerCurrentArchBlobOutputPath = ""; - var params = "--profile_deserialization"; - if (mksnapshotParams) { - // Starting from android runtime 5.3.0, the parameters passed to mksnapshot are read from the settings.json file - params = mksnapshotParams; +SnapshotGenerator.prototype.runMksnapshotTool = function (tool, mksnapshotParams, inputFile, snapshotInDocker, allToolsDir, buildCSource) { + const toolPath = tool.path; + const androidArch = this.convertToAndroidArchName(tool.arch); + if (!fs.existsSync(toolPath)) { + throw new Error("Can't find mksnapshot tool for " + androidArch + " at path " + toolPath); } + + const tempFolders = []; return new Promise((resolve, reject) => { - let snapshotToolPath = currentArchMksnapshotToolPath; + console.log("***** Generating snapshot for " + androidArch + " *****"); const inputFileDir = dirname(inputFile); - let inputFilePath = inputFile; - const appDirInDocker = "/app"; - let outputPath = currentArchBlobOutputPath; + const blobOutputDir = join(this.buildPath, "snapshots/blobs", androidArch); + createDirectory(blobOutputDir); + const toolParams = mksnapshotParams || "--profile_deserialization"; + + let command = ""; if (snapshotInDocker) { this.setupDocker(); - // create snapshots dir in docker - dockerCurrentArchBlobOutputPath = join(inputFileDir, "snapshots", androidArch); - shelljs.mkdir("-p", dockerCurrentArchBlobOutputPath); - outputPath = join(appDirInDocker, relative(inputFileDir, dockerCurrentArchBlobOutputPath)); - // calculate input in docker - inputFilePath = join(appDirInDocker, relative(inputFileDir, inputFile)); - // calculate snapshotTool path - snapshotToolPath = join(appDirInDocker, relative(snapshotToolsBasePath, currentArchMksnapshotToolPath)); - } - let command = `${snapshotToolPath} ${inputFilePath} --startup_blob ${join(outputPath, `${SNAPSHOT_BLOB_NAME}.blob`)} ${params}`; - if (snapshotInDocker) { - // we cannot mount the source tools folder as its not shared by default: - // docker: Error response from daemon: Mounts denied: - // The path /var/folders/h2/1yck52fx2mg7c790vhcw90s8087sk8/T/snapshot-tools/mksnapshot-tools - // is not shared from OS X and is not known to Docker. - const currentToolRelativeToSnapshotTools = relative(snapshotToolsBasePath, currentArchMksnapshotToolPath); - const currentToolDestination = join(inputFileDir, currentToolRelativeToSnapshotTools) - debugger; - createDirectory(dirname(currentToolDestination)); - shelljs.cp(currentArchMksnapshotToolPath, join(inputFileDir, currentToolRelativeToSnapshotTools)); - command = `docker run -v "${inputFileDir}:${appDirInDocker}" ${SNAPSHOTS_DOCKER_IMAGE} /bin/sh -c "${command}"`; + const appDirInDocker = "/app"; + const blobOutputDirInDocker = `/dist/blobs/${androidArch}`; + const toolsTempFolder = join(inputFileDir, "tmp"); + tempFolders.push(toolsTempFolder); + const toolPathInAppDir = this.copySnapshotTool(allToolsDir, toolPath, toolsTempFolder); + const toolPathInDocker = this.getPathInDocker(inputFileDir, appDirInDocker, toolPathInAppDir); + const inputFilePathInDocker = this.getPathInDocker(inputFileDir, appDirInDocker, inputFile); + const outputPathInDocker = this.getPathInDocker(blobOutputDir, blobOutputDirInDocker, blobOutputDir); + const toolCommandInDocker = this.getSnapshotToolCommand(toolPathInDocker, inputFilePathInDocker, outputPathInDocker, toolParams); + command = `docker run -v "${inputFileDir}:${appDirInDocker}" -v "${blobOutputDir}:${blobOutputDirInDocker}" ${SNAPSHOTS_DOCKER_IMAGE} /bin/sh -c "${toolCommandInDocker}"`; + } else { + command = this.getSnapshotToolCommand(toolPath, inputFilePath, outputPath, toolParams); } + + // Generate .blob file child_process.exec(command, { encoding: "utf8" }, (error, stdout, stderr) => { - const errorHeader = `Target architecture: ${androidArch}\n`; - let errorFooter = ``; - if (stderr.length || error) { - try { - require(inputFile); - } - catch (e) { - errorFooter = `\nJavaScript execution error: ${e.stack}$`; - } - } - if (stderr.length) { - const message = `${errorHeader}${stderr}${errorFooter}`; - reject(new Error(message)); - } - else if (error) { - error.message = `${errorHeader}${error.message}${errorFooter}`; - reject(error); - } - else { - console.log(stdout); - // Generate .c file - /// TODO: test in docker - if (buildCSource) { - let srcOutputPath = ""; - let dockerCurrentArchSrcOutputPath = ""; - if (snapshotInDocker) { - dockerCurrentArchSrcOutputPath = join(inputFileDir, "snapshots/src", androidArch); - shelljs.mkdir("-p", dockerCurrentArchSrcOutputPath); - srcOutputPath = join(appDirInDocker, relative(inputFileDir, dockerCurrentArchSrcOutputPath)); - } else { - srcOutputPath = join(this.buildPath, "snapshots/src", androidArch); - shelljs.mkdir("-p", srcOutputPath); - } - - const buildCSourceCommand = `xxd -i ${SNAPSHOT_BLOB_NAME}.blob > ${join(srcOutputPath, `${SNAPSHOT_BLOB_NAME}.c`)}`; - if (snapshotInDocker) { - // add vim in order to get xxd - const commandInDocker = - `docker run -v "${inputFileDir}:${appDirInDocker}" ${SNAPSHOTS_DOCKER_IMAGE} /bin/sh -c "cd ${outputPath} && apk add vim && ${buildCSourceCommand}"`; - child_process.execSync(commandInDocker); - } else { - shellJsExecuteInDir(currentArchBlobOutputPath, function () { - shelljs.exec(buildCSourceCommand); - }); - } - - if (snapshotInDocker) { - createDirectory(join(this.buildPath, "snapshots/src", androidArch)); - shelljs.cp("-R", dockerCurrentArchSrcOutputPath + "/", join(this.buildPath, "snapshots/src", androidArch)); - shelljs.rm("-rf", dockerCurrentArchSrcOutputPath); - } - } - - // TODO: move cleanup to afterPrepare? - if (snapshotInDocker) { - shelljs.cp("-R", dockerCurrentArchBlobOutputPath + "/", currentArchBlobOutputPath); - shelljs.rm("-rf", dockerCurrentArchBlobOutputPath); - } - resolve(); + tempFolders.forEach(tempFolder => { + shelljs.rm("-rf", tempFolder); + }); + + const snapshotError = this.handleSnapshotToolResult(error, stdout, stderr, androidArch); + if (snapshotError) { + return reject(error); } + + return resolve(blobOutputDir); }); + }).then((blobOutputDir) => { + // Generate .c file + if (buildCSource) { + this.buildCSource(androidArch, blobOutputDir, snapshotInDocker) + } }); } diff --git a/snapshot/android/utils.js b/snapshot/android/utils.js index 8d822e53..91c82950 100644 --- a/snapshot/android/utils.js +++ b/snapshot/android/utils.js @@ -1,6 +1,6 @@ const { chmodSync, createWriteStream, existsSync } = require("fs"); const { tmpdir, EOL } = require("os"); -const { join } = require("path"); +const { join, relative, isAbsolute } = require("path"); const os = require("os"); const { mkdir } = require("shelljs"); @@ -40,6 +40,13 @@ function has32BitArch(targetArchs) { return Array.isArray(targetArchs) && targetArchs.some(arch => arch === "arm" || arch === "ia32") } +function isSubPath(parentPath, childPath) { + const relativePath = relative(parentPath, childPath); + + return relativePath === "" || + (relativePath && !relativePath.startsWith('..') && !isAbsolute(relativePath)); +} + function isMacOSCatalinaOrHigher() { const isCatalinaOrHigher = false; const catalinaVersion = "19.0.0"; @@ -113,4 +120,5 @@ module.exports = { isMacOSCatalinaOrHigher, downloadFile, getJsonFile, + isSubPath }; From a92fce126e645750dffd924802b211e4b2090fb6 Mon Sep 17 00:00:00 2001 From: DimitarTachev Date: Thu, 19 Sep 2019 15:48:03 +0300 Subject: [PATCH 3/6] refactor: fix linting errors --- .../android/project-snapshot-generator.js | 1 - snapshot/android/snapshot-generator.js | 36 +++++++++---------- snapshot/android/utils.js | 11 +++--- 3 files changed, 22 insertions(+), 26 deletions(-) diff --git a/snapshot/android/project-snapshot-generator.js b/snapshot/android/project-snapshot-generator.js index 00053708..7ff69cd0 100644 --- a/snapshot/android/project-snapshot-generator.js +++ b/snapshot/android/project-snapshot-generator.js @@ -10,7 +10,6 @@ const { createDirectory, getJsonFile, } = require("./utils"); -const { getPackageJson } = require("../../projectHelpers"); const { ANDROID_PROJECT_DIR, ANDROID_APP_PATH, diff --git a/snapshot/android/snapshot-generator.js b/snapshot/android/snapshot-generator.js index cc1c8e62..ce514943 100644 --- a/snapshot/android/snapshot-generator.js +++ b/snapshot/android/snapshot-generator.js @@ -36,13 +36,13 @@ module.exports = SnapshotGenerator; SnapshotGenerator.SNAPSHOT_PACKAGE_NANE = "nativescript-android-snapshot"; -SnapshotGenerator.prototype.shouldSnapshotInDocker = function (hostOS) { +SnapshotGenerator.prototype.shouldSnapshotInDocker = function (hostOS, targetArchs) { let shouldSnapshotInDocker = false; const generateInDockerMessage = "The snapshots will be generated in a docker container."; if (hostOS == CONSTANTS.WIN_OS_NAME) { console.log(`The V8 snapshot tools are not supported on Windows. ${generateInDockerMessage}`); shouldSnapshotInDocker = true; - } else if (isMacOSCatalinaOrHigher() && has32BitArch(options.targetArchs)) { + } else if (isMacOSCatalinaOrHigher() && has32BitArch(targetArchs)) { console.log(`Starting from macOS Catalina, the 32-bit processes are no longer supported. ${generateInDockerMessage}`); shouldSnapshotInDocker = true; } @@ -52,8 +52,6 @@ SnapshotGenerator.prototype.shouldSnapshotInDocker = function (hostOS) { SnapshotGenerator.prototype.preprocessInputFiles = function (inputFiles, outputFile) { // Make some modifcations on the original bundle and save it on the specified path - - const bundlePreambleContent = fs.readFileSync(BUNDLE_PREAMBLE_PATH, "utf8"); const bundleEndingContent = fs.readFileSync(BUNDLE_ENDING_PATH, "utf8"); @@ -143,20 +141,20 @@ SnapshotGenerator.prototype.generateSnapshots = function (snapshotToolsPath, inp } if (shouldDownloadDockerTools) { - return this.downloadMksnapshotTools(snapshotToolsPath, v8Version, targetArchs, true).then((dockerTools) => { - console.log(`Executing '${snapshotToolPath}' in a docker container.`); + return this.downloadMksnapshotTools(snapshotToolsPath, v8Version, targetArchs, snapshotInDocker).then((dockerTools) => { + console.log(`Generating snapshots in a docker container.`); return this.runMksnapshotTools(snapshotToolsPath, dockerTools, inputFile, mksnapshotParams, buildCSource, snapshotInDocker); }); - } else { - return this.runMksnapshotTools(snapshotToolsPath, localTools, inputFile, mksnapshotParams, buildCSource, snapshotInDocker); } + + return this.runMksnapshotTools(snapshotToolsPath, localTools, inputFile, mksnapshotParams, buildCSource, snapshotInDocker); }); } SnapshotGenerator.prototype.runMksnapshotTools = function (snapshotToolsBasePath, snapshotTools, inputFile, mksnapshotParams, buildCSource, snapshotInDocker) { let currentSnapshotOperation = Promise.resolve(); - const canRunInParallel = snapshotTools.length <= 1 || !snapshotInDocker; + const canRunInParallel = !snapshotInDocker; return Promise.all(snapshotTools.map((tool) => { if (canRunInParallel) { return this.runMksnapshotTool(tool, mksnapshotParams, inputFile, snapshotInDocker, snapshotToolsBasePath, buildCSource); @@ -224,7 +222,7 @@ SnapshotGenerator.prototype.generate = function (options) { this.preprocessInputFiles(options.inputFiles, preprocessedInputFile); const hostOS = getHostOS(); - const snapshotInDocker = options.snapshotInDocker || this.shouldSnapshotInDocker(hostOS); + const snapshotInDocker = options.snapshotInDocker || this.shouldSnapshotInDocker(hostOS, options.targetArchs); // generates the actual .blob and .c files return this.generateSnapshots( @@ -263,11 +261,11 @@ SnapshotGenerator.prototype.getPathInDocker = function (mappedLocalDir, mappedDo return convertToUnixPath(pathInDocker); } -SnapshotGenerator.prototype.handleSnapshotToolResult = function (error, stdout, stderr, androidArch) { +SnapshotGenerator.prototype.handleSnapshotToolResult = function (error, stdout, stderr, inputFile, androidArch) { let toolError = null; const errorHeader = `Target architecture: ${androidArch}\n`; let errorFooter = ``; - if (stderr.length || error) { + if ((stderr && stderr.length) || error) { try { require(inputFile); } @@ -276,7 +274,7 @@ SnapshotGenerator.prototype.handleSnapshotToolResult = function (error, stdout, } } - if (stderr.length) { + if (stderr && stderr.length) { const message = `${errorHeader}${stderr}${errorFooter}`; toolError = new Error(message); } @@ -322,11 +320,11 @@ SnapshotGenerator.prototype.buildCSource = function (androidArch, blobInputDir, }); } -SnapshotGenerator.prototype.runMksnapshotTool = function (tool, mksnapshotParams, inputFile, snapshotInDocker, allToolsDir, buildCSource) { +SnapshotGenerator.prototype.runMksnapshotTool = function (tool, mksnapshotParams, inputFile, snapshotInDocker, snapshotToolsPath, buildCSource) { const toolPath = tool.path; const androidArch = this.convertToAndroidArchName(tool.arch); if (!fs.existsSync(toolPath)) { - throw new Error("Can't find mksnapshot tool for " + androidArch + " at path " + toolPath); + throw new Error(`Can't find mksnapshot tool for ${androidArch} at path ${toolPath}`); } const tempFolders = []; @@ -344,14 +342,14 @@ SnapshotGenerator.prototype.runMksnapshotTool = function (tool, mksnapshotParams const blobOutputDirInDocker = `/dist/blobs/${androidArch}`; const toolsTempFolder = join(inputFileDir, "tmp"); tempFolders.push(toolsTempFolder); - const toolPathInAppDir = this.copySnapshotTool(allToolsDir, toolPath, toolsTempFolder); + const toolPathInAppDir = this.copySnapshotTool(snapshotToolsPath, toolPath, toolsTempFolder); const toolPathInDocker = this.getPathInDocker(inputFileDir, appDirInDocker, toolPathInAppDir); const inputFilePathInDocker = this.getPathInDocker(inputFileDir, appDirInDocker, inputFile); const outputPathInDocker = this.getPathInDocker(blobOutputDir, blobOutputDirInDocker, blobOutputDir); const toolCommandInDocker = this.getSnapshotToolCommand(toolPathInDocker, inputFilePathInDocker, outputPathInDocker, toolParams); command = `docker run -v "${inputFileDir}:${appDirInDocker}" -v "${blobOutputDir}:${blobOutputDirInDocker}" ${SNAPSHOTS_DOCKER_IMAGE} /bin/sh -c "${toolCommandInDocker}"`; } else { - command = this.getSnapshotToolCommand(toolPath, inputFilePath, outputPath, toolParams); + command = this.getSnapshotToolCommand(toolPath, inputFile, blobOutputDir, toolParams); } // Generate .blob file @@ -360,9 +358,9 @@ SnapshotGenerator.prototype.runMksnapshotTool = function (tool, mksnapshotParams shelljs.rm("-rf", tempFolder); }); - const snapshotError = this.handleSnapshotToolResult(error, stdout, stderr, androidArch); + const snapshotError = this.handleSnapshotToolResult(error, stdout, stderr, inputFile, androidArch); if (snapshotError) { - return reject(error); + return reject(snapshotError); } return resolve(blobOutputDir); diff --git a/snapshot/android/utils.js b/snapshot/android/utils.js index 91c82950..aef7157b 100644 --- a/snapshot/android/utils.js +++ b/snapshot/android/utils.js @@ -1,4 +1,4 @@ -const { chmodSync, createWriteStream, existsSync } = require("fs"); +const { chmodSync, createWriteStream } = require("fs"); const { tmpdir, EOL } = require("os"); const { join, relative, isAbsolute } = require("path"); const os = require("os"); @@ -37,7 +37,8 @@ function getHostOSArch() { } function has32BitArch(targetArchs) { - return Array.isArray(targetArchs) && targetArchs.some(arch => arch === "arm" || arch === "ia32") + return (Array.isArray(targetArchs) && targetArchs.some(arch => arch === "arm" || arch === "ia32")) || + (targetArchs === "arm" || targetArchs === "ia32"); } function isSubPath(parentPath, childPath) { @@ -48,14 +49,12 @@ function isSubPath(parentPath, childPath) { } function isMacOSCatalinaOrHigher() { - const isCatalinaOrHigher = false; + let isCatalinaOrHigher = false; const catalinaVersion = "19.0.0"; const hostOS = getHostOS(); if (hostOS === CONSTANTS.MAC_OS_NAME) { const hostOSVersion = getHostOSVersion(); - if (semver.gte(hostOSVersion, catalinaVersion)) { - isCatalinaOrHigher = true; - } + isCatalinaOrHigher = semver.gte(hostOSVersion, catalinaVersion); } return isCatalinaOrHigher; From a70d64b80414d12555b63d105c2bedafbe437b0a Mon Sep 17 00:00:00 2001 From: DimitarTachev Date: Fri, 20 Sep 2019 17:32:40 +0300 Subject: [PATCH 4/6] fix: use the xxd tool from the image, instead of downloading vim --- snapshot/android/snapshot-generator.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/snapshot/android/snapshot-generator.js b/snapshot/android/snapshot-generator.js index ce514943..270524dc 100644 --- a/snapshot/android/snapshot-generator.js +++ b/snapshot/android/snapshot-generator.js @@ -248,7 +248,8 @@ SnapshotGenerator.prototype.getSnapshotToolCommand = function (snapshotToolPath, } SnapshotGenerator.prototype.getXxdCommand = function (srcOutputDir) { - return `xxd -i ${SNAPSHOT_BLOB_NAME}.blob > ${join(srcOutputDir, `${SNAPSHOT_BLOB_NAME}.c`)}`; + // https://github.com/NativeScript/docker-images/tree/master/v8-snapshot/bin + return `/bin/xxd -i ${SNAPSHOT_BLOB_NAME}.blob > ${join(srcOutputDir, `${SNAPSHOT_BLOB_NAME}.c`)}`; } SnapshotGenerator.prototype.getPathInDocker = function (mappedLocalDir, mappedDockerDir, targetPath) { @@ -309,8 +310,7 @@ SnapshotGenerator.prototype.buildCSource = function (androidArch, blobInputDir, const blobsInputInDocker = `/blobs/${androidArch}` const srcOutputDirInDocker = `/dist/src/${androidArch}`; const buildCSourceCommand = this.getXxdCommand(srcOutputDirInDocker); - // add vim in order to get xxd - command = `docker run -v "${blobInputDir}:${blobsInputInDocker}" -v "${srcOutputDir}:${srcOutputDirInDocker}" ${SNAPSHOTS_DOCKER_IMAGE} /bin/sh -c "cd ${blobsInputInDocker} && apk add vim && ${buildCSourceCommand}"`; + command = `docker run --rm -v "${blobInputDir}:${blobsInputInDocker}" -v "${srcOutputDir}:${srcOutputDirInDocker}" ${SNAPSHOTS_DOCKER_IMAGE} /bin/sh -c "cd ${blobsInputInDocker} && ${buildCSourceCommand}"`; } else { command = this.getXxdCommand(srcOutputDir); @@ -347,7 +347,7 @@ SnapshotGenerator.prototype.runMksnapshotTool = function (tool, mksnapshotParams const inputFilePathInDocker = this.getPathInDocker(inputFileDir, appDirInDocker, inputFile); const outputPathInDocker = this.getPathInDocker(blobOutputDir, blobOutputDirInDocker, blobOutputDir); const toolCommandInDocker = this.getSnapshotToolCommand(toolPathInDocker, inputFilePathInDocker, outputPathInDocker, toolParams); - command = `docker run -v "${inputFileDir}:${appDirInDocker}" -v "${blobOutputDir}:${blobOutputDirInDocker}" ${SNAPSHOTS_DOCKER_IMAGE} /bin/sh -c "${toolCommandInDocker}"`; + command = `docker run --rm -v "${inputFileDir}:${appDirInDocker}" -v "${blobOutputDir}:${blobOutputDirInDocker}" ${SNAPSHOTS_DOCKER_IMAGE} /bin/sh -c "${toolCommandInDocker}"`; } else { command = this.getSnapshotToolCommand(toolPath, inputFile, blobOutputDir, toolParams); } From 18791a5242a9e09b043e92194cd10da5313787c2 Mon Sep 17 00:00:00 2001 From: DimitarTachev Date: Fri, 20 Sep 2019 17:33:30 +0300 Subject: [PATCH 5/6] fix: add a flag for skipping the snapshot tools in order to skip them for the local part of the cloud builds --- plugins/NativeScriptSnapshotPlugin/index.js | 8 +++++++- plugins/NativeScriptSnapshotPlugin/options.json | 4 ++++ .../android/project-snapshot-generator-cli-ags-parser.js | 4 ++++ snapshot/android/project-snapshot-generator.js | 5 +++++ templates/webpack.angular.js | 6 ++++-- templates/webpack.javascript.js | 6 ++++-- templates/webpack.typescript.js | 6 ++++-- templates/webpack.vue.js | 6 ++++-- 8 files changed, 36 insertions(+), 9 deletions(-) diff --git a/plugins/NativeScriptSnapshotPlugin/index.js b/plugins/NativeScriptSnapshotPlugin/index.js index 3c583ac3..fe32ce1b 100644 --- a/plugins/NativeScriptSnapshotPlugin/index.js +++ b/plugins/NativeScriptSnapshotPlugin/index.js @@ -97,6 +97,11 @@ exports.NativeScriptSnapshotPlugin = (function () { NativeScriptSnapshotPlugin.prototype.generate = function (webpackChunks) { const options = this.options; + if (options.skipSnapshotTools) { + console.log(`Skipping snapshot tools.`); + return Promise.resolve(); + } + const inputFiles = webpackChunks.map(chunk => join(options.webpackConfig.output.path, chunk.files[0])); const preprocessedInputFile = join( this.options.projectRoot, @@ -113,7 +118,8 @@ exports.NativeScriptSnapshotPlugin = (function () { useLibs: options.useLibs, androidNdkPath: options.androidNdkPath, v8Version: options.v8Version, - snapshotInDocker: options.snapshotInDocker + snapshotInDocker: options.snapshotInDocker, + skipSnapshotTools: options.skipSnapshotTools }).then(() => { // Make the original files empty inputFiles.forEach(inputFile => diff --git a/plugins/NativeScriptSnapshotPlugin/options.json b/plugins/NativeScriptSnapshotPlugin/options.json index c82eb401..22e08d60 100644 --- a/plugins/NativeScriptSnapshotPlugin/options.json +++ b/plugins/NativeScriptSnapshotPlugin/options.json @@ -42,6 +42,10 @@ "snapshotInDocker": { "type": "boolean" }, + "skipSnapshotTools": { + "type": "boolean", + "default": false + }, "v8Version": { "type": "string" }, diff --git a/snapshot/android/project-snapshot-generator-cli-ags-parser.js b/snapshot/android/project-snapshot-generator-cli-ags-parser.js index 97cea86d..6d3b9b61 100644 --- a/snapshot/android/project-snapshot-generator-cli-ags-parser.js +++ b/snapshot/android/project-snapshot-generator-cli-ags-parser.js @@ -13,6 +13,10 @@ module.exports = function parseProjectSnapshotGeneratorArgs() { result.snapshotInDocker = parseBool(result.snapshotInDocker); } + if (result.skipSnapshotTools !== undefined) { + result.skipSnapshotTools = parseBool(result.skipSnapshotTools); + } + if (result.install !== undefined) { result.install = parseBool(result.install); } diff --git a/snapshot/android/project-snapshot-generator.js b/snapshot/android/project-snapshot-generator.js index 7ff69cd0..559c2da2 100644 --- a/snapshot/android/project-snapshot-generator.js +++ b/snapshot/android/project-snapshot-generator.js @@ -210,6 +210,11 @@ ProjectSnapshotGenerator.prototype.validateAndroidRuntimeVersion = function () { } ProjectSnapshotGenerator.prototype.generate = function (generationOptions) { + if (generationOptions.skipSnapshotTools) { + console.log("Running snapshot generation with the following arguments: "); + return Promise.resolve(); + } + generationOptions = generationOptions || {}; console.log("Running snapshot generation with the following arguments: "); diff --git a/templates/webpack.angular.js b/templates/webpack.angular.js index f15d8bea..43c3d923 100644 --- a/templates/webpack.angular.js +++ b/templates/webpack.angular.js @@ -50,7 +50,8 @@ module.exports = env => { hmr, // --env.hmr, unitTesting, // --env.unitTesting verbose, // --env.verbose - snapshotInDocker // --env.snapshotInDocker + snapshotInDocker, // --env.snapshotInDocker + skipSnapshotTools // --env.skipSnapshotTools } = env; const isAnySourceMapEnabled = !!sourceMap || !!hiddenSourceMap; @@ -309,7 +310,8 @@ module.exports = env => { ], projectRoot, webpackConfig: config, - snapshotInDocker + snapshotInDocker, + skipSnapshotTools })); } diff --git a/templates/webpack.javascript.js b/templates/webpack.javascript.js index 05142a36..da824e28 100644 --- a/templates/webpack.javascript.js +++ b/templates/webpack.javascript.js @@ -44,7 +44,8 @@ module.exports = env => { hmr, // --env.hmr, unitTesting, // --env.unitTesting, verbose, // --env.verbose - snapshotInDocker // --env.snapshotInDocker + snapshotInDocker, // --env.snapshotInDocker + skipSnapshotTools // --env.skipSnapshotTools } = env; const isAnySourceMapEnabled = !!sourceMap || !!hiddenSourceMap; @@ -250,7 +251,8 @@ module.exports = env => { ], projectRoot, webpackConfig: config, - snapshotInDocker + snapshotInDocker, + skipSnapshotTools })); } diff --git a/templates/webpack.typescript.js b/templates/webpack.typescript.js index d124be1a..b50e4d93 100644 --- a/templates/webpack.typescript.js +++ b/templates/webpack.typescript.js @@ -45,7 +45,8 @@ module.exports = env => { hmr, // --env.hmr, unitTesting, // --env.unitTesting, verbose, // --env.verbose - snapshotInDocker // --env.snapshotInDocker + snapshotInDocker, // --env.snapshotInDocker + skipSnapshotTools // --env.skipSnapshotTools } = env; const isAnySourceMapEnabled = !!sourceMap || !!hiddenSourceMap; const externals = nsWebpack.getConvertedExternals(env.externals); @@ -278,7 +279,8 @@ module.exports = env => { ], projectRoot, webpackConfig: config, - snapshotInDocker + snapshotInDocker, + skipSnapshotTools })); } diff --git a/templates/webpack.vue.js b/templates/webpack.vue.js index ba273189..5e4427f0 100644 --- a/templates/webpack.vue.js +++ b/templates/webpack.vue.js @@ -47,7 +47,8 @@ module.exports = env => { hiddenSourceMap, // --env.hiddenSourceMap unitTesting, // --env.unitTesting verbose, // --env.verbose - snapshotInDocker // --env.snapshotInDocker + snapshotInDocker, // --env.snapshotInDocker + skipSnapshotTools // --env.skipSnapshotTools } = env; const isAnySourceMapEnabled = !!sourceMap || !!hiddenSourceMap; @@ -297,7 +298,8 @@ module.exports = env => { ], projectRoot, webpackConfig: config, - snapshotInDocker + snapshotInDocker, + skipSnapshotTools })); } From 11ec7f447285c09b8cfb051b2b9834962db61c46 Mon Sep 17 00:00:00 2001 From: Fatme Date: Tue, 24 Sep 2019 09:41:38 +0300 Subject: [PATCH 6/6] chore: fix wrong console.log --- snapshot/android/project-snapshot-generator.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/snapshot/android/project-snapshot-generator.js b/snapshot/android/project-snapshot-generator.js index 559c2da2..b8a778f8 100644 --- a/snapshot/android/project-snapshot-generator.js +++ b/snapshot/android/project-snapshot-generator.js @@ -211,7 +211,7 @@ ProjectSnapshotGenerator.prototype.validateAndroidRuntimeVersion = function () { ProjectSnapshotGenerator.prototype.generate = function (generationOptions) { if (generationOptions.skipSnapshotTools) { - console.log("Running snapshot generation with the following arguments: "); + console.log("Skipping snapshot tools."); return Promise.resolve(); }