snapshot-generator.js

const fs = require("fs");
const { dirname, relative, join, EOL } = require("path");
const child_process = require("child_process");
const { convertToUnixPath, warn } = require("../../lib/utils");
const PropertiesReader = require('properties-reader');
const shelljs = require("shelljs");

const { createDirectory, downloadFile, getHostOS, getHostOSArch, CONSTANTS, has32BitArch, isMacOSCatalinaOrHigher, isSubPath } = require("./utils");

const SNAPSHOTS_DOCKER_IMAGE = "nativescript/v8-snapshot:latest";
const SNAPSHOT_TOOLS_DIR_NAME = "mksnapshot-tools";
const NDK_BUILD_SEED_PATH = join(__dirname, "snapshot-generator-tools/ndk-build");
const BUNDLE_PREAMBLE_PATH = join(__dirname, "snapshot-generator-tools/bundle-preamble.js");
const BUNDLE_ENDING_PATH = join(__dirname, "snapshot-generator-tools/bundle-ending.js");
const INCLUDE_GRADLE_PATH = join(__dirname, "snapshot-generator-tools/include.gradle");
const MKSNAPSHOT_TOOLS_DOWNLOAD_ROOT_URL = "https://raw.githubusercontent.com/NativeScript/mksnapshot-tools/production/";
const MKSNAPSHOT_TOOLS_DOWNLOAD_TIMEOUT = 60000;
const SNAPSHOT_BLOB_NAME = "TNSSnapshot";
const DOCKER_IMAGE_OS = "linux";
const DOCKER_IMAGE_ARCH = "x64";

function shellJsExecuteInDir(dir, action) {
    const currentDir = shelljs.pwd();
    shelljs.cd(dir);
    try {
        action();
    } finally {
        shelljs.cd(currentDir);
    }
}

function SnapshotGenerator(options) {
    this.buildPath = options.buildPath || join(__dirname, "build");
}
module.exports = SnapshotGenerator;

SnapshotGenerator.SNAPSHOT_PACKAGE_NANE = "nativescript-android-snapshot";

SnapshotGenerator.prototype.shouldSnapshotInDocker = function (hostOS, targetArchs) {
    let shouldSnapshotInDocker = false;
    const generateInDockerMessage = "The snapshots will be generated in a docker container.";
    if (hostOS == CONSTANTS.WIN_OS_NAME) {
        console.log(`The V8 snapshot tools are not supported on Windows. ${generateInDockerMessage}`);
        shouldSnapshotInDocker = true;
    } else if (isMacOSCatalinaOrHigher() && has32BitArch(targetArchs)) {
        console.log(`Starting from macOS Catalina, the 32-bit processes are no longer supported. ${generateInDockerMessage}`);
        shouldSnapshotInDocker = true;
    }

    return shouldSnapshotInDocker;
}

SnapshotGenerator.prototype.preprocessInputFiles = function (inputFiles, outputFile) {
    // Make some modifcations on the original bundle and save it on the specified path
    const bundlePreambleContent = fs.readFileSync(BUNDLE_PREAMBLE_PATH, "utf8");
    const bundleEndingContent = fs.readFileSync(BUNDLE_ENDING_PATH, "utf8");

    // IMPORTANT: join by "\n;" as we are joining IIFE functions and if the snapshot tool is used
    // along with Uglify configuration for replacing `;` with `/n`, we will generate invalid JavaScript
    // Example:
    // (function() {
    //  some code here
    //  })()
    //  // sourceMapUrl......
    //  ** when we join without `;` here, the next IIFE is assumed as a function call to the result of the first IIFE
    // (function() {
    //  some code here
    //  })()
    //  // sourceMapUrl......
    const inputFilesContent = inputFiles.map(file => fs.readFileSync(file, "utf8")).join("\n;");
    const snapshotFileContent = bundlePreambleContent + "\n" + inputFilesContent + "\n" + bundleEndingContent;
    fs.writeFileSync(outputFile, snapshotFileContent, { encoding: "utf8" });
}

const snapshotToolsDownloads = {};

SnapshotGenerator.prototype.downloadMksnapshotTools = function (snapshotToolsPath, v8Version, targetArchs, snapshotInDocker) {
    var toolsOS = "";
    var toolsArch = "";
    if (snapshotInDocker) {
        toolsOS = DOCKER_IMAGE_OS;
        toolsArch = DOCKER_IMAGE_ARCH;
    } else {
        toolsOS = getHostOS();
        toolsArch = getHostOSArch();
    }

    return Promise.all(targetArchs.map((arch) => {
        return this.downloadMksnapshotTool(snapshotToolsPath, v8Version, arch, toolsOS, toolsArch).then(path => {
            return { path, arch };
        });
    }));
}

SnapshotGenerator.prototype.downloadMksnapshotTool = function (snapshotToolsPath, v8Version, targetArch, hostOS, hostArch) {
    const mksnapshotToolRelativePath = join(SNAPSHOT_TOOLS_DIR_NAME, "v8-v" + v8Version, hostOS + "-" + hostArch, "mksnapshot-" + targetArch);
    const mksnapshotToolPath = join(snapshotToolsPath, mksnapshotToolRelativePath);
    if (fs.existsSync(mksnapshotToolPath))
        return Promise.resolve(mksnapshotToolPath);

    if (snapshotToolsDownloads[mksnapshotToolPath])
        return snapshotToolsDownloads[mksnapshotToolPath];

    const downloadUrl = MKSNAPSHOT_TOOLS_DOWNLOAD_ROOT_URL + mksnapshotToolRelativePath;
    createDirectory(dirname(mksnapshotToolPath));
    snapshotToolsDownloads[mksnapshotToolPath] = downloadFile(downloadUrl, mksnapshotToolPath, MKSNAPSHOT_TOOLS_DOWNLOAD_TIMEOUT);
    snapshotToolsDownloads[mksnapshotToolPath].catch(err => {
        const errorMessage = err && err.message ? err.message : "";
        let cleanupError = "";
        try {
            fs.unlinkSync(mksnapshotToolPath);
        } catch (unlinkErr) {
            if (unlinkErr && unlinkErr.code !== "ENOENT") {
                cleanupError = `${EOL}Failed to cleanup mksnapshot tool.`;
            }
        }

        throw new Error(`Failed to download mksnapshot tool. Error: ${errorMessage}.${cleanupError}`);
    });
    return snapshotToolsDownloads[mksnapshotToolPath];
}

SnapshotGenerator.prototype.convertToAndroidArchName = function (archName) {
    switch (archName) {
        case "arm": return "armeabi-v7a";
        case "arm64": return "arm64-v8a";
        case "ia32": return "x86";
        case "ia64": return "x86_64";
        default: return archName;
    }
}

SnapshotGenerator.prototype.generateSnapshots = function (snapshotToolsPath, inputFile, v8Version, targetArchs, buildCSource, mksnapshotParams, snapshotInDocker) {
    // Cleans the snapshot build folder
    shelljs.rm("-rf", join(this.buildPath, "snapshots"));
    return this.downloadMksnapshotTools(snapshotToolsPath, v8Version, targetArchs, snapshotInDocker).then((localTools) => {
        var shouldDownloadDockerTools = false;
        if (!snapshotInDocker) {
            snapshotInDocker = localTools.some(tool => !this.canUseSnapshotTool(tool.path));
            shouldDownloadDockerTools = snapshotInDocker;
        }

        if (shouldDownloadDockerTools) {
            return this.downloadMksnapshotTools(snapshotToolsPath, v8Version, targetArchs, snapshotInDocker).then((dockerTools) => {
                console.log(`Generating snapshots in a docker container.`);
                return this.runMksnapshotTools(snapshotToolsPath, dockerTools, inputFile, mksnapshotParams, buildCSource, snapshotInDocker);
            });
        }

        return this.runMksnapshotTools(snapshotToolsPath, localTools, inputFile, mksnapshotParams, buildCSource, snapshotInDocker);
    });
}


SnapshotGenerator.prototype.runMksnapshotTools = function (snapshotToolsBasePath, snapshotTools, inputFile, mksnapshotParams, buildCSource, snapshotInDocker) {
    let currentSnapshotOperation = Promise.resolve();
    const canRunInParallel = !snapshotInDocker;
    return Promise.all(snapshotTools.map((tool) => {
        if (canRunInParallel) {
            return this.runMksnapshotTool(tool, mksnapshotParams, inputFile, snapshotInDocker, snapshotToolsBasePath, buildCSource);
        } else {
            currentSnapshotOperation = currentSnapshotOperation.then(() => {
                return this.runMksnapshotTool(tool, mksnapshotParams, inputFile, snapshotInDocker, snapshotToolsBasePath, buildCSource);
            });

            return currentSnapshotOperation;
        }
    })).then(() => {
        console.log("***** Finished generating snapshots. *****");
    });
}

SnapshotGenerator.prototype.canUseSnapshotTool = function (snapshotToolPath) {
    try {
        child_process.execSync(`${snapshotToolPath} --help`);
        return true;
    }
    catch (error) {
        console.log(`Unable to execute '${snapshotToolPath}' locally.Error message: '${error.message}'`);
        return false;
    }
}

SnapshotGenerator.prototype.setupDocker = function () {
    try {
        child_process.execSync(`docker --version`);
    }
    catch (error) {
        throw new Error(`Docker installation cannot be found. Install Docker and add it to your PATH in order to build snapshots.`);
    }

    child_process.execSync(`docker pull ${SNAPSHOTS_DOCKER_IMAGE}`);
}

SnapshotGenerator.prototype.buildSnapshotLibs = function (androidNdkPath, recommendedAndroidNdkRevision, targetArchs) {
    // Compile *.c files to produce *.so libraries with ndk-build tool
    const androidNdkBuildPath = this.getAndroidNdkBuildPath(androidNdkPath, recommendedAndroidNdkRevision);
    const ndkBuildPath = join(this.buildPath, "ndk-build");
    const androidArchs = targetArchs.map(arch => this.convertToAndroidArchName(arch));
    console.log("Building native libraries for " + androidArchs.join());
    shelljs.rm("-rf", ndkBuildPath);
    shelljs.cp("-r", NDK_BUILD_SEED_PATH, ndkBuildPath);
    fs.writeFileSync(join(ndkBuildPath, "jni/Application.mk"), "APP_ABI := " + androidArchs.join(" ")); // create Application.mk file
    shelljs.mv(join(this.buildPath, "snapshots/src/*"), join(ndkBuildPath, "jni"));
    shellJsExecuteInDir(ndkBuildPath, function () {
        shelljs.exec(androidNdkBuildPath);
    });
    return join(ndkBuildPath, "libs");
}

SnapshotGenerator.prototype.getAndroidNdkBuildPath = function (androidNdkPath, recommendedAndroidNdkRevision) {
    const ndkBuildExecutableName = "ndk-build";
    // fallback for Android Runtime < 6.2.0 with the 6.1.0 value
    recommendedAndroidNdkRevision = recommendedAndroidNdkRevision || "20.0.5594570";
    let androidNdkBuildPath = "";
    if (androidNdkPath) {
        // specified by the user
        const localNdkRevision = this.getAndroidNdkRevision(androidNdkPath);
        androidNdkBuildPath = join(androidNdkPath, ndkBuildExecutableName);
        if (!fs.existsSync(androidNdkBuildPath)) {
            throw new Error(`The provided Android NDK path does not contain ${ndkBuildExecutableName} executable.`);
        } else if (localNdkRevision !== recommendedAndroidNdkRevision) {
            warn(`The provided Android NDK is v${localNdkRevision} while the recommended one is v${recommendedAndroidNdkRevision}`);
        }
    } else {
        // available globally
        let hasAndroidNdkInPath = true;
        androidNdkBuildPath = ndkBuildExecutableName;
        try {
            child_process.execSync(`${androidNdkBuildPath} --version`);
            console.log(`Cannot determine the version of the global Android NDK. The recommended versions is v${recommendedAndroidNdkRevision}`);
        } catch (_) {
            hasAndroidNdkInPath = false;
        }

        if (!hasAndroidNdkInPath) {
            // installed in ANDROID_HOME
            const androidHome = process.env.ANDROID_HOME;
            androidNdkBuildPath = join(androidHome, "ndk", recommendedAndroidNdkRevision, ndkBuildExecutableName);
            if (!fs.existsSync(androidNdkBuildPath)) {
                throw new Error(`Android NDK v${recommendedAndroidNdkRevision} is not installed. You can find installation instructions in this article: https://developer.android.com/studio/projects/install-ndk#specific-version`);
            }
        }
    }

    return androidNdkBuildPath;
}

SnapshotGenerator.prototype.getAndroidNdkRevision = function (androidNdkPath) {
    const ndkPropertiesFile = join(androidNdkPath, "source.properties");
    if (fs.existsSync(ndkPropertiesFile)) {
        const properties = PropertiesReader(ndkPropertiesFile);
        return properties.get("Pkg.Revision");
    } else {
        return null;
    }
}

SnapshotGenerator.prototype.buildIncludeGradle = function () {
    shelljs.cp(INCLUDE_GRADLE_PATH, join(this.buildPath, "include.gradle"));
}

SnapshotGenerator.prototype.generate = function (options) {
    // Arguments validation
    options = options || {};
    if (!options.v8Version) { throw new Error("No v8 version specified."); }
    if (!options.snapshotToolsPath) { throw new Error("snapshotToolsPath option is not specified."); }
    const preprocessedInputFile = options.preprocessedInputFile || join(this.buildPath, "inputFile.preprocessed");

    console.log("***** Starting snapshot generation using V8 version: ", options.v8Version);

    this.preprocessInputFiles(options.inputFiles, preprocessedInputFile);
    const hostOS = getHostOS();
    const snapshotInDocker = options.snapshotInDocker || this.shouldSnapshotInDocker(hostOS, options.targetArchs);

    // generates the actual .blob and .c files
    return this.generateSnapshots(
        options.snapshotToolsPath,
        preprocessedInputFile,
        options.v8Version,
        options.targetArchs,
        options.useLibs,
        options.mksnapshotParams,
        snapshotInDocker
    ).then(() => {
        this.buildIncludeGradle();
        if (options.useLibs) {
            this.buildSnapshotLibs(options.androidNdkPath, options.recommendedAndroidNdkRevision, options.targetArchs);
        }
        return this.buildPath;
    });
}

SnapshotGenerator.prototype.getSnapshotToolCommand = function (snapshotToolPath, inputFilePath, outputPath, toolParams) {
    return `${snapshotToolPath} ${inputFilePath} --startup_blob ${outputPath} ${toolParams}`;
}

SnapshotGenerator.prototype.getXxdCommand = function (srcOutputDir, xxdLocation) {
    // https://github.com/NativeScript/docker-images/tree/master/v8-snapshot/bin
    return `${xxdLocation || ""}xxd -i ${SNAPSHOT_BLOB_NAME}.blob > ${srcOutputDir}`;
}

SnapshotGenerator.prototype.getPathInDocker = function (mappedLocalDir, mappedDockerDir, targetPath) {
    if (!isSubPath(mappedLocalDir, targetPath)) {
        throw new Error(`Cannot determine a docker path. '${targetPath}' should be inside '${mappedLocalDir}'`)
    }

    const pathInDocker = join(mappedDockerDir, relative(mappedLocalDir, targetPath));

    return convertToUnixPath(pathInDocker);
}

SnapshotGenerator.prototype.handleSnapshotToolResult = function (error, stdout, stderr, inputFile, androidArch) {
    let toolError = null;
    const errorHeader = `Target architecture: ${androidArch}\n`;
    let errorFooter = ``;
    if ((stderr && stderr.length) || error) {
        try {
            require(inputFile);
        }
        catch (e) {
            errorFooter = `\nJavaScript execution error: ${e.stack}$`;
        }
    }

    if (stderr && stderr.length) {
        const message = `${errorHeader}${stderr}${errorFooter}`;
        toolError = new Error(message);
    }
    else if (error) {
        error.message = `${errorHeader}${error.message}${errorFooter}`;
        toolError = error;
    } else {
        console.log(stdout);
    }

    return toolError;
}

SnapshotGenerator.prototype.copySnapshotTool = function (allToolsDir, targetTool, destinationDir) {
    // we cannot mount the source tools folder as its not shared by default:
    // docker: Error response from daemon: Mounts denied:
    // The path /var/folders/h2/1yck52fx2mg7c790vhcw90s8087sk8/T/snapshot-tools/mksnapshot-tools
    // is not shared from OS X and is not known to Docker.
    const toolPathRelativeToAllToolsDir = relative(allToolsDir, targetTool);
    const toolDestinationPath = join(destinationDir, toolPathRelativeToAllToolsDir)
    createDirectory(dirname(toolDestinationPath));
    shelljs.cp(targetTool, toolDestinationPath);

    return toolDestinationPath;
}

SnapshotGenerator.prototype.buildCSource = function (androidArch, blobInputDir, snapshotInDocker) {
    const srcOutputDir = join(this.buildPath, "snapshots/src", androidArch);
    createDirectory(srcOutputDir);
    let command = "";
    if (snapshotInDocker) {
        const blobsInputInDocker = `/blobs/${androidArch}`
        const srcOutputDirInDocker = `/dist/src/${androidArch}`;
        const outputPathInDocker = this.getPathInDocker(srcOutputDir, srcOutputDirInDocker, join(srcOutputDir, `${SNAPSHOT_BLOB_NAME}.c`));
        const buildCSourceCommand = this.getXxdCommand(outputPathInDocker, "/bin/");
        command = `docker run --rm -v "${blobInputDir}:${blobsInputInDocker}" -v "${srcOutputDir}:${srcOutputDirInDocker}" ${SNAPSHOTS_DOCKER_IMAGE} /bin/sh -c "cd ${blobsInputInDocker} && ${buildCSourceCommand}"`;
    }
    else {
        command = this.getXxdCommand(join(srcOutputDir, `${SNAPSHOT_BLOB_NAME}.c`));
    }
    shellJsExecuteInDir(blobInputDir, function () {
        shelljs.exec(command);
    });
}

SnapshotGenerator.prototype.runMksnapshotTool = function (tool, mksnapshotParams, inputFile, snapshotInDocker, snapshotToolsPath, buildCSource) {
    const toolPath = tool.path;
    const androidArch = this.convertToAndroidArchName(tool.arch);
    if (!fs.existsSync(toolPath)) {
        throw new Error(`Can't find mksnapshot tool for ${androidArch} at path ${toolPath}`);
    }

    const tempFolders = [];
    return new Promise((resolve, reject) => {
        console.log("***** Generating snapshot for " + androidArch + " *****");
        const inputFileDir = dirname(inputFile);
        const blobOutputDir = join(this.buildPath, "snapshots/blobs", androidArch);
        createDirectory(blobOutputDir);
        const toolParams = mksnapshotParams || "--profile_deserialization";

        let command = "";
        if (snapshotInDocker) {
            this.setupDocker();
            const appDirInDocker = "/app";
            const blobOutputDirInDocker = `/dist/blobs/${androidArch}`;
            const toolsTempFolder = join(inputFileDir, "tmp");
            tempFolders.push(toolsTempFolder);
            const toolPathInAppDir = this.copySnapshotTool(snapshotToolsPath, toolPath, toolsTempFolder);
            const toolPathInDocker = this.getPathInDocker(inputFileDir, appDirInDocker, toolPathInAppDir);
            const inputFilePathInDocker = this.getPathInDocker(inputFileDir, appDirInDocker, inputFile);
            const outputPathInDocker = this.getPathInDocker(blobOutputDir, blobOutputDirInDocker, join(blobOutputDir, `${SNAPSHOT_BLOB_NAME}.blob`));
            const toolCommandInDocker = this.getSnapshotToolCommand(toolPathInDocker, inputFilePathInDocker, outputPathInDocker, toolParams);
            command = `docker run --rm -v "${inputFileDir}:${appDirInDocker}" -v "${blobOutputDir}:${blobOutputDirInDocker}" ${SNAPSHOTS_DOCKER_IMAGE} /bin/sh -c "${toolCommandInDocker}"`;
        } else {
            command = this.getSnapshotToolCommand(toolPath, inputFile, join(blobOutputDir, `${SNAPSHOT_BLOB_NAME}.blob`), toolParams);
        }

        // Generate .blob file
        child_process.exec(command, { encoding: "utf8" }, (error, stdout, stderr) => {
            tempFolders.forEach(tempFolder => {
                shelljs.rm("-rf", tempFolder);
            });

            const snapshotError = this.handleSnapshotToolResult(error, stdout, stderr, inputFile, androidArch);
            if (snapshotError) {
                return reject(snapshotError);
            }

            return resolve(blobOutputDir);
        });
    }).then((blobOutputDir) => {
        // Generate .c file
        if (buildCSource) {
            this.buildCSource(androidArch, blobOutputDir, snapshotInDocker)
        }

        /*
        Rename TNSSnapshot.blob files to snapshot.blob files. The xxd tool uses the file name for the name of the static array. 
        This is why the *.blob files are initially named  TNSSnapshot.blob. 
        After the xxd step, they must be renamed to snapshot.blob, because this is the filename that the Android runtime is looking for.
        */
        shelljs.mv(join(blobOutputDir, `${SNAPSHOT_BLOB_NAME}.blob`), join(blobOutputDir, `snapshot.blob`));
    });
}