Merge pull request #14677 from github/criemen/js-bazel

JS: Move the language pack build and tests to Bazel

Author: criemen

javascript/BUILD.bazel

@@ -1,3 +1,7 @@
+load("@//:dist.bzl", "dist")
+load("@rules_pkg//pkg:mappings.bzl", "pkg_files")
+load("@//buildutils-internal:zipmerge.bzl", "zipmerge")
+
 package(default_visibility = ["//visibility:public"])
 
 alias(
@@ -9,3 +13,36 @@ alias(
     name = "dbscheme-stats",
     actual = "//javascript/ql/lib:dbscheme-stats",
 )
+
+pkg_files(
+    name = "dbscheme-group",
+    srcs = [
+        ":dbscheme",
+        ":dbscheme-stats",
+    ],
+    strip_prefix = None,
+)
+
+dist(
+    name = "javascript-extractor-pack",
+    srcs = [
+        ":dbscheme-group",
+        "//javascript/downgrades",
+        "//javascript/externs",
+        "//javascript/extractor:tools-extractor",
+        "@//language-packs/javascript:resources",
+    ],
+    prefix = "javascript",
+)
+
+# We have to zipmerge in the typescript parser wrapper, as it's generated by a genrule
+# and we don't know a list of its output files. Therefore, we sidestep the
+# rules_pkg tooling here, and generate the zip for the language pack manually.
+zipmerge(
+    name = "javascript",
+    srcs = [
+        ":javascript-extractor-pack.zip",
+        "//javascript/extractor/lib/typescript",
+    ],
+    out = "javascript.zip",
+)

javascript/downgrades/BUILD.bazel

@@ -0,0 +1,11 @@
+load("@//:dist.bzl", "pack_zip")
+
+pack_zip(
+    name = "downgrades",
+    srcs = glob(
+        ["**/*"],
+        exclude = ["BUILD.bazel"],
+    ),
+    prefix = "downgrades",
+    visibility = ["//visibility:public"],
+)

javascript/externs/BUILD.bazel

@@ -0,0 +1,11 @@
+load("@//:dist.bzl", "pack_zip")
+
+pack_zip(
+    name = "externs",
+    srcs = glob(
+        ["**/*"],
+        exclude = ["BUILD.bazel"],
+    ),
+    prefix = "tools/data/externs",
+    visibility = ["//visibility:public"],
+)

javascript/extractor/BUILD.bazel

@@ -1,18 +1,63 @@
-load("@//:common.bzl", "codeql_java_project")
+load("@//:common.bzl", "codeql_fat_jar", "codeql_java_project")
+load("@rules_pkg//pkg:mappings.bzl", "pkg_files")
+
+java_library(
+    name = "deps",
+    visibility = [":__subpackages__"],
+    exports = [
+        "@//extractor:html",
+        "@//extractor:yaml",
+        "@//resources/lib/java:commons-compress",
+        "@//resources/lib/java:gson",
+        "@//resources/lib/java:jericho-html",
+        "@//resources/lib/java:slf4j-api",
+        "@//resources/lib/java:snakeyaml",
+        "@//third_party:jackson",
+        "@//third_party:logback",
+        "@//util-java7",
+        "@//util-java8",
+    ],
+)
 
 codeql_java_project(
     name = "extractor",
     deps = [
-        "@//extractor",
+        ":deps",
+    ],
+)
+
+pkg_files(
+    name = "javascript-extractor-resources",
+    srcs = glob(["resources/**"]),
+    strip_prefix = "resources",
+)
+
+codeql_fat_jar(
+    name = "extractor-javascript",
+    srcs = [
+        ":extractor",
+        "@//extractor:html",
+        "@//extractor:xml-trap-writer",
+        "@//extractor:yaml",
         "@//resources/lib/java:commons-compress",
         "@//resources/lib/java:gson",
         "@//resources/lib/java:jericho-html",
         "@//resources/lib/java:slf4j-api",
         "@//resources/lib/java:snakeyaml",
-        "@//resources/lib/java/DO_NOT_DISTRIBUTE:junit",
         "@//third_party:jackson",
         "@//third_party:logback",
         "@//util-java7",
         "@//util-java8",
     ],
+    files = [":javascript-extractor-resources"],
+    main_class = "com.semmle.js.extractor.Main",
+)
+
+pkg_files(
+    name = "tools-extractor",
+    srcs = [
+        ":extractor-javascript",
+    ],
+    prefix = "tools",
+    visibility = ["//visibility:public"],
 )

javascript/extractor/lib/typescript/BUILD.bazel

@@ -0,0 +1,55 @@
+load("@//:common.bzl", "on_windows")
+
+# Builds a zip file of the compiled typscript-parser-wrapper and its dependencies.
+genrule(
+    name = "typescript",
+    srcs = [
+        "tsconfig.json",
+        "package.json",
+        "package-lock.json",
+    ] + glob(["src/**"]),
+    outs = ["javascript.zip"],
+    cmd = "\n".join([
+        # the original working directory is not preserved anywhere, but needs to be accessible, as
+        # all paths are relative to this
+        # unfortunately, we need to change the working directory to run npm.
+        "export BAZEL_ROOT=$$(pwd)",
+        # we need a temp dir, and unfortunately, $TMPDIR is not set on Windows
+        "export TEMP=$$(mktemp -d)",
+        # Add node to the path so that npm run can find it - it's calling env node
+        "export PATH=$$PATH:$$BAZEL_ROOT/$$(dirname $(execpath @nodejs//:node_bin))",
+        "export NPM=$$BAZEL_ROOT/$(execpath @nodejs//:npm_bin)",
+        # npm has a global cache which doesn't work on macos, where absolute paths aren't filtered out by the sandbox.
+        # Therefore, set a temporary cache directory.
+        "export NPM_CONFIG_USERCONFIG=$$TEMP/npmrc",
+        "$$NPM config set cache $$TEMP/npm",
+        "$$NPM config set update-notifier false",
+        "rm -rf $(RULEDIR)/inputs",
+        "cp -L -R $$(dirname $(execpath package.json)) $(RULEDIR)/inputs",
+        "cd $(RULEDIR)/inputs",
+        "$$NPM install",
+        "$$NPM run build",
+        "rm -rf node_modules",
+        # Install again with only runtime deps
+        "$$NPM install --prod",
+        "mv node_modules build/",
+        "mkdir -p javascript/tools/typescript-parser-wrapper",
+        "mv build/* javascript/tools/typescript-parser-wrapper",
+        "",
+    ]) + on_windows(
+        " && ".join([
+            "$$BAZEL_ROOT/$(execpath @bazel_tools//tools/zip:zipper) cC $$(cygpath -w $$BAZEL_ROOT/$@) $$(find javascript -name '*' -print)",
+            "rm -rf $$TEMP",
+        ]),
+        " && ".join([
+            "$$BAZEL_ROOT/$(execpath @bazel_tools//tools/zip:zipper) cC $$BAZEL_ROOT/$@ $$(find javascript -name '*' -print)",
+            "rm -rf $$TEMP",
+        ]),
+    ),
+    tools = [
+        "@bazel_tools//tools/zip:zipper",
+        "@nodejs//:node_bin",
+        "@nodejs//:npm_bin",
+    ],
+    visibility = ["//visibility:public"],
+)

javascript/extractor/parser-tests/BUILD.bazel

@@ -0,0 +1,5 @@
+filegroup(
+    name = "parser-tests",
+    srcs = glob(["**/*"]),
+    visibility = ["//visibility:public"],
+)

javascript/extractor/src/com/semmle/js/extractor/test/ASTMatchingTests.java renamed to javascript/extractor/test/com/semmle/js/extractor/test/ASTMatchingTests.java

@@ -12,6 +12,7 @@
 import com.semmle.js.ast.AST2JSON;
 import com.semmle.js.ast.Program;
 import com.semmle.util.io.WholeIO;
+import com.semmle.util.tests.TestPaths;
 import java.io.File;
 import java.util.Iterator;
 import java.util.Map.Entry;
@@ -54,7 +55,7 @@ protected void assertMatch(String path, JsonElement expected, JsonElement actual
     }
   }
 
-  private static final File BABYLON_BASE = new File("parser-tests/babylon").getAbsoluteFile();
+  private static final File BABYLON_BASE = TestPaths.get("parser-tests/babylon").toAbsolutePath().toFile();
 
   protected void babylonTest(String dir) {
     babylonTest(dir, new Options().esnext(true));

javascript/extractor/src/com/semmle/js/extractor/test/AllTests.java renamed to javascript/extractor/test/com/semmle/js/extractor/test/AllTests.java

@@ -30,6 +30,7 @@
 import com.semmle.js.extractor.FileExtractor.FileType;
 import com.semmle.js.extractor.VirtualSourceRoot;
 import com.semmle.util.data.StringUtil;
+import com.semmle.util.exception.Exceptions;
 import com.semmle.util.exception.UserError;
 import com.semmle.util.files.FileUtil;
 import com.semmle.util.files.FileUtil8;
@@ -443,8 +444,12 @@ public void excludeByClassificationBadPath() throws IOException {
 
   /** Hide {@code p} on using {@link DosFileAttributeView} if available; otherwise do nothing. */
   private void hide(Path p) throws IOException {
+    try {
     DosFileAttributeView attrs = Files.getFileAttributeView(p, DosFileAttributeView.class);
     if (attrs != null) attrs.setHidden(true);
+    } catch (IOException e) {
+      Exceptions.ignore(e, "On Linux within the bazel sandbox, we get a DosFileAttributeView that then throws an exception upon use");
+    }
   }
 
   @Test

javascript/extractor/src/com/semmle/js/extractor/test/AutoBuildTests.java renamed to javascript/extractor/test/com/semmle/js/extractor/test/AutoBuildTests.java

@@ -0,0 +1,33 @@
+java_test(
+    name = "test_jar",
+    srcs = glob(["**/*.java"]),
+    test_class = "com.semmle.js.extractor.test.AllTests",
+    deps = [
+        "//javascript/extractor",
+        "//javascript/extractor:deps",
+        "@//resources/lib/java/DO_NOT_DISTRIBUTE:junit",
+    ],
+)
+
+# We need to unzip the typescript wrapper, and provide node on the path.
+# Therefore, we're wrapping the java_test inside a sh_test.
+sh_test(
+    name = "test",
+    size = "small",
+    srcs = ["run_tests.sh"],
+    args = [
+        "$(execpath @nodejs//:node_bin)",
+        "$(JAVABASE)/bin/java",
+        "$(rootpath //javascript/extractor/lib/typescript)",
+        "$(rootpath test_jar_deploy.jar)",
+    ],
+    data = [
+        ":test_jar_deploy.jar",
+        "//javascript/extractor/lib/typescript",
+        "//javascript/extractor/parser-tests",
+        "//javascript/extractor/tests",
+        "@bazel_tools//tools/jdk:current_java_runtime",
+        "@nodejs//:node_bin",
+    ],
+    toolchains = ["@bazel_tools//tools/jdk:current_java_runtime"],
+)

javascript/extractor/test/com/semmle/js/extractor/test/BUILD.bazel

@@ -10,6 +10,7 @@
 import com.semmle.js.ast.Program;
 import com.semmle.util.files.FileUtil;
 import com.semmle.util.io.WholeIO;
+import com.semmle.util.tests.TestPaths;
 import java.io.File;
 import java.util.ArrayList;
 import java.util.List;
@@ -25,7 +26,7 @@
  */
 @RunWith(Parameterized.class)
 public class JSXTests extends ASTMatchingTests {
-  private static final File BASE = new File("parser-tests/jcorn-jsx").getAbsoluteFile();
+  private static final File BASE = TestPaths.get("parser-tests/jcorn-jsx").toAbsolutePath().toFile();
 
   @Parameters(name = "{0}")
   public static Iterable<Object[]> tests() {

javascript/extractor/src/com/semmle/js/extractor/test/ClassPropertiesTests.java renamed to javascript/extractor/test/com/semmle/js/extractor/test/ClassPropertiesTests.java

@@ -3,6 +3,7 @@
 import com.semmle.jcorn.Options;
 import com.semmle.jcorn.Parser;
 import com.semmle.util.io.WholeIO;
+import com.semmle.util.tests.TestPaths;
 import java.io.File;
 import java.nio.charset.StandardCharsets;
 import org.junit.Test;
@@ -11,7 +12,7 @@ public class RobustnessTests {
 
   @Test
   public void letLookheadTest() {
-    File test = new File("parser-tests/robustness/letLookahead.js");
+    File test = TestPaths.get("parser-tests/robustness/letLookahead.js").toFile();
     String src = new WholeIO(StandardCharsets.UTF_8.name()).strictread(test);
     new Parser(new Options(), src, 0).parse();
   }

javascript/extractor/src/com/semmle/js/extractor/test/DecoratorTests.java renamed to javascript/extractor/test/com/semmle/js/extractor/test/DecoratorTests.java

@@ -13,6 +13,7 @@
 import com.semmle.util.io.WholeIO;
 import com.semmle.util.process.Env;
 import com.semmle.util.srcarchive.DummySourceArchive;
+import com.semmle.util.tests.TestPaths;
 import com.semmle.util.trap.ITrapWriterFactory;
 import com.semmle.util.trap.TrapWriter;
 import com.semmle.util.trap.pathtransformers.ProjectLayoutTransformer;
@@ -35,7 +36,7 @@
 
 @RunWith(Parameterized.class)
 public class TrapTests {
-  private static final File BASE = new File("tests").getAbsoluteFile();
+  private static final File BASE = TestPaths.get("tests").toAbsolutePath().toFile();
 
   @Parameters(name = "{0}:{1}")
   public static Iterable<Object[]> tests() {

javascript/extractor/src/com/semmle/js/extractor/test/ExportExtensionsTests.java renamed to javascript/extractor/test/com/semmle/js/extractor/test/ExportExtensionsTests.java

@@ -0,0 +1,31 @@
+NODE=$1
+JAVA=$2
+PARSER_WRAPPER=$3
+TEST_JAR=$4
+
+TEMP=$(mktemp -d)
+
+UNAME=$(uname -s)
+echo $UNAME
+# On Windows, the symlink set up by bazel that points at the test jar is a msys2/linux-style path
+# The JVM can't resolve that, therefore copy the jar to the temp directory, and then set the
+# windows path to it
+if [[ "$UNAME" =~ _NT ]]; then
+    cp $TEST_JAR $TEMP/test.jar
+    TEST_JAR=$(cygpath -w $TEMP/test.jar)
+    echo "On Windows, new test jar: $TEST_JAR"
+fi
+
+# unpack parser wrapper
+unzip -q $PARSER_WRAPPER -d $TEMP/parser_wrapper
+export SEMMLE_TYPESCRIPT_PARSER_WRAPPER=$TEMP/parser_wrapper/javascript/tools/typescript-parser-wrapper/main.js
+
+# setup node on path
+NODE=$(realpath $NODE)
+export PATH="$PATH:$(dirname $NODE)"
+
+$JAVA -Dbazel.test_suite=com.semmle.js.extractor.test.AllTests -jar $TEST_JAR
+EXIT_CODE=$?
+
+rm -rf $TEMP
+exit $EXIT_CODE

javascript/extractor/src/com/semmle/js/extractor/test/FunctionSentTests.java renamed to javascript/extractor/test/com/semmle/js/extractor/test/FunctionSentTests.java

@@ -0,0 +1,5 @@
+filegroup(
+    name = "tests",
+    srcs = glob(["**/*"]),
+    visibility = ["//visibility:public"],
+)