test(functional): Add real test for 'web-ext run -t chromium'

The existing unit tests for web-ext run -t chromium are at
tests/unit/test-extension-runners/test.chromium.js but are testing
implementation details without verifying that Chrome is actually going
to launch the extension.

Before refactoring the implementation as needed to support Chrome 137+
and later, this introduces a unit test that verifies the behavior
against the real Chrome (skipped by default), and a fake Chrome binary
that behaves like the real Chrome binary where relevant.

This will help us with catching regressions in the implementation and/or
downstream dependencies (chrome-launcher).

These Chrome behavior has been verified across Chrome 69, 70, 75, 77,
88, 100, 122, 125, 126, 134, 139.

The auto-reload case of the "real Chrome" test in Chrome 134 requires
manual intervention in the form of visiting chrome://extensions/,
enabling 'Developer Mode' and then manually reloading the extension, for
the following reasons:

- The bundled "web-ext Reload Manager Extension" uses Manifest Version
  2, which is disabled by default in recent Chrome.

- When 'Developer Mode' is off (by default), an attempt to reload
  extensions that were loaded with --load-extension causes the extension
  to be disabled.

These two issues will be resolved once we switch to the new CDP-based
method of loading Chrome extensions.

Author: Rob--W

src/cmd/run.js

@@ -34,6 +34,7 @@ export default async function run(
     noReload = false,
     preInstall = false,
     sourceDir,
+    verbose = false,
     watchFile,
     watchIgnored,
     startUrl,
@@ -188,6 +189,7 @@ export default async function run(
   if (target && target.includes('chromium')) {
     const chromiumRunnerParams = {
       ...commonRunnerParams,
+      verbose,
       chromiumBinary,
       chromiumProfile,
     };

src/extension-runners/chromium.js

@@ -211,6 +211,7 @@ export class ChromiumExtensionRunner {
       chromeFlags,
       startingUrl,
       userDataDir,
+      logLevel: this.params.verbose ? 'verbose' : 'silent',
       // Ignore default flags to keep the extension enabled.
       ignoreDefaultFlags: true,
     });

tests/fixtures/chrome-extension-mv3/background.js

@@ -0,0 +1,12 @@
+/* globals chrome */
+
+chrome.runtime.onInstalled.addListener(() => {
+  // Although this URL is hard-coded, it is easy to set up a test server with a
+  // random port, and let Chrome resolve this host to that local server with:
+  // --host-resolver-rules='MAP localhost:1337 localhost:12345'
+  // (where 12345 is the actual port of the local server)
+  //
+  // We are intentionally using localhost instead of another domain, to make
+  // sure that the browser does not upgrade the http:-request to https.
+  chrome.tabs.create({ url: 'http://localhost:1337/hello_from_extension' });
+});

tests/fixtures/chrome-extension-mv3/manifest.json

@@ -0,0 +1,10 @@
+{
+    "name": "Test extension opens tab on install/reload",
+    "description": "Opens localhost:1337 tab from runtime.onInstalled. Extension ID in Chrome is hgobbjbpnmemikbdbflmolpneekpflab",
+    "version": "1",
+    "manifest_version": 3,
+    "background": {
+        "service_worker": "background.js"
+    },
+    "key": "MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAmr/8tpL+H1zEIEzrOhC8tkheWH7N7FOfiSnLq9D8Uja+b4W69rPIatslIPd4a53IgLT5qv1UZz03kETaKWDYhxoWbaKWiPtJcHoWKAROZ4Ydk2eC7jDcCKYkhXSgTpDzZBhHkyKZoh10O0AEYeJ+xfCmUJT3PT/CzxrSI4rjXFOBie5FMTgjusLwjiwVSBnO9haE7HqwekGVFvixG5Ao6rXRaZx+mm2+7PLTgMSxaXk1OqBtKmz4da67zZGrnHjiz4a0JHkyRuNkyamT5HFjD9neAWjgSwWD5yXKgJKkwGHePsq2WgSZlWIaKFafcbnoIr91hk5+mRCYVm2l/wgqvwIDAQAB"
+}

tests/functional/common.js

@@ -1,3 +1,4 @@
+import { createServer } from 'http';
 import path from 'path';
 import { spawn } from 'child_process';
 import { promisify } from 'util';
@@ -31,6 +32,13 @@ export const fakeServerPath = path.join(
   'fake-amo-server.js',
 );
 
+export const chromeExtPath = path.join(fixturesDir, 'chrome-extension-mv3');
+// NOTE: Depends on preload_on_windows.cjs to load this!
+export const fakeChromePath = path.join(
+  functionalTestsDir,
+  'fake-chrome-binary.js',
+);
+
 // withTempAddonDir helper
 
 const copyDirAsPromised = promisify(copyDir);
@@ -80,7 +88,21 @@ export function execWebExt(argv, spawnOptions) {
       ...process.env,
       ...spawnOptions.env,
     };
+  } else {
+    spawnOptions.env = { ...process.env };
+  }
+
+  if (process.platform === 'win32') {
+    // See preload_on_windows.cjs for an explanation.
+    const preloadPath = path.join(functionalTestsDir, 'preload_on_windows.cjs');
+    // NODE_OPTIONS allows values to be quoted, and anything within to be escaped
+    // with a backslash: https://nodejs.org/api/cli.html#node_optionsoptions
+    // https://github.com/nodejs/node/blob/411495ee9326096e88d12d3f3efae161cbd19efd/src/node_options.cc#L1717-L1741
+    const escapedAbsolutePath = preloadPath.replace(/\\|"/g, '\\$&');
+    spawnOptions.env.NODE_OPTIONS ||= '';
+    spawnOptions.env.NODE_OPTIONS += ` --require "${escapedAbsolutePath}"`;
   }
+
   const spawnedProcess = spawn(
     process.execPath,
     [webExt, ...argv],
@@ -105,3 +127,74 @@ export function execWebExt(argv, spawnOptions) {
 
   return { argv, waitForExit, spawnedProcess };
 }
+
+export function monitorOutput(spawnedProcess) {
+  const callbacks = new Set();
+  let outputData = '';
+  function checkCallbacks() {
+    for (const callback of callbacks) {
+      const { outputTestFunc, resolve } = callback;
+      if (outputTestFunc(outputData)) {
+        callbacks.delete(callback);
+        resolve();
+      }
+    }
+  }
+  spawnedProcess.stdout.on('data', (data) => {
+    outputData += data;
+    checkCallbacks();
+  });
+
+  const waitUntilOutputMatches = (outputTestFunc) => {
+    return new Promise((resolve) => {
+      callbacks.add({ outputTestFunc, resolve });
+      checkCallbacks();
+    });
+  };
+
+  return { waitUntilOutputMatches };
+}
+
+// Test server to receive request from chrome-extension-mv3, once loaded.
+export async function startServerReceivingHelloFromExtension() {
+  let requestCount = 0;
+  let lastSeenUserAgent;
+  let resolveWaitingForHelloFromExtension;
+  const server = createServer((req, res) => {
+    if (req.url !== '/hello_from_extension') {
+      res.writeHead(404);
+      res.end('test server only handles /hello_from_extension');
+      return;
+    }
+    res.writeHead(200);
+    res.end('test server received /hello_from_extension');
+    lastSeenUserAgent = req.headers['user-agent'];
+    ++requestCount;
+    resolveWaitingForHelloFromExtension?.();
+  });
+  await new Promise((resolve) => {
+    server.listen(0, '127.0.0.1', () => resolve());
+  });
+  const testServerHost = `127.0.0.1:${server.address().port}`;
+  return {
+    get requestCount() {
+      return requestCount;
+    },
+    get lastSeenUserAgent() {
+      return lastSeenUserAgent;
+    },
+    getHostResolverRulesArgForChromeBinary() {
+      // chrome-extension-mv3 sends requests to http://localhost:1337, but our
+      // test server uses a free port to make sure that it does not conflict
+      // with an existing local server. Pass --host-resolver-rules to Chrome so
+      // that it sends requests targeting localhost:1337 to this test server.
+      return `--host-resolver-rules=MAP localhost:1337 ${testServerHost}`;
+    },
+    waitForHelloFromExtension: () => {
+      return new Promise((resolve) => {
+        resolveWaitingForHelloFromExtension = resolve;
+      });
+    },
+    close: () => new Promise((resolve) => server.close(() => resolve())),
+  };
+}