feat: auto-reload support for Chrome using CDP

This is a re-implementation of auto-reload support in Chrome,
by opening chrome://extensions/ and using a private API.
It has been confirmed to work with Chrome 69 until 139.

The functional unit tests only work in Chrome 75 until 133:
- Chrome 74 and earlier do not support MV3 background service_worker,
  and the test extension uses it. Rewriting the extension to the MV2
  variant causes it to pass in Chrome 69.
- Chrome 134 disables unpacked extensions upon reload when Developer
  Mode is disabled: https://issues.chromium.org/issues/362756477

The next commit will introduce a more reliable mechanism of reloading
extensions, using the same mechanism as loading the extension, which
works in Chrome 126+.

Tested as follows for various relevant versions of Chromium:
CHROME_PATH=/path/to/chrome75/chromium TEST_LOG_VERBOSE=1 TEST_WEBEXT_USE_REAL_CHROME=1 ./node_modules/.bin/mocha tests/functional/test.cli.run-target-chromium.js --grep='with auto-reload run real'

Unit tests with fake Chrome binary that simulates the real binary:
./node_modules/.bin/mocha tests/functional/test.cli.run-target-chromium.js --grep=auto-reload

Pre-existing unit test that tested web-ext internals:
./node_modules/.bin/mocha tests/unit/test-extension-runners/test.chromium.js

Author: Rob--W

src/extension-runners/chromium.js

@@ -28,6 +28,119 @@ export const DEFAULT_CHROME_FLAGS = ChromeLauncher.defaultFlags().filter(
   (flag) => !EXCLUDED_CHROME_FLAGS.includes(flag),
 );
 
+// This is a client for the Chrome Devtools protocol. The methods and results
+// are documented at https://chromedevtools.github.io/devtools-protocol/tot/
+class ChromeDevtoolsProtocolClient {
+  #receivedData = '';
+  #isProcessingMessage = false;
+  #lastId = 0;
+  #deferredResponses = new Map();
+  #disconnected = false;
+  #disconnectedPromise;
+  #resolveDisconnectedPromise;
+
+  // Print all exchanged CDP messages to ease debugging.
+  TEST_LOG_VERBOSE_CDP = process.env.TEST_LOG_VERBOSE_CDP;
+
+  constructor(chromiumInstance) {
+    // remoteDebuggingPipes is from chrome-launcher, see
+    // https://github.com/GoogleChrome/chrome-launcher/pull/347
+    const { incoming, outgoing } = chromiumInstance.remoteDebuggingPipes;
+    this.#disconnectedPromise = new Promise((resolve) => {
+      this.#resolveDisconnectedPromise = resolve;
+    });
+    if (incoming.closed) {
+      // Strange. Did Chrome fail to start, or exit on startup?
+      log.warn('CDP already disconnected at initialization');
+      this.#finalizeDisconnect();
+      return;
+    }
+    incoming.on('data', (data) => {
+      this.#receivedData += data;
+      this.#processNextMessage();
+    });
+    incoming.on('error', (error) => {
+      log.error(error);
+      this.#finalizeDisconnect();
+    });
+    incoming.on('close', () => this.#finalizeDisconnect());
+    this.outgoingPipe = outgoing;
+  }
+
+  waitUntilDisconnected() {
+    return this.#disconnectedPromise;
+  }
+
+  async sendCommand(method, params, sessionId = undefined) {
+    if (this.#disconnected) {
+      throw new Error(`CDP disconnected, cannot send: command ${method}`);
+    }
+    const message = {
+      id: ++this.#lastId,
+      method,
+      params,
+      sessionId,
+    };
+    const rawMessage = `${JSON.stringify(message)}\x00`;
+    if (this.TEST_LOG_VERBOSE_CDP) {
+      process.stderr.write(`[CDP] [SEND] ${rawMessage}\n`);
+    }
+    return new Promise((resolve, reject) => {
+      // CDP will always send a response.
+      this.#deferredResponses.set(message.id, { method, resolve, reject });
+      this.outgoingPipe.write(rawMessage);
+    });
+  }
+
+  #processNextMessage() {
+    if (this.#isProcessingMessage) {
+      return;
+    }
+    this.#isProcessingMessage = true;
+    let end = this.#receivedData.indexOf('\x00');
+    while (end !== -1) {
+      const rawMessage = this.#receivedData.slice(0, end);
+      this.#receivedData = this.#receivedData.slice(end + 1); // +1 skips \x00.
+      try {
+        if (this.TEST_LOG_VERBOSE_CDP) {
+          process.stderr.write(`[CDP] [RECV] ${rawMessage}\n`);
+        }
+        const { id, error, result } = JSON.parse(rawMessage);
+        const deferredResponse = this.#deferredResponses.get(id);
+        if (deferredResponse) {
+          this.#deferredResponses.delete(id);
+          if (error) {
+            const err = new Error(error.message || 'Unexpected CDP response');
+            deferredResponse.reject(err);
+          } else {
+            deferredResponse.resolve(result);
+          }
+        } else {
+          // Dropping events and non-response messages since we don't need it.
+        }
+      } catch (e) {
+        log.error(e);
+      }
+      end = this.#receivedData.indexOf('\x00');
+    }
+    this.#isProcessingMessage = false;
+    if (this.#disconnected) {
+      for (const { method, reject } of this.#deferredResponses.values()) {
+        reject(new Error(`CDP connection closed before response to ${method}`));
+      }
+      this.#deferredResponses.clear();
+      this.#resolveDisconnectedPromise();
+    }
+  }
+
+  #finalizeDisconnect() {
+    if (!this.#disconnected) {
+      this.#disconnected = true;
+      this.#processNextMessage();
+    }
+  }
+}
+
 /**
  * Implements an IExtensionRunner which manages a Chromium instance.
  */
@@ -36,13 +149,17 @@ export class ChromiumExtensionRunner {
   params;
   chromiumInstance;
   chromiumLaunch;
+  // --load-extension is deprecated, but only supported in Chrome 126+, see:
+  // https://github.com/mozilla/web-ext/issues/3388#issuecomment-2906982117
+  forceUseDeprecatedLoadExtension;
   exiting;
   _promiseSetupDone;
 
   constructor(params) {
     const { chromiumLaunch = defaultChromiumLaunch } = params;
     this.params = params;
     this.chromiumLaunch = chromiumLaunch;
+    this.forceUseDeprecatedLoadExtension = true;
     this.cleanupCallbacks = new Set();
   }
 
@@ -108,9 +225,7 @@ export class ChromiumExtensionRunner {
    */
   async setupInstance() {
     // Start chrome pointing it to a given profile dir
-    const extensions = this.params.extensions
-      .map(({ sourceDir }) => sourceDir)
-      .join(',');
+    const extensions = this.params.extensions.map(({ sourceDir }) => sourceDir);
 
     const { chromiumBinary } = this.params;
 
@@ -121,8 +236,11 @@ export class ChromiumExtensionRunner {
     }
 
     const chromeFlags = [...DEFAULT_CHROME_FLAGS];
+    chromeFlags.push('--remote-debugging-pipe');
 
-    chromeFlags.push(`--load-extension=${extensions}`);
+    if (this.forceUseDeprecatedLoadExtension) {
+      chromeFlags.push(`--load-extension=${extensions.join(',')}`);
+    }
 
     if (this.params.args) {
       chromeFlags.push(...this.params.args);
@@ -189,6 +307,7 @@ export class ChromiumExtensionRunner {
       // Ignore default flags to keep the extension enabled.
       ignoreDefaultFlags: true,
     });
+    this.cdp = new ChromeDevtoolsProtocolClient(this.chromiumInstance);
 
     this.chromiumInstance.process.once('close', () => {
       this.chromiumInstance = null;
@@ -198,6 +317,17 @@ export class ChromiumExtensionRunner {
         this.exit();
       }
     });
+
+    // Connect with the CDP to verify that we are indeed connected to Chrome.
+    // This works with Chrome 69 and later, see
+    // https://github.com/mozilla/web-ext/issues/3388#issuecomment-2906982117
+    try {
+      log.debug('Verifying chrome devtools protocol connection...');
+      const { userAgent } = await this.cdp.sendCommand('Browser.getVersion');
+      log.info(`Launched Chromium: ${userAgent}`);
+    } catch (e) {
+      log.error(e);
+    }
   }
 
   /**
@@ -207,7 +337,9 @@ export class ChromiumExtensionRunner {
   async reloadAllExtensions() {
     const runnerName = this.getName();
 
-    // TODO: Restore reload functionality using the remote debugging protocol.
+    if (this.forceUseDeprecatedLoadExtension) {
+      this.reloadAllExtensionsFallbackForChrome125andEarlier();
+    }
 
     process.stdout.write(
       `\rLast extension reload: ${new Date().toTimeString()}`,
@@ -217,6 +349,136 @@ export class ChromiumExtensionRunner {
     return [{ runnerName }];
   }
 
+  async reloadAllExtensionsFallbackForChrome125andEarlier() {
+    // Ideally, we'd like to use the "Extensions.loadUnpacked" CDP command to
+    // reload an extension, but that is unsupported in Chrome 125 and earlier.
+    //
+    // As a fallback, connect to chrome://extensions/ and reload from there.
+    // Since we are targeting old Chrome versions, we can safely use the
+    // chrome.developerPrivate APIs, because these are never going to change
+    // for the old browser versions. Do NOT use this for newer versions!
+    //
+    // Target.* CDP methods documented at: https://chromedevtools.github.io/devtools-protocol/tot/Target/
+    // developerPrivate documented at:
+    // https://source.chromium.org/chromium/chromium/src/+/main:chrome/common/extensions/api/developer_private.idl
+    //
+    // Specific revision that exposed developerPrivate to chrome://extensions/:
+    // https://source.chromium.org/chromium/chromium/src/+/main:chrome/common/extensions/api/developer_private.idl;drc=69bf75316e7ae533c0a0dccc1a56ca019aa95a1e
+    // https://chromium.googlesource.com/chromium/src.git/+/69bf75316e7ae533c0a0dccc1a56ca019aa95a1e
+    //
+    // Specific revision that introduced developerPrivate.getExtensionsInfo:
+    // https://source.chromium.org/chromium/chromium/src/+/main:chrome/common/extensions/api/developer_private.idl;drc=69bf75316e7ae533c0a0dccc1a56ca019aa95a1e
+    //
+    // The above changes are from 2015; The --remote-debugging-pipe feature
+    // that we rely on for CDP was added in 2018; this is the version of the
+    // developerPrivate API at that time:
+    // https://source.chromium.org/chromium/chromium/src/+/main:chrome/common/extensions/api/developer_private.idl;drc=c9ae59c8f37d487f1f01c222deb6b7d1f51c99c2
+
+    // Find an existing chrome://extensions/ tab, if it exists.
+    let { targetInfos: targets } = await this.cdp.sendCommand(
+      'Target.getTargets',
+      { filter: [{ type: 'tab' }] },
+    );
+    targets = targets.filter((t) => t.url.startsWith('chrome://extensions/'));
+    let targetId;
+    const hasExistingTarget = targets.length > 0;
+    if (hasExistingTarget) {
+      targetId = targets[0].targetId;
+    } else {
+      const result = await this.cdp.sendCommand('Target.createTarget', {
+        url: 'chrome://extensions/',
+        newWindow: true,
+        background: true,
+        windowState: 'minimized',
+      });
+      targetId = result.targetId;
+    }
+    const codeToEvaluateInChrome = async () => {
+      // This function is serialized and executed in Chrome. Designed for
+      // compatibility with Chrome 69 - 125. Do not use JS syntax of functions
+      // that are not supported in these versions!
+
+      // eslint-disable-next-line no-undef
+      const developerPrivate = chrome.developerPrivate;
+      if (!developerPrivate || !developerPrivate.getExtensionsInfo) {
+        // When chrome://extensions/ is still loading, its document URL may be
+        // about:blank and the chrome.developerPrivate API is not exposed.
+        return 'NOT_READY_PLEASE_RETRY';
+      }
+      const extensionIds = [];
+      await new Promise((resolve) => {
+        developerPrivate.getExtensionsInfo((extensions) => {
+          for (const extension of extensions || []) {
+            if (extension.location === 'UNPACKED') {
+              // We only care about those loaded via --load-extension.
+              extensionIds.push(extension.id);
+            }
+          }
+          resolve();
+        });
+      });
+      const reloadPromises = extensionIds.map((extensionId) => {
+        return new Promise((resolve, reject) => {
+          developerPrivate.reload(
+            extensionId,
+            // Suppress alert dialog when load fails.
+            { failQuietly: true, populateErrorForUnpacked: true },
+            (loadError) => {
+              if (loadError) {
+                reject(new Error(loadError.error));
+              } else {
+                resolve();
+              }
+            },
+          );
+        });
+      });
+      await Promise.all(reloadPromises);
+      return reloadPromises.length;
+    };
+    try {
+      const targetResult = await this.cdp.sendCommand('Target.attachToTarget', {
+        targetId,
+        flatten: true,
+      });
+      if (!targetResult.sessionId) {
+        throw new Error('Unexpectedly, no sessionId from attachToTarget');
+      }
+      // In practice, we're going to run the logic only once. But if we are
+      // unlucky, chrome://extensions is still loading, so we will then retry.
+      for (let i = 0; i < 3; ++i) {
+        const evalResult = await this.cdp.sendCommand(
+          'Runtime.evaluate',
+          {
+            expression: `(${codeToEvaluateInChrome})();`,
+            awaitPromise: true,
+          },
+          targetResult.sessionId,
+        );
+        const evalResultReturnValue = evalResult.result?.value;
+        if (evalResultReturnValue === 'NOT_READY_PLEASE_RETRY') {
+          await new Promise((r) => setTimeout(r, 200 * i));
+          continue;
+        }
+        if (evalResult.exceptionDetails) {
+          log.error(`Failed to reload: ${evalResult.exceptionDetails.text}`);
+        }
+        if (evalResultReturnValue !== this.params.extensions.length) {
+          log.warn(`Failed to reload extensions: ${evalResultReturnValue}`);
+        }
+        break;
+      }
+    } finally {
+      if (!hasExistingTarget && targetId) {
+        try {
+          await this.cdp.sendCommand('Target.closeTarget', { targetId });
+        } catch (e) {
+          log.error(e);
+        }
+      }
+    }
+  }
+
   /**
    * Reloads a single extension, collect any reload error and resolves to
    * an array composed by a single ExtensionRunnerReloadResult object.
@@ -259,6 +521,11 @@ export class ChromiumExtensionRunner {
       this.chromiumInstance = null;
     }
 
+    if (this.cdp) {
+      await this.cdp.waitUntilDisconnected();
+      this.cdp = null;
+    }
+
     // Call all the registered cleanup callbacks.
     for (const fn of this.cleanupCallbacks) {
       try {