From c1b47c32e10209d1a948b6ff5f503f60f08ce113 Mon Sep 17 00:00:00 2001 From: Daniel La Rocque Date: Thu, 5 Sep 2024 15:28:52 -0400 Subject: [PATCH 1/3] Introduce WebkitHeadless browser testing Replace Safari with WebkitHeadless in list of supported karma browsers Install playwright browsers in CI Set WEBKIT_HEADLESS_BIN Update launcher version Run firestore test changed on push Fix Don't use in test-changed-firestore Ignore failing tests --- .github/workflows/test-all.yml | 5 ++ .github/workflows/test-changed-auth.yml | 25 ++++++++ .github/workflows/test-changed-firestore.yml | 60 +++++++++++++++++++- .github/workflows/test-changed.yml | 26 +++++++++ config/karma.base.js | 37 ++++++++++-- package.json | 3 +- scripts/ci-test/testConfig.ts | 12 ++++ yarn.lock | 42 ++++++++++---- 8 files changed, 193 insertions(+), 17 deletions(-) diff --git a/.github/workflows/test-all.yml b/.github/workflows/test-all.yml index eda081d1df7..eef9bfc4224 100644 --- a/.github/workflows/test-all.yml +++ b/.github/workflows/test-all.yml @@ -91,6 +91,11 @@ jobs: run: echo "FIREBASE_CI_TEST_START_TIME=$(date +%s)" >> $GITHUB_ENV - name: Run unit tests # Ignore auth and firestore since they're handled in their own separate jobs. + # TODO (dlarocque): Fix flakes in installations, database, database-compat, storage-compat, + # and remote-config, and then run them here. + # There are several tests suites that are flaky when ran alongside the other tests in + # WebkitHeadless. It's unclear why they're only failing when ran with other tests, so let's + # skip them for now. run: | xvfb-run yarn lerna run --ignore '{firebase-messaging-integration-test,@firebase/auth*,@firebase/firestore*,firebase-firestore-integration-test}' test:ci node scripts/print_test_logs.js diff --git a/.github/workflows/test-changed-auth.yml b/.github/workflows/test-changed-auth.yml index 67d16551b3e..52b08c12e21 100644 --- a/.github/workflows/test-changed-auth.yml +++ b/.github/workflows/test-changed-auth.yml @@ -102,3 +102,28 @@ jobs: run: xvfb-run yarn test:changed auth env: BROWSERS: 'Firefox' + + test-webkit: + name: Test Auth on Webkit if Changed + runs-on: macos-latest + + steps: + - name: Checkout Repo + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Set up Node (20) + uses: actions/setup-node@v3 + with: + node-version: 20.x + - name: Test setup and yarn install + run: | + cp config/ci.config.json config/project.json + yarn + npx playwright install webkit + - name: build + run: yarn build:changed auth + - name: Run tests on changed packages + run: yarn test:changed auth + env: + BROWSERS: 'WebkitHeadless' \ No newline at end of file diff --git a/.github/workflows/test-changed-firestore.yml b/.github/workflows/test-changed-firestore.yml index e148d164909..9bd7cc33bee 100644 --- a/.github/workflows/test-changed-firestore.yml +++ b/.github/workflows/test-changed-firestore.yml @@ -16,7 +16,7 @@ name: Test Firestore on: workflow_dispatch: - pull_request: + push: env: artifactRetentionDays: 14 @@ -231,6 +231,62 @@ jobs: BROWSERS: 'Firefox' EXPERIMENTAL_MODE: true + compat-test-webkit: + name: Test Firestore Compatible on Webkit + runs-on: macos-latest + needs: build + if: ${{ needs.build.outputs.changed == 'true'}} + steps: + - name: Set up Node (20) + uses: actions/setup-node@v3 + with: + node-version: 20.x + - name: Download build archive + uses: actions/download-artifact@v3 + with: + name: build.tar.gz + - name: Unzip build artifact + run: tar xf build.tar.gz + - name: Test setup + run: | + cp config/ci.config.json config/project.json + npx playwright install webkit + - name: Run compat tests + run: cd packages/firestore-compat && yarn run test:ci + env: + BROWSERS: 'WebkitHeadless' + + test-webkit: + name: Test Firestore on Webkit + strategy: + matrix: + # TODO (dlarocque): Add test:travis once the browser tests are isolated + # Exclude test:travis for now, since it includes node tests, which are failing for + # some reason. + test-name: ["test:browser", "test:lite:browser", "test:browser:prod:nameddb", "test:lite:browser:nameddb"] + runs-on: macos-latest + needs: build + if: ${{ needs.build.outputs.changed == 'true'}} + steps: + - name: Download build archive + uses: actions/download-artifact@v3 + with: + name: build.tar.gz + - name: Unzip build artifact + run: tar xf build.tar.gz + - name: Set up Node (20) + uses: actions/setup-node@v3 + with: + node-version: 20.x + - name: Test setup + run: | + cp config/ci.config.json config/project.json + npx playwright install webkit + - name: Run tests + run: cd packages/firestore && yarn run ${{ matrix.test-name }} + env: + BROWSERS: 'WebkitHeadless' + EXPERIMENTAL_MODE: true # A job that fails if any required job in the test matrix fails, # to be used as a required check for merging. check-required-tests: @@ -241,4 +297,4 @@ jobs: steps: - name: Check test matrix if: needs.build.result == 'failure' || needs.test-chrome.result == 'failure' || needs.compat-test-chrome.result == 'failure' - run: exit 1 + run: exit 1 \ No newline at end of file diff --git a/.github/workflows/test-changed.yml b/.github/workflows/test-changed.yml index 7da82df4a26..5119c83cb37 100644 --- a/.github/workflows/test-changed.yml +++ b/.github/workflows/test-changed.yml @@ -78,3 +78,29 @@ jobs: run: xvfb-run yarn test:changed core env: BROWSERS: 'Firefox' + + + test-webkit: + name: Test Packages With Changed Files in Webkit + runs-on: macos-latest + + steps: + - name: Checkout Repo + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Set up Node (20) + uses: actions/setup-node@v3 + with: + node-version: 20.x + - name: Test setup and yarn install + run: | + cp config/ci.config.json config/project.json + yarn + npx playwright install webkit + - name: build + run: yarn build:changed core + - name: Run tests on changed packages + run: yarn test:changed core + env: + BROWSERS: 'WebkitHeadless' \ No newline at end of file diff --git a/config/karma.base.js b/config/karma.base.js index e074987f29b..a2671ab02d0 100644 --- a/config/karma.base.js +++ b/config/karma.base.js @@ -20,6 +20,34 @@ const path = require('path'); const webpackTestConfig = require('./webpack.test'); const { argv } = require('yargs'); +function determineBrowsers() { + const supportedBrowsers = ['ChromeHeadless', 'WebkitHeadless', 'Firefox']; + + if (process.env.BROWSERS) { + const browsers = process.env.BROWSERS.split(','); + + const validBrowsers = browsers.filter(browser => + supportedBrowsers.includes(browser) + ); + if (validBrowsers.length === 0) { + console.error( + `The \'BROWSER\' environment variable was set, but no supported browsers were listed. The supported browsers are ${JSON.stringify( + supportedBrowsers + )}.` + ); + return []; + } else { + return validBrowsers; + } + + } else { + console.log( + "The 'BROWSER' environment variable is undefined. Defaulting to 'ChromeHeadless'." + ); + return ['ChromeHeadless']; + } +} + const config = { // disable watcher autoWatch: false, @@ -57,10 +85,11 @@ const config = { // changes autoWatch: false, - // start these browsers - // available browser launchers: - // https://npmjs.org/browse/keyword/karma-launcher - browsers: process.env?.BROWSERS?.split(',') ?? ['ChromeHeadless'], + // Browsers to launch for testing + // To use a custom set of browsers, define the BROWSERS environment variable as a comma-seperated list. + // Supported browsers are 'ChromeHeadless', 'WebkitHeadless', and 'Firefox'. + // See: https://karma-runner.github.io/6.4/config/browsers.html + browsers: determineBrowsers(), webpack: webpackTestConfig, diff --git a/package.json b/package.json index 7c65287bb20..c301feb69de 100644 --- a/package.json +++ b/package.json @@ -120,10 +120,10 @@ "karma-firefox-launcher": "2.1.3", "karma-mocha": "2.0.1", "karma-mocha-reporter": "2.2.5", - "karma-safari-launcher": "1.0.0", "karma-sourcemap-loader": "0.4.0", "karma-spec-reporter": "0.0.36", "karma-summary-reporter": "3.1.1", + "karma-webkit-launcher": "2.6.0", "karma-webpack": "5.0.0", "lcov-result-merger": "3.3.0", "lerna": "4.0.0", @@ -139,6 +139,7 @@ "nyc": "15.1.0", "ora": "5.4.1", "patch-package": "7.0.2", + "playwright": "1.46.1", "postinstall-postinstall": "2.1.0", "prettier": "2.8.7", "protractor": "5.4.2", diff --git a/scripts/ci-test/testConfig.ts b/scripts/ci-test/testConfig.ts index 5f458a07605..1092442889d 100644 --- a/scripts/ci-test/testConfig.ts +++ b/scripts/ci-test/testConfig.ts @@ -28,11 +28,23 @@ export interface TestConfig { alwaysIncludePackages?: string[]; } +// These tests are flaky on WebkitHeadless for some reason, so skip them. +// TODO (dlarocque): Fix the flakes and remove this +const ignoredWebkitCoreTests = process.env?.BROWSERS?.includes("WebkitHeadless") ? [ + '@firebase/app-check', + '@firebase/installations', + '@firebase/storage', + '@firebase/storage-compat', + '@firebase/database', + '@firebase/database-compat', +] : []; + export const testConfig: { [key: string]: TestConfig | undefined; } = { 'core': { 'ignorePackages': [ + ...ignoredWebkitCoreTests, '@firebase/firestore', '@firebase/firestore-compat', 'firebase-firestore-integration-test', diff --git a/yarn.lock b/yarn.lock index 477e784871e..cf0282a773a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -8653,6 +8653,11 @@ fs.realpath@^1.0.0: resolved "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz" integrity sha1-FQStJSMVjKpA20onh8sBQRmU6k8= +fsevents@2.3.2, fsevents@^2.3.2, fsevents@~2.3.2: + version "2.3.2" + resolved "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz" + integrity sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA== + fsevents@^1.2.7: version "1.2.13" resolved "https://registry.npmjs.org/fsevents/-/fsevents-1.2.13.tgz" @@ -8661,11 +8666,6 @@ fsevents@^1.2.7: bindings "^1.5.0" nan "^2.12.1" -fsevents@^2.3.2, fsevents@~2.3.2: - version "2.3.2" - resolved "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz" - integrity sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA== - ftp@^0.3.10: version "0.3.10" resolved "https://registry.npmjs.org/ftp/-/ftp-0.3.10.tgz" @@ -11332,11 +11332,6 @@ karma-mocha@2.0.1: dependencies: minimist "^1.2.3" -karma-safari-launcher@1.0.0: - version "1.0.0" - resolved "https://registry.npmjs.org/karma-safari-launcher/-/karma-safari-launcher-1.0.0.tgz" - integrity sha1-lpgqLMR9BmquccVTursoMZEVos4= - karma-sourcemap-loader@0.4.0: version "0.4.0" resolved "https://registry.npmjs.org/karma-sourcemap-loader/-/karma-sourcemap-loader-0.4.0.tgz" @@ -11407,6 +11402,14 @@ karma-typescript@5.5.4: util "^0.12.1" vm-browserify "^1.1.2" +karma-webkit-launcher@2.6.0: + version "2.6.0" + resolved "https://registry.npmjs.org/karma-webkit-launcher/-/karma-webkit-launcher-2.6.0.tgz#2e3ba096b69139e608d9ce6e89816a17f358b700" + integrity sha512-IDURopxJ1SbuqnvPaE+lP2qiP2Ie7I+ojwJRBpr0tfGwObsaVdjMkUkmZ1BcXUtYRt5ogs9cyCH2Wb9sNv0BbQ== + dependencies: + is-ci "^3.0.1" + uuid "^10.0.0" + karma-webpack@5.0.0: version "5.0.0" resolved "https://registry.npmjs.org/karma-webpack/-/karma-webpack-5.0.0.tgz" @@ -14112,6 +14115,20 @@ pkg-dir@^4.1.0, pkg-dir@^4.2.0: dependencies: find-up "^4.0.0" +playwright-core@1.46.1: + version "1.46.1" + resolved "https://registry.npmjs.org/playwright-core/-/playwright-core-1.46.1.tgz#28f3ab35312135dda75b0c92a3e5c0e7edb9cc8b" + integrity sha512-h9LqIQaAv+CYvWzsZ+h3RsrqCStkBHlgo6/TJlFst3cOTlLghBQlJwPOZKQJTKNaD3QIB7aAVQ+gfWbN3NXB7A== + +playwright@1.46.1: + version "1.46.1" + resolved "https://registry.npmjs.org/playwright/-/playwright-1.46.1.tgz#ea562bc48373648e10420a10c16842f0b227c218" + integrity sha512-oPcr1yqoXLCkgKtD5eNUPLiN40rYEM39odNpIb6VE6S7/15gJmA1NzVv6zJYusV0e7tzvkU/utBFNa/Kpxmwng== + dependencies: + playwright-core "1.46.1" + optionalDependencies: + fsevents "2.3.2" + plugin-error@^1.0.1: version "1.0.1" resolved "https://registry.npmjs.org/plugin-error/-/plugin-error-1.0.1.tgz" @@ -17650,6 +17667,11 @@ utils-merge@1.0.1: resolved "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz" integrity sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM= +uuid@^10.0.0: + version "10.0.0" + resolved "https://registry.npmjs.org/uuid/-/uuid-10.0.0.tgz#5a95aa454e6e002725c79055fd42aaba30ca6294" + integrity sha512-8XkAphELsDnEGrDxUOHB3RGvXz6TeuYSGEZBOjtTtPm2lwhGBjLgOzLHB63IUWfBpNucQjND6d3AOudO+H3RWQ== + uuid@^3.3.2, uuid@^3.3.3: version "3.4.0" resolved "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz" From 4473d2bbbbb0bac551aecda5702ebad3e6bca33e Mon Sep 17 00:00:00 2001 From: Daniel La Rocque Date: Mon, 9 Sep 2024 14:01:20 -0400 Subject: [PATCH 2/3] Formatting --- config/karma.base.js | 1 - scripts/ci-test/testConfig.ts | 18 ++++++++++-------- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/config/karma.base.js b/config/karma.base.js index a2671ab02d0..49824296740 100644 --- a/config/karma.base.js +++ b/config/karma.base.js @@ -39,7 +39,6 @@ function determineBrowsers() { } else { return validBrowsers; } - } else { console.log( "The 'BROWSER' environment variable is undefined. Defaulting to 'ChromeHeadless'." diff --git a/scripts/ci-test/testConfig.ts b/scripts/ci-test/testConfig.ts index 1092442889d..0202c699154 100644 --- a/scripts/ci-test/testConfig.ts +++ b/scripts/ci-test/testConfig.ts @@ -30,14 +30,16 @@ export interface TestConfig { // These tests are flaky on WebkitHeadless for some reason, so skip them. // TODO (dlarocque): Fix the flakes and remove this -const ignoredWebkitCoreTests = process.env?.BROWSERS?.includes("WebkitHeadless") ? [ - '@firebase/app-check', - '@firebase/installations', - '@firebase/storage', - '@firebase/storage-compat', - '@firebase/database', - '@firebase/database-compat', -] : []; +const ignoredWebkitCoreTests = process.env?.BROWSERS?.includes('WebkitHeadless') + ? [ + '@firebase/app-check', + '@firebase/installations', + '@firebase/storage', + '@firebase/storage-compat', + '@firebase/database', + '@firebase/database-compat' + ] + : []; export const testConfig: { [key: string]: TestConfig | undefined; From 1f4e73d72ca51f863889cf9ba0ca3d8ce07fd054 Mon Sep 17 00:00:00 2001 From: Daniel La Rocque Date: Mon, 23 Sep 2024 12:16:33 -0400 Subject: [PATCH 3/3] Respond to review comments --- .github/workflows/test-all.yml | 5 ----- .github/workflows/test-changed-firestore.yml | 2 +- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/.github/workflows/test-all.yml b/.github/workflows/test-all.yml index eef9bfc4224..eda081d1df7 100644 --- a/.github/workflows/test-all.yml +++ b/.github/workflows/test-all.yml @@ -91,11 +91,6 @@ jobs: run: echo "FIREBASE_CI_TEST_START_TIME=$(date +%s)" >> $GITHUB_ENV - name: Run unit tests # Ignore auth and firestore since they're handled in their own separate jobs. - # TODO (dlarocque): Fix flakes in installations, database, database-compat, storage-compat, - # and remote-config, and then run them here. - # There are several tests suites that are flaky when ran alongside the other tests in - # WebkitHeadless. It's unclear why they're only failing when ran with other tests, so let's - # skip them for now. run: | xvfb-run yarn lerna run --ignore '{firebase-messaging-integration-test,@firebase/auth*,@firebase/firestore*,firebase-firestore-integration-test}' test:ci node scripts/print_test_logs.js diff --git a/.github/workflows/test-changed-firestore.yml b/.github/workflows/test-changed-firestore.yml index 9bd7cc33bee..4a924d14e25 100644 --- a/.github/workflows/test-changed-firestore.yml +++ b/.github/workflows/test-changed-firestore.yml @@ -16,7 +16,7 @@ name: Test Firestore on: workflow_dispatch: - push: + pull_request: env: artifactRetentionDays: 14