chore: notify slack of e2e failures (#370)

* chore: notify slack of e2e failures

* chore: fix config

* chore: fix results url

* chore: re-enable more tests

* chore: reenable more

* chore: don't notify Slack  on PRs

* chore: fix workflow logic

Author: ascorbic

.github/workflows/run-tests.yml

@@ -203,8 +203,18 @@ jobs:
         pattern: blob-report-${{ matrix.version }}-*
         merge-multiple: true
 
-    - name: Merge into HTML Report
-      run: npx playwright merge-reports --reporter html ./all-blob-reports 
+    - name: Merge reports
+      run: |
+        npx playwright merge-reports --reporter html ./all-blob-reports 
+        npx playwright merge-reports --reporter json ./all-blob-reports > merged_reports.json
+        
+    - name: Notify Slack
+      if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || github.head_ref == 'main'
+      run: npx playwright-slack-report -j merged_reports.json -c tests/playwright-slack-conf.json
+      env:
+        SLACK_BOT_USER_OAUTH_TOKEN: ${{ secrets.SLACK_BOT_USER_OAUTH_TOKEN }}
+        RESULTS_VERSION: ${{ matrix.version }}
+        RESULTS_URL: ${{github.server_url}}/${{github.repository}}/actions/runs/${{github.run_id}}
 
     - name: Upload HTML report
       uses: actions/upload-artifact@v4

.github/workflows/test-e2e.yml

@@ -252,7 +252,7 @@ jobs:
           payload: ${{ steps.publish-test-results.outputs.slackEvent }}
 
       - name: Publish test site
-        if: (success() || failure()) && matrix.version == 'latest' && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && github.head_ref == 'main'
+        if: (success() || failure()) && matrix.version == 'latest' && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') 
         run: |
           deno run -A tools/deno/junit2html.ts artifacts ${{needs.e2e.outputs.tag}} > report/index.html
           npx --yes netlify-cli@latest deploy --prod --cwd report

tests/netlify-deploy.ts

@@ -119,7 +119,8 @@ export class NextDeployInstance extends NextInstance {
       this._url = data.deploy_url
       this._parsedUrl = new URL(this._url)
       this._deployId = data.deploy_id
-      require('console').log(`Deployment URL: ${this._url}`)
+      require('console').log(`Deployment URL: ${data.deploy_url}`)
+      require('console').log(`Logs: ${data.logs}`)
     } catch (err) {
       console.error(err)
       throw new Error(`Failed to parse deploy output: ${deployRes.stdout}`)

tests/netlify-e2e.cjs

@@ -2,18 +2,7 @@ module.exports = {
   version: 2,
   suites: {
     'test/e2e/app-dir/app-static/app-static.test.ts': {
-      failed: [
-        'app-dir static/dynamic handling usePathname should have values from canonical url on rewrite',
-        'app-dir static/dynamic handling should have correct prerender-manifest entries',
-        'app-dir static/dynamic handling should output HTML/RSC files for static paths',
-        'app-dir static/dynamic handling should output debug info for static bailouts',
-      ],
-    },
-    'test/e2e/app-dir/app-client-cache/client-cache.test.ts': {
-      failed: [
-        'app dir client cache semantics prefetch={undefined} - default should re-use the full cache for only 30 seconds',
-        'app dir client cache semantics prefetch={undefined} - default should refetch below the fold after 30 seconds',
-      ],
+      failed: ['app-dir static/dynamic handling should warn for too many cache tags'],
     },
     'test/e2e/app-dir/headers-static-bailout/headers-static-bailout.test.ts': {
       failed: [
@@ -24,14 +13,10 @@ module.exports = {
     'test/e2e/app-dir/parallel-routes-and-interception/parallel-routes-and-interception.test.ts': {
       failed: [],
       flakey: [
+        // Uses patch file
         'parallel-routes-and-interception parallel routes should gracefully handle when two page segments match the `children` parallel slot',
       ],
     },
-    'test/e2e/app-dir/error-boundary-navigation/override-node-env.test.ts': {
-      failed: [
-        'app dir - not found navigation - with overridden node env should be able to navigate to other page from root not-found page',
-      ],
-    },
     'test/e2e/opentelemetry/opentelemetry.test.ts': {
       failed: [
         'opentelemetry root context app router should handle RSC with fetch',

tests/playwright-slack-conf.json

@@ -0,0 +1,14 @@
+{
+  "sendResults": "on-failure",
+  "slackLogLevel": "debug",
+  "sendUsingBot": {
+    "channels": ["C020GQEKC13"]
+  },
+  "meta": [
+    { "key": "version", "value": "__ENV_RESULTS_VERSION" },
+    {
+      "key": "results",
+      "value": "__ENV_RESULTS_URL"
+    }
+  ]
+}