ci: use new e2e tests manifest (#131)

* ci: run tests on canary and skip none

* DIsable one so script works

* Use manifest v2

* Remove old filter step

* Fix versions

* Exclude invalid tests

* Exclude more

* Use junit

* Set cwd for test report

* Try massive concurrency

* Fix concurrency

* Use different test reporter

* Adjust condiitons

* Typo

* Fix junit path

* Disable more

* fix: shim process in edge runtime

* Switch reporter action

* ci: fix artifact path

* Disable very slow tests

* Add test runner

* Use md formatter

* Add deno install step to task

* Fix tool path

* Disable test

* Skip bad tests

* Omit non-deploy test

* Skip tests that are failing because of whitespace in headers

* Continute on error in local tests

* Skip a non-deploy test

* Skip xff test

* Skip tests that try to patch files

* Checkout runtime in report step

* DIsable more flaky tests

* Fix script name

* Log cast totals

* Format better

.eslintignore

@@ -1,4 +1,5 @@
 dist/
 demo/
 tests/
-edge-runtime
+edge-runtime
+tools/deno

.github/workflows/test-e2e.yml

@@ -7,12 +7,11 @@ on:
     types: [opened, synchronize, reopened, labeled]
 
 env:
-  NODE_VERSION: 18.16.1
-  PNPM_VERSION: 8.7.1
+  NODE_VERSION: 18.17.1
+  PNPM_VERSION: 8.9.0
   NEXT_REPO: netlify/next.js
-  NEXT_VERSION: 13.5.1
   NEXT_TEST_MODE: deploy
-  NEXT_TEST_JOB: true
+  NEXT_JUNIT_TEST_REPORT: true
   TEST_CONCURRENCY: 8
   NEXT_E2E_TEST_TIMEOUT: 600000
   NEXT_TELEMETRY_DISABLED: 1
@@ -22,29 +21,22 @@ env:
   NETLIFY_AUTH_TOKEN: ${{ secrets.NETLIFY_AUTH_TOKEN }}
   NETLIFY_SITE_ID: 1d5a5c76-d445-4ae5-b694-b0d3f2e2c395
   NEXT_TEST_CONTINUE_ON_ERROR: 1
-
+  next-path: next.js
+  runtime-path: next-runtime-minimal
 jobs:
   e2e:
     if:
       ${{ github.event_name == ('workflow_dispatch' || 'workflow_call') ||
       contains(github.event.pull_request.labels.*.name, 'run-e2e-tests') }}
-    name: test e2e
+    name: Test group ${{ matrix.group }}/20
     runs-on: ubuntu-latest
     timeout-minutes: 120
     strategy:
       fail-fast: false
       matrix:
-        group: [1, 2, 3, 4, 5, 6, 7, 8]
-    env:
-      next-path: next.js
-      runtime-path: next-runtime-minimal
-      skipped-tests: |
-        test/e2e/app-dir/app-compilation/index.test.ts
-        test/e2e/favicon-short-circuit/favicon-short-circuit.test.ts
+        group: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20]
 
     steps:
-      - name: 'Setup jq'
-        uses: dcarbone/[email protected]
       - name: get github token
         uses: navikt/github-app-token-generator@v1
         id: token
@@ -58,7 +50,6 @@ jobs:
         with:
           repository: ${{ env.NEXT_REPO }}
           token: ${{ steps.token.outputs.token }}
-          ref: v${{ env.NEXT_VERSION }}-netlify
           path: ${{ env.next-path }}
 
       - name: checkout runtime repo
@@ -131,73 +122,42 @@ jobs:
         run: npx playwright install
         working-directory: ${{ env.next-path }}
 
-      - name: set test filter
-        run: |
-          ls test/e2e/**/*.test.* > included_tests
-          printf %s "${{ env.skipped-tests }}" > skipped_tests
-          comm -23 included_tests skipped_tests > filtered_tests
-          echo "{\"enabledTests\":[\"$(printf %s "$(< filtered_tests)" | sed -z "s/\n/\",\"/g")\"]}" > tests.json
-        working-directory: ${{ env.next-path }}
-
       - name: run tests
         env:
           NODE_ENV: production
-          NEXT_EXTERNAL_TESTS_FILTERS: ./tests.json
-        run: node run-tests.js -g ${{ matrix.group }}/8 -c ${TEST_CONCURRENCY} --type e2e
+          NEXT_EXTERNAL_TESTS_FILTERS: ../next-runtime-minimal/tests/netlify-e2e.json
+        run: node run-tests.js -g ${{ matrix.group }}/20 -c ${TEST_CONCURRENCY} --type e2e
         working-directory: ${{ env.next-path }}
 
-      - name: Test results
-        working-directory: ${{ env.next-path }}
+      - name: Upload Test Results
+        if: success() || failure()
+        uses: actions/upload-artifact@v3
+        with:
+          name: Test Results (${{ matrix.group }})
+          path: ${{ env.next-path }}/test/test-junit-report/*.xml
+  publish-test-results:
+    name: 'E2E Test Summary'
+    needs: e2e
+    runs-on: ubuntu-latest
+    permissions:
+      checks: write
+      contents: read
+      issues: read
+    if: success() || failure()
+
+    steps:
+      - name: checkout runtime repo
+        uses: actions/checkout@v4
+
+      - name: Install Deno
+        uses: denoland/setup-deno@v1
+
+      - name: Download Artifacts
+        uses: actions/download-artifact@v3
+        with:
+          path: artifacts
+
+      - name: Publish Test Report
+        if: success() || failure()
         run: |
-          for file in `ls test/e2e/**/*.results.json`; do
-            test_name=$(echo $file | sed -e 's/test\/e2e\/\(.*\/.*\).results.json/\1/')
-
-            success=$(jq -r '.success' $file)
-            if [ "$success" = "true" ] ; then
-              test_result=":white_check_mark:"
-            else
-              test_result=":x:"
-            fi
-
-            passed=$(jq -r '.numPassedTests' $file)
-            passed_total=$((passed_total + passed))
-            failed=$(jq -r '.numFailedTests' $file)
-            failed_total=$((failed_total + failed))
-            pending=$(jq -r '.numPendingTests' $file)
-            pending_total=$((pending_total + pending))
-            total=$(jq -r '.numTotalTests' $file)
-            total_total=$((total_total + total))
-
-            message=$(jq -r '.testResults[].message' $file)
-                        
-            echo "<details>" >> test_summary
-            echo "<summary>${test_result} <code>${test_name}</code></summary>" >> test_summary
-            echo "<h4>Results</h4>" >> test_summary
-            echo "<div>Passing tests: ${passed}</div>" >> test_summary
-            echo "<div>Failing tests: ${failed}</div>" >> test_summary
-            echo "<div>Pending tests: ${pending}</div>" >> test_summary
-            echo "<div>Total tests: ${total}</div>" >> test_summary
-
-            if [ -n "${message}" ] ; then
-              echo "<h4>Details</h4>" >> test_summary
-              echo "<pre><code>" >> test_summary
-              echo $message >> test_summary
-              echo "</code></pre>" >> test_summary
-            fi
-
-            echo "</details>" >> test_summary
-          done
-
-          echo "<h3>Results</h3>" >> $GITHUB_STEP_SUMMARY
-          echo "<div>Passing tests: ${passed_total}</div>" >> $GITHUB_STEP_SUMMARY
-          echo "<div>Failing tests: ${failed_total}</div>" >> $GITHUB_STEP_SUMMARY
-          echo "<div>Pending tests: ${pending_total}</div>" >> $GITHUB_STEP_SUMMARY
-          echo "<div>Total tests: ${total_total}</div>" >> $GITHUB_STEP_SUMMARY
-
-          echo "<h3>Details</h3>" >> $GITHUB_STEP_SUMMARY
-          cat test_summary >> $GITHUB_STEP_SUMMARY
-
-          if [ "$failed_total" -gt 0 ] ; then
-            echo "Failed tests. Please see action summary for details."
-            exit 1
-          fi
+          deno run -A tools/deno/junit2md.ts artifacts >> $GITHUB_STEP_SUMMARY

run-local-test.sh

@@ -0,0 +1,17 @@
+#!/bin/sh
+set -e 
+# Print usage if nothing is passed
+if [ -z "$1" ]; then
+  echo "Usage: $0 <test-pattern>"
+  exit 1
+fi
+# Check that the next.js directory exists
+if [ ! -d "../next.js" ]; then
+  echo "Error: next.js repo needs to be in ../next.js"
+  exit 1
+fi
+export NEXT_TEST_CONTINUE_ON_ERROR=1
+export NETLIFY_SITE_ID=1d5a5c76-d445-4ae5-b694-b0d3f2e2c395
+export NEXT_TEST_MODE=deploy
+cd ../next.js/
+node run-tests.js --type e2e --debug --test-pattern $1

tests/netlify-e2e.json

@@ -0,0 +1,138 @@
+{
+  "version": 2,
+  "suites": {
+    "test/e2e/app-dir/app-static/app-static.test.ts": {
+      "failed": [
+        "app-dir static/dynamic handling usePathname should have values from canonical url on rewrite",
+        "app-dir static/dynamic handling should have correct prerender-manifest entries",
+        "app-dir static/dynamic handling should output HTML/RSC files for static paths",
+        "app-dir static/dynamic handling should output debug info for static bailouts"
+      ]
+    },
+    "test/e2e/app-dir/app-client-cache/client-cache.test.ts": {
+      "failed": [
+        "app dir client cache semantics prefetch={undefined} - default should re-use the full cache for only 30 seconds",
+        "app dir client cache semantics prefetch={undefined} - default should refetch below the fold after 30 seconds"
+      ]
+    },
+    "test/e2e/app-dir/headers-static-bailout/headers-static-bailout.test.ts": {
+      "failed": [
+        "headers-static-bailout it provides a helpful link in case static generation bailout is uncaught"
+      ]
+    },
+    "test/e2e/app-dir/parallel-routes-and-interception/parallel-routes-and-interception.test.ts": {
+      "failed": [
+        "parallel-routes-and-interception route intercepting should render modal when paired with parallel routes",
+        "parallel-routes-and-interception route intercepting should support intercepting local dynamic sibling routes"
+      ]
+    },
+    "test/e2e/app-dir/error-boundary-navigation/override-node-env.test.ts": {
+      "failed": [
+        "app dir - not found navigation - with overridden node env should be able to navigate to other page from root not-found page"
+      ]
+    },
+    "test/e2e/opentelemetry/opentelemetry.test.ts": {
+      "failed": [
+        "opentelemetry root context app router should handle RSC with fetch",
+        "opentelemetry incoming context propagation app router should handle RSC with fetch",
+        "opentelemetry incoming context propagation app router should handle route handlers in app router"
+      ]
+    },
+    "test/e2e/app-dir/rsc-basic/rsc-basic.test.ts": {
+      "failed": [
+        "app dir - rsc basics should render initial styles of css-in-js in edge SSR correctly",
+        "app dir - rsc basics should render initial styles of css-in-js in nodejs SSR correctly",
+        "app dir - rsc basics should render server components correctly"
+      ],
+      "flakey": [
+        "app dir - rsc basics react@experimental should opt into the react@experimental when enabling ppr",
+        "app dir - rsc basics react@experimental should opt into the react@experimental when enabling taint"
+      ]
+    },
+    "test/e2e/app-dir/navigation/navigation.test.ts": {
+      "failed": [
+        "app dir - navigation redirect status code should respond with 308 status code if permanent flag is set",
+        "app dir - navigation redirect status code should respond with 307 status code in client component",
+        "app dir - navigation redirect status code should respond with 307 status code in server component",
+        "app dir - navigation bots should block rendering for bots and return 404 status",
+        "app dir - navigation navigation between pages and app should not continously initiate a mpa navigation to the same URL when router state changes"
+      ]
+    },
+    "test/e2e/app-dir/app-static/app-static-custom-handler.test.ts": {
+      "failed": [
+        "app-dir static/dynamic handling should output debug info for static bailouts",
+        "app-dir static/dynamic handling should have correct prerender-manifest entries",
+        "app-dir static/dynamic handling should output HTML/RSC files for static paths"
+      ]
+    },
+    "test/production/app-dir/unexpected-error/unexpected-error.test.ts": {
+      "failed": [
+        "unexpected-error should set response status to 500 for unexpected errors in ssr app route",
+        "unexpected-error should set response status to 500 for unexpected errors in isr app route"
+      ]
+    },
+    "test/e2e/skip-trailing-slash-redirect/index.test.ts": {
+      "flakey": [
+        "skip-trailing-slash-redirect should merge cookies from middleware and edge API routes correctly",
+        "skip-trailing-slash-redirect should handle external rewrite correctly /chained-rewrite-ssr",
+        "skip-trailing-slash-redirect should handle external rewrite correctly /chained-rewrite-static",
+        "skip-trailing-slash-redirect should handle external rewrite correctly /chained-rewrite-ssg"
+      ]
+    },
+    "test/e2e/module-layer/index.test.ts": {
+      "flakey": [
+        "module layer no server-only in server targets should render routes marked with restriction marks without errors",
+        "module layer with server-only in server targets should render routes marked with restriction marks without errors"
+      ]
+    },
+    "test/e2e/getserversideprops/test/index.test.ts": {
+      "flakey": [
+        "getServerSideProps should set default caching header",
+        "getServerSideProps should respect custom caching header"
+      ]
+    },
+    "test/e2e/app-dir/metadata-dynamic-routes/index.test.ts": {
+      "pending": [],
+      "flakey": [
+        "app dir - metadata dynamic routes text routes should handle robots.[ext] dynamic routes",
+        "app dir - metadata dynamic routes text routes should handle sitemap.[ext] dynamic routes",
+        "app dir - metadata dynamic routes social image routes should handle manifest.[ext] dynamic routes",
+        "app dir - metadata dynamic routes social image routes should render og image with opengraph-image dynamic routes",
+        "app dir - metadata dynamic routes social image routes should render og image with twitter-image dynamic routes",
+        "app dir - metadata dynamic routes icon image routes should render icon with dynamic routes",
+        "app dir - metadata dynamic routes icon image routes should render apple icon with dynamic routes",
+        "app dir - metadata dynamic routes should inject dynamic metadata properly to head"
+      ]
+    },
+    "test/e2e/app-dir/metadata/metadata.test.ts": {
+      "flakey": [
+        "app dir - metadata opengraph should pick up opengraph-image and twitter-image as static metadata files",
+        "app dir - metadata static routes should have /favicon.ico as route"
+      ]
+    }
+  },
+  "rules": {
+    "include": ["test/e2e/**/*.test.{t,j}s{,x}"],
+    "exclude": [
+      "test/e2e/app-dir/next-font/**/*",
+      "test/e2e/app-dir/ppr/**/*",
+      "test/e2e/app-dir/ppr-*/**/*",
+      "test/e2e/app-dir/app-prefetch*/**/*",
+      "test/e2e/app-dir/app-esm-js/index.test.ts",
+      "test/e2e/app-dir/interception-middleware-rewrite/interception-middleware-rewrite.test.ts",
+      "test/e2e/app-dir/searchparams-static-bailout/searchparams-static-bailout.test.ts",
+      "test/e2e/app-dir/app-compilation/index.test.ts",
+      "test/e2e/cancel-request/stream-cancel.test.ts",
+      "test/e2e/favicon-short-circuit/favicon-short-circuit.test.ts",
+      "test/e2e/edge-pages-support/edge-document.test.ts",
+      "test/e2e/third-parties/index.test.ts",
+      "test/e2e/swc-warnings/index.test.ts",
+      "test/e2e/app-dir/externals/externals.test.ts",
+      "test/e2e/app-dir/use-selected-layout-segment-s/use-selected-layout-segment-s.test.ts",
+      "test/e2e/repeated-forward-slashes-error/repeated-forward-slashes-error.test.ts",
+      "test/e2e/app-dir/with-exported-function-config/with-exported-function-config.test.ts",
+      "test/e2e/app-dir/x-forwarded-headers/x-forwarded-headers.test.ts",
+      "test/e2e/app-dir/third-parties/basic.test.ts"
+    ]
+  }
+}