diff --git a/.eslintignore b/.eslintignore
index feef70c86c..cd06d28d56 100644
--- a/.eslintignore
+++ b/.eslintignore
@@ -1,4 +1,5 @@
dist/
demo/
tests/
-edge-runtime
\ No newline at end of file
+edge-runtime
+tools/deno
\ No newline at end of file
diff --git a/.github/workflows/test-e2e.yml b/.github/workflows/test-e2e.yml
index ce1f60f80a..a06520505f 100644
--- a/.github/workflows/test-e2e.yml
+++ b/.github/workflows/test-e2e.yml
@@ -7,12 +7,11 @@ on:
types: [opened, synchronize, reopened, labeled]
env:
- NODE_VERSION: 18.16.1
- PNPM_VERSION: 8.7.1
+ NODE_VERSION: 18.17.1
+ PNPM_VERSION: 8.9.0
NEXT_REPO: netlify/next.js
- NEXT_VERSION: 13.5.1
NEXT_TEST_MODE: deploy
- NEXT_TEST_JOB: true
+ NEXT_JUNIT_TEST_REPORT: true
TEST_CONCURRENCY: 8
NEXT_E2E_TEST_TIMEOUT: 600000
NEXT_TELEMETRY_DISABLED: 1
@@ -22,29 +21,22 @@ env:
NETLIFY_AUTH_TOKEN: ${{ secrets.NETLIFY_AUTH_TOKEN }}
NETLIFY_SITE_ID: 1d5a5c76-d445-4ae5-b694-b0d3f2e2c395
NEXT_TEST_CONTINUE_ON_ERROR: 1
-
+ next-path: next.js
+ runtime-path: next-runtime-minimal
jobs:
e2e:
if:
${{ github.event_name == ('workflow_dispatch' || 'workflow_call') ||
contains(github.event.pull_request.labels.*.name, 'run-e2e-tests') }}
- name: test e2e
+ name: Test group ${{ matrix.group }}/20
runs-on: ubuntu-latest
timeout-minutes: 120
strategy:
fail-fast: false
matrix:
- group: [1, 2, 3, 4, 5, 6, 7, 8]
- env:
- next-path: next.js
- runtime-path: next-runtime-minimal
- skipped-tests: |
- test/e2e/app-dir/app-compilation/index.test.ts
- test/e2e/favicon-short-circuit/favicon-short-circuit.test.ts
+ group: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20]
steps:
- - name: 'Setup jq'
- uses: dcarbone/install-jq-action@v2.1.0
- name: get github token
uses: navikt/github-app-token-generator@v1
id: token
@@ -58,7 +50,6 @@ jobs:
with:
repository: ${{ env.NEXT_REPO }}
token: ${{ steps.token.outputs.token }}
- ref: v${{ env.NEXT_VERSION }}-netlify
path: ${{ env.next-path }}
- name: checkout runtime repo
@@ -131,73 +122,42 @@ jobs:
run: npx playwright install
working-directory: ${{ env.next-path }}
- - name: set test filter
- run: |
- ls test/e2e/**/*.test.* > included_tests
- printf %s "${{ env.skipped-tests }}" > skipped_tests
- comm -23 included_tests skipped_tests > filtered_tests
- echo "{\"enabledTests\":[\"$(printf %s "$(< filtered_tests)" | sed -z "s/\n/\",\"/g")\"]}" > tests.json
- working-directory: ${{ env.next-path }}
-
- name: run tests
env:
NODE_ENV: production
- NEXT_EXTERNAL_TESTS_FILTERS: ./tests.json
- run: node run-tests.js -g ${{ matrix.group }}/8 -c ${TEST_CONCURRENCY} --type e2e
+ NEXT_EXTERNAL_TESTS_FILTERS: ../next-runtime-minimal/tests/netlify-e2e.json
+ run: node run-tests.js -g ${{ matrix.group }}/20 -c ${TEST_CONCURRENCY} --type e2e
working-directory: ${{ env.next-path }}
- - name: Test results
- working-directory: ${{ env.next-path }}
+ - name: Upload Test Results
+ if: success() || failure()
+ uses: actions/upload-artifact@v3
+ with:
+ name: Test Results (${{ matrix.group }})
+ path: ${{ env.next-path }}/test/test-junit-report/*.xml
+ publish-test-results:
+ name: 'E2E Test Summary'
+ needs: e2e
+ runs-on: ubuntu-latest
+ permissions:
+ checks: write
+ contents: read
+ issues: read
+ if: success() || failure()
+
+ steps:
+ - name: checkout runtime repo
+ uses: actions/checkout@v4
+
+ - name: Install Deno
+ uses: denoland/setup-deno@v1
+
+ - name: Download Artifacts
+ uses: actions/download-artifact@v3
+ with:
+ path: artifacts
+
+ - name: Publish Test Report
+ if: success() || failure()
run: |
- for file in `ls test/e2e/**/*.results.json`; do
- test_name=$(echo $file | sed -e 's/test\/e2e\/\(.*\/.*\).results.json/\1/')
-
- success=$(jq -r '.success' $file)
- if [ "$success" = "true" ] ; then
- test_result=":white_check_mark:"
- else
- test_result=":x:"
- fi
-
- passed=$(jq -r '.numPassedTests' $file)
- passed_total=$((passed_total + passed))
- failed=$(jq -r '.numFailedTests' $file)
- failed_total=$((failed_total + failed))
- pending=$(jq -r '.numPendingTests' $file)
- pending_total=$((pending_total + pending))
- total=$(jq -r '.numTotalTests' $file)
- total_total=$((total_total + total))
-
- message=$(jq -r '.testResults[].message' $file)
-
- echo "" >> test_summary
- echo "${test_result} ${test_name}
" >> test_summary
- echo "Results
" >> test_summary
- echo "Passing tests: ${passed}
" >> test_summary
- echo "Failing tests: ${failed}
" >> test_summary
- echo "Pending tests: ${pending}
" >> test_summary
- echo "Total tests: ${total}
" >> test_summary
-
- if [ -n "${message}" ] ; then
- echo "Details
" >> test_summary
- echo "" >> test_summary
- echo $message >> test_summary
- echo "
" >> test_summary
- fi
-
- echo " " >> test_summary
- done
-
- echo "Results
" >> $GITHUB_STEP_SUMMARY
- echo "Passing tests: ${passed_total}
" >> $GITHUB_STEP_SUMMARY
- echo "Failing tests: ${failed_total}
" >> $GITHUB_STEP_SUMMARY
- echo "Pending tests: ${pending_total}
" >> $GITHUB_STEP_SUMMARY
- echo "Total tests: ${total_total}
" >> $GITHUB_STEP_SUMMARY
-
- echo "Details
" >> $GITHUB_STEP_SUMMARY
- cat test_summary >> $GITHUB_STEP_SUMMARY
-
- if [ "$failed_total" -gt 0 ] ; then
- echo "Failed tests. Please see action summary for details."
- exit 1
- fi
+ deno run -A tools/deno/junit2md.ts artifacts >> $GITHUB_STEP_SUMMARY
diff --git a/run-local-test.sh b/run-local-test.sh
new file mode 100755
index 0000000000..6dd80caae3
--- /dev/null
+++ b/run-local-test.sh
@@ -0,0 +1,17 @@
+#!/bin/sh
+set -e
+# Print usage if nothing is passed
+if [ -z "$1" ]; then
+ echo "Usage: $0 "
+ exit 1
+fi
+# Check that the next.js directory exists
+if [ ! -d "../next.js" ]; then
+ echo "Error: next.js repo needs to be in ../next.js"
+ exit 1
+fi
+export NEXT_TEST_CONTINUE_ON_ERROR=1
+export NETLIFY_SITE_ID=1d5a5c76-d445-4ae5-b694-b0d3f2e2c395
+export NEXT_TEST_MODE=deploy
+cd ../next.js/
+node run-tests.js --type e2e --debug --test-pattern $1
diff --git a/tests/netlify-e2e.json b/tests/netlify-e2e.json
new file mode 100644
index 0000000000..029c359693
--- /dev/null
+++ b/tests/netlify-e2e.json
@@ -0,0 +1,138 @@
+{
+ "version": 2,
+ "suites": {
+ "test/e2e/app-dir/app-static/app-static.test.ts": {
+ "failed": [
+ "app-dir static/dynamic handling usePathname should have values from canonical url on rewrite",
+ "app-dir static/dynamic handling should have correct prerender-manifest entries",
+ "app-dir static/dynamic handling should output HTML/RSC files for static paths",
+ "app-dir static/dynamic handling should output debug info for static bailouts"
+ ]
+ },
+ "test/e2e/app-dir/app-client-cache/client-cache.test.ts": {
+ "failed": [
+ "app dir client cache semantics prefetch={undefined} - default should re-use the full cache for only 30 seconds",
+ "app dir client cache semantics prefetch={undefined} - default should refetch below the fold after 30 seconds"
+ ]
+ },
+ "test/e2e/app-dir/headers-static-bailout/headers-static-bailout.test.ts": {
+ "failed": [
+ "headers-static-bailout it provides a helpful link in case static generation bailout is uncaught"
+ ]
+ },
+ "test/e2e/app-dir/parallel-routes-and-interception/parallel-routes-and-interception.test.ts": {
+ "failed": [
+ "parallel-routes-and-interception route intercepting should render modal when paired with parallel routes",
+ "parallel-routes-and-interception route intercepting should support intercepting local dynamic sibling routes"
+ ]
+ },
+ "test/e2e/app-dir/error-boundary-navigation/override-node-env.test.ts": {
+ "failed": [
+ "app dir - not found navigation - with overridden node env should be able to navigate to other page from root not-found page"
+ ]
+ },
+ "test/e2e/opentelemetry/opentelemetry.test.ts": {
+ "failed": [
+ "opentelemetry root context app router should handle RSC with fetch",
+ "opentelemetry incoming context propagation app router should handle RSC with fetch",
+ "opentelemetry incoming context propagation app router should handle route handlers in app router"
+ ]
+ },
+ "test/e2e/app-dir/rsc-basic/rsc-basic.test.ts": {
+ "failed": [
+ "app dir - rsc basics should render initial styles of css-in-js in edge SSR correctly",
+ "app dir - rsc basics should render initial styles of css-in-js in nodejs SSR correctly",
+ "app dir - rsc basics should render server components correctly"
+ ],
+ "flakey": [
+ "app dir - rsc basics react@experimental should opt into the react@experimental when enabling ppr",
+ "app dir - rsc basics react@experimental should opt into the react@experimental when enabling taint"
+ ]
+ },
+ "test/e2e/app-dir/navigation/navigation.test.ts": {
+ "failed": [
+ "app dir - navigation redirect status code should respond with 308 status code if permanent flag is set",
+ "app dir - navigation redirect status code should respond with 307 status code in client component",
+ "app dir - navigation redirect status code should respond with 307 status code in server component",
+ "app dir - navigation bots should block rendering for bots and return 404 status",
+ "app dir - navigation navigation between pages and app should not continously initiate a mpa navigation to the same URL when router state changes"
+ ]
+ },
+ "test/e2e/app-dir/app-static/app-static-custom-handler.test.ts": {
+ "failed": [
+ "app-dir static/dynamic handling should output debug info for static bailouts",
+ "app-dir static/dynamic handling should have correct prerender-manifest entries",
+ "app-dir static/dynamic handling should output HTML/RSC files for static paths"
+ ]
+ },
+ "test/production/app-dir/unexpected-error/unexpected-error.test.ts": {
+ "failed": [
+ "unexpected-error should set response status to 500 for unexpected errors in ssr app route",
+ "unexpected-error should set response status to 500 for unexpected errors in isr app route"
+ ]
+ },
+ "test/e2e/skip-trailing-slash-redirect/index.test.ts": {
+ "flakey": [
+ "skip-trailing-slash-redirect should merge cookies from middleware and edge API routes correctly",
+ "skip-trailing-slash-redirect should handle external rewrite correctly /chained-rewrite-ssr",
+ "skip-trailing-slash-redirect should handle external rewrite correctly /chained-rewrite-static",
+ "skip-trailing-slash-redirect should handle external rewrite correctly /chained-rewrite-ssg"
+ ]
+ },
+ "test/e2e/module-layer/index.test.ts": {
+ "flakey": [
+ "module layer no server-only in server targets should render routes marked with restriction marks without errors",
+ "module layer with server-only in server targets should render routes marked with restriction marks without errors"
+ ]
+ },
+ "test/e2e/getserversideprops/test/index.test.ts": {
+ "flakey": [
+ "getServerSideProps should set default caching header",
+ "getServerSideProps should respect custom caching header"
+ ]
+ },
+ "test/e2e/app-dir/metadata-dynamic-routes/index.test.ts": {
+ "pending": [],
+ "flakey": [
+ "app dir - metadata dynamic routes text routes should handle robots.[ext] dynamic routes",
+ "app dir - metadata dynamic routes text routes should handle sitemap.[ext] dynamic routes",
+ "app dir - metadata dynamic routes social image routes should handle manifest.[ext] dynamic routes",
+ "app dir - metadata dynamic routes social image routes should render og image with opengraph-image dynamic routes",
+ "app dir - metadata dynamic routes social image routes should render og image with twitter-image dynamic routes",
+ "app dir - metadata dynamic routes icon image routes should render icon with dynamic routes",
+ "app dir - metadata dynamic routes icon image routes should render apple icon with dynamic routes",
+ "app dir - metadata dynamic routes should inject dynamic metadata properly to head"
+ ]
+ },
+ "test/e2e/app-dir/metadata/metadata.test.ts": {
+ "flakey": [
+ "app dir - metadata opengraph should pick up opengraph-image and twitter-image as static metadata files",
+ "app dir - metadata static routes should have /favicon.ico as route"
+ ]
+ }
+ },
+ "rules": {
+ "include": ["test/e2e/**/*.test.{t,j}s{,x}"],
+ "exclude": [
+ "test/e2e/app-dir/next-font/**/*",
+ "test/e2e/app-dir/ppr/**/*",
+ "test/e2e/app-dir/ppr-*/**/*",
+ "test/e2e/app-dir/app-prefetch*/**/*",
+ "test/e2e/app-dir/app-esm-js/index.test.ts",
+ "test/e2e/app-dir/interception-middleware-rewrite/interception-middleware-rewrite.test.ts",
+ "test/e2e/app-dir/searchparams-static-bailout/searchparams-static-bailout.test.ts",
+ "test/e2e/app-dir/app-compilation/index.test.ts",
+ "test/e2e/cancel-request/stream-cancel.test.ts",
+ "test/e2e/favicon-short-circuit/favicon-short-circuit.test.ts",
+ "test/e2e/edge-pages-support/edge-document.test.ts",
+ "test/e2e/third-parties/index.test.ts",
+ "test/e2e/swc-warnings/index.test.ts",
+ "test/e2e/app-dir/externals/externals.test.ts",
+ "test/e2e/app-dir/use-selected-layout-segment-s/use-selected-layout-segment-s.test.ts",
+ "test/e2e/repeated-forward-slashes-error/repeated-forward-slashes-error.test.ts",
+ "test/e2e/app-dir/with-exported-function-config/with-exported-function-config.test.ts",
+ "test/e2e/app-dir/x-forwarded-headers/x-forwarded-headers.test.ts",
+ "test/e2e/app-dir/third-parties/basic.test.ts"
+ ]
+ }
+}
diff --git a/tools/deno/junit2md.ts b/tools/deno/junit2md.ts
new file mode 100644
index 0000000000..7425c83f29
--- /dev/null
+++ b/tools/deno/junit2md.ts
@@ -0,0 +1,156 @@
+import { expandGlob } from 'https://deno.land/std/fs/mod.ts'
+import { parse } from 'https://deno.land/x/xml/mod.ts'
+
+interface TestCase {
+ '@classname': string
+ '@name': string
+ '@time': number
+ '@file': string
+ failure?: string
+}
+
+interface TestSuite {
+ '@name': string
+ '@errors': number
+ '@failures': number
+ '@skipped': number
+ '@timestamp': string
+ '@time': number
+ '@tests': number
+ testcase: TestCase[]
+}
+
+interface TestSuites {
+ '@name': string
+ '@tests': number
+ '@failures': number
+ '@errors': number
+ '@time': number
+ testsuite: TestSuite[]
+}
+
+async function parseXMLFile(filePath: string): Promise<{ testsuites: TestSuites }> {
+ const xmlContent = await Deno.readTextFile(filePath)
+ return parse(xmlContent) as unknown as { testsuites: TestSuites }
+}
+
+const suites: Array<{
+ name: string
+ tests: number
+ failures: number
+ skipped: number
+ time: number
+}> = []
+
+const testCount = {
+ '❌': 0,
+ '⏭️': 0,
+ '✅': 0,
+}
+
+function junitToMarkdown(xmlData: { testsuites: TestSuites }) {
+ if (!xmlData.testsuites) {
+ return ''
+ }
+ let markdown = ``
+
+ const testSuites = Array.isArray(xmlData.testsuites.testsuite)
+ ? xmlData.testsuites.testsuite
+ : [xmlData.testsuites.testsuite]
+
+ for (const suite of testSuites) {
+ const {
+ '@tests': tests,
+ '@failures': failures,
+ '@skipped': skipped,
+ '@name': name,
+ '@time': time,
+ } = suite
+
+ suites.push({
+ name,
+ tests,
+ failures,
+ skipped,
+ time,
+ })
+
+ const passed = tests - failures - skipped
+
+ const testCases = Array.isArray(suite.testcase) ? suite.testcase : [suite.testcase]
+
+ const testCasesDetails = testCases
+ .map((testCase) => {
+ const status = testCase.failure ? '❌' : 'skipped' in testCase ? '⏭️' : '✅'
+
+ testCount[status]++
+ return `${status} ${testCase['@name'].slice(name.length)} (${testCase['@time'].toFixed(
+ 2,
+ )}s)`
+ })
+ .join('')
+
+ markdown += `| ${name}