From ed7eca45bae6442c32eaf143f7fb0fcbd56ade51 Mon Sep 17 00:00:00 2001 From: Matt Kane Date: Wed, 10 Jan 2024 09:24:02 +0000 Subject: [PATCH] ci: use new e2e tests manifest (#131) * ci: run tests on canary and skip none * DIsable one so script works * Use manifest v2 * Remove old filter step * Fix versions * Exclude invalid tests * Exclude more * Use junit * Set cwd for test report * Try massive concurrency * Fix concurrency * Use different test reporter * Adjust condiitons * Typo * Fix junit path * Disable more * fix: shim process in edge runtime * Switch reporter action * ci: fix artifact path * Disable very slow tests * Add test runner * Use md formatter * Add deno install step to task * Fix tool path * Disable test * Skip bad tests * Omit non-deploy test * Skip tests that are failing because of whitespace in headers * Continute on error in local tests * Skip a non-deploy test * Skip xff test * Skip tests that try to patch files * Checkout runtime in report step * DIsable more flaky tests * Fix script name * Log cast totals * Format better --- .eslintignore | 3 +- .github/workflows/test-e2e.yml | 120 +++++++++---------------- run-local-test.sh | 17 ++++ tests/netlify-e2e.json | 138 +++++++++++++++++++++++++++++ tools/deno/junit2md.ts | 156 +++++++++++++++++++++++++++++++++ 5 files changed, 353 insertions(+), 81 deletions(-) create mode 100755 run-local-test.sh create mode 100644 tests/netlify-e2e.json create mode 100644 tools/deno/junit2md.ts diff --git a/.eslintignore b/.eslintignore index feef70c86c..cd06d28d56 100644 --- a/.eslintignore +++ b/.eslintignore @@ -1,4 +1,5 @@ dist/ demo/ tests/ -edge-runtime \ No newline at end of file +edge-runtime +tools/deno \ No newline at end of file diff --git a/.github/workflows/test-e2e.yml b/.github/workflows/test-e2e.yml index ce1f60f80a..a06520505f 100644 --- a/.github/workflows/test-e2e.yml +++ b/.github/workflows/test-e2e.yml @@ -7,12 +7,11 @@ on: types: [opened, synchronize, reopened, labeled] env: - NODE_VERSION: 18.16.1 - PNPM_VERSION: 8.7.1 + NODE_VERSION: 18.17.1 + PNPM_VERSION: 8.9.0 NEXT_REPO: netlify/next.js - NEXT_VERSION: 13.5.1 NEXT_TEST_MODE: deploy - NEXT_TEST_JOB: true + NEXT_JUNIT_TEST_REPORT: true TEST_CONCURRENCY: 8 NEXT_E2E_TEST_TIMEOUT: 600000 NEXT_TELEMETRY_DISABLED: 1 @@ -22,29 +21,22 @@ env: NETLIFY_AUTH_TOKEN: ${{ secrets.NETLIFY_AUTH_TOKEN }} NETLIFY_SITE_ID: 1d5a5c76-d445-4ae5-b694-b0d3f2e2c395 NEXT_TEST_CONTINUE_ON_ERROR: 1 - + next-path: next.js + runtime-path: next-runtime-minimal jobs: e2e: if: ${{ github.event_name == ('workflow_dispatch' || 'workflow_call') || contains(github.event.pull_request.labels.*.name, 'run-e2e-tests') }} - name: test e2e + name: Test group ${{ matrix.group }}/20 runs-on: ubuntu-latest timeout-minutes: 120 strategy: fail-fast: false matrix: - group: [1, 2, 3, 4, 5, 6, 7, 8] - env: - next-path: next.js - runtime-path: next-runtime-minimal - skipped-tests: | - test/e2e/app-dir/app-compilation/index.test.ts - test/e2e/favicon-short-circuit/favicon-short-circuit.test.ts + group: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20] steps: - - name: 'Setup jq' - uses: dcarbone/install-jq-action@v2.1.0 - name: get github token uses: navikt/github-app-token-generator@v1 id: token @@ -58,7 +50,6 @@ jobs: with: repository: ${{ env.NEXT_REPO }} token: ${{ steps.token.outputs.token }} - ref: v${{ env.NEXT_VERSION }}-netlify path: ${{ env.next-path }} - name: checkout runtime repo @@ -131,73 +122,42 @@ jobs: run: npx playwright install working-directory: ${{ env.next-path }} - - name: set test filter - run: | - ls test/e2e/**/*.test.* > included_tests - printf %s "${{ env.skipped-tests }}" > skipped_tests - comm -23 included_tests skipped_tests > filtered_tests - echo "{\"enabledTests\":[\"$(printf %s "$(< filtered_tests)" | sed -z "s/\n/\",\"/g")\"]}" > tests.json - working-directory: ${{ env.next-path }} - - name: run tests env: NODE_ENV: production - NEXT_EXTERNAL_TESTS_FILTERS: ./tests.json - run: node run-tests.js -g ${{ matrix.group }}/8 -c ${TEST_CONCURRENCY} --type e2e + NEXT_EXTERNAL_TESTS_FILTERS: ../next-runtime-minimal/tests/netlify-e2e.json + run: node run-tests.js -g ${{ matrix.group }}/20 -c ${TEST_CONCURRENCY} --type e2e working-directory: ${{ env.next-path }} - - name: Test results - working-directory: ${{ env.next-path }} + - name: Upload Test Results + if: success() || failure() + uses: actions/upload-artifact@v3 + with: + name: Test Results (${{ matrix.group }}) + path: ${{ env.next-path }}/test/test-junit-report/*.xml + publish-test-results: + name: 'E2E Test Summary' + needs: e2e + runs-on: ubuntu-latest + permissions: + checks: write + contents: read + issues: read + if: success() || failure() + + steps: + - name: checkout runtime repo + uses: actions/checkout@v4 + + - name: Install Deno + uses: denoland/setup-deno@v1 + + - name: Download Artifacts + uses: actions/download-artifact@v3 + with: + path: artifacts + + - name: Publish Test Report + if: success() || failure() run: | - for file in `ls test/e2e/**/*.results.json`; do - test_name=$(echo $file | sed -e 's/test\/e2e\/\(.*\/.*\).results.json/\1/') - - success=$(jq -r '.success' $file) - if [ "$success" = "true" ] ; then - test_result=":white_check_mark:" - else - test_result=":x:" - fi - - passed=$(jq -r '.numPassedTests' $file) - passed_total=$((passed_total + passed)) - failed=$(jq -r '.numFailedTests' $file) - failed_total=$((failed_total + failed)) - pending=$(jq -r '.numPendingTests' $file) - pending_total=$((pending_total + pending)) - total=$(jq -r '.numTotalTests' $file) - total_total=$((total_total + total)) - - message=$(jq -r '.testResults[].message' $file) - - echo "
" >> test_summary - echo "${test_result} ${test_name}" >> test_summary - echo "

Results

" >> test_summary - echo "
Passing tests: ${passed}
" >> test_summary - echo "
Failing tests: ${failed}
" >> test_summary - echo "
Pending tests: ${pending}
" >> test_summary - echo "
Total tests: ${total}
" >> test_summary - - if [ -n "${message}" ] ; then - echo "

Details

" >> test_summary - echo "
" >> test_summary
-              echo $message >> test_summary
-              echo "
" >> test_summary - fi - - echo "
" >> test_summary - done - - echo "

Results

" >> $GITHUB_STEP_SUMMARY - echo "
Passing tests: ${passed_total}
" >> $GITHUB_STEP_SUMMARY - echo "
Failing tests: ${failed_total}
" >> $GITHUB_STEP_SUMMARY - echo "
Pending tests: ${pending_total}
" >> $GITHUB_STEP_SUMMARY - echo "
Total tests: ${total_total}
" >> $GITHUB_STEP_SUMMARY - - echo "

Details

" >> $GITHUB_STEP_SUMMARY - cat test_summary >> $GITHUB_STEP_SUMMARY - - if [ "$failed_total" -gt 0 ] ; then - echo "Failed tests. Please see action summary for details." - exit 1 - fi + deno run -A tools/deno/junit2md.ts artifacts >> $GITHUB_STEP_SUMMARY diff --git a/run-local-test.sh b/run-local-test.sh new file mode 100755 index 0000000000..6dd80caae3 --- /dev/null +++ b/run-local-test.sh @@ -0,0 +1,17 @@ +#!/bin/sh +set -e +# Print usage if nothing is passed +if [ -z "$1" ]; then + echo "Usage: $0 " + exit 1 +fi +# Check that the next.js directory exists +if [ ! -d "../next.js" ]; then + echo "Error: next.js repo needs to be in ../next.js" + exit 1 +fi +export NEXT_TEST_CONTINUE_ON_ERROR=1 +export NETLIFY_SITE_ID=1d5a5c76-d445-4ae5-b694-b0d3f2e2c395 +export NEXT_TEST_MODE=deploy +cd ../next.js/ +node run-tests.js --type e2e --debug --test-pattern $1 diff --git a/tests/netlify-e2e.json b/tests/netlify-e2e.json new file mode 100644 index 0000000000..029c359693 --- /dev/null +++ b/tests/netlify-e2e.json @@ -0,0 +1,138 @@ +{ + "version": 2, + "suites": { + "test/e2e/app-dir/app-static/app-static.test.ts": { + "failed": [ + "app-dir static/dynamic handling usePathname should have values from canonical url on rewrite", + "app-dir static/dynamic handling should have correct prerender-manifest entries", + "app-dir static/dynamic handling should output HTML/RSC files for static paths", + "app-dir static/dynamic handling should output debug info for static bailouts" + ] + }, + "test/e2e/app-dir/app-client-cache/client-cache.test.ts": { + "failed": [ + "app dir client cache semantics prefetch={undefined} - default should re-use the full cache for only 30 seconds", + "app dir client cache semantics prefetch={undefined} - default should refetch below the fold after 30 seconds" + ] + }, + "test/e2e/app-dir/headers-static-bailout/headers-static-bailout.test.ts": { + "failed": [ + "headers-static-bailout it provides a helpful link in case static generation bailout is uncaught" + ] + }, + "test/e2e/app-dir/parallel-routes-and-interception/parallel-routes-and-interception.test.ts": { + "failed": [ + "parallel-routes-and-interception route intercepting should render modal when paired with parallel routes", + "parallel-routes-and-interception route intercepting should support intercepting local dynamic sibling routes" + ] + }, + "test/e2e/app-dir/error-boundary-navigation/override-node-env.test.ts": { + "failed": [ + "app dir - not found navigation - with overridden node env should be able to navigate to other page from root not-found page" + ] + }, + "test/e2e/opentelemetry/opentelemetry.test.ts": { + "failed": [ + "opentelemetry root context app router should handle RSC with fetch", + "opentelemetry incoming context propagation app router should handle RSC with fetch", + "opentelemetry incoming context propagation app router should handle route handlers in app router" + ] + }, + "test/e2e/app-dir/rsc-basic/rsc-basic.test.ts": { + "failed": [ + "app dir - rsc basics should render initial styles of css-in-js in edge SSR correctly", + "app dir - rsc basics should render initial styles of css-in-js in nodejs SSR correctly", + "app dir - rsc basics should render server components correctly" + ], + "flakey": [ + "app dir - rsc basics react@experimental should opt into the react@experimental when enabling ppr", + "app dir - rsc basics react@experimental should opt into the react@experimental when enabling taint" + ] + }, + "test/e2e/app-dir/navigation/navigation.test.ts": { + "failed": [ + "app dir - navigation redirect status code should respond with 308 status code if permanent flag is set", + "app dir - navigation redirect status code should respond with 307 status code in client component", + "app dir - navigation redirect status code should respond with 307 status code in server component", + "app dir - navigation bots should block rendering for bots and return 404 status", + "app dir - navigation navigation between pages and app should not continously initiate a mpa navigation to the same URL when router state changes" + ] + }, + "test/e2e/app-dir/app-static/app-static-custom-handler.test.ts": { + "failed": [ + "app-dir static/dynamic handling should output debug info for static bailouts", + "app-dir static/dynamic handling should have correct prerender-manifest entries", + "app-dir static/dynamic handling should output HTML/RSC files for static paths" + ] + }, + "test/production/app-dir/unexpected-error/unexpected-error.test.ts": { + "failed": [ + "unexpected-error should set response status to 500 for unexpected errors in ssr app route", + "unexpected-error should set response status to 500 for unexpected errors in isr app route" + ] + }, + "test/e2e/skip-trailing-slash-redirect/index.test.ts": { + "flakey": [ + "skip-trailing-slash-redirect should merge cookies from middleware and edge API routes correctly", + "skip-trailing-slash-redirect should handle external rewrite correctly /chained-rewrite-ssr", + "skip-trailing-slash-redirect should handle external rewrite correctly /chained-rewrite-static", + "skip-trailing-slash-redirect should handle external rewrite correctly /chained-rewrite-ssg" + ] + }, + "test/e2e/module-layer/index.test.ts": { + "flakey": [ + "module layer no server-only in server targets should render routes marked with restriction marks without errors", + "module layer with server-only in server targets should render routes marked with restriction marks without errors" + ] + }, + "test/e2e/getserversideprops/test/index.test.ts": { + "flakey": [ + "getServerSideProps should set default caching header", + "getServerSideProps should respect custom caching header" + ] + }, + "test/e2e/app-dir/metadata-dynamic-routes/index.test.ts": { + "pending": [], + "flakey": [ + "app dir - metadata dynamic routes text routes should handle robots.[ext] dynamic routes", + "app dir - metadata dynamic routes text routes should handle sitemap.[ext] dynamic routes", + "app dir - metadata dynamic routes social image routes should handle manifest.[ext] dynamic routes", + "app dir - metadata dynamic routes social image routes should render og image with opengraph-image dynamic routes", + "app dir - metadata dynamic routes social image routes should render og image with twitter-image dynamic routes", + "app dir - metadata dynamic routes icon image routes should render icon with dynamic routes", + "app dir - metadata dynamic routes icon image routes should render apple icon with dynamic routes", + "app dir - metadata dynamic routes should inject dynamic metadata properly to head" + ] + }, + "test/e2e/app-dir/metadata/metadata.test.ts": { + "flakey": [ + "app dir - metadata opengraph should pick up opengraph-image and twitter-image as static metadata files", + "app dir - metadata static routes should have /favicon.ico as route" + ] + } + }, + "rules": { + "include": ["test/e2e/**/*.test.{t,j}s{,x}"], + "exclude": [ + "test/e2e/app-dir/next-font/**/*", + "test/e2e/app-dir/ppr/**/*", + "test/e2e/app-dir/ppr-*/**/*", + "test/e2e/app-dir/app-prefetch*/**/*", + "test/e2e/app-dir/app-esm-js/index.test.ts", + "test/e2e/app-dir/interception-middleware-rewrite/interception-middleware-rewrite.test.ts", + "test/e2e/app-dir/searchparams-static-bailout/searchparams-static-bailout.test.ts", + "test/e2e/app-dir/app-compilation/index.test.ts", + "test/e2e/cancel-request/stream-cancel.test.ts", + "test/e2e/favicon-short-circuit/favicon-short-circuit.test.ts", + "test/e2e/edge-pages-support/edge-document.test.ts", + "test/e2e/third-parties/index.test.ts", + "test/e2e/swc-warnings/index.test.ts", + "test/e2e/app-dir/externals/externals.test.ts", + "test/e2e/app-dir/use-selected-layout-segment-s/use-selected-layout-segment-s.test.ts", + "test/e2e/repeated-forward-slashes-error/repeated-forward-slashes-error.test.ts", + "test/e2e/app-dir/with-exported-function-config/with-exported-function-config.test.ts", + "test/e2e/app-dir/x-forwarded-headers/x-forwarded-headers.test.ts", + "test/e2e/app-dir/third-parties/basic.test.ts" + ] + } +} diff --git a/tools/deno/junit2md.ts b/tools/deno/junit2md.ts new file mode 100644 index 0000000000..7425c83f29 --- /dev/null +++ b/tools/deno/junit2md.ts @@ -0,0 +1,156 @@ +import { expandGlob } from 'https://deno.land/std/fs/mod.ts' +import { parse } from 'https://deno.land/x/xml/mod.ts' + +interface TestCase { + '@classname': string + '@name': string + '@time': number + '@file': string + failure?: string +} + +interface TestSuite { + '@name': string + '@errors': number + '@failures': number + '@skipped': number + '@timestamp': string + '@time': number + '@tests': number + testcase: TestCase[] +} + +interface TestSuites { + '@name': string + '@tests': number + '@failures': number + '@errors': number + '@time': number + testsuite: TestSuite[] +} + +async function parseXMLFile(filePath: string): Promise<{ testsuites: TestSuites }> { + const xmlContent = await Deno.readTextFile(filePath) + return parse(xmlContent) as unknown as { testsuites: TestSuites } +} + +const suites: Array<{ + name: string + tests: number + failures: number + skipped: number + time: number +}> = [] + +const testCount = { + '❌': 0, + '⏭️': 0, + '✅': 0, +} + +function junitToMarkdown(xmlData: { testsuites: TestSuites }) { + if (!xmlData.testsuites) { + return '' + } + let markdown = `` + + const testSuites = Array.isArray(xmlData.testsuites.testsuite) + ? xmlData.testsuites.testsuite + : [xmlData.testsuites.testsuite] + + for (const suite of testSuites) { + const { + '@tests': tests, + '@failures': failures, + '@skipped': skipped, + '@name': name, + '@time': time, + } = suite + + suites.push({ + name, + tests, + failures, + skipped, + time, + }) + + const passed = tests - failures - skipped + + const testCases = Array.isArray(suite.testcase) ? suite.testcase : [suite.testcase] + + const testCasesDetails = testCases + .map((testCase) => { + const status = testCase.failure ? '❌' : 'skipped' in testCase ? '⏭️' : '✅' + + testCount[status]++ + return `
  • ${status} ${testCase['@name'].slice(name.length)} (${testCase['@time'].toFixed( + 2, + )}s)
    • ` + }) + .join('') + + markdown += `|
    ${name}
      ${testCasesDetails}
    | ✅ ${passed} | ❌ ${failures} | ⏭️ ${skipped} | ${Math.round( + time, + )}s |\n` + } + + return markdown +} + +async function processJUnitFiles(directoryPath: string) { + let markdown = `| Suite | Passed | Failed | Skipped | Time |\n| ------- | ------ | ------ | ------- | ---- |\n` + for await (const file of expandGlob(`${directoryPath}/**/*.xml`)) { + const xmlData = await parseXMLFile(file.path) + markdown += junitToMarkdown(xmlData) + } + return markdown +} + +// Get the directory path from the command-line arguments +const directoryPath = Deno.args[0] + +// Check if the directory path is provided +if (!directoryPath) { + console.error('Please provide a directory path.') + Deno.exit(1) +} + +// Process the JUnit files in the provided directory +const details = await processJUnitFiles(directoryPath) + +let passedSuites = 0 +let failedSuites = 0 +let skippedSuites = 0 +let partialSuites = 0 + +suites.forEach(({ tests, failures, skipped }) => { + const unskipped = tests - skipped + const pass = unskipped - failures + if (skipped === tests) { + skippedSuites++ + } else if (failures === unskipped) { + failedSuites++ + } else if (pass === unskipped) { + passedSuites++ + } else { + partialSuites++ + } +}) + +console.log('## Test results') +console.log(`| | Suites | Tests |`) +console.log(`| --- | --- | --- |`) +console.log(`| ✅ Passed | ${passedSuites} | ${testCount['✅']} |`) +console.log(`| ❌ Failed | ${failedSuites} | ${testCount['❌']} |`) +console.log(`| ⏭️ Skipped | ${skippedSuites} | ${testCount['⏭️']} |`) +console.log(`| 🌗 Partial | ${partialSuites} | |`) +console.log( + `| **Total** | ${passedSuites + failedSuites + skippedSuites + partialSuites} | ${ + testCount['✅'] + testCount['❌'] + testCount['⏭️'] + } |`, +) + +console.log('## Test cases') + +console.log(details)