Improve the E2E testing process with Playwright (#9148)

* Add Playwright infrastucture * improve documentation * improve type * remove puppeteer tests * fix wp-env.json * add link on how run E2E tests * chore on playwright.yml * remove unnecessary flush command * improve stability E2E test * remove build:e2e-test command * Update .github/workflows/playwright.yml Co-authored-by: Niels Lange <[email protected]> * Update .github/workflows/playwright.yml Co-authored-by: Niels Lange <[email protected]> * Update docs/contributors/contributing/e2e-guidelines.md Co-authored-by: Albert Juhé Lluveras <[email protected]> * Update docs/contributors/contributing/e2e-guidelines.md Co-authored-by: Albert Juhé Lluveras <[email protected]> * Update docs/contributors/contributing/e2e-guidelines.md Co-authored-by: Albert Juhé Lluveras <[email protected]> * remove emptyline and fix JSDoc warning * add link about E2E guidelines * fix theme name * improve style * improve markdown --------- Co-authored-by: Niels Lange <[email protected]> Co-authored-by: Albert Juhé Lluveras <[email protected]>
woocommerce · May 5, 2023 · 0b52ab8 · 0b52ab8
1 parent 58d721f
commit 0b52ab8
Show file tree

Hide file tree

Showing 37 changed files with 850 additions and 344 deletions.
diff --git a/.github/workflows/php-js-e2e-tests.yml b/.github/workflows/php-js-e2e-tests.yml
@@ -37,7 +37,7 @@ jobs:
         run: npm ci --no-optional
 
       - name: Build Assets
-        run: FORCE_REDUCED_MOTION=true npm run build:e2e-test
+        run: FORCE_REDUCED_MOTION=true npm run build
 
       - name: blocks.ini setup
         run: |
@@ -124,7 +124,7 @@ jobs:
           npm install --no-optional --no-audit
 
       - name: Build Assets
-        run: FORCE_REDUCED_MOTION=true npm run build:e2e-test
+        run: FORCE_REDUCED_MOTION=true npm run build
 
       - name: blocks.ini setup
         run: |

diff --git a/.github/workflows/playwright.yml b/.github/workflows/playwright.yml
@@ -1,33 +1,17 @@
 name: Playwright Tests
-
 on:
   push:
     branches: [ trunk ]
   pull_request:
 
 jobs:
   PlaywrightE2ETests:
-    if: false
+
     name: Playwright E2E tests
     timeout-minutes: 60
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-        with:
-          ref: trunk
-
-      - name: Cache node_modules
-        id: cache-node-modules
-        uses: actions/cache@v3
-        env:
-          cache-name: cache-node-modules
-        with:
-          path: node_modules
-          key: ${{ runner.os }}-build-${{ hashFiles('**/package-lock.json') }}
-          restore-keys: |
-            ${{ runner.os }}-build-${{ env.cache-name }}-
-            ${{ runner.os }}-build-
-            ${{ runner.os }}-
 
       - name: Setup node version and npm cache
         uses: actions/setup-node@v3
@@ -36,27 +20,45 @@ jobs:
           cache: 'npm'
 
       - name: Install Node dependencies
-        if: steps.cache-node-modules.outputs.cache-hit != 'true'
-        run: npm install --no-optional --no-audit
+        run: npm ci
 
-      - name: Load wp-env
-        run: npm run env:start --update
+      - name: Build Assets
+        run: FORCE_REDUCED_MOTION=true npm run build
 
-      - name: Fix permissions # We need to figure this out https://github.com/WordPress/gutenberg/issues/22515#issuecomment-1308346256
+      - name: blocks.ini setup
+        run: |
+          echo -e 'woocommerce_blocks_phase = 3\nwoocommerce_blocks_env = tests' > blocks.ini
+      - name: Get Composer Cache Directory
+        id: composer-cache
         run: |
-          WP_ENV_DIR=$(npm run wp-env install-path --silent 2>&1 | head -1)
-          cd $WP_ENV_DIR
-          mkdir -p tests-WordPress/wp-content/languages tests-WordPress/wp-content/upgrade
-          chmod -R 767 tests-WordPress/wp-content/languages tests-WordPress/wp-content/upgrade
-          cd -
+          echo "dir=$(composer config cache-files-dir)" >> $GITHUB_OUTPUT
+      - uses: actions/cache@v3
+        with:
+          path: ${{ steps.composer-cache.outputs.dir }}
+          key: ${{ runner.os }}-composer-${{ hashFiles('**/composer.lock') }}
+          restore-keys: ${{ runner.os }}-composer-
 
-      - name: Install Playwright Browsers
+      - name: Set up PHP
+        uses: shivammathur/setup-php@v2
+        with:
+          php-version: '8.0'
+          coverage: none
+          tools: composer
+
+      - name: Composer install
+        run: composer install
+
+      - name: Install Playwright
         run: npx playwright install --with-deps
 
+
+      - name: Load wp-env
+        run: npm run env:start
+
       - name: Run Playwright tests
-        run: npx playwright test --config=tests/e2e-pw/playwright.config.ts
+        run: npm run test:e2e-pw
+
       - uses: actions/upload-artifact@v3
         with:
           name: playwright-report
-          path: playwright-report/
-          retention-days: 30
+          path: ./tests/e2e-pw/artifacts/test-results
diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml
@@ -37,7 +37,7 @@ jobs:
       - name: Npm install and build
         run: |
           npm ci --no-optional
-          FORCE_REDUCED_MOTION=true npm run build:e2e-test
+          FORCE_REDUCED_MOTION=true npm run build
           
       - name: blocks.ini setup
         run: echo -e 'woocommerce_blocks_phase = 3\nwoocommerce_blocks_env = test' > blocks.ini

diff --git a/.gitignore b/.gitignore
@@ -44,6 +44,8 @@ tests/cli/vendor
 # E2E tests
 /tests/e2e-tests/config/local-*.json
 /tests/e2e-pw/test-results/
+/tests/e2e-pw/artifacts/
+/artifacts/
 
 # Logs
 /logs

diff --git a/.wp-env.json b/.wp-env.json
@@ -3,11 +3,23 @@
 	"plugins": [
 		"https://downloads.wordpress.org/plugin/woocommerce.latest-stable.zip",
 		"https://github.com/WP-API/Basic-Auth/archive/master.zip",
-		"./tests/mocks/woo-test-helper",
-		"."
+		"https://downloads.wordpress.org/plugin/wordpress-importer.0.8.zip",
+		"./tests/mocks/woo-test-helper"
 	],
+	"env": {
+		"tests": {
+			"mappings": {
+				"wp-content/mu-plugins": "./node_modules/@wordpress/e2e-tests/mu-plugins",
+				"wp-content/plugins/gutenberg-test-plugins": "./node_modules/@wordpress/e2e-tests/plugins",
+				"wp-content/plugins/woocommerce-blocks": ".",
+				"wp-cli.yml": "./wp-cli.yml"
+			}
+		}
+	},
 	"themes": [
 		"https://downloads.wordpress.org/theme/storefront.latest-stable.zip",
+		"https://downloads.wordpress.org/theme/twentytwentyone.latest-stable.zip",
+		"https://downloads.wordpress.org/theme/twentytwentythree.latest-stable.zip",
 		"./tests/mocks/emptytheme",
 		"./tests/mocks/theme-with-woo-templates"
 	],

diff --git a/bin/wp-env-config.sh b/bin/wp-env-config.sh
@@ -13,15 +13,15 @@ else
 fi
 
 ## set permalinks for easier wp-json
-wp rewrite structure '/%postname%/'
-wp rewrite flush
+wp rewrite structure '/%postname%/' --hard
 wp core version --extra
 wp plugin list
 wp theme activate storefront
 wp wc customer update 1 --user=1 --billing='{"first_name":"John","last_name":"Doe","company":"Automattic","country":"US","address_1":"addr 1","address_2":"addr 2","city":"San Francisco","state":"CA","postcode":"94107","phone":"123456789"}' --shipping='{"first_name":"John","last_name":"Doe","company":"Automattic","country":"US","address_1":"addr 1","address_2":"addr 2","city":"San Francisco","state":"CA","postcode":"94107","phone":"123456789"}'
 ## Prepare translation for the test suite
 wp language core install nl_NL
 wp language plugin install woocommerce nl_NL
+wp plugin activate woocommerce-blocks
 ## We download a full version of .po (that has translation for js files as well).
 curl https://translate.wordpress.org/projects/wp-plugins/woo-gutenberg-products-block/stable/nl/default/export-translations/ --output ./wp-content/languages/plugins/woo-gutenberg-products-block-nl_NL.po
 sleep 5

diff --git a/docs/contributors/README.md b/docs/contributors/README.md
@@ -7,6 +7,7 @@ This folder contains documentation for developers and contributors looking to ge
 | [Getting Started](contributing/getting-started.md)         | This doc covers tooling and creating builds during development.                          |
 | [Coding Guidelines](contributing/coding-guidelines.md)     | This doc covers development best practices.                                              |
 | [JavaScript Testing](contributing/javascript-testing.md)   | This doc explains how to run automated JavaScript tests.                                 |
+| [E2E Testing Guidelines](contributing/e2e-guidelines.md)   | This doc covers development best practices about E2E tests.                              |
 | [Developing Components (& Storybook)](components.md)       | This doc outlines where our reusable components live, and how to test them in Storybook. |
 | [Block Script Assets](contributing/block-assets.md)        | This doc explains how Block Script Assets are loaded and used.                           |
 | [JS build system](contributing/javascript-build-system.md) | This doc explains how JavaScript files are built.                                        |

diff --git a/docs/contributors/contributing/README.md b/docs/contributors/contributing/README.md
@@ -2,15 +2,17 @@
 
 This folder contains documentation for developers and contributors looking to get started with WooCommerce Block Development.
 
-| Document                                              | Description                                                                              |
-| ----------------------------------------------------- | ---------------------------------------------------------------------------------------- |
-| [Getting Started](getting-started.md)                 | This doc covers tooling and creating builds during development.                          |
-| [Coding Guidelines](coding-guidelines.md)             | This doc covers development best practices.                                              |
-| [Block Script Assets](block-assets.md)                | This doc explains how Block Script Assets are loaded and used.                           |
-| [CSS Build System](css-build-system.md)               | This doc explains how CSS files are built.                                               |
-| [JavaScript Build System](javascript-build-system.md) | This doc explains how JavaScript files are built.                                        |
-| [JavaScript Testing](javascript-testing.md)           | This doc explains how to run automated JavaScript tests.                                 |
-| [Storybook & Components](storybook-and-components.md) | This doc outlines where our reusable components live, and how to test them in Storybook. |
+| Document                                                 | Description                                                                              |
+|----------------------------------------------------------|------------------------------------------------------------------------------------------|
+| [Getting Started](getting-started.md)                    | This doc covers tooling and creating builds during development.                          |
+| [Coding Guidelines](coding-guidelines.md)                | This doc covers development best practices.                                              |
+| [Block Script Assets](block-assets.md)                   | This doc explains how Block Script Assets are loaded and used.                           |
+| [CSS Build System](css-build-system.md)                  | This doc explains how CSS files are built.                                               |
+| [JavaScript Build System](javascript-build-system.md)    | This doc explains how JavaScript files are built.                                        |
+| [JavaScript Testing](javascript-testing.md)              | This doc explains how to run automated JavaScript tests.                                 |
+| [E2E Testing Guidelines](e2e-guidelines.md) 			   | This doc covers development best practices about E2E tests.                              |
+| [Storybook & Components](storybook-and-components.md)    | This doc outlines where our reusable components live, and how to test them in Storybook. |
+
 
 <!-- FEEDBACK -->
 

diff --git a/docs/contributors/contributing/e2e-guidelines.md b/docs/contributors/contributing/e2e-guidelines.md
@@ -0,0 +1,55 @@
+# E2E Guidelines <!-- omit in toc -->
+
+## Table of contents <!-- omit in toc -->
+
+-   [Structure](#structure)
+-   [Playwright](#playwright)
+    -   [Structure](#structure-1)
+
+This living document serves to prescribe coding guidelines specific to the WooCommerce Blocks project E2E tests. For more information on how to run Playwright end-to-end (E2E) tests, please refer to the [dedicated resource](../../../tests/e2e-pw/README.md).
+
+## Structure
+
+There are two folders dedicated to E2E tests.
+
+The first folder is named "e2e" and it contains all the E2E tests that were created with the deprecated infrastructure Jest + Puppetter. The "e2e-pw" folder contains all the E2E tests that were created with the current infrastructure: Playwright. These tests are actively maintained and should be used for all new E2E testing.
+
+### Playwright
+
+#### Structure
+
+There are two Playwright projects configuration:
+
+- blockTheme
+- classicTheme
+
+The blockTheme project runs the tests with the suffix *block_theme*. In this case, the theme is a block theme. The block theme is the default WordPress theme. Currently, it is Twenty-Twenty Three. You should use this configuration if you want test the block with the Site Editor.
+
+The classicTheme project runs the tests with the suffix *classic_theme*. In this case, the theme is a Twenty Twenty-One. You should use this configuration if you want test the block with a classic theme.
+
+Each block should have a dedicated folder with a scoped util file if you want share some logic related to the block.
+
+#### Code Guidelines
+
+##### Make tests as isolated as possible - Avoid side effects
+
+Each test should be completely isolated from another test and should run independently with its own local storage, session storage, data, cookies etc. Test isolation improves reproducibility, makes debugging easier and prevents cascading test failures.
+
+In order to avoid repetition for a particular part of your test you can use before and after hooks. Within your test file add a before hook to run a part of your test before each test such as going to a particular URL or logging in to a part of your app. This keeps your tests isolated as no test relies on another. However it is also ok to have a little duplication when tests are simple enough especially if it keeps your tests clearer and easier to read and maintain. Avoid using functions that impact other tests, such as the `deleteAllTemplates` function, which restores all templates and can break other tests since E2E tests run in parallel. After running a suite of tests for a specific block, it is important to clean up any changes made during the tests to ensure a clean slate for subsequent test runs.
+
+For more detail see [Make Tests as Isolated as Possible](https://playwright.dev/docs/best-practices#make-tests-as-isolated-as-possible).
+
+##### Use Locators
+
+In order to write end to end tests we need to first find elements on the webpage. We can do this by using Playwright's built in locators. Locators come with auto waiting and retry-ability. Auto waiting means that Playwright performs a range of actionability checks on the elements, such as ensuring the element is visible and enabled before it performs the click. To make tests resilient, we recommend prioritizing user-facing attributes and explicit contracts. For more detail see [Use Locators](https://playwright.dev/docs/best-practices#use-locators).
+
+##### Avoid Using Relative Imports
+
+In order to make the codebase cleaner, you should import the function from the packages:
+
+- "@woocommerce/e2e-utils": Contains generic utils for interactive with the page.
+- "@woocommerce/e2e-types": Contains generic types.
+- "@woocommerce/e2e-playwright-utils": Contains utils for playwright for example custom hooks.
+
+By using these packages, you can make your code more modular and easier to maintain.
+
diff --git a/docs/contributors/contributing/javascript-testing.md b/docs/contributors/contributing/javascript-testing.md
@@ -36,15 +36,15 @@ Additionally,
 -   `test:update` updates the snapshot tests for components, used if you change a component that has tests attached.
 -   `test:watch` keeps watch of files and automatically re-runs tests when things change.
 
-## How to run end-to-end tests
+## How to run end-to-end tests with deprecated infrastructure
 
 End-to-end tests are implemented in `tests/e2e-tests/specs/`.
 
 Since these drive the user interface, they need to run against a test environment - i.e. a web server running WordPress, Woo and blocks plugin, with a known state/configuration.
 
 To set up to run e2e tests:
 
--   `npm run build:e2e-test` builds the assets (js/css), you can exclude this step if you've already got built files to test with.
+-   `npm run build` builds the assets (js/css), you can exclude this step if you've already got built files to test with.
 -   `npm run wp-env start` to start the test environment
 
     Then, to run the tests:
@@ -60,6 +60,10 @@ When you're done, you may want to shut down the test environment:
 
 **Note:** There are a number of other useful `wp-env` commands. You can find out more in the [wp-env docs](https://github.com/WordPress/gutenberg/blob/master/packages/env/README.md).
 
+## How to run end-to-end tests
+
+Visit the [dedicated documentation](../../../tests/e2e-pw/README.md).
+
 ### Debugging e2e tests using generated reports
 
 When e2e test suites are run in a GitHub automation, a report is generated automatically for every suite that failed. This can be a useful tool to debug failing tests, as it provides a visual way to inspect the tests that failed and, additionally, it includes some screenshots.