NEW e2e environment using playground cli #299
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: E2E Tests | |
| on: | |
| workflow_dispatch: | |
| push: | |
| branches: | |
| - main | |
| pull_request: | |
| paths: | |
| - '.github/workflows/e2e-tests.yml' | |
| - 'build/**' | |
| - 'includes/**' | |
| - 'src/**' | |
| - 'test/e2e/**' | |
| - '*.php' | |
| - 'package.*' | |
| # Cancels all previous workflow runs for pull requests that have not completed. | |
| concurrency: | |
| # The concurrency group contains the workflow name and the branch name for pull requests | |
| # or the commit hash for any other events. | |
| group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.head_ref || github.sha }} | |
| cancel-in-progress: true | |
| jobs: | |
| e2e-tests: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 1 # Prevent parallel runs to make use of the caching for node_modules and playwright browsers | |
| matrix: | |
| php: [ | |
| '7.4', | |
| '8.3' | |
| ] | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - uses: actions/cache@v4 | |
| id: playwright-cache | |
| with: | |
| path: | | |
| ~/.cache/ms-playwright | |
| key: ${{ runner.os }}-playwright-${{ hashFiles('**/package-lock.json') }} | |
| - name: Setup Node | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version-file: '.nvmrc' | |
| # Enable built-in functionality for caching and restoring dependencies, which is disabled by default. | |
| # The actions/setup-node uses actions/cache under the hood. | |
| # https://github.com/actions/setup-node#caching-global-packages-data | |
| cache: 'npm' | |
| # Restoring the short lived node_modules cache | |
| # to be used across all workflows running on the last commit. | |
| # https://github.com/actions/cache/blob/main/caching-strategies.md#creating-a-short-lived-cache | |
| - uses: actions/cache/restore@v4 | |
| id: node_modules-cache | |
| with: | |
| path: | | |
| ./node_modules | |
| key: ${{ runner.os }}-node_modules-${{ github.sha }}-${{ hashFiles('package-lock.json') }} | |
| - name: NPM install | |
| if: steps.node_modules-cache.outputs.cache-hit != 'true' | |
| run: npm ci --legacy-peer-deps | |
| # Creating a short lived node_modules cache | |
| - uses: actions/cache/save@v4 | |
| if: steps.node_modules-cache.outputs.cache-hit != 'true' | |
| with: | |
| path: | | |
| ./node_modules | |
| key: ${{ steps.node_modules-cache.outputs.cache-primary-key }} | |
| - name: Install Playwright dependencies | |
| run: npx playwright install --with-deps | |
| if: steps.playwright-cache.outputs.cache-hit != 'true' | |
| - run: npx playwright install-deps | |
| if: steps.playwright-cache.outputs.cache-hit == 'true' | |
| - name: Prepare preferredVersions for blueprint | |
| # playground/cli ignores --php & --wp flags when --blueprint is set | |
| # | |
| # I created an issue at: | |
| # https://github.com/WordPress/playground-tools/issues/352 | |
| run: | | |
| preferredVersions='{ | |
| "php": "${{ matrix.php }}", | |
| "wp": "latest" | |
| }' | |
| echo "Use jq to prepend the preferred versions to the existing blueprint JSON file." | |
| jq --argjson preferredVersions "$preferredVersions" '{preferredVersions: $preferredVersions} + .' ./test/e2e/blueprint.json > temp.blueprint.json | |
| - name: Starting Playground, staring Playwright & running the tests | |
| # This is kind of a hack PART 1/2, | |
| # to make sure Playwright DOES NOT start the webserver on its own. | |
| # | |
| # Part 2/2 is the "command: undefined," declaration | |
| # in test/e2e/playwright.config.ts | |
| # | |
| # While auto-loading the webserver when needed sounded nice, it introduced a race-condition | |
| # between the setup of Playground and Playwrights own start event. | |
| # Playwright listens for the availability of the webserver relatively simple, | |
| # as soon as there is a status code 200, Playwright starts all engines. | |
| # | |
| # Unfortunately Playground is not ready at this point, it hast started WordPress | |
| # and is going to start stepping through the blueprint, but hasn't loaded GatherPress nor imported any data; | |
| # Resulting in failing tests. | |
| # | |
| # It was not possible (for me) to keep the setup of Playground in a separate step, | |
| # why this "run > sleep > run" became necessary. | |
| # The setup process usually takes about 20sec, so 60 is just a good extra, to not run into errors. | |
| # | |
| # The sleep step, should help making sure Playwright runs only, when this is DONE & READY. | |
| # | |
| # Because it seems to be a problem to "wait on webServer.command" https://github.com/microsoft/playwright/issues/11811 | |
| # & "it seems that globalSetup runs before webServer is started." https://github.com/microsoft/playwright/issues/11811#issuecomment-1040732201 | |
| run: | | |
| npm run playground:mount -- --blueprint=./temp.blueprint.json & \ | |
| sleep 60 && \ | |
| echo 'Playground is ready now, lets take some pictures.' && \ | |
| # DEBUG=pw:api,pw:webserver \ | |
| xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- \ | |
| npm run test:e2e | |
| - name: Archive debug artifacts (screenshots, traces) | |
| uses: actions/upload-artifact@v4 | |
| if: ${{ ! cancelled() }} | |
| with: | |
| name: failures-artifacts--${{ matrix.php }} | |
| path: artifacts/test-results/ | |
| if-no-files-found: ignore | |
| - name: Archive flaky tests report | |
| uses: actions/upload-artifact@v4 | |
| if: ${{ ! cancelled() }} | |
| with: | |
| name: flaky-tests-report--${{ matrix.php }} | |
| path: flaky-tests | |
| if-no-files-found: ignore | |
| merge-artifacts: | |
| name: Merge Artifacts | |
| if: ${{ ! cancelled() }} | |
| needs: [e2e-tests] | |
| runs-on: ubuntu-latest | |
| outputs: | |
| has-flaky-test-report: ${{ !!steps.merge-flaky-tests-reports.outputs.artifact-id }} | |
| steps: | |
| - name: Merge failures artifacts | |
| uses: actions/upload-artifact/merge@v4 | |
| # Don't fail the job if there aren't any artifacts to merge. | |
| continue-on-error: true | |
| with: | |
| name: failures-artifacts | |
| # Retain the merged artifacts in case of a rerun. | |
| pattern: failures-artifacts* | |
| delete-merged: true | |
| - name: Merge flaky tests reports | |
| id: merge-flaky-tests-reports | |
| uses: actions/upload-artifact/merge@v4 | |
| continue-on-error: true | |
| with: | |
| name: flaky-tests-report | |
| pattern: flaky-tests-report* | |
| delete-merged: true |