Add accessibility test job running against home page (mozilla#14773)

.github/workflows/a11y_tests.yml

@@ -0,0 +1,74 @@
+# Workflow that runs accessibility tests once per day against dev infra.
+
+name: Accessibility Tests
+run-name: Accessibility Tests for ${{ github.sha }}
+env:
+  SLACK_CHANNEL_ID: CBX0KH5GA # #www-notify in MoCo Slack
+  SLACK_BOT_TOKEN: ${{secrets.SLACK_BOT_TOKEN_FOR_MEAO_NOTIFICATIONS_APP}}
+on:
+  schedule:
+    - cron: "0 15 * * *" # 3pm daily
+  workflow_dispatch:
+    inputs:
+      mozorg_service_hostname:
+        description: The root URL to run tests against. eg "https://www.mozilla.org/"
+        required: true
+
+jobs:
+  notify-of-test-run-start:
+    if: github.repository == 'mozilla/bedrock'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Notify via Slack that tests are starting
+        uses: ./.github/actions/slack
+        with:
+          env_name: test
+          label: "Accessibility Tests [${{ github.sha }}]"
+          status: info
+          channel_id: ${{ env.SLACK_CHANNEL_ID }}
+          slack_bot_token: ${{ env.SLACK_BOT_TOKEN }}
+          ref: ${{ github.sha }}
+          message: "Accessibility Tests started"
+
+  accessibility-tests:
+    runs-on: ubuntu-latest
+    needs: notify-of-test-run-start
+    env:
+      PLAYWRIGHT_BASE_URL: ${{ github.event.inputs.mozorg_service_hostname || 'https://dev.bedrock.nonprod.webservices.mozgcp.net' }}
+      CI: true
+      CI_JOB_ID: ${{ github.run_id }}
+      DRIVER: ""
+      LABEL: accessibility-tests
+    steps:
+    - uses: actions/checkout@v4
+    - uses: actions/setup-node@v4
+      with:
+        node-version: 20
+    - name: Install dependencies
+      run: cd tests/playwright && npm ci && npm run install-deps
+    - name: Run accessibility tests
+      run: npm run a11y-tests
+    - uses: actions/upload-artifact@v4
+      if: ${{ !cancelled() }}
+      with:
+        name: a11y-report-${{ github.sha }}
+        path: a11y-report/
+        retention-days: 30
+
+  notify-of-test-run-completion:
+    if: github.repository == 'mozilla/bedrock' && always()
+    runs-on: ubuntu-latest
+    needs: [notify-of-test-run-start, accessibility-tests]
+    steps:
+      - uses: actions/checkout@v4
+      - name: Notify via Slack of test-run outcome
+        uses: ./.github/actions/slack
+        with:
+          env_name: test
+          label: "Accessibility tests [${{ github.sha }}]"
+          status: ${{ needs.accessibility-tests.result }}
+          channel_id: ${{ env.SLACK_CHANNEL_ID }}
+          slack_bot_token: ${{ env.SLACK_BOT_TOKEN }}
+          ref: ${{ github.sha }}
+          message: "Accessibility tests completed. Status: ${{ needs.accessibility-tests.result }}"

.github/workflows/integration_tests.yml

@@ -124,7 +124,7 @@ jobs:
     - name: Install dependencies
       run: cd tests/playwright && npm ci && npm run install-deps
     - name: Run Playwright tests
-      run: cd tests/playwright && npx playwright test
+      run: cd tests/playwright && npm run integration-tests
 
   notify-of-test-run-completion:
     if: always()

.gitignore

@@ -59,6 +59,7 @@ tests/functional/results.html
 tests/unit/coverage
 tests/unit/dist/
 tests/playwright/test-results/
+tests/playwright/a11y-report/
 Thumbs.db
 tmp/*
 venv

docs/testing.rst

@@ -18,6 +18,11 @@ correctly and that new changes don't break existing functionality.
   environment. These tests are run automatically as part of our CI deployment process against
   dev, stage, and prod. Playwright tests are run against Firefox, Chromium, and Webkit headless
   browsers for cross-engine coverage.
+- `Axe`_ tests are used to test for accessibility issues on key pages. These tests are not
+  run as part of our CI deployment process as they can contain a lot of information, but instead
+  run once per day via a GitHub action against dev. Axe tests are run via Playwright as a subset
+  of tests using the ``@a11y`` tag. Accessibility issues are reported in the GitHub action output,
+  which can be downloaded and reviewed.
 - `Selenium`_ tests are bedrock's older, legacy integration test suite, which will eventually be
   replaced by Playwright. These tests are run against Firefox, Chrome, and Internet Explorer
   (via a mix of both a local Selenium Grid and Sauce Labs) as part of our CI pipeline, and
@@ -33,6 +38,7 @@ The test specs for all of the above suites can be found in the root ``./tests``
 
 * ``./tests/unit/`` for Jasmine tests.
 * ``./tests/playwright/`` Playwright tests.
+* ``./tests/playwright/specs/a11y/`` Axe tests.
 * ``./tests/functional/`` for Selenium tests.
 
 Automating the browser
@@ -162,7 +168,7 @@ To run the full suite of tests (from the ``/tests/playwright/`` directory):
 
 .. code-block:: bash
 
-    $ npx playwright test
+    $ npm run integration-tests
 
 This will run all tests against three different headless browser engines (Firefox,
 Chromium, WebKit).
@@ -256,6 +262,29 @@ load a different override instead of using ``openPage``:
         await page.goto(url + '?automation=true');
     });
 
+Accessibility testing (Axe)
+===========================
+
+Axe tests are run as a subset of Playwright tests using the ``@a11y`` tag. These
+tests are run against the dev environment once per day via a GitHub action. The
+axe spec files can be found in the ``./tests/playwright/specs/a11y/`` directory.
+
+To run the Axe tests locally, you can use the following command from the
+``./tests/playwright/`` directory:
+
+.. code-block:: bash
+
+    $ npm run a11y-tests
+
+The Axe tests consist of two different test types. One that scans an entire page
+for accessibility issues, and another that scans a specific element for issues
+(such as the main navigation and footer). These tests can also be run at both
+desktop and mobile viewport sizes.
+
+Test results are output to the console, and any issues found will be created as
+HTML report files in the ``./tests/playwright/a11y-report/`` directory. When
+run via the GitHub action, the reports are also output to the GitHub action logs.
+
 Running Selenium tests
 ======================
 
@@ -509,3 +538,4 @@ via product details are well formed and return valid 200 responses.
 .. _Playwright CLI docs: https://playwright.dev/docs/test-cli
 .. _how to write tests: https://playwright.dev/docs/writing-tests
 .. _locator strategy: https://playwright.dev/docs/locators#locate-by-test-id
+.. _Axe: https://github.com/dequelabs/axe-core-npm/blob/develop/packages/playwright/README.md

tests/playwright/global-setup.js

@@ -0,0 +1,31 @@
+/*
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at https://mozilla.org/MPL/2.0/.
+ */
+
+'use strict';
+
+const fs = require('fs');
+
+/**
+ * Clean up /a11y-report/ directory before running tests.
+ */
+function cleanA11yReportDir() {
+    const dir = 'a11y-report/';
+    if (fs.existsSync(dir)) {
+        fs.rmdirSync(dir, {
+            recursive: true
+        });
+    }
+
+    if (!fs.existsSync(dir)) {
+        fs.mkdirSync(dir);
+    }
+}
+
+function globalSetup() {
+    cleanA11yReportDir();
+}
+
+module.exports = globalSetup;