test(e2e): add boilerplate and tests

.github/labeler.yml

@@ -29,6 +29,7 @@
   - any:
       - changed-files:
           - any-glob-to-any-file: '**/*.test.ts'
+          - any-glob-to-any-file: 'tests/**/*'
 
 'area: i18n':
   - any:

.github/workflows/nightly-build.yaml

@@ -14,9 +14,59 @@ defaults:
     shell: bash
 
 jobs:
+  test-e2e:
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - name: Chrome
+            project: chrome
+            target: chrome
+            runs-on: ubuntu-22.04
+          - name: Edge
+            project: msedge
+            target: chrome
+            runs-on: ubuntu-22.04
+    timeout-minutes: 15
+    name: E2E Tests - ${{ matrix.name }}
+    runs-on: ${{ matrix.runs-on }}
+    environment: test
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Environment setup
+        uses: ./.github/actions/setup
+
+      - name: Build
+        run: pnpm build ${{ matrix.target }} --channel=nightly
+
+      - name: Install Playwright Browsers
+        run: pnpm exec playwright install --with-deps
+
+      - name: Run Playwright tests
+        run: xvfb-run pnpm test:e2e:${{ matrix.project }}
+        env:
+          PLAYWRIGHT_PROJECT: ${{ matrix.project }}
+          WALLET_URL_ORIGIN: ${{ vars.E2E_WALLET_URL_ORIGIN }}
+          WALLET_USERNAME: ${{ vars.E2E_WALLET_USERNAME }}
+          WALLET_PASSWORD: ${{ secrets.E2E_WALLET_PASSWORD }}
+          CONNECT_WALLET_ADDRESS_URL: ${{ vars.E2E_CONNECT_WALLET_ADDRESS_URL }}
+          CONNECT_KEY_ID: ${{ vars.E2E_CONNECT_KEY_ID }}
+          CONNECT_PUBLIC_KEY: ${{ secrets.E2E_CONNECT_PUBLIC_KEY }}
+          CONNECT_PRIVATE_KEY: ${{ secrets.E2E_CONNECT_PRIVATE_KEY }}
+
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: playwright-report-${{ matrix.project }}
+          path: tests/e2e/playwright-report/
+          retention-days: 3
+
   build-nightly:
     name: Create Release
     runs-on: ubuntu-22.04
+    needs: test-e2e
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4

.github/workflows/tests-e2e.yml

@@ -0,0 +1,66 @@
+name: End-to-End Tests
+on:
+  pull_request_review:
+    types: [submitted]
+
+concurrency:
+  group: ${{ github.event.pull_request.number }}
+  cancel-in-progress: true
+
+jobs:
+  test-e2e:
+    if: ${{
+      github.event.review.body == 'test-e2e' &&
+      contains(fromJson('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.review.author_association)
+      }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - name: Chrome
+            project: chrome
+            target: chrome
+            runs-on: ubuntu-22.04
+          # - name: Firefox
+          #   project: firefox
+          #   target: firefox
+          #   runs-on: ubuntu-22.04
+          - name: Edge
+            project: msedge
+            target: chrome
+            runs-on: ubuntu-22.04
+
+    timeout-minutes: 15
+    name: E2E Tests - ${{ matrix.name }}
+    runs-on: ${{ matrix.runs-on }}
+    environment: test
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Environment setup
+        uses: ./.github/actions/setup
+
+      - name: Build
+        run: pnpm build ${{ matrix.target }} --channel=nightly
+
+      - name: Install Playwright Browsers
+        run: pnpm exec playwright install --with-deps
+
+      - name: Run Playwright tests
+        run: xvfb-run pnpm test:e2e:${{ matrix.project }}
+        env:
+          PLAYWRIGHT_PROJECT: ${{ matrix.project }}
+          WALLET_URL_ORIGIN: ${{ vars.E2E_WALLET_URL_ORIGIN }}
+          WALLET_USERNAME: ${{ vars.E2E_WALLET_USERNAME }}
+          WALLET_PASSWORD: ${{ secrets.E2E_WALLET_PASSWORD }}
+          CONNECT_WALLET_ADDRESS_URL: ${{ vars.E2E_CONNECT_WALLET_ADDRESS_URL }}
+          CONNECT_KEY_ID: ${{ vars.E2E_CONNECT_KEY_ID }}
+          CONNECT_PUBLIC_KEY: ${{ secrets.E2E_CONNECT_PUBLIC_KEY }}
+          CONNECT_PRIVATE_KEY: ${{ secrets.E2E_CONNECT_PRIVATE_KEY }}
+
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: playwright-report-${{ matrix.project }}
+          path: tests/e2e/playwright-report/
+          retention-days: 3

.gitignore

@@ -17,6 +17,7 @@ dist-firefox-v2
 public/manifest.json
 *.local
 coverage
+.env
 .husky
 .vscode
 
@@ -30,3 +31,9 @@ coverage
 *.njsproj
 *.sln
 *.sw?
+
+# playwright
+/tests/e2e/test-results/
+/tests/e2e/test-results/.auth
+/tests/e2e/playwright-report/
+/playwright/.cache/

cspell-dictionary.txt

@@ -27,6 +27,10 @@ iife
 backported
 autobuild
 buildscript
+networkidle
+webextensions
+firefoxUserPrefs
+textbox
 
 # packages and 3rd party tools/libraries
 awilix
@@ -35,6 +39,8 @@ loglevel
 openapi
 apidevtools
 tailwindcss
+msedge
+xvfb
 
 # user names
 raducristianpopa

cspell.json

@@ -18,6 +18,7 @@
     "*.svg",
     "pnpm-lock.yaml",
     ".eslintrc.json",
+    ".gitignore",
     "cspell-dictionary.txt"
   ]
 }

docs/TESTING.md

@@ -0,0 +1,56 @@
+# Automated Testing
+
+## Unit tests
+
+Run `pnpm test` to run unit tests locally. These tests are run automatically on every pull request.
+
+## End-to-end Tests
+
+To run end-to-end tests with chromium, run `pnpm test:e2e` in terminal.
+
+**Before you begin**, you need to setup some environment variables/secrets in `tests/.env`.
+
+1. Copy `tests/.env.example` to `tests/.env`
+2. Update `tests/.env` with your secrets.
+
+| Environment Variable         | Description                                                 | Is secret? |
+| ---------------------------- | ----------------------------------------------------------- | ---------- |
+| `WALLET_URL_ORIGIN`          | URL of the wallet (e.g. https://rafiki.money)               | No         |
+| `WALLET_USERNAME`            | Login email for the wallet                                  | No         |
+| `WALLET_PASSWORD`            | Login password for the wallet                               | Yes        |
+| `CONNECT_WALLET_ADDRESS_URL` | Your wallet address that will be connected to extension     | No         |
+| `CONNECT_KEY_ID`             | ID of the key that will be connected to extension (UUID v4) | No         |
+| `CONNECT_PRIVATE_KEY`        | Private key (hex-encoded Ed25519 private key)               | Yes        |
+| `CONNECT_PUBLIC_KEY`         | Public key (base64-encoded Ed25519 public key)              | No         |
+
+To get the `CONNECT_KEY_ID`, `CONNECT_PRIVATE_KEY` and `CONNECT_PUBLIC_KEY`:
+
+1. Load the extension in browser (via `chrome://extensions/`)
+   - Once the extension is loaded, it'll generate a key-pair that we will need to connect with our wallet.
+1. Inspect service worker with "Inspect views service worker"
+1. Run following in devtools console to copy keys to your clipboard, and paste it in `tests/.env`:
+   ```js
+   // 1. Gets generated keys from extension storage.
+   // 2. Converts result to `CONNECT_{X}="VAL"` format for use in .env file.
+   // 3. Copies result to clipboard.
+   copy(
+     Object.entries(
+       await chrome.storage.local.get(['privateKey', 'publicKey', 'keyId']),
+     )
+       .map(
+         ([k, v]) =>
+           `CONNECT_${k.replace(/([A-Z])/g, '_$1').toUpperCase()}="${v}"`,
+       )
+       .join('\n'),
+   );
+   ```
+1. Then copy `CONNECT_PUBLIC_KEY` key to https://rafiki.money/settings/developer-keys under your wallet address.
+1. Now you're ready to run the tests.
+
+### How to run in end-to-end tests in GitHub
+
+As these tests are expensive/time-consuming, these need to be triggered manually when needed, instead of on every pull request/commit.
+
+For a pull request, users with write access to repository can trigger the workflow to run end-to-end tests by adding a review-comment (from PR Files tab) with body `test-e2e` (exactly).
+
+End-to-end tests run automatically daily before creating the Nightly release. You can also trigger that workflow manually from [Actions Dashboard](https://github.com/interledger/web-monetization-extension/actions/workflows/nightly-build.yaml).

jest.config.ts

@@ -21,6 +21,7 @@ export default {
   testEnvironment: 'jsdom',
   testPathIgnorePatterns: [
     '<rootDir>/node_modules/',
+    '<rootDir>/tests/',
     '<rootDir>/jest.config.ts',
   ],
   transform: {

package.json

@@ -17,6 +17,9 @@
     "format:fix": "prettier . --write --cache --cache-location='node_modules/.cache/prettiercache' --log-level=warn",
     "typecheck": "tsc --noEmit",
     "test": "jest --maxWorkers=2 --passWithNoTests",
+    "test:e2e": "pnpm test:e2e:chrome",
+    "test:e2e:chrome": "playwright test --project=chrome",
+    "test:e2e:msedge": "playwright test --project=msedge",
     "test:ci": "pnpm test -- --reporters=default --reporters=github-actions"
   },
   "dependencies": {
@@ -42,10 +45,12 @@
   },
   "devDependencies": {
     "@jgoz/esbuild-plugin-typecheck": "^4.0.1",
+    "@playwright/test": "^1.47.0",
     "@tailwindcss/forms": "^0.5.7",
     "@testing-library/jest-dom": "^6.5.0",
     "@testing-library/react": "^16.0.1",
     "@types/archiver": "^6.0.2",
+    "@types/chrome": "^0.0.270",
     "@types/github-script": "github:actions/github-script",
     "@types/jest": "^29.5.12",
     "@types/node": "^20.16.2",
@@ -57,6 +62,7 @@
     "@typescript-eslint/parser": "^8.5.0",
     "archiver": "^7.0.1",
     "autoprefixer": "^10.4.20",
+    "dotenv": "^16.4.5",
     "esbuild": "^0.23.1",
     "esbuild-node-builtin": "^0.1.1",
     "esbuild-plugin-copy": "^2.1.1",

playwright.config.ts

@@ -0,0 +1,78 @@
+import path from 'node:path';
+import { defineConfig, devices } from '@playwright/test';
+import { testDir, authFile } from './tests/e2e/fixtures/helpers';
+
+if (!process.env.CI) {
+  // eslint-disable-next-line @typescript-eslint/no-require-imports
+  require('dotenv').config({ path: path.join(testDir, '.env') });
+}
+
+export default defineConfig({
+  testDir,
+  outputDir: path.join(testDir, 'test-results'),
+  // We don't want this set to true as that would make tests in each file to run
+  // in parallel, which will cause conflicts with the "global state". With this
+  // set to false and workers > 1, multiple test files can run in parallel, but
+  // tests within a file are run at one at a time. We make extensive use of
+  // worker-scope fixtures and beforeAll hooks to achieve best performance.
+  fullyParallel: false,
+  forbidOnly: !!process.env.CI,
+  retries: process.env.CI ? 2 : 0,
+  workers: process.env.CI ? 1 : undefined,
+  reporter: [
+    ['list'],
+    [
+      'html',
+      { open: 'never', outputFolder: path.join(testDir, 'playwright-report') },
+    ],
+  ],
+  use: { trace: 'on-first-retry' },
+
+  projects: [
+    {
+      name: 'setup',
+      testMatch: /.*\.setup\.ts/,
+    },
+
+    {
+      name: 'chrome',
+      use: {
+        ...devices['Desktop Chrome'],
+        storageState: authFile,
+        channel: 'chrome',
+      },
+      dependencies: ['setup'],
+    },
+
+    // Firefox+Playwright doesn't work well enough at the moment.
+    // {
+    //   name: 'firefox',
+    //   use: { ...devices['Desktop Firefox'], storageState: authFile },
+    //   dependencies: ['setup'],
+    // },
+
+    // Safari is surely a no-go for now
+    // {
+    //   name: 'webkit',
+    //   use: { ...devices['Desktop Safari'], storageState: authFile },
+    //   dependencies: ['setup'],
+    // },
+
+    {
+      name: 'msedge',
+      use: {
+        ...devices['Desktop Edge'],
+        channel: 'msedge',
+        storageState: authFile,
+      },
+      dependencies: ['setup'],
+    },
+  ],
+
+  /* Run your local dev server before starting the tests */
+  // webServer: {
+  //   command: 'npm run start',
+  //   url: 'http://127.0.0.1:3000',
+  //   reuseExistingServer: !process.env.CI,
+  // },
+});