diff --git a/.github/workflows/run-qit-all.yml b/.github/workflows/run-qit-all.yml new file mode 100644 index 00000000..88f4724c --- /dev/null +++ b/.github/workflows/run-qit-all.yml @@ -0,0 +1,85 @@ +name: Run QIT for all extensions + +# **What it does**: Runs a suite of tests for all Grow extensions. +# **Why we have it**: To be able to run tests for all extensions at once. For example when we want to compatibility test a new version of the platform (WP/WC). + +on: + workflow_dispatch: + inputs: + # Basic params. + version: + description: 'Version to be tested' + required: true + type: choice + options: + - latest + - dev + wait: + description: 'Should wait for results' + default: false + type: boolean + + # Configure which tests to run. + test-activation: + description: 'Should activation be tested?' + required: true + default: true + type: boolean + test-security: + description: 'Should security be tested?' + required: true + default: true + type: boolean + test-phpstan: + description: 'Should phpstan be tested?' + required: true + default: true + type: boolean + test-api: + description: 'Should API be tested?' + required: true + default: true + type: boolean + test-e2e: + description: 'Should E2E be tested? (takes a lot of time)' + required: true + default: false + type: boolean + + # Advanced customization. + ignore-fail: + description: Should pass even if any awaited test fails. + required: false + default: false + type: boolean + options: + description: 'Additional options for `qit` command, like `--optional_features=hpos`.' + required: false + + +jobs: + qit-tests: + name: Run QIT Tests + runs-on: ubuntu-latest + strategy: + # Allow to test extensions even if one of them fails. + fail-fast: false + matrix: + # List of extensions to be tested. + extension: [automatewoo, automatewoo-birthdays, automatewoo-referrals, google-listings-and-ads, woocommerce-google-analytics-integration] + steps: + - name: Run QIT + uses: woocommerce/grow/run-qit-extension@add/qit-workflows-test-build + with: + qit-partner-user: ${{ secrets.QIT_PARTNER_USER }} + qit-partner-secret: ${{ secrets.QIT_PARTNER_SECRET }} + version: ${{ inputs.version }} + wait: ${{ inputs.wait }} + test-activation: ${{ inputs.test-activation }} + test-security: ${{ inputs.test-security }} + test-phpstan: ${{ inputs.test-phpstan }} + test-api: ${{ inputs.test-api }} + test-e2e: ${{ inputs.test-e2e }} + extension: ${{ matrix.extension }} + ignore-fail: ${{ inputs.ignore-fail }} + options: ${{ inputs.options }} diff --git a/packages/github-actions/README.md b/packages/github-actions/README.md index dae3d7f2..5fd0a791 100644 --- a/packages/github-actions/README.md +++ b/packages/github-actions/README.md @@ -20,6 +20,8 @@ Custom GitHub actions that help to composite GitHub workflows across the repos m - [`prepare-node`](actions/prepare-node) - Set up Node.js with a specific version, load npm cache, install Node dependencies - [`prepare-php`](actions/prepare-php) - Set up PHP with a specific version and tools, load Composer cache, install Composer dependencies - [`publish-extension-dev-build`](actions/publish-extension-dev-build) - Publish extension development build +- [`run-qit-annotate`](actions/run-qit-annotate) - Runs QIT test and annotates the results +- [`run-qit-extension`](actions/run-qit-extension) - Run QIT tests for a given extension - [`stylelint-annotation`](actions/stylelint-annotation) - Annotate stylelint results via stylelint formatter - [`update-version-tags`](actions/update-version-tags) - Update version tags diff --git a/packages/github-actions/actions/run-qit-annotate/README.md b/packages/github-actions/actions/run-qit-annotate/README.md new file mode 100644 index 00000000..f0b1147b --- /dev/null +++ b/packages/github-actions/actions/run-qit-annotate/README.md @@ -0,0 +1,50 @@ +# Run QIT test + +This action provides the following functionality for GitHub Actions users: + +- Run `qit` test of given type for a given extension +- Annotate the results + + +## Usage + +See [action.yml](action.yml) + +### Prerequisites + +- QIT needs to be [installed](https://woocommerce.github.io/qit-documentation/#/cli/getting-started?id=installing-qit) and [authenticated](https://woocommerce.github.io/qit-documentation/#/authenticating?id=cli) + - To set up QIT, you need to set `QIT_DISABLE_ONBOARDING` env to `yes`. + + +### Basic: + +```yaml +jobs: + qit-test: + name: Run QIT Tests + runs-on: ubuntu-20.04 + env: + QIT_DISABLE_ONBOARDING: yes + steps: + - name: Install QIT via composer + run: composer require woocommerce/qit-cli + + - name: Add Partner + run: | + ./vendor/bin/qit partner:add \ + --user='${{ secrets.QIT_PARTNER_USER }}' \ + --qit_token='${{ secrets.QIT_PARTNER_SECRET }}' + + - name: Security test + id: security-test + uses: woocommerce/grow/run-qit-annotate@actions-v1 + timeout-minutes: 5 + with: + type: security + extension: automatewoo + options: '--optional_features=hpos' + + - name: Echo status + shell: bash + run: echo ${{ jobs.security-test.outputs.status }} +``` diff --git a/packages/github-actions/actions/run-qit-annotate/action.yml b/packages/github-actions/actions/run-qit-annotate/action.yml new file mode 100644 index 00000000..2d3da5dc --- /dev/null +++ b/packages/github-actions/actions/run-qit-annotate/action.yml @@ -0,0 +1,100 @@ +name: Run QIT test +description: Runs QIT test and annotates the results. + +# **What it does**: Runs a single QIT test and annotates the results. +# **Why we have it**: To reuse across other workflows to test extensions. + +inputs: + # Basic params. + extension: + description: Extension to test + required: true + extension-file: + description: Custom build of the extension to test. If not given, the latest release will be used. + required: false + ignore-fail: + description: Should pass even if the awaited test fails. + required: false + default: false + type: + description: Type of test to run + required: false + default: activation + wait: + description: Wait for the test to finish + required: false + default: false + # Advanced customization. + options: + description: Options to pass to the qit command + required: false + default: '' + +outputs: + status: + description: "Status of the test run. May be used, for example, to fail the workflow if the test fails." + value: ${{ steps.read-summary.outputs.status }} + summary: + description: "Short summary of the test" + value: ${{ steps.read-summary.outputs.summary }} + test_run_id: + description: "ID of the QIT test run" + value: ${{ steps.read-summary.outputs.test_run_id }} + +branding: + icon: 'award' + color: 'purple' + +runs: + using: composite + steps: + # Run the qit command. + - name: Run test + id: run-test + # Do not fail when the `qit` fails, so we can annotate the results. + shell: bash --noprofile --norc {0} + env: + # If the custom build is provided, pass it to the `zip` param. + zip: ${{ inputs.extension-file && format('--zip={0}', inputs.extension-file) || '' }} + wait: ${{ inputs.wait == 'true' && '--wait' || '' }} + ignore_fail: ${{ inputs.ignore-fail == 'true' && '--ignore-fail' || '' }} + run: | + json=`./vendor/bin/qit run:${{ inputs.type }} \ + ${{ inputs.extension }} \ + $zip \ + ${{ inputs.options }} \ + $wait \ + $ignore_fail \ + -n \ + --json` + echo "exitcode=${PIPESTATUS[0]}" >> "$GITHUB_OUTPUT" + json=$(echo $json | jq '. | {test_run_id, status, test_summary}') + echo $json | jq + echo "result="$json"" >> "$GITHUB_OUTPUT" + + # Parse the result json to populate GHA outputs. + - name: Read summary + id: read-summary + shell: bash + run: | + echo "test_run_id=${{fromJson(steps.run-test.outputs.result).test_run_id}}" >> $GITHUB_OUTPUT + echo "status=${{fromJson(steps.run-test.outputs.result).status}}" >> $GITHUB_OUTPUT + echo "summary=${{fromJson(steps.run-test.outputs.result).test_summary}}" >> $GITHUB_OUTPUT + + + # Annotate the results according to the status, forward qit exit code. + - name: Annotate and exit + shell: bash + run: | + summary="${{ inputs.type }} (${{ steps.read-summary.outputs.test_run_id }}): ${{ steps.read-summary.outputs.status }} - ${{ steps.read-summary.outputs.summary }} \`qit get ${{ steps.read-summary.outputs.test_run_id }}\`"; + case ${{ steps.read-summary.outputs.status }} in + "success") echo "::notice ::$summary" + ;; + "pending") echo "::notice ::$summary" + ;; + "warning") echo "::warning ::$summary" + ;; + *) echo "::error ::$summary" + ;; + esac + exit ${{ steps.run-test.outputs.exitcode }} diff --git a/packages/github-actions/actions/run-qit-extension/README.md b/packages/github-actions/actions/run-qit-extension/README.md new file mode 100644 index 00000000..db2aa99e --- /dev/null +++ b/packages/github-actions/actions/run-qit-extension/README.md @@ -0,0 +1,34 @@ +# Run QIT tests + +This action provides the following functionality for GitHub Actions users: + +- Install QIT-CLI +- Run a set of `qit` tests for a given extension +- Annotate the results + + +## Usage + +See [action.yml](action.yml) + +### Prerequisites + +- You need to obtain valid [authentication keys](https://woocommerce.github.io/qit-documentation/#/authenticating?id=cli). + + +### Basic: + +```yaml +jobs: + qit-test: + name: Run QIT Tests + runs-on: ubuntu-20.04 + steps: + - name: Delegate QIT Tests + uses: woocommerce/grow/run-qit-extension@actions-v1 + with: + qit-partner-user: ${{ secrets.QIT_PARTNER_USER }} + qit-partner-secret: ${{ secrets.QIT_PARTNER_SECRET }} + extension: 'my-extension' + test-activation: true +``` diff --git a/packages/github-actions/actions/run-qit-extension/action.yml b/packages/github-actions/actions/run-qit-extension/action.yml new file mode 100644 index 00000000..246f7aee --- /dev/null +++ b/packages/github-actions/actions/run-qit-extension/action.yml @@ -0,0 +1,151 @@ +name: Run QIT for a given extension +description: Runs many types of QIT tests and annotates the results for a given extension. + +# **What it does**: Runs a set of QIT tests for a given extension. +# **Why we have it**: To reuse across other repos, to make a full test of a single extension. + +inputs: + # Basic params. + qit-partner-user: + description: 'QIT partner username' + required: true + qit-partner-secret: + description: 'QIT partner password' + required: true + github-token: + description: 'Github token to access private repos' + required: false + default: ${{ github.token }} + extension: + description: Extension to test + required: true + ignore-fail: + description: Should pass even if any awaited test fails. + required: false + default: false + version: + description: | + Version to be tested: `latest`, `dev`, or `local`. + `latest` will test latest released version, + `dev` will make the action look up for an `{extension}.zip` file in the root + of the repository under the `gha-dev-build` tag, + `local` will look up for an `{extension}.zip` artifact in the current workflow. + required: true + default: 'latest' + wait: + description: Wait for the test to finish + required: false + default: false + + # Customize which types to run. + test-activation: + description: 'Should activation be tested?' + default: 'true' + test-security: + description: 'Should security be tested?' + default: 'true' + test-phpstan: + description: 'Should PHPStan be tested?' + default: 'true' + test-api: + description: 'Should API be tested?' + default: 'true' + test-e2e: + description: 'Should E2E be tested?' + default: 'true' + + # Advanced customization. + options: + description: 'Additional options for `qit` command, like `--optional_features=hpos`' + type: string + +runs: + using: composite + steps: + - name: Prepare env variables + shell: bash + run: | + dev_build="${{ inputs.version != 'latest' && format('{0}.zip', inputs.extension) || '' }}" + + echo "dev_build=$dev_build" >> $GITHUB_ENV + echo "QIT_DISABLE_ONBOARDING=yes" >> $GITHUB_ENV + + - name: Download `gha-dev-build` + if: ${{ inputs.version == 'dev' }} + uses: robinraju/release-downloader@v1.8 + with: + repository: "woocommerce/${{ inputs.extension }}" + tag: 'gha-dev-build' + fileName: ${{ env.dev_build }} + token: ${{ inputs.github-token }} + + - name: Install QIT via composer + shell: bash + run: composer require woocommerce/qit-cli + + - name: Add Partner + shell: bash + run: | + ./vendor/bin/qit partner:add \ + --user='${{ inputs.qit-partner-user }}' \ + --qit_token='${{ inputs.qit-partner-secret }}' + + - name: Activation test + id: activation + if: ${{ inputs.test-activation == 'true' }} + uses: woocommerce/grow/run-qit-annotate@add/qit-workflows-test-build + with: + type: activation + extension: ${{ inputs.extension }} + extension-file: ${{ env.dev_build }} + wait: ${{ inputs.wait }} + ignore-fail: ${{ inputs.ignore-fail }} + options: ${{ inputs.options }} + + - name: Security test + id: security + if: ${{ inputs.test-security == 'true' }} + uses: woocommerce/grow/run-qit-annotate@add/qit-workflows-test-build + with: + type: security + extension: ${{ inputs.extension }} + extension-file: ${{ env.dev_build }} + wait: ${{ inputs.wait }} + ignore-fail: ${{ inputs.ignore-fail }} + options: ${{ inputs.options }} + + - name: PHPStan test + id: phpstan + if: ${{ inputs.test-phpstan == 'true' }} + uses: woocommerce/grow/run-qit-annotate@add/qit-workflows-test-build + with: + type: phpstan + extension: ${{ inputs.extension }} + extension-file: ${{ env.dev_build }} + wait: ${{ inputs.wait }} + ignore-fail: ${{ inputs.ignore-fail }} + options: ${{ inputs.options }} + + - name: API test + id: api + if: ${{ inputs.test-api == 'true' }} + uses: woocommerce/grow/run-qit-annotate@add/qit-workflows-test-build + with: + type: api + extension: ${{ inputs.extension }} + extension-file: ${{ env.dev_build }} + wait: ${{ inputs.wait }} + ignore-fail: ${{ inputs.ignore-fail }} + options: ${{ inputs.options }} + + - name: E2E test + id: e2e + if: ${{ inputs.test-e2e == 'true' }} + uses: woocommerce/grow/run-qit-annotate@add/qit-workflows-test-build + with: + type: e2e + extension: ${{ inputs.extension }} + extension-file: ${{ env.dev_build }} + wait: ${{ inputs.wait }} + ignore-fail: ${{ inputs.ignore-fail }} + options: ${{ inputs.options }}