Merge pull request #81 from woocommerce/add/qit-workflows

.github/workflows/run-qit-all.yml

@@ -0,0 +1,85 @@
+name: Run QIT for all extensions
+
+# **What it does**: Runs a suite of tests for all Grow extensions.
+# **Why we have it**: To be able to run tests for all extensions at once. For example when we want to compatibility test a new version of the platform (WP/WC).
+
+on:
+  workflow_dispatch:
+    inputs:
+      # Basic params.
+      version:
+        description: 'Version to be tested'
+        required: true
+        type: choice
+        options:
+        - latest
+        - dev
+      wait:
+        description: 'Should wait for results'
+        default: false
+        type: boolean
+
+      # Configure which tests to run.
+      test-activation:
+        description: 'Should activation be tested?'
+        required: true
+        default: true
+        type: boolean
+      test-security:
+        description: 'Should security be tested?'
+        required: true
+        default: true
+        type: boolean
+      test-phpstan:
+        description: 'Should phpstan be tested?'
+        required: true
+        default: true
+        type: boolean
+      test-api:
+        description: 'Should API be tested?'
+        required: true
+        default: true
+        type: boolean
+      test-e2e:
+        description: 'Should E2E be tested? (takes a lot of time)'
+        required: true
+        default: false
+        type: boolean
+
+      # Advanced customization.
+      ignore-fail:
+        description: Should pass even if any awaited test fails.
+        required: false
+        default: false
+        type: boolean
+      options:
+        description: 'Additional options for `qit` command, like `--optional_features=hpos`.'
+        required: false
+
+
+jobs:
+    qit-tests:
+        name: Run QIT Tests
+        runs-on: ubuntu-latest
+        strategy:
+          # Allow to test extensions even if one of them fails.
+          fail-fast: false
+          matrix:
+            # List of extensions to be tested.
+            extension: [automatewoo, automatewoo-birthdays, automatewoo-referrals, google-listings-and-ads, woocommerce-google-analytics-integration]
+        steps:
+          - name: Run QIT
+            uses: woocommerce/grow/run-qit-extension@add/qit-workflows-test-build
+            with:
+              qit-partner-user: ${{ secrets.QIT_PARTNER_USER }}
+              qit-partner-secret: ${{ secrets.QIT_PARTNER_SECRET }}
+              version: ${{ inputs.version }}
+              wait: ${{ inputs.wait }}
+              test-activation: ${{ inputs.test-activation }}
+              test-security: ${{ inputs.test-security }}
+              test-phpstan: ${{ inputs.test-phpstan }}
+              test-api: ${{ inputs.test-api }}
+              test-e2e: ${{ inputs.test-e2e }}
+              extension: ${{ matrix.extension }}
+              ignore-fail: ${{ inputs.ignore-fail }}
+              options: ${{ inputs.options }}

packages/github-actions/README.md

@@ -20,6 +20,8 @@ Custom GitHub actions that help to composite GitHub workflows across the repos m
 - [`prepare-node`](actions/prepare-node) - Set up Node.js with a specific version, load npm cache, install Node dependencies
 - [`prepare-php`](actions/prepare-php) - Set up PHP with a specific version and tools, load Composer cache, install Composer dependencies
 - [`publish-extension-dev-build`](actions/publish-extension-dev-build) - Publish extension development build
+- [`run-qit-annotate`](actions/run-qit-annotate) - Runs QIT test and annotates the results
+- [`run-qit-extension`](actions/run-qit-extension) - Run QIT tests for a given extension
 - [`stylelint-annotation`](actions/stylelint-annotation) - Annotate stylelint results via stylelint formatter
 - [`update-version-tags`](actions/update-version-tags) - Update version tags
 

packages/github-actions/actions/run-qit-annotate/README.md

@@ -0,0 +1,50 @@
+# Run QIT test
+
+This action provides the following functionality for GitHub Actions users:
+
+- Run `qit` test of given type for a given extension
+- Annotate the results
+
+
+## Usage
+
+See [action.yml](action.yml)
+
+### Prerequisites
+
+- QIT needs to be [installed](https://woocommerce.github.io/qit-documentation/#/cli/getting-started?id=installing-qit) and [authenticated](https://woocommerce.github.io/qit-documentation/#/authenticating?id=cli)
+   - To set up QIT, you need to set `QIT_DISABLE_ONBOARDING` env to `yes`.
+
+
+### Basic:
+
+```yaml
+jobs:
+  qit-test:
+    name: Run QIT Tests
+    runs-on: ubuntu-20.04
+    env:
+      QIT_DISABLE_ONBOARDING: yes
+    steps:
+      - name: Install QIT via composer
+        run: composer require woocommerce/qit-cli
+
+      - name: Add Partner
+        run: |
+          ./vendor/bin/qit partner:add \
+            --user='${{ secrets.QIT_PARTNER_USER }}' \
+            --qit_token='${{ secrets.QIT_PARTNER_SECRET }}'
+
+      - name: Security test
+        id: security-test
+        uses: woocommerce/grow/run-qit-annotate@actions-v1
+        timeout-minutes: 5
+        with:
+          type: security
+          extension: automatewoo
+          options: '--optional_features=hpos'
+
+      - name: Echo status
+        shell: bash
+        run: echo ${{ jobs.security-test.outputs.status }}
+```

packages/github-actions/actions/run-qit-annotate/action.yml

@@ -0,0 +1,100 @@
+name: Run QIT test
+description: Runs QIT test and annotates the results.
+
+# **What it does**: Runs a single QIT test and annotates the results.
+# **Why we have it**: To reuse across other workflows to test extensions.
+
+inputs:
+  # Basic params.
+  extension:
+    description: Extension to test
+    required: true
+  extension-file:
+    description: Custom build of the extension to test. If not given, the latest release will be used.
+    required: false
+  ignore-fail:
+    description: Should pass even if the awaited test fails.
+    required: false
+    default: false
+  type:
+    description: Type of test to run
+    required: false
+    default: activation
+  wait:
+    description: Wait for the test to finish
+    required: false
+    default: false
+  # Advanced customization.
+  options:
+    description: Options to pass to the qit command
+    required: false
+    default: ''
+
+outputs:
+  status:
+    description: "Status of the test run. May be used, for example, to fail the workflow if the test fails."
+    value: ${{ steps.read-summary.outputs.status }}
+  summary:
+    description: "Short summary of the test"
+    value: ${{ steps.read-summary.outputs.summary }}
+  test_run_id:
+    description: "ID of the QIT test run"
+    value: ${{ steps.read-summary.outputs.test_run_id }}
+
+branding:
+  icon: 'award'
+  color: 'purple'
+
+runs:
+  using: composite
+  steps:
+    # Run the qit command.
+    - name: Run test
+      id: run-test
+      # Do not fail when the `qit` fails, so we can annotate the results.
+      shell: bash --noprofile --norc {0}
+      env:
+        # If the custom build is provided, pass it to the `zip` param.
+        zip: ${{ inputs.extension-file &&  format('--zip={0}', inputs.extension-file) || '' }}
+        wait: ${{ inputs.wait == 'true' && '--wait' || '' }}
+        ignore_fail: ${{ inputs.ignore-fail == 'true' && '--ignore-fail' || '' }}
+      run: |
+        json=`./vendor/bin/qit run:${{ inputs.type }} \
+         ${{ inputs.extension }} \
+         $zip \
+         ${{ inputs.options }} \
+         $wait \
+         $ignore_fail \
+         -n \
+         --json`
+        echo "exitcode=${PIPESTATUS[0]}" >> "$GITHUB_OUTPUT"
+        json=$(echo $json | jq '. | {test_run_id, status, test_summary}')
+        echo $json | jq
+        echo "result="$json"" >> "$GITHUB_OUTPUT"
+
+    # Parse the result json to populate GHA outputs.
+    - name: Read summary
+      id: read-summary
+      shell: bash
+      run: |
+        echo "test_run_id=${{fromJson(steps.run-test.outputs.result).test_run_id}}" >> $GITHUB_OUTPUT
+        echo "status=${{fromJson(steps.run-test.outputs.result).status}}" >> $GITHUB_OUTPUT
+        echo "summary=${{fromJson(steps.run-test.outputs.result).test_summary}}" >> $GITHUB_OUTPUT
+
+
+    # Annotate the results according to the status, forward qit exit code.
+    - name: Annotate and exit
+      shell: bash
+      run: |
+        summary="${{ inputs.type }} (${{ steps.read-summary.outputs.test_run_id }}): ${{ steps.read-summary.outputs.status }} - ${{ steps.read-summary.outputs.summary }} \`qit get ${{ steps.read-summary.outputs.test_run_id }}\`";
+        case ${{ steps.read-summary.outputs.status }} in
+          "success") echo "::notice ::$summary"
+          ;;
+          "pending") echo "::notice ::$summary"
+          ;;
+          "warning") echo "::warning ::$summary"
+          ;;
+          *) echo "::error ::$summary"
+          ;;
+        esac
+        exit ${{ steps.run-test.outputs.exitcode }}

packages/github-actions/actions/run-qit-extension/README.md

@@ -0,0 +1,34 @@
+# Run QIT tests
+
+This action provides the following functionality for GitHub Actions users:
+
+- Install QIT-CLI
+- Run a set of `qit` tests for a given extension
+- Annotate the results
+
+
+## Usage
+
+See [action.yml](action.yml)
+
+### Prerequisites
+
+- You need to obtain valid [authentication keys](https://woocommerce.github.io/qit-documentation/#/authenticating?id=cli).
+
+
+### Basic:
+
+```yaml
+jobs:
+  qit-test:
+    name: Run QIT Tests
+    runs-on: ubuntu-20.04
+    steps:    
+      - name: Delegate QIT Tests
+        uses: woocommerce/grow/run-qit-extension@actions-v1
+        with:
+          qit-partner-user: ${{ secrets.QIT_PARTNER_USER }}
+          qit-partner-secret: ${{ secrets.QIT_PARTNER_SECRET }}
+          extension: 'my-extension'
+          test-activation: true
+```