From dabc79ce25f8ac8353608ba7b674aac1ae5cd6d4 Mon Sep 17 00:00:00 2001
From: Oscar Vestlie <oxv@google.com>
Date: Tue, 1 Aug 2023 16:20:45 -0700
Subject: [PATCH] Move coverage to its own workflow (#1063) (#509)

* Move coverage to its own workflow (#1063)

The coverage job does not support sharding and takes ~1h to run. That's
~40 minutes longer than all other linux test runs. By breaking out
coverage from the linux workflow we can get unit test results from
datadog in less than half the time.

b/290997541

* Upload unit test reports to DataDog (#1059)

Unit tests produce test results in JUnit xml format. These files can be
ingested by DataDog to fuel their CI analysis, unlocking flaky test
detection, regressions, platform analysis, and more!

The upload must happen in a separately triggered workflow to have access
to upload the reports. In a forked PR workflow this access is
restricted.

Initially reports from unit tests run on-host from linux, evergreen, and
win32 will be uploaded.

b/290997541
---
 .github/actions/on_host_test/action.yaml | 34 ++++++++++++++----------
 .github/workflows/coverage.yaml          | 30 +++++++++++++++++++++
 .github/workflows/linux.yaml             | 14 ----------
 .github/workflows/main.yaml              | 20 --------------
 starboard/tools/testing/test_runner.py   |  2 +-
 5 files changed, 51 insertions(+), 49 deletions(-)
 create mode 100644 .github/workflows/coverage.yaml

diff --git a/.github/actions/on_host_test/action.yaml b/.github/actions/on_host_test/action.yaml
index 7df5e2f62e048..bdb53c6d8e6aa 100644
--- a/.github/actions/on_host_test/action.yaml
+++ b/.github/actions/on_host_test/action.yaml
@@ -60,12 +60,15 @@ runs:
       run: |
         echo "PYTHONPATH=$GITHUB_WORKSPACE" >> $GITHUB_ENV
         echo "TEST_RESULTS_DIR=${GITHUB_WORKSPACE}/unit-test-results" >> $GITHUB_ENV
+        echo "XML_FILES_DIR=${GITHUB_WORKSPACE}/unit-test-results/${{ matrix.platform }}/${{ matrix.shard }}" >> $GITHUB_ENV
         echo "COVERAGE_DIR=${GITHUB_WORKSPACE}/coverage" >> $GITHUB_ENV
-        echo "TEST_REPORT_FILE=${GITHUB_WORKSPACE}/${{matrix.platform}}-${{matrix.shard}}" >> $GITHUB_ENV
     - name: Run Tests
+      id: run-tests
       shell: bash
       run: |
         set -x
+        test_type=""
+
         # Starboard toolchains are downloaded to a different dir on github. Create a symlink to reassure our tooling that everything is fine.
         if [ -d /root/starboard-toolchains ]; then
           ln -s /root/starboard-toolchains /github/home/starboard-toolchains
@@ -85,28 +88,31 @@ runs:
         elif [[ "${{matrix.shard}}" == 'evergreen-as-blackbox' ]]; then
           xvfb-run -a --server-args="-screen 0 1920x1080x24i +render +extension GLX -noreset" python3 $GITHUB_WORKSPACE/cobalt/black_box_tests/black_box_tests.py --platform ${{matrix.target_platform}} --config ${{matrix.config}} ${loader_args} --test_set evergreen
         elif [[ "${{matrix.shard}}" == 'coverage' ]]; then
-          xvfb-run -a --server-args="-screen 0 1920x1080x24i +render +extension GLX -noreset" python3 ${GITHUB_WORKSPACE}/starboard/tools/testing/test_runner.py --platform ${{matrix.target_platform}} --config ${{matrix.config}} -r ${loader_args} --xml_output_dir=${TEST_RESULTS_DIR} --coverage_dir=${COVERAGE_DIR} --coverage_report
+          xvfb-run -a --server-args="-screen 0 1920x1080x24i +render +extension GLX -noreset" python3 ${GITHUB_WORKSPACE}/starboard/tools/testing/test_runner.py --platform ${{matrix.target_platform}} --config ${{matrix.config}} -r ${loader_args} --xml_output_dir=${XML_FILES_DIR} --coverage_dir=${COVERAGE_DIR} --coverage_report
         else
+          test_type=unit_tests
           if [[ "${{inputs.os}}" == 'windows' ]]; then
-            python3 ${GITHUB_WORKSPACE}/starboard/tools/testing/test_runner.py --platform ${{matrix.target_platform}} --config ${{matrix.config}} -s ${{matrix.shard}} -r
+            python3 ${GITHUB_WORKSPACE}/starboard/tools/testing/test_runner.py --platform ${{matrix.target_platform}} --config ${{matrix.config}} -s ${{matrix.shard}} -r --xml_output_dir=${XML_FILES_DIR}
           else
-            xvfb-run -a --server-args="-screen 0 1920x1080x24i +render +extension GLX -noreset" python3 ${GITHUB_WORKSPACE}/starboard/tools/testing/test_runner.py --platform ${{matrix.target_platform}} --config ${{matrix.config}} -s ${{matrix.shard}} -r ${loader_args} --xml_output_dir=${TEST_RESULTS_DIR}
+            xvfb-run -a --server-args="-screen 0 1920x1080x24i +render +extension GLX -noreset" python3 ${GITHUB_WORKSPACE}/starboard/tools/testing/test_runner.py --platform ${{matrix.target_platform}} --config ${{matrix.config}} -s ${{matrix.shard}} -r ${loader_args} --xml_output_dir=${XML_FILES_DIR}
           fi
         fi
-    - name: Process unit test results
-      if: failure()
+        echo "test_type=$test_type" >> $GITHUB_OUTPUT
+    - name: Populate TAGS for unit test report
+      if: ${{ steps.run-tests.outputs.test_type == 'unit_tests' }}
       shell: bash
       run: |
-        set -x
-        echo "Saving unit test report to ${TEST_REPORT_FILE}"
-        python3 ${GITHUB_WORKSPACE}/starboard/tools/testing/test_report_parser.py ${TEST_RESULTS_DIR} > ${TEST_REPORT_FILE}
-    - name: Upload unit test report
+        # Set tags for test differentiation.
+        tags="platform:${{ matrix.platform }}"
+        tags="${tags},os:${{ inputs.os }}"
+        echo $tags > ${TEST_RESULTS_DIR}/${{ matrix.platform }}/TAGS
+    - name: Archive unit test results
       uses: actions/upload-artifact@v3
-      if: failure()
+      if: ${{ steps.run-tests.outputs.test_type == 'unit_tests' }}
       with:
-        name: unit-test-reports
-        path: ${{env.TEST_REPORT_FILE}}
-    - name: Upload coverage html report
+        name: unit-test-results
+        path: ${{env.TEST_RESULTS_DIR}}/
+    - name: Archive coverage html report
       if: success() && matrix.shard == 'coverage'
       uses: actions/upload-artifact@v3
       with:
diff --git a/.github/workflows/coverage.yaml b/.github/workflows/coverage.yaml
new file mode 100644
index 0000000000000..834696dcd1123
--- /dev/null
+++ b/.github/workflows/coverage.yaml
@@ -0,0 +1,30 @@
+name: coverage
+
+on:
+  pull_request:
+    types:
+      - opened
+      - edited
+      - reopened
+      - synchronize
+  push:
+    branches:
+      - main
+      - feature/*
+
+jobs:
+  linux-coverage:
+    # TODO(b/286302961): Move back and re-enable this check once sharding is supported.
+    # Run on main branch for pushes, PRs and manual invocations.
+    # if: |
+    #  ${{ github.ref == 'refs/heads/main' &&
+    #      (github.event_name == 'push' ||
+    #      github.event_name == 'pull_request' ||
+    #      (github.event_name == 'workflow_dispatch' && inputs.nightly == 'false')) }}
+    uses: ./.github/workflows/main.yaml
+    permissions:
+      packages: write
+      pull-requests: write
+    with:
+      platform: linux-coverage
+      nightly: ${{ github.event.inputs.nightly }}
diff --git a/.github/workflows/linux.yaml b/.github/workflows/linux.yaml
index e15041dd6e616..bff0dffb3b1f1 100644
--- a/.github/workflows/linux.yaml
+++ b/.github/workflows/linux.yaml
@@ -56,17 +56,3 @@ jobs:
       platform: linux-modular
       nightly: ${{ github.event.inputs.nightly }}
       modular: true
-  linux-coverage:
-    # Run on main branch for pushes, PRs and manual invocations.
-    if: |
-     ${{ github.ref == 'refs/heads/main' &&
-         (github.event_name == 'push' ||
-         github.event_name == 'pull_request' ||
-         (github.event_name == 'workflow_dispatch' && inputs.nightly == 'false')) }}
-    uses: ./.github/workflows/main.yaml
-    permissions:
-      packages: write
-      pull-requests: write
-    with:
-      platform: linux-coverage
-      nightly: ${{ github.event.inputs.nightly }}
diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml
index b501c1d822196..e0cd9b58a3cd7 100644
--- a/.github/workflows/main.yaml
+++ b/.github/workflows/main.yaml
@@ -320,23 +320,3 @@ jobs:
         uses: ./.github/actions/on_host_test
         with:
           os: linux
-
-  # Gets unit test report from on host tests and prints it.
-  on-host-unit-test-report:
-    needs: [on-host-test]
-    permissions: {}
-    if: failure()
-    runs-on: ubuntu-latest
-    steps:
-      - name: Collect Unit Test Reports
-        uses: actions/download-artifact@v3
-        with:
-          name: unit-test-reports
-          path: unit-test-reports
-      - name: Print Unit Test Reports
-        run: |
-          for filename in ${GITHUB_WORKSPACE}/unit-test-reports/*; do
-            basename $filename
-            cat $filename
-            echo
-          done
diff --git a/starboard/tools/testing/test_runner.py b/starboard/tools/testing/test_runner.py
index 97addf1677012..75f1287518bd8 100755
--- a/starboard/tools/testing/test_runner.py
+++ b/starboard/tools/testing/test_runner.py
@@ -477,7 +477,7 @@ def MakeLauncher():
       logging.info(("Xml results for this test will "
                     "be logged to '%s'."), test_result_xml_path)
     elif self.xml_output_dir:
-      xml_output_subdir = os.path.join(self.xml_output_dir, target_name)
+      xml_output_subdir = os.path.join(self.xml_output_dir)
       try:
         os.makedirs(xml_output_subdir)
       except OSError as ose: