integration_tests.yml

name: Integration tests

on:

  workflow_dispatch:
    inputs:
      packaged_sdk_run_id:
        description: 'run # of "Firebase Unity SDK build" workflow. (If Leave it empty, the latest released Unity SDK will be used.)'
      unity_versions:
        description: 'Unity version (value: 2021, 2022. separated by commas)'
        default: '2021'
        required: true
      build_os:
        description: 'Build OS (value: windows-latest, macos-latest. Left empty will use macos-latest for iOS platform, windows-latest for the rest)'
      platforms:
        description: 'CSV of Android,iOS,Windows,macOS,Linux,Playmode'
        default: 'Android,iOS,tvOS,Windows,macOS,Linux,Playmode'
        required: true
      apis:
        description: 'CSV of apis to build and test'
        default: 'analytics,app_check,auth,crashlytics,database,dynamic_links,firestore,functions,installations,messaging,remote_config,storage'
        required: true
      mobile_test_on:
        description: 'Run mobile tests on real and/or virtual devices? (value: real, virtual. separated by commas)'
        default: 'real'
        required: true
      use_expanded_matrix:
        description: 'Use an expanded matrix? Note: above config will be ignored (unity_versions, build_os, platforms, mobile_test_on).'
        default: '0'
        required: true
      test_pull_request:
        description: 'Optional: Pull request # to build and test? (With optional commit hash, separated by a colon. Specify the FULL hash.)'

env:
  triggerLabelPrefix: "tests-requested: "
  triggerLabelFull: "tests-requested: full"
  triggerLabelQuick: "tests-requested: quick"
  pythonVersion: '3.8'
  artifactRetentionDays: 2

jobs:
  check_and_prepare:
    runs-on: ubuntu-latest
    outputs:
      trigger: ${{ steps.set_outputs.outputs.trigger }}
      github_ref: ${{ steps.set_outputs.outputs.github_ref }}
      pr_number: ${{ steps.set_outputs.outputs.pr_number }}
      apis: ${{ steps.matrix_config.outputs.apis }}
      build_matrix: ${{ steps.matrix_config.outputs.build_matrix }}
      test_matrix: ${{ steps.matrix_config.outputs.test_matrix }}
      playmode_matrix: ${{ steps.matrix_config.outputs.playmode_matrix }}
    steps:
    ### Fail the workflow if the user does not have admin access to run the tests.
    - name: Check if user has permission to trigger tests
      uses: lannonbr/repo-permission-check-action@2.0.0
      with:
        permission: "admin"
      env:
        GITHUB_TOKEN: ${{ github.token }}
    ### It sets "github_ref,trigger,pr_number,requested_tests" outputs to control the following jobs and steps
    ### trigger value: manual_trigger, scheduled_trigger, label_trigger, postsubmit_trigger
    - id: set_outputs
      run: |
        if [[ "${{ github.event.inputs.packaged_sdk_run_id }}" != "" ]]; then
          echo "::warning ::Downloading SDK package from previous run: https://github.com/${{github.repository}}/actions/runs/${{ github.event.inputs.packaged_sdk_run_id }}"
        fi
        if [[ "${{ github.event.inputs.test_pull_request }}" == *"label-quick-packaging" ]]; then
          # Triggered by build_starter SDK workflow.
          echo "trigger=label_trigger" >> $GITHUB_OUTPUT
          echo "requested_tests=auto" >> $GITHUB_OUTPUT
          echo "github_ref=${{github.sha}}" >> $GITHUB_OUTPUT
          echo "pr_number=$(echo ${{ github.event.inputs.test_pull_request }} | cut -d: -f1)" >> $GITHUB_OUTPUT
        elif [[ "${{ github.event.inputs.test_pull_request }}" == *"label-full-packaging" ]]; then
          # Triggered by build_starter SDK workflow.
          echo "trigger=label_trigger" >> $GITHUB_OUTPUT
          echo "requested_tests=expanded" >> $GITHUB_OUTPUT
          echo "github_ref=${{github.sha}}" >> $GITHUB_OUTPUT
          echo "pr_number=$(echo ${{ github.event.inputs.test_pull_request }} | cut -d: -f1)" >> $GITHUB_OUTPUT
        elif [[ "${{ github.event.inputs.test_pull_request }}" == *"pr-close-packaging" ]]; then
          # Triggered by build_starter SDK workflow.
          echo "trigger=postsubmit_trigger" >> $GITHUB_OUTPUT
          echo "requested_tests=auto" >> $GITHUB_OUTPUT
          echo "github_ref=${{github.sha}}" >> $GITHUB_OUTPUT
          echo "pr_number=$(echo ${{ github.event.inputs.test_pull_request }} | cut -d: -f1)" >> $GITHUB_OUTPUT
        elif [[ "${{ github.event.inputs.test_pull_request }}" == "nightly-packaging" ]]; then
          # Triggered by build_starter SDK workflow.
          echo "trigger=scheduled_trigger" >> $GITHUB_OUTPUT
          echo "requested_tests=expanded" >> $GITHUB_OUTPUT
          echo "github_ref=${{github.sha}}" >> $GITHUB_OUTPUT
        elif [[ "${{ github.event.inputs.test_pull_request }}" == "1" ]]; then
          # Triggered by build_starter SDK workflow.
          echo "trigger=manual_trigger" >> $GITHUB_OUTPUT
          echo "requested_tests=auto" >> $GITHUB_OUTPUT
          echo "github_ref=${{github.sha}}" >> $GITHUB_OUTPUT
        else
          # Triggered manually
          echo "trigger=manual_trigger" >> $GITHUB_OUTPUT
          if [[ "${{ github.event.inputs.use_expanded_matrix }}" == "1" ]]; then
            echo "requested_tests=expanded" >> $GITHUB_OUTPUT
          fi
          if [[ -z "${{github.event.inputs.test_pull_request}}" ]]; then
            # test_pull_request not specified
            echo "github_ref=${{github.sha}}" >> $GITHUB_OUTPUT
          elif [[ "${{github.event.inputs.test_pull_request}}" == *:* ]]; then
            # If specified as pr:commit_hash, split them.
            echo "github_ref=$(echo ${{ github.event.inputs.test_pull_request }} | cut -d: -f2)" >> $GITHUB_OUTPUT
            echo "pr_number=$(echo ${{ github.event.inputs.test_pull_request }} | cut -d: -f1)" >> $GITHUB_OUTPUT
          else
            # Just the PR specified, use refs/pull/<number>/merge as the ref.
            echo "github_ref=refs/pull/${{github.event.inputs.test_pull_request}}/merge" >> $GITHUB_OUTPUT
            echo "pr_number=${{ github.event.inputs.test_pull_request }}" >> $GITHUB_OUTPUT
          fi
        fi
    - uses: actions/checkout@v3
      with:
        ref: ${{steps.set_outputs.outputs.github_ref}}
        fetch-depth: 0
        submodules: false
    - name: Setup python
      uses: actions/setup-python@v4
      with:
        python-version: ${{ env.pythonVersion }}
    - name: Install python deps
      run: pip install -r scripts/gha/requirements.txt
    - id: matrix_config
      env:
        HEAD_REF: ${{github.event.pull_request.head.ref}}
        BASE_REF: ${{github.event.pull_request.base.ref}}
      run: |
        # Log that we ran against a given packaged SDK.
        if [[ -n "${{ github.event.inputs.packaged_sdk_run_id }}" ]]; then
          echo "::warning ::Downloading package from previous run: https://github.com/${{github.repository}}/actions/runs/${{ github.event.inputs.packaged_sdk_run_id }}"
        fi
        if [[ "${{ steps.set_outputs.outputs.requested_tests }}" == "expanded" ]]; then 
          TEST_MATRIX_PARAM=-m=expanded
          echo "::warning ::Running on the expanded matrix"
        elif [[ "${{ steps.set_outputs.outputs.requested_tests }}" == "minimal" ]]; then 
          TEST_MATRIX_PARAM=-m=minimal
          echo "::warning ::Running on the minimal matrix"
        elif [[ "${{ steps.set_outputs.outputs.requested_tests }}" == "auto" ]]; then 
          # auto-diff only apply when running in a PR. 
          # diff against the PR's base. "git merge-base main branch_name" will give the common ancestor of both branches.
          MERGE_BASE=$(git merge-base "origin/${HEAD_REF}" "origin/${BASE_REF}" || true)
          # If origin/<branch> is no longer valid, then just run all tests.
          if [[ -n "${MERGE_BASE}" ]]; then
            echo "::warning ::Auto-diff origin/${HEAD_REF}..${MERGE_BASE}"
            git diff --name-only "origin/${HEAD_REF}..${MERGE_BASE}"
            TEST_MATRIX_PARAM="--auto_diff origin/${HEAD_REF}..${MERGE_BASE}"
          fi
        fi
        # To feed input into the job matrix, we first need to convert to a JSON
        # list. Then we can use fromJson to define the field in the matrix for the tests job.
        echo "apis=$( python scripts/gha/print_matrix_configuration.py -c -w integration_tests -k apis -o "${{github.event.inputs.apis}}" ${TEST_MATRIX_PARAM} )" >> $GITHUB_OUTPUT
        echo "build_matrix=$( python scripts/gha/print_matrix_configuration.py -build_matrix -unity_versions "${{github.event.inputs.unity_versions}}" -platforms "${{github.event.inputs.platforms}}" -os "${{github.event.inputs.build_os}}" -mobile_test_on "${{github.event.inputs.mobile_test_on}}" ${TEST_MATRIX_PARAM})" >> $GITHUB_OUTPUT
        echo "test_matrix=$( python scripts/gha/print_matrix_configuration.py -test_matrix -unity_versions "${{github.event.inputs.unity_versions}}" -platforms "${{github.event.inputs.platforms}}" -os "${{github.event.inputs.build_os}}" -mobile_test_on "${{github.event.inputs.mobile_test_on}}" ${TEST_MATRIX_PARAM})" >> $GITHUB_OUTPUT
        echo "playmode_matrix=$( python scripts/gha/print_matrix_configuration.py -playmode_matrix -unity_versions "${{github.event.inputs.unity_versions}}" -platforms "${{github.event.inputs.platforms}}" -os "${{github.event.inputs.build_os}}" ${TEST_MATRIX_PARAM})" >> $GITHUB_OUTPUT
    - name: Update PR label and comment
      if: steps.set_outputs.outputs.pr_number
      run: |
        #Add the in-progress label and remove any previous labels.
        python scripts/gha/it_workflow.py --stage start \
          --token ${{github.token}} \
          --issue_number ${{steps.set_outputs.outputs.pr_number}} \
          --actor ${{github.actor}} \
          --commit ${{steps.set_outputs.outputs.github_ref}} \
          --run_id ${{github.run_id}}

  build_testapp:
    name: build-${{ matrix.unity_version }}-${{matrix.os}}-${{ matrix.platform }}-${{ matrix.ios_sdk }}
    runs-on: ${{matrix.os}}
    needs: check_and_prepare
    if: ${{ needs.check_and_prepare.outputs.build_matrix != '' && !cancelled() }}
    strategy:
      fail-fast: false
      matrix: ${{ fromJson(needs.check_and_prepare.outputs.build_matrix) }}
    env:
      xcodeVersion: "15.1"
    steps:
      - id: matrix_info
        shell: bash
        run: |
          echo "info=${{ matrix.unity_version }}-${{matrix.os}}-${{ matrix.platform }}-${{ matrix.ios_sdk }}" >> $GITHUB_OUTPUT
          echo "artifact_suffix=${{ matrix.unity_version }}-${{matrix.os}}-${{ matrix.ios_sdk }}" >> $GITHUB_OUTPUT
      - uses: actions/checkout@v3
      - name: Setup python
        uses: actions/setup-python@v4
        with:
          python-version: '3.8'
      - name: Install python deps
        timeout-minutes: 10
        shell: bash
        run: |
          pip install -r scripts/gha/requirements.txt
      - name: setup Xcode version
        if: runner.os == 'macOS'
        run: sudo xcode-select -s /Applications/Xcode_${{ env.xcodeVersion }}.app/Contents/Developer
      - id: unity_setup
        uses: ./gha/unity
        timeout-minutes: 30
        with:
          version: ${{ matrix.unity_version }}
          platforms: ${{ matrix.platform }}
          username: ${{ secrets.UNITY_USERNAME }}
          password: ${{ secrets.UNITY_PASSWORD }}
          serial_ids: ${{ secrets.SERIAL_ID }}
      - name: Workaround tvOS XCode 15 issue
        if: ${{ contains(matrix.platform, 'tvOS') && contains(matrix.unity_version, '2020') }}
        shell: bash
        run: |
          find /Applications/Unity/Hub/Editor -type f -name 'UnityViewControllerBase.h' -exec sed -i '' 's/#import <GameController\/GCController.h>/#import <GameController\/GCEventViewController.h>/g' {} \;
      - name: Prepare for integration tests
        timeout-minutes: 10
        shell: bash
        run: |
          python scripts/gha/restore_secrets.py --passphrase "${{ secrets.TEST_SECRET }}"
      - name: Setup Android keystore
        if: ${{ contains('Android', matrix.platform) }}
        shell: bash
        run: |
          # The android directory might not exist, so make it
          mkdir -p ~/.android
          # Restore the secret debug keystore
          gpg --quiet --batch --yes --decrypt --passphrase="${{ secrets.TEST_SECRET }}" \
              --output ~/.android/debug.keystore "scripts/gha-encrypted/debug_keystore.gpg"
      - name: Fetch prebuilt packaged SDK from previous run
        if: ${{ github.event.inputs.packaged_sdk_run_id != '' }}
        uses: actions/download-artifact@v4
        with:
          name: 'firebase_unity_sdk.zip'
          github-token: ${{ github.token }}
          run-id: ${{ github.event.inputs.packaged_sdk_run_id }}

      - name: Build integration tests
        timeout-minutes: 240
        shell: bash
        run: |
          if [[ -n "${{ github.event.inputs.packaged_sdk_run_id }}" ]]; then
            unzip -q firebase_unity_sdk.zip -d ~/Downloads/
          else
            curl -L "https://firebase.google.com/download/unity" -o ~/Downloads/firebase_unity_sdk.zip
            unzip -q ~/Downloads/firebase_unity_sdk.zip -d ~/Downloads/
          fi
          python scripts/gha/build_testapps.py \
            --t ${{ needs.check_and_prepare.outputs.apis }} \
            --u ${{ env.UNITY_VERSION }} \
            --p "${{ matrix.platform }}" \
            --ios_sdk "${{ matrix.ios_sdk }}" \
            --plugin_dir ~/Downloads/firebase_unity_sdk \
            --output_directory "${{ github.workspace }}" \
            --artifact_name "${{ steps.matrix_info.outputs.info }}" \
            --notimestamp \
            --force_latest_runtime \
            --ci
      - name: Return Unity license
        # Always returns true, even when job failed or canceled. But will not run when a critical failure prevents the task from running. 
        if: always()
        uses: ./gha/unity
        with:
          version: ${{ matrix.unity_version }}
          release_license: "true"
      - name: Prepare results summary artifact
        if: ${{ !cancelled() }}
        shell: bash
        run: |
          if [ ! -f build-results-${{ steps.matrix_info.outputs.info }}.log.json ]; then
            # No summary was created, make a placeholder one.
            echo "__SUMMARY_MISSING__" > build-results-${{ steps.matrix_info.outputs.info }}.log.json
          fi
      - name: Upload build results artifact
        uses: actions/upload-artifact@v4
        if: ${{ !cancelled() }}
        with:
          name: build_and_test_results-${{ matrix.platform }}-${{ matrix.ios_sdk }}
          path: build-results-${{ steps.matrix_info.outputs.info }}*
          retention-days: ${{ env.artifactRetentionDays }}
      - name: Upload Mobile integration tests artifact
        uses: actions/upload-artifact@v4
        if: ${{ contains('Android,iOS,tvOS', matrix.platform) && !cancelled() }}
        with:
          name: testapps-${{ matrix.platform }}-${{ steps.matrix_info.outputs.artifact_suffix }}
          path: testapps-${{ steps.matrix_info.outputs.info }}
          retention-days: ${{ env.artifactRetentionDays }}
      - name: Delete Mobile integration tests artifact
        if: ${{ contains('Android,iOS,tvOS', matrix.platform) && !cancelled() }}
        shell: bash
        run: rm -rf testapps-${{ steps.matrix_info.outputs.info }} || true
      - name: Upload Desktop build logs artifact  # Mobile build logs are uploaded with integration tests artifact
        uses: actions/upload-artifact@v4
        if: ${{ !contains('Android,iOS,tvOS', matrix.platform) && !cancelled() }}
        with:
          name: testapps-build-logs-${{ steps.matrix_info.outputs.info }}
          path: testapps-${{ steps.matrix_info.outputs.info }}/build-logs
          retention-days: ${{ env.artifactRetentionDays }}
      - name: Upload Linux integration tests artifact
        uses: actions/upload-artifact@v4
        if: ${{ contains(matrix.platform, 'Linux') && !cancelled() }}
        with:
          name: testapps-Linux-${{ steps.matrix_info.outputs.artifact_suffix }}
          path: testapps-${{ steps.matrix_info.outputs.info }}/Linux
          retention-days: ${{ env.artifactRetentionDays }}
      - name: Delete Linux integration tests artifact
        if: ${{ contains(matrix.platform, 'Linux') && !cancelled() }}
        shell: bash
        run: rm -rf testapps-${{ steps.matrix_info.outputs.info }}/Linux || true
      - name: Upload macOS integration tests artifact
        uses: actions/upload-artifact@v4
        if: ${{ contains(matrix.platform, 'macOS') && !cancelled() }}
        with:
          name: testapps-macOS-${{ steps.matrix_info.outputs.artifact_suffix }}
          path: testapps-${{ steps.matrix_info.outputs.info }}/macOS
          retention-days: ${{ env.artifactRetentionDays }}
      - name: Delete macOS integration tests artifact
        if: ${{ contains(matrix.platform, 'macOS') && !cancelled() }}
        shell: bash
        run: rm -rf testapps-${{ steps.matrix_info.outputs.info }}/macOS || true
      - name: Upload Windows integration tests artifact
        uses: actions/upload-artifact@v4
        if: ${{ contains(matrix.platform, 'Windows') && !cancelled() }}
        with:
          name: testapps-Windows-${{ steps.matrix_info.outputs.artifact_suffix }}
          path: testapps-${{ steps.matrix_info.outputs.info }}/Windows
          retention-days: ${{ env.artifactRetentionDays }}
      - name: Delete Windows integration tests artifact
        if: ${{ contains(matrix.platform, 'Windows') && !cancelled() }}
        shell: bash
        run: rm -rf testapps-${{ steps.matrix_info.outputs.info }}/Windows || true
      - name: Download log artifacts
        if: ${{ needs.check_and_prepare.outputs.pr_number && failure() && !cancelled() }}
        uses: actions/download-artifact@v4
        with:
          path: test_results
          name: build_and_test_results-${{ matrix.platform }}-${{ matrix.ios_sdk }}
      - name: Update PR label and comment
        if: ${{ needs.check_and_prepare.outputs.pr_number && failure() && !cancelled() }}
        shell: bash
        run: |
          python scripts/gha/it_workflow.py --stage progress \
            --token ${{github.token}} \
            --issue_number ${{needs.check_and_prepare.outputs.pr_number}}\
            --actor ${{github.actor}} \
            --commit ${{needs.check_and_prepare.outputs.github_ref}} \
            --run_id ${{github.run_id}}
      - name: Summarize build results
        if: ${{ !cancelled() }}
        shell: bash
        run: |
          cat build-results-${{ steps.matrix_info.outputs.info }}.log
          if [[ "${{ job.status }}" != "success" ]]; then
            exit 1
          fi

  playmode_test:
    name: test-${{ matrix.unity_version }}-${{matrix.os}}-Playmode
    runs-on: ${{matrix.os}}
    needs: check_and_prepare
    if: ${{ needs.check_and_prepare.outputs.playmode_matrix != '' && !cancelled() }}
    strategy:
      fail-fast: false
      matrix: ${{ fromJson(needs.check_and_prepare.outputs.playmode_matrix) }}
    steps:
      - id: matrix_info
        shell: bash
        run: |
          echo "info=${{ matrix.unity_version }}-${{matrix.os}}-Playmode-github_runner-${{ matrix.os }}" >> $GITHUB_OUTPUT
      - uses: actions/checkout@v3
      - name: Setup python
        uses: actions/setup-python@v4
        with:
          python-version: '3.8'
      - name: Install python deps
        timeout-minutes: 10
        shell: bash
        run: |
          pip install -r scripts/gha/requirements.txt
      - id: unity_setup
        uses: ./gha/unity
        timeout-minutes: 30
        with:
          version: ${{ matrix.unity_version }}
          platforms: Playmode
          username: ${{ secrets.UNITY_USERNAME }}
          password: ${{ secrets.UNITY_PASSWORD }}
          serial_ids: ${{ secrets.SERIAL_ID }}
      - name: Prepare for integration tests
        timeout-minutes: 10
        shell: bash
        run: |
          python scripts/gha/restore_secrets.py --passphrase "${{ secrets.TEST_SECRET }}"
      - name: Fetch prebuilt packaged SDK from previous run
        if: ${{ github.event.inputs.packaged_sdk_run_id != '' }}
        uses: actions/download-artifact@v4
        with:
          name: 'firebase_unity_sdk.zip'
          github-token: ${{ github.token }}
          run-id: ${{ github.event.inputs.packaged_sdk_run_id }} 
      - name: Set up Node (18)
        uses: actions/setup-node@v3
        with:
          node-version: 18.x
      - name: Setup java for Firestore emulator
        uses: actions/setup-java@v3
        with:
          distribution: 'temurin'
          java-version: '17'
      - name: Setup Firestore Emulator
        uses: nick-invision/retry@v2
        with:
          shell: bash
          timeout_minutes: 5
          max_attempts: 3
          command: npm install -g firebase-tools 
      - name: Run Playmode (in editor mode) integration tests
        shell: bash
        run: |
          firebase emulators:start --only firestore &
          if [[ -n "${{ github.event.inputs.packaged_sdk_run_id }}" ]]; then
            unzip -q firebase_unity_sdk.zip -d ~/Downloads/
          else
            curl -L "https://firebase.google.com/download/unity" -o ~/Downloads/firebase_unity_sdk.zip
            unzip -q ~/Downloads/firebase_unity_sdk.zip -d ~/Downloads/
          fi
          python scripts/gha/build_testapps.py \
            --t ${{ needs.check_and_prepare.outputs.apis }} \
            --u ${{ env.UNITY_VERSION }} \
            --p Playmode \
            --plugin_dir ~/Downloads/firebase_unity_sdk \
            --output_directory "${{ github.workspace }}" \
            --artifact_name "${{ steps.matrix_info.outputs.info }}" \
            --notimestamp \
            --force_latest_runtime \
            --ci
      - name: Return Unity license
        # Always returns true, even when job failed or canceled. But will not run when a critical failure prevents the task from running. 
        if: always()
        uses: ./gha/unity
        with:
          version: ${{ matrix.unity_version }}
          release_license: "true"
      - name: Prepare results summary artifact
        if: ${{ !cancelled() }}
        shell: bash
        run: |
          if [ ! -f test-results-${{ steps.matrix_info.outputs.info }}.log.json ]; then
            # No summary was created, make a placeholder one.
            echo "__SUMMARY_MISSING__" > test-results-${{ steps.matrix_info.outputs.info }}.log.json
          fi
      - name: Upload test results artifact
        uses: actions/upload-artifact@v4
        if: ${{ !cancelled() }}
        with:
          name: build_and_test_results-Playmode
          path: test-results-${{ steps.matrix_info.outputs.info }}*
          retention-days: ${{ env.artifactRetentionDays }}
      - name: Update PR label and comment
        if: ${{ needs.check_and_prepare.outputs.pr_number && failure() && !cancelled() }}
        shell: bash
        run: |
          python scripts/gha/it_workflow.py --stage progress \
            --token ${{github.token}} \
            --issue_number ${{needs.check_and_prepare.outputs.pr_number}}\
            --actor ${{github.actor}} \
            --commit ${{needs.check_and_prepare.outputs.github_ref}} \
            --run_id ${{github.run_id}}
      - name: Summarize test results
        if: ${{ !cancelled() }}
        shell: bash
        run: |
          cat test-results-${{ steps.matrix_info.outputs.info }}.log
          if [[ "${{ job.status }}" != "success" ]]; then
            exit 1
          fi

  integration_test:
    name: test-${{ matrix.unity_version }}-${{matrix.build_os}}-${{ matrix.platform }}-${{ matrix.test_device }}-${{ matrix.test_os }}
    runs-on: ${{ matrix.test_os }}
    if: ${{ needs.check_and_prepare.outputs.build_matrix != '' && needs.check_and_prepare.outputs.test_matrix != '' && !cancelled() }}
    needs: [check_and_prepare, build_testapp]
    strategy:
      fail-fast: false
      matrix: ${{ fromJson(needs.check_and_prepare.outputs.test_matrix) }}
    env:
      xcodeVersion: "15.1"
    steps:
      - id: matrix_info
        shell: bash
        run: |
          echo "info=${{ matrix.unity_version }}-${{matrix.build_os}}-${{ matrix.platform }}-${{ matrix.test_device }}-${{ matrix.test_os }}" >> $GITHUB_OUTPUT
          echo "artifact_path=testapps-${{ matrix.platform }}-${{ matrix.unity_version }}-${{matrix.build_os}}-${{ matrix.ios_sdk }}" >> $GITHUB_OUTPUT
      - uses: actions/checkout@v3
        with:
          ref: ${{needs.check_and_prepare.outputs.github_ref}}
      - name: Setup python
        uses: actions/setup-python@v4
        with:
          python-version: '3.8'
      - name: Install python deps
        timeout-minutes: 10
        shell: bash
        run:  pip install -r scripts/gha/requirements.txt
      - name: setup Xcode version
        if: runner.os == 'macOS'
        run: sudo xcode-select -s /Applications/Xcode_${{ env.xcodeVersion }}.app/Contents/Developer
      - name: Download Testapp artifacts
        uses: actions/download-artifact@v4
        with:
          path: testapps
          name: ${{ steps.matrix_info.outputs.artifact_path }}
      - name: Set up Node (18)
        if: ${{ matrix.test_device == 'github_runner'  }}
        uses: actions/setup-node@v3
        with:
          node-version: 18.x
      - name: Setup java for Firestore emulator
        if: ${{ matrix.test_device == 'github_runner'  }}
        uses: actions/setup-java@v3
        with:
          distribution: 'temurin'
          java-version: '17'
      - name: Setup Firestore Emulator
        if: ${{ matrix.test_device == 'github_runner'  }}
        uses: nick-invision/retry@v2
        with:
          shell: bash
          timeout_minutes: 5
          max_attempts: 3
          command: npm install -g firebase-tools  
      - name: Run Desktop integration tests
        if: ${{ matrix.test_device == 'github_runner'  }}
        timeout-minutes: 30
        shell: bash
        run: |
          firebase emulators:start --only firestore &
          python scripts/gha/desktop_tester.py --testapp_dir testapps \
            --logfile_name "${{ steps.matrix_info.outputs.info }}"
      - name: Run Mobile integration tests on real device via FTL
        id: ftl_test
        if: ${{ matrix.device_type == 'real' }}
        uses: FirebaseExtended/github-actions/firebase-test-lab@v1.4
        with:
          credentials_json: ${{ secrets.FIREBASE_SERVICE_ACCOUNT_CREDENTIALS }}
          testapp_dir: testapps
          test_type: "game-loop"
          test_devices: ${{ matrix.device_detail }}
          test_device_selection: random
          max_attempts: 3
          validator: ${GITHUB_WORKSPACE}/scripts/gha/read_ftl_test_result.py
      - name: Read FTL Test Result
        if: ${{ matrix.device_type == 'real' && !cancelled() }}
        timeout-minutes: 60
        shell: bash
        run: |
          python scripts/gha/read_ftl_test_result.py --test_result '${{ steps.ftl_test.outputs.test_summary }}' \
            --output_path testapps/test-results-"${{ steps.matrix_info.outputs.info }}".log
      - name: Print available devices
        if: ${{ matrix.device_type == 'virtual' && !cancelled() }}
        run: |
          xcrun xctrace list devices
      - name: Create keychain (macOS Simulator)
        if: ${{ contains('iOS,tvOS', matrix.platform) && matrix.device_type == 'virtual'}}
        shell: bash
        run: |
          echo "Creating temporary keychain"
          # Create a local keychain on Mac:
          # Clean up previous temp keychain, if any.
          security delete-keychain tmp-keychain 2> /dev/null || true
          # Create temp keychain file and unlock it.
          # (Avoid passing in -p on command line by using interactive mode.)
          # Also set it to default settings so there is no unlock timeout.
          security -i <<EOF
          create-keychain -p ${{ secrets.TEST_SECRET }} tmp-keychain
          set-keychain-settings tmp-keychain
          unlock-keychain -p ${{ secrets.TEST_SECRET }} tmp-keychain
          EOF
          # Change the keychain list and default keychain to the temp keychain.
          security list-keychains -d user -s tmp-keychain
          security default-keychain -s tmp-keychain
      - name: Run Mobile integration tests on virtual device locally
        timeout-minutes: 120
        if: ${{ matrix.device_type == 'virtual' && !cancelled() }}
        run: |
          python scripts/gha/test_simulator.py --testapp_dir testapps \
            --tvos_device "${{ matrix.test_device }}" \
            --ios_device "${{ matrix.test_device }}" \
            --android_device "${{ matrix.test_device }}" \
            --logfile_name ""${{ steps.matrix_info.outputs.info }}"" \
            --ci
      - name: Delete keychain (macOS Simulator)
        if: ${{ always() && contains('iOS,tvOS', matrix.platform) && matrix.device_type == 'virtual' }}
        shell: bash
        run: |
          # Remove the local keychain on Mac:
          # Set back to the default login keychain.
          security list-keychains -d user -s login.keychain
          # Delete the temp keychain, if it exists.
          security delete-keychain tmp-keychain || true
      - name: Prepare results summary artifact
        if: ${{ !cancelled() }}
        shell: bash
        run: |
          # If testapps do not exist, then it's a build error not test error. 
          if [ -d "testapps/testapps-${{ steps.matrix_info.outputs.info }}" &&  ! -f testapps/test-results-${{ steps.matrix_info.outputs.info }}.log.json ]; then
            mkdir -p testapps && echo "__SUMMARY_MISSING__" > testapps/test-results-${{ steps.matrix_info.outputs.info }}.log.json
          fi
      - name: Upload test results artifact
        if: ${{ !cancelled() }}
        uses: actions/upload-artifact@v4
        with:
          name: build_and_test_results-${{ matrix.platform }}-${{ matrix.test_device }}
          path: testapps/test-results-${{ steps.matrix_info.outputs.info }}*
          retention-days: ${{ env.artifactRetentionDays }}
      - name: Download log artifacts
        if: ${{ needs.check_and_prepare.outputs.pr_number && failure() && !cancelled() }}
        uses: actions/download-artifact@v4
        with:
          path: build_and_test_results
          name: build_and_test_results-${{ matrix.platform }}-${{ matrix.test_device }}
      - name: Update PR label and comment
        shell: bash
        if: ${{ needs.check_and_prepare.outputs.pr_number && failure() && !cancelled() }}
        run: |
          python scripts/gha/it_workflow.py --stage progress \
            --token ${{github.token}} \
            --issue_number ${{needs.check_and_prepare.outputs.pr_number}}\
            --actor ${{github.actor}} \
            --commit ${{needs.check_and_prepare.outputs.github_ref}} \
            --run_id ${{github.run_id}}
      - name: Summarize test results
        if: ${{ !cancelled() }}
        shell: bash
        run: |
          cat testapps/test-results-${{ steps.matrix_info.outputs.info }}.log
          if [[ "${{ job.status }}" != "success" ]]; then
            exit 1
          fi

  summarize_results:
    name: "summarize-results"
    needs: [check_and_prepare, build_testapp, playmode_test, integration_test]
    runs-on: ubuntu-latest
    if: ${{ !cancelled() }}
    steps:
      - uses: actions/checkout@v3
        with:
          ref: ${{needs.check_and_prepare.outputs.github_ref}}
      - name: Setup python
        uses: actions/setup-python@v4
        with:
          python-version: ${{ env.pythonVersion }}
      - name: Install python deps
        run: pip install -r scripts/gha/requirements.txt
      - name: Download log artifacts
        uses: actions/download-artifact@v4
        with:
          path: test_results
          pattern: build_and_test_results-*
          merge-multiple: true
      # Use a different token to remove the "in-progress" label,
      # to allow the removal to trigger the "Check Labels" workflow.
      - name: Generate token for GitHub API
        uses: tibdex/github-app-token@v1
        id: generate-token
        with:
          app_id: ${{ secrets.WORKFLOW_TRIGGER_APP_ID }}
          private_key: ${{ secrets.WORKFLOW_TRIGGER_APP_PRIVATE_KEY }}
      - name: Update PR label and comment
        if: ${{ needs.check_and_prepare.outputs.pr_number }}
        run: |
          python scripts/gha/it_workflow.py --stage end \
            --token ${{github.token}} \
            --issue_number ${{needs.check_and_prepare.outputs.pr_number}}\
            --actor ${{github.actor}} \
            --commit ${{needs.check_and_prepare.outputs.github_ref}} \
            --run_id ${{github.run_id}} \
            --new_token ${{steps.generate-token.outputs.token}}
      - name: Update Daily Report
        if: needs.check_and_prepare.outputs.trigger == 'scheduled_trigger'
        run: |
          python scripts/gha/it_workflow.py --stage report \
            --token ${{github.token}} \
            --actor ${{github.actor}} \
            --commit ${{needs.check_and_prepare.outputs.github_ref}} \
            --run_id ${{github.run_id}}
      - name: Print SDK package info
        run: |
          if [[ "${{ github.event.inputs.packaged_sdk_run_id }}" != "" ]]; then
            echo "run_id: ${{ github.event.inputs.packaged_sdk_run_id }}"
          fi
      - name: Summarize results into GitHub log
        run: python scripts/gha/summarize_test_results.py --dir test_results --github_log


  attempt_retry:
    name: "attempt-retry"
    needs: [check_and_prepare, summarize_results]
    runs-on: ubuntu-20.04
    if: ${{ failure() && needs.check_and_prepare.outputs.trigger == 'scheduled_trigger' }}
    steps:
      - uses: actions/checkout@v3
        with:
          ref: ${{needs.check_and_prepare.outputs.github_ref}}
      - name: Setup python
        uses: actions/setup-python@v4
        with:
          python-version: ${{ env.pythonVersion }}
      - name: Install python deps
        run: pip install -r scripts/gha/requirements.txt
      # The default token can't run workflows, so get an alternate token.
      - name: Generate token for GitHub API
        uses: tibdex/github-app-token@v1
        id: generate-token
        with:
          app_id: ${{ secrets.WORKFLOW_TRIGGER_APP_ID }}
          private_key: ${{ secrets.WORKFLOW_TRIGGER_APP_PRIVATE_KEY }}
      - name: Retry failed tests
        run: |
          echo "::warning ::Attempting to retry failed tests"
          python scripts/gha/trigger_workflow.py -t ${{ steps.generate-token.outputs.token }} \
          -w retry-test-failures.yml \
          -p run_id ${{ github.run_id }} \
          -s 10 \
          -A