PIR End to end tests: This PR exists only to test GitHub Actions

.github/workflows/pir_end_to_end_tests.yml

@@ -0,0 +1,166 @@
+name: PIR E2E Tests
+
+on:
+  workflow_dispatch:
+  schedule:
+     - cron: '0 3 * * 1-5' # 3AM UTC offsetted to legacy to avoid action-junit-report@v4 bug
+  pull_request:
+
+jobs:
+  pir-e2e-tests:
+    name: PIR e2e tests
+    runs-on: ${{ matrix.runner }}
+    strategy:
+      fail-fast: false
+      matrix:
+        runner: [macos-14-xlarge]
+        include:
+          - xcode-version: "15.4"
+            runner: macos-14-xlarge
+
+    if: |
+      startsWith(github.event.pull_request.base.ref, 'release/') ||
+      startsWith(github.event.pull_request.base.ref, 'hotfix/') ||
+      contains(toJson(github.event.pull_request.files.*.filename), 'DBPE2ETests/') ||
+      contains(toJson(github.event.pull_request.files.*.filename), 'LocalPackages/DataBrokerProtection/') ||
+      contains(toJson(github.event.pull_request.files.*.filename), 'LocalPackages/DataBrokerProtection/') ||
+      contains(toJson(github.event.pull_request.files.*.filename), 'DuckDuckGoDBPBackgroundAgent/') ||
+      contains(toJson(github.event.pull_request.files.*.filename), 'DuckDuckGo.xcodeproj/project.pbxproj') ||
+      github.event_name == 'schedule'
+
+    concurrency:
+      group: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.runner }}
+      cancel-in-progress: true
+
+    timeout-minutes: 120
+
+    steps:
+    - name: Register SSH keys for certificates repository and PIR fake broker repository access
+      uses: webfactory/[email protected]
+      with:
+        ssh-private-key: |
+                        ${{ secrets.SSH_PRIVATE_KEY_FASTLANE_MATCH }}
+                        ${{ secrets.SSH_PRIVATE_KEY_PIR_FAKE_BROKER }}
+
+    - name: Check out the PIR fake broker code
+      uses: actions/checkout@v4
+      with:
+        repository: DuckDuckGo/pir-fake-broker
+        ssh-key: ${{ secrets.SSH_PRIVATE_KEY_PIR_FAKE_BROKER }}
+        ref: loremattei/ci-integration
+        path: pir-fake-broker
+
+    - name: Start PIR Fake Broker
+      run: |
+        cd pir-fake-broker 
+        cd scripts
+        ./install-prerequisites.sh
+        ./setup-ci.sh
+        cd .. 
+        pnpm start:all &
+
+    - name: Check out the code
+      uses: actions/checkout@v4
+      with:
+        submodules: recursive
+        path: main
+
+    - name: Set up fastlane
+      run: |
+        cd main
+        bundle install
+
+    - name: Sync code signing assets
+      env:
+        APPLE_API_KEY_BASE64: ${{ secrets.APPLE_API_KEY_BASE64 }}
+        APPLE_API_KEY_ID: ${{ secrets.APPLE_API_KEY_ID }}
+        APPLE_API_KEY_ISSUER: ${{ secrets.APPLE_API_KEY_ISSUER }}
+        MATCH_PASSWORD: ${{ secrets.MATCH_PASSWORD }}
+        SSH_PRIVATE_KEY_FASTLANE_MATCH: ${{ secrets.SSH_PRIVATE_KEY_FASTLANE_MATCH }}
+      run: |
+        cd main
+        bundle exec fastlane sync_signing_ci
+
+    - name: Download and unzip artifact
+      uses: actions/download-artifact@v4
+
+    - name: Set cache key hash
+      run: |
+        cd main
+        has_only_tags=$(jq '[ .pins[].state | has("version") ] | all' DuckDuckGo.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved)
+        if [[ "$has_only_tags" == "true" ]]; then
+          echo "cache_key_hash=${{ hashFiles('DuckDuckGo.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved') }}" >> $GITHUB_ENV
+        else
+          echo "Package.resolved contains dependencies specified by branch or commit, skipping cache."
+        fi
+
+    - name: Cache SPM
+      if: env.cache_key_hash
+      uses: actions/cache@v4
+      with:
+        path: main/DerivedData/SourcePackages
+        key: ${{ runner.os }}-spm-${{ env.cache_key_hash }}
+        restore-keys: |
+          ${{ runner.os }}-spm-
+
+    - name: Select Xcode
+      run: |
+        # Override .xcode_version because 15.4 is not available on macos 13
+        echo "${{ matrix.xcode-version }}" > .xcode-version
+        sudo xcode-select -s /Applications/Xcode_$(<.xcode-version).app/Contents/Developer
+
+    - name: Run PIR e2e Tests
+      run: |
+        cd main
+        set -o pipefail && xcodebuild test \
+          -scheme "DBPE2ETests" \
+          -derivedDataPath "DerivedData" \
+          -configuration "CI" \
+          -skipPackagePluginValidation -skipMacroValidation \
+          ENABLE_TESTABILITY=true \
+          "-only-testing:DBPE2ETests" \
+          -retry-tests-on-failure \
+          | tee xcodebuild.log \
+          | tee pir-e2e-tests.log
+      env:
+          PRIVACYPRO_STAGING_TOKEN: ${{ secrets.PRIVACYPRO_STAGING_TOKEN }}
+
+    - name: Prepare test report
+      if: always()
+      run: |
+        cd main
+        xcbeautify --report junit --report-path . --junit-report-filename pir-e2e-tests.xml < pir-e2e-tests.log
+
+    - name: Publish tests report
+      uses: mikepenz/action-junit-report@v4
+      if: always() 
+      with:
+        check_name: "Test Report ${{ matrix.runner }}"
+        report_paths: pir-e2e-tests.xml
+
+    - name: Upload logs when workflow failed
+      uses: actions/upload-artifact@v4
+      if: failure() || cancelled()
+      with:
+        name: "BuildLogs ${{ matrix.runner }}"
+        path: |
+          xcodebuild.log
+          DerivedData/Logs/Test/*.xcresult
+          ~/Library/Logs/DiagnosticReports/*
+        retention-days: 7
+
+  notify-failure:
+    name: Notify on failure
+    if: ${{ always() && github.event_name == 'schedule' && (needs.pir-e2e-tests.result == 'failure' || needs.pir-e2e-tests.result == 'cancelled') }}
+    needs: [pir-e2e-tests]
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Create Asana task when workflow failed
+      uses: duckduckgo/[email protected]
+      with:
+        action: create-asana-task
+        asana-pat: ${{ secrets.ASANA_ACCESS_TOKEN }}
+        asana-project: ${{ vars.MACOS_APP_DEVELOPMENT_ASANA_PROJECT_ID }}
+        asana-task-name: GH Workflow Failure - PIR e2e Tests
+        asana-task-description: The PIR e2e Tests workflow has failed. See https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}

.github/workflows/pr.yml

@@ -2,7 +2,7 @@ name: PR Checks
 
 on: 
   push:
-    branches: [ main, "release/**" ]
+    branches: [ main, "release/**", "loremattei/**" ]
   pull_request:
   workflow_call:
     inputs:
@@ -120,33 +120,56 @@ jobs:
             flavor: Sandbox
 
     runs-on: macos-14-xlarge
-    timeout-minutes: 30
+    timeout-minutes: 50
 
     outputs:
       private-api-check-report: ${{ steps.private-api.outputs.report }}
       commit_author: ${{ steps.fetch_commit_author.outputs.commit_author }}
 
     steps:
-    - name: Register SSH key for certificates repository access
+    - name: Register SSH keys for certificates repository and PIR fake broker repository access
       uses: webfactory/[email protected]
       with:
-        ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY_FASTLANE_MATCH }}
+        ssh-private-key: |
+                        ${{ secrets.SSH_PRIVATE_KEY_FASTLANE_MATCH }}
+                        ${{ secrets.SSH_PRIVATE_KEY_PIR_FAKE_BROKER }}
+
+    - name: Check out the code
+      uses: actions/checkout@v4
+      with:
+        repository: DuckDuckGo/pir-fake-broker
+        ssh-key: ${{ secrets.SSH_PRIVATE_KEY_PIR_FAKE_BROKER }}
+        ref: loremattei/ci-integration
+        path: pir-fake-broker
+
+    - name: Start PIR Fake Broker
+      run: |
+        cd pir-fake-broker 
+        cd scripts
+        ./install-prerequisites.sh
+        ./setup-ci.sh
+        cd .. 
+        pnpm start:all &
 
     - name: Check out the code
       if: github.event_name == 'pull_request' || github.event_name == 'push'
       uses: actions/checkout@v4
       with:
         submodules: recursive
+        path: main
 
     - name: Check out the code
       if: github.event_name != 'pull_request' && github.event_name != 'push'
       uses: actions/checkout@v4
       with:
         submodules: recursive
         ref: ${{ inputs.branch || github.ref_name }}
+        path: main
 
     - name: Set up fastlane
-      run: bundle install
+      run: |
+        cd main
+        bundle install 
 
     - name: Sync code signing assets
       env:
@@ -155,10 +178,13 @@ jobs:
         APPLE_API_KEY_ISSUER: ${{ secrets.APPLE_API_KEY_ISSUER }}
         MATCH_PASSWORD: ${{ secrets.MATCH_PASSWORD }}
         SSH_PRIVATE_KEY_FASTLANE_MATCH: ${{ secrets.SSH_PRIVATE_KEY_FASTLANE_MATCH }}
-      run: bundle exec fastlane sync_signing_ci
+      run: |
+        cd main
+        bundle exec fastlane sync_signing_ci
 
     - name: Set cache key hash
       run: |
+        cd main
         has_only_tags=$(jq '[ .pins[].state | has("version") ] | all' DuckDuckGo.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved)
         if [[ "$has_only_tags" == "true" ]]; then
           echo "cache_key_hash=${{ hashFiles('DuckDuckGo.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved') }}" >> $GITHUB_ENV
@@ -170,33 +196,35 @@ jobs:
       if: env.cache_key_hash
       uses: actions/cache@v4
       with:
-        path: DerivedData/SourcePackages
+        path: main/DerivedData/SourcePackages
         key: ${{ runner.os }}-spm-${{ matrix.cache-key }}${{ env.cache_key_hash }}
         restore-keys: |
           ${{ runner.os }}-spm-${{ matrix.cache-key }}
 
     - name: Select Xcode
-      run: sudo xcode-select -s /Applications/Xcode_$(<.xcode-version).app/Contents/Developer
-
-    - name: Build and run unit tests
-      run: |
-        echo "Runner ${RUNNER_NAME} (${RUNNER_TRACKING_ID})"
-        export OS_ACTIVITY_MODE=debug
-
-        set -o pipefail && xcodebuild test \
-          -scheme "${{ matrix.scheme }}" \
-          -derivedDataPath "DerivedData" \
-          -configuration "CI" \
-          -skipPackagePluginValidation -skipMacroValidation \
-          ENABLE_TESTABILITY=true \
-          ONLY_ACTIVE_ARCH=${{ matrix.active-arch }} \
-          "-skip-testing:${{ matrix.integration-tests-target }}" \
-          | tee ${{ matrix.flavor }}-unittests-xcodebuild.log \
-          | xcbeautify --report junit --report-path . --junit-report-filename ${{ matrix.flavor }}-unittests.xml \
-          || { mv "$(grep -m 1 '.*\.xcresult' ${{ matrix.flavor }}-unittests-xcodebuild.log | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')" ./${{ matrix.flavor }}-unittests.xcresult && exit 1; }
+      run: cd main && sudo xcode-select -s /Applications/Xcode_$(<.xcode-version).app/Contents/Developer
+
+    # - name: Build and run unit tests
+    #   run: |
+    #     cd main
+    #     echo "Runner ${RUNNER_NAME} (${RUNNER_TRACKING_ID})"
+    #     export OS_ACTIVITY_MODE=debug
+
+    #     set -o pipefail && xcodebuild test \
+    #       -scheme "${{ matrix.scheme }}" \
+    #       -derivedDataPath "DerivedData" \
+    #       -configuration "CI" \
+    #       -skipPackagePluginValidation -skipMacroValidation \
+    #       ENABLE_TESTABILITY=true \
+    #       ONLY_ACTIVE_ARCH=${{ matrix.active-arch }} \
+    #       "-skip-testing:${{ matrix.integration-tests-target }}" \
+    #       | tee ${{ matrix.flavor }}-unittests-xcodebuild.log \
+    #       | xcbeautify --report junit --report-path . --junit-report-filename ${{ matrix.flavor }}-unittests.xml \
+    #       || { mv "$(grep -m 1 '.*\.xcresult' ${{ matrix.flavor }}-unittests-xcodebuild.log | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')" ./${{ matrix.flavor }}-unittests.xcresult && exit 1; }
 
     - name: Run integration tests
       run: |
+        cd main
         set -o pipefail && xcodebuild test \
           -scheme "${{ matrix.scheme }}" \
           -derivedDataPath "DerivedData" \
@@ -213,6 +241,7 @@ jobs:
     - name: Check private API usage
       id: private-api
       run: |
+        cd main
         if [[ ${{ matrix.flavor }} != "Sandbox" ]]; then
           echo "Skipping private API usage check for ${{ matrix.flavor }} build"
         else
@@ -277,15 +306,15 @@ jobs:
 
     - name: Upload failed integration tests log
       uses: actions/upload-artifact@v4
-      if: failure()
+      if: failure() || cancelled()
       with:
         name: ${{ matrix.flavor }}-integrationtests-xcodebuild.log
         path: ${{ matrix.flavor }}-integrationtests-xcodebuild.log
         retention-days: 7
 
     - name: Upload failed integration tests xcresult
       uses: actions/upload-artifact@v4
-      if: failure()
+      if: failure() || cancelled()
       with:
         name: ${{ matrix.flavor }}-integrationtests.xcresult
         path: ${{ matrix.flavor }}-integrationtests.xcresult

DBPE2ETests/DBPE2ETestsBridging.h

@@ -0,0 +1,21 @@
+//
+//  IntegrationTestsBridging.h
+//
+//  Copyright © 2021 DuckDuckGo. All rights reserved.
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+//
+
+#import "Bridging.h"
+
+#import "WKURLSchemeTask+Private.h"