Set up a CI workflow for running UI tests for the macOS browser (#2387)

Task/Issue URL:
https://app.asana.com/0/72649045549333/1206610209649715/f
Tech Design URL:
CC: @ayoy 

**Description**:
PR that enables running of sync tests via CI on macos11/12/13/14. Tests
currently not passing.
Known issues:
- xcpretty doesn't work for macos11
- Asana is not notified for failures (yet)

**Steps to test this PR**:
1. Run the workflows

<!--
Tagging instructions
If this PR isn't ready to be merged for whatever reason it should be
marked with the `DO NOT MERGE` label (particularly if it's a draft)
If it's pending Product Review/PFR, please add the `Pending Product
Review` label.

If at any point it isn't actively being worked on/ready for
review/otherwise moving forward (besides the above PR/PFR exception)
strongly consider closing it (or not opening it in the first place). If
you decide not to close it, make sure it's labelled to make it clear the
PRs state and comment with more information.
-->

---
###### Internal references:
[Pull Request Review
Checklist](https://app.asana.com/0/1202500774821704/1203764234894239/f)
[Software Engineering
Expectations](https://app.asana.com/0/59792373528535/199064865822552)
[Technical Design
Template](https://app.asana.com/0/59792373528535/184709971311943)
[Pull Request
Documentation](https://app.asana.com/0/1202500774821704/1204012835277482/f)

---------

Co-authored-by: Dominik Kapusta <[email protected]>

.github/workflows/build_notarized.yml

@@ -34,7 +34,7 @@ on:
         type: boolean
       asana-task-url:
         description: "Asana release task URL"
-        required: true
+        required: false
         type: string
       branch:
         description: "Branch name"

.github/workflows/sync_end_to_end.yml

@@ -0,0 +1,113 @@
+name: Sync-End-to-End tests
+
+on:
+  workflow_dispatch:
+  # schedule:
+  #   - cron: '0 5 * * *' # run at 5 AM UTC
+
+jobs:
+  sync-end-to-end-tests:
+    name: Sync End to end Tests
+    runs-on: ${{ matrix.runner }}
+    strategy:
+      fail-fast: false
+      matrix:
+        runner: [macos-13-xlarge, macos-14-xlarge]
+
+    timeout-minutes: 60
+
+    steps:
+    - name: Check out the code
+      uses: actions/checkout@v4
+      with:
+        submodules: recursive
+
+    - name: Set cache key hash
+      run: |
+        has_only_tags=$(jq '[ .pins[].state | has("version") ] | all' DuckDuckGo.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved)
+        if [[ "$has_only_tags" == "true" ]]; then
+          echo "cache_key_hash=${{ hashFiles('DuckDuckGo.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved') }}" >> $GITHUB_ENV
+        else
+          echo "Package.resolved contains dependencies specified by branch or commit, skipping cache."
+        fi
+
+    - name: Cache SPM
+      if: env.cache_key_hash
+      uses: actions/cache@v3
+      with:
+        path: DerivedData/SourcePackages
+        key: ${{ runner.os }}-spm-${{ env.cache_key_hash }}
+        restore-keys: |
+          ${{ runner.os }}-spm-
+
+    - name: Install Apple Developer ID Application certificate
+      uses: ./.github/actions/install-certs-and-profiles
+      with:
+        BUILD_CERTIFICATE_BASE64: ${{ secrets.BUILD_CERTIFICATE_BASE64 }}
+        P12_PASSWORD: ${{ secrets.P12_PASSWORD }}
+        KEYCHAIN_PASSWORD: ${{ secrets.KEYCHAIN_PASSWORD }}
+        REVIEW_PROVISION_PROFILE_BASE64: ${{ secrets.REVIEW_PROVISION_PROFILE_BASE64 }}
+        RELEASE_PROVISION_PROFILE_BASE64: ${{ secrets.RELEASE_PROVISION_PROFILE_BASE64 }}
+        DBP_AGENT_RELEASE_PROVISION_PROFILE_BASE64: ${{ secrets.DBP_AGENT_RELEASE_PROVISION_PROFILE_BASE64 }}
+        DBP_AGENT_REVIEW_PROVISION_PROFILE_BASE64: ${{ secrets.DBP_AGENT_REVIEW_PROVISION_PROFILE_BASE64 }}
+        NETP_SYSEX_RELEASE_PROVISION_PROFILE_BASE64: ${{ secrets.NETP_SYSEX_RELEASE_PROVISION_PROFILE_BASE64_V2 }}
+        NETP_SYSEX_REVIEW_PROVISION_PROFILE_BASE64: ${{ secrets.NETP_SYSEX_REVIEW_PROVISION_PROFILE_BASE64_V2 }}
+        NETP_AGENT_RELEASE_PROVISION_PROFILE_BASE64: ${{ secrets.NETP_AGENT_RELEASE_PROVISION_PROFILE_BASE64_V2 }}
+        NETP_AGENT_REVIEW_PROVISION_PROFILE_BASE64: ${{ secrets.NETP_AGENT_REVIEW_PROVISION_PROFILE_BASE64_V2 }}
+        NETP_NOTIFICATIONS_RELEASE_PROVISION_PROFILE_BASE64: ${{ secrets.NETP_NOTIFICATIONS_RELEASE_PROVISION_PROFILE_BASE64 }}
+        NETP_NOTIFICATIONS_REVIEW_PROVISION_PROFILE_BASE64: ${{ secrets.NETP_NOTIFICATIONS_REVIEW_PROVISION_PROFILE_BASE64 }}
+
+    - name: Select Xcode
+      run: sudo xcode-select -s /Applications/Xcode_$(<.xcode-version).app/Contents/Developer
+
+    - name: Create test account for Sync and return the recovery code
+      uses: duckduckgo/sync_crypto/action@main
+      id: sync-recovery-code
+      with:
+        debug: true
+
+    - name: Build and run Sync e2e tests
+      env:
+        CODE: ${{ steps.sync-recovery-code.outputs.recovery-code }}
+      run: |
+        defaults write com.duckduckgo.macos.browser.review moveToApplicationsFolderAlertSuppress 1
+        defaults write com.duckduckgo.macos.browser.review sync.environment Development
+        defaults write com.duckduckgo.macos.browser.review onboarding.finished -bool true
+        set -o pipefail && xcodebuild test \
+          -scheme "Sync End-to-End UI Tests" \
+          -configuration Review \
+          -derivedDataPath DerivedData \
+          -skipPackagePluginValidation \
+          -skipMacroValidation \
+          -test-iterations 2 \
+          -retry-tests-on-failure \
+        | tee xcodebuild.log \
+        | xcbeautify --report junit --report-path . --junit-report-filename ui-tests.xml \
+    
+    # - name: Create Asana task when workflow failed
+    #   if: ${{ failure() }}
+    #   run: |
+    #     curl -s "https://app.asana.com/api/1.0/tasks" \
+    #       --header "Accept: application/json" \
+    #       --header "Authorization: Bearer ${{ secrets.ASANA_ACCESS_TOKEN }}" \
+    #       --header "Content-Type: application/json" \
+    #       --data ' { "data": { "name": "GH Workflow Failure - Sync End to end tests", "projects": [ "${{ vars.MACOS_APP_DEVELOPMENT_ASANA_PROJECT_ID }}" ], "notes" : "The end to end workflow has failed. See https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" } }'
+
+    - name: Publish tests report
+      uses: mikepenz/action-junit-report@v3
+      if: always() # always run even if the previous step fails
+      with:
+        check_name: "Test Report ${{ matrix.runner }}"
+        report_paths: ui-tests.xml
+
+    - name: Upload logs when workflow failed
+      uses: actions/upload-artifact@v4
+      if: failure()
+      with:
+        name: "BuildLogs ${{ matrix.runner }}"
+        path: |
+          xcodebuild.log
+          DerivedData/Logs/Test/*.xcresult
+          ~/Library/Logs/DiagnosticReports/*
+        retention-days: 7
+

.github/workflows/sync_end_to_end_legacy_os.yml

@@ -0,0 +1,174 @@
+name: Sync-End-to-End tests MacOS11/12
+
+on:
+  workflow_dispatch:
+  # schedule:
+  #   - cron: '0 4 * * *' # run at 4 AM UTC
+
+jobs:
+  create-notarized-app:
+    name: Prepare DMG Release
+    uses: ./.github/workflows/build_notarized.yml
+    with:
+      release-type: review
+      create-dmg: false
+      branch: ${{ github.ref_name }}
+    secrets:
+      BUILD_CERTIFICATE_BASE64: ${{ secrets.BUILD_CERTIFICATE_BASE64 }}
+      P12_PASSWORD: ${{ secrets.P12_PASSWORD }}
+      KEYCHAIN_PASSWORD: ${{ secrets.KEYCHAIN_PASSWORD }}
+      REVIEW_PROVISION_PROFILE_BASE64: ${{ secrets.REVIEW_PROVISION_PROFILE_BASE64 }}
+      RELEASE_PROVISION_PROFILE_BASE64: ${{ secrets.RELEASE_PROVISION_PROFILE_BASE64 }}
+      DBP_AGENT_RELEASE_PROVISION_PROFILE_BASE64: ${{ secrets.DBP_AGENT_RELEASE_PROVISION_PROFILE_BASE64 }}
+      DBP_AGENT_REVIEW_PROVISION_PROFILE_BASE64: ${{ secrets.DBP_AGENT_REVIEW_PROVISION_PROFILE_BASE64 }}
+      NETP_SYSEX_RELEASE_PROVISION_PROFILE_BASE64_V2: ${{ secrets.NETP_SYSEX_RELEASE_PROVISION_PROFILE_BASE64_V2 }}
+      NETP_SYSEX_REVIEW_PROVISION_PROFILE_BASE64_V2: ${{ secrets.NETP_SYSEX_REVIEW_PROVISION_PROFILE_BASE64_V2 }}
+      NETP_AGENT_RELEASE_PROVISION_PROFILE_BASE64_V2: ${{ secrets.NETP_AGENT_RELEASE_PROVISION_PROFILE_BASE64_V2 }}
+      NETP_AGENT_REVIEW_PROVISION_PROFILE_BASE64_V2: ${{ secrets.NETP_AGENT_REVIEW_PROVISION_PROFILE_BASE64_V2 }}
+      NETP_NOTIFICATIONS_RELEASE_PROVISION_PROFILE_BASE64: ${{ secrets.NETP_NOTIFICATIONS_RELEASE_PROVISION_PROFILE_BASE64 }}
+      NETP_NOTIFICATIONS_REVIEW_PROVISION_PROFILE_BASE64: ${{ secrets.NETP_NOTIFICATIONS_REVIEW_PROVISION_PROFILE_BASE64 }}
+      APPLE_API_KEY_BASE64: ${{ secrets.APPLE_API_KEY_BASE64 }}
+      APPLE_API_KEY_ID: ${{ secrets.APPLE_API_KEY_ID }}
+      APPLE_API_KEY_ISSUER: ${{ secrets.APPLE_API_KEY_ISSUER }}
+      ASANA_ACCESS_TOKEN: ${{ secrets.ASANA_ACCESS_TOKEN }}
+      MM_HANDLES_BASE64: ${{ secrets.MM_HANDLES_BASE64 }}
+      MM_WEBHOOK_URL: ${{ secrets.MM_WEBHOOK_URL }}
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+      AWS_ACCESS_KEY_ID_RELEASE_S3: ${{ secrets.AWS_ACCESS_KEY_ID_RELEASE_S3 }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      AWS_SECRET_ACCESS_KEY_RELEASE_S3: ${{ secrets.AWS_SECRET_ACCESS_KEY_RELEASE_S3 }}
+
+  sync-end-to-end-tests-older-macos:
+    name: Sync E2E tests
+    needs: create-notarized-app
+    env:
+      app-version: ${{ needs.create-notarized-app.outputs.app-version }}
+
+    runs-on: ${{ matrix.runner }}
+    strategy:
+      fail-fast: false
+      matrix:
+        runner: [ macos-11, macos-12-large ]
+
+    timeout-minutes: 60
+    steps:
+    - name: Check out the code
+      uses: actions/checkout@v4
+      with:
+        submodules: recursive
+
+    - name: Download and unzip artifact
+      uses: actions/download-artifact@v4
+
+    - name: Set cache key hash
+      run: |
+        has_only_tags=$(jq '[ .pins[].state | has("version") ] | all' DuckDuckGo.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved)
+        if [[ "$has_only_tags" == "true" ]]; then
+          echo "cache_key_hash=${{ hashFiles('DuckDuckGo.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved') }}" >> $GITHUB_ENV
+        else
+          echo "Package.resolved contains dependencies specified by branch or commit, skipping cache."
+        fi
+
+    - name: Cache SPM
+      if: env.cache_key_hash
+      uses: actions/cache@v3
+      with:
+        path: DerivedData/SourcePackages
+        key: ${{ runner.os }}-spm-${{ env.cache_key_hash }}
+        restore-keys: |
+          ${{ runner.os }}-spm-
+
+    - name: Install Apple Developer ID Application certificate
+      uses: ./.github/actions/install-certs-and-profiles
+      with:
+        BUILD_CERTIFICATE_BASE64: ${{ secrets.BUILD_CERTIFICATE_BASE64 }}
+        P12_PASSWORD: ${{ secrets.P12_PASSWORD }}
+        KEYCHAIN_PASSWORD: ${{ secrets.KEYCHAIN_PASSWORD }}
+        REVIEW_PROVISION_PROFILE_BASE64: ${{ secrets.REVIEW_PROVISION_PROFILE_BASE64 }}
+        RELEASE_PROVISION_PROFILE_BASE64: ${{ secrets.RELEASE_PROVISION_PROFILE_BASE64 }}
+        DBP_AGENT_RELEASE_PROVISION_PROFILE_BASE64: ${{ secrets.DBP_AGENT_RELEASE_PROVISION_PROFILE_BASE64 }}
+        DBP_AGENT_REVIEW_PROVISION_PROFILE_BASE64: ${{ secrets.DBP_AGENT_REVIEW_PROVISION_PROFILE_BASE64 }}
+        NETP_SYSEX_RELEASE_PROVISION_PROFILE_BASE64: ${{ secrets.NETP_SYSEX_RELEASE_PROVISION_PROFILE_BASE64_V2 }}
+        NETP_SYSEX_REVIEW_PROVISION_PROFILE_BASE64: ${{ secrets.NETP_SYSEX_REVIEW_PROVISION_PROFILE_BASE64_V2 }}
+        NETP_AGENT_RELEASE_PROVISION_PROFILE_BASE64: ${{ secrets.NETP_AGENT_RELEASE_PROVISION_PROFILE_BASE64_V2 }}
+        NETP_AGENT_REVIEW_PROVISION_PROFILE_BASE64: ${{ secrets.NETP_AGENT_REVIEW_PROVISION_PROFILE_BASE64_V2 }}
+        NETP_NOTIFICATIONS_RELEASE_PROVISION_PROFILE_BASE64: ${{ secrets.NETP_NOTIFICATIONS_RELEASE_PROVISION_PROFILE_BASE64 }}
+        NETP_NOTIFICATIONS_REVIEW_PROVISION_PROFILE_BASE64: ${{ secrets.NETP_NOTIFICATIONS_REVIEW_PROVISION_PROFILE_BASE64 }}
+
+    - name: Select Xcode 14.2 for macOS 12
+      if: matrix.runner == 'macos-12-large'
+      run: sudo xcode-select -s /Applications/Xcode_14.2.app/Contents/Developer
+
+    - name: Select Xcode 13.2.1 for macOS 11
+      if: matrix.runner == 'macos-11'
+      run: sudo xcode-select -s /Applications/Xcode_13.2.1.app/Contents/Developer
+
+    - name: Create test account for Sync and return the recovery code
+      uses: duckduckgo/sync_crypto/action@main
+      id: sync-recovery-code
+      with:
+        debug: true
+
+    # This first bit is a hack to stop the app building package dependencies that isn't needed by the test runner
+    - name: Build test runner 
+      run: |
+        rm -rf LocalPackages
+        rm -rf /Users/runner/work/macos-browser/macos-browser/DuckDuckGo.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved 
+        sed -i '' '/\/\* Begin XCSwiftPackageProductDependency section \*\//,/\/\* End XCSwiftPackageProductDependency section \*\//d' DuckDuckGo.xcodeproj/project.pbxproj
+        sed -i '' '/\/\* Begin XCRemoteSwiftPackageReference section \*\//,/\/\* End XCRemoteSwiftPackageReference section \*\//d' DuckDuckGo.xcodeproj/project.pbxproj
+        set -o pipefail && xcodebuild build-for-testing \
+          -scheme "SyncE2EUITestsOldVersions" \
+          -derivedDataPath DerivedData \
+          -configuration Review 
+    
+    - name: Unzip and Copy app to /DerivedData                         
+      run: |
+        cd DuckDuckGo-review-*.app && unzip DuckDuckGo-*.zip
+        cp -R "DuckDuckGo Review.app" "../DerivedData/Build/Products/Review/DuckDuckGo Review.app"
+
+    - name: Run tests
+      env:
+        CODE: ${{ steps.sync-recovery-code.outputs.recovery-code }}
+      run: |
+        defaults write com.duckduckgo.macos.browser.review moveToApplicationsFolderAlertSuppress 1
+        defaults write com.duckduckgo.macos.browser.review sync.environment Development
+        defaults write com.duckduckgo.macos.browser.review onboarding.finished -bool true
+        set -o pipefail && xcodebuild test \
+          -scheme "SyncE2EUITestsOldVersions" \
+          -derivedDataPath DerivedData \
+          -configuration Review \
+          -test-iterations 2 \
+          -retry-tests-on-failure \
+        | tee xcodebuild.log \
+        | (if [ "${{ matrix.runner }}" = "macos-11" ]; then
+            xcpretty --report junit --output ui-tests.xml
+          else
+            xcbeautify --report junit --report-path . --junit-report-filename ui-tests.xml
+          fi)
+
+ #   - name: Create Asana task when workflow failed
+ #     if: ${{ failure() }} && github.ref == 'refs/heads/main'
+ #     run: |
+ #       curl -s "https://app.asana.com/api/1.0/tasks" \
+ #          --header "Accept: application/json" \
+ #          --header "Authorization: Bearer ${{ secrets.ASANA_ACCESS_TOKEN }}" \
+ #          --header "Content-Type: application/json" \
+ #          --data ' { "data": { "name": "GH Workflow Failure - Sync End to end tests", "projects": [ "${{ vars.MACOS_APP_DEVELOPMENT_ASANA_PROJECT_ID }}" ], "notes" : "The end to end workflow has failed. See https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" } }'
+
+    - name: Publish tests report
+      uses: mikepenz/action-junit-report@v3
+      if: always() 
+      with:
+        check_name: "Test Report ${{ matrix.runner }}"
+        report_paths: ui-tests.xml
+
+    - name: Upload logs when workflow failed
+      uses: actions/upload-artifact@v4
+      if: always() 
+      with:
+        name: "BuildLogs ${{ matrix.runner }}"
+        path: |
+          xcodebuild.log
+          DerivedData/Logs/Test/*.xcresult
+          ~/Library/Logs/DiagnosticReports/*
+        retention-days: 7

Configuration/UITests/SyncE2EUITests.xcconfig

@@ -15,12 +15,16 @@
 
 #include "../Common.xcconfig"
 
+CODE_SIGN_IDENTITY[config=Review][sdk=macosx*] =
+CODE_SIGN_STYLE[config=Review][sdk=macosx*] = Manual
+CODE_SIGNING_ALLOWED[config=Review][sdk=macosx*] = NO
 CODE_SIGN_IDENTITY[config=CI][sdk=macosx*] =
 CODE_SIGN_STYLE[config=CI][sdk=macosx*] = Manual
 CODE_SIGNING_ALLOWED[config=CI][sdk=macosx*] = NO
 COPY_PHASE_STRIP = NO
 DEAD_CODE_STRIPPING = YES
 DEVELOPMENT_TEAM[config=CI][sdk=macosx*] =
+DEVELOPMENT_TEAM[config=Review][sdk=macosx*] =
 ENABLE_USER_SCRIPT_SANDBOXING = YES
 GCC_NO_COMMON_BLOCKS = YES
 GENERATE_INFOPLIST_FILE = YES

DuckDuckGo.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved

@@ -165,7 +165,7 @@
     {
       "identity" : "trackerradarkit",
       "kind" : "remoteSourceControl",
-      "location" : "https://github.com/duckduckgo/TrackerRadarKit",
+      "location" : "https://github.com/duckduckgo/TrackerRadarKit.git",
       "state" : {
         "revision" : "a6b7ba151d9dc6684484f3785293875ec01cc1ff",
         "version" : "1.2.2"

DuckDuckGo.xcodeproj/xcshareddata/xcschemes/Sync End-to-End UI Tests.xcscheme

@@ -23,9 +23,9 @@
       </BuildActionEntries>
    </BuildAction>
    <TestAction
-      buildConfiguration = "Debug"
-      selectedDebuggerIdentifier = "Xcode.DebuggerFoundation.Debugger.LLDB"
-      selectedLauncherIdentifier = "Xcode.DebuggerFoundation.Launcher.LLDB"
+      buildConfiguration = "Review"
+      selectedDebuggerIdentifier = ""
+      selectedLauncherIdentifier = "Xcode.IDEFoundation.Launcher.PosixSpawn"
       shouldUseLaunchSchemeArgsEnv = "YES"
       codeCoverageEnabled = "YES">
       <PreActions>
@@ -69,9 +69,9 @@
       </Testables>
    </TestAction>
    <LaunchAction
-      buildConfiguration = "Debug"
-      selectedDebuggerIdentifier = "Xcode.DebuggerFoundation.Debugger.LLDB"
-      selectedLauncherIdentifier = "Xcode.DebuggerFoundation.Launcher.LLDB"
+      buildConfiguration = "Review"
+      selectedDebuggerIdentifier = ""
+      selectedLauncherIdentifier = "Xcode.IDEFoundation.Launcher.PosixSpawn"
       launchStyle = "0"
       useCustomWorkingDirectory = "NO"
       ignoresPersistentStateOnLaunch = "NO"