Update release workflow

Signed-off-by: Jeffrey Kinard <[email protected]>

.github/actions/publish-site-report/action.yml

@@ -0,0 +1,39 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Cleans up the Java environment. This is mostly intended to help keep the
+# Java cache a bit smaller by removing unnecessary dependencies or stuff that
+# we will always rebuild anyways.
+
+name: 'Publish Site Report'
+description: 'Publish site report'
+
+inputs:
+  output-zip-file:
+    description: 'Output file name'
+    type: string
+    required: true
+
+runs:
+  using: 'composite'
+  steps:
+    - name: Generate Site Report
+      shell: bash
+      run: mvn -B surefire-report:report-only -f pom.xml -Daggregate=true
+    - name: Publish Site Report
+      uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v3.1.2
+      with:
+        name: ${{ inputs.output-zip-file }}
+        path: 'target/site/'
+        retention-days: 7

.github/workflows/java-pr.yml

@@ -113,10 +113,9 @@ jobs:
           token: ${{ secrets.CODECOV_TOKEN }}
           slug: GoogleCloudPlatform/DataflowTemplates
           files: 'target/site/jacoco-aggregate/jacoco.xml'
-          # Temp fix for https://github.com/codecov/codecov-action/issues/1487
-          version: v0.6.0
       - name: Cleanup Java Environment
         uses: ./.github/actions/cleanup-java-env
+        if: always()
   java_integration_smoke_tests_templates:
     name: Dataflow Templates Integration Smoke Tests
     needs: [spotless_check, checkstyle_check, java_build, java_unit_tests]
@@ -145,6 +144,7 @@ jobs:
           retention-days: 1
       - name: Cleanup Java Environment
         uses: ./.github/actions/cleanup-java-env
+        if: always()
   java_integration_tests_templates:
     name: Dataflow Templates Integration Tests
     needs: [java_integration_smoke_tests_templates]
@@ -173,6 +173,7 @@ jobs:
           retention-days: 1
       - name: Cleanup Java Environment
         uses: ./.github/actions/cleanup-java-env
+        if: always()
   java_load_tests_templates:
     if: contains(github.event.pull_request.labels.*.name, 'run-load-tests')
     name: Dataflow Templates Load Tests
@@ -195,3 +196,4 @@ jobs:
         --it-private-connectivity="datastream-private-connect-us-central1"
     - name: Cleanup Java Environment
       uses: ./.github/actions/cleanup-java-env
+      if: always()

.github/workflows/release.yml

@@ -21,99 +21,195 @@ permissions:
   contents: write
 
 jobs:
-  release:
+  create_release_branch:
     name: Create Release
-    runs-on: [self-hosted, release]
+    runs-on: ubuntu-latest
+    outputs:
+      release_branch_name: ${{ steps.variables.outputs.release_branch_name }}
+      tag_name: ${{ steps.variables.outputs.tag_name }}
     steps:
-    - name: Get releaser identity
-      run: |
-        git config --global user.name '${{github.actor}}'
-        git config --global user.email '${{github.actor}}@users.noreply.github.com'
-    - name: Declare release branch name and tag name
-      id: variables
-      run: |
-        echo "releaseBranchName=release_${CANDIDATE_NAME,,}" >> $GITHUB_OUTPUT
-        echo "tagName=${CANDIDATE_NAME^^}" >> $GITHUB_OUTPUT
-      env:
-        CANDIDATE_NAME: ${{ inputs.candidateName }}
-    - name: Checkout code
-      uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
-      with:
-        fetch-depth: 0
-        token: ${{ secrets.RELEASE_TOKEN }}
-    - name: Create release branch
-      run: git checkout -b $RELEASE_BRANCH_NAME $BRANCH_COMMIT
-      env:
-        RELEASE_BRANCH_NAME: ${{ steps.variables.outputs.releaseBranchName }}
-        BRANCH_COMMIT: ${{ inputs.branchCommit }}
-    - name: Cherry pick commits
-      run: |
-        commits=$(echo $CHERRYPICK_COMMITS | tr "," "\n")
-        for commit in $commits
-        do
-          echo "Cherry picking $commit."
-          git cherry-pick $commit
-        done
-      env:
-        CHERRYPICK_COMMITS: ${{ inputs.cherrypickCommits }}
-    - name: Add tag to most recent commit
-      run: |
-        DATE=$(date -d"next-monday - 1week" +'%Y-%m-%d')
-        T_COMMIT=$(git log -n 1 $RELEASE_BRANCH_NAME --pretty=format:'%H')
-        git tag -a $TAG_NAME -m "Release week of $DATE" $T_COMMIT
-      env:
-        RELEASE_BRANCH_NAME: ${{ steps.variables.outputs.releaseBranchName }}
-        TAG_NAME: ${{ steps.variables.outputs.tagName }}
-    - name: Setup Environment
-      id: setup-env
-      uses: ./.github/actions/setup-env
-    - name: Run Build
-      run: ./cicd/run-build
-    - name: Run Unit Tests
-      run: ./cicd/run-unit-tests
-    - name: Run Integration Smoke Tests
-      run: |
-        ./cicd/run-it-smoke-tests \
-        --it-region="us-central1" \
-        --it-project="cloud-teleport-testing" \
-        --it-artifact-bucket="cloud-teleport-testing-it-gitactions" \
-        --it-private-connectivity="datastream-private-connect-us-central1" \
-        --it-spanner-host="https://staging-wrenchworks.sandbox.googleapis.com/" \
-        --it-release=true \
-        --it-retry-failures=2
-    - name: Run Integration Tests
-      run: |
-        ./cicd/run-it-tests \
-        --it-region="us-central1" \
-        --it-project="cloud-teleport-testing" \
-        --it-artifact-bucket="cloud-teleport-testing-it-gitactions" \
-        --it-private-connectivity="datastream-private-connect-us-central1" \
-        --it-spanner-host="https://staging-wrenchworks.sandbox.googleapis.com/" \
-        --it-release=true \
-        --it-retry-failures=2
-    - name: Upload Tests Report
-      uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
-      if: always() # always run even if previous step fails
-      with:
-        name: surefire-test-results
-        path: '**/surefire-reports/TEST-*.xml'
-        retention-days: 1
-    - name: Create artifacts and push
-      run: |
-        mvn verify -PtemplatesRelease \
-        -DprojectId="dataflow-templates" \
-        -DbucketName="dataflow-templates-staging" \
-        -DlibrariesBucketName="dataflow-templates-libraries" \
-        -DstagePrefix="${CANDIDATE_NAME}" \
-        -Dmaven.test.skip -T8 -e
-      env:
-        CANDIDATE_NAME: ${{ inputs.candidateName }}
-    - name: Push tags
-      run: |
-        git push -u origin --tags
-    - name: Release
-      run: |
-        gh release create $TAG_NAME --title "Dataflow Templates $TAG_NAME" --notes ""
-      env:
-        GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        TAG_NAME: ${{ steps.variables.outputs.tagName }}
+      - name: Get releaser identity
+        run: |
+          git config --global user.name '${{github.actor}}'
+          git config --global user.email '${{github.actor}}@users.noreply.github.com'
+      - name: Declare release branch name and tag name
+        id: variables
+        run: |
+          echo "release_branch_name=release_${CANDIDATE_NAME,,}" >> $GITHUB_OUTPUT
+          echo "tag_name=${CANDIDATE_NAME^^}" >> $GITHUB_OUTPUT
+        env:
+          CANDIDATE_NAME: ${{ inputs.candidateName }}
+      - name: Checkout code
+        uses: actions/checkout@1e31de5234b9f8995739874a8ce0492dc87873e2 # v4.0.0
+        with:
+          fetch-depth: 0
+          token: ${{ secrets.RELEASE_TOKEN }}
+      - name: Create release branch
+        run: git checkout -b $RELEASE_BRANCH_NAME $BRANCH_COMMIT
+        env:
+          RELEASE_BRANCH_NAME: ${{ steps.variables.outputs.release_branch_name }}
+          BRANCH_COMMIT: ${{ inputs.branchCommit }}
+      - name: Cherry pick commits
+        run: |
+          commits=$(echo $CHERRYPICK_COMMITS | tr "," "\n")
+          for commit in $commits
+          do
+            echo "Cherry picking $commit."
+            git cherry-pick $commit
+          done
+        env:
+          CHERRYPICK_COMMITS: ${{ inputs.cherrypickCommits }}
+      - name: Push release branch
+        run: |
+          git push origin --delete $RELEASE_BRANCH_NAME || true  
+          git push -u origin $RELEASE_BRANCH_NAME
+        env:
+          RELEASE_BRANCH_NAME: ${{ steps.variables.outputs.release_branch_name }}
+  java_unit_tests:
+    name: Unit Tests
+    needs: create_release_branch
+    timeout-minutes: 60
+    runs-on: [ self-hosted, it ]
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@1e31de5234b9f8995739874a8ce0492dc87873e2 # v4.0.0
+        with:
+          ref: ${{ needs.create_release_branch.outputs.release_branch_name }}
+          fetch-depth: 0
+          token: ${{ secrets.RELEASE_TOKEN }}
+      - name: Setup Environment
+        id: setup-env
+        uses: ./.github/actions/setup-env
+      - name: Run Unit Tests
+        run: ./cicd/run-unit-tests
+      - name: Upload Unit Tests Site Report
+        uses: ./.github/actions/publish-site-report
+        with:
+          output-zip-file: unit-test-report
+        if: always()
+      - name: Cleanup Java Environment
+        uses: ./.github/actions/cleanup-java-env
+        if: always()
+  java_integration_smoke_tests_templates:
+    name: Dataflow Templates Integration Smoke Tests
+    needs: create_release_branch
+    timeout-minutes: 60
+    # Run on any runner that matches all the specified runs-on values.
+    runs-on: [ self-hosted, it ]
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@1e31de5234b9f8995739874a8ce0492dc87873e2 # v4.0.0
+        with:
+          ref: ${{ needs.create_release_branch.outputs.release_branch_name }}
+          fetch-depth: 0
+          token: ${{ secrets.RELEASE_TOKEN }}
+      - name: Setup Environment
+        id: setup-env
+        uses: ./.github/actions/setup-env
+      - name: Run Integration Smoke Tests
+        run: |
+          ./cicd/run-it-smoke-tests \
+          --changed-files="pom.xml" \
+          --it-region="us-central1" \
+          --it-project="cloud-teleport-testing" \
+          --it-artifact-bucket="cloud-teleport-testing-it-gitactions" \
+          --it-private-connectivity="datastream-private-connect-us-central1" \
+          --it-release=true \
+          --it-retry-failures=2
+      - name: Publish Smoke Test Site Report
+        uses: ./.github/actions/publish-site-report
+        with:
+          output-zip-file: smoke-test-report
+        if: always()
+      - name: Cleanup Java Environment
+        uses: ./.github/actions/cleanup-java-env
+        if: always()
+  java_integration_tests_templates:
+    name: Dataflow Templates Integration Tests
+    needs: create_release_branch
+    timeout-minutes: 240
+    # Run on any runner that matches all the specified runs-on values.
+    runs-on: [ self-hosted, it ]
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@1e31de5234b9f8995739874a8ce0492dc87873e2 # v4.0.0
+        with:
+          ref: ${{ needs.create_release_branch.outputs.release_branch_name }}
+          fetch-depth: 0
+          token: ${{ secrets.RELEASE_TOKEN }}
+      - name: Setup Environment
+        id: setup-env
+        uses: ./.github/actions/setup-env
+      - name: Run Integration Tests
+        shell: bash
+        run: |
+          ./cicd/run-it-tests \
+            --changed-files="pom.xml" \
+            --it-region="us-central1" \
+            --it-project="cloud-teleport-testing" \
+            --it-artifact-bucket="cloud-teleport-testing-it-gitactions" \
+            --it-private-connectivity="datastream-private-connect-us-central1" \
+            --it-release=true \
+            --it-retry-failures=2
+      - name: Publish Integration Test Site Report
+        uses: ./.github/actions/publish-site-report
+        with:
+          output-zip-file: integration-test-report
+        if: always()
+      - name: Cleanup Java Environment
+        uses: ./.github/actions/cleanup-java-env
+        if: always()
+  upload_artifacts:
+    name: Upload Template Artifacts
+    needs: [ create_release_branch, java_unit_tests, java_integration_smoke_tests_templates, java_integration_tests_templates ]
+    runs-on: [ self-hosted, it ]
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@1e31de5234b9f8995739874a8ce0492dc87873e2 # v4.0.0
+        with:
+          ref: ${{ needs.create_release_branch.outputs.release_branch_name }}
+          fetch-depth: 0
+          token: ${{ secrets.RELEASE_TOKEN }}
+      - name: Create artifacts and push
+        run: |
+          ./cicd/run-release \
+            --it-project="dataflow-templates" \
+            --release-bucket-name="dataflow-templates-staging" \
+            --release-libraries-bucket-name="dataflow-templates-libraries" \
+            --release-stage-prefix="${CANDIDATE_NAME}"
+        env:
+          CANDIDATE_NAME: ${{ inputs.candidateName }}
+      - name: Cleanup Java Environment
+        uses: ./.github/actions/cleanup-java-env
+        if: always()
+  release_github:
+    name: Release on GitHub
+    needs: [ create_release_branch, upload_artifacts ]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@1e31de5234b9f8995739874a8ce0492dc87873e2 # v4.0.0
+        with:
+          ref: ${{ needs.create_release_branch.outputs.release_branch_name }}
+          fetch-depth: 0
+          token: ${{ secrets.RELEASE_TOKEN }}
+      - name: Add tag to most recent commit
+        run: |
+          DATE=$(date -d"next-monday - 1week" +'%Y-%m-%d')
+          T_COMMIT=$(git log -n 1 $RELEASE_BRANCH_NAME --pretty=format:'%H')
+          git tag -a $TAG_NAME -m "Release week of $DATE" $T_COMMIT
+        env:
+          RELEASE_BRANCH_NAME: ${{ needs.create_release_branch.outputs.release_branch_name }}
+          TAG_NAME: ${{ needs.create_release_branch.outputs.tag_name }}
+      - name: Push tags
+        run: |
+          git push --delete origin $TAG_NAME || true
+          git push -u origin --tags
+        env:
+          TAG_NAME: ${{ needs.create_release_branch.outputs.tag_name }}
+      - name: Release
+        run: |
+          gh release create $TAG_NAME --title "Dataflow Templates $TAG_NAME" --notes ""
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          TAG_NAME: ${{ needs.create_release_branch.outputs.tag_name }}

.github/workflows/spanner-pr.yml

@@ -116,8 +116,6 @@ jobs:
         token: ${{ secrets.CODECOV_TOKEN }}
         slug: GoogleCloudPlatform/DataflowTemplates
         files: 'target/site/jacoco-aggregate/jacoco.xml'
-        # Temp fix for https://github.com/codecov/codecov-action/issues/1487
-        version: v0.6.0
     - name: Cleanup Java Environment
       uses: ./.github/actions/cleanup-java-env
   java_integration_smoke_tests_templates: