From d8aac42c14b1236cd45c60be0e88debe649c7965 Mon Sep 17 00:00:00 2001 From: Anand Inguva Date: Fri, 28 Jun 2024 17:22:19 -0400 Subject: [PATCH 1/3] Add Kafka GH actions suite --- .github/workflows/java-pr.yml | 5 + .github/workflows/kafka-pr.yaml | 198 +++++++++++++++++++++++ cicd/internal/flags/common-flags.go | 4 +- cicd/internal/flags/common-flags_test.go | 4 + 4 files changed, 210 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/kafka-pr.yaml diff --git a/.github/workflows/java-pr.yml b/.github/workflows/java-pr.yml index 57d6215635..b8626f6fda 100644 --- a/.github/workflows/java-pr.yml +++ b/.github/workflows/java-pr.yml @@ -35,6 +35,11 @@ on: - '!v2/spanner-migrations-sdk/**' - '!v2/spanner-custom-shard/**' - '!v2/sourcedb-to-spanner/**' + # Exclude Kafka paths from global run (covered in https://github.com/GoogleCloudPlatform/DataflowTemplates/blob/main/.github/workflows/kafka-pr.yml) + - '!v2/kafka-to-bigquery/**' + - '!v2/kafka-to-gcs/**' + - '!v2/kafka-to-kafka/**' + - '!v2/kafka-common/**' schedule: - cron: "0 */12 * * *" workflow_dispatch: diff --git a/.github/workflows/kafka-pr.yaml b/.github/workflows/kafka-pr.yaml new file mode 100644 index 0000000000..e1485ccddc --- /dev/null +++ b/.github/workflows/kafka-pr.yaml @@ -0,0 +1,198 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Checks that are intended to run on PRs containing Java code. + +name: Kafka PR + +on: + pull_request: + branches: + - 'main' + paths: + # Include Kafka paths only + - '.github/workflows/kafka-pr.yml' + - 'v2/kafka-to-bigquery/**' + - 'v2/kafka-to-gcs/**' + - 'v2/kafka-to-kafka/**' + - 'v2/kafka-common/**' + schedule: + - cron: "0 */12 * * *" + workflow_dispatch: + +concurrency: + group: java-pr-${{ github.event.issue.number || github.run_id }} + cancel-in-progress: true + +env: + MAVEN_OPTS: -Dorg.slf4j.simpleLogger.log.org.apache.maven.plugins.shade=error + +permissions: read-all + +jobs: + spotless_check: + name: Spotless + timeout-minutes: 10 + runs-on: ubuntu-latest + steps: + - name: Checkout Code + uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0 + - name: Setup Environment + id: setup-env + uses: ./.github/actions/setup-env + - name: Run Spotless + run: | + ./cicd/run-spotless \ + --modules-to-build="KAFKA" + checkstyle_check: + name: Checkstyle + timeout-minutes: 10 + runs-on: ubuntu-latest + steps: + - name: Checkout Code + uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0 + - name: Setup Environment + id: setup-env + uses: ./.github/actions/setup-env + - name: Run Checkstyle + run: | + ./cicd/run-checkstyle \ + --modules-to-build="KAFKA" + java_build: + name: Build + timeout-minutes: 60 + runs-on: [self-hosted, it] + steps: + - name: Checkout Code + uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0 + - name: Setup Environment + id: setup-env + uses: ./.github/actions/setup-env + - name: Run Build + run: | + ./cicd/run-build \ + --modules-to-build="KAFKA" + - name: Cleanup Java Environment + uses: ./.github/actions/cleanup-java-env + java_unit_tests: + name: Unit Tests + needs: [java_build] + timeout-minutes: 60 + runs-on: [self-hosted, it] + steps: + - name: Checkout Code + uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0 + - name: Setup Environment + id: setup-env + uses: ./.github/actions/setup-env + - name: Run Unit Tests + run: | + ./cicd/run-unit-tests \ + --modules-to-build="KAFKA" + - name: Upload Unit Tests Report + uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2 + if: always() # always run even if the previous step fails + with: + name: surefire-test-results + path: '**/surefire-reports/TEST-*.xml' + retention-days: 1 + - name: Upload coverage reports to Codecov + uses: codecov/codecov-action@v4.0.1 + with: + token: ${{ secrets.CODECOV_TOKEN }} + slug: GoogleCloudPlatform/DataflowTemplates + files: 'target/site/jacoco-aggregate/jacoco.xml' + - name: Cleanup Java Environment + uses: ./.github/actions/cleanup-java-env + java_integration_smoke_tests_templates: + name: Dataflow Templates Integration Smoke Tests + needs: [spotless_check, checkstyle_check, java_build, java_unit_tests] + timeout-minutes: 60 + # Run on any runner that matches all the specified runs-on values. + runs-on: [self-hosted, it] + steps: + - name: Checkout Code + uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0 + - name: Setup Environment + id: setup-env + uses: ./.github/actions/setup-env + - name: Run Integration Smoke Tests + run: | + ./cicd/run-it-smoke-tests \ + --modules-to-build="KAFKA" \ + --it-region="us-central1" \ + --it-project="cloud-teleport-testing" \ + --it-artifact-bucket="cloud-teleport-testing-it-gitactions" \ + --it-private-connectivity="datastream-private-connect-us-central1" + - name: Upload Smoke Tests Report + uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2 + if: always() # always run even if the previous step fails + with: + name: surefire-test-results + path: '**/surefire-reports/TEST-*.xml' + retention-days: 1 + - name: Cleanup Java Environment + uses: ./.github/actions/cleanup-java-env + java_integration_tests_templates: + name: Dataflow Templates Integration Tests + needs: [java_integration_smoke_tests_templates] + timeout-minutes: 180 + # Run on any runner that matches all the specified runs-on values. + runs-on: [self-hosted, it] + steps: + - name: Checkout Code + uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0 + - name: Setup Environment + id: setup-env + uses: ./.github/actions/setup-env + - name: Run Integration Tests + run: | + ./cicd/run-it-tests \ + --modules-to-build="KAFKA" \ + --it-region="us-central1" \ + --it-project="cloud-teleport-testing" \ + --it-artifact-bucket="cloud-teleport-testing-it-gitactions" \ + --it-private-connectivity="datastream-private-connect-us-central1" + - name: Upload Integration Tests Report + uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2 + if: always() # always run even if the previous step fails + with: + name: surefire-test-results + path: '**/surefire-reports/TEST-*.xml' + retention-days: 1 + - name: Cleanup Java Environment + uses: ./.github/actions/cleanup-java-env + java_load_tests_templates: + if: contains(github.event.pull_request.labels.*.name, 'run-load-tests') + name: Dataflow Templates Load Tests + needs: [spotless_check, checkstyle_check, java_build, java_unit_tests, java_integration_tests_templates] + timeout-minutes: 600 + # Run on any runner that matches all the specified runs-on values. + runs-on: [self-hosted, perf] + steps: + - name: Checkout Code + uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0 + - name: Setup Environment + id: setup-env + uses: ./.github/actions/setup-env + - name: Run Load Tests + run: | + ./cicd/run-load-tests \ + --modules-to-build="KAFKA" \ + --it-region="us-central1" \ + --it-project="cloud-teleport-testing" \ + --it-artifact-bucket="cloud-teleport-testing-it-gitactions" \ + --it-private-connectivity="datastream-private-connect-us-central1" + - name: Cleanup Java Environment + uses: ./.github/actions/cleanup-java-env diff --git a/cicd/internal/flags/common-flags.go b/cicd/internal/flags/common-flags.go index ce69275a38..9b9e26c0f8 100644 --- a/cicd/internal/flags/common-flags.go +++ b/cicd/internal/flags/common-flags.go @@ -24,12 +24,14 @@ import ( const ( ALL = "ALL" SPANNER = "SPANNER" + + KAFKA = "KAFKA" ) // Avoid making these vars public. var ( modulesToBuild string - moduleMap = map[string]string{ALL: "", SPANNER: "v2/datastream-to-spanner/,v2/spanner-change-streams-to-sharded-file-sink/,v2/gcs-to-sourcedb/,v2/sourcedb-to-spanner/,v2/spanner-custom-shard,plugins/templates-maven-plugin"} + moduleMap = map[string]string{ALL: "", SPANNER: "v2/datastream-to-spanner/,v2/spanner-change-streams-to-sharded-file-sink/,v2/gcs-to-sourcedb/,v2/sourcedb-to-spanner/,v2/spanner-custom-shard,plugins/templates-maven-plugin", KAFKA: "v2/kafka-to-bigquery/,v2/kafka-to-gcs/,v2/kafka-to-kafka/,v2/kafka-common/"} ) // Registers all common flags. Must be called before flag.Parse(). diff --git a/cicd/internal/flags/common-flags_test.go b/cicd/internal/flags/common-flags_test.go index 63b7aaec40..f65d0df6fb 100644 --- a/cicd/internal/flags/common-flags_test.go +++ b/cicd/internal/flags/common-flags_test.go @@ -42,6 +42,10 @@ func TestModulesToBuild(t *testing.T) { input: "SPANNER", expected: []string{"v2/datastream-to-spanner/", "v2/spanner-change-streams-to-sharded-file-sink/", "v2/gcs-to-sourcedb/", "v2/sourcedb-to-spanner/", "v2/spanner-custom-shard", "plugins/templates-maven-plugin"}, }, + { + input: "KAFKA", + expected: []string{"v2/kafka-to-bigquery/", "v2/kafka-to-gcs/"," v2/kafka-to-kafka/", "v2/kafka-common"}, + }, } for _, test := range tests { From 5113183a60e77104f7342d5387d6994e1490481d Mon Sep 17 00:00:00 2001 From: Anand Inguva Date: Fri, 28 Jun 2024 17:35:11 -0400 Subject: [PATCH 2/3] Fix test --- cicd/internal/flags/common-flags_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cicd/internal/flags/common-flags_test.go b/cicd/internal/flags/common-flags_test.go index f65d0df6fb..911f887f3c 100644 --- a/cicd/internal/flags/common-flags_test.go +++ b/cicd/internal/flags/common-flags_test.go @@ -44,7 +44,7 @@ func TestModulesToBuild(t *testing.T) { }, { input: "KAFKA", - expected: []string{"v2/kafka-to-bigquery/", "v2/kafka-to-gcs/"," v2/kafka-to-kafka/", "v2/kafka-common"}, + expected: []string{"v2/kafka-to-bigquery/", "v2/kafka-to-gcs/"," v2/kafka-to-kafka/", "v2/kafka-common/"}, }, } From b5f58079ff4cc7287b149375942164c150b5d05a Mon Sep 17 00:00:00 2001 From: Anand Inguva Date: Fri, 28 Jun 2024 17:41:48 -0400 Subject: [PATCH 3/3] Fix test --- cicd/internal/flags/common-flags_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cicd/internal/flags/common-flags_test.go b/cicd/internal/flags/common-flags_test.go index 911f887f3c..6f684ca515 100644 --- a/cicd/internal/flags/common-flags_test.go +++ b/cicd/internal/flags/common-flags_test.go @@ -44,7 +44,7 @@ func TestModulesToBuild(t *testing.T) { }, { input: "KAFKA", - expected: []string{"v2/kafka-to-bigquery/", "v2/kafka-to-gcs/"," v2/kafka-to-kafka/", "v2/kafka-common/"}, + expected: []string{"v2/kafka-to-bigquery/", "v2/kafka-to-gcs/","v2/kafka-to-kafka/", "v2/kafka-common/"}, }, }