From 0741f78c26595c14adfa2ff4197cd77be366ec0a Mon Sep 17 00:00:00 2001 From: Celeste Zeng <61256376+celeste-zeng@users.noreply.github.com> Date: Thu, 29 Jun 2023 21:25:30 +0000 Subject: [PATCH 01/59] Made changes to build and test multiarch Beam Python SDK containers --- ...tCommit_Python_ValidatesContainer_Dataflow.groovy | 2 ++ .../org/apache/beam/gradle/BeamModulePlugin.groovy | 7 ++++++- sdks/go/container/build.gradle | 2 ++ sdks/python/container/common.gradle | 2 ++ sdks/python/container/run_generate_requirements.sh | 12 ++++++++++++ 5 files changed, 24 insertions(+), 1 deletion(-) diff --git a/.test-infra/jenkins/job_PostCommit_Python_ValidatesContainer_Dataflow.groovy b/.test-infra/jenkins/job_PostCommit_Python_ValidatesContainer_Dataflow.groovy index 254096448668..193c124ddd8e 100644 --- a/.test-infra/jenkins/job_PostCommit_Python_ValidatesContainer_Dataflow.groovy +++ b/.test-infra/jenkins/job_PostCommit_Python_ValidatesContainer_Dataflow.groovy @@ -38,6 +38,8 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_Py_ValCont', steps { gradle { rootBuildScriptDir(commonJobProperties.checkoutDir) + switches('-Pcontainer-architecture-list=arm64,amd64') + switches('-Ppush-multiarch-containers') tasks(':sdks:python:test-suites:dataflow:validatesContainerTests') commonJobProperties.setGradleSwitches(delegate) } diff --git a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy index 4e6620c0bbc1..d736b5be4f0f 100644 --- a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy +++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy @@ -492,7 +492,12 @@ class BeamModulePlugin implements Plugin { // Ensure we always publish the expected containers. return ["amd64"]; } else if (project.rootProject.findProperty("container-architecture-list") != null) { - return project.rootProject.findProperty("container-architecture-list").split(',') + def containerArchitectures = project.rootProject.findProperty("container-architecture-list").split(',') + if (containerArchitectures.size() > 1 && !project.rootProject.hasProperty("push-multiarch-containers")) { + throw new GradleException("A multi-arch image can't be saved in the local image store, please append the -Ppush-multiarch-containers flag and specify a repository to push in the -Pdocker-repository-root flag."); + } + return containerArchitectures + } else { return [project.nativeArchitecture()] } diff --git a/sdks/go/container/build.gradle b/sdks/go/container/build.gradle index 9a9bbb32debb..d4ced0ad8ef8 100644 --- a/sdks/go/container/build.gradle +++ b/sdks/go/container/build.gradle @@ -40,6 +40,8 @@ docker { project.rootProject.hasProperty(["isRelease"])]) buildx project.containerPlatforms() != [project.nativeArchitecture()] platform(*project.containerPlatforms()) + load project.containerPlatforms() != [project.nativeArchitecture()] && !project.rootProject.hasProperty("push-multiarch-containers") + push project.rootProject.hasProperty("push-multiarch-containers") } dockerPrepare.dependsOn tasks.named("goBuild") diff --git a/sdks/python/container/common.gradle b/sdks/python/container/common.gradle index 48c059ae5ecd..2ce8e4dbb189 100644 --- a/sdks/python/container/common.gradle +++ b/sdks/python/container/common.gradle @@ -82,6 +82,8 @@ docker { project.rootProject.hasProperty(["isRelease"])]) buildx project.containerPlatforms() != [project.nativeArchitecture()] platform(*project.containerPlatforms()) + load project.containerPlatforms() != [project.nativeArchitecture()] && !project.rootProject.hasProperty("push-multiarch-containers") + push project.rootProject.hasProperty("push-multiarch-containers") } dockerPrepare.dependsOn copyLauncherDependencies diff --git a/sdks/python/container/run_generate_requirements.sh b/sdks/python/container/run_generate_requirements.sh index fd222107dba5..fdc2f540950d 100755 --- a/sdks/python/container/run_generate_requirements.sh +++ b/sdks/python/container/run_generate_requirements.sh @@ -104,4 +104,16 @@ EOT # Remove pkg_resources to guard against # https://stackoverflow.com/questions/39577984/what-is-pkg-resources-0-0-0-in-output-of-pip-freeze-command pip freeze | grep -v pkg_resources >> "$REQUIREMENTS_FILE" + +# Check if the generated .txt file has the line "tensorflow==". +if grep -q "tensorflow==" "$REQUIREMENTS_FILE"; then + +# Get the version of tensorflow from the .txt file. +VERSION=$(grep -Po "tensorflow==\K[^;]+" "$REQUIREMENTS_FILE") + +# Write the line tensorflow-cpu-aws==${VERSION};platform_machine=="aarch64" to the .txt file. +echo "tensorflow-cpu-aws==${VERSION};platform_machine==\"aarch64\"" >> "$REQUIREMENTS_FILE" + +fi + rm -rf "$ENV_PATH" From 20fe13719868a33f7123fa56e3f401a1cce5c840 Mon Sep 17 00:00:00 2001 From: Celeste Zeng <61256376+celeste-zeng@users.noreply.github.com> Date: Fri, 30 Jun 2023 22:10:27 +0000 Subject: [PATCH 02/59] Specified directory to push the multiarch containers and changed the script for validatescontainer --- ..._Python_ValidatesContainer_Dataflow.groovy | 2 ++ .../container/run_validatescontainer.sh | 21 +++++++++---------- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/.test-infra/jenkins/job_PostCommit_Python_ValidatesContainer_Dataflow.groovy b/.test-infra/jenkins/job_PostCommit_Python_ValidatesContainer_Dataflow.groovy index 193c124ddd8e..ea89e0e0980f 100644 --- a/.test-infra/jenkins/job_PostCommit_Python_ValidatesContainer_Dataflow.groovy +++ b/.test-infra/jenkins/job_PostCommit_Python_ValidatesContainer_Dataflow.groovy @@ -40,6 +40,8 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_Py_ValCont', rootBuildScriptDir(commonJobProperties.checkoutDir) switches('-Pcontainer-architecture-list=arm64,amd64') switches('-Ppush-multiarch-containers') + switches('-Pdocker-repository-root="us.gcr.io/apache-beam-testing/jenkins"') + switches('-Pdocker-tag="latest"') tasks(':sdks:python:test-suites:dataflow:validatesContainerTests') commonJobProperties.setGradleSwitches(delegate) } diff --git a/sdks/python/container/run_validatescontainer.sh b/sdks/python/container/run_validatescontainer.sh index be16ff619ed6..3c66c8a5a79d 100755 --- a/sdks/python/container/run_validatescontainer.sh +++ b/sdks/python/container/run_validatescontainer.sh @@ -65,23 +65,22 @@ command -v gcloud docker -v gcloud -v -# Verify docker image has been built. -docker images | grep "apache/$IMAGE_NAME" | grep "$SDK_VERSION" - -TAG=$(date +%Y%m%d-%H%M%S%N) CONTAINER=us.gcr.io/$PROJECT/$USER/$IMAGE_NAME PREBUILD_SDK_CONTAINER_REGISTRY_PATH=us.gcr.io/$PROJECT/$USER/prebuild_python${PY_VERSION//.}_sdk -echo "Using container $CONTAINER" - -# Tag the docker container. -docker tag "apache/$IMAGE_NAME:$SDK_VERSION" "$CONTAINER:$TAG" +TAG="latest" -# Push the container. -gcloud docker -- push $CONTAINER:$TAG +# Verify docker image has been built and pushed during the build: +# pull and tag the x86 components; +docker pull $CONTAINER:$TAG +docker tag "$CONTAINER:$TAG" "$CONTAINER-x86:$TAG" +# pull and tag the ARM components; +docker pull --platform linux/arm64 $CONTAINER:$TAG +docker tag "$CONTAINER:$TAG" "$CONTAINER-arm:$TAG" function cleanup_container { # Delete the container locally and remotely - docker rmi $CONTAINER:$TAG || echo "Failed to remove container image" + docker rmi $CONTAINER-x86:$TAG || echo "Failed to remove x86 container image" + docker rmi $CONTAINER-arm:$TAG || echo "Failed to remove arm container image" for image in $(docker images --format '{{.Repository}}:{{.Tag}}' | grep $PREBUILD_SDK_CONTAINER_REGISTRY_PATH) do docker rmi $image || echo "Failed to remove prebuilt sdk container image" done From 38a8398f90c03f698837013a721cbf93951f8bcf Mon Sep 17 00:00:00 2001 From: Celeste Zeng <61256376+celeste-zeng@users.noreply.github.com> Date: Fri, 30 Jun 2023 23:12:26 +0000 Subject: [PATCH 03/59] Changed the tag for the multiarch containers built during the validatescontainer test. --- .../job_PostCommit_Python_ValidatesContainer_Dataflow.groovy | 2 +- sdks/python/container/run_validatescontainer.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.test-infra/jenkins/job_PostCommit_Python_ValidatesContainer_Dataflow.groovy b/.test-infra/jenkins/job_PostCommit_Python_ValidatesContainer_Dataflow.groovy index ea89e0e0980f..56d05687128d 100644 --- a/.test-infra/jenkins/job_PostCommit_Python_ValidatesContainer_Dataflow.groovy +++ b/.test-infra/jenkins/job_PostCommit_Python_ValidatesContainer_Dataflow.groovy @@ -41,7 +41,7 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_Py_ValCont', switches('-Pcontainer-architecture-list=arm64,amd64') switches('-Ppush-multiarch-containers') switches('-Pdocker-repository-root="us.gcr.io/apache-beam-testing/jenkins"') - switches('-Pdocker-tag="latest"') + switches('-Pdocker-tag="multiarch-test"') tasks(':sdks:python:test-suites:dataflow:validatesContainerTests') commonJobProperties.setGradleSwitches(delegate) } diff --git a/sdks/python/container/run_validatescontainer.sh b/sdks/python/container/run_validatescontainer.sh index 3c66c8a5a79d..be787daff66a 100755 --- a/sdks/python/container/run_validatescontainer.sh +++ b/sdks/python/container/run_validatescontainer.sh @@ -67,7 +67,7 @@ gcloud -v CONTAINER=us.gcr.io/$PROJECT/$USER/$IMAGE_NAME PREBUILD_SDK_CONTAINER_REGISTRY_PATH=us.gcr.io/$PROJECT/$USER/prebuild_python${PY_VERSION//.}_sdk -TAG="latest" +TAG="multiarch-test" # Verify docker image has been built and pushed during the build: # pull and tag the x86 components; From ea45aeea10b75600f71de070c6803a511dacf4dc Mon Sep 17 00:00:00 2001 From: Celeste Zeng <61256376+celeste-zeng@users.noreply.github.com> Date: Sat, 1 Jul 2023 06:52:06 +0000 Subject: [PATCH 04/59] Correctly formatted tag and repository so the docker pull request for images built during testing will succeed. --- .../job_PostCommit_Python_ValidatesContainer_Dataflow.groovy | 4 ++-- sdks/python/container/run_validatescontainer.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.test-infra/jenkins/job_PostCommit_Python_ValidatesContainer_Dataflow.groovy b/.test-infra/jenkins/job_PostCommit_Python_ValidatesContainer_Dataflow.groovy index 56d05687128d..f1287866fb52 100644 --- a/.test-infra/jenkins/job_PostCommit_Python_ValidatesContainer_Dataflow.groovy +++ b/.test-infra/jenkins/job_PostCommit_Python_ValidatesContainer_Dataflow.groovy @@ -40,8 +40,8 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_Py_ValCont', rootBuildScriptDir(commonJobProperties.checkoutDir) switches('-Pcontainer-architecture-list=arm64,amd64') switches('-Ppush-multiarch-containers') - switches('-Pdocker-repository-root="us.gcr.io/apache-beam-testing/jenkins"') - switches('-Pdocker-tag="multiarch-test"') + switches('-Pdocker-repository-root=us.gcr.io/apache-beam-testing/jenkins') + switches('-Pdocker-tag=latest') tasks(':sdks:python:test-suites:dataflow:validatesContainerTests') commonJobProperties.setGradleSwitches(delegate) } diff --git a/sdks/python/container/run_validatescontainer.sh b/sdks/python/container/run_validatescontainer.sh index be787daff66a..03d381ded692 100755 --- a/sdks/python/container/run_validatescontainer.sh +++ b/sdks/python/container/run_validatescontainer.sh @@ -67,7 +67,7 @@ gcloud -v CONTAINER=us.gcr.io/$PROJECT/$USER/$IMAGE_NAME PREBUILD_SDK_CONTAINER_REGISTRY_PATH=us.gcr.io/$PROJECT/$USER/prebuild_python${PY_VERSION//.}_sdk -TAG="multiarch-test" +TAG=latest # Verify docker image has been built and pushed during the build: # pull and tag the x86 components; From f5f4f44f0a066f1f3eabcd70d2b2079ec90b4a20 Mon Sep 17 00:00:00 2001 From: Celeste Zeng <61256376+celeste-zeng@users.noreply.github.com> Date: Thu, 6 Jul 2023 05:48:35 +0000 Subject: [PATCH 05/59] Made changes so that python validatesContainer will automatically build and push multi-arch container images. --- ...ob_PostCommit_Python_ValidatesContainer_Dataflow.groovy | 7 +++---- .../groovy/org/apache/beam/gradle/BeamModulePlugin.groovy | 3 ++- sdks/python/container/common.gradle | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.test-infra/jenkins/job_PostCommit_Python_ValidatesContainer_Dataflow.groovy b/.test-infra/jenkins/job_PostCommit_Python_ValidatesContainer_Dataflow.groovy index f1287866fb52..f86d22e224e7 100644 --- a/.test-infra/jenkins/job_PostCommit_Python_ValidatesContainer_Dataflow.groovy +++ b/.test-infra/jenkins/job_PostCommit_Python_ValidatesContainer_Dataflow.groovy @@ -38,11 +38,10 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_Py_ValCont', steps { gradle { rootBuildScriptDir(commonJobProperties.checkoutDir) - switches('-Pcontainer-architecture-list=arm64,amd64') - switches('-Ppush-multiarch-containers') - switches('-Pdocker-repository-root=us.gcr.io/apache-beam-testing/jenkins') - switches('-Pdocker-tag=latest') tasks(':sdks:python:test-suites:dataflow:validatesContainerTests') + switches('-Pbuild-and-push-multiarch-containers') + // Set testRCDependencies to re-generates the base image requirements. + switches('-PtestRCDependencies=true') commonJobProperties.setGradleSwitches(delegate) } } diff --git a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy index d736b5be4f0f..4e923b5f0cbe 100644 --- a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy +++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy @@ -497,7 +497,8 @@ class BeamModulePlugin implements Plugin { throw new GradleException("A multi-arch image can't be saved in the local image store, please append the -Ppush-multiarch-containers flag and specify a repository to push in the -Pdocker-repository-root flag."); } return containerArchitectures - + } else if project.hasProperty("build-and-push-multiarch-containers") { + return ["arm64","amd64"] } else { return [project.nativeArchitecture()] } diff --git a/sdks/python/container/common.gradle b/sdks/python/container/common.gradle index 2ce8e4dbb189..36c08cf0a088 100644 --- a/sdks/python/container/common.gradle +++ b/sdks/python/container/common.gradle @@ -83,7 +83,7 @@ docker { buildx project.containerPlatforms() != [project.nativeArchitecture()] platform(*project.containerPlatforms()) load project.containerPlatforms() != [project.nativeArchitecture()] && !project.rootProject.hasProperty("push-multiarch-containers") - push project.rootProject.hasProperty("push-multiarch-containers") + push project.rootProject.hasProperty("push-multiarch-containers") || project.hasProperty("build-and-push-multiarch-containers") } dockerPrepare.dependsOn copyLauncherDependencies From 5b53d5920a97bf00c81d7c1c7793da93104ae430 Mon Sep 17 00:00:00 2001 From: Celeste Zeng <61256376+celeste-zeng@users.noreply.github.com> Date: Thu, 6 Jul 2023 06:48:32 +0000 Subject: [PATCH 06/59] fix format errors --- .../apache/beam/gradle/BeamModulePlugin.groovy | 2 +- sdks/python/container/common.gradle | 15 +++++++++++---- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy index 4e923b5f0cbe..fe40cd2b3211 100644 --- a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy +++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy @@ -498,7 +498,7 @@ class BeamModulePlugin implements Plugin { } return containerArchitectures } else if project.hasProperty("build-and-push-multiarch-containers") { - return ["arm64","amd64"] + return ["arm64", "amd64"]; } else { return [project.nativeArchitecture()] } diff --git a/sdks/python/container/common.gradle b/sdks/python/container/common.gradle index 36c08cf0a088..a8d56d97a708 100644 --- a/sdks/python/container/common.gradle +++ b/sdks/python/container/common.gradle @@ -66,6 +66,13 @@ def copyLauncherDependencies = tasks.register("copyLauncherDependencies", Copy) } } +def pushContainers = project.rootProject.hasProperty("push-multiarch-containers") || project.hasProperty("build-and-push-multiarch-containers") + +def containerPlatforms = project.hasProperty("build-and-push-multiarch-containers")? +["linux/arm64","linux/amd64"] : project.containerPlatforms() + +def useBuildx = project.containerPlatforms() != [project.nativeArchitecture()] + docker { name containerImageName( name: project.docker_image_default_repo_prefix + "python${project.ext.pythonVersion}_sdk", @@ -80,10 +87,10 @@ docker { buildArgs(['py_version': "${project.ext.pythonVersion}", 'pull_licenses': project.rootProject.hasProperty(["docker-pull-licenses"]) || project.rootProject.hasProperty(["isRelease"])]) - buildx project.containerPlatforms() != [project.nativeArchitecture()] - platform(*project.containerPlatforms()) - load project.containerPlatforms() != [project.nativeArchitecture()] && !project.rootProject.hasProperty("push-multiarch-containers") - push project.rootProject.hasProperty("push-multiarch-containers") || project.hasProperty("build-and-push-multiarch-containers") + buildx useBuildx + platform(containerPlatforms) + load useBuildx && !pushContainers + push pushContainers } dockerPrepare.dependsOn copyLauncherDependencies From bec91cdf1c71059f4780371546fbf48d2acc5a30 Mon Sep 17 00:00:00 2001 From: Celeste Zeng <61256376+celeste-zeng@users.noreply.github.com> Date: Thu, 6 Jul 2023 06:59:29 +0000 Subject: [PATCH 07/59] minor errors --- .../main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy | 2 +- sdks/python/container/common.gradle | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy index fe40cd2b3211..6fcb674205bb 100644 --- a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy +++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy @@ -497,7 +497,7 @@ class BeamModulePlugin implements Plugin { throw new GradleException("A multi-arch image can't be saved in the local image store, please append the -Ppush-multiarch-containers flag and specify a repository to push in the -Pdocker-repository-root flag."); } return containerArchitectures - } else if project.hasProperty("build-and-push-multiarch-containers") { + } else if (project.hasProperty("build-and-push-multiarch-containers")) { return ["arm64", "amd64"]; } else { return [project.nativeArchitecture()] diff --git a/sdks/python/container/common.gradle b/sdks/python/container/common.gradle index a8d56d97a708..35b263ab084f 100644 --- a/sdks/python/container/common.gradle +++ b/sdks/python/container/common.gradle @@ -88,7 +88,7 @@ docker { 'pull_licenses': project.rootProject.hasProperty(["docker-pull-licenses"]) || project.rootProject.hasProperty(["isRelease"])]) buildx useBuildx - platform(containerPlatforms) + platform(*containerPlatforms) load useBuildx && !pushContainers push pushContainers } From 383c4c4c1a0b897f9285b774c5334b224901eb98 Mon Sep 17 00:00:00 2001 From: Celeste Zeng <61256376+celeste-zeng@users.noreply.github.com> Date: Thu, 6 Jul 2023 07:24:53 +0000 Subject: [PATCH 08/59] remove unnecessary changes --- .../job_PostCommit_Python_ValidatesContainer_Dataflow.groovy | 4 ++-- sdks/python/container/common.gradle | 5 +---- sdks/python/test-suites/dataflow/common.gradle | 2 +- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/.test-infra/jenkins/job_PostCommit_Python_ValidatesContainer_Dataflow.groovy b/.test-infra/jenkins/job_PostCommit_Python_ValidatesContainer_Dataflow.groovy index f86d22e224e7..1cd4a85a6a88 100644 --- a/.test-infra/jenkins/job_PostCommit_Python_ValidatesContainer_Dataflow.groovy +++ b/.test-infra/jenkins/job_PostCommit_Python_ValidatesContainer_Dataflow.groovy @@ -40,8 +40,8 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_Py_ValCont', rootBuildScriptDir(commonJobProperties.checkoutDir) tasks(':sdks:python:test-suites:dataflow:validatesContainerTests') switches('-Pbuild-and-push-multiarch-containers') - // Set testRCDependencies to re-generates the base image requirements. - switches('-PtestRCDependencies=true') + // Regenerates the base image requirements to accommodate ARM. + switches('-Pregenerate-requirements') commonJobProperties.setGradleSwitches(delegate) } } diff --git a/sdks/python/container/common.gradle b/sdks/python/container/common.gradle index 35b263ab084f..e064b79058f6 100644 --- a/sdks/python/container/common.gradle +++ b/sdks/python/container/common.gradle @@ -68,9 +68,6 @@ def copyLauncherDependencies = tasks.register("copyLauncherDependencies", Copy) def pushContainers = project.rootProject.hasProperty("push-multiarch-containers") || project.hasProperty("build-and-push-multiarch-containers") -def containerPlatforms = project.hasProperty("build-and-push-multiarch-containers")? -["linux/arm64","linux/amd64"] : project.containerPlatforms() - def useBuildx = project.containerPlatforms() != [project.nativeArchitecture()] docker { @@ -88,7 +85,7 @@ docker { 'pull_licenses': project.rootProject.hasProperty(["docker-pull-licenses"]) || project.rootProject.hasProperty(["isRelease"])]) buildx useBuildx - platform(*containerPlatforms) + platform(*project.containerPlatforms()) load useBuildx && !pushContainers push pushContainers } diff --git a/sdks/python/test-suites/dataflow/common.gradle b/sdks/python/test-suites/dataflow/common.gradle index 36fc75f2e412..fd26c965b68c 100644 --- a/sdks/python/test-suites/dataflow/common.gradle +++ b/sdks/python/test-suites/dataflow/common.gradle @@ -351,7 +351,7 @@ task chicagoTaxiExample { task validatesContainer() { def pyversion = "${project.ext.pythonVersion.replace('.', '')}" - if (project.hasProperty("testRCDependencies")) { + if (project.hasProperty("testRCDependencies") || project.hasProperty("regenerate-requirements")) { // Generate a requirements file with pre-release versions for the docker task // if testing with pre-release dependencies. dependsOn ":sdks:python:container:py${pyversion}:generatePythonRequirements" From 03759eb19cb77f5d8740d1306e7e02797879133e Mon Sep 17 00:00:00 2001 From: Celeste Zeng <61256376+celeste-zeng@users.noreply.github.com> Date: Tue, 11 Jul 2023 18:15:43 +0000 Subject: [PATCH 09/59] Made changes to build Java and Go multi-arch Beam container images --- sdks/go/container/build.gradle | 10 +++++++--- sdks/java/container/common.gradle | 9 +++++++-- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/sdks/go/container/build.gradle b/sdks/go/container/build.gradle index d4ced0ad8ef8..701379aa9650 100644 --- a/sdks/go/container/build.gradle +++ b/sdks/go/container/build.gradle @@ -27,6 +27,10 @@ goBuild { outputLocation = './build/target/${GOOS}_${GOARCH}/boot' } +def pushContainers = project.rootProject.hasProperty("push-multiarch-containers") || project.hasProperty("build-and-push-multiarch-containers") + +def useBuildx = project.containerPlatforms() != [project.nativeArchitecture()] + docker { name containerImageName( name: project.docker_image_default_repo_prefix + "go_sdk", @@ -38,10 +42,10 @@ docker { files "./build/" buildArgs(['pull_licenses': project.rootProject.hasProperty(["docker-pull-licenses"]) || project.rootProject.hasProperty(["isRelease"])]) - buildx project.containerPlatforms() != [project.nativeArchitecture()] + buildx useBuildx platform(*project.containerPlatforms()) - load project.containerPlatforms() != [project.nativeArchitecture()] && !project.rootProject.hasProperty("push-multiarch-containers") - push project.rootProject.hasProperty("push-multiarch-containers") + load useBuildx && !pushContainers + push pushContainers } dockerPrepare.dependsOn tasks.named("goBuild") diff --git a/sdks/java/container/common.gradle b/sdks/java/container/common.gradle index 5b2130f27814..edf2d48ac003 100644 --- a/sdks/java/container/common.gradle +++ b/sdks/java/container/common.gradle @@ -103,6 +103,9 @@ task validateJavaHome { } } } +def pushContainers = project.rootProject.hasProperty("push-multiarch-containers") || project.hasProperty("build-and-push-multiarch-containers") + +def useBuildx = project.containerPlatforms() != [project.nativeArchitecture()] docker { name containerImageName( @@ -121,8 +124,10 @@ docker { project.rootProject.hasProperty(["isRelease"]), 'java_version': imageJavaVersion, ]) - buildx project.containerPlatforms() != [project.nativeArchitecture()] - platform(*project.containerPlatforms()) + buildx useBuildx + platform(*project.containerPlatforms()) + load useBuildx && !pushContainers + push pushContainers } if (project.rootProject.hasProperty(["docker-pull-licenses"]) || From f5a2b7481336e9b9e76427c3d3a9b05ee1633af3 Mon Sep 17 00:00:00 2001 From: Celeste Zeng <61256376+celeste-zeng@users.noreply.github.com> Date: Tue, 11 Jul 2023 19:24:30 +0000 Subject: [PATCH 10/59] Make the postcommit go and java to build and test multi-arch containers. --- .test-infra/jenkins/job_PostCommit_Go.groovy | 1 + .test-infra/jenkins/job_PostCommit_Java.groovy | 1 + sdks/java/container/common.gradle | 8 ++++---- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.test-infra/jenkins/job_PostCommit_Go.groovy b/.test-infra/jenkins/job_PostCommit_Go.groovy index 3d1c6a3b53ff..854eb7b93fd6 100644 --- a/.test-infra/jenkins/job_PostCommit_Go.groovy +++ b/.test-infra/jenkins/job_PostCommit_Go.groovy @@ -38,6 +38,7 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_Go', 'Run Go PostCommit', tasks(':goPostCommit') commonJobProperties.setGradleSwitches(delegate) switches('--no-parallel') + switches('-Pbuild-and-push-multiarch-containers') } } } diff --git a/.test-infra/jenkins/job_PostCommit_Java.groovy b/.test-infra/jenkins/job_PostCommit_Java.groovy index cb3e0930dc89..16459014a421 100644 --- a/.test-infra/jenkins/job_PostCommit_Java.groovy +++ b/.test-infra/jenkins/job_PostCommit_Java.groovy @@ -44,6 +44,7 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java', 'Run Java PostCommit' commonJobProperties.setGradleSwitches(delegate) // Specify maven home on Jenkins, needed by Maven archetype integration tests. switches('-Pmaven_home=/home/jenkins/tools/maven/apache-maven-3.5.4') + switches('-Pbuild-and-push-multiarch-containers') } } } diff --git a/sdks/java/container/common.gradle b/sdks/java/container/common.gradle index edf2d48ac003..e44b82dd722e 100644 --- a/sdks/java/container/common.gradle +++ b/sdks/java/container/common.gradle @@ -124,10 +124,10 @@ docker { project.rootProject.hasProperty(["isRelease"]), 'java_version': imageJavaVersion, ]) - buildx useBuildx - platform(*project.containerPlatforms()) - load useBuildx && !pushContainers - push pushContainers + buildx useBuildx + platform(*project.containerPlatforms()) + load useBuildx && !pushContainers + push pushContainers } if (project.rootProject.hasProperty(["docker-pull-licenses"]) || From 8c2494d253cf65277d6e3727be4e734248348ebb Mon Sep 17 00:00:00 2001 From: Celeste Zeng <61256376+celeste-zeng@users.noreply.github.com> Date: Tue, 11 Jul 2023 21:04:29 +0000 Subject: [PATCH 11/59] Add building multiarch Java containers to the PostCommit Java job_PostCommit_Java_Examples_Dataflow_V2 test suite for diff Java versions. --- .test-infra/jenkins/job_PostCommit_Java.groovy | 1 - .../jenkins/job_PostCommit_Java_Examples_Dataflow_V2.groovy | 2 +- .../job_PostCommit_Java_Examples_Dataflow_V2_Java11.groovy | 2 +- .../job_PostCommit_Java_Examples_Dataflow_V2_Java17.groovy | 2 +- runners/google-cloud-dataflow-java/build.gradle | 1 + 5 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.test-infra/jenkins/job_PostCommit_Java.groovy b/.test-infra/jenkins/job_PostCommit_Java.groovy index 16459014a421..cb3e0930dc89 100644 --- a/.test-infra/jenkins/job_PostCommit_Java.groovy +++ b/.test-infra/jenkins/job_PostCommit_Java.groovy @@ -44,7 +44,6 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java', 'Run Java PostCommit' commonJobProperties.setGradleSwitches(delegate) // Specify maven home on Jenkins, needed by Maven archetype integration tests. switches('-Pmaven_home=/home/jenkins/tools/maven/apache-maven-3.5.4') - switches('-Pbuild-and-push-multiarch-containers') } } } diff --git a/.test-infra/jenkins/job_PostCommit_Java_Examples_Dataflow_V2.groovy b/.test-infra/jenkins/job_PostCommit_Java_Examples_Dataflow_V2.groovy index 5bfda20db3cf..76fff5ebba46 100644 --- a/.test-infra/jenkins/job_PostCommit_Java_Examples_Dataflow_V2.groovy +++ b/.test-infra/jenkins/job_PostCommit_Java_Examples_Dataflow_V2.groovy @@ -34,7 +34,7 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_Examples_Dataflow_V2', gradle { rootBuildScriptDir(commonJobProperties.checkoutDir) tasks(':runners:google-cloud-dataflow-java:examplesJavaRunnerV2IntegrationTest') - + switches('-Pbuild-and-push-multiarch-containers') // Increase parallel worker threads above processor limit since most time is // spent waiting on Dataflow jobs. ValidatesRunner tests on Dataflow are slow // because each one launches a Dataflow job with about 3 mins of overhead. diff --git a/.test-infra/jenkins/job_PostCommit_Java_Examples_Dataflow_V2_Java11.groovy b/.test-infra/jenkins/job_PostCommit_Java_Examples_Dataflow_V2_Java11.groovy index 6687ae0e6f8a..df035f14a01c 100644 --- a/.test-infra/jenkins/job_PostCommit_Java_Examples_Dataflow_V2_Java11.groovy +++ b/.test-infra/jenkins/job_PostCommit_Java_Examples_Dataflow_V2_Java11.groovy @@ -34,7 +34,7 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_Examples_Dataflow_V2_ja gradle { rootBuildScriptDir(commonJobProperties.checkoutDir) tasks(':runners:google-cloud-dataflow-java:examplesJavaRunnerV2IntegrationTest') - + switches('-Pbuild-and-push-multiarch-containers') // Increase parallel worker threads above processor limit since most time is // spent waiting on Dataflow jobs. ValidatesRunner tests on Dataflow are slow // because each one launches a Dataflow job with about 3 mins of overhead. diff --git a/.test-infra/jenkins/job_PostCommit_Java_Examples_Dataflow_V2_Java17.groovy b/.test-infra/jenkins/job_PostCommit_Java_Examples_Dataflow_V2_Java17.groovy index b275fe9276d9..8fe2c321a600 100644 --- a/.test-infra/jenkins/job_PostCommit_Java_Examples_Dataflow_V2_Java17.groovy +++ b/.test-infra/jenkins/job_PostCommit_Java_Examples_Dataflow_V2_Java17.groovy @@ -34,7 +34,7 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_Examples_Dataflow_V2_ja gradle { rootBuildScriptDir(commonJobProperties.checkoutDir) tasks(':runners:google-cloud-dataflow-java:examplesJavaRunnerV2IntegrationTest') - + switches('-Pbuild-and-push-multiarch-containers') // Increase parallel worker threads above processor limit since most time is // spent waiting on Dataflow jobs. ValidatesRunner tests on Dataflow are slow // because each one launches a Dataflow job with about 3 mins of overhead. diff --git a/runners/google-cloud-dataflow-java/build.gradle b/runners/google-cloud-dataflow-java/build.gradle index 4966b59e8d60..0612c615f63d 100644 --- a/runners/google-cloud-dataflow-java/build.gradle +++ b/runners/google-cloud-dataflow-java/build.gradle @@ -279,6 +279,7 @@ def buildAndPushDockerJavaContainer = tasks.register("buildAndPushDockerJavaCont javaVer = "java11" } dependsOn ":sdks:java:container:${javaVer}:docker" + // TODO: change the following since multi-arch Java containers have been pushed during build def defaultDockerImageName = containerImageName( name: "${project.docker_image_default_repo_prefix}${javaVer}_sdk", root: "apache", From 1fdf4cba8865397a3a5622439c2591afe79cb8e2 Mon Sep 17 00:00:00 2001 From: Celeste Zeng <61256376+celeste-zeng@users.noreply.github.com> Date: Tue, 11 Jul 2023 21:47:01 +0000 Subject: [PATCH 12/59] Add unique tag for multiarch Beam Python containers in validatesContainer test suite. --- .../job_PostCommit_Python_ValidatesContainer_Dataflow.groovy | 4 ++++ sdks/python/container/run_validatescontainer.sh | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.test-infra/jenkins/job_PostCommit_Python_ValidatesContainer_Dataflow.groovy b/.test-infra/jenkins/job_PostCommit_Python_ValidatesContainer_Dataflow.groovy index 1cd4a85a6a88..8c594889ae35 100644 --- a/.test-infra/jenkins/job_PostCommit_Python_ValidatesContainer_Dataflow.groovy +++ b/.test-infra/jenkins/job_PostCommit_Python_ValidatesContainer_Dataflow.groovy @@ -20,6 +20,7 @@ import CommonJobProperties as commonJobProperties import PostcommitJobBuilder import static PythonTestProperties.VALIDATES_CONTAINER_DATAFLOW_PYTHON_VERSIONS +import java.util.UUID // This job runs the suite of Python ValidatesContainer tests against the // Dataflow runner. @@ -34,6 +35,8 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_Py_ValCont', archiveJunit('**/pytest*.xml') } + // Generates a unique tag for the container. + def unique_tag = UUID.randomUUID() // Execute shell command to test Python SDK. steps { gradle { @@ -42,6 +45,7 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_Py_ValCont', switches('-Pbuild-and-push-multiarch-containers') // Regenerates the base image requirements to accommodate ARM. switches('-Pregenerate-requirements') + switches('-Pdocker-tag=${unique_tag}') commonJobProperties.setGradleSwitches(delegate) } } diff --git a/sdks/python/container/run_validatescontainer.sh b/sdks/python/container/run_validatescontainer.sh index 03d381ded692..65d8a4e2646c 100755 --- a/sdks/python/container/run_validatescontainer.sh +++ b/sdks/python/container/run_validatescontainer.sh @@ -67,7 +67,7 @@ gcloud -v CONTAINER=us.gcr.io/$PROJECT/$USER/$IMAGE_NAME PREBUILD_SDK_CONTAINER_REGISTRY_PATH=us.gcr.io/$PROJECT/$USER/prebuild_python${PY_VERSION//.}_sdk -TAG=latest +TAG="$(grep 'unique_tag' job_PostCommit_Python_ValidatesContainer_Dataflow.groovy | cut -d'=' -f2)" # Verify docker image has been built and pushed during the build: # pull and tag the x86 components; From cdfb58203fc6b343f5d87592be848d344d8cbb23 Mon Sep 17 00:00:00 2001 From: Celeste Zeng <61256376+celeste-zeng@users.noreply.github.com> Date: Tue, 11 Jul 2023 22:47:54 +0000 Subject: [PATCH 13/59] Hardcode the repository to push multiarch Beam Python SDK container images. --- .../job_PostCommit_Python_ValidatesContainer_Dataflow.groovy | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.test-infra/jenkins/job_PostCommit_Python_ValidatesContainer_Dataflow.groovy b/.test-infra/jenkins/job_PostCommit_Python_ValidatesContainer_Dataflow.groovy index 8c594889ae35..e9d9ab741370 100644 --- a/.test-infra/jenkins/job_PostCommit_Python_ValidatesContainer_Dataflow.groovy +++ b/.test-infra/jenkins/job_PostCommit_Python_ValidatesContainer_Dataflow.groovy @@ -45,6 +45,8 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_Py_ValCont', switches('-Pbuild-and-push-multiarch-containers') // Regenerates the base image requirements to accommodate ARM. switches('-Pregenerate-requirements') + // Push multi-arch containers to the repository set in run_validatescontainer.sh + switches('-Pdocker-repository-root=us.gcr.io/apache-beam-testing/jenkins') switches('-Pdocker-tag=${unique_tag}') commonJobProperties.setGradleSwitches(delegate) } From 5b4931a17eaa2687c25ab4622fdef3e52c3d5b07 Mon Sep 17 00:00:00 2001 From: Celeste Zeng <61256376+celeste-zeng@users.noreply.github.com> Date: Tue, 11 Jul 2023 23:21:39 +0000 Subject: [PATCH 14/59] fix format error for container tag --- ...job_PostCommit_Python_ValidatesContainer_Dataflow.groovy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.test-infra/jenkins/job_PostCommit_Python_ValidatesContainer_Dataflow.groovy b/.test-infra/jenkins/job_PostCommit_Python_ValidatesContainer_Dataflow.groovy index e9d9ab741370..e66dec229899 100644 --- a/.test-infra/jenkins/job_PostCommit_Python_ValidatesContainer_Dataflow.groovy +++ b/.test-infra/jenkins/job_PostCommit_Python_ValidatesContainer_Dataflow.groovy @@ -20,7 +20,7 @@ import CommonJobProperties as commonJobProperties import PostcommitJobBuilder import static PythonTestProperties.VALIDATES_CONTAINER_DATAFLOW_PYTHON_VERSIONS -import java.util.UUID +import java.time.LocalDateTime // This job runs the suite of Python ValidatesContainer tests against the // Dataflow runner. @@ -35,8 +35,8 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_Py_ValCont', archiveJunit('**/pytest*.xml') } - // Generates a unique tag for the container. - def unique_tag = UUID.randomUUID() + // Generates a unique tag for the container as the current time. + def unique_tag = '${now.date}${now.hour}${now.minute}${now.second}' // Execute shell command to test Python SDK. steps { gradle { From 357a3d3c508605da7c40643d5d8fe7e939b6a346 Mon Sep 17 00:00:00 2001 From: Celeste Zeng <61256376+celeste-zeng@users.noreply.github.com> Date: Wed, 12 Jul 2023 19:37:01 +0000 Subject: [PATCH 15/59] Make the Java postcommit dataflow test suite to build and use multiarch Java containers. --- ...ostCommit_Java_Examples_Dataflow_V2.groovy | 3 +- ...it_Java_Examples_Dataflow_V2_Java11.groovy | 3 +- ...it_Java_Examples_Dataflow_V2_Java17.groovy | 3 +- .../google-cloud-dataflow-java/build.gradle | 65 ++++++++++++------- 4 files changed, 47 insertions(+), 27 deletions(-) diff --git a/.test-infra/jenkins/job_PostCommit_Java_Examples_Dataflow_V2.groovy b/.test-infra/jenkins/job_PostCommit_Java_Examples_Dataflow_V2.groovy index 76fff5ebba46..9f6d8fb2ca5a 100644 --- a/.test-infra/jenkins/job_PostCommit_Java_Examples_Dataflow_V2.groovy +++ b/.test-infra/jenkins/job_PostCommit_Java_Examples_Dataflow_V2.groovy @@ -34,13 +34,14 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_Examples_Dataflow_V2', gradle { rootBuildScriptDir(commonJobProperties.checkoutDir) tasks(':runners:google-cloud-dataflow-java:examplesJavaRunnerV2IntegrationTest') - switches('-Pbuild-and-push-multiarch-containers') // Increase parallel worker threads above processor limit since most time is // spent waiting on Dataflow jobs. ValidatesRunner tests on Dataflow are slow // because each one launches a Dataflow job with about 3 mins of overhead. // 3 x num_cores strikes a good balance between maxing out parallelism without // overloading the machines. commonJobProperties.setGradleSwitches(delegate, 3 * Runtime.runtime.availableProcessors()) + switches '-Pbuild-and-push-multiarch-containers' + switches "-docker-repository-root=us.gcr.io/apache-beam-testing/java-postcommit-it" } } } diff --git a/.test-infra/jenkins/job_PostCommit_Java_Examples_Dataflow_V2_Java11.groovy b/.test-infra/jenkins/job_PostCommit_Java_Examples_Dataflow_V2_Java11.groovy index df035f14a01c..67ea3dee8614 100644 --- a/.test-infra/jenkins/job_PostCommit_Java_Examples_Dataflow_V2_Java11.groovy +++ b/.test-infra/jenkins/job_PostCommit_Java_Examples_Dataflow_V2_Java11.groovy @@ -34,7 +34,6 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_Examples_Dataflow_V2_ja gradle { rootBuildScriptDir(commonJobProperties.checkoutDir) tasks(':runners:google-cloud-dataflow-java:examplesJavaRunnerV2IntegrationTest') - switches('-Pbuild-and-push-multiarch-containers') // Increase parallel worker threads above processor limit since most time is // spent waiting on Dataflow jobs. ValidatesRunner tests on Dataflow are slow // because each one launches a Dataflow job with about 3 mins of overhead. @@ -46,6 +45,8 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_Examples_Dataflow_V2_ja switches '-PcompileAndRunTestsWithJava11' switches '-PskipCheckerFramework' switches "-Pjava11Home=${commonJobProperties.JAVA_11_HOME}" + switches '-Pbuild-and-push-multiarch-containers' + switches "-Pdocker-repository-root=us.gcr.io/apache-beam-testing/java-postcommit-it" } } } diff --git a/.test-infra/jenkins/job_PostCommit_Java_Examples_Dataflow_V2_Java17.groovy b/.test-infra/jenkins/job_PostCommit_Java_Examples_Dataflow_V2_Java17.groovy index 8fe2c321a600..d62ac142055e 100644 --- a/.test-infra/jenkins/job_PostCommit_Java_Examples_Dataflow_V2_Java17.groovy +++ b/.test-infra/jenkins/job_PostCommit_Java_Examples_Dataflow_V2_Java17.groovy @@ -34,7 +34,6 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_Examples_Dataflow_V2_ja gradle { rootBuildScriptDir(commonJobProperties.checkoutDir) tasks(':runners:google-cloud-dataflow-java:examplesJavaRunnerV2IntegrationTest') - switches('-Pbuild-and-push-multiarch-containers') // Increase parallel worker threads above processor limit since most time is // spent waiting on Dataflow jobs. ValidatesRunner tests on Dataflow are slow // because each one launches a Dataflow job with about 3 mins of overhead. @@ -46,6 +45,8 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_Examples_Dataflow_V2_ja switches '-PcompileAndRunTestsWithJava17' switches '-PskipCheckerFramework' switches "-Pjava17Home=${commonJobProperties.JAVA_17_HOME}" + switches '-Pbuild-and-push-multiarch-containers' + switches "-Pdocker-repository-root=us.gcr.io/apache-beam-testing/java-postcommit-it" } } } diff --git a/runners/google-cloud-dataflow-java/build.gradle b/runners/google-cloud-dataflow-java/build.gradle index 0612c615f63d..05a8c76903d3 100644 --- a/runners/google-cloud-dataflow-java/build.gradle +++ b/runners/google-cloud-dataflow-java/build.gradle @@ -144,11 +144,18 @@ def dataflowLegacyWorkerJar = project.findProperty('dataflowWorkerJar') ?: proje def dataflowKmsKey = project.findProperty('dataflowKmsKey') ?: "projects/apache-beam-testing/locations/global/keyRings/beam-it/cryptoKeys/test" def firestoreDb = project.findProperty('firestoreDb') ?: 'firestoredb' -def dockerImageRoot = project.findProperty('dockerImageRoot') ?: "us.gcr.io/${dataflowProject}/java-postcommit-it" -def dockerJavaImageContainer = "${dockerImageRoot}/java" -def dockerPythonImageContainer = "${dockerImageRoot}/python" -def dockerTag = new Date().format('yyyyMMddHHmmss') -ext.dockerJavaImageName = "${dockerJavaImageContainer}:${dockerTag}" +def javaVer = "java8" + if(project.hasProperty('compileAndRunTestsWithJava17')) { + javaVer = "java17" + } else if(project.hasProperty('compileAndRunTestsWithJava11')) { + javaVer = "java11" + } +def pythonVer = project.project(':sdks:python').pythonVersion +def dockerImageRoot = project.findProperty('docker-repository-root') ?: "us.gcr.io/${dataflowProject}/java-postcommit-it" +def dockerJavaMultiarchImageContainer = "${dockerImageRoot}/${project.docker_image_default_repo_prefix}java${javaVer}_sdk" +def dockerPythonImageContainer = "${dockerImageRoot}/${project.docker_image_default_repo_prefix}python${pythonVer}_sdk" +def dockerTag = project.findProperty('docker-tag') ?: new Date().format('yyyyMMddHHmmss') +ext.dockerJavaMultiarchImageName = "${dockerJavaMultiarchImageContainer}:${dockerTag}" ext.dockerPythonImageName = "${dockerPythonImageContainer}:${dockerTag}" def legacyPipelineOptions = [ @@ -165,7 +172,7 @@ def runnerV2PipelineOptions = [ "--project=${dataflowProject}", "--region=${dataflowRegion}", "--tempRoot=${dataflowValidatesTempRoot}", - "--sdkContainerImage=${dockerJavaImageContainer}:${dockerTag}", + "--sdkContainerImage=${dockerJavaMultiarchImageContainer}:${dockerTag}", "--experiments=use_unified_worker,use_runner_v2", "--firestoreDb=${firestoreDb}", ] @@ -272,39 +279,50 @@ def createRunnerV2ValidatesRunnerTest = { Map args -> // task directly ('dependsOn buildAndPushDockerJavaContainer'). This ensures the correct // task ordering such that the registry doesn't get cleaned up prior to task completion. def buildAndPushDockerJavaContainer = tasks.register("buildAndPushDockerJavaContainer") { - def javaVer = "java8" - if(project.hasProperty('compileAndRunTestsWithJava17')) { - javaVer = "java17" - } else if(project.hasProperty('compileAndRunTestsWithJava11')) { - javaVer = "java11" - } dependsOn ":sdks:java:container:${javaVer}:docker" // TODO: change the following since multi-arch Java containers have been pushed during build - def defaultDockerImageName = containerImageName( - name: "${project.docker_image_default_repo_prefix}${javaVer}_sdk", + // apache/beam_java11_sdk:tag + def localJavax86ImageName = containerImageName( + name: "${project.docker_image_default_repo_prefix}${javaVer}_sdk-x86", root: "apache", - tag: project.sdk_version) + tag: dockerTag) + def localJavaArmImageName = containerImageName( + name: "${project.docker_image_default_repo_prefix}${javaVer}_sdk-arm", + root: "apache", + tag: dockerTag) doLast { + // Verify multi-arch Java docker image has been built and pushed during the build + // by pulling and tagging the x86 and arm components: + exec { + commandLine "docker", "pull", "${dockerJavaMultiarchImageName}" + } exec { - commandLine "docker", "tag", "${defaultDockerImageName}", "${dockerJavaImageName}" + commandLine "docker", "tag", "${dockerJavaMultiarchImageName}", "${localJavax86ImageName}" } exec { - commandLine "gcloud", "docker", "--", "push", "${dockerJavaImageName}" + commandLine "docker", "pull", "--platform=linux/arm64", "${dockerJavaMultiarchImageName}" + } + exec { + commandLine "docker", "tag", "${dockerJavaMultiarchImageName}", "${localJavaArmImageName}" } } } +// TODO: change here for multi arch containers // Clean up built Java images def cleanUpDockerJavaImages = tasks.register("cleanUpDockerJavaImages") { doLast { exec { - commandLine "docker", "rmi", "--force", "${dockerJavaImageName}" + commandLine "docker", "rmi", "--force", "${localJavax86ImageName}" + } + exec { + commandLine "docker", "rmi", "--force", "${localJavaArmImageName}" } exec { - commandLine "gcloud", "--quiet", "container", "images", "untag", "${dockerJavaImageName}" + commandLine "gcloud", "--quiet", "container", "images", "untag", "${dockerJavaMultiarchImageName}" } exec { - commandLine "./scripts/cleanup_untagged_gcr_images.sh", "${dockerJavaImageContainer}" + commandLine "./scripts/cleanup_untagged_gcr_images.sh", "${dockerJavaMultiarchImageContainer}" } } } @@ -319,7 +337,6 @@ def buildAndPushDockerPythonContainer = tasks.create("buildAndPushDockerPythonCo return } project.evaluationDependsOn(":sdks:python") - def pythonVer = project.project(':sdks:python').pythonVersion dependsOn ":sdks:python:container:py"+pythonVer.replace('.', '')+":docker" def defaultDockerImageName = containerImageName( name: "${project.docker_image_default_repo_prefix}python${pythonVer}_sdk", @@ -369,7 +386,7 @@ task printRunnerV2PipelineOptions { println "To run a Dataflow job with runner V2, add the following pipeline options to your command-line:" println runnerV2PipelineOptions.join(' ') println "Please delete your image upon completion with the following command:" - println "docker rmi ${dockerJavaImageName}; gcloud container images delete --force-delete-tags ${dockerJavaImageName}" + println "docker rmi ${localJavax86ImageName}; docker rmi ${localJavaArmImageName}; gcloud container images delete --force-delete-tags ${dockerJavaMultiarchImageName}" } } @@ -429,14 +446,14 @@ createCrossLanguageValidatesRunnerTask( "--runner=TestDataflowRunner", "--project=${dataflowProject}", "--region=${dataflowRegion}", - "--sdk_harness_container_image_overrides=.*java.*,${dockerJavaImageContainer}:${dockerTag}", + "--sdk_harness_container_image_overrides=.*java.*,${dockerJavaMultiarchImageContainer}:${dockerTag}", ], javaPipelineOptions: [ "--runner=TestDataflowRunner", "--project=${dataflowProject}", "--region=${dataflowRegion}", "--tempRoot=${dataflowValidatesTempRoot}", - "--sdkContainerImage=${dockerJavaImageContainer}:${dockerTag}", + "--sdkContainerImage=${dockerJavaMultiarchImageContainer}:${dockerTag}", "--sdkHarnessContainerImageOverrides=.*python.*,${dockerPythonImageContainer}:${dockerTag}", ], pytestOptions: [ From b5cb071517a581930f201327bf4ae1430b55ca58 Mon Sep 17 00:00:00 2001 From: Celeste Zeng <61256376+celeste-zeng@users.noreply.github.com> Date: Wed, 12 Jul 2023 21:48:51 +0000 Subject: [PATCH 16/59] Add unique tag for Java and Go postcommit tests, also make the Python sdkHarnessContainerImageOverrides be multiarch. --- .test-infra/jenkins/job_PostCommit_Go.groovy | 5 +++ ...ostCommit_Java_Examples_Dataflow_V2.groovy | 6 ++- ...it_Java_Examples_Dataflow_V2_Java11.groovy | 4 ++ ...it_Java_Examples_Dataflow_V2_Java17.groovy | 4 ++ ..._Python_ValidatesContainer_Dataflow.groovy | 1 + .../google-cloud-dataflow-java/build.gradle | 37 +++++++++++++------ 6 files changed, 44 insertions(+), 13 deletions(-) diff --git a/.test-infra/jenkins/job_PostCommit_Go.groovy b/.test-infra/jenkins/job_PostCommit_Go.groovy index 854eb7b93fd6..c20e84dbc29e 100644 --- a/.test-infra/jenkins/job_PostCommit_Go.groovy +++ b/.test-infra/jenkins/job_PostCommit_Go.groovy @@ -18,6 +18,7 @@ import CommonJobProperties as commonJobProperties import PostcommitJobBuilder +import java.time.LocalDateTime // This is the Go postcommit which runs a gradle build, and the current set // of postcommit tests. @@ -32,6 +33,9 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_Go', 'Run Go PostCommit', 'master', 300) // increased to 5 hours. + // Generates a unique tag for the container as the current time. + def now = LocalDateTime.now() + def unique_tag = '${now.date}${now.hour}${now.minute}${now.second}' steps { gradle { rootBuildScriptDir(commonJobProperties.checkoutDir) @@ -39,6 +43,7 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_Go', 'Run Go PostCommit', commonJobProperties.setGradleSwitches(delegate) switches('--no-parallel') switches('-Pbuild-and-push-multiarch-containers') + switches('-Pdocker-tag=${unique_tag}') } } } diff --git a/.test-infra/jenkins/job_PostCommit_Java_Examples_Dataflow_V2.groovy b/.test-infra/jenkins/job_PostCommit_Java_Examples_Dataflow_V2.groovy index 9f6d8fb2ca5a..69107a502380 100644 --- a/.test-infra/jenkins/job_PostCommit_Java_Examples_Dataflow_V2.groovy +++ b/.test-infra/jenkins/job_PostCommit_Java_Examples_Dataflow_V2.groovy @@ -18,6 +18,7 @@ import PostcommitJobBuilder import CommonJobProperties as commonJobProperties +import java.time.LocalDateTime PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_Examples_Dataflow_V2', 'Run Java Examples on Dataflow Runner V2', 'Google Cloud Dataflow Runner V2 Examples', this) { @@ -30,6 +31,9 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_Examples_Dataflow_V2', archiveJunit('**/build/test-results/**/*.xml') } + // Generates a unique tag for the container as the current time. + def now = LocalDateTime.now() + def unique_tag = '${now.date}${now.hour}${now.minute}${now.second}' steps { gradle { rootBuildScriptDir(commonJobProperties.checkoutDir) @@ -41,7 +45,7 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_Examples_Dataflow_V2', // overloading the machines. commonJobProperties.setGradleSwitches(delegate, 3 * Runtime.runtime.availableProcessors()) switches '-Pbuild-and-push-multiarch-containers' - switches "-docker-repository-root=us.gcr.io/apache-beam-testing/java-postcommit-it" + switches "-Pdocker-repository-root=us.gcr.io/apache-beam-testing/java-postcommit-it" } } } diff --git a/.test-infra/jenkins/job_PostCommit_Java_Examples_Dataflow_V2_Java11.groovy b/.test-infra/jenkins/job_PostCommit_Java_Examples_Dataflow_V2_Java11.groovy index 67ea3dee8614..f7a03e501b28 100644 --- a/.test-infra/jenkins/job_PostCommit_Java_Examples_Dataflow_V2_Java11.groovy +++ b/.test-infra/jenkins/job_PostCommit_Java_Examples_Dataflow_V2_Java11.groovy @@ -18,6 +18,7 @@ import PostcommitJobBuilder import CommonJobProperties as commonJobProperties +import java.time.LocalDateTime PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_Examples_Dataflow_V2_java11', 'Run Java 11 Examples on Dataflow Runner V2', 'Google Cloud Dataflow Runner V2 Examples Java 11', this) { @@ -30,6 +31,9 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_Examples_Dataflow_V2_ja archiveJunit('**/build/test-results/**/*.xml') } + // Generates a unique tag for the container as the current time. + def now = LocalDateTime.now() + def unique_tag = '${now.date}${now.hour}${now.minute}${now.second}' steps { gradle { rootBuildScriptDir(commonJobProperties.checkoutDir) diff --git a/.test-infra/jenkins/job_PostCommit_Java_Examples_Dataflow_V2_Java17.groovy b/.test-infra/jenkins/job_PostCommit_Java_Examples_Dataflow_V2_Java17.groovy index d62ac142055e..d010e8413c21 100644 --- a/.test-infra/jenkins/job_PostCommit_Java_Examples_Dataflow_V2_Java17.groovy +++ b/.test-infra/jenkins/job_PostCommit_Java_Examples_Dataflow_V2_Java17.groovy @@ -18,6 +18,7 @@ import PostcommitJobBuilder import CommonJobProperties as commonJobProperties +import java.time.LocalDateTime PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_Examples_Dataflow_V2_java17', 'Run Java 17 Examples on Dataflow Runner V2', 'Google Cloud Dataflow Runner V2 Examples Java 17', this) { @@ -30,6 +31,9 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_Examples_Dataflow_V2_ja archiveJunit('**/build/test-results/**/*.xml') } + // Generates a unique tag for the container as the current time. + def now = LocalDateTime.now() + def unique_tag = '${now.date}${now.hour}${now.minute}${now.second}' steps { gradle { rootBuildScriptDir(commonJobProperties.checkoutDir) diff --git a/.test-infra/jenkins/job_PostCommit_Python_ValidatesContainer_Dataflow.groovy b/.test-infra/jenkins/job_PostCommit_Python_ValidatesContainer_Dataflow.groovy index e66dec229899..c4f76d679ce5 100644 --- a/.test-infra/jenkins/job_PostCommit_Python_ValidatesContainer_Dataflow.groovy +++ b/.test-infra/jenkins/job_PostCommit_Python_ValidatesContainer_Dataflow.groovy @@ -36,6 +36,7 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_Py_ValCont', } // Generates a unique tag for the container as the current time. + def now = LocalDateTime.now() def unique_tag = '${now.date}${now.hour}${now.minute}${now.second}' // Execute shell command to test Python SDK. steps { diff --git a/runners/google-cloud-dataflow-java/build.gradle b/runners/google-cloud-dataflow-java/build.gradle index 05a8c76903d3..068139b7c5bc 100644 --- a/runners/google-cloud-dataflow-java/build.gradle +++ b/runners/google-cloud-dataflow-java/build.gradle @@ -153,7 +153,7 @@ def javaVer = "java8" def pythonVer = project.project(':sdks:python').pythonVersion def dockerImageRoot = project.findProperty('docker-repository-root') ?: "us.gcr.io/${dataflowProject}/java-postcommit-it" def dockerJavaMultiarchImageContainer = "${dockerImageRoot}/${project.docker_image_default_repo_prefix}java${javaVer}_sdk" -def dockerPythonImageContainer = "${dockerImageRoot}/${project.docker_image_default_repo_prefix}python${pythonVer}_sdk" +def dockerPythonMultiarchImageContainer = "${dockerImageRoot}/${project.docker_image_default_repo_prefix}python${pythonVer}_sdk" def dockerTag = project.findProperty('docker-tag') ?: new Date().format('yyyyMMddHHmmss') ext.dockerJavaMultiarchImageName = "${dockerJavaMultiarchImageContainer}:${dockerTag}" ext.dockerPythonImageName = "${dockerPythonImageContainer}:${dockerTag}" @@ -280,8 +280,6 @@ def createRunnerV2ValidatesRunnerTest = { Map args -> // task ordering such that the registry doesn't get cleaned up prior to task completion. def buildAndPushDockerJavaContainer = tasks.register("buildAndPushDockerJavaContainer") { dependsOn ":sdks:java:container:${javaVer}:docker" - // TODO: change the following since multi-arch Java containers have been pushed during build - // apache/beam_java11_sdk:tag def localJavax86ImageName = containerImageName( name: "${project.docker_image_default_repo_prefix}${javaVer}_sdk-x86", root: "apache", @@ -292,7 +290,7 @@ def buildAndPushDockerJavaContainer = tasks.register("buildAndPushDockerJavaCont tag: dockerTag) doLast { // Verify multi-arch Java docker image has been built and pushed during the build - // by pulling and tagging the x86 and arm components: + // by pulling and tagging the x86 and arm components: exec { commandLine "docker", "pull", "${dockerJavaMultiarchImageName}" } @@ -338,16 +336,28 @@ def buildAndPushDockerPythonContainer = tasks.create("buildAndPushDockerPythonCo } project.evaluationDependsOn(":sdks:python") dependsOn ":sdks:python:container:py"+pythonVer.replace('.', '')+":docker" - def defaultDockerImageName = containerImageName( - name: "${project.docker_image_default_repo_prefix}python${pythonVer}_sdk", + def localPythonx86ImageName = containerImageName( + name: "${project.docker_image_default_repo_prefix}${pythonVer}_sdk-x86", root: "apache", - tag: project.sdk_version) + tag: dockerTag) + def localPythonArmImageName = containerImageName( + name: "${project.docker_image_default_repo_prefix}${pythonVer}_sdk-arm", + root: "apache", + tag: dockerTag) doLast { + // Verify multi-arch Python docker image has been built and pushed during the build + // by pulling and tagging the x86 and arm components: + exec { + commandLine "docker", "pull", "${dockerPythonMultiarchImageName}" + } exec { - commandLine "docker", "tag", "${defaultDockerImageName}", "${dockerPythonImageName}" + commandLine "docker", "tag", "${dockerPythonMultiarchImageName}", "${localPythonx86ImageName}" } exec { - commandLine "gcloud", "docker", "--", "push", "${dockerPythonImageName}" + commandLine "docker", "pull", "--platform=linux/arm64", "${dockerPythonMultiarchImageName}" + } + exec { + commandLine "docker", "tag", "${dockerPythonMultiarchImageName}", "${localPythonArmImageName}" } } } @@ -356,13 +366,16 @@ def buildAndPushDockerPythonContainer = tasks.create("buildAndPushDockerPythonCo def cleanUpDockerPythonImages = tasks.register("cleanUpDockerPythonImages") { doLast { exec { - commandLine "docker", "rmi", "--force", "${dockerPythonImageName}" + commandLine "docker", "rmi", "--force", "${localPythonx86ImageName}" + } + exec { + commandLine "docker", "rmi", "--force", "${localPythonArmImageName}" } exec { - commandLine "gcloud", "--quiet", "container", "images", "untag", "${dockerPythonImageName}" + commandLine "gcloud", "--quiet", "container", "images", "untag", "${dockerPythonMultiarchImageName}" } exec { - commandLine "./scripts/cleanup_untagged_gcr_images.sh", "${dockerPythonImageContainer}" + commandLine "./scripts/cleanup_untagged_gcr_images.sh", "${dockerPythonMultiarchImageContainer}" } } } From 3e0f47b19d882aca7639e697633b3804d55e9ac8 Mon Sep 17 00:00:00 2001 From: Celeste Zeng <61256376+celeste-zeng@users.noreply.github.com> Date: Wed, 12 Jul 2023 23:44:03 +0000 Subject: [PATCH 17/59] Made the Go PostCommit test script to build multiarch containers; remove changes in the .Groovy file. --- .test-infra/jenkins/job_PostCommit_Go.groovy | 6 ------ sdks/go/test/run_validatesrunner_tests.sh | 10 ++++------ 2 files changed, 4 insertions(+), 12 deletions(-) diff --git a/.test-infra/jenkins/job_PostCommit_Go.groovy b/.test-infra/jenkins/job_PostCommit_Go.groovy index c20e84dbc29e..3d1c6a3b53ff 100644 --- a/.test-infra/jenkins/job_PostCommit_Go.groovy +++ b/.test-infra/jenkins/job_PostCommit_Go.groovy @@ -18,7 +18,6 @@ import CommonJobProperties as commonJobProperties import PostcommitJobBuilder -import java.time.LocalDateTime // This is the Go postcommit which runs a gradle build, and the current set // of postcommit tests. @@ -33,17 +32,12 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_Go', 'Run Go PostCommit', 'master', 300) // increased to 5 hours. - // Generates a unique tag for the container as the current time. - def now = LocalDateTime.now() - def unique_tag = '${now.date}${now.hour}${now.minute}${now.second}' steps { gradle { rootBuildScriptDir(commonJobProperties.checkoutDir) tasks(':goPostCommit') commonJobProperties.setGradleSwitches(delegate) switches('--no-parallel') - switches('-Pbuild-and-push-multiarch-containers') - switches('-Pdocker-tag=${unique_tag}') } } } diff --git a/sdks/go/test/run_validatesrunner_tests.sh b/sdks/go/test/run_validatesrunner_tests.sh index 444dc1ae39a7..b4c96c141b77 100755 --- a/sdks/go/test/run_validatesrunner_tests.sh +++ b/sdks/go/test/run_validatesrunner_tests.sh @@ -358,13 +358,11 @@ if [[ "$RUNNER" == "dataflow" ]]; then TAG=$(date +%Y%m%d-%H%M%S) CONTAINER=us.gcr.io/$PROJECT/$USER/beam_go_sdk echo "Using container $CONTAINER" - ./gradlew :sdks:go:container:docker -Pdocker-repository-root=us.gcr.io/$PROJECT/$USER -Pdocker-tag=$TAG + ./gradlew :sdks:go:container:docker -Pdocker-repository-root=us.gcr.io/$PROJECT/$USER -Pdocker-tag=$TAG -Pbuild-and-push-multiarch-containers - # Verify it exists - docker images | grep $TAG - - # Push the container - gcloud docker -- push $CONTAINER:$TAG + # Verify both the x86 and ARM components of the Go multiarch container exist if the pull requests succeed + docker pull $CONTAINER:$TAG + docker pull --platform linux/arm64 $CONTAINER:$TAG if [[ -n "$TEST_EXPANSION_ADDR" || -n "$IO_EXPANSION_ADDR" || -n "$SCHEMAIO_EXPANSION_ADDR" || -n "$DEBEZIUMIO_EXPANSION_ADDR" ]]; then ARGS="$ARGS --experiments=use_portable_job_submission" From d92cc50b19c26d3af1df9b644aed0d6a5cb3502b Mon Sep 17 00:00:00 2001 From: Celeste Zeng <61256376+celeste-zeng@users.noreply.github.com> Date: Thu, 13 Jul 2023 18:09:52 +0000 Subject: [PATCH 18/59] Get rid of redundant switch options. --- .../jenkins/job_PostCommit_Java_Examples_Dataflow_V2.groovy | 3 ++- .../job_PostCommit_Java_Examples_Dataflow_V2_Java11.groovy | 3 ++- .../job_PostCommit_Java_Examples_Dataflow_V2_Java17.groovy | 3 ++- ...job_PostCommit_Python_ValidatesContainer_Dataflow.groovy | 3 ++- .../groovy/org/apache/beam/gradle/BeamModulePlugin.groovy | 6 ++---- sdks/go/container/build.gradle | 6 ++---- sdks/go/test/run_validatesrunner_tests.sh | 2 +- sdks/java/container/common.gradle | 5 ++--- sdks/python/container/common.gradle | 6 ++---- 9 files changed, 17 insertions(+), 20 deletions(-) diff --git a/.test-infra/jenkins/job_PostCommit_Java_Examples_Dataflow_V2.groovy b/.test-infra/jenkins/job_PostCommit_Java_Examples_Dataflow_V2.groovy index 69107a502380..b76ec94aaf39 100644 --- a/.test-infra/jenkins/job_PostCommit_Java_Examples_Dataflow_V2.groovy +++ b/.test-infra/jenkins/job_PostCommit_Java_Examples_Dataflow_V2.groovy @@ -44,7 +44,8 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_Examples_Dataflow_V2', // 3 x num_cores strikes a good balance between maxing out parallelism without // overloading the machines. commonJobProperties.setGradleSwitches(delegate, 3 * Runtime.runtime.availableProcessors()) - switches '-Pbuild-and-push-multiarch-containers' + switches "-Pcontainer-architecture-list=arm64,amd64" + switches '-Ppush-containers' switches "-Pdocker-repository-root=us.gcr.io/apache-beam-testing/java-postcommit-it" } } diff --git a/.test-infra/jenkins/job_PostCommit_Java_Examples_Dataflow_V2_Java11.groovy b/.test-infra/jenkins/job_PostCommit_Java_Examples_Dataflow_V2_Java11.groovy index f7a03e501b28..aa1b6f444094 100644 --- a/.test-infra/jenkins/job_PostCommit_Java_Examples_Dataflow_V2_Java11.groovy +++ b/.test-infra/jenkins/job_PostCommit_Java_Examples_Dataflow_V2_Java11.groovy @@ -49,7 +49,8 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_Examples_Dataflow_V2_ja switches '-PcompileAndRunTestsWithJava11' switches '-PskipCheckerFramework' switches "-Pjava11Home=${commonJobProperties.JAVA_11_HOME}" - switches '-Pbuild-and-push-multiarch-containers' + switches '-Ppush-containers' + switches "-Pcontainer-architecture-list=arm64,amd64" switches "-Pdocker-repository-root=us.gcr.io/apache-beam-testing/java-postcommit-it" } } diff --git a/.test-infra/jenkins/job_PostCommit_Java_Examples_Dataflow_V2_Java17.groovy b/.test-infra/jenkins/job_PostCommit_Java_Examples_Dataflow_V2_Java17.groovy index d010e8413c21..bb6790ffba59 100644 --- a/.test-infra/jenkins/job_PostCommit_Java_Examples_Dataflow_V2_Java17.groovy +++ b/.test-infra/jenkins/job_PostCommit_Java_Examples_Dataflow_V2_Java17.groovy @@ -49,7 +49,8 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_Examples_Dataflow_V2_ja switches '-PcompileAndRunTestsWithJava17' switches '-PskipCheckerFramework' switches "-Pjava17Home=${commonJobProperties.JAVA_17_HOME}" - switches '-Pbuild-and-push-multiarch-containers' + switches "-Pcontainer-architecture-list=arm64,amd64" + switches '-Ppush-containers' switches "-Pdocker-repository-root=us.gcr.io/apache-beam-testing/java-postcommit-it" } } diff --git a/.test-infra/jenkins/job_PostCommit_Python_ValidatesContainer_Dataflow.groovy b/.test-infra/jenkins/job_PostCommit_Python_ValidatesContainer_Dataflow.groovy index c4f76d679ce5..cc5c4a10a3b8 100644 --- a/.test-infra/jenkins/job_PostCommit_Python_ValidatesContainer_Dataflow.groovy +++ b/.test-infra/jenkins/job_PostCommit_Python_ValidatesContainer_Dataflow.groovy @@ -43,7 +43,8 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_Py_ValCont', gradle { rootBuildScriptDir(commonJobProperties.checkoutDir) tasks(':sdks:python:test-suites:dataflow:validatesContainerTests') - switches('-Pbuild-and-push-multiarch-containers') + switches('-Pcontainer-architecture-list=arm64,amd64') + switches('-Ppush-containers') // Regenerates the base image requirements to accommodate ARM. switches('-Pregenerate-requirements') // Push multi-arch containers to the repository set in run_validatescontainer.sh diff --git a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy index 6fcb674205bb..9a42ab330aa3 100644 --- a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy +++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy @@ -493,12 +493,10 @@ class BeamModulePlugin implements Plugin { return ["amd64"]; } else if (project.rootProject.findProperty("container-architecture-list") != null) { def containerArchitectures = project.rootProject.findProperty("container-architecture-list").split(',') - if (containerArchitectures.size() > 1 && !project.rootProject.hasProperty("push-multiarch-containers")) { - throw new GradleException("A multi-arch image can't be saved in the local image store, please append the -Ppush-multiarch-containers flag and specify a repository to push in the -Pdocker-repository-root flag."); + if (containerArchitectures.size() > 1 && !project.rootProject.hasProperty("push-containers")) { + throw new GradleException("A multi-arch image can't be saved in the local image store, please append the -Ppush-containers flag and specify a repository to push in the -Pdocker-repository-root flag."); } return containerArchitectures - } else if (project.hasProperty("build-and-push-multiarch-containers")) { - return ["arm64", "amd64"]; } else { return [project.nativeArchitecture()] } diff --git a/sdks/go/container/build.gradle b/sdks/go/container/build.gradle index 701379aa9650..5cf0e925114c 100644 --- a/sdks/go/container/build.gradle +++ b/sdks/go/container/build.gradle @@ -27,8 +27,6 @@ goBuild { outputLocation = './build/target/${GOOS}_${GOARCH}/boot' } -def pushContainers = project.rootProject.hasProperty("push-multiarch-containers") || project.hasProperty("build-and-push-multiarch-containers") - def useBuildx = project.containerPlatforms() != [project.nativeArchitecture()] docker { @@ -44,8 +42,8 @@ docker { project.rootProject.hasProperty(["isRelease"])]) buildx useBuildx platform(*project.containerPlatforms()) - load useBuildx && !pushContainers - push pushContainers + load useBuildx && !project.rootProject.hasProperty("push-containers") + push project.rootProject.hasProperty("push-containers") } dockerPrepare.dependsOn tasks.named("goBuild") diff --git a/sdks/go/test/run_validatesrunner_tests.sh b/sdks/go/test/run_validatesrunner_tests.sh index b4c96c141b77..aeec1d10ec40 100755 --- a/sdks/go/test/run_validatesrunner_tests.sh +++ b/sdks/go/test/run_validatesrunner_tests.sh @@ -358,7 +358,7 @@ if [[ "$RUNNER" == "dataflow" ]]; then TAG=$(date +%Y%m%d-%H%M%S) CONTAINER=us.gcr.io/$PROJECT/$USER/beam_go_sdk echo "Using container $CONTAINER" - ./gradlew :sdks:go:container:docker -Pdocker-repository-root=us.gcr.io/$PROJECT/$USER -Pdocker-tag=$TAG -Pbuild-and-push-multiarch-containers + ./gradlew :sdks:go:container:docker -Pdocker-repository-root=us.gcr.io/$PROJECT/$USER -Pdocker-tag=$TAG -Pcontainer-architecture-list=arm64,amd64 -Ppush-containers # Verify both the x86 and ARM components of the Go multiarch container exist if the pull requests succeed docker pull $CONTAINER:$TAG diff --git a/sdks/java/container/common.gradle b/sdks/java/container/common.gradle index e44b82dd722e..ec1c1744dfde 100644 --- a/sdks/java/container/common.gradle +++ b/sdks/java/container/common.gradle @@ -103,7 +103,6 @@ task validateJavaHome { } } } -def pushContainers = project.rootProject.hasProperty("push-multiarch-containers") || project.hasProperty("build-and-push-multiarch-containers") def useBuildx = project.containerPlatforms() != [project.nativeArchitecture()] @@ -126,8 +125,8 @@ docker { ]) buildx useBuildx platform(*project.containerPlatforms()) - load useBuildx && !pushContainers - push pushContainers + load useBuildx && !project.rootProject.hasProperty("push-containers") + push project.rootProject.hasProperty("push-containers") } if (project.rootProject.hasProperty(["docker-pull-licenses"]) || diff --git a/sdks/python/container/common.gradle b/sdks/python/container/common.gradle index e064b79058f6..b11dc8127c7d 100644 --- a/sdks/python/container/common.gradle +++ b/sdks/python/container/common.gradle @@ -66,8 +66,6 @@ def copyLauncherDependencies = tasks.register("copyLauncherDependencies", Copy) } } -def pushContainers = project.rootProject.hasProperty("push-multiarch-containers") || project.hasProperty("build-and-push-multiarch-containers") - def useBuildx = project.containerPlatforms() != [project.nativeArchitecture()] docker { @@ -86,8 +84,8 @@ docker { project.rootProject.hasProperty(["isRelease"])]) buildx useBuildx platform(*project.containerPlatforms()) - load useBuildx && !pushContainers - push pushContainers + load useBuildx && !project.rootProject.hasProperty("push-containers") + push project.rootProject.hasProperty("push-containers") } dockerPrepare.dependsOn copyLauncherDependencies From 32a410bd8c9fe86a755350f2d20ff906369549a8 Mon Sep 17 00:00:00 2001 From: Celeste Zeng <61256376+celeste-zeng@users.noreply.github.com> Date: Thu, 13 Jul 2023 18:20:49 +0000 Subject: [PATCH 19/59] Remove unnecessary changes for crosee-language tests. --- .../google-cloud-dataflow-java/build.gradle | 35 ++++++------------- 1 file changed, 10 insertions(+), 25 deletions(-) diff --git a/runners/google-cloud-dataflow-java/build.gradle b/runners/google-cloud-dataflow-java/build.gradle index 068139b7c5bc..7c6f01ca861b 100644 --- a/runners/google-cloud-dataflow-java/build.gradle +++ b/runners/google-cloud-dataflow-java/build.gradle @@ -150,10 +150,9 @@ def javaVer = "java8" } else if(project.hasProperty('compileAndRunTestsWithJava11')) { javaVer = "java11" } -def pythonVer = project.project(':sdks:python').pythonVersion def dockerImageRoot = project.findProperty('docker-repository-root') ?: "us.gcr.io/${dataflowProject}/java-postcommit-it" def dockerJavaMultiarchImageContainer = "${dockerImageRoot}/${project.docker_image_default_repo_prefix}java${javaVer}_sdk" -def dockerPythonMultiarchImageContainer = "${dockerImageRoot}/${project.docker_image_default_repo_prefix}python${pythonVer}_sdk" +def dockerPythonImageContainer = "${dockerImageRoot}/python" def dockerTag = project.findProperty('docker-tag') ?: new Date().format('yyyyMMddHHmmss') ext.dockerJavaMultiarchImageName = "${dockerJavaMultiarchImageContainer}:${dockerTag}" ext.dockerPythonImageName = "${dockerPythonImageContainer}:${dockerTag}" @@ -335,29 +334,18 @@ def buildAndPushDockerPythonContainer = tasks.create("buildAndPushDockerPythonCo return } project.evaluationDependsOn(":sdks:python") + def pythonVer = project.project(':sdks:python').pythonVersion dependsOn ":sdks:python:container:py"+pythonVer.replace('.', '')+":docker" - def localPythonx86ImageName = containerImageName( - name: "${project.docker_image_default_repo_prefix}${pythonVer}_sdk-x86", + def defaultDockerImageName = containerImageName( + name: "${project.docker_image_default_repo_prefix}python${pythonVer}_sdk", root: "apache", - tag: dockerTag) - def localPythonArmImageName = containerImageName( - name: "${project.docker_image_default_repo_prefix}${pythonVer}_sdk-arm", - root: "apache", - tag: dockerTag) + tag: project.sdk_version) doLast { - // Verify multi-arch Python docker image has been built and pushed during the build - // by pulling and tagging the x86 and arm components: - exec { - commandLine "docker", "pull", "${dockerPythonMultiarchImageName}" - } - exec { - commandLine "docker", "tag", "${dockerPythonMultiarchImageName}", "${localPythonx86ImageName}" - } exec { - commandLine "docker", "pull", "--platform=linux/arm64", "${dockerPythonMultiarchImageName}" + commandLine "docker", "tag", "${defaultDockerImageName}", "${dockerPythonImageName}" } exec { - commandLine "docker", "tag", "${dockerPythonMultiarchImageName}", "${localPythonArmImageName}" + commandLine "gcloud", "docker", "--", "push", "${dockerPythonImageName}" } } } @@ -366,16 +354,13 @@ def buildAndPushDockerPythonContainer = tasks.create("buildAndPushDockerPythonCo def cleanUpDockerPythonImages = tasks.register("cleanUpDockerPythonImages") { doLast { exec { - commandLine "docker", "rmi", "--force", "${localPythonx86ImageName}" - } - exec { - commandLine "docker", "rmi", "--force", "${localPythonArmImageName}" + commandLine "docker", "rmi", "--force", "${dockerPythonImageName}" } exec { - commandLine "gcloud", "--quiet", "container", "images", "untag", "${dockerPythonMultiarchImageName}" + commandLine "gcloud", "--quiet", "container", "images", "untag", "${dockerPythonImageName}" } exec { - commandLine "./scripts/cleanup_untagged_gcr_images.sh", "${dockerPythonMultiarchImageContainer}" + commandLine "./scripts/cleanup_untagged_gcr_images.sh", "${dockerPythonImageContainer}" } } } From 88b8f5bd45b852ea6fd85663e705ee188bfe722b Mon Sep 17 00:00:00 2001 From: Celeste Zeng <61256376+celeste-zeng@users.noreply.github.com> Date: Thu, 13 Jul 2023 18:39:29 +0000 Subject: [PATCH 20/59] Add unique tag for PostCommit Java tests. --- .../jenkins/job_PostCommit_Java_Examples_Dataflow_V2.groovy | 1 + .../job_PostCommit_Java_Examples_Dataflow_V2_Java11.groovy | 1 + .../job_PostCommit_Java_Examples_Dataflow_V2_Java17.groovy | 1 + 3 files changed, 3 insertions(+) diff --git a/.test-infra/jenkins/job_PostCommit_Java_Examples_Dataflow_V2.groovy b/.test-infra/jenkins/job_PostCommit_Java_Examples_Dataflow_V2.groovy index b76ec94aaf39..e4aae79f77df 100644 --- a/.test-infra/jenkins/job_PostCommit_Java_Examples_Dataflow_V2.groovy +++ b/.test-infra/jenkins/job_PostCommit_Java_Examples_Dataflow_V2.groovy @@ -47,6 +47,7 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_Examples_Dataflow_V2', switches "-Pcontainer-architecture-list=arm64,amd64" switches '-Ppush-containers' switches "-Pdocker-repository-root=us.gcr.io/apache-beam-testing/java-postcommit-it" + switches "-Pdocker-tag=${unique_tag}" } } } diff --git a/.test-infra/jenkins/job_PostCommit_Java_Examples_Dataflow_V2_Java11.groovy b/.test-infra/jenkins/job_PostCommit_Java_Examples_Dataflow_V2_Java11.groovy index aa1b6f444094..38791d8df5d8 100644 --- a/.test-infra/jenkins/job_PostCommit_Java_Examples_Dataflow_V2_Java11.groovy +++ b/.test-infra/jenkins/job_PostCommit_Java_Examples_Dataflow_V2_Java11.groovy @@ -52,6 +52,7 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_Examples_Dataflow_V2_ja switches '-Ppush-containers' switches "-Pcontainer-architecture-list=arm64,amd64" switches "-Pdocker-repository-root=us.gcr.io/apache-beam-testing/java-postcommit-it" + switches "-Pdocker-tag=${unique_tag}" } } } diff --git a/.test-infra/jenkins/job_PostCommit_Java_Examples_Dataflow_V2_Java17.groovy b/.test-infra/jenkins/job_PostCommit_Java_Examples_Dataflow_V2_Java17.groovy index bb6790ffba59..b8c9bfee8b9a 100644 --- a/.test-infra/jenkins/job_PostCommit_Java_Examples_Dataflow_V2_Java17.groovy +++ b/.test-infra/jenkins/job_PostCommit_Java_Examples_Dataflow_V2_Java17.groovy @@ -52,6 +52,7 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_Examples_Dataflow_V2_ja switches "-Pcontainer-architecture-list=arm64,amd64" switches '-Ppush-containers' switches "-Pdocker-repository-root=us.gcr.io/apache-beam-testing/java-postcommit-it" + switches "-Pdocker-tag=${unique_tag}" } } } From f5d2161bdcb0396c103fa181602565e429fb4865 Mon Sep 17 00:00:00 2001 From: Celeste Zeng <61256376+celeste-zeng@users.noreply.github.com> Date: Thu, 13 Jul 2023 20:31:35 +0000 Subject: [PATCH 21/59] Remove comment. --- runners/google-cloud-dataflow-java/build.gradle | 1 - 1 file changed, 1 deletion(-) diff --git a/runners/google-cloud-dataflow-java/build.gradle b/runners/google-cloud-dataflow-java/build.gradle index 7c6f01ca861b..32346a0e72c1 100644 --- a/runners/google-cloud-dataflow-java/build.gradle +++ b/runners/google-cloud-dataflow-java/build.gradle @@ -305,7 +305,6 @@ def buildAndPushDockerJavaContainer = tasks.register("buildAndPushDockerJavaCont } } -// TODO: change here for multi arch containers // Clean up built Java images def cleanUpDockerJavaImages = tasks.register("cleanUpDockerJavaImages") { doLast { From 64d503be3ec1677dc952ec6a21f34db1cb833ac2 Mon Sep 17 00:00:00 2001 From: Celeste Zeng <61256376+celeste-zeng@users.noreply.github.com> Date: Mon, 17 Jul 2023 23:38:39 +0000 Subject: [PATCH 22/59] Pull different components of the multiarch containers separately, so there will be 2 containers with the same name locally but different tags(one is untagged). Change the cleaning container method to remove all local containers with the same name, ignore tags. --- sdks/go/test/run_validatesrunner_tests.sh | 6 +++--- sdks/python/container/run_validatescontainer.sh | 9 +++------ 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/sdks/go/test/run_validatesrunner_tests.sh b/sdks/go/test/run_validatesrunner_tests.sh index aeec1d10ec40..05e16d2fe37a 100755 --- a/sdks/go/test/run_validatesrunner_tests.sh +++ b/sdks/go/test/run_validatesrunner_tests.sh @@ -361,8 +361,8 @@ if [[ "$RUNNER" == "dataflow" ]]; then ./gradlew :sdks:go:container:docker -Pdocker-repository-root=us.gcr.io/$PROJECT/$USER -Pdocker-tag=$TAG -Pcontainer-architecture-list=arm64,amd64 -Ppush-containers # Verify both the x86 and ARM components of the Go multiarch container exist if the pull requests succeed - docker pull $CONTAINER:$TAG - docker pull --platform linux/arm64 $CONTAINER:$TAG + gcloud docker -- pull $CONTAINER:$TAG + gcloud docker -- pull --platform linux/arm64 $CONTAINER:$TAG if [[ -n "$TEST_EXPANSION_ADDR" || -n "$IO_EXPANSION_ADDR" || -n "$SCHEMAIO_EXPANSION_ADDR" || -n "$DEBEZIUMIO_EXPANSION_ADDR" ]]; then ARGS="$ARGS --experiments=use_portable_job_submission" @@ -429,7 +429,7 @@ cd ../.. if [[ "$RUNNER" == "dataflow" ]]; then # Delete the container locally and remotely - docker rmi $CONTAINER:$TAG || echo "Failed to remove container" + docker rmi -q $CONTAINER || echo "Failed to remove container" gcloud --quiet container images delete $CONTAINER:$TAG || echo "Failed to delete container" if [[ -n "$TEST_EXPANSION_ADDR" || -n "$IO_EXPANSION_ADDR" || -n "$SCHEMAIO_EXPANSION_ADDR" || -n "$DEBEZIUMIO_EXPANSION_ADDR" ]]; then diff --git a/sdks/python/container/run_validatescontainer.sh b/sdks/python/container/run_validatescontainer.sh index 65d8a4e2646c..af1246cf844c 100755 --- a/sdks/python/container/run_validatescontainer.sh +++ b/sdks/python/container/run_validatescontainer.sh @@ -70,17 +70,14 @@ PREBUILD_SDK_CONTAINER_REGISTRY_PATH=us.gcr.io/$PROJECT/$USER/prebuild_python${P TAG="$(grep 'unique_tag' job_PostCommit_Python_ValidatesContainer_Dataflow.groovy | cut -d'=' -f2)" # Verify docker image has been built and pushed during the build: -# pull and tag the x86 components; +# pull the x86 components; docker pull $CONTAINER:$TAG -docker tag "$CONTAINER:$TAG" "$CONTAINER-x86:$TAG" -# pull and tag the ARM components; +# pull the ARM components; docker pull --platform linux/arm64 $CONTAINER:$TAG -docker tag "$CONTAINER:$TAG" "$CONTAINER-arm:$TAG" function cleanup_container { # Delete the container locally and remotely - docker rmi $CONTAINER-x86:$TAG || echo "Failed to remove x86 container image" - docker rmi $CONTAINER-arm:$TAG || echo "Failed to remove arm container image" + docker rmi -q $CONTAINER || echo "Failed to remove local container image" for image in $(docker images --format '{{.Repository}}:{{.Tag}}' | grep $PREBUILD_SDK_CONTAINER_REGISTRY_PATH) do docker rmi $image || echo "Failed to remove prebuilt sdk container image" done From 85aec942d07db3dfdb0a34e649deb3376bd00da9 Mon Sep 17 00:00:00 2001 From: Celeste Zeng <61256376+celeste-zeng@users.noreply.github.com> Date: Tue, 18 Jul 2023 08:15:46 +0000 Subject: [PATCH 23/59] Changed how to verify and clean Java multiarch containers. --- .../google-cloud-dataflow-java/build.gradle | 29 ++++--------------- 1 file changed, 6 insertions(+), 23 deletions(-) diff --git a/runners/google-cloud-dataflow-java/build.gradle b/runners/google-cloud-dataflow-java/build.gradle index 32346a0e72c1..bcb3ff11cdad 100644 --- a/runners/google-cloud-dataflow-java/build.gradle +++ b/runners/google-cloud-dataflow-java/build.gradle @@ -278,29 +278,15 @@ def createRunnerV2ValidatesRunnerTest = { Map args -> // task directly ('dependsOn buildAndPushDockerJavaContainer'). This ensures the correct // task ordering such that the registry doesn't get cleaned up prior to task completion. def buildAndPushDockerJavaContainer = tasks.register("buildAndPushDockerJavaContainer") { - dependsOn ":sdks:java:container:${javaVer}:docker" - def localJavax86ImageName = containerImageName( - name: "${project.docker_image_default_repo_prefix}${javaVer}_sdk-x86", - root: "apache", - tag: dockerTag) - def localJavaArmImageName = containerImageName( - name: "${project.docker_image_default_repo_prefix}${javaVer}_sdk-arm", - root: "apache", - tag: dockerTag) + dependsOn ":sdks:java:container:${javaVer}:docker" doLast { // Verify multi-arch Java docker image has been built and pushed during the build - // by pulling and tagging the x86 and arm components: - exec { - commandLine "docker", "pull", "${dockerJavaMultiarchImageName}" - } + // by pulling the x86 and arm components: exec { - commandLine "docker", "tag", "${dockerJavaMultiarchImageName}", "${localJavax86ImageName}" + commandLine "gcloud", "docker", "--", "pull", "${dockerJavaMultiarchImageName}" } exec { - commandLine "docker", "pull", "--platform=linux/arm64", "${dockerJavaMultiarchImageName}" - } - exec { - commandLine "docker", "tag", "${dockerJavaMultiarchImageName}", "${localJavaArmImageName}" + commandLine "gcloud", "docker", "--", "pull", "--platform=linux/arm64", "${dockerJavaMultiarchImageName}" } } } @@ -309,10 +295,7 @@ def buildAndPushDockerJavaContainer = tasks.register("buildAndPushDockerJavaCont def cleanUpDockerJavaImages = tasks.register("cleanUpDockerJavaImages") { doLast { exec { - commandLine "docker", "rmi", "--force", "${localJavax86ImageName}" - } - exec { - commandLine "docker", "rmi", "--force", "${localJavaArmImageName}" + commandLine "docker", "rmi", "-q", "${dockerJavaMultiarchImageName}" } exec { commandLine "gcloud", "--quiet", "container", "images", "untag", "${dockerJavaMultiarchImageName}" @@ -383,7 +366,7 @@ task printRunnerV2PipelineOptions { println "To run a Dataflow job with runner V2, add the following pipeline options to your command-line:" println runnerV2PipelineOptions.join(' ') println "Please delete your image upon completion with the following command:" - println "docker rmi ${localJavax86ImageName}; docker rmi ${localJavaArmImageName}; gcloud container images delete --force-delete-tags ${dockerJavaMultiarchImageName}" + println "docker rmi -q ${dockerJavaMultiarchImageName}; gcloud container images delete --force-delete-tags ${dockerJavaMultiarchImageName}" } } From bcf66fb95d1c67b20e415ff81a6ed65e2bbcc840 Mon Sep 17 00:00:00 2001 From: Celeste Zeng <61256376+celeste-zeng@users.noreply.github.com> Date: Tue, 18 Jul 2023 08:35:12 +0000 Subject: [PATCH 24/59] fixed typo --- runners/google-cloud-dataflow-java/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runners/google-cloud-dataflow-java/build.gradle b/runners/google-cloud-dataflow-java/build.gradle index bcb3ff11cdad..d60ec71b1d49 100644 --- a/runners/google-cloud-dataflow-java/build.gradle +++ b/runners/google-cloud-dataflow-java/build.gradle @@ -448,7 +448,7 @@ createCrossLanguageValidatesRunnerTask( "--dataflow_project ${dataflowProject}", "--region ${dataflowRegion}", "--tests \"./test/integration/xlang ./test/integration/io/xlang/...\"", - "--sdk_overrides \".*java.*,${dockerJavaImageContainer}:${dockerTag}\"", + "--sdk_overrides \".*java.*,${dockerJavaMultiarchImageContainer}:${dockerTag}\"", ], ) From 373428a042191589fc12d89a1f729ff24dbc68eb Mon Sep 17 00:00:00 2001 From: Celeste Zeng <61256376+celeste-zeng@users.noreply.github.com> Date: Tue, 18 Jul 2023 18:48:12 +0000 Subject: [PATCH 25/59] No need to pull multiarch containers to verify its existence and update the cleaning step accordingly. --- runners/google-cloud-dataflow-java/build.gradle | 14 +------------- sdks/go/test/run_validatesrunner_tests.sh | 7 +------ sdks/python/container/run_validatescontainer.sh | 7 ------- 3 files changed, 2 insertions(+), 26 deletions(-) diff --git a/runners/google-cloud-dataflow-java/build.gradle b/runners/google-cloud-dataflow-java/build.gradle index d60ec71b1d49..9841a82d22a7 100644 --- a/runners/google-cloud-dataflow-java/build.gradle +++ b/runners/google-cloud-dataflow-java/build.gradle @@ -278,25 +278,13 @@ def createRunnerV2ValidatesRunnerTest = { Map args -> // task directly ('dependsOn buildAndPushDockerJavaContainer'). This ensures the correct // task ordering such that the registry doesn't get cleaned up prior to task completion. def buildAndPushDockerJavaContainer = tasks.register("buildAndPushDockerJavaContainer") { + // The multi-arch Java container has been pushed during build. dependsOn ":sdks:java:container:${javaVer}:docker" - doLast { - // Verify multi-arch Java docker image has been built and pushed during the build - // by pulling the x86 and arm components: - exec { - commandLine "gcloud", "docker", "--", "pull", "${dockerJavaMultiarchImageName}" - } - exec { - commandLine "gcloud", "docker", "--", "pull", "--platform=linux/arm64", "${dockerJavaMultiarchImageName}" - } - } } // Clean up built Java images def cleanUpDockerJavaImages = tasks.register("cleanUpDockerJavaImages") { doLast { - exec { - commandLine "docker", "rmi", "-q", "${dockerJavaMultiarchImageName}" - } exec { commandLine "gcloud", "--quiet", "container", "images", "untag", "${dockerJavaMultiarchImageName}" } diff --git a/sdks/go/test/run_validatesrunner_tests.sh b/sdks/go/test/run_validatesrunner_tests.sh index 05e16d2fe37a..75147e0d37fd 100755 --- a/sdks/go/test/run_validatesrunner_tests.sh +++ b/sdks/go/test/run_validatesrunner_tests.sh @@ -360,10 +360,6 @@ if [[ "$RUNNER" == "dataflow" ]]; then echo "Using container $CONTAINER" ./gradlew :sdks:go:container:docker -Pdocker-repository-root=us.gcr.io/$PROJECT/$USER -Pdocker-tag=$TAG -Pcontainer-architecture-list=arm64,amd64 -Ppush-containers - # Verify both the x86 and ARM components of the Go multiarch container exist if the pull requests succeed - gcloud docker -- pull $CONTAINER:$TAG - gcloud docker -- pull --platform linux/arm64 $CONTAINER:$TAG - if [[ -n "$TEST_EXPANSION_ADDR" || -n "$IO_EXPANSION_ADDR" || -n "$SCHEMAIO_EXPANSION_ADDR" || -n "$DEBEZIUMIO_EXPANSION_ADDR" ]]; then ARGS="$ARGS --experiments=use_portable_job_submission" @@ -428,8 +424,7 @@ echo ">>> RUNNING $RUNNER integration tests with pipeline options: $ARGS" cd ../.. if [[ "$RUNNER" == "dataflow" ]]; then - # Delete the container locally and remotely - docker rmi -q $CONTAINER || echo "Failed to remove container" + # Delete the container remotely gcloud --quiet container images delete $CONTAINER:$TAG || echo "Failed to delete container" if [[ -n "$TEST_EXPANSION_ADDR" || -n "$IO_EXPANSION_ADDR" || -n "$SCHEMAIO_EXPANSION_ADDR" || -n "$DEBEZIUMIO_EXPANSION_ADDR" ]]; then diff --git a/sdks/python/container/run_validatescontainer.sh b/sdks/python/container/run_validatescontainer.sh index af1246cf844c..151f8409d5fa 100755 --- a/sdks/python/container/run_validatescontainer.sh +++ b/sdks/python/container/run_validatescontainer.sh @@ -69,15 +69,8 @@ CONTAINER=us.gcr.io/$PROJECT/$USER/$IMAGE_NAME PREBUILD_SDK_CONTAINER_REGISTRY_PATH=us.gcr.io/$PROJECT/$USER/prebuild_python${PY_VERSION//.}_sdk TAG="$(grep 'unique_tag' job_PostCommit_Python_ValidatesContainer_Dataflow.groovy | cut -d'=' -f2)" -# Verify docker image has been built and pushed during the build: -# pull the x86 components; -docker pull $CONTAINER:$TAG -# pull the ARM components; -docker pull --platform linux/arm64 $CONTAINER:$TAG - function cleanup_container { # Delete the container locally and remotely - docker rmi -q $CONTAINER || echo "Failed to remove local container image" for image in $(docker images --format '{{.Repository}}:{{.Tag}}' | grep $PREBUILD_SDK_CONTAINER_REGISTRY_PATH) do docker rmi $image || echo "Failed to remove prebuilt sdk container image" done From a078ec911420199403b063104cc84b316d047428 Mon Sep 17 00:00:00 2001 From: Celeste Zeng <61256376+celeste-zeng@users.noreply.github.com> Date: Tue, 18 Jul 2023 19:02:59 +0000 Subject: [PATCH 26/59] Revert the Multiarch Java container name changes to make the precommit test pass. --- .../google-cloud-dataflow-java/build.gradle | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/runners/google-cloud-dataflow-java/build.gradle b/runners/google-cloud-dataflow-java/build.gradle index 9841a82d22a7..b722d0146bc0 100644 --- a/runners/google-cloud-dataflow-java/build.gradle +++ b/runners/google-cloud-dataflow-java/build.gradle @@ -151,10 +151,10 @@ def javaVer = "java8" javaVer = "java11" } def dockerImageRoot = project.findProperty('docker-repository-root') ?: "us.gcr.io/${dataflowProject}/java-postcommit-it" -def dockerJavaMultiarchImageContainer = "${dockerImageRoot}/${project.docker_image_default_repo_prefix}java${javaVer}_sdk" +def dockerJavaImageContainer = "${dockerImageRoot}/${project.docker_image_default_repo_prefix}java${javaVer}_sdk" def dockerPythonImageContainer = "${dockerImageRoot}/python" def dockerTag = project.findProperty('docker-tag') ?: new Date().format('yyyyMMddHHmmss') -ext.dockerJavaMultiarchImageName = "${dockerJavaMultiarchImageContainer}:${dockerTag}" +ext.dockerJavaImageName = "${dockerJavaImageContainer}:${dockerTag}" ext.dockerPythonImageName = "${dockerPythonImageContainer}:${dockerTag}" def legacyPipelineOptions = [ @@ -171,7 +171,7 @@ def runnerV2PipelineOptions = [ "--project=${dataflowProject}", "--region=${dataflowRegion}", "--tempRoot=${dataflowValidatesTempRoot}", - "--sdkContainerImage=${dockerJavaMultiarchImageContainer}:${dockerTag}", + "--sdkContainerImage=${dockerJavaImageContainer}:${dockerTag}", "--experiments=use_unified_worker,use_runner_v2", "--firestoreDb=${firestoreDb}", ] @@ -286,10 +286,10 @@ def buildAndPushDockerJavaContainer = tasks.register("buildAndPushDockerJavaCont def cleanUpDockerJavaImages = tasks.register("cleanUpDockerJavaImages") { doLast { exec { - commandLine "gcloud", "--quiet", "container", "images", "untag", "${dockerJavaMultiarchImageName}" + commandLine "gcloud", "--quiet", "container", "images", "untag", "${dockerJavaImageName}" } exec { - commandLine "./scripts/cleanup_untagged_gcr_images.sh", "${dockerJavaMultiarchImageContainer}" + commandLine "./scripts/cleanup_untagged_gcr_images.sh", "${dockerJavaImageContainer}" } } } @@ -354,7 +354,7 @@ task printRunnerV2PipelineOptions { println "To run a Dataflow job with runner V2, add the following pipeline options to your command-line:" println runnerV2PipelineOptions.join(' ') println "Please delete your image upon completion with the following command:" - println "docker rmi -q ${dockerJavaMultiarchImageName}; gcloud container images delete --force-delete-tags ${dockerJavaMultiarchImageName}" + println "gcloud container images delete --force-delete-tags ${dockerJavaImageName}" } } @@ -414,14 +414,14 @@ createCrossLanguageValidatesRunnerTask( "--runner=TestDataflowRunner", "--project=${dataflowProject}", "--region=${dataflowRegion}", - "--sdk_harness_container_image_overrides=.*java.*,${dockerJavaMultiarchImageContainer}:${dockerTag}", + "--sdk_harness_container_image_overrides=.*java.*,${dockerJavaImageContainer}:${dockerTag}", ], javaPipelineOptions: [ "--runner=TestDataflowRunner", "--project=${dataflowProject}", "--region=${dataflowRegion}", "--tempRoot=${dataflowValidatesTempRoot}", - "--sdkContainerImage=${dockerJavaMultiarchImageContainer}:${dockerTag}", + "--sdkContainerImage=${dockerJavaImageContainer}:${dockerTag}", "--sdkHarnessContainerImageOverrides=.*python.*,${dockerPythonImageContainer}:${dockerTag}", ], pytestOptions: [ @@ -436,7 +436,7 @@ createCrossLanguageValidatesRunnerTask( "--dataflow_project ${dataflowProject}", "--region ${dataflowRegion}", "--tests \"./test/integration/xlang ./test/integration/io/xlang/...\"", - "--sdk_overrides \".*java.*,${dockerJavaMultiarchImageContainer}:${dockerTag}\"", + "--sdk_overrides \".*java.*,${dockerJavaImageContainer}:${dockerTag}\"", ], ) From 84fbc02e195da068f8460732916f93cc500adf6f Mon Sep 17 00:00:00 2001 From: Celeste Zeng <61256376+celeste-zeng@users.noreply.github.com> Date: Wed, 19 Jul 2023 05:59:37 +0000 Subject: [PATCH 27/59] Remove regenerate base image requirements for each Python validatesContainer test run. --- .../job_PostCommit_Python_ValidatesContainer_Dataflow.groovy | 2 -- sdks/python/test-suites/dataflow/common.gradle | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/.test-infra/jenkins/job_PostCommit_Python_ValidatesContainer_Dataflow.groovy b/.test-infra/jenkins/job_PostCommit_Python_ValidatesContainer_Dataflow.groovy index cc5c4a10a3b8..3397e5e7dc29 100644 --- a/.test-infra/jenkins/job_PostCommit_Python_ValidatesContainer_Dataflow.groovy +++ b/.test-infra/jenkins/job_PostCommit_Python_ValidatesContainer_Dataflow.groovy @@ -45,8 +45,6 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_Py_ValCont', tasks(':sdks:python:test-suites:dataflow:validatesContainerTests') switches('-Pcontainer-architecture-list=arm64,amd64') switches('-Ppush-containers') - // Regenerates the base image requirements to accommodate ARM. - switches('-Pregenerate-requirements') // Push multi-arch containers to the repository set in run_validatescontainer.sh switches('-Pdocker-repository-root=us.gcr.io/apache-beam-testing/jenkins') switches('-Pdocker-tag=${unique_tag}') diff --git a/sdks/python/test-suites/dataflow/common.gradle b/sdks/python/test-suites/dataflow/common.gradle index fd26c965b68c..36fc75f2e412 100644 --- a/sdks/python/test-suites/dataflow/common.gradle +++ b/sdks/python/test-suites/dataflow/common.gradle @@ -351,7 +351,7 @@ task chicagoTaxiExample { task validatesContainer() { def pyversion = "${project.ext.pythonVersion.replace('.', '')}" - if (project.hasProperty("testRCDependencies") || project.hasProperty("regenerate-requirements")) { + if (project.hasProperty("testRCDependencies")) { // Generate a requirements file with pre-release versions for the docker task // if testing with pre-release dependencies. dependsOn ":sdks:python:container:py${pyversion}:generatePythonRequirements" From bf27b0e7927d6a51aa08dabc84dcc64c23487b5f Mon Sep 17 00:00:00 2001 From: Celeste Zeng <61256376+celeste-zeng@users.noreply.github.com> Date: Thu, 20 Jul 2023 23:51:16 +0000 Subject: [PATCH 28/59] [Build/Release] Change release settings so that the release Beam SDk containers will be multiarch. --- .../groovy/org/apache/beam/gradle/BeamModulePlugin.groovy | 2 +- sdks/go/container/build.gradle | 7 ++++--- sdks/java/container/build.gradle | 6 +++--- sdks/java/container/common.gradle | 5 +++-- sdks/python/container/common.gradle | 5 +++-- 5 files changed, 14 insertions(+), 11 deletions(-) diff --git a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy index 9a42ab330aa3..2e70006eeac2 100644 --- a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy +++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy @@ -490,7 +490,7 @@ class BeamModulePlugin implements Plugin { project.ext.containerArchitectures = { if (isRelease(project)) { // Ensure we always publish the expected containers. - return ["amd64"]; + return ["amd64","arm64"]; } else if (project.rootProject.findProperty("container-architecture-list") != null) { def containerArchitectures = project.rootProject.findProperty("container-architecture-list").split(',') if (containerArchitectures.size() > 1 && !project.rootProject.hasProperty("push-containers")) { diff --git a/sdks/go/container/build.gradle b/sdks/go/container/build.gradle index 5cf0e925114c..8429de133ee7 100644 --- a/sdks/go/container/build.gradle +++ b/sdks/go/container/build.gradle @@ -28,6 +28,7 @@ goBuild { } def useBuildx = project.containerPlatforms() != [project.nativeArchitecture()] +def pushContainers = project.rootProject.hasProperty(["isRelease"]) || project.rootProject.hasProperty("push-containers") docker { name containerImageName( @@ -42,8 +43,8 @@ docker { project.rootProject.hasProperty(["isRelease"])]) buildx useBuildx platform(*project.containerPlatforms()) - load useBuildx && !project.rootProject.hasProperty("push-containers") - push project.rootProject.hasProperty("push-containers") + load useBuildx && !pushContainers + push pushContainers } dockerPrepare.dependsOn tasks.named("goBuild") @@ -67,5 +68,5 @@ if (project.rootProject.hasProperty(["docker-pull-licenses"])) { } task pushAll { - dependsOn ":sdks:go:container:dockerPush" + dependsOn ":sdks:go:container:docker" } \ No newline at end of file diff --git a/sdks/java/container/build.gradle b/sdks/java/container/build.gradle index b44addb51070..4c4b6aaa31fd 100644 --- a/sdks/java/container/build.gradle +++ b/sdks/java/container/build.gradle @@ -80,7 +80,7 @@ artifacts { } task pushAll { - dependsOn ":sdks:java:container:java8:dockerPush" - dependsOn ":sdks:java:container:java11:dockerPush" - dependsOn ":sdks:java:container:java17:dockerPush" + dependsOn ":sdks:java:container:java8:docker" + dependsOn ":sdks:java:container:java11:docker" + dependsOn ":sdks:java:container:java17:docker" } diff --git a/sdks/java/container/common.gradle b/sdks/java/container/common.gradle index ec1c1744dfde..bf4c122ca91f 100644 --- a/sdks/java/container/common.gradle +++ b/sdks/java/container/common.gradle @@ -105,6 +105,7 @@ task validateJavaHome { } def useBuildx = project.containerPlatforms() != [project.nativeArchitecture()] +def pushContainers = project.rootProject.hasProperty(["isRelease"]) || project.rootProject.hasProperty("push-containers") docker { name containerImageName( @@ -125,8 +126,8 @@ docker { ]) buildx useBuildx platform(*project.containerPlatforms()) - load useBuildx && !project.rootProject.hasProperty("push-containers") - push project.rootProject.hasProperty("push-containers") + load useBuildx && !pushContainers + push pushContainers } if (project.rootProject.hasProperty(["docker-pull-licenses"]) || diff --git a/sdks/python/container/common.gradle b/sdks/python/container/common.gradle index b11dc8127c7d..bb706aa5c5d8 100644 --- a/sdks/python/container/common.gradle +++ b/sdks/python/container/common.gradle @@ -67,6 +67,7 @@ def copyLauncherDependencies = tasks.register("copyLauncherDependencies", Copy) } def useBuildx = project.containerPlatforms() != [project.nativeArchitecture()] +def pushContainers = project.rootProject.hasProperty(["isRelease"]) || project.rootProject.hasProperty("push-containers") docker { name containerImageName( @@ -84,8 +85,8 @@ docker { project.rootProject.hasProperty(["isRelease"])]) buildx useBuildx platform(*project.containerPlatforms()) - load useBuildx && !project.rootProject.hasProperty("push-containers") - push project.rootProject.hasProperty("push-containers") + load useBuildx && !pushContainers + push pushContainers } dockerPrepare.dependsOn copyLauncherDependencies From a4adb77d180c3c7e4b7c34042d6a8b438507af87 Mon Sep 17 00:00:00 2001 From: Celeste Zeng <61256376+celeste-zeng@users.noreply.github.com> Date: Thu, 20 Jul 2023 17:57:52 -0700 Subject: [PATCH 29/59] [Build/Release]Update release task for Beam Python SDK containers --- sdks/python/container/build.gradle | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/sdks/python/container/build.gradle b/sdks/python/container/build.gradle index f6b31eec5b9b..f44187e57ca6 100644 --- a/sdks/python/container/build.gradle +++ b/sdks/python/container/build.gradle @@ -45,13 +45,13 @@ tasks.register("buildAll") { for(int i=8; i<=11; ++i) { String cur = "3" + i String prev = "3" + (i-1) - tasks.register("push" + cur) { + tasks.register("docker" + cur) { if (cur != "38") { // Enforce ordering to allow the prune step to happen between runs. // This will ensure we don't use up too much space (especially in CI environments) - mustRunAfter(":sdks:python:container:push" + prev) + mustRunAfter(":sdks:python:container:docker" + prev) } - dependsOn ':sdks:python:container:py' + cur + ':dockerPush' + dependsOn ':sdks:python:container:py' + cur + ':docker' doLast { if (project.hasProperty("prune-images")) { @@ -65,10 +65,10 @@ for(int i=8; i<=11; ++i) { } tasks.register("pushAll") { - dependsOn ':sdks:python:container:push38' - dependsOn ':sdks:python:container:push39' - dependsOn ':sdks:python:container:push310' - dependsOn ':sdks:python:container:push311' + dependsOn ':sdks:python:container:docker38' + dependsOn ':sdks:python:container:docker39' + dependsOn ':sdks:python:container:docker310' + dependsOn ':sdks:python:container:docker311' } tasks.register("generatePythonRequirementsAll") { From 66b68c6dbadfa2f347bd96b1bc13252cbbd8c3e1 Mon Sep 17 00:00:00 2001 From: Celeste Zeng <61256376+celeste-zeng@users.noreply.github.com> Date: Thu, 20 Jul 2023 18:58:49 -0700 Subject: [PATCH 30/59] [Build/Release] Update BeamModulePlugin.groovy to fix spotlessGroovyCheck --- .../main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy index d55a26564f21..27b475d9cae0 100644 --- a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy +++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy @@ -490,7 +490,7 @@ class BeamModulePlugin implements Plugin { project.ext.containerArchitectures = { if (isRelease(project)) { // Ensure we always publish the expected containers. - return ["amd64","arm64"]; + return ["amd64", "arm64"]; } else if (project.rootProject.findProperty("container-architecture-list") != null) { def containerArchitectures = project.rootProject.findProperty("container-architecture-list").split(',') if (containerArchitectures.size() > 1 && !project.rootProject.hasProperty("push-containers")) { From 3a096425ee671ecacd2792b8350580af9b6c610d Mon Sep 17 00:00:00 2001 From: Celeste Zeng <61256376+celeste-zeng@users.noreply.github.com> Date: Fri, 21 Jul 2023 19:01:22 +0000 Subject: [PATCH 31/59] [Build/Release] Remove trailing whitespaces. --- runners/google-cloud-dataflow-java/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runners/google-cloud-dataflow-java/build.gradle b/runners/google-cloud-dataflow-java/build.gradle index b722d0146bc0..0bcf79c1fe6b 100644 --- a/runners/google-cloud-dataflow-java/build.gradle +++ b/runners/google-cloud-dataflow-java/build.gradle @@ -279,7 +279,7 @@ def createRunnerV2ValidatesRunnerTest = { Map args -> // task ordering such that the registry doesn't get cleaned up prior to task completion. def buildAndPushDockerJavaContainer = tasks.register("buildAndPushDockerJavaContainer") { // The multi-arch Java container has been pushed during build. - dependsOn ":sdks:java:container:${javaVer}:docker" + dependsOn ":sdks:java:container:${javaVer}:docker" } // Clean up built Java images From 24ebacc75070e5d740cdf8f672571a3051fd0895 Mon Sep 17 00:00:00 2001 From: Celeste Zeng <61256376+celeste-zeng@users.noreply.github.com> Date: Fri, 21 Jul 2023 21:26:12 +0000 Subject: [PATCH 32/59] [Build/Release] Change the pushAll task for Python SDK containers to use the buildAll task. --- sdks/python/container/build.gradle | 27 +-------------------------- 1 file changed, 1 insertion(+), 26 deletions(-) diff --git a/sdks/python/container/build.gradle b/sdks/python/container/build.gradle index f44187e57ca6..fec9bc98c33d 100644 --- a/sdks/python/container/build.gradle +++ b/sdks/python/container/build.gradle @@ -42,33 +42,8 @@ tasks.register("buildAll") { dependsOn ':sdks:python:container:py311:docker' } -for(int i=8; i<=11; ++i) { - String cur = "3" + i - String prev = "3" + (i-1) - tasks.register("docker" + cur) { - if (cur != "38") { - // Enforce ordering to allow the prune step to happen between runs. - // This will ensure we don't use up too much space (especially in CI environments) - mustRunAfter(":sdks:python:container:docker" + prev) - } - dependsOn ':sdks:python:container:py' + cur + ':docker' - - doLast { - if (project.hasProperty("prune-images")) { - exec { - executable("docker") - args("system", "prune", "-a", "--force") - } - } - } - } -} - tasks.register("pushAll") { - dependsOn ':sdks:python:container:docker38' - dependsOn ':sdks:python:container:docker39' - dependsOn ':sdks:python:container:docker310' - dependsOn ':sdks:python:container:docker311' + dependsOn ':sdks:python:container:buildAll' } tasks.register("generatePythonRequirementsAll") { From 043875188284e48a545fa02c561dfe7ee2aae08c Mon Sep 17 00:00:00 2001 From: Celeste Zeng <61256376+celeste-zeng@users.noreply.github.com> Date: Mon, 24 Jul 2023 11:03:42 -0700 Subject: [PATCH 33/59] [New Arm Tests]Create job_PostCommit_Python_Dataflow_Arm.groovy --- .../job_PostCommit_Python_Dataflow_Arm.groovy | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 .test-infra/jenkins/job_PostCommit_Python_Dataflow_Arm.groovy diff --git a/.test-infra/jenkins/job_PostCommit_Python_Dataflow_Arm.groovy b/.test-infra/jenkins/job_PostCommit_Python_Dataflow_Arm.groovy new file mode 100644 index 000000000000..905c36ed2ffe --- /dev/null +++ b/.test-infra/jenkins/job_PostCommit_Python_Dataflow_Arm.groovy @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * This test builds Beam Python SDK multi-arch containers and use the + * Arm component to run Dataflow wordcount pipelines on Arm. + * + */ +import CommonJobProperties as commonJobProperties +import PostcommitJobBuilder + +import static PythonTestProperties.VALIDATES_CONTAINER_DATAFLOW_PYTHON_VERSIONS +import java.time.LocalDateTime + +// This job runs the suite of Python Arm tests against the +// Dataflow runner. +PostcommitJobBuilder.postCommitJob('beam_PostCommit_Py_Arm', + archiveJunit('**/pytest*.xml') + } + + // Generates a unique tag for the container as the current time. + def now = LocalDateTime.now() + def unique_tag = '${now.date}${now.hour}${now.minute}${now.second}' + // Execute shell command to test Python SDK. + steps { + gradle { + rootBuildScriptDir(commonJobProperties.checkoutDir) + tasks(':sdks:python:test-suites:dataflow:validatesContainerTests') + switches('-Pcontainer-architecture-list=arm64,amd64') + switches('-Ppush-containers') + // Push multi-arch containers to the repository set in run_validatescontainer.sh + switches('-Pdocker-repository-root=us.gcr.io/apache-beam-testing/jenkins') + switches('-Pdocker-tag=${unique_tag}') + commonJobProperties.setGradleSwitches(delegate) + } + } + } From 9820b24f95ac847c5c2eba74683bc20c359c8a4b Mon Sep 17 00:00:00 2001 From: Celeste Zeng <61256376+celeste-zeng@users.noreply.github.com> Date: Mon, 24 Jul 2023 18:23:34 +0000 Subject: [PATCH 34/59] [New Arm Tests] Changed .txt and .md files for the newly added Dataflow Python Arm test. --- .test-infra/BUILD_STATUS.md | 3 +++ .test-infra/jenkins/README.md | 1 + release/src/main/scripts/jenkins_jobs.txt | 1 + 3 files changed, 5 insertions(+) diff --git a/.test-infra/BUILD_STATUS.md b/.test-infra/BUILD_STATUS.md index 1972076df1e2..b0da79e557ff 100644 --- a/.test-infra/BUILD_STATUS.md +++ b/.test-infra/BUILD_STATUS.md @@ -133,6 +133,9 @@
Build Status +
+ + Build Status diff --git a/.test-infra/jenkins/README.md b/.test-infra/jenkins/README.md index 02cddfdc65c7..83ca6cee9a45 100644 --- a/.test-infra/jenkins/README.md +++ b/.test-infra/jenkins/README.md @@ -154,6 +154,7 @@ Beam Jenkins overview page: [link](https://ci-beam.apache.org/) | beam_PostCommit_Py_ValCont_with_RC | [cron](https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont_with_RC/), [phrase](https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont_with_RC_PR/) | `Run Python RC Dataflow ValidatesContainer` | [![Build Status](https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont_with_RC/badge/icon)](https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont_with_RC) | | beam_PostCommit_Python_Examples_Direct | [cron](https://ci-beam.apache.org/job/beam_PostCommit_Python_Examples_Direct/), [phrase](https://ci-beam.apache.org/job/beam_PostCommit_Python_Examples_Direct_PR/) | `Run Python Examples_Direct` | [![Build Status](https://ci-beam.apache.org/job/beam_PostCommit_Python_Examples_Direct/badge/icon)](https://ci-beam.apache.org/job/beam_PostCommit_Python_Examples_Direct/) | | beam_PostCommit_Python_Examples_Dataflow | [cron](https://ci-beam.apache.org/job/beam_PostCommit_Python_Examples_Dataflow/), [phrase](https://ci-beam.apache.org/job/beam_PostCommit_Python_Examples_Dataflow_PR/) | `Run Python Examples_Dataflow` | [![Build Status](https://ci-beam.apache.org/job/beam_PostCommit_Python_Examples_Dataflow/badge/icon)](https://ci-beam.apache.org/job/beam_PostCommit_Python_Examples_Dataflow/) | +| beam_PostCommit_Py_Dataflow_Arm| [cron](https://ci-beam.apache.org/job/beam_PostCommit_Py_Dataflow_Arm/), [phrase](https://ci-beam.apache.org/job/beam_PostCommit_Py_Dataflow_Arm_PR/) | `Run Python Examples_Dataflow` | [![Build Status](https://ci-beam.apache.org/job/beam_PostCommit_Py_Dataflow_Arm/badge/icon)](https://ci-beam.apache.org/job/beam_PostCommit_Py_Dataflow_Arm/) | | beam_PostCommit_Python_Examples_Flink | [cron](https://ci-beam.apache.org/job/beam_PostCommit_Python_Examples_Flink/), [phrase](https://ci-beam.apache.org/job/beam_PostCommit_Python_Examples_Flink_PR/) | `Run Python Examples_Flink` | [![Build Status](https://ci-beam.apache.org/job/beam_PostCommit_Python_Examples_Flink/badge/icon)](https://ci-beam.apache.org/job/beam_PostCommit_Python_Examples_Flink/) | | beam_PostCommit_Python_Examples_Spark | [cron](https://ci-beam.apache.org/job/beam_PostCommit_Python_Examples_Spark/), [phrase](https://ci-beam.apache.org/job/beam_PostCommit_Python_Examples_Spark_PR/) | `Run Python Examples_Spark` | [![Build Status](https://ci-beam.apache.org/job/beam_PostCommit_Python_Examples_Spark/badge/icon)](https://ci-beam.apache.org/job/beam_PostCommit_Python_Examples_Spark/) | | beam_PostCommit_Python_Chicago_Taxi_Example_Dataflow | [cron](https://ci-beam.apache.org/job/beam_PostCommit_Python_Chicago_Taxi_Dataflow/), [phrase](https://ci-beam.apache.org/job/beam_PostCommit_Python_Chicago_Taxi_Dataflow_PR/) | `Run Chicago Taxi on Dataflow` | [![Build Status](https://ci-beam.apache.org/job/beam_PostCommit_Python_Chicago_Taxi_Dataflow/badge/icon)](https://ci-beam.apache.org/job/beam_PostCommit_Python_Chicago_Taxi_Dataflow) | diff --git a/release/src/main/scripts/jenkins_jobs.txt b/release/src/main/scripts/jenkins_jobs.txt index c32b47f1dbf6..957d3440e190 100644 --- a/release/src/main/scripts/jenkins_jobs.txt +++ b/release/src/main/scripts/jenkins_jobs.txt @@ -114,6 +114,7 @@ Run Python 3.9 PostCommit,beam_PostCommit_Python39_PR Run Python Dataflow V2 ValidatesRunner,beam_PostCommit_Py_VR_Dataflow_V2_PR Run Python Dataflow ValidatesContainer,beam_PostCommit_Py_ValCont_PR Run Python Dataflow ValidatesRunner,beam_PostCommit_Py_VR_Dataflow_PR +Run Python Dataflow Arm,beam_PostCommit_Py_Dataflow_Arm_PR Run Python Direct Runner Nexmark Tests,beam_PostCommit_Python_Nexmark_Direct_PR Run Python Examples_Dataflow,beam_PostCommit_Python_Examples_Dataflow_PR Run Python Examples_Direct,beam_PostCommit_Python_Examples_Direct_PR From a91c12b9ea26895da4762eb454cf761c963f8167 Mon Sep 17 00:00:00 2001 From: Celeste Zeng <61256376+celeste-zeng@users.noreply.github.com> Date: Mon, 24 Jul 2023 18:31:00 +0000 Subject: [PATCH 35/59] [New Arm Tests] Changed .test-infra/jenkins/job_PostCommit_Python_Dataflow_Arm.groovy. --- .../jenkins/job_PostCommit_Python_Dataflow_Arm.groovy | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.test-infra/jenkins/job_PostCommit_Python_Dataflow_Arm.groovy b/.test-infra/jenkins/job_PostCommit_Python_Dataflow_Arm.groovy index 905c36ed2ffe..ce8f856931d8 100644 --- a/.test-infra/jenkins/job_PostCommit_Python_Dataflow_Arm.groovy +++ b/.test-infra/jenkins/job_PostCommit_Python_Dataflow_Arm.groovy @@ -27,7 +27,12 @@ import java.time.LocalDateTime // This job runs the suite of Python Arm tests against the // Dataflow runner. -PostcommitJobBuilder.postCommitJob('beam_PostCommit_Py_Arm', +PostcommitJobBuilder.postCommitJob('beam_PostCommit_Py_Dataflow_Arm', + 'Run Python Dataflow Arm', 'Google Cloud Dataflow Runner Python Arm Tests', this) { + description('Runs Python Arm suite on the Dataflow runner.') + // Set common parameters. + commonJobProperties.setTopLevelMainJobProperties(delegate) + publishers { archiveJunit('**/pytest*.xml') } From 150b6597cce2d213adf13c3b20e71d1b36e07b16 Mon Sep 17 00:00:00 2001 From: Celeste Zeng <61256376+celeste-zeng@users.noreply.github.com> Date: Mon, 24 Jul 2023 19:11:46 +0000 Subject: [PATCH 36/59] [New Arm Tests] Added a script file for the new Dataflow Python Arm test and changed other related files. --- .../release/test/resources/mass_comment.txt | 1 + .../apache_beam/examples/wordcount_it_test.py | 9 ++ sdks/python/container/run_dataflow_arm.sh | 114 ++++++++++++++++++ sdks/python/pytest.ini | 1 + .../python/test-suites/dataflow/common.gradle | 21 ++++ 5 files changed, 146 insertions(+) create mode 100644 sdks/python/container/run_dataflow_arm.sh diff --git a/scripts/ci/release/test/resources/mass_comment.txt b/scripts/ci/release/test/resources/mass_comment.txt index 93468b0c961b..27470ae9a540 100644 --- a/scripts/ci/release/test/resources/mass_comment.txt +++ b/scripts/ci/release/test/resources/mass_comment.txt @@ -66,6 +66,7 @@ Run Python 3.11 PostCommit Run Python Dataflow V2 ValidatesRunner Run Python Dataflow ValidatesContainer Run Python Dataflow ValidatesRunner +Run Python Dataflow Arm Run Python Examples_Dataflow Run Python Examples_Direct Run Python Examples_Flink diff --git a/sdks/python/apache_beam/examples/wordcount_it_test.py b/sdks/python/apache_beam/examples/wordcount_it_test.py index bb5859df7251..f1aa54584965 100644 --- a/sdks/python/apache_beam/examples/wordcount_it_test.py +++ b/sdks/python/apache_beam/examples/wordcount_it_test.py @@ -112,6 +112,15 @@ def test_wordcount_it_with_prebuilt_sdk_container_cloud_build(self): def test_wordcount_it_with_use_sibling_sdk_workers(self): self._run_wordcount_it(wordcount.run, experiment='use_sibling_sdk_workers') + @pytest.mark.it_postcommit + @pytest.mark.it_dataflow_arm + def test_wordcount_fnapi_it_arm(self): + self._run_wordcount_it(wordcount.run, experiment='beam_fn_api', machine_type='t2a-standard-1') + + @pytest.mark.it_dataflow_arm + def test_wordcount_it_with_use_sibling_sdk_workers_arm(self): + self._run_wordcount_it(wordcount.run, experiment='use_sibling_sdk_workers', machine_type='t2a-standard-1') + def _run_wordcount_it(self, run_wordcount, **opts): test_pipeline = TestPipeline(is_integration_test=True) extra_opts = {} diff --git a/sdks/python/container/run_dataflow_arm.sh b/sdks/python/container/run_dataflow_arm.sh new file mode 100644 index 000000000000..0176923dffd6 --- /dev/null +++ b/sdks/python/container/run_dataflow_arm.sh @@ -0,0 +1,114 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# This script will be run by Jenkins as a post commit test. In order to run +# locally make the following changes: +# +# GCS_LOCATION -> Temporary location to use for service tests. +# PROJECT -> Project name to use for dataflow and docker images. +# REGION -> Region name to use for Dataflow +# +# Execute from the root of the repository: +# test Python3.8 container: +# ./gradlew :sdks:python:test-suites:dataflow:py38:dataflowArm +# or test all supported python versions together: +# ./gradlew :sdks:python:test-suites:dataflow:dataflowArm + +echo "This script must be executed in the root of beam project. Please set GCS_LOCATION, PROJECT and REGION as desired." + +if [[ $# != 2 ]]; then + printf "Usage: \n$> ./sdks/python/container/run_dataflow_arm.sh " + printf "\n\tpython_version: [required] Python version used for container build and run tests." + printf " Sample value: 3.9" + exit 1 +fi + +set -e +set -v + +# Where to store integration test outputs. +GCS_LOCATION=${GCS_LOCATION:-gs://temp-storage-for-end-to-end-tests} + +# Project for the container and integration test +PROJECT=${PROJECT:-apache-beam-testing} +REGION=${REGION:-us-central1} +IMAGE_PREFIX="$(grep 'docker_image_default_repo_prefix' gradle.properties | cut -d'=' -f2)" +SDK_VERSION="$(grep 'sdk_version' gradle.properties | cut -d'=' -f2)" +PY_VERSION=$1 +IMAGE_NAME="${IMAGE_PREFIX}python${PY_VERSION}_sdk" +CONTAINER_PROJECT="sdks:python:container:py${PY_VERSION//.}" # Note: we substitute away the dot in the version. +PY_INTERPRETER="python${PY_VERSION}" + +XUNIT_FILE="pytest-$IMAGE_NAME.xml" + +# Verify in the root of the repository +test -d sdks/python/container + +# Verify docker and gcloud commands exist +command -v docker +command -v gcloud +docker -v +gcloud -v + +CONTAINER=us.gcr.io/$PROJECT/$USER/$IMAGE_NAME +PREBUILD_SDK_CONTAINER_REGISTRY_PATH=us.gcr.io/$PROJECT/$USER/prebuild_python${PY_VERSION//.}_sdk +TAG="$(grep 'unique_tag' job_PostCommit_Python_Dataflow_Arm.groovy | cut -d'=' -f2)" + +function cleanup_container { + # Delete the container locally and remotely + for image in $(docker images --format '{{.Repository}}:{{.Tag}}' | grep $PREBUILD_SDK_CONTAINER_REGISTRY_PATH) + do docker rmi $image || echo "Failed to remove prebuilt sdk container image" + done + gcloud --quiet container images delete $CONTAINER:$TAG || echo "Failed to delete container" + for digest in $(gcloud container images list-tags $PREBUILD_SDK_CONTAINER_REGISTRY_PATH/beam_python_prebuilt_sdk --format="get(digest)") + do gcloud container images delete $PREBUILD_SDK_CONTAINER_REGISTRY_PATH/beam_python_prebuilt_sdk@$digest --force-delete-tags --quiet || echo "Failed to remove prebuilt sdk container image" + done + + echo "Removed the container" +} +trap cleanup_container EXIT + +echo ">>> Successfully built and push container $CONTAINER" + +cd sdks/python +SDK_LOCATION=$2 + +# Run ValidatesRunner tests on Google Cloud Dataflow service +echo ">>> RUNNING DATAFLOW RUNNER PYTHON ARM TEST" +pytest -o junit_suite_name=$IMAGE_NAME \ + -m="it_dataflow_arm" \ + --show-capture=no \ + --numprocesses=1 \ + --timeout=1800 \ + --junitxml=$XUNIT_FILE \ + --ignore-glob '.*py3\d?\.py$' \ + --log-cli-level=INFO \ + --test-pipeline-options=" \ + --runner=TestDataflowRunner \ + --project=$PROJECT \ + --region=$REGION \ + --sdk_container_image=$CONTAINER:$TAG \ + --staging_location=$GCS_LOCATION/staging-dataflow-arm-test \ + --temp_location=$GCS_LOCATION/temp-dataflow-arm-test \ + --output=$GCS_LOCATION/output \ + --sdk_location=$SDK_LOCATION \ + --num_workers=1 \ + --docker_registry_push_url=$PREBUILD_SDK_CONTAINER_REGISTRY_PATH \ + --machine_type=t2a-standard-1" + +echo ">>> SUCCESS DATAFLOW RUNNER PYTHON ARM TEST" diff --git a/sdks/python/pytest.ini b/sdks/python/pytest.ini index 6e93c5f96e7f..5962c6d6e0b6 100644 --- a/sdks/python/pytest.ini +++ b/sdks/python/pytest.ini @@ -38,6 +38,7 @@ markers = no_sickbay_streaming: run without sickbay-streaming no_sickbay_batch: run without sickbay-batch it_validatescontainer: collect for ValidatesContainer integration test runs + it_dataflow_arm: collect for DataflowArm integration test runs examples_postcommit: collect for post-commit test examples runs sickbay_direct: run without sickbay-direct sickbay_spark: run without sickbay-spark diff --git a/sdks/python/test-suites/dataflow/common.gradle b/sdks/python/test-suites/dataflow/common.gradle index 12c440131855..4a59ddd3ece7 100644 --- a/sdks/python/test-suites/dataflow/common.gradle +++ b/sdks/python/test-suites/dataflow/common.gradle @@ -348,6 +348,27 @@ task validatesContainer() { } } +task dataflowArm() { + def pyversion = "${project.ext.pythonVersion.replace('.', '')}" + if (project.hasProperty("testRCDependencies")) { + // Generate a requirements file with pre-release versions for the docker task + // if testing with pre-release dependencies. + dependsOn ":sdks:python:container:py${pyversion}:generatePythonRequirements" + mustRunAfter ":sdks:python:container:py${pyversion}:generatePythonRequirements" + } + dependsOn 'initializeForDataflowJob' + dependsOn ":sdks:python:container:py${pyversion}:docker" + def runScriptsPath = "${rootDir}/sdks/python/container/run_dataflow_arm.sh" + doLast { + exec { + executable 'sh' + args '-c', ". ${envdir}/bin/activate && cd ${rootDir} && ${runScriptsPath} " + + "${project.ext.pythonVersion} " + + "${project.ext.sdkLocation}" + } + } +} + def tensorRTTests = tasks.create("tensorRTtests") { dependsOn 'installGcpTest' dependsOn ':sdks:python:sdist' From f0eec3946e5589de92b57366ee3c884035e50f11 Mon Sep 17 00:00:00 2001 From: Celeste Zeng <61256376+celeste-zeng@users.noreply.github.com> Date: Mon, 24 Jul 2023 16:19:48 -0700 Subject: [PATCH 37/59] [Existing test] Format job_PostCommit_Python_ValidatesContainer_Dataflow.groovy --- .../job_PostCommit_Python_ValidatesContainer_Dataflow.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.test-infra/jenkins/job_PostCommit_Python_ValidatesContainer_Dataflow.groovy b/.test-infra/jenkins/job_PostCommit_Python_ValidatesContainer_Dataflow.groovy index 3397e5e7dc29..c62be7aa6ed0 100644 --- a/.test-infra/jenkins/job_PostCommit_Python_ValidatesContainer_Dataflow.groovy +++ b/.test-infra/jenkins/job_PostCommit_Python_ValidatesContainer_Dataflow.groovy @@ -47,7 +47,7 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_Py_ValCont', switches('-Ppush-containers') // Push multi-arch containers to the repository set in run_validatescontainer.sh switches('-Pdocker-repository-root=us.gcr.io/apache-beam-testing/jenkins') - switches('-Pdocker-tag=${unique_tag}') + switches("-Pdocker-tag=${unique_tag}") commonJobProperties.setGradleSwitches(delegate) } } From 39c7c55ade6eb51bde34c63045cc1a7aa917837f Mon Sep 17 00:00:00 2001 From: Celeste Zeng <61256376+celeste-zeng@users.noreply.github.com> Date: Mon, 24 Jul 2023 16:25:19 -0700 Subject: [PATCH 38/59] [New Arm Tests] Fix formetts for wordcount_it_test.py --- sdks/python/apache_beam/examples/wordcount_it_test.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/sdks/python/apache_beam/examples/wordcount_it_test.py b/sdks/python/apache_beam/examples/wordcount_it_test.py index f1aa54584965..27581df78275 100644 --- a/sdks/python/apache_beam/examples/wordcount_it_test.py +++ b/sdks/python/apache_beam/examples/wordcount_it_test.py @@ -115,11 +115,15 @@ def test_wordcount_it_with_use_sibling_sdk_workers(self): @pytest.mark.it_postcommit @pytest.mark.it_dataflow_arm def test_wordcount_fnapi_it_arm(self): - self._run_wordcount_it(wordcount.run, experiment='beam_fn_api', machine_type='t2a-standard-1') + self._run_wordcount_it( + wordcount.run, experiment='beam_fn_api', machine_type='t2a-standard-1') @pytest.mark.it_dataflow_arm def test_wordcount_it_with_use_sibling_sdk_workers_arm(self): - self._run_wordcount_it(wordcount.run, experiment='use_sibling_sdk_workers', machine_type='t2a-standard-1') + self._run_wordcount_it( + wordcount.run, + experiment='use_sibling_sdk_workers', + machine_type='t2a-standard-1') def _run_wordcount_it(self, run_wordcount, **opts): test_pipeline = TestPipeline(is_integration_test=True) From 419b81b9d8e06a149cfc9f9cdb42795e7f1e145a Mon Sep 17 00:00:00 2001 From: Celeste Zeng <61256376+celeste-zeng@users.noreply.github.com> Date: Mon, 24 Jul 2023 16:31:45 -0700 Subject: [PATCH 39/59] [New Arm Tests] Update job_PostCommit_Python_Dataflow_Arm.groovy --- .test-infra/jenkins/job_PostCommit_Python_Dataflow_Arm.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.test-infra/jenkins/job_PostCommit_Python_Dataflow_Arm.groovy b/.test-infra/jenkins/job_PostCommit_Python_Dataflow_Arm.groovy index ce8f856931d8..fea2c184ae10 100644 --- a/.test-infra/jenkins/job_PostCommit_Python_Dataflow_Arm.groovy +++ b/.test-infra/jenkins/job_PostCommit_Python_Dataflow_Arm.groovy @@ -48,7 +48,7 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_Py_Dataflow_Arm', switches('-Ppush-containers') // Push multi-arch containers to the repository set in run_validatescontainer.sh switches('-Pdocker-repository-root=us.gcr.io/apache-beam-testing/jenkins') - switches('-Pdocker-tag=${unique_tag}') + switches("-Pdocker-tag=${unique_tag}") commonJobProperties.setGradleSwitches(delegate) } } From 3299c5bdf415106752134fdedfef693949202fc7 Mon Sep 17 00:00:00 2001 From: Celeste Zeng <61256376+celeste-zeng@users.noreply.github.com> Date: Tue, 25 Jul 2023 02:22:31 +0000 Subject: [PATCH 40/59] [Existing Tests] Correct image tag formats. --- .../jenkins/job_PostCommit_Java_Examples_Dataflow_V2.groovy | 2 +- .../job_PostCommit_Java_Examples_Dataflow_V2_Java11.groovy | 2 +- .../job_PostCommit_Java_Examples_Dataflow_V2_Java17.groovy | 2 +- .../job_PostCommit_Python_ValidatesContainer_Dataflow.groovy | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.test-infra/jenkins/job_PostCommit_Java_Examples_Dataflow_V2.groovy b/.test-infra/jenkins/job_PostCommit_Java_Examples_Dataflow_V2.groovy index e4aae79f77df..0a77834b8232 100644 --- a/.test-infra/jenkins/job_PostCommit_Java_Examples_Dataflow_V2.groovy +++ b/.test-infra/jenkins/job_PostCommit_Java_Examples_Dataflow_V2.groovy @@ -33,7 +33,7 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_Examples_Dataflow_V2', // Generates a unique tag for the container as the current time. def now = LocalDateTime.now() - def unique_tag = '${now.date}${now.hour}${now.minute}${now.second}' + def unique_tag = "${now.date}${now.hour}${now.minute}${now.second}" steps { gradle { rootBuildScriptDir(commonJobProperties.checkoutDir) diff --git a/.test-infra/jenkins/job_PostCommit_Java_Examples_Dataflow_V2_Java11.groovy b/.test-infra/jenkins/job_PostCommit_Java_Examples_Dataflow_V2_Java11.groovy index 38791d8df5d8..b37dbc548f3c 100644 --- a/.test-infra/jenkins/job_PostCommit_Java_Examples_Dataflow_V2_Java11.groovy +++ b/.test-infra/jenkins/job_PostCommit_Java_Examples_Dataflow_V2_Java11.groovy @@ -33,7 +33,7 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_Examples_Dataflow_V2_ja // Generates a unique tag for the container as the current time. def now = LocalDateTime.now() - def unique_tag = '${now.date}${now.hour}${now.minute}${now.second}' + def unique_tag = "${now.date}${now.hour}${now.minute}${now.second}" steps { gradle { rootBuildScriptDir(commonJobProperties.checkoutDir) diff --git a/.test-infra/jenkins/job_PostCommit_Java_Examples_Dataflow_V2_Java17.groovy b/.test-infra/jenkins/job_PostCommit_Java_Examples_Dataflow_V2_Java17.groovy index b8c9bfee8b9a..e07cb69b9ef2 100644 --- a/.test-infra/jenkins/job_PostCommit_Java_Examples_Dataflow_V2_Java17.groovy +++ b/.test-infra/jenkins/job_PostCommit_Java_Examples_Dataflow_V2_Java17.groovy @@ -33,7 +33,7 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_Examples_Dataflow_V2_ja // Generates a unique tag for the container as the current time. def now = LocalDateTime.now() - def unique_tag = '${now.date}${now.hour}${now.minute}${now.second}' + def unique_tag = "${now.date}${now.hour}${now.minute}${now.second}" steps { gradle { rootBuildScriptDir(commonJobProperties.checkoutDir) diff --git a/.test-infra/jenkins/job_PostCommit_Python_ValidatesContainer_Dataflow.groovy b/.test-infra/jenkins/job_PostCommit_Python_ValidatesContainer_Dataflow.groovy index 3397e5e7dc29..e804e0c54946 100644 --- a/.test-infra/jenkins/job_PostCommit_Python_ValidatesContainer_Dataflow.groovy +++ b/.test-infra/jenkins/job_PostCommit_Python_ValidatesContainer_Dataflow.groovy @@ -37,7 +37,7 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_Py_ValCont', // Generates a unique tag for the container as the current time. def now = LocalDateTime.now() - def unique_tag = '${now.date}${now.hour}${now.minute}${now.second}' + def unique_tag = "${now.date}${now.hour}${now.minute}${now.second}" // Execute shell command to test Python SDK. steps { gradle { From 17cdc264b2525f6e22cb348564b75414fd3fd13e Mon Sep 17 00:00:00 2001 From: Celeste Zeng <61256376+celeste-zeng@users.noreply.github.com> Date: Tue, 25 Jul 2023 03:05:26 +0000 Subject: [PATCH 41/59] [New Arm Tests] fix format for job_PostCommit_Python_Dataflow_Arm --- .test-infra/jenkins/job_PostCommit_Python_Dataflow_Arm.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.test-infra/jenkins/job_PostCommit_Python_Dataflow_Arm.groovy b/.test-infra/jenkins/job_PostCommit_Python_Dataflow_Arm.groovy index fea2c184ae10..7bf1eb4049f3 100644 --- a/.test-infra/jenkins/job_PostCommit_Python_Dataflow_Arm.groovy +++ b/.test-infra/jenkins/job_PostCommit_Python_Dataflow_Arm.groovy @@ -38,7 +38,7 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_Py_Dataflow_Arm', // Generates a unique tag for the container as the current time. def now = LocalDateTime.now() - def unique_tag = '${now.date}${now.hour}${now.minute}${now.second}' + def unique_tag = "${now.date}${now.hour}${now.minute}${now.second}" // Execute shell command to test Python SDK. steps { gradle { From 209737210bfff3761641d496a31b4a3bd1018180 Mon Sep 17 00:00:00 2001 From: Celeste Zeng <61256376+celeste-zeng@users.noreply.github.com> Date: Tue, 25 Jul 2023 22:04:17 +0000 Subject: [PATCH 42/59] [New Arm Tests] Added yml file for Go PostCommit Dataflow ARM test and added a new gradle task target to call the actual test. --- .../beam_PostCommit_Go_Dataflow_ARM.yml | 72 +++++++++++++++++++ build.gradle.kts | 4 ++ 2 files changed, 76 insertions(+) create mode 100644 .github/workflows/beam_PostCommit_Go_Dataflow_ARM.yml diff --git a/.github/workflows/beam_PostCommit_Go_Dataflow_ARM.yml b/.github/workflows/beam_PostCommit_Go_Dataflow_ARM.yml new file mode 100644 index 000000000000..953c233c0c92 --- /dev/null +++ b/.github/workflows/beam_PostCommit_Go_Dataflow_ARM.yml @@ -0,0 +1,72 @@ +name: Go PostCommit Dataflow ARM + +on: + push: + tags: ['v*'] + branches: ['master', 'release-*'] + paths: ['model/**', 'sdks/go.**', 'release/**', '.github/workflows/beam_PostCommit_Go_Dataflow_ARM.yml'] + pull_request_target: + branches: ['master', 'release-*' ] + paths: ['model/**', 'sdks/go.**', 'release/**'] + issue_comment: + types: [created] + schedule: + - cron: '* */6 * * *' +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}' + cancel-in-progress: true + +jobs: + beam_PostCommit_Go_Dataflow_ARM: + if: | + github.event_name == 'push' || + github.event_name == 'pull_request_target' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Go PostCommit Dataflow ARM' + runs-on: [self-hosted, ubuntu-20.04, main] + name: beam_PostCommit_Go_Dataflow_ARM + steps: + - uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + - name: Rerun on comment + if: github.event.comment.body == 'Run Go PostCommit Dataflow ARM' + uses: ./.github/actions/rerun-job-action + with: + pull_request_url: ${{ github.event.issue.pull_request.url }} + github_repository: ${{ github.repository }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ github.job }} + github_current_run_id: ${{ github.run_id }} + - name: Install Java + uses: actions/setup-java@v3.8.0 + with: + distribution: 'zulu' + java-version: '8' + - name: Install Go + uses: actions/setup-go@v4 + with: + go-version: '1.20' + - name: Setup Gradle + uses: gradle/gradle-build-action@v2 + with: + cache-read-only: false + - name: run goPostCommitDataflowARM test + run: ./gradlew :goPostCommitDataflowARM \ No newline at end of file diff --git a/build.gradle.kts b/build.gradle.kts index d202d9c76fbb..a76ef6576253 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -440,6 +440,10 @@ tasks.register("goPortablePreCommit") { dependsOn(":sdks:go:test:ulrValidatesRunner") } +tasks.register("goPostCommitDataflowARM") { + dependsOn(":sdks:go:test:dataflowValidatesRunnerARM64") +} + tasks.register("goPostCommit") { dependsOn(":goIntegrationTests") } From 2f69ec7c68b5945fad5e0d1a87d2de9777a4f52b Mon Sep 17 00:00:00 2001 From: Celeste Zeng <61256376+celeste-zeng@users.noreply.github.com> Date: Tue, 25 Jul 2023 22:40:09 +0000 Subject: [PATCH 43/59] [New Arm Tests] modify beam_PostCommit_Go_Dataflow_ARM and add it into the README file. --- .github/workflows/README.md | 1 + .github/workflows/beam_PostCommit_Go_Dataflow_ARM.yml | 4 ---- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 083a1326ae76..b6d10e863304 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -19,6 +19,7 @@ Please note that jobs with matrix need to have matrix element in the comment. Ex | Workflow name | Matrix | Trigger Phrase | Cron Status | |:-------------:|:------:|:--------------:|:-----------:| | [ Go PreCommit ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Go.yml) | N/A |`Run Go PreCommit`| [![Go PreCommit](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Go.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Go.yml) | +| [ Go PostCommit Dataflow ARM](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Go_Dataflow_ARM.yml) | N/A |`Run Go PostCommit Dataflow ARM`| [![Go PostCommit Dataflow ARM](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Go_Dataflow_ARM.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Go_Dataflow_ARM.yml) | | [ Python PreCommit Docker ](https://github.com/apache/beam/actions/workflows/job_PreCommit_Python_DockerBuild.yml) | ['3.8','3.9','3.10','3.11'] | `Run PythonDocker PreCommit (matrix_element)`| [![.github/workflows/job_PreCommit_Python_DockerBuild.yml](https://github.com/apache/beam/actions/workflows/job_PreCommit_Python_DockerBuild.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/job_PreCommit_Python_DockerBuild.yml) | | [ Python PreCommit Docs ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_PythonDocs.yml) | N/A | `Run PythonDocs PreCommit`| [![.github/workflows/beam_PreCommit_PythonDocs.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_PythonDocs.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_PythonDocs.yml) | | [ Python PreCommit Formatter ](https://github.com/apache/beam/actions/workflows/job_PreCommit_PythonAutoformatter.yml) | N/A | `Run PythonFormatter PreCommit`| [![.github/workflows/job_PreCommit_PythonAutoformatter.yml](https://github.com/apache/beam/actions/workflows/job_PreCommit_PythonAutoformatter.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/job_PreCommit_PythonAutoformatter.yml) | diff --git a/.github/workflows/beam_PostCommit_Go_Dataflow_ARM.yml b/.github/workflows/beam_PostCommit_Go_Dataflow_ARM.yml index 953c233c0c92..4ad32b271a44 100644 --- a/.github/workflows/beam_PostCommit_Go_Dataflow_ARM.yml +++ b/.github/workflows/beam_PostCommit_Go_Dataflow_ARM.yml @@ -5,9 +5,6 @@ on: tags: ['v*'] branches: ['master', 'release-*'] paths: ['model/**', 'sdks/go.**', 'release/**', '.github/workflows/beam_PostCommit_Go_Dataflow_ARM.yml'] - pull_request_target: - branches: ['master', 'release-*' ] - paths: ['model/**', 'sdks/go.**', 'release/**'] issue_comment: types: [created] schedule: @@ -37,7 +34,6 @@ jobs: beam_PostCommit_Go_Dataflow_ARM: if: | github.event_name == 'push' || - github.event_name == 'pull_request_target' || github.event_name == 'schedule' || github.event.comment.body == 'Run Go PostCommit Dataflow ARM' runs-on: [self-hosted, ubuntu-20.04, main] From 90055dcff3fcd23d986c89418c73e854fdecf1c8 Mon Sep 17 00:00:00 2001 From: Celeste Zeng <61256376+celeste-zeng@users.noreply.github.com> Date: Wed, 26 Jul 2023 11:39:01 -0700 Subject: [PATCH 44/59] [New ARM Tests] Update beam_PostCommit_Go_Dataflow_ARM.yml to run against local changes --- .github/workflows/beam_PostCommit_Go_Dataflow_ARM.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/beam_PostCommit_Go_Dataflow_ARM.yml b/.github/workflows/beam_PostCommit_Go_Dataflow_ARM.yml index b64496d88180..151221e341bb 100644 --- a/.github/workflows/beam_PostCommit_Go_Dataflow_ARM.yml +++ b/.github/workflows/beam_PostCommit_Go_Dataflow_ARM.yml @@ -32,9 +32,13 @@ concurrency: group: '${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}' cancel-in-progress: true +env: + USER: GithubAction + jobs: beam_PostCommit_Go_Dataflow_ARM: if: | + github.repository == 'celeste-zeng/beam' github.event_name == 'push' || github.event_name == 'schedule' || github.event.comment.body == 'Run Go PostCommit Dataflow ARM' @@ -67,4 +71,4 @@ jobs: with: cache-read-only: false - name: run goPostCommitDataflowARM test - run: ./gradlew :goPostCommitDataflowARM \ No newline at end of file + run: ./gradlew :goPostCommitDataflowARM From 85d0d20c6262a511ebf828ef9a89d3df93aa8b44 Mon Sep 17 00:00:00 2001 From: Celeste Zeng <61256376+celeste-zeng@users.noreply.github.com> Date: Wed, 26 Jul 2023 11:40:12 -0700 Subject: [PATCH 45/59] [New Arm Tests] format beam_PostCommit_Go_Dataflow_ARM.yml --- .github/workflows/beam_PostCommit_Go_Dataflow_ARM.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/beam_PostCommit_Go_Dataflow_ARM.yml b/.github/workflows/beam_PostCommit_Go_Dataflow_ARM.yml index 151221e341bb..4d21b0b65577 100644 --- a/.github/workflows/beam_PostCommit_Go_Dataflow_ARM.yml +++ b/.github/workflows/beam_PostCommit_Go_Dataflow_ARM.yml @@ -38,7 +38,7 @@ env: jobs: beam_PostCommit_Go_Dataflow_ARM: if: | - github.repository == 'celeste-zeng/beam' + github.repository == 'celeste-zeng/beam' || github.event_name == 'push' || github.event_name == 'schedule' || github.event.comment.body == 'Run Go PostCommit Dataflow ARM' From eed708019d5ee7af4dac23e2d875c7db76b0005c Mon Sep 17 00:00:00 2001 From: Celeste Zeng <61256376+celeste-zeng@users.noreply.github.com> Date: Wed, 26 Jul 2023 16:35:59 -0700 Subject: [PATCH 46/59] [New Arm Tests] Make temp changes in beam_PostCommit_Go_Dataflow_ARM.yml --- .github/workflows/beam_PostCommit_Go_Dataflow_ARM.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/beam_PostCommit_Go_Dataflow_ARM.yml b/.github/workflows/beam_PostCommit_Go_Dataflow_ARM.yml index fc3d97eb3230..30230cc13822 100644 --- a/.github/workflows/beam_PostCommit_Go_Dataflow_ARM.yml +++ b/.github/workflows/beam_PostCommit_Go_Dataflow_ARM.yml @@ -38,11 +38,11 @@ env: jobs: beam_PostCommit_Go_Dataflow_ARM: if: | - github.repository == 'celeste-zeng/beam' || github.event_name == 'push' || + github.event_name == 'pull_request_target' || github.event_name == 'schedule' || github.event.comment.body == 'Run Go PostCommit Dataflow ARM' - runs-on: [self-hosted, ubuntu-20.04, main] + runs-on: [ubuntu-20.04, main] name: beam_PostCommit_Go_Dataflow_ARM steps: - uses: actions/checkout@v3 From 83f4b2826c53245c03e5311999d3e8794a72d77c Mon Sep 17 00:00:00 2001 From: Celeste Zeng <61256376+celeste-zeng@users.noreply.github.com> Date: Wed, 26 Jul 2023 17:56:32 -0700 Subject: [PATCH 47/59] [New Arm tests] try to not skip beam_PostCommit_Go_Dataflow_ARM.yml --- .github/workflows/beam_PostCommit_Go_Dataflow_ARM.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/beam_PostCommit_Go_Dataflow_ARM.yml b/.github/workflows/beam_PostCommit_Go_Dataflow_ARM.yml index 30230cc13822..2e748f86ee83 100644 --- a/.github/workflows/beam_PostCommit_Go_Dataflow_ARM.yml +++ b/.github/workflows/beam_PostCommit_Go_Dataflow_ARM.yml @@ -42,7 +42,7 @@ jobs: github.event_name == 'pull_request_target' || github.event_name == 'schedule' || github.event.comment.body == 'Run Go PostCommit Dataflow ARM' - runs-on: [ubuntu-20.04, main] + runs-on: [ubuntu-20.04] name: beam_PostCommit_Go_Dataflow_ARM steps: - uses: actions/checkout@v3 From 89a69442a9d33bf6c897b1366bfe780e94818591 Mon Sep 17 00:00:00 2001 From: Celeste Zeng <61256376+celeste-zeng@users.noreply.github.com> Date: Wed, 26 Jul 2023 18:35:59 -0700 Subject: [PATCH 48/59] [New Arm Tests] repository only allows lower case names --- .github/workflows/beam_PostCommit_Go_Dataflow_ARM.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/beam_PostCommit_Go_Dataflow_ARM.yml b/.github/workflows/beam_PostCommit_Go_Dataflow_ARM.yml index 2e748f86ee83..69ee2ec563a3 100644 --- a/.github/workflows/beam_PostCommit_Go_Dataflow_ARM.yml +++ b/.github/workflows/beam_PostCommit_Go_Dataflow_ARM.yml @@ -33,7 +33,7 @@ concurrency: cancel-in-progress: true env: - USER: GithubAction + USER: github-actions jobs: beam_PostCommit_Go_Dataflow_ARM: From 3c339101917fc21d28617eda35979bb047a94a4c Mon Sep 17 00:00:00 2001 From: Celeste Zeng <61256376+celeste-zeng@users.noreply.github.com> Date: Wed, 26 Jul 2023 18:48:41 -0700 Subject: [PATCH 49/59] [New Arm Tests] Add Docker Buildx --- .github/workflows/beam_PostCommit_Go_Dataflow_ARM.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/beam_PostCommit_Go_Dataflow_ARM.yml b/.github/workflows/beam_PostCommit_Go_Dataflow_ARM.yml index 69ee2ec563a3..7f65d751b8b0 100644 --- a/.github/workflows/beam_PostCommit_Go_Dataflow_ARM.yml +++ b/.github/workflows/beam_PostCommit_Go_Dataflow_ARM.yml @@ -70,5 +70,7 @@ jobs: uses: gradle/gradle-build-action@v2 with: cache-read-only: false + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 - name: run goPostCommitDataflowARM test run: ./gradlew :goPostCommitDataflowARM From 1918f4b5fe9ed50e5958aeeb1db8d0be9099b2de Mon Sep 17 00:00:00 2001 From: Celeste Zeng <61256376+celeste-zeng@users.noreply.github.com> Date: Thu, 27 Jul 2023 06:52:01 +0000 Subject: [PATCH 50/59] [Go Arm Test] Revert temp changes, delete unnecessary gradle target. --- .github/workflows/beam_PostCommit_Go_Dataflow_ARM.yml | 7 +++---- build.gradle.kts | 4 ---- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/.github/workflows/beam_PostCommit_Go_Dataflow_ARM.yml b/.github/workflows/beam_PostCommit_Go_Dataflow_ARM.yml index 7f65d751b8b0..de58c3ea4ff6 100644 --- a/.github/workflows/beam_PostCommit_Go_Dataflow_ARM.yml +++ b/.github/workflows/beam_PostCommit_Go_Dataflow_ARM.yml @@ -39,10 +39,9 @@ jobs: beam_PostCommit_Go_Dataflow_ARM: if: | github.event_name == 'push' || - github.event_name == 'pull_request_target' || github.event_name == 'schedule' || github.event.comment.body == 'Run Go PostCommit Dataflow ARM' - runs-on: [ubuntu-20.04] + runs-on: [self-hosted, ubuntu-20.04, main] name: beam_PostCommit_Go_Dataflow_ARM steps: - uses: actions/checkout@v3 @@ -72,5 +71,5 @@ jobs: cache-read-only: false - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 - - name: run goPostCommitDataflowARM test - run: ./gradlew :goPostCommitDataflowARM + - name: run Go dataflowValidatesRunnerARM64 test + run: ./gradlew :sdks:go:test:dataflowValidatesRunnerARM64 diff --git a/build.gradle.kts b/build.gradle.kts index a76ef6576253..d202d9c76fbb 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -440,10 +440,6 @@ tasks.register("goPortablePreCommit") { dependsOn(":sdks:go:test:ulrValidatesRunner") } -tasks.register("goPostCommitDataflowARM") { - dependsOn(":sdks:go:test:dataflowValidatesRunnerARM64") -} - tasks.register("goPostCommit") { dependsOn(":goIntegrationTests") } From 1c63831319d1449b2070d943d005ed60a60d2e0a Mon Sep 17 00:00:00 2001 From: Celeste Zeng <61256376+celeste-zeng@users.noreply.github.com> Date: Thu, 27 Jul 2023 23:03:11 +0000 Subject: [PATCH 51/59] Delete files that are added in other PRs. --- .../job_PostCommit_Python_Dataflow_Arm.groovy | 55 --------- sdks/python/container/run_dataflow_arm.sh | 114 ------------------ 2 files changed, 169 deletions(-) delete mode 100644 .test-infra/jenkins/job_PostCommit_Python_Dataflow_Arm.groovy delete mode 100644 sdks/python/container/run_dataflow_arm.sh diff --git a/.test-infra/jenkins/job_PostCommit_Python_Dataflow_Arm.groovy b/.test-infra/jenkins/job_PostCommit_Python_Dataflow_Arm.groovy deleted file mode 100644 index 7bf1eb4049f3..000000000000 --- a/.test-infra/jenkins/job_PostCommit_Python_Dataflow_Arm.groovy +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * This test builds Beam Python SDK multi-arch containers and use the - * Arm component to run Dataflow wordcount pipelines on Arm. - * - */ -import CommonJobProperties as commonJobProperties -import PostcommitJobBuilder - -import static PythonTestProperties.VALIDATES_CONTAINER_DATAFLOW_PYTHON_VERSIONS -import java.time.LocalDateTime - -// This job runs the suite of Python Arm tests against the -// Dataflow runner. -PostcommitJobBuilder.postCommitJob('beam_PostCommit_Py_Dataflow_Arm', - 'Run Python Dataflow Arm', 'Google Cloud Dataflow Runner Python Arm Tests', this) { - description('Runs Python Arm suite on the Dataflow runner.') - // Set common parameters. - commonJobProperties.setTopLevelMainJobProperties(delegate) - publishers { - archiveJunit('**/pytest*.xml') - } - - // Generates a unique tag for the container as the current time. - def now = LocalDateTime.now() - def unique_tag = "${now.date}${now.hour}${now.minute}${now.second}" - // Execute shell command to test Python SDK. - steps { - gradle { - rootBuildScriptDir(commonJobProperties.checkoutDir) - tasks(':sdks:python:test-suites:dataflow:validatesContainerTests') - switches('-Pcontainer-architecture-list=arm64,amd64') - switches('-Ppush-containers') - // Push multi-arch containers to the repository set in run_validatescontainer.sh - switches('-Pdocker-repository-root=us.gcr.io/apache-beam-testing/jenkins') - switches("-Pdocker-tag=${unique_tag}") - commonJobProperties.setGradleSwitches(delegate) - } - } - } diff --git a/sdks/python/container/run_dataflow_arm.sh b/sdks/python/container/run_dataflow_arm.sh deleted file mode 100644 index 0176923dffd6..000000000000 --- a/sdks/python/container/run_dataflow_arm.sh +++ /dev/null @@ -1,114 +0,0 @@ -#!/bin/bash -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# This script will be run by Jenkins as a post commit test. In order to run -# locally make the following changes: -# -# GCS_LOCATION -> Temporary location to use for service tests. -# PROJECT -> Project name to use for dataflow and docker images. -# REGION -> Region name to use for Dataflow -# -# Execute from the root of the repository: -# test Python3.8 container: -# ./gradlew :sdks:python:test-suites:dataflow:py38:dataflowArm -# or test all supported python versions together: -# ./gradlew :sdks:python:test-suites:dataflow:dataflowArm - -echo "This script must be executed in the root of beam project. Please set GCS_LOCATION, PROJECT and REGION as desired." - -if [[ $# != 2 ]]; then - printf "Usage: \n$> ./sdks/python/container/run_dataflow_arm.sh " - printf "\n\tpython_version: [required] Python version used for container build and run tests." - printf " Sample value: 3.9" - exit 1 -fi - -set -e -set -v - -# Where to store integration test outputs. -GCS_LOCATION=${GCS_LOCATION:-gs://temp-storage-for-end-to-end-tests} - -# Project for the container and integration test -PROJECT=${PROJECT:-apache-beam-testing} -REGION=${REGION:-us-central1} -IMAGE_PREFIX="$(grep 'docker_image_default_repo_prefix' gradle.properties | cut -d'=' -f2)" -SDK_VERSION="$(grep 'sdk_version' gradle.properties | cut -d'=' -f2)" -PY_VERSION=$1 -IMAGE_NAME="${IMAGE_PREFIX}python${PY_VERSION}_sdk" -CONTAINER_PROJECT="sdks:python:container:py${PY_VERSION//.}" # Note: we substitute away the dot in the version. -PY_INTERPRETER="python${PY_VERSION}" - -XUNIT_FILE="pytest-$IMAGE_NAME.xml" - -# Verify in the root of the repository -test -d sdks/python/container - -# Verify docker and gcloud commands exist -command -v docker -command -v gcloud -docker -v -gcloud -v - -CONTAINER=us.gcr.io/$PROJECT/$USER/$IMAGE_NAME -PREBUILD_SDK_CONTAINER_REGISTRY_PATH=us.gcr.io/$PROJECT/$USER/prebuild_python${PY_VERSION//.}_sdk -TAG="$(grep 'unique_tag' job_PostCommit_Python_Dataflow_Arm.groovy | cut -d'=' -f2)" - -function cleanup_container { - # Delete the container locally and remotely - for image in $(docker images --format '{{.Repository}}:{{.Tag}}' | grep $PREBUILD_SDK_CONTAINER_REGISTRY_PATH) - do docker rmi $image || echo "Failed to remove prebuilt sdk container image" - done - gcloud --quiet container images delete $CONTAINER:$TAG || echo "Failed to delete container" - for digest in $(gcloud container images list-tags $PREBUILD_SDK_CONTAINER_REGISTRY_PATH/beam_python_prebuilt_sdk --format="get(digest)") - do gcloud container images delete $PREBUILD_SDK_CONTAINER_REGISTRY_PATH/beam_python_prebuilt_sdk@$digest --force-delete-tags --quiet || echo "Failed to remove prebuilt sdk container image" - done - - echo "Removed the container" -} -trap cleanup_container EXIT - -echo ">>> Successfully built and push container $CONTAINER" - -cd sdks/python -SDK_LOCATION=$2 - -# Run ValidatesRunner tests on Google Cloud Dataflow service -echo ">>> RUNNING DATAFLOW RUNNER PYTHON ARM TEST" -pytest -o junit_suite_name=$IMAGE_NAME \ - -m="it_dataflow_arm" \ - --show-capture=no \ - --numprocesses=1 \ - --timeout=1800 \ - --junitxml=$XUNIT_FILE \ - --ignore-glob '.*py3\d?\.py$' \ - --log-cli-level=INFO \ - --test-pipeline-options=" \ - --runner=TestDataflowRunner \ - --project=$PROJECT \ - --region=$REGION \ - --sdk_container_image=$CONTAINER:$TAG \ - --staging_location=$GCS_LOCATION/staging-dataflow-arm-test \ - --temp_location=$GCS_LOCATION/temp-dataflow-arm-test \ - --output=$GCS_LOCATION/output \ - --sdk_location=$SDK_LOCATION \ - --num_workers=1 \ - --docker_registry_push_url=$PREBUILD_SDK_CONTAINER_REGISTRY_PATH \ - --machine_type=t2a-standard-1" - -echo ">>> SUCCESS DATAFLOW RUNNER PYTHON ARM TEST" From 293f703b3a8b59bddd2397895b3846a467438bde Mon Sep 17 00:00:00 2001 From: Celeste Zeng <61256376+celeste-zeng@users.noreply.github.com> Date: Thu, 27 Jul 2023 23:14:05 +0000 Subject: [PATCH 52/59] Revert changes no longer needed. --- .../container/run_validatescontainer.sh | 13 +++++++++++- .../python/test-suites/dataflow/common.gradle | 21 ------------------- 2 files changed, 12 insertions(+), 22 deletions(-) diff --git a/sdks/python/container/run_validatescontainer.sh b/sdks/python/container/run_validatescontainer.sh index 151f8409d5fa..be16ff619ed6 100755 --- a/sdks/python/container/run_validatescontainer.sh +++ b/sdks/python/container/run_validatescontainer.sh @@ -65,12 +65,23 @@ command -v gcloud docker -v gcloud -v +# Verify docker image has been built. +docker images | grep "apache/$IMAGE_NAME" | grep "$SDK_VERSION" + +TAG=$(date +%Y%m%d-%H%M%S%N) CONTAINER=us.gcr.io/$PROJECT/$USER/$IMAGE_NAME PREBUILD_SDK_CONTAINER_REGISTRY_PATH=us.gcr.io/$PROJECT/$USER/prebuild_python${PY_VERSION//.}_sdk -TAG="$(grep 'unique_tag' job_PostCommit_Python_ValidatesContainer_Dataflow.groovy | cut -d'=' -f2)" +echo "Using container $CONTAINER" + +# Tag the docker container. +docker tag "apache/$IMAGE_NAME:$SDK_VERSION" "$CONTAINER:$TAG" + +# Push the container. +gcloud docker -- push $CONTAINER:$TAG function cleanup_container { # Delete the container locally and remotely + docker rmi $CONTAINER:$TAG || echo "Failed to remove container image" for image in $(docker images --format '{{.Repository}}:{{.Tag}}' | grep $PREBUILD_SDK_CONTAINER_REGISTRY_PATH) do docker rmi $image || echo "Failed to remove prebuilt sdk container image" done diff --git a/sdks/python/test-suites/dataflow/common.gradle b/sdks/python/test-suites/dataflow/common.gradle index 4a59ddd3ece7..12c440131855 100644 --- a/sdks/python/test-suites/dataflow/common.gradle +++ b/sdks/python/test-suites/dataflow/common.gradle @@ -348,27 +348,6 @@ task validatesContainer() { } } -task dataflowArm() { - def pyversion = "${project.ext.pythonVersion.replace('.', '')}" - if (project.hasProperty("testRCDependencies")) { - // Generate a requirements file with pre-release versions for the docker task - // if testing with pre-release dependencies. - dependsOn ":sdks:python:container:py${pyversion}:generatePythonRequirements" - mustRunAfter ":sdks:python:container:py${pyversion}:generatePythonRequirements" - } - dependsOn 'initializeForDataflowJob' - dependsOn ":sdks:python:container:py${pyversion}:docker" - def runScriptsPath = "${rootDir}/sdks/python/container/run_dataflow_arm.sh" - doLast { - exec { - executable 'sh' - args '-c', ". ${envdir}/bin/activate && cd ${rootDir} && ${runScriptsPath} " + - "${project.ext.pythonVersion} " + - "${project.ext.sdkLocation}" - } - } -} - def tensorRTTests = tasks.create("tensorRTtests") { dependsOn 'installGcpTest' dependsOn ':sdks:python:sdist' From 9e4922bfd2ec1443d3dcc5ec697d7a7b3522de57 Mon Sep 17 00:00:00 2001 From: Celeste Zeng <61256376+celeste-zeng@users.noreply.github.com> Date: Thu, 27 Jul 2023 23:22:06 +0000 Subject: [PATCH 53/59] Revert more Jenkins changes. --- .test-infra/BUILD_STATUS.md | 3 --- .test-infra/jenkins/README.md | 1 - release/src/main/scripts/jenkins_jobs.txt | 1 - scripts/ci/release/test/resources/mass_comment.txt | 1 - 4 files changed, 6 deletions(-) diff --git a/.test-infra/BUILD_STATUS.md b/.test-infra/BUILD_STATUS.md index b0da79e557ff..1972076df1e2 100644 --- a/.test-infra/BUILD_STATUS.md +++ b/.test-infra/BUILD_STATUS.md @@ -133,9 +133,6 @@
Build Status -
- - Build Status diff --git a/.test-infra/jenkins/README.md b/.test-infra/jenkins/README.md index 83ca6cee9a45..02cddfdc65c7 100644 --- a/.test-infra/jenkins/README.md +++ b/.test-infra/jenkins/README.md @@ -154,7 +154,6 @@ Beam Jenkins overview page: [link](https://ci-beam.apache.org/) | beam_PostCommit_Py_ValCont_with_RC | [cron](https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont_with_RC/), [phrase](https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont_with_RC_PR/) | `Run Python RC Dataflow ValidatesContainer` | [![Build Status](https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont_with_RC/badge/icon)](https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont_with_RC) | | beam_PostCommit_Python_Examples_Direct | [cron](https://ci-beam.apache.org/job/beam_PostCommit_Python_Examples_Direct/), [phrase](https://ci-beam.apache.org/job/beam_PostCommit_Python_Examples_Direct_PR/) | `Run Python Examples_Direct` | [![Build Status](https://ci-beam.apache.org/job/beam_PostCommit_Python_Examples_Direct/badge/icon)](https://ci-beam.apache.org/job/beam_PostCommit_Python_Examples_Direct/) | | beam_PostCommit_Python_Examples_Dataflow | [cron](https://ci-beam.apache.org/job/beam_PostCommit_Python_Examples_Dataflow/), [phrase](https://ci-beam.apache.org/job/beam_PostCommit_Python_Examples_Dataflow_PR/) | `Run Python Examples_Dataflow` | [![Build Status](https://ci-beam.apache.org/job/beam_PostCommit_Python_Examples_Dataflow/badge/icon)](https://ci-beam.apache.org/job/beam_PostCommit_Python_Examples_Dataflow/) | -| beam_PostCommit_Py_Dataflow_Arm| [cron](https://ci-beam.apache.org/job/beam_PostCommit_Py_Dataflow_Arm/), [phrase](https://ci-beam.apache.org/job/beam_PostCommit_Py_Dataflow_Arm_PR/) | `Run Python Examples_Dataflow` | [![Build Status](https://ci-beam.apache.org/job/beam_PostCommit_Py_Dataflow_Arm/badge/icon)](https://ci-beam.apache.org/job/beam_PostCommit_Py_Dataflow_Arm/) | | beam_PostCommit_Python_Examples_Flink | [cron](https://ci-beam.apache.org/job/beam_PostCommit_Python_Examples_Flink/), [phrase](https://ci-beam.apache.org/job/beam_PostCommit_Python_Examples_Flink_PR/) | `Run Python Examples_Flink` | [![Build Status](https://ci-beam.apache.org/job/beam_PostCommit_Python_Examples_Flink/badge/icon)](https://ci-beam.apache.org/job/beam_PostCommit_Python_Examples_Flink/) | | beam_PostCommit_Python_Examples_Spark | [cron](https://ci-beam.apache.org/job/beam_PostCommit_Python_Examples_Spark/), [phrase](https://ci-beam.apache.org/job/beam_PostCommit_Python_Examples_Spark_PR/) | `Run Python Examples_Spark` | [![Build Status](https://ci-beam.apache.org/job/beam_PostCommit_Python_Examples_Spark/badge/icon)](https://ci-beam.apache.org/job/beam_PostCommit_Python_Examples_Spark/) | | beam_PostCommit_Python_Chicago_Taxi_Example_Dataflow | [cron](https://ci-beam.apache.org/job/beam_PostCommit_Python_Chicago_Taxi_Dataflow/), [phrase](https://ci-beam.apache.org/job/beam_PostCommit_Python_Chicago_Taxi_Dataflow_PR/) | `Run Chicago Taxi on Dataflow` | [![Build Status](https://ci-beam.apache.org/job/beam_PostCommit_Python_Chicago_Taxi_Dataflow/badge/icon)](https://ci-beam.apache.org/job/beam_PostCommit_Python_Chicago_Taxi_Dataflow) | diff --git a/release/src/main/scripts/jenkins_jobs.txt b/release/src/main/scripts/jenkins_jobs.txt index 957d3440e190..c32b47f1dbf6 100644 --- a/release/src/main/scripts/jenkins_jobs.txt +++ b/release/src/main/scripts/jenkins_jobs.txt @@ -114,7 +114,6 @@ Run Python 3.9 PostCommit,beam_PostCommit_Python39_PR Run Python Dataflow V2 ValidatesRunner,beam_PostCommit_Py_VR_Dataflow_V2_PR Run Python Dataflow ValidatesContainer,beam_PostCommit_Py_ValCont_PR Run Python Dataflow ValidatesRunner,beam_PostCommit_Py_VR_Dataflow_PR -Run Python Dataflow Arm,beam_PostCommit_Py_Dataflow_Arm_PR Run Python Direct Runner Nexmark Tests,beam_PostCommit_Python_Nexmark_Direct_PR Run Python Examples_Dataflow,beam_PostCommit_Python_Examples_Dataflow_PR Run Python Examples_Direct,beam_PostCommit_Python_Examples_Direct_PR diff --git a/scripts/ci/release/test/resources/mass_comment.txt b/scripts/ci/release/test/resources/mass_comment.txt index 27470ae9a540..93468b0c961b 100644 --- a/scripts/ci/release/test/resources/mass_comment.txt +++ b/scripts/ci/release/test/resources/mass_comment.txt @@ -66,7 +66,6 @@ Run Python 3.11 PostCommit Run Python Dataflow V2 ValidatesRunner Run Python Dataflow ValidatesContainer Run Python Dataflow ValidatesRunner -Run Python Dataflow Arm Run Python Examples_Dataflow Run Python Examples_Direct Run Python Examples_Flink From f43e3650c6d7fd7a4ed8a2faf7c463ea5220df68 Mon Sep 17 00:00:00 2001 From: Celeste Zeng <61256376+celeste-zeng@users.noreply.github.com> Date: Thu, 27 Jul 2023 23:30:30 +0000 Subject: [PATCH 54/59] Revert more changes in another PR. --- sdks/python/container/common.gradle | 7 +------ sdks/python/container/run_generate_requirements.sh | 12 ------------ sdks/python/pytest.ini | 1 - 3 files changed, 1 insertion(+), 19 deletions(-) diff --git a/sdks/python/container/common.gradle b/sdks/python/container/common.gradle index bb706aa5c5d8..7669c6ecf708 100644 --- a/sdks/python/container/common.gradle +++ b/sdks/python/container/common.gradle @@ -66,9 +66,6 @@ def copyLauncherDependencies = tasks.register("copyLauncherDependencies", Copy) } } -def useBuildx = project.containerPlatforms() != [project.nativeArchitecture()] -def pushContainers = project.rootProject.hasProperty(["isRelease"]) || project.rootProject.hasProperty("push-containers") - docker { name containerImageName( name: project.docker_image_default_repo_prefix + "python${project.ext.pythonVersion}_sdk", @@ -83,10 +80,8 @@ docker { buildArgs(['py_version': "${project.ext.pythonVersion}", 'pull_licenses': project.rootProject.hasProperty(["docker-pull-licenses"]) || project.rootProject.hasProperty(["isRelease"])]) - buildx useBuildx + buildx useBuildx = project.containerPlatforms() != [project.nativeArchitecture()] platform(*project.containerPlatforms()) - load useBuildx && !pushContainers - push pushContainers } dockerPrepare.dependsOn copyLauncherDependencies diff --git a/sdks/python/container/run_generate_requirements.sh b/sdks/python/container/run_generate_requirements.sh index fdc2f540950d..fd222107dba5 100755 --- a/sdks/python/container/run_generate_requirements.sh +++ b/sdks/python/container/run_generate_requirements.sh @@ -104,16 +104,4 @@ EOT # Remove pkg_resources to guard against # https://stackoverflow.com/questions/39577984/what-is-pkg-resources-0-0-0-in-output-of-pip-freeze-command pip freeze | grep -v pkg_resources >> "$REQUIREMENTS_FILE" - -# Check if the generated .txt file has the line "tensorflow==". -if grep -q "tensorflow==" "$REQUIREMENTS_FILE"; then - -# Get the version of tensorflow from the .txt file. -VERSION=$(grep -Po "tensorflow==\K[^;]+" "$REQUIREMENTS_FILE") - -# Write the line tensorflow-cpu-aws==${VERSION};platform_machine=="aarch64" to the .txt file. -echo "tensorflow-cpu-aws==${VERSION};platform_machine==\"aarch64\"" >> "$REQUIREMENTS_FILE" - -fi - rm -rf "$ENV_PATH" diff --git a/sdks/python/pytest.ini b/sdks/python/pytest.ini index fc1fce244a2d..7c564235c581 100644 --- a/sdks/python/pytest.ini +++ b/sdks/python/pytest.ini @@ -38,7 +38,6 @@ markers = no_sickbay_streaming: run without sickbay-streaming no_sickbay_batch: run without sickbay-batch it_validatescontainer: collect for ValidatesContainer integration test runs - it_dataflow_arm: collect for DataflowArm integration test runs examples_postcommit: collect for post-commit test examples runs sickbay_direct: run without sickbay-direct sickbay_spark: run without sickbay-spark From a6e91aadcaf8bb290e4cc0566167a2659239420f Mon Sep 17 00:00:00 2001 From: Celeste Zeng <61256376+celeste-zeng@users.noreply.github.com> Date: Thu, 27 Jul 2023 23:37:58 +0000 Subject: [PATCH 55/59] Revert more remaining changes. --- .../apache_beam/examples/wordcount_it_test.py | 13 ------------- sdks/python/container/common.gradle | 2 +- 2 files changed, 1 insertion(+), 14 deletions(-) diff --git a/sdks/python/apache_beam/examples/wordcount_it_test.py b/sdks/python/apache_beam/examples/wordcount_it_test.py index 27581df78275..bb5859df7251 100644 --- a/sdks/python/apache_beam/examples/wordcount_it_test.py +++ b/sdks/python/apache_beam/examples/wordcount_it_test.py @@ -112,19 +112,6 @@ def test_wordcount_it_with_prebuilt_sdk_container_cloud_build(self): def test_wordcount_it_with_use_sibling_sdk_workers(self): self._run_wordcount_it(wordcount.run, experiment='use_sibling_sdk_workers') - @pytest.mark.it_postcommit - @pytest.mark.it_dataflow_arm - def test_wordcount_fnapi_it_arm(self): - self._run_wordcount_it( - wordcount.run, experiment='beam_fn_api', machine_type='t2a-standard-1') - - @pytest.mark.it_dataflow_arm - def test_wordcount_it_with_use_sibling_sdk_workers_arm(self): - self._run_wordcount_it( - wordcount.run, - experiment='use_sibling_sdk_workers', - machine_type='t2a-standard-1') - def _run_wordcount_it(self, run_wordcount, **opts): test_pipeline = TestPipeline(is_integration_test=True) extra_opts = {} diff --git a/sdks/python/container/common.gradle b/sdks/python/container/common.gradle index 7669c6ecf708..48c059ae5ecd 100644 --- a/sdks/python/container/common.gradle +++ b/sdks/python/container/common.gradle @@ -80,7 +80,7 @@ docker { buildArgs(['py_version': "${project.ext.pythonVersion}", 'pull_licenses': project.rootProject.hasProperty(["docker-pull-licenses"]) || project.rootProject.hasProperty(["isRelease"])]) - buildx useBuildx = project.containerPlatforms() != [project.nativeArchitecture()] + buildx project.containerPlatforms() != [project.nativeArchitecture()] platform(*project.containerPlatforms()) } From 7847711f51cad4a88ebc3a4740307ef030cc9151 Mon Sep 17 00:00:00 2001 From: Celeste Zeng <61256376+celeste-zeng@users.noreply.github.com> Date: Thu, 27 Jul 2023 23:59:00 +0000 Subject: [PATCH 56/59] Resolve conflicts. --- .github/workflows/beam_PostCommit_Go_Dataflow_ARM.yml | 7 +++---- build.gradle.kts | 4 ++++ sdks/go/test/run_validatesrunner_tests.sh | 2 +- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/.github/workflows/beam_PostCommit_Go_Dataflow_ARM.yml b/.github/workflows/beam_PostCommit_Go_Dataflow_ARM.yml index de58c3ea4ff6..65d2e9836cf4 100644 --- a/.github/workflows/beam_PostCommit_Go_Dataflow_ARM.yml +++ b/.github/workflows/beam_PostCommit_Go_Dataflow_ARM.yml @@ -32,9 +32,6 @@ concurrency: group: '${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}' cancel-in-progress: true -env: - USER: github-actions - jobs: beam_PostCommit_Go_Dataflow_ARM: if: | @@ -72,4 +69,6 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 - name: run Go dataflowValidatesRunnerARM64 test - run: ./gradlew :sdks:go:test:dataflowValidatesRunnerARM64 + run: ./gradlew :goPostCommitDataflowARM + env: + USER: github-actions diff --git a/build.gradle.kts b/build.gradle.kts index d202d9c76fbb..a76ef6576253 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -440,6 +440,10 @@ tasks.register("goPortablePreCommit") { dependsOn(":sdks:go:test:ulrValidatesRunner") } +tasks.register("goPostCommitDataflowARM") { + dependsOn(":sdks:go:test:dataflowValidatesRunnerARM64") +} + tasks.register("goPostCommit") { dependsOn(":goIntegrationTests") } diff --git a/sdks/go/test/run_validatesrunner_tests.sh b/sdks/go/test/run_validatesrunner_tests.sh index 75147e0d37fd..1f63db639252 100755 --- a/sdks/go/test/run_validatesrunner_tests.sh +++ b/sdks/go/test/run_validatesrunner_tests.sh @@ -429,7 +429,7 @@ if [[ "$RUNNER" == "dataflow" ]]; then if [[ -n "$TEST_EXPANSION_ADDR" || -n "$IO_EXPANSION_ADDR" || -n "$SCHEMAIO_EXPANSION_ADDR" || -n "$DEBEZIUMIO_EXPANSION_ADDR" ]]; then # Delete the java cross-language container locally and remotely - docker rmi $JAVA_CONTAINER:$JAVA_TAG || echo "Failed to remove container" + docker rmi $JAVA_CONTAINER:$JAVA_TAG || echo "Built container image was not removed. Possibly, it was not not saved locally." gcloud --quiet container images delete $JAVA_CONTAINER:$JAVA_TAG || echo "Failed to delete container" fi From 22a78a47ca98d33d911d2b8679f3843a8159b9d0 Mon Sep 17 00:00:00 2001 From: Celeste Zeng <61256376+celeste-zeng@users.noreply.github.com> Date: Thu, 27 Jul 2023 17:14:48 -0700 Subject: [PATCH 57/59] Remove white space From d94a5ff04f299773e0db6082494956756605041e Mon Sep 17 00:00:00 2001 From: Celeste Zeng <61256376+celeste-zeng@users.noreply.github.com> Date: Thu, 27 Jul 2023 17:17:47 -0700 Subject: [PATCH 58/59] Remove white space From b105fb48aa525b080b1cef6196bab6efcf08399f Mon Sep 17 00:00:00 2001 From: Celeste Zeng <61256376+celeste-zeng@users.noreply.github.com> Date: Fri, 28 Jul 2023 00:19:15 +0000 Subject: [PATCH 59/59] Remove blank line. --- .github/workflows/beam_PostCommit_Go_Dataflow_ARM.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/beam_PostCommit_Go_Dataflow_ARM.yml b/.github/workflows/beam_PostCommit_Go_Dataflow_ARM.yml index 65d2e9836cf4..21a202ff9f18 100644 --- a/.github/workflows/beam_PostCommit_Go_Dataflow_ARM.yml +++ b/.github/workflows/beam_PostCommit_Go_Dataflow_ARM.yml @@ -71,4 +71,4 @@ jobs: - name: run Go dataflowValidatesRunnerARM64 test run: ./gradlew :goPostCommitDataflowARM env: - USER: github-actions + USER: github-actions \ No newline at end of file