Add ValidatesContainer tests with installing release candidates(RCs)

.test-infra/jenkins/job_PostCommit_Python_ValidatesContainer_Dataflow.groovy

@@ -43,3 +43,25 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_Py_ValCont',
         }
       }
     }
+
+PostcommitJobBuilder.postCommitJob('beam_PostCommit_Py_ValCont_with_RC',
+    'Run Python RC Dataflow ValidatesContainer', 'Google Cloud Dataflow Runner Python ValidatesContainer Tests with RC Dependencies', this) {
+      description('Runs Python ValidatesContainer suite on the Dataflow runner by installing Release Candidates.')
+
+      // Set common parameters.
+      commonJobProperties.setTopLevelMainJobProperties(delegate)
+
+      publishers {
+        archiveJunit('**/pytest*.xml')
+      }
+
+      // Execute shell command to test Python SDK.
+      steps {
+        gradle {
+          rootBuildScriptDir(commonJobProperties.checkoutDir)
+          tasks(':sdks:python:test-suites:dataflow:validatesContainerTests')
+          switches('-PtestRCDependencies=true')
+          commonJobProperties.setGradleSwitches(delegate)
+        }
+      }
+    }

sdks/python/container/Dockerfile

@@ -109,5 +109,3 @@ COPY --from=third_party_licenses /opt/apache/beam/third_party_licenses /opt/apac
 RUN if [ "$pull_licenses" != "true" ] ; then \
       rm -rf /opt/apache/beam/third_party_licenses ; \
     fi
-
-

sdks/python/container/common.gradle

@@ -32,13 +32,14 @@ dependencies {
 
 def generatePythonRequirements = tasks.register("generatePythonRequirements") {
   dependsOn ':sdks:python:sdist'
+  def pipExtraOptions = project.hasProperty("testRCDependencies") ? "--pre" : ""
   def runScriptsPath = "${rootDir}/sdks/python/container/run_generate_requirements.sh"
   doLast {
     exec {
       executable 'sh'
       args '-c', "cd ${rootDir} && ${runScriptsPath} " +
               "${project.ext.pythonVersion} " +
-              "${files(configurations.sdkSourceTarball.files).singleFile}"
+              "${files(configurations.sdkSourceTarball.files).singleFile} " + "${pipExtraOptions}"
     }
   }
 }

sdks/python/container/run_generate_requirements.sh

@@ -16,7 +16,7 @@
 #    limitations under the License.
 #
 
-# Generates requirements files, which list PyPI depenedncies to install in
+# Generates requirements files, which list PyPI dependencies to install in
 # Apache Beam Python SDK container images. To generate the list,
 # we use two sources of information:
 # 1) Requirements of Apache Beam itself, as defined by setup.py.
@@ -30,39 +30,44 @@
 # You will need Python interpreters for all versions supported by Beam, see:
 # https://s.apache.org/beam-python-dev-wiki
 
-if [[ $# != 2 ]]; then
+if [[ $# -lt 2 ]]; then
   printf "Example usage: \n$> ./sdks/python/container/run_generate_requirements.sh 3.8 <sdk_tarball>"
-  printf "\n\twhere 3.8 is the Python major.minor version."
+  printf "\n\where 3.8 is the Python major.minor version."
   exit 1
 fi
 
 PY_VERSION=$1
 SDK_TARBALL=$2
+# Use the PIP_EXTRA_OPTIONS environment variable to pass additional flags to the pip install command.
+# For example, you can include the --pre flag in $PIP_EXTRA_OPTIONS to download pre-release versions of packages.
+# Note that you can modify the behavior of the pip install command in this script by passing in your own $PIP_EXTRA_OPTIONS.
+PIP_EXTRA_OPTIONS=$3
 
-if ! python$PY_VERSION --version > /dev/null 2>&1 ; then
+if ! python"$PY_VERSION" --version > /dev/null 2>&1 ; then
   echo "Please install a python${PY_VERSION} interpreter. See s.apache.org/beam-python-dev-wiki for Python installation tips."
   exit 1
 fi
 
-if ! python$PY_VERSION -m venv --help > /dev/null 2>&1 ; then
+if ! python"$PY_VERSION" -m venv --help > /dev/null 2>&1 ; then
   echo "Your python${PY_VERSION} installation does not have a required venv module. See s.apache.org/beam-python-dev-wiki for Python installation tips."
   exit 1
 fi
 
 set -ex
 
 ENV_PATH="$PWD/build/python${PY_VERSION/./}_requirements_gen"
-rm -rf $ENV_PATH 2>/dev/null || true
-python${PY_VERSION} -m venv $ENV_PATH
-source $ENV_PATH/bin/activate
+rm -rf "$ENV_PATH" 2>/dev/null || true
+python"${PY_VERSION}" -m venv "$ENV_PATH"
+source "$ENV_PATH"/bin/activate
 pip install --upgrade pip setuptools wheel
 
 # Install gcp extra deps since these deps are commonly used with Apache Beam.
 # Install dataframe deps to add have Dataframe support in released images.
 # Install test deps since some integration tests need dependencies,
 # such as pytest, installed in the runner environment.
-pip install --no-cache-dir $SDK_TARBALL[gcp,dataframe,test]
-pip install --no-cache-dir -r $PWD/sdks/python/container/base_image_requirements_manual.txt
+pip install ${PIP_EXTRA_OPTIONS:+"$PIP_EXTRA_OPTIONS"}  --no-cache-dir "$SDK_TARBALL"[gcp,dataframe,test]
+pip install ${PIP_EXTRA_OPTIONS:+"$PIP_EXTRA_OPTIONS"}  --no-cache-dir -r "$PWD"/sdks/python/container/base_image_requirements_manual.txt
+
 pip uninstall -y apache-beam
 echo "Checking for broken dependencies:"
 pip check
@@ -71,7 +76,7 @@ pip freeze
 
 PY_IMAGE="py${PY_VERSION//.}"
 REQUIREMENTS_FILE=$PWD/sdks/python/container/$PY_IMAGE/base_image_requirements.txt
-cat <<EOT > $REQUIREMENTS_FILE
+cat <<EOT > "$REQUIREMENTS_FILE"
 #    Licensed to the Apache Software Foundation (ASF) under one or more
 #    contributor license agreements.  See the NOTICE file distributed with
 #    this work for additional information regarding copyright ownership.
@@ -98,5 +103,5 @@ cat <<EOT > $REQUIREMENTS_FILE
 EOT
 # Remove pkg_resources to guard against
 # https://stackoverflow.com/questions/39577984/what-is-pkg-resources-0-0-0-in-output-of-pip-freeze-command
-pip freeze | grep -v pkg_resources >> $REQUIREMENTS_FILE
-rm -rf $ENV_PATH
+pip freeze | grep -v pkg_resources >> "$REQUIREMENTS_FILE"
+rm -rf "$ENV_PATH"

sdks/python/test-suites/dataflow/common.gradle

@@ -351,6 +351,12 @@ task chicagoTaxiExample {
 
 task validatesContainer() {
   def pyversion = "${project.ext.pythonVersion.replace('.', '')}"
+  if (project.hasProperty("testRCDependencies")) {
+    // Generate a requirements file with pre-release versions for the docker task
+    // if testing with pre-release dependencies.
+    dependsOn ":sdks:python:container:py${pyversion}:generatePythonRequirements"
+    mustRunAfter ":sdks:python:container:py${pyversion}:generatePythonRequirements"
+  }
   dependsOn 'initializeForDataflowJob'
   dependsOn ":sdks:python:container:py${pyversion}:docker"
   def runScriptsPath = "${rootDir}/sdks/python/container/run_validatescontainer.sh"
@@ -457,4 +463,4 @@ project(":sdks:python:test-suites:xlang").ext.xlangTasks.each { taskMetadata ->
       ],
       pytestOptions: basicPytestOpts
     )
-}
+}