diff --git a/.test-infra/jenkins/job_PostCommit_Python_ValidatesContainer_Dataflow.groovy b/.test-infra/jenkins/job_PostCommit_Python_ValidatesContainer_Dataflow.groovy index bc43ecb366fc..254096448668 100644 --- a/.test-infra/jenkins/job_PostCommit_Python_ValidatesContainer_Dataflow.groovy +++ b/.test-infra/jenkins/job_PostCommit_Python_ValidatesContainer_Dataflow.groovy @@ -43,3 +43,25 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_Py_ValCont', } } } + +PostcommitJobBuilder.postCommitJob('beam_PostCommit_Py_ValCont_with_RC', + 'Run Python RC Dataflow ValidatesContainer', 'Google Cloud Dataflow Runner Python ValidatesContainer Tests with RC Dependencies', this) { + description('Runs Python ValidatesContainer suite on the Dataflow runner by installing Release Candidates.') + + // Set common parameters. + commonJobProperties.setTopLevelMainJobProperties(delegate) + + publishers { + archiveJunit('**/pytest*.xml') + } + + // Execute shell command to test Python SDK. + steps { + gradle { + rootBuildScriptDir(commonJobProperties.checkoutDir) + tasks(':sdks:python:test-suites:dataflow:validatesContainerTests') + switches('-PtestRCDependencies=true') + commonJobProperties.setGradleSwitches(delegate) + } + } + } diff --git a/sdks/python/container/Dockerfile b/sdks/python/container/Dockerfile index 8dcfd5b74dd1..ff3b44849504 100644 --- a/sdks/python/container/Dockerfile +++ b/sdks/python/container/Dockerfile @@ -109,5 +109,3 @@ COPY --from=third_party_licenses /opt/apache/beam/third_party_licenses /opt/apac RUN if [ "$pull_licenses" != "true" ] ; then \ rm -rf /opt/apache/beam/third_party_licenses ; \ fi - - diff --git a/sdks/python/container/common.gradle b/sdks/python/container/common.gradle index c10211950f85..48c059ae5ecd 100644 --- a/sdks/python/container/common.gradle +++ b/sdks/python/container/common.gradle @@ -32,13 +32,14 @@ dependencies { def generatePythonRequirements = tasks.register("generatePythonRequirements") { dependsOn ':sdks:python:sdist' + def pipExtraOptions = project.hasProperty("testRCDependencies") ? "--pre" : "" def runScriptsPath = "${rootDir}/sdks/python/container/run_generate_requirements.sh" doLast { exec { executable 'sh' args '-c', "cd ${rootDir} && ${runScriptsPath} " + "${project.ext.pythonVersion} " + - "${files(configurations.sdkSourceTarball.files).singleFile}" + "${files(configurations.sdkSourceTarball.files).singleFile} " + "${pipExtraOptions}" } } } diff --git a/sdks/python/container/run_generate_requirements.sh b/sdks/python/container/run_generate_requirements.sh index 55173bb8cd93..fd222107dba5 100755 --- a/sdks/python/container/run_generate_requirements.sh +++ b/sdks/python/container/run_generate_requirements.sh @@ -16,7 +16,7 @@ # limitations under the License. # -# Generates requirements files, which list PyPI depenedncies to install in +# Generates requirements files, which list PyPI dependencies to install in # Apache Beam Python SDK container images. To generate the list, # we use two sources of information: # 1) Requirements of Apache Beam itself, as defined by setup.py. @@ -30,21 +30,25 @@ # You will need Python interpreters for all versions supported by Beam, see: # https://s.apache.org/beam-python-dev-wiki -if [[ $# != 2 ]]; then +if [[ $# -lt 2 ]]; then printf "Example usage: \n$> ./sdks/python/container/run_generate_requirements.sh 3.8 " - printf "\n\twhere 3.8 is the Python major.minor version." + printf "\n\where 3.8 is the Python major.minor version." exit 1 fi PY_VERSION=$1 SDK_TARBALL=$2 +# Use the PIP_EXTRA_OPTIONS environment variable to pass additional flags to the pip install command. +# For example, you can include the --pre flag in $PIP_EXTRA_OPTIONS to download pre-release versions of packages. +# Note that you can modify the behavior of the pip install command in this script by passing in your own $PIP_EXTRA_OPTIONS. +PIP_EXTRA_OPTIONS=$3 -if ! python$PY_VERSION --version > /dev/null 2>&1 ; then +if ! python"$PY_VERSION" --version > /dev/null 2>&1 ; then echo "Please install a python${PY_VERSION} interpreter. See s.apache.org/beam-python-dev-wiki for Python installation tips." exit 1 fi -if ! python$PY_VERSION -m venv --help > /dev/null 2>&1 ; then +if ! python"$PY_VERSION" -m venv --help > /dev/null 2>&1 ; then echo "Your python${PY_VERSION} installation does not have a required venv module. See s.apache.org/beam-python-dev-wiki for Python installation tips." exit 1 fi @@ -52,17 +56,18 @@ fi set -ex ENV_PATH="$PWD/build/python${PY_VERSION/./}_requirements_gen" -rm -rf $ENV_PATH 2>/dev/null || true -python${PY_VERSION} -m venv $ENV_PATH -source $ENV_PATH/bin/activate +rm -rf "$ENV_PATH" 2>/dev/null || true +python"${PY_VERSION}" -m venv "$ENV_PATH" +source "$ENV_PATH"/bin/activate pip install --upgrade pip setuptools wheel # Install gcp extra deps since these deps are commonly used with Apache Beam. # Install dataframe deps to add have Dataframe support in released images. # Install test deps since some integration tests need dependencies, # such as pytest, installed in the runner environment. -pip install --no-cache-dir $SDK_TARBALL[gcp,dataframe,test] -pip install --no-cache-dir -r $PWD/sdks/python/container/base_image_requirements_manual.txt +pip install ${PIP_EXTRA_OPTIONS:+"$PIP_EXTRA_OPTIONS"} --no-cache-dir "$SDK_TARBALL"[gcp,dataframe,test] +pip install ${PIP_EXTRA_OPTIONS:+"$PIP_EXTRA_OPTIONS"} --no-cache-dir -r "$PWD"/sdks/python/container/base_image_requirements_manual.txt + pip uninstall -y apache-beam echo "Checking for broken dependencies:" pip check @@ -71,7 +76,7 @@ pip freeze PY_IMAGE="py${PY_VERSION//.}" REQUIREMENTS_FILE=$PWD/sdks/python/container/$PY_IMAGE/base_image_requirements.txt -cat < $REQUIREMENTS_FILE +cat < "$REQUIREMENTS_FILE" # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. @@ -98,5 +103,5 @@ cat < $REQUIREMENTS_FILE EOT # Remove pkg_resources to guard against # https://stackoverflow.com/questions/39577984/what-is-pkg-resources-0-0-0-in-output-of-pip-freeze-command -pip freeze | grep -v pkg_resources >> $REQUIREMENTS_FILE -rm -rf $ENV_PATH +pip freeze | grep -v pkg_resources >> "$REQUIREMENTS_FILE" +rm -rf "$ENV_PATH" diff --git a/sdks/python/test-suites/dataflow/common.gradle b/sdks/python/test-suites/dataflow/common.gradle index 2552eac431a7..f47b7a5fe901 100644 --- a/sdks/python/test-suites/dataflow/common.gradle +++ b/sdks/python/test-suites/dataflow/common.gradle @@ -351,6 +351,12 @@ task chicagoTaxiExample { task validatesContainer() { def pyversion = "${project.ext.pythonVersion.replace('.', '')}" + if (project.hasProperty("testRCDependencies")) { + // Generate a requirements file with pre-release versions for the docker task + // if testing with pre-release dependencies. + dependsOn ":sdks:python:container:py${pyversion}:generatePythonRequirements" + mustRunAfter ":sdks:python:container:py${pyversion}:generatePythonRequirements" + } dependsOn 'initializeForDataflowJob' dependsOn ":sdks:python:container:py${pyversion}:docker" def runScriptsPath = "${rootDir}/sdks/python/container/run_validatescontainer.sh" @@ -457,4 +463,4 @@ project(":sdks:python:test-suites:xlang").ext.xlangTasks.each { taskMetadata -> ], pytestOptions: basicPytestOpts ) -} \ No newline at end of file +}