Improve generation of requirements (#1912)

* Improve generation of requirements

* Add generated requirements

* Feedback

plugins/core-plugin/src/main/java/com/google/cloud/teleport/plugin/DockerfileGenerator.java

@@ -42,6 +42,8 @@ public class DockerfileGenerator {
 
   public static final String BASE_CONTAINER_IMAGE =
       "gcr.io/dataflow-templates-base/java11-template-launcher-base-distroless:latest";
+  // Keep in sync with python version used in
+  // https://github.com/GoogleCloudPlatform/DataflowTemplates/blob/main/python/generate_dependencies.sh
   public static final String BASE_PYTHON_CONTAINER_IMAGE =
       "gcr.io/dataflow-templates-base/python311-template-launcher-base:latest";
   public static final String PYTHON_LAUNCHER_ENTRYPOINT =

python/src/main/python/word-count-python/base_requirements.txt → python/default_base_python_requirements.txt

@@ -1 +1,2 @@
 apache-beam[gcp]
+setuptools

python/default_base_yaml_requirements.txt

@@ -0,0 +1,2 @@
+apache-beam[dataframe,gcp,test,yaml]
+setuptools

python/generate_all_dependencies.sh

@@ -14,6 +14,18 @@
 
 SCRIPTPATH=$(dirname "$0")
 
-sh $SCRIPTPATH/generate_dependencies.sh $SCRIPTPATH/../v1/src/main/python/base_requirements.txt $SCRIPTPATH/../v1/src/main/python/requirements.txt
-sh $SCRIPTPATH/generate_dependencies.sh $SCRIPTPATH/../python/src/main/python/word-count-python/base_requirements.txt $SCRIPTPATH/../python/src/main/python/word-count-python/requirements.txt
-sh $SCRIPTPATH/generate_dependencies.sh $SCRIPTPATH/../python/src/main/python/streaming-llm/base_requirements.txt $SCRIPTPATH/../python/src/main/python/streaming-llm/requirements.txt
+sh $SCRIPTPATH/generate_dependencies.sh $SCRIPTPATH/../python/src/main/python/streaming-llm/base_requirements.txt $SCRIPTPATH/../python/src/main/python/streaming-llm/requirements.txt
+
+# Generate a base set of dependencies to use for any templates without special dependencies
+mkdir -p $SCRIPTPATH/__build__/
+sh $SCRIPTPATH/generate_dependencies.sh $SCRIPTPATH/default_base_python_requirements.txt $SCRIPTPATH/__build__/default_python_requirements.txt
+cp $SCRIPTPATH/__build__/default_python_requirements.txt $SCRIPTPATH/../python/src/main/python/word-count-python/requirements.txt
+
+sh $SCRIPTPATH/generate_dependencies.sh $SCRIPTPATH/default_base_yaml_requirements.txt $SCRIPTPATH/__build__/default_yaml_requirements.txt
+cp $SCRIPTPATH/__build__/default_yaml_requirements.txt $SCRIPTPATH/../python/src/main/python/yaml-template/requirements.txt
+cp $SCRIPTPATH/__build__/default_yaml_requirements.txt $SCRIPTPATH/../v2/googlecloud-to-elasticsearch/src/main/resources/requirements.txt
+cp $SCRIPTPATH/__build__/default_yaml_requirements.txt $SCRIPTPATH/../v2/googlecloud-to-googlecloud/src/main/resources/requirements.txt
+cp $SCRIPTPATH/__build__/default_yaml_requirements.txt $SCRIPTPATH/../v2/googlecloud-to-splunk/src/main/resources/requirements.txt
+cp $SCRIPTPATH/__build__/default_yaml_requirements.txt $SCRIPTPATH/../v2/pubsub-binary-to-bigquery/src/main/resources/requirements.txt
+cp $SCRIPTPATH/__build__/default_yaml_requirements.txt $SCRIPTPATH/../v2/pubsub-binary-to-mongodb/src/main/resources/requirements.txt
+cp $SCRIPTPATH/__build__/default_yaml_requirements.txt $SCRIPTPATH/../v2/pubsub-to-mongodb/src/main/resources/requirements.txt

python/generate_dependencies.sh

@@ -15,27 +15,32 @@
 BASE_REQUIREMENTS_PATH=$1
 TARGET_REQUIREMENTS_PATH=$2
 
-if ! python3 --version > /dev/null 2>&1 ; then
-  echo "Please install a python3 interpreter. See s.apache.org/beam-python-dev-wiki for Python installation tips."
+if ! python3.11 --version > /dev/null 2>&1 ; then
+  echo "Please install a python3.11 interpreter. See s.apache.org/beam-python-dev-wiki for Python installation tips."
   exit 1
 fi
 
-if ! python3 -m venv --help > /dev/null 2>&1 ; then
-  echo "Your python3 installation does not have a required venv module. See s.apache.org/beam-python-dev-wiki for Python installation tips."
+if ! python3.11 -m venv --help > /dev/null 2>&1 ; then
+  echo "Your python3.11 installation does not have a required venv module. See s.apache.org/beam-python-dev-wiki for Python installation tips."
   exit 1
 fi
 
 set -ex
 
 ENV_PATH="$PWD/__build__/python${PY_VERSION/./}_requirements_gen"
 rm -rf "$ENV_PATH" 2>/dev/null || true
-python3 -m venv "$ENV_PATH"
+# These python versions need to be kept in sync with our dockerfile python versions (https://github.com/GoogleCloudPlatform/DataflowTemplates/blob/0ac92513838ca525adb3f616c9e1f65237334d1e/plugins/core-plugin/src/main/java/com/google/cloud/teleport/plugin/DockerfileGenerator.java#L46)
+python3.11 -m venv "$ENV_PATH"
 source "$ENV_PATH"/bin/activate
-pip install --upgrade pip setuptools wheel
-pip install pip-tools
+
+# allow one-off executions of pip to generate requirements locally without alarming automation
+alias pip_automation="pip"
+
+pip_automation install --upgrade pip setuptools wheel
+pip_automation install pip-tools
 
 # Install requirements from base file
-pip install ${PIP_EXTRA_OPTIONS:+"$PIP_EXTRA_OPTIONS"}  --no-cache-dir -r $BASE_REQUIREMENTS_PATH
+pip_automation install ${PIP_EXTRA_OPTIONS:+"$PIP_EXTRA_OPTIONS"}  --no-cache-dir -r $BASE_REQUIREMENTS_PATH
 
 echo "Checking for broken dependencies:"
 pip check
@@ -44,7 +49,7 @@ pip freeze
 
 # Generate hashes for new requirements file
 echo "Running pip-compile to generate hashes"
-pip-compile $BASE_REQUIREMENTS_PATH -o $TARGET_REQUIREMENTS_PATH --generate-hashes
+pip-compile $BASE_REQUIREMENTS_PATH -o $TARGET_REQUIREMENTS_PATH --generate-hashes --allow-unsafe
 
 PY_IMAGE="py${PY_VERSION//.}"
 cat <<EOT > "$TARGET_REQUIREMENTS_PATH"
@@ -64,12 +69,9 @@ cat <<EOT > "$TARGET_REQUIREMENTS_PATH"
 
 # Autogenerated requirements file for Apache Beam container image.
 # From the templates base directory to update,
-# run: sh python/generate_dependencies.sh $BASE_REQUIREMENTS_PATH $TARGET_REQUIREMENTS_PATH
-# or: sh python/generate_all_dependencies.sh
+# run: sh python/generate_all_dependencies.sh
 # Do not edit manually, adjust the base requirements file, and regenerate the list.
 
 $(cat $TARGET_REQUIREMENTS_PATH)
 
-EOT
-
-rm -rf "$ENV_PATH"
+EOT

python/src/main/python/streaming-llm/base_requirements.txt

@@ -2,3 +2,4 @@ apache-beam[gcp]
 torch
 transformers
 torchvision
+setuptools

python/src/main/python/streaming-llm/requirements.txt

@@ -14,15 +14,14 @@
 
 # Autogenerated requirements file for Apache Beam container image.
 # From the templates base directory to update,
-# run: sh python/generate_dependencies.sh ./python/../python/src/main/python/streaming-llm/base_requirements.txt ./python/../python/src/main/python/streaming-llm/requirements.txt
-# or: sh python/generate_all_dependencies.sh
+# run: sh python/generate_all_dependencies.sh
 # Do not edit manually, adjust the base requirements file, and regenerate the list.
 
 #
-# This file is autogenerated by pip-compile with Python 3.8
+# This file is autogenerated by pip-compile with Python 3.11
 # by the following command:
 #
-#    pip-compile --generate-hashes --output-file=./python/../python/src/main/python/streaming-llm/requirements.txt ./python/../python/src/main/python/streaming-llm/base_requirements.txt
+#    pip-compile --allow-unsafe --generate-hashes --output-file=python/../python/src/main/python/streaming-llm/requirements.txt python/../python/src/main/python/streaming-llm/base_requirements.txt
 #
 annotated-types==0.7.0 \
     --hash=sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53 \
@@ -60,35 +59,13 @@ apache-beam[gcp]==2.59.0 \
     --hash=sha256:ecf74d123ddd10a114ca47f7b9c74b354121c1c1911da2b05e0e964e86ecbac8 \
     --hash=sha256:f1b7c019af38b4f1ff899824dbf8380a6e7245c158814546b62e8b4ab7a1d483 \
     --hash=sha256:fc0788f7313762eaedffbda4e8139e8a65b658a84dc577a5a4e106e4b0a21556
-    # via -r ./python/../python/src/main/python/streaming-llm/base_requirements.txt
-async-timeout==4.0.3 \
-    --hash=sha256:4640d96be84d82d02ed59ea2b7105a0f7b33abe8703703cd0ab0bf87c427522f \
-    --hash=sha256:7405140ff1230c310e51dc27b3145b9092d659ce68ff733fb0cefe3ee42be028
-    # via redis
+    # via -r python/../python/src/main/python/streaming-llm/base_requirements.txt
 attrs==24.2.0 \
     --hash=sha256:5cfb1b9148b5b086569baec03f20d7b6bf3bcacc9a42bebf87ffaaca362f6346 \
     --hash=sha256:81921eb96de3191c8258c199618104dd27ac608d9366f5e35d011eae1867ede2
     # via
     #   jsonschema
     #   referencing
-backports-zoneinfo==0.2.1 \
-    --hash=sha256:17746bd546106fa389c51dbea67c8b7c8f0d14b5526a579ca6ccf5ed72c526cf \
-    --hash=sha256:1b13e654a55cd45672cb54ed12148cd33628f672548f373963b0bff67b217328 \
-    --hash=sha256:1c5742112073a563c81f786e77514969acb58649bcdf6cdf0b4ed31a348d4546 \
-    --hash=sha256:4a0f800587060bf8880f954dbef70de6c11bbe59c673c3d818921f042f9954a6 \
-    --hash=sha256:5c144945a7752ca544b4b78c8c41544cdfaf9786f25fe5ffb10e838e19a27570 \
-    --hash=sha256:7b0a64cda4145548fed9efc10322770f929b944ce5cee6c0dfe0c87bf4c0c8c9 \
-    --hash=sha256:8439c030a11780786a2002261569bdf362264f605dfa4d65090b64b05c9f79a7 \
-    --hash=sha256:8961c0f32cd0336fb8e8ead11a1f8cd99ec07145ec2931122faaac1c8f7fd987 \
-    --hash=sha256:89a48c0d158a3cc3f654da4c2de1ceba85263fafb861b98b59040a5086259722 \
-    --hash=sha256:a76b38c52400b762e48131494ba26be363491ac4f9a04c1b7e92483d169f6582 \
-    --hash=sha256:da6013fd84a690242c310d77ddb8441a559e9cb3d3d59ebac9aca1a57b2e18bc \
-    --hash=sha256:e55b384612d93be96506932a786bbcde5a2db7a9e6a4bb4bffe8b733f5b9036b \
-    --hash=sha256:e81b76cace8eda1fca50e345242ba977f9be6ae3945af8d46326d776b4cf78d1 \
-    --hash=sha256:e8236383a20872c0cdf5a62b554b27538db7fa1bbec52429d8d106effbaeca08 \
-    --hash=sha256:f04e857b59d9d1ccc39ce2da1021d196e47234873820cbeaad210724b1ee28ac \
-    --hash=sha256:fadbfe37f74051d024037f223b8e001611eac868b5c5b06144ef4d8b799862f2
-    # via tzlocal
 cachetools==5.5.0 \
     --hash=sha256:02134e8439cdc2ffb62023ce1debca2944c3f289d66bb17ead3ab3dede74b292 \
     --hash=sha256:2cc24fb4cbe39633fb7badd9db9ca6295d766d9c2995f245725a46715d050f2a
@@ -571,12 +548,6 @@ idna==3.8 \
     --hash=sha256:050b4e5baadcd44d760cedbd2b8e639f2ff89bbc7a5730fcc662954303377aac \
     --hash=sha256:d838c2c0ed6fced7693d5e8ab8e734d5f8fda53a039c0164afb0b82e771e3603
     # via requests
-importlib-resources==6.4.5 \
-    --hash=sha256:980862a1d16c9e147a59603677fa2aa5fd82b87f223b6cb870695bcfce830065 \
-    --hash=sha256:ac29d5f956f01d5e4bb63102a5a19957f1b9175e45649977264a1416783bb717
-    # via
-    #   jsonschema
-    #   jsonschema-specifications
 jinja2==3.1.4 \
     --hash=sha256:4a3aee7acbbe7303aede8e9648d13b8bf88a429282aa6122a993f0ac800cb369 \
     --hash=sha256:bc5dd2abb727a5319567b7a813e6a2e7318c39f4f487cfe6c89c6f9c7d25197d
@@ -864,10 +835,6 @@ pillow==10.4.0 \
     --hash=sha256:ff25afb18123cea58a591ea0244b92eb1e61a1fd497bf6d6384f09bc3262ec3e \
     --hash=sha256:ff337c552345e95702c5fde3158acb0625111017d0e5f24bf3acdb9cc16b90d1
     # via torchvision
-pkgutil-resolve-name==1.3.10 \
-    --hash=sha256:357d6c9e6a755653cfd78893817c0853af365dd51ec97f3d358a819373bbd174 \
-    --hash=sha256:ca27cc078d25c5ad71a9de0a7a330146c4e014c2462d9af19c6b828280649c5e
-    # via jsonschema
 proto-plus==1.24.0 \
     --hash=sha256:30b72a5ecafe4406b0d339db35b56c4059064e69227b8c3bda7462397f966445 \
     --hash=sha256:402576830425e5f6ce4c2a6702400ac79897dab0b4343821aa5188b0fab81a12
@@ -1734,7 +1701,7 @@ torch==2.2.2 \
     --hash=sha256:eb4d6e9d3663e26cd27dc3ad266b34445a16b54908e74725adb241aa56987533 \
     --hash=sha256:f9ef0a648310435511e76905f9b89612e45ef2c8b023bee294f5e6f7e73a3e7c
     # via
-    #   -r ./python/../python/src/main/python/streaming-llm/base_requirements.txt
+    #   -r python/../python/src/main/python/streaming-llm/base_requirements.txt
     #   torchvision
 torchvision==0.17.2 \
     --hash=sha256:067095e87a020a7a251ac1d38483aa591c5ccb81e815527c54db88a982fc9267 \
@@ -1762,7 +1729,7 @@ torchvision==0.17.2 \
     --hash=sha256:efd6d0dd0668e15d01a2cffadc74068433b32cbcf5692e0c4aa15fc5cb250ce7 \
     --hash=sha256:f1c9ab3152cfb27f83aca072cac93a3a4c4e4ab0261cf0f2d516b9868a4e96f3 \
     --hash=sha256:f400145fc108833e7c2fc28486a04989ca742146d7a2a2cc48878ebbb40cdbbd
-    # via -r ./python/../python/src/main/python/streaming-llm/base_requirements.txt
+    # via -r python/../python/src/main/python/streaming-llm/base_requirements.txt
 tqdm==4.66.5 \
     --hash=sha256:90279a3770753eafc9194a0364852159802111925aa30eb3f9d85b0e805ac7cd \
     --hash=sha256:e1020aef2e5096702d8a025ac7d16b1577279c9d63f8375b63083e9a5f0fcbad
@@ -1772,12 +1739,11 @@ tqdm==4.66.5 \
 transformers==4.44.2 \
     --hash=sha256:1c02c65e7bfa5e52a634aff3da52138b583fc6f263c1f28d547dc144ba3d412d \
     --hash=sha256:36aa17cc92ee154058e426d951684a2dab48751b35b49437896f898931270826
-    # via -r ./python/../python/src/main/python/streaming-llm/base_requirements.txt
+    # via -r python/../python/src/main/python/streaming-llm/base_requirements.txt
 typing-extensions==4.12.2 \
     --hash=sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d \
     --hash=sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8
     # via
-    #   annotated-types
     #   apache-beam
     #   huggingface-hub
     #   pydantic
@@ -1791,10 +1757,6 @@ urllib3==2.2.3 \
     --hash=sha256:ca899ca043dcb1bafa3e262d73aa25c465bfb49e0bd9dd5d59f1d0acba2f8fac \
     --hash=sha256:e7d814a81dad81e6caf2ec9fdedb284ecc9c73076b62654547cc64ccdcae26e9
     # via requests
-zipp==3.20.2 \
-    --hash=sha256:a817ac80d6cf4b23bf7f2828b7cabf326f15a001bea8b1f9b49631780ba28350 \
-    --hash=sha256:bc9eb26f4506fda01b81bcde0ca78103b6e62f991b381fec825435c836edbc29
-    # via importlib-resources
 zstandard==0.23.0 \
     --hash=sha256:034b88913ecc1b097f528e42b539453fa82c3557e414b3de9d5632c80439a473 \
     --hash=sha256:0a7f0804bb3799414af278e9ad51be25edf67f78f916e08afdb983e74161b916 \
@@ -1895,3 +1857,9 @@ zstandard==0.23.0 \
     --hash=sha256:fe3b385d996ee0822fd46528d9f0443b880d4d05528fd26a9119a54ec3f91c69
     # via apache-beam
 
+# The following packages are considered to be unsafe in a requirements file:
+setuptools==75.1.0 \
+    --hash=sha256:35ab7fd3bcd95e6b7fd704e4a1539513edad446c097797f2985e0e4b960772f2 \
+    --hash=sha256:d59a21b17a275fb872a9c3dae73963160ae079f1049ed956880cd7c09b120538
+    # via -r python/../python/src/main/python/streaming-llm/base_requirements.txt
+