Update to Python 3.12.x (#5736)

.dockerignore

@@ -3,4 +3,5 @@
 !common.mk
 !requirements*.txt
 !bin/keys/docker-apt-keyring.pgp
+!fips_enabled
 requirements.all.txt

.gitlab-ci.yml

@@ -11,6 +11,7 @@ workflow:
 variables:
   azul_image: $CI_REGISTRY_IMAGE
   azul_image_tag: $CI_PIPELINE_ID
+  azul_proc_sys_crypto: /proc/sys/crypto
 
 stages:
   - build_image

Dockerfile

@@ -46,7 +46,11 @@ RUN mkdir terraform \
 RUN install -m 0755 -d /etc/apt/keyrings
 COPY --chmod=0644 bin/keys/docker-apt-keyring.pgp /etc/apt/keyrings/docker.gpg
 ARG azul_docker_version
-RUN set -o pipefail \
+# FIXME: Remove mounting of fips_enabled
+#        https://github.com/DataBiosphere/azul/issues/6675
+ARG azul_proc_sys_crypto=/tmp
+RUN --mount=type=bind,source=fips_enabled,target=${azul_proc_sys_crypto}/fips_enabled \
+    set -o pipefail \
     && ( \
       echo "deb [arch="$(dpkg --print-architecture)" signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/debian "$(. /etc/os-release && echo "$VERSION_CODENAME")" stable" \
       | tee /etc/apt/sources.list.d/docker.list \

Makefile

@@ -14,7 +14,7 @@ hello: check_python
 virtualenv: check_env
 	@if test -s "$$VIRTUAL_ENV"; then echo -e "\nRun 'deactivate' first\n"; false; fi
 	if test -e .venv; then rm -rf .venv/; fi
-	python3.11 -m venv .venv
+	python3.12 -m venv .venv
 	@echo -e "\nRun 'source .venv/bin/activate' now!\n"
 
 .PHONY: envhook
@@ -36,15 +36,18 @@ $(eval $(call requirements,_runtime_deps,requirements_pip,,))
 define docker
 .PHONY: docker$1
 docker$1: check_docker
+	echo 0 > fips_enabled
 	docker build \
 	       --build-arg azul_docker_registry=$$(azul_docker_registry) \
 	       --build-arg azul_python_image=$$(azul_python_image) \
 	       --build-arg azul_docker_version=$$(azul_docker_version) \
 	       --build-arg azul_terraform_version=$$(azul_terraform_version) \
 	       --build-arg PIP_DISABLE_PIP_VERSION_CHECK=$$(PIP_DISABLE_PIP_VERSION_CHECK) \
 	       --build-arg make_target=requirements$2 \
+	       --build-arg azul_proc_sys_crypto=$$(azul_proc_sys_crypto) \
 	       --tag $$(azul_image)$3:$$(azul_image_tag) \
 	       .
+	rm fips_enabled
 
 .PHONY: docker$1_push
 docker$1_push: docker$1
@@ -81,6 +84,7 @@ requirements_update: check_venv check_docker
 	rm ${azul_chalice_bin}/*
 	pip download \
 	    --platform=manylinux2014_x86_64 \
+	    --only-binary=:all: \
 	    --no-deps \
 	    -r requirements.txt \
 	    --dest=${azul_chalice_bin}

README.md

@@ -1032,7 +1032,7 @@ found in [`AzulTestCase`](test/azul_test_case.py) and commit the modifications.
 
 ## Setting up the Azul build prerequisites on macOS 12 (Monterey)
 
-The steps below are examplary for Python 3.11.6. Replace `3.11.6` with the value
+The steps below are examplary for Python 3.12.6. Replace `3.12.6` with the value
 of `azul_python_version` in [environment.py](environment.py).
 
 Make `bash` the default shell. Google it.
@@ -1048,10 +1048,10 @@ brew install zlib pyenv
 Install python
 
 ```
-pyenv install 3.11.6
+pyenv install 3.12.6
 ```
 
-Set `PYENV_VERSION` to `3.11.6` in `environment.local.py` at the project root.
+Set `PYENV_VERSION` to `3.12.6` in `environment.local.py` at the project root.
 Do not set `SYSTEM_VERSION_COMPAT`. For a more maintainable configuration use 
 `os.environ['azul_python_version']` as the value and `import os` at the top.
 
@@ -2319,10 +2319,10 @@ If you add a dependency on a package with native code, you need to build the
 wheel manually:
 
 ```
-(.venv) ~/workspace/hca/azul$ docker run -it -v ${project_root}/:/root/azul python:3.11.6-bullseye bash
+(.venv) ~/workspace/hca/azul$ docker run -it -v ${project_root}/:/root/azul python:3.12.6-slim-bookworm bash
 
 root@97804cb60d95:/# pip --version
-pip 22.0.4 from /usr/local/lib/python3.11/site-packages/pip (python 3.11)
+pip 24.2 from /usr/local/lib/python3.12/site-packages/pip (python 3.12)
 
 root@97804cb60d95:/# cd /root/azul/lambdas/.wheels
 

common.mk

@@ -49,7 +49,7 @@ check_python: check_venv
 	@if ! python -c "import sys; \
                      from chalice import chalice_version as v; \
 		             p = lambda v: tuple(map(int, v.split('.'))); \
-		             sys.exit(0 if p(v) == p('1.30.0') else 1)"; then \
+		             sys.exit(0 if p(v) == p('1.31.2') else 1)"; then \
 		echo -e "\nLooks like chalice is out of date. Please run 'make requirements'\n"; \
 		false; \
 	fi

deployments/dev/.example.environment.local.py

@@ -13,7 +13,7 @@ def env() -> Mapping[str, Optional[str]]:
     other environment variables in the form `{FOO}` where FOO is the name of an
     environment variable. See
 
-    https://docs.python.org/3.11/library/string.html#format-string-syntax
+    https://docs.python.org/3.12/library/string.html#format-string-syntax
 
     for the concrete syntax. These references will be resolved *after* the
     overall environment has been compiled by merging all relevant

deployments/dev/environment.py

@@ -189,7 +189,7 @@ def env() -> Mapping[str, Optional[str]]:
     other environment variables in the form `{FOO}` where FOO is the name of an
     environment variable. See
 
-    https://docs.python.org/3.11/library/string.html#format-string-syntax
+    https://docs.python.org/3.12/library/string.html#format-string-syntax
 
     for the concrete syntax. These references will be resolved *after* the
     overall environment has been compiled by merging all relevant

docker_images.json

@@ -15,18 +15,18 @@
             }
         }
     },
-    "docker.io/library/python:3.11.10-slim-bullseye": {
-        "digest": "sha256:f6a64ef0a5cc14855b15548056a8fc77f4c3526b79883fa6709a8e23f676ac34",
-        "mirror_digest": "sha256:13b5591078b6450547762ed1faec89f9934d69d5cffc746f8cdbc1d2ee5b44d4",
+    "docker.io/library/python:3.12.6-slim-bookworm": {
+        "digest": "sha256:15bad989b293be1dd5eb26a87ecacadaee1559f98e29f02bf6d00c8d86129f39",
+        "mirror_digest": "sha256:289f8a62caa961ebb045882e86adf8870a802f322c962078c2532858c3090756",
         "parts": {
             "linux/amd64": {
-                "digest": "sha256:99010d5cbf95418c4dbaeb3be33658bc1b65c3a6324bba447424c6768728e4f7",
-                "id": "sha256:26c608e127a5277940aedc0d8683bcd8d7df926a1ffd0aa339b6e2423b24b5b3",
+                "digest": "sha256:e0203bb78451dc5e8fe447983dc664f454068e8eea8d47b204cbe80a99b09d3e",
+                "id": "sha256:0d8d642b665d136b7a11d9f4a3d8a25b18deba7479b5897009fdee7e3fe1eb0e",
                 "platform": "linux/amd64"
             },
             "linux/arm64": {
-                "digest": "sha256:fa2d1949ad1ecc4078bc0489137366c95eb8b2cebcac759cfa16b537891539be",
-                "id": "sha256:f04f078657e27cdbcbe225b931e03dfe6417f926835b19de0a1e986d4bee02de",
+                "digest": "sha256:c0c2008864355f2263dcc690b1190540315bdd179dfbb0fae6ee58b174da4041",
+                "id": "sha256:c0cb1ac1ea30eef3bedf58895ed0abf74baac5411ea0ed177645ff5a1c0a00c8",
                 "platform": "linux/arm64"
             }
         }

docs/licenses/python.txt

@@ -1,4 +1,4 @@
-https://docs.python.org/3.11/license.html
+https://docs.python.org/3.12/license.html
 
 The MIT License
 

environment.boot

@@ -1,4 +1,4 @@
-azul_python_version=3.11.10
-azul_python_image=docker.io/library/python@sha256:13b5591078b6450547762ed1faec89f9934d69d5cffc746f8cdbc1d2ee5b44d4
+azul_python_version=3.12.6
+azul_python_image=docker.io/library/python@sha256:289f8a62caa961ebb045882e86adf8870a802f322c962078c2532858c3090756
 azul_docker_version=27.3.1
 azul_terraform_version=1.9.8

environment.py

@@ -15,7 +15,7 @@ def env() -> Mapping[str, Optional[str]]:
     other environment variables in the form `{FOO}` where FOO is the name of an
     environment variable. See
 
-    https://docs.python.org/3.11/library/string.html#format-string-syntax
+    https://docs.python.org/3.12/library/string.html#format-string-syntax
 
     for the concrete syntax. These references will be resolved *after* the
     overall environment has been compiled by merging all relevant
@@ -248,7 +248,7 @@ def env() -> Mapping[str, Optional[str]]:
         # and committing the resulting changes. It also requires redeploying the
         # `shared` component.
         #
-        'azul_python_version': '3.11.10',
+        'azul_python_version': '3.12.6',
 
         # The version of Terraform used throughout the system.
         #
@@ -279,7 +279,7 @@ def env() -> Mapping[str, Optional[str]]:
             # See `azul_python_version` above about what actions are required
             # after modifying this entry.
             'python': {
-                'ref': 'docker.io/library/python:{azul_python_version}-slim-bullseye',
+                'ref': 'docker.io/library/python:{azul_python_version}-slim-bookworm',
                 'url': 'https://hub.docker.com/_/python',
             },
             'pycharm': {

requirements.all.txt

@@ -10,12 +10,12 @@ blinker==1.8.2
 boto3==1.28.63
 boto3-stubs==1.28.63
 botocore==1.31.63
-botocore-stubs==1.35.44
+botocore-stubs==1.35.53
 brotli==1.1.0
 cachetools==5.5.0
 certifi==2024.8.30
 cffi==1.17.1
-chalice==1.30.0+12
+chalice==1.31.2+13
 charset-normalizer==3.4.0
 chevron==0.14.0
 click==8.1.7
@@ -28,9 +28,9 @@ docker==7.1.0
 docutils==0.16
 elasticsearch==7.17.9
 elasticsearch-dsl==7.4.0
-et-xmlfile==1.1.0
-fastavro==1.8.2
-flake8==5.0.4
+et_xmlfile==2.0.0
+fastavro==1.9.7
+flake8==7.1.1
 flask==3.0.3
 flask-basicauth==0.2.0
 flask-cors==5.0.0
@@ -51,21 +51,21 @@ google-crc32c==1.6.0
 google-resumable-media==2.7.2
 googleapis-common-protos==1.65.0
 greenlet==3.1.1
-grpcio==1.67.0
+grpcio==1.67.1
 grpcio-status==1.62.3
 http-message-signatures==0.4.4
 http_sfv==0.9.9
 httplib2==0.22.0
 idna==3.10
-importlib-resources==5.13.0
 inquirer==2.10.1
 itsdangerous==2.2.0
 jdcal==1.4.1
 jinja2==3.1.4
 jmespath==1.0.1
-jq==1.3.0
-jsonschema==4.17.3
-jsonschema-spec==0.1.6
+jq==1.8.0
+jsonschema==4.23.0
+jsonschema-path==0.3.3
+jsonschema-specifications==2023.12.1
 jwcrypto==1.5.6
 lazy-object-proxy==1.10.0
 locust==2.12.2
@@ -82,29 +82,28 @@ mypy-boto3-lambda==1.28.83
 mypy-boto3-s3==1.28.55
 mypy-boto3-sqs==1.28.82
 mypy-boto3-stepfunctions==1.28.36
-openapi-schema-validator==0.3.4
-openapi-spec-validator==0.5.1
+openapi-schema-validator==0.6.2
+openapi-spec-validator==0.7.1
 openpyxl==3.0.6
 orderedmultidict==1.0.1
 packaging==24.1
 pathable==0.4.3
 pip==23.3.2
 posix_ipc==1.1.1
-proto-plus==1.24.0
+proto-plus==1.25.0
 protobuf==4.25.5
 psutil==6.1.0
 py-partiql-parser==0.3.3
 pyasn1==0.6.1
 pyasn1_modules==0.4.1
-pycodestyle==2.9.1
+pycodestyle==2.12.1
 pycparser==2.22
-pyflakes==2.5.0
+pyflakes==3.2.0
 pygithub==1.56
 pyjwt==2.9.0
 pynacl==1.5.0
 pyopenssl==24.2.1
 pyparsing==3.2.0
-pyrsistent==0.20.0
 python-dateutil==2.9.0.post0
 python-dxf==11.4.0
 python-editor==1.0.4
@@ -113,11 +112,14 @@ pytz==2024.2
 pyyaml==6.0.1
 pyzmq==26.2.0
 readchar==4.2.0
+referencing==0.35.1
 requests==2.32.2
 requests-toolbelt==1.0.0
 requirements-parser==0.2.0+1
 responses==0.22.0
+rfc3339-validator==0.1.4
 roundrobin==0.0.4
+rpds-py==0.20.1
 rsa==4.7.2
 s3transfer==0.7.0
 setuptools==70.3.0
@@ -126,8 +128,8 @@ six==1.16.0
 smmap==5.0.1
 strict-rfc3339==0.7
 toml==0.10.2
-tqdm==4.66.5
-types-awscrt==0.22.0
+tqdm==4.66.6
+types-awscrt==0.23.0
 types-s3transfer==0.10.3
 types-toml==0.10.8.20240310
 typing_extensions==4.12.2
@@ -142,4 +144,4 @@ wrapt==1.16.0
 www-authenticate==0.9.2
 xmltodict==0.14.2
 zope.event==5.0
-zope.interface==7.1.0
+zope.interface==7.1.1

requirements.dev.trans.txt

@@ -1,12 +1,12 @@
 blessed==1.20.0
 blinker==1.8.2
-botocore-stubs==1.35.44
+botocore-stubs==1.35.53
 brotli==1.1.0
 click==8.1.7
 colorama==0.4.4
 configargparse==1.7
 docutils==0.16
-et-xmlfile==1.1.0
+et_xmlfile==2.0.0
 flask==3.0.3
 flask-basicauth==0.2.0
 flask-cors==5.0.0
@@ -15,13 +15,13 @@ gitdb==4.0.11
 google-auth-httplib2==0.2.0
 greenlet==3.1.1
 httplib2==0.22.0
-importlib-resources==5.13.0
 inquirer==2.10.1
 itsdangerous==2.2.0
 jdcal==1.4.1
 jinja2==3.1.4
-jsonschema==4.17.3
-jsonschema-spec==0.1.6
+jsonschema==4.23.0
+jsonschema-path==0.3.3
+jsonschema-specifications==2023.12.1
 jwcrypto==1.5.6
 lazy-object-proxy==1.10.0
 mccabe==0.7.0
@@ -33,30 +33,32 @@ mypy-boto3-lambda==1.28.83
 mypy-boto3-s3==1.28.55
 mypy-boto3-sqs==1.28.82
 mypy-boto3-stepfunctions==1.28.36
-openapi-schema-validator==0.3.4
+openapi-schema-validator==0.6.2
 pathable==0.4.3
 psutil==6.1.0
 py-partiql-parser==0.3.3
-pycodestyle==2.9.1
-pyflakes==2.5.0
+pycodestyle==2.12.1
+pyflakes==3.2.0
 pyjwt==2.9.0
 pynacl==1.5.0
 pyparsing==3.2.0
-pyrsistent==0.20.0
 python-editor==1.0.4
 pyzmq==26.2.0
 readchar==4.2.0
+referencing==0.35.1
 requests-toolbelt==1.0.0
+rfc3339-validator==0.1.4
 roundrobin==0.0.4
+rpds-py==0.20.1
 smmap==5.0.1
 toml==0.10.2
-tqdm==4.66.5
-types-awscrt==0.22.0
+tqdm==4.66.6
+types-awscrt==0.23.0
 types-s3transfer==0.10.3
 types-toml==0.10.8.20240310
 uritemplate==4.1.1
 wcwidth==0.2.13
 www-authenticate==0.9.2
 xmltodict==0.14.2
 zope.event==5.0
-zope.interface==7.1.0
+zope.interface==7.1.1