diff --git a/.dockerignore b/.dockerignore index 3fdc4b24c2..a92dc8c396 100644 --- a/.dockerignore +++ b/.dockerignore @@ -3,4 +3,5 @@ !common.mk !requirements*.txt !bin/keys/docker-apt-keyring.pgp +!fips_enabled requirements.all.txt diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 71273aaa2a..cb540b6067 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -11,6 +11,7 @@ workflow: variables: azul_image: $CI_REGISTRY_IMAGE azul_image_tag: $CI_PIPELINE_ID + azul_proc_sys_crypto: /proc/sys/crypto stages: - build_image diff --git a/Dockerfile b/Dockerfile index 06aed2ecca..415704ce81 100644 --- a/Dockerfile +++ b/Dockerfile @@ -46,7 +46,11 @@ RUN mkdir terraform \ RUN install -m 0755 -d /etc/apt/keyrings COPY --chmod=0644 bin/keys/docker-apt-keyring.pgp /etc/apt/keyrings/docker.gpg ARG azul_docker_version -RUN set -o pipefail \ +# FIXME: Remove mounting of fips_enabled +# https://github.com/DataBiosphere/azul/issues/6675 +ARG azul_proc_sys_crypto=/tmp +RUN --mount=type=bind,source=fips_enabled,target=${azul_proc_sys_crypto}/fips_enabled \ + set -o pipefail \ && ( \ echo "deb [arch="$(dpkg --print-architecture)" signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/debian "$(. /etc/os-release && echo "$VERSION_CODENAME")" stable" \ | tee /etc/apt/sources.list.d/docker.list \ diff --git a/Makefile b/Makefile index 717574bc3e..7215192445 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,7 @@ hello: check_python virtualenv: check_env @if test -s "$$VIRTUAL_ENV"; then echo -e "\nRun 'deactivate' first\n"; false; fi if test -e .venv; then rm -rf .venv/; fi - python3.11 -m venv .venv + python3.12 -m venv .venv @echo -e "\nRun 'source .venv/bin/activate' now!\n" .PHONY: envhook @@ -36,6 +36,7 @@ $(eval $(call requirements,_runtime_deps,requirements_pip,,)) define docker .PHONY: docker$1 docker$1: check_docker + echo 0 > fips_enabled docker build \ --build-arg azul_docker_registry=$$(azul_docker_registry) \ --build-arg azul_python_image=$$(azul_python_image) \ @@ -43,8 +44,10 @@ docker$1: check_docker --build-arg azul_terraform_version=$$(azul_terraform_version) \ --build-arg PIP_DISABLE_PIP_VERSION_CHECK=$$(PIP_DISABLE_PIP_VERSION_CHECK) \ --build-arg make_target=requirements$2 \ + --build-arg azul_proc_sys_crypto=$$(azul_proc_sys_crypto) \ --tag $$(azul_image)$3:$$(azul_image_tag) \ . + rm fips_enabled .PHONY: docker$1_push docker$1_push: docker$1 @@ -81,6 +84,7 @@ requirements_update: check_venv check_docker rm ${azul_chalice_bin}/* pip download \ --platform=manylinux2014_x86_64 \ + --only-binary=:all: \ --no-deps \ -r requirements.txt \ --dest=${azul_chalice_bin} diff --git a/README.md b/README.md index 0261e39932..edf11fad5a 100644 --- a/README.md +++ b/README.md @@ -1032,7 +1032,7 @@ found in [`AzulTestCase`](test/azul_test_case.py) and commit the modifications. ## Setting up the Azul build prerequisites on macOS 12 (Monterey) -The steps below are examplary for Python 3.11.6. Replace `3.11.6` with the value +The steps below are examplary for Python 3.12.6. Replace `3.12.6` with the value of `azul_python_version` in [environment.py](environment.py). Make `bash` the default shell. Google it. @@ -1048,10 +1048,10 @@ brew install zlib pyenv Install python ``` -pyenv install 3.11.6 +pyenv install 3.12.6 ``` -Set `PYENV_VERSION` to `3.11.6` in `environment.local.py` at the project root. +Set `PYENV_VERSION` to `3.12.6` in `environment.local.py` at the project root. Do not set `SYSTEM_VERSION_COMPAT`. For a more maintainable configuration use `os.environ['azul_python_version']` as the value and `import os` at the top. @@ -2319,10 +2319,10 @@ If you add a dependency on a package with native code, you need to build the wheel manually: ``` -(.venv) ~/workspace/hca/azul$ docker run -it -v ${project_root}/:/root/azul python:3.11.6-bullseye bash +(.venv) ~/workspace/hca/azul$ docker run -it -v ${project_root}/:/root/azul python:3.12.6-slim-bookworm bash root@97804cb60d95:/# pip --version -pip 22.0.4 from /usr/local/lib/python3.11/site-packages/pip (python 3.11) +pip 24.2 from /usr/local/lib/python3.12/site-packages/pip (python 3.12) root@97804cb60d95:/# cd /root/azul/lambdas/.wheels diff --git a/bin/wheels/runtime/MarkupSafe-3.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl b/bin/wheels/runtime/MarkupSafe-3.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl deleted file mode 100644 index 213b0bc8e6..0000000000 Binary files a/bin/wheels/runtime/MarkupSafe-3.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl and /dev/null differ diff --git a/bin/wheels/runtime/MarkupSafe-3.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl b/bin/wheels/runtime/MarkupSafe-3.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl new file mode 100644 index 0000000000..12902271d3 Binary files /dev/null and b/bin/wheels/runtime/MarkupSafe-3.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl differ diff --git a/bin/wheels/runtime/cffi-1.17.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl b/bin/wheels/runtime/cffi-1.17.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl deleted file mode 100644 index de6ffd6e5d..0000000000 Binary files a/bin/wheels/runtime/cffi-1.17.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl and /dev/null differ diff --git a/bin/wheels/runtime/cffi-1.17.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl b/bin/wheels/runtime/cffi-1.17.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl new file mode 100644 index 0000000000..81f076b558 Binary files /dev/null and b/bin/wheels/runtime/cffi-1.17.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl differ diff --git a/bin/wheels/runtime/charset_normalizer-3.4.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl b/bin/wheels/runtime/charset_normalizer-3.4.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl deleted file mode 100644 index 51172a4bcc..0000000000 Binary files a/bin/wheels/runtime/charset_normalizer-3.4.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl and /dev/null differ diff --git a/bin/wheels/runtime/charset_normalizer-3.4.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl b/bin/wheels/runtime/charset_normalizer-3.4.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl new file mode 100644 index 0000000000..a8a867bf98 Binary files /dev/null and b/bin/wheels/runtime/charset_normalizer-3.4.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl differ diff --git a/bin/wheels/runtime/fastavro-1.8.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl b/bin/wheels/runtime/fastavro-1.8.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl deleted file mode 100644 index 7e16742e29..0000000000 Binary files a/bin/wheels/runtime/fastavro-1.8.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl and /dev/null differ diff --git a/bin/wheels/runtime/fastavro-1.9.7-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl b/bin/wheels/runtime/fastavro-1.9.7-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl new file mode 100644 index 0000000000..8598f24be3 Binary files /dev/null and b/bin/wheels/runtime/fastavro-1.9.7-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl differ diff --git a/bin/wheels/runtime/google_crc32c-1.6.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl b/bin/wheels/runtime/google_crc32c-1.6.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl similarity index 67% rename from bin/wheels/runtime/google_crc32c-1.6.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl rename to bin/wheels/runtime/google_crc32c-1.6.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl index 4211f50b4a..301327ba05 100644 Binary files a/bin/wheels/runtime/google_crc32c-1.6.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl and b/bin/wheels/runtime/google_crc32c-1.6.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl differ diff --git a/bin/wheels/runtime/grpcio-1.67.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl b/bin/wheels/runtime/grpcio-1.67.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl deleted file mode 100644 index 38b0455b53..0000000000 Binary files a/bin/wheels/runtime/grpcio-1.67.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl and /dev/null differ diff --git a/bin/wheels/runtime/grpcio-1.67.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl b/bin/wheels/runtime/grpcio-1.67.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl new file mode 100644 index 0000000000..522320b230 Binary files /dev/null and b/bin/wheels/runtime/grpcio-1.67.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl differ diff --git a/bin/wheels/runtime/msgpack-1.0.7-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl b/bin/wheels/runtime/msgpack-1.0.7-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl deleted file mode 100644 index 893d1873d6..0000000000 Binary files a/bin/wheels/runtime/msgpack-1.0.7-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl and /dev/null differ diff --git a/bin/wheels/runtime/msgpack-1.0.7-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl b/bin/wheels/runtime/msgpack-1.0.7-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl new file mode 100644 index 0000000000..03143af5a1 Binary files /dev/null and b/bin/wheels/runtime/msgpack-1.0.7-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl differ diff --git a/bin/wheels/runtime/proto_plus-1.24.0-py3-none-any.whl b/bin/wheels/runtime/proto_plus-1.25.0-py3-none-any.whl similarity index 50% rename from bin/wheels/runtime/proto_plus-1.24.0-py3-none-any.whl rename to bin/wheels/runtime/proto_plus-1.25.0-py3-none-any.whl index 424e91edfd..13d7de1fa5 100644 Binary files a/bin/wheels/runtime/proto_plus-1.24.0-py3-none-any.whl and b/bin/wheels/runtime/proto_plus-1.25.0-py3-none-any.whl differ diff --git a/bin/wheels/runtime/setuptools-70.3.0-py3-none-any.whl b/bin/wheels/runtime/setuptools-70.3.0-py3-none-any.whl new file mode 100644 index 0000000000..2c45a90d18 Binary files /dev/null and b/bin/wheels/runtime/setuptools-70.3.0-py3-none-any.whl differ diff --git a/bin/wheels/runtime/wrapt-1.16.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl b/bin/wheels/runtime/wrapt-1.16.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl deleted file mode 100644 index 8a66b093aa..0000000000 Binary files a/bin/wheels/runtime/wrapt-1.16.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl and /dev/null differ diff --git a/bin/wheels/runtime/wrapt-1.16.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl b/bin/wheels/runtime/wrapt-1.16.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl new file mode 100644 index 0000000000..da0ebfe78d Binary files /dev/null and b/bin/wheels/runtime/wrapt-1.16.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl differ diff --git a/common.mk b/common.mk index 7c27305e84..e995e89f0e 100644 --- a/common.mk +++ b/common.mk @@ -49,7 +49,7 @@ check_python: check_venv @if ! python -c "import sys; \ from chalice import chalice_version as v; \ p = lambda v: tuple(map(int, v.split('.'))); \ - sys.exit(0 if p(v) == p('1.30.0') else 1)"; then \ + sys.exit(0 if p(v) == p('1.31.2') else 1)"; then \ echo -e "\nLooks like chalice is out of date. Please run 'make requirements'\n"; \ false; \ fi diff --git a/deployments/dev/.example.environment.local.py b/deployments/dev/.example.environment.local.py index ac71c36984..40bda46631 100644 --- a/deployments/dev/.example.environment.local.py +++ b/deployments/dev/.example.environment.local.py @@ -13,7 +13,7 @@ def env() -> Mapping[str, Optional[str]]: other environment variables in the form `{FOO}` where FOO is the name of an environment variable. See - https://docs.python.org/3.11/library/string.html#format-string-syntax + https://docs.python.org/3.12/library/string.html#format-string-syntax for the concrete syntax. These references will be resolved *after* the overall environment has been compiled by merging all relevant diff --git a/deployments/dev/environment.py b/deployments/dev/environment.py index 58b2d77e76..7d7649062e 100644 --- a/deployments/dev/environment.py +++ b/deployments/dev/environment.py @@ -189,7 +189,7 @@ def env() -> Mapping[str, Optional[str]]: other environment variables in the form `{FOO}` where FOO is the name of an environment variable. See - https://docs.python.org/3.11/library/string.html#format-string-syntax + https://docs.python.org/3.12/library/string.html#format-string-syntax for the concrete syntax. These references will be resolved *after* the overall environment has been compiled by merging all relevant diff --git a/docker_images.json b/docker_images.json index c012cdf7e0..184b4a5ebc 100644 --- a/docker_images.json +++ b/docker_images.json @@ -15,18 +15,18 @@ } } }, - "docker.io/library/python:3.11.10-slim-bullseye": { - "digest": "sha256:f6a64ef0a5cc14855b15548056a8fc77f4c3526b79883fa6709a8e23f676ac34", - "mirror_digest": "sha256:13b5591078b6450547762ed1faec89f9934d69d5cffc746f8cdbc1d2ee5b44d4", + "docker.io/library/python:3.12.6-slim-bookworm": { + "digest": "sha256:15bad989b293be1dd5eb26a87ecacadaee1559f98e29f02bf6d00c8d86129f39", + "mirror_digest": "sha256:289f8a62caa961ebb045882e86adf8870a802f322c962078c2532858c3090756", "parts": { "linux/amd64": { - "digest": "sha256:99010d5cbf95418c4dbaeb3be33658bc1b65c3a6324bba447424c6768728e4f7", - "id": "sha256:26c608e127a5277940aedc0d8683bcd8d7df926a1ffd0aa339b6e2423b24b5b3", + "digest": "sha256:e0203bb78451dc5e8fe447983dc664f454068e8eea8d47b204cbe80a99b09d3e", + "id": "sha256:0d8d642b665d136b7a11d9f4a3d8a25b18deba7479b5897009fdee7e3fe1eb0e", "platform": "linux/amd64" }, "linux/arm64": { - "digest": "sha256:fa2d1949ad1ecc4078bc0489137366c95eb8b2cebcac759cfa16b537891539be", - "id": "sha256:f04f078657e27cdbcbe225b931e03dfe6417f926835b19de0a1e986d4bee02de", + "digest": "sha256:c0c2008864355f2263dcc690b1190540315bdd179dfbb0fae6ee58b174da4041", + "id": "sha256:c0cb1ac1ea30eef3bedf58895ed0abf74baac5411ea0ed177645ff5a1c0a00c8", "platform": "linux/arm64" } } diff --git a/docs/licenses/python.txt b/docs/licenses/python.txt index 2eadb9d594..1384aa43ef 100644 --- a/docs/licenses/python.txt +++ b/docs/licenses/python.txt @@ -1,4 +1,4 @@ -https://docs.python.org/3.11/license.html +https://docs.python.org/3.12/license.html The MIT License diff --git a/environment.boot b/environment.boot index 8618d5364a..9a45901464 100644 --- a/environment.boot +++ b/environment.boot @@ -1,4 +1,4 @@ -azul_python_version=3.11.10 -azul_python_image=docker.io/library/python@sha256:13b5591078b6450547762ed1faec89f9934d69d5cffc746f8cdbc1d2ee5b44d4 +azul_python_version=3.12.6 +azul_python_image=docker.io/library/python@sha256:289f8a62caa961ebb045882e86adf8870a802f322c962078c2532858c3090756 azul_docker_version=27.3.1 azul_terraform_version=1.9.8 diff --git a/environment.py b/environment.py index a8da3691ff..fe57221bc0 100644 --- a/environment.py +++ b/environment.py @@ -15,7 +15,7 @@ def env() -> Mapping[str, Optional[str]]: other environment variables in the form `{FOO}` where FOO is the name of an environment variable. See - https://docs.python.org/3.11/library/string.html#format-string-syntax + https://docs.python.org/3.12/library/string.html#format-string-syntax for the concrete syntax. These references will be resolved *after* the overall environment has been compiled by merging all relevant @@ -248,7 +248,7 @@ def env() -> Mapping[str, Optional[str]]: # and committing the resulting changes. It also requires redeploying the # `shared` component. # - 'azul_python_version': '3.11.10', + 'azul_python_version': '3.12.6', # The version of Terraform used throughout the system. # @@ -279,7 +279,7 @@ def env() -> Mapping[str, Optional[str]]: # See `azul_python_version` above about what actions are required # after modifying this entry. 'python': { - 'ref': 'docker.io/library/python:{azul_python_version}-slim-bullseye', + 'ref': 'docker.io/library/python:{azul_python_version}-slim-bookworm', 'url': 'https://hub.docker.com/_/python', }, 'pycharm': { diff --git a/requirements.all.txt b/requirements.all.txt index 6d17985c80..b188a0d4aa 100644 --- a/requirements.all.txt +++ b/requirements.all.txt @@ -10,12 +10,12 @@ blinker==1.8.2 boto3==1.28.63 boto3-stubs==1.28.63 botocore==1.31.63 -botocore-stubs==1.35.44 +botocore-stubs==1.35.53 brotli==1.1.0 cachetools==5.5.0 certifi==2024.8.30 cffi==1.17.1 -chalice==1.30.0+12 +chalice==1.31.2+13 charset-normalizer==3.4.0 chevron==0.14.0 click==8.1.7 @@ -28,9 +28,9 @@ docker==7.1.0 docutils==0.16 elasticsearch==7.17.9 elasticsearch-dsl==7.4.0 -et-xmlfile==1.1.0 -fastavro==1.8.2 -flake8==5.0.4 +et_xmlfile==2.0.0 +fastavro==1.9.7 +flake8==7.1.1 flask==3.0.3 flask-basicauth==0.2.0 flask-cors==5.0.0 @@ -51,21 +51,21 @@ google-crc32c==1.6.0 google-resumable-media==2.7.2 googleapis-common-protos==1.65.0 greenlet==3.1.1 -grpcio==1.67.0 +grpcio==1.67.1 grpcio-status==1.62.3 http-message-signatures==0.4.4 http_sfv==0.9.9 httplib2==0.22.0 idna==3.10 -importlib-resources==5.13.0 inquirer==2.10.1 itsdangerous==2.2.0 jdcal==1.4.1 jinja2==3.1.4 jmespath==1.0.1 -jq==1.3.0 -jsonschema==4.17.3 -jsonschema-spec==0.1.6 +jq==1.8.0 +jsonschema==4.23.0 +jsonschema-path==0.3.3 +jsonschema-specifications==2023.12.1 jwcrypto==1.5.6 lazy-object-proxy==1.10.0 locust==2.12.2 @@ -82,29 +82,28 @@ mypy-boto3-lambda==1.28.83 mypy-boto3-s3==1.28.55 mypy-boto3-sqs==1.28.82 mypy-boto3-stepfunctions==1.28.36 -openapi-schema-validator==0.3.4 -openapi-spec-validator==0.5.1 +openapi-schema-validator==0.6.2 +openapi-spec-validator==0.7.1 openpyxl==3.0.6 orderedmultidict==1.0.1 packaging==24.1 pathable==0.4.3 pip==23.3.2 posix_ipc==1.1.1 -proto-plus==1.24.0 +proto-plus==1.25.0 protobuf==4.25.5 psutil==6.1.0 py-partiql-parser==0.3.3 pyasn1==0.6.1 pyasn1_modules==0.4.1 -pycodestyle==2.9.1 +pycodestyle==2.12.1 pycparser==2.22 -pyflakes==2.5.0 +pyflakes==3.2.0 pygithub==1.56 pyjwt==2.9.0 pynacl==1.5.0 pyopenssl==24.2.1 pyparsing==3.2.0 -pyrsistent==0.20.0 python-dateutil==2.9.0.post0 python-dxf==11.4.0 python-editor==1.0.4 @@ -113,11 +112,14 @@ pytz==2024.2 pyyaml==6.0.1 pyzmq==26.2.0 readchar==4.2.0 +referencing==0.35.1 requests==2.32.2 requests-toolbelt==1.0.0 requirements-parser==0.2.0+1 responses==0.22.0 +rfc3339-validator==0.1.4 roundrobin==0.0.4 +rpds-py==0.20.1 rsa==4.7.2 s3transfer==0.7.0 setuptools==70.3.0 @@ -126,8 +128,8 @@ six==1.16.0 smmap==5.0.1 strict-rfc3339==0.7 toml==0.10.2 -tqdm==4.66.5 -types-awscrt==0.22.0 +tqdm==4.66.6 +types-awscrt==0.23.0 types-s3transfer==0.10.3 types-toml==0.10.8.20240310 typing_extensions==4.12.2 @@ -142,4 +144,4 @@ wrapt==1.16.0 www-authenticate==0.9.2 xmltodict==0.14.2 zope.event==5.0 -zope.interface==7.1.0 +zope.interface==7.1.1 diff --git a/requirements.dev.trans.txt b/requirements.dev.trans.txt index db72eb193c..fc15bd4a05 100644 --- a/requirements.dev.trans.txt +++ b/requirements.dev.trans.txt @@ -1,12 +1,12 @@ blessed==1.20.0 blinker==1.8.2 -botocore-stubs==1.35.44 +botocore-stubs==1.35.53 brotli==1.1.0 click==8.1.7 colorama==0.4.4 configargparse==1.7 docutils==0.16 -et-xmlfile==1.1.0 +et_xmlfile==2.0.0 flask==3.0.3 flask-basicauth==0.2.0 flask-cors==5.0.0 @@ -15,13 +15,13 @@ gitdb==4.0.11 google-auth-httplib2==0.2.0 greenlet==3.1.1 httplib2==0.22.0 -importlib-resources==5.13.0 inquirer==2.10.1 itsdangerous==2.2.0 jdcal==1.4.1 jinja2==3.1.4 -jsonschema==4.17.3 -jsonschema-spec==0.1.6 +jsonschema==4.23.0 +jsonschema-path==0.3.3 +jsonschema-specifications==2023.12.1 jwcrypto==1.5.6 lazy-object-proxy==1.10.0 mccabe==0.7.0 @@ -33,25 +33,27 @@ mypy-boto3-lambda==1.28.83 mypy-boto3-s3==1.28.55 mypy-boto3-sqs==1.28.82 mypy-boto3-stepfunctions==1.28.36 -openapi-schema-validator==0.3.4 +openapi-schema-validator==0.6.2 pathable==0.4.3 psutil==6.1.0 py-partiql-parser==0.3.3 -pycodestyle==2.9.1 -pyflakes==2.5.0 +pycodestyle==2.12.1 +pyflakes==3.2.0 pyjwt==2.9.0 pynacl==1.5.0 pyparsing==3.2.0 -pyrsistent==0.20.0 python-editor==1.0.4 pyzmq==26.2.0 readchar==4.2.0 +referencing==0.35.1 requests-toolbelt==1.0.0 +rfc3339-validator==0.1.4 roundrobin==0.0.4 +rpds-py==0.20.1 smmap==5.0.1 toml==0.10.2 -tqdm==4.66.5 -types-awscrt==0.22.0 +tqdm==4.66.6 +types-awscrt==0.23.0 types-s3transfer==0.10.3 types-toml==0.10.8.20240310 uritemplate==4.1.1 @@ -59,4 +61,4 @@ wcwidth==0.2.13 www-authenticate==0.9.2 xmltodict==0.14.2 zope.event==5.0 -zope.interface==7.1.0 +zope.interface==7.1.1 diff --git a/requirements.dev.txt b/requirements.dev.txt index 4686a3c5a8..08b478404d 100644 --- a/requirements.dev.txt +++ b/requirements.dev.txt @@ -1,19 +1,19 @@ atomicwrites==1.4.1 awscli==1.29.63 -git+https://github.com/DataBiosphere/azul-chalice@1.30.0+12#egg=chalice +git+https://github.com/DataBiosphere/azul-chalice@1.31.2+13#egg=chalice coverage==6.5.0 docker==7.1.0 -flake8==5.0.4 +flake8==7.1.1 gevent==23.9.1 gitpython==3.1.41 google-api-python-client==2.103.0 google-cloud-storage==2.12.0 google-cloud-bigquery-reservation==1.11.3 -jq==1.3.0 +jq==1.8.0 locust==2.12.2 moto[s3,sqs,sns,dynamodb,iam]==4.1.13 # match the extras with the backends listed in AzulUnitTestCase._reset_moto boto3-stubs[s3,sqs,lambda,dynamodb,iam,ecr,stepfunctions,kms]==1.28.63 # match this with the version of the `boto3` runtime dependency -openapi-spec-validator==0.5.1 +openapi-spec-validator==0.7.1 openpyxl==3.0.6 posix_ipc==1.1.1 pygithub==1.56 diff --git a/requirements.pip.txt b/requirements.pip.txt index 8f613b849c..8bdbcd9300 100644 --- a/requirements.pip.txt +++ b/requirements.pip.txt @@ -1,3 +1,3 @@ pip==23.3.2 -setuptools==70.3.0 +setuptools==70.3.0 # keep consistent with requirements.txt wheel==0.38.4 diff --git a/requirements.trans.txt b/requirements.trans.txt index 2006a6cadd..c3e50c95a3 100644 --- a/requirements.trans.txt +++ b/requirements.trans.txt @@ -9,14 +9,14 @@ google-cloud-core==2.4.1 google-crc32c==1.6.0 google-resumable-media==2.7.2 googleapis-common-protos==1.65.0 -grpcio==1.67.0 +grpcio==1.67.1 grpcio-status==1.62.3 http_sfv==0.9.9 idna==3.10 markupsafe==3.0.2 orderedmultidict==1.0.1 packaging==24.1 -proto-plus==1.24.0 +proto-plus==1.25.0 protobuf==4.25.5 pyasn1==0.6.1 pyasn1_modules==0.4.1 diff --git a/requirements.txt b/requirements.txt index f3c6ac4185..4e090bb043 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,7 +7,7 @@ chevron==0.14.0 deprecated==1.2.13 elasticsearch-dsl==7.4.0 elasticsearch==7.17.9 -fastavro==1.8.2 +fastavro==1.9.7 furl==2.1.3 google-auth[pyopenssl]==2.23.3 google-cloud-bigquery==3.12.0 @@ -18,6 +18,7 @@ more-itertools==9.0.0 msgpack==1.0.7 requests==2.32.2 rsa==4.7.2 # resolve ambiguity with build-time dependency +setuptools==70.3.0 # Keep consistent with requirements.pip.txt. Python 3.12 removed distutils, which we depended on transitively. Luckily, setuptools includes a vendored copy. urllib3==1.26.19 werkzeug==3.0.3 -r requirements.trans.txt diff --git a/scripts/convert_environment.py b/scripts/convert_environment.py index 1e330cd27b..d2d3489f88 100644 --- a/scripts/convert_environment.py +++ b/scripts/convert_environment.py @@ -109,7 +109,7 @@ def env() -> Mapping[str, Optional[str]]: other environment variables in the form `{FOO}` where FOO is the name of an environment variable. See - https://docs.python.org/3.11/library/string.html#format-string-syntax + https://docs.python.org/3.12/library/string.html#format-string-syntax for the concrete syntax. These references will be resolved *after* the overall environment has been compiled by merging all relevant diff --git a/scripts/export_environment.py b/scripts/export_environment.py index dfe4438c36..3dc2c9ec83 100644 --- a/scripts/export_environment.py +++ b/scripts/export_environment.py @@ -57,7 +57,7 @@ def env(self) -> DraftEnvironment: references to other environment variables in the form `{FOO}` where FOO is the name of an environment variable. See - https://docs.python.org/3.11/library/string.html#format-string-syntax + https://docs.python.org/3.12/library/string.html#format-string-syntax for the concrete syntax. Any references will be resolved after the environment has been compiled by merging all environment.py files. diff --git a/scripts/sell_unused_slots.py b/scripts/sell_unused_slots.py index 5cd68f2849..e583ed398f 100644 --- a/scripts/sell_unused_slots.py +++ b/scripts/sell_unused_slots.py @@ -76,6 +76,8 @@ def _list_contribution_lambda_functions(cls) -> list[Lambda]: ] def _lambda_invocation_counts(self) -> dict[Lambda, int]: + # FIXME: DeprecationWarning for datetime methods in Python 3.12 + # https://github.com/DataBiosphere/azul/issues/5953 end = datetime.utcnow() start = end - timedelta(minutes=self.interval) lambdas_by_name = { diff --git a/src/azul/dss.py b/src/azul/dss.py index 08b67f3fde..bf1ca6baf5 100644 --- a/src/azul/dss.py +++ b/src/azul/dss.py @@ -9,6 +9,8 @@ def new_version(): + # FIXME: DeprecationWarning for datetime methods in Python 3.12 + # https://github.com/DataBiosphere/azul/issues/5953 return datetime.utcnow().strftime(version_format) diff --git a/src/azul/logging.py b/src/azul/logging.py index f5a24bb37a..d0bfeadf6e 100644 --- a/src/azul/logging.py +++ b/src/azul/logging.py @@ -56,7 +56,7 @@ def configure_app_logging(app: AzulChaliceApp, *loggers): # module can be found on GitHub, in the repository linked below. Note # that one must extract the image tarball to get to the module. # - # https://github.com/aws/aws-lambda-base-images/tree/python3.11 + # https://github.com/aws/aws-lambda-base-images/tree/python3.12 # handler = one(root_logger.handlers) root_formatter = logging.Formatter(lambda_log_format, lambda_log_date_format) diff --git a/src/azul/plugins/metadata/hca/indexer/aggregate.py b/src/azul/plugins/metadata/hca/indexer/aggregate.py index 06ff91527f..9ec34c9f4c 100644 --- a/src/azul/plugins/metadata/hca/indexer/aggregate.py +++ b/src/azul/plugins/metadata/hca/indexer/aggregate.py @@ -49,7 +49,6 @@ class HCAAggregate(Aggregate): @cached_property def cell_count(self) -> int: - cs: JSON return sum(cs['total_estimated_cells'] for cs in self.contents['cell_suspensions'] if cs['total_estimated_cells'] is not None) diff --git a/src/humancellatlas/data/metadata/helpers/schema_validation.py b/src/humancellatlas/data/metadata/helpers/schema_validation.py index a6adc96e70..ba984e7a44 100644 --- a/src/humancellatlas/data/metadata/helpers/schema_validation.py +++ b/src/humancellatlas/data/metadata/helpers/schema_validation.py @@ -7,12 +7,19 @@ from jsonschema import ( FormatChecker, ValidationError, - validate, +) +from jsonschema.validators import ( + Draft202012Validator, +) +from referencing import ( + Registry, + Resource, ) import requests from azul import ( RequirementError, + cached_property, ) from azul.types import ( JSON, @@ -25,18 +32,28 @@ class SchemaValidator: def validate_json(self, file_json: JSON, file_name: str): try: - schema = self._download_schema(file_json['describedBy']) + schema = self._download_json_file(file_json['describedBy']) except json.decoder.JSONDecodeError as e: schema_url = file_json['describedBy'] raise RequirementError('Failed to parse schema JSON', file_name, schema_url) from e + self.validator.evolve(schema=schema) try: - validate(file_json, schema, format_checker=FormatChecker()) + self.validator.validate(file_json) except ValidationError as e: raise RequirementError(*e.args, file_name) from e @lru_cache(maxsize=None) - def _download_schema(self, schema_url: str) -> JSON: - response = requests.get(schema_url, allow_redirects=False) + def _download_json_file(self, file_url: str) -> JSON: + response = requests.get(file_url, allow_redirects=False) response.raise_for_status() return response.json() + + def _retrieve_resource(self, resource_url: str) -> Resource: + file_json = self._download_json_file(resource_url) + return Resource.from_contents(file_json) + + @cached_property + def validator(self) -> Draft202012Validator: + registry = Registry(retrieve=self._retrieve_resource) + return Draft202012Validator(schema={}, registry=registry, format_checker=FormatChecker()) diff --git a/terraform/gitlab/gitlab.tf.json.template.py b/terraform/gitlab/gitlab.tf.json.template.py index 87d513b50e..a757492297 100644 --- a/terraform/gitlab/gitlab.tf.json.template.py +++ b/terraform/gitlab/gitlab.tf.json.template.py @@ -1061,11 +1061,7 @@ def merge(sets: Iterable[Iterable[str]]) -> Iterable[str]: 'egress': [ vpc.security_rule(description='Any traffic to the ' f"{'VPC' if split_tunnel else 'internet'}", - cidr_blocks=[ - '${aws_vpc.gitlab.cidr_block}' - if split_tunnel else - all_ipv4 - ], + cidr_blocks=['${aws_vpc.gitlab.cidr_block}' if split_tunnel else all_ipv4], protocol=-1, from_port=0, to_port=0), diff --git a/test/azul_test_case.py b/test/azul_test_case.py index 3818e58e1b..4500725975 100644 --- a/test/azul_test_case.py +++ b/test/azul_test_case.py @@ -147,6 +147,11 @@ class RE(str): 'The \'body\' parameter is deprecated for the \'.*\' API ' 'and will be removed in .*. Instead use .*' ), + + # FIXME: DeprecationWarning for datetime methods in Python 3.12 + # https://github.com/DataBiosphere/azul/issues/5953 + 'datetime.datetime.utcnow() is deprecated', + 'datetime.datetime.utcfromtimestamp() is deprecated' }, UserWarning: { 'https://github.com/DataBiosphere/azul/issues/2114', diff --git a/test/integration_test.py b/test/integration_test.py index 005f8d3467..a9dcb558bd 100644 --- a/test/integration_test.py +++ b/test/integration_test.py @@ -78,7 +78,7 @@ only, ) from openapi_spec_validator import ( - validate_spec, + validate, ) import requests import urllib3 @@ -1752,7 +1752,7 @@ def test_openapi(self): response = requests.get(str(url)) response.raise_for_status() spec = response.json() - validate_spec(spec) + validate(spec) class AzulChaliceLocalIntegrationTest(AzulTestCase): diff --git a/test/service/test_manifest.py b/test/service/test_manifest.py index f57b88d012..542aebda06 100644 --- a/test/service/test_manifest.py +++ b/test/service/test_manifest.py @@ -1132,6 +1132,8 @@ def test_bdbag_manifest_remove_redundant_entries(self): Test BDBagManifestGenerator._remove_redundant_entries() directly with a large set of sample data """ + # FIXME: DeprecationWarning for datetime methods in Python 3.12 + # https://github.com/DataBiosphere/azul/issues/5953 now = datetime.utcnow() def v(i): diff --git a/test/service/test_manifest_async.py b/test/service/test_manifest_async.py index e328310a62..8a4a7346ed 100644 --- a/test/service/test_manifest_async.py +++ b/test/service/test_manifest_async.py @@ -336,9 +336,9 @@ def test(self, self.assertNotEqual(token_url, response.json()['Location']) assert signed_manifest_key.encode() == manifest_url.path.segments[-1] - assert verify_manifest_key.not_called + assert not verify_manifest_key.called verify_manifest_key.return_value = manifest_key - assert get_cached_manifest_with_key.not_called + assert not get_cached_manifest_with_key.called get_cached_manifest_with_key.return_value = manifest response = requests.get(str(manifest_url), allow_redirects=False) self.assertEqual(302, response.status_code) diff --git a/test/test_log_forwarding.py b/test/test_log_forwarding.py index 9994cee56d..48f524703d 100644 --- a/test/test_log_forwarding.py +++ b/test/test_log_forwarding.py @@ -153,8 +153,8 @@ def test_s3(self): '85', '52', dq('-'), - dq('Boto3/1.24.94 Python/3.11.5 Linux/4.14.255-301-238.520.amzn2.x86_64', - 'exec-env/AWS_Lambda_python3.11 aws-chalice/1.30.0 Botocore/1.27.94'), + dq('Boto3/1.24.94 Python/3.12.6 Linux/4.14.255-301-238.520.amzn2.x86_64', + 'exec-env/AWS_Lambda_python3.12 aws-chalice/1.31.2 Botocore/1.27.94'), '-', 'jcmyLMRqqJ7dT4ovtY21rtgwmuTC3qs24vgAtLAkcad9sRV92zC90gf2zGvCkxxsLSaKm48AMjo=', 'SigV4', @@ -193,8 +193,8 @@ def test_s3(self): 'tls_version': 'TLSv1.2', 'total_time': '85', 'turn_around_time': '52', - 'user_agent': 'Boto3/1.24.94 Python/3.11.5 Linux/4.14.255-301-238.520.amzn2.x86_64 ' - 'exec-env/AWS_Lambda_python3.11 aws-chalice/1.30.0 Botocore/1.27.94', + 'user_agent': 'Boto3/1.24.94 Python/3.12.6 Linux/4.14.255-301-238.520.amzn2.x86_64 ' + 'exec-env/AWS_Lambda_python3.12 aws-chalice/1.31.2 Botocore/1.27.94', 'version_id': '-', }] self._test(self.controller.forward_s3_access_logs, input, expected_output)