diff --git a/.circleci/config.yml b/.circleci/config.yml index dc8d4debd00..f368c7f0ecc 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -219,7 +219,7 @@ jobs: deploy: docker: # The primary container is an instance of the first list image listed. Your build commands run in this container. - - image: circleci/ruby:2.4 + - image: circleci/ruby:2.6 working_directory: /tmp/deploy environment: - DISTROS: "bionic focal el7 el8" diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index fb052f79288..30873241898 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -70,11 +70,19 @@ jobs: task: 'ci-compile' python-version-short: '3.8' python-version: '3.8.10' + - name: 'Lint Checks (black, flake8, etc.)' + task: 'ci-checks' + python-version-short: '3.9' + python-version: '3.9.14' + - name: 'Compile (pip deps, pylint, etc.)' + task: 'ci-compile' + python-version-short: '3.9' + python-version: '3.9.14' env: TASK: '${{ matrix.task }}' COLUMNS: '120' - PYLINT_CONCURRENCY: '4' + PYLINT_CONCURRENCY: '6' steps: - name: Checkout repository uses: actions/checkout@v2 @@ -136,6 +144,160 @@ jobs: run: | ./scripts/ci/run-nightly-make-task-if-exists.sh "${TASK}" + self-check: + needs: pre_job + name: '${{ matrix.name }} - Python ${{ matrix.python-version-short }}' + runs-on: ubuntu-20.04 + strategy: + fail-fast: false + matrix: + include: + # TODO: Check if we want to fix the errors on Py 3.6 to have it tested as well + #- name: 'Self-check on Python 3.6' + # python-version-short: '3.6' + # python-version: '3.6.13' + - name: 'Self-check on Python 3.8' + python-version-short: '3.8' + python-version: '3.8.14' + services: + mongo: + image: mongo:4.4 + ports: + - 27017:27017 + + rabbitmq: + image: rabbitmq:3.8-management + options: >- + --name rabbitmq + ports: + - 5671:5671/tcp # AMQP SSL port + - 5672:5672/tcp # AMQP standard port + - 15672:15672/tcp # Management: HTTP, CLI + + env: + # CI st2.conf (with ST2_CI_USER user instead of stanley) + ST2_CONF: 'conf/st2.ci.conf' + + # Name of the user who is running the CI (on GitHub Actions this is 'runner') + ST2_CI_USER: 'runner' + + # GitHub is juggling how to set vars for multiple shells. Protect our PATH assumptions. + PATH: /home/runner/.local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + + # Space separated list of tests to be skipped if the self-check is running in GitHub Actions + TESTS_TO_SKIP: "tests.test_quickstart_rules tests.test_run_pack_tests_tool" + steps: + - name: Checkout repository + uses: actions/checkout@v2 + - name: Custom Environment Setup + run: | + ./scripts/github/setup-environment.sh + - name: 'Set up Python (${{ matrix.python-version }})' + uses: actions/setup-python@v2 + with: + python-version: '${{ matrix.python-version }}' + - name: Cache Python Dependencies + uses: actions/cache@v2 + with: + path: | + ~/.cache/pip + virtualenv + ~/virtualenv + # TODO: maybe make the virtualenv a partial cache to exclude st2*? + # !virtualenv/lib/python*/site-packages/st2* + # !virtualenv/bin/st2* + key: ${{ runner.os }}-v3-python-${{ matrix.python-version }}-${{ hashFiles('requirements.txt', 'test-requirements.txt') }} + restore-keys: | + ${{ runner.os }}-python-${{ matrix.python }}- + - name: Cache APT Dependencies + id: cache-apt-deps + uses: actions/cache@v2 + with: + path: | + ~/apt_cache + key: ${{ runner.os }}-apt-v5-${{ hashFiles('scripts/github/apt-packages.txt') }} + restore-keys: | + ${{ runner.os }}-apt-v5- + - name: Install APT Depedencies + env: + CACHE_HIT: ${{steps.cache-apt-deps.outputs.cache-hit}} + run: | + cat /etc/environment + # install dev dependencies for Python YAML and LDAP packages + # https://github.com/StackStorm/st2-auth-ldap + ./scripts/github/install-apt-packages-use-cache.sh + - name: Install virtualenv + run: | + ./scripts/github/install-virtualenv.sh + - name: Install requirements + run: | + ./scripts/ci/install-requirements.sh + - name: Run Redis Service Container + timeout-minutes: 2 + run: | + docker run --rm --detach -p 127.0.0.1:6379:6379/tcp --name redis redis:latest + until [ "$(docker inspect -f {{.State.Running}} redis)" == "true" ]; do sleep 0.1; done + - name: Setup Tests + run: | + # prep a ci-specific dev conf file that uses runner instead of stanley + # this user is the username of the user in GitHub actions, used for SSH, etc during + # integration tests (important) + cp conf/st2.dev.conf "${ST2_CONF}" + sed -i -e "s,/home/vagrant/.ssh/stanley_rsa,/home/stanley/.ssh/stanley_rsa," "${ST2_CONF}" + + sudo -E ./scripts/ci/add-itest-user-key.sh + - name: Permissions Workaround + run: | + sudo ST2_CI_REPO_PATH="${ST2_CI_REPO_PATH}" scripts/ci/permissions-workaround.sh + - name: Reconfigure RabbitMQ + # bitnami image allows (see bitnami/rabbitmq readme): + # Here we're copying a rabbitmq.config file which won't do anything. + # We need to switch to custom.conf or advanced.config. + timeout-minutes: 2 # may die if rabbitmq fails to start + run: | + ./scripts/github/configure-rabbitmq.sh + - name: Print versions + run: | + ./scripts/ci/print-versions.sh + - name: make + timeout-minutes: 14 # may die if rabbitmq fails to start + # use: script -e -c to print colors + run: | + script -e -c "make .ci-prepare-integration" && exit 0 + - name: Extend the path for upcoming tasks + run: | + echo ${HOME}/work/st2/st2/virtualenv/bin + echo ${HOME}/work/st2/st2/virtualenv/bin >> $GITHUB_PATH + - name: Create symlinks to find the binaries when running st2 actions + run: | + ln -s ${HOME}/work/st2/st2/virtualenv/bin/st2 /usr/local/bin/st2 + ln -s ${HOME}/work/st2/st2/virtualenv/bin/st2-run-pack-tests /usr/local/bin/st2-run-pack-tests + - name: Install st2client + timeout-minutes: 5 + run: | + cd ./st2client + pip3 install --upgrade pip + python3 setup.py develop + - name: Run self-verification script + env: + ST2_CONF: /home/runner/work/st2/st2/conf/st2.ci.conf + run: | + sudo -E ST2_AUTH_TOKEN=$(st2 auth testu -p 'testp' -t) PATH=${PATH} virtualenv/bin/st2-self-check + - name: Compress Service Logs Before upload + if: ${{ failure() }} + run: | + tar cvzpf logs.tar.gz logs/* + - name: Upload StackStorm services Logs + if: ${{ failure() }} + uses: actions/upload-artifact@v2 + with: + name: logs + path: logs.tar.gz + retention-days: 7 + - name: Stop Redis Service Container + if: "${{ always() }}" + run: docker rm --force redis || true + unit-tests: needs: pre_job # NOTE: We always want to run job on master since we run some additional checks there (code @@ -174,6 +336,18 @@ jobs: nosetests_node_index: 1 python-version-short: '3.8' python-version: '3.8.10' + - name: 'Unit Tests (chunk 1)' + task: 'ci-unit' + nosetests_node_total: 2 + nosetests_node_index: 0 + python-version-short: '3.9' + python-version: '3.9.14' + - name: 'Unit Tests (chunk 2)' + task: 'ci-unit' + nosetests_node_total: 2 + nosetests_node_index: 1 + python-version-short: '3.9' + python-version: '3.9.14' # This job is slow so we only run in on a daily basis # - name: 'Micro Benchmarks' # task: 'micro-benchmarks' @@ -197,7 +371,7 @@ jobs: env: TASK: '${{ matrix.task }}' - + PYTHON_VERSION_SHORT: '${{ matrix.python-version-short }}' NODE_TOTAL: '${{ matrix.nosetests_node_total }}' NODE_INDEX: '${{ matrix.nosetests_node_index }}' @@ -304,9 +478,11 @@ jobs: ./scripts/ci/run-nightly-make-task-if-exists.sh "${TASK}" - name: Codecov # NOTE: We only generate and submit coverage report for master and version branches and only when the build succeeds (default on GitHub Actions, this was not the case on Travis so we had to explicitly check success) - if: "${{ success() && (env.ENABLE_COVERAGE == 'yes') }}" + if: "${{ success() && (env.ENABLE_COVERAGE == 'yes') && (env.PYTHON_VERSION_SHORT == '3.8')}}" run: | ./scripts/ci/submit-codecov-coverage.sh + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} integration-tests: needs: pre_job @@ -359,6 +535,24 @@ jobs: nosetests_node_index: 1 python-version-short: '3.8' python-version: '3.8.10' + - name: 'Pack Tests' + task: 'ci-packs-tests' + nosetests_node_total: 1 + nosetests_node_index: 0 + python-version-short: '3.9' + python-version: '3.9.14' + - name: 'Integration Tests (chunk 1)' + task: 'ci-integration' + nosetests_node_total: 2 + nosetests_node_index: 0 + python-version-short: '3.9' + python-version: '3.9.14' + - name: 'Integration Tests (chunk 2)' + task: 'ci-integration' + nosetests_node_total: 2 + nosetests_node_index: 1 + python-version-short: '3.9' + python-version: '3.9.14' services: mongo: image: mongo:4.4 @@ -414,7 +608,7 @@ jobs: env: TASK: '${{ matrix.task }}' - + PYTHON_VERSION_SHORT: '${{ matrix.python-version-short }}' NODE_TOTAL: '${{ matrix.nosetests_node_total }}' NODE_INDEX: '${{ matrix.nosetests_node_index }}' @@ -523,9 +717,11 @@ jobs: script -e -c "make ${TASK}" && exit 0 - name: Codecov # NOTE: We only generate and submit coverage report for master and version branches and only when the build succeeds (default on GitHub Actions, this was not the case on Travis so we had to explicitly check success) - if: "${{ success() && env.ENABLE_COVERAGE == 'yes' && env.TASK == 'ci-integration' }}" + if: "${{ success() && (env.ENABLE_COVERAGE == 'yes') && (env.TASK == 'ci-integration') && (env.PYTHON_VERSION_SHORT == '3.8')}}" run: | ./scripts/ci/submit-codecov-coverage.sh + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} - name: Compress Service Logs Before upload if: ${{ failure() && env.TASK == 'ci-integration' }} run: | diff --git a/.github/workflows/microbenchmarks.yaml b/.github/workflows/microbenchmarks.yaml index fcc7b5cca02..9477b256f84 100644 --- a/.github/workflows/microbenchmarks.yaml +++ b/.github/workflows/microbenchmarks.yaml @@ -46,6 +46,12 @@ jobs: nosetests_node_index: 0 python-version-short: '3.8' python-version: '3.8.10' + - name: 'Microbenchmarks' + task: 'micro-benchmarks' + nosetests_node_total: 1 + nosetests_node_index: 0 + python-version-short: '3.9' + python-version: '3.9.14' services: mongo: image: mongo:4.4 diff --git a/.github/workflows/orquesta-integration-tests.yaml b/.github/workflows/orquesta-integration-tests.yaml index 5e0a4ebeef9..b45dd5fb843 100644 --- a/.github/workflows/orquesta-integration-tests.yaml +++ b/.github/workflows/orquesta-integration-tests.yaml @@ -67,6 +67,12 @@ jobs: nosetests_node_index: 0 python-version-short: '3.8' python-version: '3.8.10' + - name: 'Integration Tests (Orquesta)' + task: 'ci-orquesta' + nosetests_node_total: 1 + nosetests_node_index: 0 + python-version-short: '3.9' + python-version: '3.9.14' services: mongo: image: mongo:4.4 @@ -100,7 +106,6 @@ jobs: env: TASK: '${{ matrix.task }}' - NODE_TOTAL: '${{ matrix.nosetests_node_total }}' NODE_INDEX: '${{ matrix.nosetests_node_index }}' @@ -218,11 +223,6 @@ jobs: with: name: logs path: logs/ - - name: Codecov - # NOTE: We only generate and submit coverage report for master and version branches and only when the build succeeds (default on GitHub Actions, this was not the case on Travis so we had to explicitly check success) - if: "${{ success() && env.ENABLE_COVERAGE == 'yes' }}" - run: | - ./scripts/ci/submit-codecov-coverage.sh - name: Compress Service Logs Before upload if: ${{ failure() }} run: | diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 334479b702c..172c0cd64d0 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -38,6 +38,9 @@ jobs: - name: 'Test (pants runs: pytest)' python-version-short: '3.8' python-version: '3.8.10' + - name: 'Test (pants runs: pytest)' + python-version-short: '3.9' + python-version: '3.9.14' services: mongo: diff --git a/.gitignore b/.gitignore index 555d276e698..090159801f5 100644 --- a/.gitignore +++ b/.gitignore @@ -70,3 +70,5 @@ benchmark_histograms/ [._]sw[a-px] [._]*.sw[a-p]x [._]sw[a-p]x + +**/build/lib/** \ No newline at end of file diff --git a/ADOPTERS.md b/ADOPTERS.md index c467d4fbd1e..508bec47f69 100644 --- a/ADOPTERS.md +++ b/ADOPTERS.md @@ -1,3 +1,6 @@ +# Who uses StackStorm? +As the StackStorm Community evolves, we'd like to keep track of our users. Please submit a PR with your organization and a brief use case description below. + This is an alphabetical list of known [StackStorm](https://stackstorm.com/) adopters: diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 22035c0c410..05c1d144d08 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -6,6 +6,14 @@ in development Fixed ~~~~~ +* Additional fixes for st2 client auth when proxy auth mode enabled #6049 + Contributed by @floatingstatic + +* Fix issue with linux pack actions failed to run remotely due to incorrect python shebang. #5983 #6042 + Contributed by Ronnie Hoffmann (@ZoeLeah Schwarz IT KG) + +* Fix proxy auth mode in HA environments #5766 + Contributed by @floatingstatic * Fix CI usses #6015 Contributed by Amanda McGuinness (@amanda11 intive) @@ -16,8 +24,20 @@ Fixed * Fixed RBAC support to action-alias help and match end point. #6022 Contributed by @nzlosh +* Avoid logging sensitive information in debug (fix #5977) + +* Fix codecov failures for stackstorm/st2 tests. #6035, #6046, #6048 + +* Fix #4676, edge case where --inherit-env is skipped if the action has no parameters + +* Update cryptography 3.4.7 -> 39.0.1, pyOpenSSL 21.0.0 -> 23.1.0, paramiko 2.10.5 -> 2.11.0 (security). #6055 + +* Bumped `eventlet` to `0.33.3` and `gunicorn` to `21.2.0` to fix `RecursionError` bug in setting `SSLContext` `minimum_version` property. #6061 + Contributed by @jk464 + Added ~~~~~ + * Move `git clone` to `user_home/.st2packs` #5845 * Error on `st2ctl status` when running in Kubernetes. #5851 @@ -40,6 +60,18 @@ Added * Expose environment variable ST2_ACTION_DEBUG to all StackStorm actions. Contributed by @maxfactor1 +* Python 3.9 support. #5730 + Contributed by Amanda McGuinness (@amanda11 intive) + +* Run the st2 self-check in Github Actions and support the environment variable `TESTS_TO_SKIP` to skip tests when running st2-self-check. #5609 + + Contributed by @winem + +Changed +~~~~~~~ +* Remove `distutils` dependencies across the project. #5992 + Contributed by @AndroxxTraxxon + 3.8.0 - November 18, 2022 ------------------------- diff --git a/.codecov.yml b/codecov.yml similarity index 100% rename from .codecov.yml rename to codecov.yml diff --git a/conf/HA/nginx/st2.conf.blueprint.sample b/conf/HA/nginx/st2.conf.blueprint.sample index 91f3cdafd60..482118f6a89 100644 --- a/conf/HA/nginx/st2.conf.blueprint.sample +++ b/conf/HA/nginx/st2.conf.blueprint.sample @@ -96,6 +96,7 @@ server { proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-User $remote_user; proxy_pass_header Authorization; proxy_set_header Connection ''; diff --git a/conf/HA/nginx/st2.conf.controller.sample b/conf/HA/nginx/st2.conf.controller.sample index b7108f56c8d..9bb721cc802 100644 --- a/conf/HA/nginx/st2.conf.controller.sample +++ b/conf/HA/nginx/st2.conf.controller.sample @@ -118,6 +118,7 @@ server { proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-User $remote_user; proxy_pass_header Authorization; proxy_set_header Connection ''; diff --git a/conf/nginx/st2.conf b/conf/nginx/st2.conf index fc2243068a5..6e83d18cbea 100644 --- a/conf/nginx/st2.conf +++ b/conf/nginx/st2.conf @@ -146,6 +146,7 @@ server { proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-User $remote_user; proxy_pass_header Authorization; proxy_set_header Connection ''; diff --git a/contrib/debug/actions/print_ctx.py b/contrib/debug/actions/print_ctx.py index 99bb00d3190..42d6e3a775d 100644 --- a/contrib/debug/actions/print_ctx.py +++ b/contrib/debug/actions/print_ctx.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # Copyright 2020 The StackStorm Authors. # diff --git a/contrib/debug/actions/python_version.py b/contrib/debug/actions/python_version.py index ed044db40de..927361f7234 100644 --- a/contrib/debug/actions/python_version.py +++ b/contrib/debug/actions/python_version.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # Copyright 2020 The StackStorm Authors. # diff --git a/contrib/debug/actions/pythonpath.py b/contrib/debug/actions/pythonpath.py index 1af41a558a4..82a2f4e4807 100644 --- a/contrib/debug/actions/pythonpath.py +++ b/contrib/debug/actions/pythonpath.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # Copyright 2020 The StackStorm Authors. # diff --git a/contrib/debug/pack.yaml b/contrib/debug/pack.yaml index 41dd36b3d68..d864ac685c1 100644 --- a/contrib/debug/pack.yaml +++ b/contrib/debug/pack.yaml @@ -3,6 +3,6 @@ description: Debug utilities for StackStorm ref: debug author: StackStorm Authors email: info@stackstorm.com -version: "0.0.1" +version: "0.0.2" python_versions: - "3" diff --git a/contrib/linux/actions/checks/check_loadavg.py b/contrib/linux/actions/checks/check_loadavg.py index 41de2774cf7..04036924e81 100755 --- a/contrib/linux/actions/checks/check_loadavg.py +++ b/contrib/linux/actions/checks/check_loadavg.py @@ -1,4 +1,4 @@ -#!/opt/stackstorm/st2/bin/python +#!/usr/bin/env python # Copyright 2020 The StackStorm Authors. # Copyright 2019 Extreme Networks, Inc. diff --git a/contrib/linux/actions/checks/check_processes.py b/contrib/linux/actions/checks/check_processes.py index 4a7d9682c12..a32e009fec4 100755 --- a/contrib/linux/actions/checks/check_processes.py +++ b/contrib/linux/actions/checks/check_processes.py @@ -1,4 +1,4 @@ -#!/opt/stackstorm/st2/bin/python +#!/usr/bin/env python # Copyright 2020 The StackStorm Authors. # Copyright 2019 Extreme Networks, Inc. diff --git a/contrib/linux/actions/dig.py b/contrib/linux/actions/dig.py index c20f28259eb..737ac30133d 100644 --- a/contrib/linux/actions/dig.py +++ b/contrib/linux/actions/dig.py @@ -1,4 +1,4 @@ -#! /usr/bin/python +#!/usr/bin/env python # Copyright 2020 The StackStorm Authors. # Copyright 2019 Extreme Networks, Inc. diff --git a/contrib/linux/actions/service.py b/contrib/linux/actions/service.py index 743ffdef786..0226adeef72 100644 --- a/contrib/linux/actions/service.py +++ b/contrib/linux/actions/service.py @@ -1,4 +1,4 @@ -#!/opt/stackstorm/st2/bin/python +#!/usr/bin/env python # Copyright 2020 The StackStorm Authors. # Copyright 2019 Extreme Networks, Inc. diff --git a/contrib/linux/pack.yaml b/contrib/linux/pack.yaml index 1ca97e01b9e..c611cf950ee 100644 --- a/contrib/linux/pack.yaml +++ b/contrib/linux/pack.yaml @@ -16,7 +16,7 @@ keywords: - open ports - processes - ps -version : 1.2.0 +version : 1.2.1 python_versions: - "2" - "3" diff --git a/contrib/runners/action_chain_runner/dist_utils.py b/contrib/runners/action_chain_runner/dist_utils.py index a4e9862d2bc..fa752580a2c 100644 --- a/contrib/runners/action_chain_runner/dist_utils.py +++ b/contrib/runners/action_chain_runner/dist_utils.py @@ -24,8 +24,6 @@ import re import sys -from distutils.version import StrictVersion - # NOTE: This script can't rely on any 3rd party dependency so we need to use this code here # # TODO: Why can't this script rely on 3rd party dependencies? Is it because it has to import @@ -48,8 +46,6 @@ GET_PIP = "curl https://bootstrap.pypa.io/get-pip.py | python" __all__ = [ - "check_pip_is_installed", - "check_pip_version", "fetch_requirements", "apply_vagrant_workaround", "get_version_string", @@ -57,41 +53,6 @@ ] -def check_pip_is_installed(): - """ - Ensure that pip is installed. - """ - try: - import pip # NOQA - except ImportError as e: - print("Failed to import pip: %s" % (text_type(e))) - print("") - print("Download pip:\n%s" % (GET_PIP)) - sys.exit(1) - - return True - - -def check_pip_version(min_version="6.0.0"): - """ - Ensure that a minimum supported version of pip is installed. - """ - check_pip_is_installed() - - import pip - - if StrictVersion(pip.__version__) < StrictVersion(min_version): - print( - "Upgrade pip, your version '{0}' " - "is outdated. Minimum required version is '{1}':\n{2}".format( - pip.__version__, min_version, GET_PIP - ) - ) - sys.exit(1) - - return True - - def fetch_requirements(requirements_file_path): """ Return a list of requirements and links by parsing the provided requirements file. diff --git a/contrib/runners/announcement_runner/dist_utils.py b/contrib/runners/announcement_runner/dist_utils.py index a4e9862d2bc..fa752580a2c 100644 --- a/contrib/runners/announcement_runner/dist_utils.py +++ b/contrib/runners/announcement_runner/dist_utils.py @@ -24,8 +24,6 @@ import re import sys -from distutils.version import StrictVersion - # NOTE: This script can't rely on any 3rd party dependency so we need to use this code here # # TODO: Why can't this script rely on 3rd party dependencies? Is it because it has to import @@ -48,8 +46,6 @@ GET_PIP = "curl https://bootstrap.pypa.io/get-pip.py | python" __all__ = [ - "check_pip_is_installed", - "check_pip_version", "fetch_requirements", "apply_vagrant_workaround", "get_version_string", @@ -57,41 +53,6 @@ ] -def check_pip_is_installed(): - """ - Ensure that pip is installed. - """ - try: - import pip # NOQA - except ImportError as e: - print("Failed to import pip: %s" % (text_type(e))) - print("") - print("Download pip:\n%s" % (GET_PIP)) - sys.exit(1) - - return True - - -def check_pip_version(min_version="6.0.0"): - """ - Ensure that a minimum supported version of pip is installed. - """ - check_pip_is_installed() - - import pip - - if StrictVersion(pip.__version__) < StrictVersion(min_version): - print( - "Upgrade pip, your version '{0}' " - "is outdated. Minimum required version is '{1}':\n{2}".format( - pip.__version__, min_version, GET_PIP - ) - ) - sys.exit(1) - - return True - - def fetch_requirements(requirements_file_path): """ Return a list of requirements and links by parsing the provided requirements file. diff --git a/contrib/runners/http_runner/dist_utils.py b/contrib/runners/http_runner/dist_utils.py index a4e9862d2bc..fa752580a2c 100644 --- a/contrib/runners/http_runner/dist_utils.py +++ b/contrib/runners/http_runner/dist_utils.py @@ -24,8 +24,6 @@ import re import sys -from distutils.version import StrictVersion - # NOTE: This script can't rely on any 3rd party dependency so we need to use this code here # # TODO: Why can't this script rely on 3rd party dependencies? Is it because it has to import @@ -48,8 +46,6 @@ GET_PIP = "curl https://bootstrap.pypa.io/get-pip.py | python" __all__ = [ - "check_pip_is_installed", - "check_pip_version", "fetch_requirements", "apply_vagrant_workaround", "get_version_string", @@ -57,41 +53,6 @@ ] -def check_pip_is_installed(): - """ - Ensure that pip is installed. - """ - try: - import pip # NOQA - except ImportError as e: - print("Failed to import pip: %s" % (text_type(e))) - print("") - print("Download pip:\n%s" % (GET_PIP)) - sys.exit(1) - - return True - - -def check_pip_version(min_version="6.0.0"): - """ - Ensure that a minimum supported version of pip is installed. - """ - check_pip_is_installed() - - import pip - - if StrictVersion(pip.__version__) < StrictVersion(min_version): - print( - "Upgrade pip, your version '{0}' " - "is outdated. Minimum required version is '{1}':\n{2}".format( - pip.__version__, min_version, GET_PIP - ) - ) - sys.exit(1) - - return True - - def fetch_requirements(requirements_file_path): """ Return a list of requirements and links by parsing the provided requirements file. diff --git a/contrib/runners/inquirer_runner/dist_utils.py b/contrib/runners/inquirer_runner/dist_utils.py index a4e9862d2bc..fa752580a2c 100644 --- a/contrib/runners/inquirer_runner/dist_utils.py +++ b/contrib/runners/inquirer_runner/dist_utils.py @@ -24,8 +24,6 @@ import re import sys -from distutils.version import StrictVersion - # NOTE: This script can't rely on any 3rd party dependency so we need to use this code here # # TODO: Why can't this script rely on 3rd party dependencies? Is it because it has to import @@ -48,8 +46,6 @@ GET_PIP = "curl https://bootstrap.pypa.io/get-pip.py | python" __all__ = [ - "check_pip_is_installed", - "check_pip_version", "fetch_requirements", "apply_vagrant_workaround", "get_version_string", @@ -57,41 +53,6 @@ ] -def check_pip_is_installed(): - """ - Ensure that pip is installed. - """ - try: - import pip # NOQA - except ImportError as e: - print("Failed to import pip: %s" % (text_type(e))) - print("") - print("Download pip:\n%s" % (GET_PIP)) - sys.exit(1) - - return True - - -def check_pip_version(min_version="6.0.0"): - """ - Ensure that a minimum supported version of pip is installed. - """ - check_pip_is_installed() - - import pip - - if StrictVersion(pip.__version__) < StrictVersion(min_version): - print( - "Upgrade pip, your version '{0}' " - "is outdated. Minimum required version is '{1}':\n{2}".format( - pip.__version__, min_version, GET_PIP - ) - ) - sys.exit(1) - - return True - - def fetch_requirements(requirements_file_path): """ Return a list of requirements and links by parsing the provided requirements file. diff --git a/contrib/runners/local_runner/dist_utils.py b/contrib/runners/local_runner/dist_utils.py index a4e9862d2bc..fa752580a2c 100644 --- a/contrib/runners/local_runner/dist_utils.py +++ b/contrib/runners/local_runner/dist_utils.py @@ -24,8 +24,6 @@ import re import sys -from distutils.version import StrictVersion - # NOTE: This script can't rely on any 3rd party dependency so we need to use this code here # # TODO: Why can't this script rely on 3rd party dependencies? Is it because it has to import @@ -48,8 +46,6 @@ GET_PIP = "curl https://bootstrap.pypa.io/get-pip.py | python" __all__ = [ - "check_pip_is_installed", - "check_pip_version", "fetch_requirements", "apply_vagrant_workaround", "get_version_string", @@ -57,41 +53,6 @@ ] -def check_pip_is_installed(): - """ - Ensure that pip is installed. - """ - try: - import pip # NOQA - except ImportError as e: - print("Failed to import pip: %s" % (text_type(e))) - print("") - print("Download pip:\n%s" % (GET_PIP)) - sys.exit(1) - - return True - - -def check_pip_version(min_version="6.0.0"): - """ - Ensure that a minimum supported version of pip is installed. - """ - check_pip_is_installed() - - import pip - - if StrictVersion(pip.__version__) < StrictVersion(min_version): - print( - "Upgrade pip, your version '{0}' " - "is outdated. Minimum required version is '{1}':\n{2}".format( - pip.__version__, min_version, GET_PIP - ) - ) - sys.exit(1) - - return True - - def fetch_requirements(requirements_file_path): """ Return a list of requirements and links by parsing the provided requirements file. diff --git a/contrib/runners/noop_runner/dist_utils.py b/contrib/runners/noop_runner/dist_utils.py index a4e9862d2bc..fa752580a2c 100644 --- a/contrib/runners/noop_runner/dist_utils.py +++ b/contrib/runners/noop_runner/dist_utils.py @@ -24,8 +24,6 @@ import re import sys -from distutils.version import StrictVersion - # NOTE: This script can't rely on any 3rd party dependency so we need to use this code here # # TODO: Why can't this script rely on 3rd party dependencies? Is it because it has to import @@ -48,8 +46,6 @@ GET_PIP = "curl https://bootstrap.pypa.io/get-pip.py | python" __all__ = [ - "check_pip_is_installed", - "check_pip_version", "fetch_requirements", "apply_vagrant_workaround", "get_version_string", @@ -57,41 +53,6 @@ ] -def check_pip_is_installed(): - """ - Ensure that pip is installed. - """ - try: - import pip # NOQA - except ImportError as e: - print("Failed to import pip: %s" % (text_type(e))) - print("") - print("Download pip:\n%s" % (GET_PIP)) - sys.exit(1) - - return True - - -def check_pip_version(min_version="6.0.0"): - """ - Ensure that a minimum supported version of pip is installed. - """ - check_pip_is_installed() - - import pip - - if StrictVersion(pip.__version__) < StrictVersion(min_version): - print( - "Upgrade pip, your version '{0}' " - "is outdated. Minimum required version is '{1}':\n{2}".format( - pip.__version__, min_version, GET_PIP - ) - ) - sys.exit(1) - - return True - - def fetch_requirements(requirements_file_path): """ Return a list of requirements and links by parsing the provided requirements file. diff --git a/contrib/runners/orquesta_runner/dist_utils.py b/contrib/runners/orquesta_runner/dist_utils.py index a4e9862d2bc..fa752580a2c 100644 --- a/contrib/runners/orquesta_runner/dist_utils.py +++ b/contrib/runners/orquesta_runner/dist_utils.py @@ -24,8 +24,6 @@ import re import sys -from distutils.version import StrictVersion - # NOTE: This script can't rely on any 3rd party dependency so we need to use this code here # # TODO: Why can't this script rely on 3rd party dependencies? Is it because it has to import @@ -48,8 +46,6 @@ GET_PIP = "curl https://bootstrap.pypa.io/get-pip.py | python" __all__ = [ - "check_pip_is_installed", - "check_pip_version", "fetch_requirements", "apply_vagrant_workaround", "get_version_string", @@ -57,41 +53,6 @@ ] -def check_pip_is_installed(): - """ - Ensure that pip is installed. - """ - try: - import pip # NOQA - except ImportError as e: - print("Failed to import pip: %s" % (text_type(e))) - print("") - print("Download pip:\n%s" % (GET_PIP)) - sys.exit(1) - - return True - - -def check_pip_version(min_version="6.0.0"): - """ - Ensure that a minimum supported version of pip is installed. - """ - check_pip_is_installed() - - import pip - - if StrictVersion(pip.__version__) < StrictVersion(min_version): - print( - "Upgrade pip, your version '{0}' " - "is outdated. Minimum required version is '{1}':\n{2}".format( - pip.__version__, min_version, GET_PIP - ) - ) - sys.exit(1) - - return True - - def fetch_requirements(requirements_file_path): """ Return a list of requirements and links by parsing the provided requirements file. diff --git a/contrib/runners/python_runner/dist_utils.py b/contrib/runners/python_runner/dist_utils.py index a4e9862d2bc..fa752580a2c 100644 --- a/contrib/runners/python_runner/dist_utils.py +++ b/contrib/runners/python_runner/dist_utils.py @@ -24,8 +24,6 @@ import re import sys -from distutils.version import StrictVersion - # NOTE: This script can't rely on any 3rd party dependency so we need to use this code here # # TODO: Why can't this script rely on 3rd party dependencies? Is it because it has to import @@ -48,8 +46,6 @@ GET_PIP = "curl https://bootstrap.pypa.io/get-pip.py | python" __all__ = [ - "check_pip_is_installed", - "check_pip_version", "fetch_requirements", "apply_vagrant_workaround", "get_version_string", @@ -57,41 +53,6 @@ ] -def check_pip_is_installed(): - """ - Ensure that pip is installed. - """ - try: - import pip # NOQA - except ImportError as e: - print("Failed to import pip: %s" % (text_type(e))) - print("") - print("Download pip:\n%s" % (GET_PIP)) - sys.exit(1) - - return True - - -def check_pip_version(min_version="6.0.0"): - """ - Ensure that a minimum supported version of pip is installed. - """ - check_pip_is_installed() - - import pip - - if StrictVersion(pip.__version__) < StrictVersion(min_version): - print( - "Upgrade pip, your version '{0}' " - "is outdated. Minimum required version is '{1}':\n{2}".format( - pip.__version__, min_version, GET_PIP - ) - ) - sys.exit(1) - - return True - - def fetch_requirements(requirements_file_path): """ Return a list of requirements and links by parsing the provided requirements file. diff --git a/contrib/runners/python_runner/python_runner/python_action_wrapper.py b/contrib/runners/python_runner/python_runner/python_action_wrapper.py index 795b0114a85..d769b68d8ab 100644 --- a/contrib/runners/python_runner/python_runner/python_action_wrapper.py +++ b/contrib/runners/python_runner/python_runner/python_action_wrapper.py @@ -15,18 +15,18 @@ from __future__ import absolute_import -# Ignore CryptographyDeprecationWarning warnings which appear on older versions of Python 2.7 +# Ignore CryptographyDeprecationWarning warnings which appear on Python 3.6 +# TODO: Remove after dropping python3.6 import warnings -from cryptography.utils import CryptographyDeprecationWarning -warnings.filterwarnings("ignore", category=CryptographyDeprecationWarning) +warnings.filterwarnings("ignore", message="Python 3.6 is no longer supported") import os import sys import select import traceback -import distutils.sysconfig +import sysconfig # NOTE: We intentionally use orjson directly here instead of json_encode - orjson.dumps relies # on config option which we don't parse for the action wrapper since it speeds things down - action @@ -49,7 +49,7 @@ # This puts priority on loading virtualenv library in the pack's action. This is necessary # for the situation that both st2 and pack require to load same name libraries with different # version. Without this statement, action may call library method with unexpected dependencies. - sys.path.insert(0, distutils.sysconfig.get_python_lib()) + sys.path.insert(0, sysconfig.get_path("platlib")) import sys import argparse diff --git a/contrib/runners/python_runner/tests/integration/test_python_action_process_wrapper.py b/contrib/runners/python_runner/tests/integration/test_python_action_process_wrapper.py index c97a380fb23..da939f4aae3 100644 --- a/contrib/runners/python_runner/tests/integration/test_python_action_process_wrapper.py +++ b/contrib/runners/python_runner/tests/integration/test_python_action_process_wrapper.py @@ -37,7 +37,7 @@ import json import unittest2 -from distutils.spawn import find_executable +from shutil import which as shutil_which from st2common.util.shell import run_command from six.moves import range @@ -61,7 +61,7 @@ BASE_DIR, "../../../python_runner/python_runner/python_action_wrapper.py" ) WRAPPER_SCRIPT_PATH = os.path.abspath(WRAPPER_SCRIPT_PATH) -TIME_BINARY_PATH = find_executable("time") +TIME_BINARY_PATH = shutil_which("time") TIME_BINARY_AVAILABLE = TIME_BINARY_PATH is not None diff --git a/contrib/runners/remote_runner/dist_utils.py b/contrib/runners/remote_runner/dist_utils.py index a4e9862d2bc..fa752580a2c 100644 --- a/contrib/runners/remote_runner/dist_utils.py +++ b/contrib/runners/remote_runner/dist_utils.py @@ -24,8 +24,6 @@ import re import sys -from distutils.version import StrictVersion - # NOTE: This script can't rely on any 3rd party dependency so we need to use this code here # # TODO: Why can't this script rely on 3rd party dependencies? Is it because it has to import @@ -48,8 +46,6 @@ GET_PIP = "curl https://bootstrap.pypa.io/get-pip.py | python" __all__ = [ - "check_pip_is_installed", - "check_pip_version", "fetch_requirements", "apply_vagrant_workaround", "get_version_string", @@ -57,41 +53,6 @@ ] -def check_pip_is_installed(): - """ - Ensure that pip is installed. - """ - try: - import pip # NOQA - except ImportError as e: - print("Failed to import pip: %s" % (text_type(e))) - print("") - print("Download pip:\n%s" % (GET_PIP)) - sys.exit(1) - - return True - - -def check_pip_version(min_version="6.0.0"): - """ - Ensure that a minimum supported version of pip is installed. - """ - check_pip_is_installed() - - import pip - - if StrictVersion(pip.__version__) < StrictVersion(min_version): - print( - "Upgrade pip, your version '{0}' " - "is outdated. Minimum required version is '{1}':\n{2}".format( - pip.__version__, min_version, GET_PIP - ) - ) - sys.exit(1) - - return True - - def fetch_requirements(requirements_file_path): """ Return a list of requirements and links by parsing the provided requirements file. diff --git a/contrib/runners/winrm_runner/dist_utils.py b/contrib/runners/winrm_runner/dist_utils.py index a4e9862d2bc..fa752580a2c 100644 --- a/contrib/runners/winrm_runner/dist_utils.py +++ b/contrib/runners/winrm_runner/dist_utils.py @@ -24,8 +24,6 @@ import re import sys -from distutils.version import StrictVersion - # NOTE: This script can't rely on any 3rd party dependency so we need to use this code here # # TODO: Why can't this script rely on 3rd party dependencies? Is it because it has to import @@ -48,8 +46,6 @@ GET_PIP = "curl https://bootstrap.pypa.io/get-pip.py | python" __all__ = [ - "check_pip_is_installed", - "check_pip_version", "fetch_requirements", "apply_vagrant_workaround", "get_version_string", @@ -57,41 +53,6 @@ ] -def check_pip_is_installed(): - """ - Ensure that pip is installed. - """ - try: - import pip # NOQA - except ImportError as e: - print("Failed to import pip: %s" % (text_type(e))) - print("") - print("Download pip:\n%s" % (GET_PIP)) - sys.exit(1) - - return True - - -def check_pip_version(min_version="6.0.0"): - """ - Ensure that a minimum supported version of pip is installed. - """ - check_pip_is_installed() - - import pip - - if StrictVersion(pip.__version__) < StrictVersion(min_version): - print( - "Upgrade pip, your version '{0}' " - "is outdated. Minimum required version is '{1}':\n{2}".format( - pip.__version__, min_version, GET_PIP - ) - ) - sys.exit(1) - - return True - - def fetch_requirements(requirements_file_path): """ Return a list of requirements and links by parsing the provided requirements file. diff --git a/fixed-requirements.txt b/fixed-requirements.txt index 14f562fdecb..00ed7297b94 100644 --- a/fixed-requirements.txt +++ b/fixed-requirements.txt @@ -7,17 +7,17 @@ chardet<3.1.0 cffi<1.15.0 # NOTE: 2.0 version breaks pymongo work with hosts dnspython>=1.16.0,<2.0.0 -cryptography==3.4.7 +cryptography==39.0.1 # Note: 0.20.0 removed select.poll() on which some of our code and libraries we # depend on rely -eventlet==0.30.2 +eventlet==0.33.3 flex==6.14.1 gitpython==3.1.15 # Needed by gitpython, old versions used to bundle it gitdb==4.0.2 # Note: greenlet is used by eventlet greenlet==1.0.0 -gunicorn==20.1.0 +gunicorn==21.2.0 jsonpath-rw==1.4.0 jsonschema==2.6.0 kombu==5.0.2 @@ -36,15 +36,16 @@ decorator==4.4.2 # See https://github.com/StackStorm/st2/issues/4160#issuecomment-394386433 for details oslo.config>=1.12.1,<1.13 oslo.utils<5.0,>=4.0.0 -paramiko==2.10.5 +# paramiko 2.11.0 is needed by cryptography > 37.0.0 +paramiko==2.11.0 passlib==1.7.4 prompt-toolkit==1.0.15 pyinotify==0.9.6 ; platform_system=="Linux" pymongo==3.11.3 pyparsing<3 zstandard==0.15.2 -# pyOpenSSL 22.0.0 requires cryptography>=35.0 -pyOpenSSL<=21.0.0 +# pyOpenSSL 23.1.0 supports cryptography up to 40.0.x +pyOpenSSL==23.1.0 python-editor==1.0.4 python-keyczar==0.716 pytz==2021.1 diff --git a/lockfiles/st2.lock b/lockfiles/st2.lock index 852a04be5b1..1d97f6325d9 100644 --- a/lockfiles/st2.lock +++ b/lockfiles/st2.lock @@ -6,7 +6,7 @@ // { // "version": 3, // "valid_for_interpreter_constraints": [ -// "CPython<3.9,>=3.6" +// "CPython<3.10,>=3.6" // ], // "generated_with_requirements": [ // "PyYAML", @@ -5089,7 +5089,7 @@ "zstandard" ], "requires_python": [ - "<3.9,>=3.6" + "<3.10,>=3.6" ], "resolver_version": "pip-2020-resolver", "style": "universal", diff --git a/pants.toml b/pants.toml index 42f4a886e98..04435830d0e 100644 --- a/pants.toml +++ b/pants.toml @@ -108,8 +108,8 @@ enable_resolves = true default_resolve = "st2" interpreter_constraints = [ # python_distributions needs a single constraint (vs one line per python version). - # officially, we exclude 3.7 support, but that adds unnecessary complexity: "CPython>=3.6,!=3.7.*,<3.9", - "CPython>=3.6,<3.9", + # officially, we exclude 3.7 support, but that adds unnecessary complexity: "CPython>=3.6,!=3.7.*,<3.10", + "CPython>=3.6,<3.10", # NB: constraints for tools defined below ] diff --git a/pylint_plugins/api_models.py b/pylint_plugins/api_models.py index a5b3cf00bfc..d2b7d7e9b32 100644 --- a/pylint_plugins/api_models.py +++ b/pylint_plugins/api_models.py @@ -147,11 +147,24 @@ def transform(cls: nodes.ClassDef): if ( isinstance(target, nodes.Subscript) and target.value.value.name == "schema" - and target.value.slice.value.value == "properties" ): - property_name_node = target.slice.value + if ( + isinstance(target.value.slice.value, nodes.Const) + and target.value.slice.value.value == "properties" + ): + # python <3.9 + property_name_node = target.slice.value + elif ( + isinstance(target.value.slice, nodes.Const) + and target.value.slice.value == "properties" + ): + # python 3.9+ + property_name_node = target.slice + else: + # not schema["properties"] + continue else: - # not schema["properties"] + # not schema[...] continue except AttributeError: continue @@ -197,7 +210,12 @@ def transform(cls: nodes.ClassDef): # schema = {"properties": {"action": REQUIRED_ATTR_SCHEMAS["action"]}} if isinstance(property_data_node, nodes.Subscript): var_name = property_data_node.value.name - subscript = property_data_node.slice.value.value + if isinstance(property_data_node.slice.value, nodes.Const): # python <3.9 + subscript = property_data_node.slice.value.value + elif isinstance(property_data_node.slice, nodes.Const): # python 3.9+ + subscript = property_data_node.slice.value + else: + continue # lookup var by name (assume its at module level) var_node = next(cls.root().igetattr(var_name)) diff --git a/requirements.txt b/requirements.txt index bc2bbf4b9c8..841cf38cb34 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,15 +14,15 @@ bcrypt==3.2.0 cffi<1.15.0 chardet<3.1.0 ciso8601 -cryptography==3.4.7 +cryptography==39.0.1 decorator==4.4.2 dnspython>=1.16.0,<2.0.0 -eventlet==0.30.2 +eventlet==0.33.3 flex==6.14.1 gitdb==4.0.2 gitpython==3.1.15 greenlet==1.0.0 -gunicorn==20.1.0 +gunicorn==21.2.0 importlib-metadata==3.10.1 jinja2==2.11.3 jsonpath-rw==1.4.0 @@ -40,12 +40,12 @@ orjson==3.5.2 orquesta@ git+https://github.com/StackStorm/orquesta.git@v1.5.0 oslo.config>=1.12.1,<1.13 oslo.utils<5.0,>=4.0.0 -paramiko==2.10.5 +paramiko==2.11.0 passlib==1.7.4 prettytable==2.1.0 prompt-toolkit==1.0.15 psutil==5.8.0 -pyOpenSSL<=21.0.0 +pyOpenSSL==23.1.0 pyinotify==0.9.6 ; platform_system=="Linux" pymongo==3.11.3 pyparsing<3 diff --git a/scripts/ci/submit-codecov-coverage.sh b/scripts/ci/submit-codecov-coverage.sh index 3c8e441d3a0..76ab0f362c5 100755 --- a/scripts/ci/submit-codecov-coverage.sh +++ b/scripts/ci/submit-codecov-coverage.sh @@ -19,15 +19,12 @@ # If we're on Travis, then we need to manually check that the build succeeded. if [[ "${USER}" == "runner" || ${TRAVIS_TEST_RESULT} -eq 0 ]]; then # 1. Install codecov dependencies - # NOTE: We need eventlet installed so coverage can be correctly combined. This is needed because we are covering code which utilizes eventlet. - # Without eventlet being available to the coverage command it will fail with "Couldn't trace with concurrency=eventlet, the module isn't installed." - pip install eventlet - # NOTE: codecov only supports coverage==4.5.2 - pip install 'coverage<5.0' - pip install "codecov==2.1.11" + pip install -U pip + pip install coverage + pip install 'codecov-cli>=0.4' - # 2. Combine coverage report and submit coverage report to codecovs.io - codecov --required + # 2. Combine coverage report and submit coverage report to codecov.io + codecovcli upload-process -t "${CODECOV_TOKEN}" exit $? else echo "Build has failed, not submitting coverage" diff --git a/scripts/dist_utils.py b/scripts/dist_utils.py index 297efe16898..9c27f683ca7 100644 --- a/scripts/dist_utils.py +++ b/scripts/dist_utils.py @@ -20,8 +20,6 @@ import re import sys -from distutils.version import StrictVersion - # // NOTE: After you update this script, please run: # // # // make .sdist-requirements @@ -52,8 +50,6 @@ GET_PIP = "curl https://bootstrap.pypa.io/get-pip.py | python" __all__ = [ - "check_pip_is_installed", - "check_pip_version", "fetch_requirements", "apply_vagrant_workaround", "get_version_string", @@ -61,41 +57,6 @@ ] -def check_pip_is_installed(): - """ - Ensure that pip is installed. - """ - try: - import pip # NOQA - except ImportError as e: - print("Failed to import pip: %s" % (text_type(e))) - print("") - print("Download pip:\n%s" % (GET_PIP)) - sys.exit(1) - - return True - - -def check_pip_version(min_version="6.0.0"): - """ - Ensure that a minimum supported version of pip is installed. - """ - check_pip_is_installed() - - import pip - - if StrictVersion(pip.__version__) < StrictVersion(min_version): - print( - "Upgrade pip, your version '{0}' " - "is outdated. Minimum required version is '{1}':\n{2}".format( - pip.__version__, min_version, GET_PIP - ) - ) - sys.exit(1) - - return True - - def fetch_requirements(requirements_file_path): """ Return a list of requirements and links by parsing the provided requirements file. diff --git a/scripts/dist_utils_old.py b/scripts/dist_utils_old.py index da38f6edbf4..739970900cf 100644 --- a/scripts/dist_utils_old.py +++ b/scripts/dist_utils_old.py @@ -27,8 +27,6 @@ import re import sys -from distutils.version import StrictVersion - # NOTE: This script can't rely on any 3rd party dependency so we need to use this code here PY3 = sys.version_info[0] == 3 @@ -40,7 +38,6 @@ GET_PIP = "curl https://bootstrap.pypa.io/get-pip.py | python" try: - import pip from pip import __version__ as pip_version except ImportError as e: print("Failed to import pip: %s" % (text_type(e))) @@ -62,7 +59,6 @@ sys.exit(1) __all__ = [ - "check_pip_version", "fetch_requirements", "apply_vagrant_workaround", "get_version_string", @@ -70,20 +66,6 @@ ] -def check_pip_version(min_version="6.0.0"): - """ - Ensure that a minimum supported version of pip is installed. - """ - if StrictVersion(pip.__version__) < StrictVersion(min_version): - print( - "Upgrade pip, your version '{0}' " - "is outdated. Minimum required version is '{1}':\n{2}".format( - pip.__version__, min_version, GET_PIP - ) - ) - sys.exit(1) - - def fetch_requirements(requirements_file_path): """ Return a list of requirements and links by parsing the provided requirements file. diff --git a/scripts/fixate-requirements.py b/scripts/fixate-requirements.py index a60f43b4aa7..e7b83772975 100755 --- a/scripts/fixate-requirements.py +++ b/scripts/fixate-requirements.py @@ -34,8 +34,6 @@ import os.path import sys -from distutils.version import StrictVersion - # NOTE: This script can't rely on any 3rd party dependency so we need to use this code here PY2 = sys.version_info[0] == 2 PY3 = sys.version_info[0] == 3 @@ -49,7 +47,6 @@ GET_PIP = " curl https://bootstrap.pypa.io/get-pip.py | python" try: - import pip from pip import __version__ as pip_version except ImportError as e: print("Failed to import pip: %s" % (text_type(e))) @@ -115,15 +112,6 @@ def parse_args(): return vars(parser.parse_args()) -def check_pip_version(): - if StrictVersion(pip.__version__) < StrictVersion("6.1.0"): - print( - "Upgrade pip, your version `{0}' " "is outdated:\n".format(pip.__version__), - GET_PIP, - ) - sys.exit(1) - - def load_requirements(file_path): return tuple((r for r in parse_requirements(file_path, session=False))) @@ -282,7 +270,6 @@ def write_requirements( if __name__ == "__main__": - check_pip_version() args = parse_args() if args["skip"]: diff --git a/st2actions/dist_utils.py b/st2actions/dist_utils.py index a4e9862d2bc..fa752580a2c 100644 --- a/st2actions/dist_utils.py +++ b/st2actions/dist_utils.py @@ -24,8 +24,6 @@ import re import sys -from distutils.version import StrictVersion - # NOTE: This script can't rely on any 3rd party dependency so we need to use this code here # # TODO: Why can't this script rely on 3rd party dependencies? Is it because it has to import @@ -48,8 +46,6 @@ GET_PIP = "curl https://bootstrap.pypa.io/get-pip.py | python" __all__ = [ - "check_pip_is_installed", - "check_pip_version", "fetch_requirements", "apply_vagrant_workaround", "get_version_string", @@ -57,41 +53,6 @@ ] -def check_pip_is_installed(): - """ - Ensure that pip is installed. - """ - try: - import pip # NOQA - except ImportError as e: - print("Failed to import pip: %s" % (text_type(e))) - print("") - print("Download pip:\n%s" % (GET_PIP)) - sys.exit(1) - - return True - - -def check_pip_version(min_version="6.0.0"): - """ - Ensure that a minimum supported version of pip is installed. - """ - check_pip_is_installed() - - import pip - - if StrictVersion(pip.__version__) < StrictVersion(min_version): - print( - "Upgrade pip, your version '{0}' " - "is outdated. Minimum required version is '{1}':\n{2}".format( - pip.__version__, min_version, GET_PIP - ) - ) - sys.exit(1) - - return True - - def fetch_requirements(requirements_file_path): """ Return a list of requirements and links by parsing the provided requirements file. diff --git a/st2actions/requirements.txt b/st2actions/requirements.txt index acd17a961e1..e15c7c967e2 100644 --- a/st2actions/requirements.txt +++ b/st2actions/requirements.txt @@ -8,7 +8,7 @@ MarkupSafe<2.1.0,>=0.23 apscheduler==3.7.0 chardet<3.1.0 -eventlet==0.30.2 +eventlet==0.33.3 gitpython==3.1.15 jinja2==2.11.3 kombu==5.0.2 diff --git a/st2api/dist_utils.py b/st2api/dist_utils.py index a4e9862d2bc..fa752580a2c 100644 --- a/st2api/dist_utils.py +++ b/st2api/dist_utils.py @@ -24,8 +24,6 @@ import re import sys -from distutils.version import StrictVersion - # NOTE: This script can't rely on any 3rd party dependency so we need to use this code here # # TODO: Why can't this script rely on 3rd party dependencies? Is it because it has to import @@ -48,8 +46,6 @@ GET_PIP = "curl https://bootstrap.pypa.io/get-pip.py | python" __all__ = [ - "check_pip_is_installed", - "check_pip_version", "fetch_requirements", "apply_vagrant_workaround", "get_version_string", @@ -57,41 +53,6 @@ ] -def check_pip_is_installed(): - """ - Ensure that pip is installed. - """ - try: - import pip # NOQA - except ImportError as e: - print("Failed to import pip: %s" % (text_type(e))) - print("") - print("Download pip:\n%s" % (GET_PIP)) - sys.exit(1) - - return True - - -def check_pip_version(min_version="6.0.0"): - """ - Ensure that a minimum supported version of pip is installed. - """ - check_pip_is_installed() - - import pip - - if StrictVersion(pip.__version__) < StrictVersion(min_version): - print( - "Upgrade pip, your version '{0}' " - "is outdated. Minimum required version is '{1}':\n{2}".format( - pip.__version__, min_version, GET_PIP - ) - ) - sys.exit(1) - - return True - - def fetch_requirements(requirements_file_path): """ Return a list of requirements and links by parsing the provided requirements file. diff --git a/st2api/requirements.txt b/st2api/requirements.txt index f503ef97bc0..07aac607e98 100644 --- a/st2api/requirements.txt +++ b/st2api/requirements.txt @@ -5,8 +5,8 @@ # If you want to update depdencies for a single component, modify the # in-requirements.txt for that component and then run 'make requirements' to # update the component requirements.txt -eventlet==0.30.2 -gunicorn==20.1.0 +eventlet==0.33.3 +gunicorn==21.2.0 jsonschema==2.6.0 kombu==5.0.2 mongoengine==0.23.0 diff --git a/st2auth/dist_utils.py b/st2auth/dist_utils.py index a4e9862d2bc..fa752580a2c 100644 --- a/st2auth/dist_utils.py +++ b/st2auth/dist_utils.py @@ -24,8 +24,6 @@ import re import sys -from distutils.version import StrictVersion - # NOTE: This script can't rely on any 3rd party dependency so we need to use this code here # # TODO: Why can't this script rely on 3rd party dependencies? Is it because it has to import @@ -48,8 +46,6 @@ GET_PIP = "curl https://bootstrap.pypa.io/get-pip.py | python" __all__ = [ - "check_pip_is_installed", - "check_pip_version", "fetch_requirements", "apply_vagrant_workaround", "get_version_string", @@ -57,41 +53,6 @@ ] -def check_pip_is_installed(): - """ - Ensure that pip is installed. - """ - try: - import pip # NOQA - except ImportError as e: - print("Failed to import pip: %s" % (text_type(e))) - print("") - print("Download pip:\n%s" % (GET_PIP)) - sys.exit(1) - - return True - - -def check_pip_version(min_version="6.0.0"): - """ - Ensure that a minimum supported version of pip is installed. - """ - check_pip_is_installed() - - import pip - - if StrictVersion(pip.__version__) < StrictVersion(min_version): - print( - "Upgrade pip, your version '{0}' " - "is outdated. Minimum required version is '{1}':\n{2}".format( - pip.__version__, min_version, GET_PIP - ) - ) - sys.exit(1) - - return True - - def fetch_requirements(requirements_file_path): """ Return a list of requirements and links by parsing the provided requirements file. diff --git a/st2auth/requirements.txt b/st2auth/requirements.txt index 1d6a06de81d..e4d7f91fb94 100644 --- a/st2auth/requirements.txt +++ b/st2auth/requirements.txt @@ -6,8 +6,8 @@ # in-requirements.txt for that component and then run 'make requirements' to # update the component requirements.txt bcrypt==3.2.0 -eventlet==0.30.2 -gunicorn==20.1.0 +eventlet==0.33.3 +gunicorn==21.2.0 oslo.config>=1.12.1,<1.13 passlib==1.7.4 pymongo==3.11.3 diff --git a/st2auth/st2auth/controllers/v1/auth.py b/st2auth/st2auth/controllers/v1/auth.py index c77546141f4..b072d6c2d0b 100644 --- a/st2auth/st2auth/controllers/v1/auth.py +++ b/st2auth/st2auth/controllers/v1/auth.py @@ -67,6 +67,10 @@ def post(self, request, **kwargs): if "x-forwarded-for" in kwargs: headers["x-forwarded-for"] = kwargs.pop("x-forwarded-for") + remote_user = kwargs.pop("remote_user", None) + if not remote_user and "x-forwarded-user" in kwargs: + remote_user = kwargs.pop("x-forwarded-user", None) + authorization = kwargs.pop("authorization", None) if authorization: authorization = tuple(authorization.split(" ")) @@ -75,7 +79,7 @@ def post(self, request, **kwargs): request=request, headers=headers, remote_addr=kwargs.pop("remote_addr", None), - remote_user=kwargs.pop("remote_user", None), + remote_user=remote_user, authorization=authorization, **kwargs, ) diff --git a/st2auth/st2auth/handlers.py b/st2auth/st2auth/handlers.py index f6540bcda73..0bf9600c3e9 100644 --- a/st2auth/st2auth/handlers.py +++ b/st2auth/st2auth/handlers.py @@ -130,6 +130,25 @@ def handle_auth( remote_addr = headers.get("x-forwarded-for", remote_addr) extra = {"remote_addr": remote_addr} + # Needed to support st2client which does not connect via st2web + if authorization and not remote_user: + try: + auth_value = base64.b64decode(authorization[1]) + except Exception: + LOG.audit("Invalid authorization header", extra=extra) + abort_request() + return + + split = auth_value.split(b":", 1) + if len(split) != 2: + LOG.audit("Invalid authorization header", extra=extra) + abort_request() + return + + remote_user = split[0] + if six.PY3 and isinstance(remote_user, six.binary_type): + remote_user = remote_user.decode("utf-8") + if remote_user: ttl = getattr(request, "ttl", None) username = self._get_username_for_request(remote_user, request) diff --git a/st2auth/tests/unit/controllers/v1/test_token.py b/st2auth/tests/unit/controllers/v1/test_token.py index e56c7e9acb5..95bed14c995 100644 --- a/st2auth/tests/unit/controllers/v1/test_token.py +++ b/st2auth/tests/unit/controllers/v1/test_token.py @@ -97,6 +97,19 @@ def _test_token_post(self, path=TOKEN_V1_PATH): self.assertLess(actual_expiry, expected_expiry) return response + def test_token_post_proxy_user(self): + headers = {"X-Forwarded-For": "192.0.2.1", "X-Forwarded-User": "testuser"} + response = self.app.post_json( + TOKEN_V1_PATH, + {}, + headers=headers, + expect_errors=False, + extra_environ={"REMOTE_USER": ""}, + ) + self.assertEqual(response.status_int, 201) + self.assertIsNotNone(response.json["token"]) + self.assertEqual(response.json["user"], "testuser") + def test_token_post_unauthorized(self): response = self.app.post_json( TOKEN_V1_PATH, {}, expect_errors=True, extra_environ={"REMOTE_USER": ""} diff --git a/st2auth/tests/unit/test_handlers.py b/st2auth/tests/unit/test_handlers.py index cf00e642a69..bb297329132 100644 --- a/st2auth/tests/unit/test_handlers.py +++ b/st2auth/tests/unit/test_handlers.py @@ -48,6 +48,31 @@ def test_proxy_handler(self): ) self.assertEqual(token.user, "test_proxy_handler") + def test_proxy_handler_no_remote_user(self): + h = handlers.ProxyAuthHandler() + request = {} + token = h.handle_auth( + request, + headers={}, + remote_addr=None, + remote_user=None, + authorization=("basic", DUMMY_CREDS), + ) + self.assertEqual(token.user, "auser") + + def test_proxy_handler_bad_auth(self): + h = handlers.ProxyAuthHandler() + request = {} + + with self.assertRaises(exc.HTTPUnauthorized): + h.handle_auth( + request, + headers={}, + remote_addr=None, + remote_user=None, + authorization=None, + ) + def test_standalone_bad_auth_type(self): h = handlers.StandaloneAuthHandler() request = {} diff --git a/st2client/dist_utils.py b/st2client/dist_utils.py index a4e9862d2bc..fa752580a2c 100644 --- a/st2client/dist_utils.py +++ b/st2client/dist_utils.py @@ -24,8 +24,6 @@ import re import sys -from distutils.version import StrictVersion - # NOTE: This script can't rely on any 3rd party dependency so we need to use this code here # # TODO: Why can't this script rely on 3rd party dependencies? Is it because it has to import @@ -48,8 +46,6 @@ GET_PIP = "curl https://bootstrap.pypa.io/get-pip.py | python" __all__ = [ - "check_pip_is_installed", - "check_pip_version", "fetch_requirements", "apply_vagrant_workaround", "get_version_string", @@ -57,41 +53,6 @@ ] -def check_pip_is_installed(): - """ - Ensure that pip is installed. - """ - try: - import pip # NOQA - except ImportError as e: - print("Failed to import pip: %s" % (text_type(e))) - print("") - print("Download pip:\n%s" % (GET_PIP)) - sys.exit(1) - - return True - - -def check_pip_version(min_version="6.0.0"): - """ - Ensure that a minimum supported version of pip is installed. - """ - check_pip_is_installed() - - import pip - - if StrictVersion(pip.__version__) < StrictVersion(min_version): - print( - "Upgrade pip, your version '{0}' " - "is outdated. Minimum required version is '{1}':\n{2}".format( - pip.__version__, min_version, GET_PIP - ) - ) - sys.exit(1) - - return True - - def fetch_requirements(requirements_file_path): """ Return a list of requirements and links by parsing the provided requirements file. diff --git a/st2client/requirements.txt b/st2client/requirements.txt index faa3e4c23ea..a99071ba7f5 100644 --- a/st2client/requirements.txt +++ b/st2client/requirements.txt @@ -8,14 +8,14 @@ argcomplete==1.12.2 cffi<1.15.0 chardet<3.1.0 -cryptography==3.4.7 +cryptography==39.0.1 importlib-metadata==3.10.1 jsonpath-rw==1.4.0 jsonschema==2.6.0 orjson==3.5.2 prettytable==2.1.0 prompt-toolkit==1.0.15 -pyOpenSSL<=21.0.0 +pyOpenSSL==23.1.0 pysocks python-dateutil==2.8.1 python-editor==1.0.4 diff --git a/st2client/setup.py b/st2client/setup.py index 24040725226..1fe8ec1af39 100644 --- a/st2client/setup.py +++ b/st2client/setup.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python # Copyright 2020 The StackStorm Authors. # Copyright 2019 Extreme Networks, Inc. # @@ -18,14 +18,11 @@ from setuptools import setup, find_packages -from dist_utils import check_pip_version from dist_utils import fetch_requirements from dist_utils import apply_vagrant_workaround from st2client import __version__ -check_pip_version() - ST2_COMPONENT = "st2client" BASE_DIR = os.path.dirname(os.path.abspath(__file__)) REQUIREMENTS_FILE = os.path.join(BASE_DIR, "requirements.txt") diff --git a/st2client/st2client/commands/action.py b/st2client/st2client/commands/action.py index fb2f94c0f28..48964746a56 100644 --- a/st2client/st2client/commands/action.py +++ b/st2client/st2client/commands/action.py @@ -945,6 +945,9 @@ def normalize(name, value, action_params=None, auto_dict=False): result = {} + if args.inherit_env: + result["env"] = self._get_inherited_env_vars() + if not args.parameters: return result @@ -1008,9 +1011,6 @@ def normalize(name, value, action_params=None, auto_dict=False): del result["_file_name"] - if args.inherit_env: - result["env"] = self._get_inherited_env_vars() - return result @add_auth_token_to_kwargs_from_cli diff --git a/st2client/st2client/shell.py b/st2client/st2client/shell.py index 911795524e0..81eb7f05a89 100755 --- a/st2client/st2client/shell.py +++ b/st2client/st2client/shell.py @@ -22,11 +22,11 @@ from __future__ import print_function from __future__ import absolute_import -# Ignore CryptographyDeprecationWarning warnings which appear on older versions of Python 2.7 +# Ignore CryptographyDeprecationWarning warnings which appear on Python 3.6 +# TODO: Remove after dropping python3.6 import warnings -from cryptography.utils import CryptographyDeprecationWarning -warnings.filterwarnings("ignore", category=CryptographyDeprecationWarning) +warnings.filterwarnings("ignore", message="Python 3.6 is no longer supported") import os import sys diff --git a/st2client/tests/unit/test_command_actionrun.py b/st2client/tests/unit/test_command_actionrun.py index 1e312e0786b..afd6fb418d0 100644 --- a/st2client/tests/unit/test_command_actionrun.py +++ b/st2client/tests/unit/test_command_actionrun.py @@ -261,3 +261,135 @@ def test_get_params_from_args_with_multiple_declarations(self): # set auto_dict back to default mockarg.auto_dict = False + + def test_correctly_process_inherit_env_when_no_parameters_set(self): + """test_correctly_process_inherit_env_when_no_parameters_set + + This tests that we still correctly pass through the environment variables + when --inherit-env is set and we run an action that does not have parameters + """ + + runner = RunnerType() + runner.runner_parameters = {} + + action = Action() + action.ref = "test.action" + + subparser = mock.Mock() + command = ActionRunCommand(action, self, subparser, name="test") + + mockarg = mock.Mock() + mockarg.inherit_env = True + mockarg.auto_dict = True + mockarg.parameters = [] + + k1 = "key1" + v1 = "value1" + k2 = "key2" + v2 = "value2" + + with mock.patch("os.environ.copy") as mockCopy: + mockCopy.return_value = {k1: v1, k2: v2} + param = command._get_action_parameters_from_args( + action=action, runner=runner, args=mockarg + ) + + self.assertIn("env", param) + + env_params = param["env"] + self.assertIn(k1, env_params) + self.assertIn(k2, env_params) + self.assertEqual(v1, env_params[k1]) + self.assertEqual(v2, env_params[k2]) + + def test_correctly_process_inherit_env_when_parameters_set(self): + """test_correctly_process_inherit_env_when_parameters_set + + This tests that we still correctly pass through the environment variables + when --inherit-env is set and we run an action that has action parameters set + """ + + runner = RunnerType() + runner.runner_parameters = {} + + action = Action() + action.ref = "test.action" + action.parameters = { + "param_string": {"type": "string"}, + "param_array": {"type": "array"}, + "param_array_of_dicts": {"type": "array"}, + } + + subparser = mock.Mock() + command = ActionRunCommand(action, self, subparser, name="test") + + p_string = "param_string" + p_array = "param_array" + p_ra_dicts = "param_array_of_dicts" + mockarg = mock.Mock() + mockarg.inherit_env = True + mockarg.auto_dict = True + mockarg.parameters = [ + f"{p_string}=hoge", + f"{p_array}=foo,bar", + f"{p_ra_dicts}=foo:1,bar:2", + ] + + k1 = "key1" + v1 = "value1" + k2 = "key2" + v2 = "value2" + + with mock.patch("os.environ.copy") as mockCopy: + mockCopy.return_value = {k1: v1, k2: v2} + param = command._get_action_parameters_from_args( + action=action, runner=runner, args=mockarg + ) + + self.assertIn("env", param) + + env_params = param["env"] + self.assertIn(k1, env_params) + self.assertIn(k2, env_params) + self.assertEqual(v1, env_params[k1]) + self.assertEqual(v2, env_params[k2]) + self.assertIn(p_string, param) + self.assertEqual("hoge", param[p_string]) + self.assertIn(p_array, param) + self.assertIn("foo", param[p_array]) + self.assertIn("bar", param[p_array]) + self.assertIn(p_ra_dicts, param) + self.assertDictEqual({"foo": "1", "bar": "2"}, param[p_ra_dicts][0]) + + def test_correctly_generate_empty_params_no_inherit_empty_parameters(self): + """test_correctly_generate_empty_params_no_inherit_empty_parameters + + Verifies that we return an empty dict when we do not provide inherit env and parameters + """ + + runner = RunnerType() + runner.runner_parameters = {} + + action = Action() + action.ref = "test.action" + + subparser = mock.Mock() + command = ActionRunCommand(action, self, subparser, name="test") + + mockarg = mock.Mock() + mockarg.inherit_env = False + mockarg.auto_dict = True + mockarg.parameters = [] + + k1 = "key1" + v1 = "value1" + k2 = "key2" + v2 = "value2" + + with mock.patch("os.environ.copy") as mockCopy: + mockCopy.return_value = {k1: v1, k2: v2} + param = command._get_action_parameters_from_args( + action=action, runner=runner, args=mockarg + ) + + self.assertDictEqual({}, param) diff --git a/st2common/bin/paramiko_ssh_evenlets_tester.py b/st2common/bin/paramiko_ssh_evenlets_tester.py index ceb42060809..012e7c128b3 100755 --- a/st2common/bin/paramiko_ssh_evenlets_tester.py +++ b/st2common/bin/paramiko_ssh_evenlets_tester.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python # Copyright 2020 The StackStorm Authors. # Copyright 2019 Extreme Networks, Inc. # diff --git a/st2common/bin/st2-self-check b/st2common/bin/st2-self-check index d7d1a0b85a7..1240b13fc7f 100755 --- a/st2common/bin/st2-self-check +++ b/st2common/bin/st2-self-check @@ -29,6 +29,7 @@ function usage() { RUN_ORQUESTA_TESTS=true RUN_WINDOWS_TESTS=false ST2_TESTS_BRANCH="master" +TESTS_TO_SKIP="${TESTS_TO_SKIP:-}" while getopts "b:wo" o do @@ -101,10 +102,15 @@ if [ ${EXIT_CODE} -ne 0 ]; then fi echo "Copying asserts, fixtures, tests and examples packs." -chown -R root:st2packs st2tests/packs/ -chmod -R 775 st2tests/packs/* +if [ -n "${GITHUB_ACTIONS}" ]; then + cp -Rf --preserve /home/runner/work/st2/st2/contrib/examples /opt/stackstorm/packs/ +else + chmod -R 775 st2tests/packs/* + cp -Rf --preserve /usr/share/doc/st2/examples /opt/stackstorm/packs/ + chown -R root:st2packs st2tests/packs/ +fi + cp -R --preserve st2tests/packs/* /opt/stackstorm/packs/ -cp -Rf --preserve /usr/share/doc/st2/examples /opt/stackstorm/packs/ echo "Installing asserts, fixtures, tests and examples packs." st2 run packs.setup_virtualenv packs=examples,tests,asserts,fixtures,webui @@ -136,6 +142,11 @@ do continue fi + if [[ " ${TESTS_TO_SKIP} " =~ " ${TEST} " ]]; then + echo "Skipping ${TEST}..." + continue + fi + echo -n "Attempting Test ${TEST}..." START_TS=$(date +%s) @@ -179,6 +190,9 @@ if [ $ERRORS -ne 0 ]; then echo "SELF CHECK FAILED!" echo "st2-self-check failed. See above. Also check the execution list for details." echo "st2 execution list" + if [ -n "${GITHUB_ACTIONS}" ]; then + exit 1 + fi else echo "SELF CHECK SUCCEEDED!" echo -e "st2-self-check succeeded." diff --git a/st2common/bin/st2ctl b/st2common/bin/st2ctl index 26b404e0ebc..a26f7bfb098 100755 --- a/st2common/bin/st2ctl +++ b/st2common/bin/st2ctl @@ -15,12 +15,12 @@ # limitations under the License. COMPONENTS="st2actionrunner st2api st2stream st2auth st2garbagecollector st2notifier st2rulesengine st2sensorcontainer st2chatops st2timersengine st2workflowengine st2scheduler" -ST2_CONF="/etc/st2/st2.conf" +ST2_CONF="${ST2_CONF:-/etc/st2/st2.conf}" SYSTEMD_RELOADED="" -# Ensure global environment is sourced if exists +# Ensure global environment is sourced if exists and if not executed in the context of Github Actions # Does not happen consistently with all OSes we support. -[ -r /etc/environment ] && source /etc/environment +[ -z "${GITHUB_ACTIONS}" ] && [ -r /etc/environment ] && source /etc/environment # load in environment to allow override of COMPONENTS and ST2_CONF above # Ubuntu/Debian diff --git a/st2common/dist_utils.py b/st2common/dist_utils.py index a4e9862d2bc..fa752580a2c 100644 --- a/st2common/dist_utils.py +++ b/st2common/dist_utils.py @@ -24,8 +24,6 @@ import re import sys -from distutils.version import StrictVersion - # NOTE: This script can't rely on any 3rd party dependency so we need to use this code here # # TODO: Why can't this script rely on 3rd party dependencies? Is it because it has to import @@ -48,8 +46,6 @@ GET_PIP = "curl https://bootstrap.pypa.io/get-pip.py | python" __all__ = [ - "check_pip_is_installed", - "check_pip_version", "fetch_requirements", "apply_vagrant_workaround", "get_version_string", @@ -57,41 +53,6 @@ ] -def check_pip_is_installed(): - """ - Ensure that pip is installed. - """ - try: - import pip # NOQA - except ImportError as e: - print("Failed to import pip: %s" % (text_type(e))) - print("") - print("Download pip:\n%s" % (GET_PIP)) - sys.exit(1) - - return True - - -def check_pip_version(min_version="6.0.0"): - """ - Ensure that a minimum supported version of pip is installed. - """ - check_pip_is_installed() - - import pip - - if StrictVersion(pip.__version__) < StrictVersion(min_version): - print( - "Upgrade pip, your version '{0}' " - "is outdated. Minimum required version is '{1}':\n{2}".format( - pip.__version__, min_version, GET_PIP - ) - ) - sys.exit(1) - - return True - - def fetch_requirements(requirements_file_path): """ Return a list of requirements and links by parsing the provided requirements file. diff --git a/st2common/requirements.txt b/st2common/requirements.txt index b10f6b5fe1d..bd1a1827af9 100644 --- a/st2common/requirements.txt +++ b/st2common/requirements.txt @@ -11,10 +11,10 @@ apscheduler==3.7.0 cffi<1.15.0 chardet<3.1.0 ciso8601 -cryptography==3.4.7 +cryptography==39.0.1 decorator==4.4.2 dnspython>=1.16.0,<2.0.0 -eventlet==0.30.2 +eventlet==0.33.3 flex==6.14.1 gitdb==4.0.2 gitpython==3.1.15 @@ -29,8 +29,8 @@ networkx>=2.5.1,<2.6 orjson==3.5.2 orquesta@ git+https://github.com/StackStorm/orquesta.git@v1.5.0 oslo.config>=1.12.1,<1.13 -paramiko==2.10.5 -pyOpenSSL<=21.0.0 +paramiko==2.11.0 +pyOpenSSL==23.1.0 pymongo==3.11.3 python-dateutil==2.8.1 python-statsd==2.1.0 diff --git a/st2common/st2common/openapi.yaml b/st2common/st2common/openapi.yaml index a8ffa4ef135..f7556878186 100644 --- a/st2common/st2common/openapi.yaml +++ b/st2common/st2common/openapi.yaml @@ -4454,6 +4454,10 @@ paths: in: header description: set externally to indicate real source of the request type: string + - name: x-forwarded-user + in: header + description: set externally to indicate the remote username in the case of proxy auth + type: string - name: request in: body description: Lifespan of the token diff --git a/st2common/st2common/openapi.yaml.j2 b/st2common/st2common/openapi.yaml.j2 index 4ce83071c05..45f126275ab 100644 --- a/st2common/st2common/openapi.yaml.j2 +++ b/st2common/st2common/openapi.yaml.j2 @@ -4450,6 +4450,10 @@ paths: in: header description: set externally to indicate real source of the request type: string + - name: x-forwarded-user + in: header + description: set externally to indicate the remote username in the case of proxy auth + type: string - name: request in: body description: Lifespan of the token diff --git a/st2common/st2common/router.py b/st2common/st2common/router.py index ff4a9866f56..1200044fd68 100644 --- a/st2common/st2common/router.py +++ b/st2common/st2common/router.py @@ -328,7 +328,12 @@ def __call__(self, req): At the time of writing, the only property being utilized by middleware was `x-log-result`. """ - LOG.debug("Received call with WebOb: %s", req) + LOG.debug("Received call with WebOb: %s %s", req.method, req.url) + # if a more detailed log is required: + # loggable_req = req.copy() + # loggable_req.headers.pop('Authorization', None) + # loggable_req.headers.pop('X-Request-Id', None) + # LOG.debug("Received call with WebOb: %s", loggable_req) endpoint, path_vars = self.match(req) LOG.debug("Parsed endpoint: %s", endpoint) LOG.debug("Parsed path_vars: %s", path_vars) diff --git a/st2common/st2common/util/pack_management.py b/st2common/st2common/util/pack_management.py index ae3c2561844..c9576f6fccc 100644 --- a/st2common/st2common/util/pack_management.py +++ b/st2common/st2common/util/pack_management.py @@ -35,7 +35,7 @@ from git.repo import Repo from gitdb.exc import BadName, BadObject from lockfile import LockFile -from distutils.spawn import find_executable +from shutil import which as shutil_which from st2common import log as logging from st2common.content import utils @@ -67,7 +67,7 @@ CURRENT_STACKSTORM_VERSION = get_stackstorm_version() CURRENT_PYTHON_VERSION = get_python_version() -SUDO_BINARY = find_executable("sudo") +SUDO_BINARY = shutil_which("sudo") def download_pack( diff --git a/st2common/st2common/util/sandboxing.py b/st2common/st2common/util/sandboxing.py index 791607471fe..86409853621 100644 --- a/st2common/st2common/util/sandboxing.py +++ b/st2common/st2common/util/sandboxing.py @@ -23,7 +23,7 @@ import fnmatch import os import sys -from distutils.sysconfig import get_python_lib +from sysconfig import get_path from oslo_config import cfg @@ -31,6 +31,12 @@ from st2common.constants.pack import SYSTEM_PACK_NAMES from st2common.content.utils import get_pack_base_path + +def get_python_lib(): + """Replacement for distutil.sysconfig.get_python_lib, returns a string with the python platform lib path (to site-packages)""" + return get_path("platlib") + + __all__ = [ "get_sandbox_python_binary_path", "get_sandbox_python_path", diff --git a/st2common/st2common/util/virtualenvs.py b/st2common/st2common/util/virtualenvs.py index 20100369b00..188733e85c9 100644 --- a/st2common/st2common/util/virtualenvs.py +++ b/st2common/st2common/util/virtualenvs.py @@ -67,7 +67,7 @@ def setup_pack_virtualenv( level logger. :param no_download: Do not download and install latest version of pre-installed packages such - as pip and distutils. + as pip and setuptools. :type no_download: ``bool`` """ logger = logger or LOG @@ -170,7 +170,7 @@ def create_virtualenv( :type include_wheel : ``bool`` :param no_download: Do not download and install latest version of pre-installed packages such - as pip and distutils. + as pip and setuptools. :type no_download: ``bool`` """ diff --git a/st2common/tests/integration/log_unicode_data.py b/st2common/tests/integration/log_unicode_data.py index 9d8c6166108..1806c1d8570 100644 --- a/st2common/tests/integration/log_unicode_data.py +++ b/st2common/tests/integration/log_unicode_data.py @@ -20,6 +20,12 @@ from __future__ import absolute_import +# Ignore CryptographyDeprecationWarning warnings which appear on Python 3.6 +# TODO: Remove after dropping python3.6 +import warnings + +warnings.filterwarnings("ignore", message="Python 3.6 is no longer supported") + import os import sys diff --git a/st2common/tests/integration/test_logging.py b/st2common/tests/integration/test_logging.py index 997f96c7a57..0f7cc77228c 100644 --- a/st2common/tests/integration/test_logging.py +++ b/st2common/tests/integration/test_logging.py @@ -15,6 +15,12 @@ from __future__ import absolute_import +# Ignore CryptographyDeprecationWarning warnings which appear on Python 3.6 +# TODO: Remove after dropping python3.6 +import warnings + +warnings.filterwarnings("ignore", message="Python 3.6 is no longer supported") + import os import sys import signal diff --git a/st2common/tests/unit/test_dist_utils.py b/st2common/tests/unit/test_dist_utils.py index 1b01d4ff48b..e19df806047 100644 --- a/st2common/tests/unit/test_dist_utils.py +++ b/st2common/tests/unit/test_dist_utils.py @@ -16,7 +16,6 @@ import os import sys -import six import mock import unittest2 @@ -26,8 +25,6 @@ # Add scripts/ which contain main dist_utils.py to PYTHONPATH sys.path.insert(0, SCRIPTS_PATH) -from dist_utils import check_pip_is_installed -from dist_utils import check_pip_version from dist_utils import fetch_requirements from dist_utils import apply_vagrant_workaround from dist_utils import get_version_string @@ -51,39 +48,6 @@ def setUp(self): def tearDown(self): super(DistUtilsTestCase, self).tearDown() - def test_check_pip_is_installed_success(self): - self.assertTrue(check_pip_is_installed()) - - @mock.patch("sys.exit") - def test_check_pip_is_installed_failure(self, mock_sys_exit): - if six.PY3: - module_name = "builtins.__import__" - else: - module_name = "__builtin__.__import__" - - with mock.patch(module_name) as mock_import: - mock_import.side_effect = ImportError("not found") - - self.assertEqual(mock_sys_exit.call_count, 0) - check_pip_is_installed() - self.assertEqual(mock_sys_exit.call_count, 1) - self.assertEqual(mock_sys_exit.call_args_list[0][0], (1,)) - - def test_check_pip_version_success(self): - self.assertTrue(check_pip_version()) - - @mock.patch("sys.exit") - def test_check_pip_version_failure(self, mock_sys_exit): - - mock_pip = mock.Mock() - mock_pip.__version__ = "0.0.0" - sys.modules["pip"] = mock_pip - - self.assertEqual(mock_sys_exit.call_count, 0) - check_pip_version() - self.assertEqual(mock_sys_exit.call_count, 1) - self.assertEqual(mock_sys_exit.call_args_list[0][0], (1,)) - def test_get_version_string(self): version = get_version_string(VERSION_FILE_PATH) self.assertEqual(version, "1.2.3") diff --git a/st2reactor/dist_utils.py b/st2reactor/dist_utils.py index a4e9862d2bc..fa752580a2c 100644 --- a/st2reactor/dist_utils.py +++ b/st2reactor/dist_utils.py @@ -24,8 +24,6 @@ import re import sys -from distutils.version import StrictVersion - # NOTE: This script can't rely on any 3rd party dependency so we need to use this code here # # TODO: Why can't this script rely on 3rd party dependencies? Is it because it has to import @@ -48,8 +46,6 @@ GET_PIP = "curl https://bootstrap.pypa.io/get-pip.py | python" __all__ = [ - "check_pip_is_installed", - "check_pip_version", "fetch_requirements", "apply_vagrant_workaround", "get_version_string", @@ -57,41 +53,6 @@ ] -def check_pip_is_installed(): - """ - Ensure that pip is installed. - """ - try: - import pip # NOQA - except ImportError as e: - print("Failed to import pip: %s" % (text_type(e))) - print("") - print("Download pip:\n%s" % (GET_PIP)) - sys.exit(1) - - return True - - -def check_pip_version(min_version="6.0.0"): - """ - Ensure that a minimum supported version of pip is installed. - """ - check_pip_is_installed() - - import pip - - if StrictVersion(pip.__version__) < StrictVersion(min_version): - print( - "Upgrade pip, your version '{0}' " - "is outdated. Minimum required version is '{1}':\n{2}".format( - pip.__version__, min_version, GET_PIP - ) - ) - sys.exit(1) - - return True - - def fetch_requirements(requirements_file_path): """ Return a list of requirements and links by parsing the provided requirements file. diff --git a/st2reactor/requirements.txt b/st2reactor/requirements.txt index 3388a8214bc..d7e0d30966e 100644 --- a/st2reactor/requirements.txt +++ b/st2reactor/requirements.txt @@ -6,7 +6,7 @@ # in-requirements.txt for that component and then run 'make requirements' to # update the component requirements.txt apscheduler==3.7.0 -eventlet==0.30.2 +eventlet==0.33.3 jsonpath-rw==1.4.0 jsonschema==2.6.0 kombu==5.0.2 diff --git a/st2stream/dist_utils.py b/st2stream/dist_utils.py index a4e9862d2bc..fa752580a2c 100644 --- a/st2stream/dist_utils.py +++ b/st2stream/dist_utils.py @@ -24,8 +24,6 @@ import re import sys -from distutils.version import StrictVersion - # NOTE: This script can't rely on any 3rd party dependency so we need to use this code here # # TODO: Why can't this script rely on 3rd party dependencies? Is it because it has to import @@ -48,8 +46,6 @@ GET_PIP = "curl https://bootstrap.pypa.io/get-pip.py | python" __all__ = [ - "check_pip_is_installed", - "check_pip_version", "fetch_requirements", "apply_vagrant_workaround", "get_version_string", @@ -57,41 +53,6 @@ ] -def check_pip_is_installed(): - """ - Ensure that pip is installed. - """ - try: - import pip # NOQA - except ImportError as e: - print("Failed to import pip: %s" % (text_type(e))) - print("") - print("Download pip:\n%s" % (GET_PIP)) - sys.exit(1) - - return True - - -def check_pip_version(min_version="6.0.0"): - """ - Ensure that a minimum supported version of pip is installed. - """ - check_pip_is_installed() - - import pip - - if StrictVersion(pip.__version__) < StrictVersion(min_version): - print( - "Upgrade pip, your version '{0}' " - "is outdated. Minimum required version is '{1}':\n{2}".format( - pip.__version__, min_version, GET_PIP - ) - ) - sys.exit(1) - - return True - - def fetch_requirements(requirements_file_path): """ Return a list of requirements and links by parsing the provided requirements file. diff --git a/st2stream/requirements.txt b/st2stream/requirements.txt index faa39eafd87..8a9e422f040 100644 --- a/st2stream/requirements.txt +++ b/st2stream/requirements.txt @@ -5,8 +5,8 @@ # If you want to update depdencies for a single component, modify the # in-requirements.txt for that component and then run 'make requirements' to # update the component requirements.txt -eventlet==0.30.2 -gunicorn==20.1.0 +eventlet==0.33.3 +gunicorn==21.2.0 jsonschema==2.6.0 kombu==5.0.2 mongoengine==0.23.0 diff --git a/st2tests/dist_utils.py b/st2tests/dist_utils.py index a4e9862d2bc..fa752580a2c 100644 --- a/st2tests/dist_utils.py +++ b/st2tests/dist_utils.py @@ -24,8 +24,6 @@ import re import sys -from distutils.version import StrictVersion - # NOTE: This script can't rely on any 3rd party dependency so we need to use this code here # # TODO: Why can't this script rely on 3rd party dependencies? Is it because it has to import @@ -48,8 +46,6 @@ GET_PIP = "curl https://bootstrap.pypa.io/get-pip.py | python" __all__ = [ - "check_pip_is_installed", - "check_pip_version", "fetch_requirements", "apply_vagrant_workaround", "get_version_string", @@ -57,41 +53,6 @@ ] -def check_pip_is_installed(): - """ - Ensure that pip is installed. - """ - try: - import pip # NOQA - except ImportError as e: - print("Failed to import pip: %s" % (text_type(e))) - print("") - print("Download pip:\n%s" % (GET_PIP)) - sys.exit(1) - - return True - - -def check_pip_version(min_version="6.0.0"): - """ - Ensure that a minimum supported version of pip is installed. - """ - check_pip_is_installed() - - import pip - - if StrictVersion(pip.__version__) < StrictVersion(min_version): - print( - "Upgrade pip, your version '{0}' " - "is outdated. Minimum required version is '{1}':\n{2}".format( - pip.__version__, min_version, GET_PIP - ) - ) - sys.exit(1) - - return True - - def fetch_requirements(requirements_file_path): """ Return a list of requirements and links by parsing the provided requirements file. diff --git a/st2tests/testpacks/checks/actions/checks/check_loadavg.py b/st2tests/testpacks/checks/actions/checks/check_loadavg.py index 9439679df35..23845e7316a 100755 --- a/st2tests/testpacks/checks/actions/checks/check_loadavg.py +++ b/st2tests/testpacks/checks/actions/checks/check_loadavg.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # Copyright 2020 The StackStorm Authors. # Copyright 2019 Extreme Networks, Inc. diff --git a/test-requirements.txt b/test-requirements.txt index 56b8b7ac2a5..adf875e0cfe 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,5 +1,4 @@ -# NOTE: codecov only supports coverage==4.5.2 -coverage==4.5.2 +coverage pep8==1.7.1 st2flake8==0.1.0 astroid==2.5.6 @@ -24,7 +23,7 @@ nose-parallel==0.4.0 # Required by st2client tests pyyaml==5.4.1 RandomWords -gunicorn==20.1.0 +gunicorn==21.2.0 psutil==5.8.0 webtest==2.0.35 rstcheck>=3.3.1,<3.4