diff --git a/Makefile b/Makefile index 84679bceda..3fe34728a2 100644 --- a/Makefile +++ b/Makefile @@ -59,8 +59,8 @@ ST2TESTS_REDIS_PORT := 6379 # Pin common pip version here across all the targets # Note! Periodic maintenance pip upgrades are required to be up-to-date with the latest pip security fixes and updates -PIP_VERSION ?= 25.0.1 -SETUPTOOLS_VERSION ?= 75.3.2 +PIP_VERSION ?= 25.3 +SETUPTOOLS_VERSION ?= 80.10.2 PIP_OPTIONS := $(ST2_PIP_OPTIONS) ifndef PYLINT_CONCURRENCY @@ -174,12 +174,12 @@ install-runners: @echo "================== INSTALL RUNNERS ====================" @echo "" # NOTE: We use xargs to speed things up by installing runners in parallel - echo -e "$(COMPONENTS_RUNNERS)" | tr -d "\n" | xargs -P $(XARGS_CONCURRENCY) -d " " -n1 -i sh -c ". $(VIRTUALENV_DIR)/bin/activate; cd $$(pwd)/{} ; python setup.py develop --no-deps" + echo -e "$(COMPONENTS_RUNNERS)" | tr -d "\n" | xargs -P $(XARGS_CONCURRENCY) -d " " -n1 -i sh -c ". $(VIRTUALENV_DIR)/bin/activate; cd $$(pwd)/{} ; python -m pip install --editable . --no-deps" #@for component in $(COMPONENTS_RUNNERS); do \ # echo "==========================================================="; \ # echo "Installing runner:" $$component; \ # echo "==========================================================="; \ - # #(. $(VIRTUALENV_DIR)/bin/activate; cd $$component; python setup.py develop --no-deps); \ + # #(. $(VIRTUALENV_DIR)/bin/activate; cd $$component; python -m pip install --editable . --no-deps); \ #done .PHONY: install-mock-runners @@ -188,12 +188,12 @@ install-mock-runners: @echo "================== INSTALL MOCK RUNNERS ====================" @echo "" # NOTE: We use xargs to speed things up by installing runners in parallel - echo -e "$(MOCK_RUNNERS)" | tr -d "\n" | xargs -P $(XARGS_CONCURRENCY) -d " " -n1 -i sh -c ". $(VIRTUALENV_DIR)/bin/activate; cd $$(pwd)/{} ; python setup.py develop --no-deps" + echo -e "$(MOCK_RUNNERS)" | tr -d "\n" | xargs -P $(XARGS_CONCURRENCY) -d " " -n1 -i sh -c ". $(VIRTUALENV_DIR)/bin/activate; cd $$(pwd)/{} ; python -m pip install --editable . --no-deps" #@for component in $(MOCK_RUNNERS); do \ # echo "==========================================================="; \ # echo "Installing mock runner:" $$component; \ # echo "==========================================================="; \ - # (. $(VIRTUALENV_DIR)/bin/activate; cd $$component; python setup.py develop --no-deps); \ + # (. $(VIRTUALENV_DIR)/bin/activate; cd $$component; python -m pip install --editable . --no-deps); \ #done .PHONY: check-requirements @@ -276,7 +276,7 @@ check-python-packages-nightly: echo "==========================================================="; \ (set -e; cd $$component; ../$(VIRTUALENV_COMPONENTS_DIR)/bin/python setup.py --version) || exit 1; \ (set -e; cd $$component; ../$(VIRTUALENV_COMPONENTS_DIR)/bin/python setup.py sdist bdist_wheel) || exit 1; \ - (set -e; cd $$component; ../$(VIRTUALENV_COMPONENTS_DIR)/bin/python setup.py develop --no-deps) || exit 1; \ + (set -e; cd $$component; ../$(VIRTUALENV_COMPONENTS_DIR)/bin/python -m pip install --editable . --no-deps) || exit 1; \ ($(VIRTUALENV_COMPONENTS_DIR)/bin/python -c "import $$component") || exit 1; \ (set -e; cd $$component; rm -rf dist/; rm -rf $$component.egg-info) || exit 1; \ done @@ -505,9 +505,9 @@ flake8: requirements .flake8 touch $(VIRTUALENV_ST2CLIENT_PYPI_DIR)/bin/activate chmod +x $(VIRTUALENV_ST2CLIENT_PYPI_DIR)/bin/activate - $(VIRTUALENV_ST2CLIENT_PYPI_DIR)/bin/pip install --upgrade "pip==$(PIP_VERSION)" - $(VIRTUALENV_ST2CLIENT_PYPI_DIR)/bin/pip install --upgrade "readme_renderer" - $(VIRTUALENV_ST2CLIENT_PYPI_DIR)/bin/pip install --upgrade "restructuredtext-lint" + $(VIRTUALENV_ST2CLIENT_PYPI_DIR)/bin/python -m pip install --upgrade "pip==$(PIP_VERSION)" + $(VIRTUALENV_ST2CLIENT_PYPI_DIR)/bin/python -m pip install --upgrade "readme_renderer" + $(VIRTUALENV_ST2CLIENT_PYPI_DIR)/bin/python -m pip install --upgrade "restructuredtext-lint" # Check with readme-renderer . $(VIRTUALENV_ST2CLIENT_PYPI_DIR)/bin/activate; cd st2client ; ../$(VIRTUALENV_ST2CLIENT_PYPI_DIR)/bin/python -m readme_renderer README.rst @@ -534,10 +534,10 @@ flake8: requirements .flake8 touch $(VIRTUALENV_ST2CLIENT_DIR)/bin/activate chmod +x $(VIRTUALENV_ST2CLIENT_DIR)/bin/activate - $(VIRTUALENV_ST2CLIENT_DIR)/bin/pip install --upgrade "pip==$(PIP_VERSION)" - $(VIRTUALENV_ST2CLIENT_DIR)/bin/pip install --upgrade "setuptools==$(SETUPTOOLS_VERSION)" + $(VIRTUALENV_ST2CLIENT_DIR)/bin/python -m pip install --upgrade "pip==$(PIP_VERSION)" + $(VIRTUALENV_ST2CLIENT_DIR)/bin/python -m pip install --upgrade "setuptools==$(SETUPTOOLS_VERSION)" - $(VIRTUALENV_ST2CLIENT_DIR)/bin/activate; cd st2client ; ../$(VIRTUALENV_ST2CLIENT_DIR)/bin/pip install . ; cd .. + $(VIRTUALENV_ST2CLIENT_DIR)/bin/activate; cd st2client ; ../$(VIRTUALENV_ST2CLIENT_DIR)/bin/python -m pip install . ; cd .. $(VIRTUALENV_ST2CLIENT_DIR)/bin/st2 --version $(VIRTUALENV_ST2CLIENT_DIR)/bin/python -c "import st2client" @@ -661,7 +661,7 @@ distclean: clean .PHONY: .requirements .requirements: virtualenv - $(VIRTUALENV_DIR)/bin/pip install --upgrade "pip==$(PIP_VERSION)" + $(VIRTUALENV_DIR)/bin/python -m pip install --upgrade "pip==$(PIP_VERSION)" # Print out pip version $(VIRTUALENV_DIR)/bin/pip --version # Generate all requirements to support current CI pipeline. @@ -694,32 +694,33 @@ requirements: virtualenv .requirements .sdist-requirements install-runners insta # Note: Use the verison of virtualenv pinned in fixed-requirements.txt so we # only have to update it one place when we change the version - $(VIRTUALENV_DIR)/bin/pip install --upgrade $(shell grep "^virtualenv" fixed-requirements.txt) - $(VIRTUALENV_DIR)/bin/pip install --upgrade "setuptools==$(SETUPTOOLS_VERSION)" # workaround for pbr issue + # virtualenv is deprecated, use python -m venv + #$(VIRTUALENV_DIR)/bin/python -m pip install --upgrade $(shell grep "^virtualenv" fixed-requirements.txt) + $(VIRTUALENV_DIR)/bin/python -m pip install --upgrade "setuptools==$(SETUPTOOLS_VERSION)" # workaround for pbr issue # Install requirements for req in $(REQUIREMENTS); do \ echo "Installing $$req..." ; \ - $(VIRTUALENV_DIR)/bin/pip install $(PIP_OPTIONS) -r $$req ; \ + $(VIRTUALENV_DIR)/bin/python -m pip install $(PIP_OPTIONS) -r $$req ; \ done # Install st2common package to load drivers defined in st2common setup.py # NOTE: We pass --no-deps to the script so we don't install all the # package dependencies which are already installed as part of "requirements" # make targets. This speeds up the build - (cd ${ROOT_DIR}/st2common; ${ROOT_DIR}/$(VIRTUALENV_DIR)/bin/python setup.py develop --no-deps) + (cd ${ROOT_DIR}/st2common; ${ROOT_DIR}/$(VIRTUALENV_DIR)/bin/python -m pip install --editable . --no-deps) # Install st2common to register metrics drivers # NOTE: We pass --no-deps to the script so we don't install all the # package dependencies which are already installed as part of "requirements" # make targets. This speeds up the build - (cd ${ROOT_DIR}/st2common; ${ROOT_DIR}/$(VIRTUALENV_DIR)/bin/python setup.py develop --no-deps) + (cd ${ROOT_DIR}/st2common; ${ROOT_DIR}/$(VIRTUALENV_DIR)/bin/python -m pip install --editable . --no-deps) # Install st2auth to register SSO drivers # NOTE: We pass --no-deps to the script so we don't install all the # package dependencies which are already installed as part of "requirements" # make targets. This speeds up the build - (cd ${ROOT_DIR}/st2auth; ${ROOT_DIR}/$(VIRTUALENV_DIR)/bin/python setup.py develop --no-deps) + (cd ${ROOT_DIR}/st2auth; ${ROOT_DIR}/$(VIRTUALENV_DIR)/bin/python -m pip install --editable . --no-deps) # Some of the tests rely on submodule so we need to make sure submodules are check out git submodule update --init --recursive --remote @@ -819,22 +820,29 @@ unit-tests: requirements .unit-tests @echo @echo "----- Dropping st2-test db -----" @mongosh st2-test --eval "db.dropDatabase();" - @failed=0; \ + @failed=(); \ for component in $(COMPONENTS_TEST); do\ - echo "==========================================================="; \ - echo "Running tests in" $$component; \ - echo "-----------------------------------------------------------"; \ - . $(VIRTUALENV_DIR)/bin/activate; \ - ST2TESTS_REDIS_HOST=$(ST2TESTS_REDIS_HOST) \ - ST2TESTS_REDIS_PORT=$(ST2TESTS_REDIS_PORT) \ - pytest -rx --verbose \ - $$component/tests/unit || ((failed+=1)); \ - echo "-----------------------------------------------------------"; \ - echo "Done running tests in" $$component; \ - echo "==========================================================="; \ + if ls $$component/tests/unit/test_*.py 2>/dev/null >/dev/null; then \ + echo "==========================================================="; \ + echo "Running tests in" $$component; \ + echo "-----------------------------------------------------------"; \ + . $(VIRTUALENV_DIR)/bin/activate; \ + ST2TESTS_REDIS_HOST=$(ST2TESTS_REDIS_HOST) \ + ST2TESTS_REDIS_PORT=$(ST2TESTS_REDIS_PORT) \ + pytest -rx --verbose $$component/tests/unit; \ + RET=$$?; \ + test $$RET -eq 0 || failed+=($$component); \ + echo "-----------------------------------------------------------"; \ + echo "Done running tests in $$component ($$RET)"; \ + echo "==========================================================="; \ + else \ + echo "==========================================================="; \ + echo "Skip: no test files for $$component"; \ + echo "==========================================================="; \ + fi; \ done; \ - echo pytest runs failed=$$failed; \ - if [ $$failed -gt 0 ]; then exit 1; fi + echo "pytest runs failed=$${failed[@]}"; \ + if [[ $${#failed[@]} -gt 0 ]]; then exit 1; fi .PHONY: .run-unit-tests-coverage ifdef INCLUDE_TESTS_IN_COVERAGE @@ -1058,7 +1066,7 @@ packs-tests: requirements .packs-tests @echo "==================== packs-tests ====================" @echo # Install st2common to register metrics drivers - (cd ${ROOT_DIR}/st2common; ${ROOT_DIR}/$(VIRTUALENV_DIR)/bin/python setup.py develop --no-deps) + (cd ${ROOT_DIR}/st2common; ${ROOT_DIR}/$(VIRTUALENV_DIR)/bin/python -m pip install --editable . --no-deps) . $(VIRTUALENV_DIR)/bin/activate; find ${ROOT_DIR}/contrib/* -maxdepth 0 -type d -print0 | xargs -0 -I FILENAME ./st2common/bin/st2-run-pack-tests -c -t -x -p FILENAME @@ -1122,7 +1130,7 @@ cli: @echo @echo "=================== Building st2 client ===================" @echo - pushd $(CURDIR) && cd st2client && ((python setup.py develop || printf "\n\n!!! ERROR: BUILD FAILED !!!\n") || popd) + pushd $(CURDIR) && cd st2client && ((python -m pip install --editable . || printf "\n\n!!! ERROR: BUILD FAILED !!!\n") || popd) .PHONY: rpms rpms: diff --git a/fixed-requirements.txt b/fixed-requirements.txt index 6bb2f93314..56b067db6c 100644 --- a/fixed-requirements.txt +++ b/fixed-requirements.txt @@ -72,10 +72,10 @@ tooz==6.3.0 # Note: virtualenv embeds wheels for pip, wheel, and setuptools. So pinning virtualenv pins those as well. # virtualenv==20.30.0 (<21) has pip==25.0.1 wheel==0.45.1 setuptools==75.3.2 # lockfiles/st2.lock has pip==25.0.1 wheel==0.45.1 setuptools==75.3.2 -virtualenv==20.34.0 +pip==25.3 # This setuptools version number is in the Makefile, but CircleCI builds are pulling a version # that is incompatible with our logshipper fork. -setuptools<78 +setuptools==80.10.2 webob==1.8.9 webtest==3.0.1 zake==0.2.2 diff --git a/st2client/st2client/utils/httpclient.py b/st2client/st2client/utils/httpclient.py index ec7bebdf64..3cfea9392b 100644 --- a/st2client/st2client/utils/httpclient.py +++ b/st2client/st2client/utils/httpclient.py @@ -1,4 +1,4 @@ -# Copyright 2020 The StackStorm Authors. +# Copyright 2020-2026 The StackStorm Authors. # Copyright 2019 Extreme Networks, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -17,7 +17,7 @@ import json import logging -from pipes import quote as pquote +from shlex import quote import requests @@ -185,18 +185,18 @@ def _get_curl_line_for_request(self, request): if method in ["HEAD"]: parts.extend(["--head"]) else: - parts.extend(["-X", pquote(method)]) + parts.extend(["-X", quote(method)]) # headers for key, value in request.headers.items(): - parts.extend(["-H ", pquote("%s: %s" % (key, value))]) + parts.extend(["-H ", quote("%s: %s" % (key, value))]) # body if request.body: - parts.extend(["--data-binary", pquote(request.body)]) + parts.extend(["--data-binary", quote(request.body)]) # URL - parts.extend([pquote(request.url)]) + parts.extend([quote(request.url)]) curl_line = " ".join(parts) return curl_line diff --git a/st2common/st2common/models/utils/action_alias_utils.py b/st2common/st2common/models/utils/action_alias_utils.py index bf6d47c8b4..7dd0d9f2d0 100644 --- a/st2common/st2common/models/utils/action_alias_utils.py +++ b/st2common/st2common/models/utils/action_alias_utils.py @@ -1,4 +1,4 @@ -# Copyright 2020 The StackStorm Authors. +# Copyright 2020-2026 The StackStorm Authors. # Copyright 2019 Extreme Networks, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -17,17 +17,29 @@ import re import sys +if sys.version_info >= (3, 11): + from re._parser import ( # pylint: disable=E0611 + parse, + AT, + AT_BEGINNING, + AT_BEGINNING_STRING, + AT_END, + AT_END_STRING, + BRANCH, + SUBPATTERN, + ) +else: + from sre_parse import ( # pylint: disable=E0611 + parse, + AT, + AT_BEGINNING, + AT_BEGINNING_STRING, + AT_END, + AT_END_STRING, + BRANCH, + SUBPATTERN, + ) -from sre_parse import ( # pylint: disable=E0611 - parse, - AT, - AT_BEGINNING, - AT_BEGINNING_STRING, - AT_END, - AT_END_STRING, - BRANCH, - SUBPATTERN, -) from st2common.util.jinja import render_values from st2common.constants import keyvalue as kv_constants @@ -45,12 +57,7 @@ LOG = log.getLogger(__name__) -# Python 3 compatibility -if sys.version_info > (3,): - SUBPATTERN_INDEX = 3 -else: - SUBPATTERN_INDEX = 1 - +SUBPATTERN_INDEX = 3 class ActionAliasFormatParser(object): def __init__(self, alias_format=None, param_stream=None): diff --git a/st2common/tests/unit/test_action_alias_utils.py b/st2common/tests/unit/test_action_alias_utils.py index 1fc54359b0..ccfe204f30 100644 --- a/st2common/tests/unit/test_action_alias_utils.py +++ b/st2common/tests/unit/test_action_alias_utils.py @@ -1,4 +1,4 @@ -# Copyright 2020 The StackStorm Authors. +# Copyright 2020-2026 The StackStorm Authors. # Copyright 2019 Extreme Networks, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,14 +14,26 @@ # limitations under the License. from __future__ import absolute_import -from sre_parse import ( - parse, - AT, - AT_BEGINNING, - AT_BEGINNING_STRING, - AT_END, - AT_END_STRING, -) + +import sys +if sys.version_info >= (3, 11): + from re._parser import ( + parse, + AT, + AT_BEGINNING, + AT_BEGINNING_STRING, + AT_END, + AT_END_STRING, + ) +else: + from sre_parse import ( + parse, + AT, + AT_BEGINNING, + AT_BEGINNING_STRING, + AT_END, + AT_END_STRING, + ) from mock import Mock from unittest import TestCase from st2common.exceptions.content import ParseException