From 2961ed924f1695615b25fae1aa82dfa6dc47422b Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Wed, 20 Nov 2024 03:39:19 +0100 Subject: [PATCH] =?UTF-8?q?=F0=9F=93=A6=20Move=20plugin=20deps=20to=20cons?= =?UTF-8?q?traints?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Makefile | 5 +- requirements/README.md | 74 --- .../django-ansible-base-pinned-version.sh | 77 --- requirements/requirements.in | 73 --- requirements/requirements.txt | 538 ------------------ requirements/requirements_dev.txt | 31 - requirements/requirements_git.txt | 6 - requirements/requirements_tower_uninstall.txt | 1 - requirements/updater.sh | 111 ---- test-req/constraints.txt | 1 + test-req/direct.in | 1 + .../roles/dockerfile/templates/Dockerfile.j2 | 1 + 12 files changed, 7 insertions(+), 912 deletions(-) delete mode 100644 requirements/README.md delete mode 100755 requirements/django-ansible-base-pinned-version.sh delete mode 100644 requirements/requirements.in delete mode 100644 requirements/requirements.txt delete mode 100644 requirements/requirements_dev.txt delete mode 100644 requirements/requirements_git.txt delete mode 100644 requirements/requirements_tower_uninstall.txt delete mode 100755 requirements/updater.sh create mode 100644 test-req/constraints.txt create mode 100644 test-req/direct.in diff --git a/Makefile b/Makefile index e9a149564489..57f64f5db444 100644 --- a/Makefile +++ b/Makefile @@ -174,7 +174,10 @@ requirements_awx: virtualenv_awx if [[ "$(PIP_OPTIONS)" == *"--no-index"* ]]; then \ cat requirements/requirements.txt requirements/requirements_local.txt | $(VENV_BASE)/awx/bin/pip install $(PIP_OPTIONS) -r /dev/stdin ; \ else \ - cat requirements/requirements.txt requirements/requirements_git.txt | $(VENV_BASE)/awx/bin/pip install $(PIP_OPTIONS) --no-binary $(SRC_ONLY_PKGS) -r /dev/stdin ; \ + $(VENV_BASE)/awx/bin/pip install $(PIP_OPTIONS) --no-binary $(SRC_ONLY_PKGS) \ + -r requirements/requirements.txt \ + -r requirements/requirements_git.txt \ + -c requirements/git-constraints.txt ; \ fi $(VENV_BASE)/awx/bin/pip uninstall --yes -r requirements/requirements_tower_uninstall.txt diff --git a/requirements/README.md b/requirements/README.md deleted file mode 100644 index 4978d0bf4058..000000000000 --- a/requirements/README.md +++ /dev/null @@ -1,74 +0,0 @@ -# Dependency Management - -The `requirements.txt` file is generated from `requirements.in` and `requirements_git.txt`, using `pip-tools` and `pip-compile`. - -## How To Use - -Commands should be run in the awx container from inside the `./requirements` directory of the awx repository. - -### Upgrading or Adding Select Libraries - -If you need to add or upgrade one targeted library, then modify `requirements.in`, -then run the script: - -`./updater.sh run` - -#### Upgrading Unpinned Dependency - -If you require a new version of a dependency that does not have a pinned version -for a fix or feature, pin a minimum version in `requirements.in` and run `./updater.sh run`. For example, -replace the line `asgi-amqp` with `asgi-amqp>=1.1.4`, and consider leaving a -note. - -Then next time that a general upgrade is performed, the minimum version specifiers -can be removed, because `*.txt` files are upgraded to latest. - -### Upgrading Dependencies - -You can upgrade (`pip-compile --upgrade`) the dependencies by running - -`./updater.sh upgrade`. - -## Licenses and Source Files - -If any library has a change to its license with the upgrade, then the license for that library -inside of `licenses` needs to be updated. - -For libraries that have source distribution requirements (LGPL as an example), -a tarball of the library is kept along with the license. -To download the PyPI tarball, you can run this command: - -``` -pip download -d licenses/ --no-binary :all: --no-deps -``` - -Make sure to delete the old tarball if it is an upgrade. - -## UPGRADE BLOCKERs - -Anything pinned in `*.in` files involves additional manual work in -order to upgrade. Some information related to that work is outlined here. - -### django-oauth-toolkit - -Versions later than 1.4.1 throw an error about id_token_id, due to the -OpenID Connect work that was done in -https://github.com/jazzband/django-oauth-toolkit/pull/915. This may -be fixable by creating a migration on our end? - -### pip, setuptools and setuptools_scm - -If modifying these libraries make sure testing with the offline build is performed to confirm they are functionally working. -Versions need to match the versions used in the pip bootstrapping step -in the top-level Makefile. - -### cryptography - -If modifying this library make sure testing with the offline build is performed to confirm it is functionally working. - -## Library Notes - -### pexpect - -Version 4.8 makes us a little bit nervous with changes to `searchwindowsize` https://github.com/pexpect/pexpect/pull/579/files -Pin to `pexpect==4.7.x` until we have more time to move to `4.8` and test. diff --git a/requirements/django-ansible-base-pinned-version.sh b/requirements/django-ansible-base-pinned-version.sh deleted file mode 100755 index 84905fff4e3a..000000000000 --- a/requirements/django-ansible-base-pinned-version.sh +++ /dev/null @@ -1,77 +0,0 @@ -#!/bin/bash -set +x - -# CONSTANTS -export REGEX_LEFT='https://github.com/ansible/django-ansible-base@' -export REGEX_RIGHT='#egg=django-ansible-base' - -# GLOBALS -SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -REQ_FILE=$SCRIPT_DIR/requirements_git.txt - -# Pin Function -DESIRED_VERSION='' -Pin() -{ - export DESIRED_VERSION - perl -p -i -e 's/\Q$ENV{REGEX_LEFT}\E(.*?)\Q$ENV{REGEX_RIGHT}\E/$ENV{REGEX_LEFT}$ENV{DESIRED_VERSION}$ENV{REGEX_RIGHT}/g' $REQ_FILE -} - -# Current Function -Current() -{ - REQUIREMENTS_LINE=$(grep django-ansible-base $REQ_FILE) - - echo "$REQUIREMENTS_LINE" | perl -nE 'say $1 if /\Q$ENV{REGEX_LEFT}\E(.*?)\Q$ENV{REGEX_RIGHT}\E/' -} - - -Help() -{ - # Display Help - echo "" - echo "Help:" - echo "" - echo "Interact with django-ansible-base in $REQ_FILE." - echo "By default, output the current django-ansible-base pinned version." - echo - echo "Syntax: scriptTemplate [-s|h|v]" - echo "options:" - echo "s Set django-ansible-base version to pin to." - echo "h Print this Help." - echo "v Verbose mode." - echo -} - -if [ $# -eq 0 ]; then - Current - exit -fi - - -while getopts ":hs:" option; do - case $option in - h) # display Help - Help - exit - ;; - s) - DESIRED_VERSION=$OPTARG;; - :) - echo "Option -${OPTARG} requires an argument." - Help - exit 1 - ;; - \?) # Invalid option - echo "Error: Invalid option" - echo "" - Help - exit;; - esac -done - -if [ -n "$DESIRED_VERSION" ]; then - Pin - Current -fi - diff --git a/requirements/requirements.in b/requirements/requirements.in deleted file mode 100644 index e003a1cd59e9..000000000000 --- a/requirements/requirements.in +++ /dev/null @@ -1,73 +0,0 @@ -aiohttp>=3.9.4 # CVE-2024-30251 -ansi2html # Used to format the stdout from jobs into html for display -asciichartpy -asn1 -azure-identity -azure-keyvault -boto3 -botocore -channels -channels-redis -cryptography>=41.0.7 # CVE-2023-49083 -Cython<3 # due to https://github.com/yaml/pyyaml/pull/702 -daphne -distro -django==4.2.16 # CVE-2024-24680 -django-cors-headers -django-crum -django-extensions -django-guid -django-oauth-toolkit<2.0.0 # Version 2.0.0 has breaking changes that will need to be worked out before upgrading -django-polymorphic -django-pglocks -django-solo -django-split-settings -djangorestframework>=3.15.0 -djangorestframework-yaml -filelock -GitPython>=3.1.37 # CVE-2023-41040 -grpcio<1.63.0 # 1.63.0+ requires cython>=3 -irc -jinja2>=3.1.3 # CVE-2024-22195 -JSON-log-formatter -jsonschema -Markdown # used for formatting API help -maturin # pydantic-core build dep -msgpack<1.0.6 # 1.0.6+ requires cython>=3 -msrestazure -openshift -opentelemetry-api~=1.24 # new y streams can be drastically different, in a good way -opentelemetry-sdk~=1.24 -opentelemetry-instrumentation-logging -opentelemetry-exporter-otlp -pexpect==4.7.0 # see library notes -prometheus_client -psycopg -psutil -pygerduty -pyopenssl>=23.2.0 # resolve dep conflict from cryptography pin above -pyparsing==2.4.6 # Upgrading to v3 of pyparsing introduce errors on smart host filtering: Expected 'or' term, found 'or' (at char 15), (line:1, col:16) -python-daemon>3.0.0 -python-dsv-sdk>=1.0.4 -python-tss-sdk>=1.2.1 -pyyaml>=6.0.1 -pyzstd # otel collector log file compression library -receptorctl -sqlparse>=0.4.4 # Required by django https://github.com/ansible/awx/security/dependabot/96 -redis[hiredis] -requests -slack-sdk -twilio -twisted[tls]>=23.10.0 # CVE-2023-46137 -uWSGI -uwsgitop -wheel>=0.38.1 # CVE-2022-40898 -pip==21.2.4 # see UPGRADE BLOCKERs -setuptools # see UPGRADE BLOCKERs -setuptools_scm[toml] # see UPGRADE BLOCKERs, xmlsec build dep -setuptools-rust>=0.11.4 # cryptography build dep -pkgconfig>=1.5.1 # xmlsec build dep - needed for offline build - -# Temporarily added to use ansible-runner from git branch, to be removed -# when ansible-runner moves from requirements_git.txt to here -pbr diff --git a/requirements/requirements.txt b/requirements/requirements.txt deleted file mode 100644 index 69e323ffde1d..000000000000 --- a/requirements/requirements.txt +++ /dev/null @@ -1,538 +0,0 @@ -adal==1.2.7 - # via msrestazure -aiohttp==3.9.5 - # via - # -r /awx_devel/requirements/requirements.in - # aiohttp-retry - # twilio -aiohttp-retry==2.8.3 - # via twilio -aiosignal==1.3.1 - # via aiohttp -annotated-types==0.6.0 - # via pydantic -ansi2html==1.9.2 - # via -r /awx_devel/requirements/requirements.in -# git+https://github.com/ansible/ansible-runner.git@devel # git requirements installed separately - # via -r /awx_devel/requirements/requirements_git.txt -asciichartpy==1.5.25 - # via -r /awx_devel/requirements/requirements.in -asgiref==3.7.2 - # via - # channels - # channels-redis - # daphne - # django - # django-ansible-base - # django-cors-headers -asn1==2.7.0 - # via -r /awx_devel/requirements/requirements.in -attrs==23.2.0 - # via - # aiohttp - # automat - # jsonschema - # referencing - # service-identity - # twisted -autobahn==23.6.2 - # via daphne -autocommand==2.2.2 - # via jaraco-text -automat==22.10.0 - # via twisted -# awx-plugins-core @ git+https://git@github.com/ansible/awx-plugins.git@devel # git requirements installed separately - # via -r /awx_devel/requirements/requirements_git.txt -awx-plugins.interfaces @ git+https://github.com/ansible/awx_plugins.interfaces.git - # via -r /awx_devel/requirements/requirements_git.txt -azure-common==1.1.28 - # via - # azure-keyvault-certificates - # azure-keyvault-keys - # azure-keyvault-secrets -azure-core==1.30.0 - # via - # azure-identity - # azure-keyvault-certificates - # azure-keyvault-keys - # azure-keyvault-secrets - # msrest -azure-identity==1.15.0 - # via -r /awx_devel/requirements/requirements.in -azure-keyvault==4.2.0 - # via -r /awx_devel/requirements/requirements.in -azure-keyvault-certificates==4.7.0 - # via azure-keyvault -azure-keyvault-keys==4.8.0 - # via azure-keyvault -azure-keyvault-secrets==4.7.0 - # via azure-keyvault -boto3==1.34.47 - # via -r /awx_devel/requirements/requirements.in -botocore==1.34.47 - # via - # -r /awx_devel/requirements/requirements.in - # boto3 - # s3transfer -cachetools==5.3.2 - # via google-auth -# git+https://github.com/ansible/system-certifi.git@devel # git requirements installed separately - # via - # -r /awx_devel/requirements/requirements_git.txt - # kubernetes - # msrest - # requests -cffi==1.16.0 - # via cryptography -channels==3.0.5 - # via - # -r /awx_devel/requirements/requirements.in - # channels-redis -channels-redis==4.2.0 - # via -r /awx_devel/requirements/requirements.in -charset-normalizer==3.3.2 - # via requests -click==8.1.7 - # via receptorctl -constantly==23.10.4 - # via twisted -cryptography==41.0.7 - # via - # -r /awx_devel/requirements/requirements.in - # adal - # autobahn - # azure-identity - # azure-keyvault-keys - # django-ansible-base - # jwcrypto - # msal - # pyjwt - # pyopenssl - # service-identity -cython==0.29.37 - # via -r /awx_devel/requirements/requirements.in -daphne==3.0.2 - # via - # -r /awx_devel/requirements/requirements.in - # channels -deprecated==1.2.14 - # via - # opentelemetry-api - # opentelemetry-exporter-otlp-proto-grpc - # opentelemetry-exporter-otlp-proto-http -distro==1.9.0 - # via -r /awx_devel/requirements/requirements.in -django==4.2.16 - # via - # -r /awx_devel/requirements/requirements.in - # channels - # django-ansible-base - # django-cors-headers - # django-crum - # django-extensions - # django-guid - # django-oauth-toolkit - # django-polymorphic - # django-solo - # djangorestframework -# django-ansible-base @ git+https://github.com/ansible/django-ansible-base@devel # git requirements installed separately - # via -r /awx_devel/requirements/requirements_git.txt -django-cors-headers==4.3.1 - # via -r /awx_devel/requirements/requirements.in -django-crum==0.7.9 - # via - # -r /awx_devel/requirements/requirements.in - # django-ansible-base -django-extensions==3.2.3 - # via -r /awx_devel/requirements/requirements.in -django-guid==3.5.0 - # via -r /awx_devel/requirements/requirements.in -django-oauth-toolkit==1.7.1 - # via -r /awx_devel/requirements/requirements.in -django-pglocks==1.0.4 - # via -r /awx_devel/requirements/requirements.in -django-polymorphic==3.1.0 - # via -r /awx_devel/requirements/requirements.in -django-solo==2.2.0 - # via -r /awx_devel/requirements/requirements.in -django-split-settings==1.3.2 - # via - # -r /awx_devel/requirements/requirements.in - # django-ansible-base -djangorestframework==3.15.1 - # via - # -r /awx_devel/requirements/requirements.in - # django-ansible-base -djangorestframework-yaml==2.0.0 - # via -r /awx_devel/requirements/requirements.in -docutils==0.20.1 - # via python-daemon -enum-compat==0.0.3 - # via asn1 -filelock==3.13.1 - # via -r /awx_devel/requirements/requirements.in -frozenlist==1.4.1 - # via - # aiohttp - # aiosignal -gitdb==4.0.11 - # via gitpython -gitpython==3.1.42 - # via -r /awx_devel/requirements/requirements.in -google-auth==2.28.1 - # via kubernetes -googleapis-common-protos==1.63.0 - # via - # opentelemetry-exporter-otlp-proto-grpc - # opentelemetry-exporter-otlp-proto-http -grpcio==1.62.2 - # via - # -r /awx_devel/requirements/requirements.in - # opentelemetry-exporter-otlp-proto-grpc -hiredis==3.0.0 - # via redis -hyperlink==21.0.0 - # via - # autobahn - # twisted -idna==3.6 - # via - # hyperlink - # requests - # twisted - # yarl -importlib-metadata==6.2.1 - # via opentelemetry-api -incremental==22.10.0 - # via twisted -inflect==7.0.0 - # via jaraco-text -inflection==0.5.1 - # via django-ansible-base -irc==20.3.1 - # via -r /awx_devel/requirements/requirements.in -isodate==0.6.1 - # via - # azure-keyvault-certificates - # azure-keyvault-keys - # azure-keyvault-secrets - # msrest -jaraco-collections==5.0.0 - # via irc -jaraco-context==4.3.0 - # via jaraco-text -jaraco-functools==4.0.0 - # via - # irc - # jaraco-text - # tempora -jaraco-logging==3.3.0 - # via irc -jaraco-stream==3.0.3 - # via irc -jaraco-text==3.12.0 - # via - # irc - # jaraco-collections -jinja2==3.1.3 - # via -r /awx_devel/requirements/requirements.in -jmespath==1.0.1 - # via - # boto3 - # botocore -json-log-formatter==0.5.2 - # via -r /awx_devel/requirements/requirements.in -jsonschema==4.21.1 - # via -r /awx_devel/requirements/requirements.in -jsonschema-specifications==2023.12.1 - # via jsonschema -jwcrypto==1.5.4 - # via django-oauth-toolkit -kubernetes==29.0.0 - # via openshift -lockfile==0.12.2 - # via python-daemon -markdown==3.5.2 - # via -r /awx_devel/requirements/requirements.in -markupsafe==2.1.5 - # via jinja2 -maturin==1.5.0 - # via -r /awx_devel/requirements/requirements.in -more-itertools==10.2.0 - # via - # irc - # jaraco-functools - # jaraco-text -msal==1.26.0 - # via - # azure-identity - # msal-extensions -msal-extensions==1.1.0 - # via azure-identity -msgpack==1.0.5 - # via - # -r /awx_devel/requirements/requirements.in - # channels-redis -msrest==0.7.1 - # via msrestazure -msrestazure==0.6.4 - # via -r /awx_devel/requirements/requirements.in -multidict==6.0.5 - # via - # aiohttp - # yarl -oauthlib==3.2.2 - # via - # django-oauth-toolkit - # kubernetes - # requests-oauthlib -openshift==0.13.2 - # via -r /awx_devel/requirements/requirements.in -opentelemetry-api==1.24.0 - # via - # -r /awx_devel/requirements/requirements.in - # opentelemetry-exporter-otlp-proto-grpc - # opentelemetry-exporter-otlp-proto-http - # opentelemetry-instrumentation - # opentelemetry-instrumentation-logging - # opentelemetry-sdk -opentelemetry-exporter-otlp==1.24.0 - # via -r /awx_devel/requirements/requirements.in -opentelemetry-exporter-otlp-proto-common==1.24.0 - # via - # opentelemetry-exporter-otlp-proto-grpc - # opentelemetry-exporter-otlp-proto-http -opentelemetry-exporter-otlp-proto-grpc==1.24.0 - # via opentelemetry-exporter-otlp -opentelemetry-exporter-otlp-proto-http==1.24.0 - # via opentelemetry-exporter-otlp -opentelemetry-instrumentation==0.45b0 - # via opentelemetry-instrumentation-logging -opentelemetry-instrumentation-logging==0.45b0 - # via -r /awx_devel/requirements/requirements.in -opentelemetry-proto==1.24.0 - # via - # opentelemetry-exporter-otlp-proto-common - # opentelemetry-exporter-otlp-proto-grpc - # opentelemetry-exporter-otlp-proto-http -opentelemetry-sdk==1.24.0 - # via - # -r /awx_devel/requirements/requirements.in - # opentelemetry-exporter-otlp-proto-grpc - # opentelemetry-exporter-otlp-proto-http -opentelemetry-semantic-conventions==0.45b0 - # via opentelemetry-sdk -packaging==23.2 - # via - # ansible-runner - # msal-extensions - # setuptools-scm -pbr==6.0.0 - # via -r /awx_devel/requirements/requirements.in -pexpect==4.7.0 - # via - # -r /awx_devel/requirements/requirements.in - # ansible-runner -pkgconfig==1.5.5 - # via -r /awx_devel/requirements/requirements.in -portalocker==2.8.2 - # via msal-extensions -prometheus-client==0.20.0 - # via -r /awx_devel/requirements/requirements.in -protobuf==4.25.3 - # via - # googleapis-common-protos - # opentelemetry-proto -psutil==5.9.8 - # via -r /awx_devel/requirements/requirements.in -psycopg==3.1.18 - # via -r /awx_devel/requirements/requirements.in -ptyprocess==0.7.0 - # via pexpect -pyasn1==0.5.1 - # via - # pyasn1-modules - # rsa - # service-identity -pyasn1-modules==0.3.0 - # via - # google-auth - # service-identity -pycparser==2.21 - # via cffi -pydantic==2.5.0 - # via inflect -pydantic-core==2.14.1 - # via pydantic -pygerduty==0.38.3 - # via -r /awx_devel/requirements/requirements.in -pyjwt[crypto]==2.8.0 - # via - # adal - # django-ansible-base - # msal - # twilio -pyopenssl==24.0.0 - # via - # -r /awx_devel/requirements/requirements.in - # twisted -pyparsing==2.4.6 - # via -r /awx_devel/requirements/requirements.in -python-daemon==3.0.1 - # via - # -r /awx_devel/requirements/requirements.in - # ansible-runner -python-dateutil==2.8.2 - # via - # adal - # botocore - # kubernetes - # receptorctl -python-dsv-sdk==1.0.4 - # via -r /awx_devel/requirements/requirements.in -python-string-utils==1.0.0 - # via openshift -python-tss-sdk==1.2.2 - # via -r /awx_devel/requirements/requirements.in -pytz==2024.1 - # via - # irc - # tempora -pyyaml==6.0.1 - # via - # -r /awx_devel/requirements/requirements.in - # ansible-runner - # djangorestframework-yaml - # kubernetes - # receptorctl -pyzstd==0.15.10 - # via -r /awx_devel/requirements/requirements.in -receptorctl==1.4.4 - # via -r /awx_devel/requirements/requirements.in -redis[hiredis]==5.0.1 - # via - # -r /awx_devel/requirements/requirements.in - # channels-redis -referencing==0.33.0 - # via - # jsonschema - # jsonschema-specifications -requests==2.31.0 - # via - # -r /awx_devel/requirements/requirements.in - # adal - # azure-core - # django-ansible-base - # django-oauth-toolkit - # kubernetes - # msal - # msrest - # opentelemetry-exporter-otlp-proto-http - # python-dsv-sdk - # python-tss-sdk - # requests-oauthlib - # twilio -requests-oauthlib==1.3.1 - # via - # kubernetes - # msrest -rpds-py==0.18.0 - # via - # jsonschema - # referencing -rsa==4.9 - # via google-auth -s3transfer==0.10.0 - # via boto3 -semantic-version==2.10.0 - # via setuptools-rust -service-identity==24.1.0 - # via twisted -setuptools-rust==1.8.1 - # via -r /awx_devel/requirements/requirements.in -setuptools-scm[toml]==8.0.4 - # via -r /awx_devel/requirements/requirements.in -six==1.16.0 - # via - # automat - # azure-core - # django-pglocks - # isodate - # kubernetes - # msrestazure - # openshift - # pygerduty - # python-dateutil -slack-sdk==3.27.0 - # via -r /awx_devel/requirements/requirements.in -smmap==5.0.1 - # via gitdb -sqlparse==0.5.1 - # via - # -r /awx_devel/requirements/requirements.in - # django - # django-ansible-base -tempora==5.5.1 - # via - # irc - # jaraco-logging -twilio==8.13.0 - # via -r /awx_devel/requirements/requirements.in -twisted[tls]==23.10.0 - # via - # -r /awx_devel/requirements/requirements.in - # daphne -txaio==23.1.1 - # via autobahn -typing-extensions==4.9.0 - # via - # azure-core - # azure-keyvault-certificates - # azure-keyvault-keys - # azure-keyvault-secrets - # inflect - # jwcrypto - # opentelemetry-sdk - # psycopg - # pydantic - # pydantic-core - # setuptools-scm - # twisted -urllib3==1.26.18 - # via - # botocore - # django-ansible-base - # kubernetes - # requests -uwsgi==2.0.24 - # via -r /awx_devel/requirements/requirements.in -uwsgitop==0.11 - # via -r /awx_devel/requirements/requirements.in -websocket-client==1.7.0 - # via kubernetes -wheel==0.42.0 - # via -r /awx_devel/requirements/requirements.in -wrapt==1.16.0 - # via - # deprecated - # opentelemetry-instrumentation -yarl==1.9.4 - # via aiohttp -zipp==3.17.0 - # via importlib-metadata -zope-interface==6.2 - # via twisted - -# The following packages are considered to be unsafe in a requirements file: -pip==21.2.4 - # via -r /awx_devel/requirements/requirements.in -setuptools==69.0.2 - # via - # -r /awx_devel/requirements/requirements.in - # asciichartpy - # autobahn - # opentelemetry-instrumentation - # python-daemon - # setuptools-rust - # setuptools-scm - # zope-interface diff --git a/requirements/requirements_dev.txt b/requirements/requirements_dev.txt deleted file mode 100644 index 7dfc0f67a942..000000000000 --- a/requirements/requirements_dev.txt +++ /dev/null @@ -1,31 +0,0 @@ -build -coreapi -django-debug-toolbar==3.2.4 -django-test-migrations -drf-yasg -# pprofile - re-add once https://github.com/vpelletier/pprofile/issues/41 is addressed -ipython>=7.31.1 # https://github.com/ansible/awx/security/dependabot/30 -unittest2 -black -pytest!=7.0.0 -pytest-asyncio -pytest-cov -pytest-django -pytest-mock -pytest-timeout -pytest-xdist -tox # for awxkit -logutils -jupyter -# matplotlib - Caused issues when bumping to setuptools 58 -backports.tempfile # support in unit tests for py32+ tempfile.TemporaryDirectory -gprof2dot -atomicwrites -flake8 -yamllint -pip>=21.3,<=24.0 # PEP 660 – Editable installs for pyproject.toml based builds (wheel based) - -# python debuggers -debugpy -remote-pdb -sdb diff --git a/requirements/requirements_git.txt b/requirements/requirements_git.txt deleted file mode 100644 index f5aaccb8e3bd..000000000000 --- a/requirements/requirements_git.txt +++ /dev/null @@ -1,6 +0,0 @@ -git+https://github.com/ansible/system-certifi.git@devel#egg=certifi -# Remove pbr from requirements.in when moving ansible-runner to requirements.in -git+https://github.com/ansible/ansible-runner.git@devel#egg=ansible-runner -django-ansible-base @ git+https://github.com/ansible/django-ansible-base@devel#egg=django-ansible-base[rest_filters,jwt_consumer,resource_registry,rbac] -awx-plugins-core @ git+https://git@github.com/ansible/awx-plugins.git@devel#egg=awx-plugins-core -awx_plugins.interfaces @ git+https://github.com/ansible/awx_plugins.interfaces.git diff --git a/requirements/requirements_tower_uninstall.txt b/requirements/requirements_tower_uninstall.txt deleted file mode 100644 index 56cbaa5f19e1..000000000000 --- a/requirements/requirements_tower_uninstall.txt +++ /dev/null @@ -1 +0,0 @@ -rsa # stop adding new crypto libs diff --git a/requirements/updater.sh b/requirements/updater.sh deleted file mode 100755 index f7bbcafdd999..000000000000 --- a/requirements/updater.sh +++ /dev/null @@ -1,111 +0,0 @@ -#!/bin/sh -set -ue - -requirements_in="$(readlink -f ./requirements.in)" -requirements="$(readlink -f ./requirements.txt)" -requirements_git="$(readlink -f ./requirements_git.txt)" -requirements_dev="$(readlink -f ./requirements_dev.txt)" -pip_compile="pip-compile --no-header --quiet -r --allow-unsafe" -sanitize_git="1" - -_cleanup() { - cd / - test "${KEEP_TMP:-0}" = 1 || rm -rf "${_tmp}" -} - -generate_requirements() { - venv="`pwd`/venv" - echo $venv - /usr/bin/python3.11 -m venv "${venv}" - # shellcheck disable=SC1090 - source ${venv}/bin/activate - - # FIXME: https://github.com/jazzband/pip-tools/issues/1558 - ${venv}/bin/python3 -m pip install -U 'pip<22.0' pip-tools - - ${pip_compile} $1 --output-file requirements.txt - # consider the git requirements for purposes of resolving deps - # Then comment out any git+ lines from requirements.txt - if [[ "$sanitize_git" == "1" ]] ; then - while IFS= read -r line; do - if [[ $line != \#* ]]; then # ignore lines which are already comments - # Add # to the start of any line matched - sed -i "s!^.*${line%#*}!# ${line%#*} # git requirements installed separately!g" requirements.txt - fi - done < "${requirements_git}" - fi; -} - -main() { - base_dir=$(pwd) - dest_requirements="${requirements}" - input_requirements="${requirements_in} ${requirements_git}" - - _tmp=$(python -c "import tempfile; print(tempfile.mkdtemp(suffix='.awx-requirements', dir='/tmp'))") - - trap _cleanup INT TERM EXIT - - case $1 in - "run") - NEEDS_HELP=0 - ;; - "dev") - dest_requirements="${requirements_dev}" - input_requirements="${requirements_dev}" - sanitize_git=0 - NEEDS_HELP=0 - ;; - "upgrade") - NEEDS_HELP=0 - pip_compile="${pip_compile} --upgrade" - ;; - "help") - NEEDS_HELP=1 - ;; - *) - echo "" - echo "ERROR: Parameter $1 not valid" - echo "" - NEEDS_HELP=1 - ;; - esac - - if [[ "$NEEDS_HELP" == "1" ]] ; then - echo "This script generates requirements.txt from requirements.in and requirements_git.in" - echo "It should be run from within the awx container" - echo "" - echo "Usage: $0 [run|upgrade|dev]" - echo "" - echo "Commands:" - echo "help Print this message" - echo "run Run the process only upgrading pinned libraries from requirements.in" - echo "upgrade Upgrade all libraries to latest while respecting pinnings" - echo "dev Pin the development requirements file" - echo "" - exit - fi - - if [[ ! -d /awx_devel ]] ; then - echo "This script should be run inside the awx container" - exit - fi - - if [[ ! -z "$(tail -c 1 "${requirements_git}")" ]] - then - echo "No newline at end of ${requirements_git}, please add one" - exit - fi - - cp -vf requirements.txt "${_tmp}" - cd "${_tmp}" - - generate_requirements "${input_requirements}" - - echo "Changing $base_dir to /awx_devel/requirements" - cat requirements.txt | sed "s:$base_dir:/awx_devel/requirements:" > "${dest_requirements}" - - _cleanup -} - -# set EVAL=1 in case you want to source this script -test "${EVAL:-0}" -eq "1" || main "${1:-}" diff --git a/test-req/constraints.txt b/test-req/constraints.txt new file mode 100644 index 000000000000..b82ff555eb81 --- /dev/null +++ b/test-req/constraints.txt @@ -0,0 +1 @@ +awx-plugins.interfaces @ git+https://github.com/ansible/awx_plugins.interfaces.git diff --git a/test-req/direct.in b/test-req/direct.in new file mode 100644 index 000000000000..fd2a2e45705b --- /dev/null +++ b/test-req/direct.in @@ -0,0 +1 @@ +awx-plugins.interfaces diff --git a/tools/ansible/roles/dockerfile/templates/Dockerfile.j2 b/tools/ansible/roles/dockerfile/templates/Dockerfile.j2 index 0ab08ca6d108..065f17ad4219 100644 --- a/tools/ansible/roles/dockerfile/templates/Dockerfile.j2 +++ b/tools/ansible/roles/dockerfile/templates/Dockerfile.j2 @@ -74,6 +74,7 @@ RUN mkdir /tmp/requirements ADD requirements/requirements.txt \ requirements/requirements_tower_uninstall.txt \ requirements/requirements_git.txt \ + requirements/git-constraints.txt \ /tmp/requirements/ RUN cd /tmp && make requirements_awx