From 7047d0059d794920254956d2ef7d0fc42bd8cc05 Mon Sep 17 00:00:00 2001 From: Tim Reichard Date: Thu, 25 Jul 2024 16:37:51 -0500 Subject: [PATCH 1/3] Update mlflow, flask, flask-cors & werkzeug to remove vulnerability alert --- .pre-commit-config.yaml | 2 +- HISTORY.rst | 7 +++++++ aioradio/ds_utils.py | 27 --------------------------- aioradio/requirements.txt | 33 ++++++++++++++++----------------- conftest.py | 4 ++-- setup.py | 2 +- 6 files changed, 27 insertions(+), 48 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b52375b..dcb8cd9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,7 +2,7 @@ default_language_version: python: python3.11 repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.5.0 + rev: v4.6.0 hooks: - id: check-added-large-files - id: check-ast diff --git a/HISTORY.rst b/HISTORY.rst index 6a1b3e1..1d4f097 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -3,6 +3,13 @@ History ======= +v0.20.20 (2024-07-25) + +* Update mlflow, flask, flask-cors & werkzeug to remove vulnerability alert. +* Update python libs. +* Remove python lib dominodatalab. + + v0.20.19 (2024-05-06) * Add option to create polars LazyFrame from databricks query via pyarrow. diff --git a/aioradio/ds_utils.py b/aioradio/ds_utils.py index 6f18875..2dc0f12 100644 --- a/aioradio/ds_utils.py +++ b/aioradio/ds_utils.py @@ -529,33 +529,6 @@ def get_ftp_connection(secret_id, port=139, is_direct_tcp=False, env='sandbox'): return conn -def monitor_domino_run(domino, run_id, sleep_time=10): - """Monitor domino job run and return True/False depending if job was - successful.""" - - status = None - while status is None: - sleep(sleep_time) - result = domino.runs_status(run_id) - if result['status'] in ["Finished", "Succeeded"]: - status = True - break - if result['status'] in ["Failed", "Error"]: - status = False - break - - return status - - -def get_domino_connection(secret_id, project, host, env='sandbox'): - """Get domino connection.""" - - from domino import Domino - secret_client = get_boto3_session(env).client("secretsmanager", region_name='us-east-1') - api_key = secret_client.get_secret_value(SecretId=secret_id)['SecretString'] - return Domino(project=project, api_key=api_key, host=host) - - ######################## Postgres or MSSQL Connection Classes ####################### diff --git a/aioradio/requirements.txt b/aioradio/requirements.txt index 95ce58c..a96cbd7 100644 --- a/aioradio/requirements.txt +++ b/aioradio/requirements.txt @@ -1,29 +1,28 @@ -aioboto3==12.4.0 -aiojobs==1.2.1 +aioboto3==13.1.1 +aiojobs==1.3.0 backoff==2.2.1 -boto3==1.34.69 -botocore==1.34.69 +boto3==1.34.131 +botocore==1.34.131 cython==3.0.10 databricks-connect==14.3.1 ddtrace==2.6.5 -dominodatalab==1.4.0 -fakeredis==2.22.0 +fakeredis==2.23.3 faust-cchardet==2.1.19 -flask==2.1.2 -flask-cors==3.0.10 -haversine==2.8.0 +flask==3.0.3 +flask-cors==4.0.1 +haversine==2.8.1 httpx==0.27.0 mandrill==1.0.60 -mlflow==2.10.2 -moto==3.1.18 +mlflow==2.14.3 +moto==4.2.14 openpyxl==3.0.10 orjson==3.9.15 pandas==2.2.2 -polars==0.20.23 -pre-commit==3.7.0 +polars==1.2.1 +pre-commit==3.7.1 psycopg2-binary==2.9.9 pyarrow==15.0.2 -pylint==3.1.0 +pylint==3.2.6 pyodbc==5.1.0 --no-binary=pyodbc pysmb==1.2.9.1 pyspark==3.4.3 @@ -31,8 +30,8 @@ pytest==8.1.2 pytest-asyncio==0.21.1 pytest-cov==5.0.0 python-json-logger==2.0.7 -redis==5.0.4 -twine==5.0.0 +redis==5.0.7 +twine==5.1.1 typing_extensions==4.11.0 -werkzeug==2.1.2 +werkzeug==3.0.3 wheel==0.43.0 diff --git a/conftest.py b/conftest.py index 1b2e213..6d4723b 100644 --- a/conftest.py +++ b/conftest.py @@ -199,8 +199,8 @@ async def sqs_queue_url(sqs_client): async def _f(region_name, queue_name): nonlocal _queue_url - fifo = "true" if queue_name.lower().endswith('.fifo') else "false" - response = await sqs_client.create_queue(QueueName=queue_name, Attributes={"FifoQueue": fifo}) + attributes = {"FifoQueue": 'true', 'ContentBasedDeduplication': 'true'} if queue_name.lower().endswith('.fifo') else {} + response = await sqs_client.create_queue(QueueName=queue_name, Attributes=attributes) queue_url = response['QueueUrl'] _queue_url = queue_url assert_status_code(response, 200) diff --git a/setup.py b/setup.py index d0ed099..b4ab3b2 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ long_description = fileobj.read() setup(name='aioradio', - version='0.20.19', + version='0.20.20', description='Generic asynchronous i/o python utilities for AWS services (SQS, S3, DynamoDB, Secrets Manager), Redis, MSSQL (pyodbc), JIRA and more', long_description=long_description, long_description_content_type="text/markdown", From 50d3bd119da75b8bb03d8b5ac081ebd704e5b390 Mon Sep 17 00:00:00 2001 From: Tim Reichard Date: Thu, 25 Jul 2024 17:11:49 -0500 Subject: [PATCH 2/3] fix all dependency issues --- HISTORY.rst | 15 +++++++++++++++ aioradio/requirements.txt | 5 +++++ setup.py | 24 ++++++++++++++---------- 3 files changed, 34 insertions(+), 10 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 1d4f097..7e92487 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -3,6 +3,21 @@ History ======= +v0.20.23 (2024-07-25) + +* Update install requires grpcio==1.62.2 & grpcio-status==1.62.2. + + +v0.20.22 (2024-07-25) + +* Update install require numpy==1.26.4. + + +v0.20.21 (2024-07-25) + +* Fix broken dependencies. + + v0.20.20 (2024-07-25) * Update mlflow, flask, flask-cors & werkzeug to remove vulnerability alert. diff --git a/aioradio/requirements.txt b/aioradio/requirements.txt index a96cbd7..9942fe0 100644 --- a/aioradio/requirements.txt +++ b/aioradio/requirements.txt @@ -10,16 +10,21 @@ fakeredis==2.23.3 faust-cchardet==2.1.19 flask==3.0.3 flask-cors==4.0.1 +grpcio==1.62.2 +grpcio-status==1.62.2 haversine==2.8.1 httpx==0.27.0 mandrill==1.0.60 mlflow==2.14.3 moto==4.2.14 +numpy==1.26.4 openpyxl==3.0.10 orjson==3.9.15 pandas==2.2.2 +pkginfo==1.10.0 polars==1.2.1 pre-commit==3.7.1 +protobuf==4.25.4 psycopg2-binary==2.9.9 pyarrow==15.0.2 pylint==3.2.6 diff --git a/setup.py b/setup.py index b4ab3b2..d0bef81 100644 --- a/setup.py +++ b/setup.py @@ -3,11 +3,11 @@ from setuptools import setup -with open('README.md', 'r') as fileobj: +with open('README.md', 'r', encoding='utf8') as fileobj: long_description = fileobj.read() setup(name='aioradio', - version='0.20.20', + version='0.20.23', description='Generic asynchronous i/o python utilities for AWS services (SQS, S3, DynamoDB, Secrets Manager), Redis, MSSQL (pyodbc), JIRA and more', long_description=long_description, long_description_content_type="text/markdown", @@ -21,23 +21,27 @@ ], install_requires=[ 'cython>=0.29.33', - 'aioboto3==12.4.0', + 'aioboto3==13.1.1', 'aiojobs>=1.0.0', 'backoff>=2.1.2', - 'botocore==1.34.69', - 'boto3==1.34.69', + 'botocore==1.34.131', + 'boto3==1.34.131', 'ddtrace>=0.60.1', 'faust-cchardet>=2.1.18', 'fakeredis>=2.20.0', + 'grpcio==1.62.2', + 'grpcio-status==1.62.2', 'haversine>=2.8.0', 'httpx>=0.23.0', 'mandrill>=1.0.60', 'mlflow>=2.10.2', - 'numpy>=1.19', + 'numpy==1.26.4', 'openpyxl==3.0.10', 'orjson>=3.6.8', 'pandas>=1.3.5', + 'pkginfo==1.10.0' 'polars>=0.19.12', + 'protobuf==4.25.4', 'pyarrow>=13.0.0', 'pysmb>=1.2.7', 'python-json-logger>=2.0.2', @@ -45,16 +49,16 @@ ], include_package_data=True, tests_require=[ - 'flask==2.1.2', - 'flask-cors>=3.0.10', - 'moto==3.1.18', + 'flask==3.0.3', + 'flask-cors>=4.0.1', + 'moto==4.2.14', 'pre-commit>=2.15.0', 'pylint>=2.13.8', 'pytest>=7.0.1', 'pytest-asyncio>=0.15.1', 'pytest-cov>=3.0.0', 'typing_extensions>=4.10.0', - 'werkzeug==2.1.2' + 'werkzeug==3.0.3' ], zip_safe=False, classifiers=[ From a00037096964f7e0bfb56b31833b846e5bfe10c5 Mon Sep 17 00:00:00 2001 From: Tim Reichard Date: Thu, 25 Jul 2024 17:12:27 -0500 Subject: [PATCH 3/3] add missing comma in install requires --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index d0bef81..4dc304f 100644 --- a/setup.py +++ b/setup.py @@ -39,7 +39,7 @@ 'openpyxl==3.0.10', 'orjson>=3.6.8', 'pandas>=1.3.5', - 'pkginfo==1.10.0' + 'pkginfo==1.10.0', 'polars>=0.19.12', 'protobuf==4.25.4', 'pyarrow>=13.0.0',