From 7b7f6961c153c01bda0d99e1ab66b1c40f7a329a Mon Sep 17 00:00:00 2001 From: Matt Garber Date: Mon, 28 Oct 2024 14:33:54 -0400 Subject: [PATCH] Isolated lambdas, moved auth to secrets manager --- .github/workflows/ci.yaml | 2 +- .pre-commit-config.yaml | 7 ++- MAINTAINER.md | 14 +++-- pyproject.toml | 9 +++ scripts/credential_management.py | 30 +--------- src/dashboard/get_chart_data/__init__.py | 0 .../get_chart_data}/filter_config.py | 0 .../get_chart_data}/get_chart_data.py | 5 +- src/dashboard/get_chart_data/shared | 1 + src/dashboard/get_csv/__init__.py | 0 .../get_csv}/get_csv.py | 3 +- src/dashboard/get_csv/shared | 1 + src/dashboard/get_data_packages/__init__.py | 0 .../get_data_packages}/get_data_packages.py | 12 ++-- src/dashboard/get_data_packages/shared | 1 + src/dashboard/get_metadata/__init__.py | 0 .../get_metadata}/get_metadata.py | 5 +- src/dashboard/get_metadata/shared | 1 + src/dashboard/get_study_periods/__init__.py | 0 .../get_study_periods}/get_study_periods.py | 7 +-- src/dashboard/get_study_periods/shared | 1 + src/shared/__init__.py | 0 .../shared/awswrangler_functions.py | 2 +- src/{handlers => }/shared/decorators.py | 2 +- src/{handlers => }/shared/enums.py | 0 src/{handlers => }/shared/errors.py | 0 src/{handlers => }/shared/functions.py | 2 +- src/{handlers => }/shared/pandas_functions.py | 0 .../api_gateway_authorizer/__init__.py | 0 .../api_gateway_authorizer.py | 29 +++++++-- src/site_upload/cache_api/__init__.py | 0 .../cache_api}/cache_api.py | 3 +- src/site_upload/cache_api/shared | 1 + src/site_upload/fetch_upload_url/__init__.py | 0 .../fetch_upload_url}/fetch_upload_url.py | 7 +-- src/site_upload/fetch_upload_url/shared | 1 + src/site_upload/powerset_merge/__init__.py | 0 .../powerset_merge}/powerset_merge.py | 3 +- src/site_upload/powerset_merge/shared | 1 + src/site_upload/process_upload/__init__.py | 0 .../process_upload}/process_upload.py | 3 +- src/site_upload/process_upload/shared | 1 + src/site_upload/study_period/__init__.py | 0 src/site_upload/study_period/shared | 1 + .../study_period}/study_period.py | 9 ++- template.yaml | 59 ++++++++++++++----- tests/__init__.py | 3 + tests/conftest.py | 8 +-- tests/dashboard/test_filter_config.py | 4 +- tests/dashboard/test_get_chart_data.py | 6 +- tests/dashboard/test_get_csv.py | 4 +- tests/dashboard/test_get_data_packages.py | 14 +++-- tests/dashboard/test_get_metadata.py | 4 +- tests/dashboard/test_get_study_periods.py | 4 +- tests/mock_utils.py | 2 +- tests/shared/test_functions.py | 2 +- .../test_api_gateway_authorizer.py | 12 +++- tests/site_upload/test_cache_api.py | 4 +- tests/site_upload/test_fetch_upload_url.py | 4 +- tests/site_upload/test_powerset_merge.py | 4 +- tests/site_upload/test_process_upload.py | 33 +++++------ tests/site_upload/test_study_period.py | 13 ++-- 62 files changed, 198 insertions(+), 146 deletions(-) create mode 100644 src/dashboard/get_chart_data/__init__.py rename src/{handlers/dashboard => dashboard/get_chart_data}/filter_config.py (100%) rename src/{handlers/dashboard => dashboard/get_chart_data}/get_chart_data.py (98%) create mode 120000 src/dashboard/get_chart_data/shared create mode 100644 src/dashboard/get_csv/__init__.py rename src/{handlers/dashboard => dashboard/get_csv}/get_csv.py (98%) create mode 120000 src/dashboard/get_csv/shared create mode 100644 src/dashboard/get_data_packages/__init__.py rename src/{handlers/dashboard => dashboard/get_data_packages}/get_data_packages.py (66%) create mode 120000 src/dashboard/get_data_packages/shared create mode 100644 src/dashboard/get_metadata/__init__.py rename src/{handlers/dashboard => dashboard/get_metadata}/get_metadata.py (85%) create mode 120000 src/dashboard/get_metadata/shared create mode 100644 src/dashboard/get_study_periods/__init__.py rename src/{handlers/dashboard => dashboard/get_study_periods}/get_study_periods.py (78%) create mode 120000 src/dashboard/get_study_periods/shared create mode 100644 src/shared/__init__.py rename src/{handlers => }/shared/awswrangler_functions.py (95%) rename src/{handlers => }/shared/decorators.py (95%) rename src/{handlers => }/shared/enums.py (100%) rename src/{handlers => }/shared/errors.py (100%) rename src/{handlers => }/shared/functions.py (99%) rename src/{handlers => }/shared/pandas_functions.py (100%) create mode 100644 src/site_upload/api_gateway_authorizer/__init__.py rename src/{handlers/site_upload => site_upload/api_gateway_authorizer}/api_gateway_authorizer.py (92%) create mode 100644 src/site_upload/cache_api/__init__.py rename src/{handlers/site_upload => site_upload/cache_api}/cache_api.py (97%) create mode 120000 src/site_upload/cache_api/shared create mode 100644 src/site_upload/fetch_upload_url/__init__.py rename src/{handlers/site_upload => site_upload/fetch_upload_url}/fetch_upload_url.py (91%) create mode 120000 src/site_upload/fetch_upload_url/shared create mode 100644 src/site_upload/powerset_merge/__init__.py rename src/{handlers/site_upload => site_upload/powerset_merge}/powerset_merge.py (99%) create mode 120000 src/site_upload/powerset_merge/shared create mode 100644 src/site_upload/process_upload/__init__.py rename src/{handlers/site_upload => site_upload/process_upload}/process_upload.py (98%) create mode 120000 src/site_upload/process_upload/shared create mode 100644 src/site_upload/study_period/__init__.py create mode 120000 src/site_upload/study_period/shared rename src/{handlers/site_upload => site_upload/study_period}/study_period.py (90%) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 230bda6..2944fec 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -54,7 +54,7 @@ jobs: python -m pip install --upgrade pip pip install ".[dev]" - name: Run ruff - if: success() || failure() # still run black if above checks fails + if: success() || failure() # still run ruff if above checks fails run: | ruff check ruff format --check diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6fae921..ca474bd 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,10 +1,11 @@ default_install_hook_types: [pre-commit, pre-push] repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.2.1 + rev: v0.5.7 hooks: - - name: Ruff formatting - id: ruff-format - name: Ruff linting id: ruff + args: [ --fix ] + - name: Ruff formatting + id: ruff-format stages: [pre-push] diff --git a/MAINTAINER.md b/MAINTAINER.md index 9921029..8b1d1c3 100644 --- a/MAINTAINER.md +++ b/MAINTAINER.md @@ -14,17 +14,23 @@ If you're writing unit tests - note that moto support for Athena mocking is curr ## Managing upload credentials -In order to enable site uploads, we provide a [utility](../scripts/credential_management.py) to simplify managing S3-based dictionaries for authorizing pre-signed URLs. These artifacts should persist between cloudformation deployments, but if for some reason you need to delete the contents of the bucket and recreate, you can back up credentials by copying the contents of the admin/ virtual directory to another location. +For security reasons, we've migrated the credentials for users to AWS secrets manager. Passwords should be managed via +the secrets manager console. We provide a [utility](../scripts/credential_management.py) to simplify managing S3-based +dictionaries for authorizing pre-signed URLs, or generating the user strings for secrets manager. The S3 artifacts should +persist between cloudformation deployments, but if for some reason you need to delete the contents of the bucket and recreate, +you can back up credentials by copying the contents of the admin/ virtual directory to another location. -To create a new user, you would need to run the following two commands: +To create a user credential for secrets manager, you would need to run the following command: Creating a user: `./scripts/cumulus_upload_data.py --ca user_name auth_secret site_short_name` +To set up, or add to, a dictionary of short names to display names, you can run the following: + Associating a site with an s3 directory: `./scripts/cumulus_upload_data.py --cm site_short_name s3_folder_name` -These commands allow you to create a many to one relation of users to a given site, and a many to one relation of site to s3_upload_location, if so desired. +These commands allow you to create a many to one relation of users to a given site, if so desired. Without running these commands, no credentials are created by default, and so no access to pre-signed URLs is allowed. @@ -54,6 +60,6 @@ The SAM framework extends native cloudformation, usually with a lighter syntax, - If you modify S3 bucket permissions while in watch mode, changes to the bucket may generate a permission denied message. You'll need to delete the bucket and bring down the deployment before restarting to apply your changes. -- Similarly, if you end up in a ROLLBACK_FAILED state, usually the only recourse is to bring the deployment down and resync, or do a regular deployment deployment. +- Similarly, if you end up in a ROLLBACK_FAILED state, usually the only recourse is to bring the deployment down and resync, do a regular deployment deployment, or manually initiate a rollback from the AWS console. Using deploy is a little safer than sync in this regard, though it does take longer for each deployment. Use your best judgement. \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index cc609ea..db53087 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -57,6 +57,15 @@ omit = [ "*/api_gateway_authorizer.py", ] +[tool.pytest.ini_options] +pythonpath = [ + # we use this to get 'shared' and 'filter_config' as root level packages, which matches + # the packaging in the lambda environment, allowing unit tests to function + 'src/', + 'src/dashboard/get_chart_data', +] + + [tool.ruff] target-version = "py311" line-length = 100 diff --git a/scripts/credential_management.py b/scripts/credential_management.py index 369d54d..436b14a 100755 --- a/scripts/credential_management.py +++ b/scripts/credential_management.py @@ -26,26 +26,10 @@ def _put_s3_data(name: str, bucket_name: str, client, data: dict, path: str = "a client.upload_fileobj(Bucket=bucket_name, Key=f"{path}/{name}", Fileobj=b_data) -def create_auth(client, bucket_name: str, user: str, auth: str, site: str) -> str: +def create_auth(client, user: str, auth: str, site: str) -> str: """Adds a new entry to the auth dict used to issue pre-signed URLs""" - file = "auth.json" - auth_dict = _get_s3_data(file, bucket_name, client) site_id = _basic_auth_str(user, auth).split(" ")[1] - auth_dict[site_id] = {"site": site} - _put_s3_data(file, bucket_name, client, auth_dict) - return site_id - - -def delete_auth(client, bucket_name: str, site_id: str) -> bool: - """Removes an entry from the auth dict used to issue pre-signed urls""" - file = "auth.json" - auth_dict = _get_s3_data(file, bucket_name, client) - if site_id in auth_dict.keys(): - auth_dict.pop(site_id) - _put_s3_data(file, bucket_name, client, auth_dict) - return True - else: - return False + return f'"{site_id}"": {{"site":{site}}}' def create_meta(client, bucket_name: str, site: str, folder: str) -> None: @@ -110,19 +94,11 @@ def delete_meta(client, bucket_name: str, site: str) -> bool: bucket = f"{args.bucket}-{args.env}" if args.create_auth: id_str = create_auth( - s3_client, - bucket, args.create_auth[0], args.create_auth[1], args.create_auth[2], ) - print(f"{id_str} created") - elif args.delete_auth: - succeeded = delete_auth(s3_client, bucket, args.delete_auth) - if succeeded: - print(f"Removed {args.delete_auth}") - else: - print(f"{args.delete_auth} not found") + print(id_str) elif args.create_meta: create_meta(s3_client, bucket, args.create_meta[0], args.create_meta[1]) print(f"{args.create_meta[0]} mapped to S3 folder {args.create_meta[1]}") diff --git a/src/dashboard/get_chart_data/__init__.py b/src/dashboard/get_chart_data/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/handlers/dashboard/filter_config.py b/src/dashboard/get_chart_data/filter_config.py similarity index 100% rename from src/handlers/dashboard/filter_config.py rename to src/dashboard/get_chart_data/filter_config.py diff --git a/src/handlers/dashboard/get_chart_data.py b/src/dashboard/get_chart_data/get_chart_data.py similarity index 98% rename from src/handlers/dashboard/get_chart_data.py rename to src/dashboard/get_chart_data/get_chart_data.py index a68ba2b..e9baaa4 100644 --- a/src/handlers/dashboard/get_chart_data.py +++ b/src/dashboard/get_chart_data/get_chart_data.py @@ -7,10 +7,9 @@ import awswrangler import boto3 +import filter_config import pandas - -from src.handlers.dashboard import filter_config -from src.handlers.shared import decorators, enums, errors, functions +from shared import decorators, enums, errors, functions log_level = os.environ.get("LAMBDA_LOG_LEVEL", "INFO") logger = logging.getLogger() diff --git a/src/dashboard/get_chart_data/shared b/src/dashboard/get_chart_data/shared new file mode 120000 index 0000000..0ab0cb2 --- /dev/null +++ b/src/dashboard/get_chart_data/shared @@ -0,0 +1 @@ +../../shared \ No newline at end of file diff --git a/src/dashboard/get_csv/__init__.py b/src/dashboard/get_csv/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/handlers/dashboard/get_csv.py b/src/dashboard/get_csv/get_csv.py similarity index 98% rename from src/handlers/dashboard/get_csv.py rename to src/dashboard/get_csv/get_csv.py index 916d3b7..59000cd 100644 --- a/src/handlers/dashboard/get_csv.py +++ b/src/dashboard/get_csv/get_csv.py @@ -2,8 +2,7 @@ import boto3 import botocore - -from src.handlers.shared import decorators, enums, functions +from shared import decorators, enums, functions def _format_and_validate_key( diff --git a/src/dashboard/get_csv/shared b/src/dashboard/get_csv/shared new file mode 120000 index 0000000..0ab0cb2 --- /dev/null +++ b/src/dashboard/get_csv/shared @@ -0,0 +1 @@ +../../shared \ No newline at end of file diff --git a/src/dashboard/get_data_packages/__init__.py b/src/dashboard/get_data_packages/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/handlers/dashboard/get_data_packages.py b/src/dashboard/get_data_packages/get_data_packages.py similarity index 66% rename from src/handlers/dashboard/get_data_packages.py rename to src/dashboard/get_data_packages/get_data_packages.py index 1399a39..056861a 100644 --- a/src/handlers/dashboard/get_data_packages.py +++ b/src/dashboard/get_data_packages/get_data_packages.py @@ -2,19 +2,17 @@ import os -from src.handlers.shared.decorators import generic_error_handler -from src.handlers.shared.enums import BucketPath, JsonFilename -from src.handlers.shared.functions import get_s3_json_as_dict, http_response +from shared import decorators, enums, functions -@generic_error_handler(msg="Error retrieving data packages") +@decorators.generic_error_handler(msg="Error retrieving data packages") def data_packages_handler(event, context): """Retrieves list of data packages from S3.""" del context status = 200 - data_packages = get_s3_json_as_dict( + data_packages = functions.get_s3_json_as_dict( os.environ.get("BUCKET_NAME"), - f"{BucketPath.CACHE.value}/{JsonFilename.DATA_PACKAGES.value}.json", + f"{enums.BucketPath.CACHE.value}/{enums.JsonFilename.DATA_PACKAGES.value}.json", ) payload = data_packages if event.get("queryStringParameters"): @@ -33,5 +31,5 @@ def data_packages_handler(event, context): else: status = 404 payload = None - res = http_response(status, payload, allow_cors=True) + res = functions.http_response(status, payload, allow_cors=True) return res diff --git a/src/dashboard/get_data_packages/shared b/src/dashboard/get_data_packages/shared new file mode 120000 index 0000000..0ab0cb2 --- /dev/null +++ b/src/dashboard/get_data_packages/shared @@ -0,0 +1 @@ +../../shared \ No newline at end of file diff --git a/src/dashboard/get_metadata/__init__.py b/src/dashboard/get_metadata/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/handlers/dashboard/get_metadata.py b/src/dashboard/get_metadata/get_metadata.py similarity index 85% rename from src/handlers/dashboard/get_metadata.py rename to src/dashboard/get_metadata/get_metadata.py index b8791ba..477433d 100644 --- a/src/handlers/dashboard/get_metadata.py +++ b/src/dashboard/get_metadata/get_metadata.py @@ -3,9 +3,8 @@ import os import boto3 - -from src.handlers.shared.decorators import generic_error_handler -from src.handlers.shared.functions import http_response, read_metadata +from shared.decorators import generic_error_handler +from shared.functions import http_response, read_metadata @generic_error_handler(msg="Error retrieving metadata") diff --git a/src/dashboard/get_metadata/shared b/src/dashboard/get_metadata/shared new file mode 120000 index 0000000..0ab0cb2 --- /dev/null +++ b/src/dashboard/get_metadata/shared @@ -0,0 +1 @@ +../../shared \ No newline at end of file diff --git a/src/dashboard/get_study_periods/__init__.py b/src/dashboard/get_study_periods/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/handlers/dashboard/get_study_periods.py b/src/dashboard/get_study_periods/get_study_periods.py similarity index 78% rename from src/handlers/dashboard/get_study_periods.py rename to src/dashboard/get_study_periods/get_study_periods.py index 9ef6674..8659a11 100644 --- a/src/handlers/dashboard/get_study_periods.py +++ b/src/dashboard/get_study_periods/get_study_periods.py @@ -3,10 +3,9 @@ import os import boto3 - -from src.handlers.shared.decorators import generic_error_handler -from src.handlers.shared.enums import JsonFilename -from src.handlers.shared.functions import http_response, read_metadata +from shared.decorators import generic_error_handler +from shared.enums import JsonFilename +from shared.functions import http_response, read_metadata @generic_error_handler(msg="Error retrieving study period") diff --git a/src/dashboard/get_study_periods/shared b/src/dashboard/get_study_periods/shared new file mode 120000 index 0000000..0ab0cb2 --- /dev/null +++ b/src/dashboard/get_study_periods/shared @@ -0,0 +1 @@ +../../shared \ No newline at end of file diff --git a/src/shared/__init__.py b/src/shared/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/handlers/shared/awswrangler_functions.py b/src/shared/awswrangler_functions.py similarity index 95% rename from src/handlers/shared/awswrangler_functions.py rename to src/shared/awswrangler_functions.py index b04a8da..5ed716a 100644 --- a/src/handlers/shared/awswrangler_functions.py +++ b/src/shared/awswrangler_functions.py @@ -2,7 +2,7 @@ import awswrangler -from src.handlers.shared.enums import BucketPath +from .enums import BucketPath def get_s3_data_package_list( diff --git a/src/handlers/shared/decorators.py b/src/shared/decorators.py similarity index 95% rename from src/handlers/shared/decorators.py rename to src/shared/decorators.py index b1ae7c1..c325fc3 100644 --- a/src/handlers/shared/decorators.py +++ b/src/shared/decorators.py @@ -3,7 +3,7 @@ import functools import logging -from src.handlers.shared.functions import http_response +from .functions import http_response def generic_error_handler(msg="Internal server error"): diff --git a/src/handlers/shared/enums.py b/src/shared/enums.py similarity index 100% rename from src/handlers/shared/enums.py rename to src/shared/enums.py diff --git a/src/handlers/shared/errors.py b/src/shared/errors.py similarity index 100% rename from src/handlers/shared/errors.py rename to src/shared/errors.py diff --git a/src/handlers/shared/functions.py b/src/shared/functions.py similarity index 99% rename from src/handlers/shared/functions.py rename to src/shared/functions.py index 3be0b7b..68b40cb 100644 --- a/src/handlers/shared/functions.py +++ b/src/shared/functions.py @@ -7,7 +7,7 @@ import boto3 -from src.handlers.shared import enums +from . import enums TRANSACTION_METADATA_TEMPLATE = { enums.TransactionKeys.TRANSACTION_FORMAT_VERSION.value: "2", diff --git a/src/handlers/shared/pandas_functions.py b/src/shared/pandas_functions.py similarity index 100% rename from src/handlers/shared/pandas_functions.py rename to src/shared/pandas_functions.py diff --git a/src/site_upload/api_gateway_authorizer/__init__.py b/src/site_upload/api_gateway_authorizer/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/handlers/site_upload/api_gateway_authorizer.py b/src/site_upload/api_gateway_authorizer/api_gateway_authorizer.py similarity index 92% rename from src/handlers/site_upload/api_gateway_authorizer.py rename to src/site_upload/api_gateway_authorizer/api_gateway_authorizer.py index 71e9590..67a6414 100644 --- a/src/handlers/site_upload/api_gateway_authorizer.py +++ b/src/site_upload/api_gateway_authorizer/api_gateway_authorizer.py @@ -4,23 +4,42 @@ # pylint: disable=invalid-name,pointless-string-statement +import json import os import re -from src.handlers.shared.enums import BucketPath -from src.handlers.shared.functions import get_s3_json_as_dict +import boto3 +from botocore.exceptions import ClientError class AuthError(Exception): pass +def get_secret(): + """Retrieves a specified secret. + + This is largely unmodified boilerplate from the secrets manager recommended approach + for fetching secrets, except for getting the values from environment variables""" + + secret_name = os.environ.get("SECRET_NAME") + region_name = os.environ.get("REGION") + + session = boto3.session.Session() + client = session.client(service_name="secretsmanager", region_name=region_name) + + try: + get_secret_value_response = client.get_secret_value(SecretId=secret_name) + except ClientError as e: + raise e + + return json.loads(get_secret_value_response["SecretString"]) + + def lambda_handler(event, context): del context # ---- aggregator specific logic - user_db = get_s3_json_as_dict( - os.environ.get("BUCKET_NAME"), f"{BucketPath.ADMIN.value}/auth.json" - ) + user_db = get_secret() try: auth_header = event["headers"]["Authorization"].split(" ") auth_token = auth_header[1] diff --git a/src/site_upload/cache_api/__init__.py b/src/site_upload/cache_api/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/handlers/site_upload/cache_api.py b/src/site_upload/cache_api/cache_api.py similarity index 97% rename from src/handlers/site_upload/cache_api.py rename to src/site_upload/cache_api/cache_api.py index 641e04c..fa24a62 100644 --- a/src/handlers/site_upload/cache_api.py +++ b/src/site_upload/cache_api/cache_api.py @@ -5,8 +5,7 @@ import awswrangler import boto3 - -from src.handlers.shared import decorators, enums, functions +from shared import decorators, enums, functions def cache_api_data(s3_client, s3_bucket_name: str, db: str, target: str) -> None: diff --git a/src/site_upload/cache_api/shared b/src/site_upload/cache_api/shared new file mode 120000 index 0000000..0ab0cb2 --- /dev/null +++ b/src/site_upload/cache_api/shared @@ -0,0 +1 @@ +../../shared \ No newline at end of file diff --git a/src/site_upload/fetch_upload_url/__init__.py b/src/site_upload/fetch_upload_url/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/handlers/site_upload/fetch_upload_url.py b/src/site_upload/fetch_upload_url/fetch_upload_url.py similarity index 91% rename from src/handlers/site_upload/fetch_upload_url.py rename to src/site_upload/fetch_upload_url/fetch_upload_url.py index 4ac16ef..8dbb126 100644 --- a/src/handlers/site_upload/fetch_upload_url.py +++ b/src/site_upload/fetch_upload_url/fetch_upload_url.py @@ -6,10 +6,9 @@ import boto3 import botocore.exceptions - -from src.handlers.shared.decorators import generic_error_handler -from src.handlers.shared.enums import BucketPath -from src.handlers.shared.functions import get_s3_json_as_dict, http_response +from shared.decorators import generic_error_handler +from shared.enums import BucketPath +from shared.functions import get_s3_json_as_dict, http_response def create_presigned_post( diff --git a/src/site_upload/fetch_upload_url/shared b/src/site_upload/fetch_upload_url/shared new file mode 120000 index 0000000..0ab0cb2 --- /dev/null +++ b/src/site_upload/fetch_upload_url/shared @@ -0,0 +1 @@ +../../shared \ No newline at end of file diff --git a/src/site_upload/powerset_merge/__init__.py b/src/site_upload/powerset_merge/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/handlers/site_upload/powerset_merge.py b/src/site_upload/powerset_merge/powerset_merge.py similarity index 99% rename from src/handlers/site_upload/powerset_merge.py rename to src/site_upload/powerset_merge/powerset_merge.py index 18b6786..563f0a6 100644 --- a/src/handlers/site_upload/powerset_merge.py +++ b/src/site_upload/powerset_merge/powerset_merge.py @@ -11,8 +11,7 @@ import numpy import pandas from pandas.core.indexes.range import RangeIndex - -from src.handlers.shared import ( +from shared import ( awswrangler_functions, decorators, enums, diff --git a/src/site_upload/powerset_merge/shared b/src/site_upload/powerset_merge/shared new file mode 120000 index 0000000..0ab0cb2 --- /dev/null +++ b/src/site_upload/powerset_merge/shared @@ -0,0 +1 @@ +../../shared \ No newline at end of file diff --git a/src/site_upload/process_upload/__init__.py b/src/site_upload/process_upload/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/handlers/site_upload/process_upload.py b/src/site_upload/process_upload/process_upload.py similarity index 98% rename from src/handlers/site_upload/process_upload.py rename to src/site_upload/process_upload/process_upload.py index 1f420a5..b09aa96 100644 --- a/src/handlers/site_upload/process_upload.py +++ b/src/site_upload/process_upload/process_upload.py @@ -4,8 +4,7 @@ import os import boto3 - -from src.handlers.shared import decorators, enums, functions +from shared import decorators, enums, functions log_level = os.environ.get("LAMBDA_LOG_LEVEL", "INFO") logger = logging.getLogger() diff --git a/src/site_upload/process_upload/shared b/src/site_upload/process_upload/shared new file mode 120000 index 0000000..0ab0cb2 --- /dev/null +++ b/src/site_upload/process_upload/shared @@ -0,0 +1 @@ +../../shared \ No newline at end of file diff --git a/src/site_upload/study_period/__init__.py b/src/site_upload/study_period/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/site_upload/study_period/shared b/src/site_upload/study_period/shared new file mode 120000 index 0000000..0ab0cb2 --- /dev/null +++ b/src/site_upload/study_period/shared @@ -0,0 +1 @@ +../../shared \ No newline at end of file diff --git a/src/handlers/site_upload/study_period.py b/src/site_upload/study_period/study_period.py similarity index 90% rename from src/handlers/site_upload/study_period.py rename to src/site_upload/study_period/study_period.py index 3d20ebf..e6e4a63 100644 --- a/src/handlers/site_upload/study_period.py +++ b/src/site_upload/study_period/study_period.py @@ -5,11 +5,10 @@ import awswrangler import boto3 - -from src.handlers.shared.awswrangler_functions import get_s3_study_meta_list -from src.handlers.shared.decorators import generic_error_handler -from src.handlers.shared.enums import JsonFilename, StudyPeriodMetadataKeys -from src.handlers.shared.functions import ( +from shared.awswrangler_functions import get_s3_study_meta_list +from shared.decorators import generic_error_handler +from shared.enums import JsonFilename, StudyPeriodMetadataKeys +from shared.functions import ( http_response, read_metadata, update_metadata, diff --git a/template.yaml b/template.yaml index 2feeb5b..c940248 100644 --- a/template.yaml +++ b/template.yaml @@ -76,7 +76,8 @@ Resources: Type: AWS::Serverless::Function Properties: FunctionName: !Sub 'CumulusAggFetchAuthorizer-${DeployStage}' - Handler: src/handlers/site_upload/api_gateway_authorizer.lambda_handler + CodeUri: ./src/site_upload/api_gateway_authorizer + Handler: api_gateway_authorizer.lambda_handler Runtime: "python3.11" LoggingConfig: ApplicationLogLevel: !Ref LogLevel @@ -87,11 +88,9 @@ Resources: Description: Validates credentials before providing signed urls Environment: Variables: - BUCKET_NAME: !Sub '${BucketNameParameter}-${AWS::AccountId}-${DeployStage}' + SECRET_NAME: !Sub 'CumulusUserSecrets-${DeployStage}' REGION: !Ref "AWS::Region" Policies: - - S3ReadPolicy: - BucketName: !Sub '${BucketNameParameter}-${AWS::AccountId}-${DeployStage}' - Statement: - Sid: KMSDecryptPolicy Effect: Allow @@ -99,6 +98,14 @@ Resources: - kms:Decrypt Resource: - !ImportValue cumulus-kms-KMSKeyArn + - Statement: + - Sid: SecretsPolicy + Effect: Allow + Action: + - secretsmanager:GetSecretValue + Resource: + - !Ref UserSecrets + FetchAuthorizerLogGroup: Type: AWS::Logs::LogGroup @@ -110,7 +117,8 @@ Resources: Type: AWS::Serverless::Function Properties: FunctionName: !Sub 'CumulusAggFetchUploadUrl-${DeployStage}' - Handler: src/handlers/site_upload/fetch_upload_url.upload_url_handler + CodeUri: ./src/site_upload/fetch_upload_url + Handler: fetch_upload_url.upload_url_handler Runtime: "python3.11" LoggingConfig: ApplicationLogLevel: !Ref LogLevel @@ -151,7 +159,8 @@ Resources: Type: AWS::Serverless::Function Properties: FunctionName: !Sub 'CumulusAggProcessUpload-${DeployStage}' - Handler: src/handlers/site_upload/process_upload.process_upload_handler + CodeUri: ./src/site_upload/process_upload + Handler: process_upload.process_upload_handler Runtime: "python3.11" LoggingConfig: ApplicationLogLevel: !Ref LogLevel @@ -191,7 +200,8 @@ Resources: Properties: FunctionName: !Sub 'CumulusAggPowersetMerge-${DeployStage}' Layers: [arn:aws:lambda:us-east-1:336392948345:layer:AWSSDKPandas-Python311:17] - Handler: src/handlers/site_upload/powerset_merge.powerset_merge_handler + CodeUri: ./src/site_upload/powerset_merge + Handler: powerset_merge.powerset_merge_handler Runtime: "python3.11" LoggingConfig: ApplicationLogLevel: !Ref LogLevel @@ -233,7 +243,8 @@ Resources: Properties: FunctionName: !Sub 'CumulusAggStudyPeriod-${DeployStage}' Layers: [arn:aws:lambda:us-east-1:336392948345:layer:AWSSDKPandas-Python311:17] - Handler: src/handlers/site_upload/study_period.study_period_handler + CodeUri: ./src/site_upload/study_period + Handler: study_period.study_period_handler Runtime: "python3.11" LoggingConfig: ApplicationLogLevel: !Ref LogLevel @@ -272,7 +283,8 @@ Resources: Properties: FunctionName: !Sub 'CumulusAggCacheAPI-${DeployStage}' Layers: [arn:aws:lambda:us-east-1:336392948345:layer:AWSSDKPandas-Python311:17] - Handler: src/handlers/site_upload/cache_api.cache_api_handler + CodeUri: ./src/site_upload/cache_api + Handler: cache_api.cache_api_handler Runtime: "python3.11" LoggingConfig: ApplicationLogLevel: !Ref LogLevel @@ -331,7 +343,8 @@ Resources: Properties: FunctionName: !Sub 'CumulusAggDashboardGetChartData-${DeployStage}' Layers: [arn:aws:lambda:us-east-1:336392948345:layer:AWSSDKPandas-Python311:17] - Handler: src/handlers/dashboard/get_chart_data.chart_data_handler + CodeUri: ./src/dashboard/get_chart_data + Handler: get_chart_data.chart_data_handler Runtime: "python3.11" LoggingConfig: ApplicationLogLevel: !Ref LogLevel @@ -396,7 +409,8 @@ Resources: Type: AWS::Serverless::Function Properties: FunctionName: !Sub 'CumulusAggDashboardGetCsv-${DeployStage}' - Handler: src/handlers/dashboard/get_csv.get_csv_handler + CodeUri: ./src/dashboard/get_csv + Handler: get_csv.get_csv_handler Runtime: "python3.11" LoggingConfig: ApplicationLogLevel: !Ref LogLevel @@ -442,7 +456,8 @@ Resources: Type: AWS::Serverless::Function Properties: FunctionName: !Sub 'CumulusAggDashboardGetCsvList-${DeployStage}' - Handler: src/handlers/dashboard/get_csv.get_csv_list_handler + CodeUri: ./src/dashboard/get_csv + Handler: get_csv.get_csv_list_handler Runtime: "python3.11" LoggingConfig: ApplicationLogLevel: !Ref LogLevel @@ -489,7 +504,8 @@ Resources: Type: AWS::Serverless::Function Properties: FunctionName: !Sub 'CumulusAggDashboardGetMetadata-${DeployStage}' - Handler: src/handlers/dashboard/get_metadata.metadata_handler + CodeUri: ./src/dashboard/get_metadata + Handler: get_metadata.metadata_handler Runtime: "python3.11" LoggingConfig: ApplicationLogLevel: !Ref LogLevel @@ -554,7 +570,8 @@ Resources: Properties: FunctionName: !Sub 'CumulusAggDashboardDataPackages-${DeployStage}' Layers: [arn:aws:lambda:us-east-1:336392948345:layer:AWSSDKPandas-Python311:17] - Handler: src/handlers/dashboard/get_data_packages.data_packages_handler + CodeUri: ./src/dashboard/get_data_packages + Handler: get_data_packages.data_packages_handler Runtime: "python3.11" LoggingConfig: ApplicationLogLevel: !Ref LogLevel @@ -630,7 +647,8 @@ Resources: Type: AWS::Serverless::Function Properties: FunctionName: !Sub 'CumulusAggDashboardStudyPeriods-${DeployStage}' - Handler: src/handlers/dashboard/get_study_periods.study_periods_handler + CodeUri: ./src/dashboard/get_study_periods + Handler: get_study_periods.study_periods_handler Runtime: "python3.11" LoggingConfig: ApplicationLogLevel: !Ref LogLevel @@ -775,6 +793,17 @@ Resources: ResultConfiguration: OutputLocation: !Sub "s3://${BucketNameParameter}-${AWS::AccountId}-${DeployStage}/athena/" +### User login secrets + + UserSecrets: + Type: AWS::SecretsManager::Secret + Properties: + Name: !Sub 'CumulusUserSecrets-${DeployStage}' + Description: Upload credentials for Cumulus Users + # By default, we'll initiate this to an empty JSON object, and populate it via other methods. + SecretString: '{}' + KmsKeyId: !ImportValue cumulus-kms-KMSKeyArn + ### IAM Roles FetchAuthorizerRole: diff --git a/tests/__init__.py b/tests/__init__.py index e69de29..5ae623e 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -0,0 +1,3 @@ +from src import shared + +__all__ = ["shared"] diff --git a/tests/conftest.py b/tests/conftest.py index a7abc6e..6d39bc4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -30,7 +30,7 @@ from moto import mock_athena, mock_s3, mock_sns from scripts import credential_management -from src.handlers.shared import enums, functions +from src.shared import enums, functions from tests import mock_utils @@ -95,13 +95,13 @@ def mock_bucket(): for param_list in aggregate_params: _init_mock_data(s3_client, bucket, *param_list) - credential_management.create_auth(s3_client, bucket, "ppth_1", "test_1", "ppth") + # credential_management.create_auth(s3_client, bucket, "ppth_1", "test_1", "ppth") credential_management.create_meta( s3_client, bucket, "ppth", "princeton_plainsboro_teaching_hospital" ) - credential_management.create_auth(s3_client, bucket, "elsewhere_2", "test_2", "elsewhere") + # credential_management.create_auth(s3_client, bucket, "elsewhere_2", "test_2", "elsewhere") credential_management.create_meta(s3_client, bucket, "elsewhere", "st_elsewhere") - credential_management.create_auth(s3_client, bucket, "hope_3", "test_3", "hope") + # credential_management.create_auth(s3_client, bucket, "hope_3", "test_3", "hope") credential_management.create_meta(s3_client, bucket, "hope", "chicago_hope") metadata = mock_utils.get_mock_metadata() diff --git a/tests/dashboard/test_filter_config.py b/tests/dashboard/test_filter_config.py index d3d23dc..e4e3dd9 100644 --- a/tests/dashboard/test_filter_config.py +++ b/tests/dashboard/test_filter_config.py @@ -1,6 +1,6 @@ import pytest -from src.handlers.dashboard.filter_config import get_filter_string +from src.dashboard.get_chart_data import filter_config @pytest.mark.parametrize( @@ -136,4 +136,4 @@ ], ) def test_filter_string(input_str, output_str): - assert get_filter_string(input_str) == output_str + assert filter_config.get_filter_string(input_str) == output_str diff --git a/tests/dashboard/test_get_chart_data.py b/tests/dashboard/test_get_chart_data.py index 0f6c9c5..62a1ae7 100644 --- a/tests/dashboard/test_get_chart_data.py +++ b/tests/dashboard/test_get_chart_data.py @@ -5,7 +5,7 @@ import pandas import pytest -from src.handlers.dashboard import get_chart_data +from src.dashboard.get_chart_data import get_chart_data from tests.mock_utils import ( EXISTING_DATA_P, EXISTING_STUDY, @@ -26,7 +26,7 @@ def mock_data_frame(filter_param): return df -@mock.patch("src.handlers.dashboard.get_chart_data._get_table_cols", mock_get_table_cols) +@mock.patch("src.dashboard.get_chart_data.get_chart_data._get_table_cols", mock_get_table_cols) @mock.patch.dict(os.environ, MOCK_ENV) @pytest.mark.parametrize( "query_params,filters,path_params,query_str", @@ -113,7 +113,7 @@ def test_get_data_cols(mock_bucket): @mock.patch( - "src.handlers.dashboard.get_chart_data._build_query", + "src.dashboard.get_chart_data.get_chart_data._build_query", lambda query_params, filters, path_params: ( ( "SELECT gender, sum(cnt) as cnt" diff --git a/tests/dashboard/test_get_csv.py b/tests/dashboard/test_get_csv.py index c055b36..aa9838d 100644 --- a/tests/dashboard/test_get_csv.py +++ b/tests/dashboard/test_get_csv.py @@ -6,8 +6,8 @@ import boto3 import pytest -from src.handlers.dashboard import get_csv -from src.handlers.shared import enums +from src.dashboard.get_csv import get_csv +from src.shared import enums from tests import mock_utils # data matching these params is created via conftest diff --git a/tests/dashboard/test_get_data_packages.py b/tests/dashboard/test_get_data_packages.py index 20e1e96..43bbce8 100644 --- a/tests/dashboard/test_get_data_packages.py +++ b/tests/dashboard/test_get_data_packages.py @@ -3,7 +3,7 @@ import pathlib from unittest import mock -from src.handlers.dashboard.get_data_packages import data_packages_handler +from src.dashboard.get_data_packages import get_data_packages from tests.mock_utils import DATA_PACKAGE_COUNT, MOCK_ENV @@ -11,24 +11,28 @@ def test_get_data_packages(mock_bucket): with open(pathlib.Path(__file__).parent.parent / "./test_data/data_packages_cache.json") as f: data = json.load(f) - res = data_packages_handler({}, {}) + res = get_data_packages.data_packages_handler({}, {}) assert res["statusCode"] == 200 assert DATA_PACKAGE_COUNT == len(data) assert "Access-Control-Allow-Origin" in res["headers"] assert json.loads(res["body"]) == data - res = data_packages_handler({"queryStringParameters": {"name": "encounter"}}, {}) + res = get_data_packages.data_packages_handler( + {"queryStringParameters": {"name": "encounter"}}, {} + ) data = json.loads(res["body"]) assert res["statusCode"] == 200 assert 2 == len(json.loads(res["body"])) for item in data: assert item["name"] == "encounter" - res = data_packages_handler( + res = get_data_packages.data_packages_handler( {"pathParameters": {"data_package_id": "other_study__document__100"}}, {} ) data = json.loads(res["body"]) assert res["statusCode"] == 200 assert 9 == len(data) assert data["id"] == "other_study__document__100" - res = data_packages_handler({"pathParameters": {"data_package_id": "not_an_id"}}, {}) + res = get_data_packages.data_packages_handler( + {"pathParameters": {"data_package_id": "not_an_id"}}, {} + ) data = json.loads(res["body"]) assert res["statusCode"] == 404 diff --git a/tests/dashboard/test_get_metadata.py b/tests/dashboard/test_get_metadata.py index 18e2712..eca5c54 100644 --- a/tests/dashboard/test_get_metadata.py +++ b/tests/dashboard/test_get_metadata.py @@ -2,7 +2,7 @@ import pytest -from src.handlers.dashboard.get_metadata import metadata_handler +from src.dashboard.get_metadata import get_metadata from tests import mock_utils @@ -50,7 +50,7 @@ def test_get_metadata(mock_bucket, params, status, expected): event = {"pathParameters": params} - res = metadata_handler(event, {}) + res = get_metadata.metadata_handler(event, {}) assert res["statusCode"] == status if status == 200: assert json.loads(res["body"]) == expected diff --git a/tests/dashboard/test_get_study_periods.py b/tests/dashboard/test_get_study_periods.py index d9d2ba0..21f4325 100644 --- a/tests/dashboard/test_get_study_periods.py +++ b/tests/dashboard/test_get_study_periods.py @@ -2,7 +2,7 @@ import pytest -from src.handlers.dashboard.get_study_periods import study_periods_handler +from src.dashboard.get_study_periods import get_study_periods from tests.mock_utils import ( EXISTING_SITE, EXISTING_STUDY, @@ -32,7 +32,7 @@ ) def test_get_study_periods(mock_bucket, params, status, expected): event = {"pathParameters": params} - res = study_periods_handler(event, {}) + res = get_study_periods.study_periods_handler(event, {}) assert res["statusCode"] == status if status == 200: assert json.loads(res["body"]) == expected diff --git a/tests/mock_utils.py b/tests/mock_utils.py index 531e286..4dd5698 100644 --- a/tests/mock_utils.py +++ b/tests/mock_utils.py @@ -6,7 +6,7 @@ TEST_PROCESS_COUNTS_ARN = "arn:aws:sns:us-east-1:123456789012:test-counts" TEST_PROCESS_STUDY_META_ARN = "arn:aws:sns:us-east-1:123456789012:test-meta" TEST_CACHE_API_ARN = "arn:aws:sns:us-east-1:123456789012:test-cache" -ITEM_COUNT = 10 +ITEM_COUNT = 9 DATA_PACKAGE_COUNT = 3 EXISTING_SITE = "princeton_plainsboro_teaching_hospital" diff --git a/tests/shared/test_functions.py b/tests/shared/test_functions.py index 339a2bb..d096dde 100644 --- a/tests/shared/test_functions.py +++ b/tests/shared/test_functions.py @@ -4,7 +4,7 @@ import pandas import pytest -from src.handlers.shared import functions, pandas_functions +from src.shared import functions, pandas_functions @pytest.mark.parametrize( diff --git a/tests/site_upload/test_api_gateway_authorizer.py b/tests/site_upload/test_api_gateway_authorizer.py index 8d7a7ec..22590c2 100644 --- a/tests/site_upload/test_api_gateway_authorizer.py +++ b/tests/site_upload/test_api_gateway_authorizer.py @@ -1,8 +1,10 @@ +import json from contextlib import nullcontext as does_not_raise +from unittest import mock import pytest -from src.handlers.site_upload import api_gateway_authorizer +from src.site_upload.api_gateway_authorizer import api_gateway_authorizer from tests import mock_utils @@ -14,7 +16,12 @@ (None, pytest.raises(AttributeError)), ], ) -def test_validate_pw(auth, expects, mock_bucket): +@mock.patch("botocore.client") +def test_validate_pw(mock_client, auth, expects): + mock_secret_client = mock_client.ClientCreator.return_value.create_client.return_value + mock_secret_client.get_secret_value.return_value = { + "SecretString": json.dumps(mock_utils.get_mock_auth()) + } mock_headers = {"Authorization": auth} event = { "headers": mock_headers, @@ -22,4 +29,5 @@ def test_validate_pw(auth, expects, mock_bucket): } with expects: res = api_gateway_authorizer.lambda_handler(event, {}) + assert mock_client.is_called() assert res["policyDocument"]["Statement"][0]["Effect"] == "Allow" diff --git a/tests/site_upload/test_cache_api.py b/tests/site_upload/test_cache_api.py index d8c1d36..0b6705a 100644 --- a/tests/site_upload/test_cache_api.py +++ b/tests/site_upload/test_cache_api.py @@ -4,7 +4,7 @@ import pandas import pytest -from src.handlers.site_upload.cache_api import cache_api_handler +from src.site_upload.cache_api import cache_api from tests.mock_utils import MOCK_ENV, get_mock_data_packages_cache @@ -26,5 +26,5 @@ def test_cache_api(mocker, mock_bucket, subject, message, mock_result, status): mock_query_result = mocker.patch("awswrangler.athena.read_sql_query") mock_query_result.side_effect = mock_result event = {"Records": [{"Sns": {"Subject": subject, "Message": message}}]} - res = cache_api_handler(event, {}) + res = cache_api.cache_api_handler(event, {}) assert res["statusCode"] == status diff --git a/tests/site_upload/test_fetch_upload_url.py b/tests/site_upload/test_fetch_upload_url.py index 01e2f41..f8917e5 100644 --- a/tests/site_upload/test_fetch_upload_url.py +++ b/tests/site_upload/test_fetch_upload_url.py @@ -4,8 +4,8 @@ import botocore import pytest -from src.handlers.shared import enums -from src.handlers.site_upload import fetch_upload_url +from src.shared import enums +from src.site_upload.fetch_upload_url import fetch_upload_url from tests.mock_utils import ( EXISTING_DATA_P, EXISTING_SITE, diff --git a/tests/site_upload/test_powerset_merge.py b/tests/site_upload/test_powerset_merge.py index 106c4da..0c79fc2 100644 --- a/tests/site_upload/test_powerset_merge.py +++ b/tests/site_upload/test_powerset_merge.py @@ -10,8 +10,8 @@ from freezegun import freeze_time from pandas import DataFrame, read_parquet -from src.handlers.shared import enums, functions -from src.handlers.site_upload import powerset_merge +from src.shared import enums, functions +from src.site_upload.powerset_merge import powerset_merge from tests.mock_utils import ( EXISTING_DATA_P, EXISTING_SITE, diff --git a/tests/site_upload/test_process_upload.py b/tests/site_upload/test_process_upload.py index cee7d63..b9ede60 100644 --- a/tests/site_upload/test_process_upload.py +++ b/tests/site_upload/test_process_upload.py @@ -4,9 +4,8 @@ import pytest from freezegun import freeze_time -from src.handlers.shared.enums import BucketPath -from src.handlers.shared.functions import read_metadata -from src.handlers.site_upload.process_upload import process_upload_handler +from src.shared import enums, functions +from src.site_upload.process_upload import process_upload from tests.mock_utils import ( EXISTING_DATA_P, EXISTING_SITE, @@ -122,31 +121,31 @@ def test_process_upload( s3_client.upload_file( upload_file, TEST_BUCKET, - f"{BucketPath.UPLOAD.value}{upload_path}", + f"{enums.BucketPath.UPLOAD.value}{upload_path}", ) event = { "Records": [ { "awsRegion": "us-east-1", - "s3": {"object": {"key": f"{BucketPath.UPLOAD.value}{event_key}"}}, + "s3": {"object": {"key": f"{enums.BucketPath.UPLOAD.value}{event_key}"}}, } ] } - res = process_upload_handler(event, {}) + res = process_upload.process_upload_handler(event, {}) assert res["statusCode"] == status s3_res = s3_client.list_objects_v2(Bucket=TEST_BUCKET) assert len(s3_res["Contents"]) == expected_contents found_archive = False for item in s3_res["Contents"]: if item["Key"].endswith("aggregate.parquet"): - assert item["Key"].startswith(BucketPath.AGGREGATE.value) + assert item["Key"].startswith(enums.BucketPath.AGGREGATE.value) elif item["Key"].endswith("aggregate.csv"): - assert item["Key"].startswith(BucketPath.CSVAGGREGATE.value) + assert item["Key"].startswith(enums.BucketPath.CSVAGGREGATE.value) elif item["Key"].endswith("transactions.json"): - assert item["Key"].startswith(BucketPath.META.value) + assert item["Key"].startswith(enums.BucketPath.META.value) if upload_path is not None and "template" not in upload_path: - metadata = read_metadata(s3_client, TEST_BUCKET) + metadata = functions.read_metadata(s3_client, TEST_BUCKET) if upload_file is not None and upload_path is not None: path_params = upload_path.split("/") study = path_params[1] @@ -157,17 +156,17 @@ def test_process_upload( metadata[site][study][data_package][version]["last_upload"] == datetime.now(UTC).isoformat() ) - elif item["Key"].startswith(BucketPath.STUDY_META.value): + elif item["Key"].startswith(enums.BucketPath.STUDY_META.value): assert any(x in item["Key"] for x in ["_meta_", "/discovery__"]) - elif item["Key"].startswith(BucketPath.ARCHIVE.value): + elif item["Key"].startswith(enums.BucketPath.ARCHIVE.value): found_archive = True else: assert ( - item["Key"].startswith(BucketPath.LATEST.value) - or item["Key"].startswith(BucketPath.LAST_VALID.value) - or item["Key"].startswith(BucketPath.ERROR.value) - or item["Key"].startswith(BucketPath.ADMIN.value) - or item["Key"].startswith(BucketPath.CACHE.value) + item["Key"].startswith(enums.BucketPath.LATEST.value) + or item["Key"].startswith(enums.BucketPath.LAST_VALID.value) + or item["Key"].startswith(enums.BucketPath.ERROR.value) + or item["Key"].startswith(enums.BucketPath.ADMIN.value) + or item["Key"].startswith(enums.BucketPath.CACHE.value) or item["Key"].endswith("study_periods.json") or item["Key"].endswith("column_types.json") ) diff --git a/tests/site_upload/test_study_period.py b/tests/site_upload/test_study_period.py index a003af0..76a8266 100644 --- a/tests/site_upload/test_study_period.py +++ b/tests/site_upload/test_study_period.py @@ -5,9 +5,8 @@ import pytest from freezegun import freeze_time -from src.handlers.shared.enums import BucketPath -from src.handlers.shared.functions import read_metadata -from src.handlers.site_upload.study_period import study_period_handler +from src.shared import enums, functions +from src.site_upload.study_period import study_period from tests.mock_utils import ( EXISTING_DATA_P, EXISTING_SITE, @@ -91,12 +90,12 @@ def test_process_upload( s3_client.upload_file( upload_file, TEST_BUCKET, - f"{BucketPath.STUDY_META.value}{upload_path}", + f"{enums.BucketPath.STUDY_META.value}{upload_path}", ) - event = {"Records": [{"Sns": {"Message": f"{BucketPath.STUDY_META.value}{event_key}"}}]} - res = study_period_handler(event, {}) + event = {"Records": [{"Sns": {"Message": f"{enums.BucketPath.STUDY_META.value}{event_key}"}}]} + res = study_period.study_period_handler(event, {}) assert res["statusCode"] == status - metadata = read_metadata(s3_client, TEST_BUCKET, meta_type="study_periods") + metadata = functions.read_metadata(s3_client, TEST_BUCKET, meta_type="study_periods") if upload_file is not None and upload_path is not None: path_params = upload_path.split("/") study = path_params[1]