Isolated lambdas, moved auth to secrets manager (#137)

* Isolated lambdas, moved auth to secrets manager

* More helpful user message

.github/workflows/ci.yaml

@@ -54,7 +54,7 @@ jobs:
           python -m pip install --upgrade pip
           pip install ".[dev]"
       - name: Run ruff
-        if: success() || failure() # still run black if above checks fails
+        if: success() || failure() # still run ruff if above checks fails
         run: |
           ruff check
           ruff format --check

.pre-commit-config.yaml

@@ -1,10 +1,11 @@
 default_install_hook_types: [pre-commit, pre-push]
 repos:
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.2.1
+    rev: v0.5.7
     hooks:
-      - name: Ruff formatting
-        id: ruff-format
       - name: Ruff linting
         id: ruff
+        args: [ --fix ]
+      - name: Ruff formatting
+        id: ruff-format
         stages: [pre-push]

MAINTAINER.md

@@ -14,17 +14,23 @@ If you're writing unit tests - note that moto support for Athena mocking is curr
 
 ## Managing upload credentials
 
-In order to enable site uploads, we provide a [utility](../scripts/credential_management.py) to simplify managing S3-based dictionaries for authorizing pre-signed URLs. These artifacts should persist between cloudformation deployments, but if for some reason you need to delete the contents of the bucket and recreate, you can back up credentials by copying the contents of the admin/ virtual directory to another location.
+For security reasons, we've migrated the credentials for users to AWS secrets manager. Passwords should be managed via
+the secrets manager console. We provide a [utility](../scripts/credential_management.py) to simplify managing S3-based
+dictionaries for authorizing pre-signed URLs, or generating the user strings for secrets manager. The S3 artifacts should
+persist between cloudformation deployments, but if for some reason you need to delete the contents of the bucket and recreate,
+you can back up credentials by copying the contents of the admin/ virtual directory to another location.
 
-To create a new user, you would need to run the following two commands:
+To create a user credential for secrets manager, you would need to run the following command:
 
 Creating a user:
 `./scripts/cumulus_upload_data.py --ca user_name auth_secret site_short_name`
 
+To set up, or add to, a dictionary of short names to display names, you can run the following:
+
 Associating a site with an s3 directory:
 `./scripts/cumulus_upload_data.py --cm site_short_name s3_folder_name`
 
-These commands allow you to create a many to one relation of users to a given site, and a many to one relation of site to s3_upload_location, if so desired.
+These commands allow you to create a many to one relation of users to a given site, if so desired.
 
 Without running these commands, no credentials are created by default, and so no access to pre-signed URLs is allowed.
 
@@ -54,6 +60,6 @@ The SAM framework extends native cloudformation, usually with a lighter syntax,
 
 - If you modify S3 bucket permissions while in watch mode, changes to the bucket may generate a permission denied message. You'll need to delete the bucket and bring down the deployment before restarting to apply your changes.
 
-- Similarly, if you end up in a ROLLBACK_FAILED state, usually the only recourse is to bring the deployment down and resync, or do a regular deployment deployment.
+- Similarly, if you end up in a ROLLBACK_FAILED state, usually the only recourse is to bring the deployment down and resync, do a regular deployment deployment, or manually initiate a rollback from the AWS console.
 
 Using deploy is a little safer than sync in this regard, though it does take longer for each deployment. Use your best judgement.

pyproject.toml

@@ -57,6 +57,15 @@ omit = [
     "*/api_gateway_authorizer.py",
 ]
 
+[tool.pytest.ini_options]
+pythonpath = [
+    # we use this to get 'shared' and 'filter_config' as root level packages, which matches
+    # the packaging in the lambda environment, allowing unit tests to function
+    'src/',
+    'src/dashboard/get_chart_data',
+]
+
+
 [tool.ruff]
 target-version = "py311"
 line-length = 100

scripts/credential_management.py

@@ -26,26 +26,10 @@ def _put_s3_data(name: str, bucket_name: str, client, data: dict, path: str = "a
     client.upload_fileobj(Bucket=bucket_name, Key=f"{path}/{name}", Fileobj=b_data)
 
 
-def create_auth(client, bucket_name: str, user: str, auth: str, site: str) -> str:
+def create_auth(client, user: str, auth: str, site: str) -> str:
     """Adds a new entry to the auth dict used to issue pre-signed URLs"""
-    file = "auth.json"
-    auth_dict = _get_s3_data(file, bucket_name, client)
     site_id = _basic_auth_str(user, auth).split(" ")[1]
-    auth_dict[site_id] = {"site": site}
-    _put_s3_data(file, bucket_name, client, auth_dict)
-    return site_id
-
-
-def delete_auth(client, bucket_name: str, site_id: str) -> bool:
-    """Removes an entry from the auth dict used to issue pre-signed urls"""
-    file = "auth.json"
-    auth_dict = _get_s3_data(file, bucket_name, client)
-    if site_id in auth_dict.keys():
-        auth_dict.pop(site_id)
-        _put_s3_data(file, bucket_name, client, auth_dict)
-        return True
-    else:
-        return False
+    return f'"{site_id}"": {{"site":{site}}}'
 
 
 def create_meta(client, bucket_name: str, site: str, folder: str) -> None:
@@ -110,19 +94,11 @@ def delete_meta(client, bucket_name: str, site: str) -> bool:
         bucket = f"{args.bucket}-{args.env}"
     if args.create_auth:
         id_str = create_auth(
-            s3_client,
-            bucket,
             args.create_auth[0],
             args.create_auth[1],
             args.create_auth[2],
         )
-        print(f"{id_str} created")
-    elif args.delete_auth:
-        succeeded = delete_auth(s3_client, bucket, args.delete_auth)
-        if succeeded:
-            print(f"Removed {args.delete_auth}")
-        else:
-            print(f"{args.delete_auth} not found")
+        print(f"Add the following key/valye to secrets manager: \n {id_str}")
     elif args.create_meta:
         create_meta(s3_client, bucket, args.create_meta[0], args.create_meta[1])
         print(f"{args.create_meta[0]} mapped to S3 folder {args.create_meta[1]}")

src/handlers/dashboard/get_chart_data.py → src/dashboard/get_chart_data/get_chart_data.py

@@ -7,10 +7,9 @@
 
 import awswrangler
 import boto3
+import filter_config
 import pandas
-
-from src.handlers.dashboard import filter_config
-from src.handlers.shared import decorators, enums, errors, functions
+from shared import decorators, enums, errors, functions
 
 log_level = os.environ.get("LAMBDA_LOG_LEVEL", "INFO")
 logger = logging.getLogger()

src/dashboard/get_chart_data/shared

@@ -0,0 +1 @@
+../../shared

src/handlers/dashboard/get_csv.py → src/dashboard/get_csv/get_csv.py

@@ -2,8 +2,7 @@
 
 import boto3
 import botocore
-
-from src.handlers.shared import decorators, enums, functions
+from shared import decorators, enums, functions
 
 
 def _format_and_validate_key(

src/dashboard/get_csv/shared

@@ -0,0 +1 @@
+../../shared

src/handlers/dashboard/get_data_packages.py → src/dashboard/get_data_packages/get_data_packages.py

@@ -2,19 +2,17 @@
 
 import os
 
-from src.handlers.shared.decorators import generic_error_handler
-from src.handlers.shared.enums import BucketPath, JsonFilename
-from src.handlers.shared.functions import get_s3_json_as_dict, http_response
+from shared import decorators, enums, functions
 
 
-@generic_error_handler(msg="Error retrieving data packages")
+@decorators.generic_error_handler(msg="Error retrieving data packages")
 def data_packages_handler(event, context):
     """Retrieves list of data packages from S3."""
     del context
     status = 200
-    data_packages = get_s3_json_as_dict(
+    data_packages = functions.get_s3_json_as_dict(
         os.environ.get("BUCKET_NAME"),
-        f"{BucketPath.CACHE.value}/{JsonFilename.DATA_PACKAGES.value}.json",
+        f"{enums.BucketPath.CACHE.value}/{enums.JsonFilename.DATA_PACKAGES.value}.json",
     )
     payload = data_packages
     if event.get("queryStringParameters"):
@@ -33,5 +31,5 @@ def data_packages_handler(event, context):
         else:
             status = 404
             payload = None
-    res = http_response(status, payload, allow_cors=True)
+    res = functions.http_response(status, payload, allow_cors=True)
     return res

src/dashboard/get_data_packages/shared

@@ -0,0 +1 @@
+../../shared

src/handlers/dashboard/get_metadata.py → src/dashboard/get_metadata/get_metadata.py

@@ -3,9 +3,8 @@
 import os
 
 import boto3
-
-from src.handlers.shared.decorators import generic_error_handler
-from src.handlers.shared.functions import http_response, read_metadata
+from shared.decorators import generic_error_handler
+from shared.functions import http_response, read_metadata
 
 
 @generic_error_handler(msg="Error retrieving metadata")

src/dashboard/get_metadata/shared

@@ -0,0 +1 @@
+../../shared

src/handlers/dashboard/get_study_periods.py → src/dashboard/get_study_periods/get_study_periods.py

@@ -3,23 +3,22 @@
 import os
 
 import boto3
+from shared import decorators, enums, functions
 
-from src.handlers.shared.decorators import generic_error_handler
-from src.handlers.shared.enums import JsonFilename
-from src.handlers.shared.functions import http_response, read_metadata
 
-
-@generic_error_handler(msg="Error retrieving study period")
+@decorators.generic_error_handler(msg="Error retrieving study period")
 def study_periods_handler(event, context):
     """Retrieves the study period from S3"""
     del context
     s3_bucket = os.environ.get("BUCKET_NAME")
     s3_client = boto3.client("s3")
-    metadata = read_metadata(s3_client, s3_bucket, meta_type=JsonFilename.STUDY_PERIODS.value)
+    metadata = functions.read_metadata(
+        s3_client, s3_bucket, meta_type=enums.JsonFilename.STUDY_PERIODS.value
+    )
     if params := event["pathParameters"]:
         if "site" in params:
             metadata = metadata[params["site"]]
         if "study" in params:
             metadata = metadata[params["study"]]
-    res = http_response(200, metadata)
+    res = functions.http_response(200, metadata)
     return res

src/dashboard/get_study_periods/shared

@@ -0,0 +1 @@
+../../shared

src/handlers/shared/awswrangler_functions.py → src/shared/awswrangler_functions.py

@@ -2,7 +2,7 @@
 
 import awswrangler
 
-from src.handlers.shared.enums import BucketPath
+from .enums import BucketPath
 
 
 def get_s3_data_package_list(

src/handlers/shared/decorators.py → src/shared/decorators.py

@@ -3,7 +3,7 @@
 import functools
 import logging
 
-from src.handlers.shared.functions import http_response
+from .functions import http_response
 
 
 def generic_error_handler(msg="Internal server error"):

src/handlers/shared/functions.py → src/shared/functions.py

@@ -7,7 +7,7 @@
 
 import boto3
 
-from src.handlers.shared import enums
+from . import enums
 
 TRANSACTION_METADATA_TEMPLATE = {
     enums.TransactionKeys.TRANSACTION_FORMAT_VERSION.value: "2",

src/handlers/site_upload/api_gateway_authorizer.py → src/site_upload/api_gateway_authorizer/api_gateway_authorizer.py

@@ -4,23 +4,37 @@
 
 # pylint: disable=invalid-name,pointless-string-statement
 
+import json
 import os
 import re
 
-from src.handlers.shared.enums import BucketPath
-from src.handlers.shared.functions import get_s3_json_as_dict
+import boto3
 
 
 class AuthError(Exception):
     pass
 
 
+def get_secret():
+    """Retrieves a specified secret.
+
+    This is largely unmodified boilerplate from the secrets manager recommended approach
+    for fetching secrets, except for getting the values from environment variables"""
+
+    secret_name = os.environ.get("SECRET_NAME")
+    region_name = os.environ.get("REGION")
+
+    session = boto3.session.Session()
+    client = session.client(service_name="secretsmanager", region_name=region_name)
+
+    get_secret_value_response = client.get_secret_value(SecretId=secret_name)
+    return json.loads(get_secret_value_response["SecretString"])
+
+
 def lambda_handler(event, context):
     del context
     # ---- aggregator specific logic
-    user_db = get_s3_json_as_dict(
-        os.environ.get("BUCKET_NAME"), f"{BucketPath.ADMIN.value}/auth.json"
-    )
+    user_db = get_secret()
     try:
         auth_header = event["headers"]["Authorization"].split(" ")
         auth_token = auth_header[1]

src/handlers/site_upload/cache_api.py → src/site_upload/cache_api/cache_api.py

@@ -5,8 +5,7 @@
 
 import awswrangler
 import boto3
-
-from src.handlers.shared import decorators, enums, functions
+from shared import decorators, enums, functions
 
 
 def cache_api_data(s3_client, s3_bucket_name: str, db: str, target: str) -> None:

src/site_upload/cache_api/shared

@@ -0,0 +1 @@
+../../shared

src/handlers/site_upload/fetch_upload_url.py → src/site_upload/fetch_upload_url/fetch_upload_url.py

@@ -6,10 +6,9 @@
 
 import boto3
 import botocore.exceptions
-
-from src.handlers.shared.decorators import generic_error_handler
-from src.handlers.shared.enums import BucketPath
-from src.handlers.shared.functions import get_s3_json_as_dict, http_response
+from shared.decorators import generic_error_handler
+from shared.enums import BucketPath
+from shared.functions import get_s3_json_as_dict, http_response
 
 
 def create_presigned_post(

src/site_upload/fetch_upload_url/shared

@@ -0,0 +1 @@
+../../shared

src/handlers/site_upload/powerset_merge.py → src/site_upload/powerset_merge/powerset_merge.py

@@ -11,8 +11,7 @@
 import numpy
 import pandas
 from pandas.core.indexes.range import RangeIndex
-
-from src.handlers.shared import (
+from shared import (
     awswrangler_functions,
     decorators,
     enums,

src/site_upload/powerset_merge/shared

@@ -0,0 +1 @@
+../../shared

src/handlers/site_upload/process_upload.py → src/site_upload/process_upload/process_upload.py

@@ -4,8 +4,7 @@
 import os
 
 import boto3
-
-from src.handlers.shared import decorators, enums, functions
+from shared import decorators, enums, functions
 
 log_level = os.environ.get("LAMBDA_LOG_LEVEL", "INFO")
 logger = logging.getLogger()

src/site_upload/process_upload/shared

@@ -0,0 +1 @@
+../../shared

src/site_upload/study_period/shared

@@ -0,0 +1 @@
+../../shared