From 29a96d756f148004e2fb1c69cb26495cf856d5a1 Mon Sep 17 00:00:00 2001 From: aaronfriedman Date: Thu, 19 Sep 2024 10:18:06 -0400 Subject: [PATCH] Refactor code to run in ECS cluster --- CHANGELOG.md | 7 +++++++ Dockerfile | 25 ++++++++++--------------- config/production.yaml | 22 ++++++++++++++++++++++ config/qa.yaml | 22 ++++++++++++++++++++++ devel_requirements.txt | 1 - lambda_function.py => main.py | 23 ++++++++++------------- requirements.txt | 2 +- tests/test_lambda_function.py | 30 ------------------------------ 8 files changed, 72 insertions(+), 60 deletions(-) create mode 100644 config/production.yaml create mode 100644 config/qa.yaml rename lambda_function.py => main.py (54%) delete mode 100644 tests/test_lambda_function.py diff --git a/CHANGELOG.md b/CHANGELOG.md index a980934..a44a894 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +## 2024-09-19 +### Added +- Refactor code to run in an ECS cluster rather than as a Lambda + +### Fixed +- Update Dockerfile to run Chrome correctly + ## 2024-07-29 ### Added - Use Docker to install Google Chrome and run tests diff --git a/Dockerfile b/Dockerfile index ca05016..d923226 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,21 +1,16 @@ -FROM public.ecr.aws/lambda/python:3.12 +FROM python:3.12 -RUN dnf install -y atk cups-libs gtk3 libXcomposite alsa-lib \ - libXcursor libXdamage libXext libXi libXrandr libXScrnSaver \ - libXtst pango at-spi2-atk libXt xorg-x11-server-Xvfb \ - xorg-x11-xauth dbus-glib dbus-glib-devel nss mesa-libgbm jq unzip +RUN apt-get update && \ + apt-get install -y chromium +RUN apt-get install dumb-init -COPY ./chrome_installer.sh ./chrome_installer.sh -RUN ./chrome_installer.sh -RUN rm ./chrome_installer.sh +ADD . /src +WORKDIR /src -COPY requirements.txt ${LAMBDA_TASK_ROOT} +COPY requirements.txt ./ RUN pip install --upgrade pip && \ pip install -r requirements.txt -COPY lambda_function.py ${LAMBDA_TASK_ROOT} -COPY alarm_controller.py ${LAMBDA_TASK_ROOT} -COPY alarms ${LAMBDA_TASK_ROOT}/alarms -COPY helpers ${LAMBDA_TASK_ROOT}/helpers - -CMD [ "lambda_function.lambda_handler" ] +COPY . . +ENTRYPOINT ["/usr/bin/dumb-init", "--"] +CMD [ "python3", "./main.py"] diff --git a/config/production.yaml b/config/production.yaml new file mode 100644 index 0000000..5f5fd5c --- /dev/null +++ b/config/production.yaml @@ -0,0 +1,22 @@ +--- +PLAINTEXT_VARIABLES: + ENVIRONMENT: production + AWS_REGION: us-east-1 + SIERRA_DB_PORT: 1032 + SIERRA_DB_NAME: iii + ENVISIONWARE_DB_PORT: 3306 + ENVISIONWARE_DB_NAME: lasttwodays + REDSHIFT_DB_NAME: production + LOG_LEVEL: info + SIERRA_DB_HOST: AQECAHh7ea2tyZ6phZgT4B9BDKwguhlFtRC6hgt+7HbmeFsrsgAAAGwwagYJKoZIhvcNAQcGoF0wWwIBADBWBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDFrV5IoeP0tL98V0zgIBEIApBnZ9IWKJ/s6F++zu0rOeWwfB+Kkwh4aFt68vD7jv4LaO0zOppTOPycA= + SIERRA_DB_USER: AQECAHh7ea2tyZ6phZgT4B9BDKwguhlFtRC6hgt+7HbmeFsrsgAAAGcwZQYJKoZIhvcNAQcGoFgwVgIBADBRBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDBMTB2dgzYV4nlEtwwIBEIAky8apvCdg3fDGqaXd06Vq9U59XxG8qlcShPfW9Jp/JiaURhIH + SIERRA_DB_PASSWORD: AQECAHh7ea2tyZ6phZgT4B9BDKwguhlFtRC6hgt+7HbmeFsrsgAAAGcwZQYJKoZIhvcNAQcGoFgwVgIBADBRBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDHcUCu4t+qDgznAz5gIBEIAkL3hNCgiyGOJCAV9meNBRdDfiyfUjaejDGaDWxQsX/DUN6w6Q + ENVISIONWARE_DB_HOST: AQECAHh7ea2tyZ6phZgT4B9BDKwguhlFtRC6hgt+7HbmeFsrsgAAAGwwagYJKoZIhvcNAQcGoF0wWwIBADBWBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDAuBv/BphGCF+7eZXQIBEIApcoNy31Pd5JJDeeAmSKF8bGWMOZPLx9GgSMNxY1DTab6tICcqGK+onys= + ENVISIONWARE_DB_USER: AQECAHh7ea2tyZ6phZgT4B9BDKwguhlFtRC6hgt+7HbmeFsrsgAAAGcwZQYJKoZIhvcNAQcGoFgwVgIBADBRBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDD+XZRJPnpzgejsXUwIBEIAk+WBLH/6lD3bhgpQrT16ssVRP1r0DRcdv12oBnkI59ZX6CZza + ENVISIONWARE_DB_PASSWORD: AQECAHh7ea2tyZ6phZgT4B9BDKwguhlFtRC6hgt+7HbmeFsrsgAAAHcwdQYJKoZIhvcNAQcGoGgwZgIBADBhBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDFAfhwcljbp6QaIurQIBEIA0nG1PV7LbSH3SYR8yS5m5/59dV9FPC3DUsi08GKG/KQZnzrO6wbQH1DMKCLdTGfZSUgIRpw== + REDSHIFT_DB_HOST: AQECAHh7ea2tyZ6phZgT4B9BDKwguhlFtRC6hgt+7HbmeFsrsgAAAKIwgZ8GCSqGSIb3DQEHBqCBkTCBjgIBADCBiAYJKoZIhvcNAQcBMB4GCWCGSAFlAwQBLjARBAzfC7it2NYa1YnFy+4CARCAW1NrqGfSfPptEF4epEXl8Hr6ntrJcZ0DE1GeGt2c2mMnMt7hcim39NTgxHvi5DIldF4J2UJzgG0KBs5UGRU9dyjrQu5zLfdB8Kv50d1lHfvqM2aDQAK0SQ188BY= + REDSHIFT_DB_USER: AQECAHh7ea2tyZ6phZgT4B9BDKwguhlFtRC6hgt+7HbmeFsrsgAAAGgwZgYJKoZIhvcNAQcGoFkwVwIBADBSBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDLYhvC7M+KZDhDSGqQIBEIAllxA8jfnmi6FfWT59HuyqnQfdIAWS2lDO/imHb5B74N7wMgghkw== + REDSHIFT_DB_PASSWORD: AQECAHh7ea2tyZ6phZgT4B9BDKwguhlFtRC6hgt+7HbmeFsrsgAAAHIwcAYJKoZIhvcNAQcGoGMwYQIBADBcBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDFZ9YGTpQHPszq0t1wIBEIAvLZIm3PxGt/z5q1EbArDtz+XOZ0+1seLAL3j4SejbvxFbCnEaJc/WL42V0fUwcAs= + OVERDRIVE_USERNAME: AQECAHh7ea2tyZ6phZgT4B9BDKwguhlFtRC6hgt+7HbmeFsrsgAAAHAwbgYJKoZIhvcNAQcGoGEwXwIBADBaBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDJ4m/+sM1h1VKWLgNwIBEIAtND4XCBaP9+mnOEWceJuKC3ySzc+fZ0cObPm0Aqww3ie1ZSHFRM491Q7JTlly + OVERDRIVE_PASSWORD: AQECAHh7ea2tyZ6phZgT4B9BDKwguhlFtRC6hgt+7HbmeFsrsgAAAGowaAYJKoZIhvcNAQcGoFswWQIBADBUBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDMKCbi5y+ipM9MFnOwIBEIAnQKpRBDWG3Mnb3O8FXrx8PCYKYFAqdYMZ75qgY+R9nvooG7VXq1AC +... \ No newline at end of file diff --git a/config/qa.yaml b/config/qa.yaml new file mode 100644 index 0000000..9209593 --- /dev/null +++ b/config/qa.yaml @@ -0,0 +1,22 @@ +--- +PLAINTEXT_VARIABLES: + ENVIRONMENT: qa + AWS_REGION: us-east-1 + SIERRA_DB_PORT: 1032 + SIERRA_DB_NAME: iii + ENVISIONWARE_DB_PORT: 3306 + ENVISIONWARE_DB_NAME: lasttwodays + REDSHIFT_DB_NAME: qa + LOG_LEVEL: info + SIERRA_DB_HOST: AQECAHh7ea2tyZ6phZgT4B9BDKwguhlFtRC6hgt+7HbmeFsrsgAAAGwwagYJKoZIhvcNAQcGoF0wWwIBADBWBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDL6zq0QQYBhdW4rz8gIBEIApxucxIVAb1Ec4uHUAwxRZcvC8OAZxuj/oJAkhCPh8sPJJ08w3ECoXoKk= + SIERRA_DB_USER: AQECAHh7ea2tyZ6phZgT4B9BDKwguhlFtRC6hgt+7HbmeFsrsgAAAGcwZQYJKoZIhvcNAQcGoFgwVgIBADBRBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDBMTB2dgzYV4nlEtwwIBEIAky8apvCdg3fDGqaXd06Vq9U59XxG8qlcShPfW9Jp/JiaURhIH + SIERRA_DB_PASSWORD: AQECAHh7ea2tyZ6phZgT4B9BDKwguhlFtRC6hgt+7HbmeFsrsgAAAGcwZQYJKoZIhvcNAQcGoFgwVgIBADBRBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDHcUCu4t+qDgznAz5gIBEIAkL3hNCgiyGOJCAV9meNBRdDfiyfUjaejDGaDWxQsX/DUN6w6Q + ENVISIONWARE_DB_HOST: AQECAHh7ea2tyZ6phZgT4B9BDKwguhlFtRC6hgt+7HbmeFsrsgAAAGwwagYJKoZIhvcNAQcGoF0wWwIBADBWBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDAuBv/BphGCF+7eZXQIBEIApcoNy31Pd5JJDeeAmSKF8bGWMOZPLx9GgSMNxY1DTab6tICcqGK+onys= + ENVISIONWARE_DB_USER: AQECAHh7ea2tyZ6phZgT4B9BDKwguhlFtRC6hgt+7HbmeFsrsgAAAGcwZQYJKoZIhvcNAQcGoFgwVgIBADBRBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDD+XZRJPnpzgejsXUwIBEIAk+WBLH/6lD3bhgpQrT16ssVRP1r0DRcdv12oBnkI59ZX6CZza + ENVISIONWARE_DB_PASSWORD: AQECAHh7ea2tyZ6phZgT4B9BDKwguhlFtRC6hgt+7HbmeFsrsgAAAHcwdQYJKoZIhvcNAQcGoGgwZgIBADBhBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDFAfhwcljbp6QaIurQIBEIA0nG1PV7LbSH3SYR8yS5m5/59dV9FPC3DUsi08GKG/KQZnzrO6wbQH1DMKCLdTGfZSUgIRpw== + REDSHIFT_DB_HOST: AQECAHh7ea2tyZ6phZgT4B9BDKwguhlFtRC6hgt+7HbmeFsrsgAAAKIwgZ8GCSqGSIb3DQEHBqCBkTCBjgIBADCBiAYJKoZIhvcNAQcBMB4GCWCGSAFlAwQBLjARBAzfC7it2NYa1YnFy+4CARCAW1NrqGfSfPptEF4epEXl8Hr6ntrJcZ0DE1GeGt2c2mMnMt7hcim39NTgxHvi5DIldF4J2UJzgG0KBs5UGRU9dyjrQu5zLfdB8Kv50d1lHfvqM2aDQAK0SQ188BY= + REDSHIFT_DB_USER: AQECAHh7ea2tyZ6phZgT4B9BDKwguhlFtRC6hgt+7HbmeFsrsgAAAGgwZgYJKoZIhvcNAQcGoFkwVwIBADBSBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDLYhvC7M+KZDhDSGqQIBEIAllxA8jfnmi6FfWT59HuyqnQfdIAWS2lDO/imHb5B74N7wMgghkw== + REDSHIFT_DB_PASSWORD: AQECAHh7ea2tyZ6phZgT4B9BDKwguhlFtRC6hgt+7HbmeFsrsgAAAHIwcAYJKoZIhvcNAQcGoGMwYQIBADBcBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDFZ9YGTpQHPszq0t1wIBEIAvLZIm3PxGt/z5q1EbArDtz+XOZ0+1seLAL3j4SejbvxFbCnEaJc/WL42V0fUwcAs= + OVERDRIVE_USERNAME: AQECAHh7ea2tyZ6phZgT4B9BDKwguhlFtRC6hgt+7HbmeFsrsgAAAHAwbgYJKoZIhvcNAQcGoGEwXwIBADBaBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDJ4m/+sM1h1VKWLgNwIBEIAtND4XCBaP9+mnOEWceJuKC3ySzc+fZ0cObPm0Aqww3ie1ZSHFRM491Q7JTlly + OVERDRIVE_PASSWORD: AQECAHh7ea2tyZ6phZgT4B9BDKwguhlFtRC6hgt+7HbmeFsrsgAAAGowaAYJKoZIhvcNAQcGoFswWQIBADBUBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDMKCbi5y+ipM9MFnOwIBEIAnQKpRBDWG3Mnb3O8FXrx8PCYKYFAqdYMZ75qgY+R9nvooG7VXq1AC +... \ No newline at end of file diff --git a/devel_requirements.txt b/devel_requirements.txt index 0cb5a8c..b5a8891 100644 --- a/devel_requirements.txt +++ b/devel_requirements.txt @@ -1,5 +1,4 @@ black freezegun -nypl-py-utils[mysql-client,postgresql-client,redshift-client,config-helper]==1.1.5 pytest pytest-mock \ No newline at end of file diff --git a/lambda_function.py b/main.py similarity index 54% rename from lambda_function.py rename to main.py index 0e3e1a9..33c9999 100644 --- a/lambda_function.py +++ b/main.py @@ -1,18 +1,15 @@ -import json import os from alarm_controller import AlarmController from datetime import datetime, timedelta, timezone from nypl_py_utils.functions.log_helper import create_log +from nypl_py_utils.functions.config_helper import load_env_file -def lambda_handler(event, context): - if os.environ["ENVIRONMENT"] == "devel": - from nypl_py_utils.functions.config_helper import load_env_file - - load_env_file("devel", "config/{}.yaml") - logger = create_log("lambda_function") - logger.info("Starting lambda processing") +def main(): + load_env_file(os.environ["ENVIRONMENT"], "config/{}.yaml") + logger = create_log("main") + logger.info("Starting alarms") alarm_controller = AlarmController() yesterday = (datetime.now(timezone.utc) - timedelta(days=1)).date().isoformat() @@ -24,8 +21,8 @@ def lambda_handler(event, context): logger.error("Error running alarms: {}".format(e)) raise e - logger.info("Finished lambda processing") - return { - "statusCode": 200, - "body": json.dumps({"message": "Job ran successfully."}), - } + logger.info("Finished alarms") + + +if __name__ == "__main__": + main() diff --git a/requirements.txt b/requirements.txt index 8c79c89..9aa5ace 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,2 @@ -nypl-py-utils[mysql-client,postgresql-client,redshift-client,config-helper]==1.1.5 +nypl-py-utils[mysql-client,postgresql-client,redshift-client,config-helper]==1.3.2 selenium>=4.10.0 \ No newline at end of file diff --git a/tests/test_lambda_function.py b/tests/test_lambda_function.py deleted file mode 100644 index e3918b4..0000000 --- a/tests/test_lambda_function.py +++ /dev/null @@ -1,30 +0,0 @@ -import json -import lambda_function -import pytest - - -class TestLambdaFunction: - @pytest.fixture - def mock_alarm_controller(self, mocker): - mocker.patch("lambda_function.create_log") - mocker.patch("nypl_py_utils.functions.config_helper.load_env_file") - - mock_alarm_controller = mocker.MagicMock() - mocker.patch( - "lambda_function.AlarmController", return_value=mock_alarm_controller - ) - return mock_alarm_controller - - def test_lambda_handler_success(self, mock_alarm_controller, mocker): - assert lambda_function.lambda_handler(None, None) == { - "statusCode": 200, - "body": json.dumps({"message": "Job ran successfully."}), - } - - mock_alarm_controller.run_alarms.assert_called_once() # noqa: E501 - - def test_lambda_handler_error(self, mock_alarm_controller, mocker): - mock_alarm_controller.run_alarms.side_effect = Exception("test exception") - - with pytest.raises(Exception): - lambda_function.lambda_handler(None, None) # noqa: E501