Merge pull request #25 from NYPL/use-ecs

Refactor code to run in ECS cluster

CHANGELOG.md

@@ -1,3 +1,10 @@
+## 2024-09-19
+### Added
+- Refactor code to run in an ECS cluster rather than as a Lambda
+
+### Fixed
+- Update Dockerfile to run Chrome correctly
+
 ## 2024-07-29
 ### Added
 - Use Docker to install Google Chrome and run tests

Dockerfile

@@ -1,21 +1,16 @@
-FROM public.ecr.aws/lambda/python:3.12
+FROM python:3.12
 
-RUN dnf install -y atk cups-libs gtk3 libXcomposite alsa-lib \
-    libXcursor libXdamage libXext libXi libXrandr libXScrnSaver \
-    libXtst pango at-spi2-atk libXt xorg-x11-server-Xvfb \
-    xorg-x11-xauth dbus-glib dbus-glib-devel nss mesa-libgbm jq unzip
+RUN apt-get update && \
+	apt-get install -y chromium
+RUN apt-get install dumb-init
 
-COPY ./chrome_installer.sh ./chrome_installer.sh
-RUN ./chrome_installer.sh
-RUN rm ./chrome_installer.sh
+ADD . /src
+WORKDIR /src
 
-COPY requirements.txt ${LAMBDA_TASK_ROOT}
+COPY requirements.txt ./
 RUN pip install --upgrade pip && \
 	pip install -r requirements.txt
 
-COPY lambda_function.py ${LAMBDA_TASK_ROOT}
-COPY alarm_controller.py ${LAMBDA_TASK_ROOT}
-COPY alarms ${LAMBDA_TASK_ROOT}/alarms
-COPY helpers ${LAMBDA_TASK_ROOT}/helpers
-
-CMD [ "lambda_function.lambda_handler" ]
+COPY . .
+ENTRYPOINT ["/usr/bin/dumb-init", "--"]
+CMD [ "python3", "./main.py"]

config/production.yaml

@@ -0,0 +1,22 @@
+---
+PLAINTEXT_VARIABLES:
+    ENVIRONMENT: production
+    AWS_REGION: us-east-1
+    SIERRA_DB_PORT: 1032
+    SIERRA_DB_NAME: iii
+    ENVISIONWARE_DB_PORT: 3306
+    ENVISIONWARE_DB_NAME: lasttwodays
+    REDSHIFT_DB_NAME: production
+    LOG_LEVEL: info
+    SIERRA_DB_HOST: AQECAHh7ea2tyZ6phZgT4B9BDKwguhlFtRC6hgt+7HbmeFsrsgAAAGwwagYJKoZIhvcNAQcGoF0wWwIBADBWBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDFrV5IoeP0tL98V0zgIBEIApBnZ9IWKJ/s6F++zu0rOeWwfB+Kkwh4aFt68vD7jv4LaO0zOppTOPycA=
+    SIERRA_DB_USER: AQECAHh7ea2tyZ6phZgT4B9BDKwguhlFtRC6hgt+7HbmeFsrsgAAAGcwZQYJKoZIhvcNAQcGoFgwVgIBADBRBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDBMTB2dgzYV4nlEtwwIBEIAky8apvCdg3fDGqaXd06Vq9U59XxG8qlcShPfW9Jp/JiaURhIH
+    SIERRA_DB_PASSWORD: AQECAHh7ea2tyZ6phZgT4B9BDKwguhlFtRC6hgt+7HbmeFsrsgAAAGcwZQYJKoZIhvcNAQcGoFgwVgIBADBRBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDHcUCu4t+qDgznAz5gIBEIAkL3hNCgiyGOJCAV9meNBRdDfiyfUjaejDGaDWxQsX/DUN6w6Q
+    ENVISIONWARE_DB_HOST: AQECAHh7ea2tyZ6phZgT4B9BDKwguhlFtRC6hgt+7HbmeFsrsgAAAGwwagYJKoZIhvcNAQcGoF0wWwIBADBWBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDAuBv/BphGCF+7eZXQIBEIApcoNy31Pd5JJDeeAmSKF8bGWMOZPLx9GgSMNxY1DTab6tICcqGK+onys=
+    ENVISIONWARE_DB_USER: AQECAHh7ea2tyZ6phZgT4B9BDKwguhlFtRC6hgt+7HbmeFsrsgAAAGcwZQYJKoZIhvcNAQcGoFgwVgIBADBRBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDD+XZRJPnpzgejsXUwIBEIAk+WBLH/6lD3bhgpQrT16ssVRP1r0DRcdv12oBnkI59ZX6CZza
+    ENVISIONWARE_DB_PASSWORD: AQECAHh7ea2tyZ6phZgT4B9BDKwguhlFtRC6hgt+7HbmeFsrsgAAAHcwdQYJKoZIhvcNAQcGoGgwZgIBADBhBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDFAfhwcljbp6QaIurQIBEIA0nG1PV7LbSH3SYR8yS5m5/59dV9FPC3DUsi08GKG/KQZnzrO6wbQH1DMKCLdTGfZSUgIRpw==
+    REDSHIFT_DB_HOST: AQECAHh7ea2tyZ6phZgT4B9BDKwguhlFtRC6hgt+7HbmeFsrsgAAAKIwgZ8GCSqGSIb3DQEHBqCBkTCBjgIBADCBiAYJKoZIhvcNAQcBMB4GCWCGSAFlAwQBLjARBAzfC7it2NYa1YnFy+4CARCAW1NrqGfSfPptEF4epEXl8Hr6ntrJcZ0DE1GeGt2c2mMnMt7hcim39NTgxHvi5DIldF4J2UJzgG0KBs5UGRU9dyjrQu5zLfdB8Kv50d1lHfvqM2aDQAK0SQ188BY=
+    REDSHIFT_DB_USER: AQECAHh7ea2tyZ6phZgT4B9BDKwguhlFtRC6hgt+7HbmeFsrsgAAAGgwZgYJKoZIhvcNAQcGoFkwVwIBADBSBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDLYhvC7M+KZDhDSGqQIBEIAllxA8jfnmi6FfWT59HuyqnQfdIAWS2lDO/imHb5B74N7wMgghkw==
+    REDSHIFT_DB_PASSWORD: AQECAHh7ea2tyZ6phZgT4B9BDKwguhlFtRC6hgt+7HbmeFsrsgAAAHIwcAYJKoZIhvcNAQcGoGMwYQIBADBcBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDFZ9YGTpQHPszq0t1wIBEIAvLZIm3PxGt/z5q1EbArDtz+XOZ0+1seLAL3j4SejbvxFbCnEaJc/WL42V0fUwcAs=
+    OVERDRIVE_USERNAME: AQECAHh7ea2tyZ6phZgT4B9BDKwguhlFtRC6hgt+7HbmeFsrsgAAAHAwbgYJKoZIhvcNAQcGoGEwXwIBADBaBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDJ4m/+sM1h1VKWLgNwIBEIAtND4XCBaP9+mnOEWceJuKC3ySzc+fZ0cObPm0Aqww3ie1ZSHFRM491Q7JTlly
+    OVERDRIVE_PASSWORD: AQECAHh7ea2tyZ6phZgT4B9BDKwguhlFtRC6hgt+7HbmeFsrsgAAAGowaAYJKoZIhvcNAQcGoFswWQIBADBUBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDMKCbi5y+ipM9MFnOwIBEIAnQKpRBDWG3Mnb3O8FXrx8PCYKYFAqdYMZ75qgY+R9nvooG7VXq1AC
+...

config/qa.yaml

@@ -0,0 +1,22 @@
+---
+PLAINTEXT_VARIABLES:
+    ENVIRONMENT: qa
+    AWS_REGION: us-east-1
+    SIERRA_DB_PORT: 1032
+    SIERRA_DB_NAME: iii
+    ENVISIONWARE_DB_PORT: 3306
+    ENVISIONWARE_DB_NAME: lasttwodays
+    REDSHIFT_DB_NAME: qa
+    LOG_LEVEL: info
+    SIERRA_DB_HOST: AQECAHh7ea2tyZ6phZgT4B9BDKwguhlFtRC6hgt+7HbmeFsrsgAAAGwwagYJKoZIhvcNAQcGoF0wWwIBADBWBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDL6zq0QQYBhdW4rz8gIBEIApxucxIVAb1Ec4uHUAwxRZcvC8OAZxuj/oJAkhCPh8sPJJ08w3ECoXoKk=
+    SIERRA_DB_USER: AQECAHh7ea2tyZ6phZgT4B9BDKwguhlFtRC6hgt+7HbmeFsrsgAAAGcwZQYJKoZIhvcNAQcGoFgwVgIBADBRBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDBMTB2dgzYV4nlEtwwIBEIAky8apvCdg3fDGqaXd06Vq9U59XxG8qlcShPfW9Jp/JiaURhIH
+    SIERRA_DB_PASSWORD: AQECAHh7ea2tyZ6phZgT4B9BDKwguhlFtRC6hgt+7HbmeFsrsgAAAGcwZQYJKoZIhvcNAQcGoFgwVgIBADBRBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDHcUCu4t+qDgznAz5gIBEIAkL3hNCgiyGOJCAV9meNBRdDfiyfUjaejDGaDWxQsX/DUN6w6Q
+    ENVISIONWARE_DB_HOST: AQECAHh7ea2tyZ6phZgT4B9BDKwguhlFtRC6hgt+7HbmeFsrsgAAAGwwagYJKoZIhvcNAQcGoF0wWwIBADBWBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDAuBv/BphGCF+7eZXQIBEIApcoNy31Pd5JJDeeAmSKF8bGWMOZPLx9GgSMNxY1DTab6tICcqGK+onys=
+    ENVISIONWARE_DB_USER: AQECAHh7ea2tyZ6phZgT4B9BDKwguhlFtRC6hgt+7HbmeFsrsgAAAGcwZQYJKoZIhvcNAQcGoFgwVgIBADBRBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDD+XZRJPnpzgejsXUwIBEIAk+WBLH/6lD3bhgpQrT16ssVRP1r0DRcdv12oBnkI59ZX6CZza
+    ENVISIONWARE_DB_PASSWORD: AQECAHh7ea2tyZ6phZgT4B9BDKwguhlFtRC6hgt+7HbmeFsrsgAAAHcwdQYJKoZIhvcNAQcGoGgwZgIBADBhBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDFAfhwcljbp6QaIurQIBEIA0nG1PV7LbSH3SYR8yS5m5/59dV9FPC3DUsi08GKG/KQZnzrO6wbQH1DMKCLdTGfZSUgIRpw==
+    REDSHIFT_DB_HOST: AQECAHh7ea2tyZ6phZgT4B9BDKwguhlFtRC6hgt+7HbmeFsrsgAAAKIwgZ8GCSqGSIb3DQEHBqCBkTCBjgIBADCBiAYJKoZIhvcNAQcBMB4GCWCGSAFlAwQBLjARBAzfC7it2NYa1YnFy+4CARCAW1NrqGfSfPptEF4epEXl8Hr6ntrJcZ0DE1GeGt2c2mMnMt7hcim39NTgxHvi5DIldF4J2UJzgG0KBs5UGRU9dyjrQu5zLfdB8Kv50d1lHfvqM2aDQAK0SQ188BY=
+    REDSHIFT_DB_USER: AQECAHh7ea2tyZ6phZgT4B9BDKwguhlFtRC6hgt+7HbmeFsrsgAAAGgwZgYJKoZIhvcNAQcGoFkwVwIBADBSBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDLYhvC7M+KZDhDSGqQIBEIAllxA8jfnmi6FfWT59HuyqnQfdIAWS2lDO/imHb5B74N7wMgghkw==
+    REDSHIFT_DB_PASSWORD: AQECAHh7ea2tyZ6phZgT4B9BDKwguhlFtRC6hgt+7HbmeFsrsgAAAHIwcAYJKoZIhvcNAQcGoGMwYQIBADBcBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDFZ9YGTpQHPszq0t1wIBEIAvLZIm3PxGt/z5q1EbArDtz+XOZ0+1seLAL3j4SejbvxFbCnEaJc/WL42V0fUwcAs=
+    OVERDRIVE_USERNAME: AQECAHh7ea2tyZ6phZgT4B9BDKwguhlFtRC6hgt+7HbmeFsrsgAAAHAwbgYJKoZIhvcNAQcGoGEwXwIBADBaBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDJ4m/+sM1h1VKWLgNwIBEIAtND4XCBaP9+mnOEWceJuKC3ySzc+fZ0cObPm0Aqww3ie1ZSHFRM491Q7JTlly
+    OVERDRIVE_PASSWORD: AQECAHh7ea2tyZ6phZgT4B9BDKwguhlFtRC6hgt+7HbmeFsrsgAAAGowaAYJKoZIhvcNAQcGoFswWQIBADBUBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDMKCbi5y+ipM9MFnOwIBEIAnQKpRBDWG3Mnb3O8FXrx8PCYKYFAqdYMZ75qgY+R9nvooG7VXq1AC
+...

devel_requirements.txt

@@ -1,5 +1,4 @@
 black
 freezegun
-nypl-py-utils[mysql-client,postgresql-client,redshift-client,config-helper]==1.1.5
 pytest
 pytest-mock

lambda_function.py → main.py

@@ -1,18 +1,15 @@
-import json
 import os
 
 from alarm_controller import AlarmController
 from datetime import datetime, timedelta, timezone
 from nypl_py_utils.functions.log_helper import create_log
+from nypl_py_utils.functions.config_helper import load_env_file
 
 
-def lambda_handler(event, context):
-    if os.environ["ENVIRONMENT"] == "devel":
-        from nypl_py_utils.functions.config_helper import load_env_file
-
-        load_env_file("devel", "config/{}.yaml")
-    logger = create_log("lambda_function")
-    logger.info("Starting lambda processing")
+def main():
+    load_env_file(os.environ["ENVIRONMENT"], "config/{}.yaml")
+    logger = create_log("main")
+    logger.info("Starting alarms")
 
     alarm_controller = AlarmController()
     yesterday = (datetime.now(timezone.utc) - timedelta(days=1)).date().isoformat()
@@ -24,8 +21,8 @@ def lambda_handler(event, context):
         logger.error("Error running alarms: {}".format(e))
         raise e
 
-    logger.info("Finished lambda processing")
-    return {
-        "statusCode": 200,
-        "body": json.dumps({"message": "Job ran successfully."}),
-    }
+    logger.info("Finished alarms")
+
+
+if __name__ == "__main__":
+    main()

requirements.txt

@@ -1,2 +1,2 @@
-nypl-py-utils[mysql-client,postgresql-client,redshift-client,config-helper]==1.1.5
+nypl-py-utils[mysql-client,postgresql-client,redshift-client,config-helper]==1.3.2
 selenium>=4.10.0