From 516f7782c2e6d992d42ec935402f07f73dba4485 Mon Sep 17 00:00:00 2001 From: Michael Hall Date: Mon, 15 Apr 2024 15:30:02 -0500 Subject: [PATCH 1/9] - Added Dockerfile targets for the two images that will be built (lambda, activity), added lambda handler code, and updated the dockerfile. --- Dockerfile | 63 ++++++++++++++++++++----------- dmrpp_generator/lambda_handler.py | 15 ++++++++ dmrpp_generator/main.py | 14 ++++++- dmrpp_generator/version.py | 2 +- main.tf | 10 +++++ modules/dmrpp_lambda/main.tf | 20 ++++++++++ modules/dmrpp_lambda/output.tf | 3 ++ modules/dmrpp_lambda/variables.tf | 43 +++++++++++++++++++++ output.tf | 4 ++ variables.tf | 26 +++++++++++-- 10 files changed, 171 insertions(+), 29 deletions(-) create mode 100644 dmrpp_generator/lambda_handler.py create mode 100644 modules/dmrpp_lambda/main.tf create mode 100644 modules/dmrpp_lambda/output.tf create mode 100644 modules/dmrpp_lambda/variables.tf diff --git a/Dockerfile b/Dockerfile index 1f4d411..89531ce 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,34 +1,51 @@ -FROM opendap/besd:3.21.0-272 +ARG HOME='/home/worker' +ARG BUILD=${HOME}/build + +FROM opendap/besd:3.21.0-272 AS base +HEALTHCHECK NONE + RUN yum -y update && \ yum -y upgrade -HEALTHCHECK NONE + RUN yum install -y nano && \ yum install -y wget -# Adding a user -RUN adduser worker -USER worker -WORKDIR /home/worker -RUN wget https://repo.anaconda.com/miniconda/Miniconda3-py310_23.10.0-1-Linux-x86_64.sh && \ - bash Miniconda3-py310_23.10.0-1-Linux-x86_64.sh -b && \ - rm Miniconda3-py310_23.10.0-1-Linux-x86_64.sh -ENV HOME="/home/worker" PATH="/home/worker/miniconda3/bin:${PATH}" -RUN pip install ipython &&\ - pip install pytest &&\ + +ARG HOME +WORKDIR $HOME +RUN wget https://repo.anaconda.com/miniconda/Miniconda3-py310_23.10.0-1-Linux-x86_64.sh -O miniconda.sh && \ + bash miniconda.sh -b -p ${HOME}/miniconda && \ + rm miniconda.sh && \ + source ${HOME}/miniconda/bin/activate && \ + conda init --all +ENV PATH="${HOME}/miniconda/bin:${PATH}" + +ARG BUILD +WORKDIR ${BUILD} + +COPY setup.py requirements*txt ./ +RUN pip install -r requirements.txt +COPY dmrpp_generator ./dmrpp_generator +COPY generate_dmrpp.py ./generate_dmrpp.py +COPY tests ./tests + +RUN python setup.py install + +RUN pip install ipython && \ + pip install pytest && \ pip install coverage -RUN mkdir $HOME/build -ENV BUILD=$HOME/build -COPY --chown=worker setup.py requirements*txt $BUILD/ -RUN pip install -r $BUILD/requirements.txt -COPY --chown=worker dmrpp_generator $BUILD/dmrpp_generator -COPY --chown=worker generate_dmrpp.py $BUILD/generate_dmrpp.py -COPY --chown=worker tests $BUILD/tests -RUN \ - cd $BUILD; \ - python setup.py install -WORKDIR $BUILD RUN coverage run -m pytest RUN coverage report RUN coverage lcov -o ./coverage/lcov.info RUN rm -rf tests .coverage .pytest_cache + +FROM base AS lambda-image +ARG BUILD +RUN pip install --target $BUILD awslambdaric +COPY site.conf /etc/bes/ + +ENTRYPOINT ["/home/worker/miniconda/bin/python3.10", "-m", "awslambdaric"] +CMD ["dmrpp_generator.lambda_handler.handler"] + +FROM base AS image CMD ["python", "generate_dmrpp.py"] ENTRYPOINT [] diff --git a/dmrpp_generator/lambda_handler.py b/dmrpp_generator/lambda_handler.py new file mode 100644 index 0000000..d5f0758 --- /dev/null +++ b/dmrpp_generator/lambda_handler.py @@ -0,0 +1,15 @@ +from run_cumulus_task import run_cumulus_task + +from dmrpp_generator.main import main + + +def handler(event, context): + # print(f'CMA Event: {event}') + if 'cma' in event: + print(f'Running cumulus task...') + ret = run_cumulus_task(main, event, context) + else: + print(f'Calling main()...') + ret = main(event, context) + + return ret diff --git a/dmrpp_generator/main.py b/dmrpp_generator/main.py index 01b79b5..4cead55 100644 --- a/dmrpp_generator/main.py +++ b/dmrpp_generator/main.py @@ -172,7 +172,13 @@ def get_dmrpp_command(self, dmrpp_meta, input_path, output_filename, local=False dmrpp_options = DMRppOptions(self.path) options = dmrpp_options.get_dmrpp_option(dmrpp_meta=dmrpp_meta) local_option = f"-u file://{output_filename}" if '-u' in options else '' - dmrpp_cmd = f"get_dmrpp {options} {input_path} -o {output_filename}.dmrpp" \ + + use_lambda = dmrpp_meta.get('use_lambda', False) + s_option = '' + if isinstance(use_lambda, bool) and use_lambda: + s_option = '-s /etc/bes/site.conf' + + dmrpp_cmd = f"get_dmrpp {s_option} {options} {input_path} -o {output_filename}.dmrpp" \ f" {local_option} {os.path.basename(output_filename)}" return " ".join(dmrpp_cmd.split()) @@ -220,6 +226,12 @@ def dmrpp_generate(self, input_file, local=False, dmrpp_meta=None, args=None): return out_files +def main(event, context): + print('main event') + print(event) + return DMRPPGenerator(**event).process() + + if __name__ == "__main__": dmr = DMRPPGenerator(input=[], config={}) meta = {"options": [{"flag": "-s", "opt": "htp://localhost/config.conf", "download": "true"}, {"flag": "-M"}]} diff --git a/dmrpp_generator/version.py b/dmrpp_generator/version.py index 3e5f957..05754cf 100644 --- a/dmrpp_generator/version.py +++ b/dmrpp_generator/version.py @@ -1 +1 @@ -__version__ = "v4.4.0" +__version__ = "v5.0.0" diff --git a/main.tf b/main.tf index c87b1af..f3a1bab 100644 --- a/main.tf +++ b/main.tf @@ -12,3 +12,13 @@ module "dmrpp_service" { enable_cw_logging = var.enable_cw_logging get_dmrpp_timeout = var.get_dmrpp_timeout } + +module "dmrpp_lambda" { + source = "./modules/dmrpp_lambda" + + region = var.region + prefix = var.prefix + enable_cw_logging = var.enable_cw_logging + get_dmrpp_timeout = var.get_dmrpp_timeout + cumulus_lambda_role_arn = var.cumulus_lambda_role_arn +} diff --git a/modules/dmrpp_lambda/main.tf b/modules/dmrpp_lambda/main.tf new file mode 100644 index 0000000..2a56ba7 --- /dev/null +++ b/modules/dmrpp_lambda/main.tf @@ -0,0 +1,20 @@ +resource "aws_lambda_function" "ghrc_dmrpp" { + function_name = "${var.prefix}-ghrc-dmrpp" + package_type = "Image" + image_uri = format("%s%s", var.docker_image, "-lambda") + role = var.cumulus_lambda_role_arn + timeout = var.timeout + memory_size = var.memory_size + + ephemeral_storage { + size = var.ephemeral_storage + } + + environment { + variables = { + region = var.region + ENABLE_CW_LOGGING = var.enable_cw_logging + GET_DMRPP_TIMEOUT = var.get_dmrpp_timeout + } + } +} \ No newline at end of file diff --git a/modules/dmrpp_lambda/output.tf b/modules/dmrpp_lambda/output.tf new file mode 100644 index 0000000..3aa8db1 --- /dev/null +++ b/modules/dmrpp_lambda/output.tf @@ -0,0 +1,3 @@ +output "dmrpp_lambda_arn" { + value = aws_lambda_function.ghrc_dmrpp.arn +} \ No newline at end of file diff --git a/modules/dmrpp_lambda/variables.tf b/modules/dmrpp_lambda/variables.tf new file mode 100644 index 0000000..0962153 --- /dev/null +++ b/modules/dmrpp_lambda/variables.tf @@ -0,0 +1,43 @@ +variable "region" { + type = string + default = "us-west-2" +} + +variable "prefix" { + type = string + description = "Cumulus stack prefix" +} + +variable "cumulus_lambda_role_arn" { + type = string + nullable = true +} + +variable "timeout" { + description = "Lambda function time-out" + default = 900 +} + +variable "memory_size" { + description = "Lambda RAM limit" + default = 256 +} + +variable "ephemeral_storage" { + description = "Lambda /tmp storage limit" + default = 512 +} + +variable "enable_cw_logging" { + description = "Enable logging to cloud watch" + type = bool + default = true +} + +variable "get_dmrpp_timeout" { + description = "Duration to wait on the get_dmrpp subprocess call." + type = number + default = 60 +} + +variable "docker_image" {} \ No newline at end of file diff --git a/output.tf b/output.tf index 3fbbeca..aa1aa30 100644 --- a/output.tf +++ b/output.tf @@ -1,3 +1,7 @@ output "dmrpp_task_id" { value = module.dmrpp_service.dmrpp_task_id +} + +output "dmrpp_lambda_arn" { + value = module.dmrpp_lambda.dmrpp_lambda_arn } \ No newline at end of file diff --git a/variables.tf b/variables.tf index 16c31ed..d89e435 100644 --- a/variables.tf +++ b/variables.tf @@ -3,13 +3,11 @@ variable "aws_profile" { default = null } - variable "region" { type = string default = "us-west-2" } - variable "prefix" { type = string description = "Cumulus stack prefix" @@ -43,7 +41,6 @@ variable "volumes" { default = [] } - variable "enable_cw_logging" { description = "Enable logging to cloud watch" type = bool @@ -60,4 +57,25 @@ variable "default_log_retention_days" { type = number default = 30 description = "Default value that user chooses for their log retention periods" -} \ No newline at end of file +} + +# Lambda Variables +variable "cumulus_lambda_role_arn" { + type = string + nullable = true +} + +variable "timeout" { + description = "Lambda function time-out" + default = 900 +} + +variable "memory_size" { + description = "Lambda RAM limit" + default = 256 +} + +variable "ephemeral_storage" { + description = "Lambda /tmp storage limit" + default = 512 +} From 2627187718bb262d20d6d10802bb636f050ee723 Mon Sep 17 00:00:00 2001 From: Michael Hall Date: Mon, 15 Apr 2024 15:48:53 -0500 Subject: [PATCH 2/9] - Updating github workflow --- .github/workflows/push-to-dockerhub.yml | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/.github/workflows/push-to-dockerhub.yml b/.github/workflows/push-to-dockerhub.yml index f75d6a1..9a3fc8b 100644 --- a/.github/workflows/push-to-dockerhub.yml +++ b/.github/workflows/push-to-dockerhub.yml @@ -33,12 +33,21 @@ jobs: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_TOKEN }} - - name: Build and push Docker image + - name: Build and push lambda Docker image uses: docker/build-push-action@v2 with: context: . + target: "lambda-image" push: ${{ contains(github.ref, 'tags') }} - tags: ghrcdaac/dmrpp-generator:${{ steps.tag-name.outputs.tag_name }} + tags: ghrcdaac/dmrpp-generator:${{ steps.tag-name.outputs.tag_name }}-lambda + + - name: Build and push activity Docker image + uses: docker/build-push-action@v2 + with: + context: . + target: "image" + push: ${{ contains(github.ref, 'tags') }} + tags: ghrcdaac/dmrpp-generator:${{ steps.tag-name.outputs.tag_name }}-lambda - name: Extract coverage report run: | From 56bc22bbab61ce980656cb3149ccc562a4d69a88 Mon Sep 17 00:00:00 2001 From: Michael Hall Date: Tue, 16 Apr 2024 11:49:28 -0500 Subject: [PATCH 3/9] - Only a single docker image will be produced now and the lambda will be use an entry_point and command override. --- .github/workflows/push-to-dockerhub.yml | 11 +---------- Dockerfile | 5 ----- main.tf | 1 + modules/dmrpp_lambda/main.tf | 7 ++++++- variables.tf | 3 +++ 5 files changed, 11 insertions(+), 16 deletions(-) diff --git a/.github/workflows/push-to-dockerhub.yml b/.github/workflows/push-to-dockerhub.yml index 9a3fc8b..8bc3064 100644 --- a/.github/workflows/push-to-dockerhub.yml +++ b/.github/workflows/push-to-dockerhub.yml @@ -37,17 +37,8 @@ jobs: uses: docker/build-push-action@v2 with: context: . - target: "lambda-image" push: ${{ contains(github.ref, 'tags') }} - tags: ghrcdaac/dmrpp-generator:${{ steps.tag-name.outputs.tag_name }}-lambda - - - name: Build and push activity Docker image - uses: docker/build-push-action@v2 - with: - context: . - target: "image" - push: ${{ contains(github.ref, 'tags') }} - tags: ghrcdaac/dmrpp-generator:${{ steps.tag-name.outputs.tag_name }}-lambda + tags: ghrcdaac/dmrpp-generator:${{ steps.tag-name.outputs.tag_name }} - name: Extract coverage report run: | diff --git a/Dockerfile b/Dockerfile index 89531ce..23ddcff 100644 --- a/Dockerfile +++ b/Dockerfile @@ -38,14 +38,9 @@ RUN coverage report RUN coverage lcov -o ./coverage/lcov.info RUN rm -rf tests .coverage .pytest_cache -FROM base AS lambda-image ARG BUILD RUN pip install --target $BUILD awslambdaric COPY site.conf /etc/bes/ -ENTRYPOINT ["/home/worker/miniconda/bin/python3.10", "-m", "awslambdaric"] -CMD ["dmrpp_generator.lambda_handler.handler"] - -FROM base AS image CMD ["python", "generate_dmrpp.py"] ENTRYPOINT [] diff --git a/main.tf b/main.tf index f3a1bab..633b6a4 100644 --- a/main.tf +++ b/main.tf @@ -18,6 +18,7 @@ module "dmrpp_lambda" { region = var.region prefix = var.prefix + docker_image = var.docker_image enable_cw_logging = var.enable_cw_logging get_dmrpp_timeout = var.get_dmrpp_timeout cumulus_lambda_role_arn = var.cumulus_lambda_role_arn diff --git a/modules/dmrpp_lambda/main.tf b/modules/dmrpp_lambda/main.tf index 2a56ba7..92d91c6 100644 --- a/modules/dmrpp_lambda/main.tf +++ b/modules/dmrpp_lambda/main.tf @@ -1,11 +1,16 @@ resource "aws_lambda_function" "ghrc_dmrpp" { function_name = "${var.prefix}-ghrc-dmrpp" package_type = "Image" - image_uri = format("%s%s", var.docker_image, "-lambda") + image_uri = var.docker_image role = var.cumulus_lambda_role_arn timeout = var.timeout memory_size = var.memory_size + image_config { + entry_point = ["/home/worker/miniconda/bin/python", "-m", "awslambdaric"] + command = ["dmrpp_generator.lambda_handler.handler"] + } + ephemeral_storage { size = var.ephemeral_storage } diff --git a/variables.tf b/variables.tf index d89e435..8883b83 100644 --- a/variables.tf +++ b/variables.tf @@ -23,12 +23,15 @@ variable "cpu" { variable "memory_reservation" { default = 900 } + variable "cluster_arn" {} + variable "log_destination_arn" { type = string default = null description = "A shared AWS:Log:Destination that receives logs in log_groups" } + variable "docker_image" { description = "Docker image used to generate DMRPP" type = string From a02e7dc236bcb106447c9c8bcb533603d4665409 Mon Sep 17 00:00:00 2001 From: Michael Hall Date: Tue, 16 Apr 2024 11:55:50 -0500 Subject: [PATCH 4/9] - Adding missing config file needed for lambda configuration. --- site.conf | 1 + 1 file changed, 1 insertion(+) create mode 100644 site.conf diff --git a/site.conf b/site.conf new file mode 100644 index 0000000..c120c14 --- /dev/null +++ b/site.conf @@ -0,0 +1 @@ +BES.LogName=/tmp/bes.log \ No newline at end of file From 185c564a2d741aeef965b31cb3a5a8b3a298f4ef Mon Sep 17 00:00:00 2001 From: Michael Hall Date: Tue, 16 Apr 2024 12:46:41 -0500 Subject: [PATCH 5/9] - Changes re.match to re.search to be more flexible and easier to use. --- generate_dmrpp.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/generate_dmrpp.py b/generate_dmrpp.py index d135401..17538ae 100644 --- a/generate_dmrpp.py +++ b/generate_dmrpp.py @@ -2,7 +2,7 @@ from os import listdir, getenv from os.path import isfile, join, basename import json -from re import match +from re import search import logging from dmrpp_generator.main import DMRPPGenerator logging.basicConfig( @@ -42,7 +42,7 @@ def main(): dmrpp.path = workstation_path dmrpp.processing_regex = meta.get('dmrpp_regex', dmrpp.processing_regex) for input_file in input_files: - if match(f"{dmrpp.processing_regex}$", basename(input_file)): + if search(f"{dmrpp.processing_regex}", basename(input_file)): out_files = dmrpp.dmrpp_generate(input_file, local=True, dmrpp_meta=meta, args=args) logger.info(f'Generated: {out_files}') From bdcf601ad7c80bb3a8356e86830c0a45baab9f2d Mon Sep 17 00:00:00 2001 From: Michael Hall Date: Thu, 18 Apr 2024 14:00:06 -0500 Subject: [PATCH 6/9] - Updated terraform code to implement the ECR repository work around. --- README.md | 26 ++++++++++++++++++---- main.tf | 1 + modules/dmrpp_lambda/main.tf | 36 ++++++++++++++++++++++++++++++- modules/dmrpp_lambda/output.tf | 2 +- modules/dmrpp_lambda/variables.tf | 12 ++++++++++- variables.tf | 6 +++++- 6 files changed, 75 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 5226649..84ab481 100644 --- a/README.md +++ b/README.md @@ -9,8 +9,7 @@ ``` # Overview -This repo consists of two components. The DMR++ activity terraform module and a python CLI to the DMR++ Docker -container. +This repo consists of the code for the DMR++ ECS module, lambda, and a python CLI to interact with the DMR++ container. ## Versioning We are following `v..` versioning convention, where: @@ -44,7 +43,6 @@ module "dmrpp-generator" { region = var.region prefix = var.prefix - // Optional parameters docker_image = "ghrcdaac/dmrpp-generator:" // default to the correct release cpu = 800 // default to 800 @@ -53,8 +51,28 @@ module "dmrpp-generator" { prefix = "Cumulus stack prefix" // default Cumulus stack prefix desired_count = 1 // Default to 1 log_destination_arn = var.aws_log_mechanism // default to null + + // Optional Lambda Specific Configuration + cumulus_lambda_role_arn = module.cumulus.lambda_processing_role_arn // If provided the lambda will be provisioned + timeout = 900 + memory_size = 256 + ephemeral_storage = 512 +} +``` +Note: When the lambda is provisioned the module will create a private ECR repository for the dmrpp_generator container. +The first deployment could take +5 minutes as the image needs to be pulled from docker hub and pushed to this new ECR +repository. This is a temporary work around until a public ECR repository can be created for the lambda image. + +## Outputs +The module returns the service id and the lambda ARN: +```terraform +output "dmrpp_task_id" { + value = module.dmrpp_service.dmrpp_task_id } +output "dmrpp_lambda_arn" { + value = module.dmrpp_lambda.dmrpp_lambda_arn +} ``` In [variables.tf](https://github.com/nasa/cumulus-template-deploy/blob/master/cumulus-tf/variables.tf) file you need to define @@ -65,7 +83,7 @@ variable "dmrpp-generator-docker-image" { ``` Assuming you already defined the region and the prefix -# Add the activity to your workflow +# Add the activity/lambda to your workflow In your [workflow.tf](https://github.com/nasa/cumulus-template-deploy/blob/master/cumulus-tf/hello_world_workflow.tf) add ```code "HyraxProcessing": { diff --git a/main.tf b/main.tf index 633b6a4..4a2ad99 100644 --- a/main.tf +++ b/main.tf @@ -15,6 +15,7 @@ module "dmrpp_service" { module "dmrpp_lambda" { source = "./modules/dmrpp_lambda" + account_id = var.account_id region = var.region prefix = var.prefix diff --git a/modules/dmrpp_lambda/main.tf b/modules/dmrpp_lambda/main.tf index 92d91c6..ab3b05a 100644 --- a/modules/dmrpp_lambda/main.tf +++ b/modules/dmrpp_lambda/main.tf @@ -1,7 +1,15 @@ +locals { + image_version = regex("v\\d+.\\d+.\\d+", var.docker_image) + deploy_lambda = length(var.cumulus_lambda_role_arn) > 0 ? true : false +} + resource "aws_lambda_function" "ghrc_dmrpp" { + count = local.deploy_lambda ? 1 : 0 + depends_on = [terraform_data.build_image, aws_ecr_repository.ghrc_dmrpp_lambda] + function_name = "${var.prefix}-ghrc-dmrpp" package_type = "Image" - image_uri = var.docker_image + image_uri = "${aws_ecr_repository.ghrc_dmrpp_lambda[0].repository_url}:${local.image_version}" role = var.cumulus_lambda_role_arn timeout = var.timeout memory_size = var.memory_size @@ -22,4 +30,30 @@ resource "aws_lambda_function" "ghrc_dmrpp" { GET_DMRPP_TIMEOUT = var.get_dmrpp_timeout } } +} + +resource "aws_ecr_repository" "ghrc_dmrpp_lambda" { + count = local.deploy_lambda ? 1 : 0 + name = "${var.prefix}_ghrc_dmrpp_lambda" + image_tag_mutability = "MUTABLE" + force_delete = true + + image_scanning_configuration { + scan_on_push = true + } +} + +resource "terraform_data" "build_image" { + count = local.deploy_lambda ? 1 : 0 + depends_on = [aws_ecr_repository.ghrc_dmrpp_lambda] + triggers_replace = [var.docker_image] + + provisioner "local-exec" { + command = <<-EOT + docker pull ${var.docker_image} + aws ecr get-login-password --region ${var.region} | docker login --username AWS --password-stdin ${var.account_id}.dkr.ecr.${var.region}.amazonaws.com + docker tag ${var.docker_image} ${aws_ecr_repository.ghrc_dmrpp_lambda[0].repository_url}:${local.image_version} + docker push ${aws_ecr_repository.ghrc_dmrpp_lambda[0].repository_url}:${local.image_version} + EOT + } } \ No newline at end of file diff --git a/modules/dmrpp_lambda/output.tf b/modules/dmrpp_lambda/output.tf index 3aa8db1..7e53150 100644 --- a/modules/dmrpp_lambda/output.tf +++ b/modules/dmrpp_lambda/output.tf @@ -1,3 +1,3 @@ output "dmrpp_lambda_arn" { - value = aws_lambda_function.ghrc_dmrpp.arn + value = join("", aws_lambda_function.ghrc_dmrpp.*.arn) } \ No newline at end of file diff --git a/modules/dmrpp_lambda/variables.tf b/modules/dmrpp_lambda/variables.tf index 0962153..52bb59a 100644 --- a/modules/dmrpp_lambda/variables.tf +++ b/modules/dmrpp_lambda/variables.tf @@ -3,6 +3,10 @@ variable "region" { default = "us-west-2" } +variable "account_id" { + type = string +} + variable "prefix" { type = string description = "Cumulus stack prefix" @@ -40,4 +44,10 @@ variable "get_dmrpp_timeout" { default = 60 } -variable "docker_image" {} \ No newline at end of file +variable "docker_image" { + description = "ECR Lambda docker image" + type = string + default = "ghrcdaac/dmrpp-generator:VERSION_SUB" +} + + diff --git a/variables.tf b/variables.tf index 8883b83..1aca3ad 100644 --- a/variables.tf +++ b/variables.tf @@ -3,6 +3,10 @@ variable "aws_profile" { default = null } +variable "account_id" { + type = string +} + variable "region" { type = string default = "us-west-2" @@ -65,7 +69,7 @@ variable "default_log_retention_days" { # Lambda Variables variable "cumulus_lambda_role_arn" { type = string - nullable = true + default = "" } variable "timeout" { From 206f83c3bd9bdfdcaf78fd9258f957711d56f4ff Mon Sep 17 00:00:00 2001 From: Michael Hall Date: Thu, 18 Apr 2024 14:23:31 -0500 Subject: [PATCH 7/9] - Added required field to readme. --- README.md | 5 +++-- modules/dmrpp_lambda/main.tf | 2 +- modules/dmrpp_lambda/variables.tf | 2 -- output.tf | 2 +- 4 files changed, 5 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 84ab481..090938d 100644 --- a/README.md +++ b/README.md @@ -37,13 +37,14 @@ In [main.tf](https://github.com/nasa/cumulus-template-deploy/blob/master/cumulus (where you defined cumulus module) add ```terraform module "dmrpp-generator" { - // Required parameters + // Required Parameters source = "https://github.com/ghrcdaac/dmrpp-generator/releases/download//dmrpp-generator.zip" cluster_arn = module.cumulus.ecs_cluster_arn region = var.region prefix = var.prefix + account_id = var.account_id - // Optional parameters + // Optional Activity Parameters docker_image = "ghrcdaac/dmrpp-generator:" // default to the correct release cpu = 800 // default to 800 enable_cw_logging = False // default to False diff --git a/modules/dmrpp_lambda/main.tf b/modules/dmrpp_lambda/main.tf index ab3b05a..d676a54 100644 --- a/modules/dmrpp_lambda/main.tf +++ b/modules/dmrpp_lambda/main.tf @@ -56,4 +56,4 @@ resource "terraform_data" "build_image" { docker push ${aws_ecr_repository.ghrc_dmrpp_lambda[0].repository_url}:${local.image_version} EOT } -} \ No newline at end of file +} diff --git a/modules/dmrpp_lambda/variables.tf b/modules/dmrpp_lambda/variables.tf index 52bb59a..1f77154 100644 --- a/modules/dmrpp_lambda/variables.tf +++ b/modules/dmrpp_lambda/variables.tf @@ -49,5 +49,3 @@ variable "docker_image" { type = string default = "ghrcdaac/dmrpp-generator:VERSION_SUB" } - - diff --git a/output.tf b/output.tf index aa1aa30..7e617e9 100644 --- a/output.tf +++ b/output.tf @@ -4,4 +4,4 @@ output "dmrpp_task_id" { output "dmrpp_lambda_arn" { value = module.dmrpp_lambda.dmrpp_lambda_arn -} \ No newline at end of file +} From 661a86b75ae140a2cebc7b5b8a3b22abb0249a5d Mon Sep 17 00:00:00 2001 From: Michael Hall Date: Thu, 18 Apr 2024 15:48:48 -0500 Subject: [PATCH 8/9] - Make the application of the -s option automatic if running in AWS lambda. --- dmrpp_generator/main.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/dmrpp_generator/main.py b/dmrpp_generator/main.py index 4cead55..26ba157 100644 --- a/dmrpp_generator/main.py +++ b/dmrpp_generator/main.py @@ -173,9 +173,8 @@ def get_dmrpp_command(self, dmrpp_meta, input_path, output_filename, local=False options = dmrpp_options.get_dmrpp_option(dmrpp_meta=dmrpp_meta) local_option = f"-u file://{output_filename}" if '-u' in options else '' - use_lambda = dmrpp_meta.get('use_lambda', False) s_option = '' - if isinstance(use_lambda, bool) and use_lambda: + if os.getenv('AWS_LAMBDA_FUNCTION_NAME'): s_option = '-s /etc/bes/site.conf' dmrpp_cmd = f"get_dmrpp {s_option} {options} {input_path} -o {output_filename}.dmrpp" \ From 344f4ce32fbb531623a5e3d40550ba79c62027af Mon Sep 17 00:00:00 2001 From: Michael Hall Date: Mon, 22 Apr 2024 11:50:48 -0500 Subject: [PATCH 9/9] - Cleanud up dockerfile. --- Dockerfile | 41 ++++++++++++++++++----------------------- 1 file changed, 18 insertions(+), 23 deletions(-) diff --git a/Dockerfile b/Dockerfile index 23ddcff..4f231c6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,3 @@ -ARG HOME='/home/worker' -ARG BUILD=${HOME}/build - FROM opendap/besd:3.21.0-272 AS base HEALTHCHECK NONE @@ -10,35 +7,33 @@ RUN yum -y update && \ RUN yum install -y nano && \ yum install -y wget -ARG HOME -WORKDIR $HOME +ARG HOME='/home/worker' +WORKDIR ${HOME} RUN wget https://repo.anaconda.com/miniconda/Miniconda3-py310_23.10.0-1-Linux-x86_64.sh -O miniconda.sh && \ bash miniconda.sh -b -p ${HOME}/miniconda && \ - rm miniconda.sh && \ - source ${HOME}/miniconda/bin/activate && \ - conda init --all -ENV PATH="${HOME}/miniconda/bin:${PATH}" + rm miniconda.sh -ARG BUILD +ENV PATH="${HOME}/miniconda/bin:${PATH}" +ARG BUILD=${HOME}/build WORKDIR ${BUILD} -COPY setup.py requirements*txt ./ -RUN pip install -r requirements.txt -COPY dmrpp_generator ./dmrpp_generator -COPY generate_dmrpp.py ./generate_dmrpp.py -COPY tests ./tests - -RUN python setup.py install - RUN pip install ipython && \ pip install pytest && \ pip install coverage -RUN coverage run -m pytest -RUN coverage report -RUN coverage lcov -o ./coverage/lcov.info -RUN rm -rf tests .coverage .pytest_cache -ARG BUILD +COPY setup.py requirements*txt generate_dmrpp.py ./ +COPY dmrpp_generator ./dmrpp_generator/ +COPY tests ./tests/ +RUN pip install -r requirements.txt && \ + python setup.py install + +RUN coverage run -m pytest && \ + coverage report && \ + coverage lcov -o ./coverage/lcov.info && \ + rm -rf tests .coverage .pytest_cache && \ + pip uninstall pytest -y && \ + pip uninstall coverage -y + RUN pip install --target $BUILD awslambdaric COPY site.conf /etc/bes/