Merge pull request #62 from ghrcdaac/mlh0079-6188-6025-besd-lambda

Mlh0079 6188 6025 besd lambda

.github/workflows/push-to-dockerhub.yml

@@ -33,7 +33,7 @@ jobs:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_TOKEN }}
 
-      - name: Build and push Docker image
+      - name: Build and push lambda Docker image
         uses: docker/build-push-action@v2
         with:
           context: .

Dockerfile

@@ -1,34 +1,41 @@
-FROM opendap/besd:3.21.0-272
+FROM opendap/besd:3.21.0-272 AS base
+HEALTHCHECK NONE
+
 RUN yum -y update && \
     yum -y upgrade
-HEALTHCHECK NONE
+
 RUN yum install -y nano && \
     yum install -y wget
-# Adding a user
-RUN adduser worker
-USER worker
-WORKDIR /home/worker
-RUN wget https://repo.anaconda.com/miniconda/Miniconda3-py310_23.10.0-1-Linux-x86_64.sh && \
-    bash Miniconda3-py310_23.10.0-1-Linux-x86_64.sh -b && \
-    rm Miniconda3-py310_23.10.0-1-Linux-x86_64.sh
-ENV HOME="/home/worker" PATH="/home/worker/miniconda3/bin:${PATH}"
-RUN pip install ipython &&\
-    pip install pytest &&\
+
+ARG HOME='/home/worker'
+WORKDIR ${HOME}
+RUN wget https://repo.anaconda.com/miniconda/Miniconda3-py310_23.10.0-1-Linux-x86_64.sh -O miniconda.sh && \
+    bash miniconda.sh -b -p ${HOME}/miniconda && \
+    rm miniconda.sh
+
+ENV PATH="${HOME}/miniconda/bin:${PATH}"
+ARG BUILD=${HOME}/build
+WORKDIR ${BUILD}
+
+RUN pip install ipython  && \
+    pip install pytest  && \
     pip install coverage
-RUN mkdir $HOME/build
-ENV BUILD=$HOME/build
-COPY --chown=worker setup.py requirements*txt $BUILD/
-RUN pip install -r $BUILD/requirements.txt
-COPY --chown=worker dmrpp_generator $BUILD/dmrpp_generator
-COPY --chown=worker generate_dmrpp.py $BUILD/generate_dmrpp.py
-COPY --chown=worker tests $BUILD/tests
-RUN \
-  cd $BUILD; \
-  python setup.py install
-WORKDIR $BUILD
-RUN coverage run -m pytest
-RUN coverage report
-RUN coverage lcov -o ./coverage/lcov.info
-RUN rm -rf tests .coverage .pytest_cache
+
+COPY setup.py requirements*txt generate_dmrpp.py ./
+COPY dmrpp_generator ./dmrpp_generator/
+COPY tests ./tests/
+RUN pip install -r requirements.txt && \
+    python setup.py install
+
+RUN coverage run -m pytest && \
+    coverage report && \
+    coverage lcov -o ./coverage/lcov.info && \
+    rm -rf tests .coverage .pytest_cache && \
+    pip uninstall pytest -y && \
+    pip uninstall coverage -y
+
+RUN pip install --target $BUILD awslambdaric
+COPY site.conf /etc/bes/
+
 CMD ["python", "generate_dmrpp.py"]
 ENTRYPOINT []

README.md

@@ -9,8 +9,7 @@
 ```
 
 # Overview
-This repo consists of two components. The DMR++ activity terraform module and a python CLI to the DMR++ Docker 
-container.
+This repo consists of the code for the DMR++ ECS module, lambda, and a python CLI to interact with the DMR++ container.
 
 ## Versioning
 We are following `v<major>.<minor>.<patch>` versioning convention, where:
@@ -38,23 +37,43 @@ In [main.tf](https://github.com/nasa/cumulus-template-deploy/blob/master/cumulus
  (where you defined cumulus module) add
  ```terraform
 module "dmrpp-generator" {
-  // Required parameters
+  // Required Parameters
   source = "https://github.com/ghrcdaac/dmrpp-generator/releases/download/<tag_num>/dmrpp-generator.zip"
   cluster_arn = module.cumulus.ecs_cluster_arn
   region = var.region
   prefix = var.prefix
+  account_id = var.account_id
   
-
-  // Optional parameters
+  // Optional Activity Parameters
   docker_image = "ghrcdaac/dmrpp-generator:<tag_num>" // default to the correct release
   cpu = 800 // default to 800
   enable_cw_logging = False // default to False
   memory_reservation = 900 // default to 900
   prefix = "Cumulus stack prefix" // default Cumulus stack prefix
   desired_count = 1  // Default to 1
   log_destination_arn = var.aws_log_mechanism // default to null
+  
+  // Optional Lambda Specific Configuration  
+  cumulus_lambda_role_arn = module.cumulus.lambda_processing_role_arn // If provided the lambda will be provisioned
+  timeout = 900
+  memory_size = 256
+  ephemeral_storage = 512
+}
+```
+Note: When the lambda is provisioned the module will create a private ECR repository for the dmrpp_generator container. 
+The first deployment could take +5 minutes as the image needs to be pulled from docker hub and pushed to this new ECR
+repository. This is a temporary work around until a public ECR repository can be created for the lambda image.
+
+## Outputs
+The module returns the service id and the lambda ARN:
+```terraform
+output "dmrpp_task_id" {
+  value = module.dmrpp_service.dmrpp_task_id
 }
 
+output "dmrpp_lambda_arn" {
+  value = module.dmrpp_lambda.dmrpp_lambda_arn
+}
 ```
 In [variables.tf](https://github.com/nasa/cumulus-template-deploy/blob/master/cumulus-tf/variables.tf)
 file you need to define
@@ -65,7 +84,7 @@ variable "dmrpp-generator-docker-image" {
 ```
 Assuming you already defined the region and the prefix
 
-# Add the activity to your workflow
+# Add the activity/lambda to your workflow
 In your [workflow.tf](https://github.com/nasa/cumulus-template-deploy/blob/master/cumulus-tf/hello_world_workflow.tf) add
 ```code
    "HyraxProcessing": {

dmrpp_generator/lambda_handler.py

@@ -0,0 +1,15 @@
+from run_cumulus_task import run_cumulus_task
+
+from dmrpp_generator.main import main
+
+
+def handler(event, context):
+    # print(f'CMA Event: {event}')
+    if 'cma' in event:
+        print(f'Running cumulus task...')
+        ret = run_cumulus_task(main, event, context)
+    else:
+        print(f'Calling main()...')
+        ret = main(event, context)
+
+    return ret

dmrpp_generator/main.py

@@ -172,7 +172,12 @@ def get_dmrpp_command(self, dmrpp_meta, input_path, output_filename, local=False
         dmrpp_options = DMRppOptions(self.path)
         options = dmrpp_options.get_dmrpp_option(dmrpp_meta=dmrpp_meta)
         local_option = f"-u file://{output_filename}" if '-u' in options else ''
-        dmrpp_cmd = f"get_dmrpp {options} {input_path} -o {output_filename}.dmrpp" \
+
+        s_option = ''
+        if os.getenv('AWS_LAMBDA_FUNCTION_NAME'):
+            s_option = '-s /etc/bes/site.conf'
+
+        dmrpp_cmd = f"get_dmrpp {s_option} {options} {input_path} -o {output_filename}.dmrpp" \
                     f" {local_option} {os.path.basename(output_filename)}"
         return " ".join(dmrpp_cmd.split())
 
@@ -220,6 +225,12 @@ def dmrpp_generate(self, input_file, local=False, dmrpp_meta=None, args=None):
         return out_files
 
 
+def main(event, context):
+    print('main event')
+    print(event)
+    return DMRPPGenerator(**event).process()
+
+
 if __name__ == "__main__":
     dmr = DMRPPGenerator(input=[], config={})
     meta = {"options": [{"flag": "-s", "opt": "htp://localhost/config.conf", "download": "true"}, {"flag": "-M"}]}

dmrpp_generator/version.py

@@ -1 +1 @@
-__version__ = "v4.4.1"
+__version__ = "v5.0.0"

generate_dmrpp.py

@@ -2,7 +2,7 @@
 from os import listdir, getenv
 from os.path import isfile, join, basename
 import json
-from re import match
+from re import search
 import logging
 from dmrpp_generator.main import DMRPPGenerator
 logging.basicConfig(
@@ -42,7 +42,7 @@ def main():
     dmrpp.path = workstation_path
     dmrpp.processing_regex = meta.get('dmrpp_regex', dmrpp.processing_regex)
     for input_file in input_files:
-        if match(f"{dmrpp.processing_regex}$", basename(input_file)):
+        if search(f"{dmrpp.processing_regex}", basename(input_file)):
             out_files = dmrpp.dmrpp_generate(input_file, local=True, dmrpp_meta=meta, args=args)
             logger.info(f'Generated: {out_files}')
 

main.tf

@@ -12,3 +12,15 @@ module "dmrpp_service" {
   enable_cw_logging   = var.enable_cw_logging
   get_dmrpp_timeout   = var.get_dmrpp_timeout
 }
+
+module "dmrpp_lambda" {
+  source = "./modules/dmrpp_lambda"
+  account_id = var.account_id
+
+  region = var.region
+  prefix = var.prefix
+  docker_image = var.docker_image
+  enable_cw_logging  = var.enable_cw_logging
+  get_dmrpp_timeout  = var.get_dmrpp_timeout
+  cumulus_lambda_role_arn = var.cumulus_lambda_role_arn
+}

modules/dmrpp_lambda/main.tf

@@ -0,0 +1,59 @@
+locals {
+  image_version = regex("v\\d+.\\d+.\\d+", var.docker_image)
+  deploy_lambda = length(var.cumulus_lambda_role_arn) > 0 ? true : false
+}
+
+resource "aws_lambda_function" "ghrc_dmrpp" {
+  count = local.deploy_lambda ? 1 : 0
+  depends_on = [terraform_data.build_image, aws_ecr_repository.ghrc_dmrpp_lambda]
+
+  function_name = "${var.prefix}-ghrc-dmrpp"
+  package_type = "Image"
+  image_uri = "${aws_ecr_repository.ghrc_dmrpp_lambda[0].repository_url}:${local.image_version}"
+  role = var.cumulus_lambda_role_arn
+  timeout = var.timeout
+  memory_size = var.memory_size
+
+  image_config {
+    entry_point = ["/home/worker/miniconda/bin/python", "-m", "awslambdaric"]
+    command = ["dmrpp_generator.lambda_handler.handler"]
+  }
+
+  ephemeral_storage {
+    size = var.ephemeral_storage
+  }
+
+  environment {
+    variables = {
+      region = var.region
+      ENABLE_CW_LOGGING = var.enable_cw_logging
+      GET_DMRPP_TIMEOUT = var.get_dmrpp_timeout
+    }
+  }
+}
+
+resource "aws_ecr_repository" "ghrc_dmrpp_lambda" {
+  count = local.deploy_lambda ? 1 : 0
+  name                 = "${var.prefix}_ghrc_dmrpp_lambda"
+  image_tag_mutability = "MUTABLE"
+  force_delete = true
+
+  image_scanning_configuration {
+    scan_on_push = true
+  }
+}
+
+resource "terraform_data" "build_image" {
+  count = local.deploy_lambda ? 1 : 0
+  depends_on = [aws_ecr_repository.ghrc_dmrpp_lambda]
+  triggers_replace = [var.docker_image]
+
+  provisioner "local-exec" {
+    command = <<-EOT
+      docker pull ${var.docker_image}
+      aws ecr get-login-password --region ${var.region} | docker login --username AWS --password-stdin ${var.account_id}.dkr.ecr.${var.region}.amazonaws.com
+      docker tag ${var.docker_image} ${aws_ecr_repository.ghrc_dmrpp_lambda[0].repository_url}:${local.image_version}
+      docker push ${aws_ecr_repository.ghrc_dmrpp_lambda[0].repository_url}:${local.image_version}
+    EOT
+  }
+}

modules/dmrpp_lambda/output.tf

@@ -0,0 +1,3 @@
+output "dmrpp_lambda_arn" {
+  value = join("", aws_lambda_function.ghrc_dmrpp.*.arn)
+}

modules/dmrpp_lambda/variables.tf

@@ -0,0 +1,51 @@
+variable "region" {
+  type    = string
+  default = "us-west-2"
+}
+
+variable "account_id" {
+  type = string
+}
+
+variable "prefix" {
+  type        = string
+  description = "Cumulus stack prefix"
+}
+
+variable "cumulus_lambda_role_arn" {
+  type = string
+  nullable = true
+}
+
+variable "timeout" {
+  description = "Lambda function time-out"
+  default     = 900
+}
+
+variable "memory_size" {
+  description = "Lambda RAM limit"
+  default     = 256
+}
+
+variable "ephemeral_storage" {
+  description = "Lambda /tmp storage limit"
+  default     = 512
+}
+
+variable "enable_cw_logging" {
+  description = "Enable logging to cloud watch"
+  type        = bool
+  default     = true
+}
+
+variable "get_dmrpp_timeout" {
+  description = "Duration to wait on the get_dmrpp subprocess call."
+  type    = number
+  default = 60
+}
+
+variable "docker_image" {
+    description = "ECR Lambda docker image"
+    type    = string
+    default = "ghrcdaac/dmrpp-generator:VERSION_SUB"
+}

output.tf

@@ -1,3 +1,7 @@
 output "dmrpp_task_id" {
   value = module.dmrpp_service.dmrpp_task_id
-}
+}
+
+output "dmrpp_lambda_arn" {
+  value = module.dmrpp_lambda.dmrpp_lambda_arn
+}

site.conf

@@ -0,0 +1 @@
+BES.LogName=/tmp/bes.log