Hydrofab2ngen tools

.gitignore

@@ -0,0 +1,3 @@
+data/*
+venv9/
+tests/data/ngen_inputs/forcing/*

Dockerfile

@@ -0,0 +1,45 @@
+# Define custom function directory
+ARG FUNCTION_DIR="/function"
+
+FROM python:3.9 as build-image
+
+# Include global arg in this stage of the build
+ARG FUNCTION_DIR
+
+# Copy function code
+RUN mkdir -p ${FUNCTION_DIR}
+RUN mkdir -p ${FUNCTION_DIR}/ngen_forcing
+RUN mkdir -p ${FUNCTION_DIR}/subsetting
+RUN mkdir -p ${FUNCTION_DIR}/nwm_filenames
+COPY ./ngen_forcing ${FUNCTION_DIR}/ngen_forcing
+COPY ./subsetting ${FUNCTION_DIR}/subsetting
+COPY ./nwm_filenames ${FUNCTION_DIR}/nwm_filenames
+COPY requirements.txt ${FUNCTION_DIR}
+COPY lambda_function.py ${FUNCTION_DIR}
+
+# Install the function's dependencies
+RUN pip3 install --upgrade pip
+RUN pip3 install --no-cache-dir \
+    --target ${FUNCTION_DIR} \
+        awslambdaric
+
+# Use a slim version of the base Python image to reduce the final image size
+FROM python:3.9-slim
+
+# Include global arg in this stage of the build
+ARG FUNCTION_DIR
+
+# Set working directory to function root directory
+WORKDIR ${FUNCTION_DIR}
+
+# Copy in the built dependencies
+COPY --from=build-image ${FUNCTION_DIR} ${FUNCTION_DIR}
+
+RUN pip3 install --upgrade pip
+RUN pip3 install --no-cache-dir -r "${FUNCTION_DIR}/requirements.txt"
+
+# Set runtime interface client as default command for the container runtime
+ENTRYPOINT [ "/usr/local/bin/python", "-m", "awslambdaric" ]
+
+# Pass the name of the function handler as an argument to the runtime
+CMD [ "lambda_function.handler" ]

forcing_processor_lambda.tf

@@ -0,0 +1,151 @@
+terraform {
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = "~> 4.0"
+    }
+  }
+}
+
+# Variable declarations
+variable "region" {
+  type = string
+}
+
+variable "trigger_bucket" {
+  type = string
+}
+
+variable "ecr_repo" {
+  type = string
+}
+
+variable "function_name" {
+  type = string
+}
+
+variable "trigger_file_prefix" {
+  type = string
+}
+
+variable "trigger_file_suffix" {
+  type = string
+}
+
+variable "image_tag" {
+  type = string
+}
+
+variable "memory_size" {
+  type = number
+}
+
+provider "aws" {
+  region = var.region
+}
+
+data "aws_ecr_repository" "image_repo" {
+  name = var.ecr_repo
+}
+
+# Create function and set role
+resource "aws_lambda_function" "forcing_processor_function" {
+  function_name = "${var.function_name}"
+  timeout       = 900 # 900 is max
+  image_uri     = "${data.aws_ecr_repository.image_repo.repository_url}:${var.image_tag}"
+  package_type  = "Image"
+
+  memory_size = var.memory_size
+
+
+  role = aws_iam_role.forcing_processor_function_role.arn
+
+}
+
+resource "aws_iam_role" "forcing_processor_function_role" {
+  name = "forcing-processor"
+
+  assume_role_policy = jsonencode({
+    Statement = [
+      {
+        Action = "sts:AssumeRole"
+        Effect = "Allow"
+        Principal = {
+          Service = "lambda.amazonaws.com"
+        }
+      },
+    ]
+  })
+}
+
+# Set up the trigger
+resource "aws_s3_bucket" "trigger_bucket" {
+  bucket = var.trigger_bucket
+}
+
+resource "aws_s3_bucket_notification" "bucket_notification" {
+  bucket = aws_s3_bucket.trigger_bucket.id
+
+  lambda_function {
+    lambda_function_arn = aws_lambda_function.forcing_processor_function.arn
+    events              = ["s3:ObjectCreated:*"]
+    filter_prefix       = var.trigger_file_prefix
+    filter_suffix       = var.trigger_file_suffix
+  }
+
+  depends_on = [aws_lambda_permission.allow_bucket]
+}
+
+resource "aws_lambda_permission" "allow_bucket" {
+  statement_id  = "AllowExecutionFromS3Bucket"
+  action        = "lambda:InvokeFunction"
+  function_name = aws_lambda_function.forcing_processor_function.arn
+  principal     = "s3.amazonaws.com"
+  source_arn    = aws_s3_bucket.trigger_bucket.arn
+}
+
+resource "aws_iam_policy" "function_logging_policy" {
+  name   = "function-logging-policy"
+  policy = jsonencode({
+    "Version" : "2012-10-17",
+    "Statement" : [
+      {
+        Action : [
+            "logs:CreateLogGroup",
+            "logs:CreateLogStream",
+            "logs:PutLogEvents"
+        ],
+        Effect : "Allow",
+        Resource : "arn:aws:logs:*:*:*"
+      }
+    ]
+  })
+}
+
+resource "aws_iam_role_policy_attachment" "function_logging_policy_attachment" {
+  role = aws_iam_role.forcing_processor_function_role.id
+  policy_arn = aws_iam_policy.function_logging_policy.arn
+}
+
+# Add secret access to lambda function
+resource "aws_iam_policy" "secrets_manager_policy" {
+  name        = "secrets_manager_access_policy"
+  description = "Allows access to Secrets Manager"
+
+  policy = jsonencode({
+    Version   = "2012-10-17"
+    Statement = [
+      {
+        Sid       = "AllowSecretsManagerAccess"
+        Effect    = "Allow"
+        Action    = ["secretsmanager:GetSecretValue"]
+        Resource  = "*"
+      }
+    ]
+  })
+}
+
+resource "aws_iam_role_policy_attachment" "secrets_manager_attachment" {
+  role       = aws_iam_role.forcing_processor_function_role.name
+  policy_arn = aws_iam_policy.secrets_manager_policy.arn
+}

jl_dev.tfvars

@@ -0,0 +1,8 @@
+region = "us-east-2"
+trigger_bucket = "ngenresources"
+ecr_repo = "nextgenforcing"
+function_name = "forcingprocessor"
+trigger_file_prefix = ""
+trigger_file_suffix = "02.conus.nc.txt"
+image_tag = "forcingprocessor"
+memory_size = 4096

lambda_function.py

@@ -0,0 +1,15 @@
+import sys, json
+# from aws_lambda_powertools.utilities import parameters
+
+def handler(event, context):
+
+    # load template config
+    conf = json.load(open('/data_access_examples/ngen_forcing/ngen_forcings_lambda.json'))
+
+    # get date from event
+
+    # call function
+    from ngen_forcing import prep_hydrofab_forcings_ngen
+    prep_hydrofab_forcings_ngen.prep_ngen_data(conf)
+
+    return 'Done!'

ngen_forcing/filenames.txt

@@ -0,0 +1,3 @@
+https://storage.googleapis.com/national-water-model/nwm.20220822/forcing_medium_range/nwm.t00z.medium_range.forcing.f006.conus.nc
+https://storage.googleapis.com/national-water-model/nwm.20220822/forcing_medium_range/nwm.t00z.medium_range.forcing.f007.conus.nc
+https://storage.googleapis.com/national-water-model/nwm.20220822/forcing_medium_range/nwm.t00z.medium_range.forcing.f008.conus.nc

ngen_forcing/ngen_forcings_lambda.json

@@ -0,0 +1,42 @@
+{
+    "forcing"  : {
+        "forcing_type" : "operational_archive",
+        "start_date"   : "20230705",
+        "end_date"     : "20230705",
+        "cache"        : false,        
+        "nwm_file"     : "",
+        "path_override": "",
+        "runinput"     : 1,
+        "varinput"     : 5,
+        "geoinput"     : 1,
+        "meminput"     : 0,
+        "urlbaseinput" : 3,
+        "fcst_cycle"   : [0],
+        "lead_time"    : [1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18]
+    },
+
+    "hydrofab" : {
+        "version"      : "v1.2",
+        "vpu"          : "ENV",
+        "geopkg_file"  : "",
+        "catch_subset" : "",
+        "weights_only" : false
+    },
+
+    "storage":{
+        "storage_type"       : "S3",
+        "output_bucket"      : "ngenforcingdev1",
+        "output_bucket_path" : "",
+        "cache_bucket"       : "ngenforcingresources",
+        "cache_bucket_path"  : "",
+        "output_file_type"   : "csv"
+    },    
+
+    "run" : {
+        "verbose"       : true,
+        "dl_threads"    : 1,
+        "collect_stats" : true,
+        "secret_name"   : "jlaser_creds",
+        "region_name"   : "us-west-2"
+    }
+}