From f1591a0832b50420cd823b5b62f0d087e65e4af2 Mon Sep 17 00:00:00 2001
From: Chuck Daniels <chuck@developmentseed.org>
Date: Mon, 13 Jan 2025 17:14:23 -0500
Subject: [PATCH] Add lifecycle policy to Sentinel input buckets

Whole number of days after which object expire must be set via the
environment variable `HLS_SENTINEL_INPUT_BUCKET_EXPIRATION_DAYS`.

Fixes #294
---
 environment.sh.sample | 22 +++++++++++++---------
 stack/stack.py        | 18 ++++++++++++++++++
 2 files changed, 31 insertions(+), 9 deletions(-)

diff --git a/environment.sh.sample b/environment.sh.sample
index b38ddd5..8bc458b 100755
--- a/environment.sh.sample
+++ b/environment.sh.sample
@@ -1,9 +1,11 @@
 #!/bin/bash
 
 # Make sure that necessary executables are installed
-for b in jq aws
-do
-    command -v $b >/dev/null 2>&1 || { echo >&2 "I require $b but it's not installed.  Aborting."; exit 1; }
+for b in jq aws; do
+    command -v $b >/dev/null 2>&1 || {
+        echo >&2 "I require $b but it's not installed.  Aborting."
+        exit 1
+    }
 done
 
 # Set allexport mode, all variables defined in this block will get exported
@@ -19,7 +21,6 @@ HLS_LAADS_TOKEN="<your token>"
 HLS_OUTPUT_BUCKET=hls-global
 HLS_OUTPUT_BUCKET_HISTORIC=hls-gobal-historic
 
-
 # Role for copying to output bucket
 HLS_OUTPUT_BUCKET_ROLE_ARN=arn:aws:iam::611670965994:role/gcc-S3Test
 
@@ -27,7 +28,6 @@ HLS_OUTPUT_BUCKET_ROLE_ARN=arn:aws:iam::611670965994:role/gcc-S3Test
 HLS_LANDSAT_SNS_TOPIC=arn:aws:sns:us-west-2:673253540267:public-c2-notify
 HLS_LANDASAT_HISTORIC_SNS_TOPIC=arn:aws:sns:us-west-2:018923174646:landsat-historic-LandsatHistoricTopic643F0596-1TIGFB893SX3B
 
-
 # Bucket for merged GIBS tile output.
 HLS_GIBS_OUTPUT_BUCKET=hls-browse-imagery
 HLS_LAADS_BUCKET_BOOTSTRAP=hls-development-laads-bucket
@@ -76,6 +76,9 @@ HLS_SSH_KEYNAME=hls-mount
 # Sentinel serverless downloader function role arn.
 HLS_DOWNLOADER_FUNCTION_ARN=something
 
+# Number of days after which objects in the Sentinel input buckets expire
+HLS_SENTINEL_INPUT_BUCKET_EXPIRATION_DAYS=60
+
 # GCC Specific environment settings.
 GCC=false
 HLS_GCC_ACCOUNT=account_id
@@ -89,13 +92,14 @@ HLS_GCC_BOUNDARY_ARN=boudary_policy_arn
 set +a
 
 # Set environment variables for all outputs set up in cloud formation
-stack_info=$(aws cloudformation describe-stacks --stack-name ${HLS_STACKNAME} --output json)
+stack_info=$(aws cloudformation describe-stacks --stack-name "${HLS_STACKNAME}" --output json)
 if [[ "$stack_info" =~ "OutputKey" ]]; then
     l=$(echo "$stack_info" | jq ".Stacks[].Outputs | length")
-    for ((i=0;i<$l;++i)); do
-        key=$(echo "$stack_info" | jq ".Stacks[].Outputs[$i].OutputKey" | sed -e 's/^"//'  -e 's/"$//')
+
+    for ((i = 0; i < l; ++i)); do
+        key=$(echo "$stack_info" | jq ".Stacks[].Outputs[$i].OutputKey" | sed -e 's/^"//' -e 's/"$//')
         keyupper=$(echo "$key" | awk '{print toupper($0)}')
-        val=$(echo "$stack_info" | jq ".Stacks[].Outputs[$i].OutputValue" | sed -e 's/^"//'  -e 's/"$//')
+        val=$(echo "$stack_info" | jq ".Stacks[].Outputs[$i].OutputValue" | sed -e 's/^"//' -e 's/"$//')
         export "HLSSTACK_$keyupper"="$val"
     done
 fi
diff --git a/stack/stack.py b/stack/stack.py
index cad0ba3..72a4cde 100644
--- a/stack/stack.py
+++ b/stack/stack.py
@@ -149,12 +149,23 @@ def __init__(self, scope: Construct, id: str, **kwargs) -> None:
             self, "landsat_output_bucket", OUTPUT_BUCKET
         )
 
+        sentinel_input_bucket_expiration_days = int(
+            os.environ["HLS_SENTINEL_INPUT_BUCKET_EXPIRATION_DAYS"]
+        )
+
         # Must be created as part of the stack due to trigger requirements
         self.sentinel_input_bucket = aws_s3.Bucket(
             self,
             "SentinelInputBucket",
             bucket_name=SENTINEL_INPUT_BUCKET,
             removal_policy=RemovalPolicy.DESTROY,
+            lifecycle_rules=[
+                aws_s3.LifecycleRule(
+                    expiration=Duration.days(sentinel_input_bucket_expiration_days),
+                    expired_object_delete_marker=True,
+                    noncurrent_version_expiration=Duration.days(1),
+                )
+            ],
         )
 
         self.sentinel_input_bucket_historic = aws_s3.Bucket(
@@ -162,6 +173,13 @@ def __init__(self, scope: Construct, id: str, **kwargs) -> None:
             "SentinelInputBucketHistoric",
             bucket_name=SENTINEL_INPUT_BUCKET_HISTORIC,
             removal_policy=RemovalPolicy.DESTROY,
+            lifecycle_rules=[
+                aws_s3.LifecycleRule(
+                    expiration=Duration.days(sentinel_input_bucket_expiration_days),
+                    expired_object_delete_marker=True,
+                    noncurrent_version_expiration=Duration.days(1),
+                )
+            ],
         )
 
         self.landsat_input_bucket_historic = aws_s3.Bucket(