Add lifecycle policy to Sentinel input buckets (#304)

* Whole number of days after which objects expire must be set via the
environment variable `HLS_SENTINEL_INPUT_BUCKET_EXPIRATION_DAYS`.
* Abort incomplete multipart uploads

Fixes #294

environment.sh.sample

@@ -1,9 +1,11 @@
 #!/bin/bash
 
 # Make sure that necessary executables are installed
-for b in jq aws
-do
-    command -v $b >/dev/null 2>&1 || { echo >&2 "I require $b but it's not installed.  Aborting."; exit 1; }
+for b in jq aws; do
+    command -v $b >/dev/null 2>&1 || {
+        echo >&2 "I require $b but it's not installed.  Aborting."
+        exit 1
+    }
 done
 
 # Set allexport mode, all variables defined in this block will get exported
@@ -19,15 +21,13 @@ HLS_LAADS_TOKEN="<your token>"
 HLS_OUTPUT_BUCKET=hls-global
 HLS_OUTPUT_BUCKET_HISTORIC=hls-gobal-historic
 
-
 # Role for copying to output bucket
 HLS_OUTPUT_BUCKET_ROLE_ARN=arn:aws:iam::611670965994:role/gcc-S3Test
 
 # Landsat SNS topic
 HLS_LANDSAT_SNS_TOPIC=arn:aws:sns:us-west-2:673253540267:public-c2-notify
 HLS_LANDASAT_HISTORIC_SNS_TOPIC=arn:aws:sns:us-west-2:018923174646:landsat-historic-LandsatHistoricTopic643F0596-1TIGFB893SX3B
 
-
 # Bucket for merged GIBS tile output.
 HLS_GIBS_OUTPUT_BUCKET=hls-browse-imagery
 HLS_LAADS_BUCKET_BOOTSTRAP=hls-development-laads-bucket
@@ -76,6 +76,9 @@ HLS_SSH_KEYNAME=hls-mount
 # Sentinel serverless downloader function role arn.
 HLS_DOWNLOADER_FUNCTION_ARN=something
 
+# Number of days after which objects in the Sentinel input buckets expire
+HLS_SENTINEL_INPUT_BUCKET_EXPIRATION_DAYS=60
+
 # GCC Specific environment settings.
 GCC=false
 HLS_GCC_ACCOUNT=account_id
@@ -89,13 +92,14 @@ HLS_GCC_BOUNDARY_ARN=boudary_policy_arn
 set +a
 
 # Set environment variables for all outputs set up in cloud formation
-stack_info=$(aws cloudformation describe-stacks --stack-name ${HLS_STACKNAME} --output json)
+stack_info=$(aws cloudformation describe-stacks --stack-name "${HLS_STACKNAME}" --output json)
 if [[ "$stack_info" =~ "OutputKey" ]]; then
     l=$(echo "$stack_info" | jq ".Stacks[].Outputs | length")
-    for ((i=0;i<$l;++i)); do
-        key=$(echo "$stack_info" | jq ".Stacks[].Outputs[$i].OutputKey" | sed -e 's/^"//'  -e 's/"$//')
+
+    for ((i = 0; i < l; ++i)); do
+        key=$(echo "$stack_info" | jq ".Stacks[].Outputs[$i].OutputKey" | sed -e 's/^"//' -e 's/"$//')
         keyupper=$(echo "$key" | awk '{print toupper($0)}')
-        val=$(echo "$stack_info" | jq ".Stacks[].Outputs[$i].OutputValue" | sed -e 's/^"//'  -e 's/"$//')
+        val=$(echo "$stack_info" | jq ".Stacks[].Outputs[$i].OutputValue" | sed -e 's/^"//' -e 's/"$//')
         export "HLSSTACK_$keyupper"="$val"
     done
 fi

stack/stack.py

@@ -149,19 +149,39 @@ def __init__(self, scope: Construct, id: str, **kwargs) -> None:
             self, "landsat_output_bucket", OUTPUT_BUCKET
         )
 
+        sentinel_input_bucket_expiration_days = int(
+            os.environ["HLS_SENTINEL_INPUT_BUCKET_EXPIRATION_DAYS"]
+        )
+
         # Must be created as part of the stack due to trigger requirements
         self.sentinel_input_bucket = aws_s3.Bucket(
             self,
             "SentinelInputBucket",
             bucket_name=SENTINEL_INPUT_BUCKET,
             removal_policy=RemovalPolicy.DESTROY,
+            lifecycle_rules=[
+                aws_s3.LifecycleRule(
+                    abort_incomplete_multipart_upload_after=Duration.days(1),
+                    expiration=Duration.days(sentinel_input_bucket_expiration_days),
+                    expired_object_delete_marker=True,
+                    noncurrent_version_expiration=Duration.days(1),
+                )
+            ],
         )
 
         self.sentinel_input_bucket_historic = aws_s3.Bucket(
             self,
             "SentinelInputBucketHistoric",
             bucket_name=SENTINEL_INPUT_BUCKET_HISTORIC,
             removal_policy=RemovalPolicy.DESTROY,
+            lifecycle_rules=[
+                aws_s3.LifecycleRule(
+                    abort_incomplete_multipart_upload_after=Duration.days(1),
+                    expiration=Duration.days(sentinel_input_bucket_expiration_days),
+                    expired_object_delete_marker=True,
+                    noncurrent_version_expiration=Duration.days(1),
+                )
+            ],
         )
 
         self.landsat_input_bucket_historic = aws_s3.Bucket(
@@ -176,15 +196,25 @@ def __init__(self, scope: Construct, id: str, **kwargs) -> None:
             "LandsatIntermediateBucket",
             bucket_name=LANDSAT_INTERMEDIATE_OUTPUT_BUCKET,
             removal_policy=RemovalPolicy.DESTROY,
-            lifecycle_rules=[aws_s3.LifecycleRule(expiration=Duration.days(60))],
+            lifecycle_rules=[
+                aws_s3.LifecycleRule(
+                    abort_incomplete_multipart_upload_after=Duration.days(1),
+                    expiration=Duration.days(60),
+                )
+            ],
         )
 
         self.gibs_intermediate_output_bucket = aws_s3.Bucket(
             self,
             "GibsIntermediateBucket",
             bucket_name=GIBS_INTERMEDIATE_OUTPUT_BUCKET,
             removal_policy=RemovalPolicy.DESTROY,
-            lifecycle_rules=[aws_s3.LifecycleRule(expiration=Duration.days(60))],
+            lifecycle_rules=[
+                aws_s3.LifecycleRule(
+                    abort_incomplete_multipart_upload_after=Duration.days(1),
+                    expiration=Duration.days(60),
+                )
+            ],
         )
 
         self.efs = Efs(self, "Efs", network=self.network)