Join sms_usage and notifications in cloud formation (#1231)

* rough in cloudformation join

* tweak manifeset file for different environments

* works

* remove projection

aws/quicksight/data_source.tf

@@ -36,10 +36,10 @@ resource "aws_quicksight_data_source" "rds" {
 
 resource "aws_s3_object" "manifest_file" {
   bucket = var.s3_bucket_sms_usage_id
-  key    = "quicksight/s3-manifest-sms-usage.json"     # replace with desired object key
-  source = "./s3-manifests/s3-manifest-sms-usage.json" # replace with path to local file
+  key    = "quicksight/s3-manifest-sms-usage-${var.env}.json"     # replace with desired object key
+  source = "./s3-manifests/s3-manifest-sms-usage-${var.env}.json" # replace with path to local file
   acl    = "private"
-  etag   = filemd5("./s3-manifests/s3-manifest-sms-usage.json")
+  etag   = filemd5("./s3-manifests/s3-manifest-sms-usage-${var.env}.json")
 }
 
 resource "aws_quicksight_data_source" "s3_sms_usage" {

aws/quicksight/dataset_refresh_order.txt

@@ -4,6 +4,8 @@ For example, we refresh the notifications before the services so we can be confi
 
 All times are UTC.
 
+0500 - sms_usage
+
 0510 - notifications (takes about 1.5 hours. Rest of tables are under 5 minutes to refresh)
 
 0715 - jobs
@@ -14,3 +16,5 @@ All times are UTC.
 
 0735 - login_events
 0740 - users
+
+0745 - sms_usage_notifications

aws/quicksight/dataset_sms_usage.tf

@@ -106,7 +106,7 @@ resource "aws_quicksight_refresh_schedule" "sms_usage" {
     # SMS usage reports are generated around 01:00 UTC.
     schedule_frequency {
       interval        = "DAILY"
-      time_of_the_day = "02:30"
+      time_of_the_day = "05:00"
     }
   }
 }

aws/quicksight/dataset_sms_usage_notifications.tf

@@ -0,0 +1,100 @@
+# valid column types are [STRING INTEGER DECIMAL DATETIME BIT BOOLEAN JSON]
+# We have to use a cloudformation stack here because the provider has a bug in it
+# Ref: https://github.com/hashicorp/terraform-provider-aws/issues/34199
+
+resource "aws_cloudformation_stack" "sms-usage-notifications" {
+  name              = "sms-usage-notifications"
+  notification_arns = ["arn:aws:sns:ca-central-1:${var.account_id}:aws-controltower-SecurityNotifications"]
+
+  template_body = jsonencode({
+
+    Resources = {
+      smsusageJoined = {
+        Type = "AWS::QuickSight::DataSet"
+        Properties = {
+          AwsAccountId = var.account_id
+          DataSetId    = "sms-usage-notifications"
+          Name         = "Notifications with SMS pricing"
+          Permissions = [
+            {
+              Actions = [
+                "quicksight:DescribeDataSet",
+                "quicksight:DescribeDataSetPermissions",
+                "quicksight:DescribeIngestion",
+                "quicksight:ListIngestions",
+                "quicksight:PassDataSet"
+              ],
+              Principal = aws_quicksight_group.dataset_viewer.arn
+            },
+            {
+              Actions = [
+                "quicksight:DescribeDataSet",
+                "quicksight:DescribeDataSetPermissions",
+                "quicksight:DescribeIngestion",
+                "quicksight:ListIngestions",
+                "quicksight:PassDataSet",
+                "quicksight:DeleteDataSet",
+                "quicksight:UpdateDataSet",
+                "quicksight:UpdateDataSetPermissions",
+                "quicksight:CreateIngestion",
+                "quicksight:CancelIngestion"
+              ],
+              Principal = aws_quicksight_group.dataset_owner.arn
+            }
+          ],
+
+
+          LogicalTableMap = {
+
+            sms-usage-notifications = {
+              Alias = "sms-usage-notifications",
+              Source = {
+                JoinInstruction = {
+                  LeftOperand  = "sms-usage",
+                  RightOperand = "notifications",
+                  Type         = "LEFT",
+                  OnClause     = "{MessageId} = {notification_reference}"
+                }
+              }
+            },
+
+            notifications = {
+              Alias = "notifications",
+              Source = {
+                DataSetArn = aws_quicksight_data_set.notifications.arn
+              }
+            },
+
+            sms-usage = {
+              Alias = "sms-usage",
+              Source = {
+                DataSetArn = aws_quicksight_data_set.sms_usage.arn
+              }
+            },
+          },
+          DataSetUsageConfiguration = {
+            DisableUseAsDirectQuerySource = false,
+            DisableUseAsImportedSource    = false
+          },
+          ImportMode = "SPICE"
+        }
+      }
+    }
+  })
+}
+
+
+resource "aws_quicksight_refresh_schedule" "sms-usage-notifications" {
+  data_set_id = "sms-usage-notifications"
+  schedule_id = "schedule-sms-usage-notifications"
+  depends_on  = [aws_cloudformation_stack.sms-usage-notifications]
+
+  schedule {
+    refresh_type = "FULL_REFRESH"
+
+    schedule_frequency {
+      interval        = "DAILY"
+      time_of_the_day = "07:45"
+    }
+  }
+}

aws/quicksight/s3-manifests/s3-manifest-sms-usage-dev.json

@@ -0,0 +1,15 @@
+{
+    "fileLocations": [
+        {
+            "URIPrefixes": [
+                "s3://notification-canada-ca-dev-sms-usage-logs/SMSUsageReports/ca-central-1/"
+            ]
+        }
+    ],
+    "globalUploadSettings": {
+        "format": "CSV",
+        "delimiter": ",",
+        "textqualifier": "'",
+        "containsHeader": "true"
+    }
+}

aws/quicksight/s3-manifests/s3-manifest-sms-usage-production.json

@@ -0,0 +1,15 @@
+{
+    "fileLocations": [
+        {
+            "URIPrefixes": [
+                "s3://notification-canada-ca-production-sms-usage-logs/SMSUsageReports/ca-central-1/"
+            ]
+        }
+    ],
+    "globalUploadSettings": {
+        "format": "CSV",
+        "delimiter": ",",
+        "textqualifier": "'",
+        "containsHeader": "true"
+    }
+}