diff --git a/aws/quicksight/data_source.tf b/aws/quicksight/data_source.tf index 29c751a0f..c103ca093 100644 --- a/aws/quicksight/data_source.tf +++ b/aws/quicksight/data_source.tf @@ -36,10 +36,10 @@ resource "aws_quicksight_data_source" "rds" { resource "aws_s3_object" "manifest_file" { bucket = var.s3_bucket_sms_usage_id - key = "quicksight/s3-manifest-sms-usage.json" # replace with desired object key - source = "./s3-manifests/s3-manifest-sms-usage.json" # replace with path to local file + key = "quicksight/s3-manifest-sms-usage-${var.env}.json" # replace with desired object key + source = "./s3-manifests/s3-manifest-sms-usage-${var.env}.json" # replace with path to local file acl = "private" - etag = filemd5("./s3-manifests/s3-manifest-sms-usage.json") + etag = filemd5("./s3-manifests/s3-manifest-sms-usage-${var.env}.json") } resource "aws_quicksight_data_source" "s3_sms_usage" { diff --git a/aws/quicksight/dataset_refresh_order.txt b/aws/quicksight/dataset_refresh_order.txt index 9a5aacb8d..873ee9d3f 100644 --- a/aws/quicksight/dataset_refresh_order.txt +++ b/aws/quicksight/dataset_refresh_order.txt @@ -4,6 +4,8 @@ For example, we refresh the notifications before the services so we can be confi All times are UTC. +0500 - sms_usage + 0510 - notifications (takes about 1.5 hours. Rest of tables are under 5 minutes to refresh) 0715 - jobs @@ -14,3 +16,5 @@ All times are UTC. 0735 - login_events 0740 - users + +0745 - sms_usage_notifications \ No newline at end of file diff --git a/aws/quicksight/dataset_sms_usage.tf b/aws/quicksight/dataset_sms_usage.tf index fa717c655..6e4959131 100644 --- a/aws/quicksight/dataset_sms_usage.tf +++ b/aws/quicksight/dataset_sms_usage.tf @@ -106,7 +106,7 @@ resource "aws_quicksight_refresh_schedule" "sms_usage" { # SMS usage reports are generated around 01:00 UTC. schedule_frequency { interval = "DAILY" - time_of_the_day = "02:30" + time_of_the_day = "05:00" } } } diff --git a/aws/quicksight/dataset_sms_usage_notifications.tf b/aws/quicksight/dataset_sms_usage_notifications.tf new file mode 100644 index 000000000..1af51713c --- /dev/null +++ b/aws/quicksight/dataset_sms_usage_notifications.tf @@ -0,0 +1,100 @@ +# valid column types are [STRING INTEGER DECIMAL DATETIME BIT BOOLEAN JSON] +# We have to use a cloudformation stack here because the provider has a bug in it +# Ref: https://github.com/hashicorp/terraform-provider-aws/issues/34199 + +resource "aws_cloudformation_stack" "sms-usage-notifications" { + name = "sms-usage-notifications" + notification_arns = ["arn:aws:sns:ca-central-1:${var.account_id}:aws-controltower-SecurityNotifications"] + + template_body = jsonencode({ + + Resources = { + smsusageJoined = { + Type = "AWS::QuickSight::DataSet" + Properties = { + AwsAccountId = var.account_id + DataSetId = "sms-usage-notifications" + Name = "Notifications with SMS pricing" + Permissions = [ + { + Actions = [ + "quicksight:DescribeDataSet", + "quicksight:DescribeDataSetPermissions", + "quicksight:DescribeIngestion", + "quicksight:ListIngestions", + "quicksight:PassDataSet" + ], + Principal = aws_quicksight_group.dataset_viewer.arn + }, + { + Actions = [ + "quicksight:DescribeDataSet", + "quicksight:DescribeDataSetPermissions", + "quicksight:DescribeIngestion", + "quicksight:ListIngestions", + "quicksight:PassDataSet", + "quicksight:DeleteDataSet", + "quicksight:UpdateDataSet", + "quicksight:UpdateDataSetPermissions", + "quicksight:CreateIngestion", + "quicksight:CancelIngestion" + ], + Principal = aws_quicksight_group.dataset_owner.arn + } + ], + + + LogicalTableMap = { + + sms-usage-notifications = { + Alias = "sms-usage-notifications", + Source = { + JoinInstruction = { + LeftOperand = "sms-usage", + RightOperand = "notifications", + Type = "LEFT", + OnClause = "{MessageId} = {notification_reference}" + } + } + }, + + notifications = { + Alias = "notifications", + Source = { + DataSetArn = aws_quicksight_data_set.notifications.arn + } + }, + + sms-usage = { + Alias = "sms-usage", + Source = { + DataSetArn = aws_quicksight_data_set.sms_usage.arn + } + }, + }, + DataSetUsageConfiguration = { + DisableUseAsDirectQuerySource = false, + DisableUseAsImportedSource = false + }, + ImportMode = "SPICE" + } + } + } + }) +} + + +resource "aws_quicksight_refresh_schedule" "sms-usage-notifications" { + data_set_id = "sms-usage-notifications" + schedule_id = "schedule-sms-usage-notifications" + depends_on = [aws_cloudformation_stack.sms-usage-notifications] + + schedule { + refresh_type = "FULL_REFRESH" + + schedule_frequency { + interval = "DAILY" + time_of_the_day = "07:45" + } + } +} diff --git a/aws/quicksight/s3-manifests/s3-manifest-sms-usage-dev.json b/aws/quicksight/s3-manifests/s3-manifest-sms-usage-dev.json new file mode 100644 index 000000000..14b22140a --- /dev/null +++ b/aws/quicksight/s3-manifests/s3-manifest-sms-usage-dev.json @@ -0,0 +1,15 @@ +{ + "fileLocations": [ + { + "URIPrefixes": [ + "s3://notification-canada-ca-dev-sms-usage-logs/SMSUsageReports/ca-central-1/" + ] + } + ], + "globalUploadSettings": { + "format": "CSV", + "delimiter": ",", + "textqualifier": "'", + "containsHeader": "true" + } +} \ No newline at end of file diff --git a/aws/quicksight/s3-manifests/s3-manifest-sms-usage-production.json b/aws/quicksight/s3-manifests/s3-manifest-sms-usage-production.json new file mode 100644 index 000000000..c3048157e --- /dev/null +++ b/aws/quicksight/s3-manifests/s3-manifest-sms-usage-production.json @@ -0,0 +1,15 @@ +{ + "fileLocations": [ + { + "URIPrefixes": [ + "s3://notification-canada-ca-production-sms-usage-logs/SMSUsageReports/ca-central-1/" + ] + } + ], + "globalUploadSettings": { + "format": "CSV", + "delimiter": ",", + "textqualifier": "'", + "containsHeader": "true" + } +} \ No newline at end of file diff --git a/aws/quicksight/s3-manifests/s3-manifest-sms-usage.json b/aws/quicksight/s3-manifests/s3-manifest-sms-usage-staging.json similarity index 100% rename from aws/quicksight/s3-manifests/s3-manifest-sms-usage.json rename to aws/quicksight/s3-manifests/s3-manifest-sms-usage-staging.json