From a95404053180ab6a9c623a9cd1e265b7efe366eb Mon Sep 17 00:00:00 2001
From: Steve Astels <sastels@users.noreply.github.com>
Date: Thu, 8 Feb 2024 15:28:43 -0500
Subject: [PATCH] Refresh all QuickSight datasets (#1132)

* refresh all the things

* order refresh by dependancies
---
 aws/quicksight/dataset_jobs.tf           | 15 +++++++++++++++
 aws/quicksight/dataset_login_events.tf   |  4 ++--
 aws/quicksight/dataset_notifications.tf  | 16 ++++++++++++++++
 aws/quicksight/dataset_organisation.tf   | 16 ++++++++++++++++
 aws/quicksight/dataset_refresh_order.txt | 17 +++++++++++++++++
 aws/quicksight/dataset_services.tf       | 15 +++++++++++++++
 aws/quicksight/dataset_templates.tf      | 15 +++++++++++++++
 7 files changed, 96 insertions(+), 2 deletions(-)
 create mode 100644 aws/quicksight/dataset_refresh_order.txt

diff --git a/aws/quicksight/dataset_jobs.tf b/aws/quicksight/dataset_jobs.tf
index ebde2666d..2ad7d6447 100644
--- a/aws/quicksight/dataset_jobs.tf
+++ b/aws/quicksight/dataset_jobs.tf
@@ -81,3 +81,18 @@ resource "aws_quicksight_data_set" "jobs" {
     principal = aws_quicksight_group.dataset_owner.arn
   }
 }
+
+resource "aws_quicksight_refresh_schedule" "jobs" {
+  data_set_id = "jobs"
+  schedule_id = "schedule-jobs"
+  depends_on  = [aws_quicksight_data_set.jobs]
+
+  schedule {
+    refresh_type = "FULL_REFRESH"
+
+    schedule_frequency {
+      interval        = "DAILY"
+      time_of_the_day = "07:15"
+    }
+  }
+}
diff --git a/aws/quicksight/dataset_login_events.tf b/aws/quicksight/dataset_login_events.tf
index 9064fb83a..89967cb96 100644
--- a/aws/quicksight/dataset_login_events.tf
+++ b/aws/quicksight/dataset_login_events.tf
@@ -52,11 +52,11 @@ resource "aws_quicksight_refresh_schedule" "login_events" {
   depends_on  = [aws_quicksight_data_set.login_events]
 
   schedule {
-    refresh_type = "INCREMENTAL_REFRESH"
+    refresh_type = "FULL_REFRESH"
 
     schedule_frequency {
       interval        = "DAILY"
-      time_of_the_day = "07:10"
+      time_of_the_day = "07:35"
     }
   }
 }
diff --git a/aws/quicksight/dataset_notifications.tf b/aws/quicksight/dataset_notifications.tf
index aa9cae56d..6c1fc4a81 100644
--- a/aws/quicksight/dataset_notifications.tf
+++ b/aws/quicksight/dataset_notifications.tf
@@ -384,3 +384,19 @@ resource "aws_cloudformation_stack" "notifications" {
     }
   })
 }
+
+
+resource "aws_quicksight_refresh_schedule" "notifications" {
+  data_set_id = "notifications"
+  schedule_id = "schedule-notifications"
+  depends_on  = [aws_cloudformation_stack.notifications]
+
+  schedule {
+    refresh_type = "FULL_REFRESH"
+
+    schedule_frequency {
+      interval        = "DAILY"
+      time_of_the_day = "07:10"
+    }
+  }
+}
diff --git a/aws/quicksight/dataset_organisation.tf b/aws/quicksight/dataset_organisation.tf
index a8f66f2cc..caf73d734 100644
--- a/aws/quicksight/dataset_organisation.tf
+++ b/aws/quicksight/dataset_organisation.tf
@@ -41,3 +41,19 @@ resource "aws_quicksight_data_set" "organisation" {
     principal = aws_quicksight_group.dataset_owner.arn
   }
 }
+
+
+resource "aws_quicksight_refresh_schedule" "organisation" {
+  data_set_id = "organisation"
+  schedule_id = "schedule-organisation"
+  depends_on  = [aws_quicksight_data_set.organisation]
+
+  schedule {
+    refresh_type = "FULL_REFRESH"
+
+    schedule_frequency {
+      interval        = "DAILY"
+      time_of_the_day = "07:30"
+    }
+  }
+}
diff --git a/aws/quicksight/dataset_refresh_order.txt b/aws/quicksight/dataset_refresh_order.txt
new file mode 100644
index 000000000..73b768f52
--- /dev/null
+++ b/aws/quicksight/dataset_refresh_order.txt
@@ -0,0 +1,17 @@
+
+Datasets are refreshed nightly. The order doesn't matter too much since we do it in the middle of the night, but we have ordered the refreshes by dependancy. 
+For example, we refresh the notifications before the services so we can be confident that if we have the notification in QuickSight then we have the corresponding service as well.
+
+All times are UTC.
+
+0510 - notification_history (takes about 1.5 hours. Rest of tables are under 5 minutes to refresh)
+0710 - notifications
+
+0715 - jobs
+0720 - templates
+
+0725 - services
+0730 - organisations
+
+0735 - login_events
+
diff --git a/aws/quicksight/dataset_services.tf b/aws/quicksight/dataset_services.tf
index c5a12553b..a3c924fcb 100644
--- a/aws/quicksight/dataset_services.tf
+++ b/aws/quicksight/dataset_services.tf
@@ -69,3 +69,18 @@ resource "aws_quicksight_data_set" "services" {
     principal = aws_quicksight_group.dataset_owner.arn
   }
 }
+
+resource "aws_quicksight_refresh_schedule" "services" {
+  data_set_id = "services"
+  schedule_id = "schedule-services"
+  depends_on  = [aws_quicksight_data_set.services]
+
+  schedule {
+    refresh_type = "FULL_REFRESH"
+
+    schedule_frequency {
+      interval        = "DAILY"
+      time_of_the_day = "07:25"
+    }
+  }
+}
diff --git a/aws/quicksight/dataset_templates.tf b/aws/quicksight/dataset_templates.tf
index f76df5459..f043c5d0b 100644
--- a/aws/quicksight/dataset_templates.tf
+++ b/aws/quicksight/dataset_templates.tf
@@ -53,3 +53,18 @@ resource "aws_quicksight_data_set" "templates" {
     principal = aws_quicksight_group.dataset_owner.arn
   }
 }
+
+resource "aws_quicksight_refresh_schedule" "templates" {
+  data_set_id = "templatesv2"
+  schedule_id = "schedule-templates"
+  depends_on  = [aws_quicksight_data_set.templates]
+
+  schedule {
+    refresh_type = "FULL_REFRESH"
+
+    schedule_frequency {
+      interval        = "DAILY"
+      time_of_the_day = "07:20"
+    }
+  }
+}