New relic terraform alert POC (#1409)

* starting new relic stuff

* Successful terragrunt deploy of a NR alert condition

* adding aert condition

* adding workflow vars

* fixing the missing dns files

* Update terragrunt.hcl

just removing an unnecessary line

* Update alert_conditions.tf

adding this for a test

* updating secrets

* updating all environments

* removing uneccessary variable

* atttempting to fix formatting

* formatting

* formatting

* migrating the specific values for the new relic provider to the TF side so we can consume the TFVars while working locally.

* adding back variable

.github/workflows/merge_to_main_production.yml

@@ -17,6 +17,8 @@ env:
   AWS_ACCESS_KEY_ID: ${{ secrets.PRODUCTION_AWS_ACCESS_KEY_ID }}
   AWS_SECRET_ACCESS_KEY: ${{ secrets.PRODUCTION_AWS_SECRET_ACCESS_KEY }}
   AWS_REGION: ca-central-1
+  TF_VAR_new_relic_api_key: ${{ secrets.PRODUCTION_NEW_RELIC_API_KEY }}
+  TF_VAR_new_relic_account_id: ${{ secrets.PRODUCTION_NEW_RELIC_ACCOUNT_ID }}
   TF_VAR_base_domain: ${{secrets.PRODUCTION_BASE_DOMAIN}}
   TF_VAR_alt_base_domain: ${{secrets.PRODUCTION_ALT_BASE_DOMAIN}}  
   TF_VAR_dbtools_password: ${{ secrets.PRODUCTION_DBTOOLS_PASSWORD }}

.github/workflows/merge_to_main_staging.yml

@@ -20,6 +20,8 @@ env:
   AWS_ACCESS_KEY_ID: ${{ secrets.STAGING_AWS_ACCESS_KEY_ID }}
   AWS_SECRET_ACCESS_KEY: ${{ secrets.STAGING_AWS_SECRET_ACCESS_KEY }}
   AWS_REGION: ca-central-1
+  TF_VAR_new_relic_api_key: ${{ secrets.PRODUCTION_NEW_RELIC_API_KEY }}
+  TF_VAR_new_relic_account_id: ${{ secrets.PRODUCTION_NEW_RELIC_ACCOUNT_ID }}
   TF_VAR_base_domain: ${{secrets.STAGING_BASE_DOMAIN}}
   TF_VAR_alt_base_domain: ${{secrets.STAGING_ALT_BASE_DOMAIN}}
   TF_VAR_dbtools_password: ${{ secrets.STAGING_DBTOOLS_PASSWORD }}

.github/workflows/terragrunt_plan_production.yml

@@ -12,6 +12,8 @@ env:
   AWS_REGION: ca-central-1
   TERRAFORM_VERSION: 0.14.4
   TERRAGRUNT_VERSION: 0.35.13
+  TF_VAR_new_relic_api_key: ${{ secrets.PRODUCTION_NEW_RELIC_API_KEY }}
+  TF_VAR_new_relic_account_id: ${{ secrets.PRODUCTION_NEW_RELIC_ACCOUNT_ID }}
   TF_VAR_base_domain: ${{secrets.PRODUCTION_BASE_DOMAIN}}
   TF_VAR_alt_base_domain: ${{secrets.PRODUCTION_ALT_BASE_DOMAIN}}
   TF_VAR_dbtools_password: ${{ secrets.PRODUCTION_DBTOOLS_PASSWORD }}

.github/workflows/terragrunt_plan_staging.yml

@@ -17,6 +17,8 @@ env:
   AWS_REGION: ca-central-1
   TERRAFORM_VERSION: 0.14.4
   TERRAGRUNT_VERSION: 0.35.13
+  TF_VAR_new_relic_api_key: ${{ secrets.PRODUCTION_NEW_RELIC_API_KEY }}
+  TF_VAR_new_relic_account_id: ${{ secrets.PRODUCTION_NEW_RELIC_ACCOUNT_ID }}
   TF_VAR_base_domain: ${{secrets.STAGING_BASE_DOMAIN}}
   TF_VAR_alt_base_domain: ${{secrets.STAGING_ALT_BASE_DOMAIN}}
   TF_VAR_dbtools_password: ${{ secrets.STAGING_DBTOOLS_PASSWORD }}

aws/newrelic/alert_conditions.tf

@@ -0,0 +1,31 @@
+resource "newrelic_nrql_alert_condition" "tf_lambda_api_errors_count_anomaly_unexpected_errors" {
+  account_id                   = var.new_relic_account_id
+  policy_id                    = 2801728
+  type                         = "baseline"
+  name                         = "Staging - Terraform - [Lambda API] Errors count anomaly (Unexpected Errors)"
+  enabled                      = true
+  violation_time_limit_seconds = 86400
+
+  nrql {
+    query = "SELECT count(*) FROM AwsLambdaInvocationError WHERE (`entityGuid`='MjY5MTk3NHxJTkZSQXxOQXwtNzgwNDUyNTc5NzAyODI1NTcyNw') and error.class NOT IN ('app.v2.errors:BadRequestError','jsonschema.exceptions:ValidationError', 'sqlalchemy.exc:NoResultFound', 'app.authentication.auth:AuthError', 'werkzeug.exceptions:MethodNotAllowed') and error.message NOT LIKE '{\\'result\\': \\'error\\', \\'message\\': {\\'password\\': [\\'Incorrect password\\']}}'"
+  }
+
+  critical {
+    operator              = "above"
+    threshold             = 6
+    threshold_duration    = 300
+    threshold_occurrences = "all"
+  }
+
+  warning {
+    operator              = "above"
+    threshold             = 3
+    threshold_duration    = 300
+    threshold_occurrences = "all"
+  }
+  fill_option        = "none"
+  aggregation_window = 60
+  aggregation_method = "event_flow"
+  aggregation_delay  = 300
+  baseline_direction = "upper_and_lower"
+}

aws/newrelic/alert_policies.tf

@@ -0,0 +1,5 @@
+# resource "newrelic_alert_policy" "notify_terraform_policy" {
+#   name     = "Pond Example staging"
+#   provider = newrelic
+# }
+

aws/newrelic/data.tf

@@ -0,0 +1,7 @@
+# data "newrelic_entity" "notification-admin-script-staging" {
+#   name = "notification-admin-script"
+#   provider = newrelic
+#   domain = "BROWSER" # or BROWSER, INFRA, MOBILE, SYNTH, depending on your entity's domain
+#   type = "APPLICATION"
+# }
+

aws/newrelic/entities.tf

@@ -0,0 +1 @@
+

aws/newrelic/main.tf

@@ -0,0 +1,5 @@
+provider "newrelic" {
+  account_id = var.new_relic_account_id
+  api_key    = var.new_relic_api_key
+  region     = "US"
+}

aws/newrelic/variables.tf

@@ -0,0 +1,8 @@
+variable "new_relic_account_id" {
+  type        = string
+  description = "New Relic Account ID"
+}
+variable "new_relic_api_key" {
+  type        = string
+  description = "New Relic API Key"
+}

env/dev/newrelic/terragrunt.hcl

@@ -0,0 +1,7 @@
+terraform {
+  source = "../../../aws//newrelic"
+}
+
+include {
+  path = find_in_parent_folders()
+}

env/production/env_vars.hcl

@@ -7,5 +7,4 @@ inputs = {
   account_budget_limit                = 10000
   log_retention_period_days           = 0
   sensitive_log_retention_period_days = 7
-
 }

env/terragrunt.hcl

@@ -16,13 +16,14 @@ inputs = {
   log_retention_period_days             = local.vars.inputs.log_retention_period_days
   sensitive_log_retention_period_days   = local.vars.inputs.sensitive_log_retention_period_days
   account_budget_limit                  = local.vars.inputs.account_budget_limit
+
 
   region             = "ca-central-1"
   # See https://docs.aws.amazon.com/elasticloadbalancing/latest/application/load-balancer-access-logs.html#access-logging-bucket-permissions
   elb_account_ids = {
     "ca-central-1" = "985666609251"
   }
-  new_relic_account_id      = "2691974"
+
   cbs_satellite_bucket_name = "cbs-satellite-${local.vars.inputs.account_id}"
 }
 
@@ -50,8 +51,11 @@ terraform {
       source  = "hashicorp/tls"
       version = "~> 4.0"
     }
+    newrelic = {
+      source  = "newrelic/newrelic"
+      version = "~> 2.0"
+    }
   }
-
 }
 
 provider "aws" {
@@ -84,8 +88,6 @@ provider "aws" {
   }
 }
 
-
-
 EOF
 }