Create health events relay function

PRX · Oct 24, 2023 · ae1372a · ae1372a
1 parent 362be48
commit ae1372a
Show file tree

Hide file tree

Showing 3 changed files with 139 additions and 0 deletions.
diff --git a/devops/tooling-cd-pipeline.yml b/devops/tooling-cd-pipeline.yml
@@ -41,6 +41,9 @@ Parameters:
   kStatuspageStackName:
     Type: String
     Default: prx-devops-statuspage
+  kHealthEventsStackName:
+    Type: String
+    Default: prx-devops-health-events
   ######
   GitHubCodeStarConnectionArn: { Type: String }
   OrganizationId:
@@ -223,6 +226,9 @@ Resources:
 
                 - sam build --parallel --template-file ./devops/tooling/statuspage/template.yml
                 - sam package --s3-bucket ${PipelineArtifactStore} --output-template-file ./${kStatuspageStackName}
+
+                - sam build --parallel --template-file ./devops/tooling/chat-ops/health-events/template.yml
+                - sam package --s3-bucket ${PipelineArtifactStore} --output-template-file ./${kHealthEventsStackName}
           artifacts:
             files:
               - ${kCloudFormationMacroFunctionsStackName}
@@ -231,6 +237,7 @@ Resources:
               - ${kCloudWatchAlarmRemindersStackName}
               - ${kSlackAppStackName}
               - ${kStatuspageStackName}
+              - ${kHealthEventsStackName}
             name: SamBuildArtifact
         Type: CODEPIPELINE
       Tags:
@@ -366,6 +373,7 @@ Resources:
                   - !Sub arn:${AWS::Partition}:cloudformation:${AWS::Region}:${AWS::AccountId}:stack/${kCloudFormationMacroFunctionsStackName}/*
                   - !Sub arn:${AWS::Partition}:cloudformation:${AWS::Region}:${AWS::AccountId}:stack/${kCloudFormationNotificationsStackName}/*
                   - !Sub arn:${AWS::Partition}:cloudformation:${AWS::Region}:${AWS::AccountId}:stack/${kStatuspageStackName}/*
+                  - !Sub arn:${AWS::Partition}:cloudformation:${AWS::Region}:${AWS::AccountId}:stack/${kHealthEventsStackName}/*
                   # StackSets
                   - !Sub arn:${AWS::Partition}:cloudformation:${AWS::Region}:${AWS::AccountId}:stackset/${kCloudFormationNotificationTopicsStacksetName}:*
             Version: "2012-10-17"
@@ -497,6 +505,26 @@ Resources:
                 - Name: SamBuildArtifact
               Namespace: statuspage_deploy
               RunOrder: 1
+            - Name: HealthEvents
+              ActionTypeId:
+                Category: Deploy
+                Owner: AWS
+                Provider: CloudFormation
+                Version: "1"
+              Configuration:
+                ActionMode: CREATE_UPDATE
+                Capabilities: CAPABILITY_IAM,CAPABILITY_AUTO_EXPAND
+                ParameterOverrides: !Sub |
+                  {
+                    "SlackMessageRelaySnsTopicArn": "#{slack_app_deploy.SlackMessageRelaySnsTopicArn}"
+                  }
+                RoleArn: !GetAtt CloudFormationRole.Arn
+                StackName: !Ref kHealthEventsStackName
+                TemplatePath: !Sub SamBuildArtifact::${kHealthEventsStackName}
+              InputArtifacts:
+                - Name: SamBuildArtifact
+              Namespace: healthevents_deploy
+              RunOrder: 1
             - Name: CloudWatch_Alarms
               ActionTypeId:
                 Category: Deploy

diff --git a/devops/tooling/chat-ops/health-events/src/index.mjs b/devops/tooling/chat-ops/health-events/src/index.mjs
@@ -0,0 +1,31 @@
+import { SNS, PublishCommand } from '@aws-sdk/client-sns';
+
+const sns = new SNS({ region: process.env.AWS_REGION });
+
+export const handler = async (event) => {
+  sns.send(
+    new PublishCommand({
+      TopicArn: process.env.SLACK_MESSAGE_RELAY_SNS_TOPIC_ARN,
+      Message: JSON.stringify({
+        username: 'AWS Health Events',
+        icon_emoji: ':ops-aws-health:',
+        channel: 'G2QHC2N7K', // #ops-warn
+        attachments: [
+          {
+            color: '#a30200',
+            fallback: event.eventArn.eventTypeCode,
+            blocks: [
+              {
+                type: 'section',
+                text: {
+                  type: 'mrkdwn',
+                  text: event.eventArn.eventDescription[0].latestDescription,
+                },
+              },
+            ],
+          },
+        ],
+      }),
+    }),
+  );
+};
diff --git a/devops/tooling/chat-ops/health-events/template.yml b/devops/tooling/chat-ops/health-events/template.yml
@@ -0,0 +1,80 @@
+# devops/tooling/chat-ops/health-events/template.yml
+# This template is continuously deployed by the DevOps CD pipeline
+#
+AWSTemplateFormatVersion: "2010-09-09"
+Transform: AWS::Serverless-2016-10-31
+Description: >-
+  Receives EventBridge events for AWS Health Events and sends them to Slack
+
+Parameters:
+  SlackMessageRelaySnsTopicArn:
+    Type: String
+
+Resources:
+  HealthEventsFunction:
+    Type: AWS::Serverless::Function
+    Properties:
+      CodeUri: src/
+      Description: >-
+        Sends messages to Slack in response to AWS Health Events
+      Environment:
+        Variables:
+          AWS_NODEJS_CONNECTION_REUSE_ENABLED: "1"
+          SLACK_MESSAGE_RELAY_SNS_TOPIC_ARN: !Ref SlackMessageRelaySnsTopicArn
+      Events:
+        PublicHealthEvent:
+          Properties:
+            Pattern:
+              detail-type:
+                - AWS Health Event
+              source:
+                - aws.health
+          Type: EventBridgeRule
+      Handler: index.handler
+      MemorySize: 192
+      Policies:
+        - Statement:
+            - Action: sns:Publish
+              Effect: Allow
+              Resource: !Ref SlackMessageRelaySnsTopicArn
+          Version: "2012-10-17"
+      Runtime: nodejs18.x
+      Tags:
+        prx:meta:tagging-version: "2021-04-07"
+        prx:cloudformation:stack-name: !Ref AWS::StackName
+        prx:cloudformation:stack-id: !Ref AWS::StackId
+        prx:ops:environment: Production
+        prx:dev:application: DevOps
+      Timeout: 20
+  HealthEventsFunctionLogGroup:
+    Type: AWS::Logs::LogGroup
+    DeletionPolicy: Delete
+    UpdateReplacePolicy: Delete
+    Properties:
+      LogGroupName: !Sub /aws/lambda/${HealthEventsFunction}
+      RetentionInDays: 14
+      Tags:
+        - { Key: prx:meta:tagging-version, Value: "2021-04-07" }
+        - { Key: prx:cloudformation:stack-name, Value: !Ref AWS::StackName }
+        - { Key: prx:cloudformation:stack-id, Value: !Ref AWS::StackId }
+        - { Key: prx:ops:environment, Value: Production }
+        - { Key: prx:dev:application, Value: DevOps }
+  HealthEventsErrorAlarm:
+    Type: AWS::CloudWatch::Alarm
+    Properties:
+      AlarmName: !Sub ERROR [DevOps] AWS Health Events Relay <prod> FUNCTION ERRORS (${AWS::StackName})
+      AlarmDescription: >-
+        The Lambda function that relays AWS Health Events to Slack is
+        experiencing errors, which means there could be health events that are
+        not being seen. Check the Health Dashboard in AWS Console.
+      ComparisonOperator: GreaterThanThreshold
+      Dimensions:
+        - Name: FunctionName
+          Value: !Ref HealthEventsFunction
+      EvaluationPeriods: 1
+      MetricName: Errors
+      Namespace: AWS/Lambda
+      Period: 60
+      Statistic: Sum
+      Threshold: 1
+      TreatMissingData: notBreaching