feat: add a retry with delay when db not ready

If the database is not ready when we attempt to enable insights, wait and retry.

.eslintrc.cjs

@@ -250,9 +250,6 @@ const rules = {
   // sometimes these are much clearer than if else blocks
   'no-nested-ternary': 'off',
 
-  // avoid unknown horrors
-  complexity: ['warn', { max: 9 }],
-
   'no-underscore-dangle': [
     'error',
     {

package.json

@@ -172,7 +172,9 @@
     "@aws-sdk/client-lambda": "^3.148.0",
     "@aws-sdk/client-rds": "^3.145.0",
     "@aws-sdk/client-sns": "^3.145.0",
-    "@aws-sdk/client-sqs": "^3.145.0"
+    "@aws-sdk/client-sqs": "^3.145.0",
+    "@aws-sdk/smithy-client": "^3.180.0",
+    "delay": "^5.0.0"
   },
   "ava": {
     "files": [

src/main/AuroraTools.ts

@@ -15,8 +15,10 @@ import {
   StartDBClusterCommand,
   StopDBClusterCommand,
   ModifyDBInstanceCommand,
+  RDSServiceException,
 } from '@aws-sdk/client-rds';
 import type { SNSEventRecord } from 'aws-lambda';
+import delay from 'delay';
 
 import type { AuroraConfig } from './@types/index.js';
 import { ClusterState, StackReference } from './constants.js';
@@ -276,7 +278,9 @@ export class AuroraTools {
    */
   public async enablePerformanceInsights(
     record: SNSEventRecord,
-    reEnableIfDisabled = true
+    reEnableIfDisabled = true,
+    retryDelay = 60e3,
+    retryAttempts = 5
   ): Promise<void> {
     const message = JSON.parse(record.Sns.Message);
     if (
@@ -299,12 +303,30 @@ export class AuroraTools {
           ) ?? {};
         const { PerformanceInsightsEnabled } = description;
         if (description && !PerformanceInsightsEnabled) {
-          await this.rds.send(
-            new ModifyDBInstanceCommand({
-              DBInstanceIdentifier,
-              EnablePerformanceInsights: true,
-            })
-          );
+          let complete = false;
+          let attempts = 0;
+          while (!complete && attempts < retryAttempts) {
+            try {
+              await this.rds.send(
+                new ModifyDBInstanceCommand({
+                  DBInstanceIdentifier,
+                  EnablePerformanceInsights: true,
+                })
+              );
+              complete = true;
+            } catch (e) {
+              console.error(e);
+              if (
+                e instanceof RDSServiceException &&
+                (e as any).Code === 'InvalidDBInstanceState'
+              ) {
+                attempts += 1;
+                await delay(retryDelay);
+              } else {
+                throw e;
+              }
+            }
+          }
         }
       }
     }

src/test/AuroraTools.test.js

@@ -1,3 +1,7 @@
+import {
+  ModifyDBInstanceCommand,
+  RDSServiceException,
+} from '@aws-sdk/client-rds';
 import test from 'ava';
 
 import {
@@ -89,6 +93,38 @@ test.serial(
   }
 );
 
+test.serial(
+  'AuroraTools > enablePerformanceInsights > retries after 60s if instance not ready',
+  async (t) => {
+    t.timeout(5e3);
+    const e = new RDSServiceException({
+      message:
+        'InvalidDBInstanceState: Database instance is not in available state.',
+    });
+    e.name = 'InvalidDBInstanceState';
+    e.Code = 'InvalidDBInstanceState';
+    awsMocks.mockRds.on(ModifyDBInstanceCommand).rejectsOnce(e);
+    const startTime = Date.now();
+    await auroraTools.enablePerformanceInsights(
+      {
+        Sns: {
+          Message:
+            '{"Event Source":"db-instance","Event Time":"2022-08-26 13:29:19.857","Identifier Link":"https://console.aws.amazon.com/rds/home?region=us-east-1#dbinstance:id=application-autoscaling-d1583f39-ab0b-4d73-87e4-4db2d83476b3","Source ID":"application-autoscaling-d1583f39-ab0b-4d73-87e4-4db2d83476b3","Source ARN":"arn:aws:rds:us-east-1:000000000000:db:application-autoscaling-d1583f39-ab0b-4d73-87e4-4db2d83476b3","Event ID":"http://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/USER_Events.html#RDS-EVENT-0003","Event Message":"DB instance created"}',
+        },
+      },
+      true,
+      2e3
+    );
+    t.is(awsMocks.mockRds.commandCalls(ModifyDBInstanceCommand).length, 2);
+    t.snapshot(
+      awsMocks.mockRds
+        .commandCalls(ModifyDBInstanceCommand)
+        .map((el) => el.args[0].input)
+    );
+    t.true(Date.now() - startTime > 2e3);
+  }
+);
+
 test.serial(
   'AuroraTools > enablePerformanceInsights > calls ModifyDBInstanceCommand with expected params when triggered from a disable event',
   async (t) => {

src/test/snapshots/AuroraTools.test.js.md

@@ -43,6 +43,21 @@ Generated by [AVA](https://avajs.dev).
       EnablePerformanceInsights: true,
     }
 
+## AuroraTools > enablePerformanceInsights > retries after 60s if instance not ready
+
+> Snapshot 1
+
+    [
+      {
+        DBInstanceIdentifier: 'application-autoscaling-d1583f39-ab0b-4d73-87e4-4db2d83476b3',
+        EnablePerformanceInsights: true,
+      },
+      {
+        DBInstanceIdentifier: 'application-autoscaling-d1583f39-ab0b-4d73-87e4-4db2d83476b3',
+        EnablePerformanceInsights: true,
+      },
+    ]
+
 ## AuroraTools > enablePerformanceInsights > calls ModifyDBInstanceCommand with expected params when triggered from a disable event
 
 > Snapshot 1