Update scripting20231128 (#183)

* Bump openssl from 0.10.57 to 0.10.60 (#181) Bumps [openssl](https://github.com/sfackler/rust-openssl) from 0.10.57 to 0.10.60. - [Release notes](https://github.com/sfackler/rust-openssl/releases) - [Commits](sfackler/rust-openssl@openssl-v0.10.57...openssl-v0.10.60) --- updated-dependencies: - dependency-name: openssl dependency-type: direct:production ... Signed-off-by: dependabot[bot] <[email protected]> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Reduced log levels to reduce logging to Splunk * Added the ability to upate the tags on an S3File * Added code so that the delete API will also delete the files from S3 - We're changing the recurring tests to not be automatically deleted after 6 months so we need to have the delete schedule delete the files from s3 * Added code to re-tag files on the last run of a recurring set * Updated the postTest recurring to not tag for deletion - postTest with a recurring schedule will add the tag recurring=true instead of test=true - Changing from recurring to non-recurring or vice-versa will update the tags accordingly - Recurring tests should no longer delete after 6 months * Fixed some issues from the previous shadow variable bug * Cleaned up a ton of shadow variable bugs * Cleaned up more shadow variables * Fixed the last of the shadow variable bugs and added the lint rule * Updated version and dependencies * Updated additional dependencies: typescript, next, storybook, etc * Update 2023-11-28 (#182) * Reduced log levels to reduce logging to Splunk * Added the ability to upate the tags on an S3File * Added code so that the delete API will also delete the files from S3 - We're changing the recurring tests to not be automatically deleted after 6 months so we need to have the delete schedule delete the files from s3 * Added code to re-tag files on the last run of a recurring set * Updated the postTest recurring to not tag for deletion - postTest with a recurring schedule will add the tag recurring=true instead of test=true - Changing from recurring to non-recurring or vice-versa will update the tags accordingly - Recurring tests should no longer delete after 6 months * Fixed some issues from the previous shadow variable bug * Cleaned up a ton of shadow variable bugs * Cleaned up more shadow variables * Fixed the last of the shadow variable bugs and added the lint rule * Updated version and dependencies * Updated additional dependencies: typescript, next, storybook, etc * Fixed the copy s3File to use the new tags rather than always re-using the old tags * Updated the remove test code to return a 500 error if we can't delete any of the files from s3 --------- Signed-off-by: dependabot[bot] <[email protected]> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
FamilySearch · Nov 30, 2023 · b89f50d · b89f50d
1 parent b396e79
commit b89f50d
Show file tree

Hide file tree

Showing 33 changed files with 3,189 additions and 3,644 deletions.
diff --git a/.eslintrc b/.eslintrc
@@ -68,6 +68,8 @@
     "camelcase": 1,
     "no-irregular-whitespace": 1,
     "object-shorthand": 1,
+    "no-shadow": "off",
+    "@typescript-eslint/no-shadow": "warn",
     "@typescript-eslint/await-thenable": 1,
     "quotes": ["warn", "double"]
   }

diff --git a/Cargo.lock b/Cargo.lock
diff --git a/agent/package.json b/agent/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@fs/ppaas-agent",
-  "version": "3.0.2",
+  "version": "3.0.3",
   "description": "Agent Service for running pewpew tests",
   "main": "dist/src/app.js",
   "scripts": {
@@ -58,6 +58,6 @@
     "eslint": "^8.40.0",
     "mocha": "^10.2.0",
     "nyc": "^15.1.0",
-    "typescript": "~5.2.0"
+    "typescript": "~5.3.0"
   }
 }
diff --git a/agent/src/pewpewtest.ts b/agent/src/pewpewtest.ts
@@ -102,27 +102,27 @@ export function copyTestStatus (ppaasTestStatus: PpaasTestStatus, s3Status: Test
 }
 
 /** instanceId doesn't change, so we'll save it after the first call as the instanceId or an Error */
-let instanceId: string | Error | undefined;
+let globalInstanceId: string | Error | undefined;
 async function getInstanceId (): Promise<string> {
-  if (typeof instanceId === "string") {
-    log("instanceId string: " + instanceId, LogLevel.DEBUG, instanceId);
-    return instanceId;
+  if (typeof globalInstanceId === "string") {
+    log("instanceId string: " + globalInstanceId, LogLevel.DEBUG, globalInstanceId);
+    return globalInstanceId;
   }
-  if (instanceId instanceof Error) {
-    log("instanceId instanceof Error", LogLevel.DEBUG, instanceId);
-    throw instanceId;
+  if (globalInstanceId instanceof Error) {
+    log("instanceId instanceof Error", LogLevel.DEBUG, globalInstanceId);
+    throw globalInstanceId;
   }
 
   try {
-    log("instanceId getInstanceId", LogLevel.DEBUG, instanceId);
-    instanceId = await ec2.getInstanceId();
-    log("instanceId new string: " + instanceId, LogLevel.DEBUG, instanceId);
+    log("instanceId getInstanceId", LogLevel.DEBUG, globalInstanceId);
+    globalInstanceId = await ec2.getInstanceId();
+    log("instanceId new string: " + globalInstanceId, LogLevel.DEBUG, globalInstanceId);
   } catch (error) {
-    instanceId = error;
-    log("instanceId new Error: " + instanceId, LogLevel.DEBUG, instanceId);
-    throw instanceId;
+    globalInstanceId = error;
+    log("instanceId new Error: " + globalInstanceId, LogLevel.DEBUG, globalInstanceId);
+    throw globalInstanceId;
   }
-  return instanceId;
+  return globalInstanceId;
 }
 if (IS_RUNNING_IN_AWS) {
   getInstanceId().catch((error: unknown) => log("Could not retrieve instanceId", LogLevel.ERROR, error));

diff --git a/common/integration/s3.spec.ts b/common/integration/s3.spec.ts
@@ -48,13 +48,36 @@ export const UNIT_TEST_LOCAL_FILE_LOCATION: string = process.env.UNIT_TEST_LOCAL
 export const MAX_POLL_WAIT: number = parseInt(process.env.MAX_POLL_WAIT || "0", 10) || 500;
 // const LARGE_FILE_SIZE: number = parseInt(process.env.LARGE_FILE_SIZE || "0", 10) || 500000000;
 
-export const tagKey: string = "unittest";
-export const tagValue: string = "true";
-export const testTags = new Map<string, string>([[tagKey, tagValue]]);
+export const unittestTagKey: string = "unittest";
+export const unittestTagValue: string = "true";
+export const testTags = new Map<string, string>([[unittestTagKey, unittestTagValue]]);
 // These are set by the before after init()
 export const defaultTags = new Map<string, string>();
 export const fullTestTags = new Map<string, string>([...testTags]);
 
+export function initTags (): string {
+  let defaultKey: string | undefined;
+  let defaultValue: string | undefined;
+  if (ADDITIONAL_TAGS_ON_ALL.size > 0) {
+    for (const [key, value] of ADDITIONAL_TAGS_ON_ALL) {
+      if (!defaultKey) {
+        defaultKey = key;
+        defaultValue = value;
+      }
+      defaultTags.set(key, value);
+      fullTestTags.set(key, value);
+    }
+  } else {
+    defaultKey = "application";
+    defaultValue = util.APPLICATION_NAME;
+    defaultTags.set(defaultKey, defaultValue);
+    fullTestTags.set(defaultKey, defaultValue);
+  }
+  log("tags", LogLevel.DEBUG, { defaultKey, tags: Array.from(testTags.entries()), defaultTags: Array.from(defaultTags.entries()), allTags: Array.from(fullTestTags.entries()) });
+  expect(defaultKey, "defaultKey").to.not.equal(undefined);
+  return defaultKey!;
+}
+
 export const validateTagMap = (actual: Map<string, string>, expected: Map<string, string>) => {
   try {
     expect(actual.size, "validateTagMap actual.size").to.equal(expected.size);
@@ -87,29 +110,14 @@ export const validateTagSet = (actual: S3Tag[], expected: Map<string, string>) =
 describe("S3Util Integration", () => {
   let s3FileKey: string | undefined;
   let healthCheckDate: Date | undefined;
-  let tagKey: string;
-  let tagValue: string;
+  let defaultTagKey: string;
 
   before (async () => {
     // This test was failing until we reset everything. I don't know why and it bothers me.
     s3Config.s3Client = undefined as any;
     initS3();
-    if (ADDITIONAL_TAGS_ON_ALL.size > 0) {
-      for (const [key, value] of ADDITIONAL_TAGS_ON_ALL) {
-        if (!tagKey) {
-          tagKey = key;
-          tagValue = value;
-        }
-        defaultTags.set(key, value);
-        fullTestTags.set(key, value);
-      }
-    } else {
-      tagKey = "application";
-      tagValue = util.APPLICATION_NAME;
-      defaultTags.set(tagKey, tagValue);
-      fullTestTags.set(tagKey, tagValue);
-  }
-    log("tags", LogLevel.DEBUG, { tags: Array.from(testTags.entries()), defaultTags: Array.from(defaultTags.entries()), allTags: Array.from(fullTestTags.entries()) });
+    defaultTagKey = initTags();
+    expect(defaultTagKey, "defaultTagKey").to.not.equal(undefined);
     // Set the access callback to test that healthchecks will be updated
     setAccessCallback((date: Date) => healthCheckDate = date);
     try {
@@ -685,7 +693,7 @@ describe("S3Util Integration", () => {
         };
 
         // Change tags
-        expectedTags.set(tagKey, "pewpewagent");
+        expectedTags.set(defaultTagKey, "pewpewagent");
         expectedTags.set("unittest", "false");
         expectedTags.set("additionaltag", "additionalvalue");
         const tags = new Map(expectedTags);
@@ -794,8 +802,8 @@ describe("S3Util Integration", () => {
     it("Copy File should change name", (done: Mocha.Done) => {
       if (s3FileKey) {
         // Change tags
-        expectedTags.set(tagKey, "pewpewagent");
-        expectedTags.set("unittest", "false");
+        expectedTags.set(defaultTagKey, "pewpewagent");
+        expectedTags.set(unittestTagKey, "false");
         expectedTags.set("additionaltag", "additionalvalue");
         const tags = new Map(expectedTags);
         expect(expectedTags.size, "expectedTags.size before").to.equal(3); // Make sure there aren't some others we don't know about
@@ -939,7 +947,7 @@ describe("S3Util Integration", () => {
       try {
         const uploadTags = new Map(testTags);
         // Change it so clearing will set it back
-        uploadTags.set(tagKey, "pewpewagent");
+        uploadTags.set(defaultTagKey, "pewpewagent");
         const url: string = await uploadFile({
           filepath: UNIT_TEST_FILEPATH,
           s3Folder,
@@ -961,7 +969,7 @@ describe("S3Util Integration", () => {
         validateTagSet(tagging.TagSet!, uploadTags);
         expectedTags = new Map(uploadTags);
       } catch (error) {
-        log("copyObject beforeEach error", LogLevel.ERROR, error);
+        log("putObjectTagging beforeEach error", LogLevel.ERROR, error);
         throw error;
       }
     });
@@ -974,13 +982,14 @@ describe("S3Util Integration", () => {
           validateTagSet(tagging.TagSet!, expectedTags);
         }
       } catch (error) {
-        log("copyObject beforeEach error", LogLevel.ERROR, error);
+        log("putObjectTagging beforeEach error", LogLevel.ERROR, error);
         throw error;
       }
     });
 
     it("putObjectTagging should put a tag", (done: Mocha.Done) => {
       expectedTags.set("additionalTag", "additionalValue");
+      log("putObjectTagging should put a tag", LogLevel.DEBUG, expectedTags);
       const tags = new Map(expectedTags);
       putObjectTagging({ key: s3FileKey!, tags }).then((result: PutObjectTaggingCommandOutput) => {
         expect(result).to.not.equal(undefined);
@@ -990,6 +999,7 @@ describe("S3Util Integration", () => {
 
     it("putTags should put a tag", (done: Mocha.Done) => {
       expectedTags.set("additionalTag", "additionalValue");
+      log("putTags should put a tag", LogLevel.DEBUG, expectedTags);
       const tags = new Map(expectedTags);
       putTags({ filename, s3Folder, tags }).then(() => {
         done();
@@ -999,6 +1009,7 @@ describe("S3Util Integration", () => {
     it("putObjectTagging should clear tags", (done: Mocha.Done) => {
       const tags = new Map();
       expectedTags = new Map(defaultTags); // default will be set back
+      log("putObjectTagging should clear tags", LogLevel.DEBUG, expectedTags);
       putObjectTagging({ key: s3FileKey!, tags }).then((result: PutObjectTaggingCommandOutput) => {
         expect(result).to.not.equal(undefined);
         done();
@@ -1008,6 +1019,7 @@ describe("S3Util Integration", () => {
     it("putTags should clear tags", (done: Mocha.Done) => {
       const tags = new Map();
       expectedTags = new Map(defaultTags); // default will be set back
+      log("putTags should clear tags", LogLevel.DEBUG, expectedTags);
       putTags({ filename, s3Folder, tags }).then(() => {
         done();
       }).catch((error) => done(error));

diff --git a/common/integration/s3file.spec.ts b/common/integration/s3file.spec.ts
@@ -1,29 +1,33 @@
 import * as path from "path";
 import {
+  ADDITIONAL_TAGS_ON_ALL,
   BUCKET_URL,
   KEYSPACE_PREFIX,
   deleteObject,
   getObjectTagging,
   listFiles
 } from "../src/util/s3";
+import { GetObjectTaggingCommandOutput, _Object as S3Object } from "@aws-sdk/client-s3";
 import {
   LogLevel,
   PpaasS3File,
   PpaasS3FileOptions,
   PpaasTestId,
-  log
+  log,
+  s3
 } from "../src/index";
 import {
   MAX_POLL_WAIT,
   UNIT_TEST_FILENAME,
   UNIT_TEST_FILEPATH,
   UNIT_TEST_LOCAL_FILE_LOCATION,
+  defaultTags,
   fullTestTags,
+  initTags,
   testTags,
   validateTagMap,
   validateTagSet
 } from "./s3.spec";
-import { _Object as S3Object } from "@aws-sdk/client-s3";
 import { Stats } from "fs";
 import { expect } from "chai";
 import fs from "fs/promises";
@@ -76,6 +80,12 @@ describe("PpaasS3File Integration", () => {
       localDirectory: UNIT_TEST_LOCAL_FILE_LOCATION,
       tags: testTags
     });
+    if (ADDITIONAL_TAGS_ON_ALL.size > 0) {
+      for (const [key, value] of ADDITIONAL_TAGS_ON_ALL) {
+        defaultTags.set(key, value);
+        fullTestTags.set(key, value);
+      }
+    }
     expectedTags = new Map(fullTestTags);
   });
 
@@ -455,4 +465,56 @@ describe("PpaasS3File Integration", () => {
       }
     });
   });
+
+  describe("Update Tagging in S3", () => {
+    const clearedTags = new Map<string, string>();
+
+    beforeEach (async () => {
+      try {
+        const defaultTagKey = initTags();
+        const uploadTags = new Map(testTags);
+        // Change it so we verify clearing won't set it back
+        uploadTags.set(defaultTagKey, "pewpewagent");
+        clearedTags.set(defaultTagKey, "pewpewagent");
+        testPpaasS3FileUpload.tags = uploadTags;
+        await testPpaasS3FileUpload.upload(true);
+        log("testPpaasS3FileUpload.upload() succeeded", LogLevel.DEBUG);
+        s3FileKey = testPpaasS3FileUpload.key;
+        // As long as we don't throw, it passes
+        // Need time for eventual consistency to complete
+        await poll(async (): Promise<boolean | undefined> => {
+          const objects = await s3.listObjects(s3FileKey!);
+          return (objects && objects.Contents && objects.Contents.length > 0);
+        }, MAX_POLL_WAIT, (errMsg: string) => `${errMsg} Could not find the ${s3FileKey} in s3`);
+        const expectedObject = await s3.getObject(s3FileKey);
+        expect(expectedObject, "actualObject").to.not.equal(undefined);
+        expect(expectedObject.TagCount, "TagCount").to.equal(uploadTags.size);
+        const tagging: GetObjectTaggingCommandOutput = await getObjectTagging(s3FileKey);
+        expect(tagging.TagSet, "tagging.TagSet").to.not.equal(undefined);
+        validateTagSet(tagging.TagSet!, uploadTags);
+        expectedTags = new Map(uploadTags);
+      } catch (error) {
+        log("updateTags beforeEach error", LogLevel.ERROR, error);
+        throw error;
+      }
+    });
+
+    it("updateTags should put a tag", (done: Mocha.Done) => {
+      expectedTags.set("additionalTag", "additionalValue");
+      log("updateTags should put a tag", LogLevel.DEBUG, expectedTags);
+      testPpaasS3FileUpload.tags = new Map(expectedTags);
+      testPpaasS3FileUpload.updateTags().then(() => {
+        done();
+      }).catch((error) => done(error));
+    });
+
+    it("updateTags should clear tags", (done: Mocha.Done) => {
+      testPpaasS3FileUpload.tags = new Map();
+      expectedTags = new Map(clearedTags); // default will be set back
+      log("updateTags should clear tags", LogLevel.DEBUG, expectedTags);
+      testPpaasS3FileUpload.updateTags().then(() => {
+        done();
+      }).catch((error) => done(error));
+    });
+  });
 });
diff --git a/common/package.json b/common/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@fs/ppaas-common",
-  "version": "3.0.2",
+  "version": "3.0.3",
   "description": "Common Code for the PewPewController and PewPewAgent",
   "main": "dist/src/index.js",
   "types": "dist/src/index.d.ts",
@@ -56,6 +56,6 @@
     "eslint": "^8.40.0",
     "mocha": "^10.2.0",
     "nyc": "^15.1.0",
-    "typescript": "~5.2.0"
+    "typescript": "~5.3.0"
   }
 }