Update Dependencies 2023-12-28 Scripting (#190)

* Update readme (#187) * Moved the pewpew README under the src folder since it's pewpew binary specific * Added new README for overarching design * Fixed link to examples after move * Added fix so we only slice the key if we have a KEYSPACE_PREFIX * Retain tests longer (#188) * Added new tags for extra files from tests - Test files will be the yaml file, status file, and results files (these will be kept longer) - Test Extra files will be log files, environment variables, and all other files needed by the yaml or logged by the yaml * Added code to tag extra files correctly - Stdout, stderr, and all other files generated by the yaml will be tagged as Test Extra files and be tagged for deletion accordingly * Added code to tag extra files correctly - Additional files uploaded to the test, or additional files being copied from the old s3 location will be tagged as Test Extra files - Environment variable files will be tagged as Test Extra files - When changing from recurring to non-recurring, only yaml and test status files will be tagged as Test files, all others will be tagged as Extra files * Updated the bucket expiration so that test files are 2 years and all extra files are 6 months * Updated the integration tests to check the new test extra tags * Added code to validate that the logger files are tagged as extra * Added code to clear tests off of the calendar after 1 year (or configured) - if RUN_HISTORICAL_DELETE is true, then it will remove things off the calendar after DELETE_OLD_FILES_DAYS (365 default) days. * Cleaned up some of the logging on the historical delete * Made changes to allow testing faster locally * Update Dependencies 2023-12-28 (#189) * Updated dependencies and lock file - Held back Chai due to breaking change requiring ESM chaijs/chai#1561 - Updated date-fns and react-datepicker both working * Updated PPaaS version * Updated guide dependencies
FamilySearch · Dec 28, 2023 · 220141c · 220141c
1 parent 8a1b73e
commit 220141c
Show file tree

Hide file tree

Showing 18 changed files with 3,572 additions and 3,032 deletions.
diff --git a/README.md b/README.md
diff --git a/agent/createtest/pewpewtest.spec.ts b/agent/createtest/pewpewtest.spec.ts
@@ -11,6 +11,7 @@ import {
   TestStatus,
   TestStatusMessage,
   log,
+  logger,
   s3,
   sqs,
   util
@@ -149,11 +150,21 @@ describe("PewPewTest Create Test", () => {
             // Validate S3
           const filename: string = basename(test!.getResultsFile()!);
           const result = await s3.getObject(`${ppaasTestId!.s3Folder}/${filename}`);
-            expect(result).to.not.equal(undefined);
-            expect(result.ContentType).to.equal("application/json");
-            done();
-      })
-      .catch((error) => {
+          expect(result, "result").to.not.equal(undefined);
+          expect(result.ContentType, "result.ContentType").to.equal("application/json");
+          const [tagKeyExtra, tagValueExtra]: [string, string] = s3.defaultTestExtraFileTags().entries().next().value;
+          const stdoutFilename = logger.pewpewStdOutFilename(ppaasTestId!.testId);
+          const stderrFilename = logger.pewpewStdErrFilename(ppaasTestId!.testId);
+          const stdouttags = await s3.getTags({ s3Folder: ppaasTestId!.s3Folder, filename: stdoutFilename });
+          expect(stdouttags, "stdouttags").to.not.equal(undefined);
+          expect(stdouttags!.size, "stdouttags.size").to.be.greaterThan(0);
+          expect(stdouttags!.get(tagKeyExtra), `stdouttags!.get("${tagKeyExtra}")`).to.equal(tagValueExtra);
+          const stderrtags = await s3.getTags({ s3Folder: ppaasTestId!.s3Folder, filename: stderrFilename });
+          expect(stderrtags, "stderrtags").to.not.equal(undefined);
+          expect(stderrtags!.size, "stderrtags.size").to.be.greaterThan(0);
+          expect(stderrtags!.get(tagKeyExtra), `stderrtags!.get("${tagKeyExtra}")`).to.equal(tagValueExtra);
+          done();
+      }).catch((error) => {
         done(error);
       });
     });

diff --git a/agent/package.json b/agent/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@fs/ppaas-agent",
-  "version": "3.1.0",
+  "version": "3.1.1",
   "description": "Agent Service for running pewpew tests",
   "main": "dist/src/app.js",
   "scripts": {

diff --git a/agent/src/pewpewtest.ts b/agent/src/pewpewtest.ts
@@ -15,6 +15,7 @@ import {
   ec2,
   log,
   logger,
+  s3,
   sqs,
   util
 } from "@fs/ppaas-common";
@@ -418,13 +419,15 @@ export class PewPewTest {
       this.pewpewStdOutS3File = new PpaasS3File({
         filename: logger.pewpewStdOutFilename(this.testMessage.testId),
         s3Folder,
-        localDirectory: logConfig.LogFileLocation
+        localDirectory: logConfig.LogFileLocation,
+        tags: s3.defaultTestExtraFileTags()
       });
       this.log(`pewpewStdOutFilename = ${this.pewpewStdOutS3File.localFilePath}`, LogLevel.DEBUG);
       this.pewpewStdErrS3File =  new PpaasS3File({
         filename: logger.pewpewStdErrFilename(this.testMessage.testId),
         s3Folder,
-        localDirectory: logConfig.LogFileLocation
+        localDirectory: logConfig.LogFileLocation,
+        tags: s3.defaultTestExtraFileTags()
       });
       this.log(`pewpewStdErrS3File = ${this.pewpewStdErrS3File.localFilePath}`, LogLevel.DEBUG);
 
@@ -734,7 +737,8 @@ export class PewPewTest {
                 const foundS3File: PpaasS3File = new PpaasS3File({
                   filename: foundFile,
                   s3Folder: this.testMessage.s3Folder,
-                  localDirectory: this.localPath
+                  localDirectory: this.localPath,
+                  tags: s3.defaultTestExtraFileTags()
                 });
                 yamlCreatedFiles.set(foundFile, foundS3File);
               }

diff --git a/common/package.json b/common/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@fs/ppaas-common",
-  "version": "3.1.0",
+  "version": "3.1.1",
   "description": "Common Code for the PewPewController and PewPewAgent",
   "main": "dist/src/index.js",
   "types": "dist/src/index.d.ts",

diff --git a/common/src/s3file.ts b/common/src/s3file.ts
@@ -135,7 +135,7 @@ export class PpaasS3File implements S3File {
         .map(async (s3File: S3Object) => {
         // find the part after the s3Folder. We may have a prefix added to us so it may not be at the beginning
         // If s3Folder is part of a folder we need to split on the / not on the folder name
-        const key: string = s3File.Key!.startsWith(KEYSPACE_PREFIX) ? s3File.Key!.slice(KEYSPACE_PREFIX.length) : s3File.Key!;
+        const key: string = KEYSPACE_PREFIX && s3File.Key!.startsWith(KEYSPACE_PREFIX) ? s3File.Key!.slice(KEYSPACE_PREFIX.length) : s3File.Key!;
         const s3KeySplit = key.split("/");
         const realFolder = s3KeySplit.slice(0, -1).join("/");
         const filename = s3KeySplit[s3KeySplit.length - 1];

diff --git a/common/src/util/s3.ts b/common/src/util/s3.ts
@@ -56,10 +56,14 @@ export const config: { s3Client: S3Client } = {
  * has tags passed in of "key2=value3" then only key1=value1 would be added. WARNING: Is only initialized after s3.init() called.
  */
 export const ADDITIONAL_TAGS_ON_ALL = new Map<string, string>();
-// Don't export so that the original can't be modified
+// Don't export so that the original can't be modified,
+// we'll return a new copy of the Map each time so the original can't be modified
 const TEST_FILE_TAGS_INTERNAL = new Map<string, string>([["test", "true"]]);
-/** Returns a new copy of the Map each time so the original can't be modified */
+const TEST_EXTRA_FILE_TAGS_INTERNAL = new Map<string, string>([["test", "false"]]);
+/** Default tags on all test files (yaml, results, status) from the tests */
 export const defaultTestFileTags = (): Map<string, string> => new Map(TEST_FILE_TAGS_INTERNAL);
+/** Default tags on all extra files from the tests */
+export const defaultTestExtraFileTags = (): Map<string, string> => new Map(TEST_EXTRA_FILE_TAGS_INTERNAL);
 
 /**
  * Initializes the S3 object using environment variables. Runs later so it doesn't throw on start-up

diff --git a/controller/integration/testmanager.spec.ts b/controller/integration/testmanager.spec.ts
@@ -25,12 +25,12 @@ import {
   s3
 } from "@fs/ppaas-common";
 import { TestManager, defaultRecurringFileTags} from "../pages/api/util/testmanager";
+import { isYamlFile, latestPewPewVersion } from "../pages/api/util/clientutil";
 import { EventInput } from "@fullcalendar/core";
 import { PpaasEncryptEnvironmentFile } from "../pages/api/util/ppaasencryptenvfile";
 import { TestScheduler } from "../pages/api/util/testscheduler";
 import { expect } from "chai";
 import { getPewPewVersionsInS3 } from "../pages/api/util/pewpew";
-import { latestPewPewVersion } from "../pages/api/util/clientutil";
 import path from "path";
 
 logger.config.LogFileName = "ppaas-controller";
@@ -202,10 +202,15 @@ describe("TestManager Integration", () => {
           expect(s3Files.length, "s3Files.length").to.equal(3);
           // Check that the test=true tag is added
           const [tagKey, tagValue]: [string, string] = s3.defaultTestFileTags().entries().next().value;
+          const [tagKeyExtra, tagValueExtra]: [string, string] = s3.defaultTestExtraFileTags().entries().next().value;
           expect(typeof tagKey, "typeof tagKey").to.equal("string");
           for (const s3File of s3Files) {
             expect(s3File.tags, "s3File.tags").to.not.equal(undefined);
-            expect(s3File.tags?.get(tagKey), `${s3File.filename}.tags?.get("${tagKey}")`).to.equal(tagValue);
+            if (isYamlFile(s3File.filename) || s3File.filename.endsWith(".info")) {
+              expect(s3File.tags?.get(tagKey), `${s3File.filename}.tags?.get("${tagKey}")`).to.equal(tagValue);
+            } else {
+              expect(s3File.tags?.get(tagKeyExtra), `${s3File.filename}.tags?.get("${tagKeyExtra}")`).to.equal(tagValueExtra);
+            }
           }
           done();
         }).catch((error) => {
@@ -556,10 +561,15 @@ describe("TestManager Integration", () => {
           expect(s3Files.length, "s3Files.length").to.equal(5);
           // Check that the test=true tag is added
           const [tagKey, tagValue]: [string, string] = s3.defaultTestFileTags().entries().next().value;
+          const [tagKeyExtra, tagValueExtra]: [string, string] = s3.defaultTestExtraFileTags().entries().next().value;
           expect(typeof tagKey, "typeof tagKey").to.equal("string");
           for (const s3File of s3Files) {
             expect(s3File.tags, "s3File.tags").to.not.equal(undefined);
-            expect(s3File.tags?.get(tagKey), `${s3File.filename}.tags?.get("${tagKey}")`).to.equal(tagValue);
+            if (isYamlFile(s3File.filename) || s3File.filename.endsWith(".info")) {
+              expect(s3File.tags?.get(tagKey), `${s3File.filename}.tags?.get("${tagKey}")`).to.equal(tagValue);
+            } else {
+              expect(s3File.tags?.get(tagKeyExtra), `${s3File.filename}.tags?.get("${tagKeyExtra}")`).to.equal(tagValueExtra);
+            }
           }
           done();
         }).catch((error) => {
@@ -655,10 +665,15 @@ describe("TestManager Integration", () => {
           expect(s3Files.length, "s3Files.length").to.equal(3);
           // Check that the recurring=true tag is added
           const [tagKey, tagValue]: [string, string] = s3.defaultTestFileTags().entries().next().value;
+          const [tagKeyExtra, tagValueExtra]: [string, string] = s3.defaultTestExtraFileTags().entries().next().value;
           expect(typeof tagKey, "typeof tagKey").to.equal("string");
           for (const s3File of s3Files) {
             expect(s3File.tags, "s3File.tags").to.not.equal(undefined);
-            expect(s3File.tags?.get(tagKey), `${s3File.filename}.tags?.get("${tagKey}")`).to.equal(tagValue);
+            if (isYamlFile(s3File.filename) || s3File.filename.endsWith(".info")) {
+              expect(s3File.tags?.get(tagKey), `${s3File.filename}.tags?.get("${tagKey}")`).to.equal(tagValue);
+            } else {
+              expect(s3File.tags?.get(tagKeyExtra), `${s3File.filename}.tags?.get("${tagKeyExtra}")`).to.equal(tagValueExtra);
+            }
           }
           done();
         }).catch((error) => {
@@ -753,10 +768,15 @@ describe("TestManager Integration", () => {
           expect(s3Files.length, "s3Files.length").to.equal(5);
           // Check that the recurring=true tag is added
           const [tagKey, tagValue]: [string, string] = s3.defaultTestFileTags().entries().next().value;
+          const [tagKeyExtra, tagValueExtra]: [string, string] = s3.defaultTestExtraFileTags().entries().next().value;
           expect(typeof tagKey, "typeof tagKey").to.equal("string");
           for (const s3File of s3Files) {
             expect(s3File.tags, "s3File.tags").to.not.equal(undefined);
-            expect(s3File.tags?.get(tagKey), `${s3File.filename}.tags?.get("${tagKey}")`).to.equal(tagValue);
+            if (isYamlFile(s3File.filename) || s3File.filename.endsWith(".info")) {
+              expect(s3File.tags?.get(tagKey), `${s3File.filename}.tags?.get("${tagKey}")`).to.equal(tagValue);
+            } else {
+              expect(s3File.tags?.get(tagKeyExtra), `${s3File.filename}.tags?.get("${tagKeyExtra}")`).to.equal(tagValueExtra);
+            }
           }
           done();
         }).catch((error) => {
@@ -1795,12 +1815,17 @@ describe("TestManager Integration", () => {
             expect(s3Files.length, "s3Files.length").to.equal(3);
             // Check that the test=true tag is added and recurring=true is removed
             const [testTagKey, testTagValue]: [string, string] = s3.defaultTestFileTags().entries().next().value;
-            // eslint-disable-next-line @typescript-eslint/no-unused-vars
+            const [tagKeyExtra, tagValueExtra]: [string, string] = s3.defaultTestExtraFileTags().entries().next().value;
+              // eslint-disable-next-line @typescript-eslint/no-unused-vars
             const [recurringTagKey, recurringTagValue]: [string, string] = defaultRecurringFileTags().entries().next().value;
             expect(typeof testTagKey, "typeof tagKey").to.equal("string");
             for (const s3File of s3Files) {
               expect(s3File.tags, "s3File.tags").to.not.equal(undefined);
-              expect(s3File.tags?.get(testTagKey), `${s3File.filename}.tags?.get("${testTagKey}")`).to.equal(testTagValue);
+              if (isYamlFile(s3File.filename) || s3File.filename.endsWith(".info")) {
+                expect(s3File.tags?.get(testTagKey), `${s3File.filename}.tags?.get("${testTagKey}")`).to.equal(testTagValue);
+              } else {
+                expect(s3File.tags?.get(tagKeyExtra), `${s3File.filename}.tags?.get("${tagKeyExtra}")`).to.equal(tagValueExtra);
+              }
               expect(s3File.tags?.get(recurringTagKey), `${s3File.filename}.tags?.get("${recurringTagKey}")`).to.equal(undefined);
             }
             done();

diff --git a/controller/package.json b/controller/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@fs/ppaas-controller",
-  "version": "3.1.0",
+  "version": "3.1.1",
   "description": "Controller Service for running pewpew tests",
   "private": true,
   "scripts": {
@@ -53,7 +53,7 @@
     "chart.js": "~4.4.0",
     "chartjs-adapter-date-fns": "^3.0.0",
     "cookie": "^0.6.0",
-    "date-fns": "~2.30.0",
+    "date-fns": "~3.0.0",
     "express": "^4.18.2",
     "file-saver": "^2.0.5",
     "formidable": "~2.1.2",
@@ -65,7 +65,7 @@
     "rc-progress": "^3.4.2",
     "react": "^18.2.0",
     "react-accessible-accordion": "^5.0.0",
-    "react-datepicker": "~4.24.0",
+    "react-datepicker": "~4.25.0",
     "react-dom": "^18.2.0",
     "react-dropzone": "^14.2.3",
     "react-transition-group": "^4.4.5",

diff --git a/controller/pages/api/util/testmanager.ts b/controller/pages/api/util/testmanager.ts
@@ -1349,23 +1349,28 @@ export abstract class TestManager {
         // Upload files
         const uploadPromises: Promise<PpaasS3File | void>[] = [uploadFile(yamlFile, s3Folder, fileTags)];
         // additionalFiles - Do this last so we can upload them at the same time
-        uploadPromises.push(...(additionalFiles.map((file: File) => uploadFile(file, s3Folder, fileTags))));
+        uploadPromises.push(...(additionalFiles.map((file: File) => uploadFile(file, s3Folder, fileTags || s3.defaultTestExtraFileTags()))));
         if (!editSchedule) {
           // copyFiles, just copy them from the old s3 location to the new one
           uploadPromises.push(...(copyFiles.map((file: PpaasS3File) => {
-            file.tags = fileTags;
+            file.tags = fileTags || s3.defaultTestExtraFileTags();
             return file.copy({ destinationS3Folder: s3Folder });
           })));
         } else {
-          // If we're changing from non-recurring to recurring or vice-versa we need to edit the existing file tags.
-          const updateTags: Map<string, string> = fileTags || s3.defaultTestFileTags(); // If fileTags is truthy it's recurring
+          const s3StatusFilename: string = createS3StatusFilename(ppaasTestId);
           uploadPromises.push(...(copyFiles.map((file: PpaasS3File) => {
-            file.tags = updateTags;
+            // If we're changing from non-recurring to recurring or vice-versa we need to edit the existing file tags.
+            // yaml and status files need defaultTestFileTags, all others should be defaultTestExtraFileTags
+            file.tags = fileTags
+              ? fileTags
+              : isYamlFile(file.filename) || file.filename === s3StatusFilename // yaml and status files are test files
+                ? s3.defaultTestFileTags()
+                : s3.defaultTestExtraFileTags();
             return file.updateTags();
           })));
         }
         // Store encrypted environment variables in s3
-        uploadPromises.push(new PpaasEncryptEnvironmentFile({ s3Folder, environmentVariablesFile, tags: fileTags }).upload());
+        uploadPromises.push(new PpaasEncryptEnvironmentFile({ s3Folder, environmentVariablesFile, tags: fileTags || s3.defaultTestExtraFileTags() }).upload());
         // Wait for all uploads to complete
         await Promise.all(uploadPromises);