Test mac and windows executables scripting (#210)

* Added eslint key-spacing rule (#204) * Adding toaster, dynamically checking pewpewVersion (#201) * adding files to dynamically show latest pewpew version on Toaster * cleaning up code for lintter to pass * refactoring code to use tags in S3 * removing files from tsconfig * fixing lint rules * fixing spacing as local linter didnt catch those but the build did * missed one space * adding tests, cleaning up files * removing trailing spaces failing on build * adding DS_Store - make specific file to gitignore * updating storybook to show pewpew latest version output * Added common constants for the different os executables to common - To differentiate Mac in S3, we'll use pewpew.mac as the file name. - These constants will be used by common, agent, and controller * Updated the agent to remove hardcoded pewpew references - The agent will now use os specific versions of the pewpew executable: pewpew (Linux), pewpew.exe (Windows), and pewpew.mac (Mac) - Updated the README to add instructions for Mac and Windows Users on how to add an OS specific file and where * Updated the controller to remove hardcoded pewpew and pewpew.zip references - The controller will now use os specific versions of the pewpew executable: pewpew (Linux), pewpew.exe (Windows), and pewpew.mac (Mac) - Updated the README to add instructions for Mac and Windows Users on how to add an OS specific file(s) and where * Updated pewpew to the 0.5.13 release * Removed deprecated fs and log calls from util - Removed fs and logger calls from util so it can be used by client functions - Added default exports of common util functions and constants * Fixed issue with import from common bringing in fs into controller * Added fixes for filtering out pre-release versions * Fixed test that was only ignoring one pewpew executable * Added sleep on windows to avoid race condition after unzipping files * Fixed issue with integration tests - POST /pewpew tests need to run before the GET tests to make sure the data is there - After the deletes are completed we need to put back any versions we deleted * Added fix for compiling next.js on Windows - Symlinks do not work on Windows for compiling so we have to copy the file * Added fix for acceptance tests on Windows - Just like the integration tests, we must wait for the unzip to release the lock before we can access the files * Fixed warning during Next.js build on invalid config * Fixed security vulnerabilities * Added fixes missed from merge * Updated README to add Windows and Mac instructions to include scripting pewpew version * Test mac and windows executables (#209) * Added common constants for the different os executables to common - To differentiate Mac in S3, we'll use pewpew.mac as the file name. - These constants will be used by common, agent, and controller * Updated the agent to remove hardcoded pewpew references - The agent will now use os specific versions of the pewpew executable: pewpew (Linux), pewpew.exe (Windows), and pewpew.mac (Mac) - Updated the README to add instructions for Mac and Windows Users on how to add an OS specific file and where * Updated the controller to remove hardcoded pewpew and pewpew.zip references - The controller will now use os specific versions of the pewpew executable: pewpew (Linux), pewpew.exe (Windows), and pewpew.mac (Mac) - Updated the README to add instructions for Mac and Windows Users on how to add an OS specific file(s) and where * Updated pewpew to the 0.5.13 release * Removed deprecated fs and log calls from util - Removed fs and logger calls from util so it can be used by client functions - Added default exports of common util functions and constants * Fixed issue with import from common bringing in fs into controller * Added fixes for filtering out pre-release versions * Fixed test that was only ignoring one pewpew executable * Added sleep on windows to avoid race condition after unzipping files * Fixed issue with integration tests - POST /pewpew tests need to run before the GET tests to make sure the data is there - After the deletes are completed we need to put back any versions we deleted * Added fix for compiling next.js on Windows - Symlinks do not work on Windows for compiling so we have to copy the file * Added fix for acceptance tests on Windows - Just like the integration tests, we must wait for the unzip to release the lock before we can access the files * Fixed warning during Next.js build on invalid config * Fixed security vulnerabilities * Removed additional hard-coded latest and pewpew strings * Fixed the getTags call to get the latest tags (#211) * Fixed the getTags call to get the latest tags - Unit and Integration tests were not catching the bug, fixed the tests, then fixed the code * Cleaned up the logging of currentLatestVersion --------- Co-authored-by: Bryan <[email protected]>
FamilySearch · Mar 26, 2024 · 48d31d2 · 48d31d2
1 parent f605a80
commit 48d31d2
Show file tree

Hide file tree

Showing 33 changed files with 580 additions and 206 deletions.
diff --git a/agent/.gitignore b/agent/.gitignore
@@ -9,6 +9,7 @@
 /coverage
 setaws.sh
 /.nyc_output/
+/test/pewpew.*
 
 # production
 /build

diff --git a/agent/README.md b/agent/README.md
@@ -15,6 +15,11 @@ $ npm i && npm run build
 
 ```
 
+## Mac and Windows Testing
+The unit tests, integration, and acceptance tests are designed to run on Linux. As such, the pewpew executable files required for running on Linux are checked into the tree in the test server so that the files are available for our Github Actions (`test/pewpew`).
+
+To override these tests for mac or windows, the pewpew exectuable must be named `pewpew.exe` for Windows and `pewpew.mac` for Mac. These files should then be dropped in the `test/` folder.
+
 ## Test
 
 ```bash

diff --git a/agent/createtest/pewpewtest.spec.ts b/agent/createtest/pewpewtest.spec.ts
@@ -1,6 +1,7 @@
 import {
   LogLevel,
   MessageType,
+  PEWPEW_VERSION_LATEST,
   PpaasS3File,
   PpaasS3Message,
   PpaasTestId,
@@ -85,7 +86,7 @@ describe("PewPewTest Create Test", () => {
         s3Folder,
         yamlFile: createTestFilename,
         testRunTimeMn: 2,
-        version: "latest",
+        version: PEWPEW_VERSION_LATEST,
         envVariables: { SERVICE_URL_AGENT: "127.0.0.1:8080" },
         restartOnFailure: false,
         additionalFiles: [],
@@ -311,7 +312,7 @@ describe("PewPewTest Create Test", () => {
         testId: ppaasTestId.testId,
         s3Folder,
         yamlFile: createTestFilename,
-        version: "latest",
+        version: PEWPEW_VERSION_LATEST,
         envVariables: {
           SERVICE_URL_AGENT: "127.0.0.1:8080",
           RUST_LOG: "info",

diff --git a/agent/src/pewpewtest.ts b/agent/src/pewpewtest.ts
@@ -38,7 +38,7 @@ const DEFAULT_PEWPEW_PARAMS = [
   "-w"
 ];
 
-const PEWPEW_PATH: string = process.env.PEWPEW_PATH || "pewpew";
+const PEWPEW_PATH: string = process.env.PEWPEW_PATH || util.PEWPEW_BINARY_EXECUTABLE;
 const DOWNLOAD_PEWPEW: boolean = (process.env.DOWNLOAD_PEWPEW || "false") === "true";
 const LOCAL_FILE_LOCATION: string = process.env.LOCAL_FILE_LOCATION || process.env.TEMP || "/tmp";
 const RESULTS_FILE_MAX_WAIT: number = parseInt(process.env.RESULTS_FILE_MAX_WAIT || "0", 10) || 5000; // S3_UPLOAD_INTERVAL + this Could take up to a minute
@@ -70,7 +70,7 @@ export async function findYamlCreatedFiles (localPath: string, yamlFile: string,
       additionalFiles = additionalFiles || [];
       // Find files that aren't the yamlFile or additionalFile, a results file, or the pewpew executable
       const YamlCreatedFiles = files.filter((file) => file !== yamlFile && !additionalFiles!.includes(file)
-          && !(file.startsWith("stats-") && file.endsWith(".json")) && file !== "pewpew" && file !== "pewpew.exe");
+        && !(file.startsWith("stats-") && file.endsWith(".json")) && !util.PEWPEW_BINARY_EXECUTABLE_NAMES.includes(file));
       log(`YamlCreatedFiles: ${YamlCreatedFiles}}`, LogLevel.DEBUG);
       if (YamlCreatedFiles.length > 0) {
         return YamlCreatedFiles; // Don't return the joined path
@@ -86,9 +86,9 @@ export async function findYamlCreatedFiles (localPath: string, yamlFile: string,
 // Export for testing
 export function versionGreaterThan (currentVersion: string, compareVersion: string): boolean {
   // If the current version is latest then we're always greater than or equal to
-  if (currentVersion === "latest") { return true; }
+  if (currentVersion === util.PEWPEW_VERSION_LATEST) { return true; }
   // If the compareVersion is latest, then only currrentVersion=latest is greater
-  if (compareVersion === "latest") { return false; }
+  if (compareVersion === util.PEWPEW_VERSION_LATEST) { return false; }
 
   return semver.gt(currentVersion, compareVersion);
 }
@@ -373,28 +373,20 @@ export class PewPewTest {
       // Download the pewpew executable if needed
       if (DOWNLOAD_PEWPEW) {
         // version check in the test message
-        const version = this.testMessage.version || "latest";
+        const version = this.testMessage.version || util.PEWPEW_VERSION_LATEST;
         const localDirectory = this.localPath;
-        const s3Folder = "pewpew/" + version;
-        this.log(`os.platform() = ${platform()}`, LogLevel.DEBUG, { version, s3Folder });
-        if (platform() === "win32") {
-          const pewpewS3File: PpaasS3File = new PpaasS3File({
-            filename: "pewpew.exe",
-            s3Folder,
-            localDirectory
-          });
-          pewpewPath = await pewpewS3File.download(true);
-          this.log(`getFile(pewpew.exe) result = ${pewpewPath}`, LogLevel.DEBUG);
-        } else {
-          // If the version isn't there, this will throw (since we can't find it)
-          const pewpewS3File: PpaasS3File = new PpaasS3File({
-            filename: "pewpew",
-            s3Folder,
-            localDirectory
-          });
-          pewpewPath = await pewpewS3File.download(true);
-          this.log(`getFile(pewpew) result = ${pewpewPath}`, LogLevel.DEBUG);
-          // We need to make it executable
+        const s3Folder = `${util.PEWPEW_BINARY_FOLDER}/${version}`;
+        this.log(`os.platform() = ${platform()}`, LogLevel.DEBUG, { version, s3Folder, filename: util.PEWPEW_BINARY_EXECUTABLE });
+        const pewpewS3File: PpaasS3File = new PpaasS3File({
+          filename: util.PEWPEW_BINARY_EXECUTABLE,
+          s3Folder,
+          localDirectory
+        });
+        pewpewPath = await pewpewS3File.download(true);
+        this.log(`getFile("${util.PEWPEW_BINARY_EXECUTABLE}") result = ${pewpewPath}`, LogLevel.DEBUG);
+        // If the version isn't there, this will throw (since we can't find it)
+        if (platform() !== "win32") {
+          // We need to make it executable for non-windows
           await fs.chmod(pewpewPath, 0o775);
         }
         // Always call this even if it isn't logged to make sure the file downloaded

diff --git a/agent/src/tests.ts b/agent/src/tests.ts
@@ -1,5 +1,7 @@
 import {
   LogLevel,
+  PEWPEW_BINARY_FOLDER,
+  PEWPEW_VERSION_LATEST,
   PpaasTestId,
   PpaasTestMessage,
   TestMessage,
@@ -19,8 +21,8 @@ logger.config.LogFileName = "ppaas-agent";
 
 const UNIT_TEST_FOLDER = process.env.UNIT_TEST_FOLDER || "test";
 const yamlFile = "basicwithenv.yaml";
-const version = "latest";
-const PEWPEW_PATH = process.env.PEWPEW_PATH || pathJoin(UNIT_TEST_FOLDER, "pewpew");
+const version = PEWPEW_VERSION_LATEST;
+const PEWPEW_PATH = process.env.PEWPEW_PATH || pathJoin(UNIT_TEST_FOLDER, util.PEWPEW_BINARY_EXECUTABLE);
 const buildTestContents = `
 vars:
   rampTime: 10s
@@ -72,7 +74,7 @@ export async function buildTest ({
       }),
       s3.uploadFile({
         filepath: PEWPEW_PATH,
-        s3Folder: `pewpew/${version}`,
+        s3Folder: `${PEWPEW_BINARY_FOLDER}/${version}`,
         publicRead: false,
         contentType: "application/octet-stream"
       })

diff --git a/agent/test/pewpew b/agent/test/pewpew
diff --git a/agent/test/pewpewtest.spec.ts b/agent/test/pewpewtest.spec.ts
@@ -1,11 +1,13 @@
 import {
   LogLevel,
+  PEWPEW_VERSION_LATEST,
   PpaasTestId,
   PpaasTestStatus,
   TestStatus,
   TestStatusMessage,
   log,
-  logger
+  logger,
+  util
 } from "@fs/ppaas-common";
 import {
   copyTestStatus,
@@ -26,17 +28,9 @@ describe("PewPewTest", () => {
     let localFiles: string[];
 
     before (async () => {
-      localFiles = await readdir(UNIT_TEST_FILEDIR);
+      localFiles = (await readdir(UNIT_TEST_FILEDIR))
+      .filter((filename) => filename !== UNIT_TEST_FILENAME && !util.PEWPEW_BINARY_EXECUTABLE_NAMES.includes(filename));
       log(`localFiles = ${JSON.stringify(localFiles)}`, LogLevel.DEBUG);
-      const unitTestFound = localFiles.indexOf(UNIT_TEST_FILENAME);
-      if (unitTestFound >= 0) {
-        localFiles.splice(unitTestFound, 1);
-      }
-      const pewpewFound = localFiles.indexOf("pewpew");
-      if (pewpewFound >= 0) {
-        localFiles.splice(pewpewFound, 1);
-      }
-      log(`localFiles removed = ${JSON.stringify(localFiles)}`, LogLevel.DEBUG);
     });
 
     it("Find Yaml should find nothing when everything passed", (done: Mocha.Done) => {
@@ -85,17 +79,17 @@ describe("PewPewTest", () => {
 
   describe("versionGreaterThan", () => {
     it("latest is always greater", (done: Mocha.Done) => {
-      expect(versionGreaterThan("latest", "")).to.equal(true);
+      expect(versionGreaterThan(PEWPEW_VERSION_LATEST, "")).to.equal(true);
       done();
     });
 
     it("latest is always greater than latest", (done: Mocha.Done) => {
-      expect(versionGreaterThan("latest", "latest")).to.equal(true);
+      expect(versionGreaterThan(PEWPEW_VERSION_LATEST, PEWPEW_VERSION_LATEST)).to.equal(true);
       done();
     });
 
     it("greater than latest is false", (done: Mocha.Done) => {
-      expect(versionGreaterThan("0.5.5", "latest")).to.equal(false);
+      expect(versionGreaterThan("0.5.5", PEWPEW_VERSION_LATEST)).to.equal(false);
       done();
     });
 

diff --git a/common/integration/ppaasteststatus.spec.ts b/common/integration/ppaasteststatus.spec.ts
@@ -1,5 +1,6 @@
 import {
   LogLevel,
+  PEWPEW_VERSION_LATEST,
   PpaasTestId,
   PpaasTestStatus,
   TestStatus,
@@ -33,7 +34,7 @@ describe("PpaasTestStatus", () => {
       resultsFilename: [ppaasTestId.testId + ".json"],
       status: TestStatus.Running,
       errors: ["Test Error"],
-      version: "latest",
+      version: PEWPEW_VERSION_LATEST,
       queueName: "unittest",
       userId: "unittestuser"
     };

diff --git a/common/src/index.ts b/common/src/index.ts
@@ -3,6 +3,16 @@ import * as logger from "./util/log";
 import * as s3 from "./util/s3";
 import * as sqs from "./util/sqs";
 import * as util from "./util/util";
+import {
+  APPLICATION_NAME,
+  PEWPEW_BINARY_EXECUTABLE,
+  PEWPEW_BINARY_EXECUTABLE_NAMES,
+  PEWPEW_BINARY_FOLDER,
+  PEWPEW_VERSION_LATEST,
+  SYSTEM_NAME,
+  poll,
+  sleep
+ } from "./util/util";
 import { LogLevel, log } from "./util/log";
 import { MakeTestIdOptions, PpaasTestId } from "./ppaastestid";
 import { PpaasS3File, PpaasS3FileCopyOptions, PpaasS3FileOptions } from "./s3file";
@@ -29,6 +39,14 @@ export {
   util,
   log,
   LogLevel,
+  APPLICATION_NAME,
+  PEWPEW_BINARY_EXECUTABLE,
+  PEWPEW_BINARY_EXECUTABLE_NAMES,
+  PEWPEW_BINARY_FOLDER,
+  PEWPEW_VERSION_LATEST,
+  SYSTEM_NAME,
+  poll,
+  sleep,
   PpaasCommunicationsMessage,
   PpaasS3Message,
   PpaasTestId,

diff --git a/common/src/ppaastestid.ts b/common/src/ppaastestid.ts
@@ -68,7 +68,7 @@ export class PpaasTestId {
     const yamlname: string = (path.basename(yamlFile, path.extname(yamlFile)).toLocaleLowerCase()
     + (profile || "").toLocaleLowerCase())
     .replace(/[^a-z0-9]/g, "");
-    if (yamlname === "pewpew") {
+    if (yamlname.startsWith("pewpew")) {
       throw new Error("Yaml File cannot be named PewPew");
     }
     const newTestId: PpaasTestId = new PpaasTestId(yamlname, dateString || this.getDateString());

diff --git a/common/src/s3file.ts b/common/src/s3file.ts
@@ -14,12 +14,12 @@ import {
   uploadFile
 } from "./util/s3";
 import { LogLevel, log } from "./util/log";
+import { PEWPEW_BINARY_EXECUTABLE_NAMES, sleep } from "./util/util";
 import { PutObjectCommandInput, _Object as S3Object } from "@aws-sdk/client-s3";
 import { access, stat } from "fs/promises";
 import { S3File } from "../types";
 import { Stats } from "fs";
 import { URL } from "url";
-import { sleep } from "./util/util";
 
 const RESULTS_UPLOAD_RETRY: number = parseInt(process.env.RESULTS_UPLOAD_RETRY || "0", 10) || 5;
 
@@ -105,7 +105,7 @@ export class PpaasS3File implements S3File {
         this.contentType = "text/plain";
         break;
     }
-    if (filename === "pewpew" || filename === "pewpew.exe") {
+    if (PEWPEW_BINARY_EXECUTABLE_NAMES.includes(filename)) {
       this.contentType = "application/octet-stream";
     }
     log(`contentType: ${this.contentType}`, LogLevel.DEBUG);

diff --git a/common/src/util/util.ts b/common/src/util/util.ts
@@ -1,6 +1,4 @@
-import * as _fs from "fs/promises";
 import * as os from "os";
-import { LogLevel, log } from "./log";
 
 export const APPLICATION_NAME: string = process.env.APPLICATION_NAME || "pewpewagent";
 export const CONTROLLER_APPLICATION_NAME: string = process.env.CONTROLLER_APPLICATION_NAME || "pewpewcontroller";
@@ -10,6 +8,24 @@ export const SYSTEM_NAME: string = process.env.SYSTEM_NAME || "unittests";
 export const CONTROLLER_ENV = process.env.CONTROLLER_ENV;
 export const AGENT_ENV = process.env.AGENT_ENV;
 
+export const PEWPEW_BINARY_FOLDER: string = "pewpew";
+export const PEWPEW_VERSION_LATEST: string = "latest";
+export const PEWPEW_BINARY_EXECUTABLE_LINUX = "pewpew";
+export const PEWPEW_BINARY_EXECUTABLE_WINDOWS = "pewpew.exe";
+export const PEWPEW_BINARY_EXECUTABLE_MAC = "pewpew.mac";
+export const PEWPEW_BINARY_EXECUTABLE = process.env.PEWPEW_BINARY_EXECUTABLE
+  || os.platform() === "win32"
+  ? PEWPEW_BINARY_EXECUTABLE_WINDOWS
+  : os.platform() === "darwin"
+    ? PEWPEW_BINARY_EXECUTABLE_MAC
+    : PEWPEW_BINARY_EXECUTABLE_LINUX;
+export const PEWPEW_BINARY_EXECUTABLE_NAMES = [
+  PEWPEW_BINARY_EXECUTABLE_LINUX,
+  PEWPEW_BINARY_EXECUTABLE_WINDOWS,
+  PEWPEW_BINARY_EXECUTABLE_MAC
+];
+
+
 /** This applications PREFIX. No overrides */
 export const PREFIX_DEFAULT: string = `${APPLICATION_NAME}-${SYSTEM_NAME}`.toUpperCase().replace(/-/g, "_");
 let PREFIX_CONTROLLER_ENV: string | undefined;
@@ -34,33 +50,12 @@ export const getPrefix = (controllerEnv?: boolean | string): string => {
   return PREFIX_DEFAULT;
 };
 
-/** @deprecated Use `fs/promises` */
-export const fs = {
-  /** @deprecated Use `fs/promises` */
-  access: _fs.access,
-  /** @deprecated Use `fs/promises` */
-  chmod: _fs.chmod,
-  /** @deprecated Use `fs/promises` */
-  mkdir: _fs.mkdir,
-  /** @deprecated Use `fs/promises` */
-  readdir: _fs.readdir,
-  /** @deprecated Use `fs/promises` */
-  readFile: _fs.readFile,
-  /** @deprecated Use `fs/promises` */
-  rename: _fs.rename,
-  /** @deprecated Use `fs/promises` */
-  unlink: _fs.unlink,
-  /** @deprecated Use `rimraf` or `fs/promises` */
-  rmdir: _fs.rmdir,
-  /** @deprecated Use `fs/promises` */
-  stat: _fs.stat
-};
-
 export async function sleep (ms: number): Promise<void> {
   try {
     await new Promise((resolve) => setTimeout(resolve, ms));
   } catch (error: unknown) {
-    log("sleep Error", LogLevel.ERROR, error); // swallow it
+    // eslint-disable-next-line no-console
+    console.error("sleep Error", error); // swallow it
   }
 }
-Original file line number
+Diff line change
@@ Expand Up / @@ -15,6 +15,11 @@ $ npm i && npm run build @@
     ```
+    ## Mac and Windows Testing
+    The unit tests, integration, and acceptance tests are designed to run on Linux. As such, the pewpew executable files required for running on Linux are checked into the tree in the test server so that the files are available for our Github Actions (`test/pewpew`).
+    To override these tests for mac or windows, the pewpew exectuable must be named `pewpew.exe` for Windows and `pewpew.mac` for Mac. These files should then be dropped in the `test/` folder.
     ## Test
     ```bash
@@ Expand Down @@