Fixed an issue where searching for old tests that were yml failed to find

common/src/s3file.ts

@@ -34,7 +34,7 @@ export interface PpaasS3FileOptions {
 export interface GetAllFilesInS3Options {
   s3Folder: string;
   localDirectory: string;
-  extension?: string;
+  extension?: string | string[];
   maxFiles?: number;
 }
 
@@ -123,8 +123,9 @@ export class PpaasS3File implements S3File {
   }
 
   public static async getAllFilesInS3 ({ s3Folder, localDirectory, extension, maxFiles }: GetAllFilesInS3Options): Promise<PpaasS3File[]> {
-    log(`Finding in s3Folder: ${s3Folder}, extension: ${extension}, maxFiles: ${maxFiles}`, LogLevel.DEBUG);
+    log(`PpaasS3File Finding in s3Folder: ${s3Folder}, extension: ${extension}, maxFiles: ${maxFiles}`, LogLevel.DEBUG);
     const s3Files: S3Object[] = await listFiles({ s3Folder, maxKeys: maxFiles, extension });
+    log(`Found S3Files: ${s3Folder}, extension: ${extension}, maxFiles: ${maxFiles}`, LogLevel.DEBUG, s3Files.map((s3File) => s3File.Key));
     if (s3Files.length === 0) {
       return [];
     }

common/src/util/s3.ts

@@ -146,15 +146,15 @@ export interface FileOptions {
 export interface ListFilesOptions {
   s3Folder: string;
   maxKeys?: number;
-  extension?: string;
+  extension?: string | string[];
 }
 
 export async function listFiles ({ s3Folder, extension, maxKeys }: ListFilesOptions): Promise<S3Object[]>;
 export async function listFiles (s3Folder: string): Promise<S3Object[]>;
 export async function listFiles (options: string | ListFilesOptions): Promise<S3Object[]> {
   let s3Folder: string;
   let maxKeys: number | undefined;
-  let extension: string | undefined;
+  let extension: string | string[] | undefined;
   if (typeof options === "string") {
     s3Folder = options;
   } else {
@@ -166,8 +166,12 @@ export async function listFiles (options: string | ListFilesOptions): Promise<S3
   do {
     result = await listObjects({ prefix: s3Folder, maxKeys, continuationToken: result && result.NextContinuationToken});
     if (result.Contents) {
-      if (extension && result.Contents.length > 0) {
-        const filtered: S3Object[] = result.Contents.filter((s3File: S3Object) => s3File.Key!.endsWith(extension!));
+      if (extension && extension.length > 0 && result.Contents.length > 0) {
+        const filtered: S3Object[] = result.Contents.filter((s3File: S3Object) => Array.isArray(extension)
+          ? extension.findIndex((thisExtension: string) => s3File.Key!.endsWith(thisExtension)) >= 0
+          : s3File.Key!.endsWith(extension!)
+        );
+        log(`listFiles(${s3Folder}, ${maxKeys}, ${extension}) results`, LogLevel.DEBUG, { results: result.Contents.length, filtered: filtered.length });
         files.push(...filtered);
       } else {
         files.push(...(result.Contents));

common/test/s3.spec.ts

@@ -166,7 +166,7 @@ describe("S3Util", () => {
         s3Folder: UNIT_TEST_KEY_PREFIX,
         extension: UNIT_TEST_FILENAME.slice(-3)
       }).then((result: S3Object[]) => {
-        log(`listFiles("${UNIT_TEST_KEY_PREFIX}", undefined, ${UNIT_TEST_FILENAME.slice(-3)}) result = ${JSON.stringify(result)}`, LogLevel.DEBUG);
+        log(`listFiles("${UNIT_TEST_KEY_PREFIX}", undefined, "${UNIT_TEST_FILENAME.slice(-3)}") result = ${JSON.stringify(result)}`, LogLevel.DEBUG);
         expect(result).to.not.equal(undefined);
         expect(result.length).to.equal(1);
         done();
@@ -189,6 +189,51 @@ describe("S3Util", () => {
         done(error);
       });
     });
+
+    it("List Files with extension array first should return files", (done: Mocha.Done) => {
+      mockListObjects([s3TestObject]);
+      listFiles({
+        s3Folder: UNIT_TEST_KEY_PREFIX,
+        extension: [UNIT_TEST_FILENAME.slice(-3), "bogus"]
+      }).then((result: S3Object[]) => {
+        log(`listFiles("${UNIT_TEST_KEY_PREFIX}", undefined, ["${UNIT_TEST_FILENAME.slice(-3)}", "bogus"]) result = ${JSON.stringify(result)}`, LogLevel.DEBUG);
+        expect(result).to.not.equal(undefined);
+        expect(result.length).to.equal(1);
+        done();
+      }).catch((error) => {
+        done(error);
+      });
+    });
+
+    it("List Files with extension array second should return files", (done: Mocha.Done) => {
+      mockListObjects([s3TestObject]);
+      listFiles({
+        s3Folder: UNIT_TEST_KEY_PREFIX,
+        extension: ["bogus", UNIT_TEST_FILENAME.slice(-3)]
+      }).then((result: S3Object[]) => {
+        log(`listFiles("${UNIT_TEST_KEY_PREFIX}", undefined, ["bogus", "${UNIT_TEST_FILENAME.slice(-3)}"]) result = ${JSON.stringify(result)}`, LogLevel.DEBUG);
+        expect(result).to.not.equal(undefined);
+        expect(result.length).to.equal(1);
+        done();
+      }).catch((error) => {
+        done(error);
+      });
+    });
+
+    it("List Files with not found extension array should not return files", (done: Mocha.Done) => {
+      mockListObjects([s3TestObject]);
+      listFiles({
+        s3Folder: UNIT_TEST_KEY_PREFIX,
+        extension: ["bad", "bogus"]
+      }).then((result: S3Object[]) => {
+        log(`listFiles("${UNIT_TEST_KEY_PREFIX}", undefined, ["bad", "bogus"]) result = ${JSON.stringify(result)}`, LogLevel.DEBUG);
+        expect(result).to.not.equal(undefined);
+        expect(result.length).to.equal(0);
+        done();
+      }).catch((error) => {
+        done(error);
+      });
+    });
   });
 
   describe("Upload Object to S3", () => {

common/test/s3file.spec.ts

@@ -391,7 +391,7 @@ describe("PpaasS3File", () => {
         localDirectory: UNIT_TEST_LOCAL_FILE_LOCATION,
         extension: UNIT_TEST_FILENAME.slice(-3)
       }).then((result: PpaasS3File[]) => {
-        log(`PpaasS3File.getAllFilesInS3("${unitTestKeyPrefix}", "${UNIT_TEST_LOCAL_FILE_LOCATION}") result = ${JSON.stringify(result)}`, LogLevel.DEBUG);
+        log(`PpaasS3File.getAllFilesInS3("${unitTestKeyPrefix}", "${UNIT_TEST_LOCAL_FILE_LOCATION}", "${UNIT_TEST_FILENAME.slice(-3)}") result = ${JSON.stringify(result)}`, LogLevel.DEBUG);
         expect(result).to.not.equal(undefined);
         expect(result.length).to.equal(1);
         // getAllFilesInS3 should set the remote date so we can sort
@@ -414,7 +414,70 @@ describe("PpaasS3File", () => {
         extension: "bad",
         maxFiles: 1000
       }).then((result: PpaasS3File[]) => {
-        log(`PpaasS3File.getAllFilesInS3("${unitTestKeyPrefix}", "${UNIT_TEST_LOCAL_FILE_LOCATION}", 1000) result = ${JSON.stringify(result)}`, LogLevel.DEBUG);
+        log(`PpaasS3File.getAllFilesInS3("${unitTestKeyPrefix}", "${UNIT_TEST_LOCAL_FILE_LOCATION}", "bad", 1000) result = ${JSON.stringify(result)}`, LogLevel.DEBUG);
+        expect(result).to.not.equal(undefined);
+        expect(result.length).to.equal(0);
+        done();
+      }).catch((error) => {
+        done(error);
+      });
+    });
+
+    it("getAllFilesInS3 partial folder by extension array first should return files", (done: Mocha.Done) => {
+      mockListObject(UNIT_TEST_FILENAME, unitTestKeyPrefix);
+      mockGetObjectTagging(tags);
+      PpaasS3File.getAllFilesInS3({
+        s3Folder: unitTestKeyPrefix.slice(0, -2),
+        localDirectory: UNIT_TEST_LOCAL_FILE_LOCATION,
+        extension: [UNIT_TEST_FILENAME.slice(-3), "bogus"]
+      }).then((result: PpaasS3File[]) => {
+        log(`PpaasS3File.getAllFilesInS3("${unitTestKeyPrefix}", "${UNIT_TEST_LOCAL_FILE_LOCATION}", ["${UNIT_TEST_FILENAME.slice(-3)}", "bogus"]) result = ${JSON.stringify(result)}`, LogLevel.DEBUG);
+        expect(result).to.not.equal(undefined);
+        expect(result.length).to.equal(1);
+        // getAllFilesInS3 should set the remote date so we can sort
+        expect(result[0].getLastModifiedRemote()).to.be.greaterThan(new Date(0));
+        expect(result[0].tags).to.not.equal(undefined);
+        expect(result[0].tags?.size).to.equal(1);
+        expect(result[0].tags?.has("test")).to.equal(false);
+        expect(result[0].tags?.get("unittest")).to.equal("true");
+        done();
+      }).catch((error) => {
+        done(error);
+      });
+    });
+
+    it("getAllFilesInS3 partial folder by extension array second should return files", (done: Mocha.Done) => {
+      mockListObject(UNIT_TEST_FILENAME, unitTestKeyPrefix);
+      mockGetObjectTagging(tags);
+      PpaasS3File.getAllFilesInS3({
+        s3Folder: unitTestKeyPrefix.slice(0, -2),
+        localDirectory: UNIT_TEST_LOCAL_FILE_LOCATION,
+        extension: ["bogus", UNIT_TEST_FILENAME.slice(-3)]
+      }).then((result: PpaasS3File[]) => {
+        log(`PpaasS3File.getAllFilesInS3("${unitTestKeyPrefix}", "${UNIT_TEST_LOCAL_FILE_LOCATION}", ["bogus", "${UNIT_TEST_FILENAME.slice(-3)}"]) result = ${JSON.stringify(result)}`, LogLevel.DEBUG);
+        expect(result).to.not.equal(undefined);
+        expect(result.length).to.equal(1);
+        // getAllFilesInS3 should set the remote date so we can sort
+        expect(result[0].getLastModifiedRemote()).to.be.greaterThan(new Date(0));
+        expect(result[0].tags).to.not.equal(undefined);
+        expect(result[0].tags?.size).to.equal(1);
+        expect(result[0].tags?.has("test")).to.equal(false);
+        expect(result[0].tags?.get("unittest")).to.equal("true");
+        done();
+      }).catch((error) => {
+        done(error);
+      });
+    });
+
+    it("getAllFilesInS3 partial folder wrong extension array should not return files", (done: Mocha.Done) => {
+      mockListObjects([]);
+      PpaasS3File.getAllFilesInS3({
+        s3Folder: unitTestKeyPrefix.slice(0, -2),
+        localDirectory: UNIT_TEST_LOCAL_FILE_LOCATION,
+        extension: ["bad", "bogus"],
+        maxFiles: 1000
+      }).then((result: PpaasS3File[]) => {
+        log(`PpaasS3File.getAllFilesInS3("${unitTestKeyPrefix}", "${UNIT_TEST_LOCAL_FILE_LOCATION}", ["bad", "bogus"], 1000) result = ${JSON.stringify(result)}`, LogLevel.DEBUG);
         expect(result).to.not.equal(undefined);
         expect(result.length).to.equal(0);
         done();

controller/components/Layout/index.tsx

@@ -105,7 +105,7 @@ export const Layout = ({
     authPermission,
     otherControllers = OTHER_CONTROLLERS
   }: LayoutProps) => {
-    // There seems to be a bug when usling our LinkButton that when the button goes
+    // There seems to be a bug when using our LinkButton that when the button goes
     // to "/" it removes the "/" from the url and refresh breaks if there is a query param
     // Ctrl-click keeps the trailing slash, but the click routing removes it.
     // Two possible fixes. 1) Fix the routing in nginx. I don't think we can fix the routing by

controller/components/LinkButton/index.tsx

@@ -42,7 +42,9 @@ export const LinkButton = ({
     <React.Fragment>
       {/* https://nextjs.org/docs/messages/invalid-new-link-with-extra-anchor */}
       <Link href={href} as={formatPageHref(href)} title={title} legacyBehavior>
-        <a href={formatPageHref(href)} title={title}><Button name={name} theme={{...defaultButtonTheme, ...theme}} onClick={onClick}>{children}</Button></a>
+        <a href={formatPageHref(href)} title={title}>
+          <Button name={name} theme={{...defaultButtonTheme, ...theme}} onClick={onClick}>{children}</Button>
+        </a>
       </Link>
     </React.Fragment>
   );

controller/pages/api/search.ts

@@ -17,7 +17,7 @@ export default async (req: NextApiRequest, res: NextApiResponse): Promise<void>
       return;
     }
     try {
-      const testManagerResponse: TestManagerResponse = await TestManager.searchTests(req.query.s3Folder, req.query.maxResults);
+      const testManagerResponse: TestManagerResponse = await TestManager.searchTests(req.query.s3Folder, req.query.maxResults, req.query.extension);
       res.status(testManagerResponse.status).json(testManagerResponse.json);
     } catch (error) {
       // If we get here it's a 500. All the "bad requests" are handled above

controller/pages/api/util/ppaasencrypts3file.ts

@@ -63,7 +63,7 @@ export class PpaasEncryptS3File implements s3.S3File {
   }
 
   public static async getAllFilesInS3 (s3Folder: string, extension?: string, maxFiles?: number): Promise<PpaasEncryptS3File[]> {
-    log(`Finding in s3Folder: ${s3Folder}, extension: ${extension}, maxFiles: ${maxFiles}`, LogLevel.DEBUG);
+    log(`PpaasEncryptS3File Finding in s3Folder: ${s3Folder}, extension: ${extension}, maxFiles: ${maxFiles}`, LogLevel.DEBUG);
     const s3Files: S3Object[] = await listFiles({ s3Folder, maxKeys: maxFiles, extension });
     if (s3Files.length === 0) {
       return [];

controller/pages/api/util/testmanager.ts

@@ -1614,7 +1614,11 @@ export abstract class TestManager {
   }
 
   // define the put stop route to put a message on the queue to stop a test
-  public static async searchTests (s3FolderQuery: string | string[] | undefined, maxResultsQuery: string | string[] | undefined): Promise<ErrorResponse | TestListResponse> {
+  public static async searchTests (
+    s3FolderQuery: string | string[] | undefined,
+    maxResultsQuery: string | string[] | undefined,
+    extension: string | string[] = [".yaml", ".yml"]
+  ): Promise<ErrorResponse | TestListResponse> {
     try {
       // Check if either one is empty
       if (s3FolderQuery === undefined) {
@@ -1647,13 +1651,13 @@ export abstract class TestManager {
         const s3YamlFiles: PpaasS3File[] = await PpaasS3File.getAllFilesInS3({
           s3Folder: s3FolderPartial,
           localDirectory,
-          extension: "yaml",
+          extension,
           maxFiles: 1000
         });
         if (s3YamlFiles.length === 0) {
           return { json: [], status: 204 };
         }
-        log(`Found files for s3FolderSearch: ${s3FolderPartial}`, LogLevel.DEBUG, s3YamlFiles);
+        log(`Found files for s3FolderSearch: ${s3FolderPartial}`, LogLevel.DEBUG, s3YamlFiles.map((s3YamlFile) => s3YamlFile.key));
 
         let tests: TestData[] = [];
         for (const s3YamlFile of s3YamlFiles) {

controller/pages/index.tsx

@@ -76,7 +76,8 @@ export interface TestStatusState {
   error: string | undefined;
 }
 
-const noTestsFoundEror = (searchString: string) => "No s3Folders found starting with: " + searchString;
+const noTestsFoundEror = (searchString: string, searchExtension?: string | string[]) =>
+  `No s3Folders found starting with: "${searchString}"` + (searchExtension ? ` and extension ${JSON.stringify(searchExtension)}` : "");
 
 const TestStatusPage = ({
   testData,
@@ -117,7 +118,7 @@ const TestStatusPage = ({
       }
       setState({ error: undefined }); // Force a redraw
     } catch (error) {
-      log("Error loading test data", LogLevel.ERROR, error);
+      log("Error loading test data", LogLevel.WARN, error);
       setState({ error: formatError(error) });
     }
   };
@@ -211,7 +212,7 @@ const TestStatusPage = ({
         });
         return;
       }
-      // PUT /api/search
+      // PUT /api/search - Don't include the extension here. Only on page loads
       const url = formatPageHref(`${API_SEARCH}?s3Folder=${searchString}`);
       const response: AxiosResponse = await axios.get(url);
       // Update the URL to include the search param `?search=${searchString}`
@@ -273,9 +274,10 @@ const TestStatusPage = ({
 
 export const getServerSideProps: GetServerSideProps =
   async (ctx: GetServerSidePropsContext): Promise<GetServerSidePropsResult<TestStatusProps>> => {
+  let authPermissions: AuthPermissions | string | undefined;
   try {
     // Authenticate
-    const authPermissions: AuthPermissions | string = await authPage(ctx, AuthPermission.ReadOnly);
+    authPermissions = await authPage(ctx, AuthPermission.ReadOnly);
     // If we have a authPermissions we're authorized, if we're not, we'l redirect
     if (typeof authPermissions === "string") {
       return {
@@ -354,9 +356,10 @@ export const getServerSideProps: GetServerSideProps =
       let searchTestResult: TestData[] | undefined;
       let errorLoading: string | undefined;
       const searchString = typeof ctx.query?.search === "string" ? ctx.query.search : undefined;
-      if (searchString) {
+      const searchExtension = ctx.query.extension;
+      if (searchString || searchExtension) {
         // Check for search param and do search
-        const testManagerResponse: ErrorResponse | TestListResponse = await TestManager.searchTests(searchString, ctx.query.maxResults);
+        const testManagerResponse: ErrorResponse | TestListResponse = await TestManager.searchTests(searchString, ctx.query.maxResults, searchExtension);
         if ("message" in testManagerResponse.json) {
           return {
             props: {
@@ -371,7 +374,7 @@ export const getServerSideProps: GetServerSideProps =
         searchTestResult = testManagerResponse.json;
         if (searchTestResult.length === 0) {
           searchTestResult = undefined;
-          errorLoading = noTestsFoundEror(searchString);
+          errorLoading = noTestsFoundEror(searchString || "", searchExtension);
         }
       }
       return {
@@ -387,7 +390,10 @@ export const getServerSideProps: GetServerSideProps =
     }
   } catch (error) {
     const errorLoading = formatError(error);
-    logServer("Error loading test data", LogLevelServer.ERROR, error);
+    logServer(
+      "TestStatusPage Error loading test data", LogLevelServer.WARN, error,
+      typeof authPermissions === "string" ? authPermissions : authPermissions?.userId
+    );
     return {
       props: { testData: undefined, allTests: undefined, errorLoading, authPermission: undefined }
     };