diff --git a/common/src/s3file.ts b/common/src/s3file.ts index eaea68e7..7702f76e 100644 --- a/common/src/s3file.ts +++ b/common/src/s3file.ts @@ -34,7 +34,7 @@ export interface PpaasS3FileOptions { export interface GetAllFilesInS3Options { s3Folder: string; localDirectory: string; - extension?: string; + extension?: string | string[]; maxFiles?: number; } @@ -123,8 +123,9 @@ export class PpaasS3File implements S3File { } public static async getAllFilesInS3 ({ s3Folder, localDirectory, extension, maxFiles }: GetAllFilesInS3Options): Promise { - log(`Finding in s3Folder: ${s3Folder}, extension: ${extension}, maxFiles: ${maxFiles}`, LogLevel.DEBUG); + log(`PpaasS3File Finding in s3Folder: ${s3Folder}, extension: ${extension}, maxFiles: ${maxFiles}`, LogLevel.DEBUG); const s3Files: S3Object[] = await listFiles({ s3Folder, maxKeys: maxFiles, extension }); + log(`Found S3Files: ${s3Folder}, extension: ${extension}, maxFiles: ${maxFiles}`, LogLevel.DEBUG, s3Files.map((s3File) => s3File.Key)); if (s3Files.length === 0) { return []; } diff --git a/common/src/util/s3.ts b/common/src/util/s3.ts index b0ce8b8a..8373ce2c 100644 --- a/common/src/util/s3.ts +++ b/common/src/util/s3.ts @@ -146,7 +146,7 @@ export interface FileOptions { export interface ListFilesOptions { s3Folder: string; maxKeys?: number; - extension?: string; + extension?: string | string[]; } export async function listFiles ({ s3Folder, extension, maxKeys }: ListFilesOptions): Promise; @@ -154,7 +154,7 @@ export async function listFiles (s3Folder: string): Promise; export async function listFiles (options: string | ListFilesOptions): Promise { let s3Folder: string; let maxKeys: number | undefined; - let extension: string | undefined; + let extension: string | string[] | undefined; if (typeof options === "string") { s3Folder = options; } else { @@ -166,8 +166,12 @@ export async function listFiles (options: string | ListFilesOptions): Promise 0) { - const filtered: S3Object[] = result.Contents.filter((s3File: S3Object) => s3File.Key!.endsWith(extension!)); + if (extension && extension.length > 0 && result.Contents.length > 0) { + const filtered: S3Object[] = result.Contents.filter((s3File: S3Object) => Array.isArray(extension) + ? extension.findIndex((thisExtension: string) => s3File.Key!.endsWith(thisExtension)) >= 0 + : s3File.Key!.endsWith(extension!) + ); + log(`listFiles(${s3Folder}, ${maxKeys}, ${extension}) results`, LogLevel.DEBUG, { results: result.Contents.length, filtered: filtered.length }); files.push(...filtered); } else { files.push(...(result.Contents)); diff --git a/common/test/s3.spec.ts b/common/test/s3.spec.ts index 661a216d..eb9be6c2 100644 --- a/common/test/s3.spec.ts +++ b/common/test/s3.spec.ts @@ -166,7 +166,7 @@ describe("S3Util", () => { s3Folder: UNIT_TEST_KEY_PREFIX, extension: UNIT_TEST_FILENAME.slice(-3) }).then((result: S3Object[]) => { - log(`listFiles("${UNIT_TEST_KEY_PREFIX}", undefined, ${UNIT_TEST_FILENAME.slice(-3)}) result = ${JSON.stringify(result)}`, LogLevel.DEBUG); + log(`listFiles("${UNIT_TEST_KEY_PREFIX}", undefined, "${UNIT_TEST_FILENAME.slice(-3)}") result = ${JSON.stringify(result)}`, LogLevel.DEBUG); expect(result).to.not.equal(undefined); expect(result.length).to.equal(1); done(); @@ -189,6 +189,51 @@ describe("S3Util", () => { done(error); }); }); + + it("List Files with extension array first should return files", (done: Mocha.Done) => { + mockListObjects([s3TestObject]); + listFiles({ + s3Folder: UNIT_TEST_KEY_PREFIX, + extension: [UNIT_TEST_FILENAME.slice(-3), "bogus"] + }).then((result: S3Object[]) => { + log(`listFiles("${UNIT_TEST_KEY_PREFIX}", undefined, ["${UNIT_TEST_FILENAME.slice(-3)}", "bogus"]) result = ${JSON.stringify(result)}`, LogLevel.DEBUG); + expect(result).to.not.equal(undefined); + expect(result.length).to.equal(1); + done(); + }).catch((error) => { + done(error); + }); + }); + + it("List Files with extension array second should return files", (done: Mocha.Done) => { + mockListObjects([s3TestObject]); + listFiles({ + s3Folder: UNIT_TEST_KEY_PREFIX, + extension: ["bogus", UNIT_TEST_FILENAME.slice(-3)] + }).then((result: S3Object[]) => { + log(`listFiles("${UNIT_TEST_KEY_PREFIX}", undefined, ["bogus", "${UNIT_TEST_FILENAME.slice(-3)}"]) result = ${JSON.stringify(result)}`, LogLevel.DEBUG); + expect(result).to.not.equal(undefined); + expect(result.length).to.equal(1); + done(); + }).catch((error) => { + done(error); + }); + }); + + it("List Files with not found extension array should not return files", (done: Mocha.Done) => { + mockListObjects([s3TestObject]); + listFiles({ + s3Folder: UNIT_TEST_KEY_PREFIX, + extension: ["bad", "bogus"] + }).then((result: S3Object[]) => { + log(`listFiles("${UNIT_TEST_KEY_PREFIX}", undefined, ["bad", "bogus"]) result = ${JSON.stringify(result)}`, LogLevel.DEBUG); + expect(result).to.not.equal(undefined); + expect(result.length).to.equal(0); + done(); + }).catch((error) => { + done(error); + }); + }); }); describe("Upload Object to S3", () => { diff --git a/common/test/s3file.spec.ts b/common/test/s3file.spec.ts index 12ac4a6c..d0a91b4a 100644 --- a/common/test/s3file.spec.ts +++ b/common/test/s3file.spec.ts @@ -391,7 +391,7 @@ describe("PpaasS3File", () => { localDirectory: UNIT_TEST_LOCAL_FILE_LOCATION, extension: UNIT_TEST_FILENAME.slice(-3) }).then((result: PpaasS3File[]) => { - log(`PpaasS3File.getAllFilesInS3("${unitTestKeyPrefix}", "${UNIT_TEST_LOCAL_FILE_LOCATION}") result = ${JSON.stringify(result)}`, LogLevel.DEBUG); + log(`PpaasS3File.getAllFilesInS3("${unitTestKeyPrefix}", "${UNIT_TEST_LOCAL_FILE_LOCATION}", "${UNIT_TEST_FILENAME.slice(-3)}") result = ${JSON.stringify(result)}`, LogLevel.DEBUG); expect(result).to.not.equal(undefined); expect(result.length).to.equal(1); // getAllFilesInS3 should set the remote date so we can sort @@ -414,7 +414,70 @@ describe("PpaasS3File", () => { extension: "bad", maxFiles: 1000 }).then((result: PpaasS3File[]) => { - log(`PpaasS3File.getAllFilesInS3("${unitTestKeyPrefix}", "${UNIT_TEST_LOCAL_FILE_LOCATION}", 1000) result = ${JSON.stringify(result)}`, LogLevel.DEBUG); + log(`PpaasS3File.getAllFilesInS3("${unitTestKeyPrefix}", "${UNIT_TEST_LOCAL_FILE_LOCATION}", "bad", 1000) result = ${JSON.stringify(result)}`, LogLevel.DEBUG); + expect(result).to.not.equal(undefined); + expect(result.length).to.equal(0); + done(); + }).catch((error) => { + done(error); + }); + }); + + it("getAllFilesInS3 partial folder by extension array first should return files", (done: Mocha.Done) => { + mockListObject(UNIT_TEST_FILENAME, unitTestKeyPrefix); + mockGetObjectTagging(tags); + PpaasS3File.getAllFilesInS3({ + s3Folder: unitTestKeyPrefix.slice(0, -2), + localDirectory: UNIT_TEST_LOCAL_FILE_LOCATION, + extension: [UNIT_TEST_FILENAME.slice(-3), "bogus"] + }).then((result: PpaasS3File[]) => { + log(`PpaasS3File.getAllFilesInS3("${unitTestKeyPrefix}", "${UNIT_TEST_LOCAL_FILE_LOCATION}", ["${UNIT_TEST_FILENAME.slice(-3)}", "bogus"]) result = ${JSON.stringify(result)}`, LogLevel.DEBUG); + expect(result).to.not.equal(undefined); + expect(result.length).to.equal(1); + // getAllFilesInS3 should set the remote date so we can sort + expect(result[0].getLastModifiedRemote()).to.be.greaterThan(new Date(0)); + expect(result[0].tags).to.not.equal(undefined); + expect(result[0].tags?.size).to.equal(1); + expect(result[0].tags?.has("test")).to.equal(false); + expect(result[0].tags?.get("unittest")).to.equal("true"); + done(); + }).catch((error) => { + done(error); + }); + }); + + it("getAllFilesInS3 partial folder by extension array second should return files", (done: Mocha.Done) => { + mockListObject(UNIT_TEST_FILENAME, unitTestKeyPrefix); + mockGetObjectTagging(tags); + PpaasS3File.getAllFilesInS3({ + s3Folder: unitTestKeyPrefix.slice(0, -2), + localDirectory: UNIT_TEST_LOCAL_FILE_LOCATION, + extension: ["bogus", UNIT_TEST_FILENAME.slice(-3)] + }).then((result: PpaasS3File[]) => { + log(`PpaasS3File.getAllFilesInS3("${unitTestKeyPrefix}", "${UNIT_TEST_LOCAL_FILE_LOCATION}", ["bogus", "${UNIT_TEST_FILENAME.slice(-3)}"]) result = ${JSON.stringify(result)}`, LogLevel.DEBUG); + expect(result).to.not.equal(undefined); + expect(result.length).to.equal(1); + // getAllFilesInS3 should set the remote date so we can sort + expect(result[0].getLastModifiedRemote()).to.be.greaterThan(new Date(0)); + expect(result[0].tags).to.not.equal(undefined); + expect(result[0].tags?.size).to.equal(1); + expect(result[0].tags?.has("test")).to.equal(false); + expect(result[0].tags?.get("unittest")).to.equal("true"); + done(); + }).catch((error) => { + done(error); + }); + }); + + it("getAllFilesInS3 partial folder wrong extension array should not return files", (done: Mocha.Done) => { + mockListObjects([]); + PpaasS3File.getAllFilesInS3({ + s3Folder: unitTestKeyPrefix.slice(0, -2), + localDirectory: UNIT_TEST_LOCAL_FILE_LOCATION, + extension: ["bad", "bogus"], + maxFiles: 1000 + }).then((result: PpaasS3File[]) => { + log(`PpaasS3File.getAllFilesInS3("${unitTestKeyPrefix}", "${UNIT_TEST_LOCAL_FILE_LOCATION}", ["bad", "bogus"], 1000) result = ${JSON.stringify(result)}`, LogLevel.DEBUG); expect(result).to.not.equal(undefined); expect(result.length).to.equal(0); done(); diff --git a/controller/components/Layout/index.tsx b/controller/components/Layout/index.tsx index ab3dab0d..de830eb5 100644 --- a/controller/components/Layout/index.tsx +++ b/controller/components/Layout/index.tsx @@ -105,7 +105,7 @@ export const Layout = ({ authPermission, otherControllers = OTHER_CONTROLLERS }: LayoutProps) => { - // There seems to be a bug when usling our LinkButton that when the button goes + // There seems to be a bug when using our LinkButton that when the button goes // to "/" it removes the "/" from the url and refresh breaks if there is a query param // Ctrl-click keeps the trailing slash, but the click routing removes it. // Two possible fixes. 1) Fix the routing in nginx. I don't think we can fix the routing by diff --git a/controller/components/LinkButton/index.tsx b/controller/components/LinkButton/index.tsx index f600605a..9d2b0381 100644 --- a/controller/components/LinkButton/index.tsx +++ b/controller/components/LinkButton/index.tsx @@ -42,7 +42,9 @@ export const LinkButton = ({ {/* https://nextjs.org/docs/messages/invalid-new-link-with-extra-anchor */} - + + + ); diff --git a/controller/pages/api/search.ts b/controller/pages/api/search.ts index 3963bf54..45bcd79f 100644 --- a/controller/pages/api/search.ts +++ b/controller/pages/api/search.ts @@ -17,7 +17,7 @@ export default async (req: NextApiRequest, res: NextApiResponse): Promise return; } try { - const testManagerResponse: TestManagerResponse = await TestManager.searchTests(req.query.s3Folder, req.query.maxResults); + const testManagerResponse: TestManagerResponse = await TestManager.searchTests(req.query.s3Folder, req.query.maxResults, req.query.extension); res.status(testManagerResponse.status).json(testManagerResponse.json); } catch (error) { // If we get here it's a 500. All the "bad requests" are handled above diff --git a/controller/pages/api/util/ppaasencrypts3file.ts b/controller/pages/api/util/ppaasencrypts3file.ts index 8996c7f8..1edc6f15 100644 --- a/controller/pages/api/util/ppaasencrypts3file.ts +++ b/controller/pages/api/util/ppaasencrypts3file.ts @@ -63,7 +63,7 @@ export class PpaasEncryptS3File implements s3.S3File { } public static async getAllFilesInS3 (s3Folder: string, extension?: string, maxFiles?: number): Promise { - log(`Finding in s3Folder: ${s3Folder}, extension: ${extension}, maxFiles: ${maxFiles}`, LogLevel.DEBUG); + log(`PpaasEncryptS3File Finding in s3Folder: ${s3Folder}, extension: ${extension}, maxFiles: ${maxFiles}`, LogLevel.DEBUG); const s3Files: S3Object[] = await listFiles({ s3Folder, maxKeys: maxFiles, extension }); if (s3Files.length === 0) { return []; diff --git a/controller/pages/api/util/testmanager.ts b/controller/pages/api/util/testmanager.ts index 260d595c..24af92c9 100644 --- a/controller/pages/api/util/testmanager.ts +++ b/controller/pages/api/util/testmanager.ts @@ -1614,7 +1614,11 @@ export abstract class TestManager { } // define the put stop route to put a message on the queue to stop a test - public static async searchTests (s3FolderQuery: string | string[] | undefined, maxResultsQuery: string | string[] | undefined): Promise { + public static async searchTests ( + s3FolderQuery: string | string[] | undefined, + maxResultsQuery: string | string[] | undefined, + extension: string | string[] = [".yaml", ".yml"] + ): Promise { try { // Check if either one is empty if (s3FolderQuery === undefined) { @@ -1647,13 +1651,13 @@ export abstract class TestManager { const s3YamlFiles: PpaasS3File[] = await PpaasS3File.getAllFilesInS3({ s3Folder: s3FolderPartial, localDirectory, - extension: "yaml", + extension, maxFiles: 1000 }); if (s3YamlFiles.length === 0) { return { json: [], status: 204 }; } - log(`Found files for s3FolderSearch: ${s3FolderPartial}`, LogLevel.DEBUG, s3YamlFiles); + log(`Found files for s3FolderSearch: ${s3FolderPartial}`, LogLevel.DEBUG, s3YamlFiles.map((s3YamlFile) => s3YamlFile.key)); let tests: TestData[] = []; for (const s3YamlFile of s3YamlFiles) { diff --git a/controller/pages/index.tsx b/controller/pages/index.tsx index 2ca1585e..43b5788a 100644 --- a/controller/pages/index.tsx +++ b/controller/pages/index.tsx @@ -76,7 +76,8 @@ export interface TestStatusState { error: string | undefined; } -const noTestsFoundEror = (searchString: string) => "No s3Folders found starting with: " + searchString; +const noTestsFoundEror = (searchString: string, searchExtension?: string | string[]) => + `No s3Folders found starting with: "${searchString}"` + (searchExtension ? ` and extension ${JSON.stringify(searchExtension)}` : ""); const TestStatusPage = ({ testData, @@ -117,7 +118,7 @@ const TestStatusPage = ({ } setState({ error: undefined }); // Force a redraw } catch (error) { - log("Error loading test data", LogLevel.ERROR, error); + log("Error loading test data", LogLevel.WARN, error); setState({ error: formatError(error) }); } }; @@ -211,7 +212,7 @@ const TestStatusPage = ({ }); return; } - // PUT /api/search + // PUT /api/search - Don't include the extension here. Only on page loads const url = formatPageHref(`${API_SEARCH}?s3Folder=${searchString}`); const response: AxiosResponse = await axios.get(url); // Update the URL to include the search param `?search=${searchString}` @@ -273,9 +274,10 @@ const TestStatusPage = ({ export const getServerSideProps: GetServerSideProps = async (ctx: GetServerSidePropsContext): Promise> => { + let authPermissions: AuthPermissions | string | undefined; try { // Authenticate - const authPermissions: AuthPermissions | string = await authPage(ctx, AuthPermission.ReadOnly); + authPermissions = await authPage(ctx, AuthPermission.ReadOnly); // If we have a authPermissions we're authorized, if we're not, we'l redirect if (typeof authPermissions === "string") { return { @@ -354,9 +356,10 @@ export const getServerSideProps: GetServerSideProps = let searchTestResult: TestData[] | undefined; let errorLoading: string | undefined; const searchString = typeof ctx.query?.search === "string" ? ctx.query.search : undefined; - if (searchString) { + const searchExtension = ctx.query.extension; + if (searchString || searchExtension) { // Check for search param and do search - const testManagerResponse: ErrorResponse | TestListResponse = await TestManager.searchTests(searchString, ctx.query.maxResults); + const testManagerResponse: ErrorResponse | TestListResponse = await TestManager.searchTests(searchString, ctx.query.maxResults, searchExtension); if ("message" in testManagerResponse.json) { return { props: { @@ -371,7 +374,7 @@ export const getServerSideProps: GetServerSideProps = searchTestResult = testManagerResponse.json; if (searchTestResult.length === 0) { searchTestResult = undefined; - errorLoading = noTestsFoundEror(searchString); + errorLoading = noTestsFoundEror(searchString || "", searchExtension); } } return { @@ -387,7 +390,10 @@ export const getServerSideProps: GetServerSideProps = } } catch (error) { const errorLoading = formatError(error); - logServer("Error loading test data", LogLevelServer.ERROR, error); + logServer( + "TestStatusPage Error loading test data", LogLevelServer.WARN, error, + typeof authPermissions === "string" ? authPermissions : authPermissions?.userId + ); return { props: { testData: undefined, allTests: undefined, errorLoading, authPermission: undefined } };