Skip to content

Commit

Permalink
Fixed an issue where searching for old tests that were yml failed to …
Browse files Browse the repository at this point in the history 
…find

- Changed the s3 listFiles by extension and PPaasS3File getAllFilesInS3 to allow an extension array
- Changed the TestManager searchTests to search for both .yaml and .yml files when searching for old tests
  • Loading branch information
@tkmcmaster
tkmcmaster committed Dec 12, 2023
1 parent fe11763 commit 199f635
Show file tree
Hide file tree
Showing 6 changed files with 123 additions and 10 deletions.
5 changes: 3 additions & 2 deletions common/src/s3file.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ export interface PpaasS3FileOptions {
export interface GetAllFilesInS3Options {
s3Folder: string;
localDirectory: string;
extension?: string;
extension?: string | string[];
maxFiles?: number;
}

Expand Down Expand Up @@ -123,8 +123,9 @@ export class PpaasS3File implements S3File {
}

public static async getAllFilesInS3 ({ s3Folder, localDirectory, extension, maxFiles }: GetAllFilesInS3Options): Promise<PpaasS3File[]> {
log(`Finding in s3Folder: ${s3Folder}, extension: ${extension}, maxFiles: ${maxFiles}`, LogLevel.DEBUG);
log(`PpaasS3File Finding in s3Folder: ${s3Folder}, extension: ${extension}, maxFiles: ${maxFiles}`, LogLevel.DEBUG);
const s3Files: S3Object[] = await listFiles({ s3Folder, maxKeys: maxFiles, extension });
log(`Found S3Files: ${s3Folder}, extension: ${extension}, maxFiles: ${maxFiles}`, LogLevel.DEBUG, s3Files.map((s3File) => s3File.Key));
if (s3Files.length === 0) {
return [];
}
Expand Down
10 changes: 7 additions & 3 deletions common/src/util/s3.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -146,15 +146,15 @@ export interface FileOptions {
export interface ListFilesOptions {
s3Folder: string;
maxKeys?: number;
extension?: string;
extension?: string | string[];
}

export async function listFiles ({ s3Folder, extension, maxKeys }: ListFilesOptions): Promise<S3Object[]>;
export async function listFiles (s3Folder: string): Promise<S3Object[]>;
export async function listFiles (options: string | ListFilesOptions): Promise<S3Object[]> {
let s3Folder: string;
let maxKeys: number | undefined;
let extension: string | undefined;
let extension: string | string[] | undefined;
if (typeof options === "string") {
s3Folder = options;
} else {
Expand All @@ -167,7 +167,11 @@ export async function listFiles (options: string | ListFilesOptions): Promise<S3
result = await listObjects({ prefix: s3Folder, maxKeys, continuationToken: result && result.NextContinuationToken});
if (result.Contents) {
if (extension && result.Contents.length > 0) {
const filtered: S3Object[] = result.Contents.filter((s3File: S3Object) => s3File.Key!.endsWith(extension!));
const filtered: S3Object[] = result.Contents.filter((s3File: S3Object) => Array.isArray(extension)
? extension.findIndex((thisExtension: string) => s3File.Key!.endsWith(thisExtension)) >= 0
: s3File.Key!.endsWith(extension!)
);
log(`listFiles(${s3Folder}, ${maxKeys}, ${extension}) results`, LogLevel.DEBUG, { results: result.Contents.length, filtered: filtered.length });
files.push(...filtered);
} else {
files.push(...(result.Contents));
Expand Down
47 changes: 46 additions & 1 deletion common/test/s3.spec.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -166,7 +166,7 @@ describe("S3Util", () => {
s3Folder: UNIT_TEST_KEY_PREFIX,
extension: UNIT_TEST_FILENAME.slice(-3)
}).then((result: S3Object[]) => {
log(`listFiles("${UNIT_TEST_KEY_PREFIX}", undefined, ${UNIT_TEST_FILENAME.slice(-3)}) result = ${JSON.stringify(result)}`, LogLevel.DEBUG);
log(`listFiles("${UNIT_TEST_KEY_PREFIX}", undefined, "${UNIT_TEST_FILENAME.slice(-3)}") result = ${JSON.stringify(result)}`, LogLevel.DEBUG);
expect(result).to.not.equal(undefined);
expect(result.length).to.equal(1);
done();
Expand All @@ -189,6 +189,51 @@ describe("S3Util", () => {
done(error);
});
});

it("List Files with extension array first should return files", (done: Mocha.Done) => {
mockListObjects([s3TestObject]);
listFiles({
s3Folder: UNIT_TEST_KEY_PREFIX,
extension: [UNIT_TEST_FILENAME.slice(-3), "bogus"]
}).then((result: S3Object[]) => {
log(`listFiles("${UNIT_TEST_KEY_PREFIX}", undefined, ["${UNIT_TEST_FILENAME.slice(-3)}", "bogus"]) result = ${JSON.stringify(result)}`, LogLevel.DEBUG);
expect(result).to.not.equal(undefined);
expect(result.length).to.equal(1);
done();
}).catch((error) => {
done(error);
});
});

it("List Files with extension array second should return files", (done: Mocha.Done) => {
mockListObjects([s3TestObject]);
listFiles({
s3Folder: UNIT_TEST_KEY_PREFIX,
extension: ["bogus", UNIT_TEST_FILENAME.slice(-3)]
}).then((result: S3Object[]) => {
log(`listFiles("${UNIT_TEST_KEY_PREFIX}", undefined, ["bogus", "${UNIT_TEST_FILENAME.slice(-3)}"]) result = ${JSON.stringify(result)}`, LogLevel.DEBUG);
expect(result).to.not.equal(undefined);
expect(result.length).to.equal(1);
done();
}).catch((error) => {
done(error);
});
});

it("List Files with not found extension array should not return files", (done: Mocha.Done) => {
mockListObjects([s3TestObject]);
listFiles({
s3Folder: UNIT_TEST_KEY_PREFIX,
extension: ["bad", "bogus"]
}).then((result: S3Object[]) => {
log(`listFiles("${UNIT_TEST_KEY_PREFIX}", undefined, ["bad", "bogus"]) result = ${JSON.stringify(result)}`, LogLevel.DEBUG);
expect(result).to.not.equal(undefined);
expect(result.length).to.equal(0);
done();
}).catch((error) => {
done(error);
});
});
});

describe("Upload Object to S3", () => {
Expand Down
67 changes: 65 additions & 2 deletions common/test/s3file.spec.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -391,7 +391,7 @@ describe("PpaasS3File", () => {
localDirectory: UNIT_TEST_LOCAL_FILE_LOCATION,
extension: UNIT_TEST_FILENAME.slice(-3)
}).then((result: PpaasS3File[]) => {
log(`PpaasS3File.getAllFilesInS3("${unitTestKeyPrefix}", "${UNIT_TEST_LOCAL_FILE_LOCATION}") result = ${JSON.stringify(result)}`, LogLevel.DEBUG);
log(`PpaasS3File.getAllFilesInS3("${unitTestKeyPrefix}", "${UNIT_TEST_LOCAL_FILE_LOCATION}", "${UNIT_TEST_FILENAME.slice(-3)}") result = ${JSON.stringify(result)}`, LogLevel.DEBUG);
expect(result).to.not.equal(undefined);
expect(result.length).to.equal(1);
// getAllFilesInS3 should set the remote date so we can sort
Expand All @@ -414,7 +414,70 @@ describe("PpaasS3File", () => {
extension: "bad",
maxFiles: 1000
}).then((result: PpaasS3File[]) => {
log(`PpaasS3File.getAllFilesInS3("${unitTestKeyPrefix}", "${UNIT_TEST_LOCAL_FILE_LOCATION}", 1000) result = ${JSON.stringify(result)}`, LogLevel.DEBUG);
log(`PpaasS3File.getAllFilesInS3("${unitTestKeyPrefix}", "${UNIT_TEST_LOCAL_FILE_LOCATION}", "bad", 1000) result = ${JSON.stringify(result)}`, LogLevel.DEBUG);
expect(result).to.not.equal(undefined);
expect(result.length).to.equal(0);
done();
}).catch((error) => {
done(error);
});
});

it("getAllFilesInS3 partial folder by extension array first should return files", (done: Mocha.Done) => {
mockListObject(UNIT_TEST_FILENAME, unitTestKeyPrefix);
mockGetObjectTagging(tags);
PpaasS3File.getAllFilesInS3({
s3Folder: unitTestKeyPrefix.slice(0, -2),
localDirectory: UNIT_TEST_LOCAL_FILE_LOCATION,
extension: [UNIT_TEST_FILENAME.slice(-3), "bogus"]
}).then((result: PpaasS3File[]) => {
log(`PpaasS3File.getAllFilesInS3("${unitTestKeyPrefix}", "${UNIT_TEST_LOCAL_FILE_LOCATION}", ["${UNIT_TEST_FILENAME.slice(-3)}", "bogus"]) result = ${JSON.stringify(result)}`, LogLevel.DEBUG);
expect(result).to.not.equal(undefined);
expect(result.length).to.equal(1);
// getAllFilesInS3 should set the remote date so we can sort
expect(result[0].getLastModifiedRemote()).to.be.greaterThan(new Date(0));
expect(result[0].tags).to.not.equal(undefined);
expect(result[0].tags?.size).to.equal(1);
expect(result[0].tags?.has("test")).to.equal(false);
expect(result[0].tags?.get("unittest")).to.equal("true");
done();
}).catch((error) => {
done(error);
});
});

it("getAllFilesInS3 partial folder by extension array second should return files", (done: Mocha.Done) => {
mockListObject(UNIT_TEST_FILENAME, unitTestKeyPrefix);
mockGetObjectTagging(tags);
PpaasS3File.getAllFilesInS3({
s3Folder: unitTestKeyPrefix.slice(0, -2),
localDirectory: UNIT_TEST_LOCAL_FILE_LOCATION,
extension: ["bogus", UNIT_TEST_FILENAME.slice(-3)]
}).then((result: PpaasS3File[]) => {
log(`PpaasS3File.getAllFilesInS3("${unitTestKeyPrefix}", "${UNIT_TEST_LOCAL_FILE_LOCATION}", ["bogus", "${UNIT_TEST_FILENAME.slice(-3)}"]) result = ${JSON.stringify(result)}`, LogLevel.DEBUG);
expect(result).to.not.equal(undefined);
expect(result.length).to.equal(1);
// getAllFilesInS3 should set the remote date so we can sort
expect(result[0].getLastModifiedRemote()).to.be.greaterThan(new Date(0));
expect(result[0].tags).to.not.equal(undefined);
expect(result[0].tags?.size).to.equal(1);
expect(result[0].tags?.has("test")).to.equal(false);
expect(result[0].tags?.get("unittest")).to.equal("true");
done();
}).catch((error) => {
done(error);
});
});

it("getAllFilesInS3 partial folder wrong extension array should not return files", (done: Mocha.Done) => {
mockListObjects([]);
PpaasS3File.getAllFilesInS3({
s3Folder: unitTestKeyPrefix.slice(0, -2),
localDirectory: UNIT_TEST_LOCAL_FILE_LOCATION,
extension: ["bad", "bogus"],
maxFiles: 1000
}).then((result: PpaasS3File[]) => {
log(`PpaasS3File.getAllFilesInS3("${unitTestKeyPrefix}", "${UNIT_TEST_LOCAL_FILE_LOCATION}", ["bad", "bogus"], 1000) result = ${JSON.stringify(result)}`, LogLevel.DEBUG);
expect(result).to.not.equal(undefined);
expect(result.length).to.equal(0);
done();
Expand Down
2 changes: 1 addition & 1 deletion controller/pages/api/util/ppaasencrypts3file.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@ export class PpaasEncryptS3File implements s3.S3File {
}

public static async getAllFilesInS3 (s3Folder: string, extension?: string, maxFiles?: number): Promise<PpaasEncryptS3File[]> {
log(`Finding in s3Folder: ${s3Folder}, extension: ${extension}, maxFiles: ${maxFiles}`, LogLevel.DEBUG);
log(`PpaasEncryptS3File Finding in s3Folder: ${s3Folder}, extension: ${extension}, maxFiles: ${maxFiles}`, LogLevel.DEBUG);
const s3Files: S3Object[] = await listFiles({ s3Folder, maxKeys: maxFiles, extension });
if (s3Files.length === 0) {
return [];
Expand Down
2 changes: 1 addition & 1 deletion controller/pages/api/util/testmanager.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1647,7 +1647,7 @@ export abstract class TestManager {
const s3YamlFiles: PpaasS3File[] = await PpaasS3File.getAllFilesInS3({
s3Folder: s3FolderPartial,
localDirectory,
extension: "yaml",
extension: [".yaml", ".yml"],
maxFiles: 1000
});
if (s3YamlFiles.length === 0) {
Expand Down

0 comments on commit 199f635

Please sign in to comment.