Skip to content

Commit

Permalink
Added code to clear tests off of the calendar after 1 year (or config…
Browse files Browse the repository at this point in the history 
…ured)

- if RUN_HISTORICAL_DELETE is true, then it will remove things off the calendar after DELETE_OLD_FILES_DAYS (365 default) days.
  • Loading branch information
@tkmcmaster
tkmcmaster committed Dec 27, 2023
1 parent bc4e211 commit ede686c
Show file tree
Hide file tree
Showing 2 changed files with 149 additions and 4 deletions.
66 changes: 63 additions & 3 deletions controller/pages/api/util/testscheduler.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,8 +42,10 @@ const { sleep } = util;
logger.config.LogFileName = "ppaas-controller";

const TEST_SCHEDULER_POLL_INTERVAL_MS: number = parseInt(process.env.TEST_SCHEDULER_POLL_INTERVAL_MS || "0", 10) || 60000;
const RUN_HISTORICAL_SEARCH: boolean = process.env.RUN_HISTORICAL_SEARCH === "true";
const RUN_HISTORICAL_SEARCH: boolean = process.env.RUN_HISTORICAL_SEARCH?.toLowerCase() === "true";
const HISTORICAL_SEARCH_MAX_FILES: number = parseInt(process.env.HISTORICAL_SEARCH_MAX_FILES || "0", 10) || 100000;
const RUN_HISTORICAL_DELETE: boolean = process.env.RUN_HISTORICAL_DELETE?.toLowerCase() === "true";
const DELETE_OLD_FILES_DAYS: number = parseInt(process.env.DELETE_OLD_FILES_DAYS || "0") || 365;
const ONE_DAY: number = 24 * 60 * 60000;
export const AUTH_PERMISSIONS_SCHEDULER: AuthPermissions = { authPermission: AuthPermission.Admin, token: "startTestSchedulerLoop", userId: "controller" };
export const TEST_HISTORY_FILENAME = "testhistory.json";
Expand Down Expand Up @@ -391,6 +393,37 @@ export class TestScheduler implements TestSchedulerItem {
if (RUN_HISTORICAL_SEARCH) {
TestScheduler.runHistoricalSearch().catch(() => { /* already logs error, swallow */ });
}
if (RUN_HISTORICAL_DELETE) {
// Start background task to delete old files
(async () => {
// Don't start right away, delay to sometime within today
let nextLoop: number = Date.now() + Math.floor(Math.random() * ONE_DAY);
if (nextLoop > Date.now()) {
const delay = nextLoop - Date.now();
log("Delete Historical Loop: nextLoop: " + new Date(nextLoop), LogLevel.DEBUG, { delay, nextLoop });
await sleep(delay);
}
while (global.testSchedulerLoopRunning) {
const loopStart = Date.now();
try {
await TestScheduler.runHistoricalDelete();
} catch (error) {
log("Delete Historical Loop: Error running runHistoricalDelete", LogLevel.ERROR, error);
}
// If Date.now() is exactly the same time we need to check the next one
nextLoop += ONE_DAY;
const delay = nextLoop - Date.now();
log("Delete Historical Loop: nextLoop: " + new Date(nextLoop), LogLevel.DEBUG, { loopStart, delay, nextLoop });
if (delay > 0) {
await sleep(delay);
}
}
// We'll only reach here if we got some kind of sigterm message or an unhandled exception. Shut down this loop so we can be restarted or replaced
log("Shutting Down Delete Historical Loop.", LogLevel.INFO);
})().catch((err) => {
log("Error during Delete Historical Loop", LogLevel.ERROR, err);
});
}
(async () => {
// We'll never set this to false unless something really bad happens
while (global.testSchedulerLoopRunning) {
Expand Down Expand Up @@ -424,9 +457,9 @@ export class TestScheduler implements TestSchedulerItem {
const nextStartTime = TestScheduler.nextStart || Number.MAX_VALUE;
const delay = Math.min(nextPollTime - Date.now(), nextStartTime - Date.now(), TEST_SCHEDULER_POLL_INTERVAL_MS);
log(
"Test Scheduler Loop: nextPollTime: " + nextPollTime,
"Test Scheduler Loop: nextPollTime: " + new Date(nextPollTime),
LogLevel.DEBUG,
{ loopStart, nextPollTime, nextStartTime, now: Date.now(), delay, TEST_SCHEDULER_POLL_INTERVAL_MS }
{ loopStart, nextPollTime, nextStartTime, delay, TEST_SCHEDULER_POLL_INTERVAL_MS }
);
if (delay > 0) {
await sleep(delay);
Expand Down Expand Up @@ -828,6 +861,33 @@ export class TestScheduler implements TestSchedulerItem {
}
}

protected static async runHistoricalDelete (deleteOldFilesDays: number = DELETE_OLD_FILES_DAYS): Promise<number> {
try {
let deletedCount: number = 0;
// Load existing ones
await TestScheduler.loadHistoricalFromS3();
const oldDatetime: number = Date.now() - (deleteOldFilesDays * ONE_DAY);
log("Starting Test Historical Delete", LogLevel.INFO, { sizeBefore: TestScheduler.historicalTests!.size, oldDatetime: new Date(oldDatetime), deleteOldFilesDays });

// Delete old ones off the historical Calendar. These will be cleaned up in S3 by Bucket Expiration Policy
for (const [testId, eventInput] of TestScheduler.historicalTests!) {
if ((typeof eventInput.end === "number" && eventInput.end < oldDatetime)
|| (eventInput.end instanceof Date && (eventInput as Date).getTime() < oldDatetime)) {
log("Deleting Historical Test " + testId, LogLevel.INFO, eventInput);
// Delete
TestScheduler.historicalTests!.delete(testId);
deletedCount++;
}
}
await TestScheduler.saveHistoricalToS3();
log("Finished Test Historical Delete", LogLevel.INFO, { sizeAfter: TestScheduler.historicalTests!.size });
return deletedCount;
} catch (error) {
log("Error running Historical Delete", LogLevel.ERROR, error);
throw error; // Throw for testing, but the loop will catch and noop
}
}

protected static async loadHistoricalFromS3 (force?: boolean): Promise<void> {
log("TestScheduler: loadHistoricalFromS3", LogLevel.DEBUG, {
thisScheduledTests: (TestScheduler.historicalTests !== undefined)
Expand Down
87 changes: 86 additions & 1 deletion controller/test/testscheduler.spec.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,10 @@ import {
PpaasTestId,
PpaasTestStatus,
TestMessage,
TestStatus,
log,
logger
logger,
util
} from "@fs/ppaas-common";
import {
TestScheduler,
Expand Down Expand Up @@ -103,6 +105,7 @@ export class TestSchedulerIntegration extends TestScheduler {
}

public static getScheduledTests (): Map<string, TestSchedulerItem> {
TestScheduler.loadTestsFromS3();
return TestScheduler.scheduledTests!;
}

Expand All @@ -111,6 +114,7 @@ export class TestSchedulerIntegration extends TestScheduler {
}

public static getHistoricalTests (): Map<string, EventInput> {
TestScheduler.loadHistoricalFromS3();
return TestScheduler.historicalTests!;
}

Expand All @@ -137,6 +141,11 @@ export class TestSchedulerIntegration extends TestScheduler {
public static setNextStart (nextStart: number | undefined) {
TestScheduler.nextStart = nextStart;
}

/** public so we can call it for testing */
public static runHistoricalDelete (deleteOldFilesDays?: number) {
return TestScheduler.runHistoricalDelete(deleteOldFilesDays);
}
}

class PpaasTestStatusIntegration extends PpaasTestStatus {
Expand Down Expand Up @@ -1279,4 +1288,80 @@ describe("TestScheduler", () => {
}
});
});

describe("runHistoricalDelete", () => {
const deleteOldFilesDays = 7;

// Empty
it("should succeed even if empty", async () => {
try {
const historicalTests: Map<string, EventInput> = TestSchedulerIntegration.getHistoricalTests();
historicalTests.clear();
const deletedCount: number = await TestSchedulerIntegration.runHistoricalDelete(deleteOldFilesDays);
expect(deletedCount).to.equal(0);
expect(historicalTests.size, "historicalTests.size").to.equal(0);
} catch (error) {
log("should succeed even if empty error", LogLevel.ERROR, error);
throw error;
}
});

// One today should not be deleted
it("should not remove new", async () => {
try {
const historicalTests: Map<string, EventInput> = TestSchedulerIntegration.getHistoricalTests();
historicalTests.clear();
const startTime = Date.now() - (2 * ONE_HOUR);
const endTime = Date.now() - ONE_HOUR;
await TestSchedulerIntegration.addHistoricalTest(testId, yamlFile, startTime, endTime, TestStatus.Finished);
const deletedCount: number = await TestSchedulerIntegration.runHistoricalDelete(deleteOldFilesDays);
expect(deletedCount, "deletedCount").to.equal(0);
expect(historicalTests.size, "historicalTests.size").to.equal(1);
} catch (error) {
log("should not remove new error", LogLevel.ERROR, error);
throw error;
}
});

// One last week should be deleted
it("should remove old", async () => {
try {
const historicalTests: Map<string, EventInput> = TestSchedulerIntegration.getHistoricalTests();
historicalTests.clear();
const startTime = Date.now() - (2 * ONE_HOUR + ONE_WEEK);
const endTime = Date.now() - (ONE_HOUR + ONE_WEEK);
await TestSchedulerIntegration.addHistoricalTest(testId, yamlFile, startTime, endTime, TestStatus.Finished);
const deletedCount: number = await TestSchedulerIntegration.runHistoricalDelete(deleteOldFilesDays);
expect(deletedCount, "deletedCount").to.equal(1);
expect(historicalTests.size, "historicalTests.size").to.equal(0);
} catch (error) {
log("should remove old error", LogLevel.ERROR, error);
throw error;
}
});

// One today, one yesterday, one 1 week ago, delete one week
it("should remove old, but not new", async () => {
try {
const historicalTests: Map<string, EventInput> = TestSchedulerIntegration.getHistoricalTests();
historicalTests.clear();
const startTime = Date.now() - (2 * ONE_HOUR);
const endTime = Date.now() - ONE_HOUR;
// Today
await TestSchedulerIntegration.addHistoricalTest(testId, yamlFile, startTime, endTime, TestStatus.Finished);
// Yesterday
await TestSchedulerIntegration.addHistoricalTest(PpaasTestId.makeTestId(yamlFile).testId, yamlFile, startTime - ONE_DAY, endTime - ONE_DAY, TestStatus.Finished);
await util.sleep(5); // Need to get a different timestamp for this next test.
// Last week
await TestSchedulerIntegration.addHistoricalTest(PpaasTestId.makeTestId(yamlFile).testId, yamlFile, startTime - ONE_WEEK, endTime - ONE_WEEK, TestStatus.Finished);
expect(historicalTests.size, "historicalTests.size before").to.equal(3);
const deletedCount: number = await TestSchedulerIntegration.runHistoricalDelete(deleteOldFilesDays);
expect(deletedCount, "deletedCount").to.equal(1);
expect(historicalTests.size, "historicalTests.size").to.equal(2);
} catch (error) {
log("should remove old, but not new error", LogLevel.ERROR, error);
throw error;
}
});
});
});

0 comments on commit ede686c

Please sign in to comment.