Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: reload chaos experiments on service startup #642

Merged
merged 6 commits into from
Oct 1, 2023
Merged
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 5 additions & 3 deletions src/jobs/helpers/jobVerifier.js
Original file line number Diff line number Diff line change
@@ -81,10 +81,12 @@ module.exports.verifyExperimentsExist = async (req, res, next) => {
let errorToThrow;
const jobPlatform = await configHandler.getConfigValue(CONFIG.JOB_PLATFORM);
const experiments = jobBody.experiments;
if (jobPlatform.toUpperCase() !== KUBERNETES) {
next();
} else if (!experiments || experiments.length === 0) {
if (!experiments || experiments.length === 0) {
next();
} else if (experiments.length > 0 && jobPlatform.toUpperCase() !== KUBERNETES) {
errorToThrow = new Error(ERROR_MESSAGES.CHAOS_EXPERIMENT_SUPPORTED_ONLY_IN_KUBERNETES);
errorToThrow.statusCode = 400;
next(errorToThrow);
} else {
const uniqueExperimentIds = [...new Set(experiments.map(experiment => experiment.experiment_id))];
const chaosExperiments = await choasExperimentsManager.getChaosExperimentsByIds(uniqueExperimentIds, ['kubeObject']);
22 changes: 15 additions & 7 deletions src/jobs/models/jobManager.js
Original file line number Diff line number Diff line change
@@ -38,11 +38,11 @@ module.exports.reloadCronJobs = async () => {
const configData = await configHandler.getConfig();
try {
const jobs = await databaseConnector.getJobs(contextId);
for (const job of jobs) {
jobs.forEach(function (job) {
if (job.cron_expression !== null) {
await addCron(job, job.cron_expression, configData);
addCron(job, job.cron_expression, configData);
}
};
});
} catch (error) {
throw new Error('Unable to reload scheduled jobs, error: ' + error);
}
@@ -55,12 +55,10 @@ module.exports.reloadChaosExperiments = async () => {
return;
}
try {
const timestamp = new Date().valueOf();
const timestamp = Date.now();
const futureJobExperiments = await chaosExperimentsManager.getFutureJobExperiments(timestamp, contextId);
for (const futureJobExperiment of futureJobExperiments) {
const calculatedStartAfter = futureJobExperiment.start_time - timestamp;
const chaosExperiment = await chaosExperimentsManager.getChaosExperimentById(futureJobExperiment.experiment_id);
jobExperimentHandler.scheduleChaosExperiment(chaosExperiment.kubeObject, futureJobExperiment.job_id, futureJobExperiment.id, calculatedStartAfter);
await reloadSingleChaosExperiment(futureJobExperiment, timestamp);
}
GuyAb marked this conversation as resolved.
Show resolved Hide resolved
} catch (error) {
throw new Error('Unable to reload job experiments , error: ' + error);
@@ -404,3 +402,13 @@ function produceJobToStreamingPlatform(jobResponse) {
};
streamingManager.produce({}, STREAMING_EVENT_TYPES.JOB_CREATED, streamingResource);
}

async function reloadSingleChaosExperiment(futureJobExperiment, timestamp){
try {
const calculatedStartAfter = futureJobExperiment.start_time - timestamp;
const chaosExperiment = await chaosExperimentsManager.getChaosExperimentById(futureJobExperiment.experiment_id);
jobExperimentHandler.scheduleChaosExperiment(chaosExperiment.kubeObject, futureJobExperiment.job_id, futureJobExperiment.id, calculatedStartAfter);
} catch (error) {
throw new Error('Unable to reload job experiments ' + futureJobExperiment.id + ' , error: ' + error);
}
}
10 changes: 7 additions & 3 deletions tests/unit-tests/jobs/helpers/jobVerifier-test.js
Original file line number Diff line number Diff line change
@@ -195,22 +195,19 @@ describe('Jobs verifier tests', function () {
await jobVerifier.verifyExperimentsExist(req, res, nextStub);
should(nextStub.args[0][0]).eql(undefined);
});

it('if job platform is not KUBERNETES, should pass', async () => {
configHandlerStub.withArgs(consts.CONFIG.JOB_PLATFORM).resolves('DOCKER');
req = { body: { run_immediately: true, cron_expression: '* * * * *', enabled: false } };
await jobVerifier.verifyExperimentsExist(req, res, nextStub);
should(nextStub.args[0][0]).eql(undefined);
});

it('if chaos experiments mentioned in the job exist, should pass', async () => {
configHandlerStub.withArgs(consts.CONFIG.JOB_PLATFORM).resolves('KUBERNETES');
req = { body: { run_immediately: true, cron_expression: '* * * * *', enabled: true, experiments: [{ experiment_id: '1234', start_after: 1000 }] } };
getChaosExperimentsByIdsStub.resolves([{ id: '1234' }]);
await jobVerifier.verifyExperimentsExist(req, res, nextStub);
should(nextStub.args[0][0]).eql(undefined);
});

it('if chaos experiments mentioned in the job do not exist, should fail', async () => {
configHandlerStub.withArgs(consts.CONFIG.JOB_PLATFORM).resolves('KUBERNETES');
req = { body: { run_immediately: true, cron_expression: '* * * * *', enabled: true, experiments: [{ experiment_id: '1234', start_after: 1000 }] } };
@@ -219,5 +216,12 @@ describe('Jobs verifier tests', function () {
should(nextStub.args[0][0].message).eql('One or more chaos experiments are not configured. Job can not be created');
should(nextStub.args[0][0].statusCode).eql(400);
});
it('if job platform is not KUBERNETES and job has experiments job does not have experiments, should pass', async () => {
configHandlerStub.withArgs(consts.CONFIG.JOB_PLATFORM).resolves('DOCKER');
req = { body: { run_immediately: true, cron_expression: '* * * * *', enabled: false, experiments: [{ experiment_id: '1234', start_after: 1000 }] } };
await jobVerifier.verifyExperimentsExist(req, res, nextStub);
should(nextStub.args[0][0].message).eql('Chaos experiment is supported only in kubernetes jobs');
should(nextStub.args[0][0].statusCode).eql(400);
});
});
});