Skip to content

Commit

Permalink
Merge pull request #648 from Zooz/aggregate-report-experiments
Browse files Browse the repository at this point in the history 
feat: aggregate experiments in the report
  • Loading branch information
@kerenfi
kerenfi authored Oct 2, 2023
2 parents b3db7f1 + 6ebe684 commit 1b21ec3
Show file tree
Hide file tree
Showing 7 changed files with 186 additions and 25 deletions.
6 changes: 5 additions & 1 deletion src/chaos-experiments/models/chaosExperimentsManager.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -92,5 +92,9 @@ module.exports.runChaosExperiment = async (kubernetesChaosConfig, jobExperimentI
};

module.exports.getFutureJobExperiments = async function (timestamp, contextId) {
return databaseConnector.getFutureJobExperiments(contextId);
return databaseConnector.getFutureJobExperiments(timestamp, contextId);
};

module.exports.getChaosJobExperimentsByJobId = async function (jobId, contextId) {
return databaseConnector.getChaosJobExperimentsByJobId(jobId, contextId);
};
10 changes: 5 additions & 5 deletions src/chaos-experiments/models/database/databaseConnector.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ module.exports = {
getChaosExperimentsByIds,
deleteChaosExperiment,
insertChaosJobExperiment,
getChaosJobExperimentById,
getChaosJobExperimentsByJobId,
getChaosJobExperimentByJobId,
getFutureJobExperiments,
setChaosJobExperimentTriggered,
Expand Down Expand Up @@ -57,16 +57,16 @@ async function insertChaosJobExperiment(jobExperimentId, jobId, experimentId, st
return databaseConnector.insertChaosJobExperiment(jobExperimentId, jobId, experimentId, startTime, endTime, contextId);
}

async function getChaosJobExperimentById(jobExperimentId, contextId) {
return databaseConnector.getChaosJobExperimentById(jobExperimentId, contextId);
async function getChaosJobExperimentsByJobId(jobExperimentId, contextId) {
return databaseConnector.getChaosJobExperimentsByJobId(jobExperimentId, contextId);
}

async function getChaosJobExperimentByJobId(jobId, contextId) {
return databaseConnector.getChaosJobExperimentById(jobId, contextId);
}

async function getFutureJobExperiments(contextId) {
return databaseConnector.getFutureJobExperiments(contextId);
async function getFutureJobExperiments(timestamp, contextId) {
return databaseConnector.getFutureJobExperiments(timestamp, contextId);
}

async function setChaosJobExperimentTriggered(jobExperimentId, isTriggered, contextId) {
Expand Down
24 changes: 10 additions & 14 deletions src/chaos-experiments/models/database/sequelize/sequelizeConnector.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ module.exports = {
updateChaosExperiment,
insertChaosJobExperiment,
getChaosJobExperimentById,
getChaosJobExperimentByJobId,
getChaosJobExperimentsByJobId,
getFutureJobExperiments,
setChaosJobExperimentTriggered
};
Expand Down Expand Up @@ -70,11 +70,10 @@ async function getChaosExperimentById(experimentId, contextId) {
options.where.context_id = contextId;
}

let chaosExperiment = await _getChaosExperiment(options);
const chaosExperiment = await _getChaosExperiment(options);
if (chaosExperiment) {
chaosExperiment = chaosExperiment.get();
return chaosExperiment.get();
}
return chaosExperiment;
}

async function getChaosExperimentsByIds(experimentIds, exclude, contextId) {
Expand Down Expand Up @@ -152,14 +151,13 @@ async function getChaosJobExperimentById(jobExperimentId, contextId) {
options.where.context_id = contextId;
}

let chaosExperiment = await _getChaosJobExperiment(options);
if (chaosExperiment) {
chaosExperiment = chaosExperiment.get();
const chaosJobExperiment = await _getChaosJobExperiment(options);
if (chaosJobExperiment) {
return chaosJobExperiment.get();
}
return chaosExperiment;
}

async function getChaosJobExperimentByJobId(jobId, contextId) {
async function getChaosJobExperimentsByJobId(jobId, contextId) {
const options = {
where: { job_id: jobId }
};
Expand All @@ -168,11 +166,9 @@ async function getChaosJobExperimentByJobId(jobId, contextId) {
options.where.context_id = contextId;
}

let chaosExperiment = await _getChaosJobExperiment(options);
if (chaosExperiment) {
chaosExperiment = chaosExperiment.get();
}
return chaosExperiment;
const chaosJobExperimentModel = client.model(CHAOS_JOB_EXPERIMENTS_TABLE_NAME);
const allChaosJobExperiments = await chaosJobExperimentModel.findAll(options);
return allChaosJobExperiments;
}

async function getFutureJobExperiments(timestamp, contextId) {
Expand Down
25 changes: 25 additions & 0 deletions src/reports/models/aggregateReportManager.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ const math = require('mathjs');

const logger = require('../../common/logger');
const databaseConnector = require('./databaseConnector');
const chaosExperimentsManager = require('../../chaos-experiments/models/chaosExperimentsManager');
const constants = require('../utils/constants');

const STATS_INTERVAL = 30;
Expand All @@ -15,6 +16,7 @@ module.exports = {

async function aggregateReport(report) {
let stats = await databaseConnector.getStats(report.test_id, report.report_id);
const experiments = await getChaosExperimentsByJobId(report.job_id);

if (stats.length === 0) {
const errorMessage = `Can not generate aggregate report as there are no statistics yet for testId: ${report.test_id} and reportId: ${report.report_id}`;
Expand All @@ -35,6 +37,7 @@ async function aggregateReport(report) {
reportInput.revision_id = report.revision_id;
reportInput.score = report.score;
reportInput.benchmark_weights_data = report.benchmark_weights_data;
reportInput.experiments = experiments;
reportInput.notes = report.notes;

reportInput.status = mapReportStatus(report.status);
Expand Down Expand Up @@ -63,6 +66,28 @@ async function aggregateReport(report) {
return reportInput;
}

async function getChaosExperimentsByJobId(jobId) {
const chaosJobExperiments = await chaosExperimentsManager.getChaosJobExperimentsByJobId(jobId);
if (!chaosJobExperiments) {
return;
}
const uniqueExperimentIds = [...new Set(chaosJobExperiments.map(jobExperiment => jobExperiment.experiment_id))];
const chaosExperiments = await chaosExperimentsManager.getChaosExperimentsByIds(uniqueExperimentIds);
const mappedChaosJobExperiments = chaosJobExperiments.map((jobExperiment) => {
const chaosExperiment = chaosExperiments.find((experiment) => experiment.id === jobExperiment.experiment_id && jobExperiment.is_triggered);
if (chaosExperiment) {
return {
kind: chaosExperiment.kubeObject.kind,
name: chaosExperiment.name,
id: chaosExperiment.id,
start_time: jobExperiment.start_time,
end_time: jobExperiment.end_time
};
}
});
return mappedChaosJobExperiments;
}

function createAggregateManually(listOfStats) {
const requestMedians = [], requestMaxs = [], requestMins = [], scenario95 = [], scenario99 = [], request95 = [],
request99 = [], scenarioMins = [], scenarioMaxs = [], scenarioMedians = [];
Expand Down
21 changes: 18 additions & 3 deletions tests/unit-tests/chaos-experiments/sequelize/sequelizeConnector-test.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -140,7 +140,7 @@ describe('Sequelize client tests', function () {
should(sequelizeGetStub.args[0][0]).containDeep({ where: { name: experimentName } });
});
});
describe('Get ChaosExperimentsById', () => {
describe('Get ChaosExperimentsByIds', () => {
it('Validate sequelize passed arguments', async () => {
sequelizeGetStub.returns([experiment]);
const experimentIds = ['1234', '4321'];
Expand Down Expand Up @@ -226,15 +226,30 @@ describe('Sequelize client tests', function () {
should(sequelizeGetStub.args[0][0]).containDeep({ where: { id: jobExperimentId } });
});
});
describe('getChaosJobExperimentByJobId', function() {
describe('getChaosJobExperimentsByJobId', function() {
it('Validate sequelize passed arguments', async () => {
sequelizeGetStub.returns([jobExperiment]);
const experimentJobId = experiment.job_id;
await sequelizeConnector.getChaosJobExperimentByJobId(experimentJobId);
await sequelizeConnector.getChaosJobExperimentsByJobId(experimentJobId);
should(sequelizeGetStub.calledOnce).eql(true);
should(sequelizeGetStub.args[0][0]).containDeep({ where: { job_id: experimentJobId } });
});
});
describe('getFutureJobExperiments', function() {
it('Validate sequelize passed arguments', async () => {
sequelizeGetStub.returns([jobExperiment]);
const timestamp = Date.now();
await sequelizeConnector.getFutureJobExperiments(timestamp, 'contextId');
should(sequelizeGetStub.calledOnce).eql(true);
should(sequelizeGetStub.args[0][0]).deepEqual({
where: {
is_triggered: false,
start_time: {},
context_id: 'contextId'
}
});
});
});
});

describe('Set job experiment is triggered', () => {
Expand Down
123 changes: 122 additions & 1 deletion tests/unit-tests/reporter/models/finalReportGenerator-test.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,11 @@ const rewire = require('rewire');
const logger = require('../../../../src/common/logger');
const aggregateReportGenerator = rewire('../../../../src/reports/models/aggregateReportGenerator');
const aggregateReportManager = rewire('../../../../src/reports/models/aggregateReportManager');
const chaosExperimentsManager = require('../../../../src/chaos-experiments/models/chaosExperimentsManager');
const databaseConnector = require('../../../../src/reports/models/databaseConnector');
const reportsManager = require('../../../../src/reports/models/reportsManager');
const configHandler = require('../../../../src/configManager/models/configHandler');
const consts = require('../../../../src/common/consts');

const REPORT = {
test_id: 'test_id',
Expand All @@ -17,17 +20,28 @@ const REPORT = {
test_name: 'some_test_name',
webhooks: ['http://www.zooz.com'],
arrival_rate: 100,
job_id: 'job_id',
duration: 10,
environment: 'test'
};

describe('Artillery report generator test', () => {
let sandbox, databaseConnectorGetStatsStub, loggerErrorStub, loggerWarnStub, reportsManagerGetReportStub;
let sandbox,
configHandlerStub,
databaseConnectorGetStatsStub,
getJobExperimentsByJobIdStub,
getChaosExperimentsByIdsStub,
loggerErrorStub,
loggerWarnStub,
reportsManagerGetReportStub;

before(() => {
sandbox = sinon.sandbox.create();
configHandlerStub = sandbox.stub(configHandler, 'getConfigValue');
databaseConnectorGetStatsStub = sandbox.stub(databaseConnector, 'getStats');
reportsManagerGetReportStub = sandbox.stub(reportsManager, 'getReport');
getJobExperimentsByJobIdStub = sandbox.stub(chaosExperimentsManager, 'getChaosJobExperimentsByJobId');
getChaosExperimentsByIdsStub = sandbox.stub(chaosExperimentsManager, 'getChaosExperimentsByIds');
loggerErrorStub = sandbox.stub(logger, 'error');
loggerWarnStub = sandbox.stub(logger, 'warn');
});
Expand All @@ -48,6 +62,7 @@ describe('Artillery report generator test', () => {

it('create aggregate report when there is only intermediate rows', async () => {
databaseConnectorGetStatsStub.resolves(SINGLE_RUNNER_INTERMEDIATE_ROWS);
getJobExperimentsByJobIdStub.resolves([]);

const reportOutput = await aggregateReportGenerator.createAggregateReport(REPORT.test_id, REPORT.report_id);
should(reportOutput.parallelism).eql(1);
Expand All @@ -57,6 +72,7 @@ describe('Artillery report generator test', () => {
const statsWithUnknownData = JSON.parse(JSON.stringify(SINGLE_RUNNER_INTERMEDIATE_ROWS));
statsWithUnknownData.push({ phase_status: 'some_unknown_phase', data: JSON.stringify({}) });
databaseConnectorGetStatsStub.resolves(statsWithUnknownData);
getJobExperimentsByJobIdStub.resolves([]);

const reportOutput = await aggregateReportGenerator.createAggregateReport(REPORT.test_id, REPORT.report_id);
should(reportOutput.parallelism).eql(1);
Expand All @@ -72,6 +88,39 @@ describe('Artillery report generator test', () => {

loggerWarnStub.callCount.should.eql(1);
});

it('create final report successfully with chaos experiments', async function() {
configHandlerStub.withArgs(consts.CONFIG.JOB_PLATFORM).resolves('KUBERNETES');
const statsWithUnknownData = JSON.parse(JSON.stringify(SINGLE_RUNNER_INTERMEDIATE_ROWS));
statsWithUnknownData.push({ phase_status: 'intermediate', data: 'unsupported data type' });
databaseConnectorGetStatsStub.resolves(statsWithUnknownData);
getJobExperimentsByJobIdStub.resolves(JOB_EXPERIMENTS_ROWS);
getChaosExperimentsByIdsStub.resolves(CHAOS_EXPERIMENTS_ROWS);
const reportOutput = await aggregateReportGenerator.createAggregateReport(REPORT.test_id, REPORT.report_id);
should(reportOutput.experiments).deepEqual([
{
kind: CHAOS_EXPERIMENTS_ROWS[0].kubeObject.kind,
name: CHAOS_EXPERIMENTS_ROWS[0].name,
id: JOB_EXPERIMENTS_ROWS[0].experiment_id,
start_time: JOB_EXPERIMENTS_ROWS[0].start_time,
end_time: JOB_EXPERIMENTS_ROWS[0].end_time
},
{
kind: CHAOS_EXPERIMENTS_ROWS[1].kubeObject.kind,
name: CHAOS_EXPERIMENTS_ROWS[1].name,
id: JOB_EXPERIMENTS_ROWS[1].experiment_id,
start_time: JOB_EXPERIMENTS_ROWS[1].start_time,
end_time: JOB_EXPERIMENTS_ROWS[1].end_time
},
{
kind: CHAOS_EXPERIMENTS_ROWS[2].kubeObject.kind,
name: CHAOS_EXPERIMENTS_ROWS[2].name,
id: JOB_EXPERIMENTS_ROWS[2].experiment_id,
start_time: JOB_EXPERIMENTS_ROWS[2].start_time,
end_time: JOB_EXPERIMENTS_ROWS[2].end_time
}
]);
});
});

describe('Happy flows - With parallelism', function () {
Expand All @@ -86,6 +135,7 @@ describe('Artillery report generator test', () => {
const firstStatsTimestamp = JSON.parse(PARALLEL_INTERMEDIATE_ROWS[0].data).timestamp;

reportsManagerGetReportStub.resolves(REPORT);
getJobExperimentsByJobIdStub.resolves([]);
REPORT.start_time = new Date(new Date(firstStatsTimestamp).getTime() - (STATS_INTERVAL * 1000));
databaseConnectorGetStatsStub.resolves(PARALLEL_INTERMEDIATE_ROWS);
const reportOutput = await aggregateReportGenerator.createAggregateReport(REPORT.test_id, REPORT.report_id);
Expand Down Expand Up @@ -233,9 +283,42 @@ describe('Artillery report generator test', () => {

testShouldFail.should.eql(false, 'Test action was supposed to get exception');
});

it('create final report fails when get experiments returns error on get job experiments', async () => {
databaseConnectorGetStatsStub.resolves(SINGLE_RUNNER_INTERMEDIATE_ROWS);
reportsManagerGetReportStub.rejects(new Error('Database failure'));

let testShouldFail = true;
try {
await aggregateReportGenerator.createAggregateReport('testId', 'reportId');
} catch (error) {
testShouldFail = false;
error.message.should.eql('Database failure');
}

testShouldFail.should.eql(false, 'Test action was supposed to get exception');
});

it('create final report fails when get experiments returns error on get chaos experiments', async () => {
databaseConnectorGetStatsStub.resolves(SINGLE_RUNNER_INTERMEDIATE_ROWS);
getJobExperimentsByJobIdStub.resolves(JOB_EXPERIMENTS_ROWS);
getChaosExperimentsByIdsStub.rejects(new Error('Database failure'));

let testShouldFail = true;
try {
await aggregateReportGenerator.createAggregateReport('testId', 'reportId');
} catch (error) {
testShouldFail = false;
error.message.should.eql('Database failure');
}

testShouldFail.should.eql(false, 'Test action was supposed to get exception');
});
});
});

const timestamp = Date.now();

const SINGLE_RUNNER_INTERMEDIATE_ROWS = [{
test_id: 'cb7d7862-55c2-4a9b-bcec-d41d54101836',
report_id: 'b6489011-2073-4998-91cc-fd62f8b927f7',
Expand Down Expand Up @@ -337,3 +420,41 @@ const PARALLEL_INTERMEDIATE_ROWS = [
data: '{"timestamp":"2019-03-10T17:24:33.043Z","scenariosCreated":300,"scenariosCompleted":300,"requestsCompleted":300,"latency":{"min":59.5,"max":98.3,"median":61.3,"p95":72.9,"p99":84},"rps":{"count":300,"mean":20},"scenarioDuration":{"min":60,"max":98.9,"median":61.9,"p95":73.5,"p99":84.5},"scenarioCounts":{"Get response code 200":300},"errors":{},"codes":{"200":300},"matches":0,"customStats":{},"counters":{},"concurrency":1,"pendingRequests":1,"scenariosAvoided":0}'
}
];

const JOB_EXPERIMENTS_ROWS = [{
job_id: REPORT.job_id,
experiment_id: '1234-abc-5678',
start_time: timestamp,
end_time: timestamp + 100,
is_triggered: true
},
{
job_id: REPORT.job_id,
experiment_id: 'abcd-1234-efgh',
start_time: timestamp,
end_time: timestamp + 200,
is_triggered: true
},
{
job_id: REPORT.job_id,
experiment_id: '4321-abc-5678',
start_time: timestamp,
end_time: timestamp + 300,
is_triggered: true
}];

const CHAOS_EXPERIMENTS_ROWS = [{
id: '1234-abc-5678',
name: 'first-experiment',
kubeObject: { kind: 'PodChaos' }
},
{
id: 'abcd-1234-efgh',
name: 'second-experiment',
kubeObject: { kind: 'DNSChaos' }
},
{
id: '4321-abc-5678',
name: 'third-experiment',
kubeObject: { kind: 'IOChaos' }
}];
2 changes: 1 addition & 1 deletion ui/src/features/components/JobForm/index.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -212,7 +212,7 @@ class Form extends React.Component {
<span className={style['list-item__title']}>experiment name:</span>
<span className={style['list-item']}> {experiment.experiment_name}</span>
<span className={style['list-item__title']}>start after:</span>
<span className={style['list-item']}> {experiment.start_after / ONE_MIN_MS} seconds</span>
<span className={style['list-item']}> {experiment.start_after / ONE_MIN_MS} minutes</span>
<FontAwesomeIcon
icon={faTimes}
size='1px'
Expand Down

0 comments on commit 1b21ec3

Please sign in to comment.