From 763afbfc4bd598fc1a57b53627f0c5018e91a95f Mon Sep 17 00:00:00 2001 From: NivLipetz Date: Sun, 22 Nov 2020 23:09:54 +0200 Subject: [PATCH 01/10] feat: report trends --- docs/openapi3.yaml | 117 ++++++++++++++++-- src/configManager/helpers/configDataMap.js | 1 - .../migrations/12_reports_reults_summary.js | 15 +++ .../database/sequelize/sequelizeConnector.js | 20 ++- src/reports/models/databaseConnector.js | 7 +- src/reports/models/statsManager.js | 38 +++--- src/tests/controllers/testsController.js | 13 +- src/tests/models/trends.js | 56 +++++++++ src/tests/routes/testsRoute.js | 1 + 9 files changed, 243 insertions(+), 25 deletions(-) create mode 100644 src/database/sequlize-handler/migrations/12_reports_reults_summary.js create mode 100644 src/tests/models/trends.js diff --git a/docs/openapi3.yaml b/docs/openapi3.yaml index a122022c2..ceffde83a 100644 --- a/docs/openapi3.yaml +++ b/docs/openapi3.yaml @@ -1154,7 +1154,7 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/benchmark_request' + $ref: '#/components/schemas/results_summary' '400': description: Bad request content: @@ -1171,8 +1171,8 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/benchmark_request' - description: The benchmark to add + $ref: '#/components/schemas/results_summary' + description: The results summary to mark as the benchmark required: true get: tags: @@ -1186,15 +1186,100 @@ paths: type: string format: uuid example: 4bf5d7ab-f310-4a64-8ec2-d65c06188ec1 - summary: Get a benchmark for test - description: Get a benchmark for a test + summary: Get the benchmark for test + description: Get the benchmark for a test responses: '200': description: Success content: application/json: schema: - $ref: '#/components/schemas/benchmark_request' + $ref: '#/components/schemas/results_summary' + '404': + description: Not found + content: + application/json: + schema: + $ref: '#/components/schemas/error_response' + '500': + description: Internal server error + content: + application/json: + schema: + $ref: '#/components/schemas/error_response' + /v1/tests/{test_id}/trends: + parameters: + - $ref: '#/components/parameters/context_id' + get: + tags: + - Trends + parameters: + - in: path + name: test_id + description: The test id. + required: true + schema: + type: string + format: uuid + example: 4bf5d7ab-f310-4a64-8ec2-d65c06188ec1 + - in: query + name: from + description: from when to analyze trends of a test + required: false + schema: + type: string + default: now-7d + example: now-7d + - in: query + name: to + description: until when to analyze trends of a test + required: false + schema: + type: string + default: now + example: now + - in: query + name: limit + description: maximum number of runs to include in the trend analysis for the timeframe given + required: false + schema: + type: integer + default: 250 + example: 250 + - in: query + name: threshold + description: minimum difference between different test run configurations that should be calculated in a specific trend + required: false + schema: + type: number + format: float + default: 0.9 + example: 0.9 + - in: query + name: arrival_rate + description: arrival rate of test run configuration that should be compared to + required: false + schema: + type: integer + default: 100 + example: 100 + - in: query + name: duration + description: duration of test run configuration that should be compared to + required: false + schema: + type: integer + default: 100 + example: 100 + summary: Get trends for a test + description: Get trends for a test + responses: + '200': + description: Success + content: + application/json: + schema: + $ref: '#/components/schemas/trends' '404': description: Not found content: @@ -1999,7 +2084,7 @@ components: - basic example: dsl - benchmark_request: + results_summary: type: object additionalProperties: false required: @@ -2041,6 +2126,22 @@ components: type: number p95: type: number + trends: + type: object + additionalProperties: false + required: + - latency + properties: + latency: + type: object + required: + - median + - p95 + properties: + median: + type: number + p95: + type: number dsl: description: A test that is made of scenarios base on domain specific language @@ -2612,6 +2713,8 @@ components: benchmark_weights_data: type: object description: The way score calualated + results_summary: + $ref: '#/components/schemas/results_summary' arrival_rate: type: number description: The arrival rate that was set for the test. This is the number of times per second that the test scenarios will run. diff --git a/src/configManager/helpers/configDataMap.js b/src/configManager/helpers/configDataMap.js index 6ec557e0a..facc203ae 100644 --- a/src/configManager/helpers/configDataMap.js +++ b/src/configManager/helpers/configDataMap.js @@ -48,7 +48,6 @@ const configDataMap = { type: 'json' }, [constConfig.CUSTOM_RUNNER_DEFINITION]: { value: process.env.CUSTOM_RUNNER_DEFINITION, type: 'json' } - }; module.exports.getConstType = (configValue) => { diff --git a/src/database/sequlize-handler/migrations/12_reports_reults_summary.js b/src/database/sequlize-handler/migrations/12_reports_reults_summary.js new file mode 100644 index 000000000..b3c223907 --- /dev/null +++ b/src/database/sequlize-handler/migrations/12_reports_reults_summary.js @@ -0,0 +1,15 @@ +const Sequelize = require('sequelize'); + +module.exports.up = async (query) => { + const reportsTable = await query.describeTable('reports'); + + if (!reportsTable.results_summary) { + await query.addColumn( + 'reports', 'results_summary', + Sequelize.DataTypes.TEXT('long')); + } +}; + +module.exports.down = async (query) => { + await query.removeColumn('reports', 'results_summary'); +}; diff --git a/src/reports/models/database/sequelize/sequelizeConnector.js b/src/reports/models/database/sequelize/sequelizeConnector.js index 63f136138..eaeda78d2 100644 --- a/src/reports/models/database/sequelize/sequelizeConnector.js +++ b/src/reports/models/database/sequelize/sequelizeConnector.js @@ -20,7 +20,8 @@ module.exports = { subscribeRunner, updateSubscriberWithStats, updateSubscriber, - updateReportBenchmark + updateReportBenchmark, + updateResultsSummary }; async function init(sequlizeClient) { @@ -120,6 +121,7 @@ async function deleteReport(testId, reportId) { } } +// TODO update only score async function updateReportBenchmark(testId, reportId, score, benchmarkData) { const benchmark = client.model('report'); const params = { score: score, benchmark_weights_data: benchmarkData }; @@ -133,6 +135,19 @@ async function updateReportBenchmark(testId, reportId, score, benchmarkData) { return res; } +async function updateResultsSummary(testId, reportId, resultsSummary) { + const benchmark = client.model('report'); + const params = { results_summary: resultsSummary }; + const options = { + where: { + test_id: testId, + report_id: reportId + } + }; + const res = await benchmark.update(params, options); + return res; +} + async function subscribeRunner(testId, reportId, runnerId, phaseStatus = constants.SUBSCRIBER_INITIALIZING_STAGE) { const newSubscriber = { runner_id: runnerId, @@ -344,6 +359,9 @@ async function initSchemas() { benchmark_weights_data: { type: Sequelize.DataTypes.TEXT('long') }, + results_summary: { + type: Sequelize.DataTypes.TEXT('long') + }, score: { type: Sequelize.DataTypes.FLOAT }, diff --git a/src/reports/models/databaseConnector.js b/src/reports/models/databaseConnector.js index 6e7b13dfd..974891f80 100644 --- a/src/reports/models/databaseConnector.js +++ b/src/reports/models/databaseConnector.js @@ -15,7 +15,8 @@ module.exports = { subscribeRunner, updateSubscriberWithStats, updateSubscriber, - updateReportBenchmark + updateReportBenchmark, + updateResultsSummary }; @@ -39,6 +40,10 @@ function updateReportBenchmark(testId, reportId, score, benchmarkData) { return databaseConnector.updateReportBenchmark(testId, reportId, score, benchmarkData); } +function updateResultsSummary(testId, reportId, resultsSummary) { + return databaseConnector.updateResultsSummary(testId, reportId, resultsSummary); +} + function getLastReports(limit, filter, contextId) { return databaseConnector.getLastReports(limit, filter, contextId); } diff --git a/src/reports/models/statsManager.js b/src/reports/models/statsManager.js index 5149c5e63..33ec0e0cc 100644 --- a/src/reports/models/statsManager.js +++ b/src/reports/models/statsManager.js @@ -26,7 +26,17 @@ module.exports.postStats = async (report, stats) => { } await databaseConnector.updateReport(report.test_id, report.report_id, { phase: report.phase, last_updated_at: statsTime }); report = await reportsManager.getReport(report.test_id, report.report_id); - const reportBenchmark = await updateReportBenchmarkIfNeeded(report); + + let reportBenchmark; + if (reportUtil.isAllRunnersInExpectedPhase(report, constants.SUBSCRIBER_DONE_STAGE)) { + await updateResultsSummary(report); + + const testBenchmarkData = await extractBenchmark(report.test_id); + if (testBenchmarkData) { + reportBenchmark = await updateReportBenchmark(report); + } + } + notifier.notifyIfNeeded(report, stats, reportBenchmark); return stats; @@ -43,24 +53,18 @@ async function updateSubscriberWithStatsInternal(report, stats) { await databaseConnector.updateSubscriberWithStats(report.test_id, report.report_id, stats.runner_id, stats.phase_status, JSON.stringify(parseData)); } -async function updateReportBenchmarkIfNeeded(report) { - if (!reportUtil.isAllRunnersInExpectedPhase(report, constants.SUBSCRIBER_DONE_STAGE)) { - return; - } +async function updateReportBenchmark(report, testBenchmarkData) { const config = await configHandler.getConfig(); const configBenchmark = { weights: config[configConsts.BENCHMARK_WEIGHTS], threshold: config[configConsts.BENCHMARK_THRESHOLD] }; - const testBenchmarkData = await extractBenchmark(report.test_id); - if (testBenchmarkData) { - const reportAggregate = await aggregateReportManager.aggregateReport(report); - const reportBenchmark = benchmarkCalculator.calculate(testBenchmarkData, reportAggregate.aggregate, configBenchmark.weights); - const { data, score } = reportBenchmark; - data[configConsts.BENCHMARK_THRESHOLD] = configBenchmark.threshold; - await databaseConnector.updateReportBenchmark(report.test_id, report.report_id, score, JSON.stringify(data)); - return reportBenchmark; - } + const reportAggregate = await aggregateReportManager.aggregateReport(report); + const reportBenchmark = benchmarkCalculator.calculate(testBenchmarkData, reportAggregate.aggregate, configBenchmark.weights); + const { data, score } = reportBenchmark; + data[configConsts.BENCHMARK_THRESHOLD] = configBenchmark.threshold; + await databaseConnector.updateReportBenchmark(report.test_id, report.report_id, score, JSON.stringify(data)); + return reportBenchmark; } async function extractBenchmark(testId) { @@ -71,3 +75,9 @@ async function extractBenchmark(testId) { return undefined; } } + +async function updateResultsSummary(report) { + const resultsSummary = report.data; // TODO + await databaseConnector.updateResultsSummary(report.test_id, report.report_id, JSON.stringify(resultsSummary)); + return resultsSummary; +} diff --git a/src/tests/controllers/testsController.js b/src/tests/controllers/testsController.js index 42c419749..fefbd2347 100644 --- a/src/tests/controllers/testsController.js +++ b/src/tests/controllers/testsController.js @@ -2,6 +2,7 @@ const manager = require('../models/manager'); const jobManager = require('../../jobs/models/jobManager'); +const trends = require('../models/trends'); module.exports = { upsertTest, @@ -10,7 +11,8 @@ module.exports = { getTests, getTestRevisions, insertTestBenchmark, - getBenchmark + getBenchmark, + getTrends }; async function insertTestBenchmark(req, res, next) { @@ -30,6 +32,15 @@ async function getBenchmark(req, res, next) { } } +async function getTrends(req, res, next) { + try { + const result = await trends.getTrends(req.params.test_id, req.query); + return res.status(200).json(result); + } catch (err){ + return next(err); + } +} + async function upsertTest(req, res, next) { try { const result = await manager.upsertTest(req.body, req.params.test_id); diff --git a/src/tests/models/trends.js b/src/tests/models/trends.js new file mode 100644 index 000000000..3c27689a8 --- /dev/null +++ b/src/tests/models/trends.js @@ -0,0 +1,56 @@ +'use strict'; +const reportsManager = require('../../reports/models/reportsManager'); + +module.exports = { + getTrends +} + +async function getTrends(testId, queryParams) { + const reports = reportsManager.getReports(testId); + const filteredReports = filterRelevantReports(reports, queryParams) + const trends = calculateTrends(filteredReports); + return trends; +} + +async function calculateTrends(report) { + // Todo get reports only in the last X days + // duration/ arrival_rate + const reports = await reportsManager.getReports(report.test_id); + const relevantReports = filterRelevantReports(reports, report.arrival_rate, report.duration); + + const allAggregatedReport = []; + + const promises = relevantReports.map(async report => { + const aggregatedReport = await aggregateReportManager.aggregateReport((report)); + allAggregatedReport.push(aggregatedReport); + }); + + await Promise.all(promises); + + const trends = []; + allAggregatedReport.forEach(aggregatedReport => { + trends.push({ + start_time: aggregatedReport.start_time, + latency: aggregatedReport.aggregate.latency, + rps: aggregatedReport.aggregate.rps + }); + }) + + return trends; +} + +function filterRelevantReports(reports, queryParams) { + const { to, from, threshold, arrival_rate: arrivalRate, duration } = queryParams; + // + // + // + // const minimumDate = new Date(); + // minimumDate.setDate(minimumDate.getDate() - configConsts.TREND_BACK_IN_DAYS); + // + // const relevantReports = reports.filter((report) => { + // const arrivalRateChange = Math.min(report.arrival_rate, arrivalRate) / Math.max(report.arrival_rate, arrivalRate); + // const durationRateChange = Math.min(report.duration, duration) / Math.max(report.duration, duration); + // return durationRateChange >= 0.9 && arrivalRateChange >= configConsts.TREND_THRESHOLD && report.start_time >= minimumDate; + // }); + // return relevantReports; +} \ No newline at end of file diff --git a/src/tests/routes/testsRoute.js b/src/tests/routes/testsRoute.js index c3638ab42..c6c3facc2 100644 --- a/src/tests/routes/testsRoute.js +++ b/src/tests/routes/testsRoute.js @@ -15,4 +15,5 @@ router.put('/:test_id', swaggerValidator.validate, testsVerifier.verifyProcessor router.post('/:test_id/benchmark', swaggerValidator.validate, testsVerifier.verifyTestExist, tests.insertTestBenchmark); router.get('/:test_id/benchmark', swaggerValidator.validate, testsVerifier.verifyTestExist, tests.getBenchmark); router.get('/:test_id/revisions', swaggerValidator.validate, tests.getTestRevisions); +router.get('/:test_id/trends', swaggerValidator.validate, testsVerifier.verifyTestExist, tests.getTrends); module.exports = router; From f14721bd88805a62a2bd33ec1ec1bc533d83d4d8 Mon Sep 17 00:00:00 2001 From: NivLipetz Date: Sat, 28 Nov 2020 19:40:52 +0200 Subject: [PATCH 02/10] feat: report trends --- docs/openapi3.yaml | 33 ++++++++---- .../database/sequelize/sequelizeConnector.js | 4 +- src/reports/models/reportsManager.js | 3 +- src/reports/models/statsManager.js | 27 +++++++--- src/tests/models/trends.js | 52 +++++++++---------- 5 files changed, 72 insertions(+), 47 deletions(-) diff --git a/docs/openapi3.yaml b/docs/openapi3.yaml index ceffde83a..28bdcb114 100644 --- a/docs/openapi3.yaml +++ b/docs/openapi3.yaml @@ -2126,22 +2126,33 @@ components: type: number p95: type: number + p99: + type: number trends: type: object additionalProperties: false required: - - latency + - shift + - latencies properties: - latency: - type: object - required: - - median - - p95 - properties: - median: - type: number - p95: - type: number + shift: + type: number + description: represents the incline/decline in the performance results calculated for the queried timeframe + latencies: + type: array + items: + type: object + required: + - median + - p95 + - p99 + properties: + median: + type: number + p95: + type: number + p99: + type: number dsl: description: A test that is made of scenarios base on domain specific language diff --git a/src/reports/models/database/sequelize/sequelizeConnector.js b/src/reports/models/database/sequelize/sequelizeConnector.js index eaeda78d2..332197ca0 100644 --- a/src/reports/models/database/sequelize/sequelizeConnector.js +++ b/src/reports/models/database/sequelize/sequelizeConnector.js @@ -136,7 +136,7 @@ async function updateReportBenchmark(testId, reportId, score, benchmarkData) { } async function updateResultsSummary(testId, reportId, resultsSummary) { - const benchmark = client.model('report'); + const report = client.model('report'); const params = { results_summary: resultsSummary }; const options = { where: { @@ -144,7 +144,7 @@ async function updateResultsSummary(testId, reportId, resultsSummary) { report_id: reportId } }; - const res = await benchmark.update(params, options); + const res = await report.update(params, options); return res; } diff --git a/src/reports/models/reportsManager.js b/src/reports/models/reportsManager.js index 5261fe2c3..2a0040e2d 100644 --- a/src/reports/models/reportsManager.js +++ b/src/reports/models/reportsManager.js @@ -199,7 +199,8 @@ function getReportResponse(summaryRow, config) { avg_rps: Number((totalRequests / reportDurationSeconds).toFixed(2)) || 0, last_success_rate: successRate, score: summaryRow.score ? summaryRow.score : undefined, - benchmark_weights_data: summaryRow.benchmark_weights_data ? JSON.parse(summaryRow.benchmark_weights_data) : undefined + benchmark_weights_data: summaryRow.benchmark_weights_data ? JSON.parse(summaryRow.benchmark_weights_data) : undefined, + results_summary: summaryRow.results_summary ? JSON.parse(summaryRow.results_summary) : undefined }; report.status = reportsStatusCalculator.calculateReportStatus(report, config); diff --git a/src/reports/models/statsManager.js b/src/reports/models/statsManager.js index 33ec0e0cc..d01def2a7 100644 --- a/src/reports/models/statsManager.js +++ b/src/reports/models/statsManager.js @@ -29,11 +29,12 @@ module.exports.postStats = async (report, stats) => { let reportBenchmark; if (reportUtil.isAllRunnersInExpectedPhase(report, constants.SUBSCRIBER_DONE_STAGE)) { - await updateResultsSummary(report); + const reportAggregate = await aggregateReportManager.aggregateReport(report); + await updateResultsSummary(reportAggregate); const testBenchmarkData = await extractBenchmark(report.test_id); if (testBenchmarkData) { - reportBenchmark = await updateReportBenchmark(report); + reportBenchmark = await updateReportBenchmark(reportAggregate); } } @@ -53,17 +54,16 @@ async function updateSubscriberWithStatsInternal(report, stats) { await databaseConnector.updateSubscriberWithStats(report.test_id, report.report_id, stats.runner_id, stats.phase_status, JSON.stringify(parseData)); } -async function updateReportBenchmark(report, testBenchmarkData) { +async function updateReportBenchmark(reportAggregate, testBenchmarkData) { const config = await configHandler.getConfig(); const configBenchmark = { weights: config[configConsts.BENCHMARK_WEIGHTS], threshold: config[configConsts.BENCHMARK_THRESHOLD] }; - const reportAggregate = await aggregateReportManager.aggregateReport(report); const reportBenchmark = benchmarkCalculator.calculate(testBenchmarkData, reportAggregate.aggregate, configBenchmark.weights); const { data, score } = reportBenchmark; data[configConsts.BENCHMARK_THRESHOLD] = configBenchmark.threshold; - await databaseConnector.updateReportBenchmark(report.test_id, report.report_id, score, JSON.stringify(data)); + await databaseConnector.updateReportBenchmark(reportAggregate.test_id, reportAggregate.report_id, score, JSON.stringify(data)); return reportBenchmark; } @@ -76,8 +76,21 @@ async function extractBenchmark(testId) { } } -async function updateResultsSummary(report) { - const resultsSummary = report.data; // TODO +async function updateResultsSummary(reportAggregate) { + const aggregatedResults = reportAggregate.aggregate; + const resultsSummary = { + errors: aggregatedResults.errors, + codes: aggregatedResults.codes, + rps: { + mean: aggregatedResults.rps.mean, + count: aggregatedResults.rps.count + }, + latency: { + median: aggregatedResults.latency.median, + p95: aggregatedResults.latency.p95, + p99: aggregatedResults.latency.p99 + } + }; await databaseConnector.updateResultsSummary(report.test_id, report.report_id, JSON.stringify(resultsSummary)); return resultsSummary; } diff --git a/src/tests/models/trends.js b/src/tests/models/trends.js index 3c27689a8..c2102e171 100644 --- a/src/tests/models/trends.js +++ b/src/tests/models/trends.js @@ -6,43 +6,43 @@ module.exports = { } async function getTrends(testId, queryParams) { - const reports = reportsManager.getReports(testId); + const reports = await reportsManager.getReports(testId); const filteredReports = filterRelevantReports(reports, queryParams) const trends = calculateTrends(filteredReports); return trends; } -async function calculateTrends(report) { - // Todo get reports only in the last X days - // duration/ arrival_rate - const reports = await reportsManager.getReports(report.test_id); - const relevantReports = filterRelevantReports(reports, report.arrival_rate, report.duration); - - const allAggregatedReport = []; - - const promises = relevantReports.map(async report => { - const aggregatedReport = await aggregateReportManager.aggregateReport((report)); - allAggregatedReport.push(aggregatedReport); - }); - - await Promise.all(promises); +async function calculateTrends(reports) { + const trends = { + shift: 1, + latencies: [] + } + reports.forEach((report) => trends.latencies.push(report.results_summary.latency)); + return trends; - const trends = []; - allAggregatedReport.forEach(aggregatedReport => { - trends.push({ - start_time: aggregatedReport.start_time, - latency: aggregatedReport.aggregate.latency, - rps: aggregatedReport.aggregate.rps - }); - }) + // const allAggregatedReport = []; - return trends; + // const promises = reports.map(async report => { + // const aggregatedReport = await aggregateReportManager.aggregateReport((report)); + // allAggregatedReport.push(aggregatedReport); + // }); + // + // await Promise.all(promises); + // + // const trends = []; + // allAggregatedReport.forEach(aggregatedReport => { + // trends.push({ + // start_time: aggregatedReport.start_time, + // latency: aggregatedReport.aggregate.latency, + // rps: aggregatedReport.aggregate.rps + // }); + // }); } function filterRelevantReports(reports, queryParams) { const { to, from, threshold, arrival_rate: arrivalRate, duration } = queryParams; - // - // + return reports; + // // const minimumDate = new Date(); // minimumDate.setDate(minimumDate.getDate() - configConsts.TREND_BACK_IN_DAYS); From 39eb2f9347ba888fc131015e9f8cb5e4c2c4baf4 Mon Sep 17 00:00:00 2001 From: NivLipetz Date: Sat, 28 Nov 2020 19:54:29 +0200 Subject: [PATCH 03/10] feat: report trends --- docs/openapi3.yaml | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/docs/openapi3.yaml b/docs/openapi3.yaml index 28bdcb114..860980ab7 100644 --- a/docs/openapi3.yaml +++ b/docs/openapi3.yaml @@ -2133,27 +2133,15 @@ components: additionalProperties: false required: - shift - - latencies + - report_summaries properties: shift: type: number description: represents the incline/decline in the performance results calculated for the queried timeframe - latencies: + report_summaries: type: array items: - type: object - required: - - median - - p95 - - p99 - properties: - median: - type: number - p95: - type: number - p99: - type: number - + $ref: '#/components/schemas/results_summary' dsl: description: A test that is made of scenarios base on domain specific language required: From f168daad3a14b4a7cb4f67749daef6e1ee6b5294 Mon Sep 17 00:00:00 2001 From: NivLipetz Date: Sat, 28 Nov 2020 20:40:47 +0200 Subject: [PATCH 04/10] feat: report trends --- src/reports/models/statsManager.js | 2 +- src/tests/models/trends.js | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/reports/models/statsManager.js b/src/reports/models/statsManager.js index d01def2a7..f00ebb2b4 100644 --- a/src/reports/models/statsManager.js +++ b/src/reports/models/statsManager.js @@ -91,6 +91,6 @@ async function updateResultsSummary(reportAggregate) { p99: aggregatedResults.latency.p99 } }; - await databaseConnector.updateResultsSummary(report.test_id, report.report_id, JSON.stringify(resultsSummary)); + await databaseConnector.updateResultsSummary(reportAggregate.test_id, reportAggregate.report_id, JSON.stringify(resultsSummary)); return resultsSummary; } diff --git a/src/tests/models/trends.js b/src/tests/models/trends.js index c2102e171..4a116e65b 100644 --- a/src/tests/models/trends.js +++ b/src/tests/models/trends.js @@ -15,9 +15,9 @@ async function getTrends(testId, queryParams) { async function calculateTrends(reports) { const trends = { shift: 1, - latencies: [] + report_summaries: [] } - reports.forEach((report) => trends.latencies.push(report.results_summary.latency)); + reports.forEach((report) => trends.report_summaries.push(report.results_summary)); return trends; // const allAggregatedReport = []; From 62bdf4a160d5910d450760398512829c242db9e2 Mon Sep 17 00:00:00 2001 From: NivLipetz Date: Sun, 29 Nov 2020 23:33:05 +0200 Subject: [PATCH 05/10] fix: add to trends api report and job ids --- docs/openapi3.yaml | 21 +++++++++++++++++++-- src/tests/models/trends.js | 11 +++++++++-- 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/docs/openapi3.yaml b/docs/openapi3.yaml index 860980ab7..f8066b13b 100644 --- a/docs/openapi3.yaml +++ b/docs/openapi3.yaml @@ -2133,14 +2133,31 @@ components: additionalProperties: false required: - shift - - report_summaries + - reports properties: shift: type: number description: represents the incline/decline in the performance results calculated for the queried timeframe - report_summaries: + reports: type: array items: + $ref: '#/components/schemas/trends_reports_summary' + trends_reports_summary: + type: object + additionalProperties: false + required: + - report_id + - job_id + - results_summary + properties: + job_id: + type: string + format: uuid + description: id of the job that created the report + report_id: + type: string + format: uuid + results_summary: $ref: '#/components/schemas/results_summary' dsl: description: A test that is made of scenarios base on domain specific language diff --git a/src/tests/models/trends.js b/src/tests/models/trends.js index 4a116e65b..eb348cdc1 100644 --- a/src/tests/models/trends.js +++ b/src/tests/models/trends.js @@ -15,9 +15,16 @@ async function getTrends(testId, queryParams) { async function calculateTrends(reports) { const trends = { shift: 1, - report_summaries: [] + reports: [] } - reports.forEach((report) => trends.report_summaries.push(report.results_summary)); + reports.forEach((report) => { + const reportSummary = { + report_id: report.report_id, + job_id: report.job_id, + results_summary: report.results_summary + } + trends.reports.push(reportSummary) + }); return trends; // const allAggregatedReport = []; From c476f310c31f8a1ebcfa625ac9b9fe29e19d428d Mon Sep 17 00:00:00 2001 From: NivLipetz Date: Wed, 2 Dec 2020 00:07:14 +0200 Subject: [PATCH 06/10] fix: filter report trends based on query params in API --- docs/openapi3.yaml | 55 ++++++++++++++---------- src/tests/models/trends.js | 85 ++++++++++++++++++++++---------------- 2 files changed, 82 insertions(+), 58 deletions(-) diff --git a/docs/openapi3.yaml b/docs/openapi3.yaml index f8066b13b..e245d1ddd 100644 --- a/docs/openapi3.yaml +++ b/docs/openapi3.yaml @@ -1224,20 +1224,20 @@ paths: example: 4bf5d7ab-f310-4a64-8ec2-d65c06188ec1 - in: query name: from - description: from when to analyze trends of a test + description: Epoch time in milliseconds from when to analyze trends of a test required: false schema: - type: string - default: now-7d - example: now-7d + type: integer + format: int64 + example: 1464829513623 - in: query name: to - description: until when to analyze trends of a test + description: Epoch time in milliseconds until when to analyze trends of a test required: false schema: - type: string - default: now - example: now + type: integer + format: int64 + example: 1464829513623 - in: query name: limit description: maximum number of runs to include in the trend analysis for the timeframe given @@ -1247,30 +1247,41 @@ paths: default: 250 example: 250 - in: query - name: threshold - description: minimum difference between different test run configurations that should be calculated in a specific trend + name: min_rate + description: minimum rate of test run configuration that should be compared to required: false schema: - type: number - format: float - default: 0.9 - example: 0.9 + type: integer - in: query - name: arrival_rate - description: arrival rate of test run configuration that should be compared to + name: max_rate + description: minimum rate of test run configuration that should be compared to required: false schema: type: integer - default: 100 - example: 100 - in: query - name: duration - description: duration of test run configuration that should be compared to + name: min_duration + description: minimum rate of test run configuration that should be compared to + required: false + schema: + type: integer + - in: query + name: max_duration + description: minimum rate of test run configuration that should be compared to + required: false + schema: + type: integer + - in: query + name: min_virtual_users + description: minimum rate of test run configuration that should be compared to + required: false + schema: + type: integer + - in: query + name: max_virtual_users + description: minimum rate of test run configuration that should be compared to required: false schema: type: integer - default: 100 - example: 100 summary: Get trends for a test description: Get trends for a test responses: diff --git a/src/tests/models/trends.js b/src/tests/models/trends.js index eb348cdc1..7d652b8b5 100644 --- a/src/tests/models/trends.js +++ b/src/tests/models/trends.js @@ -1,5 +1,6 @@ 'use strict'; const reportsManager = require('../../reports/models/reportsManager'); +const FOURTEEN_DAYS_IN_MS = 12096e5; module.exports = { getTrends @@ -18,46 +19,58 @@ async function calculateTrends(reports) { reports: [] } reports.forEach((report) => { - const reportSummary = { - report_id: report.report_id, - job_id: report.job_id, - results_summary: report.results_summary + if (report.results_summary) { + const reportSummary = { + report_id: report.report_id, + job_id: report.job_id, + start_time: report.start_time, + results_summary: report.results_summary + } + trends.reports.push(reportSummary) } - trends.reports.push(reportSummary) }); return trends; - - // const allAggregatedReport = []; - - // const promises = reports.map(async report => { - // const aggregatedReport = await aggregateReportManager.aggregateReport((report)); - // allAggregatedReport.push(aggregatedReport); - // }); - // - // await Promise.all(promises); - // - // const trends = []; - // allAggregatedReport.forEach(aggregatedReport => { - // trends.push({ - // start_time: aggregatedReport.start_time, - // latency: aggregatedReport.aggregate.latency, - // rps: aggregatedReport.aggregate.rps - // }); - // }); } function filterRelevantReports(reports, queryParams) { - const { to, from, threshold, arrival_rate: arrivalRate, duration } = queryParams; - return reports; - - // - // const minimumDate = new Date(); - // minimumDate.setDate(minimumDate.getDate() - configConsts.TREND_BACK_IN_DAYS); - // - // const relevantReports = reports.filter((report) => { - // const arrivalRateChange = Math.min(report.arrival_rate, arrivalRate) / Math.max(report.arrival_rate, arrivalRate); - // const durationRateChange = Math.min(report.duration, duration) / Math.max(report.duration, duration); - // return durationRateChange >= 0.9 && arrivalRateChange >= configConsts.TREND_THRESHOLD && report.start_time >= minimumDate; - // }); - // return relevantReports; + const { + to = Date.now(), + from = Date.now() - FOURTEEN_DAYS_IN_MS, + min_duration: minDuration, + max_duration: maxDuration, + min_rate: minRate, + max_rate: maxRate, + min_virtual_users: maxVUsers, + max_virtual_users: minVUsers + } = queryParams; + + const fromDate = new Date(from); + const toDate = new Date(to); + + const relevantReports = reports.filter((report) => { + const afterFromDate = report.start_time >= fromDate; + const beforeToDate = report.start_time < toDate; + const matchesDuration = matchReportProperty(report, 'duration', minDuration, maxDuration); + const matchesRate = matchReportProperty(report, 'arrival_rate', minRate, maxRate); + const matchesVUsers = matchReportProperty(report, 'max_virtual_users', minVUsers, maxVUsers); + + return afterFromDate && beforeToDate && matchesDuration && matchesRate && matchesVUsers; + }); + + return relevantReports; +} + +function matchReportProperty(report, property, min, max) { + let match; + if (!min && !max) { + match = true + } else if (min && !max) { + match = report[property] >= min + } else if (!min && max) { + match = report[property] < max + } else { + match = report[property] >= min && report[property] < max + } + + return match; } \ No newline at end of file From 85054329a959928759ac30635dd130b02c580dbb Mon Sep 17 00:00:00 2001 From: NivLipetz Date: Wed, 2 Dec 2020 00:38:04 +0200 Subject: [PATCH 07/10] fix: calculate shift in trends --- docs/openapi3.yaml | 13 ++++++++++++- src/tests/models/trends.js | 18 +++++++++++++++--- 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/docs/openapi3.yaml b/docs/openapi3.yaml index e245d1ddd..428fa05fa 100644 --- a/docs/openapi3.yaml +++ b/docs/openapi3.yaml @@ -1282,6 +1282,17 @@ paths: required: false schema: type: integer + - in: query + name: shift_comparator + description: minimum rate of test run configuration that should be compared to + required: false + schema: + type: string + default: p95 + enum: + - median + - p95 + - p99 summary: Get trends for a test description: Get trends for a test responses: @@ -2148,7 +2159,7 @@ components: properties: shift: type: number - description: represents the incline/decline in the performance results calculated for the queried timeframe + description: represents the incline/decline in the performance results calculated for the queried timeframe in % reports: type: array items: diff --git a/src/tests/models/trends.js b/src/tests/models/trends.js index 7d652b8b5..6c798074c 100644 --- a/src/tests/models/trends.js +++ b/src/tests/models/trends.js @@ -9,15 +9,17 @@ module.exports = { async function getTrends(testId, queryParams) { const reports = await reportsManager.getReports(testId); const filteredReports = filterRelevantReports(reports, queryParams) - const trends = calculateTrends(filteredReports); + const trends = calculateTrends(filteredReports, queryParams); return trends; } -async function calculateTrends(reports) { +async function calculateTrends(reports, queryParams) { + const { shift_comparator: shiftComparator } = queryParams; const trends = { - shift: 1, + shift: 0, reports: [] } + reports.forEach((report) => { if (report.results_summary) { const reportSummary = { @@ -29,9 +31,19 @@ async function calculateTrends(reports) { trends.reports.push(reportSummary) } }); + + trends.shift = calculateShift(trends.reports, shiftComparator); + return trends; } +function calculateShift(reports, shiftComparator) { + const mostRecentResult = reports[0].results_summary.latency[shiftComparator]; + const leastRecentResult = reports[reports.length-1].results_summary.latency[shiftComparator]; + const shift = (leastRecentResult - mostRecentResult) / leastRecentResult; + return shift; +} + function filterRelevantReports(reports, queryParams) { const { to = Date.now(), From 6652ba9a78aeed1497ad84600833ade544d1d40f Mon Sep 17 00:00:00 2001 From: NivLipetz Date: Wed, 2 Dec 2020 00:43:37 +0200 Subject: [PATCH 08/10] fix: add limit reports to trends response --- docs/openapi3.yaml | 4 ++-- src/tests/models/trends.js | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/openapi3.yaml b/docs/openapi3.yaml index 428fa05fa..35f9b579b 100644 --- a/docs/openapi3.yaml +++ b/docs/openapi3.yaml @@ -1244,8 +1244,8 @@ paths: required: false schema: type: integer - default: 250 - example: 250 + default: 100 + example: 100 - in: query name: min_rate description: minimum rate of test run configuration that should be compared to diff --git a/src/tests/models/trends.js b/src/tests/models/trends.js index 6c798074c..4527d2606 100644 --- a/src/tests/models/trends.js +++ b/src/tests/models/trends.js @@ -48,6 +48,7 @@ function filterRelevantReports(reports, queryParams) { const { to = Date.now(), from = Date.now() - FOURTEEN_DAYS_IN_MS, + limit, min_duration: minDuration, max_duration: maxDuration, min_rate: minRate, @@ -69,7 +70,7 @@ function filterRelevantReports(reports, queryParams) { return afterFromDate && beforeToDate && matchesDuration && matchesRate && matchesVUsers; }); - return relevantReports; + return relevantReports.slice(0, limit); } function matchReportProperty(report, property, min, max) { From bb05d2d8ef826f3f2a2405100c14575d4063ea16 Mon Sep 17 00:00:00 2001 From: NivLipetz Date: Wed, 2 Dec 2020 08:23:10 +0200 Subject: [PATCH 09/10] fix: return shift as percentage --- src/tests/models/trends.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tests/models/trends.js b/src/tests/models/trends.js index 4527d2606..c30b3dc5e 100644 --- a/src/tests/models/trends.js +++ b/src/tests/models/trends.js @@ -40,7 +40,7 @@ async function calculateTrends(reports, queryParams) { function calculateShift(reports, shiftComparator) { const mostRecentResult = reports[0].results_summary.latency[shiftComparator]; const leastRecentResult = reports[reports.length-1].results_summary.latency[shiftComparator]; - const shift = (leastRecentResult - mostRecentResult) / leastRecentResult; + const shift = (leastRecentResult - mostRecentResult) / leastRecentResult * 100; return shift; } From 4cacc96aafc2151bb17feb708d2e383e7a01f611 Mon Sep 17 00:00:00 2001 From: NivLipetz Date: Wed, 2 Dec 2020 21:24:43 +0200 Subject: [PATCH 10/10] test: add results_summary to reports api test --- src/reports/models/statsManager.js | 4 +- .../reports/reportsApi-test.js | 9 ++++ .../reporter/models/reportsManager-test.js | 42 +++++++++++++++++-- 3 files changed, 50 insertions(+), 5 deletions(-) diff --git a/src/reports/models/statsManager.js b/src/reports/models/statsManager.js index f00ebb2b4..d1f7850a0 100644 --- a/src/reports/models/statsManager.js +++ b/src/reports/models/statsManager.js @@ -34,7 +34,7 @@ module.exports.postStats = async (report, stats) => { const testBenchmarkData = await extractBenchmark(report.test_id); if (testBenchmarkData) { - reportBenchmark = await updateReportBenchmark(reportAggregate); + reportBenchmark = await updateReportBenchmark(testBenchmarkData, reportAggregate); } } @@ -54,7 +54,7 @@ async function updateSubscriberWithStatsInternal(report, stats) { await databaseConnector.updateSubscriberWithStats(report.test_id, report.report_id, stats.runner_id, stats.phase_status, JSON.stringify(parseData)); } -async function updateReportBenchmark(reportAggregate, testBenchmarkData) { +async function updateReportBenchmark(testBenchmarkData, reportAggregate) { const config = await configHandler.getConfig(); const configBenchmark = { weights: config[configConsts.BENCHMARK_WEIGHTS], diff --git a/tests/integration-tests/reports/reportsApi-test.js b/tests/integration-tests/reports/reportsApi-test.js index fd5f1d33b..682188e26 100644 --- a/tests/integration-tests/reports/reportsApi-test.js +++ b/tests/integration-tests/reports/reportsApi-test.js @@ -787,6 +787,11 @@ const jobPlatform = process.env.JOB_PLATFORM; await assertPostStats(testId, reportId, runnerId, constants.SUBSCRIBER_INTERMEDIATE_STAGE); await assertReportStatus(testId, reportId, constants.REPORT_IN_PROGRESS_STATUS); await assertPostStats(testId, reportId, runnerId, constants.SUBSCRIBER_DONE_STAGE); + + const getReportResponse = await reportsRequestCreator.getReport(testId, reportId); + expect(getReportResponse.status).to.be.equal(200); + expect(getReportResponse.body).to.have.property('results_summary'); + expect(getReportResponse.body).to.have.property('status').and.to.be.equal(constants.REPORT_FINISHED_STATUS); }); it('should fetch a report successfully with status failed after a runner subscribed and aborted', async function () { const jobName = 'jobName'; @@ -817,6 +822,10 @@ const jobPlatform = process.env.JOB_PLATFORM; await assertRunnerSubscriptionToReport(testId, reportId, runnerId); await assertPostStats(testId, reportId, runnerId, constants.REPORT_ABORTED_STATUS); await assertReportStatus(testId, reportId, constants.REPORT_ABORTED_STATUS); + + const getReportResponse = await reportsRequestCreator.getReport(testId, reportId); + expect(getReportResponse.status).to.be.equal(200); + expect(getReportResponse.body).to.not.have.property('results_summary'); }); it('should create a report successfully - a complete happy flow cycle', async function () { const jobName = 'jobName'; diff --git a/tests/unit-tests/reporter/models/reportsManager-test.js b/tests/unit-tests/reporter/models/reportsManager-test.js index cef1c3257..f5ca890ae 100644 --- a/tests/unit-tests/reporter/models/reportsManager-test.js +++ b/tests/unit-tests/reporter/models/reportsManager-test.js @@ -94,6 +94,7 @@ describe('Reports manager tests', function () { let benchmarkCalculatorStub; let updateReportBenchmarkStub; let testManagerGetTestStub; + let updateResultsSummaryStub; before(() => { sandbox = sinon.sandbox.create(); @@ -110,6 +111,7 @@ describe('Reports manager tests', function () { aggregateReportManagerStub = sandbox.stub(aggregateReportManager, 'aggregateReport'); benchmarkCalculatorStub = sandbox.stub(benchmarkCalculator, 'calculate'); updateReportBenchmarkStub = sandbox.stub(databaseConnector, 'updateReportBenchmark'); + updateResultsSummaryStub = sandbox.stub(databaseConnector, 'updateResultsSummary'); databaseUpdateReportStub = sandbox.stub(databaseConnector, 'updateReport'); databaseDeleteReportStub = sandbox.stub(databaseConnector, 'deleteReport'); testManagerGetTestStub = sandbox.stub(testManager, 'getTest'); @@ -708,8 +710,26 @@ describe('Reports manager tests', function () { it('when report done and have benchmark data ', async () => { databaseGetReportStub.resolves([REPORT_DONE]); + updateResultsSummaryStub.resolves(); getBenchmarkStub.resolves({ test: 'some benchmark data' }); - aggregateReportManagerStub.resolves({ aggregate: { test: 'some aggregate data' } }); + const aggregateReport = { + test_id: 'test_id', + report_id: 'report_id', + aggregate: { + errors: { }, + codes: { }, + rps: { + mean: 1, + count: 10 + }, + latency: { + median: 1, + p95: 2, + p99: 3 + } + } + }; + aggregateReportManagerStub.resolves(aggregateReport); benchmarkCalculatorStub.returns({ score: 5.5, data: { test: 'some calculate data' } }); updateReportBenchmarkStub.resolves(); const stats = { phase_status: 'done', data: JSON.stringify({ median: 1 }) }; @@ -721,7 +741,7 @@ describe('Reports manager tests', function () { updateReportBenchmarkStub.callCount.should.eql(1); should(getBenchmarkStub.args).eql([['test_id']]); - should(benchmarkCalculatorStub.args).eql([[{ test: 'some benchmark data' }, { test: 'some aggregate data' }, { + should(benchmarkCalculatorStub.args).eql([[{ test: 'some benchmark data' }, aggregateReport.aggregate, { config: 'some value' }]]); should(updateReportBenchmarkStub.args[0][0]).eql('test_id'); @@ -731,12 +751,28 @@ describe('Reports manager tests', function () { }); it('when report done and dont have benchmark data ', async () => { databaseGetReportStub.resolves([REPORT_DONE]); + updateResultsSummaryStub.resolves(); getBenchmarkStub.resolves(); + aggregateReportManagerStub.resolves({ + aggregate: { + errors: { }, + codes: { }, + rps: { + mean: 1, + count: 10 + }, + latency: { + median: 1, + p95: 2, + p99: 3 + } + } + }); const stats = { phase_status: 'done', data: JSON.stringify({ median: 1 }) }; await statsManager.postStats('test_id', stats); getBenchmarkStub.callCount.should.eql(1); - aggregateReportManagerStub.callCount.should.eql(0); + aggregateReportManagerStub.callCount.should.eql(1); benchmarkCalculatorStub.callCount.should.eql(0); updateReportBenchmarkStub.callCount.should.eql(0);