Feat/dynamic trends

docs/openapi3.yaml

@@ -1154,7 +1154,7 @@ paths:
           content:
             application/json:
               schema:
-                $ref: '#/components/schemas/benchmark_request'
+                $ref: '#/components/schemas/results_summary'
         '400':
           description: Bad request
           content:
@@ -1171,8 +1171,8 @@ paths:
         content:
           application/json:
             schema:
-              $ref: '#/components/schemas/benchmark_request'
-        description: The benchmark to add
+              $ref: '#/components/schemas/results_summary'
+        description: The results summary to mark as the benchmark
         required: true
     get:
       tags:
@@ -1186,15 +1186,122 @@ paths:
             type: string
             format: uuid
             example: 4bf5d7ab-f310-4a64-8ec2-d65c06188ec1
-      summary: Get a benchmark for test
-      description:  Get a benchmark for a test
+      summary: Get the benchmark for test
+      description:  Get the benchmark for a test
       responses:
         '200':
           description: Success
           content:
             application/json:
               schema:
-                $ref: '#/components/schemas/benchmark_request'
+                $ref: '#/components/schemas/results_summary'
+        '404':
+          description: Not found
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/error_response'
+        '500':
+          description: Internal server error
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/error_response'
+  /v1/tests/{test_id}/trends:
+    parameters:
+      - $ref: '#/components/parameters/context_id'
+    get:
+      tags:
+        - Trends
+      parameters:
+        - in: path
+          name: test_id
+          description: The test id.
+          required: true
+          schema:
+            type: string
+            format: uuid
+            example: 4bf5d7ab-f310-4a64-8ec2-d65c06188ec1
+        - in: query
+          name: from
+          description: Epoch time in milliseconds from when to analyze trends of a test
+          required: false
+          schema:
+            type: integer
+            format: int64
+            example: 1464829513623
+        - in: query
+          name: to
+          description: Epoch time in milliseconds until when to analyze trends of a test
+          required: false
+          schema:
+            type: integer
+            format: int64
+            example: 1464829513623
+        - in: query
+          name: limit
+          description: maximum number of runs to include in the trend analysis for the timeframe given
+          required: false
+          schema:
+            type: integer
+            default: 100
+            example: 100
+        - in: query
+          name: min_rate
+          description: minimum rate of test run configuration that should be compared to
+          required: false
+          schema:
+            type: integer
+        - in: query
+          name: max_rate
+          description: minimum rate of test run configuration that should be compared to
+          required: false
+          schema:
+            type: integer
+        - in: query
+          name: min_duration
+          description: minimum rate of test run configuration that should be compared to
+          required: false
+          schema:
+            type: integer
+        - in: query
+          name: max_duration
+          description: minimum rate of test run configuration that should be compared to
+          required: false
+          schema:
+            type: integer
+        - in: query
+          name: min_virtual_users
+          description: minimum rate of test run configuration that should be compared to
+          required: false
+          schema:
+            type: integer
+        - in: query
+          name: max_virtual_users
+          description: minimum rate of test run configuration that should be compared to
+          required: false
+          schema:
+            type: integer
+        - in: query
+          name: shift_comparator
+          description: minimum rate of test run configuration that should be compared to
+          required: false
+          schema:
+            type: string
+            default: p95
+            enum:
+              - median
+              - p95
+              - p99
+      summary: Get trends for a test
+      description:  Get trends for a test
+      responses:
+        '200':
+          description: Success
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/trends'
         '404':
           description: Not found
           content:
@@ -1999,7 +2106,7 @@ components:
         - basic
         example: dsl
 
-    benchmark_request:
+    results_summary:
       type: object
       additionalProperties: false
       required:
@@ -2041,7 +2148,39 @@ components:
               type: number
             p95:
               type: number
-
+            p99:
+              type: number
+    trends:
+      type: object
+      additionalProperties: false
+      required:
+        - shift
+        - reports
+      properties:
+        shift:
+          type: number
+          description: represents the incline/decline in the performance results calculated for the queried timeframe in %
+        reports:
+          type: array
+          items:
+            $ref: '#/components/schemas/trends_reports_summary'
+    trends_reports_summary:
+      type: object
+      additionalProperties: false
+      required:
+        - report_id
+        - job_id
+        - results_summary
+      properties:
+        job_id:
+          type: string
+          format: uuid
+          description: id of the job that created the report
+        report_id:
+          type: string
+          format: uuid
+        results_summary:
+            $ref: '#/components/schemas/results_summary'
     dsl:
       description: A test that is made of scenarios base on domain specific language
       required:
@@ -2612,6 +2751,8 @@ components:
         benchmark_weights_data:
           type: object
           description: The way score calualated
+        results_summary:
+          $ref: '#/components/schemas/results_summary'
         arrival_rate:
           type: number
           description: The arrival rate that was set for the test. This is the number of times per second that the test scenarios will run.

src/configManager/helpers/configDataMap.js

@@ -48,7 +48,6 @@ const configDataMap = {
         type: 'json'
     },
     [constConfig.CUSTOM_RUNNER_DEFINITION]: { value: process.env.CUSTOM_RUNNER_DEFINITION, type: 'json' }
-
 };
 
 module.exports.getConstType = (configValue) => {

src/database/sequlize-handler/migrations/12_reports_reults_summary.js

@@ -0,0 +1,15 @@
+const Sequelize = require('sequelize');
+
+module.exports.up = async (query) => {
+    const reportsTable = await query.describeTable('reports');
+
+    if (!reportsTable.results_summary) {
+        await query.addColumn(
+            'reports', 'results_summary',
+            Sequelize.DataTypes.TEXT('long'));
+    }
+};
+
+module.exports.down = async (query) => {
+    await query.removeColumn('reports', 'results_summary');
+};

src/reports/models/database/sequelize/sequelizeConnector.js

@@ -20,7 +20,8 @@ module.exports = {
     subscribeRunner,
     updateSubscriberWithStats,
     updateSubscriber,
-    updateReportBenchmark
+    updateReportBenchmark,
+    updateResultsSummary
 };
 
 async function init(sequlizeClient) {
@@ -120,6 +121,7 @@ async function deleteReport(testId, reportId) {
     }
 }
 
+// TODO update only score
 async function updateReportBenchmark(testId, reportId, score, benchmarkData) {
     const benchmark = client.model('report');
     const params = { score: score, benchmark_weights_data: benchmarkData };
@@ -133,6 +135,19 @@ async function updateReportBenchmark(testId, reportId, score, benchmarkData) {
     return res;
 }
 
+async function updateResultsSummary(testId, reportId, resultsSummary) {
+    const report = client.model('report');
+    const params = { results_summary: resultsSummary };
+    const options = {
+        where: {
+            test_id: testId,
+            report_id: reportId
+        }
+    };
+    const res = await report.update(params, options);
+    return res;
+}
+
 async function subscribeRunner(testId, reportId, runnerId, phaseStatus = constants.SUBSCRIBER_INITIALIZING_STAGE) {
     const newSubscriber = {
         runner_id: runnerId,
@@ -344,6 +359,9 @@ async function initSchemas() {
         benchmark_weights_data: {
             type: Sequelize.DataTypes.TEXT('long')
         },
+        results_summary: {
+            type: Sequelize.DataTypes.TEXT('long')
+        },
         score: {
             type: Sequelize.DataTypes.FLOAT
         },

src/reports/models/databaseConnector.js

@@ -15,7 +15,8 @@ module.exports = {
     subscribeRunner,
     updateSubscriberWithStats,
     updateSubscriber,
-    updateReportBenchmark
+    updateReportBenchmark,
+    updateResultsSummary
 
 };
 
@@ -39,6 +40,10 @@ function updateReportBenchmark(testId, reportId, score, benchmarkData) {
     return databaseConnector.updateReportBenchmark(testId, reportId, score, benchmarkData);
 }
 
+function updateResultsSummary(testId, reportId, resultsSummary) {
+    return databaseConnector.updateResultsSummary(testId, reportId, resultsSummary);
+}
+
 function getLastReports(limit, filter, contextId) {
     return databaseConnector.getLastReports(limit, filter, contextId);
 }

src/reports/models/reportsManager.js

@@ -199,7 +199,8 @@ function getReportResponse(summaryRow, config) {
         avg_rps: Number((totalRequests / reportDurationSeconds).toFixed(2)) || 0,
         last_success_rate: successRate,
         score: summaryRow.score ? summaryRow.score : undefined,
-        benchmark_weights_data: summaryRow.benchmark_weights_data ? JSON.parse(summaryRow.benchmark_weights_data) : undefined
+        benchmark_weights_data: summaryRow.benchmark_weights_data ? JSON.parse(summaryRow.benchmark_weights_data) : undefined,
+        results_summary: summaryRow.results_summary ? JSON.parse(summaryRow.results_summary) : undefined
     };
 
     report.status = reportsStatusCalculator.calculateReportStatus(report, config);

src/reports/models/statsManager.js

@@ -26,7 +26,18 @@ module.exports.postStats = async (report, stats) => {
     }
     await databaseConnector.updateReport(report.test_id, report.report_id, { phase: report.phase, last_updated_at: statsTime });
     report = await reportsManager.getReport(report.test_id, report.report_id);
-    const reportBenchmark = await updateReportBenchmarkIfNeeded(report);
+
+    let reportBenchmark;
+    if (reportUtil.isAllRunnersInExpectedPhase(report, constants.SUBSCRIBER_DONE_STAGE)) {
+        const reportAggregate = await aggregateReportManager.aggregateReport(report);
+        await updateResultsSummary(reportAggregate);
+
+        const testBenchmarkData = await extractBenchmark(report.test_id);
+        if (testBenchmarkData) {
+            reportBenchmark = await updateReportBenchmark(testBenchmarkData, reportAggregate);
+        }
+    }
+
     notifier.notifyIfNeeded(report, stats, reportBenchmark);
 
     return stats;
@@ -43,24 +54,17 @@ async function updateSubscriberWithStatsInternal(report, stats) {
     await databaseConnector.updateSubscriberWithStats(report.test_id, report.report_id, stats.runner_id, stats.phase_status, JSON.stringify(parseData));
 }
 
-async function updateReportBenchmarkIfNeeded(report) {
-    if (!reportUtil.isAllRunnersInExpectedPhase(report, constants.SUBSCRIBER_DONE_STAGE)) {
-        return;
-    }
+async function updateReportBenchmark(testBenchmarkData, reportAggregate) {
     const config = await configHandler.getConfig();
     const configBenchmark = {
         weights: config[configConsts.BENCHMARK_WEIGHTS],
         threshold: config[configConsts.BENCHMARK_THRESHOLD]
     };
-    const testBenchmarkData = await extractBenchmark(report.test_id);
-    if (testBenchmarkData) {
-        const reportAggregate = await aggregateReportManager.aggregateReport(report);
-        const reportBenchmark = benchmarkCalculator.calculate(testBenchmarkData, reportAggregate.aggregate, configBenchmark.weights);
-        const { data, score } = reportBenchmark;
-        data[configConsts.BENCHMARK_THRESHOLD] = configBenchmark.threshold;
-        await databaseConnector.updateReportBenchmark(report.test_id, report.report_id, score, JSON.stringify(data));
-        return reportBenchmark;
-    }
+    const reportBenchmark = benchmarkCalculator.calculate(testBenchmarkData, reportAggregate.aggregate, configBenchmark.weights);
+    const { data, score } = reportBenchmark;
+    data[configConsts.BENCHMARK_THRESHOLD] = configBenchmark.threshold;
+    await databaseConnector.updateReportBenchmark(reportAggregate.test_id, reportAggregate.report_id, score, JSON.stringify(data));
+    return reportBenchmark;
 }
 
 async function extractBenchmark(testId) {
@@ -71,3 +75,22 @@ async function extractBenchmark(testId) {
         return undefined;
     }
 }
+
+async function updateResultsSummary(reportAggregate) {
+    const aggregatedResults = reportAggregate.aggregate;
+    const resultsSummary = {
+        errors: aggregatedResults.errors,
+        codes: aggregatedResults.codes,
+        rps: {
+            mean: aggregatedResults.rps.mean,
+            count: aggregatedResults.rps.count
+        },
+        latency: {
+            median: aggregatedResults.latency.median,
+            p95: aggregatedResults.latency.p95,
+            p99: aggregatedResults.latency.p99
+        }
+    };
+    await databaseConnector.updateResultsSummary(reportAggregate.test_id, reportAggregate.report_id, JSON.stringify(resultsSummary));
+    return resultsSummary;
+}