From 065b6ad50af744604a8c71bf02819b0a43cd302f Mon Sep 17 00:00:00 2001 From: Suchit Sahoo <38322563+LDrago27@users.noreply.github.com> Date: Mon, 10 Jun 2024 13:23:47 -0700 Subject: [PATCH] Add functional test for UI Metric (#1267) (#1370) Signed-off-by: Suchit Sahoo (cherry picked from commit b2e2498789e016cceea4be39fcca539399a20a22) --- ...cypress-workflow-bundle-snapshot-based.yml | 28 +++++ cypress.json | 3 +- .../telemetry/uiReport.json | 1 + .../apps/telemetry/server.spec.js | 113 ++++++++++++++++++ .../apps/telemetry/ui_metric.spec.js | 59 +++++++++ package.json | 2 +- 6 files changed, 204 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/cypress-workflow-bundle-snapshot-based.yml create mode 100644 cypress/fixtures/dashboard/opensearch_dashboards/telemetry/uiReport.json create mode 100644 cypress/integration/core-opensearch-dashboards/opensearch-dashboards/apps/telemetry/server.spec.js create mode 100644 cypress/integration/core-opensearch-dashboards/opensearch-dashboards/apps/telemetry/ui_metric.spec.js diff --git a/.github/workflows/cypress-workflow-bundle-snapshot-based.yml b/.github/workflows/cypress-workflow-bundle-snapshot-based.yml new file mode 100644 index 000000000..db297dacb --- /dev/null +++ b/.github/workflows/cypress-workflow-bundle-snapshot-based.yml @@ -0,0 +1,28 @@ +name: Bundle Snapshot based E2E Cypress tests workflow for core Dashboards +on: + pull_request: + branches: [ '**' ] + paths: + - 'cypress/**/core-opensearch-dashboards/**' + - 'cypress/utils/dashboards/**' + push: + branches: [ '**' ] + paths: + - 'cypress/**/core-opensearch-dashboards/**' + - 'cypress/utils/dashboards/**' + +jobs: + tests-with-security: + uses: ./.github/workflows/release-e2e-workflow-template.yml + with: + test-name: Core Dashboards using Bundle Snapshot + test-command: env CYPRESS_NO_COMMAND_LOG=1 CYPRESS_ML_COMMONS_DASHBOARDS_ENABLED=true CYPRESS_VISBUILDER_ENABLED=true CYPRESS_UIMETRIC_ENABLED=true CYPRESS_DATASOURCE_MANAGEMENT_ENABLED=true yarn cypress:run-with-security --browser chromium --spec 'cypress/integration/core-opensearch-dashboards/opensearch-dashboards/**/*.js' + osd-serve-args: --data_source.enabled=true --data_source.ssl.verificationMode=none --vis_builder.enabled=true --ml_commons_dashboards.enabled=true --usageCollection.uiMetric.enabled=true + + tests-without-security: + uses: ./.github/workflows/release-e2e-workflow-template.yml + with: + test-name: Core Dashboards using Bundle Snapshot + test-command: env CYPRESS_NO_COMMAND_LOG=1 CYPRESS_ML_COMMONS_DASHBOARDS_ENABLED=true CYPRESS_VISBUILDER_ENABLED=true CYPRESS_UIMETRIC_ENABLED=true CYPRESS_DATASOURCE_MANAGEMENT_ENABLED=true yarn cypress:run-without-security --browser chromium --spec 'cypress/integration/core-opensearch-dashboards/opensearch-dashboards/**/*.js' + osd-serve-args: --data_source.enabled=true --data_source.ssl.verificationMode=none --vis_builder.enabled=true --ml_commons_dashboards.enabled=true --usageCollection.uiMetric.enabled=true + security-enabled: false diff --git a/cypress.json b/cypress.json index 9b74aa613..dc9cb08ad 100644 --- a/cypress.json +++ b/cypress.json @@ -31,6 +31,7 @@ "DISABLE_LOCAL_CLUSTER": false, "browserPermissions": { "clipboard": "allow" - } + }, + "UIMETRIC_ENABLED": false } } diff --git a/cypress/fixtures/dashboard/opensearch_dashboards/telemetry/uiReport.json b/cypress/fixtures/dashboard/opensearch_dashboards/telemetry/uiReport.json new file mode 100644 index 000000000..28bc0da80 --- /dev/null +++ b/cypress/fixtures/dashboard/opensearch_dashboards/telemetry/uiReport.json @@ -0,0 +1 @@ +{"report":{"reportVersion":1,"uiStatsMetrics":{"console-count-GET_cat.indices":{"key":"console-count-GET_cat.indices","appName":"console","eventName":"GET_cat.indices","type":"count","stats":{"min":0,"max":1,"avg":0.5,"sum":21}}}}} diff --git a/cypress/integration/core-opensearch-dashboards/opensearch-dashboards/apps/telemetry/server.spec.js b/cypress/integration/core-opensearch-dashboards/opensearch-dashboards/apps/telemetry/server.spec.js new file mode 100644 index 000000000..58f5d624c --- /dev/null +++ b/cypress/integration/core-opensearch-dashboards/opensearch-dashboards/apps/telemetry/server.spec.js @@ -0,0 +1,113 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { CURRENT_TENANT } from '../../../../../utils/commands'; +import report from '../../../../../fixtures/dashboard/opensearch_dashboards/telemetry/uiReport.json'; + +describe('server', () => { + before(() => { + CURRENT_TENANT.newTenant = 'global'; + }); + + if (Cypress.env('UIMETRIC_ENABLED')) { + it('test server side batching', function () { + cy.wait(60000); // Intentional Wait to burst previous batching + // verify we don't have any entries forGET_cat.indices + cy.request( + 'GET', + `${ + Cypress.config().baseUrl + }/api/stats?extended=true&legacy=true&exclude_usage=false` + ).then((res) => { + expect(res.status).to.eq(200); + const usageMetric = res.body.usage.ui_metric.console || []; + expect(usageMetric).to.not.include({ key: 'GET_cat.indices' }); + }); + + // Send the first UI metric report + cy.request({ + method: 'POST', + url: `${Cypress.config().baseUrl}/api/ui_metric/report`, + headers: { + 'Osd-Xsrf': 'osd-fetch', + }, + body: report, + }).then((res) => { + expect(res.status).to.eq(200); + }); + + // Verify that the above report has been written + cy.request( + 'GET', + `${ + Cypress.config().baseUrl + }/api/stats?extended=true&legacy=true&exclude_usage=false` + ).then((res) => { + expect(res.status).to.eq(200); + const usageMetric = res.body.usage.ui_metric.console || []; // eslint-disable-line no-console + expect(usageMetric).to.deep.include({ + key: 'GET_cat.indices', + value: 21, + }); + }); + + // Send the second UI metric report + cy.request({ + method: 'POST', + url: `${Cypress.config().baseUrl}/api/ui_metric/report`, + headers: { + 'Osd-Xsrf': 'osd-fetch', + }, + body: report, + }).then((res) => { + expect(res.status).to.eq(200); + }); + + // Verify that the above report has not been written and count is same as before + cy.request( + 'GET', + `${ + Cypress.config().baseUrl + }/api/stats?extended=true&legacy=true&exclude_usage=false` + ).then((res) => { + expect(res.status).to.eq(200); + const usageMetric = res.body.usage.ui_metric.console || []; // eslint-disable-line no-console + expect(usageMetric).to.deep.include({ + key: 'GET_cat.indices', + value: 21, + }); + }); + + cy.wait(60000); // Intentional wait to exceed batching interval + + // Send the third UI metric report, since the time interval is greater than batching interval it will write this and previous report + cy.request({ + method: 'POST', + url: `${Cypress.config().baseUrl}/api/ui_metric/report`, + headers: { + 'Osd-Xsrf': 'osd-fetch', + }, + body: report, + }).then((res) => { + expect(res.status).to.eq(200); + }); + + // Verify all the 3 Ui metric report have been written + cy.request( + 'GET', + `${ + Cypress.config().baseUrl + }/api/stats?extended=true&legacy=true&exclude_usage=false` + ).then((res) => { + expect(res.status).to.eq(200); + const usageMetric = res.body.usage.ui_metric.console || []; // eslint-disable-line + expect(usageMetric).to.deep.include({ + key: 'GET_cat.indices', + value: 63, + }); + }); + }); + } +}); diff --git a/cypress/integration/core-opensearch-dashboards/opensearch-dashboards/apps/telemetry/ui_metric.spec.js b/cypress/integration/core-opensearch-dashboards/opensearch-dashboards/apps/telemetry/ui_metric.spec.js new file mode 100644 index 000000000..9c9741058 --- /dev/null +++ b/cypress/integration/core-opensearch-dashboards/opensearch-dashboards/apps/telemetry/ui_metric.spec.js @@ -0,0 +1,59 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { MiscUtils } from '@opensearch-dashboards-test/opensearch-dashboards-test-library'; +import { CURRENT_TENANT } from '../../../../../utils/commands'; + +const miscUtils = new MiscUtils(cy); +describe('dev_console_ui_metric', () => { + before(() => { + CURRENT_TENANT.newTenant = 'global'; + miscUtils.visitPage('app/dev_tools#/console'); + + cy.get('[data-test-subj="help-close-button"]', { timeout: 30000 }).then( + ($btn) => { + if ($btn.is(':visible')) { + cy.wrap($btn).click({ force: true }); + } else { + cy.get('[type="button"]').contains('Console').click({ force: true }); + } + } + ); + + cy.intercept('POST', 'api/ui_metric/report').as('reportreq'); + + cy.wait(5000); // Intentional wait + }); + if (Cypress.env('UIMETRIC_ENABLED')) { + it('check UI Metric are being recorded', function () { + miscUtils.visitPage('app/home#/'); + + cy.wait('@reportreq', { timeout: 100000 }) + .its('response.statusCode') + .should('equal', 200); + + // Now verify the response of api/stat + + cy.request( + 'GET', + `${ + Cypress.config().baseUrl + }/api/stats?extended=true&legacy=true&exclude_usage=false` + ).then((res) => { + expect(res.status).to.eq(200); + expect(res.body) + .to.have.property('usage') + .that.has.property('application_usage') + .that.has.property('dev_tools'); + expect(res.body) + .to.have.property('usage') + .that.has.property('ui_metric') + .that.has.property('console') + .that.has.property('length') + .that.is.gt(0); + }); + }); + } +}); diff --git a/package.json b/package.json index 2a23da2f1..ef0873d03 100644 --- a/package.json +++ b/package.json @@ -28,7 +28,7 @@ "osd:ciGroup1": "echo \"apps/vis_builder/*.js\"", "osd:ciGroup2": "echo \"apps/vis_type_table/*.js\"", "osd:ciGroup3": "echo \"apps/vis-augmenter/*.js\"", - "osd:ciGroup4": "echo \"dashboard_sample_data_with_datasource_spec.js,dashboard_sanity_test_spec.js\"", + "osd:ciGroup4": "echo \"dashboard_sample_data_with_datasource_spec.js,dashboard_sanity_test_spec.js,apps/telemetry/*.js\"", "osd:ciGroup5": "echo \"datasource-management-plugin/*.js\"", "osd:ciGroup6": "echo \"apps/data_explorer/aaa_before.spec.js,apps/data_explorer/data_source_selector.spec.js,apps/data_explorer/date_nanos_mixed.spec.js,apps/data_explorer/date_nanos.spec.js,apps/data_explorer/discover_histogram.spec.js,apps/data_explorer/discover.spec.js,apps/data_explorer/zzz_after.spec.js\"", "osd:ciGroup7": "echo \"apps/data_explorer/aaa_before.spec.js,apps/data_explorer/doc_navigation.spec.js,apps/data_explorer/doc_table.spec.js,apps/data_explorer/errors.spec.js,apps/data_explorer/field_data.spec.js,apps/data_explorer/zzz_after.spec.js\"",