Add functional test for UI Metric (#1267) (#1370)

Signed-off-by: Suchit Sahoo <[email protected]>
(cherry picked from commit b2e2498)

.github/workflows/cypress-workflow-bundle-snapshot-based.yml

@@ -0,0 +1,28 @@
+name: Bundle Snapshot based E2E Cypress tests workflow for core Dashboards
+on:
+  pull_request:
+    branches: [ '**' ]
+    paths:
+      - 'cypress/**/core-opensearch-dashboards/**'
+      - 'cypress/utils/dashboards/**'
+  push:
+    branches: [ '**' ]
+    paths:
+      - 'cypress/**/core-opensearch-dashboards/**'
+      - 'cypress/utils/dashboards/**'
+
+jobs:
+  tests-with-security:
+    uses: ./.github/workflows/release-e2e-workflow-template.yml
+    with:
+      test-name: Core Dashboards using Bundle Snapshot
+      test-command: env CYPRESS_NO_COMMAND_LOG=1 CYPRESS_ML_COMMONS_DASHBOARDS_ENABLED=true CYPRESS_VISBUILDER_ENABLED=true CYPRESS_UIMETRIC_ENABLED=true CYPRESS_DATASOURCE_MANAGEMENT_ENABLED=true yarn cypress:run-with-security --browser chromium --spec 'cypress/integration/core-opensearch-dashboards/opensearch-dashboards/**/*.js'
+      osd-serve-args: --data_source.enabled=true --data_source.ssl.verificationMode=none --vis_builder.enabled=true --ml_commons_dashboards.enabled=true --usageCollection.uiMetric.enabled=true
+
+  tests-without-security:
+    uses: ./.github/workflows/release-e2e-workflow-template.yml
+    with:
+      test-name: Core Dashboards using Bundle Snapshot
+      test-command: env CYPRESS_NO_COMMAND_LOG=1 CYPRESS_ML_COMMONS_DASHBOARDS_ENABLED=true CYPRESS_VISBUILDER_ENABLED=true CYPRESS_UIMETRIC_ENABLED=true CYPRESS_DATASOURCE_MANAGEMENT_ENABLED=true yarn cypress:run-without-security --browser chromium --spec 'cypress/integration/core-opensearch-dashboards/opensearch-dashboards/**/*.js'
+      osd-serve-args: --data_source.enabled=true --data_source.ssl.verificationMode=none --vis_builder.enabled=true --ml_commons_dashboards.enabled=true --usageCollection.uiMetric.enabled=true
+      security-enabled: false

cypress.json

@@ -31,6 +31,7 @@
     "DISABLE_LOCAL_CLUSTER": false,
     "browserPermissions": {
       "clipboard": "allow"
-    }
+    },
+    "UIMETRIC_ENABLED": false
   }
 }

cypress/fixtures/dashboard/opensearch_dashboards/telemetry/uiReport.json

@@ -0,0 +1 @@
+{"report":{"reportVersion":1,"uiStatsMetrics":{"console-count-GET_cat.indices":{"key":"console-count-GET_cat.indices","appName":"console","eventName":"GET_cat.indices","type":"count","stats":{"min":0,"max":1,"avg":0.5,"sum":21}}}}}

cypress/integration/core-opensearch-dashboards/opensearch-dashboards/apps/telemetry/server.spec.js

@@ -0,0 +1,113 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { CURRENT_TENANT } from '../../../../../utils/commands';
+import report from '../../../../../fixtures/dashboard/opensearch_dashboards/telemetry/uiReport.json';
+
+describe('server', () => {
+  before(() => {
+    CURRENT_TENANT.newTenant = 'global';
+  });
+
+  if (Cypress.env('UIMETRIC_ENABLED')) {
+    it('test server side batching', function () {
+      cy.wait(60000); // Intentional Wait to burst previous batching
+      // verify we don't have any entries forGET_cat.indices
+      cy.request(
+        'GET',
+        `${
+          Cypress.config().baseUrl
+        }/api/stats?extended=true&legacy=true&exclude_usage=false`
+      ).then((res) => {
+        expect(res.status).to.eq(200);
+        const usageMetric = res.body.usage.ui_metric.console || [];
+        expect(usageMetric).to.not.include({ key: 'GET_cat.indices' });
+      });
+
+      // Send the first UI metric report
+      cy.request({
+        method: 'POST',
+        url: `${Cypress.config().baseUrl}/api/ui_metric/report`,
+        headers: {
+          'Osd-Xsrf': 'osd-fetch',
+        },
+        body: report,
+      }).then((res) => {
+        expect(res.status).to.eq(200);
+      });
+
+      // Verify that the above report has been written
+      cy.request(
+        'GET',
+        `${
+          Cypress.config().baseUrl
+        }/api/stats?extended=true&legacy=true&exclude_usage=false`
+      ).then((res) => {
+        expect(res.status).to.eq(200);
+        const usageMetric = res.body.usage.ui_metric.console || []; // eslint-disable-line no-console
+        expect(usageMetric).to.deep.include({
+          key: 'GET_cat.indices',
+          value: 21,
+        });
+      });
+
+      // Send the second UI metric report
+      cy.request({
+        method: 'POST',
+        url: `${Cypress.config().baseUrl}/api/ui_metric/report`,
+        headers: {
+          'Osd-Xsrf': 'osd-fetch',
+        },
+        body: report,
+      }).then((res) => {
+        expect(res.status).to.eq(200);
+      });
+
+      // Verify that the above report has not been written and count is same as before
+      cy.request(
+        'GET',
+        `${
+          Cypress.config().baseUrl
+        }/api/stats?extended=true&legacy=true&exclude_usage=false`
+      ).then((res) => {
+        expect(res.status).to.eq(200);
+        const usageMetric = res.body.usage.ui_metric.console || []; // eslint-disable-line no-console
+        expect(usageMetric).to.deep.include({
+          key: 'GET_cat.indices',
+          value: 21,
+        });
+      });
+
+      cy.wait(60000); // Intentional wait to exceed batching interval
+
+      // Send the third UI metric report, since the time interval is greater than batching interval it will write this and previous report
+      cy.request({
+        method: 'POST',
+        url: `${Cypress.config().baseUrl}/api/ui_metric/report`,
+        headers: {
+          'Osd-Xsrf': 'osd-fetch',
+        },
+        body: report,
+      }).then((res) => {
+        expect(res.status).to.eq(200);
+      });
+
+      // Verify all the 3 Ui metric report have been written
+      cy.request(
+        'GET',
+        `${
+          Cypress.config().baseUrl
+        }/api/stats?extended=true&legacy=true&exclude_usage=false`
+      ).then((res) => {
+        expect(res.status).to.eq(200);
+        const usageMetric = res.body.usage.ui_metric.console || []; // eslint-disable-line 
+        expect(usageMetric).to.deep.include({
+          key: 'GET_cat.indices',
+          value: 63,
+        });
+      });
+    });
+  }
+});

cypress/integration/core-opensearch-dashboards/opensearch-dashboards/apps/telemetry/ui_metric.spec.js

@@ -0,0 +1,59 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { MiscUtils } from '@opensearch-dashboards-test/opensearch-dashboards-test-library';
+import { CURRENT_TENANT } from '../../../../../utils/commands';
+
+const miscUtils = new MiscUtils(cy);
+describe('dev_console_ui_metric', () => {
+  before(() => {
+    CURRENT_TENANT.newTenant = 'global';
+    miscUtils.visitPage('app/dev_tools#/console');
+
+    cy.get('[data-test-subj="help-close-button"]', { timeout: 30000 }).then(
+      ($btn) => {
+        if ($btn.is(':visible')) {
+          cy.wrap($btn).click({ force: true });
+        } else {
+          cy.get('[type="button"]').contains('Console').click({ force: true });
+        }
+      }
+    );
+
+    cy.intercept('POST', 'api/ui_metric/report').as('reportreq');
+
+    cy.wait(5000); // Intentional wait
+  });
+  if (Cypress.env('UIMETRIC_ENABLED')) {
+    it('check UI Metric are being recorded', function () {
+      miscUtils.visitPage('app/home#/');
+
+      cy.wait('@reportreq', { timeout: 100000 })
+        .its('response.statusCode')
+        .should('equal', 200);
+
+      // Now verify the response of api/stat
+
+      cy.request(
+        'GET',
+        `${
+          Cypress.config().baseUrl
+        }/api/stats?extended=true&legacy=true&exclude_usage=false`
+      ).then((res) => {
+        expect(res.status).to.eq(200);
+        expect(res.body)
+          .to.have.property('usage')
+          .that.has.property('application_usage')
+          .that.has.property('dev_tools');
+        expect(res.body)
+          .to.have.property('usage')
+          .that.has.property('ui_metric')
+          .that.has.property('console')
+          .that.has.property('length')
+          .that.is.gt(0);
+      });
+    });
+  }
+});

package.json

@@ -28,7 +28,7 @@
     "osd:ciGroup1": "echo \"apps/vis_builder/*.js\"",
     "osd:ciGroup2": "echo \"apps/vis_type_table/*.js\"",
     "osd:ciGroup3": "echo \"apps/vis-augmenter/*.js\"",
-    "osd:ciGroup4": "echo \"dashboard_sample_data_with_datasource_spec.js,dashboard_sanity_test_spec.js\"",
+    "osd:ciGroup4": "echo \"dashboard_sample_data_with_datasource_spec.js,dashboard_sanity_test_spec.js,apps/telemetry/*.js\"",
     "osd:ciGroup5": "echo \"datasource-management-plugin/*.js\"",
     "osd:ciGroup6": "echo \"apps/data_explorer/aaa_before.spec.js,apps/data_explorer/data_source_selector.spec.js,apps/data_explorer/date_nanos_mixed.spec.js,apps/data_explorer/date_nanos.spec.js,apps/data_explorer/discover_histogram.spec.js,apps/data_explorer/discover.spec.js,apps/data_explorer/zzz_after.spec.js\"",
     "osd:ciGroup7": "echo \"apps/data_explorer/aaa_before.spec.js,apps/data_explorer/doc_navigation.spec.js,apps/data_explorer/doc_table.spec.js,apps/data_explorer/errors.spec.js,apps/data_explorer/field_data.spec.js,apps/data_explorer/zzz_after.spec.js\"",