From cde949f34a7f45ca0df7281f90705d9410f9ab53 Mon Sep 17 00:00:00 2001 From: Yulong Ruan Date: Sat, 14 Sep 2024 13:46:48 +0800 Subject: [PATCH 1/6] Add metrics for text2viz Signed-off-by: Yulong Ruan --- opensearch_dashboards.json | 2 +- public/components/feedback_thumbs.test.tsx | 79 +++++++++++++++++++ public/components/feedback_thumbs.tsx | 59 ++++++++++++++ public/components/visualization/text2viz.scss | 7 ++ public/components/visualization/text2viz.tsx | 47 ++++++++++- public/types.ts | 4 + 6 files changed, 194 insertions(+), 4 deletions(-) create mode 100644 public/components/feedback_thumbs.test.tsx create mode 100644 public/components/feedback_thumbs.tsx diff --git a/opensearch_dashboards.json b/opensearch_dashboards.json index 3bfa5670..be11a53b 100644 --- a/opensearch_dashboards.json +++ b/opensearch_dashboards.json @@ -24,4 +24,4 @@ "configPath": [ "assistant" ] -} \ No newline at end of file +} diff --git a/public/components/feedback_thumbs.test.tsx b/public/components/feedback_thumbs.test.tsx new file mode 100644 index 00000000..c5055c5c --- /dev/null +++ b/public/components/feedback_thumbs.test.tsx @@ -0,0 +1,79 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import React from 'react'; +import { render, screen, fireEvent, waitFor } from '@testing-library/react'; +import { METRIC_TYPE } from '@osd/analytics'; + +import { FeedbackThumbs } from './feedback_thumbs'; + +describe('', () => { + it('should report thumbs up metric', () => { + const usageCollectionMock = { + reportUiStats: jest.fn(), + METRIC_TYPE, + }; + + render(); + fireEvent.click(screen.getByLabelText('ThumbsUp')); + expect(usageCollectionMock.reportUiStats).toHaveBeenCalledWith( + 'test-app', + METRIC_TYPE.CLICK, + expect.stringMatching(/thumbs_up.*/) + ); + }); + + it('should report thumbs down metric', () => { + const usageCollectionMock = { + reportUiStats: jest.fn(), + METRIC_TYPE, + }; + + render(); + fireEvent.click(screen.getByLabelText('ThumbsDown')); + expect(usageCollectionMock.reportUiStats).toHaveBeenCalledWith( + 'test-app', + METRIC_TYPE.CLICK, + expect.stringMatching(/thumbs_down.*/) + ); + }); + + it('should only report metric only once', () => { + const usageCollectionMock = { + reportUiStats: jest.fn(), + METRIC_TYPE, + }; + + render(); + // click the button two times + fireEvent.click(screen.getByLabelText('ThumbsDown')); + fireEvent.click(screen.getByLabelText('ThumbsDown')); + expect(usageCollectionMock.reportUiStats).toHaveBeenCalledTimes(1); + }); + + it('should hide thumbs down button after thumbs up been clicked', () => { + const usageCollectionMock = { + reportUiStats: jest.fn(), + METRIC_TYPE, + }; + + render(); + + fireEvent.click(screen.getByLabelText('ThumbsUp')); + expect(screen.queryByLabelText('ThumbsDown')).toBeNull(); + }); + + it('should hide thumbs up button after thumbs down been clicked', () => { + const usageCollectionMock = { + reportUiStats: jest.fn(), + METRIC_TYPE, + }; + + render(); + + fireEvent.click(screen.getByLabelText('ThumbsDown')); + expect(screen.queryByLabelText('ThumbsUp')).toBeNull(); + }); +}); diff --git a/public/components/feedback_thumbs.tsx b/public/components/feedback_thumbs.tsx new file mode 100644 index 00000000..d7ee3343 --- /dev/null +++ b/public/components/feedback_thumbs.tsx @@ -0,0 +1,59 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { EuiButtonIcon, EuiFlexGroup, EuiFlexItem } from '@elastic/eui'; +import React, { useState } from 'react'; +import { v4 as uuidv4 } from 'uuid'; + +import { UsageCollectionStart } from '../../../../src/plugins/usage_collection/public'; + +interface Props { + appName: string; + usageCollection: UsageCollectionStart; + className?: string; +} + +export const FeedbackThumbs = ({ usageCollection, appName, className }: Props) => { + const [feedback, setFeedback] = useState<'thumbs_up' | 'thumbs_down' | undefined>(); + + const onFeedback = (eventName: 'thumbs_up' | 'thumbs_down') => { + // Only send metric if no current feedback set + if (!feedback) { + usageCollection.reportUiStats( + appName, + usageCollection.METRIC_TYPE.CLICK, + `${eventName}-${uuidv4()}` + ); + setFeedback(eventName); + } + }; + + return ( + + {(!feedback || feedback === 'thumbs_up') && ( + + onFeedback('thumbs_up')} + /> + + )} + {(!feedback || feedback === 'thumbs_down') && ( + + onFeedback('thumbs_down')} + /> + + )} + + ); +}; diff --git a/public/components/visualization/text2viz.scss b/public/components/visualization/text2viz.scss index f2b0f84b..c917471e 100644 --- a/public/components/visualization/text2viz.scss +++ b/public/components/visualization/text2viz.scss @@ -16,4 +16,11 @@ padding-top: 15px; padding-left: 30px; } + + .feedback_thumbs { + position: absolute; + right: 16px; + top: 4px; + z-index: 9999; + } } diff --git a/public/components/visualization/text2viz.tsx b/public/components/visualization/text2viz.tsx index 99ee35e8..1efb507d 100644 --- a/public/components/visualization/text2viz.tsx +++ b/public/components/visualization/text2viz.tsx @@ -18,6 +18,7 @@ import { } from '@elastic/eui'; import React, { useEffect, useMemo, useRef, useState } from 'react'; import { i18n } from '@osd/i18n'; +import { v4 as uuidv4 } from 'uuid'; import { useCallback } from 'react'; import { useObservable } from 'react-use'; @@ -46,9 +47,10 @@ import { getIndexPatterns } from '../../services'; import { NLQ_VISUALIZATION_EMBEDDABLE_TYPE } from './embeddable/nlq_vis_embeddable'; import { NLQVisualizationInput } from './embeddable/types'; import { EditorPanel } from './editor_panel'; -import { VIS_NLQ_SAVED_OBJECT } from '../../../common/constants/vis_type_nlq'; +import { VIS_NLQ_APP_ID, VIS_NLQ_SAVED_OBJECT } from '../../../common/constants/vis_type_nlq'; import { HeaderVariant } from '../../../../../src/core/public'; import { TEXT2VEGA_INPUT_SIZE_LIMIT } from '../../../common/constants/llm'; +import { FeedbackThumbs } from '../feedback_thumbs'; export const Text2Viz = () => { const { savedObjectId } = useParams<{ savedObjectId?: string }>(); @@ -68,9 +70,23 @@ export const Text2Viz = () => { uiSettings, savedObjects, config, + usageCollection, }, } = useOpenSearchDashboards(); + /** + * Report metrics when the application is loaded + */ + useEffect(() => { + if (usageCollection) { + usageCollection.reportUiStats( + VIS_NLQ_APP_ID, + usageCollection.METRIC_TYPE.LOADED, + `app_loaded-${uuidv4()}` + ); + } + }, [usageCollection]); + const useUpdatedUX = uiSettings.get('home:useNewHomePage'); const [input, setInput] = useState(''); @@ -112,6 +128,15 @@ export const Text2Viz = () => { }); } else { setEditorInput(JSON.stringify(result, undefined, 4)); + + // Report metric when visualization generated successfully + if (usageCollection) { + usageCollection.reportUiStats( + VIS_NLQ_APP_ID, + usageCollection.METRIC_TYPE.LOADED, + `generated-${uuidv4()}` + ); + } } } }); @@ -119,7 +144,7 @@ export const Text2Viz = () => { return () => { subscription.unsubscribe(); }; - }, [http, notifications]); + }, [http, notifications, usageCollection]); /** * Loads the saved object from id when editing an existing visualization @@ -243,6 +268,15 @@ export const Text2Viz = () => { }), }); dialog.close(); + + // Report metric when a new visualization is saved. + if (usageCollection) { + usageCollection.reportUiStats( + VIS_NLQ_APP_ID, + usageCollection.METRIC_TYPE.LOADED, + `saved-${uuidv4()}` + ); + } } } catch (e) { notifications.toasts.addDanger({ @@ -270,7 +304,7 @@ export const Text2Viz = () => { /> ) ); - }, [notifications, vegaSpec, input, overlays, selectedSource, savedObjectId]); + }, [notifications, vegaSpec, input, overlays, selectedSource, savedObjectId, usageCollection]); const pageTitle = savedObjectId ? i18n.translate('dashboardAssistant.feature.text2viz.breadcrumbs.editVisualization', { @@ -412,6 +446,13 @@ export const Text2Viz = () => { paddingSize="none" scrollable={false} > + {usageCollection ? ( + + ) : null} diff --git a/public/types.ts b/public/types.ts index 317f78c2..eeff84f5 100644 --- a/public/types.ts +++ b/public/types.ts @@ -20,6 +20,8 @@ import { AssistantClient } from './services/assistant_client'; import { UiActionsSetup, UiActionsStart } from '../../../src/plugins/ui_actions/public'; import { ExpressionsSetup, ExpressionsStart } from '../../../src/plugins/expressions/public'; import { SavedObjectsStart } from '../../../src/plugins/saved_objects/public'; +import { UsageCollectionSetup } from '../../../src/plugins/usage_collection/server'; +import { UsageCollectionStart } from '../../../src/plugins/usage_collection/public'; import { UsageCollectionSetup } from '../../../src/plugins/usage_collection/public'; import { ConfigSchema } from '../common/types/config'; @@ -49,6 +51,7 @@ export interface AssistantPluginStartDependencies { uiActions: UiActionsStart; expressions: ExpressionsStart; savedObjects: SavedObjectsStart; + usageCollection?: UsageCollectionStart; } export interface AssistantPluginSetupDependencies { @@ -59,6 +62,7 @@ export interface AssistantPluginSetupDependencies { usageCollection?: UsageCollectionSetup; uiActions: UiActionsSetup; expressions: ExpressionsSetup; + usageCollection?: UsageCollectionSetup; } export interface AssistantSetup { From 4559e634df1cbf0f0f28db7492b82ade91610f0b Mon Sep 17 00:00:00 2001 From: Yulong Ruan Date: Sat, 14 Sep 2024 13:52:27 +0800 Subject: [PATCH 2/6] remove new line at the end of file Signed-off-by: Yulong Ruan --- opensearch_dashboards.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opensearch_dashboards.json b/opensearch_dashboards.json index be11a53b..3bfa5670 100644 --- a/opensearch_dashboards.json +++ b/opensearch_dashboards.json @@ -24,4 +24,4 @@ "configPath": [ "assistant" ] -} +} \ No newline at end of file From 4f107e946848e22a97d34420e9d7c3413513eb02 Mon Sep 17 00:00:00 2001 From: Yulong Ruan Date: Sat, 14 Sep 2024 15:03:44 +0800 Subject: [PATCH 3/6] fix imported types Signed-off-by: Yulong Ruan --- public/types.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/public/types.ts b/public/types.ts index eeff84f5..192c5bb7 100644 --- a/public/types.ts +++ b/public/types.ts @@ -20,8 +20,10 @@ import { AssistantClient } from './services/assistant_client'; import { UiActionsSetup, UiActionsStart } from '../../../src/plugins/ui_actions/public'; import { ExpressionsSetup, ExpressionsStart } from '../../../src/plugins/expressions/public'; import { SavedObjectsStart } from '../../../src/plugins/saved_objects/public'; -import { UsageCollectionSetup } from '../../../src/plugins/usage_collection/server'; -import { UsageCollectionStart } from '../../../src/plugins/usage_collection/public'; +import { + UsageCollectionStart, + UsageCollectionSetup, +} from '../../../src/plugins/usage_collection/public'; import { UsageCollectionSetup } from '../../../src/plugins/usage_collection/public'; import { ConfigSchema } from '../common/types/config'; From 25442d1eecec18c72ff87850d0206d013660793f Mon Sep 17 00:00:00 2001 From: Yulong Ruan Date: Sat, 14 Sep 2024 15:09:25 +0800 Subject: [PATCH 4/6] update changelogs Signed-off-by: Yulong Ruan --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c776b2c4..a0f28e0b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - fix: incorrect string escaping of vega schema([325](https://github.com/opensearch-project/dashboards-assistant/pull/325)) - feat: register the AI actions to query controls in discover([#327](https://github.com/opensearch-project/dashboards-assistant/pull/327)) - fix: t2viz ux improvements([#330](https://github.com/opensearch-project/dashboards-assistant/pull/330)) +- feat: report metrics for text to visualization([#312](https://github.com/opensearch-project/dashboards-assistant/pull/312)) ### 📈 Features/Enhancements From 947805619a25f14afec94f5e3645e4c559bcfde0 Mon Sep 17 00:00:00 2001 From: Yulong Ruan Date: Wed, 18 Sep 2024 14:54:38 +0800 Subject: [PATCH 5/6] fix UI flashing Signed-off-by: Yulong Ruan --- public/components/visualization/text2vega.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/public/components/visualization/text2vega.ts b/public/components/visualization/text2vega.ts index aba0e388..ea8b0936 100644 --- a/public/components/visualization/text2vega.ts +++ b/public/components/visualization/text2vega.ts @@ -38,8 +38,8 @@ export class Text2Vega { this.result$ = this.input$ .pipe( filter((v) => v.prompt.length > 0), - debounceTime(200), - tap(() => this.status$.next('RUNNING')) + tap(() => this.status$.next('RUNNING')), + debounceTime(200) ) .pipe( switchMap((v) => From c56f0d30deee2d932d3f79c3fd3af4275220466d Mon Sep 17 00:00:00 2001 From: Yulong Ruan Date: Fri, 27 Sep 2024 15:42:02 +0800 Subject: [PATCH 6/6] cleanup duplicate imports Signed-off-by: Yulong Ruan --- public/types.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/public/types.ts b/public/types.ts index 192c5bb7..6a95238f 100644 --- a/public/types.ts +++ b/public/types.ts @@ -25,7 +25,6 @@ import { UsageCollectionSetup, } from '../../../src/plugins/usage_collection/public'; -import { UsageCollectionSetup } from '../../../src/plugins/usage_collection/public'; import { ConfigSchema } from '../common/types/config'; export interface RenderProps { @@ -64,7 +63,6 @@ export interface AssistantPluginSetupDependencies { usageCollection?: UsageCollectionSetup; uiActions: UiActionsSetup; expressions: ExpressionsSetup; - usageCollection?: UsageCollectionSetup; } export interface AssistantSetup {