From a74ff7f77fb45ace8c9583c48692a962773dccec Mon Sep 17 00:00:00 2001 From: SuZhou-Joe Date: Mon, 27 Nov 2023 17:42:42 +0800 Subject: [PATCH 1/5] feat: add implement visualization in customized parser Signed-off-by: SuZhou-Joe --- server/parsers/visualization_card_parser.ts | 41 +++++++++++++++++++++ server/plugin.ts | 3 +- 2 files changed, 43 insertions(+), 1 deletion(-) create mode 100644 server/parsers/visualization_card_parser.ts diff --git a/server/parsers/visualization_card_parser.ts b/server/parsers/visualization_card_parser.ts new file mode 100644 index 00000000..f833dfa8 --- /dev/null +++ b/server/parsers/visualization_card_parser.ts @@ -0,0 +1,41 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { IMessage, Interaction } from '../../common/types/chat_saved_object_attributes'; + +const extractNthColumn = (csv: string, column: number) => { + const lines = csv.split(/\r?\n/).slice(1); + return lines + .map((line) => line.split(',').at(column)) + .filter((v: T | null | undefined): v is T => v !== null && v !== undefined); +}; + +export const VisualizationCardParser = { + id: 'core_visualization', + async parserProvider(interaction: Interaction) { + const additionalInfo = interaction.additional_info as { + 'VisualizationTool.output': string[]; + }; + const visualizationOutputs = additionalInfo?.['VisualizationTool.output']; + if (!visualizationOutputs) { + return []; + } + const visualizationIds = visualizationOutputs.flatMap((output) => extractNthColumn(output, 1)); // second column is id field + + const visOutputs: IMessage[] = visualizationIds.map((id) => ({ + type: 'output', + content: id, + contentType: 'visualization', + suggestedActions: [ + { + message: 'View in Visualize', + actionType: 'view_in_dashboards', + }, + ], + })); + + return visOutputs; + }, +}; diff --git a/server/plugin.ts b/server/plugin.ts index 745e3b0c..521d855c 100644 --- a/server/plugin.ts +++ b/server/plugin.ts @@ -17,13 +17,13 @@ import { import { OpenSearchAlertingPlugin } from './adaptors/opensearch_alerting_plugin'; import { OpenSearchObservabilityPlugin } from './adaptors/opensearch_observability_plugin'; import { PPLPlugin } from './adaptors/ppl_plugin'; -import { AssistantServerConfig } from './config/schema'; import './fetch-polyfill'; import { setupRoutes } from './routes/index'; import { chatSavedObject } from './saved_objects/chat_saved_object'; import { AssistantPluginSetup, AssistantPluginStart, MessageParser } from './types'; import { chatConfigSavedObject } from './saved_objects/chat_config_saved_object'; import { BasicInputOutputParser } from './parsers/basic_input_output_parser'; +import { VisualizationCardParser } from './parsers/visualization_card_parser'; export class AssistantPlugin implements Plugin { private readonly logger: Logger; @@ -79,6 +79,7 @@ export class AssistantPlugin implements Plugin Date: Tue, 28 Nov 2023 16:46:32 +0800 Subject: [PATCH 2/5] feat: add csv-parser lib Signed-off-by: SuZhou-Joe --- common/types/chat_saved_object_attributes.ts | 2 +- package.json | 1 + server/parsers/visualization_card_parser.ts | 40 ++++++++++++-------- yarn.lock | 9 ++++- 4 files changed, 34 insertions(+), 18 deletions(-) diff --git a/common/types/chat_saved_object_attributes.ts b/common/types/chat_saved_object_attributes.ts index 0421cd54..acfc8d71 100644 --- a/common/types/chat_saved_object_attributes.ts +++ b/common/types/chat_saved_object_attributes.ts @@ -12,7 +12,7 @@ export interface Interaction { conversation_id: string; interaction_id: string; create_time: string; - additional_info: Record; + additional_info?: Record; parent_interaction_id?: string; } diff --git a/package.json b/package.json index 1aaf6fab..d79d30a9 100644 --- a/package.json +++ b/package.json @@ -19,6 +19,7 @@ }, "dependencies": { "autosize": "^6.0.1", + "csv-parser": "^3.0.0", "dompurify": "^2.4.1", "jsdom": "^22.1.0", "langchain": "^0.0.164", diff --git a/server/parsers/visualization_card_parser.ts b/server/parsers/visualization_card_parser.ts index f833dfa8..574bc251 100644 --- a/server/parsers/visualization_card_parser.ts +++ b/server/parsers/visualization_card_parser.ts @@ -4,11 +4,12 @@ */ import { IMessage, Interaction } from '../../common/types/chat_saved_object_attributes'; +import { getJsonFromString } from '../utils/csv-parser-helper'; -const extractNthColumn = (csv: string, column: number) => { - const lines = csv.split(/\r?\n/).slice(1); +const extractNthColumn = async (csv: string, column: number) => { + const lines = (await getJsonFromString(csv)) as Array<{ Id: string }>; return lines - .map((line) => line.split(',').at(column)) + .map((line) => line.Id) .filter((v: T | null | undefined): v is T => v !== null && v !== undefined); }; @@ -17,24 +18,31 @@ export const VisualizationCardParser = { async parserProvider(interaction: Interaction) { const additionalInfo = interaction.additional_info as { 'VisualizationTool.output': string[]; - }; + } | null; const visualizationOutputs = additionalInfo?.['VisualizationTool.output']; if (!visualizationOutputs) { return []; } - const visualizationIds = visualizationOutputs.flatMap((output) => extractNthColumn(output, 1)); // second column is id field + const visualizationIds = ( + await Promise.all(visualizationOutputs.map((output) => extractNthColumn(output, 1))) + ).flatMap((id) => id); // second column is id field - const visOutputs: IMessage[] = visualizationIds.map((id) => ({ - type: 'output', - content: id, - contentType: 'visualization', - suggestedActions: [ - { - message: 'View in Visualize', - actionType: 'view_in_dashboards', - }, - ], - })); + const visOutputs: IMessage[] = visualizationIds + /** + * Empty id will be filtered + */ + .filter((id) => id) + .map((id) => ({ + type: 'output', + content: id, + contentType: 'visualization', + suggestedActions: [ + { + message: 'View in Visualize', + actionType: 'view_in_dashboards', + }, + ], + })); return visOutputs; }, diff --git a/yarn.lock b/yarn.lock index 3f22bbf1..290e4f09 100644 --- a/yarn.lock +++ b/yarn.lock @@ -545,6 +545,13 @@ csstype@^3.0.2: resolved "https://registry.yarnpkg.com/csstype/-/csstype-3.1.2.tgz#1d4bf9d572f11c14031f0436e1c10bc1f571f50b" integrity sha512-I7K1Uu0MBPzaFKg4nI5Q7Vs2t+3gWWW648spaF+Rg7pI9ds18Ugn+lvg4SHczUdKlHI5LWBXyqfS8+DufyBsgQ== +csv-parser@^3.0.0: + version "3.0.0" + resolved "https://registry.npmmirror.com/csv-parser/-/csv-parser-3.0.0.tgz#b88a6256d79e090a97a1b56451f9327b01d710e7" + integrity sha512-s6OYSXAK3IdKqYO33y09jhypG/bSDHPuyCme/IdEHfWpLf/jKcpitVFyOC6UemgGk8v7Q5u2XE0vvwmanxhGlQ== + dependencies: + minimist "^1.2.0" + data-urls@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/data-urls/-/data-urls-4.0.0.tgz#333a454eca6f9a5b7b0f1013ff89074c3f522dd4" @@ -1354,7 +1361,7 @@ minimatch@^3.0.4, minimatch@^3.1.1: dependencies: brace-expansion "^1.1.7" -minimist@^1.2.5, minimist@^1.2.6: +minimist@^1.2.0, minimist@^1.2.5, minimist@^1.2.6: version "1.2.8" resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.8.tgz#c1a464e7693302e082a075cee0c057741ac4772c" integrity sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA== From 85b5146cbb9fd5edb841543232d093a27f0d9b3d Mon Sep 17 00:00:00 2001 From: SuZhou-Joe Date: Tue, 28 Nov 2023 16:47:19 +0800 Subject: [PATCH 3/5] feat: add csv-parser lib Signed-off-by: SuZhou-Joe --- .../parsers/basic_input_output_parser.test.ts | 32 +++++ .../parsers/visualization_card_parser.test.ts | 116 ++++++++++++++++++ server/utils/csv-parser-helper.ts | 25 ++++ 3 files changed, 173 insertions(+) create mode 100644 server/parsers/basic_input_output_parser.test.ts create mode 100644 server/parsers/visualization_card_parser.test.ts create mode 100644 server/utils/csv-parser-helper.ts diff --git a/server/parsers/basic_input_output_parser.test.ts b/server/parsers/basic_input_output_parser.test.ts new file mode 100644 index 00000000..9f72f941 --- /dev/null +++ b/server/parsers/basic_input_output_parser.test.ts @@ -0,0 +1,32 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { BasicInputOutputParser } from './basic_input_output_parser'; + +describe('BasicInputOutputParser', () => { + it('return input and output', async () => { + expect( + await BasicInputOutputParser.parserProvider({ + input: 'input', + response: 'response', + conversation_id: '', + interaction_id: 'interaction_id', + create_time: '', + }) + ).toEqual([ + { + type: 'input', + contentType: 'text', + content: 'input', + }, + { + type: 'output', + contentType: 'markdown', + content: 'response', + traceId: 'interaction_id', + }, + ]); + }); +}); diff --git a/server/parsers/visualization_card_parser.test.ts b/server/parsers/visualization_card_parser.test.ts new file mode 100644 index 00000000..63fd66fe --- /dev/null +++ b/server/parsers/visualization_card_parser.test.ts @@ -0,0 +1,116 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { VisualizationCardParser } from './visualization_card_parser'; + +describe('VisualizationCardParser', () => { + it('return visualizations when there is VisualizationTool.output', async () => { + expect( + await VisualizationCardParser.parserProvider({ + input: 'input', + response: 'response', + conversation_id: '', + interaction_id: 'interaction_id', + create_time: '', + additional_info: { + 'VisualizationTool.output': [ + 'row_number,Id,title\n' + + '1,id1,[Flights] Total Flights\n' + + '2,id2,[Flights] Controls\n' + + '3,id3,[Flights] Airline Carrier', + ], + }, + }) + ).toEqual([ + { + content: 'id1', + contentType: 'visualization', + suggestedActions: [{ actionType: 'view_in_dashboards', message: 'View in Visualize' }], + type: 'output', + }, + { + content: 'id2', + contentType: 'visualization', + suggestedActions: [{ actionType: 'view_in_dashboards', message: 'View in Visualize' }], + type: 'output', + }, + { + content: 'id3', + contentType: 'visualization', + suggestedActions: [{ actionType: 'view_in_dashboards', message: 'View in Visualize' }], + type: 'output', + }, + ]); + }); + + it('return visualizations when there are multiple VisualizationTool.outputs', async () => { + expect( + await VisualizationCardParser.parserProvider({ + input: 'input', + response: 'response', + conversation_id: '', + interaction_id: 'interaction_id', + create_time: '', + additional_info: { + 'VisualizationTool.output': [ + 'row_number,Id,title\n' + '1,id1,[Flights] Total Flights\n', + 'row_number,Id,title\n' + '2,id2,[Flights] Controls\n', + ], + }, + }) + ).toEqual([ + { + content: 'id1', + contentType: 'visualization', + suggestedActions: [{ actionType: 'view_in_dashboards', message: 'View in Visualize' }], + type: 'output', + }, + { + content: 'id2', + contentType: 'visualization', + suggestedActions: [{ actionType: 'view_in_dashboards', message: 'View in Visualize' }], + type: 'output', + }, + ]); + }); + + it('do not return visualizations when VisualizationTool.output is null', async () => { + expect( + await VisualizationCardParser.parserProvider({ + input: 'input', + response: 'response', + conversation_id: '', + interaction_id: 'interaction_id', + create_time: '', + additional_info: {}, + }) + ).toEqual([]); + }); + + it('do not return visualizations when VisualizationTool.output is not in correct format', async () => { + expect( + await VisualizationCardParser.parserProvider({ + input: 'input', + response: 'response', + conversation_id: '', + interaction_id: 'interaction_id', + create_time: '', + additional_info: { + 'VisualizationTool.output': [ + 'row_number\n' + '1', + 'row_number,Id,title\n' + '2,id2,[Flights] Controls\n', + ], + }, + }) + ).toEqual([ + { + content: 'id2', + contentType: 'visualization', + suggestedActions: [{ actionType: 'view_in_dashboards', message: 'View in Visualize' }], + type: 'output', + }, + ]); + }); +}); diff --git a/server/utils/csv-parser-helper.ts b/server/utils/csv-parser-helper.ts new file mode 100644 index 00000000..690d4aee --- /dev/null +++ b/server/utils/csv-parser-helper.ts @@ -0,0 +1,25 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { Readable } from 'stream'; +import csvParser from 'csv-parser'; + +export const getJsonFromString = ( + csvString: string, + options?: csvParser.Options +): Promise> | string[][]> => { + const results: string[][] | Array> = []; + return new Promise((resolve, reject) => { + Readable.from(csvString) + .pipe(csvParser(options)) + .on('data', (data) => results.push(data)) + .on('end', () => { + resolve(results); + }) + .on('error', (err) => { + reject(err); + }); + }); +}; From 2be2bb494d0acb5eb2f731bc9dfcaadc4978c525 Mon Sep 17 00:00:00 2001 From: SuZhou-Joe Date: Tue, 28 Nov 2023 16:53:21 +0800 Subject: [PATCH 4/5] feat: add test cases Signed-off-by: SuZhou-Joe --- server/utils/csv-parser-helper.test.ts | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 server/utils/csv-parser-helper.test.ts diff --git a/server/utils/csv-parser-helper.test.ts b/server/utils/csv-parser-helper.test.ts new file mode 100644 index 00000000..6671ae39 --- /dev/null +++ b/server/utils/csv-parser-helper.test.ts @@ -0,0 +1,21 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { getJsonFromString } from './csv-parser-helper'; + +describe('getJsonFromString', () => { + it('return correct answer', async () => { + expect(await getJsonFromString('title,id\n1,2')).toEqual([ + { + title: '1', + id: '2', + }, + ]); + }); + + it('return empty array when string is not in correct format', async () => { + expect(await getJsonFromString('1,2')).toEqual([]); + }); +}); From 13b799be48115573ab7a66d6b90bd8edf53e0cf2 Mon Sep 17 00:00:00 2001 From: SuZhou-Joe Date: Wed, 29 Nov 2023 10:52:57 +0800 Subject: [PATCH 5/5] feat: optimize Signed-off-by: SuZhou-Joe --- server/parsers/visualization_card_parser.ts | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/server/parsers/visualization_card_parser.ts b/server/parsers/visualization_card_parser.ts index 574bc251..b6afb731 100644 --- a/server/parsers/visualization_card_parser.ts +++ b/server/parsers/visualization_card_parser.ts @@ -6,7 +6,7 @@ import { IMessage, Interaction } from '../../common/types/chat_saved_object_attributes'; import { getJsonFromString } from '../utils/csv-parser-helper'; -const extractNthColumn = async (csv: string, column: number) => { +const extractIdsFromCsvString = async (csv: string) => { const lines = (await getJsonFromString(csv)) as Array<{ Id: string }>; return lines .map((line) => line.Id) @@ -16,16 +16,15 @@ const extractNthColumn = async (csv: string, column: number) => { export const VisualizationCardParser = { id: 'core_visualization', async parserProvider(interaction: Interaction) { - const additionalInfo = interaction.additional_info as { - 'VisualizationTool.output': string[]; - } | null; - const visualizationOutputs = additionalInfo?.['VisualizationTool.output']; + const visualizationOutputs = interaction.additional_info?.['VisualizationTool.output'] as + | string[] + | undefined; if (!visualizationOutputs) { return []; } const visualizationIds = ( - await Promise.all(visualizationOutputs.map((output) => extractNthColumn(output, 1))) - ).flatMap((id) => id); // second column is id field + await Promise.all(visualizationOutputs.map((output) => extractIdsFromCsvString(output))) + ).flatMap((id) => id); const visOutputs: IMessage[] = visualizationIds /**