Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: Add @powerpages feedback telemetry #1000

Merged
merged 9 commits into from
Aug 26, 2024
Merged

feat: Add @powerpages feedback telemetry #1000

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
eacee49
feat: Add @powerpages feedback telemetry
Jul 16, 2024
f6519b1
Merge branch 'main' into users/amitjoshi/@powerpagesFeedbackTelemetry
Aug 14, 2024
9dd42a9
refactor: response return and feedback
Aug 14, 2024
1fe3a10
Merge branch 'main' into users/amitjoshi/@powerpagesFeedbackTelemetry
Aug 21, 2024
5e8f596
refactor:ADX_WEBPAGE CONSTANT
Aug 21, 2024
8a92117
Merge branch 'main' into users/amitjoshi/@powerpagesFeedbackTelemetry
amitjoshi438 Aug 21, 2024
38a028f
refactor: Update PowerPagesChatParticipantConstants formatting
Aug 22, 2024
e5b8a4c
Merge branch 'users/amitjoshi/@powerpagesFeedbackTelemetry' of https:…
Aug 22, 2024
d0044a9
refactor: Improve PowerPagesChatParticipant logic and error handling
Aug 25, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
173 changes: 80 additions & 93 deletions src/common/chat-participants/powerpages/PowerPagesChatParticipant.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,14 +12,14 @@ import { sendApiRequest } from '../../copilot/IntelligenceApiService';
import { PacWrapper } from '../../../client/pac/PacWrapper';
import { intelligenceAPIAuthentication } from '../../services/AuthenticationProvider';
import { ActiveOrgOutput } from '../../../client/pac/PacTypes';
import { AUTHENTICATION_FAILED_MSG, COPILOT_NOT_AVAILABLE_MSG, DISCLAIMER_MESSAGE, NO_PROMPT_MESSAGE, PAC_AUTH_NOT_FOUND, POWERPAGES_CHAT_PARTICIPANT_ID, RESPONSE_AWAITED_MSG, SKIP_CODES, STATER_PROMPTS, VSCODE_EXTENSION_GITHUB_POWER_PAGES_AGENT_INVOKED, VSCODE_EXTENSION_GITHUB_POWER_PAGES_AGENT_ORG_DETAILS, VSCODE_EXTENSION_GITHUB_POWER_PAGES_AGENT_ORG_DETAILS_NOT_FOUND, VSCODE_EXTENSION_GITHUB_POWER_PAGES_AGENT_SCENARIO, WELCOME_MESSAGE, WELCOME_PROMPT } from './PowerPagesChatParticipantConstants';
import { AUTHENTICATION_FAILED_MSG, COPILOT_NOT_AVAILABLE_MSG, DISCLAIMER_MESSAGE, INVALID_RESPONSE, NO_PROMPT_MESSAGE, PAC_AUTH_NOT_FOUND, POWERPAGES_CHAT_PARTICIPANT_ID, RESPONSE_AWAITED_MSG, RESPONSE_SCENARIOS, SKIP_CODES, STATER_PROMPTS, VSCODE_EXTENSION_GITHUB_POWER_PAGES_AGENT_ERROR, VSCODE_EXTENSION_GITHUB_POWER_PAGES_AGENT_INVOKED, VSCODE_EXTENSION_GITHUB_POWER_PAGES_AGENT_ORG_DETAILS, VSCODE_EXTENSION_GITHUB_POWER_PAGES_AGENT_ORG_DETAILS_NOT_FOUND, VSCODE_EXTENSION_GITHUB_POWER_PAGES_AGENT_SCENARIO, WELCOME_MESSAGE, WELCOME_PROMPT } from './PowerPagesChatParticipantConstants';
import { ORG_DETAILS_KEY, handleOrgChangeSuccess, initializeOrgDetails } from '../../utilities/OrgHandlerUtils';
import { createAndReferenceLocation, getComponentInfo, getEndpoint, provideChatParticipantFollowups } from './PowerPagesChatParticipantUtils';
import { createAndReferenceLocation, getComponentInfo, getEndpoint, provideChatParticipantFollowups, handleChatParticipantFeedback, createErrorResult, createSuccessResult } from './PowerPagesChatParticipantUtils';
import { checkCopilotAvailability, fetchRelatedFiles, getActiveEditorContent } from '../../utilities/Utils';
import { IIntelligenceAPIEndpointInformation } from '../../services/Interfaces';
import { v4 as uuidv4 } from 'uuid';
import { orgChangeErrorEvent, orgChangeEvent } from '../../../client/OrgChangeNotifier';
import { IRelatedFiles } from '../../constants';
import { ADX_WEBPAGE, IRelatedFiles } from '../../constants';

export class PowerPagesChatParticipant {
private static instance: PowerPagesChatParticipant | null = null;
Expand All @@ -43,6 +43,10 @@ export class PowerPagesChatParticipant {
//TODO: Check the icon image
this.chatParticipant.iconPath = vscode.Uri.joinPath(context.extensionUri, 'src', 'common', 'chat-participants', 'powerpages', 'assets', 'copilot.png');

this.chatParticipant.onDidReceiveFeedback((feedback: vscode.ChatResultFeedback) => {
handleChatParticipantFeedback(feedback, this.powerPagesAgentSessionId, this.telemetry);
}
);
this.chatParticipant.followupProvider = {
provideFollowups: provideChatParticipantFollowups
};
Expand Down Expand Up @@ -82,89 +86,59 @@ export class PowerPagesChatParticipant {
stream: vscode.ChatResponseStream,
//_token: vscode.CancellationToken
): Promise<IPowerPagesChatResult> => {
// Handle chat requests here

stream.progress(RESPONSE_AWAITED_MSG)

this.telemetry.sendTelemetryEvent(VSCODE_EXTENSION_GITHUB_POWER_PAGES_AGENT_INVOKED, { sessionId: this.powerPagesAgentSessionId });
try {
stream.progress(RESPONSE_AWAITED_MSG);
this.telemetry.sendTelemetryEvent(VSCODE_EXTENSION_GITHUB_POWER_PAGES_AGENT_INVOKED, { sessionId: this.powerPagesAgentSessionId });

await this.initializeOrgDetails();
await this.initializeOrgDetails();

if (!this.orgID || !this.environmentID) {
stream.markdown(PAC_AUTH_NOT_FOUND);
this.telemetry.sendTelemetryEvent(VSCODE_EXTENSION_GITHUB_POWER_PAGES_AGENT_ORG_DETAILS_NOT_FOUND, { sessionId: this.powerPagesAgentSessionId });
return {
metadata: {
command: ''
}
};
}

this.telemetry.sendTelemetryEvent(VSCODE_EXTENSION_GITHUB_POWER_PAGES_AGENT_ORG_DETAILS, { orgID: this.orgID, environmentID: this.environmentID, sessionId: this.powerPagesAgentSessionId });

const intelligenceApiAuthResponse = await intelligenceAPIAuthentication(this.telemetry, this.powerPagesAgentSessionId, this.orgID, true);

if (!intelligenceApiAuthResponse) {
stream.markdown(AUTHENTICATION_FAILED_MSG);
return {
metadata: {
command: '',
}
};
}
if (!this.orgID || !this.environmentID) {
this.telemetry.sendTelemetryEvent(VSCODE_EXTENSION_GITHUB_POWER_PAGES_AGENT_ORG_DETAILS_NOT_FOUND, { sessionId: this.powerPagesAgentSessionId });
return createErrorResult(PAC_AUTH_NOT_FOUND, RESPONSE_SCENARIOS.PAC_AUTH_NOT_FOUND, '');
}

const intelligenceApiToken = intelligenceApiAuthResponse.accessToken;
this.telemetry.sendTelemetryEvent(VSCODE_EXTENSION_GITHUB_POWER_PAGES_AGENT_ORG_DETAILS, { orgID: this.orgID, environmentID: this.environmentID, sessionId: this.powerPagesAgentSessionId });

const intelligenceAPIEndpointInfo = await getEndpoint(this.orgID, this.environmentID, this.telemetry, this.cachedEndpoint, this.powerPagesAgentSessionId);
const intelligenceApiAuthResponse = await intelligenceAPIAuthentication(this.telemetry, this.powerPagesAgentSessionId, this.orgID, true);

const copilotAvailabilityStatus = checkCopilotAvailability(intelligenceAPIEndpointInfo.intelligenceEndpoint, this.orgID, this.telemetry, this.powerPagesAgentSessionId);
if (!intelligenceApiAuthResponse) {
return createErrorResult(AUTHENTICATION_FAILED_MSG, RESPONSE_SCENARIOS.AUTHENTICATION_FAILED, this.orgID);
}

if (!copilotAvailabilityStatus) {
stream.markdown(COPILOT_NOT_AVAILABLE_MSG)
return {
metadata: {
command: ''
}
};
}
const intelligenceApiToken = intelligenceApiAuthResponse.accessToken;
const intelligenceAPIEndpointInfo = await getEndpoint(this.orgID, this.environmentID, this.telemetry, this.cachedEndpoint, this.powerPagesAgentSessionId);
amitjoshi438 marked this conversation as resolved.
Show resolved Hide resolved

const userPrompt = request.prompt;
const copilotAvailabilityStatus = checkCopilotAvailability(intelligenceAPIEndpointInfo.intelligenceEndpoint, this.orgID, this.telemetry, this.powerPagesAgentSessionId);

if (userPrompt === WELCOME_PROMPT) {
stream.markdown(WELCOME_MESSAGE);
return {
metadata: {
command: STATER_PROMPTS
}
if (!copilotAvailabilityStatus || !intelligenceAPIEndpointInfo.intelligenceEndpoint) {
return createErrorResult(COPILOT_NOT_AVAILABLE_MSG, RESPONSE_SCENARIOS.COPILOT_NOT_AVAILABLE, this.orgID);
}
}

if (!userPrompt) {
stream.markdown(NO_PROMPT_MESSAGE);
return {
metadata: {
command: STATER_PROMPTS
}
};
}

const { activeFileContent, activeFileUri, startLine, endLine, activeFileParams } = getActiveEditorContent();
const userPrompt = request.prompt;

const location = activeFileUri ? createAndReferenceLocation(activeFileUri, startLine, endLine) : undefined;
if (userPrompt === WELCOME_PROMPT) {
stream.markdown(WELCOME_MESSAGE);
return createSuccessResult(STATER_PROMPTS, RESPONSE_SCENARIOS.WELCOME_PROMPT, this.orgID);
} else if (!userPrompt) {
stream.markdown(NO_PROMPT_MESSAGE);
return createSuccessResult('', RESPONSE_SCENARIOS.NO_PROMPT, this.orgID);
}

if (location) {
stream.reference(location);
}
const { activeFileContent, activeFileUri, startLine, endLine, activeFileParams } = getActiveEditorContent();
const location = activeFileUri ? createAndReferenceLocation(activeFileUri, startLine, endLine) : undefined;

if (request.command) {
//TODO: Handle command scenarios
if (request.command) {
//TODO: Handle command scenarios
} else {
if (location) {
stream.reference(location);
}

} else {
const relatedFiles: IRelatedFiles[] = [];
const relatedFiles: IRelatedFiles[] = [];

// Based on dataverse entity fetch required context for the active file
switch (activeFileParams.dataverseEntity) {
case 'adx_webpage':
case ADX_WEBPAGE:
if (activeFileUri) {
const files = await fetchRelatedFiles(activeFileUri, activeFileParams.dataverseEntity, activeFileParams.fieldType, this.telemetry, this.powerPagesAgentSessionId);
relatedFiles.push(...files);
Expand All @@ -174,34 +148,47 @@ export class PowerPagesChatParticipant {
break;
}

const { componentInfo, entityName }: IComponentInfo = await getComponentInfo(this.telemetry, this.orgUrl, activeFileParams, this.powerPagesAgentSessionId);

const llmResponse = await sendApiRequest([{ displayText: userPrompt, code: activeFileContent }], activeFileParams, this.orgID, intelligenceApiToken, this.powerPagesAgentSessionId, entityName, componentInfo, this.telemetry, intelligenceAPIEndpointInfo.intelligenceEndpoint, intelligenceAPIEndpointInfo.geoName, intelligenceAPIEndpointInfo.crossGeoDataMovementEnabledPPACFlag, relatedFiles);

const scenario = llmResponse.length > 1 ? llmResponse[llmResponse.length - 1] : llmResponse[0].displayText;

this.telemetry.sendTelemetryEvent(VSCODE_EXTENSION_GITHUB_POWER_PAGES_AGENT_SCENARIO, { scenario: scenario, sessionId: this.powerPagesAgentSessionId, orgId: this.orgID, environmentId: this.environmentID })

llmResponse.forEach((response: { displayText: string | vscode.MarkdownString; code: string; }) => {

if (response.displayText) {
stream.markdown(response.displayText);
const { componentInfo, entityName }: IComponentInfo = await getComponentInfo(this.telemetry, this.orgUrl, activeFileParams, this.powerPagesAgentSessionId);

const llmResponse = await sendApiRequest(
[{ displayText: userPrompt, code: activeFileContent }],
activeFileParams,
this.orgID,
intelligenceApiToken,
this.powerPagesAgentSessionId,
entityName,
componentInfo,
this.telemetry,
intelligenceAPIEndpointInfo.intelligenceEndpoint,
intelligenceAPIEndpointInfo.geoName,
intelligenceAPIEndpointInfo.crossGeoDataMovementEnabledPPACFlag,
relatedFiles
);

const scenario = llmResponse.length > 1 ? llmResponse[llmResponse.length - 1] : llmResponse[0].displayText;

this.telemetry.sendTelemetryEvent(VSCODE_EXTENSION_GITHUB_POWER_PAGES_AGENT_SCENARIO, { scenario: scenario, sessionId: this.powerPagesAgentSessionId, orgId: this.orgID, environmentId: this.environmentID });

llmResponse.forEach((response: { displayText: string | vscode.MarkdownString; code: string; }) => {
if (response.displayText) {
stream.markdown(response.displayText);
stream.markdown('\n');
}
if (response.code && !SKIP_CODES.includes(response.code)) {
stream.markdown('\n```javascript\n' + response.code + '\n```');
}
stream.markdown('\n');
}
if (response.code && !SKIP_CODES.includes(response.code)) {
stream.markdown('\n```javascript\n' + response.code + '\n```');
}
stream.markdown('\n');
});
stream.markdown(DISCLAIMER_MESSAGE);
}
});

return {
metadata: {
command: ''
stream.markdown(DISCLAIMER_MESSAGE);
return createSuccessResult('', scenario.toString(), this.orgID);
}
};

return createSuccessResult('', '', this.orgID);
} catch (error) {
this.telemetry.sendTelemetryEvent(VSCODE_EXTENSION_GITHUB_POWER_PAGES_AGENT_ERROR, { sessionId: this.powerPagesAgentSessionId, error: error as string });
amitjoshi438 marked this conversation as resolved.
Show resolved Hide resolved
return createErrorResult(INVALID_RESPONSE, RESPONSE_SCENARIOS.INVALID_RESPONSE, this.orgID ? this.orgID : '');
}
};

private async initializeOrgDetails(): Promise<void> {
Expand Down
17 changes: 17 additions & 0 deletions src/common/chat-participants/powerpages/PowerPagesChatParticipantConstants.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,13 +12,17 @@ export const RESPONSE_AWAITED_MSG = vscode.l10n.t('Working on it...');
export const AUTHENTICATION_FAILED_MSG = vscode.l10n.t('Authentication failed. Please try again.');
export const COPILOT_NOT_AVAILABLE_MSG = vscode.l10n.t('Copilot is not available. Please contact your administrator.');
export const PAC_AUTH_NOT_FOUND = vscode.l10n.t('Active auth profile is not found or has expired. Please try again.');
export const INVALID_RESPONSE = vscode.l10n.t('Something went wrong. Don’t worry, you can try again.');
export const DISCLAIMER_MESSAGE = vscode.l10n.t('Make sure AI-generated content is accurate and appropriate before using. [Learn more](https://go.microsoft.com/fwlink/?linkid=2240145) | [View terms](https://go.microsoft.com/fwlink/?linkid=2189520)');
export const SUPPORTED_ENTITIES = [ADX_ENTITYFORM, ADX_ENTITYLIST];
// Telemetry Event Names
export const VSCODE_EXTENSION_GITHUB_POWER_PAGES_AGENT_INVOKED = 'GitHubPowerPagesAgentInvoked';
export const VSCODE_EXTENSION_GITHUB_POWER_PAGES_AGENT_ORG_DETAILS = 'GitHubPowerPagesAgentOrgDetails';
export const VSCODE_EXTENSION_GITHUB_POWER_PAGES_AGENT_ORG_DETAILS_NOT_FOUND = 'GitHubPowerPagesAgentOrgDetailsNotFound';
export const VSCODE_EXTENSION_GITHUB_POWER_PAGES_AGENT_SCENARIO = 'GitHubPowerPagesAgentScenario';
export const VSCODE_EXTENSION_GITHUB_POWER_PAGES_AGENT_SCENARIO_FEEDBACK_THUMBSUP = 'GitHubPowerPagesAgentScenarioFeedbackThumbsUp';
export const VSCODE_EXTENSION_GITHUB_POWER_PAGES_AGENT_SCENARIO_FEEDBACK_THUMBSDOWN = 'GitHubPowerPagesAgentScenarioFeedbackThumbsDown';
export const VSCODE_EXTENSION_GITHUB_POWER_PAGES_AGENT_ERROR = 'GitHubPowerPagesAgentError';
export const SKIP_CODES = ["", null, undefined, "violation", "unclear", "explain"];
export const EXPLAIN_CODE_PROMPT = vscode.l10n.t('Explain the following code {% include \'Page Copy\'%}');
export const WEB_API_PROMPT = vscode.l10n.t('Write web API code to query active contact records.');
Expand All @@ -27,5 +31,18 @@ export const LIST_PROMPT = vscode.l10n.t('Write JavaScript code to highlight the
export const STATER_PROMPTS = "starterPrompts"
export const WELCOME_PROMPT = 'how can you help with coding for my website?'
export const WELCOME_MESSAGE = vscode.l10n.t('Hi! @powerpages can help you write, edit, and even summarize your website code.')
export const RESPONSE_SCENARIOS = {
PAC_AUTH_NOT_FOUND: 'PAC_AUTH_NOT_FOUND',
AUTHENTICATION_FAILED: 'AUTHENTICATION_FAILED',
COPILOT_NOT_AVAILABLE: 'COPILOT_NOT_AVAILABLE',
INVALID_RESPONSE: 'INVALID_RESPONSE',
RESPONSE_AWAITED: 'RESPONSE_AWAITED',
NO_PROMPT: 'NO_PROMPT',
EXPLAIN_CODE_PROMPT: 'EXPLAIN_CODE_PROMPT',
FORM_PROMPT: 'FORM_PROMPT',
LIST_PROMPT: 'LIST_PROMPT',
WEB_API_PROMPT: 'WEB_API_PROMPT',
WELCOME_PROMPT: 'WELCOME_PROMPT'
};


2 changes: 2 additions & 0 deletions src/common/chat-participants/powerpages/PowerPagesChatParticipantTypes.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@ import * as vscode from 'vscode';
export interface IPowerPagesChatResult extends vscode.ChatResult {
metadata: {
command: string;
scenario: string;
amitjoshi438 marked this conversation as resolved.
Show resolved Hide resolved
orgId?: string;
}
}
export interface IOrgDetails {
Expand Down
35 changes: 34 additions & 1 deletion src/common/chat-participants/powerpages/PowerPagesChatParticipantUtils.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ import { ITelemetry } from "../../OneDSLoggerTelemetry/telemetry/ITelemetry";
import { ArtemisService } from "../../services/ArtemisService";
import { dataverseAuthentication } from "../../services/AuthenticationProvider";
import { IIntelligenceAPIEndpointInformation } from "../../services/Interfaces";
import { EXPLAIN_CODE_PROMPT, FORM_PROMPT, LIST_PROMPT, STATER_PROMPTS, SUPPORTED_ENTITIES, WEB_API_PROMPT } from "./PowerPagesChatParticipantConstants";
import { SUPPORTED_ENTITIES, VSCODE_EXTENSION_GITHUB_POWER_PAGES_AGENT_SCENARIO_FEEDBACK_THUMBSDOWN, VSCODE_EXTENSION_GITHUB_POWER_PAGES_AGENT_SCENARIO_FEEDBACK_THUMBSUP, EXPLAIN_CODE_PROMPT, FORM_PROMPT, LIST_PROMPT, STATER_PROMPTS, WEB_API_PROMPT } from "./PowerPagesChatParticipantConstants";
import { IComponentInfo, IPowerPagesChatResult } from "./PowerPagesChatParticipantTypes";
import * as vscode from 'vscode';

Expand Down Expand Up @@ -57,6 +57,15 @@ export function isEntityInSupportedList(entity: string): boolean {
return SUPPORTED_ENTITIES.includes(entity);
}

export function handleChatParticipantFeedback (feedback: vscode.ChatResultFeedback, sessionId: string, telemetry: ITelemetry) {
const scenario = feedback.result.metadata?.scenario;
const orgId = feedback.result.metadata?.orgId;
if (feedback.kind === 1) {
telemetry.sendTelemetryEvent(VSCODE_EXTENSION_GITHUB_POWER_PAGES_AGENT_SCENARIO_FEEDBACK_THUMBSUP, { feedback: feedback.kind.toString(), scenario: scenario, orgId:orgId, sessionId: sessionId });
} else if (feedback.kind === 0) {
telemetry.sendTelemetryEvent(VSCODE_EXTENSION_GITHUB_POWER_PAGES_AGENT_SCENARIO_FEEDBACK_THUMBSDOWN, { feedback: feedback.kind.toString(), scenario: scenario, orgId: orgId, sessionId: sessionId});
}
}
export function createAndReferenceLocation(activeFileUri: vscode.Uri, startLine: number, endLine: number): vscode.Location {

const positionStart = new vscode.Position(startLine, 0),
Expand All @@ -77,3 +86,27 @@ export function provideChatParticipantFollowups(result: IPowerPagesChatResult, _
];
}
}

export function createErrorResult(message: string, scenario: string, orgId: string): IPowerPagesChatResult {
return {
metadata: {
command: '',
scenario: scenario,
orgId: orgId
},
errorDetails: {
message: message
}
};
}

export function createSuccessResult(command: string, scenario: string, orgId: string): IPowerPagesChatResult {
return {
metadata: {
command: command,
scenario: scenario,
orgId: orgId
}
};
}

1 change: 1 addition & 0 deletions src/common/constants.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,3 +65,4 @@ export interface IRelatedFiles {
}

export const COPILOT_RELATED_FILES_FETCH_FAILED = "CopilotRelatedFilesFetchFailed";
export const ADX_WEBPAGE = 'adx_webpage'
Loading