Skip to content

Commit

Permalink
Merge branch 'main' into genai-integration-metrics
Browse files Browse the repository at this point in the history
  • Loading branch information
kartik727 authored Jan 26, 2024
2 parents 3f05ead + 66e33cb commit 330c79b
Show file tree
Hide file tree
Showing 23 changed files with 275 additions and 79 deletions.
6 changes: 6 additions & 0 deletions .github/workflows/CI-python.yml
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,12 @@ jobs:
pip install -v -e .
working-directory: ${{ matrix.packageDirectory }}

- if: ${{ (matrix.packageDirectory == 'erroranalysis') || (matrix.packageDirectory == 'responsibleai') }}
name: Install rai_test_utils locally until next version is released
run: |
pip install -v -e .
working-directory: rai_test_utils

- name: Pip freeze
run: |
pip freeze > installed-requirements-dev.txt
Expand Down
5 changes: 5 additions & 0 deletions apps/dashboard/src/app/textApplications.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ import {
emotionModelExplanationData
} from "../model-assessment-text/__mock_data__/emotion";
import { squad } from "../model-assessment-text/__mock_data__/squad";
import { squadGenai } from "../model-assessment-text/__mock_data__/squadGenai";

import {
IDataSet,
Expand Down Expand Up @@ -65,6 +66,10 @@ export const textApplications: ITextApplications = <const>{
squad: {
classDimension: 3,
dataset: squad
} as IModelAssessmentDataSet,
squadGenai: {
classDimension: 3,
dataset: squadGenai
} as IModelAssessmentDataSet
},
versions: { "1": 1, "2:Static-View": 2 }
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,9 @@ mockForecastingDataSingleTimeSeries.predicted_y =
startingIndexBobsSandwichesTimeSeries,
endingIndexBobsSandwichesTimeSeries
);
mockForecastingDataSingleTimeSeries.true_y = mockForecastingData.true_y.slice(
startingIndexBobsSandwichesTimeSeries,
endingIndexBobsSandwichesTimeSeries
);
if (mockForecastingData.true_y) {
mockForecastingDataSingleTimeSeries.true_y = mockForecastingData.true_y.slice(
startingIndexBobsSandwichesTimeSeries,
endingIndexBobsSandwichesTimeSeries
);
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.

import { DatasetTaskType, IDataset } from "@responsible-ai/core-ui";

export const squadGenai: IDataset = {
categorical_features: [],
class_names: undefined,
feature_names: [
"context",
"prompt",
"positive_words",
"negative_words",
"negation_words",
"negated_entities",
"named_persons",
"sentence_length"
],
features: [
[
'Architecturally, the school has a Catholic character. Atop the Main Building\'s gold dome is a golden statue of the Virgin Mary. Immediately in front of the Main Building and facing it, is a copper statue of Christ with arms upraised with the legend "Venite Ad Me Omnes". Next to the Main Building is the Basilica of the Sacred Heart. Immediately behind the basilica is the Grotto, a Marian place of prayer and reflection. It is a replica of the grotto at Lourdes, France where the Virgin Mary reputedly appeared to Saint Bernadette Soubirous in 1858. At the end of the main drive (and in a direct line that connects through 3 statues and the Gold Dome), is a simple, modern stone statue of Mary.',
'Answer the question given the context.\n\ncontext:\nArchitecturally, the school has a Catholic character. Atop the Main Building\'s gold dome is a golden statue of the Virgin Mary. Immediately in front of the Main Building and facing it, is a copper statue of Christ with arms upraised with the legend "Venite Ad Me Omnes". Next to the Main Building is the Basilica of the Sacred Heart. Immediately behind the basilica is the Grotto, a Marian place of prayer and reflection. It is a replica of the grotto at Lourdes, France where the Virgin Mary reputedly appeared to Saint Bernadette Soubirous in 1858. At the end of the main drive (and in a direct line that connects through 3 statues and the Gold Dome), is a simple, modern stone statue of Mary.\n\nquestion:\nTo whom did the Virgin Mary allegedly appear in 1858 in Lourdes France?',
50,
0,
0,
0,
3,
827
],
[
'Architecturally, the school has a Catholic character. Atop the Main Building\'s gold dome is a golden statue of the Virgin Mary. Immediately in front of the Main Building and facing it, is a copper statue of Christ with arms upraised with the legend "Venite Ad Me Omnes". Next to the Main Building is the Basilica of the Sacred Heart. Immediately behind the basilica is the Grotto, a Marian place of prayer and reflection. It is a replica of the grotto at Lourdes, France where the Virgin Mary reputedly appeared to Saint Bernadette Soubirous in 1858. At the end of the main drive (and in a direct line that connects through 3 statues and the Gold Dome), is a simple, modern stone statue of Mary.',
'Answer the question given the context.\n\ncontext:\nArchitecturally, the school has a Catholic character. Atop the Main Building\'s gold dome is a golden statue of the Virgin Mary. Immediately in front of the Main Building and facing it, is a copper statue of Christ with arms upraised with the legend "Venite Ad Me Omnes". Next to the Main Building is the Basilica of the Sacred Heart. Immediately behind the basilica is the Grotto, a Marian place of prayer and reflection. It is a replica of the grotto at Lourdes, France where the Virgin Mary reputedly appeared to Saint Bernadette Soubirous in 1858. At the end of the main drive (and in a direct line that connects through 3 statues and the Gold Dome), is a simple, modern stone statue of Mary.\n\nquestion:\nWhat is in front of the Notre Dame Main Building?',
50,
0,
0,
0,
2,
805
],
[
'Architecturally, the school has a Catholic character. Atop the Main Building\'s gold dome is a golden statue of the Virgin Mary. Immediately in front of the Main Building and facing it, is a copper statue of Christ with arms upraised with the legend "Venite Ad Me Omnes". Next to the Main Building is the Basilica of the Sacred Heart. Immediately behind the basilica is the Grotto, a Marian place of prayer and reflection. It is a replica of the grotto at Lourdes, France where the Virgin Mary reputedly appeared to Saint Bernadette Soubirous in 1858. At the end of the main drive (and in a direct line that connects through 3 statues and the Gold Dome), is a simple, modern stone statue of Mary.',
'Answer the question given the context.\n\ncontext:\nArchitecturally, the school has a Catholic character. Atop the Main Building\'s gold dome is a golden statue of the Virgin Mary. Immediately in front of the Main Building and facing it, is a copper statue of Christ with arms upraised with the legend "Venite Ad Me Omnes". Next to the Main Building is the Basilica of the Sacred Heart. Immediately behind the basilica is the Grotto, a Marian place of prayer and reflection. It is a replica of the grotto at Lourdes, France where the Virgin Mary reputedly appeared to Saint Bernadette Soubirous in 1858. At the end of the main drive (and in a direct line that connects through 3 statues and the Gold Dome), is a simple, modern stone statue of Mary.\n\nquestion:\nThe Basilica of the Sacred heart at Notre Dame is beside to which structure?',
52,
0,
0,
0,
3,
832
]
],
predicted_y: [
"This is a dummy answer",
"This is a dummy answer",
"This is a dummy answer"
],
target_column: undefined,
task_type: DatasetTaskType.GenerativeText,
true_y: undefined
};
2 changes: 1 addition & 1 deletion libs/core-ui/src/lib/DatasetCohort.ts
Original file line number Diff line number Diff line change
Expand Up @@ -120,7 +120,7 @@ export class DatasetCohort {
dataDict[index][featureName] = val;
});
});
this.dataset.true_y.forEach((val, index) => {
this.dataset.true_y?.forEach((val, index) => {
if (Array.isArray(val)) {
val.forEach((subVal, subIndex) => {
dataDict[index][DatasetCohortColumns.TrueY + subIndex.toString()] =
Expand Down
5 changes: 3 additions & 2 deletions libs/core-ui/src/lib/Interfaces/IDataset.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,8 @@ export enum DatasetTaskType {
MultilabelImageClassification = "multilabel_image_classification",
Forecasting = "forecasting",
ObjectDetection = "object_detection",
QuestionAnswering = "question_answering"
QuestionAnswering = "question_answering",
GenerativeText = "generative_text"
}

export interface ITabularDatasetMetadata {
Expand All @@ -31,7 +32,7 @@ export interface IObjectDetectionLabelType {

export interface IDataset {
task_type: DatasetTaskType;
true_y: number[] | number[][] | string[];
true_y?: number[] | number[][] | string[];
predicted_y?: number[] | number[][] | string[];
probability_y?: number[][];
features: unknown[][];
Expand Down
6 changes: 5 additions & 1 deletion libs/core-ui/src/lib/util/datasetUtils/getColumnRanges.ts
Original file line number Diff line number Diff line change
Expand Up @@ -161,7 +161,11 @@ function getRegressionErrorFeatureRange(
dataset: IDataset,
modelType: ModelTypes
): IColumnRange | undefined {
if (modelType === ModelTypes.Regression && dataset.predicted_y) {
if (
modelType === ModelTypes.Regression &&
dataset.predicted_y &&
dataset.true_y
) {
const regressionErrors = [];
for (let index = 0; index < dataset.features.length; index++) {
const trueY = dataset.true_y[index];
Expand Down
10 changes: 7 additions & 3 deletions libs/core-ui/src/lib/util/datasetUtils/getPropertyValues.ts
Original file line number Diff line number Diff line change
Expand Up @@ -44,9 +44,12 @@ export function getPropertyValues(
});
}
if (property === DatasetCohortColumns.TrueY) {
return indexes.map((index) => {
return dataset.true_y[index];
});
const trueYs = dataset.true_y;
if (trueYs) {
return indexes.map((index) => {
return trueYs[index];
});
}
}
if (dataset.predicted_y && dataset.true_y) {
return getErrors(property, indexes, dataset, modelType);
Expand All @@ -62,6 +65,7 @@ function getErrors(
): unknown[] {
if (
dataset.predicted_y &&
dataset.true_y &&
!Array.isArray(dataset.true_y) &&
!Array.isArray(dataset.predicted_y)
) {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.

import { Stack, Text } from "@fluentui/react";
import { IVisionListItem } from "@responsible-ai/core-ui";
import { localization } from "@responsible-ai/localization";
import React from "react";

import { stackTokens } from "./FlyoutObjectDetectionUtils";

interface IDetectionDetailsProps {
item: IVisionListItem; // replace with actual type
correctDetections: string;
incorrectDetections: string;
}
export class DetectionDetails extends React.Component<IDetectionDetailsProps> {
public render(): React.ReactNode {
return (
<Stack
tokens={stackTokens.large}
horizontalAlign="start"
verticalAlign="start"
>
<Stack
horizontal
tokens={stackTokens.medium}
horizontalAlign="center"
verticalAlign="center"
/>
<Stack.Item>
<Text variant="large">
{localization.InterpretVision.Dashboard.indexLabel}
{this.props.item?.index}
</Text>
</Stack.Item>
<Stack.Item>
<Text variant="large">
{localization.InterpretVision.Dashboard.correctDetections}
{this.props.correctDetections}
</Text>
</Stack.Item>
<Stack.Item>
<Text variant="large">
{localization.InterpretVision.Dashboard.incorrectDetections}
{this.props.incorrectDetections}
</Text>
</Stack.Item>
</Stack>
);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import {
IComboBox,
IComboBoxOption,
Image,
Label,
List,
Panel,
PanelType,
Expand All @@ -24,6 +25,7 @@ import * as FlyoutStyles from "../utils/FlyoutUtils";
import { getObjectDetectionImageAltText } from "../utils/getAltTextUtils";
import { getJoinedLabelString } from "../utils/labelUtils";

import { DetectionDetails } from "./DetectionDetails";
import {
flyoutStyles,
explanationImage,
Expand Down Expand Up @@ -55,7 +57,8 @@ export class FlyoutObjectDetection extends React.Component<
const selectableObjectIndexes =
FlyoutStyles.generateSelectableObjectDetectionIndexes(
localization.InterpretVision.Dashboard.prefix,
item
item,
this.props.dataset.class_names
);
this.setState({ item, metadata, selectableObjectIndexes });
}
Expand All @@ -73,7 +76,8 @@ export class FlyoutObjectDetection extends React.Component<
const selectableObjectIndexes =
FlyoutStyles.generateSelectableObjectDetectionIndexes(
localization.InterpretVision.Dashboard.prefix,
item
item,
this.props.dataset.class_names
);
this.setState({
item: this.props.item,
Expand Down Expand Up @@ -122,42 +126,11 @@ export class FlyoutObjectDetection extends React.Component<
verticalAlign="center"
>
<Stack.Item>
<Stack
tokens={FlyoutODUtils.stackTokens.large}
horizontalAlign="start"
verticalAlign="start"
>
<Stack
horizontal
tokens={FlyoutODUtils.stackTokens.medium}
horizontalAlign="center"
verticalAlign="center"
/>
<Stack.Item>
<Text variant="large">
{localization.InterpretVision.Dashboard.indexLabel}
{item?.index}
</Text>
</Stack.Item>
<Stack.Item>
<Text variant="large">
{
localization.InterpretVision.Dashboard
.correctDetections
}
{correctDetections}
</Text>
</Stack.Item>
<Stack.Item>
<Text variant="large">
{
localization.InterpretVision.Dashboard
.incorrectDetections
}
{incorrectDetections}
</Text>
</Stack.Item>
</Stack>
<DetectionDetails
item={item}
correctDetections={correctDetections}
incorrectDetections={incorrectDetections}
/>
</Stack.Item>
</Stack>
</Stack.Item>
Expand All @@ -173,13 +146,17 @@ export class FlyoutObjectDetection extends React.Component<
{localization.InterpretVision.Dashboard.panelInformation}
</Text>
</Stack.Item>
<Stack.Item className={classNames.featureListContainer}>
<Stack.Item
className={classNames.featureListContainer}
tabIndex={0}
>
<List
items={this.state.metadata}
onRenderCell={FlyoutStyles.onRenderCell}
/>
</Stack.Item>
</Stack>
<FlyoutODUtils.ColorLegend />
<Stack>
<Stack.Item className={classNames.imageContainer}>
<Stack.Item id="canvasToolsDiv">
Expand All @@ -200,17 +177,27 @@ export class FlyoutObjectDetection extends React.Component<
</Text>
</Stack.Item>
<Stack>
{
<ComboBox
id={localization.InterpretVision.Dashboard.objectSelect}
label={localization.InterpretVision.Dashboard.chooseObject}
onChange={this.selectODChoiceFromDropdown}
selectedKey={this.state.odSelectedKey}
options={this.state.selectableObjectIndexes}
className={"classNames.dropdown"}
styles={FluentUIStyles.smallDropdownStyle}
/>
}
<Label
id={
localization.InterpretVision.Dashboard.objectSelectionLabel
}
>
{localization.InterpretVision.Dashboard.chooseObject}
</Label>
<ComboBox
id={localization.InterpretVision.Dashboard.objectSelect}
onChange={this.selectODChoiceFromDropdown}
selectedKey={this.state.odSelectedKey}
options={this.state.selectableObjectIndexes}
className="classNames.dropdown"
styles={FluentUIStyles.smallDropdownStyle}
ariaLabel={
localization.InterpretVision.Dashboard.chooseObject
}
aria-labelledby={
localization.InterpretVision.Dashboard.objectSelectionLabel
}
/>
<Stack>
{!this.props.loadingExplanation[item.index][
+this.state.odSelectedKey.slice(
Expand Down
Loading

0 comments on commit 330c79b

Please sign in to comment.