Calibrate ensemble chart to allow variables not mapped (#6163)

Co-authored-by: Yohann Paris <[email protected]>

packages/client/hmi-client/src/components/workflow/ops/calibrate-ensemble-ciemss/calibrate-ensemble-util.ts

@@ -13,6 +13,7 @@ import { EnsembleModelConfigs, ModelConfiguration } from '@/types/Types';
 import { WorkflowNode } from '@/types/workflow';
 import { getActiveOutput } from '@/components/workflow/util';
 import { CalibrateMap, setupModelInput } from '@/services/calibrate-workflow';
+import { getAsConfiguredModel } from '@/services/model-configurations';
 import {
 	CalibrateEnsembleCiemssOperationState,
 	CalibrateEnsembleMappingRow,
@@ -85,8 +86,9 @@ export function formatCalibrateModelConfigurations(
 	return [...Object.values(ensembleModelConfigMap)];
 }
 
-export function getSelectedOutputEnsembleMapping(
+export function getChartEnsembleMapping(
 	node: WorkflowNode<CalibrateEnsembleCiemssOperationState>,
+	stateToModelConfigMap: { [key: string]: string[] },
 	hasTimestampCol = true
 ) {
 	const wfOutputState = getActiveOutput(node)?.state;
@@ -97,6 +99,22 @@ export function getSelectedOutputEnsembleMapping(
 			datasetMapping: wfOutputState?.timestampColName ?? '',
 			modelConfigurationMappings: {}
 		});
+
+	// For every State Variable that has not been mapped in the ensembleMapping
+	// We will fill in here so the user can still see these if they want
+	Object.keys(stateToModelConfigMap).forEach((state) => {
+		if (!mapping.find((map) => map.newName === state)) {
+			const modelConfigurationsMap = {};
+			stateToModelConfigMap[state].forEach((id) => {
+				modelConfigurationsMap[id] = state;
+			});
+			mapping.push({
+				newName: state,
+				datasetMapping: '',
+				modelConfigurationMappings: modelConfigurationsMap
+			});
+		}
+	});
 	return mapping;
 }
 
@@ -213,3 +231,37 @@ export function getEnsembleErrorData(
 	});
 	return errorData;
 }
+
+// This will grab all of the variables in each model configuration and place them into a dictionary.
+// The key will be the variable, the value will be a list of uuids that this variable is found in.
+// An example output with two model config ids uuid-1 and uuid-2 may look like where model 1 is SIRD, and model 2 is SIR
+// {
+// 		S: ["uuid-1","uuid-2"]
+// 		I: ["uuid-1","uuid-2"]
+// 		R: ["uuid-1","uuid-2"]
+// 		D: ["uuid-1"]
+// }
+
+export async function setStateToModelConfigMap(modelConfigurationIds: string[]) {
+	const stateToModelConfigMap: { [key: string]: string[] } = {};
+	const models: any[] = [];
+	// Model configuration input
+	await Promise.all(
+		modelConfigurationIds.map(async (id) => {
+			const model = await getAsConfiguredModel(id);
+			models.push({ ...model, configId: id });
+		})
+	);
+
+	models.forEach((model) => {
+		const modelConfigId = model.configId as string;
+		model.model.states.forEach((state) => {
+			const key = state.id;
+			if (!stateToModelConfigMap[key]) {
+				stateToModelConfigMap[key] = [];
+			}
+			stateToModelConfigMap[key].push(modelConfigId);
+		});
+	});
+	return stateToModelConfigMap;
+}

packages/client/hmi-client/src/components/workflow/ops/calibrate-ensemble-ciemss/tera-calibrate-ensemble-ciemss-drilldown.vue

@@ -390,12 +390,13 @@ import {
 	updateLossChartSpec,
 	getLossValuesFromSimulation,
 	formatCalibrateModelConfigurations,
-	getSelectedOutputEnsembleMapping,
+	getChartEnsembleMapping,
 	fetchOutputData,
 	buildChartData,
 	getEnsembleErrorData,
 	EnsembleErrorData,
-	fetchModelConfigurations
+	fetchModelConfigurations,
+	setStateToModelConfigMap
 } from './calibrate-ensemble-util';
 
 const props = defineProps<{
@@ -456,15 +457,23 @@ const isRunInProgress = computed(() => Boolean(inProgressCalibrationId.value ||
 
 const datasetId = computed(() => props.node.inputs[0].value?.[0] as string | undefined);
 const currentDatasetFileName = ref<string>();
-const datasetColumnNames = computed(() => dataset.value?.columns?.map((col) => col.name) ?? ([] as string[]));
+const datasetColumnNames = computed(
+	() =>
+		dataset.value?.columns?.filter((col) => col.fileName === currentDatasetFileName.value).map((col) => col.name) ??
+		([] as string[])
+);
 // Loss Chart:
 const lossChartRef = ref<InstanceType<typeof VegaChart>>();
 const lossChartSpec = ref();
 const lossValues = ref<{ [key: string]: number }[]>([]);
 const LOSS_CHART_DATA_SOURCE = 'lossData';
 // Model:
+const modelConfigIds = computed(() =>
+	props.node.inputs.filter((input) => input.type === 'modelConfigId' && input.value).map((input) => input.value?.[0])
+);
 const listModelLabels = ref<string[]>([]);
 const allModelConfigurations = ref<ModelConfiguration[]>([]);
+const stateToModelConfigMap = ref<{ [key: string]: string[] }>({});
 
 const tableHeaders = computed(() => {
 	const headers = ['Ensemble model'];
@@ -617,6 +626,7 @@ const runEnsemble = async () => {
 };
 
 onMounted(async () => {
+	stateToModelConfigMap.value = await setStateToModelConfigMap(modelConfigIds.value as string[]);
 	const configs = await fetchModelConfigurations(props.node.inputs);
 	if (!configs) return;
 	allModelConfigurations.value = configs.allModelConfigurations;
@@ -661,7 +671,7 @@ const outputData = ref<{
 } | null>(null);
 const groundTruthData = computed<DataArray>(() => parseCsvAsset(csvAsset.value as CsvAsset));
 const chartSize = useDrilldownChartSize(chartWidthDiv);
-const selectedOutputMapping = computed(() => getSelectedOutputEnsembleMapping(props.node));
+const selectedOutputMapping = computed(() => getChartEnsembleMapping(props.node, stateToModelConfigMap.value));
 const {
 	activeChartSettings,
 	chartSettings,
@@ -699,7 +709,9 @@ const errorData = computed<EnsembleErrorData>(() =>
 	)
 );
 
-const ensembleVariables = computed(() => getSelectedOutputEnsembleMapping(props.node, false).map((d) => d.newName));
+const ensembleVariables = computed(() =>
+	getChartEnsembleMapping(props.node, stateToModelConfigMap.value, false).map((d) => d.newName)
+);
 const ensembleVariableCharts = useEnsembleVariableCharts(selectedEnsembleVariableSettings, groundTruthData);
 const weightsDistributionCharts = useWeightsDistributionCharts();
 const { errorCharts, onExpandErrorChart } = useEnsembleErrorCharts(selectedErrorVariableSettings, errorData);

packages/client/hmi-client/src/components/workflow/ops/calibrate-ensemble-ciemss/tera-calibrate-ensemble-node-ciemss.vue

@@ -59,7 +59,7 @@ import {
 	updateLossChartSpec,
 	getLossValuesFromSimulation,
 	formatCalibrateModelConfigurations,
-	getSelectedOutputEnsembleMapping,
+	getChartEnsembleMapping,
 	buildChartData,
 	fetchModelConfigurations,
 	fetchOutputData
@@ -93,7 +93,7 @@ const outputData = ref<{
 	resultGroupByTimepoint: GroupedDataArray;
 } | null>(null);
 const groundTruthData = computed<DataArray>(() => parseCsvAsset(csvAsset.value as CsvAsset));
-const selectedOutputMapping = computed(() => getSelectedOutputEnsembleMapping(props.node));
+const selectedOutputMapping = computed(() => getChartEnsembleMapping(props.node, {}));
 const { selectedEnsembleVariableSettings } = useChartSettings(props, emit);
 const { useEnsembleVariableCharts } = useCharts(
 	props.node.id,

packages/client/hmi-client/src/components/workflow/ops/funman/tera-funman-drilldown.vue

@@ -834,7 +834,7 @@ onMounted(async () => {
 	const modelConfigurationId = props.node.inputs[0].value?.[0];
 	if (!modelConfigurationId) return;
 	const modelConfiguration = await getModelConfigurationById(modelConfigurationId);
-	configuredInputModel = await getAsConfiguredModel(modelConfiguration);
+	configuredInputModel = await getAsConfiguredModel(modelConfigurationId);
 
 	setModelOptions();
 

packages/client/hmi-client/src/composables/useCharts.ts

@@ -193,6 +193,7 @@ export function useCharts(
 		multiVariable = false,
 		showBaseLines = false
 	) => {
+		setting.hideInNode = true;
 		const ensembleVarName = setting.selectedVariables[0];
 		const options: ForecastChartOptions = {
 			title: getModelConfigName(<ModelConfiguration[]>modelConfig?.value ?? [], modelConfigId) || ensembleVarName,

packages/client/hmi-client/src/services/model-configurations.ts

@@ -46,8 +46,8 @@ export const deleteModelConfiguration = async (id: string) => {
 	return response?.data ?? null;
 };
 
-export const getAsConfiguredModel = async (modelConfiguration: ModelConfiguration): Promise<Model> => {
-	const response = await API.get<Model>(`model-configurations/as-configured-model/${modelConfiguration.id}`);
+export const getAsConfiguredModel = async (modelConfigurationId: string): Promise<Model> => {
+	const response = await API.get<Model>(`model-configurations/as-configured-model/${modelConfigurationId}`);
 	return response?.data ?? null;
 };