added error bars

MadcowD · Oct 13, 2024 · 26abbf6 · 26abbf6
1 parent 6fbc927
commit 26abbf6
Show file tree

Hide file tree

Showing 12 changed files with 313 additions and 108 deletions.
diff --git a/ell-studio/package-lock.json b/ell-studio/package-lock.json
diff --git a/ell-studio/package.json b/ell-studio/package.json
@@ -17,6 +17,7 @@
     "axios": "^1.6.0",
     "base64-js": "^1.5.1",
     "chart.js": "^4.4.4",
+    "chartjs-chart-error-bars": "^4.4.2",
     "class-variance-authority": "^0.7.0",
     "clsx": "^2.1.1",
     "d3-force": "^3.0.0",

diff --git a/ell-studio/src/components/depgraph/graphUtils.js b/ell-studio/src/components/depgraph/graphUtils.js
@@ -38,6 +38,7 @@ const calculateNodeDimensions = (nodeType, data) => {
  * @returns {Object} - Contains initial nodes and edges.
  */
 export const getInitialGraph = (lmps, traces, evals) => {
+  if(!lmps || !traces || !evals) return { initialNodes: [], initialEdges: [] };
   const lmpIds = new Set(lmps.map(lmp => lmp.lmp_id));
   const evalLmpIds = new Set();
   const lmpToEvalMap = new Map();

diff --git a/ell-studio/src/components/evaluations/EvaluationOverview.js b/ell-studio/src/components/evaluations/EvaluationOverview.js
@@ -1,12 +1,19 @@
-import React from 'react';
+import React, { useState } from 'react';
 import { FiBarChart2, FiClock, FiDatabase, FiTag, FiZap } from 'react-icons/fi';
 import { Card, CardContent } from '../common/Card';
 import RunSummary from './RunSummary';
 import VersionBadge from '../VersionBadge';
 import { getTimeAgo } from '../../utils/lmpUtils';
 import MetricGraphGrid from './MetricGraphGrid';
 
-function EvaluationOverview({ evaluation, groupedRuns }) {
+function EvaluationOverview({ evaluation, groupedRuns, onActiveIndexChange }) {
+  const [activeIndex, setActiveIndex] = useState(null);
+
+  const handleActiveIndexChange = (index) => {
+    setActiveIndex(index);
+    onActiveIndexChange(index);
+  };
+
   return (
     <>
       <div className="mb-6">
@@ -15,7 +22,19 @@ function EvaluationOverview({ evaluation, groupedRuns }) {
           <span className="text-muted-foreground mx-2">•</span>
           <VersionBadge version={evaluation.version_number} />
         </h2>
-        <MetricGraphGrid evaluation={evaluation} groupedRuns={groupedRuns} />
+        {evaluation.labelers ? (
+          <MetricGraphGrid 
+            evaluation={evaluation} 
+            groupedRuns={groupedRuns} 
+            onActiveIndexChange={handleActiveIndexChange}
+          />
+        ) : (
+          <div className="animate-pulse">
+            <div className="h-4 bg-gray-300 rounded w-3/4 mb-2"></div>
+            <div className="h-4 bg-gray-300 rounded w-1/2 mb-2"></div>
+            <div className="h-4 bg-gray-300 rounded w-1/4"></div>
+          </div>
+        )}
       </div>
     </>
   );

diff --git a/ell-studio/src/components/evaluations/EvaluationRunsTable.js b/ell-studio/src/components/evaluations/EvaluationRunsTable.js
@@ -6,20 +6,21 @@ import { Card } from '../common/Card';
 import { getTimeAgo } from '../../utils/lmpUtils';
 import VersionBadge from '../VersionBadge';
 
-const EvaluationRunsTable = ({ runs, currentPage, setCurrentPage, pageSize, onSelectRun, currentlySelectedRun }) => {
+const EvaluationRunsTable = ({ runs, currentPage, setCurrentPage, pageSize, onSelectRun, currentlySelectedRun, activeIndex }) => {
   const navigate = useNavigate();
 
   const onClickLMP = (run) => {
     navigate(`/lmp/${run.evaluated_lmp.name}/${run.evaluated_lmp.lmp_id}`);
   };
 
   const runsTableData = useMemo(() => {
-    return runs.map(run => ({
+    return runs.map((run, index) => ({
       ...run,
       id: run.id,
       name: run.evaluated_lmp.name,
       version: run.evaluated_lmp.version_number + 1,
       created_at: new Date(run.end_time),
+      runIndex: index, // Add this line to keep track of the run's index
     }));
   }, [runs]);
 
@@ -99,9 +100,18 @@ const EvaluationRunsTable = ({ runs, currentPage, setCurrentPage, pageSize, onSe
       data={runsTableData}
       onRowClick={onSelectRun}
       initialSortConfig={initialSortConfig}
-      rowClassName={(item) => 
-        item.id === currentlySelectedRun?.id ? 'bg-blue-600 bg-opacity-30' : ''
-      }
+      rowClassName={(item) => {
+        let className = '';
+        if (item.runIndex === activeIndex) {
+          className += 'bg-blue-600 bg-opacity-30 ';
+        } else if (activeIndex !== null) {
+          className += 'opacity-50 ';
+        }
+        if (item.id === currentlySelectedRun?.id) {
+          className += 'border-2 border-blue-600 ';
+        }
+        return className.trim();
+      }}
       currentPage={currentPage}
       onPageChange={setCurrentPage}
       pageSize={pageSize}

diff --git a/ell-studio/src/components/evaluations/MetricDisplay.js b/ell-studio/src/components/evaluations/MetricDisplay.js
@@ -29,7 +29,7 @@ const MetricDisplay = ({ currentValue, previousValue, label, showTooltip = true
   }, [currentValue]);
 
   const content = (
-    <div className="text-right min-w-[5rem]">
+    <div className="text-right min-w-[3rem]">
       <div className={`font-bold font-mono overflow-hidden`}>
         <span className={`inline-block transition-all duration-100 ease-in-out ${isHighlighted ? `${trendColorClass} transform scale-105` : 'transform scale-100'}`}>
           {currentValue.toFixed(2)}
@@ -50,16 +50,16 @@ const MetricDisplay = ({ currentValue, previousValue, label, showTooltip = true
     <TooltipProvider delayDuration={300}>
       <Tooltip>
         <TooltipTrigger asChild>
-          <div>
+          <div >
             {content}
           </div>
         </TooltipTrigger>
         <TooltipContent sideOffset={5}>
           <div className="text-xs">
             <p className="font-medium">{label}</p>
             <p>Current: {currentValue.toFixed(4)}</p>
-            <p>Previous: {previousValue !== undefined ? previousValue.toFixed(4) : 'N/A'}</p>
-            <p>Change: {previousValue !== undefined ? (currentValue - previousValue).toFixed(4) : 'N/A'}</p>
+            <p>Previous: {previousValue !== undefined && previousValue !== null ? previousValue.toFixed(4) : 'N/A'}</p>
+            <p>Change: {previousValue !== undefined && previousValue !== null ? (currentValue - previousValue).toFixed(4) : 'N/A'}</p>
           </div>
         </TooltipContent>
       </Tooltip>

diff --git a/ell-studio/src/components/evaluations/MetricGraphGrid.js b/ell-studio/src/components/evaluations/MetricGraphGrid.js
@@ -5,27 +5,54 @@ import Graph from '../graphing/Graph';
 import { GraphProvider } from '../graphing/GraphSystem';
 import MetricDisplay from './MetricDisplay';
 
-const MetricGraphGrid = ({ evaluation, groupedRuns }) => {
+const MetricGraphGrid = ({ evaluation, groupedRuns, onActiveIndexChange }) => {
   const [activeIndex, setActiveIndex] = useState(null);
 
   const getHistoricalData = (labeler) => {
-    return Object.values(groupedRuns).flatMap(runs => 
-      runs.map(run => {
+    if (!labeler) return { means: [], stdDevs: [], errors: [], confidenceIntervals: [] };
+    return Object.values(groupedRuns).reduce((acc, runs) => {
+      runs.forEach(run => {
         const summary = run.labeler_summaries.find(s => s.evaluation_labeler_id === labeler.id);
-        return summary ? summary.data.mean : null;
-      }).filter(Boolean)
-    );
+        if (summary) {
+          const { mean, std, min, max } = summary.data;
+          const count = summary.count;
+
+          // Calculate Standard Error of the Mean (SEM)
+          const sem = std / Math.sqrt(count);
+
+          // Z-score for 95% confidence
+          const zScore = 1.96;
+
+          // Margin of Error
+          let marginOfError = zScore * sem;
+
+          // Bounded Confidence Interval
+          let lowerBound = Math.max(mean - marginOfError, min);
+          let upperBound = Math.min(mean + marginOfError, max);
+
+          acc.means.push(mean);
+          acc.stdDevs.push(std);
+          acc.errors.push(marginOfError);
+          acc.confidenceIntervals.push({ low: lowerBound, high: upperBound });
+        }
+      });
+      return acc;
+    }, { means: [], stdDevs: [], errors: [], confidenceIntervals: [] });
   };
 
-  const xData = Array.from({ length: getHistoricalData(evaluation.labelers[0]).length }, (_, i) => `Run ${i + 1}`);
+  const xData = Array.from({ length: getHistoricalData(evaluation.labelers?.[0]).means.length}, (_, i) => `Run ${i + 1}`);
 
   const handleHover = useCallback((index) => {
     setActiveIndex(index);
-  }, []);
+    onActiveIndexChange(index);
+  }, [onActiveIndexChange]);
 
   const handleLeave = useCallback(() => {
     setActiveIndex(null);
-  }, []);
+    onActiveIndexChange(null);
+  }, [onActiveIndexChange]);
+
+  const hasMultipleValues = getHistoricalData(evaluation.labelers[0]).means.length > 1;
 
   return (
     <GraphProvider 
@@ -36,6 +63,12 @@ const MetricGraphGrid = ({ evaluation, groupedRuns }) => {
           plugins: {
             title: { display: false },
             legend: { display: false },
+            tooltip: { 
+              enabled: true, 
+              intersect: false, 
+              position: 'average', 
+              // TODO: Make the label custom so when we click it takes us to that run id.
+            },
           }
         } 
       }} 
@@ -44,39 +77,15 @@ const MetricGraphGrid = ({ evaluation, groupedRuns }) => {
     >
       <div className="grid grid-cols-1 md:grid-cols-2 lg:grid-cols-3 gap-3">
         {evaluation.labelers.map((labeler) => {
-          const historicalData = getHistoricalData(labeler);
+          const { means: historicalData, stdDevs, confidenceIntervals } = getHistoricalData(labeler);
           if (historicalData.length === 0) return null;
 
-          const graphId = `labeler-${labeler.id}`;
-          const trend = historicalData[historicalData.length - 1] - historicalData[0];
-          const trendColor = trend > 0 ? 'rgba(52, 211, 153, 0.8)' : 'rgba(239, 68, 68, 0.8)';
-          const fillColor = trend > 0 ? 'rgba(52, 211, 153, 0.2)' : 'rgba(239, 68, 68, 0.2)';
-
-          const metrics = [
-            {
-              label: labeler.name,
-              yData: historicalData,
-              color: trendColor,
-              config: {
-                backgroundColor: fillColor,
-                borderColor: trendColor,
-                fill: true,
-                tension: 0.4,
-                borderWidth: 1,
-                pointRadius: 3,
-              }
-            }
-          ];
-          const currentValue = activeIndex !== null && activeIndex < historicalData.length
-            ? historicalData[activeIndex]
-            : historicalData[historicalData.length - 1] || null;
-          const previousValue = activeIndex !== null && activeIndex > 0
-            ? historicalData[activeIndex - 1]
-            : currentValue;
+          const currentValue = activeIndex !== null ? historicalData[activeIndex] : historicalData[historicalData.length - 1];
+          const previousValue = activeIndex !== null && activeIndex > 0 ? historicalData[activeIndex - 1] : historicalData[historicalData.length - 2];
 
           return (
             <Card key={labeler.id}>
-              <div className="border-b border-gray-800 flex justify-between items-center p-2">
+              <div className={`flex justify-between items-center p-2 ${hasMultipleValues ? 'border-b border-gray-800' : ''}`}>
                 <LMPCardTitle
                   lmp={labeler.labeling_lmp}
                   nameOverridePrint={labeler.name}
@@ -92,12 +101,29 @@ const MetricGraphGrid = ({ evaluation, groupedRuns }) => {
                   label={labeler.name}
                 />
               </div>
-              <div className="p-4">
-                <Graph
-                  graphId={graphId}
-                  metrics={metrics}
-                />
-              </div>
+              {hasMultipleValues && (
+                <div className="p-4">
+                  <Graph
+                    graphId={`labeler-${labeler.id}`}
+                    metrics={[
+                      {
+                        label: labeler.name,
+                        yData: historicalData,
+                        errorBars: confidenceIntervals,
+                        color: currentValue > historicalData[0] ? 'rgba(52, 211, 153, 0.8)' : 'rgba(239, 68, 68, 0.8)',
+                        config: {
+                          backgroundColor: currentValue > historicalData[0] ? 'rgba(52, 211, 153, 0.2)' : 'rgba(239, 68, 68, 0.2)',
+                          borderColor: currentValue > historicalData[0] ? 'rgba(52, 211, 153, 0.8)' : 'rgba(239, 68, 68, 0.8)',
+                          fill: true,
+                          tension: 0.4,
+                          borderWidth: 1,
+                          pointRadius: 3,
+                        }
+                      }
+                    ]}
+                  />
+                </div>
+              )}
             </Card>
           );
         })}

diff --git a/ell-studio/src/components/evaluations/MetricTable.js b/ell-studio/src/components/evaluations/MetricTable.js
@@ -14,7 +14,7 @@ const MetricTable = ({ summaries, historicalData, isVertical }) => {
 
         return (
           <React.Fragment key={summary.evaluation_labeler_id}>
-            <div className={`flex flex-col space-y-1 text-xs py-1 hover:bg-accent/50 transition-colors duration-200`}>
+            <div className={`flex flex-col space-y-1 text-xs py-1 hover:bg-accent/50 transition-colors duration-200 pr-1`}>
               <div className={`w-full font-medium truncate flex items-center`} title={summary.evaluation_labeler.name}>
                 <FiBarChart2 className="mr-1 h-3 w-3 text-muted-foreground flex-shrink-0" />
                 <code className="metric-label text-xs font-medium truncate max-w-[calc(100%-1.5rem)]">