truncate data in datapoints and eval-datapoints

app-server/src/db/datapoints.rs

@@ -13,8 +13,8 @@ pub struct DatapointView {
     id: Uuid,
     created_at: DateTime<Utc>,
     dataset_id: Uuid,
-    data: Value,
-    target: Option<Value>,
+    data: String,
+    target: Option<String>,
     metadata: Option<Value>,
 }
 
@@ -95,7 +95,13 @@ pub async fn get_datapoints(
     offset: i64,
 ) -> Result<Vec<DatapointView>> {
     let datapoints = sqlx::query_as::<_, DatapointView>(
-        "SELECT id, dataset_id, data, target, metadata, created_at
+        "SELECT
+            id,
+            dataset_id,
+            SUBSTRING(data::text, 0, 100) as data,
+            SUBSTRING(target::text, 0, 100) as target,
+            metadata,
+            created_at
         FROM dataset_datapoints
         WHERE dataset_id = $1
         ORDER BY

frontend/app/api/projects/[projectId]/datasets/[datasetId]/datapoints/[datapointId]/route.ts

@@ -7,6 +7,26 @@ import { db } from '@/lib/db/drizzle';
 import { datasetDatapoints } from '@/lib/db/migrations/schema';
 import { fetcher } from '@/lib/utils';
 
+export async function GET(
+  req: Request,
+  {
+    params
+  }: { params: { projectId: string; datasetId: string; datapointId: string } }
+) {
+  const datapoint = await db.query.datasetDatapoints.findFirst({
+    where: and(
+      eq(datasetDatapoints.id, params.datapointId),
+      eq(datasetDatapoints.datasetId, params.datasetId)
+    )
+  });
+
+  if (!datapoint) {
+    return new Response('Datapoint not found', { status: 404 });
+  }
+
+  return new Response(JSON.stringify(datapoint), { status: 200 });
+}
+
 export async function POST(
   req: Request,
   {

frontend/app/api/projects/[projectId]/evaluations/[evaluationId]/route.ts

@@ -3,7 +3,7 @@ import { and, asc, eq, sql } from 'drizzle-orm';
 import { db } from '@/lib/db/drizzle';
 import {
   evaluationResults,
-  evaluations,evaluationScores
+  evaluations, evaluationScores
 } from '@/lib/db/migrations/schema';
 
 export async function GET(
@@ -39,11 +39,10 @@ export async function GET(
       id: evaluationResults.id,
       createdAt: evaluationResults.createdAt,
       evaluationId: evaluationResults.evaluationId,
-      data: evaluationResults.data,
-      target: evaluationResults.target,
+      data: sql<string>`SUBSTRING(${evaluationResults.data}::text, 0, 100)`.as('data'),
+      target: sql<string>`SUBSTRING(${evaluationResults.target}::text, 0, 100)`.as('target'),
       executorOutput: evaluationResults.executorOutput,
       scores: subQueryScoreCte.cteScores,
-      traceId: evaluationResults.traceId
     })
     .from(evaluationResults)
     .leftJoin(

frontend/app/project/[projectId]/evaluations/[evaluationId]/page.tsx

@@ -6,7 +6,7 @@ import Evaluation from '@/components/evaluation/evaluation';
 import { db } from '@/lib/db/drizzle';
 import {
   evaluationResults,
-  evaluations,evaluationScores
+  evaluations, evaluationScores
 } from '@/lib/db/migrations/schema';
 import { EvaluationResultsInfo } from '@/lib/evaluation/types';
 
@@ -70,8 +70,8 @@ async function getEvaluationInfo(
       id: evaluationResults.id,
       createdAt: evaluationResults.createdAt,
       evaluationId: evaluationResults.evaluationId,
-      data: evaluationResults.data,
-      target: evaluationResults.target,
+      data: sql<string>`SUBSTRING(${evaluationResults.data}::text, 0, 100)`.as('data'),
+      target: sql<string>`SUBSTRING(${evaluationResults.target}::text, 0, 100)`.as('target'),
       executorOutput: evaluationResults.executorOutput,
       scores: subQueryScoreCte.cteScores,
       traceId: evaluationResults.traceId

frontend/components/dataset/dataset-panel.tsx

@@ -1,9 +1,10 @@
 import { ChevronsRight, Loader2 } from 'lucide-react';
 import { useEffect, useRef, useState } from 'react';
+import useSWR from 'swr';
 
 import { useProjectContext } from '@/contexts/project-context';
-import { Datapoint } from '@/lib/dataset/types';
 import { useToast } from '@/lib/hooks/use-toast';
+import { swrFetcher } from '@/lib/utils';
 
 import { Button } from '../ui/button';
 import Formatter from '../ui/formatter';
@@ -15,7 +16,7 @@ import { Skeleton } from '../ui/skeleton';
 interface DatasetPanelProps {
   datasetId: string;
   indexedOn: string | null;
-  datapoint: Datapoint;
+  datapointId: string;
   onClose: () => void;
 }
 
@@ -24,17 +25,21 @@ const AUTO_SAVE_TIMEOUT_MS = 750;
 export default function DatasetPanel({
   datasetId,
   indexedOn,
-  datapoint,
+  datapointId,
   onClose,
 }: DatasetPanelProps) {
   const { projectId } = useProjectContext();
+  const { data: datapoint, isLoading } = useSWR(
+    `/api/projects/${projectId}/datasets/${datasetId}/datapoints/${datapointId}`,
+    swrFetcher
+  );
   // datapoint is DatasetDatapoint, i.e. result of one execution on a data point
-  const [newData, setNewData] = useState<Record<string, any> | null>(datapoint.data);
+  const [newData, setNewData] = useState<Record<string, any> | null>(datapoint?.data);
   const [newTarget, setNewTarget] = useState<Record<string, any> | null>(
-    datapoint.target
+    datapoint?.target
   );
   const [newMetadata, setNewMetadata] = useState<Record<string, any> | null>(
-    datapoint.metadata
+    datapoint?.metadata
   );
   const [isValidJsonData, setIsValidJsonData] = useState(true);
   const [isValidJsonTarget, setIsValidJsonTarget] = useState(true);
@@ -52,7 +57,7 @@ export default function DatasetPanel({
     }
     setSaving(true);
     const res = await fetch(
-      `/api/projects/${projectId}/datasets/${datasetId}/datapoints/${datapoint.id}`,
+      `/api/projects/${projectId}/datasets/${datasetId}/datapoints/${datapointId}`,
       {
         method: 'POST',
         headers: {
@@ -92,12 +97,13 @@ export default function DatasetPanel({
   }, [newData, newTarget, newMetadata]);
 
   useEffect(() => {
+    if (!datapoint) return;
     setNewData(datapoint.data);
     setNewTarget(datapoint.target);
     setNewMetadata(datapoint.metadata);
   }, [datapoint]);
 
-  return (
+  return isLoading ? (<div>Loading...</div>) : (
     <div className="flex flex-col h-full w-full">
       <div className="h-12 flex flex-none space-x-2 px-3 items-center border-b">
         <Button

frontend/components/dataset/dataset.tsx

@@ -267,7 +267,7 @@ export default function Dataset({ dataset }: DatasetProps) {
             <div className="w-full h-full flex">
               <DatasetPanel
                 datasetId={dataset.id}
-                datapoint={selectedDatapoint}
+                datapointId={selectedDatapoint.id}
                 onClose={() => {
                   handleDatapointSelect(null);
                   mutate();

frontend/lib/evaluation/types.ts

@@ -24,9 +24,9 @@ export type EvaluationDatapointPreview = {
   evaluationId: string;
   createdAt: string;
   scores?: Record<string, any>;
-  data: Record<string, any>;
-  target: Record<string, any>;
-  executorOutput: Record<string, any>;
+  data: any;
+  target: any;
+  executorOutput: any;
   traceId: string;
 };