Merge branch 'main' into main

.github/workflows/fe-build-check.yml

@@ -0,0 +1,44 @@
+name: Check Frontend Build
+
+on:
+  pull_request:
+    types:
+      - synchronize
+      - opened
+      - reopened
+    paths:
+      - 'frontend/**' # Trigger only if files in the `/frontend` directory change
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+      # Step 1: Checkout the code
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      # Step 2: Install pnpm
+      - name: Install pnpm
+        uses: pnpm/action-setup@v4
+        with:
+          version: 9
+
+      # Step 3: Set up Node.js
+      - name: Set up Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: 'lts/*' # Use the latest LTS version of Node.js
+          cache: 'pnpm'
+          # Specify the subdirectory containing the lockfile
+          cache-dependency-path: ./frontend/pnpm-lock.yaml
+
+      # Step 4: Install dependencies
+      - name: Install dependencies
+        working-directory: ./frontend
+        run: pnpm install
+
+      # Step 5: Run the build
+      - name: Build frontend
+        working-directory: ./frontend
+        run: pnpm build

app-server/src/main.rs

@@ -1,7 +1,7 @@
 use actix_service::Service;
 use actix_web::{
     middleware::{Logger, NormalizePath},
-    web::{self, PayloadConfig},
+    web::{self, JsonConfig, PayloadConfig},
     App, HttpMessage, HttpServer,
 };
 use actix_web_httpauth::middleware::HttpAuthentication;
@@ -74,6 +74,8 @@ mod storage;
 mod traces;
 
 const DEFAULT_CACHE_SIZE: u64 = 100; // entries
+const HTTP_PAYLOAD_LIMIT: usize = 100 * 1024 * 1024; // 100MB
+const GRPC_PAYLOAD_DECODING_LIMIT: usize = 100 * 1024 * 1024; // 100MB
 
 fn tonic_error_to_io_error(err: tonic::transport::Error) -> io::Error {
     io::Error::new(io::ErrorKind::Other, err)
@@ -397,14 +399,15 @@ fn main() -> anyhow::Result<()> {
                         .service(
                             web::scope("/v1")
                                 .wrap(project_auth.clone())
+                                .app_data(PayloadConfig::new(HTTP_PAYLOAD_LIMIT))
+                                .app_data(JsonConfig::default().limit(HTTP_PAYLOAD_LIMIT))
                                 .service(api::v1::pipelines::run_pipeline_graph)
                                 .service(api::v1::pipelines::ping_healthcheck)
                                 .service(api::v1::traces::process_traces)
                                 .service(api::v1::datasets::get_datapoints)
                                 .service(api::v1::evaluations::create_evaluation)
                                 .service(api::v1::metrics::process_metrics)
-                                .service(api::v1::semantic_search::semantic_search)
-                                .app_data(PayloadConfig::new(10 * 1024 * 1024)),
+                                .service(api::v1::semantic_search::semantic_search),
                         )
                         // Scopes with generic auth
                         .service(
@@ -537,7 +540,10 @@ fn main() -> anyhow::Result<()> {
                 );
 
                 Server::builder()
-                    .add_service(TraceServiceServer::new(process_traces_service))
+                    .add_service(
+                        TraceServiceServer::new(process_traces_service)
+                            .max_decoding_message_size(GRPC_PAYLOAD_DECODING_LIMIT),
+                    )
                     .serve_with_shutdown(grpc_address, async {
                         wait_stop_signal("gRPC service").await;
                     })

app-server/src/traces/span_attributes.rs

@@ -25,7 +25,7 @@ pub const GEN_AI_INPUT_COST: &str = "gen_ai.usage.input_cost";
 pub const GEN_AI_OUTPUT_COST: &str = "gen_ai.usage.output_cost";
 
 // Custom lmnr attributes
-pub const ASSOCIATION_PROPERTIES_PREFIX: &str = "lmnr.association.properties.";
+pub const ASSOCIATION_PROPERTIES_PREFIX: &str = "lmnr.association.properties";
 pub const SPAN_TYPE: &str = "lmnr.span.type";
 pub const SPAN_PATH: &str = "lmnr.span.path";
 pub const LLM_NODE_RENDERED_PROMPT: &str = "lmnr.span.prompt";

app-server/src/traces/spans.rs

@@ -59,7 +59,7 @@ impl SpanAttributes {
     pub fn session_id(&self) -> Option<String> {
         match self
             .attributes
-            .get(format!("{ASSOCIATION_PROPERTIES_PREFIX}session_id").as_str())
+            .get(format!("{ASSOCIATION_PROPERTIES_PREFIX}.session_id").as_str())
         {
             Some(Value::String(s)) => Some(s.clone()),
             _ => None,
@@ -69,7 +69,7 @@ impl SpanAttributes {
     pub fn user_id(&self) -> Option<String> {
         match self
             .attributes
-            .get(format!("{ASSOCIATION_PROPERTIES_PREFIX}user_id").as_str())
+            .get(format!("{ASSOCIATION_PROPERTIES_PREFIX}.user_id").as_str())
         {
             Some(Value::String(s)) => Some(s.clone()),
             _ => None,
@@ -78,7 +78,7 @@ impl SpanAttributes {
 
     pub fn trace_type(&self) -> Option<TraceType> {
         self.attributes
-            .get(format!("{ASSOCIATION_PROPERTIES_PREFIX}trace_type").as_str())
+            .get(format!("{ASSOCIATION_PROPERTIES_PREFIX}.trace_type").as_str())
             .and_then(|s| serde_json::from_value(s.clone()).ok())
     }
 
@@ -131,7 +131,7 @@ impl SpanAttributes {
             if provider == "Langchain" {
                 let ls_provider = self
                     .attributes
-                    .get(format!("{ASSOCIATION_PROPERTIES_PREFIX}ls_provider").as_str())
+                    .get(format!("{ASSOCIATION_PROPERTIES_PREFIX}.ls_provider").as_str())
                     .and_then(|s| serde_json::from_value(s.clone()).ok());
                 if let Some(ls_provider) = ls_provider {
                     ls_provider
@@ -222,21 +222,32 @@ impl SpanAttributes {
     }
 
     pub fn metadata(&self) -> Option<HashMap<String, String>> {
-        let res = self.get_flattened_association_properties("metadata");
-        if res.is_empty() {
+        let mut metadata = self.get_flattened_association_properties("metadata");
+        let ai_sdk_metadata = self.get_flattened_properties("ai", "telemetry.metadata");
+        metadata.extend(ai_sdk_metadata);
+        if metadata.is_empty() {
             None
         } else {
             Some(
-                res.into_iter()
+                metadata
+                    .into_iter()
                     .map(|(k, v)| (k, json_value_to_string(v)))
                     .collect(),
             )
         }
     }
 
-    fn get_flattened_association_properties(&self, prefix: &str) -> HashMap<String, Value> {
+    fn get_flattened_association_properties(&self, entity: &str) -> HashMap<String, Value> {
+        self.get_flattened_properties(ASSOCIATION_PROPERTIES_PREFIX, entity)
+    }
+
+    fn get_flattened_properties(
+        &self,
+        attribute_prefix: &str,
+        entity: &str,
+    ) -> HashMap<String, Value> {
         let mut res = HashMap::new();
-        let prefix = format!("{ASSOCIATION_PROPERTIES_PREFIX}{prefix}.");
+        let prefix = format!("{attribute_prefix}.{entity}.");
         for (key, value) in self.attributes.iter() {
             if key.starts_with(&prefix) {
                 res.insert(
@@ -364,6 +375,16 @@ impl Span {
                 );
             }
         }
+
+        // Vercel AI SDK wraps "raw" LLM spans in an additional `ai.generateText` span.
+        // Which is not really an LLM span, but it has the prompt in its attributes.
+        // Set the input to the prompt.
+        if let Some(serde_json::Value::String(s)) = attributes.get("ai.prompt") {
+            span.input = Some(
+                serde_json::from_str::<Value>(s).unwrap_or(serde_json::Value::String(s.clone())),
+            );
+        }
+
         // If an LLM span is sent manually, we prefer `lmnr.span.input` and `lmnr.span.output`
         // attributes over gen_ai/vercel/LiteLLM attributes.
         // Therefore this block is outside and after the LLM span type check.
@@ -442,7 +463,7 @@ impl Span {
         };
         let mut attributes = HashMap::new();
         attributes.insert(
-            format!("{ASSOCIATION_PROPERTIES_PREFIX}trace_type",),
+            format!("{ASSOCIATION_PROPERTIES_PREFIX}.trace_type",),
             json!(trace_type),
         );
         attributes.insert(SPAN_PATH.to_string(), json!(path));
@@ -614,6 +635,12 @@ fn should_keep_attribute(attribute: &str) -> bool {
         return false;
     }
 
+    // AI SDK
+    // remove ai.prompt.messages as it is stored in LLM span's input
+    if attribute == "ai.prompt.messages" {
+        return false;
+    }
+
     true
 }
 

frontend/app/api/projects/[projectId]/datasets/[datasetId]/download/route.ts → frontend/app/api/projects/[projectId]/datasets/[datasetId]/download/[format]/route.ts

@@ -1,20 +1,30 @@
 import { and, asc, eq } from 'drizzle-orm';
+import { json2csv } from 'json-2-csv';
 
 import { db } from '@/lib/db/drizzle';
 import { datasetDatapoints, datasets } from '@/lib/db/migrations/schema';
+import { DownloadFormat } from '@/lib/types';
 
 export async function GET(
   req: Request,
   {
     params
   }: {
-    params: { projectId: string; datasetId: string; };
+    params: { projectId: string; datasetId: string; format: DownloadFormat };
   }
 ): Promise<Response> {
 
 
   const projectId = params.projectId;
   const datasetId = params.datasetId;
+  const format = params.format;
+
+  if (!Object.values(DownloadFormat).includes(format)) {
+    return Response.json(
+      { error: 'Invalid format. Supported formats are: csv, json' },
+      { status: 400 }
+    );
+  }
 
   const dataset = await db.query.datasets.findFirst({
     where: and(
@@ -37,12 +47,28 @@ export async function GET(
     }
   });
 
+  // if the format is csv, convert the datapoints to csv
+  if (format === 'csv') {
+    const csv = await json2csv(datapoints, {
+      emptyFieldValue: '',
+      expandNestedObjects: false
+    });
+    const contentType = 'text/csv';
+    const filename = `${dataset.name.replace(/[^a-zA-Z0-9-_\.]/g, '_')}-${datasetId}.csv`;
+    const headers = new Headers();
+    headers.set('Content-Type', contentType);
+    headers.set('Content-Disposition', `attachment; filename="${filename}"`);
+
+    return new Response(csv, {
+      headers
+    });
+  }
+  // if the format is json, return the datapoints as json
   const contentType = 'application/json';
   const filename = `${dataset.name.replace(/[^a-zA-Z0-9-_\.]/g, '_')}-${datasetId}.json`;
   const headers = new Headers();
   headers.set('Content-Type', contentType);
   headers.set('Content-Disposition', `attachment; filename="${filename}"`);
-
   return new Response(JSON.stringify(datapoints, null, 2), {
     headers
   });

frontend/app/api/projects/[projectId]/evaluations/[evaluationId]/download/route.ts → frontend/app/api/projects/[projectId]/evaluations/[evaluationId]/download/[format]/route.ts

@@ -3,19 +3,26 @@ import { json2csv } from 'json-2-csv';
 
 import { db } from '@/lib/db/drizzle';
 import { evaluationResults, evaluations, evaluationScores } from '@/lib/db/migrations/schema';
+import { DownloadFormat } from '@/lib/types';
 
 export async function GET(
   req: Request,
   {
     params
   }: {
-    params: { projectId: string; evaluationId: string; };
+    params: { projectId: string; evaluationId: string; format: DownloadFormat };
   }
 ): Promise<Response> {
-
-
   const projectId = params.projectId;
   const evaluationId = params.evaluationId;
+  const format = params.format as DownloadFormat;
+
+  if (!Object.values(DownloadFormat).includes(format)) {
+    return Response.json(
+      { error: 'Invalid format. Supported formats are: csv, json' },
+      { status: 400 }
+    );
+  }
 
   const evaluation = await db.query.evaluations.findFirst({
     where: and(
@@ -66,9 +73,21 @@ export async function GET(
       ...scores
     };
   });
+
+  // else the format is json, return the results as json
+  if (format === DownloadFormat.JSON) {
+    const json = JSON.stringify(flattenedResults);
+    const contentType = 'application/json';
+    const filename = `${evaluation.name.replace(/[^a-zA-Z0-9-_\.]/g, '_')}-${evaluationId}.json`;
+    return new Response(json, {
+      headers: { 'Content-Type': contentType, 'Content-Disposition': `attachment; filename="${filename}"` }
+    });
+  }
+
+  // if the format is csv, convert the results to csv
   const csv = await json2csv(flattenedResults, {
     emptyFieldValue: '',
-    expandNestedObjects: false // we only expand the scores object manually
+    expandNestedObjects: false
   });
   const contentType = 'text/csv';
   const filename = `${evaluation.name.replace(/[^a-zA-Z0-9-_\.]/g, '_')}-${evaluationId}.csv`;

frontend/app/blog/[slug]/page.tsx

@@ -40,14 +40,9 @@ export default async function BlogPostPage({ params }: { params: { slug: string
   return (
     <>
       <LandingHeader hasSession={session !== null && session !== undefined} />
-      <div className="mt-32 h-full flex justify-center">
-        {/* <div className="w-1/4 flex justify-end">
-          <Link href="/blog" className="text-secondary-foreground hover:text-primary h-0">Back to all posts</Link>
-        </div> */}
-        <article className="flex flex-col z-30 py-16 md:w-[1000px] w-full px-8 md:px-0">
-          {/* <ScrollArea className="h-full flex-grow w-full mx-auto bg-background px-16">
-            <div className="h-0"> */}
-          <BlogMeta data={data} />
+      <div className="mt-48 h-full flex justify-center flex-col items-center">
+        <BlogMeta data={data} />
+        <article className="flex flex-col z-30 md:w-[700px] w-full px-8 md:px-0">
           <div className="pt-4 pb-48">
             <MDXRemote
               source={content}
@@ -56,26 +51,21 @@ export default async function BlogPostPage({ params }: { params: { slug: string
                 h2: (props) => <MDHeading props={props} level={1} />,
                 h3: (props) => <MDHeading props={props} level={2} />,
                 h4: (props) => <MDHeading props={props} level={3} />,
-                p: (props) => <p className="py-2 text-secondary-foreground" {...props} />,
-                a: (props) => <a className="text-primary underline" target="_blank" rel="noopener noreferrer" {...props} />,
+                p: (props) => <p className="py-2 text-white/85" {...props} />,
+                a: (props) => <a className="text-white underline" target="_blank" rel="noopener noreferrer" {...props} />,
                 blockquote: (props) => <blockquote className="border-l-2 border-primary pl-4 py-2" {...props} />,
                 // codeblock
                 pre: (props) => <PreHighlighter className="pl-4 py-4" {...props} />,
                 // inline code
-                code: (props) => <span className="text-sm bg-secondary text-primary font-mono px-0.5" {...props} />,
-                ul: (props) => <ul className="list-disc pl-4 text-secondary-foreground" {...props} />,
-                ol: (props) => <ol className="list-decimal pl-4 text-secondary-foreground" {...props} />,
-                img: (props) => <img className="w-full border rounded-lg" {...props} />,
+                code: (props) => <span className="text-sm bg-secondary rounded text-white font-mono px-1.5 py-0.5" {...props} />,
+                ul: (props) => <ul className="list-disc pl-4 text-white/85" {...props} />,
+                ol: (props) => <ol className="list-decimal pl-4 text-white/85" {...props} />,
+                img: (props) => <img className="md:w-[1000px] relative w-full border rounded-lg" {...props} />,
               }}
             />
           </div>
           <Footer />
-          {/* </div>
-          </ScrollArea> */}
         </article>
-        {/* <div className="w-1/5 right-0 top-120 hidden 2xl:block fixed">
-          <TableOfContents headings={parseHeadings(content)} />
-        </div> */}
       </div>
     </>
   );

frontend/app/blog/page.tsx

@@ -30,11 +30,11 @@ export default async function BlogsPage() {
   return <>
     <div className="h-full">
       <LandingHeader hasSession={session !== null && session !== undefined} />
-      <div className="mt-32 pb-48 grid grid-cols-1 gap-4 md:w-[1000px] w-full md:grid-cols-3 mx-auto">
+      <div className="px-4 md:px-0 mt-32 pb-48 grid grid-cols-1 gap-4 md:w-[1200px] w-full md:grid-cols-3 mx-auto">
         {posts.map((post, index) => (
           <Link href={`/blog/${post.slug}`} key={index}>
             <Card key={index} className="overflow-hidden h-[350px]">
-              {post.data.image && <Image src={post.data.image} alt={post.data.title} width={400} height={200} className="object-cover mx-auto"/>}
+              {post.data.image && <Image src={post.data.image} alt={post.data.title} width={400} height={200} className="object-cover mx-auto" />}
               <CardHeader>
                 <CardTitle>
                   {post.data.title}