wip

Arize-ai · Jan 13, 2024 · a578c29 · a578c29
1 parent 7c120d3
commit a578c29
Show file tree

Hide file tree

Showing 2 changed files with 77 additions and 9 deletions.
diff --git a/js/packages/openinference-instrumentation-openai/src/instrumentation.ts b/js/packages/openinference-instrumentation-openai/src/instrumentation.ts
@@ -25,6 +25,7 @@ import {
   ChatCompletion,
   ChatCompletionChunk,
   ChatCompletionCreateParamsBase,
+  ChatCompletionMessage,
   ChatCompletionMessageParam,
 } from "openai/resources/chat/completions";
 import { CompletionCreateParamsBase } from "openai/resources/completions";
@@ -356,7 +357,7 @@ function getChatCompletionInputMessageAttributes(
   switch (role) {
     case "user":
       // Add the content only if it is a string
-      if (typeof message.content == "string")
+      if (typeof message.content === "string")
         attributes[SemanticConventions.MESSAGE_CONTENT] = message.content;
       break;
     case "assistant":
@@ -375,16 +376,16 @@ function getChatCompletionInputMessageAttributes(
       break;
     case "function":
       // Add the content only if it is a string
-      if (typeof message.content == "string")
+      if (typeof message.content === "string")
         attributes[SemanticConventions.MESSAGE_CONTENT] = message.content;
       attributes[SemanticConventions.MESSAGE_FUNCTION_CALL_NAME] = message.name;
       break;
     case "tool":
-      if (typeof message.content == "string")
+      if (typeof message.content === "string")
         attributes[SemanticConventions.MESSAGE_CONTENT] = message.content;
       break;
     case "system":
-      if (typeof message.content == "string")
+      if (typeof message.content === "string")
         attributes[SemanticConventions.MESSAGE_CONTENT] = message.content;
       break;
     default:
@@ -451,14 +452,21 @@ function getChatCompletionLLMOutputMessagesAttributes(
   }
   return [choice.message].reduce((acc, message, index) => {
     const indexPrefix = `${SemanticConventions.LLM_OUTPUT_MESSAGES}.${index}`;
-    acc[`${indexPrefix}.${SemanticConventions.MESSAGE_CONTENT}`] = String(
-      message.content,
-    );
-    acc[`${indexPrefix}.${SemanticConventions.MESSAGE_ROLE}`] = message.role;
+    const messageAttributes = getChatCompletionOutputMessageAttributes(message);
+    // Flatten the attributes on the index prefix
+    for (const [key, value] of Object.entries(messageAttributes)) {
+      acc[`${indexPrefix}.${key}`] = value;
+    }
     return acc;
   }, {} as Attributes);
 }
 
+function getChatCompletionOutputMessageAttributes(
+  _message: ChatCompletionMessage,
+): Attributes {
+  return {};
+}
+
 /**
  * Converts the completion result to output attributes
  */

diff --git a/spec/llm_spans.md b/spec/llm_spans.md
@@ -4,7 +4,7 @@ LLM spans capture the API parameters sent to a LLM provider such as OpenAI or Co
 
 ## Examples
 
-A span that includes tool calls and messages
+A span for a tool call with OpenAI
 
 ```json
 {
@@ -49,3 +49,63 @@ A span that includes tool calls and messages
     "events": []
 }
 ```
+
+A synthesis call using a function call output
+
+```json
+{
+    "name": "llm",
+    "context": {
+        "trace_id": "409df945-e058-4829-b240-cfbdd2ff4488",
+        "span_id": "f26d1f26-9671-435d-9716-14a87a3f228b"
+    },
+    "span_kind": "LLM",
+    "parent_id": "2fe8a793-2cf1-42d7-a1df-bd7d46e017ef",
+    "start_time": "2024-01-11T16:45:18.519427-07:00",
+    "end_time": "2024-01-11T16:45:19.159145-07:00",
+    "status_code": "OK",
+    "status_message": "",
+    "attributes": {
+        "llm.input_messages": [
+            {
+                "message.role": "system",
+                "message.content": "You are a Shakespearean writing assistant who speaks in a Shakespearean style. You help people come up with creative ideas and content like stories, poems, and songs that use Shakespearean style of writing style, including words like \"thou\" and \"hath\u201d.\nHere are some example of Shakespeare's style:\n - Romeo, Romeo! Wherefore art thou Romeo?\n - Love looks not with the eyes, but with the mind; and therefore is winged Cupid painted blind.\n - Shall I compare thee to a summer's day? Thou art more lovely and more temperate.\n"
+            },
+            {
+                "message.role": "user",
+                "message.content": "what is 23 times 87"
+            },
+            {
+                "message.role": "assistant",
+                "message.content": null,
+                "message.tool_calls": [
+                    {
+                        "tool_call.function.name": "multiply",
+                        "tool_call.function.arguments": "{\n  \"a\": 23,\n  \"b\": 87\n}"
+                    }
+                ]
+            },
+            {
+                "message.role": "tool",
+                "message.content": "2001",
+                "message.name": "multiply"
+            }
+        ],
+        "llm.model_name": "gpt-3.5-turbo-0613",
+        "llm.invocation_parameters": "{\"model\": \"gpt-3.5-turbo-0613\", \"temperature\": 0.1, \"max_tokens\": null}",
+        "output.value": "The product of 23 times 87 is 2001.",
+        "output.mime_type": "text/plain",
+        "llm.output_messages": [
+            {
+                "message.role": "assistant",
+                "message.content": "The product of 23 times 87 is 2001."
+            }
+        ],
+        "llm.token_count.prompt": 259,
+        "llm.token_count.completion": 14,
+        "llm.token_count.total": 273
+    },
+    "events": [],
+    "conversation": null
+}
+```