add observation

LLMAgentBuilder · Oct 5, 2024 · 7f238ce · 7f238ce
1 parent 70ff89b
commit 7f238ce
Show file tree

Hide file tree

Showing 12 changed files with 218 additions and 16 deletions.
diff --git a/bootstrap/pom.xml b/bootstrap/pom.xml
@@ -43,6 +43,21 @@
       <artifactId>launcher-ktor</artifactId>
       <version>${project.version}</version>
     </dependency>
+    <dependency>
+      <groupId>io.github.llmagentbuilder</groupId>
+      <artifactId>open-telemetry</artifactId>
+      <version>${project.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>io.github.llmagentbuilder</groupId>
+      <artifactId>llm-openai</artifactId>
+      <version>${project.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>io.github.llmagentbuilder</groupId>
+      <artifactId>agent-planner-react-json</artifactId>
+      <version>${project.version}</version>
+    </dependency>
     <dependency>
       <groupId>org.slf4j</groupId>
       <artifactId>slf4j-api</artifactId>

diff --git a/bootstrap/src/main/kotlin/io/github/llmagentbuilder/bootstrap/AgentBootstrap.kt b/bootstrap/src/main/kotlin/io/github/llmagentbuilder/bootstrap/AgentBootstrap.kt
@@ -6,6 +6,8 @@ import io.github.llmagentbuilder.core.*
 import io.github.llmagentbuilder.core.tool.AgentToolFunctionCallbackContext
 import io.github.llmagentbuilder.core.tool.AgentToolsProviderFactory
 import io.github.llmagentbuilder.launcher.ktor.server.KtorLauncher
+import io.github.llmagentbuilder.plugin.observation.opentelemetry.OpenTelemetryPlugin
+import io.micrometer.observation.ObservationRegistry
 import org.slf4j.LoggerFactory
 import org.springframework.ai.chat.client.ChatClient
 import org.springframework.ai.chat.client.advisor.MessageChatMemoryAdvisor
@@ -47,6 +49,8 @@ object AgentBootstrap {
             AgentToolFunctionCallbackContext(
                 agentToolsProvider,
             )
+        val observationRegistry = ObservationRegistry.create()
+        OpenTelemetryPlugin().install(agentConfig, observationRegistry)
         val llmConfigs = agentConfig.llm
         val chatModel = ServiceLoader.load(ChatModelProvider::class.java)
             .stream()
@@ -60,8 +64,9 @@ object AgentBootstrap {
             .firstOrNull()?.also {
                 logger.info("Loaded ChatModel $it")
             } ?: throw RuntimeException("No ChatModel found")
-        val chatClientBuilder = ChatClient.builder(chatModel)
-            .defaultAdvisors(advisors)
+        val chatClientBuilder =
+            ChatClient.builder(chatModel, observationRegistry, null)
+                .defaultAdvisors(advisors)
         val plannerConfigs = agentConfig.planner
         val planner = ServiceLoader.load(PlannerProvider::class.java)
             .stream()
@@ -82,9 +87,10 @@ object AgentBootstrap {
             metadata?.description,
             metadata?.usageInstruction,
             agentToolsProvider,
+            UUID.randomUUID().toString(),
+            observationRegistry,
         )
         KtorLauncher.launch(chatAgent)
-//        JdkHttpSyncLauncher().launch(chatAgent, agentToolsProvider)
     }
 
     private fun profileAdvisor(agentConfig: AgentConfig): Advisor? {

diff --git a/bootstrap/src/main/resources/agent.yaml b/bootstrap/src/main/resources/agent.yaml
@@ -1,5 +1,5 @@
 metadata:
-  name: Test agent
+  name: TestAgent
 llm:
   openai:
     enabled: true
@@ -11,6 +11,12 @@ memory:
 planner:
   reActJson:
     enabled: true
+observation:
+  tracing:
+    exporter:
+      endpoint: "https://api.honeycomb.io/v1/traces"
+      headers:
+        "x-honeycomb-team": "{{env.HONEYCOMB_API_KEY}}"
 tools:
   - id: writeLocalFile
     config:

diff --git a/cli/pom.xml b/cli/pom.xml
@@ -28,7 +28,6 @@
     <dependency>
       <groupId>com.github.jknack</groupId>
       <artifactId>handlebars</artifactId>
-      <version>4.4.0</version>
     </dependency>
     <dependency>
       <groupId>org.yaml</groupId>

diff --git a/core/pom.xml b/core/pom.xml
@@ -64,6 +64,10 @@
       <groupId>org.yaml</groupId>
       <artifactId>snakeyaml</artifactId>
     </dependency>
+    <dependency>
+      <groupId>com.github.jknack</groupId>
+      <artifactId>handlebars</artifactId>
+    </dependency>
 
     <dependency>
       <groupId>uk.org.webcompere</groupId>

diff --git a/core/src/main/kotlin/io/github/llmagentbuilder/core/AgentConfig.kt b/core/src/main/kotlin/io/github/llmagentbuilder/core/AgentConfig.kt
@@ -1,5 +1,6 @@
 package io.github.llmagentbuilder.core
 
+import com.github.jknack.handlebars.Handlebars
 import org.yaml.snakeyaml.LoaderOptions
 import org.yaml.snakeyaml.Yaml
 import org.yaml.snakeyaml.constructor.Constructor
@@ -36,13 +37,29 @@ class AgentMetadata {
     var usageInstruction: String? = null
 }
 
+class TracingExporterConfig {
+    lateinit var endpoint: String
+    var headers: Map<String, String>? = null
+}
+
+class TracingConfig {
+    var enabled: Boolean? = false
+    var exporter: TracingExporterConfig? = null
+}
+
+class ObservationConfig {
+    var enabled: Boolean? = false
+    var tracing: TracingConfig? = null
+}
+
 class AgentConfig {
     var metadata: AgentMetadata? = null
     var profile: ProfileConfig? = null
     var memory: MemoryConfig? = null
     var llm: Map<String, Any?>? = null
     var planner: Map<String, Any?>? = null
     var tools: List<ToolConfig>? = null
+    var observation: ObservationConfig? = null
 }
 
 object AgentConfigLoader {
@@ -62,4 +79,16 @@ object AgentConfigLoader {
             )
         ).load(reader)
     }
+}
+
+object EvaluationHelper {
+    private val handlebars = Handlebars()
+
+    fun evaluate(input: String): String {
+        return handlebars.compileInline(input).apply(
+            mapOf(
+                "env" to System.getenv(),
+            )
+        )
+    }
 }
diff --git a/core/src/main/kotlin/io/github/llmagentbuilder/core/ObservationPlugin.kt b/core/src/main/kotlin/io/github/llmagentbuilder/core/ObservationPlugin.kt
@@ -0,0 +1,10 @@
+package io.github.llmagentbuilder.core
+
+import io.micrometer.observation.ObservationRegistry
+
+interface ObservationPlugin {
+    fun install(
+        agentConfig: AgentConfig,
+        observationRegistry: ObservationRegistry
+    )
+}
diff --git a/...hers/ktor/src/main/kotlin/io/github/llmagentbuilder/launcher/ktor/server/apis/AgentApi.kt b/...hers/ktor/src/main/kotlin/io/github/llmagentbuilder/launcher/ktor/server/apis/AgentApi.kt
@@ -1,14 +1,3 @@
-/**
- * Agent Protocol
- * Specification of the API protocol for communication with an agent.
- *
- * The version of the OpenAPI document: v1
- *
- *
- * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
- * https://openapi-generator.tech
- * Do not edit the class manually.
- */
 package io.github.llmagentbuilder.launcher.ktor.server.apis
 
 import io.github.llmagentbuilder.core.ChatAgent
@@ -28,6 +17,11 @@ import kotlin.collections.set
 fun Route.AgentApi(chatAgent: ChatAgent) {
     val tasks = ConcurrentHashMap<String, String>()
 
+    post("/chat") {
+        val request = call.receive(ChatAgentRequest::class)
+        call.respond(chatAgent.call(request))
+    }
+
     post("/ap/v1/agent/tasks") {
         val taskId = UUID.randomUUID().toString()
         val request = call.receive(TaskRequestBody::class)

diff --git a/observation/open-telemetry/pom.xml b/observation/open-telemetry/pom.xml
@@ -0,0 +1,47 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <parent>
+    <groupId>io.github.llmagentbuilder</groupId>
+    <artifactId>observation</artifactId>
+    <version>0.2.1</version>
+  </parent>
+
+  <artifactId>open-telemetry</artifactId>
+
+  <properties>
+    <maven.compiler.source>21</maven.compiler.source>
+    <maven.compiler.target>21</maven.compiler.target>
+    <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+    <opentelemetry.version>1.42.1</opentelemetry.version>
+  </properties>
+
+  <dependencies>
+    <dependency>
+      <groupId>io.github.llmagentbuilder</groupId>
+      <artifactId>core</artifactId>
+      <version>${project.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>io.opentelemetry</groupId>
+      <artifactId>opentelemetry-sdk</artifactId>
+      <version>${opentelemetry.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>io.opentelemetry</groupId>
+      <artifactId>opentelemetry-exporter-otlp</artifactId>
+      <version>${opentelemetry.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>io.micrometer</groupId>
+      <artifactId>micrometer-registry-otlp</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>io.micrometer</groupId>
+      <artifactId>micrometer-tracing-bridge-otel</artifactId>
+    </dependency>
+  </dependencies>
+
+</project>
diff --git a/.../kotlin/io/github/llmagentbuilder/plugin/observation/opentelemetry/OpenTelemetryPlugin.kt b/.../kotlin/io/github/llmagentbuilder/plugin/observation/opentelemetry/OpenTelemetryPlugin.kt
@@ -0,0 +1,60 @@
+package io.github.llmagentbuilder.plugin.observation.opentelemetry
+
+import io.github.llmagentbuilder.core.AgentConfig
+import io.github.llmagentbuilder.core.EvaluationHelper
+import io.github.llmagentbuilder.core.ObservationPlugin
+import io.micrometer.observation.ObservationRegistry
+import io.micrometer.tracing.handler.DefaultTracingObservationHandler
+import io.micrometer.tracing.otel.bridge.EventListener
+import io.micrometer.tracing.otel.bridge.OtelCurrentTraceContext
+import io.micrometer.tracing.otel.bridge.OtelTracer
+import io.micrometer.tracing.otel.bridge.OtelTracer.EventPublisher
+import io.micrometer.tracing.otel.bridge.Slf4JEventListener
+import io.opentelemetry.exporter.otlp.http.trace.OtlpHttpSpanExporter
+import io.opentelemetry.sdk.resources.Resource
+import io.opentelemetry.sdk.trace.SdkTracerProvider
+import io.opentelemetry.sdk.trace.export.BatchSpanProcessor
+
+class OpenTelemetryPlugin : ObservationPlugin {
+    override fun install(
+        agentConfig: AgentConfig,
+        observationRegistry: ObservationRegistry
+    ) {
+        val builder = Resource.builder()
+        builder.put("service.name", agentConfig.metadata?.name)
+        val exporterBuilder = OtlpHttpSpanExporter.builder()
+        agentConfig.observation?.tracing?.exporter?.let {
+            exporterBuilder.setEndpoint(it.endpoint)
+            it.headers?.entries?.forEach { (k, v) ->
+                exporterBuilder.addHeader(k, EvaluationHelper.evaluate(v))
+            }
+        }
+        val resource = builder.build()
+        val tracer = SdkTracerProvider.builder()
+            .setResource(resource)
+            .addSpanProcessor(
+                BatchSpanProcessor.builder(
+                    exporterBuilder.build()
+                ).build()
+            )
+            .build()
+            .get("llm-agent")
+        val context = OtelCurrentTraceContext()
+        val otelTracer = OtelTracer(
+            tracer, context, OTelEventPublisher(
+                listOf(
+                    Slf4JEventListener()
+                )
+            )
+        )
+        val handler = DefaultTracingObservationHandler(otelTracer)
+        observationRegistry.observationConfig().observationHandler(handler)
+    }
+
+    private class OTelEventPublisher(private val listeners: List<EventListener>) :
+        EventPublisher {
+        override fun publishEvent(event: Any?) {
+            listeners.forEach { it.onEvent(event) }
+        }
+    }
+}
diff --git a/observation/pom.xml b/observation/pom.xml
@@ -0,0 +1,25 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <parent>
+    <groupId>io.github.llmagentbuilder</groupId>
+    <artifactId>llm-agent-builder</artifactId>
+    <version>0.2.1</version>
+  </parent>
+
+  <artifactId>observation</artifactId>
+  <packaging>pom</packaging>
+  <name>LLM Observation</name>
+  <modules>
+    <module>open-telemetry</module>
+  </modules>
+
+  <properties>
+    <maven.compiler.source>21</maven.compiler.source>
+    <maven.compiler.target>21</maven.compiler.target>
+    <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+  </properties>
+
+</project>
diff --git a/pom.xml b/pom.xml
@@ -17,6 +17,7 @@
     <module>bootstrap</module>
     <module>llm</module>
     <module>cli</module>
+    <module>observation</module>
   </modules>
 
   <description>LLM Agent Builder</description>
@@ -57,6 +58,7 @@
     <jackson.version>2.16.1</jackson.version>
     <micrometer.version>1.12.4</micrometer.version>
     <micrometer-tracing.version>1.2.4</micrometer-tracing.version>
+    <handlebars.version>4.4.0</handlebars.version>
   </properties>
 
   <build>
@@ -290,6 +292,11 @@
         <type>pom</type>
         <scope>import</scope>
       </dependency>
+      <dependency>
+        <groupId>com.github.jknack</groupId>
+        <artifactId>handlebars</artifactId>
+        <version>${handlebars.version}</version>
+      </dependency>
     </dependencies>
   </dependencyManagement>