Anthropic prompt caching

docs/providers/anthropic.md

@@ -27,7 +27,12 @@ While Anthropic models don't have native JSON mode or structured output like som
 ## Limitations
 ### Messages
 
-Does not support the `SystemMessage` message type, we automatically convert `SystemMessage` to `UserMessage`.
+Most providers' API include system messages in the messages array with a "system" role. Anthropic does not support the system role, and instead has a "system" property, separate from messages.
+
+Therefore, for Anthropic we:
+* Filter all `SystemMessage`s out, omitting them from messages.
+* Always submit the prompt defined with `->withSystemPrompt()` at the top of the system prompts array.
+* Move all `SystemMessage`s to the system prompts array in the order they were declared.
 
 ### Images
 

src/Concerns/HasProviderMeta.php

@@ -0,0 +1,29 @@
+<?php
+
+namespace EchoLabs\Prism\Concerns;
+
+use EchoLabs\Prism\Enums\Provider;
+
+trait HasProviderMeta
+{
+    /** @var array<string, array<string, mixed>> */
+    protected $providerMeta = [];
+
+    /**
+     * @param  array<string, mixed>  $meta
+     */
+    public function withProviderMeta(Provider $provider, array $meta): self
+    {
+        $this->providerMeta[$provider->value] = $meta;
+
+        return $this;
+    }
+
+    /**
+     * @return array<string, mixed>> $meta
+     */
+    public function providerMeta(Provider $provider): array
+    {
+        return data_get($this->providerMeta, $provider->value, []);
+    }
+}

src/Providers/Anthropic/Anthropic.php

@@ -20,6 +20,7 @@ class Anthropic implements Provider
     public function __construct(
         #[\SensitiveParameter] public readonly string $apiKey,
         public readonly string $apiVersion,
+        public readonly ?string $betaFeatures = null
     ) {}
 
     #[\Override]
@@ -56,10 +57,11 @@ public function embeddings(EmbeddingRequest $request): EmbeddingResponse
      */
     protected function client(array $options = [], array $retry = []): PendingRequest
     {
-        return Http::withHeaders([
+        return Http::withHeaders(array_filter([
             'x-api-key' => $this->apiKey,
             'anthropic-version' => $this->apiVersion,
-        ])
+            'anthropic-beta' => $this->betaFeatures,
+        ]))
             ->withOptions($options)
             ->retry(...$retry)
             ->baseUrl('https://api.anthropic.com/v1');

src/Providers/Anthropic/Enums/AnthropicCacheType.php

@@ -0,0 +1,8 @@
+<?php
+
+namespace EchoLabs\Prism\Providers\Anthropic\Enums;
+
+enum AnthropicCacheType
+{
+    case ephemeral;
+}

src/Providers/Anthropic/Handlers/Structured.php

@@ -47,8 +47,10 @@ public function handle(Request $request): ProviderResponse
             text: $this->extractText($data),
             toolCalls: $this->extractToolCalls($data),
             usage: new Usage(
-                data_get($data, 'usage.input_tokens'),
-                data_get($data, 'usage.output_tokens'),
+                promptTokens: data_get($data, 'usage.input_tokens'),
+                completionTokens: data_get($data, 'usage.output_tokens'),
+                cacheWriteInputTokens: data_get($data, 'usage.cache_creation_input_tokens', null),
+                cacheReadInputTokens: data_get($data, 'usage.cache_read_input_tokens', null)
             ),
             finishReason: FinishReasonMap::map(data_get($data, 'stop_reason', '')),
             response: [
@@ -67,7 +69,7 @@ public function sendRequest(Request $request): Response
                 'messages' => MessageMap::map($request->messages),
                 'max_tokens' => $request->maxTokens ?? 2048,
             ], array_filter([
-                'system' => $request->systemPrompt,
+                'system' => MessageMap::mapSystemMessages($request->messages, $request->systemPrompt),
                 'temperature' => $request->temperature,
                 'top_p' => $request->topP,
                 'tools' => ToolMap::map($request->tools),

src/Providers/Anthropic/Handlers/Text.php

@@ -45,8 +45,10 @@ public function handle(Request $request): ProviderResponse
             text: $this->extractText($data),
             toolCalls: $this->extractToolCalls($data),
             usage: new Usage(
-                data_get($data, 'usage.input_tokens'),
-                data_get($data, 'usage.output_tokens'),
+                promptTokens: data_get($data, 'usage.input_tokens'),
+                completionTokens: data_get($data, 'usage.output_tokens'),
+                cacheWriteInputTokens: data_get($data, 'usage.cache_creation_input_tokens'),
+                cacheReadInputTokens: data_get($data, 'usage.cache_read_input_tokens')
             ),
             finishReason: FinishReasonMap::map(data_get($data, 'stop_reason', '')),
             response: [
@@ -65,7 +67,7 @@ public function sendRequest(Request $request): Response
                 'messages' => MessageMap::map($request->messages),
                 'max_tokens' => $request->maxTokens ?? 2048,
             ], array_filter([
-                'system' => $request->systemPrompt,
+                'system' => MessageMap::mapSystemMessages($request->messages, $request->systemPrompt),
                 'temperature' => $request->temperature,
                 'top_p' => $request->topP,
                 'tools' => ToolMap::map($request->tools),

src/Providers/Anthropic/Maps/MessageMap.php

@@ -5,6 +5,7 @@
 namespace EchoLabs\Prism\Providers\Anthropic\Maps;
 
 use EchoLabs\Prism\Contracts\Message;
+use EchoLabs\Prism\Enums\Provider;
 use EchoLabs\Prism\ValueObjects\Messages\AssistantMessage;
 use EchoLabs\Prism\ValueObjects\Messages\Support\Image;
 use EchoLabs\Prism\ValueObjects\Messages\SystemMessage;
@@ -14,6 +15,7 @@
 use EchoLabs\Prism\ValueObjects\ToolResult;
 use Exception;
 use InvalidArgumentException;
+use UnitEnum;
 
 class MessageMap
 {
@@ -23,7 +25,25 @@ class MessageMap
      */
     public static function map(array $messages): array
     {
-        return array_map(fn (Message $message): array => self::mapMessage($message), $messages);
+        return array_values(array_map(
+            fn (Message $message): array => self::mapMessage($message),
+            array_filter($messages, fn (Message $message): bool => ! $message instanceof SystemMessage)
+        ));
+    }
+
+    /**
+     * @param  array<int, Message>  $messages
+     * @return array<int, mixed>
+     */
+    public static function mapSystemMessages(array $messages, ?string $systemPrompt): array
+    {
+        return array_values(array_merge(
+            $systemPrompt !== null ? [self::mapSystemMessage(new SystemMessage($systemPrompt))] : [],
+            array_map(
+                fn (Message $message): array => self::mapMessage($message),
+                array_filter($messages, fn (Message $message): bool => $message instanceof SystemMessage)
+            )
+        ));
     }
 
     /**
@@ -45,10 +65,13 @@ protected static function mapMessage(Message $message): array
      */
     protected static function mapSystemMessage(SystemMessage $systemMessage): array
     {
-        return [
-            'role' => 'user',
-            'content' => $systemMessage->content,
-        ];
+        $cacheType = data_get($systemMessage->providerMeta(Provider::Anthropic), 'cacheType', null);
+
+        return array_filter([
+            'type' => 'text',
+            'text' => $systemMessage->content,
+            'cache_control' => $cacheType ? ['type' => $cacheType instanceof UnitEnum ? $cacheType->name : $cacheType] : null,
+        ]);
     }
 
     /**
@@ -71,28 +94,19 @@ protected static function mapToolResultMessage(ToolResultMessage $message): arra
      */
     protected static function mapUserMessage(UserMessage $message): array
     {
-        $imageParts = array_map(function (Image $image): array {
-            if ($image->isUrl()) {
-                throw new InvalidArgumentException('URL image type is not supported by Anthropic');
-            }
-
-            return [
-                'type' => 'image',
-                'source' => [
-                    'type' => 'base64',
-                    'media_type' => $image->mimeType,
-                    'data' => $image->image,
-                ],
-            ];
-        }, $message->images());
+        $cacheType = data_get($message->providerMeta(Provider::Anthropic), 'cacheType', null);
+        $cache_control = $cacheType ? ['type' => $cacheType instanceof UnitEnum ? $cacheType->name : $cacheType] : null;
 
         return [
             'role' => 'user',
             'content' => [
-                ['type' => 'text', 'text' => $message->text()],
-                ...$imageParts,
+                array_filter([
+                    'type' => 'text',
+                    'text' => $message->text(),
+                    'cache_control' => $cache_control,
+                ]),
+                ...self::mapImageParts($message->images(), $cache_control),
             ],
-
         ];
     }
 
@@ -101,32 +115,54 @@ protected static function mapUserMessage(UserMessage $message): array
      */
     protected static function mapAssistantMessage(AssistantMessage $message): array
     {
-        if ($message->toolCalls) {
-            $content = [];
+        $content = [];
 
-            if ($message->content !== '' && $message->content !== '0') {
-                $content[] = [
-                    'type' => 'text',
-                    'text' => $message->content,
-                ];
-            }
+        if ($message->content !== '' && $message->content !== '0') {
+            $cacheType = data_get($message->providerMeta(Provider::Anthropic), 'cacheType', null);
+
+            $content[] = array_filter([
+                'type' => 'text',
+                'text' => $message->content,
+                'cache_control' => $cacheType ? ['type' => $cacheType instanceof UnitEnum ? $cacheType->name : $cacheType] : null,
+            ]);
+        }
 
-            $toolCalls = array_map(fn (ToolCall $toolCall): array => [
+        $toolCalls = $message->toolCalls
+            ? array_map(fn (ToolCall $toolCall): array => [
                 'type' => 'tool_use',
                 'id' => $toolCall->id,
                 'name' => $toolCall->name,
                 'input' => $toolCall->arguments(),
-            ], $message->toolCalls);
-
-            return [
-                'role' => 'assistant',
-                'content' => array_merge($content, $toolCalls),
-            ];
-        }
+            ], $message->toolCalls)
+            : [];
 
         return [
             'role' => 'assistant',
-            'content' => $message->content,
+            'content' => array_merge($content, $toolCalls),
         ];
     }
+
+    /**
+     * @param  Image[]  $parts
+     * @param  array<string, mixed>|null  $cache_control
+     * @return array<string, mixed>
+     */
+    protected static function mapImageParts(array $parts, ?array $cache_control = null): array
+    {
+        return array_map(function (Image $image) use ($cache_control): array {
+            if ($image->isUrl()) {
+                throw new InvalidArgumentException('URL image type is not supported by Anthropic');
+            }
+
+            return array_filter([
+                'type' => 'image',
+                'source' => [
+                    'type' => 'base64',
+                    'media_type' => $image->mimeType,
+                    'data' => $image->image,
+                ],
+                'cache_control' => $cache_control,
+            ]);
+        }, $parts);
+    }
 }

src/Providers/Anthropic/Maps/ToolMap.php

@@ -4,7 +4,9 @@
 
 namespace EchoLabs\Prism\Providers\Anthropic\Maps;
 
+use EchoLabs\Prism\Enums\Provider;
 use EchoLabs\Prism\Tool as PrismTool;
+use UnitEnum;
 
 class ToolMap
 {
@@ -14,14 +16,19 @@ class ToolMap
      */
     public static function map(array $tools): array
     {
-        return array_map(fn (PrismTool $tool): array => [
-            'name' => $tool->name(),
-            'description' => $tool->description(),
-            'input_schema' => [
-                'type' => 'object',
-                'properties' => $tool->parameters(),
-                'required' => $tool->requiredParameters(),
-            ],
-        ], $tools);
+        return array_map(function (PrismTool $tool): array {
+            $cacheType = data_get($tool->providerMeta(Provider::Anthropic), 'cacheType', null);
+
+            return array_filter([
+                'name' => $tool->name(),
+                'description' => $tool->description(),
+                'input_schema' => [
+                    'type' => 'object',
+                    'properties' => $tool->parameters(),
+                    'required' => $tool->requiredParameters(),
+                ],
+                'cache_control' => $cacheType ? ['type' => $cacheType instanceof UnitEnum ? $cacheType->name : $cacheType] : null,
+            ]);
+        }, $tools);
     }
 }

src/Structured/ResponseBuilder.php

@@ -73,12 +73,18 @@ protected function decodeObject(string $responseText): ?array
     protected function calculateTotalUsage(): Usage
     {
         return new Usage(
-            $this
+            promptTokens: $this
                 ->steps
                 ->sum(fn (Step $result): int => $result->usage->promptTokens),
-            $this
+            completionTokens: $this
                 ->steps
-                ->sum(fn (Step $result): int => $result->usage->completionTokens)
+                ->sum(fn (Step $result): int => $result->usage->completionTokens),
+            cacheWriteInputTokens: $this->steps->contains(fn (Step $result): bool => $result->usage->cacheWriteInputTokens !== null)
+                ? $this->steps->sum(fn (Step $result): int => $result->usage->cacheWriteInputTokens ?? 0)
+                : null,
+            cacheReadInputTokens: $this->steps->contains(fn (Step $result): bool => $result->usage->cacheReadInputTokens !== null)
+                ? $this->steps->sum(fn (Step $result): int => $result->usage->cacheReadInputTokens ?? 0)
+                : null,
         );
     }
 }

src/Text/ResponseBuilder.php

@@ -56,12 +56,18 @@ public function toResponse(): Response
     protected function calculateTotalUsage(): Usage
     {
         return new Usage(
-            $this
+            promptTokens: $this
                 ->steps
                 ->sum(fn (Step $result): int => $result->usage->promptTokens),
-            $this
+            completionTokens: $this
                 ->steps
-                ->sum(fn (Step $result): int => $result->usage->completionTokens)
+                ->sum(fn (Step $result): int => $result->usage->completionTokens),
+            cacheWriteInputTokens: $this->steps->contains(fn (Step $result): bool => $result->usage->cacheWriteInputTokens !== null)
+                ? $this->steps->sum(fn (Step $result): int => $result->usage->cacheWriteInputTokens ?? 0)
+                : null,
+            cacheReadInputTokens: $this->steps->contains(fn (Step $result): bool => $result->usage->cacheReadInputTokens !== null)
+                ? $this->steps->sum(fn (Step $result): int => $result->usage->cacheReadInputTokens ?? 0)
+                : null,
         );
     }
 }