From b851875041ac1b83dc8effc139db4d3b284affb0 Mon Sep 17 00:00:00 2001 From: Chris Bridges Date: Wed, 15 Jan 2025 13:48:21 +0000 Subject: [PATCH 1/8] allow for multiple system prompts --- docs/providers/anthropic.md | 7 ++++- .../Anthropic/Handlers/Structured.php | 2 +- src/Providers/Anthropic/Handlers/Text.php | 2 +- src/Providers/Anthropic/Maps/MessageMap.php | 24 ++++++++++++-- tests/Providers/Anthropic/MessageMapTest.php | 31 +++++++++++++++---- 5 files changed, 54 insertions(+), 12 deletions(-) diff --git a/docs/providers/anthropic.md b/docs/providers/anthropic.md index b388b72..ee3787a 100644 --- a/docs/providers/anthropic.md +++ b/docs/providers/anthropic.md @@ -27,7 +27,12 @@ While Anthropic models don't have native JSON mode or structured output like som ## Limitations ### Messages -Does not support the `SystemMessage` message type, we automatically convert `SystemMessage` to `UserMessage`. +Most providers' API include system messages in the messages array with a "system" role. Anthropic does not support the system role, and instead has a "system" property, separate from messages. + +Therefore, for Anthropic we: +* Filter all `SystemMessage`s out, omitting them from messages. +* Always submit the prompt defined with `->withSystemPrompt()` at the top of the system prompts array. +* Move all `SystemMessage`s to the system prompts array in the order they were declared. ### Images diff --git a/src/Providers/Anthropic/Handlers/Structured.php b/src/Providers/Anthropic/Handlers/Structured.php index c30b0e9..d79b532 100644 --- a/src/Providers/Anthropic/Handlers/Structured.php +++ b/src/Providers/Anthropic/Handlers/Structured.php @@ -64,7 +64,7 @@ public function sendRequest(Request $request): Response 'messages' => MessageMap::map($request->messages), 'max_tokens' => $request->maxTokens ?? 2048, ], array_filter([ - 'system' => $request->systemPrompt, + 'system' => MessageMap::mapSystemMessages($request->messages, $request->systemPrompt), 'temperature' => $request->temperature, 'top_p' => $request->topP, ])) diff --git a/src/Providers/Anthropic/Handlers/Text.php b/src/Providers/Anthropic/Handlers/Text.php index 7f49492..f622227 100644 --- a/src/Providers/Anthropic/Handlers/Text.php +++ b/src/Providers/Anthropic/Handlers/Text.php @@ -65,7 +65,7 @@ public function sendRequest(Request $request): Response 'messages' => MessageMap::map($request->messages), 'max_tokens' => $request->maxTokens ?? 2048, ], array_filter([ - 'system' => $request->systemPrompt, + 'system' => MessageMap::mapSystemMessages($request->messages, $request->systemPrompt), 'temperature' => $request->temperature, 'top_p' => $request->topP, 'tools' => ToolMap::map($request->tools), diff --git a/src/Providers/Anthropic/Maps/MessageMap.php b/src/Providers/Anthropic/Maps/MessageMap.php index 7fee966..397f0d9 100644 --- a/src/Providers/Anthropic/Maps/MessageMap.php +++ b/src/Providers/Anthropic/Maps/MessageMap.php @@ -23,7 +23,25 @@ class MessageMap */ public static function map(array $messages): array { - return array_map(fn (Message $message): array => self::mapMessage($message), $messages); + return array_values(array_map( + fn (Message $message): array => self::mapMessage($message), + array_filter($messages, fn (Message $message): bool => ! $message instanceof SystemMessage) + )); + } + + /** + * @param array $messages + * @return array + */ + public static function mapSystemMessages(array $messages, ?string $systemPrompt): array + { + return array_values(array_merge( + $systemPrompt !== null ? [self::mapSystemMessage(new SystemMessage($systemPrompt))] : [], + array_map( + fn (Message $message): array => self::mapMessage($message), + array_filter($messages, fn (Message $message): bool => $message instanceof SystemMessage) + ) + )); } /** @@ -46,8 +64,8 @@ protected static function mapMessage(Message $message): array protected static function mapSystemMessage(SystemMessage $systemMessage): array { return [ - 'role' => 'user', - 'content' => $systemMessage->content, + 'type' => 'text', + 'text' => $systemMessage->content, ]; } diff --git a/tests/Providers/Anthropic/MessageMapTest.php b/tests/Providers/Anthropic/MessageMapTest.php index e61d405..af5e6af 100644 --- a/tests/Providers/Anthropic/MessageMapTest.php +++ b/tests/Providers/Anthropic/MessageMapTest.php @@ -25,6 +25,18 @@ ]]); }); +it('filters system messages out when calling map', function (): void { + expect(MessageMap::map([ + new UserMessage('Who are you?'), + new SystemMessage('I am Groot.'), + ]))->toBe([[ + 'role' => 'user', + 'content' => [ + ['type' => 'text', 'text' => 'Who are you?'], + ], + ]]); +}); + it('maps user messages with images from path', function (): void { $mappedMessage = MessageMap::map([ new UserMessage('Who are you?', [ @@ -136,10 +148,17 @@ }); it('maps system messages', function (): void { - expect(MessageMap::map([ - new SystemMessage('Who are you?'), - ]))->toBe([[ - 'role' => 'user', - 'content' => 'Who are you?', - ]]); + expect(MessageMap::mapSystemMessages( + [new SystemMessage('Who are you?'), new UserMessage('I am rocket.')], + 'I am Thanos. Me first.' + ))->toBe([ + [ + 'type' => 'text', + 'text' => 'I am Thanos. Me first.', + ], + [ + 'type' => 'text', + 'text' => 'Who are you?', + ], + ]); }); From c2f0e55df4e414ca854a2da3459f1a16002b96c2 Mon Sep 17 00:00:00 2001 From: Chris Bridges Date: Wed, 15 Jan 2025 13:57:45 +0000 Subject: [PATCH 2/8] support prompt caching --- src/Concerns/HasProviderMeta.php | 29 +++++++++ src/Providers/Anthropic/Anthropic.php | 6 +- .../Anthropic/Enums/AnthropicCacheType.php | 8 +++ .../Anthropic/Handlers/Structured.php | 6 +- src/Providers/Anthropic/Handlers/Text.php | 6 +- src/Providers/Anthropic/Maps/MessageMap.php | 49 ++++++++------- src/Providers/Anthropic/Maps/ToolMap.php | 25 +++++--- src/Structured/ResponseBuilder.php | 12 +++- src/Text/ResponseBuilder.php | 12 +++- src/Tool.php | 25 +------- .../Messages/AssistantMessage.php | 3 + src/ValueObjects/Messages/SystemMessage.php | 5 +- src/ValueObjects/Messages/UserMessage.php | 3 + src/ValueObjects/Usage.php | 4 +- .../anthropic/calculate-cache-usage-1.json | 1 + .../Providers/Anthropic/AnthropicTextTest.php | 20 +++++++ tests/Providers/Anthropic/MessageMapTest.php | 60 ++++++++++++++++++- tests/Providers/Anthropic/ToolMapTest.php | 30 ++++++++++ 18 files changed, 238 insertions(+), 66 deletions(-) create mode 100644 src/Concerns/HasProviderMeta.php create mode 100644 src/Providers/Anthropic/Enums/AnthropicCacheType.php create mode 100644 tests/Fixtures/anthropic/calculate-cache-usage-1.json diff --git a/src/Concerns/HasProviderMeta.php b/src/Concerns/HasProviderMeta.php new file mode 100644 index 0000000..6e2bb56 --- /dev/null +++ b/src/Concerns/HasProviderMeta.php @@ -0,0 +1,29 @@ +> */ + protected $providerMeta = []; + + /** + * @param array $meta + */ + public function withProviderMeta(Provider $provider, array $meta): self + { + $this->providerMeta[$provider->value] = $meta; + + return $this; + } + + /** + * @return array> $meta + */ + public function providerMeta(Provider $provider): array + { + return data_get($this->providerMeta, $provider->value, []); + } +} diff --git a/src/Providers/Anthropic/Anthropic.php b/src/Providers/Anthropic/Anthropic.php index 4e17eaa..115651f 100644 --- a/src/Providers/Anthropic/Anthropic.php +++ b/src/Providers/Anthropic/Anthropic.php @@ -20,6 +20,7 @@ class Anthropic implements Provider public function __construct( #[\SensitiveParameter] public readonly string $apiKey, public readonly string $apiVersion, + public readonly ?string $betaFeatures = null ) {} #[\Override] @@ -56,10 +57,11 @@ public function embeddings(EmbeddingRequest $request): EmbeddingResponse */ protected function client(array $options = [], array $retry = []): PendingRequest { - return Http::withHeaders([ + return Http::withHeaders(array_filter([ 'x-api-key' => $this->apiKey, 'anthropic-version' => $this->apiVersion, - ]) + 'anthropic-beta' => $this->betaFeatures, + ])) ->withOptions($options) ->retry(...$retry) ->baseUrl('https://api.anthropic.com/v1'); diff --git a/src/Providers/Anthropic/Enums/AnthropicCacheType.php b/src/Providers/Anthropic/Enums/AnthropicCacheType.php new file mode 100644 index 0000000..e3581e9 --- /dev/null +++ b/src/Providers/Anthropic/Enums/AnthropicCacheType.php @@ -0,0 +1,8 @@ +extractText($data), toolCalls: [], usage: new Usage( - data_get($data, 'usage.input_tokens'), - data_get($data, 'usage.output_tokens'), + promptTokens: data_get($data, 'usage.input_tokens'), + completionTokens: data_get($data, 'usage.output_tokens'), + cacheWriteInputTokens: data_get($data, 'usage.cache_creation_input_tokens', null), + cacheReadInputTokens: data_get($data, 'usage.cache_read_input_tokens', null) ), finishReason: FinishReasonMap::map(data_get($data, 'stop_reason', '')), response: [ diff --git a/src/Providers/Anthropic/Handlers/Text.php b/src/Providers/Anthropic/Handlers/Text.php index f622227..4e1d8f9 100644 --- a/src/Providers/Anthropic/Handlers/Text.php +++ b/src/Providers/Anthropic/Handlers/Text.php @@ -45,8 +45,10 @@ public function handle(Request $request): ProviderResponse text: $this->extractText($data), toolCalls: $this->extractToolCalls($data), usage: new Usage( - data_get($data, 'usage.input_tokens'), - data_get($data, 'usage.output_tokens'), + promptTokens: data_get($data, 'usage.input_tokens'), + completionTokens: data_get($data, 'usage.output_tokens'), + cacheWriteInputTokens: data_get($data, 'usage.cache_creation_input_tokens'), + cacheReadInputTokens: data_get($data, 'usage.cache_read_input_tokens') ), finishReason: FinishReasonMap::map(data_get($data, 'stop_reason', '')), response: [ diff --git a/src/Providers/Anthropic/Maps/MessageMap.php b/src/Providers/Anthropic/Maps/MessageMap.php index 397f0d9..163f6f1 100644 --- a/src/Providers/Anthropic/Maps/MessageMap.php +++ b/src/Providers/Anthropic/Maps/MessageMap.php @@ -5,6 +5,7 @@ namespace EchoLabs\Prism\Providers\Anthropic\Maps; use EchoLabs\Prism\Contracts\Message; +use EchoLabs\Prism\Enums\Provider; use EchoLabs\Prism\ValueObjects\Messages\AssistantMessage; use EchoLabs\Prism\ValueObjects\Messages\Support\Image; use EchoLabs\Prism\ValueObjects\Messages\SystemMessage; @@ -14,6 +15,7 @@ use EchoLabs\Prism\ValueObjects\ToolResult; use Exception; use InvalidArgumentException; +use UnitEnum; class MessageMap { @@ -63,10 +65,13 @@ protected static function mapMessage(Message $message): array */ protected static function mapSystemMessage(SystemMessage $systemMessage): array { - return [ + $cacheType = data_get($systemMessage->providerMeta(Provider::Anthropic), 'cacheType', null); + + return array_filter([ 'type' => 'text', 'text' => $systemMessage->content, - ]; + 'cache_control' => $cacheType ? ['type' => $cacheType instanceof UnitEnum ? $cacheType->name : $cacheType] : null, + ]); } /** @@ -104,10 +109,16 @@ protected static function mapUserMessage(UserMessage $message): array ]; }, $message->images()); + $cacheType = data_get($message->providerMeta(Provider::Anthropic), 'cacheType', null); + return [ 'role' => 'user', 'content' => [ - ['type' => 'text', 'text' => $message->text()], + array_filter([ + 'type' => 'text', + 'text' => $message->text(), + 'cache_control' => $cacheType ? ['type' => $cacheType instanceof UnitEnum ? $cacheType->name : $cacheType] : null, + ]), ...$imageParts, ], @@ -119,32 +130,30 @@ protected static function mapUserMessage(UserMessage $message): array */ protected static function mapAssistantMessage(AssistantMessage $message): array { - if ($message->toolCalls) { - $content = []; + $content = []; - if ($message->content !== '' && $message->content !== '0') { - $content[] = [ - 'type' => 'text', - 'text' => $message->content, - ]; - } + if ($message->content !== '' && $message->content !== '0') { + $cacheType = data_get($message->providerMeta(Provider::Anthropic), 'cacheType', null); - $toolCalls = array_map(fn (ToolCall $toolCall): array => [ + $content[] = array_filter([ + 'type' => 'text', + 'text' => $message->content, + 'cache_control' => $cacheType ? ['type' => $cacheType instanceof UnitEnum ? $cacheType->name : $cacheType] : null, + ]); + } + + $toolCalls = $message->toolCalls + ? array_map(fn (ToolCall $toolCall): array => [ 'type' => 'tool_use', 'id' => $toolCall->id, 'name' => $toolCall->name, 'input' => $toolCall->arguments(), - ], $message->toolCalls); - - return [ - 'role' => 'assistant', - 'content' => array_merge($content, $toolCalls), - ]; - } + ], $message->toolCalls) + : []; return [ 'role' => 'assistant', - 'content' => $message->content, + 'content' => array_merge($content, $toolCalls), ]; } } diff --git a/src/Providers/Anthropic/Maps/ToolMap.php b/src/Providers/Anthropic/Maps/ToolMap.php index 75ade55..292e194 100644 --- a/src/Providers/Anthropic/Maps/ToolMap.php +++ b/src/Providers/Anthropic/Maps/ToolMap.php @@ -4,7 +4,9 @@ namespace EchoLabs\Prism\Providers\Anthropic\Maps; +use EchoLabs\Prism\Enums\Provider; use EchoLabs\Prism\Tool as PrismTool; +use UnitEnum; class ToolMap { @@ -14,14 +16,19 @@ class ToolMap */ public static function map(array $tools): array { - return array_map(fn (PrismTool $tool): array => [ - 'name' => $tool->name(), - 'description' => $tool->description(), - 'input_schema' => [ - 'type' => 'object', - 'properties' => $tool->parameters(), - 'required' => $tool->requiredParameters(), - ], - ], $tools); + return array_map(function (PrismTool $tool): array { + $cacheType = data_get($tool->providerMeta(Provider::Anthropic), 'cacheType', null); + + return array_filter([ + 'name' => $tool->name(), + 'description' => $tool->description(), + 'input_schema' => [ + 'type' => 'object', + 'properties' => $tool->parameters(), + 'required' => $tool->requiredParameters(), + ], + 'cache_control' => $cacheType ? ['type' => $cacheType instanceof UnitEnum ? $cacheType->name : $cacheType] : null, + ]); + }, $tools); } } diff --git a/src/Structured/ResponseBuilder.php b/src/Structured/ResponseBuilder.php index 4eef54f..9b2f956 100644 --- a/src/Structured/ResponseBuilder.php +++ b/src/Structured/ResponseBuilder.php @@ -71,12 +71,18 @@ protected function decodeObject(string $responseText): ?array protected function calculateTotalUsage(): Usage { return new Usage( - $this + promptTokens: $this ->steps ->sum(fn (Step $result): int => $result->usage->promptTokens), - $this + completionTokens: $this ->steps - ->sum(fn (Step $result): int => $result->usage->completionTokens) + ->sum(fn (Step $result): int => $result->usage->completionTokens), + cacheWriteInputTokens: $this->steps->contains(fn (Step $result): bool => $result->usage->cacheWriteInputTokens !== null) + ? $this->steps->sum(fn (Step $result): int => $result->usage->cacheWriteInputTokens ?? 0) + : null, + cacheReadInputTokens: $this->steps->contains(fn (Step $result): bool => $result->usage->cacheReadInputTokens !== null) + ? $this->steps->sum(fn (Step $result): int => $result->usage->cacheReadInputTokens ?? 0) + : null, ); } } diff --git a/src/Text/ResponseBuilder.php b/src/Text/ResponseBuilder.php index 2bf5c58..86588c6 100644 --- a/src/Text/ResponseBuilder.php +++ b/src/Text/ResponseBuilder.php @@ -56,12 +56,18 @@ public function toResponse(): Response protected function calculateTotalUsage(): Usage { return new Usage( - $this + promptTokens: $this ->steps ->sum(fn (Step $result): int => $result->usage->promptTokens), - $this + completionTokens: $this ->steps - ->sum(fn (Step $result): int => $result->usage->completionTokens) + ->sum(fn (Step $result): int => $result->usage->completionTokens), + cacheWriteInputTokens: $this->steps->contains(fn (Step $result): bool => $result->usage->cacheWriteInputTokens !== null) + ? $this->steps->sum(fn (Step $result): int => $result->usage->cacheWriteInputTokens ?? 0) + : null, + cacheReadInputTokens: $this->steps->contains(fn (Step $result): bool => $result->usage->cacheReadInputTokens !== null) + ? $this->steps->sum(fn (Step $result): int => $result->usage->cacheReadInputTokens ?? 0) + : null, ); } } diff --git a/src/Tool.php b/src/Tool.php index bbb1494..c1993f1 100644 --- a/src/Tool.php +++ b/src/Tool.php @@ -6,8 +6,8 @@ use ArgumentCountError; use Closure; +use EchoLabs\Prism\Concerns\HasProviderMeta; use EchoLabs\Prism\Contracts\Schema; -use EchoLabs\Prism\Enums\Provider; use EchoLabs\Prism\Exceptions\PrismException; use EchoLabs\Prism\Schema\ArraySchema; use EchoLabs\Prism\Schema\BooleanSchema; @@ -21,6 +21,8 @@ class Tool { + use HasProviderMeta; + protected string $name = ''; protected string $description; @@ -34,9 +36,6 @@ class Tool /** @var Closure():string|callable():string */ protected $fn; - /** @var array> */ - protected $providerMeta = []; - public function as(string $name): self { $this->name = $name; @@ -139,24 +138,6 @@ public function withEnumParameter( return $this; } - /** - * @param array $meta - */ - public function withProviderMeta(Provider $provider, array $meta): self - { - $this->providerMeta[$provider->value] = $meta; - - return $this; - } - - /** - * @return array> $meta - */ - public function providerMeta(Provider $provider): array - { - return data_get($this->providerMeta, $provider->value, []); - } - /** @return array */ public function requiredParameters(): array { diff --git a/src/ValueObjects/Messages/AssistantMessage.php b/src/ValueObjects/Messages/AssistantMessage.php index 2793d50..3f78bee 100644 --- a/src/ValueObjects/Messages/AssistantMessage.php +++ b/src/ValueObjects/Messages/AssistantMessage.php @@ -4,11 +4,14 @@ namespace EchoLabs\Prism\ValueObjects\Messages; +use EchoLabs\Prism\Concerns\HasProviderMeta; use EchoLabs\Prism\Contracts\Message; use EchoLabs\Prism\ValueObjects\ToolCall; class AssistantMessage implements Message { + use HasProviderMeta; + /** * @param ToolCall[] $toolCalls */ diff --git a/src/ValueObjects/Messages/SystemMessage.php b/src/ValueObjects/Messages/SystemMessage.php index 95e88be..e9edd40 100644 --- a/src/ValueObjects/Messages/SystemMessage.php +++ b/src/ValueObjects/Messages/SystemMessage.php @@ -4,11 +4,14 @@ namespace EchoLabs\Prism\ValueObjects\Messages; +use EchoLabs\Prism\Concerns\HasProviderMeta; use EchoLabs\Prism\Contracts\Message; class SystemMessage implements Message { + use HasProviderMeta; + public function __construct( - public readonly string $content, + public readonly string $content ) {} } diff --git a/src/ValueObjects/Messages/UserMessage.php b/src/ValueObjects/Messages/UserMessage.php index 13da895..58790d0 100644 --- a/src/ValueObjects/Messages/UserMessage.php +++ b/src/ValueObjects/Messages/UserMessage.php @@ -4,12 +4,15 @@ namespace EchoLabs\Prism\ValueObjects\Messages; +use EchoLabs\Prism\Concerns\HasProviderMeta; use EchoLabs\Prism\Contracts\Message; use EchoLabs\Prism\ValueObjects\Messages\Support\Image; use EchoLabs\Prism\ValueObjects\Messages\Support\Text; class UserMessage implements Message { + use HasProviderMeta; + /** * @param array $additionalContent */ diff --git a/src/ValueObjects/Usage.php b/src/ValueObjects/Usage.php index e7a2b03..533a575 100644 --- a/src/ValueObjects/Usage.php +++ b/src/ValueObjects/Usage.php @@ -8,6 +8,8 @@ class Usage { public function __construct( public readonly int $promptTokens, - public readonly int $completionTokens + public readonly int $completionTokens, + public readonly ?int $cacheWriteInputTokens = null, + public readonly ?int $cacheReadInputTokens = null ) {} } diff --git a/tests/Fixtures/anthropic/calculate-cache-usage-1.json b/tests/Fixtures/anthropic/calculate-cache-usage-1.json new file mode 100644 index 0000000..dd2f1da --- /dev/null +++ b/tests/Fixtures/anthropic/calculate-cache-usage-1.json @@ -0,0 +1 @@ +{"id":"msg_01X2Qk7LtNEh4HB9xpYU57XU","type":"message","role":"assistant","model":"claude-3-5-sonnet-20240620","content":[{"type":"text","text":"I am an AI assistant created by Anthropic to be helpful, harmless, and honest. I don't have a physical form or avatar - I'm a language model trained to engage in conversation and help with tasks. How can I assist you today?"}],"stop_reason":"end_turn","stop_sequence":null,"usage":{"input_tokens":11,"output_tokens":55, "cache_creation_input_tokens" : 200, "cache_read_input_tokens": 100}} \ No newline at end of file diff --git a/tests/Providers/Anthropic/AnthropicTextTest.php b/tests/Providers/Anthropic/AnthropicTextTest.php index 2d404ca..d8eb1f8 100644 --- a/tests/Providers/Anthropic/AnthropicTextTest.php +++ b/tests/Providers/Anthropic/AnthropicTextTest.php @@ -8,6 +8,7 @@ use EchoLabs\Prism\Facades\Tool; use EchoLabs\Prism\Prism; use EchoLabs\Prism\ValueObjects\Messages\Support\Image; +use EchoLabs\Prism\ValueObjects\Messages\SystemMessage; use EchoLabs\Prism\ValueObjects\Messages\UserMessage; use Illuminate\Http\Client\Request; use Illuminate\Support\Facades\Http; @@ -27,6 +28,8 @@ expect($response->usage->promptTokens)->toBe(11); expect($response->usage->completionTokens)->toBe(55); + expect($response->usage->cacheWriteInputTokens)->toBeNull(); + expect($response->usage->cacheReadInputTokens)->toBeNull(); expect($response->response['id'])->toBe('msg_01X2Qk7LtNEh4HB9xpYU57XU'); expect($response->response['model'])->toBe('claude-3-5-sonnet-20240620'); expect($response->text)->toBe( @@ -159,3 +162,20 @@ expect($response->toolCalls[0]->name)->toBe('weather'); }); + +it('can calculate cache usage correctly', function (): void { + config()->set('prism.providers.anthropic.beta_features', 'prompt-caching-2024-07-31'); + + FixtureResponse::fakeResponseSequence('v1/messages', 'anthropic/calculate-cache-usage'); + + $response = Prism::text() + ->using('anthropic', 'claude-3-5-sonnet-20240620') + ->withMessages([ + (new SystemMessage('Old context'))->withProviderMeta(Provider::Anthropic, ['cacheType' => 'ephemeral']), + (new UserMessage('New context'))->withProviderMeta(Provider::Anthropic, ['cacheType' => 'ephemeral']), + ]) + ->generate(); + + expect($response->usage->cacheWriteInputTokens)->toBe(200); + expect($response->usage->cacheReadInputTokens)->ToBe(100); +}); diff --git a/tests/Providers/Anthropic/MessageMapTest.php b/tests/Providers/Anthropic/MessageMapTest.php index af5e6af..ecdbf63 100644 --- a/tests/Providers/Anthropic/MessageMapTest.php +++ b/tests/Providers/Anthropic/MessageMapTest.php @@ -4,6 +4,8 @@ namespace Tests\Providers\Anthropic; +use EchoLabs\Prism\Enums\Provider; +use EchoLabs\Prism\Providers\Anthropic\Enums\AnthropicCacheType; use EchoLabs\Prism\Providers\Anthropic\Maps\MessageMap; use EchoLabs\Prism\ValueObjects\Messages\AssistantMessage; use EchoLabs\Prism\ValueObjects\Messages\Support\Image; @@ -85,7 +87,12 @@ new AssistantMessage('I am Nyx'), ]))->toContain([ 'role' => 'assistant', - 'content' => 'I am Nyx', + 'content' => [ + [ + 'type' => 'text', + 'text' => 'I am Nyx', + ], + ], ]); }); @@ -162,3 +169,54 @@ ], ]); }); + +it('sets the cache type on a UserMessage if cacheType providerMeta is set on message', function (mixed $cacheType): void { + expect(MessageMap::map([ + (new UserMessage(content: 'Who are you?'))->withProviderMeta(Provider::Anthropic, ['cacheType' => $cacheType]), + ]))->toBe([[ + 'role' => 'user', + 'content' => [ + [ + 'type' => 'text', + 'text' => 'Who are you?', + 'cache_control' => ['type' => 'ephemeral'], + ], + ], + ]]); +})->with([ + 'ephemeral', + AnthropicCacheType::ephemeral, +]); + +it('sets the cache type on an AssistantMessage if cacheType providerMeta is set on message using an enum', function (mixed $cacheType): void { + expect(MessageMap::map([ + (new AssistantMessage(content: 'Who are you?'))->withProviderMeta(Provider::Anthropic, ['cacheType' => $cacheType]), + ]))->toBe([[ + 'role' => 'assistant', + 'content' => [ + [ + 'type' => 'text', + 'text' => 'Who are you?', + 'cache_control' => ['type' => AnthropicCacheType::ephemeral->name], + ], + ], + ]]); +})->with([ + 'ephemeral', + AnthropicCacheType::ephemeral, +]); + +it('sets the cache type on a SystemMessage if cacheType providerMeta is set on message using an enum', function (mixed $cacheType): void { + expect(MessageMap::mapSystemMessages([ + (new SystemMessage(content: 'Who are you?'))->withProviderMeta(Provider::Anthropic, ['cacheType' => $cacheType]), + ], null))->toBe([ + [ + 'type' => 'text', + 'text' => 'Who are you?', + 'cache_control' => ['type' => AnthropicCacheType::ephemeral->name], + ], + ]); +})->with([ + 'ephemeral', + AnthropicCacheType::ephemeral, +]); diff --git a/tests/Providers/Anthropic/ToolMapTest.php b/tests/Providers/Anthropic/ToolMapTest.php index b61de83..0de144d 100644 --- a/tests/Providers/Anthropic/ToolMapTest.php +++ b/tests/Providers/Anthropic/ToolMapTest.php @@ -4,6 +4,8 @@ namespace Tests\Providers\Anthropic; +use EchoLabs\Prism\Enums\Provider; +use EchoLabs\Prism\Providers\Anthropic\Enums\AnthropicCacheType; use EchoLabs\Prism\Providers\Anthropic\Maps\ToolMap; use EchoLabs\Prism\Tool; @@ -29,3 +31,31 @@ ], ]]); }); + +it('sets the cache typeif cacheType providerMeta is set on tool', function (mixed $cacheType): void { + $tool = (new Tool) + ->as('search') + ->for('Searching the web') + ->withStringParameter('query', 'the detailed search query') + ->using(fn (): string => '[Search results]') + ->withProviderMeta(Provider::Anthropic, ['cacheType' => $cacheType]); + + expect(ToolMap::map([$tool]))->toBe([[ + 'name' => 'search', + 'description' => 'Searching the web', + 'input_schema' => [ + 'type' => 'object', + 'properties' => [ + 'query' => [ + 'description' => 'the detailed search query', + 'type' => 'string', + ], + ], + 'required' => ['query'], + ], + 'cache_control' => ['type' => 'ephemeral'], + ]]); +})->with([ + 'ephemeral', + AnthropicCacheType::ephemeral, +]); From d3cb90c778abdc6a8d6f70f337dbeb51a3b57f50 Mon Sep 17 00:00:00 2001 From: Chris Bridges Date: Wed, 15 Jan 2025 16:24:24 +0000 Subject: [PATCH 3/8] caching for images --- src/Providers/Anthropic/Maps/MessageMap.php | 45 ++++++++++++-------- tests/Providers/Anthropic/MessageMapTest.php | 27 ++++++++++++ 2 files changed, 54 insertions(+), 18 deletions(-) diff --git a/src/Providers/Anthropic/Maps/MessageMap.php b/src/Providers/Anthropic/Maps/MessageMap.php index 163f6f1..0e9d7a1 100644 --- a/src/Providers/Anthropic/Maps/MessageMap.php +++ b/src/Providers/Anthropic/Maps/MessageMap.php @@ -94,22 +94,8 @@ protected static function mapToolResultMessage(ToolResultMessage $message): arra */ protected static function mapUserMessage(UserMessage $message): array { - $imageParts = array_map(function (Image $image): array { - if ($image->isUrl()) { - throw new InvalidArgumentException('URL image type is not supported by Anthropic'); - } - - return [ - 'type' => 'image', - 'source' => [ - 'type' => 'base64', - 'media_type' => $image->mimeType, - 'data' => $image->image, - ], - ]; - }, $message->images()); - $cacheType = data_get($message->providerMeta(Provider::Anthropic), 'cacheType', null); + $cache_control = $cacheType ? ['type' => $cacheType instanceof UnitEnum ? $cacheType->name : $cacheType] : null; return [ 'role' => 'user', @@ -117,11 +103,10 @@ protected static function mapUserMessage(UserMessage $message): array array_filter([ 'type' => 'text', 'text' => $message->text(), - 'cache_control' => $cacheType ? ['type' => $cacheType instanceof UnitEnum ? $cacheType->name : $cacheType] : null, + 'cache_control' => $cache_control, ]), - ...$imageParts, + ...self::mapImageParts($message->images(), $cache_control), ], - ]; } @@ -156,4 +141,28 @@ protected static function mapAssistantMessage(AssistantMessage $message): array 'content' => array_merge($content, $toolCalls), ]; } + + /** + * @param Image[] $parts + * @param array|null $cache_control + * @return array + */ + protected static function mapImageParts(array $parts, ?array $cache_control = null): array + { + return array_map(function (Image $image) use ($cache_control): array { + if ($image->isUrl()) { + throw new InvalidArgumentException('URL image type is not supported by Anthropic'); + } + + return array_filter([ + 'type' => 'image', + 'source' => [ + 'type' => 'base64', + 'media_type' => $image->mimeType, + 'data' => $image->image, + ], + 'cache_control' => $cache_control, + ]); + }, $parts); + } } diff --git a/tests/Providers/Anthropic/MessageMapTest.php b/tests/Providers/Anthropic/MessageMapTest.php index ecdbf63..fef8636 100644 --- a/tests/Providers/Anthropic/MessageMapTest.php +++ b/tests/Providers/Anthropic/MessageMapTest.php @@ -188,6 +188,33 @@ AnthropicCacheType::ephemeral, ]); +it('sets the cache type on a UserMessage image if cacheType providerMeta is set on message', function (): void { + expect(MessageMap::map([ + (new UserMessage( + content: 'Who are you?', + additionalContent: [Image::fromPath('tests/Fixtures/test-image.png')] + ))->withProviderMeta(Provider::Anthropic, ['cacheType' => 'ephemeral']), + ]))->toBe([[ + 'role' => 'user', + 'content' => [ + [ + 'type' => 'text', + 'text' => 'Who are you?', + 'cache_control' => ['type' => 'ephemeral'], + ], + [ + 'type' => 'image', + 'source' => [ + 'type' => 'base64', + 'media_type' => 'image/png', + 'data' => base64_encode(file_get_contents('tests/Fixtures/test-image.png')), + ], + 'cache_control' => ['type' => 'ephemeral'], + ], + ], + ]]); +}); + it('sets the cache type on an AssistantMessage if cacheType providerMeta is set on message using an enum', function (mixed $cacheType): void { expect(MessageMap::map([ (new AssistantMessage(content: 'Who are you?'))->withProviderMeta(Provider::Anthropic, ['cacheType' => $cacheType]), From e84110e20b515134200c7f846b696206f9d5774c Mon Sep 17 00:00:00 2001 From: Chris Bridges Date: Wed, 15 Jan 2025 17:50:56 +0000 Subject: [PATCH 4/8] tweak test names --- tests/Providers/Anthropic/MessageMapTest.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/Providers/Anthropic/MessageMapTest.php b/tests/Providers/Anthropic/MessageMapTest.php index fef8636..3f547b1 100644 --- a/tests/Providers/Anthropic/MessageMapTest.php +++ b/tests/Providers/Anthropic/MessageMapTest.php @@ -215,7 +215,7 @@ ]]); }); -it('sets the cache type on an AssistantMessage if cacheType providerMeta is set on message using an enum', function (mixed $cacheType): void { +it('sets the cache type on an AssistantMessage if cacheType providerMeta is set on message', function (mixed $cacheType): void { expect(MessageMap::map([ (new AssistantMessage(content: 'Who are you?'))->withProviderMeta(Provider::Anthropic, ['cacheType' => $cacheType]), ]))->toBe([[ @@ -233,7 +233,7 @@ AnthropicCacheType::ephemeral, ]); -it('sets the cache type on a SystemMessage if cacheType providerMeta is set on message using an enum', function (mixed $cacheType): void { +it('sets the cache type on a SystemMessage if cacheType providerMeta is set on message', function (mixed $cacheType): void { expect(MessageMap::mapSystemMessages([ (new SystemMessage(content: 'Who are you?'))->withProviderMeta(Provider::Anthropic, ['cacheType' => $cacheType]), ], null))->toBe([ From bf8d32d1b40751c6332ed018d4040aaf8ee935b8 Mon Sep 17 00:00:00 2001 From: sixlive Date: Fri, 17 Jan 2025 13:25:37 -0500 Subject: [PATCH 5/8] Refactor to use PendingRequests to use the HasProviderMeta trait --- src/Concerns/ConfiguresProviders.php | 15 --------------- src/Embeddings/PendingRequest.php | 2 ++ src/Structured/PendingRequest.php | 2 ++ src/Text/PendingRequest.php | 2 ++ 4 files changed, 6 insertions(+), 15 deletions(-) diff --git a/src/Concerns/ConfiguresProviders.php b/src/Concerns/ConfiguresProviders.php index a9e4600..24a8aa0 100644 --- a/src/Concerns/ConfiguresProviders.php +++ b/src/Concerns/ConfiguresProviders.php @@ -16,9 +16,6 @@ trait ConfiguresProviders protected string $model; - /** @var array> */ - protected $providerMeta = []; - public function using(string|ProviderEnum $provider, string $model): self { $this->providerKey = is_string($provider) ? $provider : $provider->value; @@ -35,18 +32,6 @@ public function provider(): Provider return $this->provider; } - /** - * @param array $meta - */ - public function withProviderMeta(string|ProviderEnum $provider, array $meta): self - { - $key = is_string($provider) ? $provider : $provider->value; - - $this->providerMeta[$key] = $meta; - - return $this; - } - /** * @param array $config */ diff --git a/src/Embeddings/PendingRequest.php b/src/Embeddings/PendingRequest.php index bb7e575..959360f 100644 --- a/src/Embeddings/PendingRequest.php +++ b/src/Embeddings/PendingRequest.php @@ -6,12 +6,14 @@ use EchoLabs\Prism\Concerns\ConfiguresClient; use EchoLabs\Prism\Concerns\ConfiguresProviders; +use EchoLabs\Prism\Concerns\HasProviderMeta; use EchoLabs\Prism\Exceptions\PrismException; class PendingRequest { use ConfiguresClient; use ConfiguresProviders; + use HasProviderMeta; protected string $input = ''; diff --git a/src/Structured/PendingRequest.php b/src/Structured/PendingRequest.php index 4455888..85a793b 100644 --- a/src/Structured/PendingRequest.php +++ b/src/Structured/PendingRequest.php @@ -10,6 +10,7 @@ use EchoLabs\Prism\Concerns\ConfiguresStructuredOutput; use EchoLabs\Prism\Concerns\HasMessages; use EchoLabs\Prism\Concerns\HasPrompts; +use EchoLabs\Prism\Concerns\HasProviderMeta; use EchoLabs\Prism\Concerns\HasSchema; use EchoLabs\Prism\Exceptions\PrismException; use EchoLabs\Prism\ValueObjects\Messages\UserMessage; @@ -22,6 +23,7 @@ class PendingRequest use ConfiguresStructuredOutput; use HasMessages; use HasPrompts; + use HasProviderMeta; use HasSchema; public function generate(): Response diff --git a/src/Text/PendingRequest.php b/src/Text/PendingRequest.php index abb833d..0456595 100644 --- a/src/Text/PendingRequest.php +++ b/src/Text/PendingRequest.php @@ -11,6 +11,7 @@ use EchoLabs\Prism\Concerns\ConfiguresTools; use EchoLabs\Prism\Concerns\HasMessages; use EchoLabs\Prism\Concerns\HasPrompts; +use EchoLabs\Prism\Concerns\HasProviderMeta; use EchoLabs\Prism\Concerns\HasTools; use EchoLabs\Prism\Exceptions\PrismException; use EchoLabs\Prism\ValueObjects\Messages\UserMessage; @@ -24,6 +25,7 @@ class PendingRequest use ConfiguresTools; use HasMessages; use HasPrompts; + use HasProviderMeta; use HasTools; public function generate(): Response From 291d015d66c7d3afd542133b3dfee08011a0ccf7 Mon Sep 17 00:00:00 2001 From: sixlive Date: Fri, 17 Jan 2025 14:56:41 -0500 Subject: [PATCH 6/8] Use title case for enums --- src/Providers/Anthropic/Enums/AnthropicCacheType.php | 4 ++-- src/Providers/Anthropic/Maps/MessageMap.php | 6 +++--- tests/Providers/Anthropic/MessageMapTest.php | 10 +++++----- tests/Providers/Anthropic/ToolMapTest.php | 2 +- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/Providers/Anthropic/Enums/AnthropicCacheType.php b/src/Providers/Anthropic/Enums/AnthropicCacheType.php index e3581e9..a74ccac 100644 --- a/src/Providers/Anthropic/Enums/AnthropicCacheType.php +++ b/src/Providers/Anthropic/Enums/AnthropicCacheType.php @@ -2,7 +2,7 @@ namespace EchoLabs\Prism\Providers\Anthropic\Enums; -enum AnthropicCacheType +enum AnthropicCacheType: string { - case ephemeral; + case Ephemeral = 'ephemeral'; } diff --git a/src/Providers/Anthropic/Maps/MessageMap.php b/src/Providers/Anthropic/Maps/MessageMap.php index 0e9d7a1..a9ffd15 100644 --- a/src/Providers/Anthropic/Maps/MessageMap.php +++ b/src/Providers/Anthropic/Maps/MessageMap.php @@ -70,7 +70,7 @@ protected static function mapSystemMessage(SystemMessage $systemMessage): array return array_filter([ 'type' => 'text', 'text' => $systemMessage->content, - 'cache_control' => $cacheType ? ['type' => $cacheType instanceof UnitEnum ? $cacheType->name : $cacheType] : null, + 'cache_control' => $cacheType ? ['type' => $cacheType instanceof UnitEnum ? $cacheType->value : $cacheType] : null, ]); } @@ -95,7 +95,7 @@ protected static function mapToolResultMessage(ToolResultMessage $message): arra protected static function mapUserMessage(UserMessage $message): array { $cacheType = data_get($message->providerMeta(Provider::Anthropic), 'cacheType', null); - $cache_control = $cacheType ? ['type' => $cacheType instanceof UnitEnum ? $cacheType->name : $cacheType] : null; + $cache_control = $cacheType ? ['type' => $cacheType instanceof UnitEnum ? $cacheType->value : $cacheType] : null; return [ 'role' => 'user', @@ -123,7 +123,7 @@ protected static function mapAssistantMessage(AssistantMessage $message): array $content[] = array_filter([ 'type' => 'text', 'text' => $message->content, - 'cache_control' => $cacheType ? ['type' => $cacheType instanceof UnitEnum ? $cacheType->name : $cacheType] : null, + 'cache_control' => $cacheType ? ['type' => $cacheType instanceof UnitEnum ? $cacheType->value : $cacheType] : null, ]); } diff --git a/tests/Providers/Anthropic/MessageMapTest.php b/tests/Providers/Anthropic/MessageMapTest.php index 3f547b1..1c5642f 100644 --- a/tests/Providers/Anthropic/MessageMapTest.php +++ b/tests/Providers/Anthropic/MessageMapTest.php @@ -185,7 +185,7 @@ ]]); })->with([ 'ephemeral', - AnthropicCacheType::ephemeral, + AnthropicCacheType::Ephemeral, ]); it('sets the cache type on a UserMessage image if cacheType providerMeta is set on message', function (): void { @@ -224,13 +224,13 @@ [ 'type' => 'text', 'text' => 'Who are you?', - 'cache_control' => ['type' => AnthropicCacheType::ephemeral->name], + 'cache_control' => ['type' => AnthropicCacheType::Ephemeral->value], ], ], ]]); })->with([ 'ephemeral', - AnthropicCacheType::ephemeral, + AnthropicCacheType::Ephemeral, ]); it('sets the cache type on a SystemMessage if cacheType providerMeta is set on message', function (mixed $cacheType): void { @@ -240,10 +240,10 @@ [ 'type' => 'text', 'text' => 'Who are you?', - 'cache_control' => ['type' => AnthropicCacheType::ephemeral->name], + 'cache_control' => ['type' => AnthropicCacheType::Ephemeral->value], ], ]); })->with([ 'ephemeral', - AnthropicCacheType::ephemeral, + AnthropicCacheType::Ephemeral, ]); diff --git a/tests/Providers/Anthropic/ToolMapTest.php b/tests/Providers/Anthropic/ToolMapTest.php index 0de144d..acfeee0 100644 --- a/tests/Providers/Anthropic/ToolMapTest.php +++ b/tests/Providers/Anthropic/ToolMapTest.php @@ -57,5 +57,5 @@ ]]); })->with([ 'ephemeral', - AnthropicCacheType::ephemeral, + AnthropicCacheType::Ephemeral->value, ]); From 45ddd39e202087907970105c26edfd1efff30e06 Mon Sep 17 00:00:00 2001 From: sixlive Date: Fri, 17 Jan 2025 15:49:17 -0500 Subject: [PATCH 7/8] Types --- src/Providers/Anthropic/Maps/MessageMap.php | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Providers/Anthropic/Maps/MessageMap.php b/src/Providers/Anthropic/Maps/MessageMap.php index a9ffd15..6aecbb4 100644 --- a/src/Providers/Anthropic/Maps/MessageMap.php +++ b/src/Providers/Anthropic/Maps/MessageMap.php @@ -4,6 +4,7 @@ namespace EchoLabs\Prism\Providers\Anthropic\Maps; +use BackedEnum; use EchoLabs\Prism\Contracts\Message; use EchoLabs\Prism\Enums\Provider; use EchoLabs\Prism\ValueObjects\Messages\AssistantMessage; @@ -15,7 +16,6 @@ use EchoLabs\Prism\ValueObjects\ToolResult; use Exception; use InvalidArgumentException; -use UnitEnum; class MessageMap { @@ -70,7 +70,7 @@ protected static function mapSystemMessage(SystemMessage $systemMessage): array return array_filter([ 'type' => 'text', 'text' => $systemMessage->content, - 'cache_control' => $cacheType ? ['type' => $cacheType instanceof UnitEnum ? $cacheType->value : $cacheType] : null, + 'cache_control' => $cacheType ? ['type' => $cacheType instanceof BackedEnum ? $cacheType->value : $cacheType] : null, ]); } @@ -95,7 +95,7 @@ protected static function mapToolResultMessage(ToolResultMessage $message): arra protected static function mapUserMessage(UserMessage $message): array { $cacheType = data_get($message->providerMeta(Provider::Anthropic), 'cacheType', null); - $cache_control = $cacheType ? ['type' => $cacheType instanceof UnitEnum ? $cacheType->value : $cacheType] : null; + $cache_control = $cacheType ? ['type' => $cacheType instanceof BackedEnum ? $cacheType->value : $cacheType] : null; return [ 'role' => 'user', @@ -123,7 +123,7 @@ protected static function mapAssistantMessage(AssistantMessage $message): array $content[] = array_filter([ 'type' => 'text', 'text' => $message->content, - 'cache_control' => $cacheType ? ['type' => $cacheType instanceof UnitEnum ? $cacheType->value : $cacheType] : null, + 'cache_control' => $cacheType ? ['type' => $cacheType instanceof BackedEnum ? $cacheType->value : $cacheType] : null, ]); } From 1f220294dfd59a5b2a601b596bbc7590edafd734 Mon Sep 17 00:00:00 2001 From: Chris Bridges Date: Sun, 19 Jan 2025 10:02:11 +0000 Subject: [PATCH 8/8] Add usage instructions to docs --- docs/providers/anthropic.md | 47 +++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/docs/providers/anthropic.md b/docs/providers/anthropic.md index ee3787a..df6e86a 100644 --- a/docs/providers/anthropic.md +++ b/docs/providers/anthropic.md @@ -7,6 +7,53 @@ 'version' => env('ANTHROPIC_API_VERSION', '2023-06-01'), ] ``` +## Prompt caching + +Anthropic's prompt caching feature allows you to drastically reduce latency and your API bill when repeatedly re-using blocks of content within five minutes of each other. + +We support Anthropic prompt caching on: + +- System Messages (text only) +- User Messages (Text and Image) +- Tools + +The API for enable prompt caching is the same for all, enabled via the `withProviderMeta()` method. Where a UserMessage contains both text and an image, both will be cached. + +```php +use EchoLabs\Enums\Provider; +use EchoLabs\Prism\Prism; +use EchoLabs\Prism\Tool; +use EchoLabs\Prism\ValueObjects\Messages\UserMessage; +use EchoLabs\Prism\ValueObjects\Messages\SystemMessage; + +Prism::text() + ->using(Provider::Anthropic, 'claude-3-5-sonnet-20241022') + ->withMessages([ + (new SystemMessage('I am a long re-usable system message.')) + ->withProviderMeta(Provider::Anthropic, ['cacheType' => 'ephemeral']), + + (new UserMessage('I am a long re-usable user message.')) + ->withProviderMeta(Provider::Anthropic, ['cacheType' => 'ephemeral']) + ]) + ->withTools([ + Tool::as('cache me') + ->withProviderMeta(Provider::Anthropic, ['cacheType' => 'ephemeral']) + ]) + ->generate(); +``` + +If you prefer, you can use the `AnthropicCacheType` Enum like so: + +```php +use EchoLabs\Enums\Provider; +use EchoLabs\Prism\Providers\Anthropic\Enums\AnthropicCacheType; +use EchoLabs\Prism\ValueObjects\Messages\UserMessage; + +(new UserMessage('I am a long re-usable user message.'))->withProviderMeta(Provider::Anthropic, ['cacheType' => AnthropicCacheType::ephemeral]) +``` +Note that you must use the `withMessages()` method in order to enable prompt caching, rather than `withPrompt()` or `withSystemPrompt()`. + +Please ensure you read Anthropic's [prompt caching documentation](https://docs.anthropic.com/en/docs/build-with-claude/prompt-caching), which covers some important information on e.g. minimum cacheable tokens and message order consistency. ## Considerations ### Message Order