From 39e5c1f91ef6ef6fbf35e08542d86def86edadda Mon Sep 17 00:00:00 2001 From: Walter Korman Date: Sat, 25 Jan 2025 22:22:14 -0800 Subject: [PATCH] feat (provider/luma): add Luma provider (#4516) --- .changeset/thin-rice-drum.md | 5 + .changeset/weak-bobcats-wink.md | 6 + CHANGELOG.md | 1 + .../03-ai-sdk-core/35-image-generation.mdx | 2 + .../providers/01-ai-sdk-providers/80-luma.mdx | 243 +++++++++ examples/ai-core/package.json | 1 + .../ai-core/src/e2e/feature-test-suite.ts | 19 + examples/ai-core/src/e2e/luma.test.ts | 27 + .../luma-character-reference.ts | 32 ++ .../generate-image/luma-image-reference.ts | 31 ++ .../src/generate-image/luma-modify-image.ts | 29 + .../generate-image/luma-style-reference.ts | 31 ++ examples/ai-core/src/generate-image/luma.ts | 25 + .../fireworks/src/fireworks-image-model.ts | 70 +-- packages/luma/CHANGELOG.md | 1 + packages/luma/README.md | 50 ++ packages/luma/package.json | 63 +++ packages/luma/src/index.ts | 3 + packages/luma/src/luma-image-model.test.ts | 513 ++++++++++++++++++ packages/luma/src/luma-image-model.ts | 234 ++++++++ packages/luma/src/luma-image-settings.ts | 30 + packages/luma/src/luma-provider.test.ts | 77 +++ packages/luma/src/luma-provider.ts | 70 +++ packages/luma/tsconfig.json | 5 + packages/luma/tsup.config.ts | 10 + packages/luma/turbo.json | 12 + packages/luma/vitest.edge.config.js | 10 + packages/luma/vitest.node.config.js | 10 + .../provider-utils/src/get-from-api.test.ts | 181 ++++++ packages/provider-utils/src/get-from-api.ts | 107 ++++ packages/provider-utils/src/index.ts | 1 + .../src/response-handler.test.ts | 54 ++ .../provider-utils/src/response-handler.ts | 54 ++ pnpm-lock.yaml | 28 + turbo.json | 1 + 35 files changed, 1971 insertions(+), 65 deletions(-) create mode 100644 .changeset/thin-rice-drum.md create mode 100644 .changeset/weak-bobcats-wink.md create mode 100644 content/providers/01-ai-sdk-providers/80-luma.mdx create mode 100644 examples/ai-core/src/e2e/luma.test.ts create mode 100644 examples/ai-core/src/generate-image/luma-character-reference.ts create mode 100644 examples/ai-core/src/generate-image/luma-image-reference.ts create mode 100644 examples/ai-core/src/generate-image/luma-modify-image.ts create mode 100644 examples/ai-core/src/generate-image/luma-style-reference.ts create mode 100644 examples/ai-core/src/generate-image/luma.ts create mode 100644 packages/luma/CHANGELOG.md create mode 100644 packages/luma/README.md create mode 100644 packages/luma/package.json create mode 100644 packages/luma/src/index.ts create mode 100644 packages/luma/src/luma-image-model.test.ts create mode 100644 packages/luma/src/luma-image-model.ts create mode 100644 packages/luma/src/luma-image-settings.ts create mode 100644 packages/luma/src/luma-provider.test.ts create mode 100644 packages/luma/src/luma-provider.ts create mode 100644 packages/luma/tsconfig.json create mode 100644 packages/luma/tsup.config.ts create mode 100644 packages/luma/turbo.json create mode 100644 packages/luma/vitest.edge.config.js create mode 100644 packages/luma/vitest.node.config.js create mode 100644 packages/provider-utils/src/get-from-api.test.ts create mode 100644 packages/provider-utils/src/get-from-api.ts diff --git a/.changeset/thin-rice-drum.md b/.changeset/thin-rice-drum.md new file mode 100644 index 000000000000..c1c7e6ba747e --- /dev/null +++ b/.changeset/thin-rice-drum.md @@ -0,0 +1,5 @@ +--- +'@ai-sdk/luma': patch +--- + +feat (provider/luma): add Luma provider diff --git a/.changeset/weak-bobcats-wink.md b/.changeset/weak-bobcats-wink.md new file mode 100644 index 000000000000..14d1ce937a03 --- /dev/null +++ b/.changeset/weak-bobcats-wink.md @@ -0,0 +1,6 @@ +--- +'@ai-sdk/provider-utils': patch +'@ai-sdk/fireworks': patch +--- + +feat (provider-utils): add getFromApi and response handlers for binary responses and status-code errors diff --git a/CHANGELOG.md b/CHANGELOG.md index 72f67577a343..eb164d61b548 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ You can find the changelogs for the individual packages in their respective `CHA - [@ai-sdk/google](./packages/google/CHANGELOG.md) - [@ai-sdk/google-vertex](./packages/google-vertex/CHANGELOG.md) - [@ai-sdk/groq](./packages/groq/CHANGELOG.md) +- [@ai-sdk/luma](./packages/luma/CHANGELOG.md) - [@ai-sdk/mistral](./packages/mistral/CHANGELOG.md) - [@ai-sdk/openai](./packages/openai/CHANGELOG.md) - [@ai-sdk/openai-compatible](./packages/openai-compatible/CHANGELOG.md) diff --git a/content/docs/03-ai-sdk-core/35-image-generation.mdx b/content/docs/03-ai-sdk-core/35-image-generation.mdx index c2a206975cb0..237ed596ced7 100644 --- a/content/docs/03-ai-sdk-core/35-image-generation.mdx +++ b/content/docs/03-ai-sdk-core/35-image-generation.mdx @@ -225,3 +225,5 @@ try { | [Fireworks](/providers/ai-sdk-providers/fireworks#image-models) | `accounts/fireworks/models/playground-v2-1024px-aesthetic` | 640x1536, 768x1344, 832x1216, 896x1152, 1024x1024, 1152x896, 1216x832, 1344x768, 1536x640 | | [Fireworks](/providers/ai-sdk-providers/fireworks#image-models) | `accounts/fireworks/models/SSD-1B` | 640x1536, 768x1344, 832x1216, 896x1152, 1024x1024, 1152x896, 1216x832, 1344x768, 1536x640 | | [Fireworks](/providers/ai-sdk-providers/fireworks#image-models) | `accounts/fireworks/models/stable-diffusion-xl-1024-v1-0` | 640x1536, 768x1344, 832x1216, 896x1152, 1024x1024, 1152x896, 1216x832, 1344x768, 1536x640 | +| [Luma](/providers/ai-sdk-providers/luma#image-models) | `photon-1` | 1:1, 3:4, 4:3, 9:16, 16:9, 9:21, 21:9 | +| [Luma](/providers/ai-sdk-providers/luma#image-models) | `photon-flash-1` | 1:1, 3:4, 4:3, 9:16, 16:9, 9:21, 21:9 | diff --git a/content/providers/01-ai-sdk-providers/80-luma.mdx b/content/providers/01-ai-sdk-providers/80-luma.mdx new file mode 100644 index 000000000000..425139f9a782 --- /dev/null +++ b/content/providers/01-ai-sdk-providers/80-luma.mdx @@ -0,0 +1,243 @@ +--- +title: Luma +description: Learn how to use Luma AI models with the AI SDK. +--- + +# Luma Provider + +[Luma AI](https://lumalabs.ai/) provides state-of-the-art image generation models through their Dream Machine platform. Their models offer ultra-high quality image generation with superior prompt understanding and unique capabilities like character consistency and multi-image reference support. + +## Setup + +The Luma provider is available via the `@ai-sdk/luma` module. You can install it with + + + + + + + + + + + + + +## Provider Instance + +You can import the default provider instance `luma` from `@ai-sdk/luma`: + +```ts +import { luma } from '@ai-sdk/luma'; +``` + +If you need a customized setup, you can import `createLuma` and create a provider instance with your settings: + +```ts +import { createLuma } from '@ai-sdk/luma'; + +const luma = createLuma({ + apiKey: 'your-api-key', // optional, defaults to LUMA_API_KEY environment variable + baseURL: 'custom-url', // optional + headers: { + /* custom headers */ + }, // optional +}); +``` + +You can use the following optional settings to customize the Luma provider instance: + +- **baseURL** _string_ + + Use a different URL prefix for API calls, e.g. to use proxy servers. + The default prefix is `https://api.lumalabs.ai`. + +- **apiKey** _string_ + + API key that is being sent using the `Authorization` header. + It defaults to the `LUMA_API_KEY` environment variable. + +- **headers** _Record<string,string>_ + + Custom headers to include in the requests. + +- **fetch** _(input: RequestInfo, init?: RequestInit) => Promise<Response>_ + + Custom [fetch](https://developer.mozilla.org/en-US/docs/Web/API/fetch) implementation. + You can use it as a middleware to intercept requests, + or to provide a custom fetch implementation for e.g. testing. + +## Image Models + +You can create Luma image models using the `.image()` factory method. +For more on image generation with the AI SDK see [generateImage()](/docs/reference/ai-sdk-core/generate-image). + +### Basic Usage + +```ts +import { luma } from '@ai-sdk/luma'; +import { experimental_generateImage as generateImage } from 'ai'; +import fs from 'fs'; + +const { image } = await generateImage({ + model: luma.image('photon-1'), + prompt: 'A serene mountain landscape at sunset', + aspectRatio: '16:9', +}); + +const filename = `image-${Date.now()}.png`; +fs.writeFileSync(filename, image.uint8Array); +console.log(`Image saved to ${filename}`); +``` + +### Image Model Settings + +When creating an image model, you can customize the generation behavior with optional settings: + +```ts +const model = luma.image('photon-1', { + maxImagesPerCall: 1, // Maximum number of images to generate per API call + pollIntervalMillis: 5000, // How often to check for completed images (in ms) + maxPollAttempts: 10, // Maximum number of polling attempts before timeout +}); +``` + +Since Luma processes images through an asynchronous queue system, these settings allow you to tune the polling behavior: + +- **maxImagesPerCall** _number_ + + Override the maximum number of images generated per API call. Defaults to 1. + +- **pollIntervalMillis** _number_ + + Control how frequently the API is checked for completed images while they are + being processed. Defaults to 500ms. + +- **maxPollAttempts** _number_ + + Limit how long to wait for results before timing out, since image generation + is queued asynchronously. Defaults to 120 attempts. + +### Model Capabilities + +Luma offers two main models: + +| Model | Description | +| ---------------- | ---------------------------------------------------------------- | +| `photon-1` | High-quality image generation with superior prompt understanding | +| `photon-flash-1` | Faster generation optimized for speed while maintaining quality | + +Both models support the following aspect ratios: + +- 1:1 +- 3:4 +- 4:3 +- 9:16 +- 16:9 (default) +- 9:21 +- 21:9 + +For more details about supported aspect ratios, see the [Luma Image Generation documentation](https://docs.lumalabs.ai/docs/image-generation). + +Key features of Luma models include: + +- Ultra-high quality image generation +- 10x higher cost efficiency compared to similar models +- Superior prompt understanding and adherence +- Unique character consistency capabilities from single reference images +- Multi-image reference support for precise style matching + +### Advanced Options + +Luma models support several advanced features through the `providerOptions.luma` parameter. + +#### Image Reference + +Use up to 4 reference images to guide your generation. Useful for creating variations or visualizing complex concepts. Adjust the `weight` (0-1) to control the influence of reference images. + +```ts +// Example: Generate a salamander with reference +await generateImage({ + model: luma.image('photon-1'), + prompt: 'A salamander at dusk in a forest pond, in the style of ukiyo-e', + providerOptions: { + luma: { + image_ref: [ + { + url: 'https://example.com/reference.jpg', + weight: 0.85, + }, + ], + }, + }, +}); +``` + +#### Style Reference + +Apply specific visual styles to your generations using reference images. Control the style influence using the `weight` parameter. + +```ts +// Example: Generate with style reference +await generateImage({ + model: luma.image('photon-1'), + prompt: 'A blue cream Persian cat launching its website on Vercel', + providerOptions: { + luma: { + style_ref: [ + { + url: 'https://example.com/style.jpg', + weight: 0.8, + }, + ], + }, + }, +}); +``` + +#### Character Reference + +Create consistent and personalized characters using up to 4 reference images of the same subject. More reference images improve character representation. + +```ts +// Example: Generate character-based image +await generateImage({ + model: luma.image('photon-1'), + prompt: 'A woman with a cat riding a broomstick in a forest', + providerOptions: { + luma: { + character_ref: { + identity0: { + images: ['https://example.com/character.jpg'], + }, + }, + }, + }, +}); +``` + +#### Modify Image + +Transform existing images using text prompts. Use the `weight` parameter to control how closely the result matches the input image (higher weight = closer to input but less creative). + + + For color changes, it's recommended to use a lower weight value (0.0-0.1). + + +```ts +// Example: Modify existing image +await generateImage({ + model: luma.image('photon-1'), + prompt: 'transform the bike to a boat', + providerOptions: { + luma: { + modify_image_ref: { + url: 'https://example.com/image.jpg', + weight: 1.0, + }, + }, + }, +}); +``` + +For more details about Luma's capabilities and features, visit the [Luma Image Generation documentation](https://docs.lumalabs.ai/docs/image-generation). diff --git a/examples/ai-core/package.json b/examples/ai-core/package.json index 4d17274e72f8..0e565000676f 100644 --- a/examples/ai-core/package.json +++ b/examples/ai-core/package.json @@ -14,6 +14,7 @@ "@ai-sdk/google": "1.1.2", "@ai-sdk/google-vertex": "2.1.2", "@ai-sdk/groq": "1.1.2", + "@ai-sdk/luma": "0.0.0", "@ai-sdk/mistral": "1.1.2", "@ai-sdk/openai": "1.1.2", "@ai-sdk/openai-compatible": "0.1.3", diff --git a/examples/ai-core/src/e2e/feature-test-suite.ts b/examples/ai-core/src/e2e/feature-test-suite.ts index 08bb1554c3c5..dc0a694412b1 100644 --- a/examples/ai-core/src/e2e/feature-test-suite.ts +++ b/examples/ai-core/src/e2e/feature-test-suite.ts @@ -77,6 +77,7 @@ export interface ModelVariants { invalidModel?: LanguageModelV1; languageModels?: ModelWithCapabilities[]; embeddingModels?: ModelWithCapabilities>[]; + invalidImageModel?: ImageModelV1; imageModels?: ModelWithCapabilities[]; } @@ -1021,6 +1022,24 @@ export function createFeatureTestSuite({ }); } + if (models.invalidImageModel) { + describe('Image Model Error Handling:', () => { + const invalidModel = models.invalidImageModel!; + + it('should throw error on generate image attempt with invalid model ID', async () => { + try { + await generateImage({ + model: invalidModel, + prompt: 'This should fail', + }); + } catch (error) { + expect(error).toBeInstanceOf(APICallError); + errorValidator(error as APICallError); + } + }); + }); + } + if (models.embeddingModels && models.embeddingModels.length > 0) { describe.each(createModelObjects(models.embeddingModels))( 'Embedding Model: $modelId', diff --git a/examples/ai-core/src/e2e/luma.test.ts b/examples/ai-core/src/e2e/luma.test.ts new file mode 100644 index 000000000000..9d970ec22a54 --- /dev/null +++ b/examples/ai-core/src/e2e/luma.test.ts @@ -0,0 +1,27 @@ +import { expect } from 'vitest'; +import { luma as provider, LumaErrorData } from '@ai-sdk/luma'; +import { APICallError } from '@ai-sdk/provider'; +import { + createFeatureTestSuite, + createImageModelWithCapabilities, +} from './feature-test-suite'; +import 'dotenv/config'; + +createFeatureTestSuite({ + name: 'Luma', + models: { + invalidImageModel: provider.image('no-such-model'), + imageModels: [ + createImageModelWithCapabilities(provider.image('photon-flash-1')), + createImageModelWithCapabilities(provider.image('photon-1')), + ], + }, + timeout: 30000, + customAssertions: { + errorValidator: (error: APICallError) => { + expect((error.data as LumaErrorData).detail[0].msg).toMatch( + /Input should be/i, + ); + }, + }, +})(); diff --git a/examples/ai-core/src/generate-image/luma-character-reference.ts b/examples/ai-core/src/generate-image/luma-character-reference.ts new file mode 100644 index 000000000000..d39b593870af --- /dev/null +++ b/examples/ai-core/src/generate-image/luma-character-reference.ts @@ -0,0 +1,32 @@ +import { luma } from '@ai-sdk/luma'; +import { experimental_generateImage as generateImage } from 'ai'; +import 'dotenv/config'; +import fs from 'fs'; + +async function main() { + const result = await generateImage({ + model: luma.image('photon-flash-1'), + prompt: 'A woman with a cat riding a broomstick in a forest', + aspectRatio: '1:1', + providerOptions: { + luma: { + // https://docs.lumalabs.ai/docs/image-generation#character-reference + character_ref: { + identity0: { + images: [ + 'https://hebbkx1anhila5yf.public.blob.vercel-storage.com/future-me-8hcBWcZOkbE53q3gshhEm16S87qDpF.jpeg', + ], + }, + }, + }, + }, + }); + + for (const [index, image] of result.images.entries()) { + const filename = `image-${Date.now()}-${index}.png`; + fs.writeFileSync(filename, image.uint8Array); + console.log(`Image saved to ${filename}`); + } +} + +main().catch(console.error); diff --git a/examples/ai-core/src/generate-image/luma-image-reference.ts b/examples/ai-core/src/generate-image/luma-image-reference.ts new file mode 100644 index 000000000000..187aa4d114f1 --- /dev/null +++ b/examples/ai-core/src/generate-image/luma-image-reference.ts @@ -0,0 +1,31 @@ +import { luma } from '@ai-sdk/luma'; +import { experimental_generateImage as generateImage } from 'ai'; +import 'dotenv/config'; +import fs from 'fs'; + +async function main() { + const result = await generateImage({ + model: luma.image('photon-flash-1'), + prompt: 'A salamander at dusk in a forest pond, in the style of ukiyo-e', + aspectRatio: '1:1', + providerOptions: { + luma: { + // https://docs.lumalabs.ai/docs/image-generation#image-reference + image_ref: [ + { + url: 'https://hebbkx1anhila5yf.public.blob.vercel-storage.com/future-me-8hcBWcZOkbE53q3gshhEm16S87qDpF.jpeg', + weight: 0.8, + }, + ], + }, + }, + }); + + for (const [index, image] of result.images.entries()) { + const filename = `image-${Date.now()}-${index}.png`; + fs.writeFileSync(filename, image.uint8Array); + console.log(`Image saved to ${filename}`); + } +} + +main().catch(console.error); diff --git a/examples/ai-core/src/generate-image/luma-modify-image.ts b/examples/ai-core/src/generate-image/luma-modify-image.ts new file mode 100644 index 000000000000..c33502245c62 --- /dev/null +++ b/examples/ai-core/src/generate-image/luma-modify-image.ts @@ -0,0 +1,29 @@ +import { luma } from '@ai-sdk/luma'; +import { experimental_generateImage as generateImage } from 'ai'; +import 'dotenv/config'; +import fs from 'fs'; + +async function main() { + const result = await generateImage({ + model: luma.image('photon-flash-1'), + prompt: 'transform the bike to a boat', + aspectRatio: '1:1', + providerOptions: { + luma: { + // https://docs.lumalabs.ai/docs/image-generation#modify-image + modify_image_ref: { + url: 'https://hebbkx1anhila5yf.public.blob.vercel-storage.com/future-me-8hcBWcZOkbE53q3gshhEm16S87qDpF.jpeg', + weight: 1.0, + }, + }, + }, + }); + + for (const [index, image] of result.images.entries()) { + const filename = `image-${Date.now()}-${index}.png`; + fs.writeFileSync(filename, image.uint8Array); + console.log(`Image saved to ${filename}`); + } +} + +main().catch(console.error); diff --git a/examples/ai-core/src/generate-image/luma-style-reference.ts b/examples/ai-core/src/generate-image/luma-style-reference.ts new file mode 100644 index 000000000000..5c39a44f9d0a --- /dev/null +++ b/examples/ai-core/src/generate-image/luma-style-reference.ts @@ -0,0 +1,31 @@ +import { luma } from '@ai-sdk/luma'; +import { experimental_generateImage as generateImage } from 'ai'; +import 'dotenv/config'; +import fs from 'fs'; + +async function main() { + const result = await generateImage({ + model: luma.image('photon-flash-1'), + prompt: 'A blue cream Persian cat launching its website on Vercel', + aspectRatio: '1:1', + providerOptions: { + luma: { + // https://docs.lumalabs.ai/docs/image-generation#style-reference + style_ref: [ + { + url: 'https://hebbkx1anhila5yf.public.blob.vercel-storage.com/future-me-8hcBWcZOkbE53q3gshhEm16S87qDpF.jpeg', + weight: 0.8, + }, + ], + }, + }, + }); + + for (const [index, image] of result.images.entries()) { + const filename = `image-${Date.now()}-${index}.png`; + fs.writeFileSync(filename, image.uint8Array); + console.log(`Image saved to ${filename}`); + } +} + +main().catch(console.error); diff --git a/examples/ai-core/src/generate-image/luma.ts b/examples/ai-core/src/generate-image/luma.ts new file mode 100644 index 000000000000..9ffe2c98da3f --- /dev/null +++ b/examples/ai-core/src/generate-image/luma.ts @@ -0,0 +1,25 @@ +import { luma } from '@ai-sdk/luma'; +import { experimental_generateImage as generateImage } from 'ai'; +import 'dotenv/config'; +import fs from 'fs'; + +async function main() { + const result = await generateImage({ + model: luma.image('photon-flash-1'), + prompt: 'A salamander at dusk in a forest pond, in the style of ukiyo-e', + aspectRatio: '1:1', + providerOptions: { + luma: { + // add'l options here + }, + }, + }); + + for (const [index, image] of result.images.entries()) { + const filename = `image-${Date.now()}-${index}.png`; + fs.writeFileSync(filename, image.uint8Array); + console.log(`Image saved to ${filename}`); + } +} + +main().catch(console.error); diff --git a/packages/fireworks/src/fireworks-image-model.ts b/packages/fireworks/src/fireworks-image-model.ts index 303ca259a56d..d3075c1137ad 100644 --- a/packages/fireworks/src/fireworks-image-model.ts +++ b/packages/fireworks/src/fireworks-image-model.ts @@ -1,14 +1,10 @@ -import { - APICallError, - ImageModelV1, - ImageModelV1CallWarning, -} from '@ai-sdk/provider'; +import { ImageModelV1, ImageModelV1CallWarning } from '@ai-sdk/provider'; import { combineHeaders, - extractResponseHeaders, + createBinaryResponseHandler, + createStatusCodeErrorResponseHandler, FetchFunction, postJsonToApi, - ResponseHandler, } from '@ai-sdk/provider-utils'; import { FireworksImageModelId, @@ -74,62 +70,6 @@ interface FireworksImageModelConfig { }; } -const createBinaryResponseHandler = - (): ResponseHandler => - async ({ response, url, requestBodyValues }) => { - const responseHeaders = extractResponseHeaders(response); - - if (!response.body) { - throw new APICallError({ - message: 'Response body is empty', - url, - requestBodyValues, - statusCode: response.status, - responseHeaders, - responseBody: undefined, - }); - } - - try { - const buffer = await response.arrayBuffer(); - return { - responseHeaders, - value: buffer, - }; - } catch (error) { - throw new APICallError({ - message: 'Failed to read response as array buffer', - url, - requestBodyValues, - statusCode: response.status, - responseHeaders, - responseBody: undefined, - cause: error, - }); - } - }; - -const statusCodeErrorResponseHandler: ResponseHandler = async ({ - response, - url, - requestBodyValues, -}) => { - const responseHeaders = extractResponseHeaders(response); - const responseBody = await response.text(); - - return { - responseHeaders, - value: new APICallError({ - message: response.statusText, - url, - requestBodyValues: requestBodyValues as Record, - statusCode: response.status, - responseHeaders, - responseBody, - }), - }; -}; - export class FireworksImageModel implements ImageModelV1 { readonly specificationVersion = 'v1'; @@ -194,14 +134,14 @@ export class FireworksImageModel implements ImageModelV1 { ...(splitSize && { width: splitSize[0], height: splitSize[1] }), ...(providerOptions.fireworks ?? {}), }, - failedResponseHandler: statusCodeErrorResponseHandler, + failedResponseHandler: createStatusCodeErrorResponseHandler(), successfulResponseHandler: createBinaryResponseHandler(), abortSignal, fetch: this.config.fetch, }); return { - images: [new Uint8Array(response)], + images: [response], warnings, response: { timestamp: currentDate, diff --git a/packages/luma/CHANGELOG.md b/packages/luma/CHANGELOG.md new file mode 100644 index 000000000000..d1d8fa012b28 --- /dev/null +++ b/packages/luma/CHANGELOG.md @@ -0,0 +1 @@ +# @ai-sdk/luma diff --git a/packages/luma/README.md b/packages/luma/README.md new file mode 100644 index 000000000000..14aeafa7c964 --- /dev/null +++ b/packages/luma/README.md @@ -0,0 +1,50 @@ +# AI SDK - Luma Provider + +The **Luma provider** for the [AI SDK](https://sdk.vercel.ai/docs) contains support for Luma AI's state-of-the-art image generation models - Photon and Photon Flash. + +## About Luma Photon Models + +Luma Photon and Photon Flash are groundbreaking image generation models that deliver: + +- Ultra-high quality image generation +- 10x higher cost efficiency compared to similar models +- Superior prompt understanding and adherence +- Unique character consistency capabilities from single reference images +- Multi-image reference support for precise style matching + +## Setup + +The Luma provider is available in the `@ai-sdk/luma` module. You can install it with: + +```bash +npm i @ai-sdk/luma +``` + +## Provider Instance + +You can import the default provider instance `luma` from `@ai-sdk/luma`: + +```ts +import { luma } from '@ai-sdk/luma'; +``` + +## Image Generation Example + +```ts +import { luma } from '@ai-sdk/luma'; +import { experimental_generateImage as generateImage } from 'ai'; +import fs from 'fs'; + +const { image } = await generateImage({ + model: luma.image('photon'), + prompt: 'A serene mountain landscape at sunset', +}); + +const filename = `image-${Date.now()}.png`; +fs.writeFileSync(filename, image.uint8Array); +console.log(`Image saved to ${filename}`); +``` + +## Documentation + +For more detailed information about the Luma models and their capabilities, please visit [Luma AI](https://lumalabs.ai/). diff --git a/packages/luma/package.json b/packages/luma/package.json new file mode 100644 index 000000000000..30c2def213d3 --- /dev/null +++ b/packages/luma/package.json @@ -0,0 +1,63 @@ +{ + "name": "@ai-sdk/luma", + "version": "0.0.0", + "license": "Apache-2.0", + "sideEffects": false, + "main": "./dist/index.js", + "module": "./dist/index.mjs", + "types": "./dist/index.d.ts", + "files": [ + "dist/**/*", + "CHANGELOG.md" + ], + "scripts": { + "build": "tsup", + "build:watch": "tsup --watch", + "clean": "rm -rf dist", + "lint": "eslint \"./**/*.ts*\"", + "type-check": "tsc --noEmit", + "prettier-check": "prettier --check \"./**/*.ts*\"", + "test": "pnpm test:node && pnpm test:edge", + "test:edge": "vitest --config vitest.edge.config.js --run", + "test:node": "vitest --config vitest.node.config.js --run" + }, + "exports": { + "./package.json": "./package.json", + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.mjs", + "require": "./dist/index.js" + } + }, + "dependencies": { + "@ai-sdk/provider": "1.0.6", + "@ai-sdk/provider-utils": "2.1.2" + }, + "devDependencies": { + "@types/node": "^18", + "@vercel/ai-tsconfig": "workspace:*", + "tsup": "^8", + "typescript": "5.6.3", + "zod": "3.23.8" + }, + "peerDependencies": { + "zod": "^3.0.0" + }, + "engines": { + "node": ">=18" + }, + "publishConfig": { + "access": "public" + }, + "homepage": "https://sdk.vercel.ai/docs", + "repository": { + "type": "git", + "url": "git+https://github.com/vercel/ai.git" + }, + "bugs": { + "url": "https://github.com/vercel/ai/issues" + }, + "keywords": [ + "ai" + ] +} diff --git a/packages/luma/src/index.ts b/packages/luma/src/index.ts new file mode 100644 index 000000000000..57e5f4561e47 --- /dev/null +++ b/packages/luma/src/index.ts @@ -0,0 +1,3 @@ +export { createLuma, luma } from './luma-provider'; +export type { LumaProvider, LumaProviderSettings } from './luma-provider'; +export type { LumaErrorData } from './luma-image-model'; diff --git a/packages/luma/src/luma-image-model.test.ts b/packages/luma/src/luma-image-model.test.ts new file mode 100644 index 000000000000..a224a7bbde93 --- /dev/null +++ b/packages/luma/src/luma-image-model.test.ts @@ -0,0 +1,513 @@ +import { FetchFunction } from '@ai-sdk/provider-utils'; +import { createTestServer } from '@ai-sdk/provider-utils/test'; +import { describe, expect, it } from 'vitest'; +import { LumaImageModel } from './luma-image-model'; +import { InvalidResponseDataError } from '@ai-sdk/provider'; + +const prompt = 'A cute baby sea otter'; + +function createBasicModel({ + headers, + fetch, + currentDate, + settings, +}: { + headers?: () => Record; + fetch?: FetchFunction; + currentDate?: () => Date; + settings?: any; +} = {}) { + return new LumaImageModel('test-model', settings ?? {}, { + provider: 'luma', + baseURL: 'https://api.example.com', + headers: headers ?? (() => ({ 'api-key': 'test-key' })), + fetch, + _internal: { + currentDate, + }, + }); +} + +describe('LumaImageModel', () => { + const server = createTestServer({ + 'https://api.example.com/dream-machine/v1/generations/image': { + response: { + type: 'json-value', + body: { + id: 'test-generation-id', + generation_type: 'image', + state: 'queued', + created_at: '2024-01-01T00:00:00Z', + model: 'test-model', + request: { + generation_type: 'image', + model: 'test-model', + prompt: 'A cute baby sea otter', + }, + }, + }, + }, + 'https://api.example.com/dream-machine/v1/generations/test-generation-id': { + response: { + type: 'json-value', + body: { + id: 'test-generation-id', + generation_type: 'image', + state: 'completed', + created_at: '2024-01-01T00:00:00Z', + assets: { + image: 'https://api.example.com/image.png', + }, + model: 'test-model', + request: { + generation_type: 'image', + model: 'test-model', + prompt: 'A cute baby sea otter', + }, + }, + }, + }, + 'https://api.example.com/image.png': { + response: { + type: 'binary', + body: Buffer.from('test-binary-content'), + }, + }, + }); + + describe('doGenerate', () => { + it('should pass the correct parameters including aspect ratio', async () => { + const model = createBasicModel(); + + await model.doGenerate({ + prompt, + n: 1, + size: undefined, + aspectRatio: '16:9', + seed: undefined, + providerOptions: { luma: { additional_param: 'value' } }, + }); + + expect(await server.calls[0].requestBody).toStrictEqual({ + prompt, + aspect_ratio: '16:9', + model: 'test-model', + additional_param: 'value', + }); + }); + + it('should call the correct urls in sequence', async () => { + const model = createBasicModel(); + + await model.doGenerate({ + prompt, + n: 1, + aspectRatio: '16:9', + providerOptions: {}, + size: undefined, + seed: undefined, + }); + + expect(server.calls[0].requestMethod).toBe('POST'); + expect(server.calls[0].requestUrl).toBe( + 'https://api.example.com/dream-machine/v1/generations/image', + ); + expect(server.calls[1].requestMethod).toBe('GET'); + expect(server.calls[1].requestUrl).toBe( + 'https://api.example.com/dream-machine/v1/generations/test-generation-id', + ); + expect(server.calls[2].requestMethod).toBe('GET'); + expect(server.calls[2].requestUrl).toBe( + 'https://api.example.com/image.png', + ); + }); + + it('should pass headers', async () => { + const modelWithHeaders = createBasicModel({ + headers: () => ({ + 'Custom-Provider-Header': 'provider-header-value', + }), + }); + + await modelWithHeaders.doGenerate({ + prompt, + n: 1, + providerOptions: {}, + headers: { + 'Custom-Request-Header': 'request-header-value', + }, + size: undefined, + seed: undefined, + aspectRatio: undefined, + }); + + expect(server.calls[0].requestHeaders).toStrictEqual({ + 'content-type': 'application/json', + 'custom-provider-header': 'provider-header-value', + 'custom-request-header': 'request-header-value', + }); + }); + + it('should handle API errors', async () => { + server.urls[ + 'https://api.example.com/dream-machine/v1/generations/image' + ].response = { + type: 'error', + status: 400, + body: 'Bad Request', + }; + + const model = createBasicModel(); + await expect( + model.doGenerate({ + prompt, + n: 1, + providerOptions: {}, + size: undefined, + seed: undefined, + aspectRatio: undefined, + }), + ).rejects.toMatchObject({ + message: 'Bad Request', + statusCode: 400, + url: 'https://api.example.com/dream-machine/v1/generations/image', + requestBodyValues: { + prompt: 'A cute baby sea otter', + }, + responseBody: 'Bad Request', + }); + }); + + it('should handle failed generation state', async () => { + server.urls[ + 'https://api.example.com/dream-machine/v1/generations/test-generation-id' + ].response = { + type: 'json-value', + body: { + id: 'test-generation-id', + generation_type: 'image', + state: 'failed', + failure_reason: 'Generation failed', + created_at: '2024-01-01T00:00:00Z', + model: 'test-model', + request: { + generation_type: 'image', + model: 'test-model', + prompt: 'A cute baby sea otter', + }, + }, + }; + + const model = createBasicModel(); + await expect( + model.doGenerate({ + prompt, + n: 1, + providerOptions: {}, + size: undefined, + seed: undefined, + aspectRatio: undefined, + }), + ).rejects.toThrow(InvalidResponseDataError); + }); + + describe('warnings', () => { + it('should return warnings for unsupported parameters', async () => { + const model = createBasicModel(); + + const result = await model.doGenerate({ + prompt, + n: 1, + size: '1024x1024', + seed: 123, + providerOptions: {}, + aspectRatio: undefined, + }); + + expect(result.warnings).toContainEqual({ + type: 'unsupported-setting', + setting: 'seed', + details: 'This model does not support the `seed` option.', + }); + + expect(result.warnings).toContainEqual({ + type: 'unsupported-setting', + setting: 'size', + details: + 'This model does not support the `size` option. Use `aspectRatio` instead.', + }); + }); + }); + + describe('response metadata', () => { + it('should include timestamp, headers and modelId in response', async () => { + const testDate = new Date('2024-01-01T00:00:00Z'); + const model = createBasicModel({ + currentDate: () => testDate, + }); + + const result = await model.doGenerate({ + prompt, + n: 1, + providerOptions: {}, + size: undefined, + seed: undefined, + aspectRatio: undefined, + }); + + expect(result.response).toStrictEqual({ + timestamp: testDate, + modelId: 'test-model', + headers: expect.any(Object), + }); + }); + }); + }); + + describe('constructor', () => { + it('should expose correct provider and model information', () => { + const model = createBasicModel(); + + expect(model.provider).toBe('luma'); + expect(model.modelId).toBe('test-model'); + expect(model.specificationVersion).toBe('v1'); + expect(model.maxImagesPerCall).toBe(1); + }); + + it('should use maxImagesPerCall from settings', () => { + const model = createBasicModel({ + settings: { + maxImagesPerCall: 4, + }, + }); + + expect(model.maxImagesPerCall).toBe(4); + }); + + it('should default maxImagesPerCall to 1 when not specified', () => { + const model = createBasicModel(); + + expect(model.maxImagesPerCall).toBe(1); + }); + }); + + describe('response schema validation', () => { + it('should parse response with image references', async () => { + server.urls[ + 'https://api.example.com/dream-machine/v1/generations/test-generation-id' + ].response = { + type: 'json-value', + body: { + id: 'test-generation-id', + generation_type: 'image', + state: 'completed', + created_at: '2024-01-01T00:00:00Z', + assets: { + image: 'https://api.example.com/image.png', + }, + model: 'test-model', + request: { + generation_type: 'image', + model: 'test-model', + prompt: 'A cute baby sea otter', + image_ref: [ + { + url: 'https://example.com/ref1.jpg', + weight: 0.85, + }, + ], + }, + }, + }; + + const model = createBasicModel(); + const result = await model.doGenerate({ + prompt, + n: 1, + providerOptions: {}, + size: undefined, + seed: undefined, + aspectRatio: undefined, + }); + + // If schema validation fails, this won't get reached + expect(result.images).toBeDefined(); + }); + + it('should parse response with style references', async () => { + server.urls[ + 'https://api.example.com/dream-machine/v1/generations/test-generation-id' + ].response = { + type: 'json-value', + body: { + id: 'test-generation-id', + generation_type: 'image', + state: 'completed', + created_at: '2024-01-01T00:00:00Z', + assets: { + image: 'https://api.example.com/image.png', + }, + model: 'test-model', + request: { + generation_type: 'image', + model: 'test-model', + prompt: 'A cute baby sea otter', + style_ref: [ + { + url: 'https://example.com/style1.jpg', + weight: 0.8, + }, + ], + }, + }, + }; + + const model = createBasicModel(); + const result = await model.doGenerate({ + prompt, + n: 1, + providerOptions: {}, + size: undefined, + seed: undefined, + aspectRatio: undefined, + }); + + expect(result.images).toBeDefined(); + }); + + it('should parse response with character references', async () => { + server.urls[ + 'https://api.example.com/dream-machine/v1/generations/test-generation-id' + ].response = { + type: 'json-value', + body: { + id: 'test-generation-id', + generation_type: 'image', + state: 'completed', + created_at: '2024-01-01T00:00:00Z', + assets: { + image: 'https://api.example.com/image.png', + }, + model: 'test-model', + request: { + generation_type: 'image', + model: 'test-model', + prompt: 'A cute baby sea otter', + character_ref: { + identity0: { + images: ['https://example.com/character1.jpg'], + }, + }, + }, + }, + }; + + const model = createBasicModel(); + const result = await model.doGenerate({ + prompt, + n: 1, + providerOptions: {}, + size: undefined, + seed: undefined, + aspectRatio: undefined, + }); + + expect(result.images).toBeDefined(); + }); + + it('should parse response with modify image reference', async () => { + server.urls[ + 'https://api.example.com/dream-machine/v1/generations/test-generation-id' + ].response = { + type: 'json-value', + body: { + id: 'test-generation-id', + generation_type: 'image', + state: 'completed', + created_at: '2024-01-01T00:00:00Z', + assets: { + image: 'https://api.example.com/image.png', + }, + model: 'test-model', + request: { + generation_type: 'image', + model: 'test-model', + prompt: 'A cute baby sea otter', + modify_image_ref: { + url: 'https://example.com/modify.jpg', + weight: 1.0, + }, + }, + }, + }; + + const model = createBasicModel(); + const result = await model.doGenerate({ + prompt, + n: 1, + providerOptions: {}, + size: undefined, + seed: undefined, + aspectRatio: undefined, + }); + + expect(result.images).toBeDefined(); + }); + + it('should parse response with multiple reference types', async () => { + server.urls[ + 'https://api.example.com/dream-machine/v1/generations/test-generation-id' + ].response = { + type: 'json-value', + body: { + id: 'test-generation-id', + generation_type: 'image', + state: 'completed', + created_at: '2024-01-01T00:00:00Z', + assets: { + image: 'https://api.example.com/image.png', + }, + model: 'test-model', + request: { + generation_type: 'image', + model: 'test-model', + prompt: 'A cute baby sea otter', + image_ref: [ + { + url: 'https://example.com/ref1.jpg', + weight: 0.85, + }, + ], + style_ref: [ + { + url: 'https://example.com/style1.jpg', + weight: 0.8, + }, + ], + character_ref: { + identity0: { + images: ['https://example.com/character1.jpg'], + }, + }, + modify_image_ref: { + url: 'https://example.com/modify.jpg', + weight: 1.0, + }, + }, + }, + }; + + const model = createBasicModel(); + const result = await model.doGenerate({ + prompt, + n: 1, + providerOptions: {}, + size: undefined, + seed: undefined, + aspectRatio: undefined, + }); + + expect(result.images).toBeDefined(); + }); + }); +}); diff --git a/packages/luma/src/luma-image-model.ts b/packages/luma/src/luma-image-model.ts new file mode 100644 index 000000000000..5a256f0cfff0 --- /dev/null +++ b/packages/luma/src/luma-image-model.ts @@ -0,0 +1,234 @@ +import { + ImageModelV1, + ImageModelV1CallWarning, + InvalidResponseDataError, +} from '@ai-sdk/provider'; +import { + FetchFunction, + combineHeaders, + createBinaryResponseHandler, + createJsonResponseHandler, + createJsonErrorResponseHandler, + createStatusCodeErrorResponseHandler, + getFromApi, + postJsonToApi, +} from '@ai-sdk/provider-utils'; +import { LumaImageSettings } from './luma-image-settings'; +import { z } from 'zod'; + +const DEFAULT_POLL_INTERVAL_MILLIS = 500; +const DEFAULT_MAX_POLL_ATTEMPTS = 60000 / DEFAULT_POLL_INTERVAL_MILLIS; + +interface LumaImageModelConfig { + provider: string; + baseURL: string; + headers: () => Record; + fetch?: FetchFunction; + _internal?: { + currentDate?: () => Date; + }; +} + +async function delay(delayInMs?: number | null): Promise { + return delayInMs == null + ? Promise.resolve() + : new Promise(resolve => setTimeout(resolve, delayInMs)); +} + +export class LumaImageModel implements ImageModelV1 { + readonly specificationVersion = 'v1'; + + private readonly pollIntervalMillis: number; + private readonly maxPollAttempts: number; + + get provider(): string { + return this.config.provider; + } + + get maxImagesPerCall(): number { + return this.settings.maxImagesPerCall ?? 1; + } + + constructor( + readonly modelId: string, + private readonly settings: LumaImageSettings, + private readonly config: LumaImageModelConfig, + ) { + this.pollIntervalMillis = + settings.pollIntervalMillis ?? DEFAULT_POLL_INTERVAL_MILLIS; + this.maxPollAttempts = + settings.maxPollAttempts ?? DEFAULT_MAX_POLL_ATTEMPTS; + } + + async doGenerate({ + prompt, + n, + size, + aspectRatio, + seed, + providerOptions, + headers, + abortSignal, + }: Parameters[0]): Promise< + Awaited> + > { + const warnings: Array = []; + + if (seed != null) { + warnings.push({ + type: 'unsupported-setting', + setting: 'seed', + details: 'This model does not support the `seed` option.', + }); + } + + if (size != null) { + warnings.push({ + type: 'unsupported-setting', + setting: 'size', + details: + 'This model does not support the `size` option. Use `aspectRatio` instead.', + }); + } + + const currentDate = this.config._internal?.currentDate?.() ?? new Date(); + const fullHeaders = combineHeaders(this.config.headers(), headers); + const { value: generationResponse, responseHeaders } = await postJsonToApi({ + url: this.getLumaGenerationsUrl(), + headers: fullHeaders, + body: { + prompt, + ...(aspectRatio ? { aspect_ratio: aspectRatio } : {}), + model: this.modelId, + ...(providerOptions.luma ?? {}), + }, + abortSignal, + fetch: this.config.fetch, + failedResponseHandler: this.createLumaErrorHandler(), + successfulResponseHandler: createJsonResponseHandler( + lumaGenerationResponseSchema, + ), + }); + + const imageUrl = await this.pollForImageUrl( + generationResponse.id, + fullHeaders, + abortSignal, + ); + + const downloadedImage = await this.downloadImage(imageUrl, abortSignal); + + return { + images: [downloadedImage], + warnings, + response: { + modelId: this.modelId, + timestamp: currentDate, + headers: responseHeaders, + }, + }; + } + + private async pollForImageUrl( + generationId: string, + headers: Record, + abortSignal: AbortSignal | undefined, + ): Promise { + let attemptCount = 0; + const url = this.getLumaGenerationsUrl(generationId); + for (let i = 0; i < this.maxPollAttempts; i++) { + const { value: statusResponse } = await getFromApi({ + url, + headers, + abortSignal, + fetch: this.config.fetch, + failedResponseHandler: this.createLumaErrorHandler(), + successfulResponseHandler: createJsonResponseHandler( + lumaGenerationResponseSchema, + ), + }); + + switch (statusResponse.state) { + case 'completed': + if (!statusResponse.assets?.image) { + throw new InvalidResponseDataError({ + data: statusResponse, + message: `Image generation completed but no image was found.`, + }); + } + return statusResponse.assets.image; + case 'failed': + throw new InvalidResponseDataError({ + data: statusResponse, + message: `Image generation failed.`, + }); + } + await delay(this.pollIntervalMillis); + } + + throw new Error( + `Image generation timed out after ${this.maxPollAttempts} attempts.`, + ); + } + + private createLumaErrorHandler() { + return createJsonErrorResponseHandler({ + errorSchema: lumaErrorSchema, + errorToMessage: (error: LumaErrorData) => + error.detail[0].msg ?? 'Unknown error', + }); + } + + private getLumaGenerationsUrl(generationId?: string) { + return `${this.config.baseURL}/dream-machine/v1/generations/${ + generationId ?? 'image' + }`; + } + + private async downloadImage( + url: string, + abortSignal: AbortSignal | undefined, + ): Promise { + const { value: response } = await getFromApi({ + url, + // No specific headers should be needed for this request as it's a + // generated image provided by Luma. + abortSignal, + failedResponseHandler: createStatusCodeErrorResponseHandler(), + successfulResponseHandler: createBinaryResponseHandler(), + fetch: this.config.fetch, + }); + return response; + } +} + +// limited version of the schema, focussed on what is needed for the implementation +// this approach limits breakages when the API changes and increases efficiency +const lumaGenerationResponseSchema = z.object({ + id: z.string(), + state: z.enum(['queued', 'dreaming', 'completed', 'failed']), + failure_reason: z.string().nullish(), + assets: z + .object({ + image: z.string(), // URL of the generated image + }) + .nullish(), +}); + +const lumaErrorSchema = z.object({ + detail: z.array( + z.object({ + type: z.string(), + loc: z.array(z.string()), + msg: z.string(), + input: z.string(), + ctx: z + .object({ + expected: z.string(), + }) + .nullish(), + }), + ), +}); + +export type LumaErrorData = z.infer; diff --git a/packages/luma/src/luma-image-settings.ts b/packages/luma/src/luma-image-settings.ts new file mode 100644 index 000000000000..6231504cf942 --- /dev/null +++ b/packages/luma/src/luma-image-settings.ts @@ -0,0 +1,30 @@ +// https://luma.ai/models?type=image +export type LumaImageModelId = 'photon-1' | 'photon-flash-1' | (string & {}); + +/** +Configuration settings for Luma image generation. + +Since the Luma API processes images through an asynchronous queue system, these +settings allow you to tune the polling behavior when waiting for image +generation to complete. + */ +export interface LumaImageSettings { + /** +Override the maximum number of images per call (default 1) + */ + maxImagesPerCall?: number; + + /** +Override the polling interval in milliseconds (default 500). This controls how +frequently the API is checked for completed images while they are being +processed in Luma's queue. + */ + pollIntervalMillis?: number; + + /** +Override the maximum number of polling attempts (default 120). Since image +generation is queued and processed asynchronously, this limits how long to wait +for results before timing out. + */ + maxPollAttempts?: number; +} diff --git a/packages/luma/src/luma-provider.test.ts b/packages/luma/src/luma-provider.test.ts new file mode 100644 index 000000000000..a804e4c26ec6 --- /dev/null +++ b/packages/luma/src/luma-provider.test.ts @@ -0,0 +1,77 @@ +import { describe, it, expect, beforeEach, vi } from 'vitest'; +import { createLuma } from './luma-provider'; +import { LumaImageModel } from './luma-image-model'; + +vi.mock('./luma-image-model', () => ({ + LumaImageModel: vi.fn(), +})); + +describe('createLuma', () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + describe('image', () => { + it('should construct an image model with default configuration', () => { + const provider = createLuma(); + const modelId = 'luma-v1'; + + const model = provider.image(modelId); + + expect(model).toBeInstanceOf(LumaImageModel); + expect(LumaImageModel).toHaveBeenCalledWith( + modelId, + {}, + expect.objectContaining({ + provider: 'luma.image', + baseURL: 'https://api.lumalabs.ai', + }), + ); + }); + + it('should construct an image model with custom settings', () => { + const provider = createLuma(); + const modelId = 'luma-v1'; + const settings = { maxImagesPerCall: 2 }; + + const model = provider.image(modelId, settings); + + expect(model).toBeInstanceOf(LumaImageModel); + expect(LumaImageModel).toHaveBeenCalledWith( + modelId, + settings, + expect.objectContaining({ + provider: 'luma.image', + baseURL: 'https://api.lumalabs.ai', + }), + ); + }); + + it('should respect custom configuration options', () => { + const customBaseURL = 'https://custom-api.lumalabs.ai'; + const customHeaders = { 'X-Custom-Header': 'value' }; + const mockFetch = vi.fn(); + + const provider = createLuma({ + apiKey: 'custom-api-key', + baseURL: customBaseURL, + headers: customHeaders, + fetch: mockFetch, + }); + const modelId = 'luma-v1'; + + provider.image(modelId); + + expect(LumaImageModel).toHaveBeenCalledWith( + modelId, + {}, + expect.objectContaining({ + baseURL: customBaseURL, + headers: expect.any(Function), + fetch: mockFetch, + provider: 'luma.image', + }), + ); + }); + }); +}); diff --git a/packages/luma/src/luma-provider.ts b/packages/luma/src/luma-provider.ts new file mode 100644 index 000000000000..f8e179749e19 --- /dev/null +++ b/packages/luma/src/luma-provider.ts @@ -0,0 +1,70 @@ +import { ImageModelV1 } from '@ai-sdk/provider'; +import { + FetchFunction, + loadApiKey, + withoutTrailingSlash, +} from '@ai-sdk/provider-utils'; +import { LumaImageModel } from './luma-image-model'; +import { LumaImageModelId, LumaImageSettings } from './luma-image-settings'; + +export interface LumaProviderSettings { + /** +Luma API key. Default value is taken from the `LUMA_API_KEY` environment +variable. + */ + apiKey?: string; + /** +Base URL for the API calls. + */ + baseURL?: string; + /** +Custom headers to include in the requests. + */ + headers?: Record; + /** +Custom fetch implementation. You can use it as a middleware to intercept requests, +or to provide a custom fetch implementation for e.g. testing. + */ + fetch?: FetchFunction; +} + +export interface LumaProvider { + /** +Creates a model for image generation. + */ + image(modelId: LumaImageModelId, settings?: LumaImageSettings): ImageModelV1; +} + +const defaultBaseURL = 'https://api.lumalabs.ai'; + +export function createLuma(options: LumaProviderSettings = {}): LumaProvider { + const baseURL = withoutTrailingSlash(options.baseURL ?? defaultBaseURL); + const getHeaders = () => ({ + Authorization: `Bearer ${loadApiKey({ + apiKey: options.apiKey, + environmentVariableName: 'LUMA_API_KEY', + description: 'Luma', + })}`, + ...options.headers, + }); + + const createImageModel = ( + modelId: LumaImageModelId, + settings: LumaImageSettings = {}, + ) => + new LumaImageModel(modelId, settings, { + provider: 'luma.image', + baseURL: baseURL ?? defaultBaseURL, + headers: getHeaders, + fetch: options.fetch, + }); + + const provider = (modelId: LumaImageModelId, settings?: LumaImageSettings) => + createImageModel(modelId, settings); + + provider.image = createImageModel; + + return provider as LumaProvider; +} + +export const luma = createLuma(); diff --git a/packages/luma/tsconfig.json b/packages/luma/tsconfig.json new file mode 100644 index 000000000000..8eee8f9f6a82 --- /dev/null +++ b/packages/luma/tsconfig.json @@ -0,0 +1,5 @@ +{ + "extends": "./node_modules/@vercel/ai-tsconfig/ts-library.json", + "include": ["."], + "exclude": ["*/dist", "dist", "build", "node_modules"] +} diff --git a/packages/luma/tsup.config.ts b/packages/luma/tsup.config.ts new file mode 100644 index 000000000000..3f92041b987c --- /dev/null +++ b/packages/luma/tsup.config.ts @@ -0,0 +1,10 @@ +import { defineConfig } from 'tsup'; + +export default defineConfig([ + { + entry: ['src/index.ts'], + format: ['cjs', 'esm'], + dts: true, + sourcemap: true, + }, +]); diff --git a/packages/luma/turbo.json b/packages/luma/turbo.json new file mode 100644 index 000000000000..620b8380e744 --- /dev/null +++ b/packages/luma/turbo.json @@ -0,0 +1,12 @@ +{ + "extends": [ + "//" + ], + "tasks": { + "build": { + "outputs": [ + "**/dist/**" + ] + } + } +} diff --git a/packages/luma/vitest.edge.config.js b/packages/luma/vitest.edge.config.js new file mode 100644 index 000000000000..700660e913f5 --- /dev/null +++ b/packages/luma/vitest.edge.config.js @@ -0,0 +1,10 @@ +import { defineConfig } from 'vite'; + +// https://vitejs.dev/config/ +export default defineConfig({ + test: { + environment: 'edge-runtime', + globals: true, + include: ['**/*.test.ts', '**/*.test.tsx'], + }, +}); diff --git a/packages/luma/vitest.node.config.js b/packages/luma/vitest.node.config.js new file mode 100644 index 000000000000..b1d14b21fc11 --- /dev/null +++ b/packages/luma/vitest.node.config.js @@ -0,0 +1,10 @@ +import { defineConfig } from 'vite'; + +// https://vitejs.dev/config/ +export default defineConfig({ + test: { + environment: 'node', + globals: true, + include: ['**/*.test.ts', '**/*.test.tsx'], + }, +}); diff --git a/packages/provider-utils/src/get-from-api.test.ts b/packages/provider-utils/src/get-from-api.test.ts new file mode 100644 index 000000000000..3a90fe6437af --- /dev/null +++ b/packages/provider-utils/src/get-from-api.test.ts @@ -0,0 +1,181 @@ +import { APICallError } from '@ai-sdk/provider'; +import { describe, it, expect, vi } from 'vitest'; +import { getFromApi } from './get-from-api'; +import { + createJsonResponseHandler, + createStatusCodeErrorResponseHandler, +} from './response-handler'; +import { z } from 'zod'; + +describe('getFromApi', () => { + const mockSuccessResponse = { + name: 'test', + value: 123, + }; + + const mockResponseSchema = z.object({ + name: z.string(), + value: z.number(), + }); + + const mockHeaders = { + 'Content-Type': 'application/json', + Authorization: 'Bearer test', + }; + + it('should successfully fetch and parse data', async () => { + const mockFetch = vi.fn().mockResolvedValue( + new Response(JSON.stringify(mockSuccessResponse), { + status: 200, + headers: mockHeaders, + }), + ); + + const result = await getFromApi({ + url: 'https://api.test.com/data', + headers: { Authorization: 'Bearer test' }, + successfulResponseHandler: createJsonResponseHandler(mockResponseSchema), + failedResponseHandler: createStatusCodeErrorResponseHandler(), + fetch: mockFetch, + }); + + expect(result.value).toEqual(mockSuccessResponse); + expect(mockFetch).toHaveBeenCalledWith( + 'https://api.test.com/data', + expect.objectContaining({ + method: 'GET', + headers: { Authorization: 'Bearer test' }, + }), + ); + }); + + it('should handle API errors', async () => { + const errorResponse = { error: 'Not Found' }; + const mockFetch = vi.fn().mockResolvedValue( + new Response(JSON.stringify(errorResponse), { + status: 404, + statusText: 'Not Found', + headers: mockHeaders, + }), + ); + + await expect( + getFromApi({ + url: 'https://api.test.com/data', + successfulResponseHandler: + createJsonResponseHandler(mockResponseSchema), + failedResponseHandler: createStatusCodeErrorResponseHandler(), + fetch: mockFetch, + }), + ).rejects.toThrow(APICallError); + }); + + it('should handle network errors', async () => { + const mockFetch = vi.fn().mockRejectedValue( + Object.assign(new TypeError('fetch failed'), { + cause: new Error('Failed to connect'), + }), + ); + + await expect( + getFromApi({ + url: 'https://api.test.com/data', + successfulResponseHandler: + createJsonResponseHandler(mockResponseSchema), + failedResponseHandler: createStatusCodeErrorResponseHandler(), + fetch: mockFetch, + }), + ).rejects.toThrow('Cannot connect to API: Failed to connect'); + }); + + it('should handle abort signals', async () => { + const abortController = new AbortController(); + const mockFetch = vi.fn().mockImplementation(() => { + abortController.abort(); + return Promise.reject(new DOMException('Aborted', 'AbortError')); + }); + + await expect( + getFromApi({ + url: 'https://api.test.com/data', + successfulResponseHandler: + createJsonResponseHandler(mockResponseSchema), + failedResponseHandler: createStatusCodeErrorResponseHandler(), + fetch: mockFetch, + abortSignal: abortController.signal, + }), + ).rejects.toThrow('Aborted'); + }); + + it('should remove undefined header entries', async () => { + const mockFetch = vi.fn().mockResolvedValue( + new Response(JSON.stringify(mockSuccessResponse), { + status: 200, + headers: mockHeaders, + }), + ); + + await getFromApi({ + url: 'https://api.test.com/data', + headers: { + Authorization: 'Bearer test', + 'X-Custom-Header': undefined, + }, + successfulResponseHandler: createJsonResponseHandler(mockResponseSchema), + failedResponseHandler: createStatusCodeErrorResponseHandler(), + fetch: mockFetch, + }); + + expect(mockFetch).toHaveBeenCalledWith( + 'https://api.test.com/data', + expect.objectContaining({ + headers: { + Authorization: 'Bearer test', + }, + }), + ); + }); + + it('should handle errors in response handlers', async () => { + const mockFetch = vi.fn().mockResolvedValue( + new Response('invalid json', { + status: 200, + headers: mockHeaders, + }), + ); + + await expect( + getFromApi({ + url: 'https://api.test.com/data', + successfulResponseHandler: + createJsonResponseHandler(mockResponseSchema), + failedResponseHandler: createStatusCodeErrorResponseHandler(), + fetch: mockFetch, + }), + ).rejects.toThrow(APICallError); + }); + + it('should use default fetch when not provided', async () => { + const originalFetch = global.fetch; + const mockFetch = vi.fn().mockResolvedValue( + new Response(JSON.stringify(mockSuccessResponse), { + status: 200, + headers: mockHeaders, + }), + ); + global.fetch = mockFetch; + + try { + await getFromApi({ + url: 'https://api.test.com/data', + successfulResponseHandler: + createJsonResponseHandler(mockResponseSchema), + failedResponseHandler: createStatusCodeErrorResponseHandler(), + }); + + expect(mockFetch).toHaveBeenCalled(); + } finally { + global.fetch = originalFetch; + } + }); +}); diff --git a/packages/provider-utils/src/get-from-api.ts b/packages/provider-utils/src/get-from-api.ts new file mode 100644 index 000000000000..58b10c9dde2e --- /dev/null +++ b/packages/provider-utils/src/get-from-api.ts @@ -0,0 +1,107 @@ +import { APICallError } from '@ai-sdk/provider'; +import { FetchFunction } from './fetch-function'; +import { removeUndefinedEntries } from './remove-undefined-entries'; +import { ResponseHandler } from './response-handler'; +import { isAbortError } from './is-abort-error'; +import { extractResponseHeaders } from './extract-response-headers'; + +// use function to allow for mocking in tests: +const getOriginalFetch = () => globalThis.fetch; + +export const getFromApi = async ({ + url, + headers = {}, + successfulResponseHandler, + failedResponseHandler, + abortSignal, + fetch = getOriginalFetch(), +}: { + url: string; + headers?: Record; + failedResponseHandler: ResponseHandler; + successfulResponseHandler: ResponseHandler; + abortSignal?: AbortSignal; + fetch?: FetchFunction; +}) => { + try { + const response = await fetch(url, { + method: 'GET', + headers: removeUndefinedEntries(headers), + signal: abortSignal, + }); + + const responseHeaders = extractResponseHeaders(response); + + if (!response.ok) { + let errorInformation: { + value: Error; + responseHeaders?: Record | undefined; + }; + + try { + errorInformation = await failedResponseHandler({ + response, + url, + requestBodyValues: {}, + }); + } catch (error) { + if (isAbortError(error) || APICallError.isInstance(error)) { + throw error; + } + + throw new APICallError({ + message: 'Failed to process error response', + cause: error, + statusCode: response.status, + url, + responseHeaders, + requestBodyValues: {}, + }); + } + + throw errorInformation.value; + } + + try { + return await successfulResponseHandler({ + response, + url, + requestBodyValues: {}, + }); + } catch (error) { + if (error instanceof Error) { + if (isAbortError(error) || APICallError.isInstance(error)) { + throw error; + } + } + + throw new APICallError({ + message: 'Failed to process successful response', + cause: error, + statusCode: response.status, + url, + responseHeaders, + requestBodyValues: {}, + }); + } + } catch (error) { + if (isAbortError(error)) { + throw error; + } + + if (error instanceof TypeError && error.message === 'fetch failed') { + const cause = (error as any).cause; + if (cause != null) { + throw new APICallError({ + message: `Cannot connect to API: ${cause.message}`, + cause, + url, + isRetryable: true, + requestBodyValues: {}, + }); + } + } + + throw error; + } +}; diff --git a/packages/provider-utils/src/index.ts b/packages/provider-utils/src/index.ts index 9b5fc3ab0d0c..f89ad5721daa 100644 --- a/packages/provider-utils/src/index.ts +++ b/packages/provider-utils/src/index.ts @@ -4,6 +4,7 @@ export * from './extract-response-headers'; export * from './fetch-function'; export { createIdGenerator, generateId } from './generate-id'; export * from './get-error-message'; +export * from './get-from-api'; export * from './is-abort-error'; export * from './load-api-key'; export { loadOptionalSetting } from './load-optional-setting'; diff --git a/packages/provider-utils/src/response-handler.test.ts b/packages/provider-utils/src/response-handler.test.ts index bb1e7a0b6175..8e54f4864268 100644 --- a/packages/provider-utils/src/response-handler.test.ts +++ b/packages/provider-utils/src/response-handler.test.ts @@ -6,6 +6,8 @@ import { import { createJsonResponseHandler, createJsonStreamResponseHandler, + createBinaryResponseHandler, + createStatusCodeErrorResponseHandler, } from './response-handler'; describe('createJsonStreamResponseHandler', () => { @@ -82,3 +84,55 @@ describe('createJsonResponseHandler', () => { expect(result.rawValue).toEqual(rawData); }); }); + +describe('createBinaryResponseHandler', () => { + it('should handle binary response successfully', async () => { + const binaryData = new Uint8Array([1, 2, 3, 4]); + const response = new Response(binaryData); + const handler = createBinaryResponseHandler(); + + const result = await handler({ + url: 'test-url', + requestBodyValues: {}, + response, + }); + + expect(result.value).toBeInstanceOf(Uint8Array); + expect(result.value).toEqual(binaryData); + }); + + it('should throw APICallError when response body is null', async () => { + const response = new Response(null); + const handler = createBinaryResponseHandler(); + + await expect( + handler({ + url: 'test-url', + requestBodyValues: {}, + response, + }), + ).rejects.toThrow('Response body is empty'); + }); +}); + +describe('createStatusCodeErrorResponseHandler', () => { + it('should create error with status text and response body', async () => { + const response = new Response('Error message', { + status: 404, + statusText: 'Not Found', + }); + const handler = createStatusCodeErrorResponseHandler(); + + const result = await handler({ + url: 'test-url', + requestBodyValues: { some: 'data' }, + response, + }); + + expect(result.value.message).toBe('Not Found'); + expect(result.value.statusCode).toBe(404); + expect(result.value.responseBody).toBe('Error message'); + expect(result.value.url).toBe('test-url'); + expect(result.value.requestBodyValues).toEqual({ some: 'data' }); + }); +}); diff --git a/packages/provider-utils/src/response-handler.ts b/packages/provider-utils/src/response-handler.ts index a1e95aff1094..8e48b93b755d 100644 --- a/packages/provider-utils/src/response-handler.ts +++ b/packages/provider-utils/src/response-handler.ts @@ -184,3 +184,57 @@ export const createJsonResponseHandler = rawValue: parsedResult.rawValue, }; }; + +export const createBinaryResponseHandler = + (): ResponseHandler => + async ({ response, url, requestBodyValues }) => { + const responseHeaders = extractResponseHeaders(response); + + if (!response.body) { + throw new APICallError({ + message: 'Response body is empty', + url, + requestBodyValues, + statusCode: response.status, + responseHeaders, + responseBody: undefined, + }); + } + + try { + const buffer = await response.arrayBuffer(); + return { + responseHeaders, + value: new Uint8Array(buffer), + }; + } catch (error) { + throw new APICallError({ + message: 'Failed to read response as array buffer', + url, + requestBodyValues, + statusCode: response.status, + responseHeaders, + responseBody: undefined, + cause: error, + }); + } + }; + +export const createStatusCodeErrorResponseHandler = + (): ResponseHandler => + async ({ response, url, requestBodyValues }) => { + const responseHeaders = extractResponseHeaders(response); + const responseBody = await response.text(); + + return { + responseHeaders, + value: new APICallError({ + message: response.statusText, + url, + requestBodyValues: requestBodyValues as Record, + statusCode: response.status, + responseHeaders, + responseBody, + }), + }; + }; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b5f3f8fbc35a..49c102f51819 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -89,6 +89,9 @@ importers: '@ai-sdk/groq': specifier: 1.1.2 version: link:../../packages/groq + '@ai-sdk/luma': + specifier: 0.0.0 + version: link:../../packages/luma '@ai-sdk/mistral': specifier: 1.1.2 version: link:../../packages/mistral @@ -1502,6 +1505,31 @@ importers: specifier: 3.23.8 version: 3.23.8 + packages/luma: + dependencies: + '@ai-sdk/provider': + specifier: 1.0.6 + version: link:../provider + '@ai-sdk/provider-utils': + specifier: 2.1.2 + version: link:../provider-utils + devDependencies: + '@types/node': + specifier: ^18 + version: 18.19.54 + '@vercel/ai-tsconfig': + specifier: workspace:* + version: link:../../tools/tsconfig + tsup: + specifier: ^8 + version: 8.3.0(jiti@2.4.0)(postcss@8.4.49)(tsx@4.19.2)(typescript@5.6.3)(yaml@2.5.0) + typescript: + specifier: 5.6.3 + version: 5.6.3 + zod: + specifier: 3.23.8 + version: 3.23.8 + packages/mistral: dependencies: '@ai-sdk/provider': diff --git a/turbo.json b/turbo.json index 5da362a585fe..4f524c6efcce 100644 --- a/turbo.json +++ b/turbo.json @@ -25,6 +25,7 @@ "GOOGLE_VERTEX_LOCATION", "GOOGLE_VERTEX_PROJECT", "GROQ_API_KEY", + "LUMA_API_KEY", "MISTRAL_API_KEY", "NEXT_RUNTIME", "NODE_ENV",