From 4cb6acf79889612817081cef8fed16aa7b70a139 Mon Sep 17 00:00:00 2001 From: juliusmarminge Date: Thu, 25 Jan 2024 16:58:43 +0100 Subject: [PATCH 01/14] init --- package.json | 7 ++- pnpm-lock.yaml | 29 ++++++++++++ src/drivers/uploadthing.ts | 90 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 125 insertions(+), 1 deletion(-) create mode 100644 src/drivers/uploadthing.ts diff --git a/package.json b/package.json index a47dd1b5..69b97cdd 100644 --- a/package.json +++ b/package.json @@ -92,6 +92,7 @@ "types-cloudflare-worker": "^1.2.0", "typescript": "^5.3.3", "unbuild": "^2.0.0", + "uploadthing": "^6.3.0", "vite": "^5.0.11", "vitest": "^1.2.1", "vue": "^3.4.14" @@ -108,7 +109,8 @@ "@planetscale/database": "^1.13.0", "@upstash/redis": "^1.28.1", "@vercel/kv": "^0.2.4", - "idb-keyval": "^6.2.1" + "idb-keyval": "^6.2.1", + "uploadthing": "^6.0.0" }, "peerDependenciesMeta": { "@azure/app-configuration": { @@ -146,6 +148,9 @@ }, "idb-keyval": { "optional": true + }, + "uploadthing": { + "optional": true } }, "packageManager": "pnpm@8.14.1" diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 17b7920a..0cc00e5f 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -148,6 +148,9 @@ devDependencies: unbuild: specifier: ^2.0.0 version: 2.0.0(typescript@5.3.3) + uploadthing: + specifier: ^6.3.0 + version: 6.3.0 vite: specifier: ^5.0.11 version: 5.0.11(@types/node@20.11.5) @@ -1743,6 +1746,22 @@ packages: resolution: {integrity: sha512-zuVdFrMJiuCDQUMCzQaD6KL28MjnqqN8XnAqiEq9PNm/hCPTSGfrXCOfwj1ow4LFb/tNymJPwsNbVePc1xFqrQ==} dev: true + /@uploadthing/mime-types@0.2.2: + resolution: {integrity: sha512-ZUo1JHOPPMZDsUw1mOhhVDIvJGlsjj6T0xJ/YJtulyJwL43S9B5pxg1cHcRuTEgjaxj7B55jiqQ6r9mDrrjH9A==} + dev: true + + /@uploadthing/shared@6.2.0(@uploadthing/mime-types@0.2.2): + resolution: {integrity: sha512-GMC1gwZa9X48RZwuY7gGse7E8TXYMAMUNaYWEQL2ctVhMZv7viCPyMyZ0yMqBsQmOeGfbmycPnUeNE9IuesJGg==} + peerDependencies: + '@uploadthing/mime-types': ^0.2.2 + peerDependenciesMeta: + '@uploadthing/mime-types': + optional: true + dependencies: + '@uploadthing/mime-types': 0.2.2 + std-env: 3.7.0 + dev: true + /@upstash/redis@1.24.3: resolution: {integrity: sha512-gw6d4IA1biB4eye5ESaXc0zOlVQI94aptsBvVcTghYWu1kRmOrJFoMFEDCa8p5uzluyYAOFCuY2GWLR6O4ZoIw==} dependencies: @@ -7420,6 +7439,16 @@ packages: picocolors: 1.0.0 dev: true + /uploadthing@6.3.0: + resolution: {integrity: sha512-Herfy6a59jFv+5pAKuyUHZrOGcDrtfo7jqKOOhfd80/5vae8LCESVauJj37ZIoUVaPQzNs6sqvF3g86O4+JNFg==} + engines: {node: '>=18.13.0'} + dependencies: + '@uploadthing/mime-types': 0.2.2 + '@uploadthing/shared': 6.2.0(@uploadthing/mime-types@0.2.2) + consola: 3.2.3 + std-env: 3.7.0 + dev: true + /uqr@0.1.2: resolution: {integrity: sha512-MJu7ypHq6QasgF5YRTjqscSzQp/W11zoUk6kvmlH+fmWEs63Y0Eib13hYFwAzagRJcVY8WVnlV+eBDUGMJ5IbA==} dev: false diff --git a/src/drivers/uploadthing.ts b/src/drivers/uploadthing.ts new file mode 100644 index 00000000..762ebbd2 --- /dev/null +++ b/src/drivers/uploadthing.ts @@ -0,0 +1,90 @@ +import { defineDriver } from "./utils"; +import { $fetch, ofetch, $Fetch } from "ofetch"; +import { UTApi } from "uploadthing/server"; + +export interface UploadThingOptions { + apiKey: string; +} + +export default defineDriver((opts) => { + let client: UTApi; + let utFetch: $Fetch; + + const getClient = () => { + return (client ??= new UTApi({ + apiKey: opts.apiKey, + fetch: $fetch.native, + })); + }; + + const getUTFetch = () => { + return (utFetch ??= ofetch.create({ + baseURL: "https://uploadthing.com/api", + headers: { + "x-uploadthing-api-key": opts.apiKey, + }, + })); + }; + + return { + hasItem(key) { + // This is the best endpoint we got currently... + return getUTFetch()("/getFileUrl", { + body: { fileKeys: [key] }, + }).then((res) => res.ok); + }, + getItem(key) { + return ofetch(`https://utfs.io/f/${key}`); + }, + getItemRaw(key) { + return ofetch + .native(`https://utfs.io/f/${key}`) + .then((res) => res.arrayBuffer()); + }, + getItems(items) { + return Promise.all( + items.map((item) => + ofetch(`https://utfs.io/f/${item.key}`).then((res) => ({ + key: item.key, + value: res, + })) + ) + ); + }, + getKeys() { + return getClient() + .listFiles({}) + .then((res) => res.map((file) => file.key)); + }, + setItem(key, value, opts) { + return getClient() + .uploadFiles(new Blob([value]), { + metadata: opts.metadata, + }) + .then(() => {}); + }, + setItems(items, opts) { + return getClient() + .uploadFiles( + items.map((item) => new Blob([item.value])), + { + metadata: opts?.metadata, + } + ) + .then(() => {}); + }, + removeItem(key, opts) { + return getClient() + .deleteFiles([key]) + .then(() => {}); + }, + async clear() { + const client = getClient(); + const keys = await client.listFiles({}).then((r) => r.map((f) => f.key)); + return client.deleteFiles(keys).then(() => {}); + }, + // getMeta(key, opts) { + // // TODO: We don't currently have an endpoint to fetch metadata, but it does exist + // }, + }; +}); From cac6a066bc2afea1c3336321155339b1285612a9 Mon Sep 17 00:00:00 2001 From: juliusmarminge Date: Thu, 25 Jan 2024 17:04:33 +0100 Subject: [PATCH 02/14] nit --- src/drivers/uploadthing.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/drivers/uploadthing.ts b/src/drivers/uploadthing.ts index 762ebbd2..d3fd2740 100644 --- a/src/drivers/uploadthing.ts +++ b/src/drivers/uploadthing.ts @@ -1,5 +1,5 @@ import { defineDriver } from "./utils"; -import { $fetch, ofetch, $Fetch } from "ofetch"; +import { ofetch, $Fetch } from "ofetch"; import { UTApi } from "uploadthing/server"; export interface UploadThingOptions { @@ -13,7 +13,7 @@ export default defineDriver((opts) => { const getClient = () => { return (client ??= new UTApi({ apiKey: opts.apiKey, - fetch: $fetch.native, + fetch: ofetch.native, })); }; From f9a5a15216f55ae732e21957793540350457f39e Mon Sep 17 00:00:00 2001 From: juliusmarminge Date: Thu, 25 Jan 2024 17:54:41 +0100 Subject: [PATCH 03/14] rm getItems --- src/drivers/uploadthing.ts | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/src/drivers/uploadthing.ts b/src/drivers/uploadthing.ts index d3fd2740..64631d9e 100644 --- a/src/drivers/uploadthing.ts +++ b/src/drivers/uploadthing.ts @@ -4,6 +4,16 @@ import { UTApi } from "uploadthing/server"; export interface UploadThingOptions { apiKey: string; + /** + * Primarily used for testing + * @default "https://uploadthing.com/api" + */ + uploadthignApiUrl?: string; + /** + * Primarily used for testing + * @default "https://utfs.io/f" + */ + uploadthingFileUrl?: string; } export default defineDriver((opts) => { @@ -19,7 +29,7 @@ export default defineDriver((opts) => { const getUTFetch = () => { return (utFetch ??= ofetch.create({ - baseURL: "https://uploadthing.com/api", + baseURL: opts.uploadthignApiUrl ?? "https://uploadthing.com/api", headers: { "x-uploadthing-api-key": opts.apiKey, }, @@ -34,23 +44,15 @@ export default defineDriver((opts) => { }).then((res) => res.ok); }, getItem(key) { - return ofetch(`https://utfs.io/f/${key}`); + return ofetch(`/${key}`, { + baseURL: opts.uploadthingFileUrl ?? "https://utfs.io/f", + }); }, getItemRaw(key) { return ofetch .native(`https://utfs.io/f/${key}`) .then((res) => res.arrayBuffer()); }, - getItems(items) { - return Promise.all( - items.map((item) => - ofetch(`https://utfs.io/f/${item.key}`).then((res) => ({ - key: item.key, - value: res, - })) - ) - ); - }, getKeys() { return getClient() .listFiles({}) From 8d46dab0809afc719f2dc51fd845b93ad5bebfbb Mon Sep 17 00:00:00 2001 From: juliusmarminge Date: Thu, 25 Jan 2024 22:35:36 +0100 Subject: [PATCH 04/14] track internal keys --- src/drivers/uploadthing.ts | 102 +++++++++++++++++++------------ test/drivers/uploadthing.test.ts | 81 ++++++++++++++++++++++++ 2 files changed, 144 insertions(+), 39 deletions(-) create mode 100644 test/drivers/uploadthing.test.ts diff --git a/src/drivers/uploadthing.ts b/src/drivers/uploadthing.ts index 64631d9e..2f22ae7f 100644 --- a/src/drivers/uploadthing.ts +++ b/src/drivers/uploadthing.ts @@ -4,87 +4,111 @@ import { UTApi } from "uploadthing/server"; export interface UploadThingOptions { apiKey: string; - /** - * Primarily used for testing - * @default "https://uploadthing.com/api" - */ - uploadthignApiUrl?: string; - /** - * Primarily used for testing - * @default "https://utfs.io/f" - */ - uploadthingFileUrl?: string; } export default defineDriver((opts) => { let client: UTApi; - let utFetch: $Fetch; + let utApiFetch: $Fetch; + let utFsFetch: $Fetch; + + const internalToUTKeyMap = new Map(); + const fromUTKey = (utKey: string) => { + for (const [key, value] of internalToUTKeyMap.entries()) { + if (value === utKey) { + return key; + } + } + }; const getClient = () => { return (client ??= new UTApi({ apiKey: opts.apiKey, fetch: ofetch.native, + logLevel: "debug", })); }; - const getUTFetch = () => { - return (utFetch ??= ofetch.create({ - baseURL: opts.uploadthignApiUrl ?? "https://uploadthing.com/api", + // The UTApi doesn't have all methods we need right now, so use raw fetch + const getUTApiFetch = () => { + return (utApiFetch ??= ofetch.create({ + method: "POST", + baseURL: "https://uploadthing.com/api", headers: { "x-uploadthing-api-key": opts.apiKey, }, })); }; + const getUTFsFetch = () => { + return (utFsFetch ??= ofetch.create({ + baseURL: "https://utfs.io/f", + })); + }; + return { hasItem(key) { + const utkey = internalToUTKeyMap.get(key); + if (!utkey) return false; // This is the best endpoint we got currently... - return getUTFetch()("/getFileUrl", { - body: { fileKeys: [key] }, - }).then((res) => res.ok); - }, - getItem(key) { - return ofetch(`/${key}`, { - baseURL: opts.uploadthingFileUrl ?? "https://utfs.io/f", + return getUTApiFetch()("/getFileUrl", { + body: { fileKeys: [utkey] }, + }).then((res) => { + return !!res?.data?.length; }); }, - getItemRaw(key) { - return ofetch - .native(`https://utfs.io/f/${key}`) - .then((res) => res.arrayBuffer()); + getItem(key) { + const utkey = internalToUTKeyMap.get(key); + if (!utkey) return null; + return getUTFsFetch()(`/${utkey}`).then((r) => r.text()); }, getKeys() { return getClient() .listFiles({}) - .then((res) => res.map((file) => file.key)); + .then((res) => res.map((file) => fromUTKey(file.key) ?? file.key)); }, setItem(key, value, opts) { return getClient() - .uploadFiles(new Blob([value]), { - metadata: opts.metadata, + .uploadFiles(Object.assign(new Blob([value]), { name: key }), { + metadata: opts?.metadata, }) - .then(() => {}); + .then((response) => { + if (response.error) throw response.error; + internalToUTKeyMap.set(key, response.data.key); + }); }, setItems(items, opts) { return getClient() .uploadFiles( - items.map((item) => new Blob([item.value])), - { - metadata: opts?.metadata, - } + items.map((item) => + Object.assign(new Blob([item.value]), { name: item.key }) + ), + { metadata: opts?.metadata } ) - .then(() => {}); + .then((responses) => { + responses.map((response) => { + if (response.error) throw response.error; + internalToUTKeyMap.set(response.data.name, response.data.key); + }); + }); }, removeItem(key, opts) { + const utkey = internalToUTKeyMap.get(key); + if (!utkey) throw new Error(`Unknown key: ${key}`); return getClient() - .deleteFiles([key]) - .then(() => {}); + .deleteFiles([utkey]) + .then(() => { + internalToUTKeyMap.delete(key); + }); }, async clear() { - const client = getClient(); - const keys = await client.listFiles({}).then((r) => r.map((f) => f.key)); - return client.deleteFiles(keys).then(() => {}); + const utkeys = Array.from(internalToUTKeyMap.values()); + return getClient() + .deleteFiles(utkeys) + .then(() => { + internalToUTKeyMap.clear(); + }); }, + // getMeta(key, opts) { // // TODO: We don't currently have an endpoint to fetch metadata, but it does exist // }, diff --git a/test/drivers/uploadthing.test.ts b/test/drivers/uploadthing.test.ts new file mode 100644 index 00000000..f1ca59dc --- /dev/null +++ b/test/drivers/uploadthing.test.ts @@ -0,0 +1,81 @@ +import { afterAll, beforeAll, describe, it } from "vitest"; +import driver from "../../src/drivers/uploadthing"; +import { testDriver } from "./utils"; +import { setupServer } from "msw/node"; +import { rest } from "msw"; + +const store: Record = {}; + +const utapiUrl = "https://uploadthing.com/api"; +const utfsUrl = "https://utfs.io/f"; + +const server = setupServer( + rest.post(`${utapiUrl}/getFileUrl`, async (req, res, ctx) => { + const { fileKeys } = await req.json(); + const key = fileKeys[0]; + if (!(key in store)) { + return res(ctx.status(401), ctx.json({ error: "Unauthorized" })); + } + return res( + ctx.status(200), + ctx.json({ + result: { + [key]: `https://utfs.io/f/${key}`, + }, + }) + ); + }), + rest.get(`${utfsUrl}/:key`, (req, res, ctx) => { + const key = req.params.key as string; + if (!(key in store)) { + return res(ctx.status(404), ctx.json(null)); + } + return res( + ctx.status(200), + ctx.set("content-type", "application/octet-stream"), + ctx.body(store[key]) + ); + }), + rest.post(`${utapiUrl}/uploadFiles`, async (req, res, ctx) => { + console.log("intercepted request"); + return res( + ctx.status(200), + ctx.json({ + data: [ + { + presignedUrls: [`https://my-s3-server.com/:key`], + }, + ], + }) + ); + }), + rest.post(`${utapiUrl}/deleteFile`, async (req, res, ctx) => { + console.log("hello????"); + const { fileKeys } = await req.json(); + for (const key of fileKeys) { + delete store[key]; + } + return res(ctx.status(200), ctx.json({ success: true })); + }) +); + +describe( + "drivers: uploadthing", + () => { + // beforeAll(() => { + // server.listen(); + // }); + // afterAll(() => { + // server.close(); + // }); + + testDriver({ + driver: driver({ + apiKey: + "sk_live_4603822c7c4574cc90495ff3b31204adf20311bc953903d8081be7a5176f31aa", + }), + async additionalTests(ctx) {}, + }); + }, + { timeout: 30e3 } +); From 6444a42326ee0ab873de159c7aaa75bee2f31299 Mon Sep 17 00:00:00 2001 From: juliusmarminge Date: Thu, 25 Jan 2024 23:04:49 +0100 Subject: [PATCH 05/14] fix --- src/drivers/uploadthing.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/drivers/uploadthing.ts b/src/drivers/uploadthing.ts index 2f22ae7f..0837438c 100644 --- a/src/drivers/uploadthing.ts +++ b/src/drivers/uploadthing.ts @@ -24,7 +24,6 @@ export default defineDriver((opts) => { return (client ??= new UTApi({ apiKey: opts.apiKey, fetch: ofetch.native, - logLevel: "debug", })); }; @@ -64,7 +63,9 @@ export default defineDriver((opts) => { getKeys() { return getClient() .listFiles({}) - .then((res) => res.map((file) => fromUTKey(file.key) ?? file.key)); + .then((res) => + res.map((file) => fromUTKey(file.key)).filter((k): k is string => !!k) + ); }, setItem(key, value, opts) { return getClient() From e98edabc52dc2f40046899e16b1dcb3b9fc09de7 Mon Sep 17 00:00:00 2001 From: juliusmarminge Date: Fri, 26 Jan 2024 00:00:43 +0100 Subject: [PATCH 06/14] remove my apikey --- test/drivers/uploadthing.test.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/drivers/uploadthing.test.ts b/test/drivers/uploadthing.test.ts index f1ca59dc..a54ab863 100644 --- a/test/drivers/uploadthing.test.ts +++ b/test/drivers/uploadthing.test.ts @@ -71,8 +71,7 @@ describe( testDriver({ driver: driver({ - apiKey: - "sk_live_4603822c7c4574cc90495ff3b31204adf20311bc953903d8081be7a5176f31aa", + apiKey: "sk_live_xxx", }), async additionalTests(ctx) {}, }); From 4d55f54aefc989042ae04d6a08305a608ec4004a Mon Sep 17 00:00:00 2001 From: juliusmarminge Date: Fri, 26 Jan 2024 00:06:12 +0100 Subject: [PATCH 07/14] stub out docs --- docs/content/2.drivers/uploadthing.md | 32 +++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 docs/content/2.drivers/uploadthing.md diff --git a/docs/content/2.drivers/uploadthing.md b/docs/content/2.drivers/uploadthing.md new file mode 100644 index 00000000..061355b7 --- /dev/null +++ b/docs/content/2.drivers/uploadthing.md @@ -0,0 +1,32 @@ +# UploadThing + +Store data using UploadThing. + +::note{to="https://uploadthing.com/"} +Learn more about UploadThing. +:: + +```js +import { createStorage } from "unstorage"; +import uploadthingDriver from "unstorage/drivers/uploadthing"; + +const storage = createStorage({ + driver: uploadthingDriver({ + // apiKey: "", + }), +}); +``` + +To use, you will need to install `uploadthing` dependency in your project: + +```json +{ + "dependencies": { + "uploadthing": "latest" + } +} +``` + +**Options:** + +- `apiKey`: Your UploadThing API key. Will be automatically inferred from the `UPLOADTHING_SECRET` environment variable if not provided. From 4c9a1885eb350c2ff5d4f204b2007f338e12c660 Mon Sep 17 00:00:00 2001 From: juliusmarminge Date: Tue, 30 Jan 2024 12:26:31 +0100 Subject: [PATCH 08/14] use custom ids --- package.json | 2 +- pnpm-lock.yaml | 8 ++-- src/drivers/uploadthing.ts | 98 ++++++++++++++++---------------------- src/storage.ts | 2 +- 4 files changed, 46 insertions(+), 64 deletions(-) diff --git a/package.json b/package.json index 69b97cdd..3005d7ec 100644 --- a/package.json +++ b/package.json @@ -92,7 +92,7 @@ "types-cloudflare-worker": "^1.2.0", "typescript": "^5.3.3", "unbuild": "^2.0.0", - "uploadthing": "^6.3.0", + "uploadthing": "6.3.2-canary.a35b49f", "vite": "^5.0.11", "vitest": "^1.2.1", "vue": "^3.4.14" diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 0cc00e5f..f98c61d9 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -149,8 +149,8 @@ devDependencies: specifier: ^2.0.0 version: 2.0.0(typescript@5.3.3) uploadthing: - specifier: ^6.3.0 - version: 6.3.0 + specifier: 6.3.2-canary.a35b49f + version: 6.3.2-canary.a35b49f vite: specifier: ^5.0.11 version: 5.0.11(@types/node@20.11.5) @@ -7439,8 +7439,8 @@ packages: picocolors: 1.0.0 dev: true - /uploadthing@6.3.0: - resolution: {integrity: sha512-Herfy6a59jFv+5pAKuyUHZrOGcDrtfo7jqKOOhfd80/5vae8LCESVauJj37ZIoUVaPQzNs6sqvF3g86O4+JNFg==} + /uploadthing@6.3.2-canary.a35b49f: + resolution: {integrity: sha512-62FAyXOsz9PGX4pzpCwXKmW0hgLrd/NgYyx5nfmKZXZMNT4ceA83PA/8MAZ0M2l3ySCbyJikbt4ZFnA+xbEZHw==} engines: {node: '>=18.13.0'} dependencies: '@uploadthing/mime-types': 0.2.2 diff --git a/src/drivers/uploadthing.ts b/src/drivers/uploadthing.ts index 0837438c..4b42412c 100644 --- a/src/drivers/uploadthing.ts +++ b/src/drivers/uploadthing.ts @@ -9,16 +9,6 @@ export interface UploadThingOptions { export default defineDriver((opts) => { let client: UTApi; let utApiFetch: $Fetch; - let utFsFetch: $Fetch; - - const internalToUTKeyMap = new Map(); - const fromUTKey = (utKey: string) => { - for (const [key, value] of internalToUTKeyMap.entries()) { - if (value === utKey) { - return key; - } - } - }; const getClient = () => { return (client ??= new UTApi({ @@ -38,76 +28,68 @@ export default defineDriver((opts) => { })); }; - const getUTFsFetch = () => { - return (utFsFetch ??= ofetch.create({ - baseURL: "https://utfs.io/f", - })); - }; + function getKeys() { + return getClient() + .listFiles({}) + .then((res) => + res.map((file) => file.customId).filter((k): k is string => !!k) + ); + } return { - hasItem(key) { - const utkey = internalToUTKeyMap.get(key); - if (!utkey) return false; - // This is the best endpoint we got currently... - return getUTApiFetch()("/getFileUrl", { - body: { fileKeys: [utkey] }, - }).then((res) => { - return !!res?.data?.length; - }); + hasItem(id) { + return getClient() + .getFileUrls(id, { keyType: "customId" }) + .then((res) => { + return !!res.length; + }); }, - getItem(key) { - const utkey = internalToUTKeyMap.get(key); - if (!utkey) return null; - return getUTFsFetch()(`/${utkey}`).then((r) => r.text()); + async getItem(id) { + const url = await getClient() + .getFileUrls(id, { keyType: "customId" }) + .then((res) => { + return res[0]?.url; + }); + if (!url) return null; + return ofetch(url).then((res) => res.text()); }, getKeys() { - return getClient() - .listFiles({}) - .then((res) => - res.map((file) => fromUTKey(file.key)).filter((k): k is string => !!k) - ); + return getKeys(); }, setItem(key, value, opts) { return getClient() - .uploadFiles(Object.assign(new Blob([value]), { name: key }), { - metadata: opts?.metadata, - }) - .then((response) => { - if (response.error) throw response.error; - internalToUTKeyMap.set(key, response.data.key); - }); + .uploadFiles( + Object.assign(new Blob([value]), { + name: key, + customId: key, + }), + { metadata: opts?.metadata } + ) + .then(() => {}); }, setItems(items, opts) { return getClient() .uploadFiles( items.map((item) => - Object.assign(new Blob([item.value]), { name: item.key }) + Object.assign(new Blob([item.value]), { + name: item.key, + customId: item.key, + }) ), { metadata: opts?.metadata } ) - .then((responses) => { - responses.map((response) => { - if (response.error) throw response.error; - internalToUTKeyMap.set(response.data.name, response.data.key); - }); - }); + .then(() => {}); }, removeItem(key, opts) { - const utkey = internalToUTKeyMap.get(key); - if (!utkey) throw new Error(`Unknown key: ${key}`); return getClient() - .deleteFiles([utkey]) - .then(() => { - internalToUTKeyMap.delete(key); - }); + .deleteFiles([key], { keyType: "customId" }) + .then(() => {}); }, async clear() { - const utkeys = Array.from(internalToUTKeyMap.values()); + const keys = await getKeys(); return getClient() - .deleteFiles(utkeys) - .then(() => { - internalToUTKeyMap.clear(); - }); + .deleteFiles(keys, { keyType: "customId" }) + .then(() => {}); }, // getMeta(key, opts) { diff --git a/src/storage.ts b/src/storage.ts index d53c755d..fe435525 100644 --- a/src/storage.ts +++ b/src/storage.ts @@ -231,7 +231,7 @@ export function createStorage( async setItems(items, commonOptions) { await runBatch(items, commonOptions, async (batch) => { if (batch.driver.setItems) { - await asyncCall( + return asyncCall( batch.driver.setItems, batch.items.map((item) => ({ key: item.relativeKey, From a54c6485aaef43aca523301e56653410fd697fe9 Mon Sep 17 00:00:00 2001 From: juliusmarminge Date: Tue, 30 Jan 2024 12:32:29 +0100 Subject: [PATCH 09/14] ref --- src/drivers/uploadthing.ts | 87 +++++++++++++------------------------- 1 file changed, 30 insertions(+), 57 deletions(-) diff --git a/src/drivers/uploadthing.ts b/src/drivers/uploadthing.ts index 4b42412c..83aee6d7 100644 --- a/src/drivers/uploadthing.ts +++ b/src/drivers/uploadthing.ts @@ -8,8 +8,6 @@ export interface UploadThingOptions { export default defineDriver((opts) => { let client: UTApi; - let utApiFetch: $Fetch; - const getClient = () => { return (client ??= new UTApi({ apiKey: opts.apiKey, @@ -17,34 +15,17 @@ export default defineDriver((opts) => { })); }; - // The UTApi doesn't have all methods we need right now, so use raw fetch - const getUTApiFetch = () => { - return (utApiFetch ??= ofetch.create({ - method: "POST", - baseURL: "https://uploadthing.com/api", - headers: { - "x-uploadthing-api-key": opts.apiKey, - }, - })); - }; - - function getKeys() { - return getClient() - .listFiles({}) - .then((res) => - res.map((file) => file.customId).filter((k): k is string => !!k) - ); + async function getKeys() { + const res = await getClient().listFiles({}); + return res.map((file) => file.customId).filter((k): k is string => !!k); } return { - hasItem(id) { - return getClient() - .getFileUrls(id, { keyType: "customId" }) - .then((res) => { - return !!res.length; - }); + hasItem: async (id) => { + const res = await getClient().getFileUrls(id, { keyType: "customId" }); + return res.length > 0; }, - async getItem(id) { + getItem: async (id) => { const url = await getClient() .getFileUrls(id, { keyType: "customId" }) .then((res) => { @@ -53,43 +34,35 @@ export default defineDriver((opts) => { if (!url) return null; return ofetch(url).then((res) => res.text()); }, - getKeys() { + getKeys: () => { return getKeys(); }, - setItem(key, value, opts) { - return getClient() - .uploadFiles( - Object.assign(new Blob([value]), { - name: key, - customId: key, - }), - { metadata: opts?.metadata } - ) - .then(() => {}); + setItem: async (key, value, opts) => { + await getClient().uploadFiles( + Object.assign(new Blob([value]), { + name: key, + customId: key, + }), + { metadata: opts?.metadata } + ); }, - setItems(items, opts) { - return getClient() - .uploadFiles( - items.map((item) => - Object.assign(new Blob([item.value]), { - name: item.key, - customId: item.key, - }) - ), - { metadata: opts?.metadata } - ) - .then(() => {}); + setItems: async (items, opts) => { + await getClient().uploadFiles( + items.map((item) => + Object.assign(new Blob([item.value]), { + name: item.key, + customId: item.key, + }) + ), + { metadata: opts?.metadata } + ); }, - removeItem(key, opts) { - return getClient() - .deleteFiles([key], { keyType: "customId" }) - .then(() => {}); + removeItem: async (key, opts) => { + await getClient().deleteFiles([key], { keyType: "customId" }); }, - async clear() { + clear: async () => { const keys = await getKeys(); - return getClient() - .deleteFiles(keys, { keyType: "customId" }) - .then(() => {}); + await getClient().deleteFiles(keys, { keyType: "customId" }); }, // getMeta(key, opts) { From 73a651b8e89c6bf1a4ccb21326c4f617c7b822ff Mon Sep 17 00:00:00 2001 From: Pooya Parsa Date: Sun, 15 Dec 2024 23:07:50 +0100 Subject: [PATCH 10/14] update impl --- .env.example | 2 + .github/workflows/ci.yml | 1 + src/drivers/uploadthing.ts | 124 +++++++++++++++++++------------ test/drivers/uploadthing.test.ts | 72 ++---------------- vite.config.ts | 2 +- 5 files changed, 87 insertions(+), 114 deletions(-) diff --git a/.env.example b/.env.example index b6193989..51e03c32 100644 --- a/.env.example +++ b/.env.example @@ -6,3 +6,5 @@ VITE_VERCEL_BLOB_READ_WRITE_TOKEN= VITE_CLOUDFLARE_ACC_ID= VITE_CLOUDFLARE_KV_NS_ID= VITE_CLOUDFLARE_TOKEN= + +VITE_UPLOADTHING_TOKEN= diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6ba70092..826f07e1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,6 +38,7 @@ jobs: VITE_CLOUDFLARE_ACC_ID: ${{ secrets.VITE_CLOUDFLARE_ACC_ID }} VITE_CLOUDFLARE_KV_NS_ID: ${{ secrets.VITE_CLOUDFLARE_KV_NS_ID }} VITE_CLOUDFLARE_TOKEN: ${{ secrets.VITE_CLOUDFLARE_TOKEN }} + VITE_UPLOADTHING_TOKEN: ${{ secrets.VITE_UPLOADTHING_TOKEN }} - uses: codecov/codecov-action@v5 - name: nightly release if: | diff --git a/src/drivers/uploadthing.ts b/src/drivers/uploadthing.ts index 83aee6d7..8214ab91 100644 --- a/src/drivers/uploadthing.ts +++ b/src/drivers/uploadthing.ts @@ -1,70 +1,102 @@ -import { defineDriver } from "./utils"; -import { ofetch, $Fetch } from "ofetch"; +import { defineDriver, normalizeKey } from "./utils"; import { UTApi } from "uploadthing/server"; -export interface UploadThingOptions { - apiKey: string; +// Reference: https://docs.uploadthing.com + +type UTApiOptions = Omit< + Exclude[0], undefined>, + "defaultKeyType" +>; + +type FileEsque = Parameters[0][0]; + +export interface UploadThingOptions extends UTApiOptions { + /** base key to add to keys */ + base?: string; } -export default defineDriver((opts) => { +const DRIVER_NAME = "uploadthing"; + +export default defineDriver((opts = {}) => { let client: UTApi; + + const base = opts.base ? normalizeKey(opts.base) : ""; + const r = (key: string) => (base ? `${base}:${key}` : key); + const getClient = () => { return (client ??= new UTApi({ - apiKey: opts.apiKey, - fetch: ofetch.native, + ...opts, + defaultKeyType: "customId", })); }; - async function getKeys() { - const res = await getClient().listFiles({}); - return res.map((file) => file.customId).filter((k): k is string => !!k); - } + const getKeys = async (base: string) => { + const client = getClient(); + const { files } = await client.listFiles({}); + return files + .map((file) => file.customId) + .filter((k) => k && k.startsWith(base)) as string[]; + }; + + const toFile = (key: string, value: BlobPart) => { + return Object.assign(new Blob([value]), { + name: key, + customId: key, + }) satisfies FileEsque; + }; return { - hasItem: async (id) => { - const res = await getClient().getFileUrls(id, { keyType: "customId" }); - return res.length > 0; + name: DRIVER_NAME, + getInstance() { + return getClient(); + }, + getKeys(base) { + return getKeys(r(base)); }, - getItem: async (id) => { - const url = await getClient() - .getFileUrls(id, { keyType: "customId" }) - .then((res) => { - return res[0]?.url; - }); + async hasItem(key) { + const client = getClient(); + const res = await client.getFileUrls(r(key)); + return res.data.length > 0; + }, + async getItem(key) { + const client = getClient(); + const url = await client + .getFileUrls(r(key)) + .then((res) => res.data[0]?.url); if (!url) return null; - return ofetch(url).then((res) => res.text()); + return fetch(url).then((res) => res.text()); }, - getKeys: () => { - return getKeys(); + async getItemRaw(key) { + const client = getClient(); + const url = await client + .getFileUrls(r(key)) + .then((res) => res.data[0]?.url); + if (!url) return null; + return fetch(url).then((res) => res.arrayBuffer()); }, - setItem: async (key, value, opts) => { - await getClient().uploadFiles( - Object.assign(new Blob([value]), { - name: key, - customId: key, - }), - { metadata: opts?.metadata } - ); + async setItem(key, value) { + const client = getClient(); + await client.uploadFiles(toFile(r(key), value)); + }, + async setItemRaw(key, value) { + const client = getClient(); + await client.uploadFiles(toFile(r(key), value)); }, - setItems: async (items, opts) => { - await getClient().uploadFiles( - items.map((item) => - Object.assign(new Blob([item.value]), { - name: item.key, - customId: item.key, - }) - ), - { metadata: opts?.metadata } + async setItems(items) { + const client = getClient(); + await client.uploadFiles( + items.map((item) => toFile(r(item.key), item.value)) ); }, - removeItem: async (key, opts) => { - await getClient().deleteFiles([key], { keyType: "customId" }); + async removeItem(key) { + const client = getClient(); + await client.deleteFiles([r(key)]); }, - clear: async () => { - const keys = await getKeys(); - await getClient().deleteFiles(keys, { keyType: "customId" }); + async clear(base) { + const client = getClient(); + const keys = await getKeys(r(base)); + await client.deleteFiles(keys); }, - // getMeta(key, opts) { // // TODO: We don't currently have an endpoint to fetch metadata, but it does exist // }, diff --git a/test/drivers/uploadthing.test.ts b/test/drivers/uploadthing.test.ts index a54ab863..72a50c0d 100644 --- a/test/drivers/uploadthing.test.ts +++ b/test/drivers/uploadthing.test.ts @@ -1,79 +1,17 @@ -import { afterAll, beforeAll, describe, it } from "vitest"; +import { describe } from "vitest"; import driver from "../../src/drivers/uploadthing"; import { testDriver } from "./utils"; -import { setupServer } from "msw/node"; -import { rest } from "msw"; -const store: Record = {}; +const utfsToken = process.env.VITE_UPLOADTHING_TOKEN; -const utapiUrl = "https://uploadthing.com/api"; -const utfsUrl = "https://utfs.io/f"; - -const server = setupServer( - rest.post(`${utapiUrl}/getFileUrl`, async (req, res, ctx) => { - const { fileKeys } = await req.json(); - const key = fileKeys[0]; - if (!(key in store)) { - return res(ctx.status(401), ctx.json({ error: "Unauthorized" })); - } - return res( - ctx.status(200), - ctx.json({ - result: { - [key]: `https://utfs.io/f/${key}`, - }, - }) - ); - }), - rest.get(`${utfsUrl}/:key`, (req, res, ctx) => { - const key = req.params.key as string; - if (!(key in store)) { - return res(ctx.status(404), ctx.json(null)); - } - return res( - ctx.status(200), - ctx.set("content-type", "application/octet-stream"), - ctx.body(store[key]) - ); - }), - rest.post(`${utapiUrl}/uploadFiles`, async (req, res, ctx) => { - console.log("intercepted request"); - return res( - ctx.status(200), - ctx.json({ - data: [ - { - presignedUrls: [`https://my-s3-server.com/:key`], - }, - ], - }) - ); - }), - rest.post(`${utapiUrl}/deleteFile`, async (req, res, ctx) => { - console.log("hello????"); - const { fileKeys } = await req.json(); - for (const key of fileKeys) { - delete store[key]; - } - return res(ctx.status(200), ctx.json({ success: true })); - }) -); - -describe( +describe.skipIf(!utfsToken)( "drivers: uploadthing", () => { - // beforeAll(() => { - // server.listen(); - // }); - // afterAll(() => { - // server.close(); - // }); - + process.env.UPLOADTHING_TOKEN = utfsToken; testDriver({ driver: driver({ - apiKey: "sk_live_xxx", + base: Math.round(Math.random() * 1_000_000).toString(16), }), - async additionalTests(ctx) {}, }); }, { timeout: 30e3 } diff --git a/vite.config.ts b/vite.config.ts index 7fda8351..0d9f9da2 100644 --- a/vite.config.ts +++ b/vite.config.ts @@ -3,7 +3,7 @@ import { defineConfig } from "vitest/config"; export default defineConfig({ test: { testTimeout: 10_000, - retry: 3, + retry: process.env.CI ? 3 : undefined, typecheck: { enabled: true, }, From 7b8e4907badf4478e50f82648c8b0e9e570f45f4 Mon Sep 17 00:00:00 2001 From: Pooya Parsa Date: Sun, 15 Dec 2024 23:24:54 +0100 Subject: [PATCH 11/14] Update docs/2.drivers/uploadthing.md Co-authored-by: Julius Marminge --- docs/2.drivers/uploadthing.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/2.drivers/uploadthing.md b/docs/2.drivers/uploadthing.md index 061355b7..8fd375e0 100644 --- a/docs/2.drivers/uploadthing.md +++ b/docs/2.drivers/uploadthing.md @@ -12,7 +12,7 @@ import uploadthingDriver from "unstorage/drivers/uploadthing"; const storage = createStorage({ driver: uploadthingDriver({ - // apiKey: "", + // token: "", }), }); ``` From 76d981ae8248c5944d39465e88aa0003c0495d05 Mon Sep 17 00:00:00 2001 From: Pooya Parsa Date: Wed, 18 Dec 2024 15:47:27 +0100 Subject: [PATCH 12/14] update drivers list --- src/_drivers.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/_drivers.ts b/src/_drivers.ts index ada0fbb0..7780ba69 100644 --- a/src/_drivers.ts +++ b/src/_drivers.ts @@ -27,11 +27,12 @@ import type { PlanetscaleDriverOptions as PlanetscaleOptions } from "unstorage/d import type { RedisOptions as RedisOptions } from "unstorage/drivers/redis"; import type { S3DriverOptions as S3Options } from "unstorage/drivers/s3"; import type { SessionStorageOptions as SessionStorageOptions } from "unstorage/drivers/session-storage"; +import type { UploadThingOptions as UploadthingOptions } from "unstorage/drivers/uploadthing"; import type { UpstashOptions as UpstashOptions } from "unstorage/drivers/upstash"; import type { VercelBlobOptions as VercelBlobOptions } from "unstorage/drivers/vercel-blob"; import type { VercelKVOptions as VercelKVOptions } from "unstorage/drivers/vercel-kv"; -export type BuiltinDriverName = "azure-app-configuration" | "azureAppConfiguration" | "azure-cosmos" | "azureCosmos" | "azure-key-vault" | "azureKeyVault" | "azure-storage-blob" | "azureStorageBlob" | "azure-storage-table" | "azureStorageTable" | "capacitor-preferences" | "capacitorPreferences" | "cloudflare-kv-binding" | "cloudflareKVBinding" | "cloudflare-kv-http" | "cloudflareKVHttp" | "cloudflare-r2-binding" | "cloudflareR2Binding" | "db0" | "deno-kv-node" | "denoKVNode" | "deno-kv" | "denoKV" | "fs-lite" | "fsLite" | "fs" | "github" | "http" | "indexedb" | "localstorage" | "lru-cache" | "lruCache" | "memory" | "mongodb" | "netlify-blobs" | "netlifyBlobs" | "null" | "overlay" | "planetscale" | "redis" | "s3" | "session-storage" | "sessionStorage" | "upstash" | "vercel-blob" | "vercelBlob" | "vercel-kv" | "vercelKV"; +export type BuiltinDriverName = "azure-app-configuration" | "azureAppConfiguration" | "azure-cosmos" | "azureCosmos" | "azure-key-vault" | "azureKeyVault" | "azure-storage-blob" | "azureStorageBlob" | "azure-storage-table" | "azureStorageTable" | "capacitor-preferences" | "capacitorPreferences" | "cloudflare-kv-binding" | "cloudflareKVBinding" | "cloudflare-kv-http" | "cloudflareKVHttp" | "cloudflare-r2-binding" | "cloudflareR2Binding" | "db0" | "deno-kv-node" | "denoKVNode" | "deno-kv" | "denoKV" | "fs-lite" | "fsLite" | "fs" | "github" | "http" | "indexedb" | "localstorage" | "lru-cache" | "lruCache" | "memory" | "mongodb" | "netlify-blobs" | "netlifyBlobs" | "null" | "overlay" | "planetscale" | "redis" | "s3" | "session-storage" | "sessionStorage" | "uploadthing" | "upstash" | "vercel-blob" | "vercelBlob" | "vercel-kv" | "vercelKV"; export type BuiltinDriverOptions = { "azure-app-configuration": AzureAppConfigurationOptions; @@ -75,6 +76,7 @@ export type BuiltinDriverOptions = { "s3": S3Options; "session-storage": SessionStorageOptions; "sessionStorage": SessionStorageOptions; + "uploadthing": UploadthingOptions; "upstash": UpstashOptions; "vercel-blob": VercelBlobOptions; "vercelBlob": VercelBlobOptions; @@ -126,6 +128,7 @@ export const builtinDrivers = { "s3": "unstorage/drivers/s3", "session-storage": "unstorage/drivers/session-storage", "sessionStorage": "unstorage/drivers/session-storage", + "uploadthing": "unstorage/drivers/uploadthing", "upstash": "unstorage/drivers/upstash", "vercel-blob": "unstorage/drivers/vercel-blob", "vercelBlob": "unstorage/drivers/vercel-blob", From 7ad2e71a617fadb9bcbb1c77ab62938c6321d174 Mon Sep 17 00:00:00 2001 From: Pooya Parsa Date: Wed, 18 Dec 2024 15:57:30 +0100 Subject: [PATCH 13/14] update docs --- docs/2.drivers/uploadthing.md | 28 +++++++++++++++++----------- test/drivers/uploadthing.test.ts | 20 ++++++++------------ 2 files changed, 25 insertions(+), 23 deletions(-) diff --git a/docs/2.drivers/uploadthing.md b/docs/2.drivers/uploadthing.md index 8fd375e0..62c4883d 100644 --- a/docs/2.drivers/uploadthing.md +++ b/docs/2.drivers/uploadthing.md @@ -1,11 +1,27 @@ +--- +icon: qlementine-icons:cloud-16 +--- + # UploadThing -Store data using UploadThing. +> Store data using UploadThing. ::note{to="https://uploadthing.com/"} Learn more about UploadThing. :: +::warning +UploadThing support is currently experimental! +
+There is a known issue that same key, if deleted cannot be used again [tracker issue](https://github.com/pingdotgg/uploadthing/issues/948). +:: + +## Usage + +To use, you will need to install `uploadthing` dependency in your project: + +:pm-install{name="uploadthing"} + ```js import { createStorage } from "unstorage"; import uploadthingDriver from "unstorage/drivers/uploadthing"; @@ -17,16 +33,6 @@ const storage = createStorage({ }); ``` -To use, you will need to install `uploadthing` dependency in your project: - -```json -{ - "dependencies": { - "uploadthing": "latest" - } -} -``` - **Options:** - `apiKey`: Your UploadThing API key. Will be automatically inferred from the `UPLOADTHING_SECRET` environment variable if not provided. diff --git a/test/drivers/uploadthing.test.ts b/test/drivers/uploadthing.test.ts index 72a50c0d..3fe3fa08 100644 --- a/test/drivers/uploadthing.test.ts +++ b/test/drivers/uploadthing.test.ts @@ -4,15 +4,11 @@ import { testDriver } from "./utils"; const utfsToken = process.env.VITE_UPLOADTHING_TOKEN; -describe.skipIf(!utfsToken)( - "drivers: uploadthing", - () => { - process.env.UPLOADTHING_TOKEN = utfsToken; - testDriver({ - driver: driver({ - base: Math.round(Math.random() * 1_000_000).toString(16), - }), - }); - }, - { timeout: 30e3 } -); +describe.skipIf(!utfsToken)("drivers: uploadthing", { timeout: 30e3 }, () => { + process.env.UPLOADTHING_TOKEN = utfsToken; + testDriver({ + driver: driver({ + base: Math.round(Math.random() * 1_000_000).toString(16), + }), + }); +}); From e6a34fbe5824870bac629218bd617bb54358d5b4 Mon Sep 17 00:00:00 2001 From: Pooya Parsa Date: Wed, 18 Dec 2024 16:06:01 +0100 Subject: [PATCH 14/14] update docs --- docs/2.drivers/uploadthing.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/2.drivers/uploadthing.md b/docs/2.drivers/uploadthing.md index 62c4883d..fa2d6dec 100644 --- a/docs/2.drivers/uploadthing.md +++ b/docs/2.drivers/uploadthing.md @@ -28,11 +28,11 @@ import uploadthingDriver from "unstorage/drivers/uploadthing"; const storage = createStorage({ driver: uploadthingDriver({ - // token: "", + // token: "", // UPLOADTHING_SECRET environment variable will be used if not provided. }), }); ``` **Options:** -- `apiKey`: Your UploadThing API key. Will be automatically inferred from the `UPLOADTHING_SECRET` environment variable if not provided. +- `token`: Your UploadThing API key. Will be automatically inferred from the `UPLOADTHING_SECRET` environment variable if not provided.