diff --git a/packages/tre/src/plugins/r2/errors.ts b/packages/tre/src/plugins/r2/errors.ts index a43422795..42cd45ea9 100644 --- a/packages/tre/src/plugins/r2/errors.ts +++ b/packages/tre/src/plugins/r2/errors.ts @@ -14,6 +14,7 @@ enum CfCode { InternalError = 10001, NoSuchObjectKey = 10007, EntityTooLarge = 100100, + MetadataTooLarge = 10012, InvalidObjectName = 10020, InvalidMaxKeys = 10022, InvalidArgument = 10029, @@ -109,6 +110,16 @@ export class EntityTooLarge extends R2Error { } } +export class MetadataTooLarge extends R2Error { + constructor() { + super( + Status.BadRequest, + "Your metadata headers exceed the maximum allowed metadata size.", + CfCode.MetadataTooLarge + ); + } +} + export class BadDigest extends R2Error { constructor( algorithm: "MD5" | "SHA-1" | "SHA-256" | "SHA-384" | "SHA-512", diff --git a/packages/tre/src/plugins/r2/gateway.ts b/packages/tre/src/plugins/r2/gateway.ts index 88ac2a559..be0bee5c4 100644 --- a/packages/tre/src/plugins/r2/gateway.ts +++ b/packages/tre/src/plugins/r2/gateway.ts @@ -111,6 +111,7 @@ export class R2Gateway { const checksums = validate .key(key) .size(value) + .metadataSize(options.customMetadata) .condition(meta, options.onlyIf) .hash(value, options); diff --git a/packages/tre/src/plugins/r2/validator.ts b/packages/tre/src/plugins/r2/validator.ts index acebe5498..73cee5484 100644 --- a/packages/tre/src/plugins/r2/validator.ts +++ b/packages/tre/src/plugins/r2/validator.ts @@ -5,6 +5,7 @@ import { EntityTooLarge, InvalidMaxKeys, InvalidObjectName, + MetadataTooLarge, PreconditionFailed, } from "./errors"; import { R2Object, R2ObjectMetadata } from "./r2Object"; @@ -12,7 +13,9 @@ import { R2Conditional } from "./schemas"; export const MAX_LIST_KEYS = 1_000; const MAX_KEY_SIZE = 1024; -const MAX_VALUE_SIZE = 5 * 1_000 * 1_000 * 1_000 - 5 * 1_000 * 1_000; +// https://developers.cloudflare.com/r2/platform/limits/ +const MAX_VALUE_SIZE = 5_000_000_000 - 5_000_000; // 5GB - 5MB +const MAX_METADATA_SIZE = 2048; // 2048B function identity(ms: number) { return ms; @@ -67,6 +70,14 @@ export type R2Hashes = Record< Buffer | undefined >; +function serialisedLength(x: string) { + // Adapted from internal R2 gateway implementation + for (let i = 0; i < x.length; i++) { + if (x.charCodeAt(i) >= 256) return x.length * 2; + } + return x.length; +} + export class Validator { hash(value: Uint8Array, hashes: R2Hashes): R2StringChecksums { const checksums: R2StringChecksums = {}; @@ -95,13 +106,24 @@ export class Validator { } size(value: Uint8Array): Validator { - // TODO: should we be validating httpMetadata/customMetadata size too if (value.byteLength > MAX_VALUE_SIZE) { throw new EntityTooLarge(); } return this; } + metadataSize(customMetadata?: Record): Validator { + if (customMetadata === undefined) return this; + let metadataLength = 0; + for (const [key, value] of Object.entries(customMetadata)) { + metadataLength += serialisedLength(key) + serialisedLength(value); + } + if (metadataLength > MAX_METADATA_SIZE) { + throw new MetadataTooLarge(); + } + return this; + } + key(key: string): Validator { const keyLength = Buffer.byteLength(key); if (keyLength >= MAX_KEY_SIZE) { diff --git a/packages/tre/test/plugins/r2/index.spec.ts b/packages/tre/test/plugins/r2/index.spec.ts index 91a26638a..9c75fdfac 100644 --- a/packages/tre/test/plugins/r2/index.spec.ts +++ b/packages/tre/test/plugins/r2/index.spec.ts @@ -146,13 +146,31 @@ class TestR2Bucket implements R2Bucket { options?: R2PutOptions ): Promise { const url = `http://localhost/${encodeURIComponent(this.ns + key)}`; + + let valueBlob: Blob; + if (value === null) { + valueBlob = new Blob([]); + } else if (value instanceof ArrayBuffer) { + valueBlob = new Blob([new Uint8Array(value)]); + } else if (ArrayBuffer.isView(value)) { + valueBlob = new Blob([viewToArray(value)]); + } else if (value instanceof ReadableStream) { + // @ts-expect-error `ReadableStream` is an `AsyncIterable` + valueBlob = await blob(value); + } else { + valueBlob = new Blob([value]); + } + + // We can't store options in headers as some put() tests include extended + // characters in them, and `undici` validates all headers are byte strings, + // so use a form data body instead + const formData = new FormData(); + formData.set("options", maybeJsonStringify(options)); + formData.set("value", valueBlob); const res = await this.mf.dispatchFetch(url, { method: "PUT", - headers: { - Accept: "multipart/form-data", - "Test-Options": maybeJsonStringify(options), - }, - body: ArrayBuffer.isView(value) ? viewToArray(value) : value, + headers: { Accept: "multipart/form-data" }, + body: formData, }); return deconstructResponse(res); } @@ -376,11 +394,12 @@ const test = miniflareTest<{ BUCKET: R2Bucket }, Context>( const options = maybeJsonParse(optionsHeader); return constructResponse(await env.BUCKET.get(key, options)); } else if (method === "PUT") { - const optionsHeader = request.headers.get("Test-Options"); - const options = maybeJsonParse(optionsHeader); - return constructResponse( - await env.BUCKET.put(key, await request.arrayBuffer(), options) - ); + const formData = await request.formData(); + const optionsData = formData.get("options"); + if (typeof optionsData !== "string") throw new TypeError(); + const options = maybeJsonParse(optionsData); + const value = formData.get("value"); + return constructResponse(await env.BUCKET.put(key, value, options)); } else if (method === "DELETE") { const keys = await request.json(); await env.BUCKET.delete(keys); @@ -796,6 +815,39 @@ test("put: stores only if passes onlyIf", async (t) => { const object = await r2.put("no-key", "2", { onlyIf: { etagMatches: etag } }); t.is(object as R2Object | null, null); }); +test("put: validates metadata size", async (t) => { + const { r2 } = t.context; + + // TODO(soon): add check for max value size once we have streaming support + // (don't really want to allocate 5GB buffers in tests :sweat_smile:) + + const expectations: ThrowsExpectation = { + instanceOf: Error, + message: + "put: Your metadata headers exceed the maximum allowed metadata size. (10012)", + }; + + // Check with ASCII characters + await r2.put("key", "value", { customMetadata: { key: "x".repeat(2045) } }); + await t.throwsAsync( + r2.put("key", "value", { customMetadata: { key: "x".repeat(2046) } }), + expectations + ); + await r2.put("key", "value", { customMetadata: { hi: "x".repeat(2046) } }); + + // Check with extended characters: note "🙂" is 2 UTF-16 code units, so + // `"🙂".length === 2`, and it requires 4 bytes to store + await r2.put("key", "value", { customMetadata: { key: "🙂".repeat(511) } }); // 3 + 4*511 = 2047 + await r2.put("key", "value", { customMetadata: { key1: "🙂".repeat(511) } }); // 4 + 4*511 = 2048 + await t.throwsAsync( + r2.put("key", "value", { customMetadata: { key12: "🙂".repeat(511) } }), // 5 + 4*511 = 2049 + expectations + ); + await t.throwsAsync( + r2.put("key", "value", { customMetadata: { key: "🙂".repeat(512) } }), // 3 + 4*512 = 2051 + expectations + ); +}); test("delete: deletes existing keys", async (t) => { const { r2 } = t.context;