add .bytes() method to various readables (#11104)

Co-authored-by: Jarred Sumner <[email protected]>
Co-authored-by: nektro <[email protected]>

bench/snippets/react-dom-render.bun.js

@@ -26,6 +26,11 @@ group("new Response(stream).arrayBuffer()", () => {
   bench("react-dom/server.bun", async () => await new Response(await renderToReadableStreamBun(<App />)).arrayBuffer());
 });
 
+group("new Response(stream).bytes()", () => {
+  bench("react-dom/server.browser", async () => await new Response(await renderToReadableStream(<App />)).bytes());
+  bench("react-dom/server.bun", async () => await new Response(await renderToReadableStreamBun(<App />)).bytes());
+});
+
 group("new Response(stream).blob()", () => {
   bench("react-dom/server.browser", async () => await new Response(await renderToReadableStream(<App />)).blob());
   bench("react-dom/server.bun", async () => await new Response(await renderToReadableStreamBun(<App />)).blob());

docs/api/binary-data.md

@@ -886,15 +886,25 @@ new Response(stream).arrayBuffer();
 Bun.readableStreamToArrayBuffer(stream);
 ```
 
+#### To `Uint8Array`
+
+```ts
+// with Response
+new Response(stream).bytes();
+
+// with Bun function
+Bun.readableStreamToBytes(stream);
+```
+
 #### To `TypedArray`
 
 ```ts
 // with Response
 const buf = await new Response(stream).arrayBuffer();
-new Uint8Array(buf);
+new Int8Array(buf);
 
 // with Bun function
-new Uint8Array(Bun.readableStreamToArrayBuffer(stream));
+new Int8Array(Bun.readableStreamToArrayBuffer(stream));
 ```
 
 #### To `DataView`

docs/api/file-io.md

@@ -28,6 +28,7 @@ const foo = Bun.file("foo.txt");
 await foo.text(); // contents as a string
 await foo.stream(); // contents as ReadableStream
 await foo.arrayBuffer(); // contents as ArrayBuffer
+await foo.bytes(); // contents as Uint8Array
 ```
 
 File references can also be created using numerical [file descriptors](https://en.wikipedia.org/wiki/File_descriptor) or `file://` URLs.

docs/api/utils.md

@@ -275,6 +275,7 @@ Bun.stringWidth("\u001b[31mhello\u001b[0m", { countAnsiEscapeCodes: true }); //
 ```
 
 This is useful for:
+
 - Aligning text in a terminal
 - Quickly checking if a string contains ANSI escape codes
 - Measuring the width of a string in a terminal
@@ -372,7 +373,6 @@ npm/string-width 95,000 chars ansi+emoji+ascii     3.68 s/iter        (3.66 s 
 
 {% /details %}
 
-
 TypeScript definition:
 
 ```ts
@@ -400,7 +400,6 @@ namespace Bun {
 }
 ```
 
-
 <!-- ## `Bun.enableANSIColors()` -->
 
 ## `Bun.fileURLToPath()`
@@ -603,6 +602,9 @@ stream; // => ReadableStream
 await Bun.readableStreamToArrayBuffer(stream);
 // => ArrayBuffer
 
+await Bun.readableStreamToBytes(stream);
+// => Uint8Array
+
 await Bun.readableStreamToBlob(stream);
 // => Blob
 

docs/bundler/index.md

@@ -1096,6 +1096,7 @@ const build = await Bun.build({
 
 for (const output of build.outputs) {
   await output.arrayBuffer(); // => ArrayBuffer
+  await output.bytes(); // => Uint8Array
   await output.text(); // string
 }
 ```

docs/guides/read-file/arraybuffer.md

@@ -13,11 +13,11 @@ const buffer = await file.arrayBuffer();
 
 ---
 
-The binary content in the `ArrayBuffer` can then be read as a typed array, such as `Uint8Array`.
+The binary content in the `ArrayBuffer` can then be read as a typed array, such as `Int8Array`. For `Uint8Array`, use [`.bytes()`](./uint8array).
 
 ```ts
 const buffer = await file.arrayBuffer();
-const bytes = new Uint8Array(buffer);
+const bytes = new Int8Array(buffer);
 
 bytes[0];
 bytes.length;

docs/guides/read-file/uint8array.md

@@ -4,14 +4,13 @@ name: Read a file to a Uint8Array
 
 The `Bun.file()` function accepts a path and returns a `BunFile` instance. The `BunFile` class extends `Blob` and allows you to lazily read the file in a variety of formats.
 
-To read the file into a `Uint8Array` instance, retrieve the contents of the `BunFile` as an `ArrayBuffer` with `.arrayBuffer()`, then pass it into the `Uint8Array` constructor.
+To read the file into a `Uint8Array` instance, retrieve the contents of the `BunFile` with `.bytes()`.
 
 ```ts
 const path = "/path/to/package.json";
 const file = Bun.file(path);
 
-const arrBuffer = await file.arrayBuffer();
-const byteArray = new Uint8Array(arrBuffer);
+const byteArray = await file.bytes();
 
 byteArray[0]; // first byteArray
 byteArray.length; // length of byteArray

docs/guides/streams/node-readable-to-uint8array.md

@@ -0,0 +1,11 @@
+---
+name: Convert a Node.js Readable to an Uint8Array
+---
+
+To convert a Node.js `Readable` stream to an `Uint8Array` in Bun, you can create a new `Response` object with the stream as the body, then use `bytes()` to read the stream into an `Uint8Array`.
+
+```ts
+import { Readable } from "stream";
+const stream = Readable.from(["Hello, ", "world!"]);
+const buf = await new Response(stream).bytes();
+```

docs/guides/streams/to-typedarray.md

@@ -10,6 +10,13 @@ const buf = await Bun.readableStreamToArrayBuffer(stream);
 const uint8 = new Uint8Array(buf);
 ```
 
+Additionally, there is a convenience method to convert to `Uint8Array` directly.
+
+```ts
+const stream = new ReadableStream();
+const uint8 = await Bun.readableStreamToBytes(stream);
+```
+
 ---
 
 See [Docs > API > Utils](/docs/api/utils#bun-readablestreamto) for documentation on Bun's other `ReadableStream` conversion functions.

packages/bun-polyfills/src/modules/bun.ts

@@ -410,6 +410,21 @@ export const readableStreamToArrayBuffer = ((stream: ReadableStream<ArrayBufferV
         return sink.end() as ArrayBuffer;
     })();
 }) satisfies typeof Bun.readableStreamToArrayBuffer;
+
+export const readableStreamToBytes = ((stream: ReadableStream<ArrayBufferView | ArrayBufferLike>): Uint8Array | Promise<Uint8Array> => {
+    return (async () => {
+        const sink = new ArrayBufferSink();
+        sink.start({ asUint8Array: true });
+        const reader = stream.getReader();
+        while (true) {
+            const { done, value } = await reader.read();
+            if (done) break;
+            sink.write(value);
+        }
+        return sink.end() as Uint8Array;
+    })();
+}) satisfies typeof Bun.readableStreamToBytes;
+
 export const readableStreamToText = (async (stream: ReadableStream<ArrayBufferView | ArrayBuffer>) => {
     let result = '';
     const reader = stream.pipeThrough(new TextDecoderStream()).getReader(); ReadableStreamDefaultReader
@@ -445,16 +460,19 @@ export const readableStreamToJSON = (async <T = unknown>(stream: ReadableStream<
     }
 }) satisfies typeof Bun.readableStreamToJSON;
 
-export const concatArrayBuffers = ((buffers) => {
+export const concatArrayBuffers = ((buffers, maxLength = Infinity, asUint8Array = false) => {
     let size = 0;
     for (const chunk of buffers) size += chunk.byteLength;
+    size = Math.min(size, maxLength);
     const buffer = new ArrayBuffer(size);
     const view = new Uint8Array(buffer);
     let offset = 0;
     for (const chunk of buffers) {
+        if (offset > size) break;
         view.set(new Uint8Array(chunk instanceof ArrayBuffer || chunk instanceof SharedArrayBuffer ? chunk : chunk.buffer), offset);
         offset += chunk.byteLength;
     }
+    if (asUint8Array) return view;
     return buffer;
 }) satisfies typeof Bun.concatArrayBuffers;
 

packages/bun-types/bun.d.ts

@@ -410,6 +410,19 @@ declare module "bun" {
      */
     arrayBuffer(): ArrayBuffer;
 
+    /**
+     * Read from stdout as an Uint8Array
+     *
+     * @returns Stdout as an Uint8Array
+     * @example
+     *
+     * ```ts
+     * const output = await $`echo hello`;
+     * console.log(output.bytes()); // Uint8Array { byteLength: 6 }
+     * ```
+     */
+    bytes(): Uint8Array;
+
     /**
      * Read from stdout as a Blob
      *
@@ -688,7 +701,17 @@ declare module "bun" {
    * This function is faster because it uses uninitialized memory when copying. Since the entire
    * length of the buffer is known, it is safe to use uninitialized memory.
    */
-  function concatArrayBuffers(buffers: Array<ArrayBufferView | ArrayBufferLike>): ArrayBuffer;
+  function concatArrayBuffers(buffers: Array<ArrayBufferView | ArrayBufferLike>, maxLength?: number): ArrayBuffer;
+  function concatArrayBuffers(
+    buffers: Array<ArrayBufferView | ArrayBufferLike>,
+    maxLength: number,
+    asUint8Array: false,
+  ): ArrayBuffer;
+  function concatArrayBuffers(
+    buffers: Array<ArrayBufferView | ArrayBufferLike>,
+    maxLength: number,
+    asUint8Array: true,
+  ): Uint8Array;
 
   /**
    * Consume all data from a {@link ReadableStream} until it closes or errors.
@@ -705,6 +728,21 @@ declare module "bun" {
     stream: ReadableStream<ArrayBufferView | ArrayBufferLike>,
   ): Promise<ArrayBuffer> | ArrayBuffer;
 
+  /**
+   * Consume all data from a {@link ReadableStream} until it closes or errors.
+   *
+   * Concatenate the chunks into a single {@link ArrayBuffer}.
+   *
+   * Each chunk must be a TypedArray or an ArrayBuffer. If you need to support
+   * chunks of different types, consider {@link readableStreamToBlob}
+   *
+   * @param stream The stream to consume.
+   * @returns A promise that resolves with the concatenated chunks or the concatenated chunks as a {@link Uint8Array}.
+   */
+  function readableStreamToBytes(
+    stream: ReadableStream<ArrayBufferView | ArrayBufferLike>,
+  ): Promise<Uint8Array> | Uint8Array;
+
   /**
    * Consume all data from a {@link ReadableStream} until it closes or errors.
    *

packages/bun-types/test/globals.test.ts

@@ -6,6 +6,7 @@ import { expectAssignable, expectType } from "./utilities.test";
 // FileBlob
 expectType<ReadableStream<Uint8Array>>(Bun.file("index.test-d.ts").stream());
 expectType<Promise<ArrayBuffer>>(Bun.file("index.test-d.ts").arrayBuffer());
+expectType<Promise<Uint8Array>>(Bun.file("index.test-d.ts").bytes());
 expectType<Promise<string>>(Bun.file("index.test-d.ts").text());
 
 expectType<number>(Bun.file("index.test-d.ts").size);

src/bun.js/api/BunObject.zig

@@ -2418,7 +2418,7 @@ pub const Crypto = struct {
                 return output_buf.value;
             } else {
                 // Clone to GC-managed memory
-                return JSC.ArrayBuffer.create(globalThis, output_digest_slice[0..len], .Buffer);
+                return JSC.ArrayBuffer.createBuffer(globalThis, output_digest_slice[0..len]);
             }
         }
 
@@ -2590,7 +2590,7 @@ pub const Crypto = struct {
                 return output_buf.value;
             } else {
                 // Clone to GC-managed memory
-                return JSC.ArrayBuffer.create(globalThis, result, .Buffer);
+                return JSC.ArrayBuffer.createBuffer(globalThis, result);
             }
         }
 
@@ -2695,7 +2695,7 @@ pub const Crypto = struct {
                 var out: [Algorithm.digest_length]u8 = undefined;
                 h.final(&out);
                 // Clone to GC-managed memory
-                return JSC.ArrayBuffer.create(globalThis, &out, .Buffer);
+                return JSC.ArrayBuffer.createBuffer(globalThis, &out);
             }
         }
 

src/bun.js/api/brotli.zig

@@ -92,7 +92,7 @@ pub const BrotliEncoder = struct {
         defer this.output_lock.unlock();
 
         defer this.output.clearRetainingCapacity();
-        return JSC.ArrayBuffer.create(this.globalThis, this.output.items, .Buffer);
+        return JSC.ArrayBuffer.createBuffer(this.globalThis, this.output.items);
     }
 
     pub fn runFromJSThread(this: *BrotliEncoder) void {
@@ -383,7 +383,7 @@ pub const BrotliDecoder = struct {
         defer this.output_lock.unlock();
 
         defer this.output.clearRetainingCapacity();
-        return JSC.ArrayBuffer.create(this.globalThis, this.output.items, .Buffer);
+        return JSC.ArrayBuffer.createBuffer(this.globalThis, this.output.items);
     }
 
     pub fn runFromJSThread(this: *BrotliDecoder) void {

src/bun.js/api/server.zig

@@ -4064,10 +4064,9 @@ pub const ServerWebSocket = struct {
 
     fn binaryToJS(this: *const ServerWebSocket, globalThis: *JSC.JSGlobalObject, data: []const u8) JSC.JSValue {
         return switch (this.flags.binary_type) {
-            .Buffer => JSC.ArrayBuffer.create(
+            .Buffer => JSC.ArrayBuffer.createBuffer(
                 globalThis,
                 data,
-                .Buffer,
             ),
             .Uint8Array => JSC.ArrayBuffer.create(
                 globalThis,

src/bun.js/base.zig

@@ -312,7 +312,7 @@ pub const ArrayBuffer = extern struct {
         return buffer_value;
     }
 
-    extern fn ArrayBuffer__fromSharedMemfd(fd: i64, globalObject: *JSC.JSGlobalObject, byte_offset: usize, byte_length: usize, total_size: usize) JSC.JSValue;
+    extern fn ArrayBuffer__fromSharedMemfd(fd: i64, globalObject: *JSC.JSGlobalObject, byte_offset: usize, byte_length: usize, total_size: usize, JSC.JSValue.JSType) JSC.JSValue;
     pub const toArrayBufferFromSharedMemfd = ArrayBuffer__fromSharedMemfd;
 
     pub fn toJSBufferFromMemfd(fd: bun.FileDescriptor, globalObject: *JSC.JSGlobalObject) JSC.JSValue {
@@ -384,11 +384,10 @@ pub const ArrayBuffer = extern struct {
         return Stream{ .pos = 0, .buf = this.slice() };
     }
 
-    pub fn create(globalThis: *JSC.JSGlobalObject, bytes: []const u8, comptime kind: BinaryType) JSValue {
+    pub fn create(globalThis: *JSC.JSGlobalObject, bytes: []const u8, comptime kind: JSValue.JSType) JSValue {
         JSC.markBinding(@src());
         return switch (comptime kind) {
             .Uint8Array => Bun__createUint8ArrayForCopy(globalThis, bytes.ptr, bytes.len, false),
-            .Buffer => Bun__createUint8ArrayForCopy(globalThis, bytes.ptr, bytes.len, true),
             .ArrayBuffer => Bun__createArrayBufferForCopy(globalThis, bytes.ptr, bytes.len),
             else => @compileError("Not implemented yet"),
         };