feat: support http range header

packages/verified-fetch/src/types.ts

@@ -1 +1,3 @@
 export type RequestFormatShorthand = 'raw' | 'car' | 'tar' | 'ipns-record' | 'dag-json' | 'dag-cbor' | 'json' | 'cbor'
+
+export type SupportedBodyTypes = string | ArrayBuffer | Blob | ReadableStream<Uint8Array> | null

packages/verified-fetch/src/utils/request-headers.ts

@@ -0,0 +1,86 @@
+import { type CatOptions } from '@helia/unixfs'
+import { CodeError } from '@libp2p/interface'
+import { type ExporterOptions } from 'ipfs-unixfs-exporter'
+
+export function getHeader (headers: HeadersInit | undefined, header: string): string | undefined {
+  if (headers == null) {
+    return undefined
+  }
+  if (headers instanceof Headers) {
+    return headers.get(header) ?? undefined
+  }
+  if (Array.isArray(headers)) {
+    const entry = headers.find(([key]) => key.toLowerCase() === header.toLowerCase())
+    return entry?.[1]
+  }
+  const key = Object.keys(headers).find(k => k.toLowerCase() === header.toLowerCase())
+  if (key == null) {
+    return undefined
+  }
+
+  return headers[key]
+}
+
+/**
+ * Type abstraction that will break the build if the supported range options change.
+ */
+export interface HeliaRangeOptions extends Pick<ExporterOptions | CatOptions, 'offset' | 'length'> {
+  suffixLength?: number
+}
+
+/**
+ * Converts a range request header into helia/unixfs supported range options
+ * Note that the gateway specification says we "MAY" support multiple ranges (https://specs.ipfs.tech/http-gateways/path-gateway/#range-request-header) but we don't
+ *
+ * Also note that @helia/unixfs and ipfs-unixfs-exporter expect length and offset to be numbers, the range header is a string, and the size of the resource is likely a bigint.
+ *
+ * SUPPORTED:
+ * Range: bytes=<range-start>-<range-end>
+ * Range: bytes=<range-start>-
+ * Range: bytes=-<suffix-length> // must pass size so we can calculate the offset
+ *
+ * NOT SUPPORTED:
+ * Range: bytes=<range-start>-<range-end>, <range-start>-<range-end>
+ * Range: bytes=<range-start>-<range-end>, <range-start>-<range-end>, <range-start>-<range-end>
+ *
+ * @see https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Range#directives
+ */
+export function getRequestRange (headers: Headers | HeadersInit | undefined, size?: bigint): HeliaRangeOptions | undefined {
+  const range = getHeader(headers, 'Range')
+  if (range == null) {
+    return undefined
+  }
+
+  /**
+   * Range: bytes=<start>-<end> | bytes=<start2>- | bytes=-<end2>
+   */
+  const match = range.match(/^bytes=((?<start>\d+)-(?<end>\d+)|(?<start2>\d+)-|-(?<end2>\d+))$/)
+  if (match?.groups == null) {
+    throw new CodeError('ERR_INVALID_RANGE_REQUEST', 'Invalid range request')
+  }
+  const { start, end, start2, end2 } = match.groups
+
+  let offset: number | undefined
+  let length: number | undefined
+  let suffixLength: number | undefined
+  if (end2 != null) {
+    if (size == null) {
+      suffixLength = Number(end2)
+    } else {
+      const stop = BigInt(end2)
+      offset = Number(size - stop)
+      length = Number(stop)
+    }
+  } else if (start2 != null) {
+    offset = parseInt(start2)
+  } else {
+    offset = parseInt(start)
+    length = parseInt(end) - offset + 1
+  }
+
+  return {
+    offset,
+    length,
+    suffixLength
+  }
+}

packages/verified-fetch/src/utils/response-headers.ts

@@ -0,0 +1,53 @@
+import { CodeError } from '@libp2p/interface'
+import type { SupportedBodyTypes } from '../types.js'
+
+/**
+ * Gets the body size of a given body if it's possible to calculate it synchronously.
+ */
+function syncBodySize (body: SupportedBodyTypes): number | null {
+  if (typeof body === 'string') {
+    return body.length
+  }
+  if (body instanceof ArrayBuffer || body instanceof Uint8Array) {
+    return body.byteLength
+  }
+  if (body instanceof Blob) {
+    return body.size
+  }
+
+  if (body instanceof ReadableStream) {
+    return null
+  }
+
+  return null
+}
+
+/**
+ * This function returns the value of the `Content-Range` header for a given range.
+ * If you know the total size of the body, you should pass it in the `options` object.
+ *
+ * offset and length are 0-based, and the range is inclusive.
+ *
+ * @see https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Range
+ */
+export function getContentRangeHeader ({ body, ...opts }: { offset?: number, length?: number, total?: number, body: SupportedBodyTypes }): string {
+  const totalSizeNum = opts.total ?? syncBodySize(body)
+  const rangeStart = opts.offset ?? 0
+  let rangeEnd = opts.length
+
+  if (rangeEnd == null) {
+    if (totalSizeNum == null) {
+      throw new CodeError('Cannot calculate content range without total size or length', 'ERR_INVALID_CONTENT_RANGE')
+    }
+    rangeEnd = totalSizeNum - rangeStart + 1
+  }
+  let end: number
+  if (rangeStart != null && rangeEnd != null) {
+    end = rangeStart + rangeEnd - 1
+  } else {
+    end = totalSizeNum ?? 0
+  }
+  const total = totalSizeNum ?? '*' // if we don't know the total size, we should use *
+
+  return `bytes ${rangeStart}-${end}/${total}`
+}

packages/verified-fetch/src/utils/responses.ts

@@ -1,3 +1,6 @@
+import { getContentRangeHeader } from './response-headers.js'
+import type { SupportedBodyTypes } from '../types.js'
+
 function setField (response: Response, name: string, value: string | boolean): void {
   Object.defineProperty(response, name, {
     enumerable: true,
@@ -23,7 +26,7 @@
   redirected?: boolean
 }
 
-export function okResponse (url: string, body?: BodyInit | null, init?: ResponseOptions): Response {
+export function okResponse (url: string, body?: SupportedBodyTypes, init?: ResponseOptions): Response {
   const response = new Response(body, {
     ...(init ?? {}),
     status: 200,
@@ -40,7 +43,20 @@
   return response
 }
 
-export function notSupportedResponse (url: string, body?: BodyInit | null, init?: ResponseInit): Response {
+export function internalServerErrorResponse (url: string, body?: SupportedBodyTypes, init?: ResponseInit): Response {
+  const response = new Response(body, {
+    ...(init ?? {}),
+    status: 500,
+    statusText: 'Internal Server Error'
+  })
+
+  setType(response, 'basic')
+  setUrl(response, url)
+
+  return response
+}
+
+export function notSupportedResponse (url: string, body?: SupportedBodyTypes, init?: ResponseInit): Response {
   const response = new Response(body, {
     ...(init ?? {}),
     status: 501,
@@ -54,7 +70,7 @@
   return response
 }
 
-export function notAcceptableResponse (url: string, body?: BodyInit | null, init?: ResponseInit): Response {
+export function notAcceptableResponse (url: string, body?: SupportedBodyTypes, init?: ResponseInit): Response {
   const response = new Response(body, {
     ...(init ?? {}),
     status: 406,
@@ -67,7 +83,7 @@
   return response
 }
 
-export function badRequestResponse (url: string, body?: BodyInit | null, init?: ResponseInit): Response {
+export function badRequestResponse (url: string, body?: SupportedBodyTypes, init?: ResponseInit): Response {
   const response = new Response(body, {
     ...(init ?? {}),
     status: 400,
@@ -96,3 +112,63 @@
 
   return response
 }
+
+type RangeOptions = { offset: number, length: number, totalSize?: number } | { offset: number, length?: number, totalSize: number }
+
+/**
+ * Some caveats about range responses here:
+ * * We only support single range requests (multi-range is optional), see https://specs.ipfs.tech/http-gateways/path-gateway/#range-request-header
+ * * Range responses are only supported for unixfs and raw data, see https://specs.ipfs.tech/http-gateways/path-gateway/#range-request-header.
+ *
+ * If the user requests something other than unixfs or raw data, we should not call this method and ignore the range header (200 OK). See https://developer.mozilla.org/en-US/docs/Web/HTTP/Range_requests#partial_request_responses
+ *
+ * TODO: Supporting multiple range requests will require additional changes to the `handleDagPb` and `handleRaw` functions in `src/verified-fetch.js`
+ */
+export function okRangeResponse (url: string, body: SupportedBodyTypes, range: RangeOptions, init?: ResponseOptions): Response {
+  // if we know the full size of the body, we should use it in the content-range header. See https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Range
+  let contentRangeHeader: string | undefined
+
+  try {
+    contentRangeHeader = getContentRangeHeader({ body, total: range.totalSize, offset: range.offset, length: range.length })
+  } catch (e) {
+    return badRangeResponse(url, body, init)
+  }
+
+  const response = new Response(body, {
+    ...(init ?? {}),
+    status: 206,
+    statusText: 'Partial Content',
+    headers: {
+      ...(init?.headers ?? {}),
+      'content-range': contentRangeHeader
+    }
+  })
+
+  if (init?.redirected === true) {
+    setRedirected(response)
+  }
+
+  setType(response, 'basic')
+  setUrl(response, url)
+
+  return response
+}
+
+/**
+ * We likely need to catch errors handled by upstream helia libraries if range-request throws an error. Some examples:
+ * * The range is out of bounds
+ * * The range is invalid
+ * * The range is not supported for the given type
+ */
+export function badRangeResponse (url: string, body?: SupportedBodyTypes, init?: ResponseInit): Response {
+  const response = new Response(body, {
+    ...(init ?? {}),
+    status: 416,
+    statusText: 'Requested Range Not Satisfiable'
+  })
+
+  setType(response, 'basic')
+  setUrl(response, url)
+
+  return response
+}