feat: support redirects for UnixFS directories

Adds support for simulating redirects for UnixFS directories.

We're somewhat in uncharted water here because window.fetch does this
transparently unless you specify a `redirect` option, none of which
actually allow you to follow a redirect.

The states we can be in are:

1. URL: `ipfs://QmFoo/dir/`
  - Happy path
  - 200 response
  - `response.redirected = false`
  - `response.url = 'ipfs://QmFoo/dir'`

2: URL: `ipfs://QmFoo/dir`, `redirect: 'follow'`
  - The default value
  - Simulates automatically following a redirect
  - 200 response
  - `response.redirected = true`
  - `response.url = 'ipfs://QmFoo/dir/'`

3: URL: `ipfs://QmFoo/dir`, `redirect: 'error'`
  - Return an error if a redirect would take place
  - Throws `TypeError('Failed to Fetch')` same as `window.fetch`

4: URL: `ipfs://QmFoo/dir`, `redirect: 'manual'`
  - Allows a caller to take action on the redirect
  - 301 response
  - `response.redirected = false`
  - `response.url = 'ipfs://QmFoo/dir`
  - `response.headers.get('location') = 'ipfs://QmFoo/dir/'`

Number 4 is the furthest from [the fetch spec](https://fetch.spec.whatwg.org/#concept-request-redirect-mode)
but to follow the spec would make it impossible to actually follow a
redirect.

packages/verified-fetch/src/utils/responses.ts

@@ -1,29 +1,98 @@
-export function okResponse (body?: BodyInit | null): Response {
-  return new Response(body, {
+function setField (response: Response, name: string, value: string | boolean): void {
+  Object.defineProperty(response, name, {
+    enumerable: true,
+    configurable: false,
+    set: () => {},
+    get: () => value
+  })
+}
+
+function setType (response: Response, value: 'basic' | 'cors' | 'error' | 'opaque' | 'opaqueredirect'): void {
+  setField(response, 'type', value)
+}
+
+function setUrl (response: Response, value: string): void {
+  setField(response, 'url', value)
+}
+
+function setRedirected (response: Response): void {
+  setField(response, 'redirected', true)
+}
+
+export interface ResponseOptions extends ResponseInit {
+  redirected?: boolean
+}
+
+export function okResponse (url: string, body?: BodyInit | null, init?: ResponseOptions): Response {
+  const response = new Response(body, {
+    ...(init ?? {}),
     status: 200,
     statusText: 'OK'
   })
+
+  if (init?.redirected === true) {
+    setRedirected(response)
+  }
+
+  setType(response, 'basic')
+  setUrl(response, url)
+
+  return response
 }
 
-export function notSupportedResponse (body?: BodyInit | null): Response {
+export function notSupportedResponse (url: string, body?: BodyInit | null, init?: ResponseInit): Response {
   const response = new Response(body, {
+    ...(init ?? {}),
     status: 501,
     statusText: 'Not Implemented'
   })
   response.headers.set('X-Content-Type-Options', 'nosniff') // see https://specs.ipfs.tech/http-gateways/path-gateway/#x-content-type-options-response-header
+
+  setType(response, 'basic')
+  setUrl(response, url)
+
   return response
 }
 
-export function notAcceptableResponse (body?: BodyInit | null): Response {
-  return new Response(body, {
+export function notAcceptableResponse (url: string, body?: BodyInit | null, init?: ResponseInit): Response {
+  const response = new Response(body, {
+    ...(init ?? {}),
     status: 406,
     statusText: 'Not Acceptable'
   })
+
+  setType(response, 'basic')
+  setUrl(response, url)
+
+  return response
 }
 
-export function badRequestResponse (body?: BodyInit | null): Response {
-  return new Response(body, {
+export function badRequestResponse (url: string, body?: BodyInit | null, init?: ResponseInit): Response {
+  const response = new Response(body, {
+    ...(init ?? {}),
     status: 400,
     statusText: 'Bad Request'
   })
+
+  setType(response, 'basic')
+  setUrl(response, url)
+
+  return response
+}
+
+export function movedPermanentlyResponse (url: string, location: string, init?: ResponseInit): Response {
+  const response = new Response(null, {
+    ...(init ?? {}),
+    status: 301,
+    statusText: 'Moved Permanently',
+    headers: {
+      ...(init?.headers ?? {}),
+      location
+    }
+  })
+
+  setType(response, 'basic')
+  setUrl(response, url)
+
+  return response
 }

packages/verified-fetch/src/verified-fetch.ts

@@ -22,7 +22,7 @@ import { getETag } from './utils/get-e-tag.js'
 import { getStreamFromAsyncIterable } from './utils/get-stream-from-async-iterable.js'
 import { tarStream } from './utils/get-tar-stream.js'
 import { parseResource } from './utils/parse-resource.js'
-import { badRequestResponse, notAcceptableResponse, notSupportedResponse, okResponse } from './utils/responses.js'
+import { badRequestResponse, movedPermanentlyResponse, notAcceptableResponse, notSupportedResponse, okResponse } from './utils/responses.js'
 import { selectOutputType, queryFormatToAcceptHeader } from './utils/select-output-type.js'
 import { walkPath } from './utils/walk-path.js'
 import type { CIDDetail, ContentTypeParser, Resource, VerifiedFetchInit as VerifiedFetchOptions } from './index.js'
@@ -167,7 +167,7 @@ export class VerifiedFetch {
     const buf = await this.helia.datastore.get(datastoreKey, options)
     const record = DHTRecord.deserialize(buf)
 
-    const response = okResponse(record.value)
+    const response = okResponse(resource, record.value)
     response.headers.set('content-type', 'application/vnd.ipfs.ipns-record')
 
     return response
@@ -177,11 +177,11 @@ export class VerifiedFetch {
    * Accepts a `CID` and returns a `Response` with a body stream that is a CAR
    * of the `DAG` referenced by the `CID`.
    */
-  private async handleCar ({ cid, options }: FetchHandlerFunctionArg): Promise<Response> {
+  private async handleCar ({ resource, cid, options }: FetchHandlerFunctionArg): Promise<Response> {
     const c = car(this.helia)
     const stream = toBrowserReadableStream(c.stream(cid, options))
 
-    const response = okResponse(stream)
+    const response = okResponse(resource, stream)
     response.headers.set('content-type', 'application/vnd.ipld.car; version=1')
 
     return response
@@ -191,20 +191,20 @@ export class VerifiedFetch {
    * Accepts a UnixFS `CID` and returns a `.tar` file containing the file or
    * directory structure referenced by the `CID`.
    */
-  private async handleTar ({ cid, path, options }: FetchHandlerFunctionArg): Promise<Response> {
+  private async handleTar ({ resource, cid, path, options }: FetchHandlerFunctionArg): Promise<Response> {
     if (cid.code !== dagPbCode && cid.code !== rawCode) {
       return notAcceptableResponse('only UnixFS data can be returned in a TAR file')
     }
 
     const stream = toBrowserReadableStream<Uint8Array>(tarStream(`/ipfs/${cid}/${path}`, this.helia.blockstore, options))
 
-    const response = okResponse(stream)
+    const response = okResponse(resource, stream)
     response.headers.set('content-type', 'application/x-tar')
 
     return response
   }
 
-  private async handleJson ({ cid, path, accept, options }: FetchHandlerFunctionArg): Promise<Response> {
+  private async handleJson ({ resource, cid, path, accept, options }: FetchHandlerFunctionArg): Promise<Response> {
     this.log.trace('fetching %c/%s', cid, path)
     const block = await this.helia.blockstore.get(cid, options)
     let body: string | Uint8Array
@@ -218,19 +218,19 @@ export class VerifiedFetch {
         body = ipldDagCbor.encode(obj)
       } catch (err) {
         this.log.error('could not transform %c to application/vnd.ipld.dag-cbor', err)
-        return notAcceptableResponse()
+        return notAcceptableResponse(resource)
       }
     } else {
       // skip decoding
       body = block
     }
 
-    const response = okResponse(body)
+    const response = okResponse(resource, body)
     response.headers.set('content-type', accept ?? 'application/json')
     return response
   }
 
-  private async handleDagCbor ({ cid, path, accept, options }: FetchHandlerFunctionArg): Promise<Response> {
+  private async handleDagCbor ({ resource, cid, path, accept, options }: FetchHandlerFunctionArg): Promise<Response> {
     this.log.trace('fetching %c/%s', cid, path)
 
     const block = await this.helia.blockstore.get(cid, options)
@@ -248,7 +248,7 @@ export class VerifiedFetch {
         body = ipldDagJson.encode(obj)
       } catch (err) {
         this.log.error('could not transform %c to application/vnd.ipld.dag-json', err)
-        return notAcceptableResponse()
+        return notAcceptableResponse(resource)
       }
     } else {
       try {
@@ -257,15 +257,15 @@ export class VerifiedFetch {
         if (accept === 'application/json') {
           this.log('could not decode DAG-CBOR as JSON-safe, but the client sent "Accept: application/json"', err)
 
-          return notAcceptableResponse()
+          return notAcceptableResponse(resource)
         }
 
         this.log('could not decode DAG-CBOR as JSON-safe, falling back to `application/octet-stream`', err)
         body = block
       }
     }
 
-    const response = okResponse(body)
+    const response = okResponse(resource, body)
 
     if (accept == null) {
       accept = body instanceof Uint8Array ? 'application/octet-stream' : 'application/json'
@@ -276,9 +276,10 @@ export class VerifiedFetch {
     return response
   }
 
-  private async handleDagPb ({ cid, path, options }: FetchHandlerFunctionArg): Promise<Response> {
+  private async handleDagPb ({ cid, path, resource, options }: FetchHandlerFunctionArg): Promise<Response> {
     let terminalElement: UnixFSEntry | undefined
     let ipfsRoots: CID[] | undefined
+    let redirected = false
 
     try {
       const pathDetails = await walkPath(this.helia.blockstore, `${cid.toString()}/${path}`, options)
@@ -293,6 +294,21 @@ export class VerifiedFetch {
     if (terminalElement?.type === 'directory') {
       const dirCid = terminalElement.cid
 
+      // https://specs.ipfs.tech/http-gateways/path-gateway/#use-in-directory-url-normalization
+      if (path !== '' && !path.endsWith('/')) {
+        if (options?.redirect === 'error') {
+          this.log('could not redirect to %s/ as redirect option was set to "error"', resource)
+          throw new TypeError('Failed to fetch')
+        } else if (options?.redirect === 'manual') {
+          this.log('returning 301 permanent redirect to %s/', resource)
+          return movedPermanentlyResponse(resource, `${resource}/`)
+        }
+
+        // fall-through simulates following the redirect?
+        resource = `${resource}/`
+        redirected = true
+      }
+
       const rootFilePath = 'index.html'
       try {
         this.log.trace('found directory at %c/%s, looking for index.html', cid, path)
@@ -304,7 +320,6 @@ export class VerifiedFetch {
         this.log.trace('found root file at %c/%s with cid %c', dirCid, rootFilePath, stat.cid)
         path = rootFilePath
         resolvedCID = stat.cid
-        // terminalElement = stat
       } catch (err: any) {
         this.log('error loading path %c/%s', dirCid, rootFilePath, err)
         return notSupportedResponse('Unable to find index.html for directory at given path. Support for directories with implicit root is not implemented')
@@ -322,7 +337,9 @@ export class VerifiedFetch {
     const { stream, firstChunk } = await getStreamFromAsyncIterable(asyncIter, path ?? '', this.helia.logger, {
       onProgress: options?.onProgress
     })
-    const response = okResponse(stream)
+    const response = okResponse(resource, stream, {
+      redirected
+    })
     await this.setContentType(firstChunk, path, response)
 
     if (ipfsRoots != null) {
@@ -332,9 +349,9 @@ export class VerifiedFetch {
     return response
   }
 
-  private async handleRaw ({ cid, path, options }: FetchHandlerFunctionArg): Promise<Response> {
+  private async handleRaw ({ resource, cid, path, options }: FetchHandlerFunctionArg): Promise<Response> {
     const result = await this.helia.blockstore.get(cid, options)
-    const response = okResponse(result)
+    const response = okResponse(resource, result)
 
     // if the user has specified an `Accept` header that corresponds to a raw
     // type, honour that header, so for example they don't request
@@ -418,7 +435,7 @@ export class VerifiedFetch {
     this.log('output type %s', accept)
 
     if (acceptHeader != null && accept == null) {
-      return notAcceptableResponse()
+      return notAcceptableResponse(resource.toString())
     }
 
     let response: Response

packages/verified-fetch/test/verified-fetch.spec.ts

@@ -133,6 +133,84 @@ describe('@helia/verifed-fetch', () => {
       expect(new Uint8Array(data)).to.equalBytes(finalRootFileContent)
     })
 
+    it('should return a 301 with a trailing slash when a directory is requested without a trailing slash', async () => {
+      const finalRootFileContent = new Uint8Array([0x01, 0x02, 0x03])
+
+      const fs = unixfs(helia)
+      const res = await last(fs.addAll([{
+        path: 'foo/index.html',
+        content: finalRootFileContent
+      }], {
+        wrapWithDirectory: true
+      }))
+
+      if (res == null) {
+        throw new Error('Import failed')
+      }
+
+      const stat = await fs.stat(res.cid)
+      expect(stat.type).to.equal('directory')
+
+      const ipfsResponse = await verifiedFetch.fetch(`ipfs://${res.cid}/foo`, {
+        redirect: 'manual'
+      })
+      expect(ipfsResponse).to.be.ok()
+      expect(ipfsResponse.status).to.equal(301)
+      expect(ipfsResponse.headers.get('location')).to.equal(`ipfs://${res.cid}/foo/`)
+      expect(ipfsResponse.url).to.equal(`ipfs://${res.cid}/foo`)
+    })
+
+    it('should simulate following a redirect to a path with a slash when a directory is requested without a trailing slash', async () => {
+      const finalRootFileContent = new Uint8Array([0x01, 0x02, 0x03])
+
+      const fs = unixfs(helia)
+      const res = await last(fs.addAll([{
+        path: 'foo/index.html',
+        content: finalRootFileContent
+      }], {
+        wrapWithDirectory: true
+      }))
+
+      if (res == null) {
+        throw new Error('Import failed')
+      }
+
+      const stat = await fs.stat(res.cid)
+      expect(stat.type).to.equal('directory')
+
+      const ipfsResponse = await verifiedFetch.fetch(`ipfs://${res.cid}/foo`)
+      expect(ipfsResponse).to.be.ok()
+      expect(ipfsResponse.type).to.equal('basic')
+      expect(ipfsResponse.status).to.equal(200)
+      expect(ipfsResponse.redirected).to.be.true()
+      expect(ipfsResponse.url).to.equal(`ipfs://${res.cid}/foo/`)
+    })
+
+    it('should not redirect when a directory is requested with a trailing slash', async () => {
+      const finalRootFileContent = new Uint8Array([0x01, 0x02, 0x03])
+
+      const fs = unixfs(helia)
+      const res = await last(fs.addAll([{
+        path: 'foo/index.html',
+        content: finalRootFileContent
+      }], {
+        wrapWithDirectory: true
+      }))
+
+      if (res == null) {
+        throw new Error('Import failed')
+      }
+
+      const stat = await fs.stat(res.cid)
+      expect(stat.type).to.equal('directory')
+
+      const ipfsResponse = await verifiedFetch.fetch(`ipfs://${res.cid}/foo/`)
+      expect(ipfsResponse).to.be.ok()
+      expect(ipfsResponse.status).to.equal(200)
+      expect(ipfsResponse.redirected).to.be.false()
+      expect(ipfsResponse.url).to.equal(`ipfs://${res.cid}/foo/`)
+    })
+
     it('should allow use as a stream', async () => {
       const content = new Uint8Array([0x01, 0x02, 0x03])