From 99668cef249f223e143e359fb282c00d98b82f28 Mon Sep 17 00:00:00 2001 From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com> Date: Mon, 8 Apr 2024 11:26:36 -0700 Subject: [PATCH] fix: walking dag-cbor paths (#39) * fix: implicit accept header can be overridden by format query * chore: some cleanup and optimizations * chore: forgot to add files * fix: walking dag-cbor paths * chore: suggestions from code review Co-authored-by: Alex Potsides * chore: use signal.throwIfAborted --------- Co-authored-by: Alex Potsides --- .../verified-fetch/src/utils/responses.ts | 13 +++++ .../verified-fetch/src/utils/walk-path.ts | 9 ++- packages/verified-fetch/src/verified-fetch.ts | 55 +++++++++++++------ .../test/verified-fetch.spec.ts | 31 +++++++++++ 4 files changed, 89 insertions(+), 19 deletions(-) diff --git a/packages/verified-fetch/src/utils/responses.ts b/packages/verified-fetch/src/utils/responses.ts index dda0230d..1c2d85e2 100644 --- a/packages/verified-fetch/src/utils/responses.ts +++ b/packages/verified-fetch/src/utils/responses.ts @@ -85,6 +85,19 @@ export function notAcceptableResponse (url: string, body?: SupportedBodyTypes, i return response } +export function notFoundResponse (url: string, body?: SupportedBodyTypes, init?: ResponseInit): Response { + const response = new Response(body, { + ...(init ?? {}), + status: 404, + statusText: 'Not Found' + }) + + setType(response, 'basic') + setUrl(response, url) + + return response +} + /** * if body is an Error, it will be converted to a string containing the error message. */ diff --git a/packages/verified-fetch/src/utils/walk-path.ts b/packages/verified-fetch/src/utils/walk-path.ts index 45f2066e..be46fdf2 100644 --- a/packages/verified-fetch/src/utils/walk-path.ts +++ b/packages/verified-fetch/src/utils/walk-path.ts @@ -1,4 +1,5 @@ -import { walkPath as exporterWalk, type ExporterOptions, type ReadableStorage, type UnixFSEntry } from 'ipfs-unixfs-exporter' +import { CodeError } from '@libp2p/interface' +import { walkPath as exporterWalk, type ExporterOptions, type ReadableStorage, type ObjectNode, type UnixFSEntry } from 'ipfs-unixfs-exporter' import type { CID } from 'multiformats/cid' export interface PathWalkerOptions extends ExporterOptions { @@ -24,7 +25,7 @@ export async function walkPath (blockstore: ReadableStorage, path: string, optio } if (terminalElement == null) { - throw new Error('No terminal element found') + throw new CodeError('No terminal element found', 'ERR_NO_TERMINAL_ELEMENT') } return { @@ -32,3 +33,7 @@ export async function walkPath (blockstore: ReadableStorage, path: string, optio terminalElement } } + +export function isObjectNode (node: UnixFSEntry): node is ObjectNode { + return node.type === 'object' +} diff --git a/packages/verified-fetch/src/verified-fetch.ts b/packages/verified-fetch/src/verified-fetch.ts index 901479e3..8e744d69 100644 --- a/packages/verified-fetch/src/verified-fetch.ts +++ b/packages/verified-fetch/src/verified-fetch.ts @@ -4,7 +4,7 @@ import { unixfs as heliaUnixFs, type UnixFS as HeliaUnixFs } from '@helia/unixfs import * as ipldDagCbor from '@ipld/dag-cbor' import * as ipldDagJson from '@ipld/dag-json' import { code as dagPbCode } from '@ipld/dag-pb' -import { AbortError, type AbortOptions, type Logger, type PeerId } from '@libp2p/interface' +import { type AbortOptions, type Logger, type PeerId } from '@libp2p/interface' import { Record as DHTRecord } from '@libp2p/kad-dht' import { peerIdFromString } from '@libp2p/peer-id' import { Key } from 'interface-datastore' @@ -25,15 +25,15 @@ import { getStreamFromAsyncIterable } from './utils/get-stream-from-async-iterab import { tarStream } from './utils/get-tar-stream.js' import { parseResource } from './utils/parse-resource.js' import { setCacheControlHeader } from './utils/response-headers.js' -import { badRequestResponse, movedPermanentlyResponse, notAcceptableResponse, notSupportedResponse, okResponse, badRangeResponse, okRangeResponse, badGatewayResponse } from './utils/responses.js' +import { badRequestResponse, movedPermanentlyResponse, notAcceptableResponse, notSupportedResponse, okResponse, badRangeResponse, okRangeResponse, badGatewayResponse, notFoundResponse } from './utils/responses.js' import { selectOutputType } from './utils/select-output-type.js' -import { walkPath } from './utils/walk-path.js' +import { isObjectNode, walkPath } from './utils/walk-path.js' import type { CIDDetail, ContentTypeParser, Resource, VerifiedFetchInit as VerifiedFetchOptions } from './index.js' import type { RequestFormatShorthand } from './types.js' import type { ParsedUrlStringResults } from './utils/parse-url-string' import type { Helia } from '@helia/interface' import type { DNSResolver } from '@multiformats/dns/resolvers' -import type { UnixFSEntry } from 'ipfs-unixfs-exporter' +import type { ObjectNode, UnixFSEntry } from 'ipfs-unixfs-exporter' import type { CID } from 'multiformats/cid' interface VerifiedFetchComponents { @@ -236,8 +236,31 @@ export class VerifiedFetch { private async handleDagCbor ({ resource, cid, path, accept, options }: FetchHandlerFunctionArg): Promise { this.log.trace('fetching %c/%s', cid, path) + let terminalElement: ObjectNode | undefined + let ipfsRoots: CID[] | undefined + + // need to walk path, if it exists, to get the terminal element + try { + const pathDetails = await walkPath(this.helia.blockstore, `${cid.toString()}/${path}`, options) + ipfsRoots = pathDetails.ipfsRoots + const potentialTerminalElement = pathDetails.terminalElement + if (potentialTerminalElement == null) { + return notFoundResponse(resource) + } + if (isObjectNode(potentialTerminalElement)) { + terminalElement = potentialTerminalElement + } + } catch (err: any) { + options?.signal?.throwIfAborted() + if (['ERR_NO_PROP', 'ERR_NO_TERMINAL_ELEMENT'].includes(err.code)) { + return notFoundResponse(resource) + } + + this.log.error('error walking path %s', path, err) + return badGatewayResponse(resource, 'Error walking path') + } + const block = terminalElement?.node ?? await this.helia.blockstore.get(cid, options) - const block = await this.helia.blockstore.get(cid, options) let body: string | Uint8Array if (accept === 'application/octet-stream' || accept === 'application/vnd.ipld.dag-cbor' || accept === 'application/cbor') { @@ -277,6 +300,10 @@ export class VerifiedFetch { response.headers.set('content-type', accept) + if (ipfsRoots != null) { + response.headers.set('X-Ipfs-Roots', ipfsRoots.map(cid => cid.toV1().toString()).join(',')) // https://specs.ipfs.tech/http-gateways/path-gateway/#x-ipfs-roots-response-header + } + return response } @@ -291,8 +318,9 @@ export class VerifiedFetch { ipfsRoots = pathDetails.ipfsRoots terminalElement = pathDetails.terminalElement } catch (err: any) { - if (options?.signal?.aborted === true) { - throw new AbortError('signal aborted by user') + options?.signal?.throwIfAborted() + if (['ERR_NO_PROP', 'ERR_NO_TERMINAL_ELEMENT'].includes(err.code)) { + return notFoundResponse(resource.toString()) } this.log.error('error walking path %s', path, err) @@ -331,9 +359,7 @@ export class VerifiedFetch { path = rootFilePath resolvedCID = stat.cid } catch (err: any) { - if (options?.signal?.aborted === true) { - throw new AbortError('signal aborted by user') - } + options?.signal?.throwIfAborted() this.log('error loading path %c/%s', dirCid, rootFilePath, err) return notSupportedResponse('Unable to find index.html for directory at given path. Support for directories with implicit root is not implemented') } finally { @@ -377,9 +403,7 @@ export class VerifiedFetch { return response } catch (err: any) { - if (options?.signal?.aborted === true) { - throw new AbortError('signal aborted by user') - } + options?.signal?.throwIfAborted() this.log.error('error streaming %c/%s', cid, path, err) if (byteRangeContext.isRangeRequest && err.code === 'ERR_INVALID_PARAMS') { return badRangeResponse(resource) @@ -455,7 +479,6 @@ export class VerifiedFetch { * TODO: move operations called by fetch to a queue of operations where we can * always exit early (and cleanly) if a given signal is aborted */ - // eslint-disable-next-line complexity async fetch (resource: Resource, opts?: VerifiedFetchOptions): Promise { this.log('fetch %s', resource) @@ -477,9 +500,7 @@ export class VerifiedFetch { ttl = result.ttl protocol = result.protocol } catch (err: any) { - if (options?.signal?.aborted === true) { - throw new AbortError('signal aborted by user') - } + options?.signal?.throwIfAborted() this.log.error('error parsing resource %s', resource, err) return badRequestResponse(resource.toString(), err) diff --git a/packages/verified-fetch/test/verified-fetch.spec.ts b/packages/verified-fetch/test/verified-fetch.spec.ts index 52b086b9..9e202749 100644 --- a/packages/verified-fetch/test/verified-fetch.spec.ts +++ b/packages/verified-fetch/test/verified-fetch.spec.ts @@ -807,4 +807,35 @@ describe('@helia/verifed-fetch', () => { expect(new Uint8Array(data)).to.equalBytes(finalRootFileContent) }) }) + + describe('404 paths', () => { + let helia: Helia + let verifiedFetch: VerifiedFetch + let contentTypeParser: Sinon.SinonStub + + beforeEach(async () => { + contentTypeParser = Sinon.stub() + helia = await createHelia() + verifiedFetch = new VerifiedFetch({ + helia + }, { + contentTypeParser + }) + }) + + afterEach(async () => { + await stop(helia, verifiedFetch) + }) + + it('returns a 404 when walking dag-cbor for non-existent path', async () => { + const obj = { + hello: 'world' + } + const c = dagCbor(helia) + const cid = await c.add(obj) + + const resp = await verifiedFetch.fetch(`http://example.com/ipfs/${cid}/foo/i-do-not-exist`) + expect(resp.status).to.equal(404) + }) + }) })