diff --git a/packages/verified-fetch/src/utils/parse-resource.ts b/packages/verified-fetch/src/utils/parse-resource.ts index 49e0b6d3..4b3b000c 100644 --- a/packages/verified-fetch/src/utils/parse-resource.ts +++ b/packages/verified-fetch/src/utils/parse-resource.ts @@ -32,8 +32,9 @@ export async function parseResource (resource: Resource, { ipns, logger }: Parse cid, protocol: 'ipfs', path: '', - query: {} - } + query: {}, + ttl: 29030400 // 1 year for ipfs content + } satisfies ParsedUrlStringResults } throw new TypeError(`Invalid resource. Cannot determine CID from resource: ${resource}`) diff --git a/packages/verified-fetch/src/utils/parse-url-string.ts b/packages/verified-fetch/src/utils/parse-url-string.ts index 6866f7f1..c87a422f 100644 --- a/packages/verified-fetch/src/utils/parse-url-string.ts +++ b/packages/verified-fetch/src/utils/parse-url-string.ts @@ -2,11 +2,11 @@ import { peerIdFromString } from '@libp2p/peer-id' import { CID } from 'multiformats/cid' import { TLRU } from './tlru.js' import type { RequestFormatShorthand } from '../types.js' -import type { IPNS, ResolveDNSLinkProgressEvents, ResolveResult } from '@helia/ipns' +import type { DNSLinkResolveResult, IPNS, IPNSResolveResult, ResolveDNSLinkProgressEvents, ResolveResult } from '@helia/ipns' import type { ComponentLogger } from '@libp2p/interface' import type { ProgressOptions } from 'progress-events' -const ipnsCache = new TLRU(1000) +const ipnsCache = new TLRU(1000) export interface ParseUrlStringInput { urlString: string @@ -23,24 +23,32 @@ export interface ParsedUrlQuery extends Record { filename?: string } -export interface ParsedUrlStringResults { - protocol: string - path: string - cid: CID +interface ParsedUrlStringResultsBase extends ResolveResult { + protocol: 'ipfs' | 'ipns' query: ParsedUrlQuery + ttl?: number } +export type ParsedUrlStringResults = ParsedUrlStringResultsBase // | DNSLinkResolveResult | IPNSResolveResult + const URL_REGEX = /^(?ip[fn]s):\/\/(?[^/?]+)\/?(?[^?]*)\??(?.*)$/ const PATH_REGEX = /^\/(?ip[fn]s)\/(?[^/?]+)\/?(?[^?]*)\??(?.*)$/ const PATH_GATEWAY_REGEX = /^https?:\/\/(.*[^/])\/(?ip[fn]s)\/(?[^/?]+)\/?(?[^?]*)\??(?.*)$/ const SUBDOMAIN_GATEWAY_REGEX = /^https?:\/\/(?[^/?]+)\.(?ip[fn]s)\.([^/?]+)\/?(?[^?]*)\??(?.*)$/ -function matchURLString (urlString: string): Record { +interface MatchUrlGroups { + protocol: 'ipfs' | 'ipns' + cidOrPeerIdOrDnsLink: string + path?: string + queryString?: string + +} +function matchURLString (urlString: string): MatchUrlGroups { for (const pattern of [URL_REGEX, PATH_REGEX, PATH_GATEWAY_REGEX, SUBDOMAIN_GATEWAY_REGEX]) { const match = urlString.match(pattern) if (match?.groups != null) { - return match.groups + return match.groups as unknown as MatchUrlGroups // force cast to MatchUrlGroups, because if it matches, it has to contain this structure. } } @@ -89,16 +97,19 @@ export async function parseUrlString ({ urlString, ipns, logger }: ParseUrlStrin let cid: CID | undefined let resolvedPath: string | undefined const errors: Error[] = [] + let resolveResult: IPNSResolveResult | DNSLinkResolveResult | undefined + let ttl: number = 5 * 60 // 5 minutes based on https://github.com/ipfs/boxo/issues/329#issuecomment-1995236409 if (protocol === 'ipfs') { try { cid = CID.parse(cidOrPeerIdOrDnsLink) + ttl = 29030400 // 1 year for ipfs content } catch (err) { log.error(err) errors.push(new TypeError('Invalid CID for ipfs:// URL')) } } else { - let resolveResult = ipnsCache.get(cidOrPeerIdOrDnsLink) + resolveResult = ipnsCache.get(cidOrPeerIdOrDnsLink) if (resolveResult != null) { cid = resolveResult.cid @@ -113,6 +124,13 @@ export async function parseUrlString ({ urlString, ipns, logger }: ParseUrlStrin resolveResult = await ipns.resolve(peerId, { onProgress: options?.onProgress }) cid = resolveResult?.cid resolvedPath = resolveResult?.path + /** + * For TTL nuances, see + * + * @see https://github.com/ipfs/js-ipns/blob/16e0e10682fa9a663e0bb493a44d3e99a5200944/src/index.ts#L200 + * @see https://github.com/ipfs/js-ipns/pull/308 + */ + ttl = Number(resolveResult.record.ttl) log.trace('resolved %s to %c', cidOrPeerIdOrDnsLink, cid) ipnsCache.set(cidOrPeerIdOrDnsLink, resolveResult, 60 * 1000 * 2) } catch (err) { @@ -137,6 +155,7 @@ export async function parseUrlString ({ urlString, ipns, logger }: ParseUrlStrin resolveResult = await ipns.resolveDNSLink(decodedDnsLinkLabel, { onProgress: options?.onProgress }) cid = resolveResult?.cid resolvedPath = resolveResult?.path + ttl = resolveResult.answer.TTL log.trace('resolved %s to %c', decodedDnsLinkLabel, cid) ipnsCache.set(cidOrPeerIdOrDnsLink, resolveResult, 60 * 1000 * 2) } catch (err: any) { @@ -177,9 +196,10 @@ export async function parseUrlString ({ urlString, ipns, logger }: ParseUrlStrin return { protocol, cid, - path: joinPaths(resolvedPath, urlPath), - query - } + path: joinPaths(resolvedPath, urlPath ?? ''), + query, + ttl + } satisfies ParsedUrlStringResults } /** diff --git a/packages/verified-fetch/src/utils/response-headers.ts b/packages/verified-fetch/src/utils/response-headers.ts new file mode 100644 index 00000000..464a33e4 --- /dev/null +++ b/packages/verified-fetch/src/utils/response-headers.ts @@ -0,0 +1,31 @@ +interface CacheControlHeaderOptions { + ttl?: number + protocol: 'ipfs' | 'ipns' + response: Response +} +/** + * Implementations may place an upper bound on any TTL received, as noted in Section 8 of [rfc2181]. + * If TTL value is unknown, implementations should not send a Cache-Control + * No matter if TTL value is known or not, implementations should always send a Last-Modified header with the timestamp of the record resolution. + * + * @see https://specs.ipfs.tech/http-gateways/path-gateway/#cache-control-response-header + */ +export function setCacheControlHeader ({ ttl, protocol, response }: CacheControlHeaderOptions): void { + let headerValue: string + if (protocol === 'ipfs') { + headerValue = 'public, max-age=29030400, immutable' + } else if (ttl == null) { + /** + * default limit for unknown TTL: "use 5 minute as default fallback when it is not available." + * + * @see https://github.com/ipfs/boxo/issues/329#issuecomment-1995236409 + */ + headerValue = 'public, max-age=300' + } else { + headerValue = `public, max-age=${ttl}` + } + + if (headerValue != null) { + response.headers.set('cache-control', headerValue) + } +} diff --git a/packages/verified-fetch/src/verified-fetch.ts b/packages/verified-fetch/src/verified-fetch.ts index a8354ccd..0b2bca7d 100644 --- a/packages/verified-fetch/src/verified-fetch.ts +++ b/packages/verified-fetch/src/verified-fetch.ts @@ -21,11 +21,13 @@ import { getETag } from './utils/get-e-tag.js' import { getStreamFromAsyncIterable } from './utils/get-stream-from-async-iterable.js' import { tarStream } from './utils/get-tar-stream.js' import { parseResource } from './utils/parse-resource.js' +import { setCacheControlHeader } from './utils/response-headers.js' import { badRequestResponse, movedPermanentlyResponse, notAcceptableResponse, notSupportedResponse, okResponse } from './utils/responses.js' import { selectOutputType, queryFormatToAcceptHeader } from './utils/select-output-type.js' import { walkPath } from './utils/walk-path.js' import type { CIDDetail, ContentTypeParser, Resource, VerifiedFetchInit as VerifiedFetchOptions } from './index.js' import type { RequestFormatShorthand } from './types.js' +import type { ParsedUrlStringResults } from './utils/parse-url-string' import type { Helia } from '@helia/interface' import type { AbortOptions, Logger, PeerId } from '@libp2p/interface' import type { DNSResolver } from '@multiformats/dns/resolvers' @@ -408,7 +410,23 @@ export class VerifiedFetch { options?.onProgress?.(new CustomProgressEvent('verified-fetch:request:start', { resource })) // resolve the CID/path from the requested resource - const { path, query, cid, protocol } = await parseResource(resource, { ipns: this.ipns, logger: this.helia.logger }, options) + let cid: ParsedUrlStringResults['cid'] + let path: ParsedUrlStringResults['path'] + let query: ParsedUrlStringResults['query'] + let ttl: ParsedUrlStringResults['ttl'] + let protocol: ParsedUrlStringResults['protocol'] + try { + const result = await parseResource(resource, { ipns: this.ipns, logger: this.helia.logger }, options) + cid = result.cid + path = result.path + query = result.query + ttl = result.ttl + protocol = result.protocol + } catch (err) { + this.log.error('error parsing resource %s', resource, err) + + return badRequestResponse('Invalid resource') + } options?.onProgress?.(new CustomProgressEvent('verified-fetch:request:resolve', { cid, path })) @@ -473,9 +491,7 @@ export class VerifiedFetch { response.headers.set('etag', getETag({ cid, reqFormat, weak: false })) - if (protocol === 'ipfs') { - response.headers.set('cache-control', 'public, max-age=29030400, immutable') - } + setCacheControlHeader({ response, ttl, protocol }) // https://specs.ipfs.tech/http-gateways/path-gateway/#x-ipfs-path-response-header response.headers.set('X-Ipfs-Path', resource.toString()) diff --git a/packages/verified-fetch/test/cache-control-header.spec.ts b/packages/verified-fetch/test/cache-control-header.spec.ts index bae88476..3403453f 100644 --- a/packages/verified-fetch/test/cache-control-header.spec.ts +++ b/packages/verified-fetch/test/cache-control-header.spec.ts @@ -2,20 +2,41 @@ import { dagCbor } from '@helia/dag-cbor' import { ipns } from '@helia/ipns' import { stop } from '@libp2p/interface' import { createEd25519PeerId } from '@libp2p/peer-id-factory' +import { dns } from '@multiformats/dns' import { expect } from 'aegir/chai' import Sinon from 'sinon' +import { stubInterface } from 'sinon-ts' import { VerifiedFetch } from '../src/verified-fetch.js' import { createHelia } from './fixtures/create-offline-helia.js' import type { Helia } from '@helia/interface' import type { IPNS } from '@helia/ipns' - +import type { DNSResponse } from '@multiformats/dns' + +function answerFake (data: string, TTL: number, name: string, type: number): DNSResponse { + const fake = stubInterface() + fake.Answer = [{ + data, + TTL, + name, + type + }] + return fake +} describe('cache-control header', () => { let helia: Helia let name: IPNS let verifiedFetch: VerifiedFetch + let customDnsResolver: Sinon.SinonStub> beforeEach(async () => { - helia = await createHelia() + customDnsResolver = Sinon.stub() + helia = await createHelia({ + dns: dns({ + resolvers: { + '.': customDnsResolver + } + }) + }) name = ipns(helia) verifiedFetch = new VerifiedFetch({ helia @@ -60,7 +81,7 @@ describe('cache-control header', () => { expect(resp.headers.get('Cache-Control')).to.not.containIgnoreCase('immutable') }) - it.skip('should return the correct max-age in the cache-control header for an IPNS name', async () => { + it('should return the correct max-age in the cache-control header for an IPNS name', async () => { const obj = { hello: 'world' } @@ -70,19 +91,24 @@ describe('cache-control header', () => { const oneHourInMs = 1000 * 60 * 60 const peerId = await createEd25519PeerId() - // ipns currently only allows customising the lifetime which is also used as the TTL - await name.publish(peerId, cid, { lifetime: oneHourInMs }) + /** + * ipns currently only allows customising the lifetime which is also used as the TTL + * + * lifetime is coming back as 100000 times larger than expected + * + * @see https://github.com/ipfs/js-ipns/blob/16e0e10682fa9a663e0bb493a44d3e99a5200944/src/index.ts#L200 + * @see https://github.com/ipfs/js-ipns/pull/308 + */ + await name.publish(peerId, cid, { lifetime: oneHourInMs / 100000 }) const resp = await verifiedFetch.fetch(`ipns://${peerId}`) expect(resp).to.be.ok() expect(resp.status).to.equal(200) - expect(resp.headers.get('Cache-Control')).to.equal(`public, max-age=${oneHourInMs.toString()}`) + expect(resp.headers.get('Cache-Control')).to.equal(`public, max-age=${oneHourInMs}`) }) it('should not contain immutable in the cache-control header for a DNSLink name', async () => { - const customDnsResolver = Sinon.stub() - verifiedFetch = new VerifiedFetch({ helia }, { @@ -94,12 +120,12 @@ describe('cache-control header', () => { } const c = dagCbor(helia) const cid = await c.add(obj) - customDnsResolver.returns(Promise.resolve(`/ipfs/${cid.toString()}`)) + customDnsResolver.withArgs('_dnslink.example-domain.com').resolves(answerFake(`dnslink=/ipfs/${cid}`, 666, '_dnslink.example-domain.com', 16)) const resp = await verifiedFetch.fetch('ipns://example-domain.com') expect(resp).to.be.ok() expect(resp.status).to.equal(200) - expect(resp.headers.get('Cache-Control')).to.not.containIgnoreCase('immutable') + expect(resp.headers.get('Cache-Control')).to.equal('public, max-age=666') }) }) diff --git a/packages/verified-fetch/test/custom-dns-resolvers.spec.ts b/packages/verified-fetch/test/custom-dns-resolvers.spec.ts index ae44cde8..eb382680 100644 --- a/packages/verified-fetch/test/custom-dns-resolvers.spec.ts +++ b/packages/verified-fetch/test/custom-dns-resolvers.spec.ts @@ -19,9 +19,11 @@ describe('custom dns-resolvers', () => { }) it('is used when passed to createVerifiedFetch', async () => { - const customDnsResolver = Sinon.stub() - - customDnsResolver.returns(Promise.resolve('/ipfs/QmVP2ip92jQuMDezVSzQBWDqWFbp9nyCHNQSiciRauPLDg')) + const customDnsResolver = Sinon.stub().withArgs('_dnslink.some-non-cached-domain.com').resolves({ + Answer: [{ + data: 'dnslink=/ipfs/QmVP2ip92jQuMDezVSzQBWDqWFbp9nyCHNQSiciRauPLDg' + }] + }) const fetch = await createVerifiedFetch({ gateways: ['http://127.0.0.1:8080'], diff --git a/packages/verified-fetch/test/utils/parse-url-string.spec.ts b/packages/verified-fetch/test/utils/parse-url-string.spec.ts index cd91d8cd..703da0ff 100644 --- a/packages/verified-fetch/test/utils/parse-url-string.spec.ts +++ b/packages/verified-fetch/test/utils/parse-url-string.spec.ts @@ -76,7 +76,10 @@ describe('parseUrlString', () => { logger }) ).to.eventually.be.rejected - .with.property('message', 'Could not parse PeerId in ipns url "mydomain.com", Non-base64 character') + .and.to.have.property('errors').that.deep.equals([ + new TypeError('Could not parse PeerId in ipns url "mydomain.com", Non-base64 character'), + new TypeError('Cannot read properties of undefined (reading \'answer\')') + ]) }) })