chore: debugging conformance failures

packages/gateway-conformance/package.json

@@ -56,7 +56,7 @@
     "@helia/http": "^2.0.1",
     "@helia/interface": "^5.0.0",
     "@helia/routers": "^2.1.0",
-    "@helia/verified-fetch": "2.1.0",
+    "@helia/verified-fetch": "../verified-fetch",
     "@libp2p/kad-dht": "^14.1.0",
     "@libp2p/logger": "^5.1.3",
     "@libp2p/peer-id": "^5.0.7",
@@ -69,10 +69,11 @@
     "fast-glob": "^3.3.2",
     "interface-blockstore": "^5.2.10",
     "interface-datastore": "^8.2.11",
-    "ipfsd-ctl": "^14.1.0",
+    "ipfsd-ctl": "^15.0.1",
     "ipns": "^10.0.0",
     "kubo": "^0.32.0",
-    "kubo-rpc-client": "^4.1.1",
+    "kubo-rpc-client": "^5.0.2",
+    "multiformats": "^13.3.1",
     "uint8arrays": "^5.1.0",
     "undici": "^6.18.1"
   },

packages/gateway-conformance/src/fixtures/basic-server.ts

@@ -1,6 +1,7 @@
 import { createServer, type IncomingMessage, type ServerResponse } from 'node:http'
 import { trustlessGateway } from '@helia/block-brokers'
 import { createHeliaHTTP } from '@helia/http'
+// eslint-disable-next-line @typescript-eslint/no-unused-vars
 import { httpGatewayRouting } from '@helia/routers'
 import { logger } from '@libp2p/logger'
 import { dns } from '@multiformats/dns'
@@ -57,9 +58,9 @@ async function createHelia (init: CreateHeliaOptions): Promise<ReturnType<typeof
       })
     ],
     routers: [
-      httpGatewayRouting({
-        gateways: init.gateways
-      })
+      // httpGatewayRouting({
+      //   gateways: init.gateways
+      // })
     ],
     dns: dns({
       resolvers: {
@@ -135,9 +136,29 @@ async function callVerifiedFetch (req: IncomingMessage, res: Response, { serverP
   }
   res.on('finish', onResFinish)
 
+  // // test that the content can be requested from the trustless gateway directly
+  // void fetch(fullUrlHref.toString().replace('3441', '3440')).then(async (res) => {
+  //   if (!res.ok) {
+  //     log.error('Error fetching directly from trustless gateway', res.status, res.statusText)
+  //   }
+  //   log('Fetched directly from trustless gateway', res.status, res.statusText)
+  //   log('content: %s', await res.text())
+  // }).catch((err) => {
+  //   log.error('Error fetching directly from trustless gateway', err)
+  // })
   try {
     urlLog.trace('calling verified-fetch')
-    const resp = await verifiedFetch(fullUrlHref.toString(), { redirect: 'manual', signal: requestController.signal, session: useSessions, allowInsecure: true, allowLocal: true, headers: convertNodeJsHeadersToFetchHeaders(req.headers) })
+    const resp = await verifiedFetch(fullUrlHref.toString(), {
+      redirect: 'manual',
+      signal: requestController.signal,
+      session: useSessions,
+      allowInsecure: true,
+      allowLocal: true,
+      headers: convertNodeJsHeadersToFetchHeaders(req.headers),
+      onProgress: (progress) => {
+        urlLog('progress: %O', progress)
+      }
+    })
     urlLog.trace('verified-fetch response status: %d', resp.status)
 
     const headers = convertFetchHeadersToNodeJsHeaders({ resp, log: urlLog, fixingGwcAnnoyance, serverPort })
@@ -192,6 +213,7 @@ export async function startVerifiedFetchGateway ({ kuboGateway, serverPort, IPFS
   if (kuboGateway == null) {
     throw new Error('options.kuboGateway or KUBO_GATEWAY env var is required')
   }
+  log('Using trustless Kubo gateway: %s', kuboGateway)
 
   const blockstore = new MemoryBlockstore()
   const datastore = getIpnsRecordDatastore()

packages/gateway-conformance/src/fixtures/create-kubo.ts

@@ -32,8 +32,8 @@ export async function createKuboNode (listenPort?: number): Promise<KuboNodeDeta
           }
         }
       }
-    },
-    args: ['--enable-pubsub-experiment', '--enable-namesys-pubsub']
+    }
+    // args: ['--enable-pubsub-experiment', '--enable-namesys-pubsub']
   })
   const info = await controller.info()
 

packages/verified-fetch/src/types.ts

@@ -1,4 +1,4 @@
-import { type AbortOptions } from '@libp2p/interface'
+import { type AbortOptions, type PeerId } from '@libp2p/interface'
 import { type CID } from 'multiformats/cid'
 import type { VerifiedFetchInit } from './index.js'
 
@@ -8,6 +8,7 @@ export type SupportedBodyTypes = string | ArrayBuffer | Blob | ReadableStream<Ui
 
 export interface FetchHandlerFunctionArg {
   cid: CID
+  peerId?: PeerId
   path: string
 
   /**

packages/verified-fetch/src/utils/parse-url-string.ts

@@ -1,5 +1,5 @@
-import { peerIdFromCID, peerIdFromString } from '@libp2p/peer-id'
 import { CID } from 'multiformats/cid'
+import { peerIdFromString } from './peer-id-from-string.js'
 import { TLRU } from './tlru.js'
 import type { RequestFormatShorthand } from '../types.js'
 import type { DNSLinkResolveResult, IPNS, IPNSResolveResult, IPNSRoutingEvents, ResolveDNSLinkProgressEvents, ResolveProgressEvents, ResolveResult } from '@helia/ipns'
@@ -41,6 +41,11 @@ interface ParsedUrlStringResultsBase extends ResolveResult {
    * seconds as a number
    */
   ttl?: number
+
+  /**
+   * The resolved PeerId if the URL was an IPNS URL
+   */
+  peerId?: PeerId
 }
 
 export type ParsedUrlStringResults = ParsedUrlStringResultsBase
@@ -151,6 +156,7 @@ export async function parseUrlString ({ urlString, ipns, logger }: ParseUrlStrin
 
   let cid: CID | undefined
   let resolvedPath: string | undefined
+  let peerId: PeerId | undefined
   const errors: Error[] = []
   let resolveResult: IPNSResolveResult | DNSLinkResolveResult | undefined
 
@@ -174,16 +180,9 @@ export async function parseUrlString ({ urlString, ipns, logger }: ParseUrlStrin
       log.trace('resolved %s to %c from cache', cidOrPeerIdOrDnsLink, cid)
     } else {
       log.trace('Attempting to resolve PeerId for %s', cidOrPeerIdOrDnsLink)
-      let peerId: PeerId | undefined
       try {
         // try resolving as an IPNS name
-
-        if (cidOrPeerIdOrDnsLink.charAt(0) === '1' || cidOrPeerIdOrDnsLink.charAt(0) === 'Q') {
-          peerId = peerIdFromString(cidOrPeerIdOrDnsLink)
-        } else {
-          // try resolving as a base36 CID
-          peerId = peerIdFromCID(CID.parse(cidOrPeerIdOrDnsLink))
-        }
+        peerId = peerIdFromString(cidOrPeerIdOrDnsLink, log)
         if (peerId.publicKey == null) {
           throw new TypeError('cidOrPeerIdOrDnsLink contains no public key')
         }
@@ -263,6 +262,7 @@ export async function parseUrlString ({ urlString, ipns, logger }: ParseUrlStrin
   }
 
   return {
+    peerId,
     protocol,
     cid,
     path: joinPaths(resolvedPath, urlPath ?? ''),

packages/verified-fetch/src/utils/peer-id-from-string.ts

@@ -0,0 +1,26 @@
+import { peerIdFromCID, peerIdFromString as libp2pPeerIdFromString } from '@libp2p/peer-id'
+import { CID } from 'multiformats/cid'
+import type { Logger, PeerId } from '@libp2p/interface'
+
+export function peerIdFromString (peerIdStr: string, log?: Logger): PeerId {
+  let peerId: PeerId
+  if (peerIdStr.charAt(0) === '1' || peerIdStr.charAt(0) === 'Q') {
+    peerId = libp2pPeerIdFromString(peerIdStr)
+  } else { // if (peerIdStr.startsWith('k')) {
+    /**
+     * libp2p CID peerID
+     *
+     * @see https://github.com/libp2p/js-libp2p/blob/2feaeddb40712a5d58aee158021a10b9b9bbf660/doc/migrations/v1.0.0-v2.0.0.md?plain=1#L255
+     */
+    let cid: CID
+    try {
+      cid = CID.parse(peerIdStr)
+    } catch (err: any) {
+      log?.error('Could not parse CID from string "%s"', peerIdStr, err)
+      throw new Error(`Could not parse CID from string: ${peerIdStr}`)
+    }
+    peerId = peerIdFromCID(cid)
+  }
+
+  return peerId
+}

packages/verified-fetch/src/verified-fetch.ts

@@ -3,14 +3,13 @@ import { ipns as heliaIpns, type IPNS } from '@helia/ipns'
 import * as ipldDagCbor from '@ipld/dag-cbor'
 import * as ipldDagJson from '@ipld/dag-json'
 import { code as dagPbCode } from '@ipld/dag-pb'
-import { type AbortOptions, type Logger, type PeerId } from '@libp2p/interface'
+import { type AbortOptions, type Logger } from '@libp2p/interface'
 import { Record as DHTRecord } from '@libp2p/kad-dht'
-import { peerIdFromCID, peerIdFromString } from '@libp2p/peer-id'
 import { Key } from 'interface-datastore'
 import { exporter } from 'ipfs-unixfs-exporter'
 import toBrowserReadableStream from 'it-to-browser-readablestream'
 import { LRUCache } from 'lru-cache'
-import { CID } from 'multiformats/cid'
+import { type CID } from 'multiformats/cid'
 import { code as jsonCode } from 'multiformats/codecs/json'
 import { code as rawCode } from 'multiformats/codecs/raw'
 import { identity } from 'multiformats/hashes/identity'
@@ -147,36 +146,32 @@ export class VerifiedFetch {
    * Accepts an `ipns://...` or `https?://<ipnsname>.ipns.<domain>` URL as a string and returns a `Response` containing
    * a raw IPNS record.
    */
-  private async handleIPNSRecord ({ resource, cid, path, options }: FetchHandlerFunctionArg): Promise<Response> {
+  private async handleIPNSRecord ({ resource, peerId, path, options }: FetchHandlerFunctionArg): Promise<Response> {
     if (path !== '' || !(resource.startsWith('ipns://') || resource.includes('.ipns.'))) {
       this.log.error('invalid request for IPNS name "%s" and path "%s"', resource, path)
       return badRequestResponse(resource, 'Invalid IPNS name')
     }
 
-    let peerId: PeerId
-
-    try {
-      if (resource.startsWith('ipns://')) {
-        const peerIdString = resource.replace('ipns://', '')
-        this.log.trace('trying to parse peer id from "%s"', peerIdString)
-        peerId = peerIdFromString(peerIdString)
-      } else {
-        const peerIdString = resource.split('.ipns.')[0].split('://')[1]
-        this.log.trace('trying to parse peer id from "%s"', peerIdString)
-        let cid: CID
-        try {
-          cid = CID.parse(peerIdString)
-        } catch (err: any) {
-          this.log.error('could not construct CID from peerId string "%s"', resource, err)
-          return badRequestResponse(resource, err)
-        }
-
-        peerId = peerIdFromCID(cid)
-      }
-    } catch (err: any) {
-      this.log.error('could not parse peer id from IPNS url %s', resource, err)
-
-      return badRequestResponse(resource, err)
+    // let peerId: PeerId
+    // let peerIdString: string
+    // if (resource.startsWith('ipns://')) {
+    //   peerIdString = resource.replace('ipns://', '')
+    // } else {
+    //   peerIdString = resource.split('.ipns.')[0].split('://')[1]
+    // }
+
+    // this.log.trace('trying to parse peer id from "%s"', peerIdString)
+
+    // try {
+    //   peerId = peerIdFromString(peerIdString)
+    // } catch (err: any) {
+    //   this.log.error('could not parse peer id from IPNS url %s', resource, err)
+
+    //   return badRequestResponse(resource, err)
+    // }
+    if (peerId == null) {
+      this.log.error('no peerId passed to handleIPNSRecord for request of "%s"', resource)
+      return badRequestResponse(resource, 'Could not determine peer ID')
     }
 
     // since this call happens after parseResource, we've already resolved the
@@ -232,7 +227,9 @@ export class VerifiedFetch {
   private async handleJson ({ resource, cid, path, accept, session, options }: FetchHandlerFunctionArg): Promise<Response> {
     this.log.trace('fetching %c/%s', cid, path)
     const blockstore = this.getBlockstore(cid, resource, session, options)
+    this.log.trace('got blockstore. getting block %c, with options %O', cid, options)
     const block = await blockstore.get(cid, options)
+    this.log.trace('got block for %c', cid)
     let body: string | Uint8Array
 
     if (accept === 'application/vnd.ipld.dag-cbor' || accept === 'application/cbor') {
@@ -506,6 +503,7 @@ export class VerifiedFetch {
     let ttl: ParsedUrlStringResults['ttl']
     let protocol: ParsedUrlStringResults['protocol']
     let ipfsPath: string
+    let peerId: ParsedUrlStringResults['peerId']
     try {
       const result = await parseResource(resource, { ipns: this.ipns, logger: this.helia.logger }, options)
       cid = result.cid
@@ -514,6 +512,7 @@ export class VerifiedFetch {
       ttl = result.ttl
       protocol = result.protocol
       ipfsPath = result.ipfsPath
+      peerId = result.peerId
     } catch (err: any) {
       options?.signal?.throwIfAborted()
       this.log.error('error parsing resource %s', resource, err)
@@ -540,7 +539,7 @@ export class VerifiedFetch {
       return redirectResponse
     }
 
-    const handlerArgs: FetchHandlerFunctionArg = { resource: resource.toString(), cid, path, accept, session: options?.session ?? true, options }
+    const handlerArgs: FetchHandlerFunctionArg = { resource: resource.toString(), cid, path, accept, session: options?.session ?? true, peerId, options }
 
     if (accept === 'application/vnd.ipfs.ipns-record') {
       // the user requested a raw IPNS record