Merge branch 'main' into 9-heliaverified-fetch-http-range-request-sup…

…port

FYI: some tests were failing after `npm run reset` and `npm run test` was run. I had to fix them manually.

packages/interop/CHANGELOG.md

@@ -1,3 +1,26 @@
+## @helia/verified-fetch-interop [1.6.0](https://github.com/ipfs/helia-verified-fetch/compare/@helia/verified-fetch-interop-1.5.1...@helia/verified-fetch-interop-1.6.0) (2024-03-14)
+
+
+
+### Dependencies
+
+* **@helia/verified-fetch:** upgraded to 1.1.3
+
+## @helia/verified-fetch-interop [1.5.1](https://github.com/ipfs/helia-verified-fetch/compare/@helia/verified-fetch-interop-1.5.0...@helia/verified-fetch-interop-1.5.1) (2024-03-12)
+
+
+### Dependencies
+
+* bump kubo from 0.26.0 to 0.27.0 ([#12](https://github.com/ipfs/helia-verified-fetch/issues/12)) ([92cad49](https://github.com/ipfs/helia-verified-fetch/commit/92cad49de60a34cad031a07ee89f5c046004982f))
+
+## @helia/verified-fetch-interop [1.5.0](https://github.com/ipfs/helia-verified-fetch/compare/@helia/verified-fetch-interop-1.4.0...@helia/verified-fetch-interop-1.5.0) (2024-03-11)
+
+
+
+### Dependencies
+
+* **@helia/verified-fetch:** upgraded to 1.1.2
+
 ## @helia/verified-fetch-interop [1.4.0](https://github.com/ipfs/helia-verified-fetch/compare/@helia/verified-fetch-interop-1.3.0...@helia/verified-fetch-interop-1.4.0) (2024-03-08)
 
 

packages/interop/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@helia/verified-fetch-interop",
-  "version": "1.4.0",
+  "version": "1.6.0",
   "description": "Interop tests for @helia/verified-fetch",
   "license": "Apache-2.0 OR MIT",
   "homepage": "https://github.com/ipfs/helia-verified-fetch/tree/main/packages/interop#readme",
@@ -57,11 +57,11 @@
     "test:electron-main": "aegir test -t electron-main"
   },
   "dependencies": {
-    "@helia/verified-fetch": "1.1.1",
+    "@helia/verified-fetch": "1.1.3",
     "aegir": "^42.2.5",
     "ipfsd-ctl": "^13.0.0",
     "it-drain": "^3.0.5",
-    "kubo": "^0.26.0",
+    "kubo": "^0.27.0",
     "kubo-rpc-client": "^3.0.3",
     "magic-bytes.js": "^1.8.0",
     "multiformats": "^13.1.0"

packages/verified-fetch/CHANGELOG.md

@@ -1,3 +1,17 @@
+## @helia/verified-fetch [1.1.3](https://github.com/ipfs/helia-verified-fetch/compare/@helia/verified-fetch-1.1.2...@helia/verified-fetch-1.1.3) (2024-03-14)
+
+
+### Bug Fixes
+
+* update @helia/ipns and dns config ([#18](https://github.com/ipfs/helia-verified-fetch/issues/18)) ([9f88c54](https://github.com/ipfs/helia-verified-fetch/commit/9f88c5492f3418143c9b69907b212d29ecec4f91))
+
+## @helia/verified-fetch [1.1.2](https://github.com/ipfs/helia-verified-fetch/compare/@helia/verified-fetch-1.1.1...@helia/verified-fetch-1.1.2) (2024-03-11)
+
+
+### Bug Fixes
+
+* support https?://<dnsLink>.ipns.<gateway> urls ([#16](https://github.com/ipfs/helia-verified-fetch/issues/16)) ([0ece19a](https://github.com/ipfs/helia-verified-fetch/commit/0ece19a4bd355d75e52390bb5c8fdb477e99293b))
+
 ## @helia/verified-fetch [1.1.1](https://github.com/ipfs/helia-verified-fetch/compare/@helia/verified-fetch-1.1.0...@helia/verified-fetch-1.1.1) (2024-03-08)
 
 

packages/verified-fetch/README.md

@@ -177,7 +177,7 @@ Note that you do not need to provide both a DNS-over-HTTPS and a DNS-over-JSON r
 
 ```typescript
 import { createVerifiedFetch } from '@helia/verified-fetch'
-import { dnsJsonOverHttps, dnsOverHttps } from '@helia/ipns/dns-resolvers'
+import { dnsJsonOverHttps, dnsOverHttps } from '@multiformats/dns/resolvers'
 
 const fetch = await createVerifiedFetch({
   gateways: ['https://trustless-gateway.link'],
@@ -189,6 +189,29 @@ const fetch = await createVerifiedFetch({
 })
 ```
 
+## Example - Customizing DNS per-TLD resolvers
+
+DNS resolvers can be configured to only service DNS queries for specific
+TLDs:
+
+```typescript
+import { createVerifiedFetch } from '@helia/verified-fetch'
+import { dnsJsonOverHttps, dnsOverHttps } from '@multiformats/dns/resolvers'
+
+const fetch = await createVerifiedFetch({
+  gateways: ['https://trustless-gateway.link'],
+  routers: ['http://delegated-ipfs.dev'],
+  dnsResolvers: {
+    // this resolver will only be used for `.com` domains (note - this could
+    // also be an array of resolvers)
+    'com.': dnsJsonOverHttps('https://my-dns-resolver.example.com/dns-json'),
+    // this resolver will be used for everything else (note - this could
+    // also be an array of resolvers)
+    '.': dnsOverHttps('https://my-dns-resolver.example.com/dns-query')
+  }
+})
+```
+
 ### IPLD codec handling
 
 IPFS supports several data formats (typically referred to as codecs) which are included in the CID. `@helia/verified-fetch` attempts to abstract away some of the details for easier consumption.

packages/verified-fetch/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@helia/verified-fetch",
-  "version": "1.1.1",
+  "version": "1.1.3",
   "description": "A fetch-like API for obtaining verified & trustless IPFS content on the web",
   "license": "Apache-2.0 OR MIT",
   "homepage": "https://github.com/ipfs/helia-verified-fetch/tree/main/packages/verified-fetch#readme",
@@ -61,7 +61,7 @@
     "@helia/car": "^3.1.0",
     "@helia/http": "^1.0.2",
     "@helia/interface": "^4.0.1",
-    "@helia/ipns": "^6.0.1",
+    "@helia/ipns": "^7.0.0",
     "@helia/routers": "^1.0.1",
     "@helia/unixfs": "^3.0.1",
     "@ipld/dag-cbor": "^9.2.0",
@@ -70,6 +70,7 @@
     "@libp2p/interface": "^1.1.4",
     "@libp2p/kad-dht": "^12.0.8",
     "@libp2p/peer-id": "^4.0.7",
+    "@multiformats/dns": "^1.0.2",
     "cborg": "^4.0.9",
     "hashlru": "^2.3.0",
     "interface-blockstore": "^5.2.10",
@@ -88,7 +89,7 @@
     "@helia/dag-cbor": "^3.0.1",
     "@helia/dag-json": "^3.0.1",
     "@helia/json": "^3.0.1",
-    "@helia/utils": "^0.0.2",
+    "@helia/utils": "^0.1.0",
     "@ipld/car": "^5.3.0",
     "@libp2p/interface-compliance-tests": "^5.3.2",
     "@libp2p/logger": "^4.0.7",

packages/verified-fetch/src/index.ts

@@ -148,7 +148,7 @@
  *
  * ```typescript
  * import { createVerifiedFetch } from '@helia/verified-fetch'
- * import { dnsJsonOverHttps, dnsOverHttps } from '@helia/ipns/dns-resolvers'
+ * import { dnsJsonOverHttps, dnsOverHttps } from '@multiformats/dns/resolvers'
  *
  * const fetch = await createVerifiedFetch({
  *   gateways: ['https://trustless-gateway.link'],
@@ -160,6 +160,29 @@
  * })
  * ```
  *
+ * @example Customizing DNS per-TLD resolvers
+ *
+ * DNS resolvers can be configured to only service DNS queries for specific
+ * TLDs:
+ *
+ * ```typescript
+ * import { createVerifiedFetch } from '@helia/verified-fetch'
+ * import { dnsJsonOverHttps, dnsOverHttps } from '@multiformats/dns/resolvers'
+ *
+ * const fetch = await createVerifiedFetch({
+ *   gateways: ['https://trustless-gateway.link'],
+ *   routers: ['http://delegated-ipfs.dev'],
+ *   dnsResolvers: {
+ *     // this resolver will only be used for `.com` domains (note - this could
+ *     // also be an array of resolvers)
+ *     'com.': dnsJsonOverHttps('https://my-dns-resolver.example.com/dns-json'),
+ *     // this resolver will be used for everything else (note - this could
+ *     // also be an array of resolvers)
+ *     '.': dnsOverHttps('https://my-dns-resolver.example.com/dns-query')
+ *   }
+ * })
+ * ```
+ *
  * ### IPLD codec handling
  *
  * IPFS supports several data formats (typically referred to as codecs) which are included in the CID. `@helia/verified-fetch` attempts to abstract away some of the details for easier consumption.
@@ -569,10 +592,13 @@
 import { trustlessGateway } from '@helia/block-brokers'
 import { createHeliaHTTP } from '@helia/http'
 import { delegatedHTTPRouting } from '@helia/routers'
+import { dns } from '@multiformats/dns'
 import { VerifiedFetch as VerifiedFetchClass } from './verified-fetch.js'
 import type { Helia } from '@helia/interface'
-import type { DNSResolver, IPNSRoutingEvents, ResolveDnsLinkProgressEvents, ResolveProgressEvents } from '@helia/ipns'
+import type { ResolveDNSLinkProgressEvents } from '@helia/ipns'
 import type { GetEvents } from '@helia/unixfs'
+import type { DNSResolvers, DNS } from '@multiformats/dns'
+import type { DNSResolver } from '@multiformats/dns/resolvers'
 import type { CID } from 'multiformats/cid'
 import type { ProgressEvent, ProgressOptions } from 'progress-events'
 
@@ -618,7 +644,7 @@ export interface CreateVerifiedFetchInit {
    *
    * @default [dnsJsonOverHttps('https://mozilla.cloudflare-dns.com/dns-query'),dnsJsonOverHttps('https://dns.google/resolve')]
    */
-  dnsResolvers?: DNSResolver[]
+  dnsResolvers?: DNSResolver[] | DNSResolvers
 }
 
 export interface CreateVerifiedFetchOptions {
@@ -651,7 +677,7 @@ export type BubbledProgressEvents =
   // unixfs
   GetEvents |
   // ipns
-  ResolveProgressEvents | ResolveDnsLinkProgressEvents | IPNSRoutingEvents
+  ResolveDNSLinkProgressEvents
 
 export type VerifiedFetchProgressEvents =
   ProgressEvent<'verified-fetch:request:start', CIDDetail> |
@@ -674,20 +700,19 @@ export interface VerifiedFetchInit extends RequestInit, ProgressOptions<BubbledP
  * Create and return a Helia node
  */
 export async function createVerifiedFetch (init?: Helia | CreateVerifiedFetchInit, options?: CreateVerifiedFetchOptions): Promise<VerifiedFetch> {
-  let dnsResolvers: DNSResolver[] | undefined
   if (!isHelia(init)) {
-    dnsResolvers = init?.dnsResolvers
     init = await createHeliaHTTP({
       blockBrokers: [
         trustlessGateway({
           gateways: init?.gateways
         })
       ],
-      routers: (init?.routers ?? ['https://delegated-ipfs.dev']).map((routerUrl) => delegatedHTTPRouting(routerUrl))
+      routers: (init?.routers ?? ['https://delegated-ipfs.dev']).map((routerUrl) => delegatedHTTPRouting(routerUrl)),
+      dns: createDns(init?.dnsResolvers)
     })
   }
 
-  const verifiedFetchInstance = new VerifiedFetchClass({ helia: init }, { dnsResolvers, ...options })
+  const verifiedFetchInstance = new VerifiedFetchClass({ helia: init }, options)
   async function verifiedFetch (resource: Resource, options?: VerifiedFetchInit): Promise<Response> {
     return verifiedFetchInstance.fetch(resource, options)
   }
@@ -707,3 +732,19 @@ function isHelia (obj: any): obj is Helia {
     obj?.stop != null &&
     obj?.start != null
 }
+
+function createDns (resolvers?: DNSResolver[] | DNSResolvers): DNS | undefined {
+  if (resolvers == null) {
+    return
+  }
+
+  if (Array.isArray(resolvers)) {
+    return dns({
+      resolvers: {
+        '.': resolvers
+      }
+    })
+  }
+
+  return dns({ resolvers })
+}

packages/verified-fetch/src/utils/parse-resource.ts

@@ -2,7 +2,7 @@ import { CID } from 'multiformats/cid'
 import { parseUrlString } from './parse-url-string.js'
 import type { ParsedUrlStringResults } from './parse-url-string.js'
 import type { Resource } from '../index.js'
-import type { IPNS, IPNSRoutingEvents, ResolveDnsLinkProgressEvents, ResolveProgressEvents } from '@helia/ipns'
+import type { IPNS, IPNSRoutingEvents, ResolveDNSLinkProgressEvents, ResolveProgressEvents } from '@helia/ipns'
 import type { ComponentLogger } from '@libp2p/interface'
 import type { ProgressOptions } from 'progress-events'
 
@@ -11,7 +11,7 @@ export interface ParseResourceComponents {
   logger: ComponentLogger
 }
 
-export interface ParseResourceOptions extends ProgressOptions<ResolveProgressEvents | IPNSRoutingEvents | ResolveDnsLinkProgressEvents> {
+export interface ParseResourceOptions extends ProgressOptions<ResolveProgressEvents | IPNSRoutingEvents | ResolveDNSLinkProgressEvents> {
 
 }
 /**

packages/verified-fetch/src/utils/parse-url-string.ts

@@ -2,7 +2,7 @@ import { peerIdFromString } from '@libp2p/peer-id'
 import { CID } from 'multiformats/cid'
 import { TLRU } from './tlru.js'
 import type { RequestFormatShorthand } from '../types.js'
-import type { IPNS, IPNSRoutingEvents, ResolveDnsLinkProgressEvents, ResolveProgressEvents, ResolveResult } from '@helia/ipns'
+import type { IPNS, ResolveDNSLinkProgressEvents, ResolveResult } from '@helia/ipns'
 import type { ComponentLogger } from '@libp2p/interface'
 import type { ProgressOptions } from 'progress-events'
 
@@ -13,7 +13,7 @@ export interface ParseUrlStringInput {
   ipns: IPNS
   logger: ComponentLogger
 }
-export interface ParseUrlStringOptions extends ProgressOptions<ResolveProgressEvents | IPNSRoutingEvents | ResolveDnsLinkProgressEvents> {
+export interface ParseUrlStringOptions extends ProgressOptions<ResolveDNSLinkProgressEvents> {
 
 }
 
@@ -33,7 +33,7 @@ export interface ParsedUrlStringResults {
 const URL_REGEX = /^(?<protocol>ip[fn]s):\/\/(?<cidOrPeerIdOrDnsLink>[^/?]+)\/?(?<path>[^?]*)\??(?<queryString>.*)$/
 const PATH_REGEX = /^\/(?<protocol>ip[fn]s)\/(?<cidOrPeerIdOrDnsLink>[^/?]+)\/?(?<path>[^?]*)\??(?<queryString>.*)$/
 const PATH_GATEWAY_REGEX = /^https?:\/\/(.*[^/])\/(?<protocol>ip[fn]s)\/(?<cidOrPeerIdOrDnsLink>[^/?]+)\/?(?<path>[^?]*)\??(?<queryString>.*)$/
-const SUBDOMAIN_GATEWAY_REGEX = /^https?:\/\/(?<cidOrPeerIdOrDnsLink>[^/.?]+)\.(?<protocol>ip[fn]s)\.([^/?]+)\/?(?<path>[^?]*)\??(?<queryString>.*)$/
+const SUBDOMAIN_GATEWAY_REGEX = /^https?:\/\/(?<cidOrPeerIdOrDnsLink>[^/?]+)\.(?<protocol>ip[fn]s)\.([^/?]+)\/?(?<path>[^?]*)\??(?<queryString>.*)$/
 
 function matchURLString (urlString: string): Record<string, string> {
   for (const pattern of [URL_REGEX, PATH_REGEX, PATH_GATEWAY_REGEX, SUBDOMAIN_GATEWAY_REGEX]) {
@@ -47,6 +47,34 @@ function matchURLString (urlString: string): Record<string, string> {
   throw new TypeError(`Invalid URL: ${urlString}, please use ipfs://, ipns://, or gateway URLs only`)
 }
 
+/**
+ * For dnslinks see https://specs.ipfs.tech/http-gateways/subdomain-gateway/#host-request-header
+ * DNSLink names include . which means they must be inlined into a single DNS label to provide unique origin and work with wildcard TLS certificates.
+ */
+
+// DNS label can have up to 63 characters, consisting of alphanumeric
+// characters or hyphens -, but it must not start or end with a hyphen.
+const dnsLabelRegex = /^[a-zA-Z0-9]([a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?$/
+
+/**
+ * Checks if label looks like inlined DNSLink.
+ * (https://specs.ipfs.tech/http-gateways/subdomain-gateway/#host-request-header)
+ */
+function isInlinedDnsLink (label: string): boolean {
+  return dnsLabelRegex.test(label) && label.includes('-') && !label.includes('.')
+}
+
+/**
+ * DNSLink label decoding
+ * * Every standalone - is replaced with .
+ * * Every remaining -- is replaced with -
+ *
+ * @example en-wikipedia--on--ipfs-org.ipns.example.net -> example.net/ipns/en.wikipedia-on-ipfs.org
+ */
+function dnsLinkLabelDecoder (linkLabel: string): string {
+  return linkLabel.replace(/--/g, '%').replace(/-/g, '.').replace(/%/g, '-')
+}
+
 /**
  * A function that parses ipfs:// and ipns:// URLs, returning an object with easily recognizable properties.
  *
@@ -80,7 +108,6 @@ export async function parseUrlString ({ urlString, ipns, logger }: ParseUrlStrin
       // protocol is ipns
       log.trace('attempting to resolve PeerId for %s', cidOrPeerIdOrDnsLink)
       let peerId = null
-
       try {
         peerId = peerIdFromString(cidOrPeerIdOrDnsLink)
         resolveResult = await ipns.resolve(peerId, { onProgress: options?.onProgress })
@@ -99,13 +126,18 @@ export async function parseUrlString ({ urlString, ipns, logger }: ParseUrlStrin
       }
 
       if (cid == null) {
-        log.trace('attempting to resolve DNSLink for %s', cidOrPeerIdOrDnsLink)
+        let decodedDnsLinkLabel = cidOrPeerIdOrDnsLink
+        if (isInlinedDnsLink(cidOrPeerIdOrDnsLink)) {
+          decodedDnsLinkLabel = dnsLinkLabelDecoder(cidOrPeerIdOrDnsLink)
+          log.trace('decoded dnslink from "%s" to "%s"', cidOrPeerIdOrDnsLink, decodedDnsLinkLabel)
+        }
+        log.trace('Attempting to resolve DNSLink for %s', decodedDnsLinkLabel)
 
         try {
-          resolveResult = await ipns.resolveDns(cidOrPeerIdOrDnsLink, { onProgress: options?.onProgress })
+          resolveResult = await ipns.resolveDNSLink(decodedDnsLinkLabel, { onProgress: options?.onProgress })
           cid = resolveResult?.cid
           resolvedPath = resolveResult?.path
-          log.trace('resolved %s to %c', cidOrPeerIdOrDnsLink, cid)
+          log.trace('resolved %s to %c', decodedDnsLinkLabel, cid)
           ipnsCache.set(cidOrPeerIdOrDnsLink, resolveResult, 60 * 1000 * 2)
         } catch (err: any) {
           log.error('could not resolve DnsLink for "%s"', cidOrPeerIdOrDnsLink, err)