Skip to content

Commit

Permalink
Merge branch 'main' into feat/use-blockstore-sessions
Browse files Browse the repository at this point in the history
  • Loading branch information
achingbrain authored Apr 15, 2024
2 parents d744c0d + 4532bf1 commit ed8b611
Show file tree
Hide file tree
Showing 9 changed files with 246 additions and 81 deletions.
24 changes: 24 additions & 0 deletions packages/interop/CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,3 +1,27 @@
## @helia/verified-fetch-interop [1.18.0](https://github.com/ipfs/helia-verified-fetch/compare/@helia/verified-fetch-interop-1.17.0...@helia/verified-fetch-interop-1.18.0) (2024-04-15)



### Dependencies

* **@helia/verified-fetch:** upgraded to 1.3.11

## @helia/verified-fetch-interop [1.17.0](https://github.com/ipfs/helia-verified-fetch/compare/@helia/verified-fetch-interop-1.16.0...@helia/verified-fetch-interop-1.17.0) (2024-04-12)



### Dependencies

* **@helia/verified-fetch:** upgraded to 1.3.10

## @helia/verified-fetch-interop [1.16.0](https://github.com/ipfs/helia-verified-fetch/compare/@helia/verified-fetch-interop-1.15.1...@helia/verified-fetch-interop-1.16.0) (2024-04-11)



### Dependencies

* **@helia/verified-fetch:** upgraded to 1.3.9

## @helia/verified-fetch-interop [1.15.1](https://github.com/ipfs/helia-verified-fetch/compare/@helia/verified-fetch-interop-1.15.0...@helia/verified-fetch-interop-1.15.1) (2024-04-09)


Expand Down
4 changes: 2 additions & 2 deletions packages/interop/package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "@helia/verified-fetch-interop",
"version": "1.15.1",
"version": "1.18.0",
"description": "Interop tests for @helia/verified-fetch",
"license": "Apache-2.0 OR MIT",
"homepage": "https://github.com/ipfs/helia-verified-fetch/tree/main/packages/interop#readme",
Expand Down Expand Up @@ -57,7 +57,7 @@
"test:electron-main": "aegir test -t electron-main"
},
"dependencies": {
"@helia/verified-fetch": "1.3.8",
"@helia/verified-fetch": "1.3.11",
"aegir": "^42.2.5",
"ipfsd-ctl": "^13.0.0",
"it-drain": "^3.0.5",
Expand Down
21 changes: 21 additions & 0 deletions packages/verified-fetch/CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,3 +1,24 @@
## @helia/verified-fetch [1.3.11](https://github.com/ipfs/helia-verified-fetch/compare/@helia/verified-fetch-1.3.10...@helia/verified-fetch-1.3.11) (2024-04-15)


### Documentation

* update documented default value ([161a470](https://github.com/ipfs/helia-verified-fetch/commit/161a4707a6e06411eed86ee68d5a07994574f00a)), closes [/github.com/multiformats/js-dns/blob/a56c9e0b953d644392cf10fd0792757da0d61c32/src/resolvers/default.browser.ts#L6-L7](https://github.com/ipfs//github.com/multiformats/js-dns/blob/a56c9e0b953d644392cf10fd0792757da0d61c32/src/resolvers/default.browser.ts/issues/L6-L7)

## @helia/verified-fetch [1.3.10](https://github.com/ipfs/helia-verified-fetch/compare/@helia/verified-fetch-1.3.9...@helia/verified-fetch-1.3.10) (2024-04-12)


### Documentation

* fix readme link ([#51](https://github.com/ipfs/helia-verified-fetch/issues/51)) ([8a41c57](https://github.com/ipfs/helia-verified-fetch/commit/8a41c5701f800ddac25ae59d790836e5b1c48d93))

## @helia/verified-fetch [1.3.9](https://github.com/ipfs/helia-verified-fetch/compare/@helia/verified-fetch-1.3.8...@helia/verified-fetch-1.3.9) (2024-04-11)


### Bug Fixes

* identity CIDs use contentTypeParser ([#49](https://github.com/ipfs/helia-verified-fetch/issues/49)) ([3014498](https://github.com/ipfs/helia-verified-fetch/commit/30144981b5253b3269cbf186a24f1ef9dd04a452))

## @helia/verified-fetch [1.3.8](https://github.com/ipfs/helia-verified-fetch/compare/@helia/verified-fetch-1.3.7...@helia/verified-fetch-1.3.8) (2024-04-09)


Expand Down
31 changes: 16 additions & 15 deletions packages/verified-fetch/package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "@helia/verified-fetch",
"version": "1.3.8",
"version": "1.3.11",
"description": "A fetch-like API for obtaining verified & trustless IPFS content on the web",
"license": "Apache-2.0 OR MIT",
"homepage": "https://github.com/ipfs/helia-verified-fetch/tree/main/packages/verified-fetch#readme",
Expand Down Expand Up @@ -57,13 +57,12 @@
"release": "aegir release"
},
"dependencies": {
"@helia/block-brokers": "^2.0.3",
"@helia/car": "^3.1.2",
"@helia/http": "^1.0.3",
"@helia/interface": "^4.1.0",
"@helia/ipns": "^7.2.0",
"@helia/routers": "^1.0.2",
"@helia/unixfs": "^3.0.3",
"@helia/block-brokers": "^2.1.0",
"@helia/car": "^3.1.3",
"@helia/http": "^1.0.4",
"@helia/interface": "^4.2.0",
"@helia/ipns": "^7.2.1",
"@helia/routers": "^1.0.3",
"@ipld/dag-cbor": "^9.2.0",
"@ipld/dag-json": "^10.2.0",
"@ipld/dag-pb": "^4.1.0",
Expand All @@ -85,25 +84,27 @@
"uint8arrays": "^5.0.3"
},
"devDependencies": {
"@helia/car": "^3.1.2",
"@helia/dag-cbor": "^3.0.2",
"@helia/dag-json": "^3.0.2",
"@helia/json": "^3.0.2",
"@helia/utils": "^0.1.0",
"@helia/car": "^3.1.3",
"@helia/dag-cbor": "^3.0.3",
"@helia/dag-json": "^3.0.3",
"@helia/json": "^3.0.3",
"@helia/unixfs": "^3.0.4",
"@helia/utils": "^0.2.0",
"@ipld/car": "^5.3.0",
"@libp2p/interface-compliance-tests": "^5.3.4",
"@libp2p/logger": "^4.0.9",
"@libp2p/peer-id-factory": "^4.0.9",
"@sgtpooki/file-type": "^1.0.1",
"@types/sinon": "^17.0.3",
"aegir": "^42.2.5",
"blockstore-core": "^4.4.0",
"blockstore-core": "^4.4.1",
"browser-readablestream-to-it": "^2.0.5",
"datastore-core": "^9.2.9",
"helia": "^4.1.0",
"helia": "^4.1.1",
"ipfs-unixfs-importer": "^15.2.5",
"ipns": "^9.1.0",
"it-all": "^3.0.4",
"it-drain": "^3.0.5",
"it-last": "^3.0.4",
"it-to-buffer": "^4.0.5",
"magic-bytes.js": "^1.10.0",
Expand Down
14 changes: 8 additions & 6 deletions packages/verified-fetch/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,7 @@
*
* The [helia](https://www.npmjs.com/package/helia) module is configured with a libp2p node that is suited for decentralized applications, alternatively [@helia/http](https://www.npmjs.com/package/@helia/http) is available which uses HTTP gateways for all network operations.
*
* You can see variations of Helia and js-libp2p configuration options at https://helia.io/interfaces/helia.index.HeliaInit.html.
* See variations of [Helia and js-libp2p configuration options](https://helia.io/interfaces/helia.HeliaInit.html)
*
* ```typescript
* import { trustlessGateway } from '@helia/block-brokers'
Expand Down Expand Up @@ -594,11 +594,11 @@ import { createHeliaHTTP } from '@helia/http'
import { delegatedHTTPRouting } from '@helia/routers'
import { dns } from '@multiformats/dns'
import { VerifiedFetch as VerifiedFetchClass } from './verified-fetch.js'
import type { Helia } from '@helia/interface'
import type { GetBlockProgressEvents, Helia } from '@helia/interface'
import type { ResolveDNSLinkProgressEvents } from '@helia/ipns'
import type { GetEvents } from '@helia/unixfs'
import type { DNSResolvers, DNS } from '@multiformats/dns'
import type { DNSResolver } from '@multiformats/dns/resolvers'
import type { ExporterProgressEvents } from 'ipfs-unixfs-exporter'
import type { CID } from 'multiformats/cid'
import type { ProgressEvent, ProgressOptions } from 'progress-events'

Expand Down Expand Up @@ -642,7 +642,7 @@ export interface CreateVerifiedFetchInit {
*
* We use cloudflare and google's dnsJsonOverHttps resolvers by default.
*
* @default [dnsJsonOverHttps('https://mozilla.cloudflare-dns.com/dns-query'),dnsJsonOverHttps('https://dns.google/resolve')]
* @default [dnsJsonOverHttps('https://cloudflare-dns.com/dns-query'),dnsJsonOverHttps('https://dns.google/resolve')]
*/
dnsResolvers?: DNSResolver[] | DNSResolvers
}
Expand Down Expand Up @@ -690,8 +690,10 @@ export interface ContentTypeParser {
}

export type BubbledProgressEvents =
// unixfs
GetEvents |
// unixfs-exporter
ExporterProgressEvents |
// helia blockstore
GetBlockProgressEvents |
// ipns
ResolveDNSLinkProgressEvents

Expand Down
59 changes: 34 additions & 25 deletions packages/verified-fetch/src/verified-fetch.ts
Original file line number Diff line number Diff line change
@@ -1,13 +1,13 @@
import { car } from '@helia/car'
import { ipns as heliaIpns, type IPNS } from '@helia/ipns'
import { unixfs } from '@helia/unixfs'
import * as ipldDagCbor from '@ipld/dag-cbor'
import * as ipldDagJson from '@ipld/dag-json'
import { code as dagPbCode } from '@ipld/dag-pb'
import { type AbortOptions, type Logger, type PeerId } from '@libp2p/interface'
import { Record as DHTRecord } from '@libp2p/kad-dht'
import { peerIdFromString } from '@libp2p/peer-id'
import { Key } from 'interface-datastore'
import { exporter } from 'ipfs-unixfs-exporter'
import toBrowserReadableStream from 'it-to-browser-readablestream'
import { LRUCache } from 'lru-cache'
import { code as jsonCode } from 'multiformats/codecs/json'
Expand Down Expand Up @@ -46,6 +46,14 @@ interface VerifiedFetchComponents {
ipns?: IPNS
}

/**
* Potential future options for the VerifiedFetch constructor.
*/
interface VerifiedFetchInit {
contentTypeParser?: ContentTypeParser
dnsResolvers?: DNSResolver[]
}

interface FetchHandlerFunctionArg {
cid: CID
path: string
Expand Down Expand Up @@ -141,7 +149,7 @@ export class VerifiedFetch {
private readonly contentTypeParser: ContentTypeParser | undefined
private readonly blockstoreSessions: LRUCache<string, SessionBlockstore>

constructor ({ helia, ipns }: VerifiedFetchComponents, init?: CreateVerifiedFetchOptions) {
constructor ({ helia, ipns }: VerifiedFetchComponents, init?: VerifiedFetchInit) {
this.helia = helia
this.log = helia.logger.forComponent('helia:verified-fetch')
this.ipns = ipns ?? heliaIpns(helia)
Expand Down Expand Up @@ -386,14 +394,15 @@ export class VerifiedFetch {
const rootFilePath = 'index.html'
try {
this.log.trace('found directory at %c/%s, looking for index.html', cid, path)
const stat = await fs.stat(dirCid, {
path: rootFilePath,

const entry = await exporter(`/ipfs/${dirCid}/${rootFilePath}`, this.helia.blockstore, {
signal: options?.signal,
onProgress: options?.onProgress
})
this.log.trace('found root file at %c/%s with cid %c', dirCid, rootFilePath, stat.cid)

this.log.trace('found root file at %c/%s with cid %c', dirCid, rootFilePath, entry.cid)
path = rootFilePath
resolvedCID = stat.cid
resolvedCID = entry.cid
} catch (err: any) {
options?.signal?.throwIfAborted()
this.log('error loading path %c/%s', dirCid, rootFilePath, err)
Expand All @@ -411,16 +420,22 @@ export class VerifiedFetch {
}
const offset = byteRangeContext.offset
const length = byteRangeContext.length
this.log.trace('calling unixfs.cat for %c/%s with offset=%o & length=%o', resolvedCID, path, offset, length)
const asyncIter = fs.cat(resolvedCID, {
signal: options?.signal,
onProgress: options?.onProgress,
offset,
length
})
this.log('got async iterator for %c/%s', cid, path)
this.log.trace('calling exporter for %c/%s with offset=%o & length=%o', resolvedCID, path, offset, length)

try {
const entry = await exporter(resolvedCID, this.helia.blockstore, {
signal: options?.signal,
onProgress: options?.onProgress
})

const asyncIter = entry.content({
signal: options?.signal,
onProgress: options?.onProgress,
offset,
length
})
this.log('got async iterator for %c/%s', cid, path)

const { stream, firstChunk } = await getStreamFromAsyncIterable(asyncIter, path ?? '', this.helia.logger, {
onProgress: options?.onProgress,
signal: options?.signal
Expand All @@ -436,7 +451,6 @@ export class VerifiedFetch {
if (ipfsRoots != null) {
response.headers.set('X-Ipfs-Roots', ipfsRoots.map(cid => cid.toV1().toString()).join(',')) // https://specs.ipfs.tech/http-gateways/path-gateway/#x-ipfs-roots-response-header
}

return response
} catch (err: any) {
options?.signal?.throwIfAborted()
Expand All @@ -460,18 +474,13 @@ export class VerifiedFetch {
// if the user has specified an `Accept` header that corresponds to a raw
// type, honour that header, so for example they don't request
// `application/vnd.ipld.raw` but get `application/octet-stream`
const overriddenContentType = getOverridenRawContentType({ headers: options?.headers, accept })
if (overriddenContentType != null) {
response.headers.set('content-type', overriddenContentType)
} else {
await this.setContentType(result, path, response)
}
await this.setContentType(result, path, response, getOverridenRawContentType({ headers: options?.headers, accept }))

return response
}

private async setContentType (bytes: Uint8Array, path: string, response: Response): Promise<void> {
let contentType = 'application/octet-stream'
private async setContentType (bytes: Uint8Array, path: string, response: Response, defaultContentType = 'application/octet-stream'): Promise<void> {
let contentType: string | undefined

if (this.contentTypeParser != null) {
try {
Expand All @@ -492,8 +501,8 @@ export class VerifiedFetch {
this.log.error('error parsing content type', err)
}
}
this.log.trace('setting content type to "%s"', contentType)
response.headers.set('content-type', contentType)
this.log.trace('setting content type to "%s"', contentType ?? defaultContentType)
response.headers.set('content-type', contentType ?? defaultContentType)
}

/**
Expand Down
Loading

0 comments on commit ed8b611

Please sign in to comment.