Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

fix: use ipfs-unixfs-exporter directly #42

Merged
merged 4 commits into from
Apr 15, 2024
Merged

fix: use ipfs-unixfs-exporter directly #42

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
ac1f7c5
fix: use ipfs-unixfs-exporter directly
achingbrain Apr 5, 2024
8230eb0
Merge remote-tracking branch 'origin/main' into fix/use-exporter-dire…
achingbrain Apr 15, 2024
7f29e5b
chore: update utils
achingbrain Apr 15, 2024
a263f60
chore: update comment
achingbrain Apr 15, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 14 additions & 13 deletions packages/verified-fetch/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,13 +57,12 @@
"release": "aegir release"
},
"dependencies": {
"@helia/block-brokers": "^2.0.3",
"@helia/car": "^3.1.2",
"@helia/http": "^1.0.3",
"@helia/interface": "^4.1.0",
"@helia/ipns": "^7.2.0",
"@helia/routers": "^1.0.2",
"@helia/unixfs": "^3.0.3",
"@helia/block-brokers": "^2.1.0",
"@helia/car": "^3.1.3",
"@helia/http": "^1.0.4",
"@helia/interface": "^4.2.0",
"@helia/ipns": "^7.2.1",
"@helia/routers": "^1.0.3",
"@ipld/dag-cbor": "^9.2.0",
"@ipld/dag-json": "^10.2.0",
"@ipld/dag-pb": "^4.1.0",
Expand All @@ -85,11 +84,12 @@
"uint8arrays": "^5.0.3"
},
"devDependencies": {
"@helia/car": "^3.1.2",
"@helia/dag-cbor": "^3.0.2",
"@helia/dag-json": "^3.0.2",
"@helia/json": "^3.0.2",
"@helia/utils": "^0.1.0",
"@helia/car": "^3.1.3",
"@helia/dag-cbor": "^3.0.3",
"@helia/dag-json": "^3.0.3",
"@helia/json": "^3.0.3",
"@helia/unixfs": "^3.0.4",
"@helia/utils": "^0.2.0",
"@ipld/car": "^5.3.0",
"@libp2p/interface-compliance-tests": "^5.3.4",
"@libp2p/logger": "^4.0.9",
Expand All @@ -100,10 +100,11 @@
"blockstore-core": "^4.4.1",
"browser-readablestream-to-it": "^2.0.5",
"datastore-core": "^9.2.9",
"helia": "^4.1.0",
"helia": "^4.1.1",
"ipfs-unixfs-importer": "^15.2.5",
"ipns": "^9.1.0",
"it-all": "^3.0.4",
"it-drain": "^3.0.5",
"it-last": "^3.0.4",
"it-to-buffer": "^4.0.5",
"magic-bytes.js": "^1.10.0",
Expand Down
10 changes: 6 additions & 4 deletions packages/verified-fetch/src/index.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -594,11 +594,11 @@ import { createHeliaHTTP } from '@helia/http'
import { delegatedHTTPRouting } from '@helia/routers'
import { dns } from '@multiformats/dns'
import { VerifiedFetch as VerifiedFetchClass } from './verified-fetch.js'
import type { Helia } from '@helia/interface'
import type { GetBlockProgressEvents, Helia } from '@helia/interface'
import type { ResolveDNSLinkProgressEvents } from '@helia/ipns'
import type { GetEvents } from '@helia/unixfs'
import type { DNSResolvers, DNS } from '@multiformats/dns'
import type { DNSResolver } from '@multiformats/dns/resolvers'
import type { ExporterProgressEvents } from 'ipfs-unixfs-exporter'
import type { CID } from 'multiformats/cid'
import type { ProgressEvent, ProgressOptions } from 'progress-events'

Expand Down Expand Up @@ -674,8 +674,10 @@ export interface ContentTypeParser {
}

export type BubbledProgressEvents =
// unixfs
GetEvents |
// unixfs-exporter
ExporterProgressEvents |
// helia blockstore
GetBlockProgressEvents |
// ipns
ResolveDNSLinkProgressEvents

Expand Down
39 changes: 21 additions & 18 deletions packages/verified-fetch/src/verified-fetch.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,13 +1,13 @@
import { car } from '@helia/car'
import { ipns as heliaIpns, type IPNS } from '@helia/ipns'
import { unixfs as heliaUnixFs, type UnixFS as HeliaUnixFs } from '@helia/unixfs'
import * as ipldDagCbor from '@ipld/dag-cbor'
import * as ipldDagJson from '@ipld/dag-json'
import { code as dagPbCode } from '@ipld/dag-pb'
import { type AbortOptions, type Logger, type PeerId } from '@libp2p/interface'
import { Record as DHTRecord } from '@libp2p/kad-dht'
import { peerIdFromString } from '@libp2p/peer-id'
import { Key } from 'interface-datastore'
import { exporter } from 'ipfs-unixfs-exporter'
import toBrowserReadableStream from 'it-to-browser-readablestream'
import { code as jsonCode } from 'multiformats/codecs/json'
import { code as rawCode } from 'multiformats/codecs/raw'
Expand Down Expand Up @@ -39,7 +39,6 @@ import type { CID } from 'multiformats/cid'
interface VerifiedFetchComponents {
helia: Helia
ipns?: IPNS
unixfs?: HeliaUnixFs
}

/**
Expand Down Expand Up @@ -128,15 +127,13 @@ function getOverridenRawContentType ({ headers, accept }: { headers?: HeadersIni
export class VerifiedFetch {
private readonly helia: Helia
private readonly ipns: IPNS
private readonly unixfs: HeliaUnixFs
private readonly log: Logger
private readonly contentTypeParser: ContentTypeParser | undefined

constructor ({ helia, ipns, unixfs }: VerifiedFetchComponents, init?: VerifiedFetchInit) {
constructor ({ helia, ipns }: VerifiedFetchComponents, init?: VerifiedFetchInit) {
this.helia = helia
this.log = helia.logger.forComponent('helia:verified-fetch')
this.ipns = ipns ?? heliaIpns(helia)
this.unixfs = unixfs ?? heliaUnixFs(helia)
this.contentTypeParser = init?.contentTypeParser
this.log.trace('created VerifiedFetch instance')
}
Expand Down Expand Up @@ -350,14 +347,15 @@ export class VerifiedFetch {
const rootFilePath = 'index.html'
try {
this.log.trace('found directory at %c/%s, looking for index.html', cid, path)
const stat = await this.unixfs.stat(dirCid, {
path: rootFilePath,

const entry = await exporter(`/ipfs/${dirCid}/${rootFilePath}`, this.helia.blockstore, {
signal: options?.signal,
onProgress: options?.onProgress
})
this.log.trace('found root file at %c/%s with cid %c', dirCid, rootFilePath, stat.cid)

this.log.trace('found root file at %c/%s with cid %c', dirCid, rootFilePath, entry.cid)
path = rootFilePath
resolvedCID = stat.cid
resolvedCID = entry.cid
} catch (err: any) {
options?.signal?.throwIfAborted()
this.log('error loading path %c/%s', dirCid, rootFilePath, err)
Expand All @@ -375,16 +373,22 @@ export class VerifiedFetch {
}
const offset = byteRangeContext.offset
const length = byteRangeContext.length
this.log.trace('calling unixfs.cat for %c/%s with offset=%o & length=%o', resolvedCID, path, offset, length)
const asyncIter = this.unixfs.cat(resolvedCID, {
signal: options?.signal,
onProgress: options?.onProgress,
offset,
length
})
this.log('got async iterator for %c/%s', cid, path)
this.log.trace('calling exporter for %c/%s with offset=%o & length=%o', resolvedCID, path, offset, length)

try {
const entry = await exporter(resolvedCID, this.helia.blockstore, {
signal: options?.signal,
onProgress: options?.onProgress
})

const asyncIter = entry.content({
signal: options?.signal,
onProgress: options?.onProgress,
offset,
length
})
this.log('got async iterator for %c/%s', cid, path)

const { stream, firstChunk } = await getStreamFromAsyncIterable(asyncIter, path ?? '', this.helia.logger, {
onProgress: options?.onProgress,
signal: options?.signal
Expand All @@ -400,7 +404,6 @@ export class VerifiedFetch {
if (ipfsRoots != null) {
response.headers.set('X-Ipfs-Roots', ipfsRoots.map(cid => cid.toV1().toString()).join(',')) // https://specs.ipfs.tech/http-gateways/path-gateway/#x-ipfs-roots-response-header
}

return response
} catch (err: any) {
options?.signal?.throwIfAborted()
Expand Down
Loading
Loading