diff --git a/README.md b/README.md index 58479f98..8602e433 100644 --- a/README.md +++ b/README.md @@ -116,6 +116,14 @@ DID of the filecoin aggregator service. URL of the filecoin aggregator service. +#### `CONTENT_CLAIMS_DID` + +DID of the [content claims service](https://github.com/web3-storage/content-claims). + +#### `CONTENT_CLAIMS_URL` + +URL of the [content claims service](https://github.com/web3-storage/content-claims). + #### `UPLOAD_API_DID` [DID](https://www.w3.org/TR/did-core/) of the upload-api ucanto server. e.g. `did:web:up.web3.storage`. Optional: if omitted, a `did:key` will be derrived from `PRIVATE_KEY` @@ -208,6 +216,14 @@ Generated by [@ucanto/principal `EdSigner`](https://github.com/web3-storage/ucan _Example:_ `MgCZG7EvaA...1pX9as=` +#### `CONTENT_CLAIMS_PRIVATE_KEY` + +The `base64pad` [`multibase`](https://github.com/multiformats/multibase) encoded ED25519 keypair used as the signing key for [content-claims](https://github.com/web3-storage/content-claims). + +Generated by [@ucanto/principal `EdSigner`](https://github.com/web3-storage/ucanto) via [`ucan-key`](https://www.npmjs.com/package/ucan-key) + +_Example:_ `MgCZG7EvaA...1pX9as=` + #### `UCAN_INVOCATION_POST_BASIC_AUTH` The HTTP Basic auth token for the UCAN Invocation entrypoint, where UCAN invocations can be stored and proxied to the UCAN Stream. diff --git a/filecoin/functions/piece-cid-report.js b/filecoin/functions/piece-cid-report.js index f50c12a8..1aee46fe 100644 --- a/filecoin/functions/piece-cid-report.js +++ b/filecoin/functions/piece-cid-report.js @@ -2,6 +2,10 @@ import * as Sentry from '@sentry/serverless' import { Config } from '@serverless-stack/node/config/index.js' import { unmarshall } from '@aws-sdk/util-dynamodb' import { Piece } from '@web3-storage/data-segment' +import { CID } from 'multiformats/cid' +import * as Delegation from '@ucanto/core/delegation' +import { fromString } from 'uint8arrays/from-string' +import * as DID from '@ipld/dag-ucan/did' import { reportPieceCid } from '../index.js' import { getServiceConnection, getServiceSigner } from '../service.js' @@ -17,8 +21,8 @@ Sentry.AWSLambda.init({ * @param {import('aws-lambda').DynamoDBStreamEvent} event */ async function pieceCidReport (event) { - const { aggregatorDid, aggregatorUrl } = getEnv() - const { PRIVATE_KEY: privateKey } = Config + const { aggregatorDid, aggregatorUrl, contentClaimsDid, contentClaimsUrl, contentClaimsProof } = getEnv() + const { PRIVATE_KEY: privateKey, CONTENT_CLAIMS_PRIVATE_KEY: contentClaimsPrivateKey } = Config const records = parseDynamoDbEvent(event) if (records.length > 1) { @@ -28,30 +32,53 @@ async function pieceCidReport (event) { // @ts-expect-error can't figure out type of new const pieceRecord = unmarshall(records[0].new) const piece = Piece.fromString(pieceRecord.piece).link + const content = CID.parse(pieceRecord.link) const aggregateServiceConnection = getServiceConnection({ did: aggregatorDid, url: aggregatorUrl }) - const issuer = getServiceSigner({ + const claimsServiceConnection = getServiceConnection({ + did: contentClaimsDid, + url: contentClaimsUrl + }) + const storefrontIssuer = getServiceSigner({ privateKey }) - const audience = aggregateServiceConnection.id - /** @type {import('@web3-storage/filecoin-client/types').InvocationConfig} */ - const invocationConfig = { - issuer, - audience, - with: issuer.did(), + let claimsIssuer = getServiceSigner({ + privateKey: contentClaimsPrivateKey + }) + const claimsProofs = [] + if (contentClaimsProof) { + const proof = await Delegation.extract(fromString(contentClaimsProof, 'base64pad')) + if (!proof.ok) throw new Error('failed to extract proof', { cause: proof.error }) + claimsProofs.push(proof.ok) + } else { + // if no proofs, we must be using the service private key to sign + claimsIssuer = claimsIssuer.withDID(DID.parse(contentClaimsDid).did()) } const { ok, error } = await reportPieceCid({ piece, - group: issuer.did(), + content, + group: storefrontIssuer.did(), aggregateServiceConnection, - invocationConfig + aggregateInvocationConfig: /** @type {import('@web3-storage/filecoin-client/types').InvocationConfig} */ ({ + issuer: storefrontIssuer, + audience: aggregateServiceConnection.id, + with: storefrontIssuer.did(), + }), + claimsServiceConnection, + claimsInvocationConfig: /** @type {import('../types').ClaimsInvocationConfig} */ ({ + issuer: claimsIssuer, + audience: claimsServiceConnection.id, + with: claimsIssuer.did(), + }) }) if (error) { + console.error(error) + return { statusCode: 500, body: error.message || 'failed to add aggregate' @@ -73,6 +100,9 @@ function getEnv() { return { aggregatorDid: mustGetEnv('AGGREGATOR_DID'), aggregatorUrl: mustGetEnv('AGGREGATOR_URL'), + contentClaimsDid: mustGetEnv('CONTENT_CLAIMS_DID'), + contentClaimsUrl: mustGetEnv('CONTENT_CLAIMS_URL'), + contentClaimsProof: process.env.CONTENT_CLAIMS_PROOF, } } diff --git a/filecoin/index.js b/filecoin/index.js index 4a7286d4..a0488027 100644 --- a/filecoin/index.js +++ b/filecoin/index.js @@ -4,6 +4,7 @@ import * as Hasher from 'fr32-sha2-256-trunc254-padded-binary-tree-multihash' import * as Digest from 'multiformats/hashes/digest' import { Piece } from '@web3-storage/data-segment' import { CID } from 'multiformats/cid' +import { Assert } from '@web3-storage/content-claims/capability' import { Aggregator } from '@web3-storage/filecoin-client' import { GetCarFailed, ComputePieceFailed } from './errors.js' @@ -86,19 +87,43 @@ export async function computePieceCid({ /** * @param {object} props * @param {import('@web3-storage/data-segment').PieceLink} props.piece + * @param {import('multiformats').CID} props.content * @param {string} props.group - * @param {import('@web3-storage/filecoin-client/types').InvocationConfig} props.invocationConfig * @param {import('@ucanto/principal/ed25519').ConnectionView} props.aggregateServiceConnection + * @param {import('@web3-storage/filecoin-client/types').InvocationConfig} props.aggregateInvocationConfig + * @param {import('@ucanto/principal/ed25519').ConnectionView} props.claimsServiceConnection + * @param {import('./types.js').ClaimsInvocationConfig} props.claimsInvocationConfig */ export async function reportPieceCid ({ piece, + content, group, - invocationConfig, - aggregateServiceConnection + aggregateServiceConnection, + aggregateInvocationConfig, + claimsServiceConnection, + claimsInvocationConfig }) { + // Add claim for reading + const claimResult = await Assert.equals + .invoke({ + issuer: claimsInvocationConfig.issuer, + audience: claimsInvocationConfig.audience, + with: claimsInvocationConfig.with, + nb: { + content, + equals: piece + } + }) + .execute(claimsServiceConnection) + if (claimResult.out.error) { + return { + error: claimResult.out.error + } + } + // Add piece for aggregation const aggregateQueue = await Aggregator.aggregateQueue( - invocationConfig, + aggregateInvocationConfig, piece, group, { connection: aggregateServiceConnection } @@ -109,6 +134,7 @@ export async function reportPieceCid ({ error: aggregateQueue.out.error } } + return { ok: {}, } diff --git a/filecoin/package.json b/filecoin/package.json index 281fc354..b944a2df 100644 --- a/filecoin/package.json +++ b/filecoin/package.json @@ -11,12 +11,16 @@ "@aws-sdk/client-sqs": "^3.226.0", "@sentry/serverless": "^7.22.0", "@ucanto/client": "^8.0.1", + "@ucanto/core": "^8.2.0", + "@ucanto/interface": "^8.1.0", "@ucanto/principal": "^8.1.0", "@ucanto/transport": "^8.0.0", + "@web3-storage/content-claims": "^3.1.0", "@web3-storage/data-segment": "^3.0.1", "@web3-storage/filecoin-client": "^1.3.0", "fr32-sha2-256-trunc254-padded-binary-tree-multihash": "^1.0.0", - "multiformats": "^12.1.1" + "multiformats": "^12.1.1", + "uint8arrays": "4.0.6" }, "devDependencies": { "@serverless-stack/resources": "*", diff --git a/filecoin/test/helpers/mocks.js b/filecoin/test/helpers/mocks.js index d6dff7b2..cd33c8e2 100644 --- a/filecoin/test/helpers/mocks.js +++ b/filecoin/test/helpers/mocks.js @@ -6,7 +6,8 @@ const notImplemented = () => { /** * @param {Partial< - * import('@web3-storage/filecoin-client/types').AggregatorService + * import('@web3-storage/filecoin-client/types').AggregatorService & + * { assert: Partial } * >} impl */ export function mockService(impl) { @@ -15,6 +16,9 @@ export function mockService(impl) { add: withCallCount(impl.aggregate?.add ?? notImplemented), queue: withCallCount(impl.aggregate?.queue ?? notImplemented), }, + assert: { + equals: withCallCount(impl.assert?.equals ?? notImplemented) + } } } diff --git a/filecoin/test/helpers/ucanto.js b/filecoin/test/helpers/ucanto.js index c34dd151..30c34a52 100644 --- a/filecoin/test/helpers/ucanto.js +++ b/filecoin/test/helpers/ucanto.js @@ -3,12 +3,68 @@ import * as Server from '@ucanto/server' import * as Client from '@ucanto/client' import * as CAR from '@ucanto/transport/car' import * as FilecoinCapabilities from '@web3-storage/capabilities/filecoin' +import { Assert } from '@web3-storage/content-claims/capability' import { OperationFailed } from './errors.js' import { mockService } from './mocks.js' const nop = (/** @type {any} */ invCap) => {} +/** + * @param {any} serviceProvider + * @param {object} [options] + * @param {(inCap: any) => void} [options.onCall] + * @param {boolean} [options.mustFail] + */ +export async function getClaimsServiceServer (serviceProvider, options = {}) { + const onCall = options.onCall || nop + const equalsStore = new Map() + + const service = mockService({ + assert: { + equals: Server.provide(Assert.equals, async ({ capability, invocation }) => { + const invCap = invocation.capabilities[0] + const { content, equals } = capability.nb + + if (options.mustFail) { + return { + error: new OperationFailed( + 'failed to add to aggregate', + // @ts-ignore wrong dep + invCap.nb?.content + ) + } + } + + equalsStore.set(content.toString(), equals.toString()) + equalsStore.set(equals.toString(), content.toString()) + + onCall(invCap) + + return { + ok: {} + } + }) + } + }) + + const server = Server.create({ + id: serviceProvider, + service, + codec: CAR.inbound, + }) + const connection = Client.connect({ + id: serviceProvider, + codec: CAR.outbound, + channel: server, + }) + + return { + service, + connection + } +} + /** * @param {any} serviceProvider * @param {object} [options] @@ -112,9 +168,10 @@ export async function getAggregatorServiceServer (serviceProvider, options = {}) } } -export async function getAggregatorServiceCtx () { +export async function getServiceCtx () { const storefront = await Signer.generate() const aggregator = await Signer.generate() + const claims = await Signer.generate() return { storefront: { @@ -126,6 +183,11 @@ export async function getAggregatorServiceCtx () { did: aggregator.did(), privateKey: Signer.format(aggregator), raw: aggregator + }, + claims: { + did: claims.did(), + privateKey: Signer.format(claims), + raw: claims } } } diff --git a/filecoin/test/report-piece-cid.test.js b/filecoin/test/report-piece-cid.test.js index 7d4a50e2..524a7e1b 100644 --- a/filecoin/test/report-piece-cid.test.js +++ b/filecoin/test/report-piece-cid.test.js @@ -5,77 +5,146 @@ import pDefer from 'p-defer' import { reportPieceCid } from '../index.js' import { getServiceSigner } from '../service.js' -import { getAggregatorServiceServer, getAggregatorServiceCtx } from './helpers/ucanto.js' +import { getAggregatorServiceServer, getClaimsServiceServer, getServiceCtx } from './helpers/ucanto.js' import { createCar } from './helpers/car.js' test('reports piece cid from a piece written to the piece table', async t => { - const { piece } = await createCar() + const { piece, link } = await createCar() const aggregatorQueueCall = pDefer() - const { invocationConfig, aggregatorService } = await getService({ - onCall: aggregatorQueueCall + const claimsEqualsCall = pDefer() + const { aggregateInvocationConfig, aggregatorService, claimsInvocationConfig, claimsService } = await getService({ + aggregator: { + onCall: aggregatorQueueCall + }, + claims: { + onCall: claimsEqualsCall + } }) const reportPieceCidResponse = await reportPieceCid({ piece, - group: invocationConfig.issuer.did(), - invocationConfig, - aggregateServiceConnection: aggregatorService.connection + content: link, + group: aggregateInvocationConfig.issuer.did(), + aggregateInvocationConfig, + aggregateServiceConnection: aggregatorService.connection, + claimsInvocationConfig, + claimsServiceConnection: claimsService.connection, }) t.truthy(reportPieceCidResponse.ok) t.falsy(reportPieceCidResponse.error) - // Validate ucanto server call + // Validate ucanto server calls + t.is(claimsService.service.assert.equals.callCount, 1) + const invCapClaims = await claimsEqualsCall.promise + t.is(invCapClaims.can, 'assert/equals') + t.is(aggregatorService.service.aggregate.queue.callCount, 1) - const invCap = await aggregatorQueueCall.promise - t.is(invCap.can, 'aggregate/queue') + const invCapAggregator = await aggregatorQueueCall.promise + t.is(invCapAggregator.can, 'aggregate/queue') +}) + +test('fails reporting piece cid if fails to claim equals', async t => { + const { piece, link } = await createCar() + const aggregatorQueueCall = pDefer() + const claimEqualsCall = pDefer() + const { aggregateInvocationConfig, aggregatorService, claimsInvocationConfig, claimsService } = await getService({ + aggregator: { + onCall: aggregatorQueueCall + }, + claims: { + onCall: claimEqualsCall, + mustFail: true + } + }) + + const reportPieceCidResponse = await reportPieceCid({ + piece, + content: link, + group: aggregateInvocationConfig.issuer.did(), + aggregateInvocationConfig, + aggregateServiceConnection: aggregatorService.connection, + claimsInvocationConfig, + claimsServiceConnection: claimsService.connection, + }) + + t.falsy(reportPieceCidResponse.ok) + t.truthy(reportPieceCidResponse.error) + + // Validate ucanto server calls + t.is(claimsService.service.assert.equals.callCount, 1) + t.is(aggregatorService.service.aggregate.queue.callCount, 0) }) test('fails reporting piece cid if fails to queue to aggregator', async t => { - const { piece } = await createCar() + const { piece, link } = await createCar() const aggregatorQueueCall = pDefer() - const { invocationConfig, aggregatorService } = await getService({ - onCall: aggregatorQueueCall, - mustFail: true + const claimEqualsCall = pDefer() + const { aggregateInvocationConfig, aggregatorService, claimsInvocationConfig, claimsService } = await getService({ + aggregator: { + onCall: aggregatorQueueCall, + mustFail: true + }, + claims: { + onCall: claimEqualsCall + } }) const reportPieceCidResponse = await reportPieceCid({ piece, - group: invocationConfig.issuer.did(), - invocationConfig, - aggregateServiceConnection: aggregatorService.connection + content: link, + group: aggregateInvocationConfig.issuer.did(), + aggregateInvocationConfig, + aggregateServiceConnection: aggregatorService.connection, + claimsInvocationConfig, + claimsServiceConnection: claimsService.connection, }) t.falsy(reportPieceCidResponse.ok) t.truthy(reportPieceCidResponse.error) + // Validate ucanto server calls + t.is(claimsService.service.assert.equals.callCount, 1) t.is(aggregatorService.service.aggregate.queue.callCount, 1) }) /** - * @param {object} options - * @param {import('p-defer').DeferredPromise} options.onCall - * @param {boolean} [options.mustFail] + * @typedef {object} Props + * @property {import('p-defer').DeferredPromise} onCall + * @property {boolean} [mustFail] + * + * @param {Record<'aggregator' | 'claims', Props>} options */ async function getService (options) { - const { storefront, aggregator } = await getAggregatorServiceCtx() + const { storefront, aggregator, claims } = await getServiceCtx() const aggregatorService = await getAggregatorServiceServer(aggregator.raw, { onCall: (invCap) => { - options.onCall.resolve(invCap) + options.aggregator.onCall.resolve(invCap) }, - mustFail: options.mustFail + mustFail: options.aggregator.mustFail }) + + const claimsService = await getClaimsServiceServer(claims.raw, { + onCall: (invCap) => { + options.claims.onCall.resolve(invCap) + }, + mustFail: options.claims.mustFail + }) + const issuer = getServiceSigner(storefront) - const audience = aggregatorService.connection.id - /** @type {import('@web3-storage/filecoin-client/types').InvocationConfig} */ - const invocationConfig = { - issuer, - audience, - with: issuer.did(), - } return { - invocationConfig, - aggregatorService + aggregateInvocationConfig: /** @type {import('@web3-storage/filecoin-client/types').InvocationConfig} */({ + issuer, + audience: aggregatorService.connection.id, + with: issuer.did(), + }), + aggregatorService, + claimsInvocationConfig:/** @type {import('../types').ClaimsInvocationConfig} */({ + issuer, + audience: claimsService.connection.id, + with: issuer.did(), + }), + claimsService, } } diff --git a/filecoin/types.ts b/filecoin/types.ts index 4b86adb7..7da04c42 100644 --- a/filecoin/types.ts +++ b/filecoin/types.ts @@ -1,3 +1,9 @@ +import { + Signer, + DID, + Principal, + Proof, +} from '@ucanto/interface' import { UnknownLink } from 'multiformats' import { PieceLink } from '@web3-storage/data-segment' @@ -22,6 +28,25 @@ export interface ComputePieceError extends Error { name: 'ComputePieceFailed' } +export interface ClaimsInvocationConfig { + /** + * Signing authority that is issuing the UCAN invocation(s). + */ + issuer: Signer + /** + * The principal delegated to in the current UCAN. + */ + audience: Principal + /** + * The resource the invocation applies to. + */ + with: DID + /** + * Proof(s) the issuer has the capability to perform the action. + */ + proofs?: Proof[] +} + export class Failure extends Error { describe() { return this.toString() @@ -79,3 +104,17 @@ export type Variant> = { [K in Key]: U[Key] } }[keyof U] + +// would be generated by sst, but requires `sst build` to be run, which calls out to aws; not great for CI +declare module '@serverless-stack/node/config' { + export interface SecretResources { + PRIVATE_KEY: { + value: string + }, + CONTENT_CLAIMS_PRIVATE_KEY: { + value: string + } + } +} + +export {} diff --git a/package-lock.json b/package-lock.json index ebd5fe19..a16ebb79 100644 --- a/package-lock.json +++ b/package-lock.json @@ -74,12 +74,16 @@ "@aws-sdk/client-sqs": "^3.226.0", "@sentry/serverless": "^7.22.0", "@ucanto/client": "^8.0.1", + "@ucanto/core": "^8.2.0", + "@ucanto/interface": "^8.1.0", "@ucanto/principal": "^8.1.0", "@ucanto/transport": "^8.0.0", + "@web3-storage/content-claims": "^3.1.0", "@web3-storage/data-segment": "^3.0.1", "@web3-storage/filecoin-client": "^1.3.0", "fr32-sha2-256-trunc254-padded-binary-tree-multihash": "^1.0.0", - "multiformats": "^12.1.1" + "multiformats": "^12.1.1", + "uint8arrays": "4.0.6" }, "devDependencies": { "@serverless-stack/resources": "*", @@ -4277,13 +4281,14 @@ } }, "node_modules/@ucanto/core": { - "version": "8.1.0", - "license": "(Apache-2.0 AND MIT)", + "version": "8.2.0", + "resolved": "https://registry.npmjs.org/@ucanto/core/-/core-8.2.0.tgz", + "integrity": "sha512-Yw1NZySQFS2+JPUuWGlRpq4HMB1npXYq1WBo6B1tINGRabKAujq3HrZ/iidYL2MuoHn0CGfSvCUzlXrcD4obNg==", "dependencies": { "@ipld/car": "^5.1.0", "@ipld/dag-cbor": "^9.0.0", - "@ipld/dag-ucan": "^3.3.2", - "@ucanto/interface": "^8.0.0", + "@ipld/dag-ucan": "^3.4.0", + "@ucanto/interface": "^8.1.0", "multiformats": "^11.0.2" } }, @@ -4448,6 +4453,27 @@ "@web3-storage/data-segment": "^3.0.1" } }, + "node_modules/@web3-storage/content-claims": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/@web3-storage/content-claims/-/content-claims-3.1.0.tgz", + "integrity": "sha512-ieoEGL+bqmAjxS2NDcxF8jo0IKpBDhDu/bnNXRbhgQLUtGMxv+2PT0n35PM69AKnw8QgzEQCM8FibbZAdw0I/Q==", + "dependencies": { + "@ucanto/client": "^8.0.0", + "@ucanto/server": "^8.0.1", + "@ucanto/transport": "^8.0.0", + "carstream": "^1.0.2", + "multiformats": "^12.0.1" + } + }, + "node_modules/@web3-storage/content-claims/node_modules/multiformats": { + "version": "12.1.1", + "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-12.1.1.tgz", + "integrity": "sha512-GBSToTmri2vJYs8wqcZQ8kB21dCaeTOzHTIAlr8J06C1eL6UbzqURXFZ5Fl0EYm9GAFz1IlYY8SxGOs9G9NJRg==", + "engines": { + "node": ">=16.0.0", + "npm": ">=7.0.0" + } + }, "node_modules/@web3-storage/data-segment": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/@web3-storage/data-segment/-/data-segment-3.0.1.tgz", @@ -6015,6 +6041,25 @@ "cardex": "bin.js" } }, + "node_modules/carstream": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/carstream/-/carstream-1.1.0.tgz", + "integrity": "sha512-tbf8FOnGX1+0kOe77nm9MG53REiqQopDwzwbXYVxUcsKOAHG2KSD++qy95v1vrtRt1Q6L0Sb01it7QwJ+Yt1sQ==", + "dependencies": { + "@ipld/dag-cbor": "^9.0.3", + "multiformats": "^12.0.1", + "uint8arraylist": "^2.4.3" + } + }, + "node_modules/carstream/node_modules/multiformats": { + "version": "12.1.1", + "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-12.1.1.tgz", + "integrity": "sha512-GBSToTmri2vJYs8wqcZQ8kB21dCaeTOzHTIAlr8J06C1eL6UbzqURXFZ5Fl0EYm9GAFz1IlYY8SxGOs9G9NJRg==", + "engines": { + "node": ">=16.0.0", + "npm": ">=7.0.0" + } + }, "node_modules/cbor": { "version": "8.1.0", "dev": true, @@ -13446,6 +13491,18 @@ "node": ">=4.2.0" } }, + "node_modules/uint8arraylist": { + "version": "2.4.3", + "resolved": "https://registry.npmjs.org/uint8arraylist/-/uint8arraylist-2.4.3.tgz", + "integrity": "sha512-oEVZr4/GrH87K0kjNce6z8pSCzLEPqHNLNR5sj8cJOySrTP8Vb/pMIbZKLJGhQKxm1TiZ31atNrpn820Pyqpow==", + "dependencies": { + "uint8arrays": "^4.0.2" + }, + "engines": { + "node": ">=16.0.0", + "npm": ">=7.0.0" + } + }, "node_modules/uint8arrays": { "version": "4.0.6", "license": "Apache-2.0 OR MIT", @@ -17325,12 +17382,14 @@ } }, "@ucanto/core": { - "version": "8.1.0", + "version": "8.2.0", + "resolved": "https://registry.npmjs.org/@ucanto/core/-/core-8.2.0.tgz", + "integrity": "sha512-Yw1NZySQFS2+JPUuWGlRpq4HMB1npXYq1WBo6B1tINGRabKAujq3HrZ/iidYL2MuoHn0CGfSvCUzlXrcD4obNg==", "requires": { "@ipld/car": "^5.1.0", "@ipld/dag-cbor": "^9.0.0", - "@ipld/dag-ucan": "^3.3.2", - "@ucanto/interface": "^8.0.0", + "@ipld/dag-ucan": "^3.4.0", + "@ucanto/interface": "^8.1.0", "multiformats": "^11.0.2" } }, @@ -17480,6 +17539,25 @@ "@web3-storage/data-segment": "^3.0.1" } }, + "@web3-storage/content-claims": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/@web3-storage/content-claims/-/content-claims-3.1.0.tgz", + "integrity": "sha512-ieoEGL+bqmAjxS2NDcxF8jo0IKpBDhDu/bnNXRbhgQLUtGMxv+2PT0n35PM69AKnw8QgzEQCM8FibbZAdw0I/Q==", + "requires": { + "@ucanto/client": "^8.0.0", + "@ucanto/server": "^8.0.1", + "@ucanto/transport": "^8.0.0", + "carstream": "^1.0.2", + "multiformats": "^12.0.1" + }, + "dependencies": { + "multiformats": { + "version": "12.1.1", + "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-12.1.1.tgz", + "integrity": "sha512-GBSToTmri2vJYs8wqcZQ8kB21dCaeTOzHTIAlr8J06C1eL6UbzqURXFZ5Fl0EYm9GAFz1IlYY8SxGOs9G9NJRg==" + } + } + }, "@web3-storage/data-segment": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/@web3-storage/data-segment/-/data-segment-3.0.1.tgz", @@ -17618,8 +17696,11 @@ "@sentry/serverless": "^7.22.0", "@serverless-stack/resources": "*", "@ucanto/client": "^8.0.1", + "@ucanto/core": "^8.2.0", + "@ucanto/interface": "^8.1.0", "@ucanto/principal": "^8.1.0", "@ucanto/transport": "^8.0.0", + "@web3-storage/content-claims": "^3.1.0", "@web3-storage/data-segment": "^3.0.1", "@web3-storage/filecoin-client": "^1.3.0", "ava": "^4.3.3", @@ -17627,7 +17708,8 @@ "multiformats": "^12.1.1", "nanoid": "^4.0.0", "p-defer": "^4.0.0", - "testcontainers": "^8.13.0" + "testcontainers": "^8.13.0", + "uint8arrays": "4.0.6" }, "dependencies": { "multiformats": { @@ -18692,6 +18774,23 @@ "varint": "^6.0.0" } }, + "carstream": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/carstream/-/carstream-1.1.0.tgz", + "integrity": "sha512-tbf8FOnGX1+0kOe77nm9MG53REiqQopDwzwbXYVxUcsKOAHG2KSD++qy95v1vrtRt1Q6L0Sb01it7QwJ+Yt1sQ==", + "requires": { + "@ipld/dag-cbor": "^9.0.3", + "multiformats": "^12.0.1", + "uint8arraylist": "^2.4.3" + }, + "dependencies": { + "multiformats": { + "version": "12.1.1", + "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-12.1.1.tgz", + "integrity": "sha512-GBSToTmri2vJYs8wqcZQ8kB21dCaeTOzHTIAlr8J06C1eL6UbzqURXFZ5Fl0EYm9GAFz1IlYY8SxGOs9G9NJRg==" + } + } + }, "cbor": { "version": "8.1.0", "dev": true, @@ -23258,6 +23357,14 @@ "version": "4.9.5", "dev": true }, + "uint8arraylist": { + "version": "2.4.3", + "resolved": "https://registry.npmjs.org/uint8arraylist/-/uint8arraylist-2.4.3.tgz", + "integrity": "sha512-oEVZr4/GrH87K0kjNce6z8pSCzLEPqHNLNR5sj8cJOySrTP8Vb/pMIbZKLJGhQKxm1TiZ31atNrpn820Pyqpow==", + "requires": { + "uint8arrays": "^4.0.2" + } + }, "uint8arrays": { "version": "4.0.6", "requires": { diff --git a/stacks/config.js b/stacks/config.js index 65005b19..e63fc745 100644 --- a/stacks/config.js +++ b/stacks/config.js @@ -143,6 +143,10 @@ export function getEnv() { UPLOAD_API_DID: mustGetEnv('UPLOAD_API_DID'), AGGREGATOR_DID: mustGetEnv('AGGREGATOR_DID'), AGGREGATOR_URL: mustGetEnv('AGGREGATOR_URL'), + CONTENT_CLAIMS_DID: mustGetEnv('CONTENT_CLAIMS_DID'), + CONTENT_CLAIMS_URL: mustGetEnv('CONTENT_CLAIMS_URL'), + // Not required + CONTENT_CLAIMS_PROOF: process.env.CONTENT_CLAIMS_PROOF ?? '' } } diff --git a/stacks/filecoin-stack.js b/stacks/filecoin-stack.js index 277671ec..80b27b04 100644 --- a/stacks/filecoin-stack.js +++ b/stacks/filecoin-stack.js @@ -20,7 +20,7 @@ export function FilecoinStack({ stack, app }) { srcPath: 'filecoin' }) - const { AGGREGATOR_DID, AGGREGATOR_URL } = getEnv() + const { AGGREGATOR_DID, AGGREGATOR_URL, CONTENT_CLAIMS_DID, CONTENT_CLAIMS_URL, CONTENT_CLAIMS_PROOF } = getEnv() // Setup app monitoring with Sentry setupSentry(app, stack) @@ -30,7 +30,7 @@ export function FilecoinStack({ stack, app }) { // Get eventBus reference const { eventBus } = use(BusStack) // Get store table reference - const { pieceTable, privateKey } = use(UploadDbStack) + const { pieceTable, privateKey, contentClaimsPrivateKey } = use(UploadDbStack) // piece-cid reporting pieceTable.addConsumers(stack, { @@ -40,10 +40,14 @@ export function FilecoinStack({ stack, app }) { environment: { AGGREGATOR_DID, AGGREGATOR_URL, + CONTENT_CLAIMS_DID, + CONTENT_CLAIMS_URL, + CONTENT_CLAIMS_PROOF }, timeout: 3 * 60, bind: [ privateKey, + contentClaimsPrivateKey ] }, cdk: { diff --git a/stacks/upload-db-stack.js b/stacks/upload-db-stack.js index a92a6d31..7a4200d0 100644 --- a/stacks/upload-db-stack.js +++ b/stacks/upload-db-stack.js @@ -25,8 +25,13 @@ export function UploadDbStack({ stack, app }) { // Setup app monitoring with Sentry setupSentry(app, stack) + // Upload API private key const privateKey = new Config.Secret(stack, 'PRIVATE_KEY') + // Content claims private key + // TODO: we should look into creating a trust layer for content claims + const contentClaimsPrivateKey = new Config.Secret(stack, 'CONTENT_CLAIMS_PRIVATE_KEY') + /** * This table takes a stored CAR and makes an entry in the store table * Used by the store/* service capabilities. @@ -100,6 +105,7 @@ export function UploadDbStack({ stack, app }) { delegationTable, adminMetricsTable, spaceMetricsTable, - privateKey + privateKey, + contentClaimsPrivateKey } } diff --git a/test/integration.test.js b/test/integration.test.js index 8e53db48..cfbe0908 100644 --- a/test/integration.test.js +++ b/test/integration.test.js @@ -142,6 +142,7 @@ test('w3infra integration flow', async t => { const fileLink = await client.uploadFile(file, { onShardStored: (meta) => { shards.push(meta.cid) + console.log('shard file written', meta.cid) } }) t.truthy(fileLink) @@ -243,6 +244,8 @@ test('w3infra integration flow', async t => { t.is(pieces?.length, 1) t.truthy(pieces?.[0].piece) + console.log('piece written', pieces?.[0].piece) + // Check metrics were updated if (beforeStoreAddSizeTotal && spaceBeforeUploadAddMetrics && spaceBeforeStoreAddSizeMetrics && beforeUploadAddTotal) { await pWaitFor(async () => {