From 74c2de62cc827ecd6ad50f1b6e72d6c87c2a539d Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Mon, 18 Sep 2023 18:57:01 +0200 Subject: [PATCH 1/7] feat: trigger content claims equals on piece cid computed --- filecoin/functions/piece-cid-report.js | 28 ++++-- filecoin/index.js | 35 ++++++- filecoin/package.json | 2 + filecoin/test/helpers/mocks.js | 6 +- filecoin/test/helpers/ucanto.js | 64 +++++++++++- filecoin/test/report-piece-cid.test.js | 133 +++++++++++++++++++------ filecoin/types.ts | 20 ++++ package-lock.json | 99 ++++++++++++++++++ stacks/config.js | 2 + stacks/filecoin-stack.js | 4 +- test/integration.test.js | 2 +- 11 files changed, 347 insertions(+), 48 deletions(-) diff --git a/filecoin/functions/piece-cid-report.js b/filecoin/functions/piece-cid-report.js index f50c12a8..43102294 100644 --- a/filecoin/functions/piece-cid-report.js +++ b/filecoin/functions/piece-cid-report.js @@ -2,6 +2,7 @@ import * as Sentry from '@sentry/serverless' import { Config } from '@serverless-stack/node/config/index.js' import { unmarshall } from '@aws-sdk/util-dynamodb' import { Piece } from '@web3-storage/data-segment' +import { CID } from 'multiformats/cid' import { reportPieceCid } from '../index.js' import { getServiceConnection, getServiceSigner } from '../service.js' @@ -28,27 +29,36 @@ async function pieceCidReport (event) { // @ts-expect-error can't figure out type of new const pieceRecord = unmarshall(records[0].new) const piece = Piece.fromString(pieceRecord.piece).link + const content = CID.parse(pieceRecord.link) const aggregateServiceConnection = getServiceConnection({ did: aggregatorDid, url: aggregatorUrl }) + const claimsServiceConnection = getServiceConnection({ + did: aggregatorDid, + url: aggregatorUrl + }) const issuer = getServiceSigner({ privateKey }) - const audience = aggregateServiceConnection.id - /** @type {import('@web3-storage/filecoin-client/types').InvocationConfig} */ - const invocationConfig = { - issuer, - audience, - with: issuer.did(), - } const { ok, error } = await reportPieceCid({ piece, + content, group: issuer.did(), aggregateServiceConnection, - invocationConfig + aggregateInvocationConfig: /** @type {import('@web3-storage/filecoin-client/types').InvocationConfig} */ ({ + issuer, + audience: aggregateServiceConnection.id, + with: issuer.did(), + }), + claimsServiceConnection, + claimsInvocationConfig: /** @type {import('../types').ClaimsInvocationConfig} */ ({ + issuer, + audience: claimsServiceConnection.id, + with: issuer.did(), + }) }) if (error) { @@ -73,6 +83,8 @@ function getEnv() { return { aggregatorDid: mustGetEnv('AGGREGATOR_DID'), aggregatorUrl: mustGetEnv('AGGREGATOR_URL'), + contentClaimsDid: mustGetEnv('CONTENT_CLAIMS_DID'), + contentClaimsUrl: mustGetEnv('CONTENT_CLAIMS_URL'), } } diff --git a/filecoin/index.js b/filecoin/index.js index 4a7286d4..7421b817 100644 --- a/filecoin/index.js +++ b/filecoin/index.js @@ -4,6 +4,7 @@ import * as Hasher from 'fr32-sha2-256-trunc254-padded-binary-tree-multihash' import * as Digest from 'multiformats/hashes/digest' import { Piece } from '@web3-storage/data-segment' import { CID } from 'multiformats/cid' +import { Assert } from '@web3-storage/content-claims/capability' import { Aggregator } from '@web3-storage/filecoin-client' import { GetCarFailed, ComputePieceFailed } from './errors.js' @@ -86,19 +87,44 @@ export async function computePieceCid({ /** * @param {object} props * @param {import('@web3-storage/data-segment').PieceLink} props.piece + * @param {import('multiformats').CID} props.content * @param {string} props.group - * @param {import('@web3-storage/filecoin-client/types').InvocationConfig} props.invocationConfig * @param {import('@ucanto/principal/ed25519').ConnectionView} props.aggregateServiceConnection + * @param {import('@web3-storage/filecoin-client/types').InvocationConfig} props.aggregateInvocationConfig + * @param {import('@ucanto/principal/ed25519').ConnectionView} props.claimsServiceConnection + * @param {import('./types.js').ClaimsInvocationConfig} props.claimsInvocationConfig */ export async function reportPieceCid ({ piece, + content, group, - invocationConfig, - aggregateServiceConnection + aggregateServiceConnection, + aggregateInvocationConfig, + claimsServiceConnection, + claimsInvocationConfig }) { + // Add claim for reading + const claimResult = await Assert.equals + .invoke({ + issuer: claimsInvocationConfig.issuer, + audience: claimsInvocationConfig.audience, + with: claimsInvocationConfig.with, + nb: { + content, + equals: piece + } + }) + .execute(claimsServiceConnection) + + if (claimResult.out.error) { + return { + error: claimResult.out.error + } + } + // Add piece for aggregation const aggregateQueue = await Aggregator.aggregateQueue( - invocationConfig, + aggregateInvocationConfig, piece, group, { connection: aggregateServiceConnection } @@ -109,6 +135,7 @@ export async function reportPieceCid ({ error: aggregateQueue.out.error } } + return { ok: {}, } diff --git a/filecoin/package.json b/filecoin/package.json index 281fc354..6307dfac 100644 --- a/filecoin/package.json +++ b/filecoin/package.json @@ -11,8 +11,10 @@ "@aws-sdk/client-sqs": "^3.226.0", "@sentry/serverless": "^7.22.0", "@ucanto/client": "^8.0.1", + "@ucanto/interface": "^8.1.0", "@ucanto/principal": "^8.1.0", "@ucanto/transport": "^8.0.0", + "@web3-storage/content-claims": "^3.0.1", "@web3-storage/data-segment": "^3.0.1", "@web3-storage/filecoin-client": "^1.3.0", "fr32-sha2-256-trunc254-padded-binary-tree-multihash": "^1.0.0", diff --git a/filecoin/test/helpers/mocks.js b/filecoin/test/helpers/mocks.js index d6dff7b2..cd33c8e2 100644 --- a/filecoin/test/helpers/mocks.js +++ b/filecoin/test/helpers/mocks.js @@ -6,7 +6,8 @@ const notImplemented = () => { /** * @param {Partial< - * import('@web3-storage/filecoin-client/types').AggregatorService + * import('@web3-storage/filecoin-client/types').AggregatorService & + * { assert: Partial } * >} impl */ export function mockService(impl) { @@ -15,6 +16,9 @@ export function mockService(impl) { add: withCallCount(impl.aggregate?.add ?? notImplemented), queue: withCallCount(impl.aggregate?.queue ?? notImplemented), }, + assert: { + equals: withCallCount(impl.assert?.equals ?? notImplemented) + } } } diff --git a/filecoin/test/helpers/ucanto.js b/filecoin/test/helpers/ucanto.js index c34dd151..30c34a52 100644 --- a/filecoin/test/helpers/ucanto.js +++ b/filecoin/test/helpers/ucanto.js @@ -3,12 +3,68 @@ import * as Server from '@ucanto/server' import * as Client from '@ucanto/client' import * as CAR from '@ucanto/transport/car' import * as FilecoinCapabilities from '@web3-storage/capabilities/filecoin' +import { Assert } from '@web3-storage/content-claims/capability' import { OperationFailed } from './errors.js' import { mockService } from './mocks.js' const nop = (/** @type {any} */ invCap) => {} +/** + * @param {any} serviceProvider + * @param {object} [options] + * @param {(inCap: any) => void} [options.onCall] + * @param {boolean} [options.mustFail] + */ +export async function getClaimsServiceServer (serviceProvider, options = {}) { + const onCall = options.onCall || nop + const equalsStore = new Map() + + const service = mockService({ + assert: { + equals: Server.provide(Assert.equals, async ({ capability, invocation }) => { + const invCap = invocation.capabilities[0] + const { content, equals } = capability.nb + + if (options.mustFail) { + return { + error: new OperationFailed( + 'failed to add to aggregate', + // @ts-ignore wrong dep + invCap.nb?.content + ) + } + } + + equalsStore.set(content.toString(), equals.toString()) + equalsStore.set(equals.toString(), content.toString()) + + onCall(invCap) + + return { + ok: {} + } + }) + } + }) + + const server = Server.create({ + id: serviceProvider, + service, + codec: CAR.inbound, + }) + const connection = Client.connect({ + id: serviceProvider, + codec: CAR.outbound, + channel: server, + }) + + return { + service, + connection + } +} + /** * @param {any} serviceProvider * @param {object} [options] @@ -112,9 +168,10 @@ export async function getAggregatorServiceServer (serviceProvider, options = {}) } } -export async function getAggregatorServiceCtx () { +export async function getServiceCtx () { const storefront = await Signer.generate() const aggregator = await Signer.generate() + const claims = await Signer.generate() return { storefront: { @@ -126,6 +183,11 @@ export async function getAggregatorServiceCtx () { did: aggregator.did(), privateKey: Signer.format(aggregator), raw: aggregator + }, + claims: { + did: claims.did(), + privateKey: Signer.format(claims), + raw: claims } } } diff --git a/filecoin/test/report-piece-cid.test.js b/filecoin/test/report-piece-cid.test.js index 7d4a50e2..524a7e1b 100644 --- a/filecoin/test/report-piece-cid.test.js +++ b/filecoin/test/report-piece-cid.test.js @@ -5,77 +5,146 @@ import pDefer from 'p-defer' import { reportPieceCid } from '../index.js' import { getServiceSigner } from '../service.js' -import { getAggregatorServiceServer, getAggregatorServiceCtx } from './helpers/ucanto.js' +import { getAggregatorServiceServer, getClaimsServiceServer, getServiceCtx } from './helpers/ucanto.js' import { createCar } from './helpers/car.js' test('reports piece cid from a piece written to the piece table', async t => { - const { piece } = await createCar() + const { piece, link } = await createCar() const aggregatorQueueCall = pDefer() - const { invocationConfig, aggregatorService } = await getService({ - onCall: aggregatorQueueCall + const claimsEqualsCall = pDefer() + const { aggregateInvocationConfig, aggregatorService, claimsInvocationConfig, claimsService } = await getService({ + aggregator: { + onCall: aggregatorQueueCall + }, + claims: { + onCall: claimsEqualsCall + } }) const reportPieceCidResponse = await reportPieceCid({ piece, - group: invocationConfig.issuer.did(), - invocationConfig, - aggregateServiceConnection: aggregatorService.connection + content: link, + group: aggregateInvocationConfig.issuer.did(), + aggregateInvocationConfig, + aggregateServiceConnection: aggregatorService.connection, + claimsInvocationConfig, + claimsServiceConnection: claimsService.connection, }) t.truthy(reportPieceCidResponse.ok) t.falsy(reportPieceCidResponse.error) - // Validate ucanto server call + // Validate ucanto server calls + t.is(claimsService.service.assert.equals.callCount, 1) + const invCapClaims = await claimsEqualsCall.promise + t.is(invCapClaims.can, 'assert/equals') + t.is(aggregatorService.service.aggregate.queue.callCount, 1) - const invCap = await aggregatorQueueCall.promise - t.is(invCap.can, 'aggregate/queue') + const invCapAggregator = await aggregatorQueueCall.promise + t.is(invCapAggregator.can, 'aggregate/queue') +}) + +test('fails reporting piece cid if fails to claim equals', async t => { + const { piece, link } = await createCar() + const aggregatorQueueCall = pDefer() + const claimEqualsCall = pDefer() + const { aggregateInvocationConfig, aggregatorService, claimsInvocationConfig, claimsService } = await getService({ + aggregator: { + onCall: aggregatorQueueCall + }, + claims: { + onCall: claimEqualsCall, + mustFail: true + } + }) + + const reportPieceCidResponse = await reportPieceCid({ + piece, + content: link, + group: aggregateInvocationConfig.issuer.did(), + aggregateInvocationConfig, + aggregateServiceConnection: aggregatorService.connection, + claimsInvocationConfig, + claimsServiceConnection: claimsService.connection, + }) + + t.falsy(reportPieceCidResponse.ok) + t.truthy(reportPieceCidResponse.error) + + // Validate ucanto server calls + t.is(claimsService.service.assert.equals.callCount, 1) + t.is(aggregatorService.service.aggregate.queue.callCount, 0) }) test('fails reporting piece cid if fails to queue to aggregator', async t => { - const { piece } = await createCar() + const { piece, link } = await createCar() const aggregatorQueueCall = pDefer() - const { invocationConfig, aggregatorService } = await getService({ - onCall: aggregatorQueueCall, - mustFail: true + const claimEqualsCall = pDefer() + const { aggregateInvocationConfig, aggregatorService, claimsInvocationConfig, claimsService } = await getService({ + aggregator: { + onCall: aggregatorQueueCall, + mustFail: true + }, + claims: { + onCall: claimEqualsCall + } }) const reportPieceCidResponse = await reportPieceCid({ piece, - group: invocationConfig.issuer.did(), - invocationConfig, - aggregateServiceConnection: aggregatorService.connection + content: link, + group: aggregateInvocationConfig.issuer.did(), + aggregateInvocationConfig, + aggregateServiceConnection: aggregatorService.connection, + claimsInvocationConfig, + claimsServiceConnection: claimsService.connection, }) t.falsy(reportPieceCidResponse.ok) t.truthy(reportPieceCidResponse.error) + // Validate ucanto server calls + t.is(claimsService.service.assert.equals.callCount, 1) t.is(aggregatorService.service.aggregate.queue.callCount, 1) }) /** - * @param {object} options - * @param {import('p-defer').DeferredPromise} options.onCall - * @param {boolean} [options.mustFail] + * @typedef {object} Props + * @property {import('p-defer').DeferredPromise} onCall + * @property {boolean} [mustFail] + * + * @param {Record<'aggregator' | 'claims', Props>} options */ async function getService (options) { - const { storefront, aggregator } = await getAggregatorServiceCtx() + const { storefront, aggregator, claims } = await getServiceCtx() const aggregatorService = await getAggregatorServiceServer(aggregator.raw, { onCall: (invCap) => { - options.onCall.resolve(invCap) + options.aggregator.onCall.resolve(invCap) }, - mustFail: options.mustFail + mustFail: options.aggregator.mustFail }) + + const claimsService = await getClaimsServiceServer(claims.raw, { + onCall: (invCap) => { + options.claims.onCall.resolve(invCap) + }, + mustFail: options.claims.mustFail + }) + const issuer = getServiceSigner(storefront) - const audience = aggregatorService.connection.id - /** @type {import('@web3-storage/filecoin-client/types').InvocationConfig} */ - const invocationConfig = { - issuer, - audience, - with: issuer.did(), - } return { - invocationConfig, - aggregatorService + aggregateInvocationConfig: /** @type {import('@web3-storage/filecoin-client/types').InvocationConfig} */({ + issuer, + audience: aggregatorService.connection.id, + with: issuer.did(), + }), + aggregatorService, + claimsInvocationConfig:/** @type {import('../types').ClaimsInvocationConfig} */({ + issuer, + audience: claimsService.connection.id, + with: issuer.did(), + }), + claimsService, } } diff --git a/filecoin/types.ts b/filecoin/types.ts index 4b86adb7..127f9f9f 100644 --- a/filecoin/types.ts +++ b/filecoin/types.ts @@ -1,3 +1,8 @@ +import { + Signer, + DID, + Principal, +} from '@ucanto/interface' import { UnknownLink } from 'multiformats' import { PieceLink } from '@web3-storage/data-segment' @@ -22,6 +27,21 @@ export interface ComputePieceError extends Error { name: 'ComputePieceFailed' } +export interface ClaimsInvocationConfig { + /** + * Signing authority that is issuing the UCAN invocation(s). + */ + issuer: Signer + /** + * The principal delegated to in the current UCAN. + */ + audience: Principal + /** + * The resource the invocation applies to. + */ + with: DID +} + export class Failure extends Error { describe() { return this.toString() diff --git a/package-lock.json b/package-lock.json index ebd5fe19..a323cc03 100644 --- a/package-lock.json +++ b/package-lock.json @@ -76,6 +76,7 @@ "@ucanto/client": "^8.0.1", "@ucanto/principal": "^8.1.0", "@ucanto/transport": "^8.0.0", + "@web3-storage/content-claims": "^3.0.1", "@web3-storage/data-segment": "^3.0.1", "@web3-storage/filecoin-client": "^1.3.0", "fr32-sha2-256-trunc254-padded-binary-tree-multihash": "^1.0.0", @@ -4448,6 +4449,28 @@ "@web3-storage/data-segment": "^3.0.1" } }, + "node_modules/@web3-storage/content-claims": { + "version": "3.0.1", + "resolved": "https://gitpkg.now.sh/web3-storage/content-claims/packages/core?main", + "integrity": "sha512-Y9ad2BPHLrgMipb4ynPpkrL8Gf+Pr3KwyF+MlExT8WuZn3Vgk5UbqDodEH3WFph+AVh3AYNnxzV2hUqRS6N/XQ==", + "license": "Apache-2.0 OR MIT", + "dependencies": { + "@ucanto/client": "^8.0.0", + "@ucanto/server": "^8.0.1", + "@ucanto/transport": "^8.0.0", + "carstream": "^1.0.2", + "multiformats": "^12.0.1" + } + }, + "node_modules/@web3-storage/content-claims/node_modules/multiformats": { + "version": "12.1.1", + "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-12.1.1.tgz", + "integrity": "sha512-GBSToTmri2vJYs8wqcZQ8kB21dCaeTOzHTIAlr8J06C1eL6UbzqURXFZ5Fl0EYm9GAFz1IlYY8SxGOs9G9NJRg==", + "engines": { + "node": ">=16.0.0", + "npm": ">=7.0.0" + } + }, "node_modules/@web3-storage/data-segment": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/@web3-storage/data-segment/-/data-segment-3.0.1.tgz", @@ -6015,6 +6038,25 @@ "cardex": "bin.js" } }, + "node_modules/carstream": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/carstream/-/carstream-1.1.0.tgz", + "integrity": "sha512-tbf8FOnGX1+0kOe77nm9MG53REiqQopDwzwbXYVxUcsKOAHG2KSD++qy95v1vrtRt1Q6L0Sb01it7QwJ+Yt1sQ==", + "dependencies": { + "@ipld/dag-cbor": "^9.0.3", + "multiformats": "^12.0.1", + "uint8arraylist": "^2.4.3" + } + }, + "node_modules/carstream/node_modules/multiformats": { + "version": "12.1.1", + "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-12.1.1.tgz", + "integrity": "sha512-GBSToTmri2vJYs8wqcZQ8kB21dCaeTOzHTIAlr8J06C1eL6UbzqURXFZ5Fl0EYm9GAFz1IlYY8SxGOs9G9NJRg==", + "engines": { + "node": ">=16.0.0", + "npm": ">=7.0.0" + } + }, "node_modules/cbor": { "version": "8.1.0", "dev": true, @@ -13446,6 +13488,18 @@ "node": ">=4.2.0" } }, + "node_modules/uint8arraylist": { + "version": "2.4.3", + "resolved": "https://registry.npmjs.org/uint8arraylist/-/uint8arraylist-2.4.3.tgz", + "integrity": "sha512-oEVZr4/GrH87K0kjNce6z8pSCzLEPqHNLNR5sj8cJOySrTP8Vb/pMIbZKLJGhQKxm1TiZ31atNrpn820Pyqpow==", + "dependencies": { + "uint8arrays": "^4.0.2" + }, + "engines": { + "node": ">=16.0.0", + "npm": ">=7.0.0" + } + }, "node_modules/uint8arrays": { "version": "4.0.6", "license": "Apache-2.0 OR MIT", @@ -17480,6 +17534,25 @@ "@web3-storage/data-segment": "^3.0.1" } }, + "@web3-storage/content-claims": { + "version": "3.0.1", + "resolved": "https://gitpkg.now.sh/web3-storage/content-claims/packages/core?main", + "integrity": "sha512-Y9ad2BPHLrgMipb4ynPpkrL8Gf+Pr3KwyF+MlExT8WuZn3Vgk5UbqDodEH3WFph+AVh3AYNnxzV2hUqRS6N/XQ==", + "requires": { + "@ucanto/client": "^8.0.0", + "@ucanto/server": "^8.0.1", + "@ucanto/transport": "^8.0.0", + "carstream": "^1.0.2", + "multiformats": "^12.0.1" + }, + "dependencies": { + "multiformats": { + "version": "12.1.1", + "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-12.1.1.tgz", + "integrity": "sha512-GBSToTmri2vJYs8wqcZQ8kB21dCaeTOzHTIAlr8J06C1eL6UbzqURXFZ5Fl0EYm9GAFz1IlYY8SxGOs9G9NJRg==" + } + } + }, "@web3-storage/data-segment": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/@web3-storage/data-segment/-/data-segment-3.0.1.tgz", @@ -17620,6 +17693,7 @@ "@ucanto/client": "^8.0.1", "@ucanto/principal": "^8.1.0", "@ucanto/transport": "^8.0.0", + "@web3-storage/content-claims": "^3.0.1", "@web3-storage/data-segment": "^3.0.1", "@web3-storage/filecoin-client": "^1.3.0", "ava": "^4.3.3", @@ -18692,6 +18766,23 @@ "varint": "^6.0.0" } }, + "carstream": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/carstream/-/carstream-1.1.0.tgz", + "integrity": "sha512-tbf8FOnGX1+0kOe77nm9MG53REiqQopDwzwbXYVxUcsKOAHG2KSD++qy95v1vrtRt1Q6L0Sb01it7QwJ+Yt1sQ==", + "requires": { + "@ipld/dag-cbor": "^9.0.3", + "multiformats": "^12.0.1", + "uint8arraylist": "^2.4.3" + }, + "dependencies": { + "multiformats": { + "version": "12.1.1", + "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-12.1.1.tgz", + "integrity": "sha512-GBSToTmri2vJYs8wqcZQ8kB21dCaeTOzHTIAlr8J06C1eL6UbzqURXFZ5Fl0EYm9GAFz1IlYY8SxGOs9G9NJRg==" + } + } + }, "cbor": { "version": "8.1.0", "dev": true, @@ -23258,6 +23349,14 @@ "version": "4.9.5", "dev": true }, + "uint8arraylist": { + "version": "2.4.3", + "resolved": "https://registry.npmjs.org/uint8arraylist/-/uint8arraylist-2.4.3.tgz", + "integrity": "sha512-oEVZr4/GrH87K0kjNce6z8pSCzLEPqHNLNR5sj8cJOySrTP8Vb/pMIbZKLJGhQKxm1TiZ31atNrpn820Pyqpow==", + "requires": { + "uint8arrays": "^4.0.2" + } + }, "uint8arrays": { "version": "4.0.6", "requires": { diff --git a/stacks/config.js b/stacks/config.js index 65005b19..0bb288a2 100644 --- a/stacks/config.js +++ b/stacks/config.js @@ -143,6 +143,8 @@ export function getEnv() { UPLOAD_API_DID: mustGetEnv('UPLOAD_API_DID'), AGGREGATOR_DID: mustGetEnv('AGGREGATOR_DID'), AGGREGATOR_URL: mustGetEnv('AGGREGATOR_URL'), + CONTENT_CLAIMS_DID: mustGetEnv('CONTENT_CLAIMS_DID'), + CONTENT_CLAIMS_URL: mustGetEnv('CONTENT_CLAIMS_URL'), } } diff --git a/stacks/filecoin-stack.js b/stacks/filecoin-stack.js index 277671ec..5305efbd 100644 --- a/stacks/filecoin-stack.js +++ b/stacks/filecoin-stack.js @@ -20,7 +20,7 @@ export function FilecoinStack({ stack, app }) { srcPath: 'filecoin' }) - const { AGGREGATOR_DID, AGGREGATOR_URL } = getEnv() + const { AGGREGATOR_DID, AGGREGATOR_URL, CONTENT_CLAIMS_DID, CONTENT_CLAIMS_URL } = getEnv() // Setup app monitoring with Sentry setupSentry(app, stack) @@ -40,6 +40,8 @@ export function FilecoinStack({ stack, app }) { environment: { AGGREGATOR_DID, AGGREGATOR_URL, + CONTENT_CLAIMS_DID, + CONTENT_CLAIMS_URL, }, timeout: 3 * 60, bind: [ diff --git a/test/integration.test.js b/test/integration.test.js index 8e53db48..1524de35 100644 --- a/test/integration.test.js +++ b/test/integration.test.js @@ -114,7 +114,7 @@ test('authorizations can be blocked by email or domain', async t => { }) // Integration test for all flow from uploading a file to Kinesis events consumers and replicator -test('w3infra integration flow', async t => { +test.skip('w3infra integration flow', async t => { const client = await setupNewClient(t.context.apiEndpoint) const spaceDid = client.currentSpace()?.did() if (!spaceDid) { From 73223f3c68e8f69f1131815c6baea94d528c47ef Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Tue, 19 Sep 2023 15:26:18 +0200 Subject: [PATCH 2/7] chore: address review comments --- README.md | 16 ++++++++++++++++ filecoin/functions/piece-cid-report.js | 23 +++++++++++++---------- stacks/filecoin-stack.js | 3 ++- stacks/upload-db-stack.js | 8 +++++++- 4 files changed, 38 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 58479f98..9860fcc6 100644 --- a/README.md +++ b/README.md @@ -116,6 +116,14 @@ DID of the filecoin aggregator service. URL of the filecoin aggregator service. +#### `CONTENT_CLAIMS_DID` + +DID of the [content claims service](https://github.com/web3-storage/content-claims). + +#### `CONTENT_CLAIMS_URL` + +URL of the [content claims service](https://github.com/web3-storage/content-claims). + #### `UPLOAD_API_DID` [DID](https://www.w3.org/TR/did-core/) of the upload-api ucanto server. e.g. `did:web:up.web3.storage`. Optional: if omitted, a `did:key` will be derrived from `PRIVATE_KEY` @@ -208,6 +216,14 @@ Generated by [@ucanto/principal `EdSigner`](https://github.com/web3-storage/ucan _Example:_ `MgCZG7EvaA...1pX9as=` +#### `CONTENT_CLAIMS_PRIVATE_KEY` + +The [`multibase`](https://github.com/multiformats/multibase) encoded ED25519 keypair used as the signing key for the [content-claims api](https://github.com/web3-storage/content-claims). + +Generated by [@ucanto/principal `EdSigner`](https://github.com/web3-storage/ucanto) via [`ucan-key`](https://www.npmjs.com/package/ucan-key) + +_Example:_ `MgCZG7EvaA...1pX9as=` + #### `UCAN_INVOCATION_POST_BASIC_AUTH` The HTTP Basic auth token for the UCAN Invocation entrypoint, where UCAN invocations can be stored and proxied to the UCAN Stream. diff --git a/filecoin/functions/piece-cid-report.js b/filecoin/functions/piece-cid-report.js index 43102294..2b5ea2f1 100644 --- a/filecoin/functions/piece-cid-report.js +++ b/filecoin/functions/piece-cid-report.js @@ -18,8 +18,8 @@ Sentry.AWSLambda.init({ * @param {import('aws-lambda').DynamoDBStreamEvent} event */ async function pieceCidReport (event) { - const { aggregatorDid, aggregatorUrl } = getEnv() - const { PRIVATE_KEY: privateKey } = Config + const { aggregatorDid, aggregatorUrl, contentClaimsDid, contentClaimsUrl } = getEnv() + const { PRIVATE_KEY: privateKey, CONTENT_CLAIMS_PRIVATE_KEY: contentClaimsPrivateKey } = Config const records = parseDynamoDbEvent(event) if (records.length > 1) { @@ -36,28 +36,31 @@ async function pieceCidReport (event) { url: aggregatorUrl }) const claimsServiceConnection = getServiceConnection({ - did: aggregatorDid, - url: aggregatorUrl + did: contentClaimsDid, + url: contentClaimsUrl }) - const issuer = getServiceSigner({ + const storefrontIssuer = getServiceSigner({ privateKey }) + const claimsIssuer = getServiceSigner({ + privateKey: contentClaimsPrivateKey + }) const { ok, error } = await reportPieceCid({ piece, content, - group: issuer.did(), + group: storefrontIssuer.did(), aggregateServiceConnection, aggregateInvocationConfig: /** @type {import('@web3-storage/filecoin-client/types').InvocationConfig} */ ({ - issuer, + issuer: storefrontIssuer, audience: aggregateServiceConnection.id, - with: issuer.did(), + with: storefrontIssuer.did(), }), claimsServiceConnection, claimsInvocationConfig: /** @type {import('../types').ClaimsInvocationConfig} */ ({ - issuer, + issuer: claimsIssuer, audience: claimsServiceConnection.id, - with: issuer.did(), + with: claimsIssuer.did(), }) }) diff --git a/stacks/filecoin-stack.js b/stacks/filecoin-stack.js index 5305efbd..677dd8fb 100644 --- a/stacks/filecoin-stack.js +++ b/stacks/filecoin-stack.js @@ -30,7 +30,7 @@ export function FilecoinStack({ stack, app }) { // Get eventBus reference const { eventBus } = use(BusStack) // Get store table reference - const { pieceTable, privateKey } = use(UploadDbStack) + const { pieceTable, privateKey, contentClaimsPrivateKey } = use(UploadDbStack) // piece-cid reporting pieceTable.addConsumers(stack, { @@ -46,6 +46,7 @@ export function FilecoinStack({ stack, app }) { timeout: 3 * 60, bind: [ privateKey, + contentClaimsPrivateKey ] }, cdk: { diff --git a/stacks/upload-db-stack.js b/stacks/upload-db-stack.js index a92a6d31..7a4200d0 100644 --- a/stacks/upload-db-stack.js +++ b/stacks/upload-db-stack.js @@ -25,8 +25,13 @@ export function UploadDbStack({ stack, app }) { // Setup app monitoring with Sentry setupSentry(app, stack) + // Upload API private key const privateKey = new Config.Secret(stack, 'PRIVATE_KEY') + // Content claims private key + // TODO: we should look into creating a trust layer for content claims + const contentClaimsPrivateKey = new Config.Secret(stack, 'CONTENT_CLAIMS_PRIVATE_KEY') + /** * This table takes a stored CAR and makes an entry in the store table * Used by the store/* service capabilities. @@ -100,6 +105,7 @@ export function UploadDbStack({ stack, app }) { delegationTable, adminMetricsTable, spaceMetricsTable, - privateKey + privateKey, + contentClaimsPrivateKey } } From 3af33b13752cfe124d7269d22eb84ffcf2881a98 Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Tue, 19 Sep 2023 15:27:54 +0200 Subject: [PATCH 3/7] chore: use final version of content-claims client --- filecoin/package.json | 2 +- package-lock.json | 19 ++++++++++--------- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/filecoin/package.json b/filecoin/package.json index 6307dfac..957003b1 100644 --- a/filecoin/package.json +++ b/filecoin/package.json @@ -14,7 +14,7 @@ "@ucanto/interface": "^8.1.0", "@ucanto/principal": "^8.1.0", "@ucanto/transport": "^8.0.0", - "@web3-storage/content-claims": "^3.0.1", + "@web3-storage/content-claims": "^3.1.0", "@web3-storage/data-segment": "^3.0.1", "@web3-storage/filecoin-client": "^1.3.0", "fr32-sha2-256-trunc254-padded-binary-tree-multihash": "^1.0.0", diff --git a/package-lock.json b/package-lock.json index a323cc03..782a60f7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -74,9 +74,10 @@ "@aws-sdk/client-sqs": "^3.226.0", "@sentry/serverless": "^7.22.0", "@ucanto/client": "^8.0.1", + "@ucanto/interface": "^8.1.0", "@ucanto/principal": "^8.1.0", "@ucanto/transport": "^8.0.0", - "@web3-storage/content-claims": "^3.0.1", + "@web3-storage/content-claims": "^3.1.0", "@web3-storage/data-segment": "^3.0.1", "@web3-storage/filecoin-client": "^1.3.0", "fr32-sha2-256-trunc254-padded-binary-tree-multihash": "^1.0.0", @@ -4450,10 +4451,9 @@ } }, "node_modules/@web3-storage/content-claims": { - "version": "3.0.1", - "resolved": "https://gitpkg.now.sh/web3-storage/content-claims/packages/core?main", - "integrity": "sha512-Y9ad2BPHLrgMipb4ynPpkrL8Gf+Pr3KwyF+MlExT8WuZn3Vgk5UbqDodEH3WFph+AVh3AYNnxzV2hUqRS6N/XQ==", - "license": "Apache-2.0 OR MIT", + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/@web3-storage/content-claims/-/content-claims-3.1.0.tgz", + "integrity": "sha512-ieoEGL+bqmAjxS2NDcxF8jo0IKpBDhDu/bnNXRbhgQLUtGMxv+2PT0n35PM69AKnw8QgzEQCM8FibbZAdw0I/Q==", "dependencies": { "@ucanto/client": "^8.0.0", "@ucanto/server": "^8.0.1", @@ -17535,9 +17535,9 @@ } }, "@web3-storage/content-claims": { - "version": "3.0.1", - "resolved": "https://gitpkg.now.sh/web3-storage/content-claims/packages/core?main", - "integrity": "sha512-Y9ad2BPHLrgMipb4ynPpkrL8Gf+Pr3KwyF+MlExT8WuZn3Vgk5UbqDodEH3WFph+AVh3AYNnxzV2hUqRS6N/XQ==", + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/@web3-storage/content-claims/-/content-claims-3.1.0.tgz", + "integrity": "sha512-ieoEGL+bqmAjxS2NDcxF8jo0IKpBDhDu/bnNXRbhgQLUtGMxv+2PT0n35PM69AKnw8QgzEQCM8FibbZAdw0I/Q==", "requires": { "@ucanto/client": "^8.0.0", "@ucanto/server": "^8.0.1", @@ -17691,9 +17691,10 @@ "@sentry/serverless": "^7.22.0", "@serverless-stack/resources": "*", "@ucanto/client": "^8.0.1", + "@ucanto/interface": "^8.1.0", "@ucanto/principal": "^8.1.0", "@ucanto/transport": "^8.0.0", - "@web3-storage/content-claims": "^3.0.1", + "@web3-storage/content-claims": "^3.1.0", "@web3-storage/data-segment": "^3.0.1", "@web3-storage/filecoin-client": "^1.3.0", "ava": "^4.3.3", From 7625e2c8c2245683b1c003f214521469c3b8d007 Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Tue, 19 Sep 2023 15:36:16 +0200 Subject: [PATCH 4/7] chore: satisfy types for generated secrets --- filecoin/types.ts | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/filecoin/types.ts b/filecoin/types.ts index 127f9f9f..239b19bd 100644 --- a/filecoin/types.ts +++ b/filecoin/types.ts @@ -99,3 +99,17 @@ export type Variant> = { [K in Key]: U[Key] } }[keyof U] + +// would be generated by sst, but requires `sst build` to be run, which calls out to aws; not great for CI +declare module '@serverless-stack/node/config' { + export interface SecretResources { + PRIVATE_KEY: { + value: string + }, + CONTENT_CLAIMS_PRIVATE_KEY: { + value: string + } + } +} + +export {} From 16a9099dc8afba257897562c98a1f0d179ae168b Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Tue, 19 Sep 2023 16:04:52 +0200 Subject: [PATCH 5/7] chore: run full interop tests suite --- test/integration.test.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration.test.js b/test/integration.test.js index 1524de35..8e53db48 100644 --- a/test/integration.test.js +++ b/test/integration.test.js @@ -114,7 +114,7 @@ test('authorizations can be blocked by email or domain', async t => { }) // Integration test for all flow from uploading a file to Kinesis events consumers and replicator -test.skip('w3infra integration flow', async t => { +test('w3infra integration flow', async t => { const client = await setupNewClient(t.context.apiEndpoint) const spaceDid = client.currentSpace()?.did() if (!spaceDid) { From 0354887a1a6ee3fbb49f2b3bede6b086f178cfaa Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Wed, 20 Sep 2023 11:11:16 +0200 Subject: [PATCH 6/7] chore: apply suggestions from code review Co-authored-by: Alan Shaw --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 9860fcc6..8602e433 100644 --- a/README.md +++ b/README.md @@ -218,7 +218,7 @@ _Example:_ `MgCZG7EvaA...1pX9as=` #### `CONTENT_CLAIMS_PRIVATE_KEY` -The [`multibase`](https://github.com/multiformats/multibase) encoded ED25519 keypair used as the signing key for the [content-claims api](https://github.com/web3-storage/content-claims). +The `base64pad` [`multibase`](https://github.com/multiformats/multibase) encoded ED25519 keypair used as the signing key for [content-claims](https://github.com/web3-storage/content-claims). Generated by [@ucanto/principal `EdSigner`](https://github.com/web3-storage/ucanto) via [`ucan-key`](https://www.npmjs.com/package/ucan-key) From a8b24c5ce6716ee0d3c783089076acaffaed6e71 Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Wed, 20 Sep 2023 11:54:06 +0200 Subject: [PATCH 7/7] fix: support optional proof and use did:web --- filecoin/functions/piece-cid-report.js | 19 +++++++++++++++++-- filecoin/index.js | 1 - filecoin/package.json | 4 +++- filecoin/types.ts | 5 +++++ package-lock.json | 25 ++++++++++++++++--------- stacks/config.js | 2 ++ stacks/filecoin-stack.js | 3 ++- test/integration.test.js | 3 +++ 8 files changed, 48 insertions(+), 14 deletions(-) diff --git a/filecoin/functions/piece-cid-report.js b/filecoin/functions/piece-cid-report.js index 2b5ea2f1..1aee46fe 100644 --- a/filecoin/functions/piece-cid-report.js +++ b/filecoin/functions/piece-cid-report.js @@ -3,6 +3,9 @@ import { Config } from '@serverless-stack/node/config/index.js' import { unmarshall } from '@aws-sdk/util-dynamodb' import { Piece } from '@web3-storage/data-segment' import { CID } from 'multiformats/cid' +import * as Delegation from '@ucanto/core/delegation' +import { fromString } from 'uint8arrays/from-string' +import * as DID from '@ipld/dag-ucan/did' import { reportPieceCid } from '../index.js' import { getServiceConnection, getServiceSigner } from '../service.js' @@ -18,7 +21,7 @@ Sentry.AWSLambda.init({ * @param {import('aws-lambda').DynamoDBStreamEvent} event */ async function pieceCidReport (event) { - const { aggregatorDid, aggregatorUrl, contentClaimsDid, contentClaimsUrl } = getEnv() + const { aggregatorDid, aggregatorUrl, contentClaimsDid, contentClaimsUrl, contentClaimsProof } = getEnv() const { PRIVATE_KEY: privateKey, CONTENT_CLAIMS_PRIVATE_KEY: contentClaimsPrivateKey } = Config const records = parseDynamoDbEvent(event) @@ -42,9 +45,18 @@ async function pieceCidReport (event) { const storefrontIssuer = getServiceSigner({ privateKey }) - const claimsIssuer = getServiceSigner({ + let claimsIssuer = getServiceSigner({ privateKey: contentClaimsPrivateKey }) + const claimsProofs = [] + if (contentClaimsProof) { + const proof = await Delegation.extract(fromString(contentClaimsProof, 'base64pad')) + if (!proof.ok) throw new Error('failed to extract proof', { cause: proof.error }) + claimsProofs.push(proof.ok) + } else { + // if no proofs, we must be using the service private key to sign + claimsIssuer = claimsIssuer.withDID(DID.parse(contentClaimsDid).did()) + } const { ok, error } = await reportPieceCid({ piece, @@ -65,6 +77,8 @@ async function pieceCidReport (event) { }) if (error) { + console.error(error) + return { statusCode: 500, body: error.message || 'failed to add aggregate' @@ -88,6 +102,7 @@ function getEnv() { aggregatorUrl: mustGetEnv('AGGREGATOR_URL'), contentClaimsDid: mustGetEnv('CONTENT_CLAIMS_DID'), contentClaimsUrl: mustGetEnv('CONTENT_CLAIMS_URL'), + contentClaimsProof: process.env.CONTENT_CLAIMS_PROOF, } } diff --git a/filecoin/index.js b/filecoin/index.js index 7421b817..a0488027 100644 --- a/filecoin/index.js +++ b/filecoin/index.js @@ -115,7 +115,6 @@ export async function reportPieceCid ({ } }) .execute(claimsServiceConnection) - if (claimResult.out.error) { return { error: claimResult.out.error diff --git a/filecoin/package.json b/filecoin/package.json index 957003b1..b944a2df 100644 --- a/filecoin/package.json +++ b/filecoin/package.json @@ -11,6 +11,7 @@ "@aws-sdk/client-sqs": "^3.226.0", "@sentry/serverless": "^7.22.0", "@ucanto/client": "^8.0.1", + "@ucanto/core": "^8.2.0", "@ucanto/interface": "^8.1.0", "@ucanto/principal": "^8.1.0", "@ucanto/transport": "^8.0.0", @@ -18,7 +19,8 @@ "@web3-storage/data-segment": "^3.0.1", "@web3-storage/filecoin-client": "^1.3.0", "fr32-sha2-256-trunc254-padded-binary-tree-multihash": "^1.0.0", - "multiformats": "^12.1.1" + "multiformats": "^12.1.1", + "uint8arrays": "4.0.6" }, "devDependencies": { "@serverless-stack/resources": "*", diff --git a/filecoin/types.ts b/filecoin/types.ts index 239b19bd..7da04c42 100644 --- a/filecoin/types.ts +++ b/filecoin/types.ts @@ -2,6 +2,7 @@ import { Signer, DID, Principal, + Proof, } from '@ucanto/interface' import { UnknownLink } from 'multiformats' import { PieceLink } from '@web3-storage/data-segment' @@ -40,6 +41,10 @@ export interface ClaimsInvocationConfig { * The resource the invocation applies to. */ with: DID + /** + * Proof(s) the issuer has the capability to perform the action. + */ + proofs?: Proof[] } export class Failure extends Error { diff --git a/package-lock.json b/package-lock.json index 782a60f7..a16ebb79 100644 --- a/package-lock.json +++ b/package-lock.json @@ -74,6 +74,7 @@ "@aws-sdk/client-sqs": "^3.226.0", "@sentry/serverless": "^7.22.0", "@ucanto/client": "^8.0.1", + "@ucanto/core": "^8.2.0", "@ucanto/interface": "^8.1.0", "@ucanto/principal": "^8.1.0", "@ucanto/transport": "^8.0.0", @@ -81,7 +82,8 @@ "@web3-storage/data-segment": "^3.0.1", "@web3-storage/filecoin-client": "^1.3.0", "fr32-sha2-256-trunc254-padded-binary-tree-multihash": "^1.0.0", - "multiformats": "^12.1.1" + "multiformats": "^12.1.1", + "uint8arrays": "4.0.6" }, "devDependencies": { "@serverless-stack/resources": "*", @@ -4279,13 +4281,14 @@ } }, "node_modules/@ucanto/core": { - "version": "8.1.0", - "license": "(Apache-2.0 AND MIT)", + "version": "8.2.0", + "resolved": "https://registry.npmjs.org/@ucanto/core/-/core-8.2.0.tgz", + "integrity": "sha512-Yw1NZySQFS2+JPUuWGlRpq4HMB1npXYq1WBo6B1tINGRabKAujq3HrZ/iidYL2MuoHn0CGfSvCUzlXrcD4obNg==", "dependencies": { "@ipld/car": "^5.1.0", "@ipld/dag-cbor": "^9.0.0", - "@ipld/dag-ucan": "^3.3.2", - "@ucanto/interface": "^8.0.0", + "@ipld/dag-ucan": "^3.4.0", + "@ucanto/interface": "^8.1.0", "multiformats": "^11.0.2" } }, @@ -17379,12 +17382,14 @@ } }, "@ucanto/core": { - "version": "8.1.0", + "version": "8.2.0", + "resolved": "https://registry.npmjs.org/@ucanto/core/-/core-8.2.0.tgz", + "integrity": "sha512-Yw1NZySQFS2+JPUuWGlRpq4HMB1npXYq1WBo6B1tINGRabKAujq3HrZ/iidYL2MuoHn0CGfSvCUzlXrcD4obNg==", "requires": { "@ipld/car": "^5.1.0", "@ipld/dag-cbor": "^9.0.0", - "@ipld/dag-ucan": "^3.3.2", - "@ucanto/interface": "^8.0.0", + "@ipld/dag-ucan": "^3.4.0", + "@ucanto/interface": "^8.1.0", "multiformats": "^11.0.2" } }, @@ -17691,6 +17696,7 @@ "@sentry/serverless": "^7.22.0", "@serverless-stack/resources": "*", "@ucanto/client": "^8.0.1", + "@ucanto/core": "^8.2.0", "@ucanto/interface": "^8.1.0", "@ucanto/principal": "^8.1.0", "@ucanto/transport": "^8.0.0", @@ -17702,7 +17708,8 @@ "multiformats": "^12.1.1", "nanoid": "^4.0.0", "p-defer": "^4.0.0", - "testcontainers": "^8.13.0" + "testcontainers": "^8.13.0", + "uint8arrays": "4.0.6" }, "dependencies": { "multiformats": { diff --git a/stacks/config.js b/stacks/config.js index 0bb288a2..e63fc745 100644 --- a/stacks/config.js +++ b/stacks/config.js @@ -145,6 +145,8 @@ export function getEnv() { AGGREGATOR_URL: mustGetEnv('AGGREGATOR_URL'), CONTENT_CLAIMS_DID: mustGetEnv('CONTENT_CLAIMS_DID'), CONTENT_CLAIMS_URL: mustGetEnv('CONTENT_CLAIMS_URL'), + // Not required + CONTENT_CLAIMS_PROOF: process.env.CONTENT_CLAIMS_PROOF ?? '' } } diff --git a/stacks/filecoin-stack.js b/stacks/filecoin-stack.js index 677dd8fb..80b27b04 100644 --- a/stacks/filecoin-stack.js +++ b/stacks/filecoin-stack.js @@ -20,7 +20,7 @@ export function FilecoinStack({ stack, app }) { srcPath: 'filecoin' }) - const { AGGREGATOR_DID, AGGREGATOR_URL, CONTENT_CLAIMS_DID, CONTENT_CLAIMS_URL } = getEnv() + const { AGGREGATOR_DID, AGGREGATOR_URL, CONTENT_CLAIMS_DID, CONTENT_CLAIMS_URL, CONTENT_CLAIMS_PROOF } = getEnv() // Setup app monitoring with Sentry setupSentry(app, stack) @@ -42,6 +42,7 @@ export function FilecoinStack({ stack, app }) { AGGREGATOR_URL, CONTENT_CLAIMS_DID, CONTENT_CLAIMS_URL, + CONTENT_CLAIMS_PROOF }, timeout: 3 * 60, bind: [ diff --git a/test/integration.test.js b/test/integration.test.js index 8e53db48..cfbe0908 100644 --- a/test/integration.test.js +++ b/test/integration.test.js @@ -142,6 +142,7 @@ test('w3infra integration flow', async t => { const fileLink = await client.uploadFile(file, { onShardStored: (meta) => { shards.push(meta.cid) + console.log('shard file written', meta.cid) } }) t.truthy(fileLink) @@ -243,6 +244,8 @@ test('w3infra integration flow', async t => { t.is(pieces?.length, 1) t.truthy(pieces?.[0].piece) + console.log('piece written', pieces?.[0].piece) + // Check metrics were updated if (beforeStoreAddSizeTotal && spaceBeforeUploadAddMetrics && spaceBeforeStoreAddSizeMetrics && beforeUploadAddTotal) { await pWaitFor(async () => {