From 2bcf5465a5426fde64e9c09ac5135a392254dc4f Mon Sep 17 00:00:00 2001 From: Travis Vachon Date: Fri, 11 Aug 2023 11:47:18 -0700 Subject: [PATCH 1/6] fix: re-enable support for multiple providers we lost this in the D1 migration work - go back to parsing the PROVIDERS env var and use the result as the list of supported providers --- upload-api/config.js | 14 +++++++++++++- upload-api/functions/ucan-invocation-router.js | 9 +++------ upload-api/functions/validate-email.jsx | 9 +++------ 3 files changed, 19 insertions(+), 13 deletions(-) diff --git a/upload-api/config.js b/upload-api/config.js index 16248930..0d8369d4 100644 --- a/upload-api/config.js +++ b/upload-api/config.js @@ -9,7 +9,7 @@ import * as DID from '@ipld/dag-ucan/did' * @param {string} [config.UPLOAD_API_DID] - public DID for the upload service (did:key:... derived from PRIVATE_KEY if not set) * @returns {import('@ucanto/principal/ed25519').Signer.Signer} */ - export function getServiceSigner(config) { +export function getServiceSigner(config) { const signer = ed25519.parse(config.PRIVATE_KEY) if (config.UPLOAD_API_DID) { const did = DID.parse(config.UPLOAD_API_DID).did() @@ -17,3 +17,15 @@ import * as DID from '@ipld/dag-ucan/did' } return signer } + +/** + * Given a string, parse into provider service DIDs. + * + * @param {string} providersEnvVar + * @return {import('@web3-storage/upload-api').ServiceDID[]} + */ +export function parseProviders(providersEnvVar) { + return /** @type {import('@web3-storage/upload-api').ServiceDID[]} */( + providersEnvVar.split(',').map(s => s.trim()) + ) +} \ No newline at end of file diff --git a/upload-api/functions/ucan-invocation-router.js b/upload-api/functions/ucan-invocation-router.js index 46097f1c..356a3dc0 100644 --- a/upload-api/functions/ucan-invocation-router.js +++ b/upload-api/functions/ucan-invocation-router.js @@ -12,7 +12,7 @@ import { createTaskStore } from '../buckets/task-store.js' import { createWorkflowStore } from '../buckets/workflow-store.js' import { createStoreTable } from '../tables/store.js' import { createUploadTable } from '../tables/upload.js' -import { getServiceSigner } from '../config.js' +import { getServiceSigner, parseProviders } from '../config.js' import { createUcantoServer } from '../service.js' import { Config } from '@serverless-stack/node/config/index.js' import { CAR, Legacy, Codec } from '@ucanto/transport' @@ -94,9 +94,9 @@ export async function ucanInvocationRouter(request) { WORKFLOW_BUCKET_NAME: workflowBucketName = '', UCAN_LOG_STREAM_NAME: streamName = '', POSTMARK_TOKEN: postmarkToken = '', + PROVIDERS: providers = '', // set for testing DYNAMO_DB_ENDPOINT: dbEndpoint, - ACCESS_SERVICE_DID: accessServiceDID = '', ACCESS_SERVICE_URL: accessServiceURL = '', } = process.env @@ -123,10 +123,7 @@ export async function ucanInvocationRouter(request) { const consumerTable = createConsumerTable(AWS_REGION, consumerTableName, { endpoint: dbEndpoint }); - const provisionsStorage = useProvisionStore(subscriptionTable, consumerTable, [ - /** @type {import('@web3-storage/upload-api').ServiceDID} */ - (accessServiceDID) - ]) + const provisionsStorage = useProvisionStore(subscriptionTable, consumerTable, parseProviders(providers)) const delegationsStorage = createDelegationsTable(AWS_REGION, delegationTableName, { bucket: delegationBucket, invocationBucket, workflowBucket }) const server = createUcantoServer(serviceSigner, { diff --git a/upload-api/functions/validate-email.jsx b/upload-api/functions/validate-email.jsx index 969a611f..0c2d5ad4 100644 --- a/upload-api/functions/validate-email.jsx +++ b/upload-api/functions/validate-email.jsx @@ -1,7 +1,7 @@ import * as Sentry from '@sentry/serverless' import { authorize } from '@web3-storage/upload-api/validate' import { Config } from '@serverless-stack/node/config/index.js' -import { getServiceSigner } from '../config.js' +import { getServiceSigner, parseProviders } from '../config.js' import { Email } from '../email.js' import { createDelegationsTable } from '../tables/delegations.js' import { createDelegationsStore } from '../buckets/delegations-store.js' @@ -63,7 +63,6 @@ export const preValidateEmail = Sentry.AWSLambda.wrapHandler(validateEmailGet) function createAuthorizeContext () { const { ACCESS_SERVICE_URL = '', - ACCESS_SERVICE_DID = '', AWS_REGION = '', DELEGATION_TABLE_NAME = '', R2_ENDPOINT = '', @@ -76,6 +75,7 @@ function createAuthorizeContext () { SUBSCRIPTION_TABLE_NAME = '', CONSUMER_TABLE_NAME = '', UPLOAD_API_DID = '', + PROVIDERS = '', // set for testing DYNAMO_DB_ENDPOINT: dbEndpoint, } = process.env @@ -98,10 +98,7 @@ function createAuthorizeContext () { email: new Email({ token: POSTMARK_TOKEN }), signer: getServiceSigner({ UPLOAD_API_DID, PRIVATE_KEY }), delegationsStorage: createDelegationsTable(AWS_REGION, DELEGATION_TABLE_NAME, { bucket: delegationBucket, invocationBucket, workflowBucket }), - provisionsStorage: useProvisionStore(subscriptionTable, consumerTable, [ - /** @type {import('@web3-storage/upload-api').ServiceDID} */ - (ACCESS_SERVICE_DID) - ]), + provisionsStorage: useProvisionStore(subscriptionTable, consumerTable, parseProviders(PROVIDERS)), } } From 7055412a48b7f60f1594acc06ec852d64912c7ac Mon Sep 17 00:00:00 2001 From: Travis Vachon Date: Fri, 11 Aug 2023 12:33:31 -0700 Subject: [PATCH 2/6] fix: revive accessServiceDID I've cleaned this up in a different PR --- upload-api/functions/ucan-invocation-router.js | 1 + 1 file changed, 1 insertion(+) diff --git a/upload-api/functions/ucan-invocation-router.js b/upload-api/functions/ucan-invocation-router.js index 356a3dc0..766f2e06 100644 --- a/upload-api/functions/ucan-invocation-router.js +++ b/upload-api/functions/ucan-invocation-router.js @@ -97,6 +97,7 @@ export async function ucanInvocationRouter(request) { PROVIDERS: providers = '', // set for testing DYNAMO_DB_ENDPOINT: dbEndpoint, + ACCESS_SERVICE_DID: accessServiceDID = '', ACCESS_SERVICE_URL: accessServiceURL = '', } = process.env From 010244e3e1729aacbd7bf8955f52dc43c1835e5d Mon Sep 17 00:00:00 2001 From: Travis Vachon Date: Fri, 11 Aug 2023 12:38:55 -0700 Subject: [PATCH 3/6] fix: one more jsdoc tweak local was in a weird state, finally bit the bullet and clean/installed --- upload-api/config.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/upload-api/config.js b/upload-api/config.js index 0d8369d4..132bd85b 100644 --- a/upload-api/config.js +++ b/upload-api/config.js @@ -22,7 +22,7 @@ export function getServiceSigner(config) { * Given a string, parse into provider service DIDs. * * @param {string} providersEnvVar - * @return {import('@web3-storage/upload-api').ServiceDID[]} + * @returns {import('@web3-storage/upload-api').ServiceDID[]} */ export function parseProviders(providersEnvVar) { return /** @type {import('@web3-storage/upload-api').ServiceDID[]} */( From f1a846b522c09fd3abd8c1c95746eba895e04307 Mon Sep 17 00:00:00 2001 From: Travis Vachon Date: Fri, 11 Aug 2023 14:11:45 -0700 Subject: [PATCH 4/6] feat: add tests and better typechecking good idea to make sure these are actually ServiceDIDs since that's what the types want --- upload-api/config.js | 10 ++++++-- upload-api/test/config.test.js | 45 ++++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 2 deletions(-) diff --git a/upload-api/config.js b/upload-api/config.js index 132bd85b..80f51813 100644 --- a/upload-api/config.js +++ b/upload-api/config.js @@ -19,13 +19,19 @@ export function getServiceSigner(config) { } /** - * Given a string, parse into provider service DIDs. + * Given a string, parse into provider ServiceDIDs. * * @param {string} providersEnvVar * @returns {import('@web3-storage/upload-api').ServiceDID[]} */ export function parseProviders(providersEnvVar) { return /** @type {import('@web3-storage/upload-api').ServiceDID[]} */( - providersEnvVar.split(',').map(s => s.trim()) + providersEnvVar.split(',').map(s => { + const did = DID.parse(s.trim()).did() + if (!did.startsWith('did:web:')) { + throw new Error(`Invalid ServiceDID - ServiceDID must be a did:web: ${did}`) + } + return did + }) ) } \ No newline at end of file diff --git a/upload-api/test/config.test.js b/upload-api/test/config.test.js index 773654ea..28b1a907 100644 --- a/upload-api/test/config.test.js +++ b/upload-api/test/config.test.js @@ -57,3 +57,48 @@ test('getServiceSigner errors if config.UPLOAD_API_DID is provided but not a DID }) }, { message: /^Invalid DID/ }) }) + +test('parseProviders parses one DID', async (t) => { + t.deepEqual( + configModule.parseProviders('did:web:example.com'), + ['did:web:example.com'] + ) +}) + +test('parseProviders parses more than one DID', async (t) => { + t.deepEqual( + configModule.parseProviders('did:web:example.com,did:web:two.example.com'), + ['did:web:example.com', 'did:web:two.example.com'] + ) + + t.deepEqual( + configModule.parseProviders('did:web:example.com,did:web:two.example.com,did:web:three.example.com'), + ['did:web:example.com', 'did:web:two.example.com', 'did:web:three.example.com'] + ) +}) + +test('parseProviders trims space around dids', async (t) => { + t.deepEqual( + configModule.parseProviders(' did:web:example.com, did:web:two.example.com '), + ['did:web:example.com', 'did:web:two.example.com'] + ) +}) + +test('parseProviders throws an exception if a non-DID is provided', async (t) => { + t.throws( + () => configModule.parseProviders('http://example.com'), + { message: /^Invalid DID/} + ) +}) + +test('parseProviders throws an exception if a non-ServiceDID is provided', async (t) => { + t.throws( + () => configModule.parseProviders('did:mailto:abc123'), + { message: /^Invalid ServiceDID/} + ) + + t.throws( + () => configModule.parseProviders('did:key:z6Mkfy8k2JJUdNWCJtvzYrko5QRc7GXP6pksKDG19gxYzyi4'), + { message: /^Invalid ServiceDID/} + ) +}) \ No newline at end of file From 7d7b51b6741e865b03abcf4628990728b5798f4a Mon Sep 17 00:00:00 2001 From: Travis Vachon Date: Fri, 11 Aug 2023 15:07:15 -0700 Subject: [PATCH 5/6] fix: set PROVIDERS in the upload api stack --- stacks/upload-api-stack.js | 1 + 1 file changed, 1 insertion(+) diff --git a/stacks/upload-api-stack.js b/stacks/upload-api-stack.js index da12a66e..3132d3ce 100644 --- a/stacks/upload-api-stack.js +++ b/stacks/upload-api-stack.js @@ -72,6 +72,7 @@ export function UploadApiStack({ stack, app }) { ACCESS_SERVICE_DID: process.env.ACCESS_SERVICE_DID ?? '', ACCESS_SERVICE_URL: process.env.ACCESS_SERVICE_URL ?? '', POSTMARK_TOKEN: process.env.POSTMARK_TOKEN ?? '', + PROVIDERS: process.env.PROVIDERS ?? '', R2_ACCESS_KEY_ID: process.env.R2_ACCESS_KEY_ID ?? '', R2_SECRET_ACCESS_KEY: process.env.R2_SECRET_ACCESS_KEY ?? '', R2_REGION: process.env.R2_REGION ?? '', From 54d3950b32a74a3a9c65bf4b237e5dcd7bfffad6 Mon Sep 17 00:00:00 2001 From: Travis Vachon Date: Mon, 14 Aug 2023 09:27:55 -0700 Subject: [PATCH 6/6] chore: rename parseProviders per PR comment, make the name better reflect the implementation --- upload-api/config.js | 6 ++--- .../functions/ucan-invocation-router.js | 4 ++-- upload-api/functions/validate-email.jsx | 4 ++-- upload-api/test/config.test.js | 24 +++++++++---------- 4 files changed, 19 insertions(+), 19 deletions(-) diff --git a/upload-api/config.js b/upload-api/config.js index 80f51813..2fb04e9f 100644 --- a/upload-api/config.js +++ b/upload-api/config.js @@ -21,12 +21,12 @@ export function getServiceSigner(config) { /** * Given a string, parse into provider ServiceDIDs. * - * @param {string} providersEnvVar + * @param {string} serviceDids a comma-separated string of ServiceDIDs * @returns {import('@web3-storage/upload-api').ServiceDID[]} */ -export function parseProviders(providersEnvVar) { +export function parseServiceDids(serviceDids) { return /** @type {import('@web3-storage/upload-api').ServiceDID[]} */( - providersEnvVar.split(',').map(s => { + serviceDids.split(',').map(s => { const did = DID.parse(s.trim()).did() if (!did.startsWith('did:web:')) { throw new Error(`Invalid ServiceDID - ServiceDID must be a did:web: ${did}`) diff --git a/upload-api/functions/ucan-invocation-router.js b/upload-api/functions/ucan-invocation-router.js index 7eb1c5ea..3a45dd94 100644 --- a/upload-api/functions/ucan-invocation-router.js +++ b/upload-api/functions/ucan-invocation-router.js @@ -11,7 +11,7 @@ import { createTaskStore } from '../buckets/task-store.js' import { createWorkflowStore } from '../buckets/workflow-store.js' import { createStoreTable } from '../tables/store.js' import { createUploadTable } from '../tables/upload.js' -import { getServiceSigner, parseProviders } from '../config.js' +import { getServiceSigner, parseServiceDids } from '../config.js' import { createUcantoServer } from '../service.js' import { Config } from '@serverless-stack/node/config/index.js' import { CAR, Legacy, Codec } from '@ucanto/transport' @@ -125,7 +125,7 @@ export async function ucanInvocationRouter(request) { endpoint: dbEndpoint }); const rateLimitsStorage = createRateLimitTable(AWS_REGION, rateLimitTableName) - const provisionsStorage = useProvisionStore(subscriptionTable, consumerTable, parseProviders(providers)) + const provisionsStorage = useProvisionStore(subscriptionTable, consumerTable, parseServiceDids(providers)) const delegationsStorage = createDelegationsTable(AWS_REGION, delegationTableName, { bucket: delegationBucket, invocationBucket, workflowBucket }) const server = createUcantoServer(serviceSigner, { diff --git a/upload-api/functions/validate-email.jsx b/upload-api/functions/validate-email.jsx index 1a6072f3..da231d59 100644 --- a/upload-api/functions/validate-email.jsx +++ b/upload-api/functions/validate-email.jsx @@ -1,7 +1,7 @@ import * as Sentry from '@sentry/serverless' import { authorize } from '@web3-storage/upload-api/validate' import { Config } from '@serverless-stack/node/config/index.js' -import { getServiceSigner, parseProviders } from '../config.js' +import { getServiceSigner, parseServiceDids } from '../config.js' import { Email } from '../email.js' import { createDelegationsTable } from '../tables/delegations.js' import { createDelegationsStore } from '../buckets/delegations-store.js' @@ -100,7 +100,7 @@ function createAuthorizeContext () { email: new Email({ token: POSTMARK_TOKEN }), signer: getServiceSigner({ UPLOAD_API_DID, PRIVATE_KEY }), delegationsStorage: createDelegationsTable(AWS_REGION, DELEGATION_TABLE_NAME, { bucket: delegationBucket, invocationBucket, workflowBucket }), - provisionsStorage: useProvisionStore(subscriptionTable, consumerTable, parseProviders(PROVIDERS)), + provisionsStorage: useProvisionStore(subscriptionTable, consumerTable, parseServiceDids(PROVIDERS)), rateLimitsStorage: createRateLimitTable(AWS_REGION, RATE_LIMIT_TABLE_NAME) } } diff --git a/upload-api/test/config.test.js b/upload-api/test/config.test.js index 28b1a907..775c73af 100644 --- a/upload-api/test/config.test.js +++ b/upload-api/test/config.test.js @@ -58,47 +58,47 @@ test('getServiceSigner errors if config.UPLOAD_API_DID is provided but not a DID }, { message: /^Invalid DID/ }) }) -test('parseProviders parses one DID', async (t) => { +test('parseServiceDids parses one DID', async (t) => { t.deepEqual( - configModule.parseProviders('did:web:example.com'), + configModule.parseServiceDids('did:web:example.com'), ['did:web:example.com'] ) }) -test('parseProviders parses more than one DID', async (t) => { +test('parseServiceDids parses more than one DID', async (t) => { t.deepEqual( - configModule.parseProviders('did:web:example.com,did:web:two.example.com'), + configModule.parseServiceDids('did:web:example.com,did:web:two.example.com'), ['did:web:example.com', 'did:web:two.example.com'] ) t.deepEqual( - configModule.parseProviders('did:web:example.com,did:web:two.example.com,did:web:three.example.com'), + configModule.parseServiceDids('did:web:example.com,did:web:two.example.com,did:web:three.example.com'), ['did:web:example.com', 'did:web:two.example.com', 'did:web:three.example.com'] ) }) -test('parseProviders trims space around dids', async (t) => { +test('parseServiceDids trims space around dids', async (t) => { t.deepEqual( - configModule.parseProviders(' did:web:example.com, did:web:two.example.com '), + configModule.parseServiceDids(' did:web:example.com, did:web:two.example.com '), ['did:web:example.com', 'did:web:two.example.com'] ) }) -test('parseProviders throws an exception if a non-DID is provided', async (t) => { +test('parseServiceDids throws an exception if a non-DID is provided', async (t) => { t.throws( - () => configModule.parseProviders('http://example.com'), + () => configModule.parseServiceDids('http://example.com'), { message: /^Invalid DID/} ) }) -test('parseProviders throws an exception if a non-ServiceDID is provided', async (t) => { +test('parseServiceDids throws an exception if a non-ServiceDID is provided', async (t) => { t.throws( - () => configModule.parseProviders('did:mailto:abc123'), + () => configModule.parseServiceDids('did:mailto:abc123'), { message: /^Invalid ServiceDID/} ) t.throws( - () => configModule.parseProviders('did:key:z6Mkfy8k2JJUdNWCJtvzYrko5QRc7GXP6pksKDG19gxYzyi4'), + () => configModule.parseServiceDids('did:key:z6Mkfy8k2JJUdNWCJtvzYrko5QRc7GXP6pksKDG19gxYzyi4'), { message: /^Invalid ServiceDID/} ) }) \ No newline at end of file