-
Notifications
You must be signed in to change notification settings - Fork 7
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat: implement RateLimitStorage and integrate it into upload-api (#216)
Implement the new RateLimitStorage interface from storacha/w3up#832 and integrate it into the upload-api. This gives us the ability to block uploads to a space and to block an email or domain name from authorizing with our service. Note that we leave a couple of the new capabilities unimplemented for now since we don't need them urgently. We are actively hoping to block some abusive users ASAP, so I'm getting the blocking part of this work in now and will then turn my attention to the remaining capabilities. TODO - [x] remove file dependencies from package.json and point at latest versions of deps once storacha/w3up#832 is merged and released
- Loading branch information
Showing
20 changed files
with
16,633 additions
and
19,654 deletions.
There are no files selected for viewing
Large diffs are not rendered by default.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -2,6 +2,10 @@ import { fetch } from '@web-std/fetch' | |
| import git from 'git-rev-sync' | ||
| import pWaitFor from 'p-wait-for' | ||
| import { HeadObjectCommand } from '@aws-sdk/client-s3' | ||
| import { | ||
| PutItemCommand | ||
| } from '@aws-sdk/client-dynamodb' | ||
| import { marshall } from '@aws-sdk/util-dynamodb' | ||
|
|
||
| import { METRICS_NAMES, SPACE_METRICS_NAMES } from '../ucan-invocation/constants.js' | ||
| import { test } from './helpers/context.js' | ||
|
|
@@ -14,15 +18,16 @@ import { | |
| getCarparkBucketInfo, | ||
| getDynamoDb | ||
| } from './helpers/deployment.js' | ||
| import { setupNewClient } from './helpers/up-client.js' | ||
| import { createNewClient, setupNewClient } from './helpers/up-client.js' | ||
| import { randomFile } from './helpers/random.js' | ||
| import { getTableItem, getAllTableRows } from './helpers/table.js' | ||
|
|
||
| test.before(t => { | ||
| t.context = { | ||
| apiEndpoint: getApiEndpoint(), | ||
| metricsDynamo: getDynamoDb('admin-metrics'), | ||
| spaceMetricsDynamo: getDynamoDb('space-metrics') | ||
| spaceMetricsDynamo: getDynamoDb('space-metrics'), | ||
| rateLimitsDynamo: getDynamoDb('rate-limit') | ||
| } | ||
| }) | ||
|
|
||
|
|
@@ -65,6 +70,48 @@ test('upload-api /metrics', async t => { | |
| t.is((body.match(/w3up_invocations_total/g) || []).length, 6) | ||
| }) | ||
|
|
||
| test('authorizations can be blocked by email or domain', async t => { | ||
| const client = await createNewClient(t.context.apiEndpoint) | ||
|
|
||
| // test email blocking | ||
| await t.context.rateLimitsDynamo.client.send(new PutItemCommand({ | ||
| TableName: t.context.rateLimitsDynamo.tableName, | ||
| Item: marshall({ | ||
| id: Math.random().toString(10), | ||
| subject: '[email protected]', | ||
| rate: 0 | ||
| }) | ||
| })) | ||
|
|
||
| // it would be nice to use t.throwsAsync here, but that doesn't work with errors that aren't exceptions: https://github.com/avajs/ava/issues/2517 | ||
| try { | ||
| await client.authorize('[email protected]') | ||
| t.fail('authorize should fail with a blocked email address') | ||
| } catch (e) { | ||
| t.is(e.name, 'AccountBlocked') | ||
| t.is(e.message, 'Account identified by did:mailto:example.com:travis is blocked') | ||
| } | ||
|
|
||
| // test domain blocking | ||
| await t.context.rateLimitsDynamo.client.send(new PutItemCommand({ | ||
| TableName: t.context.rateLimitsDynamo.tableName, | ||
| Item: marshall({ | ||
| id: Math.random().toString(10), | ||
| subject: 'example2.com', | ||
| rate: 0 | ||
| }) | ||
| })) | ||
|
|
||
| // it would be nice to use t.throwsAsync here, but that doesn't work with errors that aren't exceptions: https://github.com/avajs/ava/issues/2517 | ||
| try { | ||
| await client.authorize('[email protected]') | ||
| t.fail('authorize should fail with a blocked domain') | ||
| } catch (e) { | ||
| t.is(e.name, 'AccountBlocked') | ||
| t.is(e.message, 'Account identified by did:mailto:example2.com:travis is blocked') | ||
| } | ||
| }) | ||
|
|
||
| // Integration test for all flow from uploading a file to Kinesis events consumers and replicator | ||
| test('w3infra integration flow', async t => { | ||
| const client = await setupNewClient(t.context.apiEndpoint) | ||
|
|
@@ -222,6 +269,21 @@ test('w3infra integration flow', async t => { | |
| ) | ||
| }) | ||
| } | ||
|
|
||
| // verify that blocking a space makes it impossible to upload a file to it | ||
| await t.context.rateLimitsDynamo.client.send(new PutItemCommand({ | ||
| TableName: t.context.rateLimitsDynamo.tableName, | ||
| Item: marshall({ | ||
| id: Math.random().toString(10), | ||
| subject: client.currentSpace().did(), | ||
| rate: 0 | ||
| }) | ||
| })) | ||
| const uploadError = await t.throwsAsync(async () => { | ||
| await client.uploadFile(await randomFile(100)) | ||
| }) | ||
|
|
||
| t.is(uploadError.message, 'failed store/add invocation') | ||
| }) | ||
|
|
||
| /** | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.