From 27563039dbc9ebb96abde37814ef21aa07b118a4 Mon Sep 17 00:00:00 2001 From: Thomas Hargrove <15239392+thomashargrove@users.noreply.github.com> Date: Fri, 7 Jun 2024 12:11:00 -0700 Subject: [PATCH] Speed up dynamodb List() by only getting keys (#21159) * Speed up ddb list by only getting keys * Add same optimization to has_children * Add changeling --------- Co-authored-by: Violet Hynes --- changelog/21159.txt | 4 ++++ physical/dynamodb/dynamodb.go | 10 ++++++++++ 2 files changed, 14 insertions(+) create mode 100644 changelog/21159.txt diff --git a/changelog/21159.txt b/changelog/21159.txt new file mode 100644 index 000000000000..25c1e53d90d5 --- /dev/null +++ b/changelog/21159.txt @@ -0,0 +1,4 @@ +```release-note:improvement +storage/dynamodb: Speed up list and delete of large directories by only requesting keys from DynamoDB +``` + diff --git a/physical/dynamodb/dynamodb.go b/physical/dynamodb/dynamodb.go index 7b6413f143a0..c4484d20d446 100644 --- a/physical/dynamodb/dynamodb.go +++ b/physical/dynamodb/dynamodb.go @@ -410,6 +410,11 @@ func (d *DynamoDBBackend) List(ctx context.Context, prefix string) ([]string, er }}, }, }, + ProjectionExpression: aws.String("#key, #path"), + ExpressionAttributeNames: map[string]*string{ + "#key": aws.String("Key"), + "#path": aws.String("Path"), + }, } d.permitPool.Acquire() @@ -453,6 +458,11 @@ func (d *DynamoDBBackend) hasChildren(prefix string, exclude []string) (bool, er }}, }, }, + ProjectionExpression: aws.String("#key, #path"), + ExpressionAttributeNames: map[string]*string{ + "#key": aws.String("Key"), + "#path": aws.String("Path"), + }, // Avoid fetching too many items from DynamoDB for performance reasons. // We want to know if there are any children we don't expect to see. // Answering that question requires fetching a minimum of one more item