From b48b52ccdc931155ed063711e373ff1099b290bb Mon Sep 17 00:00:00 2001 From: Benjamin Bengfort Date: Sat, 30 Sep 2023 13:24:55 -0700 Subject: [PATCH 1/2] Pagination Module --- pagination/pagination.pb.go | 59 +++++++++++++++++++--------- proto/buf.lock | 8 ++++ proto/buf.yaml | 2 + proto/pagination/v1/pagination.proto | 19 +++++++-- 4 files changed, 65 insertions(+), 23 deletions(-) create mode 100644 proto/buf.lock diff --git a/pagination/pagination.pb.go b/pagination/pagination.pb.go index 4bd7eb0..ef27e9b 100644 --- a/pagination/pagination.pb.go +++ b/pagination/pagination.pb.go @@ -9,6 +9,7 @@ package pagination import ( protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" + timestamppb "google.golang.org/protobuf/types/known/timestamppb" reflect "reflect" sync "sync" ) @@ -31,9 +32,14 @@ type PageCursor struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - PageSize int32 `protobuf:"varint,1,opt,name=page_size,json=pageSize,proto3" json:"page_size,omitempty"` // the number of results returned on each iteration. - NextKey []byte `protobuf:"bytes,2,opt,name=next_key,json=nextKey,proto3" json:"next_key,omitempty"` // the key to start the iteration from - Namespace string `protobuf:"bytes,3,opt,name=namespace,proto3" json:"namespace,omitempty"` // the namespace the cursor is iterating on + // The number of results returned on each iteration. + PageSize int32 `protobuf:"varint,1,opt,name=page_size,json=pageSize,proto3" json:"page_size,omitempty"` + // The key to start the iteration from for forward iteration (e.g. the seek key). + NextKey []byte `protobuf:"bytes,2,opt,name=next_key,json=nextKey,proto3" json:"next_key,omitempty"` + // The namespace the cursor is iterating on + Namespace string `protobuf:"bytes,3,opt,name=namespace,proto3" json:"namespace,omitempty"` + // The timestamp when the cursor is no longer valid. + Expires *timestamppb.Timestamp `protobuf:"bytes,4,opt,name=expires,proto3" json:"expires,omitempty"` } func (x *PageCursor) Reset() { @@ -89,22 +95,35 @@ func (x *PageCursor) GetNamespace() string { return "" } +func (x *PageCursor) GetExpires() *timestamppb.Timestamp { + if x != nil { + return x.Expires + } + return nil +} + var File_pagination_v1_pagination_proto protoreflect.FileDescriptor var file_pagination_v1_pagination_proto_rawDesc = []byte{ 0x0a, 0x1e, 0x70, 0x61, 0x67, 0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2f, 0x76, 0x31, 0x2f, 0x70, 0x61, 0x67, 0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x12, 0x68, 0x6f, 0x6e, 0x75, 0x2e, 0x70, 0x61, 0x67, 0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, - 0x6e, 0x2e, 0x76, 0x31, 0x22, 0x62, 0x0a, 0x0a, 0x50, 0x61, 0x67, 0x65, 0x43, 0x75, 0x72, 0x73, - 0x6f, 0x72, 0x12, 0x1b, 0x0a, 0x09, 0x70, 0x61, 0x67, 0x65, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x08, 0x70, 0x61, 0x67, 0x65, 0x53, 0x69, 0x7a, 0x65, 0x12, - 0x19, 0x0a, 0x08, 0x6e, 0x65, 0x78, 0x74, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x0c, 0x52, 0x07, 0x6e, 0x65, 0x78, 0x74, 0x4b, 0x65, 0x79, 0x12, 0x1c, 0x0a, 0x09, 0x6e, 0x61, - 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6e, - 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x42, 0x29, 0x5a, 0x27, 0x67, 0x69, 0x74, 0x68, - 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x72, 0x6f, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x61, - 0x6c, 0x69, 0x6f, 0x2f, 0x68, 0x6f, 0x6e, 0x75, 0x2f, 0x70, 0x61, 0x67, 0x69, 0x6e, 0x61, 0x74, - 0x69, 0x6f, 0x6e, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x6e, 0x2e, 0x76, 0x31, 0x1a, 0x1f, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x98, 0x01, 0x0a, 0x0a, 0x50, 0x61, 0x67, 0x65, 0x43, 0x75, + 0x72, 0x73, 0x6f, 0x72, 0x12, 0x1b, 0x0a, 0x09, 0x70, 0x61, 0x67, 0x65, 0x5f, 0x73, 0x69, 0x7a, + 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x08, 0x70, 0x61, 0x67, 0x65, 0x53, 0x69, 0x7a, + 0x65, 0x12, 0x19, 0x0a, 0x08, 0x6e, 0x65, 0x78, 0x74, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x0c, 0x52, 0x07, 0x6e, 0x65, 0x78, 0x74, 0x4b, 0x65, 0x79, 0x12, 0x1c, 0x0a, 0x09, + 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x09, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x12, 0x34, 0x0a, 0x07, 0x65, 0x78, + 0x70, 0x69, 0x72, 0x65, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, + 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, + 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x07, 0x65, 0x78, 0x70, 0x69, 0x72, 0x65, 0x73, + 0x42, 0x29, 0x5a, 0x27, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x72, + 0x6f, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x69, 0x6f, 0x2f, 0x68, 0x6f, 0x6e, 0x75, + 0x2f, 0x70, 0x61, 0x67, 0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x62, 0x06, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x33, } var ( @@ -121,14 +140,16 @@ func file_pagination_v1_pagination_proto_rawDescGZIP() []byte { var file_pagination_v1_pagination_proto_msgTypes = make([]protoimpl.MessageInfo, 1) var file_pagination_v1_pagination_proto_goTypes = []interface{}{ - (*PageCursor)(nil), // 0: honu.pagination.v1.PageCursor + (*PageCursor)(nil), // 0: honu.pagination.v1.PageCursor + (*timestamppb.Timestamp)(nil), // 1: google.protobuf.Timestamp } var file_pagination_v1_pagination_proto_depIdxs = []int32{ - 0, // [0:0] is the sub-list for method output_type - 0, // [0:0] is the sub-list for method input_type - 0, // [0:0] is the sub-list for extension type_name - 0, // [0:0] is the sub-list for extension extendee - 0, // [0:0] is the sub-list for field type_name + 1, // 0: honu.pagination.v1.PageCursor.expires:type_name -> google.protobuf.Timestamp + 1, // [1:1] is the sub-list for method output_type + 1, // [1:1] is the sub-list for method input_type + 1, // [1:1] is the sub-list for extension type_name + 1, // [1:1] is the sub-list for extension extendee + 0, // [0:1] is the sub-list for field type_name } func init() { file_pagination_v1_pagination_proto_init() } diff --git a/proto/buf.lock b/proto/buf.lock new file mode 100644 index 0000000..6c4355d --- /dev/null +++ b/proto/buf.lock @@ -0,0 +1,8 @@ +# Generated by buf. DO NOT EDIT. +version: v1 +deps: + - remote: buf.build + owner: googleapis + repository: googleapis + commit: 28151c0d0a1641bf938a7672c500e01d + digest: shake256:49215edf8ef57f7863004539deff8834cfb2195113f0b890dd1f67815d9353e28e668019165b9d872395871eeafcbab3ccfdb2b5f11734d3cca95be9e8d139de diff --git a/proto/buf.yaml b/proto/buf.yaml index c72c02c..b5a6b11 100644 --- a/proto/buf.yaml +++ b/proto/buf.yaml @@ -1,5 +1,7 @@ version: v1 name: buf.build/rotational/honudb +deps: + - buf.build/googleapis/googleapis breaking: use: - FILE diff --git a/proto/pagination/v1/pagination.proto b/proto/pagination/v1/pagination.proto index b07ec18..490d4fe 100644 --- a/proto/pagination/v1/pagination.proto +++ b/proto/pagination/v1/pagination.proto @@ -1,6 +1,9 @@ syntax = "proto3"; package honu.pagination.v1; + +import "google/protobuf/timestamp.proto"; + option go_package = "github.com/rotationalio/honu/pagination"; // Implements a protocol buffer struct for state managed pagination. This struct will be @@ -10,7 +13,15 @@ option go_package = "github.com/rotationalio/honu/pagination"; // size in the cursor matches the page size in the request. // See https://cloud.google.com/apis/design/design_patterns#list_pagination for more. message PageCursor { - int32 page_size = 1; // the number of results returned on each iteration. - bytes next_key = 2; // the key to start the iteration from - string namespace = 3; // the namespace the cursor is iterating on -} \ No newline at end of file + // The number of results returned on each iteration. + int32 page_size = 1; + + // The key to start the iteration from for forward iteration (e.g. the seek key). + bytes next_key = 2; + + // The namespace the cursor is iterating on + string namespace = 3; + + // The timestamp when the cursor is no longer valid. + google.protobuf.Timestamp expires = 4; +} From 5ff00441fac6d8038ad3fa314a967d2372ae74f5 Mon Sep 17 00:00:00 2001 From: Benjamin Bengfort Date: Sat, 30 Sep 2023 13:39:48 -0700 Subject: [PATCH 2/2] Pagination Module --- go.mod | 2 +- pagination/pagination.go | 88 +++++++++++++++++++++++++++++++++++ pagination/pagination_test.go | 75 +++++++++++++++++++++++++++++ 3 files changed, 164 insertions(+), 1 deletion(-) create mode 100644 pagination/pagination.go create mode 100644 pagination/pagination_test.go diff --git a/go.mod b/go.mod index ceece5b..e638bcf 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,6 @@ go 1.19 require ( github.com/cockroachdb/pebble v0.0.0-20230906203007-2129a6e99d0f - github.com/golang/protobuf v1.5.3 github.com/stretchr/testify v1.8.4 github.com/syndtr/goleveldb v1.0.0 google.golang.org/grpc v1.58.0 @@ -22,6 +21,7 @@ require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/getsentry/sentry-go v0.24.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/protobuf v1.5.3 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/klauspost/compress v1.16.7 // indirect github.com/kr/pretty v0.3.1 // indirect diff --git a/pagination/pagination.go b/pagination/pagination.go new file mode 100644 index 0000000..d299c92 --- /dev/null +++ b/pagination/pagination.go @@ -0,0 +1,88 @@ +package pagination + +import ( + "encoding/base64" + "errors" + "time" + + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/timestamppb" +) + +const ( + DefaultPageSize int32 = 100 + MaximumPageSize int32 = 5000 + CursorDuration = 24 * time.Hour +) + +var ( + ErrMissingExpiration = errors.New("cursor does not have an expires timestamp") + ErrCursorExpired = errors.New("cursor has expired and is no longer useable") + ErrUnparsableToken = errors.New("could not parse the next page token") + ErrTokenQueryMismatch = errors.New("cannot change query parameters during pagination") +) + +func New(nextKey []byte, namespace string, pageSize int32) *PageCursor { + if pageSize == 0 { + pageSize = DefaultPageSize + } + + return &PageCursor{ + PageSize: pageSize, + NextKey: nextKey, + Namespace: namespace, + Expires: timestamppb.New(time.Now().Add(CursorDuration)), + } +} + +func Parse(token string) (cursor *PageCursor, err error) { + var data []byte + if data, err = base64.RawURLEncoding.DecodeString(token); err != nil { + return nil, ErrUnparsableToken + } + + cursor = &PageCursor{} + if err = proto.Unmarshal(data, cursor); err != nil { + return nil, ErrUnparsableToken + } + + var expired bool + if expired, err = cursor.HasExpired(); err != nil { + return nil, err + } + + if expired { + return nil, ErrCursorExpired + } + + return cursor, nil +} + +func (c *PageCursor) NextPageToken() (_ string, err error) { + var expired bool + if expired, err = c.HasExpired(); err != nil { + return "", err + } + + if expired { + return "", ErrCursorExpired + } + + var data []byte + if data, err = proto.Marshal(c); err != nil { + return "", err + } + + return base64.RawURLEncoding.EncodeToString(data), nil +} + +func (c *PageCursor) HasExpired() (bool, error) { + if c.Expires == nil { + return false, ErrMissingExpiration + } + return time.Now().After(c.Expires.AsTime()), nil +} + +func (c *PageCursor) IsZero() bool { + return c.PageSize == 0 && len(c.NextKey) == 0 && c.Namespace == "" && (c.Expires == nil || c.Expires.AsTime().IsZero()) +} diff --git a/pagination/pagination_test.go b/pagination/pagination_test.go new file mode 100644 index 0000000..abb0c82 --- /dev/null +++ b/pagination/pagination_test.go @@ -0,0 +1,75 @@ +package pagination_test + +import ( + "encoding/base64" + "testing" + "time" + + "github.com/rotationalio/honu/pagination" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/timestamppb" +) + +func TestPaginationToken(t *testing.T) { + cursor := pagination.New([]byte("foo"), "bar", 0) + token, err := cursor.NextPageToken() + require.NoError(t, err, "could not create next page token") + require.Greater(t, len(token), 32, "the token created above should at least be 32 characters") + require.Less(t, len(token), 64, "the token create above should be less than 64 characters") + + parsed, err := pagination.Parse(token) + require.NoError(t, err, "could not parse token") + require.True(t, proto.Equal(cursor, parsed), "parsed token should match cursor") + + cursor.Expires = nil + _, err = cursor.NextPageToken() + require.ErrorIs(t, err, pagination.ErrMissingExpiration, "should not be able to create a token with no expiration") + + data, err := proto.Marshal(cursor) + require.NoError(t, err, "could not marshal protocol buffers") + token = base64.RawURLEncoding.EncodeToString(data) + _, err = pagination.Parse(token) + require.ErrorIs(t, err, pagination.ErrMissingExpiration, "should not be able to parse a token with no expiration") + + _, err = pagination.Parse("ab") + require.ErrorIs(t, err, pagination.ErrUnparsableToken, "should not be able to parse an invalid token") + + _, err = pagination.Parse(base64.RawStdEncoding.EncodeToString([]byte("badtokendata"))) + require.ErrorIs(t, err, pagination.ErrUnparsableToken, "should not be able to parse an invalid token") + + cursor.Expires = timestamppb.New(time.Now().Add(-5 * time.Minute)) + _, err = cursor.NextPageToken() + require.ErrorIs(t, err, pagination.ErrCursorExpired, "should not be able to create an expired token") + + data, err = proto.Marshal(cursor) + require.NoError(t, err, "could not marshal protocol buffers") + token = base64.RawURLEncoding.EncodeToString(data) + _, err = pagination.Parse(token) + require.ErrorIs(t, err, pagination.ErrCursorExpired, "should not be able to parse an expired token") +} + +func TestPaginationExpired(t *testing.T) { + cursor := &pagination.PageCursor{} + expired, err := cursor.HasExpired() + require.ErrorIs(t, err, pagination.ErrMissingExpiration) + require.False(t, expired, "if err is not nil, expired should be false") + + cursor.Expires = timestamppb.New(time.Now().Add(5 * time.Minute)) + expired, err = cursor.HasExpired() + require.NoError(t, err, "cursor should compute expiration without error") + require.False(t, expired, "cusor should not be expired for 5 minutes") + + cursor.Expires = timestamppb.New(time.Now().Add(-5 * time.Minute)) + expired, err = cursor.HasExpired() + require.NoError(t, err, "cursor should compute expiration without error") + require.True(t, expired, "cusor should have expired 5 minutes ago") +} + +func TestPaginationIsZero(t *testing.T) { + cursor := &pagination.PageCursor{} + require.True(t, cursor.IsZero(), "empty cursor should be zero valued") + + cursor = pagination.New([]byte("foo"), "bar", 0) + require.False(t, cursor.IsZero(), "new cursor should not be zero valued") +}