From bd349d93a4c26b17bea0065c64c8d4612b1dfa6c Mon Sep 17 00:00:00 2001 From: Jonathan Fung <121899091+jonfung-dydx@users.noreply.github.com> Date: Thu, 1 Aug 2024 15:36:12 -0400 Subject: [PATCH 01/33] Post only order breaks out of matching loop + Add post only crosses maker order as first class order status field. (#1996) --- protocol/x/clob/memclob/memclob.go | 17 ++++++++++++----- .../x/clob/memclob/memclob_place_order_test.go | 7 ++----- protocol/x/clob/types/orderbook.go | 3 +++ 3 files changed, 17 insertions(+), 10 deletions(-) diff --git a/protocol/x/clob/memclob/memclob.go b/protocol/x/clob/memclob/memclob.go index 2b3b18c72b..6c9779571f 100644 --- a/protocol/x/clob/memclob/memclob.go +++ b/protocol/x/clob/memclob/memclob.go @@ -810,12 +810,9 @@ func (m *MemClobPriceTimePriority) matchOrder( var matchingErr error - // If the order is post only and it's not the rewind step, then it cannot be filled. + // If the order is post only and crosses the book, // Set the matching error so that the order is canceled. - // TODO(DEC-998): Determine if allowing post-only orders to match in rewind step is valid. - if len(newMakerFills) > 0 && - !order.IsLiquidation() && - order.MustGetOrder().TimeInForce == types.Order_TIME_IN_FORCE_POST_ONLY { + if !order.IsLiquidation() && takerOrderStatus.OrderStatus == types.PostOnlyWouldCrossMakerOrder { matchingErr = types.ErrPostOnlyWouldCrossMakerOrder } @@ -1757,6 +1754,16 @@ func (m *MemClobPriceTimePriority) mustPerformTakerOrderMatching( continue } + // If a valid match has been generated but the taker order is a post only order, + // end the matching loop. Because of this, post-only orders can cause + // undercollateralized maker orders to be removed from the book up to the first valid match. + if takerOrderCrossesMakerOrder && + !newTakerOrder.IsLiquidation() && + newTakerOrder.MustGetOrder().TimeInForce == types.Order_TIME_IN_FORCE_POST_ONLY { + takerOrderStatus.OrderStatus = types.PostOnlyWouldCrossMakerOrder + break + } + // The orders have matched successfully, and the state has been updated. // To mark the orders as matched, perform the following actions: // 1. Deduct `matchedAmount` from the taker order's remaining quantums, and add the matched diff --git a/protocol/x/clob/memclob/memclob_place_order_test.go b/protocol/x/clob/memclob/memclob_place_order_test.go index 51c5a6a250..1b942201ab 100644 --- a/protocol/x/clob/memclob/memclob_place_order_test.go +++ b/protocol/x/clob/memclob/memclob_place_order_test.go @@ -2834,17 +2834,14 @@ func TestPlaceOrder_PostOnly(t *testing.T) { }, }, expectedRemainingAsks: []OrderWithRemainingSize{}, + // Second order is not collat check'd since the first order generates a valid + // match, so the matching loop ends. expectedCollatCheck: []expectedMatch{ { makerOrder: &constants.Order_Bob_Num0_Id11_Clob1_Buy5_Price40_GTB32, takerOrder: &constants.Order_Alice_Num1_Id1_Clob1_Sell10_Price15_GTB20_PO, matchedQuantums: 5, }, - { - makerOrder: &constants.Order_Bob_Num0_Id4_Clob1_Buy20_Price35_GTB22, - takerOrder: &constants.Order_Alice_Num1_Id1_Clob1_Sell10_Price15_GTB20_PO, - matchedQuantums: 5, - }, }, expectedExistingMatches: []expectedMatch{}, expectedOperations: []types.Operation{}, diff --git a/protocol/x/clob/types/orderbook.go b/protocol/x/clob/types/orderbook.go index 9c640308c0..7650c2869b 100644 --- a/protocol/x/clob/types/orderbook.go +++ b/protocol/x/clob/types/orderbook.go @@ -136,6 +136,9 @@ const ( // with either multiple positions in isolated perpetuals or both an isolated and a cross perpetual // position. ViolatesIsolatedSubaccountConstraints + // PostOnlyWouldCrossMakerOrder indicates that matching the post only taker order would cross the + // orderbook, and was therefore canceled. + PostOnlyWouldCrossMakerOrder ) // String returns a string representation of this `OrderStatus` enum. From b01af24cedd8d7957426726639947f687592c2ea Mon Sep 17 00:00:00 2001 From: Jonathan Fung <121899091+jonfung-dydx@users.noreply.github.com> Date: Fri, 2 Aug 2024 14:22:48 -0400 Subject: [PATCH 02/33] Full Node Status taker order status protos (#2003) --- proto/dydxprotocol/clob/order.proto | 20 + proto/dydxprotocol/clob/query.proto | 35 + protocol/x/clob/types/liquidation_order.go | 12 + protocol/x/clob/types/order.pb.go | 466 ++++++++++-- protocol/x/clob/types/orderbook.go | 10 + protocol/x/clob/types/query.pb.go | 808 ++++++++++++++++++--- 6 files changed, 1198 insertions(+), 153 deletions(-) diff --git a/proto/dydxprotocol/clob/order.proto b/proto/dydxprotocol/clob/order.proto index ac80fec3c1..4357913474 100644 --- a/proto/dydxprotocol/clob/order.proto +++ b/proto/dydxprotocol/clob/order.proto @@ -3,6 +3,7 @@ package dydxprotocol.clob; import "gogoproto/gogo.proto"; import "dydxprotocol/subaccounts/subaccount.proto"; +import "dydxprotocol/clob/liquidations.proto"; option go_package = "github.com/dydxprotocol/v4-chain/protocol/x/clob/types"; @@ -227,3 +228,22 @@ message TransactionOrdering { // Within the block, the unique transaction index. uint32 transaction_index = 2; } + +// StreamLiquidationOrder represents an protocol-generated IOC liquidation order. +// Used in full node streaming. +message StreamLiquidationOrder { + // Information about this liquidation order. + PerpetualLiquidationInfo liquidation_info = 1; + + // CLOB pair ID of the CLOB pair the liquidation order will be matched against. + uint32 clob_pair_id = 2; + + // True if this is a buy order liquidating a short position, false if vice versa. + bool is_buy = 3; + + // The number of base quantums for this liquidation order. + uint64 quantums = 4; + + // The subticks this liquidation order will be submitted at. + uint64 subticks = 5; +} \ No newline at end of file diff --git a/proto/dydxprotocol/clob/query.proto b/proto/dydxprotocol/clob/query.proto index cca523bcc6..c7eb3d3aba 100644 --- a/proto/dydxprotocol/clob/query.proto +++ b/proto/dydxprotocol/clob/query.proto @@ -220,3 +220,38 @@ message StreamOrderbookFill { // Resulting fill amounts for each order in the orders array. repeated uint64 fill_amounts = 3; } + +// StreamTakerOrder provides information on a taker order that was attempted +// to be matched on the orderbook. +// It is intended to be used only in full node streaming. +message StreamTakerOrder { + // The taker order that was matched on the orderbook. Can be a + // regular order or a liquidation order. + oneof taker_order { + Order order = 1; + StreamLiquidationOrder liquidation_order = 2; + } + + // Information on the taker order after it is matched on the book, + // either successfully or unsuccessfully. + StreamTakerOrderStatus taker_order_status = 3; +} + +// StreamTakerOrderStatus is a representation of a taker order +// after it is attempted to be matched on the orderbook. +// It is intended to be used only in full node streaming. +message StreamTakerOrderStatus { + // The state of the taker order after attempting to match it against the orderbook. + // Possible enum values can be found here: + // https://github.com/dydxprotocol/v4-chain/blob/main/protocol/x/clob/types/orderbook.go#L105 + uint32 order_status = 1; + + // The amount of remaining (non-matched) base quantums of this taker order. + uint64 remaining_quantums = 2; + + // The amount of base quantums that were *optimistically* filled for this taker order + // when the order is matched against the orderbook. Note that if any quantums of this order + // were optimistically filled or filled in state before this invocation of the matching loop, + // this value will not include them. + uint64 optimistically_filled_quantums = 3; +} diff --git a/protocol/x/clob/types/liquidation_order.go b/protocol/x/clob/types/liquidation_order.go index 182c0e84f8..20aa56fe3d 100644 --- a/protocol/x/clob/types/liquidation_order.go +++ b/protocol/x/clob/types/liquidation_order.go @@ -51,6 +51,18 @@ func NewLiquidationOrder( } } +// ToStreamLiquidationOrder converts the LiquidationOrder to a StreamLiquidationOrder +// to be emitted by full node streaming. +func (lo *LiquidationOrder) ToStreamLiquidationOrder() *StreamLiquidationOrder { + return &StreamLiquidationOrder{ + LiquidationInfo: &lo.perpetualLiquidationInfo, + ClobPairId: uint32(lo.clobPairId), + IsBuy: lo.isBuy, + Quantums: lo.quantums.ToUint64(), + Subticks: lo.subticks.ToUint64(), + } +} + // IsBuy returns true if this is a buy order, false if not. // This function is necessary for the `LiquidationOrder` type to implement the `MatchableOrder` interface. func (lo *LiquidationOrder) IsBuy() bool { diff --git a/protocol/x/clob/types/order.pb.go b/protocol/x/clob/types/order.pb.go index f1e239d1bd..ad17410e45 100644 --- a/protocol/x/clob/types/order.pb.go +++ b/protocol/x/clob/types/order.pb.go @@ -807,6 +807,89 @@ func (m *TransactionOrdering) GetTransactionIndex() uint32 { return 0 } +// StreamLiquidationOrder represents an protocol-generated IOC liquidation order. +// Used in full node streaming. +type StreamLiquidationOrder struct { + // Information about this liquidation order. + LiquidationInfo *PerpetualLiquidationInfo `protobuf:"bytes,1,opt,name=liquidation_info,json=liquidationInfo,proto3" json:"liquidation_info,omitempty"` + // CLOB pair ID of the CLOB pair the liquidation order will be matched against. + ClobPairId uint32 `protobuf:"varint,2,opt,name=clob_pair_id,json=clobPairId,proto3" json:"clob_pair_id,omitempty"` + // True if this is a buy order liquidating a short position, false if vice versa. + IsBuy bool `protobuf:"varint,3,opt,name=is_buy,json=isBuy,proto3" json:"is_buy,omitempty"` + // The number of base quantums for this liquidation order. + Quantums uint64 `protobuf:"varint,4,opt,name=quantums,proto3" json:"quantums,omitempty"` + // The subticks this liquidation order will be submitted at. + Subticks uint64 `protobuf:"varint,5,opt,name=subticks,proto3" json:"subticks,omitempty"` +} + +func (m *StreamLiquidationOrder) Reset() { *m = StreamLiquidationOrder{} } +func (m *StreamLiquidationOrder) String() string { return proto.CompactTextString(m) } +func (*StreamLiquidationOrder) ProtoMessage() {} +func (*StreamLiquidationOrder) Descriptor() ([]byte, []int) { + return fileDescriptor_673c6f4faa93736b, []int{9} +} +func (m *StreamLiquidationOrder) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *StreamLiquidationOrder) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_StreamLiquidationOrder.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *StreamLiquidationOrder) XXX_Merge(src proto.Message) { + xxx_messageInfo_StreamLiquidationOrder.Merge(m, src) +} +func (m *StreamLiquidationOrder) XXX_Size() int { + return m.Size() +} +func (m *StreamLiquidationOrder) XXX_DiscardUnknown() { + xxx_messageInfo_StreamLiquidationOrder.DiscardUnknown(m) +} + +var xxx_messageInfo_StreamLiquidationOrder proto.InternalMessageInfo + +func (m *StreamLiquidationOrder) GetLiquidationInfo() *PerpetualLiquidationInfo { + if m != nil { + return m.LiquidationInfo + } + return nil +} + +func (m *StreamLiquidationOrder) GetClobPairId() uint32 { + if m != nil { + return m.ClobPairId + } + return 0 +} + +func (m *StreamLiquidationOrder) GetIsBuy() bool { + if m != nil { + return m.IsBuy + } + return false +} + +func (m *StreamLiquidationOrder) GetQuantums() uint64 { + if m != nil { + return m.Quantums + } + return 0 +} + +func (m *StreamLiquidationOrder) GetSubticks() uint64 { + if m != nil { + return m.Subticks + } + return 0 +} + func init() { proto.RegisterEnum("dydxprotocol.clob.Order_Side", Order_Side_name, Order_Side_value) proto.RegisterEnum("dydxprotocol.clob.Order_TimeInForce", Order_TimeInForce_name, Order_TimeInForce_value) @@ -820,75 +903,82 @@ func init() { proto.RegisterType((*ConditionalOrderPlacement)(nil), "dydxprotocol.clob.ConditionalOrderPlacement") proto.RegisterType((*Order)(nil), "dydxprotocol.clob.Order") proto.RegisterType((*TransactionOrdering)(nil), "dydxprotocol.clob.TransactionOrdering") + proto.RegisterType((*StreamLiquidationOrder)(nil), "dydxprotocol.clob.StreamLiquidationOrder") } func init() { proto.RegisterFile("dydxprotocol/clob/order.proto", fileDescriptor_673c6f4faa93736b) } var fileDescriptor_673c6f4faa93736b = []byte{ - // 996 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xcc, 0x56, 0xdd, 0x6e, 0xe3, 0x44, - 0x14, 0x8e, 0xdb, 0xec, 0x36, 0x3d, 0xf9, 0x59, 0xef, 0x74, 0x17, 0xdc, 0x94, 0xa6, 0xc1, 0x42, - 0xa5, 0x08, 0x91, 0x88, 0xb2, 0x42, 0x42, 0x88, 0x8b, 0x6d, 0x9b, 0xa8, 0x56, 0xd3, 0x3a, 0xd8, - 0x2e, 0x52, 0x57, 0x88, 0x91, 0x63, 0x4f, 0xdd, 0xd1, 0x4e, 0x3c, 0xc1, 0x1e, 0xa3, 0xe6, 0x9e, - 0x07, 0xe0, 0x25, 0x78, 0x0b, 0x1e, 0x60, 0x2f, 0xf7, 0x92, 0x2b, 0x84, 0xda, 0x67, 0xe0, 0x8a, - 0x1b, 0x34, 0x63, 0x37, 0x4d, 0xba, 0x5b, 0x21, 0xb4, 0x37, 0xdc, 0x79, 0xbe, 0xf3, 0xcd, 0x37, - 0xe7, 0x9c, 0xf9, 0xce, 0xc8, 0xb0, 0x19, 0x4e, 0xc3, 0xcb, 0x49, 0xc2, 0x05, 0x0f, 0x38, 0xeb, - 0x06, 0x8c, 0x8f, 0xba, 0x3c, 0x09, 0x49, 0xd2, 0x51, 0x18, 0x7a, 0x3c, 0x1f, 0xee, 0xc8, 0x70, - 0xf3, 0x49, 0xc4, 0x23, 0xae, 0xa0, 0xae, 0xfc, 0xca, 0x89, 0xcd, 0x4f, 0x16, 0x74, 0xd2, 0x6c, - 0xe4, 0x07, 0x01, 0xcf, 0x62, 0x91, 0xce, 0x7d, 0xe7, 0x54, 0xf3, 0x37, 0x0d, 0x56, 0x6c, 0x79, - 0x86, 0x15, 0xa2, 0x6f, 0xa1, 0x7e, 0x1b, 0xc7, 0x34, 0x34, 0xb4, 0xb6, 0xb6, 0x53, 0xdd, 0xdd, - 0xee, 0x2c, 0x9c, 0x3b, 0x27, 0xd7, 0x71, 0x67, 0xdf, 0x56, 0xb8, 0x57, 0x7e, 0xf5, 0xc7, 0x56, - 0xc9, 0xa9, 0xa5, 0x73, 0x18, 0xda, 0x80, 0xd5, 0x80, 0x51, 0x92, 0xcb, 0x2d, 0xb5, 0xb5, 0x9d, - 0x15, 0xa7, 0x92, 0x03, 0x56, 0x88, 0xb6, 0xa0, 0xaa, 0xca, 0xc3, 0xe7, 0xcc, 0x8f, 0x52, 0x63, - 0xb9, 0xad, 0xed, 0xd4, 0x1d, 0x50, 0x50, 0x5f, 0x22, 0xa8, 0x0d, 0x35, 0x59, 0x25, 0x9e, 0xf8, - 0x34, 0x91, 0x02, 0xe5, 0x9c, 0x21, 0xb1, 0xa1, 0x4f, 0x13, 0x2b, 0x34, 0x7f, 0x80, 0x4d, 0x95, - 0x7d, 0xda, 0xa7, 0x8c, 0x91, 0xf0, 0x20, 0x4b, 0x68, 0x1c, 0x0d, 0x7c, 0x41, 0x52, 0xb1, 0xc7, - 0x78, 0xf0, 0x12, 0x7d, 0x03, 0xab, 0xf9, 0x19, 0x34, 0x4c, 0x0d, 0xad, 0xbd, 0xbc, 0x53, 0xdd, - 0x6d, 0x76, 0xde, 0xe8, 0x63, 0xa7, 0x68, 0x41, 0x51, 0x43, 0x85, 0xe7, 0xcb, 0xd4, 0x7c, 0x01, - 0xeb, 0x43, 0x2e, 0x48, 0x2c, 0xa8, 0xcf, 0xd8, 0x74, 0x98, 0x64, 0xb1, 0x3f, 0x62, 0x24, 0x3f, - 0xf2, 0x5d, 0xb5, 0x09, 0x34, 0x54, 0x48, 0xa6, 0xee, 0x0a, 0x5f, 0x10, 0xd9, 0x90, 0x73, 0xca, - 0x18, 0xf6, 0xc7, 0xb2, 0x7d, 0xaa, 0xfd, 0x65, 0x07, 0x24, 0xf4, 0x5c, 0x21, 0x68, 0x17, 0x9e, - 0x4e, 0x8a, 0x1c, 0xf0, 0x48, 0xd6, 0x87, 0x2f, 0x08, 0x8d, 0x2e, 0x84, 0x6a, 0x6d, 0xdd, 0x59, - 0xbb, 0x09, 0xaa, 0xda, 0x0f, 0x55, 0xc8, 0xfc, 0x1e, 0x36, 0x94, 0xfa, 0x79, 0xc6, 0xd4, 0x71, - 0x1e, 0x1d, 0x13, 0x97, 0xd1, 0x80, 0x7c, 0xe7, 0xb3, 0x8c, 0xbc, 0x6b, 0x11, 0xbf, 0x6a, 0xf0, - 0xde, 0x80, 0xc7, 0x91, 0x47, 0x92, 0xb1, 0xe2, 0x0c, 0x99, 0x1f, 0x90, 0x31, 0x89, 0x05, 0x7a, - 0x06, 0x0f, 0x14, 0xad, 0xb0, 0x91, 0x71, 0x9f, 0x6a, 0xa1, 0x99, 0x93, 0xd1, 0x29, 0x3c, 0x9a, - 0xdc, 0x48, 0x60, 0x1a, 0x87, 0xe4, 0x52, 0x15, 0xf7, 0x86, 0x0d, 0xd5, 0x7e, 0x2f, 0xf1, 0xe3, - 0xd4, 0x0f, 0x04, 0xe5, 0xb1, 0x92, 0xa2, 0x71, 0x54, 0xa8, 0x35, 0x66, 0x22, 0x96, 0xd4, 0x30, - 0xff, 0xd2, 0x60, 0x7d, 0x9f, 0xc7, 0x21, 0x95, 0x5c, 0x9f, 0xfd, 0x8f, 0x53, 0x45, 0x47, 0x50, - 0x17, 0x09, 0x8d, 0x22, 0x79, 0x27, 0x4a, 0x74, 0xf9, 0xbf, 0x88, 0x3a, 0xb5, 0x62, 0x73, 0x5e, - 0xf7, 0xdf, 0x0f, 0xe1, 0x81, 0x0a, 0xa1, 0xaf, 0xa1, 0x72, 0x73, 0xd1, 0x45, 0x99, 0xff, 0x7e, - 0xcf, 0x2b, 0xc5, 0x3d, 0xa3, 0xcf, 0xa1, 0x9c, 0xd2, 0x90, 0xa8, 0xfa, 0x1a, 0xbb, 0x9b, 0xf7, - 0x6d, 0xec, 0xb8, 0x34, 0x24, 0x8e, 0xa2, 0xa2, 0x26, 0x54, 0x7e, 0xcc, 0xfc, 0x58, 0x64, 0xe3, - 0x7c, 0xb4, 0xcb, 0xce, 0x6c, 0x2d, 0x63, 0x69, 0x36, 0x12, 0x34, 0x78, 0x99, 0xaa, 0xa1, 0x2e, - 0x3b, 0xb3, 0x35, 0xda, 0x86, 0x46, 0xc4, 0x79, 0x88, 0x05, 0x65, 0xb9, 0xc7, 0x8d, 0x07, 0xd2, - 0xdc, 0x87, 0x25, 0xa7, 0x26, 0x71, 0x8f, 0xb2, 0x7c, 0xb2, 0xbb, 0xb0, 0xb6, 0xc8, 0xc3, 0x82, - 0x8e, 0x89, 0xf1, 0x50, 0x3e, 0x32, 0x87, 0x25, 0x47, 0x9f, 0x27, 0x4b, 0xcf, 0xa3, 0x43, 0xa8, - 0x4b, 0x06, 0xa6, 0x31, 0x3e, 0xe7, 0x49, 0x40, 0x8c, 0x15, 0x55, 0xcc, 0x47, 0xf7, 0x16, 0x23, - 0x77, 0x59, 0x71, 0x5f, 0x72, 0x9d, 0xaa, 0xb8, 0x5d, 0xc8, 0x39, 0x4d, 0x48, 0x98, 0x05, 0x04, - 0xf3, 0x98, 0x4d, 0x8d, 0x4a, 0x5b, 0xdb, 0xa9, 0x38, 0x90, 0x43, 0x76, 0xcc, 0xa6, 0xe8, 0x63, - 0x78, 0x54, 0x3c, 0x7b, 0x63, 0x22, 0xfc, 0xd0, 0x17, 0xbe, 0xb1, 0xaa, 0x26, 0xb4, 0x91, 0xc3, - 0xc7, 0x05, 0x8a, 0x8e, 0xa1, 0x11, 0xdc, 0xb8, 0x12, 0x8b, 0xe9, 0x84, 0x18, 0xa0, 0x92, 0xda, - 0xbe, 0x37, 0xa9, 0x99, 0x89, 0xbd, 0xe9, 0x84, 0x38, 0xf5, 0x60, 0x7e, 0x89, 0x8e, 0xc0, 0x0c, - 0x6e, 0x4d, 0x8e, 0xf3, 0xfb, 0xbe, 0x31, 0xd3, 0xac, 0xe3, 0x55, 0xd5, 0xf1, 0xad, 0xe0, 0xce, - 0x38, 0x78, 0x39, 0xcf, 0x2d, 0x68, 0xe6, 0x57, 0x50, 0x96, 0xd7, 0x89, 0x9e, 0x80, 0xee, 0x5a, - 0x07, 0x3d, 0x7c, 0x7a, 0xe2, 0x0e, 0x7b, 0xfb, 0x56, 0xdf, 0xea, 0x1d, 0xe8, 0x25, 0x54, 0x83, - 0x8a, 0x42, 0xf7, 0x4e, 0xcf, 0x74, 0x0d, 0xd5, 0x61, 0x55, 0xad, 0xdc, 0xde, 0x60, 0xa0, 0x2f, - 0x99, 0x3f, 0x6b, 0x50, 0x9d, 0xeb, 0x1e, 0xda, 0x84, 0x75, 0xcf, 0x3a, 0xee, 0x61, 0xeb, 0x04, - 0xf7, 0x6d, 0x67, 0xff, 0xae, 0xd6, 0x53, 0x78, 0xbc, 0x18, 0xb6, 0xec, 0x7d, 0x5d, 0x43, 0x1b, - 0xf0, 0xfe, 0x22, 0x3c, 0xb4, 0x5d, 0x0f, 0xdb, 0x27, 0x83, 0x33, 0x7d, 0x09, 0x99, 0xd0, 0x5c, - 0x0c, 0xf6, 0xad, 0xc1, 0x00, 0xdb, 0x0e, 0x3e, 0xb2, 0x06, 0x03, 0x7d, 0xb9, 0xb9, 0x54, 0xd1, - 0xcc, 0x31, 0xd4, 0x17, 0xda, 0x85, 0x5a, 0xd0, 0xdc, 0xb7, 0x4f, 0x0e, 0x2c, 0xcf, 0xb2, 0x4f, - 0xb0, 0x77, 0x36, 0xbc, 0x9b, 0xc8, 0x07, 0x60, 0xdc, 0x89, 0xbb, 0x9e, 0x3d, 0xc4, 0x03, 0xdb, - 0x75, 0x75, 0xed, 0x2d, 0xbb, 0xbd, 0xe7, 0x47, 0x3d, 0x3c, 0x74, 0xec, 0xbe, 0xe5, 0xe9, 0x4b, - 0x7b, 0xfa, 0x9c, 0x73, 0x79, 0x4c, 0xf8, 0xb9, 0x49, 0x60, 0xed, 0x2d, 0x23, 0x8a, 0x3e, 0x84, - 0xda, 0xc2, 0xeb, 0xad, 0x29, 0x6f, 0x54, 0x47, 0xb7, 0xaf, 0x36, 0xfa, 0x14, 0x1e, 0x8b, 0xdb, - 0x9d, 0x73, 0xaf, 0x4b, 0xdd, 0xd1, 0xe7, 0x02, 0x6a, 0xc8, 0xf7, 0x86, 0xaf, 0xae, 0x5a, 0xda, - 0xeb, 0xab, 0x96, 0xf6, 0xe7, 0x55, 0x4b, 0xfb, 0xe5, 0xba, 0x55, 0x7a, 0x7d, 0xdd, 0x2a, 0xfd, - 0x7e, 0xdd, 0x2a, 0xbd, 0xf8, 0x32, 0xa2, 0xe2, 0x22, 0x1b, 0x75, 0x02, 0x3e, 0xee, 0x2e, 0xfc, - 0x14, 0xfc, 0xf4, 0xec, 0xb3, 0xe0, 0xc2, 0xa7, 0x71, 0x77, 0x86, 0x5c, 0xe6, 0x3f, 0x1c, 0xd2, - 0x84, 0xe9, 0xe8, 0xa1, 0x82, 0xbf, 0xf8, 0x27, 0x00, 0x00, 0xff, 0xff, 0x85, 0x48, 0x2a, 0xbd, - 0x92, 0x08, 0x00, 0x00, + // 1091 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xcc, 0x56, 0xcd, 0x6e, 0xdb, 0xc6, + 0x13, 0x17, 0x65, 0xd9, 0x96, 0x47, 0x1f, 0x61, 0x36, 0x1f, 0x7f, 0x45, 0xf9, 0x5b, 0x56, 0x85, + 0xc0, 0x75, 0x11, 0x54, 0x42, 0xdd, 0xa0, 0x40, 0x51, 0xf4, 0x10, 0xd9, 0x12, 0x4c, 0x58, 0x36, + 0x55, 0x92, 0x0e, 0xe0, 0xa0, 0xe8, 0x82, 0x22, 0x57, 0xf2, 0x22, 0x2b, 0x52, 0x21, 0x97, 0x85, + 0x75, 0xef, 0x03, 0xf4, 0x25, 0xfa, 0x16, 0x7d, 0x80, 0x1c, 0x73, 0xec, 0xa9, 0x68, 0xed, 0x67, + 0xe8, 0xa9, 0x97, 0x62, 0x97, 0xb4, 0x44, 0xfa, 0xa3, 0x45, 0x91, 0x4b, 0x6f, 0xdc, 0xdf, 0xfc, + 0xf6, 0xb7, 0x33, 0xb3, 0x33, 0xb3, 0x84, 0x4d, 0x77, 0xee, 0x9e, 0xcf, 0x02, 0x9f, 0xfb, 0x8e, + 0xcf, 0x3a, 0x0e, 0xf3, 0x47, 0x1d, 0x3f, 0x70, 0x49, 0xd0, 0x96, 0x18, 0xba, 0x9f, 0x36, 0xb7, + 0x85, 0xb9, 0xfe, 0x70, 0xe2, 0x4f, 0x7c, 0x09, 0x75, 0xc4, 0x57, 0x4c, 0xac, 0x7f, 0x92, 0xd1, + 0x09, 0xa3, 0x91, 0xed, 0x38, 0x7e, 0xe4, 0xf1, 0x30, 0xf5, 0x9d, 0x50, 0x9f, 0xdd, 0x3c, 0x92, + 0xd1, 0xb7, 0x11, 0x75, 0x6d, 0x4e, 0x7d, 0x2f, 0x8c, 0x59, 0xad, 0x9f, 0x15, 0x58, 0xd7, 0x85, + 0x27, 0x9a, 0x8b, 0xbe, 0x81, 0xca, 0x52, 0x05, 0x53, 0xb7, 0xa6, 0x34, 0x95, 0x9d, 0xd2, 0xee, + 0x76, 0x3b, 0xe3, 0x5d, 0xea, 0xd0, 0xb6, 0xb9, 0xf8, 0xd6, 0xdc, 0x6e, 0xe1, 0xdd, 0xaf, 0x5b, + 0x39, 0xa3, 0x1c, 0xa6, 0x30, 0xf4, 0x14, 0x36, 0x1c, 0x46, 0x49, 0x2c, 0x97, 0x6f, 0x2a, 0x3b, + 0xeb, 0x46, 0x31, 0x06, 0x34, 0x17, 0x6d, 0x41, 0x49, 0x26, 0x01, 0x8f, 0x99, 0x3d, 0x09, 0x6b, + 0x2b, 0x4d, 0x65, 0xa7, 0x62, 0x80, 0x84, 0xfa, 0x02, 0x41, 0x4d, 0x28, 0x0b, 0xbf, 0xf1, 0xcc, + 0xa6, 0x81, 0x10, 0x28, 0xc4, 0x0c, 0x81, 0x0d, 0x6d, 0x1a, 0x68, 0x6e, 0xeb, 0x3b, 0xd8, 0x94, + 0xde, 0x87, 0x7d, 0xca, 0x18, 0x71, 0xf7, 0xa3, 0x80, 0x7a, 0x93, 0x81, 0xcd, 0x49, 0xc8, 0xbb, + 0xcc, 0x77, 0xde, 0xa0, 0xaf, 0x61, 0x23, 0x3e, 0x83, 0xba, 0x61, 0x4d, 0x69, 0xae, 0xec, 0x94, + 0x76, 0xeb, 0xed, 0x1b, 0xd9, 0x6e, 0x27, 0x29, 0x48, 0x62, 0x28, 0xfa, 0xf1, 0x32, 0x6c, 0xbd, + 0x86, 0x27, 0x43, 0x9f, 0x13, 0x8f, 0x53, 0x9b, 0xb1, 0xf9, 0x30, 0x88, 0x3c, 0x7b, 0xc4, 0x48, + 0x7c, 0xe4, 0x87, 0x6a, 0x13, 0xa8, 0x4a, 0x93, 0x70, 0xdd, 0xe4, 0x36, 0x27, 0x22, 0x21, 0x63, + 0xca, 0x18, 0xb6, 0xa7, 0x22, 0x7d, 0x32, 0xfd, 0x05, 0x03, 0x04, 0xf4, 0x52, 0x22, 0x68, 0x17, + 0x1e, 0xcd, 0x12, 0x1f, 0xf0, 0x48, 0xc4, 0x87, 0xcf, 0x08, 0x9d, 0x9c, 0x71, 0x99, 0xda, 0x8a, + 0xf1, 0xe0, 0xca, 0x28, 0x63, 0x3f, 0x90, 0xa6, 0xd6, 0xb7, 0xf0, 0x54, 0xaa, 0x8f, 0x23, 0x26, + 0x8f, 0xb3, 0xe8, 0x94, 0x98, 0x8c, 0x3a, 0xe4, 0x95, 0xcd, 0x22, 0xf2, 0xa1, 0x41, 0xfc, 0xa4, + 0xc0, 0xe3, 0x81, 0xef, 0x4d, 0x2c, 0x12, 0x4c, 0x25, 0x67, 0xc8, 0x6c, 0x87, 0x4c, 0x89, 0xc7, + 0xd1, 0x0b, 0x58, 0x95, 0xb4, 0xa4, 0x8c, 0x6a, 0x77, 0xa9, 0x26, 0x9a, 0x31, 0x19, 0x9d, 0xc0, + 0xbd, 0xd9, 0x95, 0x04, 0xa6, 0x9e, 0x4b, 0xce, 0x65, 0x70, 0x37, 0xca, 0x50, 0xee, 0xb7, 0x02, + 0xdb, 0x0b, 0x6d, 0x47, 0x14, 0xb4, 0x94, 0xa2, 0xde, 0x24, 0x51, 0xab, 0x2e, 0x44, 0x34, 0xa1, + 0xd1, 0xfa, 0x43, 0x81, 0x27, 0x7b, 0xbe, 0xe7, 0x52, 0xc1, 0xb5, 0xd9, 0x7f, 0xd8, 0x55, 0x74, + 0x08, 0x15, 0x1e, 0xd0, 0xc9, 0x44, 0xdc, 0x89, 0x14, 0x5d, 0xf9, 0x37, 0xa2, 0x46, 0x39, 0xd9, + 0x1c, 0xc7, 0xfd, 0xe7, 0x1a, 0xac, 0x4a, 0x13, 0xfa, 0x0a, 0x8a, 0x57, 0x17, 0x9d, 0x84, 0xf9, + 0xcf, 0xf7, 0xbc, 0x9e, 0xdc, 0x33, 0xfa, 0x0c, 0x0a, 0x21, 0x75, 0x89, 0x8c, 0xaf, 0xba, 0xbb, + 0x79, 0xd7, 0xc6, 0xb6, 0x49, 0x5d, 0x62, 0x48, 0x2a, 0xaa, 0x43, 0xf1, 0x6d, 0x64, 0x7b, 0x3c, + 0x9a, 0xc6, 0xad, 0x5d, 0x30, 0x16, 0x6b, 0x61, 0x0b, 0xa3, 0x11, 0xa7, 0xce, 0x9b, 0x50, 0x36, + 0x75, 0xc1, 0x58, 0xac, 0xd1, 0x36, 0x54, 0x27, 0xbe, 0xef, 0x62, 0x4e, 0x59, 0x5c, 0xe3, 0xb5, + 0x55, 0x51, 0xdc, 0x07, 0x39, 0xa3, 0x2c, 0x70, 0x8b, 0xb2, 0xb8, 0xb3, 0x3b, 0xf0, 0x20, 0xcb, + 0xc3, 0x9c, 0x4e, 0x49, 0x6d, 0x4d, 0x0c, 0x99, 0x83, 0x9c, 0xa1, 0xa6, 0xc9, 0xa2, 0xe6, 0xd1, + 0x01, 0x54, 0x04, 0x03, 0x53, 0x0f, 0x8f, 0xfd, 0xc0, 0x21, 0xb5, 0x75, 0x19, 0xcc, 0xb3, 0x3b, + 0x83, 0x11, 0xbb, 0x34, 0xaf, 0x2f, 0xb8, 0x46, 0x89, 0x2f, 0x17, 0xa2, 0x4f, 0x03, 0xe2, 0x46, + 0x0e, 0xc1, 0xbe, 0xc7, 0xe6, 0xb5, 0x62, 0x53, 0xd9, 0x29, 0x1a, 0x10, 0x43, 0xba, 0xc7, 0xe6, + 0xe8, 0x63, 0xb8, 0x97, 0x8c, 0xbd, 0x29, 0xe1, 0xb6, 0x6b, 0x73, 0xbb, 0xb6, 0x21, 0x3b, 0xb4, + 0x1a, 0xc3, 0x47, 0x09, 0x8a, 0x8e, 0xa0, 0xea, 0x5c, 0x55, 0x25, 0xe6, 0xf3, 0x19, 0xa9, 0x81, + 0x74, 0x6a, 0xfb, 0x4e, 0xa7, 0x16, 0x45, 0x6c, 0xcd, 0x67, 0xc4, 0xa8, 0x38, 0xe9, 0x25, 0x3a, + 0x84, 0x96, 0xb3, 0x2c, 0x72, 0x1c, 0xdf, 0xf7, 0x55, 0x31, 0x2d, 0x32, 0x5e, 0x92, 0x19, 0xdf, + 0x72, 0xae, 0xb5, 0x83, 0x15, 0xf3, 0xcc, 0x84, 0xd6, 0xfa, 0x12, 0x0a, 0xe2, 0x3a, 0xd1, 0x43, + 0x50, 0x4d, 0x6d, 0xbf, 0x87, 0x4f, 0x8e, 0xcd, 0x61, 0x6f, 0x4f, 0xeb, 0x6b, 0xbd, 0x7d, 0x35, + 0x87, 0xca, 0x50, 0x94, 0x68, 0xf7, 0xe4, 0x54, 0x55, 0x50, 0x05, 0x36, 0xe4, 0xca, 0xec, 0x0d, + 0x06, 0x6a, 0xbe, 0xf5, 0x83, 0x02, 0xa5, 0x54, 0xf6, 0xd0, 0x26, 0x3c, 0xb1, 0xb4, 0xa3, 0x1e, + 0xd6, 0x8e, 0x71, 0x5f, 0x37, 0xf6, 0xae, 0x6b, 0x3d, 0x82, 0xfb, 0x59, 0xb3, 0xa6, 0xef, 0xa9, + 0x0a, 0x7a, 0x0a, 0xff, 0xcb, 0xc2, 0x43, 0xdd, 0xb4, 0xb0, 0x7e, 0x3c, 0x38, 0x55, 0xf3, 0xa8, + 0x05, 0xf5, 0xac, 0xb1, 0xaf, 0x0d, 0x06, 0x58, 0x37, 0xf0, 0xa1, 0x36, 0x18, 0xa8, 0x2b, 0xf5, + 0x7c, 0x51, 0x69, 0x4d, 0xa1, 0x92, 0x49, 0x17, 0x6a, 0x40, 0x7d, 0x4f, 0x3f, 0xde, 0xd7, 0x2c, + 0x4d, 0x3f, 0xc6, 0xd6, 0xe9, 0xf0, 0xba, 0x23, 0xff, 0x87, 0xda, 0x35, 0xbb, 0x69, 0xe9, 0x43, + 0x3c, 0xd0, 0x4d, 0x53, 0x55, 0x6e, 0xd9, 0x6d, 0xbd, 0x3c, 0xec, 0xe1, 0xa1, 0xa1, 0xf7, 0x35, + 0x4b, 0xcd, 0x77, 0xd5, 0x54, 0xe5, 0xfa, 0x1e, 0xf1, 0xc7, 0x2d, 0x02, 0x0f, 0x6e, 0x69, 0x51, + 0xf4, 0x11, 0x94, 0x33, 0xd3, 0x5b, 0x91, 0xb5, 0x51, 0x1a, 0x2d, 0xa7, 0x36, 0x7a, 0x0e, 0xf7, + 0xf9, 0x72, 0x67, 0x6a, 0xba, 0x54, 0x0c, 0x35, 0x65, 0x88, 0x9b, 0xfc, 0x77, 0x05, 0x1e, 0x9b, + 0x3c, 0x20, 0xf6, 0x74, 0xb0, 0x7c, 0xe1, 0xe3, 0xae, 0x7f, 0x05, 0x6a, 0xea, 0xd5, 0xc7, 0xd4, + 0x1b, 0xfb, 0x49, 0xf7, 0x3f, 0xbf, 0xa5, 0xc4, 0x86, 0x24, 0x98, 0x11, 0x1e, 0xd9, 0x2c, 0xa5, + 0xa3, 0x79, 0x63, 0xdf, 0xb8, 0xc7, 0xb2, 0xc0, 0x8d, 0xa7, 0x39, 0x7f, 0xfd, 0x69, 0x46, 0x8f, + 0x60, 0x8d, 0x86, 0x78, 0x14, 0xcd, 0x65, 0xf7, 0x17, 0x8d, 0x55, 0x1a, 0x76, 0xa3, 0x79, 0x66, + 0x2c, 0x14, 0xfe, 0x66, 0x2c, 0xac, 0x66, 0xc7, 0x42, 0x77, 0xf8, 0xee, 0xa2, 0xa1, 0xbc, 0xbf, + 0x68, 0x28, 0xbf, 0x5d, 0x34, 0x94, 0x1f, 0x2f, 0x1b, 0xb9, 0xf7, 0x97, 0x8d, 0xdc, 0x2f, 0x97, + 0x8d, 0xdc, 0xeb, 0x2f, 0x26, 0x94, 0x9f, 0x45, 0xa3, 0xb6, 0xe3, 0x4f, 0x3b, 0x99, 0x7f, 0x9e, + 0xef, 0x5f, 0x7c, 0xea, 0x9c, 0xd9, 0xd4, 0xeb, 0x2c, 0x90, 0xf3, 0xf8, 0x3f, 0x48, 0x34, 0x5a, + 0x38, 0x5a, 0x93, 0xf0, 0xe7, 0x7f, 0x05, 0x00, 0x00, 0xff, 0xff, 0x9c, 0x15, 0xc3, 0x1c, 0x9c, + 0x09, 0x00, 0x00, } func (m *OrderId) Marshal() (dAtA []byte, err error) { @@ -1327,6 +1417,66 @@ func (m *TransactionOrdering) MarshalToSizedBuffer(dAtA []byte) (int, error) { return len(dAtA) - i, nil } +func (m *StreamLiquidationOrder) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *StreamLiquidationOrder) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *StreamLiquidationOrder) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.Subticks != 0 { + i = encodeVarintOrder(dAtA, i, uint64(m.Subticks)) + i-- + dAtA[i] = 0x28 + } + if m.Quantums != 0 { + i = encodeVarintOrder(dAtA, i, uint64(m.Quantums)) + i-- + dAtA[i] = 0x20 + } + if m.IsBuy { + i-- + if m.IsBuy { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i-- + dAtA[i] = 0x18 + } + if m.ClobPairId != 0 { + i = encodeVarintOrder(dAtA, i, uint64(m.ClobPairId)) + i-- + dAtA[i] = 0x10 + } + if m.LiquidationInfo != nil { + { + size, err := m.LiquidationInfo.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintOrder(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + func encodeVarintOrder(dAtA []byte, offset int, v uint64) int { offset -= sovOrder(v) base := offset @@ -1519,6 +1669,31 @@ func (m *TransactionOrdering) Size() (n int) { return n } +func (m *StreamLiquidationOrder) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.LiquidationInfo != nil { + l = m.LiquidationInfo.Size() + n += 1 + l + sovOrder(uint64(l)) + } + if m.ClobPairId != 0 { + n += 1 + sovOrder(uint64(m.ClobPairId)) + } + if m.IsBuy { + n += 2 + } + if m.Quantums != 0 { + n += 1 + sovOrder(uint64(m.Quantums)) + } + if m.Subticks != 0 { + n += 1 + sovOrder(uint64(m.Subticks)) + } + return n +} + func sovOrder(x uint64) (n int) { return (math_bits.Len64(x|1) + 6) / 7 } @@ -2619,6 +2794,169 @@ func (m *TransactionOrdering) Unmarshal(dAtA []byte) error { } return nil } +func (m *StreamLiquidationOrder) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowOrder + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: StreamLiquidationOrder: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: StreamLiquidationOrder: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field LiquidationInfo", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowOrder + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthOrder + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthOrder + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.LiquidationInfo == nil { + m.LiquidationInfo = &PerpetualLiquidationInfo{} + } + if err := m.LiquidationInfo.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field ClobPairId", wireType) + } + m.ClobPairId = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowOrder + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.ClobPairId |= uint32(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field IsBuy", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowOrder + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.IsBuy = bool(v != 0) + case 4: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Quantums", wireType) + } + m.Quantums = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowOrder + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Quantums |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 5: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Subticks", wireType) + } + m.Subticks = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowOrder + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Subticks |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := skipOrder(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthOrder + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func skipOrder(dAtA []byte) (n int, err error) { l := len(dAtA) iNdEx := 0 diff --git a/protocol/x/clob/types/orderbook.go b/protocol/x/clob/types/orderbook.go index 7650c2869b..ef4cf65e93 100644 --- a/protocol/x/clob/types/orderbook.go +++ b/protocol/x/clob/types/orderbook.go @@ -102,6 +102,16 @@ type TakerOrderStatus struct { OrderOptimisticallyFilledQuantums satypes.BaseQuantums } +// ToStreamingTakerOrderStatus converts the TakerOrderStatus to a StreamTakerOrderStatus +// to be emitted by full node streaming. +func (tos *TakerOrderStatus) ToStreamingTakerOrderStatus() *StreamTakerOrderStatus { + return &StreamTakerOrderStatus{ + OrderStatus: uint32(tos.OrderStatus), + RemainingQuantums: tos.RemainingQuantums.ToUint64(), + OptimisticallyFilledQuantums: tos.OrderOptimisticallyFilledQuantums.ToUint64(), + } +} + // OrderStatus represents the status of an order after attempting to place it on the orderbook. type OrderStatus uint diff --git a/protocol/x/clob/types/query.pb.go b/protocol/x/clob/types/query.pb.go index b4c4596eae..1d66576185 100644 --- a/protocol/x/clob/types/query.pb.go +++ b/protocol/x/clob/types/query.pb.go @@ -1095,6 +1095,179 @@ func (m *StreamOrderbookFill) GetFillAmounts() []uint64 { return nil } +// StreamTakerOrder provides information on a taker order that was attempted +// to be matched on the orderbook. +// It is intended to be used only in full node streaming. +type StreamTakerOrder struct { + // The taker order that was matched on the orderbook. Can be a + // regular order or a liquidation order. + // + // Types that are valid to be assigned to TakerOrder: + // + // *StreamTakerOrder_Order + // *StreamTakerOrder_LiquidationOrder + TakerOrder isStreamTakerOrder_TakerOrder `protobuf_oneof:"taker_order"` + // Information on the taker order after it is matched on the book, + // either successfully or unsuccessfully. + TakerOrderStatus *StreamTakerOrderStatus `protobuf:"bytes,3,opt,name=taker_order_status,json=takerOrderStatus,proto3" json:"taker_order_status,omitempty"` +} + +func (m *StreamTakerOrder) Reset() { *m = StreamTakerOrder{} } +func (m *StreamTakerOrder) String() string { return proto.CompactTextString(m) } +func (*StreamTakerOrder) ProtoMessage() {} +func (*StreamTakerOrder) Descriptor() ([]byte, []int) { + return fileDescriptor_3365c195b25c5bc0, []int{19} +} +func (m *StreamTakerOrder) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *StreamTakerOrder) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_StreamTakerOrder.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *StreamTakerOrder) XXX_Merge(src proto.Message) { + xxx_messageInfo_StreamTakerOrder.Merge(m, src) +} +func (m *StreamTakerOrder) XXX_Size() int { + return m.Size() +} +func (m *StreamTakerOrder) XXX_DiscardUnknown() { + xxx_messageInfo_StreamTakerOrder.DiscardUnknown(m) +} + +var xxx_messageInfo_StreamTakerOrder proto.InternalMessageInfo + +type isStreamTakerOrder_TakerOrder interface { + isStreamTakerOrder_TakerOrder() + MarshalTo([]byte) (int, error) + Size() int +} + +type StreamTakerOrder_Order struct { + Order *Order `protobuf:"bytes,1,opt,name=order,proto3,oneof" json:"order,omitempty"` +} +type StreamTakerOrder_LiquidationOrder struct { + LiquidationOrder *StreamLiquidationOrder `protobuf:"bytes,2,opt,name=liquidation_order,json=liquidationOrder,proto3,oneof" json:"liquidation_order,omitempty"` +} + +func (*StreamTakerOrder_Order) isStreamTakerOrder_TakerOrder() {} +func (*StreamTakerOrder_LiquidationOrder) isStreamTakerOrder_TakerOrder() {} + +func (m *StreamTakerOrder) GetTakerOrder() isStreamTakerOrder_TakerOrder { + if m != nil { + return m.TakerOrder + } + return nil +} + +func (m *StreamTakerOrder) GetOrder() *Order { + if x, ok := m.GetTakerOrder().(*StreamTakerOrder_Order); ok { + return x.Order + } + return nil +} + +func (m *StreamTakerOrder) GetLiquidationOrder() *StreamLiquidationOrder { + if x, ok := m.GetTakerOrder().(*StreamTakerOrder_LiquidationOrder); ok { + return x.LiquidationOrder + } + return nil +} + +func (m *StreamTakerOrder) GetTakerOrderStatus() *StreamTakerOrderStatus { + if m != nil { + return m.TakerOrderStatus + } + return nil +} + +// XXX_OneofWrappers is for the internal use of the proto package. +func (*StreamTakerOrder) XXX_OneofWrappers() []interface{} { + return []interface{}{ + (*StreamTakerOrder_Order)(nil), + (*StreamTakerOrder_LiquidationOrder)(nil), + } +} + +// StreamTakerOrderStatus is a representation of a taker order +// after it is attempted to be matched on the orderbook. +// It is intended to be used only in full node streaming. +type StreamTakerOrderStatus struct { + // The state of the taker order after attempting to match it against the orderbook. + // Possible enum values can be found here: + // https://github.com/dydxprotocol/v4-chain/blob/main/protocol/x/clob/types/orderbook.go#L105 + OrderStatus uint32 `protobuf:"varint,1,opt,name=order_status,json=orderStatus,proto3" json:"order_status,omitempty"` + // The amount of remaining (non-matched) base quantums of this taker order. + RemainingQuantums uint64 `protobuf:"varint,2,opt,name=remaining_quantums,json=remainingQuantums,proto3" json:"remaining_quantums,omitempty"` + // The amount of base quantums that were *optimistically* filled for this taker order + // when the order is matched against the orderbook. Note that if any quantums of this order + // were optimistically filled or filled in state before this invocation of the matching loop, + // this value will not include them. + OptimisticallyFilledQuantums uint64 `protobuf:"varint,3,opt,name=optimistically_filled_quantums,json=optimisticallyFilledQuantums,proto3" json:"optimistically_filled_quantums,omitempty"` +} + +func (m *StreamTakerOrderStatus) Reset() { *m = StreamTakerOrderStatus{} } +func (m *StreamTakerOrderStatus) String() string { return proto.CompactTextString(m) } +func (*StreamTakerOrderStatus) ProtoMessage() {} +func (*StreamTakerOrderStatus) Descriptor() ([]byte, []int) { + return fileDescriptor_3365c195b25c5bc0, []int{20} +} +func (m *StreamTakerOrderStatus) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *StreamTakerOrderStatus) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_StreamTakerOrderStatus.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *StreamTakerOrderStatus) XXX_Merge(src proto.Message) { + xxx_messageInfo_StreamTakerOrderStatus.Merge(m, src) +} +func (m *StreamTakerOrderStatus) XXX_Size() int { + return m.Size() +} +func (m *StreamTakerOrderStatus) XXX_DiscardUnknown() { + xxx_messageInfo_StreamTakerOrderStatus.DiscardUnknown(m) +} + +var xxx_messageInfo_StreamTakerOrderStatus proto.InternalMessageInfo + +func (m *StreamTakerOrderStatus) GetOrderStatus() uint32 { + if m != nil { + return m.OrderStatus + } + return 0 +} + +func (m *StreamTakerOrderStatus) GetRemainingQuantums() uint64 { + if m != nil { + return m.RemainingQuantums + } + return 0 +} + +func (m *StreamTakerOrderStatus) GetOptimisticallyFilledQuantums() uint64 { + if m != nil { + return m.OptimisticallyFilledQuantums + } + return 0 +} + func init() { proto.RegisterType((*QueryGetClobPairRequest)(nil), "dydxprotocol.clob.QueryGetClobPairRequest") proto.RegisterType((*QueryClobPairResponse)(nil), "dydxprotocol.clob.QueryClobPairResponse") @@ -1116,100 +1289,112 @@ func init() { proto.RegisterType((*StreamUpdate)(nil), "dydxprotocol.clob.StreamUpdate") proto.RegisterType((*StreamOrderbookUpdate)(nil), "dydxprotocol.clob.StreamOrderbookUpdate") proto.RegisterType((*StreamOrderbookFill)(nil), "dydxprotocol.clob.StreamOrderbookFill") + proto.RegisterType((*StreamTakerOrder)(nil), "dydxprotocol.clob.StreamTakerOrder") + proto.RegisterType((*StreamTakerOrderStatus)(nil), "dydxprotocol.clob.StreamTakerOrderStatus") } func init() { proto.RegisterFile("dydxprotocol/clob/query.proto", fileDescriptor_3365c195b25c5bc0) } var fileDescriptor_3365c195b25c5bc0 = []byte{ - // 1407 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x57, 0xcf, 0x6f, 0xdc, 0xc4, - 0x17, 0x5f, 0x27, 0xf9, 0xb6, 0x9b, 0x97, 0xfe, 0xfa, 0x4e, 0x9a, 0x76, 0xeb, 0xa4, 0x9b, 0xd4, - 0xd0, 0x74, 0x93, 0xd2, 0x75, 0x93, 0x56, 0x55, 0x69, 0x50, 0x51, 0x12, 0xd1, 0x1f, 0x52, 0x43, - 0x83, 0xfb, 0x83, 0x0a, 0x2a, 0x59, 0xb3, 0xf6, 0xac, 0x63, 0xd5, 0xf6, 0x6c, 0xec, 0xd9, 0x55, - 0x22, 0x84, 0x40, 0x1c, 0xb8, 0x00, 0x12, 0x12, 0x07, 0x0e, 0x48, 0x5c, 0x38, 0x73, 0xe4, 0x88, - 0x80, 0x5b, 0x8f, 0x95, 0xb8, 0x70, 0x40, 0x08, 0xb5, 0x9c, 0xf9, 0x1b, 0x90, 0x67, 0xc6, 0x9b, - 0x75, 0x6c, 0xef, 0x26, 0xb9, 0xec, 0xda, 0x6f, 0xde, 0x7b, 0xf3, 0x79, 0xef, 0x7d, 0x66, 0xde, - 0x33, 0x9c, 0xb5, 0xb7, 0xed, 0xad, 0x56, 0x48, 0x19, 0xb5, 0xa8, 0xa7, 0x5b, 0x1e, 0x6d, 0xe8, - 0x9b, 0x6d, 0x12, 0x6e, 0xd7, 0xb9, 0x0c, 0xfd, 0xbf, 0x77, 0xb9, 0x1e, 0x2f, 0xab, 0x27, 0x1d, - 0xea, 0x50, 0x2e, 0xd2, 0xe3, 0x27, 0xa1, 0xa8, 0x4e, 0x39, 0x94, 0x3a, 0x1e, 0xd1, 0x71, 0xcb, - 0xd5, 0x71, 0x10, 0x50, 0x86, 0x99, 0x4b, 0x83, 0x48, 0xae, 0xce, 0x5b, 0x34, 0xf2, 0x69, 0xa4, - 0x37, 0x70, 0x44, 0x84, 0x7f, 0xbd, 0xb3, 0xd0, 0x20, 0x0c, 0x2f, 0xe8, 0x2d, 0xec, 0xb8, 0x01, - 0x57, 0x96, 0xba, 0x7a, 0x16, 0x51, 0xc3, 0xa3, 0xd6, 0x33, 0x33, 0xc4, 0x8c, 0x98, 0x9e, 0xeb, - 0xbb, 0xcc, 0xb4, 0x68, 0xd0, 0x74, 0x1d, 0x69, 0x70, 0x2e, 0x6b, 0x10, 0xff, 0x98, 0x2d, 0xec, - 0x86, 0x52, 0xe5, 0x72, 0x56, 0x85, 0x6c, 0xb6, 0x5d, 0xb6, 0x6d, 0x32, 0x97, 0x84, 0x79, 0x4e, - 0x73, 0xf2, 0x42, 0x43, 0x9b, 0x24, 0x0e, 0xa7, 0xb3, 0xcb, 0x3e, 0x66, 0xd6, 0x06, 0x49, 0x22, - 0xbe, 0x98, 0x55, 0xf0, 0xdc, 0xcd, 0xb6, 0x6b, 0x8b, 0xbc, 0xa4, 0x37, 0x9b, 0xcc, 0xf1, 0x46, - 0x3a, 0x72, 0xf1, 0x66, 0x6a, 0xd1, 0x0d, 0x6c, 0xb2, 0x45, 0x42, 0x9d, 0x36, 0x9b, 0xa6, 0xb5, - 0x81, 0xdd, 0xc0, 0x6c, 0xb7, 0x6c, 0xcc, 0x48, 0x94, 0x95, 0x08, 0x7b, 0x6d, 0x0e, 0x4e, 0xbf, - 0x17, 0x67, 0xfc, 0x36, 0x61, 0xab, 0x1e, 0x6d, 0xac, 0x63, 0x37, 0x34, 0xc8, 0x66, 0x9b, 0x44, - 0x0c, 0x1d, 0x83, 0x21, 0xd7, 0xae, 0x28, 0x33, 0x4a, 0xed, 0xa8, 0x31, 0xe4, 0xda, 0xda, 0xfb, - 0x30, 0xc1, 0x55, 0x77, 0xf4, 0xa2, 0x16, 0x0d, 0x22, 0x82, 0x6e, 0xc2, 0x68, 0x37, 0xa5, 0x5c, - 0x7f, 0x6c, 0x71, 0xb2, 0x9e, 0xa1, 0x46, 0x3d, 0xb1, 0x5b, 0x19, 0x79, 0xfe, 0xd7, 0x74, 0xc9, - 0x28, 0x5b, 0xf2, 0x5d, 0xc3, 0x12, 0xc3, 0xb2, 0xe7, 0xed, 0xc6, 0x70, 0x0b, 0x60, 0x87, 0x02, - 0xd2, 0xf7, 0x6c, 0x5d, 0xf0, 0xa5, 0x1e, 0xf3, 0xa5, 0x2e, 0xf8, 0x28, 0xf9, 0x52, 0x5f, 0xc7, - 0x0e, 0x91, 0xb6, 0x46, 0x8f, 0xa5, 0xf6, 0x83, 0x02, 0x95, 0x14, 0xf8, 0x65, 0xcf, 0x2b, 0xc2, - 0x3f, 0xbc, 0x4f, 0xfc, 0xe8, 0x76, 0x0a, 0xe4, 0x10, 0x07, 0x79, 0x61, 0x20, 0x48, 0xb1, 0x79, - 0x0a, 0xe5, 0x9f, 0x0a, 0x4c, 0xaf, 0x91, 0xce, 0xbb, 0xd4, 0x26, 0x0f, 0x69, 0xfc, 0xbb, 0x8a, - 0x3d, 0xab, 0xed, 0xf1, 0xc5, 0x24, 0x23, 0x4f, 0xe1, 0x94, 0x20, 0x7c, 0x2b, 0xa4, 0x2d, 0x1a, - 0x91, 0xd0, 0x94, 0xd4, 0xea, 0x66, 0x27, 0x8b, 0xfc, 0x31, 0xf6, 0x62, 0x6a, 0xd1, 0x70, 0x8d, - 0x74, 0xd6, 0x84, 0xb6, 0x71, 0x92, 0x7b, 0x59, 0x97, 0x4e, 0xa4, 0x14, 0x7d, 0x08, 0x13, 0x9d, - 0x44, 0xd9, 0xf4, 0x49, 0xc7, 0xf4, 0x09, 0x0b, 0x5d, 0x2b, 0xea, 0x46, 0x95, 0x75, 0x9e, 0x02, - 0xbc, 0x26, 0xd4, 0x8d, 0xf1, 0x4e, 0xef, 0x96, 0x42, 0xa8, 0xfd, 0xab, 0xc0, 0x4c, 0x71, 0x78, - 0xb2, 0x18, 0x0e, 0x1c, 0x0e, 0x49, 0xd4, 0xf6, 0x58, 0x24, 0x4b, 0x71, 0x7b, 0xd0, 0x9e, 0x39, - 0x5e, 0x62, 0x85, 0xe5, 0xc0, 0x7e, 0x4c, 0xbd, 0xb6, 0x4f, 0xd6, 0x49, 0x18, 0x97, 0x4e, 0x96, - 0x2d, 0xf1, 0xae, 0x62, 0x18, 0xcf, 0xd1, 0x42, 0x33, 0x70, 0xa4, 0x4b, 0x06, 0xb3, 0xcb, 0x7f, - 0x48, 0x8a, 0x7d, 0xd7, 0x46, 0x27, 0x60, 0xd8, 0x27, 0x1d, 0x9e, 0x91, 0x21, 0x23, 0x7e, 0x44, - 0xa7, 0xe0, 0x50, 0x87, 0x3b, 0xa9, 0x0c, 0xcf, 0x28, 0xb5, 0x11, 0x43, 0xbe, 0x69, 0xf3, 0x50, - 0xe3, 0xa4, 0x7b, 0x87, 0xdf, 0x26, 0x0f, 0x5d, 0x12, 0xde, 0x8b, 0xef, 0x92, 0x55, 0x7e, 0xba, - 0xdb, 0x61, 0x6f, 0x5d, 0xb5, 0xef, 0x14, 0x98, 0xdb, 0x83, 0xb2, 0xcc, 0x52, 0x00, 0x95, 0xa2, - 0x2b, 0x4a, 0xf2, 0x40, 0xcf, 0x49, 0x5b, 0x3f, 0xd7, 0x32, 0x3d, 0x13, 0x24, 0x4f, 0x47, 0x9b, - 0x83, 0x0b, 0x1c, 0xdc, 0x4a, 0x4c, 0x1a, 0x03, 0x33, 0x52, 0x1c, 0xc8, 0xb7, 0x8a, 0x8c, 0xba, - 0xaf, 0xae, 0x8c, 0xe3, 0x19, 0x9c, 0x2e, 0xb8, 0xbe, 0x65, 0x18, 0xf5, 0x9c, 0x30, 0xfa, 0x38, - 0x96, 0x51, 0x08, 0x72, 0xef, 0x52, 0xd1, 0x9e, 0xc0, 0x19, 0x0e, 0xec, 0x01, 0xc3, 0x8c, 0x34, - 0xdb, 0xde, 0xfd, 0xf8, 0xca, 0x4e, 0xce, 0xd5, 0x12, 0x94, 0xf9, 0x15, 0x9e, 0xd4, 0x7c, 0x6c, - 0x51, 0xcd, 0xd9, 0x9a, 0x9b, 0xdc, 0xb5, 0x13, 0x2e, 0x51, 0xf1, 0xaa, 0xfd, 0xa4, 0x80, 0x9a, - 0xe7, 0x5a, 0x46, 0xf9, 0x04, 0x8e, 0x0b, 0xdf, 0x2d, 0x0f, 0x5b, 0xc4, 0x27, 0x01, 0x93, 0x5b, - 0xcc, 0xe5, 0x6c, 0x71, 0x8f, 0x06, 0xce, 0x43, 0x12, 0xfa, 0xdc, 0xc5, 0x7a, 0x62, 0x20, 0x77, - 0x3c, 0x46, 0x53, 0x52, 0x34, 0x0d, 0x63, 0x4d, 0xd7, 0xf3, 0x4c, 0xec, 0xd3, 0x76, 0xc0, 0x38, - 0x27, 0x47, 0x0c, 0x88, 0x45, 0xcb, 0x5c, 0x82, 0xa6, 0x60, 0x94, 0x85, 0xae, 0xe3, 0x90, 0x90, - 0xd8, 0x9c, 0x9d, 0x65, 0x63, 0x47, 0xa0, 0x5d, 0x80, 0xf3, 0x1c, 0xf6, 0xbd, 0x9e, 0xe6, 0x93, - 0x5b, 0xd4, 0xcf, 0x15, 0x98, 0x1d, 0xa4, 0x29, 0x83, 0x7d, 0x0a, 0xe3, 0x39, 0xbd, 0x4c, 0x06, - 0x7c, 0x3e, 0x2f, 0xe0, 0x8c, 0x4b, 0x19, 0x2c, 0xf2, 0x32, 0x2b, 0xda, 0x32, 0x9c, 0x7d, 0xc0, - 0x42, 0x82, 0x45, 0x7a, 0x1a, 0x94, 0x3e, 0x7b, 0x24, 0xfa, 0x59, 0x52, 0xc7, 0xec, 0xf9, 0x1d, - 0x4e, 0x9f, 0x5f, 0x0d, 0x43, 0xb5, 0xc8, 0x85, 0x0c, 0xe1, 0x6d, 0x38, 0x2c, 0xbb, 0xa4, 0xbc, - 0x83, 0xa6, 0x73, 0x60, 0x0b, 0x1f, 0xc2, 0x34, 0xe1, 0x83, 0xb4, 0xd2, 0x3e, 0x1d, 0x82, 0x23, - 0xbd, 0xeb, 0xe8, 0x11, 0x9c, 0xa0, 0xc9, 0x6e, 0xb2, 0x03, 0xcb, 0x8c, 0xd4, 0x0a, 0x5d, 0xef, - 0x82, 0x77, 0xa7, 0x64, 0x1c, 0xa7, 0x69, 0x51, 0xdc, 0x79, 0x04, 0xb1, 0xe2, 0x8a, 0xcb, 0x3b, - 0x7a, 0x76, 0xb0, 0xc3, 0x5b, 0xae, 0xe7, 0xdd, 0x29, 0x19, 0xa3, 0xdc, 0x36, 0x7e, 0x41, 0xe7, - 0xe0, 0x88, 0x38, 0x87, 0x1b, 0xc4, 0x75, 0x36, 0x18, 0x67, 0xca, 0x51, 0x63, 0x8c, 0xcb, 0xee, - 0x70, 0x11, 0x9a, 0x84, 0x51, 0xb2, 0x45, 0x2c, 0xd3, 0xa7, 0x36, 0xa9, 0x8c, 0xf0, 0xf5, 0x72, - 0x2c, 0x58, 0xa3, 0x36, 0x59, 0x39, 0x01, 0xc7, 0x44, 0x54, 0xa6, 0x4f, 0xa2, 0x08, 0x3b, 0x44, - 0xfb, 0x4a, 0x81, 0x89, 0xdc, 0x38, 0xd0, 0x93, 0xdd, 0xd9, 0xbd, 0x9e, 0x46, 0x2c, 0x87, 0x98, - 0x7a, 0x76, 0x64, 0xb9, 0xdf, 0x6c, 0xae, 0xc6, 0x02, 0xe1, 0xe8, 0xf1, 0xc2, 0xae, 0xb4, 0x23, - 0x15, 0xca, 0x51, 0x80, 0x5b, 0xd1, 0x06, 0x15, 0x47, 0xa1, 0x6c, 0x74, 0xdf, 0xb5, 0x1f, 0x15, - 0x18, 0xcf, 0x49, 0x03, 0x5a, 0x02, 0xce, 0x0d, 0xd1, 0x45, 0x65, 0x4d, 0xa6, 0x0a, 0xba, 0x3f, - 0xef, 0x92, 0x06, 0x1f, 0x16, 0xf8, 0x23, 0xba, 0x06, 0x87, 0x78, 0x0e, 0xe3, 0xfe, 0x18, 0x47, - 0x52, 0x29, 0xba, 0x32, 0x24, 0x52, 0xa9, 0x1d, 0xa7, 0xbb, 0xe7, 0xd8, 0x46, 0x95, 0xe1, 0x99, - 0xe1, 0xda, 0x88, 0x31, 0xb6, 0x73, 0x6e, 0xa3, 0xc5, 0xef, 0x01, 0xfe, 0xc7, 0x4f, 0x1c, 0xfa, - 0x42, 0x81, 0x72, 0x32, 0x7b, 0xa0, 0xf9, 0x9c, 0x1d, 0x0a, 0x06, 0x38, 0xb5, 0x56, 0xa4, 0xbb, - 0x7b, 0x82, 0xd3, 0xe6, 0x3e, 0xfb, 0xfd, 0x9f, 0x6f, 0x86, 0x5e, 0x43, 0xe7, 0xf4, 0x3e, 0xd3, - 0xb2, 0xfe, 0x91, 0x6b, 0x7f, 0x8c, 0xbe, 0x54, 0x60, 0xac, 0x67, 0x88, 0x2a, 0x06, 0x94, 0x9d, - 0xe6, 0xd4, 0x8b, 0x83, 0x00, 0xf5, 0x4c, 0x65, 0xda, 0xeb, 0x1c, 0x53, 0x15, 0x4d, 0xf5, 0xc3, - 0x84, 0x7e, 0x51, 0xa0, 0x52, 0x34, 0x0d, 0xa0, 0xc5, 0x7d, 0x8d, 0x0e, 0x02, 0xe3, 0x95, 0x03, - 0x8c, 0x1b, 0xda, 0x0d, 0x8e, 0xf5, 0xea, 0x0d, 0x65, 0x5e, 0xd3, 0xf5, 0xdc, 0x71, 0xdd, 0x0c, - 0xa8, 0x4d, 0x4c, 0x46, 0xc5, 0xbf, 0xd5, 0x03, 0xf2, 0x37, 0x05, 0xa6, 0xfa, 0x35, 0x66, 0xb4, - 0x54, 0x94, 0xb5, 0x3d, 0x8c, 0x15, 0xea, 0x5b, 0x07, 0x33, 0x96, 0x71, 0xcd, 0xf2, 0xb8, 0x66, - 0x50, 0x55, 0xef, 0xfb, 0x89, 0x84, 0x7e, 0x56, 0x60, 0xb2, 0x4f, 0x57, 0x46, 0x37, 0x8a, 0x50, - 0x0c, 0x9e, 0x27, 0xd4, 0xa5, 0x03, 0xd9, 0xca, 0x00, 0xce, 0xf3, 0x00, 0xa6, 0xd1, 0xd9, 0xbe, - 0xdf, 0x8d, 0xe8, 0x57, 0x05, 0xce, 0x14, 0x76, 0x36, 0x74, 0xbd, 0x08, 0xc1, 0xa0, 0xb6, 0xa9, - 0xbe, 0x79, 0x00, 0x4b, 0x89, 0xbc, 0xce, 0x91, 0xd7, 0xd0, 0xac, 0xbe, 0xa7, 0x6f, 0x45, 0x14, - 0xc0, 0xd1, 0xd4, 0xf0, 0x81, 0xde, 0x28, 0xda, 0x3b, 0x6f, 0xfc, 0x51, 0x2f, 0xed, 0x51, 0x5b, - 0xa2, 0x2b, 0xa1, 0x4f, 0xe0, 0x54, 0x7e, 0x17, 0x45, 0x97, 0xf7, 0xda, 0xd1, 0x92, 0x9e, 0xad, - 0x2e, 0xec, 0xc3, 0x42, 0x00, 0xb8, 0xac, 0xac, 0xac, 0x3f, 0x7f, 0x59, 0x55, 0x5e, 0xbc, 0xac, - 0x2a, 0x7f, 0xbf, 0xac, 0x2a, 0x5f, 0xbf, 0xaa, 0x96, 0x5e, 0xbc, 0xaa, 0x96, 0xfe, 0x78, 0x55, - 0x2d, 0x7d, 0x70, 0xcd, 0x71, 0xd9, 0x46, 0xbb, 0x51, 0xb7, 0xa8, 0x9f, 0x4e, 0x5e, 0xe7, 0xea, - 0x25, 0xde, 0x50, 0xf4, 0xae, 0x64, 0x4b, 0x24, 0x94, 0x6d, 0xb7, 0x48, 0xd4, 0x38, 0xc4, 0xc5, - 0x57, 0xfe, 0x0b, 0x00, 0x00, 0xff, 0xff, 0xd1, 0xc6, 0xcc, 0x0a, 0xf6, 0x10, 0x00, 0x00, + // 1557 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x57, 0xcf, 0x6f, 0xdc, 0x44, + 0x14, 0x5e, 0x6f, 0xd2, 0x36, 0x79, 0xdb, 0xb4, 0xe9, 0xa4, 0x69, 0xb7, 0x9b, 0x74, 0x93, 0x1a, + 0x9a, 0x26, 0x29, 0x5d, 0x27, 0x69, 0x55, 0x95, 0x06, 0x15, 0x25, 0x81, 0x36, 0x95, 0x1a, 0x9a, + 0xba, 0x69, 0x1b, 0x41, 0x25, 0x6b, 0xd6, 0x9e, 0x38, 0x56, 0x6c, 0xcf, 0xc6, 0x1e, 0xaf, 0x12, + 0x21, 0x04, 0xe2, 0xd0, 0x0b, 0x20, 0x21, 0x71, 0xe0, 0x80, 0xc4, 0x85, 0x33, 0x12, 0x17, 0x8e, + 0x08, 0xb8, 0xf5, 0x58, 0x89, 0x0b, 0x07, 0x84, 0x50, 0xcb, 0x99, 0xbf, 0x01, 0x79, 0x66, 0xbc, + 0xd9, 0x1f, 0xf6, 0x26, 0xcd, 0x65, 0xd7, 0x7e, 0xf3, 0xbd, 0x6f, 0xbe, 0xf7, 0xe6, 0xcd, 0xcc, + 0x33, 0x9c, 0xb7, 0x76, 0xad, 0x9d, 0x5a, 0x40, 0x19, 0x35, 0xa9, 0xab, 0x99, 0x2e, 0xad, 0x6a, + 0xdb, 0x11, 0x09, 0x76, 0x2b, 0xdc, 0x86, 0x4e, 0x35, 0x0f, 0x57, 0xe2, 0xe1, 0xd2, 0x69, 0x9b, + 0xda, 0x94, 0x9b, 0xb4, 0xf8, 0x49, 0x00, 0x4b, 0xa3, 0x36, 0xa5, 0xb6, 0x4b, 0x34, 0x5c, 0x73, + 0x34, 0xec, 0xfb, 0x94, 0x61, 0xe6, 0x50, 0x3f, 0x94, 0xa3, 0xd3, 0x26, 0x0d, 0x3d, 0x1a, 0x6a, + 0x55, 0x1c, 0x12, 0xc1, 0xaf, 0xd5, 0x67, 0xab, 0x84, 0xe1, 0x59, 0xad, 0x86, 0x6d, 0xc7, 0xe7, + 0x60, 0x89, 0xd5, 0x3a, 0x15, 0x55, 0x5d, 0x6a, 0x6e, 0x19, 0x01, 0x66, 0xc4, 0x70, 0x1d, 0xcf, + 0x61, 0x86, 0x49, 0xfd, 0x0d, 0xc7, 0x96, 0x0e, 0x17, 0x3a, 0x1d, 0xe2, 0x1f, 0xa3, 0x86, 0x9d, + 0x40, 0x42, 0x66, 0x3a, 0x21, 0x64, 0x3b, 0x72, 0xd8, 0xae, 0xc1, 0x1c, 0x12, 0xa4, 0x91, 0xa6, + 0xe4, 0x85, 0x06, 0x16, 0x49, 0x08, 0xc7, 0x3a, 0x87, 0x3d, 0xcc, 0xcc, 0x4d, 0x92, 0x44, 0x7c, + 0xb9, 0x13, 0xe0, 0x3a, 0xdb, 0x91, 0x63, 0x89, 0xbc, 0xb4, 0x4e, 0x36, 0x92, 0xc2, 0x46, 0xea, + 0x72, 0xf0, 0x56, 0xcb, 0xa0, 0xe3, 0x5b, 0x64, 0x87, 0x04, 0x1a, 0xdd, 0xd8, 0x30, 0xcc, 0x4d, + 0xec, 0xf8, 0x46, 0x54, 0xb3, 0x30, 0x23, 0x61, 0xa7, 0x45, 0xf8, 0xab, 0x53, 0x70, 0xf6, 0x41, + 0x9c, 0xf1, 0x3b, 0x84, 0x2d, 0xb9, 0xb4, 0xba, 0x8a, 0x9d, 0x40, 0x27, 0xdb, 0x11, 0x09, 0x19, + 0x3a, 0x01, 0x79, 0xc7, 0x2a, 0x2a, 0xe3, 0xca, 0xe4, 0x80, 0x9e, 0x77, 0x2c, 0xf5, 0x09, 0x0c, + 0x73, 0xe8, 0x1e, 0x2e, 0xac, 0x51, 0x3f, 0x24, 0xe8, 0x16, 0xf4, 0x37, 0x52, 0xca, 0xf1, 0x85, + 0xb9, 0x91, 0x4a, 0x47, 0x69, 0x54, 0x12, 0xbf, 0xc5, 0xde, 0xe7, 0x7f, 0x8f, 0xe5, 0xf4, 0x3e, + 0x53, 0xbe, 0xab, 0x58, 0x6a, 0x58, 0x70, 0xdd, 0x76, 0x0d, 0xb7, 0x01, 0xf6, 0x4a, 0x40, 0x72, + 0x4f, 0x54, 0x44, 0xbd, 0x54, 0xe2, 0x7a, 0xa9, 0x88, 0x7a, 0x94, 0xf5, 0x52, 0x59, 0xc5, 0x36, + 0x91, 0xbe, 0x7a, 0x93, 0xa7, 0xfa, 0x83, 0x02, 0xc5, 0x16, 0xf1, 0x0b, 0xae, 0x9b, 0xa5, 0xbf, + 0xe7, 0x35, 0xf5, 0xa3, 0x3b, 0x2d, 0x22, 0xf3, 0x5c, 0xe4, 0xa5, 0x7d, 0x45, 0x8a, 0xc9, 0x5b, + 0x54, 0xfe, 0xa5, 0xc0, 0xd8, 0x0a, 0xa9, 0x7f, 0x40, 0x2d, 0xb2, 0x46, 0xe3, 0xdf, 0x25, 0xec, + 0x9a, 0x91, 0xcb, 0x07, 0x93, 0x8c, 0x3c, 0x85, 0x33, 0xa2, 0xe0, 0x6b, 0x01, 0xad, 0xd1, 0x90, + 0x04, 0x86, 0x2c, 0xad, 0x46, 0x76, 0x3a, 0x95, 0x3f, 0xc6, 0x6e, 0x5c, 0x5a, 0x34, 0x58, 0x21, + 0xf5, 0x15, 0x81, 0xd6, 0x4f, 0x73, 0x96, 0x55, 0x49, 0x22, 0xad, 0xe8, 0x23, 0x18, 0xae, 0x27, + 0x60, 0xc3, 0x23, 0x75, 0xc3, 0x23, 0x2c, 0x70, 0xcc, 0xb0, 0x11, 0x55, 0x27, 0x79, 0x8b, 0xe0, + 0x15, 0x01, 0xd7, 0x87, 0xea, 0xcd, 0x53, 0x0a, 0xa3, 0xfa, 0x9f, 0x02, 0xe3, 0xd9, 0xe1, 0xc9, + 0xc5, 0xb0, 0xe1, 0x58, 0x40, 0xc2, 0xc8, 0x65, 0xa1, 0x5c, 0x8a, 0x3b, 0xfb, 0xcd, 0x99, 0xc2, + 0x12, 0x03, 0x16, 0x7c, 0xeb, 0x31, 0x75, 0x23, 0x8f, 0xac, 0x92, 0x20, 0x5e, 0x3a, 0xb9, 0x6c, + 0x09, 0x7b, 0x09, 0xc3, 0x50, 0x0a, 0x0a, 0x8d, 0xc3, 0xf1, 0x46, 0x31, 0x18, 0x8d, 0xfa, 0x87, + 0x64, 0xb1, 0xef, 0x5a, 0x68, 0x10, 0x7a, 0x3c, 0x52, 0xe7, 0x19, 0xc9, 0xeb, 0xf1, 0x23, 0x3a, + 0x03, 0x47, 0xeb, 0x9c, 0xa4, 0xd8, 0x33, 0xae, 0x4c, 0xf6, 0xea, 0xf2, 0x4d, 0x9d, 0x86, 0x49, + 0x5e, 0x74, 0xef, 0xf3, 0xd3, 0x64, 0xcd, 0x21, 0xc1, 0xbd, 0xf8, 0x2c, 0x59, 0xe2, 0xbb, 0x3b, + 0x0a, 0x9a, 0xd7, 0x55, 0xfd, 0x4e, 0x81, 0xa9, 0x03, 0x80, 0x65, 0x96, 0x7c, 0x28, 0x66, 0x1d, + 0x51, 0xb2, 0x0e, 0xb4, 0x94, 0xb4, 0x75, 0xa3, 0x96, 0xe9, 0x19, 0x26, 0x69, 0x18, 0x75, 0x0a, + 0x2e, 0x71, 0x71, 0x8b, 0x71, 0xd1, 0xe8, 0x98, 0x91, 0xec, 0x40, 0xbe, 0x55, 0x64, 0xd4, 0x5d, + 0xb1, 0x32, 0x8e, 0x2d, 0x38, 0x9b, 0x71, 0x7c, 0xcb, 0x30, 0x2a, 0x29, 0x61, 0x74, 0x21, 0x96, + 0x51, 0x88, 0xe2, 0x6e, 0x83, 0xa8, 0xeb, 0x70, 0x8e, 0x0b, 0x7b, 0xc8, 0x30, 0x23, 0x1b, 0x91, + 0x7b, 0x3f, 0x3e, 0xb2, 0x93, 0x7d, 0x35, 0x0f, 0x7d, 0xfc, 0x08, 0x4f, 0xd6, 0xbc, 0x30, 0x57, + 0x4a, 0x99, 0x9a, 0xbb, 0xdc, 0xb5, 0x92, 0x5a, 0xa2, 0xe2, 0x55, 0xfd, 0x59, 0x81, 0x52, 0x1a, + 0xb5, 0x8c, 0x72, 0x1d, 0x4e, 0x0a, 0xee, 0x9a, 0x8b, 0x4d, 0xe2, 0x11, 0x9f, 0xc9, 0x29, 0xa6, + 0x52, 0xa6, 0xb8, 0x47, 0x7d, 0x7b, 0x8d, 0x04, 0x1e, 0xa7, 0x58, 0x4d, 0x1c, 0xe4, 0x8c, 0x27, + 0x68, 0x8b, 0x15, 0x8d, 0x41, 0x61, 0xc3, 0x71, 0x5d, 0x03, 0x7b, 0x34, 0xf2, 0x19, 0xaf, 0xc9, + 0x5e, 0x1d, 0x62, 0xd3, 0x02, 0xb7, 0xa0, 0x51, 0xe8, 0x67, 0x81, 0x63, 0xdb, 0x24, 0x20, 0x16, + 0xaf, 0xce, 0x3e, 0x7d, 0xcf, 0xa0, 0x5e, 0x82, 0x8b, 0x5c, 0xf6, 0xbd, 0xa6, 0xcb, 0x27, 0x75, + 0x51, 0x9f, 0x29, 0x30, 0xb1, 0x1f, 0x52, 0x06, 0xfb, 0x14, 0x86, 0x52, 0xee, 0x32, 0x19, 0xf0, + 0xc5, 0xb4, 0x80, 0x3b, 0x28, 0x65, 0xb0, 0xc8, 0xed, 0x18, 0x51, 0x17, 0xe0, 0xfc, 0x43, 0x16, + 0x10, 0x2c, 0xd2, 0x53, 0xa5, 0x74, 0xeb, 0x91, 0xb8, 0xcf, 0x92, 0x75, 0xec, 0xdc, 0xbf, 0x3d, + 0xad, 0xfb, 0x57, 0xc5, 0x50, 0xce, 0xa2, 0x90, 0x21, 0xbc, 0x0b, 0xc7, 0xe4, 0x2d, 0x29, 0xcf, + 0xa0, 0xb1, 0x14, 0xd9, 0x82, 0x43, 0xb8, 0x26, 0xf5, 0x20, 0xbd, 0xd4, 0xcf, 0xf2, 0x70, 0xbc, + 0x79, 0x1c, 0x3d, 0x82, 0x41, 0x9a, 0xcc, 0x26, 0x6f, 0x60, 0x99, 0x91, 0xc9, 0x4c, 0xea, 0x36, + 0x79, 0xcb, 0x39, 0xfd, 0x24, 0x6d, 0x35, 0xc5, 0x37, 0x8f, 0x28, 0xac, 0x78, 0xc5, 0xe5, 0x19, + 0x3d, 0xb1, 0x3f, 0xe1, 0x6d, 0xc7, 0x75, 0x97, 0x73, 0x7a, 0x3f, 0xf7, 0x8d, 0x5f, 0xd0, 0x05, + 0x38, 0x2e, 0xf6, 0xe1, 0x26, 0x71, 0xec, 0x4d, 0xc6, 0x2b, 0x65, 0x40, 0x2f, 0x70, 0xdb, 0x32, + 0x37, 0xa1, 0x11, 0xe8, 0x27, 0x3b, 0xc4, 0x34, 0x3c, 0x6a, 0x91, 0x62, 0x2f, 0x1f, 0xef, 0x8b, + 0x0d, 0x2b, 0xd4, 0x22, 0x8b, 0x83, 0x70, 0x42, 0x44, 0x65, 0x78, 0x24, 0x0c, 0xb1, 0x4d, 0xd4, + 0xaf, 0x14, 0x18, 0x4e, 0x8d, 0x03, 0xad, 0xb7, 0x67, 0xf7, 0x46, 0xab, 0x62, 0xd9, 0xc4, 0x54, + 0x3a, 0x5b, 0x96, 0xfb, 0x1b, 0x1b, 0x4b, 0xb1, 0x41, 0x10, 0x3d, 0x9e, 0x6d, 0x4b, 0x3b, 0x2a, + 0x41, 0x5f, 0xe8, 0xe3, 0x5a, 0xb8, 0x49, 0xc5, 0x56, 0xe8, 0xd3, 0x1b, 0xef, 0xea, 0x8f, 0x0a, + 0x0c, 0xa5, 0xa4, 0x01, 0xcd, 0x03, 0xaf, 0x0d, 0x71, 0x8b, 0xca, 0x35, 0x19, 0xcd, 0xb8, 0xfd, + 0xf9, 0x2d, 0xa9, 0xf3, 0x66, 0x81, 0x3f, 0xa2, 0xeb, 0x70, 0x94, 0xe7, 0x30, 0xbe, 0x1f, 0xe3, + 0x48, 0x8a, 0x59, 0x47, 0x86, 0x54, 0x2a, 0xd1, 0x71, 0xba, 0x9b, 0xb6, 0x6d, 0x58, 0xec, 0x19, + 0xef, 0x99, 0xec, 0xd5, 0x0b, 0x7b, 0xfb, 0x36, 0x54, 0x9f, 0xe5, 0x61, 0x50, 0xe8, 0x5d, 0xc3, + 0x5b, 0x24, 0xe0, 0x2c, 0x68, 0x06, 0x8e, 0x70, 0x06, 0xa9, 0x33, 0x73, 0xba, 0xe5, 0x9c, 0x2e, + 0x80, 0x68, 0x1d, 0x4e, 0x35, 0xed, 0x22, 0x43, 0x78, 0xe7, 0x33, 0x0f, 0x1f, 0x31, 0x63, 0xd3, + 0x8e, 0x4c, 0xe8, 0x06, 0xdd, 0x36, 0x1b, 0x7a, 0x02, 0x88, 0xc5, 0xca, 0x04, 0xa7, 0x11, 0x32, + 0xcc, 0xa2, 0x90, 0x17, 0x4e, 0x37, 0xea, 0xbd, 0x60, 0x1e, 0x72, 0x07, 0x7d, 0x90, 0xb5, 0x59, + 0x16, 0x07, 0xa0, 0xd0, 0x44, 0xac, 0xfe, 0xa4, 0xc0, 0x99, 0x74, 0xdf, 0x38, 0x8d, 0x2d, 0x93, + 0x8b, 0xbb, 0xba, 0x40, 0x9b, 0x20, 0x57, 0x00, 0x05, 0xc4, 0xc3, 0x8e, 0xef, 0xf8, 0xb6, 0xb1, + 0x1d, 0x61, 0x9f, 0x45, 0x5e, 0x28, 0xcf, 0xc9, 0x53, 0x8d, 0x91, 0x07, 0x72, 0x00, 0xbd, 0x07, + 0x65, 0x5a, 0x63, 0x8e, 0xe7, 0x84, 0xcc, 0x31, 0xb1, 0xeb, 0xee, 0xf2, 0x9d, 0x45, 0xac, 0x3d, + 0x57, 0x71, 0xc3, 0x8f, 0xb6, 0xa2, 0x6e, 0x73, 0x50, 0xc2, 0x32, 0xf7, 0x3d, 0xc0, 0x11, 0x7e, + 0x5a, 0xa2, 0x2f, 0x14, 0xe8, 0x4b, 0xfa, 0x46, 0x34, 0x9d, 0x92, 0x95, 0x8c, 0xe6, 0xbb, 0x34, + 0x99, 0x85, 0x6d, 0xef, 0xbe, 0xd5, 0xa9, 0xcf, 0xff, 0xf8, 0xf7, 0x9b, 0xfc, 0x1b, 0xe8, 0x82, + 0xd6, 0xe5, 0x4b, 0x47, 0xfb, 0xd8, 0xb1, 0x3e, 0x41, 0x5f, 0x2a, 0x50, 0x68, 0x6a, 0x80, 0xb3, + 0x05, 0x75, 0x76, 0xe2, 0xa5, 0xcb, 0xfb, 0x09, 0x6a, 0xea, 0xa8, 0xd5, 0x37, 0xb9, 0xa6, 0x32, + 0x1a, 0xed, 0xa6, 0x09, 0xfd, 0xaa, 0x40, 0x31, 0xab, 0x93, 0x43, 0x73, 0xaf, 0xd5, 0xf6, 0x09, + 0x8d, 0x57, 0x0f, 0xd1, 0x2a, 0xaa, 0x37, 0xb9, 0xd6, 0x6b, 0x37, 0x95, 0x69, 0x55, 0xd3, 0x52, + 0x3f, 0xb5, 0x0c, 0x9f, 0x5a, 0xc4, 0x60, 0x54, 0xfc, 0x9b, 0x4d, 0x22, 0x7f, 0x57, 0x60, 0xb4, + 0x5b, 0x53, 0x85, 0xe6, 0xb3, 0xb2, 0x76, 0x80, 0x96, 0xb0, 0xf4, 0xce, 0xe1, 0x9c, 0x65, 0x5c, + 0x13, 0x3c, 0xae, 0x71, 0x54, 0xd6, 0xba, 0x7e, 0xde, 0xa2, 0x5f, 0x14, 0x18, 0xe9, 0xd2, 0x51, + 0xa1, 0x9b, 0x59, 0x2a, 0xf6, 0xef, 0x05, 0x4b, 0xf3, 0x87, 0xf2, 0x95, 0x01, 0x5c, 0xe4, 0x01, + 0x8c, 0xa1, 0xf3, 0x5d, 0xbf, 0xf9, 0xd1, 0x6f, 0x0a, 0x9c, 0xcb, 0xec, 0x4a, 0xd0, 0x8d, 0x2c, + 0x05, 0xfb, 0xb5, 0x3c, 0xa5, 0xb7, 0x0f, 0xe1, 0x29, 0x95, 0x57, 0xb8, 0xf2, 0x49, 0x34, 0xa1, + 0x1d, 0xe8, 0x3b, 0x1f, 0xf9, 0x30, 0xd0, 0xd2, 0x38, 0xa2, 0xb7, 0xb2, 0xe6, 0x4e, 0x6b, 0x5d, + 0x4b, 0x57, 0x0e, 0x88, 0x96, 0xea, 0x72, 0xe8, 0xd3, 0xe4, 0x44, 0x6d, 0xef, 0x80, 0xd0, 0xcc, + 0x41, 0xbb, 0x91, 0xa4, 0xdf, 0x2a, 0xcd, 0xbe, 0x86, 0x87, 0x10, 0x30, 0xa3, 0x2c, 0xae, 0x3e, + 0x7f, 0x59, 0x56, 0x5e, 0xbc, 0x2c, 0x2b, 0xff, 0xbc, 0x2c, 0x2b, 0x5f, 0xbf, 0x2a, 0xe7, 0x5e, + 0xbc, 0x2a, 0xe7, 0xfe, 0x7c, 0x55, 0xce, 0x7d, 0x78, 0xdd, 0x76, 0xd8, 0x66, 0x54, 0xad, 0x98, + 0xd4, 0x6b, 0x4d, 0x5e, 0xfd, 0xda, 0x15, 0xde, 0x0c, 0x68, 0x0d, 0xcb, 0x8e, 0x48, 0x28, 0xdb, + 0xad, 0x91, 0xb0, 0x7a, 0x94, 0x9b, 0xaf, 0xfe, 0x1f, 0x00, 0x00, 0xff, 0xff, 0xe0, 0x10, 0x44, + 0xf0, 0xb2, 0x12, 0x00, 0x00, } // Reference imports to suppress errors if they are not otherwise used. @@ -2383,6 +2568,130 @@ func (m *StreamOrderbookFill) MarshalToSizedBuffer(dAtA []byte) (int, error) { return len(dAtA) - i, nil } +func (m *StreamTakerOrder) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *StreamTakerOrder) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *StreamTakerOrder) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.TakerOrderStatus != nil { + { + size, err := m.TakerOrderStatus.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintQuery(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x1a + } + if m.TakerOrder != nil { + { + size := m.TakerOrder.Size() + i -= size + if _, err := m.TakerOrder.MarshalTo(dAtA[i:]); err != nil { + return 0, err + } + } + } + return len(dAtA) - i, nil +} + +func (m *StreamTakerOrder_Order) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *StreamTakerOrder_Order) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + if m.Order != nil { + { + size, err := m.Order.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintQuery(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} +func (m *StreamTakerOrder_LiquidationOrder) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *StreamTakerOrder_LiquidationOrder) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + if m.LiquidationOrder != nil { + { + size, err := m.LiquidationOrder.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintQuery(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x12 + } + return len(dAtA) - i, nil +} +func (m *StreamTakerOrderStatus) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *StreamTakerOrderStatus) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *StreamTakerOrderStatus) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.OptimisticallyFilledQuantums != 0 { + i = encodeVarintQuery(dAtA, i, uint64(m.OptimisticallyFilledQuantums)) + i-- + dAtA[i] = 0x18 + } + if m.RemainingQuantums != 0 { + i = encodeVarintQuery(dAtA, i, uint64(m.RemainingQuantums)) + i-- + dAtA[i] = 0x10 + } + if m.OrderStatus != 0 { + i = encodeVarintQuery(dAtA, i, uint64(m.OrderStatus)) + i-- + dAtA[i] = 0x8 + } + return len(dAtA) - i, nil +} + func encodeVarintQuery(dAtA []byte, offset int, v uint64) int { offset -= sovQuery(v) base := offset @@ -2704,6 +3013,64 @@ func (m *StreamOrderbookFill) Size() (n int) { return n } +func (m *StreamTakerOrder) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.TakerOrder != nil { + n += m.TakerOrder.Size() + } + if m.TakerOrderStatus != nil { + l = m.TakerOrderStatus.Size() + n += 1 + l + sovQuery(uint64(l)) + } + return n +} + +func (m *StreamTakerOrder_Order) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.Order != nil { + l = m.Order.Size() + n += 1 + l + sovQuery(uint64(l)) + } + return n +} +func (m *StreamTakerOrder_LiquidationOrder) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.LiquidationOrder != nil { + l = m.LiquidationOrder.Size() + n += 1 + l + sovQuery(uint64(l)) + } + return n +} +func (m *StreamTakerOrderStatus) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.OrderStatus != 0 { + n += 1 + sovQuery(uint64(m.OrderStatus)) + } + if m.RemainingQuantums != 0 { + n += 1 + sovQuery(uint64(m.RemainingQuantums)) + } + if m.OptimisticallyFilledQuantums != 0 { + n += 1 + sovQuery(uint64(m.OptimisticallyFilledQuantums)) + } + return n +} + func sovQuery(x uint64) (n int) { return (math_bits.Len64(x|1) + 6) / 7 } @@ -4645,6 +5012,269 @@ func (m *StreamOrderbookFill) Unmarshal(dAtA []byte) error { } return nil } +func (m *StreamTakerOrder) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowQuery + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: StreamTakerOrder: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: StreamTakerOrder: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Order", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowQuery + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthQuery + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthQuery + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + v := &Order{} + if err := v.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + m.TakerOrder = &StreamTakerOrder_Order{v} + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field LiquidationOrder", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowQuery + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthQuery + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthQuery + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + v := &StreamLiquidationOrder{} + if err := v.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + m.TakerOrder = &StreamTakerOrder_LiquidationOrder{v} + iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field TakerOrderStatus", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowQuery + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthQuery + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthQuery + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.TakerOrderStatus == nil { + m.TakerOrderStatus = &StreamTakerOrderStatus{} + } + if err := m.TakerOrderStatus.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipQuery(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthQuery + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *StreamTakerOrderStatus) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowQuery + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: StreamTakerOrderStatus: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: StreamTakerOrderStatus: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field OrderStatus", wireType) + } + m.OrderStatus = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowQuery + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.OrderStatus |= uint32(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field RemainingQuantums", wireType) + } + m.RemainingQuantums = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowQuery + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.RemainingQuantums |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field OptimisticallyFilledQuantums", wireType) + } + m.OptimisticallyFilledQuantums = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowQuery + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.OptimisticallyFilledQuantums |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := skipQuery(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthQuery + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func skipQuery(dAtA []byte) (n int, err error) { l := len(dAtA) iNdEx := 0 From 63951ce8a66190c9afd2ff79036c9ae7154774f7 Mon Sep 17 00:00:00 2001 From: Jonathan Fung <121899091+jonfung-dydx@users.noreply.github.com> Date: Fri, 2 Aug 2024 16:39:34 -0400 Subject: [PATCH 03/33] Fix proto formatting (#2020) --- .../src/codegen/dydxprotocol/clob/order.ts | 140 ++++++++++++ .../src/codegen/dydxprotocol/clob/query.ts | 216 +++++++++++++++++- proto/dydxprotocol/clob/order.proto | 10 +- proto/dydxprotocol/clob/query.proto | 13 +- protocol/x/clob/types/order.pb.go | 10 +- protocol/x/clob/types/query.pb.go | 13 +- 6 files changed, 381 insertions(+), 21 deletions(-) diff --git a/indexer/packages/v4-protos/src/codegen/dydxprotocol/clob/order.ts b/indexer/packages/v4-protos/src/codegen/dydxprotocol/clob/order.ts index aeaaa06aa6..6d166bff82 100644 --- a/indexer/packages/v4-protos/src/codegen/dydxprotocol/clob/order.ts +++ b/indexer/packages/v4-protos/src/codegen/dydxprotocol/clob/order.ts @@ -1,4 +1,5 @@ import { SubaccountId, SubaccountIdSDKType } from "../subaccounts/subaccount"; +import { PerpetualLiquidationInfo, PerpetualLiquidationInfoSDKType } from "./liquidations"; import * as _m0 from "protobufjs/minimal"; import { DeepPartial, Long } from "../../helpers"; /** @@ -700,6 +701,60 @@ export interface TransactionOrderingSDKType { transaction_index: number; } +/** + * StreamLiquidationOrder represents an protocol-generated IOC liquidation + * order. Used in full node streaming. + */ + +export interface StreamLiquidationOrder { + /** Information about this liquidation order. */ + liquidationInfo?: PerpetualLiquidationInfo; + /** + * CLOB pair ID of the CLOB pair the liquidation order will be matched + * against. + */ + + clobPairId: number; + /** + * True if this is a buy order liquidating a short position, false if vice + * versa. + */ + + isBuy: boolean; + /** The number of base quantums for this liquidation order. */ + + quantums: Long; + /** The subticks this liquidation order will be submitted at. */ + + subticks: Long; +} +/** + * StreamLiquidationOrder represents an protocol-generated IOC liquidation + * order. Used in full node streaming. + */ + +export interface StreamLiquidationOrderSDKType { + /** Information about this liquidation order. */ + liquidation_info?: PerpetualLiquidationInfoSDKType; + /** + * CLOB pair ID of the CLOB pair the liquidation order will be matched + * against. + */ + + clob_pair_id: number; + /** + * True if this is a buy order liquidating a short position, false if vice + * versa. + */ + + is_buy: boolean; + /** The number of base quantums for this liquidation order. */ + + quantums: Long; + /** The subticks this liquidation order will be submitted at. */ + + subticks: Long; +} function createBaseOrderId(): OrderId { return { @@ -1284,4 +1339,89 @@ export const TransactionOrdering = { return message; } +}; + +function createBaseStreamLiquidationOrder(): StreamLiquidationOrder { + return { + liquidationInfo: undefined, + clobPairId: 0, + isBuy: false, + quantums: Long.UZERO, + subticks: Long.UZERO + }; +} + +export const StreamLiquidationOrder = { + encode(message: StreamLiquidationOrder, writer: _m0.Writer = _m0.Writer.create()): _m0.Writer { + if (message.liquidationInfo !== undefined) { + PerpetualLiquidationInfo.encode(message.liquidationInfo, writer.uint32(10).fork()).ldelim(); + } + + if (message.clobPairId !== 0) { + writer.uint32(16).uint32(message.clobPairId); + } + + if (message.isBuy === true) { + writer.uint32(24).bool(message.isBuy); + } + + if (!message.quantums.isZero()) { + writer.uint32(32).uint64(message.quantums); + } + + if (!message.subticks.isZero()) { + writer.uint32(40).uint64(message.subticks); + } + + return writer; + }, + + decode(input: _m0.Reader | Uint8Array, length?: number): StreamLiquidationOrder { + const reader = input instanceof _m0.Reader ? input : new _m0.Reader(input); + let end = length === undefined ? reader.len : reader.pos + length; + const message = createBaseStreamLiquidationOrder(); + + while (reader.pos < end) { + const tag = reader.uint32(); + + switch (tag >>> 3) { + case 1: + message.liquidationInfo = PerpetualLiquidationInfo.decode(reader, reader.uint32()); + break; + + case 2: + message.clobPairId = reader.uint32(); + break; + + case 3: + message.isBuy = reader.bool(); + break; + + case 4: + message.quantums = (reader.uint64() as Long); + break; + + case 5: + message.subticks = (reader.uint64() as Long); + break; + + default: + reader.skipType(tag & 7); + break; + } + } + + return message; + }, + + fromPartial(object: DeepPartial): StreamLiquidationOrder { + const message = createBaseStreamLiquidationOrder(); + message.liquidationInfo = object.liquidationInfo !== undefined && object.liquidationInfo !== null ? PerpetualLiquidationInfo.fromPartial(object.liquidationInfo) : undefined; + message.clobPairId = object.clobPairId ?? 0; + message.isBuy = object.isBuy ?? false; + message.quantums = object.quantums !== undefined && object.quantums !== null ? Long.fromValue(object.quantums) : Long.UZERO; + message.subticks = object.subticks !== undefined && object.subticks !== null ? Long.fromValue(object.subticks) : Long.UZERO; + return message; + } + }; \ No newline at end of file diff --git a/indexer/packages/v4-protos/src/codegen/dydxprotocol/clob/query.ts b/indexer/packages/v4-protos/src/codegen/dydxprotocol/clob/query.ts index e3ebe88ee0..5377900d5b 100644 --- a/indexer/packages/v4-protos/src/codegen/dydxprotocol/clob/query.ts +++ b/indexer/packages/v4-protos/src/codegen/dydxprotocol/clob/query.ts @@ -1,6 +1,6 @@ import { PageRequest, PageRequestSDKType, PageResponse, PageResponseSDKType } from "../../cosmos/base/query/v1beta1/pagination"; import { ValidatorMevMatches, ValidatorMevMatchesSDKType, MevNodeToNodeMetrics, MevNodeToNodeMetricsSDKType } from "./mev"; -import { OrderId, OrderIdSDKType, LongTermOrderPlacement, LongTermOrderPlacementSDKType, Order, OrderSDKType } from "./order"; +import { OrderId, OrderIdSDKType, LongTermOrderPlacement, LongTermOrderPlacementSDKType, Order, OrderSDKType, StreamLiquidationOrder, StreamLiquidationOrderSDKType } from "./order"; import { ClobPair, ClobPairSDKType } from "./clob_pair"; import { EquityTierLimitConfiguration, EquityTierLimitConfigurationSDKType } from "./equity_tier_limit_config"; import { BlockRateLimitConfiguration, BlockRateLimitConfigurationSDKType } from "./block_rate_limit_config"; @@ -391,6 +391,90 @@ export interface StreamOrderbookFillSDKType { fill_amounts: Long[]; } +/** + * StreamTakerOrder provides information on a taker order that was attempted + * to be matched on the orderbook. + * It is intended to be used only in full node streaming. + */ + +export interface StreamTakerOrder { + order?: Order; + liquidationOrder?: StreamLiquidationOrder; + /** + * Information on the taker order after it is matched on the book, + * either successfully or unsuccessfully. + */ + + takerOrderStatus?: StreamTakerOrderStatus; +} +/** + * StreamTakerOrder provides information on a taker order that was attempted + * to be matched on the orderbook. + * It is intended to be used only in full node streaming. + */ + +export interface StreamTakerOrderSDKType { + order?: OrderSDKType; + liquidation_order?: StreamLiquidationOrderSDKType; + /** + * Information on the taker order after it is matched on the book, + * either successfully or unsuccessfully. + */ + + taker_order_status?: StreamTakerOrderStatusSDKType; +} +/** + * StreamTakerOrderStatus is a representation of a taker order + * after it is attempted to be matched on the orderbook. + * It is intended to be used only in full node streaming. + */ + +export interface StreamTakerOrderStatus { + /** + * The state of the taker order after attempting to match it against the + * orderbook. Possible enum values can be found here: + * https://github.com/dydxprotocol/v4-chain/blob/main/protocol/x/clob/types/orderbook.go#L105 + */ + orderStatus: number; + /** The amount of remaining (non-matched) base quantums of this taker order. */ + + remainingQuantums: Long; + /** + * The amount of base quantums that were *optimistically* filled for this + * taker order when the order is matched against the orderbook. Note that if + * any quantums of this order were optimistically filled or filled in state + * before this invocation of the matching loop, this value will not include + * them. + */ + + optimisticallyFilledQuantums: Long; +} +/** + * StreamTakerOrderStatus is a representation of a taker order + * after it is attempted to be matched on the orderbook. + * It is intended to be used only in full node streaming. + */ + +export interface StreamTakerOrderStatusSDKType { + /** + * The state of the taker order after attempting to match it against the + * orderbook. Possible enum values can be found here: + * https://github.com/dydxprotocol/v4-chain/blob/main/protocol/x/clob/types/orderbook.go#L105 + */ + order_status: number; + /** The amount of remaining (non-matched) base quantums of this taker order. */ + + remaining_quantums: Long; + /** + * The amount of base quantums that were *optimistically* filled for this + * taker order when the order is matched against the orderbook. Note that if + * any quantums of this order were optimistically filled or filled in state + * before this invocation of the matching loop, this value will not include + * them. + */ + + optimistically_filled_quantums: Long; +} function createBaseQueryGetClobPairRequest(): QueryGetClobPairRequest { return { @@ -1401,4 +1485,134 @@ export const StreamOrderbookFill = { return message; } +}; + +function createBaseStreamTakerOrder(): StreamTakerOrder { + return { + order: undefined, + liquidationOrder: undefined, + takerOrderStatus: undefined + }; +} + +export const StreamTakerOrder = { + encode(message: StreamTakerOrder, writer: _m0.Writer = _m0.Writer.create()): _m0.Writer { + if (message.order !== undefined) { + Order.encode(message.order, writer.uint32(10).fork()).ldelim(); + } + + if (message.liquidationOrder !== undefined) { + StreamLiquidationOrder.encode(message.liquidationOrder, writer.uint32(18).fork()).ldelim(); + } + + if (message.takerOrderStatus !== undefined) { + StreamTakerOrderStatus.encode(message.takerOrderStatus, writer.uint32(26).fork()).ldelim(); + } + + return writer; + }, + + decode(input: _m0.Reader | Uint8Array, length?: number): StreamTakerOrder { + const reader = input instanceof _m0.Reader ? input : new _m0.Reader(input); + let end = length === undefined ? reader.len : reader.pos + length; + const message = createBaseStreamTakerOrder(); + + while (reader.pos < end) { + const tag = reader.uint32(); + + switch (tag >>> 3) { + case 1: + message.order = Order.decode(reader, reader.uint32()); + break; + + case 2: + message.liquidationOrder = StreamLiquidationOrder.decode(reader, reader.uint32()); + break; + + case 3: + message.takerOrderStatus = StreamTakerOrderStatus.decode(reader, reader.uint32()); + break; + + default: + reader.skipType(tag & 7); + break; + } + } + + return message; + }, + + fromPartial(object: DeepPartial): StreamTakerOrder { + const message = createBaseStreamTakerOrder(); + message.order = object.order !== undefined && object.order !== null ? Order.fromPartial(object.order) : undefined; + message.liquidationOrder = object.liquidationOrder !== undefined && object.liquidationOrder !== null ? StreamLiquidationOrder.fromPartial(object.liquidationOrder) : undefined; + message.takerOrderStatus = object.takerOrderStatus !== undefined && object.takerOrderStatus !== null ? StreamTakerOrderStatus.fromPartial(object.takerOrderStatus) : undefined; + return message; + } + +}; + +function createBaseStreamTakerOrderStatus(): StreamTakerOrderStatus { + return { + orderStatus: 0, + remainingQuantums: Long.UZERO, + optimisticallyFilledQuantums: Long.UZERO + }; +} + +export const StreamTakerOrderStatus = { + encode(message: StreamTakerOrderStatus, writer: _m0.Writer = _m0.Writer.create()): _m0.Writer { + if (message.orderStatus !== 0) { + writer.uint32(8).uint32(message.orderStatus); + } + + if (!message.remainingQuantums.isZero()) { + writer.uint32(16).uint64(message.remainingQuantums); + } + + if (!message.optimisticallyFilledQuantums.isZero()) { + writer.uint32(24).uint64(message.optimisticallyFilledQuantums); + } + + return writer; + }, + + decode(input: _m0.Reader | Uint8Array, length?: number): StreamTakerOrderStatus { + const reader = input instanceof _m0.Reader ? input : new _m0.Reader(input); + let end = length === undefined ? reader.len : reader.pos + length; + const message = createBaseStreamTakerOrderStatus(); + + while (reader.pos < end) { + const tag = reader.uint32(); + + switch (tag >>> 3) { + case 1: + message.orderStatus = reader.uint32(); + break; + + case 2: + message.remainingQuantums = (reader.uint64() as Long); + break; + + case 3: + message.optimisticallyFilledQuantums = (reader.uint64() as Long); + break; + + default: + reader.skipType(tag & 7); + break; + } + } + + return message; + }, + + fromPartial(object: DeepPartial): StreamTakerOrderStatus { + const message = createBaseStreamTakerOrderStatus(); + message.orderStatus = object.orderStatus ?? 0; + message.remainingQuantums = object.remainingQuantums !== undefined && object.remainingQuantums !== null ? Long.fromValue(object.remainingQuantums) : Long.UZERO; + message.optimisticallyFilledQuantums = object.optimisticallyFilledQuantums !== undefined && object.optimisticallyFilledQuantums !== null ? Long.fromValue(object.optimisticallyFilledQuantums) : Long.UZERO; + return message; + } + }; \ No newline at end of file diff --git a/proto/dydxprotocol/clob/order.proto b/proto/dydxprotocol/clob/order.proto index 4357913474..7045122ace 100644 --- a/proto/dydxprotocol/clob/order.proto +++ b/proto/dydxprotocol/clob/order.proto @@ -229,16 +229,18 @@ message TransactionOrdering { uint32 transaction_index = 2; } -// StreamLiquidationOrder represents an protocol-generated IOC liquidation order. -// Used in full node streaming. +// StreamLiquidationOrder represents an protocol-generated IOC liquidation +// order. Used in full node streaming. message StreamLiquidationOrder { // Information about this liquidation order. PerpetualLiquidationInfo liquidation_info = 1; - // CLOB pair ID of the CLOB pair the liquidation order will be matched against. + // CLOB pair ID of the CLOB pair the liquidation order will be matched + // against. uint32 clob_pair_id = 2; - // True if this is a buy order liquidating a short position, false if vice versa. + // True if this is a buy order liquidating a short position, false if vice + // versa. bool is_buy = 3; // The number of base quantums for this liquidation order. diff --git a/proto/dydxprotocol/clob/query.proto b/proto/dydxprotocol/clob/query.proto index c7eb3d3aba..1642ad15cf 100644 --- a/proto/dydxprotocol/clob/query.proto +++ b/proto/dydxprotocol/clob/query.proto @@ -241,17 +241,18 @@ message StreamTakerOrder { // after it is attempted to be matched on the orderbook. // It is intended to be used only in full node streaming. message StreamTakerOrderStatus { - // The state of the taker order after attempting to match it against the orderbook. - // Possible enum values can be found here: + // The state of the taker order after attempting to match it against the + // orderbook. Possible enum values can be found here: // https://github.com/dydxprotocol/v4-chain/blob/main/protocol/x/clob/types/orderbook.go#L105 uint32 order_status = 1; // The amount of remaining (non-matched) base quantums of this taker order. uint64 remaining_quantums = 2; - // The amount of base quantums that were *optimistically* filled for this taker order - // when the order is matched against the orderbook. Note that if any quantums of this order - // were optimistically filled or filled in state before this invocation of the matching loop, - // this value will not include them. + // The amount of base quantums that were *optimistically* filled for this + // taker order when the order is matched against the orderbook. Note that if + // any quantums of this order were optimistically filled or filled in state + // before this invocation of the matching loop, this value will not include + // them. uint64 optimistically_filled_quantums = 3; } diff --git a/protocol/x/clob/types/order.pb.go b/protocol/x/clob/types/order.pb.go index ad17410e45..cad93b7f8c 100644 --- a/protocol/x/clob/types/order.pb.go +++ b/protocol/x/clob/types/order.pb.go @@ -807,14 +807,16 @@ func (m *TransactionOrdering) GetTransactionIndex() uint32 { return 0 } -// StreamLiquidationOrder represents an protocol-generated IOC liquidation order. -// Used in full node streaming. +// StreamLiquidationOrder represents an protocol-generated IOC liquidation +// order. Used in full node streaming. type StreamLiquidationOrder struct { // Information about this liquidation order. LiquidationInfo *PerpetualLiquidationInfo `protobuf:"bytes,1,opt,name=liquidation_info,json=liquidationInfo,proto3" json:"liquidation_info,omitempty"` - // CLOB pair ID of the CLOB pair the liquidation order will be matched against. + // CLOB pair ID of the CLOB pair the liquidation order will be matched + // against. ClobPairId uint32 `protobuf:"varint,2,opt,name=clob_pair_id,json=clobPairId,proto3" json:"clob_pair_id,omitempty"` - // True if this is a buy order liquidating a short position, false if vice versa. + // True if this is a buy order liquidating a short position, false if vice + // versa. IsBuy bool `protobuf:"varint,3,opt,name=is_buy,json=isBuy,proto3" json:"is_buy,omitempty"` // The number of base quantums for this liquidation order. Quantums uint64 `protobuf:"varint,4,opt,name=quantums,proto3" json:"quantums,omitempty"` diff --git a/protocol/x/clob/types/query.pb.go b/protocol/x/clob/types/query.pb.go index 1d66576185..ebbe6e5f4d 100644 --- a/protocol/x/clob/types/query.pb.go +++ b/protocol/x/clob/types/query.pb.go @@ -1201,16 +1201,17 @@ func (*StreamTakerOrder) XXX_OneofWrappers() []interface{} { // after it is attempted to be matched on the orderbook. // It is intended to be used only in full node streaming. type StreamTakerOrderStatus struct { - // The state of the taker order after attempting to match it against the orderbook. - // Possible enum values can be found here: + // The state of the taker order after attempting to match it against the + // orderbook. Possible enum values can be found here: // https://github.com/dydxprotocol/v4-chain/blob/main/protocol/x/clob/types/orderbook.go#L105 OrderStatus uint32 `protobuf:"varint,1,opt,name=order_status,json=orderStatus,proto3" json:"order_status,omitempty"` // The amount of remaining (non-matched) base quantums of this taker order. RemainingQuantums uint64 `protobuf:"varint,2,opt,name=remaining_quantums,json=remainingQuantums,proto3" json:"remaining_quantums,omitempty"` - // The amount of base quantums that were *optimistically* filled for this taker order - // when the order is matched against the orderbook. Note that if any quantums of this order - // were optimistically filled or filled in state before this invocation of the matching loop, - // this value will not include them. + // The amount of base quantums that were *optimistically* filled for this + // taker order when the order is matched against the orderbook. Note that if + // any quantums of this order were optimistically filled or filled in state + // before this invocation of the matching loop, this value will not include + // them. OptimisticallyFilledQuantums uint64 `protobuf:"varint,3,opt,name=optimistically_filled_quantums,json=optimisticallyFilledQuantums,proto3" json:"optimistically_filled_quantums,omitempty"` } From c20ef9925432c9e4e433b1933f96604cd9832155 Mon Sep 17 00:00:00 2001 From: Jonathan Fung <121899091+jonfung-dydx@users.noreply.github.com> Date: Mon, 5 Aug 2024 11:11:56 -0400 Subject: [PATCH 04/33] FNS protos - add taker order in stream update oneof (#2021) --- .../src/codegen/dydxprotocol/clob/query.ts | 18 +- proto/dydxprotocol/clob/query.proto | 7 +- protocol/x/clob/types/query.pb.go | 310 +++++++++++------- 3 files changed, 215 insertions(+), 120 deletions(-) diff --git a/indexer/packages/v4-protos/src/codegen/dydxprotocol/clob/query.ts b/indexer/packages/v4-protos/src/codegen/dydxprotocol/clob/query.ts index 5377900d5b..4097359e36 100644 --- a/indexer/packages/v4-protos/src/codegen/dydxprotocol/clob/query.ts +++ b/indexer/packages/v4-protos/src/codegen/dydxprotocol/clob/query.ts @@ -287,6 +287,7 @@ export interface StreamOrderbookUpdatesResponseSDKType { export interface StreamUpdate { orderbookUpdate?: StreamOrderbookUpdate; orderFill?: StreamOrderbookFill; + takerOrder?: StreamTakerOrder; /** Block height of the update. */ blockHeight: number; @@ -302,6 +303,7 @@ export interface StreamUpdate { export interface StreamUpdateSDKType { orderbook_update?: StreamOrderbookUpdateSDKType; order_fill?: StreamOrderbookFillSDKType; + taker_order?: StreamTakerOrderSDKType; /** Block height of the update. */ block_height: number; @@ -1284,6 +1286,7 @@ function createBaseStreamUpdate(): StreamUpdate { return { orderbookUpdate: undefined, orderFill: undefined, + takerOrder: undefined, blockHeight: 0, execMode: 0 }; @@ -1299,12 +1302,16 @@ export const StreamUpdate = { StreamOrderbookFill.encode(message.orderFill, writer.uint32(18).fork()).ldelim(); } + if (message.takerOrder !== undefined) { + StreamTakerOrder.encode(message.takerOrder, writer.uint32(26).fork()).ldelim(); + } + if (message.blockHeight !== 0) { - writer.uint32(24).uint32(message.blockHeight); + writer.uint32(32).uint32(message.blockHeight); } if (message.execMode !== 0) { - writer.uint32(32).uint32(message.execMode); + writer.uint32(40).uint32(message.execMode); } return writer; @@ -1328,10 +1335,14 @@ export const StreamUpdate = { break; case 3: - message.blockHeight = reader.uint32(); + message.takerOrder = StreamTakerOrder.decode(reader, reader.uint32()); break; case 4: + message.blockHeight = reader.uint32(); + break; + + case 5: message.execMode = reader.uint32(); break; @@ -1348,6 +1359,7 @@ export const StreamUpdate = { const message = createBaseStreamUpdate(); message.orderbookUpdate = object.orderbookUpdate !== undefined && object.orderbookUpdate !== null ? StreamOrderbookUpdate.fromPartial(object.orderbookUpdate) : undefined; message.orderFill = object.orderFill !== undefined && object.orderFill !== null ? StreamOrderbookFill.fromPartial(object.orderFill) : undefined; + message.takerOrder = object.takerOrder !== undefined && object.takerOrder !== null ? StreamTakerOrder.fromPartial(object.takerOrder) : undefined; message.blockHeight = object.blockHeight ?? 0; message.execMode = object.execMode ?? 0; return message; diff --git a/proto/dydxprotocol/clob/query.proto b/proto/dydxprotocol/clob/query.proto index 1642ad15cf..a00be1aaa1 100644 --- a/proto/dydxprotocol/clob/query.proto +++ b/proto/dydxprotocol/clob/query.proto @@ -178,17 +178,18 @@ message StreamOrderbookUpdatesResponse { // GRPC stream. message StreamUpdate { // Contains one of an StreamOrderbookUpdate, - // StreamOrderbookFill. + // StreamOrderbookFill, StreamTakerOrderStatus. oneof update_message { StreamOrderbookUpdate orderbook_update = 1; StreamOrderbookFill order_fill = 2; + StreamTakerOrder taker_order = 3; } // Block height of the update. - uint32 block_height = 3; + uint32 block_height = 4; // Exec mode of the update. - uint32 exec_mode = 4; + uint32 exec_mode = 5; } // StreamOrderbookUpdate provides information on an orderbook update. Used in diff --git a/protocol/x/clob/types/query.pb.go b/protocol/x/clob/types/query.pb.go index ebbe6e5f4d..287d4c1e48 100644 --- a/protocol/x/clob/types/query.pb.go +++ b/protocol/x/clob/types/query.pb.go @@ -863,17 +863,18 @@ func (m *StreamOrderbookUpdatesResponse) GetUpdates() []StreamUpdate { // GRPC stream. type StreamUpdate struct { // Contains one of an StreamOrderbookUpdate, - // StreamOrderbookFill. + // StreamOrderbookFill, StreamTakerOrderStatus. // // Types that are valid to be assigned to UpdateMessage: // // *StreamUpdate_OrderbookUpdate // *StreamUpdate_OrderFill + // *StreamUpdate_TakerOrder UpdateMessage isStreamUpdate_UpdateMessage `protobuf_oneof:"update_message"` // Block height of the update. - BlockHeight uint32 `protobuf:"varint,3,opt,name=block_height,json=blockHeight,proto3" json:"block_height,omitempty"` + BlockHeight uint32 `protobuf:"varint,4,opt,name=block_height,json=blockHeight,proto3" json:"block_height,omitempty"` // Exec mode of the update. - ExecMode uint32 `protobuf:"varint,4,opt,name=exec_mode,json=execMode,proto3" json:"exec_mode,omitempty"` + ExecMode uint32 `protobuf:"varint,5,opt,name=exec_mode,json=execMode,proto3" json:"exec_mode,omitempty"` } func (m *StreamUpdate) Reset() { *m = StreamUpdate{} } @@ -921,9 +922,13 @@ type StreamUpdate_OrderbookUpdate struct { type StreamUpdate_OrderFill struct { OrderFill *StreamOrderbookFill `protobuf:"bytes,2,opt,name=order_fill,json=orderFill,proto3,oneof" json:"order_fill,omitempty"` } +type StreamUpdate_TakerOrder struct { + TakerOrder *StreamTakerOrder `protobuf:"bytes,3,opt,name=taker_order,json=takerOrder,proto3,oneof" json:"taker_order,omitempty"` +} func (*StreamUpdate_OrderbookUpdate) isStreamUpdate_UpdateMessage() {} func (*StreamUpdate_OrderFill) isStreamUpdate_UpdateMessage() {} +func (*StreamUpdate_TakerOrder) isStreamUpdate_UpdateMessage() {} func (m *StreamUpdate) GetUpdateMessage() isStreamUpdate_UpdateMessage { if m != nil { @@ -946,6 +951,13 @@ func (m *StreamUpdate) GetOrderFill() *StreamOrderbookFill { return nil } +func (m *StreamUpdate) GetTakerOrder() *StreamTakerOrder { + if x, ok := m.GetUpdateMessage().(*StreamUpdate_TakerOrder); ok { + return x.TakerOrder + } + return nil +} + func (m *StreamUpdate) GetBlockHeight() uint32 { if m != nil { return m.BlockHeight @@ -965,6 +977,7 @@ func (*StreamUpdate) XXX_OneofWrappers() []interface{} { return []interface{}{ (*StreamUpdate_OrderbookUpdate)(nil), (*StreamUpdate_OrderFill)(nil), + (*StreamUpdate_TakerOrder)(nil), } } @@ -1297,105 +1310,106 @@ func init() { func init() { proto.RegisterFile("dydxprotocol/clob/query.proto", fileDescriptor_3365c195b25c5bc0) } var fileDescriptor_3365c195b25c5bc0 = []byte{ - // 1557 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x57, 0xcf, 0x6f, 0xdc, 0x44, - 0x14, 0x5e, 0x6f, 0xd2, 0x36, 0x79, 0xdb, 0xb4, 0xe9, 0xa4, 0x69, 0xb7, 0x9b, 0x74, 0x93, 0x1a, - 0x9a, 0x26, 0x29, 0x5d, 0x27, 0x69, 0x55, 0x95, 0x06, 0x15, 0x25, 0x81, 0x36, 0x95, 0x1a, 0x9a, - 0xba, 0x69, 0x1b, 0x41, 0x25, 0x6b, 0xd6, 0x9e, 0x38, 0x56, 0x6c, 0xcf, 0xc6, 0x1e, 0xaf, 0x12, - 0x21, 0x04, 0xe2, 0xd0, 0x0b, 0x20, 0x21, 0x71, 0xe0, 0x80, 0xc4, 0x85, 0x33, 0x12, 0x17, 0x8e, - 0x08, 0xb8, 0xf5, 0x58, 0x89, 0x0b, 0x07, 0x84, 0x50, 0xcb, 0x99, 0xbf, 0x01, 0x79, 0x66, 0xbc, - 0xd9, 0x1f, 0xf6, 0x26, 0xcd, 0x65, 0xd7, 0x7e, 0xf3, 0xbd, 0x6f, 0xbe, 0xf7, 0xe6, 0xcd, 0xcc, - 0x33, 0x9c, 0xb7, 0x76, 0xad, 0x9d, 0x5a, 0x40, 0x19, 0x35, 0xa9, 0xab, 0x99, 0x2e, 0xad, 0x6a, - 0xdb, 0x11, 0x09, 0x76, 0x2b, 0xdc, 0x86, 0x4e, 0x35, 0x0f, 0x57, 0xe2, 0xe1, 0xd2, 0x69, 0x9b, - 0xda, 0x94, 0x9b, 0xb4, 0xf8, 0x49, 0x00, 0x4b, 0xa3, 0x36, 0xa5, 0xb6, 0x4b, 0x34, 0x5c, 0x73, - 0x34, 0xec, 0xfb, 0x94, 0x61, 0xe6, 0x50, 0x3f, 0x94, 0xa3, 0xd3, 0x26, 0x0d, 0x3d, 0x1a, 0x6a, - 0x55, 0x1c, 0x12, 0xc1, 0xaf, 0xd5, 0x67, 0xab, 0x84, 0xe1, 0x59, 0xad, 0x86, 0x6d, 0xc7, 0xe7, - 0x60, 0x89, 0xd5, 0x3a, 0x15, 0x55, 0x5d, 0x6a, 0x6e, 0x19, 0x01, 0x66, 0xc4, 0x70, 0x1d, 0xcf, - 0x61, 0x86, 0x49, 0xfd, 0x0d, 0xc7, 0x96, 0x0e, 0x17, 0x3a, 0x1d, 0xe2, 0x1f, 0xa3, 0x86, 0x9d, - 0x40, 0x42, 0x66, 0x3a, 0x21, 0x64, 0x3b, 0x72, 0xd8, 0xae, 0xc1, 0x1c, 0x12, 0xa4, 0x91, 0xa6, - 0xe4, 0x85, 0x06, 0x16, 0x49, 0x08, 0xc7, 0x3a, 0x87, 0x3d, 0xcc, 0xcc, 0x4d, 0x92, 0x44, 0x7c, - 0xb9, 0x13, 0xe0, 0x3a, 0xdb, 0x91, 0x63, 0x89, 0xbc, 0xb4, 0x4e, 0x36, 0x92, 0xc2, 0x46, 0xea, - 0x72, 0xf0, 0x56, 0xcb, 0xa0, 0xe3, 0x5b, 0x64, 0x87, 0x04, 0x1a, 0xdd, 0xd8, 0x30, 0xcc, 0x4d, - 0xec, 0xf8, 0x46, 0x54, 0xb3, 0x30, 0x23, 0x61, 0xa7, 0x45, 0xf8, 0xab, 0x53, 0x70, 0xf6, 0x41, - 0x9c, 0xf1, 0x3b, 0x84, 0x2d, 0xb9, 0xb4, 0xba, 0x8a, 0x9d, 0x40, 0x27, 0xdb, 0x11, 0x09, 0x19, - 0x3a, 0x01, 0x79, 0xc7, 0x2a, 0x2a, 0xe3, 0xca, 0xe4, 0x80, 0x9e, 0x77, 0x2c, 0xf5, 0x09, 0x0c, - 0x73, 0xe8, 0x1e, 0x2e, 0xac, 0x51, 0x3f, 0x24, 0xe8, 0x16, 0xf4, 0x37, 0x52, 0xca, 0xf1, 0x85, - 0xb9, 0x91, 0x4a, 0x47, 0x69, 0x54, 0x12, 0xbf, 0xc5, 0xde, 0xe7, 0x7f, 0x8f, 0xe5, 0xf4, 0x3e, - 0x53, 0xbe, 0xab, 0x58, 0x6a, 0x58, 0x70, 0xdd, 0x76, 0x0d, 0xb7, 0x01, 0xf6, 0x4a, 0x40, 0x72, - 0x4f, 0x54, 0x44, 0xbd, 0x54, 0xe2, 0x7a, 0xa9, 0x88, 0x7a, 0x94, 0xf5, 0x52, 0x59, 0xc5, 0x36, - 0x91, 0xbe, 0x7a, 0x93, 0xa7, 0xfa, 0x83, 0x02, 0xc5, 0x16, 0xf1, 0x0b, 0xae, 0x9b, 0xa5, 0xbf, - 0xe7, 0x35, 0xf5, 0xa3, 0x3b, 0x2d, 0x22, 0xf3, 0x5c, 0xe4, 0xa5, 0x7d, 0x45, 0x8a, 0xc9, 0x5b, - 0x54, 0xfe, 0xa5, 0xc0, 0xd8, 0x0a, 0xa9, 0x7f, 0x40, 0x2d, 0xb2, 0x46, 0xe3, 0xdf, 0x25, 0xec, - 0x9a, 0x91, 0xcb, 0x07, 0x93, 0x8c, 0x3c, 0x85, 0x33, 0xa2, 0xe0, 0x6b, 0x01, 0xad, 0xd1, 0x90, - 0x04, 0x86, 0x2c, 0xad, 0x46, 0x76, 0x3a, 0x95, 0x3f, 0xc6, 0x6e, 0x5c, 0x5a, 0x34, 0x58, 0x21, - 0xf5, 0x15, 0x81, 0xd6, 0x4f, 0x73, 0x96, 0x55, 0x49, 0x22, 0xad, 0xe8, 0x23, 0x18, 0xae, 0x27, - 0x60, 0xc3, 0x23, 0x75, 0xc3, 0x23, 0x2c, 0x70, 0xcc, 0xb0, 0x11, 0x55, 0x27, 0x79, 0x8b, 0xe0, - 0x15, 0x01, 0xd7, 0x87, 0xea, 0xcd, 0x53, 0x0a, 0xa3, 0xfa, 0x9f, 0x02, 0xe3, 0xd9, 0xe1, 0xc9, - 0xc5, 0xb0, 0xe1, 0x58, 0x40, 0xc2, 0xc8, 0x65, 0xa1, 0x5c, 0x8a, 0x3b, 0xfb, 0xcd, 0x99, 0xc2, - 0x12, 0x03, 0x16, 0x7c, 0xeb, 0x31, 0x75, 0x23, 0x8f, 0xac, 0x92, 0x20, 0x5e, 0x3a, 0xb9, 0x6c, - 0x09, 0x7b, 0x09, 0xc3, 0x50, 0x0a, 0x0a, 0x8d, 0xc3, 0xf1, 0x46, 0x31, 0x18, 0x8d, 0xfa, 0x87, - 0x64, 0xb1, 0xef, 0x5a, 0x68, 0x10, 0x7a, 0x3c, 0x52, 0xe7, 0x19, 0xc9, 0xeb, 0xf1, 0x23, 0x3a, - 0x03, 0x47, 0xeb, 0x9c, 0xa4, 0xd8, 0x33, 0xae, 0x4c, 0xf6, 0xea, 0xf2, 0x4d, 0x9d, 0x86, 0x49, - 0x5e, 0x74, 0xef, 0xf3, 0xd3, 0x64, 0xcd, 0x21, 0xc1, 0xbd, 0xf8, 0x2c, 0x59, 0xe2, 0xbb, 0x3b, - 0x0a, 0x9a, 0xd7, 0x55, 0xfd, 0x4e, 0x81, 0xa9, 0x03, 0x80, 0x65, 0x96, 0x7c, 0x28, 0x66, 0x1d, - 0x51, 0xb2, 0x0e, 0xb4, 0x94, 0xb4, 0x75, 0xa3, 0x96, 0xe9, 0x19, 0x26, 0x69, 0x18, 0x75, 0x0a, - 0x2e, 0x71, 0x71, 0x8b, 0x71, 0xd1, 0xe8, 0x98, 0x91, 0xec, 0x40, 0xbe, 0x55, 0x64, 0xd4, 0x5d, - 0xb1, 0x32, 0x8e, 0x2d, 0x38, 0x9b, 0x71, 0x7c, 0xcb, 0x30, 0x2a, 0x29, 0x61, 0x74, 0x21, 0x96, - 0x51, 0x88, 0xe2, 0x6e, 0x83, 0xa8, 0xeb, 0x70, 0x8e, 0x0b, 0x7b, 0xc8, 0x30, 0x23, 0x1b, 0x91, - 0x7b, 0x3f, 0x3e, 0xb2, 0x93, 0x7d, 0x35, 0x0f, 0x7d, 0xfc, 0x08, 0x4f, 0xd6, 0xbc, 0x30, 0x57, - 0x4a, 0x99, 0x9a, 0xbb, 0xdc, 0xb5, 0x92, 0x5a, 0xa2, 0xe2, 0x55, 0xfd, 0x59, 0x81, 0x52, 0x1a, - 0xb5, 0x8c, 0x72, 0x1d, 0x4e, 0x0a, 0xee, 0x9a, 0x8b, 0x4d, 0xe2, 0x11, 0x9f, 0xc9, 0x29, 0xa6, - 0x52, 0xa6, 0xb8, 0x47, 0x7d, 0x7b, 0x8d, 0x04, 0x1e, 0xa7, 0x58, 0x4d, 0x1c, 0xe4, 0x8c, 0x27, - 0x68, 0x8b, 0x15, 0x8d, 0x41, 0x61, 0xc3, 0x71, 0x5d, 0x03, 0x7b, 0x34, 0xf2, 0x19, 0xaf, 0xc9, - 0x5e, 0x1d, 0x62, 0xd3, 0x02, 0xb7, 0xa0, 0x51, 0xe8, 0x67, 0x81, 0x63, 0xdb, 0x24, 0x20, 0x16, - 0xaf, 0xce, 0x3e, 0x7d, 0xcf, 0xa0, 0x5e, 0x82, 0x8b, 0x5c, 0xf6, 0xbd, 0xa6, 0xcb, 0x27, 0x75, - 0x51, 0x9f, 0x29, 0x30, 0xb1, 0x1f, 0x52, 0x06, 0xfb, 0x14, 0x86, 0x52, 0xee, 0x32, 0x19, 0xf0, - 0xc5, 0xb4, 0x80, 0x3b, 0x28, 0x65, 0xb0, 0xc8, 0xed, 0x18, 0x51, 0x17, 0xe0, 0xfc, 0x43, 0x16, - 0x10, 0x2c, 0xd2, 0x53, 0xa5, 0x74, 0xeb, 0x91, 0xb8, 0xcf, 0x92, 0x75, 0xec, 0xdc, 0xbf, 0x3d, - 0xad, 0xfb, 0x57, 0xc5, 0x50, 0xce, 0xa2, 0x90, 0x21, 0xbc, 0x0b, 0xc7, 0xe4, 0x2d, 0x29, 0xcf, - 0xa0, 0xb1, 0x14, 0xd9, 0x82, 0x43, 0xb8, 0x26, 0xf5, 0x20, 0xbd, 0xd4, 0xcf, 0xf2, 0x70, 0xbc, - 0x79, 0x1c, 0x3d, 0x82, 0x41, 0x9a, 0xcc, 0x26, 0x6f, 0x60, 0x99, 0x91, 0xc9, 0x4c, 0xea, 0x36, - 0x79, 0xcb, 0x39, 0xfd, 0x24, 0x6d, 0x35, 0xc5, 0x37, 0x8f, 0x28, 0xac, 0x78, 0xc5, 0xe5, 0x19, - 0x3d, 0xb1, 0x3f, 0xe1, 0x6d, 0xc7, 0x75, 0x97, 0x73, 0x7a, 0x3f, 0xf7, 0x8d, 0x5f, 0xd0, 0x05, - 0x38, 0x2e, 0xf6, 0xe1, 0x26, 0x71, 0xec, 0x4d, 0xc6, 0x2b, 0x65, 0x40, 0x2f, 0x70, 0xdb, 0x32, - 0x37, 0xa1, 0x11, 0xe8, 0x27, 0x3b, 0xc4, 0x34, 0x3c, 0x6a, 0x91, 0x62, 0x2f, 0x1f, 0xef, 0x8b, - 0x0d, 0x2b, 0xd4, 0x22, 0x8b, 0x83, 0x70, 0x42, 0x44, 0x65, 0x78, 0x24, 0x0c, 0xb1, 0x4d, 0xd4, - 0xaf, 0x14, 0x18, 0x4e, 0x8d, 0x03, 0xad, 0xb7, 0x67, 0xf7, 0x46, 0xab, 0x62, 0xd9, 0xc4, 0x54, - 0x3a, 0x5b, 0x96, 0xfb, 0x1b, 0x1b, 0x4b, 0xb1, 0x41, 0x10, 0x3d, 0x9e, 0x6d, 0x4b, 0x3b, 0x2a, - 0x41, 0x5f, 0xe8, 0xe3, 0x5a, 0xb8, 0x49, 0xc5, 0x56, 0xe8, 0xd3, 0x1b, 0xef, 0xea, 0x8f, 0x0a, - 0x0c, 0xa5, 0xa4, 0x01, 0xcd, 0x03, 0xaf, 0x0d, 0x71, 0x8b, 0xca, 0x35, 0x19, 0xcd, 0xb8, 0xfd, - 0xf9, 0x2d, 0xa9, 0xf3, 0x66, 0x81, 0x3f, 0xa2, 0xeb, 0x70, 0x94, 0xe7, 0x30, 0xbe, 0x1f, 0xe3, - 0x48, 0x8a, 0x59, 0x47, 0x86, 0x54, 0x2a, 0xd1, 0x71, 0xba, 0x9b, 0xb6, 0x6d, 0x58, 0xec, 0x19, - 0xef, 0x99, 0xec, 0xd5, 0x0b, 0x7b, 0xfb, 0x36, 0x54, 0x9f, 0xe5, 0x61, 0x50, 0xe8, 0x5d, 0xc3, - 0x5b, 0x24, 0xe0, 0x2c, 0x68, 0x06, 0x8e, 0x70, 0x06, 0xa9, 0x33, 0x73, 0xba, 0xe5, 0x9c, 0x2e, - 0x80, 0x68, 0x1d, 0x4e, 0x35, 0xed, 0x22, 0x43, 0x78, 0xe7, 0x33, 0x0f, 0x1f, 0x31, 0x63, 0xd3, - 0x8e, 0x4c, 0xe8, 0x06, 0xdd, 0x36, 0x1b, 0x7a, 0x02, 0x88, 0xc5, 0xca, 0x04, 0xa7, 0x11, 0x32, - 0xcc, 0xa2, 0x90, 0x17, 0x4e, 0x37, 0xea, 0xbd, 0x60, 0x1e, 0x72, 0x07, 0x7d, 0x90, 0xb5, 0x59, - 0x16, 0x07, 0xa0, 0xd0, 0x44, 0xac, 0xfe, 0xa4, 0xc0, 0x99, 0x74, 0xdf, 0x38, 0x8d, 0x2d, 0x93, - 0x8b, 0xbb, 0xba, 0x40, 0x9b, 0x20, 0x57, 0x00, 0x05, 0xc4, 0xc3, 0x8e, 0xef, 0xf8, 0xb6, 0xb1, - 0x1d, 0x61, 0x9f, 0x45, 0x5e, 0x28, 0xcf, 0xc9, 0x53, 0x8d, 0x91, 0x07, 0x72, 0x00, 0xbd, 0x07, - 0x65, 0x5a, 0x63, 0x8e, 0xe7, 0x84, 0xcc, 0x31, 0xb1, 0xeb, 0xee, 0xf2, 0x9d, 0x45, 0xac, 0x3d, - 0x57, 0x71, 0xc3, 0x8f, 0xb6, 0xa2, 0x6e, 0x73, 0x50, 0xc2, 0x32, 0xf7, 0x3d, 0xc0, 0x11, 0x7e, - 0x5a, 0xa2, 0x2f, 0x14, 0xe8, 0x4b, 0xfa, 0x46, 0x34, 0x9d, 0x92, 0x95, 0x8c, 0xe6, 0xbb, 0x34, - 0x99, 0x85, 0x6d, 0xef, 0xbe, 0xd5, 0xa9, 0xcf, 0xff, 0xf8, 0xf7, 0x9b, 0xfc, 0x1b, 0xe8, 0x82, - 0xd6, 0xe5, 0x4b, 0x47, 0xfb, 0xd8, 0xb1, 0x3e, 0x41, 0x5f, 0x2a, 0x50, 0x68, 0x6a, 0x80, 0xb3, - 0x05, 0x75, 0x76, 0xe2, 0xa5, 0xcb, 0xfb, 0x09, 0x6a, 0xea, 0xa8, 0xd5, 0x37, 0xb9, 0xa6, 0x32, - 0x1a, 0xed, 0xa6, 0x09, 0xfd, 0xaa, 0x40, 0x31, 0xab, 0x93, 0x43, 0x73, 0xaf, 0xd5, 0xf6, 0x09, - 0x8d, 0x57, 0x0f, 0xd1, 0x2a, 0xaa, 0x37, 0xb9, 0xd6, 0x6b, 0x37, 0x95, 0x69, 0x55, 0xd3, 0x52, - 0x3f, 0xb5, 0x0c, 0x9f, 0x5a, 0xc4, 0x60, 0x54, 0xfc, 0x9b, 0x4d, 0x22, 0x7f, 0x57, 0x60, 0xb4, - 0x5b, 0x53, 0x85, 0xe6, 0xb3, 0xb2, 0x76, 0x80, 0x96, 0xb0, 0xf4, 0xce, 0xe1, 0x9c, 0x65, 0x5c, - 0x13, 0x3c, 0xae, 0x71, 0x54, 0xd6, 0xba, 0x7e, 0xde, 0xa2, 0x5f, 0x14, 0x18, 0xe9, 0xd2, 0x51, - 0xa1, 0x9b, 0x59, 0x2a, 0xf6, 0xef, 0x05, 0x4b, 0xf3, 0x87, 0xf2, 0x95, 0x01, 0x5c, 0xe4, 0x01, - 0x8c, 0xa1, 0xf3, 0x5d, 0xbf, 0xf9, 0xd1, 0x6f, 0x0a, 0x9c, 0xcb, 0xec, 0x4a, 0xd0, 0x8d, 0x2c, - 0x05, 0xfb, 0xb5, 0x3c, 0xa5, 0xb7, 0x0f, 0xe1, 0x29, 0x95, 0x57, 0xb8, 0xf2, 0x49, 0x34, 0xa1, - 0x1d, 0xe8, 0x3b, 0x1f, 0xf9, 0x30, 0xd0, 0xd2, 0x38, 0xa2, 0xb7, 0xb2, 0xe6, 0x4e, 0x6b, 0x5d, - 0x4b, 0x57, 0x0e, 0x88, 0x96, 0xea, 0x72, 0xe8, 0xd3, 0xe4, 0x44, 0x6d, 0xef, 0x80, 0xd0, 0xcc, - 0x41, 0xbb, 0x91, 0xa4, 0xdf, 0x2a, 0xcd, 0xbe, 0x86, 0x87, 0x10, 0x30, 0xa3, 0x2c, 0xae, 0x3e, - 0x7f, 0x59, 0x56, 0x5e, 0xbc, 0x2c, 0x2b, 0xff, 0xbc, 0x2c, 0x2b, 0x5f, 0xbf, 0x2a, 0xe7, 0x5e, - 0xbc, 0x2a, 0xe7, 0xfe, 0x7c, 0x55, 0xce, 0x7d, 0x78, 0xdd, 0x76, 0xd8, 0x66, 0x54, 0xad, 0x98, - 0xd4, 0x6b, 0x4d, 0x5e, 0xfd, 0xda, 0x15, 0xde, 0x0c, 0x68, 0x0d, 0xcb, 0x8e, 0x48, 0x28, 0xdb, - 0xad, 0x91, 0xb0, 0x7a, 0x94, 0x9b, 0xaf, 0xfe, 0x1f, 0x00, 0x00, 0xff, 0xff, 0xe0, 0x10, 0x44, - 0xf0, 0xb2, 0x12, 0x00, 0x00, + // 1578 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x58, 0xcf, 0x4f, 0x1b, 0x47, + 0x14, 0xf6, 0x1a, 0x92, 0xc0, 0x73, 0x7e, 0x90, 0x21, 0x24, 0x8e, 0x21, 0x86, 0x6c, 0x1a, 0x02, + 0xa4, 0xf1, 0x02, 0x89, 0xa2, 0x34, 0x54, 0xa9, 0x80, 0x96, 0x10, 0x29, 0x34, 0x64, 0x43, 0x12, + 0xd4, 0x46, 0x5a, 0x8d, 0xbd, 0xc3, 0xb2, 0x62, 0x77, 0xc7, 0xec, 0xce, 0x5a, 0xa0, 0xaa, 0xaa, + 0xd4, 0x43, 0x2e, 0x6d, 0xa5, 0x4a, 0x3d, 0xf4, 0x50, 0xa9, 0x97, 0x9e, 0x2b, 0xf5, 0xd2, 0x63, + 0xd5, 0xf6, 0x96, 0x63, 0xa4, 0x5e, 0x7a, 0xa8, 0xaa, 0x2a, 0xe9, 0xb9, 0x7f, 0x43, 0xb5, 0x33, + 0xb3, 0xc6, 0x6b, 0xef, 0x1a, 0xc2, 0x05, 0x3c, 0x6f, 0xde, 0x7b, 0xf3, 0xbd, 0xf7, 0xbe, 0x79, + 0xf3, 0x6c, 0xb8, 0x60, 0xee, 0x9a, 0x3b, 0x75, 0x9f, 0x32, 0x5a, 0xa3, 0x8e, 0x56, 0x73, 0x68, + 0x55, 0xdb, 0x0e, 0x89, 0xbf, 0x5b, 0xe1, 0x32, 0x74, 0xba, 0x75, 0xbb, 0x12, 0x6d, 0x97, 0xce, + 0x58, 0xd4, 0xa2, 0x5c, 0xa4, 0x45, 0x9f, 0x84, 0x62, 0x69, 0xc4, 0xa2, 0xd4, 0x72, 0x88, 0x86, + 0xeb, 0xb6, 0x86, 0x3d, 0x8f, 0x32, 0xcc, 0x6c, 0xea, 0x05, 0x72, 0x77, 0xaa, 0x46, 0x03, 0x97, + 0x06, 0x5a, 0x15, 0x07, 0x44, 0xf8, 0xd7, 0x1a, 0x33, 0x55, 0xc2, 0xf0, 0x8c, 0x56, 0xc7, 0x96, + 0xed, 0x71, 0x65, 0xa9, 0xab, 0x75, 0x22, 0xaa, 0x3a, 0xb4, 0xb6, 0x65, 0xf8, 0x98, 0x11, 0xc3, + 0xb1, 0x5d, 0x9b, 0x19, 0x35, 0xea, 0x6d, 0xd8, 0x96, 0x34, 0xb8, 0xd8, 0x69, 0x10, 0xfd, 0x31, + 0xea, 0xd8, 0xf6, 0xa5, 0xca, 0x74, 0xa7, 0x0a, 0xd9, 0x0e, 0x6d, 0xb6, 0x6b, 0x30, 0x9b, 0xf8, + 0x69, 0x4e, 0x53, 0xf2, 0x42, 0x7d, 0x93, 0xc4, 0x0e, 0x47, 0x3b, 0xb7, 0x5d, 0xcc, 0x6a, 0x9b, + 0x24, 0x8e, 0xf8, 0x6a, 0xa7, 0x82, 0x63, 0x6f, 0x87, 0xb6, 0x29, 0xf2, 0x92, 0x3c, 0x6c, 0x38, + 0xc5, 0x1b, 0x69, 0xc8, 0xcd, 0x3b, 0x89, 0x4d, 0xdb, 0x33, 0xc9, 0x0e, 0xf1, 0x35, 0xba, 0xb1, + 0x61, 0xd4, 0x36, 0xb1, 0xed, 0x19, 0x61, 0xdd, 0xc4, 0x8c, 0x04, 0x9d, 0x12, 0x61, 0xaf, 0x4e, + 0xc2, 0xb9, 0x87, 0x51, 0xc6, 0xef, 0x12, 0xb6, 0xe8, 0xd0, 0xea, 0x2a, 0xb6, 0x7d, 0x9d, 0x6c, + 0x87, 0x24, 0x60, 0xe8, 0x24, 0xe4, 0x6d, 0xb3, 0xa8, 0x8c, 0x29, 0x13, 0x27, 0xf4, 0xbc, 0x6d, + 0xaa, 0x4f, 0x61, 0x88, 0xab, 0xee, 0xe9, 0x05, 0x75, 0xea, 0x05, 0x04, 0xdd, 0x81, 0xfe, 0x66, + 0x4a, 0xb9, 0x7e, 0x61, 0x76, 0xb8, 0xd2, 0x41, 0x8d, 0x4a, 0x6c, 0xb7, 0xd0, 0xfb, 0xe2, 0xef, + 0xd1, 0x9c, 0xde, 0x57, 0x93, 0x6b, 0x15, 0x4b, 0x0c, 0xf3, 0x8e, 0xd3, 0x8e, 0x61, 0x09, 0x60, + 0x8f, 0x02, 0xd2, 0xf7, 0x78, 0x45, 0xf0, 0xa5, 0x12, 0xf1, 0xa5, 0x22, 0xf8, 0x28, 0xf9, 0x52, + 0x59, 0xc5, 0x16, 0x91, 0xb6, 0x7a, 0x8b, 0xa5, 0xfa, 0x83, 0x02, 0xc5, 0x04, 0xf8, 0x79, 0xc7, + 0xc9, 0xc2, 0xdf, 0xf3, 0x86, 0xf8, 0xd1, 0xdd, 0x04, 0xc8, 0x3c, 0x07, 0x79, 0x65, 0x5f, 0x90, + 0xe2, 0xf0, 0x04, 0xca, 0xbf, 0x14, 0x18, 0x5d, 0x21, 0x8d, 0x0f, 0xa9, 0x49, 0xd6, 0x68, 0xf4, + 0x77, 0x11, 0x3b, 0xb5, 0xd0, 0xe1, 0x9b, 0x71, 0x46, 0x9e, 0xc1, 0x59, 0x41, 0xf8, 0xba, 0x4f, + 0xeb, 0x34, 0x20, 0xbe, 0x21, 0xa9, 0xd5, 0xcc, 0x4e, 0x27, 0xf2, 0x27, 0xd8, 0x89, 0xa8, 0x45, + 0xfd, 0x15, 0xd2, 0x58, 0x11, 0xda, 0xfa, 0x19, 0xee, 0x65, 0x55, 0x3a, 0x91, 0x52, 0xf4, 0x31, + 0x0c, 0x35, 0x62, 0x65, 0xc3, 0x25, 0x0d, 0xc3, 0x25, 0xcc, 0xb7, 0x6b, 0x41, 0x33, 0xaa, 0x4e, + 0xe7, 0x09, 0xc0, 0x2b, 0x42, 0x5d, 0x1f, 0x6c, 0xb4, 0x1e, 0x29, 0x84, 0xea, 0x7f, 0x0a, 0x8c, + 0x65, 0x87, 0x27, 0x8b, 0x61, 0xc1, 0x31, 0x9f, 0x04, 0xa1, 0xc3, 0x02, 0x59, 0x8a, 0xbb, 0xfb, + 0x9d, 0x99, 0xe2, 0x25, 0x52, 0x98, 0xf7, 0xcc, 0x27, 0xd4, 0x09, 0x5d, 0xb2, 0x4a, 0xfc, 0xa8, + 0x74, 0xb2, 0x6c, 0xb1, 0xf7, 0x12, 0x86, 0xc1, 0x14, 0x2d, 0x34, 0x06, 0xc7, 0x9b, 0x64, 0x30, + 0x9a, 0xfc, 0x87, 0xb8, 0xd8, 0xf7, 0x4c, 0x34, 0x00, 0x3d, 0x2e, 0x69, 0xf0, 0x8c, 0xe4, 0xf5, + 0xe8, 0x23, 0x3a, 0x0b, 0x47, 0x1b, 0xdc, 0x49, 0xb1, 0x67, 0x4c, 0x99, 0xe8, 0xd5, 0xe5, 0x4a, + 0x9d, 0x82, 0x09, 0x4e, 0xba, 0x0f, 0x78, 0x37, 0x59, 0xb3, 0x89, 0x7f, 0x3f, 0xea, 0x25, 0x8b, + 0xfc, 0x76, 0x87, 0x7e, 0x6b, 0x5d, 0xd5, 0xef, 0x14, 0x98, 0x3c, 0x80, 0xb2, 0xcc, 0x92, 0x07, + 0xc5, 0xac, 0x16, 0x25, 0x79, 0xa0, 0xa5, 0xa4, 0xad, 0x9b, 0x6b, 0x99, 0x9e, 0x21, 0x92, 0xa6, + 0xa3, 0x4e, 0xc2, 0x15, 0x0e, 0x6e, 0x21, 0x22, 0x8d, 0x8e, 0x19, 0xc9, 0x0e, 0xe4, 0x5b, 0x45, + 0x46, 0xdd, 0x55, 0x57, 0xc6, 0xb1, 0x05, 0xe7, 0x32, 0xda, 0xb7, 0x0c, 0xa3, 0x92, 0x12, 0x46, + 0x17, 0xc7, 0x32, 0x0a, 0x41, 0xee, 0x36, 0x15, 0x75, 0x1d, 0xce, 0x73, 0x60, 0x8f, 0x18, 0x66, + 0x64, 0x23, 0x74, 0x1e, 0x44, 0x2d, 0x3b, 0xbe, 0x57, 0x73, 0xd0, 0xc7, 0x5b, 0x78, 0x5c, 0xf3, + 0xc2, 0x6c, 0x29, 0xe5, 0x68, 0x6e, 0x72, 0xcf, 0x8c, 0xb9, 0x44, 0xc5, 0x52, 0xfd, 0x59, 0x81, + 0x52, 0x9a, 0x6b, 0x19, 0xe5, 0x3a, 0x9c, 0x12, 0xbe, 0xeb, 0x0e, 0xae, 0x11, 0x97, 0x78, 0x4c, + 0x1e, 0x31, 0x99, 0x72, 0xc4, 0x7d, 0xea, 0x59, 0x6b, 0xc4, 0x77, 0xb9, 0x8b, 0xd5, 0xd8, 0x40, + 0x9e, 0x78, 0x92, 0x26, 0xa4, 0x68, 0x14, 0x0a, 0x1b, 0xb6, 0xe3, 0x18, 0xd8, 0xa5, 0xa1, 0xc7, + 0x38, 0x27, 0x7b, 0x75, 0x88, 0x44, 0xf3, 0x5c, 0x82, 0x46, 0xa0, 0x9f, 0xf9, 0xb6, 0x65, 0x11, + 0x9f, 0x98, 0x9c, 0x9d, 0x7d, 0xfa, 0x9e, 0x40, 0xbd, 0x02, 0x97, 0x39, 0xec, 0xfb, 0x2d, 0x8f, + 0x4f, 0x6a, 0x51, 0x9f, 0x2b, 0x30, 0xbe, 0x9f, 0xa6, 0x0c, 0xf6, 0x19, 0x0c, 0xa6, 0xbc, 0x65, + 0x32, 0xe0, 0xcb, 0x69, 0x01, 0x77, 0xb8, 0x94, 0xc1, 0x22, 0xa7, 0x63, 0x47, 0x9d, 0x87, 0x0b, + 0x8f, 0x98, 0x4f, 0xb0, 0x48, 0x4f, 0x95, 0xd2, 0xad, 0xc7, 0xe2, 0x3d, 0x8b, 0xeb, 0xd8, 0x79, + 0x7f, 0x7b, 0x92, 0xf7, 0x57, 0xc5, 0x50, 0xce, 0x72, 0x21, 0x43, 0x78, 0x0f, 0x8e, 0xc9, 0x57, + 0x52, 0xf6, 0xa0, 0xd1, 0x14, 0xd8, 0xc2, 0x87, 0x30, 0x8d, 0xf9, 0x20, 0xad, 0xd4, 0x17, 0x79, + 0x38, 0xde, 0xba, 0x8f, 0x1e, 0xc3, 0x00, 0x8d, 0x4f, 0x93, 0x2f, 0xb0, 0xcc, 0xc8, 0x44, 0xa6, + 0xeb, 0x36, 0x78, 0xcb, 0x39, 0xfd, 0x14, 0x4d, 0x8a, 0xa2, 0x97, 0x47, 0x10, 0x2b, 0xaa, 0xb8, + 0xec, 0xd1, 0xe3, 0xfb, 0x3b, 0x5c, 0xb2, 0x1d, 0x67, 0x39, 0xa7, 0xf7, 0x73, 0xdb, 0x68, 0x81, + 0x96, 0xa0, 0xc0, 0xf0, 0x16, 0xf1, 0x0d, 0x2e, 0xe2, 0x44, 0x29, 0xcc, 0x5e, 0xca, 0xf4, 0xb4, + 0x16, 0xe9, 0x72, 0x77, 0xcb, 0x39, 0x1d, 0x58, 0x73, 0x85, 0x2e, 0xc2, 0x71, 0x71, 0x9f, 0x37, + 0x89, 0x6d, 0x6d, 0xb2, 0x62, 0x2f, 0xef, 0x9e, 0x05, 0x2e, 0x5b, 0xe6, 0x22, 0x34, 0x0c, 0xfd, + 0x64, 0x87, 0xd4, 0x0c, 0x97, 0x9a, 0xa4, 0x78, 0x84, 0xef, 0xf7, 0x45, 0x82, 0x15, 0x6a, 0x92, + 0x85, 0x01, 0x38, 0x29, 0xb2, 0x63, 0xb8, 0x24, 0x08, 0xb0, 0x45, 0xd4, 0xaf, 0x14, 0x18, 0x4a, + 0xcd, 0x07, 0x5a, 0x6f, 0xaf, 0xd2, 0xad, 0x24, 0x5e, 0x39, 0x0c, 0x55, 0x3a, 0x47, 0x9f, 0x07, + 0x1b, 0x1b, 0x8b, 0x91, 0x40, 0x38, 0x7a, 0x32, 0xd3, 0x56, 0x3e, 0x54, 0x82, 0xbe, 0xc0, 0xc3, + 0xf5, 0x60, 0x93, 0x8a, 0x2b, 0xd5, 0xa7, 0x37, 0xd7, 0xea, 0x8f, 0x0a, 0x0c, 0xa6, 0xa4, 0x13, + 0xcd, 0x01, 0xe7, 0x98, 0x78, 0x8d, 0x65, 0x6d, 0x47, 0x32, 0xa6, 0x08, 0xfe, 0xda, 0xea, 0x7c, + 0xe8, 0xe0, 0x1f, 0xd1, 0x4d, 0x38, 0xca, 0x13, 0x1f, 0xbd, 0xb3, 0x51, 0x24, 0xc5, 0xac, 0xd6, + 0x23, 0x91, 0x4a, 0xed, 0x28, 0xdd, 0x2d, 0xd7, 0x3f, 0x28, 0xf6, 0x8c, 0xf5, 0x4c, 0xf4, 0xea, + 0x85, 0xbd, 0xfb, 0x1f, 0xa8, 0xcf, 0xf3, 0x30, 0xd0, 0x5e, 0x34, 0x34, 0x0d, 0x47, 0x44, 0xa1, + 0x05, 0xce, 0xcc, 0xe3, 0x96, 0x73, 0xba, 0x50, 0x44, 0xeb, 0x70, 0xba, 0xe5, 0x36, 0x4a, 0x9a, + 0xe4, 0x33, 0x9b, 0x98, 0x38, 0xb1, 0xe5, 0x66, 0xc7, 0xee, 0x06, 0x9c, 0x36, 0x19, 0x7a, 0x0a, + 0xa8, 0x85, 0x7a, 0x46, 0xc0, 0x30, 0x0b, 0x03, 0xc9, 0xc0, 0xc9, 0x03, 0x30, 0xf0, 0x11, 0x37, + 0xd0, 0x07, 0x58, 0x9b, 0x64, 0xe1, 0x44, 0x82, 0xd3, 0xea, 0x4f, 0x0a, 0x9c, 0x4d, 0xb7, 0x8d, + 0xd2, 0x98, 0x38, 0x5c, 0xbc, 0xf9, 0x05, 0xda, 0xa2, 0x72, 0x0d, 0x90, 0x4f, 0x5c, 0x6c, 0x7b, + 0xb6, 0x67, 0x19, 0xdb, 0x21, 0xf6, 0x58, 0xe8, 0x06, 0xb2, 0xdf, 0x9e, 0x6e, 0xee, 0x3c, 0x94, + 0x1b, 0xe8, 0x7d, 0x28, 0xd3, 0x3a, 0xb3, 0x5d, 0x3b, 0x60, 0x76, 0x0d, 0x3b, 0xce, 0x2e, 0xbf, + 0xa1, 0xc4, 0xdc, 0x33, 0x15, 0x93, 0xc2, 0x48, 0x52, 0x6b, 0x89, 0x2b, 0xc5, 0x5e, 0x66, 0xbf, + 0x07, 0x38, 0xc2, 0xbb, 0x2e, 0xfa, 0x42, 0x81, 0xbe, 0x78, 0xfe, 0x44, 0x53, 0x29, 0x59, 0xc9, + 0x18, 0xe2, 0x4b, 0x13, 0x59, 0xba, 0xed, 0x53, 0xbc, 0x3a, 0xf9, 0xf9, 0x1f, 0xff, 0x7e, 0x93, + 0xbf, 0x84, 0x2e, 0x6a, 0x5d, 0xbe, 0x31, 0x69, 0x9f, 0xd8, 0xe6, 0xa7, 0xe8, 0x4b, 0x05, 0x0a, + 0x2d, 0x83, 0x74, 0x36, 0xa0, 0xce, 0x89, 0xbe, 0x74, 0x75, 0x3f, 0x40, 0x2d, 0x93, 0xb9, 0xfa, + 0x16, 0xc7, 0x54, 0x46, 0x23, 0xdd, 0x30, 0xa1, 0x5f, 0x15, 0x28, 0x66, 0x4d, 0x84, 0x68, 0xf6, + 0x8d, 0xc6, 0x47, 0x81, 0xf1, 0xfa, 0x21, 0x46, 0x4e, 0xf5, 0x36, 0xc7, 0x7a, 0xe3, 0xb6, 0x32, + 0xa5, 0x6a, 0x5a, 0xea, 0x57, 0x36, 0xc3, 0xa3, 0x26, 0x31, 0x18, 0x15, 0xff, 0x6b, 0x2d, 0x20, + 0x7f, 0x57, 0x60, 0xa4, 0xdb, 0x70, 0x86, 0xe6, 0xb2, 0xb2, 0x76, 0x80, 0xd1, 0xb2, 0xf4, 0xee, + 0xe1, 0x8c, 0x65, 0x5c, 0xe3, 0x3c, 0xae, 0x31, 0x54, 0xd6, 0xba, 0x7e, 0x4d, 0x46, 0xbf, 0x28, + 0x30, 0xdc, 0x65, 0x32, 0x43, 0xb7, 0xb3, 0x50, 0xec, 0x3f, 0x53, 0x96, 0xe6, 0x0e, 0x65, 0x2b, + 0x03, 0xb8, 0xcc, 0x03, 0x18, 0x45, 0x17, 0xba, 0xfe, 0x76, 0x80, 0x7e, 0x53, 0xe0, 0x7c, 0xe6, + 0x74, 0x83, 0x6e, 0x65, 0x21, 0xd8, 0x6f, 0x74, 0x2a, 0xbd, 0x73, 0x08, 0x4b, 0x89, 0xbc, 0xc2, + 0x91, 0x4f, 0xa0, 0x71, 0xed, 0x40, 0xbf, 0x17, 0x20, 0x0f, 0x4e, 0x24, 0x06, 0x50, 0xf4, 0x76, + 0xd6, 0xd9, 0x69, 0x23, 0x70, 0xe9, 0xda, 0x01, 0xb5, 0x25, 0xba, 0x1c, 0xfa, 0x2c, 0xee, 0xa8, + 0xed, 0x93, 0x14, 0x9a, 0x3e, 0xe8, 0x54, 0x13, 0xcf, 0x6d, 0xa5, 0x99, 0x37, 0xb0, 0x10, 0x00, + 0xa6, 0x95, 0x85, 0xd5, 0x17, 0xaf, 0xca, 0xca, 0xcb, 0x57, 0x65, 0xe5, 0x9f, 0x57, 0x65, 0xe5, + 0xeb, 0xd7, 0xe5, 0xdc, 0xcb, 0xd7, 0xe5, 0xdc, 0x9f, 0xaf, 0xcb, 0xb9, 0x8f, 0x6e, 0x5a, 0x36, + 0xdb, 0x0c, 0xab, 0x95, 0x1a, 0x75, 0x93, 0xc9, 0x6b, 0xdc, 0xb8, 0xc6, 0x87, 0x01, 0xad, 0x29, + 0xd9, 0x11, 0x09, 0x65, 0xbb, 0x75, 0x12, 0x54, 0x8f, 0x72, 0xf1, 0xf5, 0xff, 0x03, 0x00, 0x00, + 0xff, 0xff, 0xe3, 0x8b, 0xbe, 0xb6, 0xfa, 0x12, 0x00, 0x00, } // Reference imports to suppress errors if they are not otherwise used. @@ -2394,12 +2408,12 @@ func (m *StreamUpdate) MarshalToSizedBuffer(dAtA []byte) (int, error) { if m.ExecMode != 0 { i = encodeVarintQuery(dAtA, i, uint64(m.ExecMode)) i-- - dAtA[i] = 0x20 + dAtA[i] = 0x28 } if m.BlockHeight != 0 { i = encodeVarintQuery(dAtA, i, uint64(m.BlockHeight)) i-- - dAtA[i] = 0x18 + dAtA[i] = 0x20 } if m.UpdateMessage != nil { { @@ -2455,6 +2469,27 @@ func (m *StreamUpdate_OrderFill) MarshalToSizedBuffer(dAtA []byte) (int, error) } return len(dAtA) - i, nil } +func (m *StreamUpdate_TakerOrder) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *StreamUpdate_TakerOrder) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + if m.TakerOrder != nil { + { + size, err := m.TakerOrder.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintQuery(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x1a + } + return len(dAtA) - i, nil +} func (m *StreamOrderbookUpdate) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) @@ -2523,20 +2558,20 @@ func (m *StreamOrderbookFill) MarshalToSizedBuffer(dAtA []byte) (int, error) { var l int _ = l if len(m.FillAmounts) > 0 { - dAtA16 := make([]byte, len(m.FillAmounts)*10) - var j15 int + dAtA17 := make([]byte, len(m.FillAmounts)*10) + var j16 int for _, num := range m.FillAmounts { for num >= 1<<7 { - dAtA16[j15] = uint8(uint64(num)&0x7f | 0x80) + dAtA17[j16] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 - j15++ + j16++ } - dAtA16[j15] = uint8(num) - j15++ + dAtA17[j16] = uint8(num) + j16++ } - i -= j15 - copy(dAtA[i:], dAtA16[:j15]) - i = encodeVarintQuery(dAtA, i, uint64(j15)) + i -= j16 + copy(dAtA[i:], dAtA17[:j16]) + i = encodeVarintQuery(dAtA, i, uint64(j16)) i-- dAtA[i] = 0x1a } @@ -2970,6 +3005,18 @@ func (m *StreamUpdate_OrderFill) Size() (n int) { } return n } +func (m *StreamUpdate_TakerOrder) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.TakerOrder != nil { + l = m.TakerOrder.Size() + n += 1 + l + sovQuery(uint64(l)) + } + return n +} func (m *StreamOrderbookUpdate) Size() (n int) { if m == nil { return 0 @@ -4655,6 +4702,41 @@ func (m *StreamUpdate) Unmarshal(dAtA []byte) error { m.UpdateMessage = &StreamUpdate_OrderFill{v} iNdEx = postIndex case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field TakerOrder", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowQuery + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthQuery + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthQuery + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + v := &StreamTakerOrder{} + if err := v.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + m.UpdateMessage = &StreamUpdate_TakerOrder{v} + iNdEx = postIndex + case 4: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field BlockHeight", wireType) } @@ -4673,7 +4755,7 @@ func (m *StreamUpdate) Unmarshal(dAtA []byte) error { break } } - case 4: + case 5: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field ExecMode", wireType) } From df696c97b3b9dfb4b1f2c1824d8873c1bf73aaa2 Mon Sep 17 00:00:00 2001 From: dydxwill <119354122+dydxwill@users.noreply.github.com> Date: Tue, 6 Aug 2024 15:05:38 -0400 Subject: [PATCH 05/33] add subaccount support for grpc stream (#1992) --- .../src/codegen/dydxprotocol/clob/query.ts | 321 ++++- proto/dydxprotocol/clob/query.proto | 45 +- protocol/x/clob/types/query.pb.go | 1245 ++++++++++++++--- 3 files changed, 1432 insertions(+), 179 deletions(-) diff --git a/indexer/packages/v4-protos/src/codegen/dydxprotocol/clob/query.ts b/indexer/packages/v4-protos/src/codegen/dydxprotocol/clob/query.ts index 4097359e36..3eb7ca7df2 100644 --- a/indexer/packages/v4-protos/src/codegen/dydxprotocol/clob/query.ts +++ b/indexer/packages/v4-protos/src/codegen/dydxprotocol/clob/query.ts @@ -1,6 +1,7 @@ import { PageRequest, PageRequestSDKType, PageResponse, PageResponseSDKType } from "../../cosmos/base/query/v1beta1/pagination"; import { ValidatorMevMatches, ValidatorMevMatchesSDKType, MevNodeToNodeMetrics, MevNodeToNodeMetricsSDKType } from "./mev"; import { OrderId, OrderIdSDKType, LongTermOrderPlacement, LongTermOrderPlacementSDKType, Order, OrderSDKType, StreamLiquidationOrder, StreamLiquidationOrderSDKType } from "./order"; +import { SubaccountId, SubaccountIdSDKType } from "../subaccounts/subaccount"; import { ClobPair, ClobPairSDKType } from "./clob_pair"; import { EquityTierLimitConfiguration, EquityTierLimitConfigurationSDKType } from "./equity_tier_limit_config"; import { BlockRateLimitConfiguration, BlockRateLimitConfigurationSDKType } from "./block_rate_limit_config"; @@ -251,6 +252,9 @@ export interface QueryLiquidationsConfigurationResponseSDKType { export interface StreamOrderbookUpdatesRequest { /** Clob pair ids to stream orderbook updates for. */ clobPairId: number[]; + /** Subaccount ids to stream subaccount updates for. */ + + subaccountIds: SubaccountId[]; } /** * StreamOrderbookUpdatesRequest is a request message for the @@ -260,6 +264,9 @@ export interface StreamOrderbookUpdatesRequest { export interface StreamOrderbookUpdatesRequestSDKType { /** Clob pair ids to stream orderbook updates for. */ clob_pair_id: number[]; + /** Subaccount ids to stream subaccount updates for. */ + + subaccount_ids: SubaccountIdSDKType[]; } /** * StreamOrderbookUpdatesResponse is a response message for the @@ -288,6 +295,7 @@ export interface StreamUpdate { orderbookUpdate?: StreamOrderbookUpdate; orderFill?: StreamOrderbookFill; takerOrder?: StreamTakerOrder; + subaccountUpdate?: StreamSubaccountUpdate; /** Block height of the update. */ blockHeight: number; @@ -304,6 +312,7 @@ export interface StreamUpdateSDKType { orderbook_update?: StreamOrderbookUpdateSDKType; order_fill?: StreamOrderbookFillSDKType; taker_order?: StreamTakerOrderSDKType; + subaccount_update?: StreamSubaccountUpdateSDKType; /** Block height of the update. */ block_height: number; @@ -311,6 +320,104 @@ export interface StreamUpdateSDKType { exec_mode: number; } +/** + * SubaccountPerpetualPosition provides information on a subaccount's updated + * perpetual positions. + */ + +export interface SubaccountPerpetualPosition { + /** The `Id` of the `Perpetual`. */ + perpetualId: number; + /** The size of the position in base quantums. */ + + quantums: Long; +} +/** + * SubaccountPerpetualPosition provides information on a subaccount's updated + * perpetual positions. + */ + +export interface SubaccountPerpetualPositionSDKType { + /** The `Id` of the `Perpetual`. */ + perpetual_id: number; + /** The size of the position in base quantums. */ + + quantums: Long; +} +/** + * SubaccountAssetPosition provides information on a subaccount's updated asset + * positions. + */ + +export interface SubaccountAssetPosition { + /** The `Id` of the `Asset`. */ + assetId: number; + /** The absolute size of the position in base quantums. */ + + quantums: Long; +} +/** + * SubaccountAssetPosition provides information on a subaccount's updated asset + * positions. + */ + +export interface SubaccountAssetPositionSDKType { + /** The `Id` of the `Asset`. */ + asset_id: number; + /** The absolute size of the position in base quantums. */ + + quantums: Long; +} +/** + * StreamSubaccountUpdate provides information on a subaccount update. Used in + * the full node GRPC stream. + */ + +export interface StreamSubaccountUpdate { + subaccountId?: SubaccountId; + /** updated_perpetual_positions will each be for unique perpetuals. */ + + updatedPerpetualPositions: SubaccountPerpetualPosition[]; + /** updated_asset_positions will each be for unique assets. */ + + updatedAssetPositions: SubaccountAssetPosition[]; + /** + * Snapshot indicates if the response is from a snapshot of the subaccount. + * All updates should be ignored until snapshot is received. + * If the snapshot is true, then all previous entries should be + * discarded and the subaccount should be resynced. + * For a snapshot subaccount update, the `updated_perpetual_positions` and + * `updated_asset_positions` fields will contain the full state of the + * subaccount. + */ + + snapshot: boolean; +} +/** + * StreamSubaccountUpdate provides information on a subaccount update. Used in + * the full node GRPC stream. + */ + +export interface StreamSubaccountUpdateSDKType { + subaccount_id?: SubaccountIdSDKType; + /** updated_perpetual_positions will each be for unique perpetuals. */ + + updated_perpetual_positions: SubaccountPerpetualPositionSDKType[]; + /** updated_asset_positions will each be for unique assets. */ + + updated_asset_positions: SubaccountAssetPositionSDKType[]; + /** + * Snapshot indicates if the response is from a snapshot of the subaccount. + * All updates should be ignored until snapshot is received. + * If the snapshot is true, then all previous entries should be + * discarded and the subaccount should be resynced. + * For a snapshot subaccount update, the `updated_perpetual_positions` and + * `updated_asset_positions` fields will contain the full state of the + * subaccount. + */ + + snapshot: boolean; +} /** * StreamOrderbookUpdate provides information on an orderbook update. Used in * the full node GRPC stream. @@ -1182,7 +1289,8 @@ export const QueryLiquidationsConfigurationResponse = { function createBaseStreamOrderbookUpdatesRequest(): StreamOrderbookUpdatesRequest { return { - clobPairId: [] + clobPairId: [], + subaccountIds: [] }; } @@ -1195,6 +1303,11 @@ export const StreamOrderbookUpdatesRequest = { } writer.ldelim(); + + for (const v of message.subaccountIds) { + SubaccountId.encode(v!, writer.uint32(18).fork()).ldelim(); + } + return writer; }, @@ -1220,6 +1333,10 @@ export const StreamOrderbookUpdatesRequest = { break; + case 2: + message.subaccountIds.push(SubaccountId.decode(reader, reader.uint32())); + break; + default: reader.skipType(tag & 7); break; @@ -1232,6 +1349,7 @@ export const StreamOrderbookUpdatesRequest = { fromPartial(object: DeepPartial): StreamOrderbookUpdatesRequest { const message = createBaseStreamOrderbookUpdatesRequest(); message.clobPairId = object.clobPairId?.map(e => e) || []; + message.subaccountIds = object.subaccountIds?.map(e => SubaccountId.fromPartial(e)) || []; return message; } @@ -1287,6 +1405,7 @@ function createBaseStreamUpdate(): StreamUpdate { orderbookUpdate: undefined, orderFill: undefined, takerOrder: undefined, + subaccountUpdate: undefined, blockHeight: 0, execMode: 0 }; @@ -1306,12 +1425,16 @@ export const StreamUpdate = { StreamTakerOrder.encode(message.takerOrder, writer.uint32(26).fork()).ldelim(); } + if (message.subaccountUpdate !== undefined) { + StreamSubaccountUpdate.encode(message.subaccountUpdate, writer.uint32(34).fork()).ldelim(); + } + if (message.blockHeight !== 0) { - writer.uint32(32).uint32(message.blockHeight); + writer.uint32(40).uint32(message.blockHeight); } if (message.execMode !== 0) { - writer.uint32(40).uint32(message.execMode); + writer.uint32(48).uint32(message.execMode); } return writer; @@ -1339,10 +1462,14 @@ export const StreamUpdate = { break; case 4: - message.blockHeight = reader.uint32(); + message.subaccountUpdate = StreamSubaccountUpdate.decode(reader, reader.uint32()); break; case 5: + message.blockHeight = reader.uint32(); + break; + + case 6: message.execMode = reader.uint32(); break; @@ -1360,6 +1487,7 @@ export const StreamUpdate = { message.orderbookUpdate = object.orderbookUpdate !== undefined && object.orderbookUpdate !== null ? StreamOrderbookUpdate.fromPartial(object.orderbookUpdate) : undefined; message.orderFill = object.orderFill !== undefined && object.orderFill !== null ? StreamOrderbookFill.fromPartial(object.orderFill) : undefined; message.takerOrder = object.takerOrder !== undefined && object.takerOrder !== null ? StreamTakerOrder.fromPartial(object.takerOrder) : undefined; + message.subaccountUpdate = object.subaccountUpdate !== undefined && object.subaccountUpdate !== null ? StreamSubaccountUpdate.fromPartial(object.subaccountUpdate) : undefined; message.blockHeight = object.blockHeight ?? 0; message.execMode = object.execMode ?? 0; return message; @@ -1367,6 +1495,191 @@ export const StreamUpdate = { }; +function createBaseSubaccountPerpetualPosition(): SubaccountPerpetualPosition { + return { + perpetualId: 0, + quantums: Long.UZERO + }; +} + +export const SubaccountPerpetualPosition = { + encode(message: SubaccountPerpetualPosition, writer: _m0.Writer = _m0.Writer.create()): _m0.Writer { + if (message.perpetualId !== 0) { + writer.uint32(8).uint32(message.perpetualId); + } + + if (!message.quantums.isZero()) { + writer.uint32(16).uint64(message.quantums); + } + + return writer; + }, + + decode(input: _m0.Reader | Uint8Array, length?: number): SubaccountPerpetualPosition { + const reader = input instanceof _m0.Reader ? input : new _m0.Reader(input); + let end = length === undefined ? reader.len : reader.pos + length; + const message = createBaseSubaccountPerpetualPosition(); + + while (reader.pos < end) { + const tag = reader.uint32(); + + switch (tag >>> 3) { + case 1: + message.perpetualId = reader.uint32(); + break; + + case 2: + message.quantums = (reader.uint64() as Long); + break; + + default: + reader.skipType(tag & 7); + break; + } + } + + return message; + }, + + fromPartial(object: DeepPartial): SubaccountPerpetualPosition { + const message = createBaseSubaccountPerpetualPosition(); + message.perpetualId = object.perpetualId ?? 0; + message.quantums = object.quantums !== undefined && object.quantums !== null ? Long.fromValue(object.quantums) : Long.UZERO; + return message; + } + +}; + +function createBaseSubaccountAssetPosition(): SubaccountAssetPosition { + return { + assetId: 0, + quantums: Long.UZERO + }; +} + +export const SubaccountAssetPosition = { + encode(message: SubaccountAssetPosition, writer: _m0.Writer = _m0.Writer.create()): _m0.Writer { + if (message.assetId !== 0) { + writer.uint32(8).uint32(message.assetId); + } + + if (!message.quantums.isZero()) { + writer.uint32(16).uint64(message.quantums); + } + + return writer; + }, + + decode(input: _m0.Reader | Uint8Array, length?: number): SubaccountAssetPosition { + const reader = input instanceof _m0.Reader ? input : new _m0.Reader(input); + let end = length === undefined ? reader.len : reader.pos + length; + const message = createBaseSubaccountAssetPosition(); + + while (reader.pos < end) { + const tag = reader.uint32(); + + switch (tag >>> 3) { + case 1: + message.assetId = reader.uint32(); + break; + + case 2: + message.quantums = (reader.uint64() as Long); + break; + + default: + reader.skipType(tag & 7); + break; + } + } + + return message; + }, + + fromPartial(object: DeepPartial): SubaccountAssetPosition { + const message = createBaseSubaccountAssetPosition(); + message.assetId = object.assetId ?? 0; + message.quantums = object.quantums !== undefined && object.quantums !== null ? Long.fromValue(object.quantums) : Long.UZERO; + return message; + } + +}; + +function createBaseStreamSubaccountUpdate(): StreamSubaccountUpdate { + return { + subaccountId: undefined, + updatedPerpetualPositions: [], + updatedAssetPositions: [], + snapshot: false + }; +} + +export const StreamSubaccountUpdate = { + encode(message: StreamSubaccountUpdate, writer: _m0.Writer = _m0.Writer.create()): _m0.Writer { + if (message.subaccountId !== undefined) { + SubaccountId.encode(message.subaccountId, writer.uint32(10).fork()).ldelim(); + } + + for (const v of message.updatedPerpetualPositions) { + SubaccountPerpetualPosition.encode(v!, writer.uint32(18).fork()).ldelim(); + } + + for (const v of message.updatedAssetPositions) { + SubaccountAssetPosition.encode(v!, writer.uint32(26).fork()).ldelim(); + } + + if (message.snapshot === true) { + writer.uint32(32).bool(message.snapshot); + } + + return writer; + }, + + decode(input: _m0.Reader | Uint8Array, length?: number): StreamSubaccountUpdate { + const reader = input instanceof _m0.Reader ? input : new _m0.Reader(input); + let end = length === undefined ? reader.len : reader.pos + length; + const message = createBaseStreamSubaccountUpdate(); + + while (reader.pos < end) { + const tag = reader.uint32(); + + switch (tag >>> 3) { + case 1: + message.subaccountId = SubaccountId.decode(reader, reader.uint32()); + break; + + case 2: + message.updatedPerpetualPositions.push(SubaccountPerpetualPosition.decode(reader, reader.uint32())); + break; + + case 3: + message.updatedAssetPositions.push(SubaccountAssetPosition.decode(reader, reader.uint32())); + break; + + case 4: + message.snapshot = reader.bool(); + break; + + default: + reader.skipType(tag & 7); + break; + } + } + + return message; + }, + + fromPartial(object: DeepPartial): StreamSubaccountUpdate { + const message = createBaseStreamSubaccountUpdate(); + message.subaccountId = object.subaccountId !== undefined && object.subaccountId !== null ? SubaccountId.fromPartial(object.subaccountId) : undefined; + message.updatedPerpetualPositions = object.updatedPerpetualPositions?.map(e => SubaccountPerpetualPosition.fromPartial(e)) || []; + message.updatedAssetPositions = object.updatedAssetPositions?.map(e => SubaccountAssetPosition.fromPartial(e)) || []; + message.snapshot = object.snapshot ?? false; + return message; + } + +}; + function createBaseStreamOrderbookUpdate(): StreamOrderbookUpdate { return { updates: [], diff --git a/proto/dydxprotocol/clob/query.proto b/proto/dydxprotocol/clob/query.proto index a00be1aaa1..34f9899801 100644 --- a/proto/dydxprotocol/clob/query.proto +++ b/proto/dydxprotocol/clob/query.proto @@ -12,6 +12,7 @@ import "dydxprotocol/clob/matches.proto"; import "dydxprotocol/clob/liquidations_config.proto"; import "dydxprotocol/clob/mev.proto"; import "dydxprotocol/indexer/off_chain_updates/off_chain_updates.proto"; +import "dydxprotocol/subaccounts/subaccount.proto"; option go_package = "github.com/dydxprotocol/v4-chain/protocol/x/clob/types"; @@ -165,6 +166,9 @@ message QueryLiquidationsConfigurationResponse { message StreamOrderbookUpdatesRequest { // Clob pair ids to stream orderbook updates for. repeated uint32 clob_pair_id = 1; + + // Subaccount ids to stream subaccount updates for. + repeated dydxprotocol.subaccounts.SubaccountId subaccount_ids = 2; } // StreamOrderbookUpdatesResponse is a response message for the @@ -183,13 +187,50 @@ message StreamUpdate { StreamOrderbookUpdate orderbook_update = 1; StreamOrderbookFill order_fill = 2; StreamTakerOrder taker_order = 3; + StreamSubaccountUpdate subaccount_update = 4; } // Block height of the update. - uint32 block_height = 4; + uint32 block_height = 5; // Exec mode of the update. - uint32 exec_mode = 5; + uint32 exec_mode = 6; +} + +// SubaccountPerpetualPosition provides information on a subaccount's updated +// perpetual positions. +message SubaccountPerpetualPosition { + // The `Id` of the `Perpetual`. + uint32 perpetual_id = 1; + // The size of the position in base quantums. + uint64 quantums = 2; +} + +// SubaccountAssetPosition provides information on a subaccount's updated asset +// positions. +message SubaccountAssetPosition { + // The `Id` of the `Asset`. + uint32 asset_id = 1; + // The absolute size of the position in base quantums. + uint64 quantums = 2; +} + +// StreamSubaccountUpdate provides information on a subaccount update. Used in +// the full node GRPC stream. +message StreamSubaccountUpdate { + dydxprotocol.subaccounts.SubaccountId subaccount_id = 1; + // updated_perpetual_positions will each be for unique perpetuals. + repeated SubaccountPerpetualPosition updated_perpetual_positions = 2; + // updated_asset_positions will each be for unique assets. + repeated SubaccountAssetPosition updated_asset_positions = 3; + // Snapshot indicates if the response is from a snapshot of the subaccount. + // All updates should be ignored until snapshot is received. + // If the snapshot is true, then all previous entries should be + // discarded and the subaccount should be resynced. + // For a snapshot subaccount update, the `updated_perpetual_positions` and + // `updated_asset_positions` fields will contain the full state of the + // subaccount. + bool snapshot = 4; } // StreamOrderbookUpdate provides information on an orderbook update. Used in diff --git a/protocol/x/clob/types/query.pb.go b/protocol/x/clob/types/query.pb.go index 287d4c1e48..260961811e 100644 --- a/protocol/x/clob/types/query.pb.go +++ b/protocol/x/clob/types/query.pb.go @@ -11,7 +11,8 @@ import ( _ "github.com/cosmos/gogoproto/gogoproto" grpc1 "github.com/cosmos/gogoproto/grpc" proto "github.com/cosmos/gogoproto/proto" - types "github.com/dydxprotocol/v4-chain/protocol/indexer/off_chain_updates/types" + types1 "github.com/dydxprotocol/v4-chain/protocol/indexer/off_chain_updates/types" + types "github.com/dydxprotocol/v4-chain/protocol/x/subaccounts/types" _ "google.golang.org/genproto/googleapis/api/annotations" grpc "google.golang.org/grpc" codes "google.golang.org/grpc/codes" @@ -770,6 +771,8 @@ func (m *QueryLiquidationsConfigurationResponse) GetLiquidationsConfig() Liquida type StreamOrderbookUpdatesRequest struct { // Clob pair ids to stream orderbook updates for. ClobPairId []uint32 `protobuf:"varint,1,rep,packed,name=clob_pair_id,json=clobPairId,proto3" json:"clob_pair_id,omitempty"` + // Subaccount ids to stream subaccount updates for. + SubaccountIds []*types.SubaccountId `protobuf:"bytes,2,rep,name=subaccount_ids,json=subaccountIds,proto3" json:"subaccount_ids,omitempty"` } func (m *StreamOrderbookUpdatesRequest) Reset() { *m = StreamOrderbookUpdatesRequest{} } @@ -812,6 +815,13 @@ func (m *StreamOrderbookUpdatesRequest) GetClobPairId() []uint32 { return nil } +func (m *StreamOrderbookUpdatesRequest) GetSubaccountIds() []*types.SubaccountId { + if m != nil { + return m.SubaccountIds + } + return nil +} + // StreamOrderbookUpdatesResponse is a response message for the // StreamOrderbookUpdates method. type StreamOrderbookUpdatesResponse struct { @@ -870,11 +880,12 @@ type StreamUpdate struct { // *StreamUpdate_OrderbookUpdate // *StreamUpdate_OrderFill // *StreamUpdate_TakerOrder + // *StreamUpdate_SubaccountUpdate UpdateMessage isStreamUpdate_UpdateMessage `protobuf_oneof:"update_message"` // Block height of the update. - BlockHeight uint32 `protobuf:"varint,4,opt,name=block_height,json=blockHeight,proto3" json:"block_height,omitempty"` + BlockHeight uint32 `protobuf:"varint,5,opt,name=block_height,json=blockHeight,proto3" json:"block_height,omitempty"` // Exec mode of the update. - ExecMode uint32 `protobuf:"varint,5,opt,name=exec_mode,json=execMode,proto3" json:"exec_mode,omitempty"` + ExecMode uint32 `protobuf:"varint,6,opt,name=exec_mode,json=execMode,proto3" json:"exec_mode,omitempty"` } func (m *StreamUpdate) Reset() { *m = StreamUpdate{} } @@ -925,10 +936,14 @@ type StreamUpdate_OrderFill struct { type StreamUpdate_TakerOrder struct { TakerOrder *StreamTakerOrder `protobuf:"bytes,3,opt,name=taker_order,json=takerOrder,proto3,oneof" json:"taker_order,omitempty"` } +type StreamUpdate_SubaccountUpdate struct { + SubaccountUpdate *StreamSubaccountUpdate `protobuf:"bytes,4,opt,name=subaccount_update,json=subaccountUpdate,proto3,oneof" json:"subaccount_update,omitempty"` +} -func (*StreamUpdate_OrderbookUpdate) isStreamUpdate_UpdateMessage() {} -func (*StreamUpdate_OrderFill) isStreamUpdate_UpdateMessage() {} -func (*StreamUpdate_TakerOrder) isStreamUpdate_UpdateMessage() {} +func (*StreamUpdate_OrderbookUpdate) isStreamUpdate_UpdateMessage() {} +func (*StreamUpdate_OrderFill) isStreamUpdate_UpdateMessage() {} +func (*StreamUpdate_TakerOrder) isStreamUpdate_UpdateMessage() {} +func (*StreamUpdate_SubaccountUpdate) isStreamUpdate_UpdateMessage() {} func (m *StreamUpdate) GetUpdateMessage() isStreamUpdate_UpdateMessage { if m != nil { @@ -958,6 +973,13 @@ func (m *StreamUpdate) GetTakerOrder() *StreamTakerOrder { return nil } +func (m *StreamUpdate) GetSubaccountUpdate() *StreamSubaccountUpdate { + if x, ok := m.GetUpdateMessage().(*StreamUpdate_SubaccountUpdate); ok { + return x.SubaccountUpdate + } + return nil +} + func (m *StreamUpdate) GetBlockHeight() uint32 { if m != nil { return m.BlockHeight @@ -978,7 +1000,199 @@ func (*StreamUpdate) XXX_OneofWrappers() []interface{} { (*StreamUpdate_OrderbookUpdate)(nil), (*StreamUpdate_OrderFill)(nil), (*StreamUpdate_TakerOrder)(nil), + (*StreamUpdate_SubaccountUpdate)(nil), + } +} + +// SubaccountPerpetualPosition provides information on a subaccount's updated +// perpetual positions. +type SubaccountPerpetualPosition struct { + // The `Id` of the `Perpetual`. + PerpetualId uint32 `protobuf:"varint,1,opt,name=perpetual_id,json=perpetualId,proto3" json:"perpetual_id,omitempty"` + // The size of the position in base quantums. + Quantums uint64 `protobuf:"varint,2,opt,name=quantums,proto3" json:"quantums,omitempty"` +} + +func (m *SubaccountPerpetualPosition) Reset() { *m = SubaccountPerpetualPosition{} } +func (m *SubaccountPerpetualPosition) String() string { return proto.CompactTextString(m) } +func (*SubaccountPerpetualPosition) ProtoMessage() {} +func (*SubaccountPerpetualPosition) Descriptor() ([]byte, []int) { + return fileDescriptor_3365c195b25c5bc0, []int{17} +} +func (m *SubaccountPerpetualPosition) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *SubaccountPerpetualPosition) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_SubaccountPerpetualPosition.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *SubaccountPerpetualPosition) XXX_Merge(src proto.Message) { + xxx_messageInfo_SubaccountPerpetualPosition.Merge(m, src) +} +func (m *SubaccountPerpetualPosition) XXX_Size() int { + return m.Size() +} +func (m *SubaccountPerpetualPosition) XXX_DiscardUnknown() { + xxx_messageInfo_SubaccountPerpetualPosition.DiscardUnknown(m) +} + +var xxx_messageInfo_SubaccountPerpetualPosition proto.InternalMessageInfo + +func (m *SubaccountPerpetualPosition) GetPerpetualId() uint32 { + if m != nil { + return m.PerpetualId + } + return 0 +} + +func (m *SubaccountPerpetualPosition) GetQuantums() uint64 { + if m != nil { + return m.Quantums + } + return 0 +} + +// SubaccountAssetPosition provides information on a subaccount's updated asset +// positions. +type SubaccountAssetPosition struct { + // The `Id` of the `Asset`. + AssetId uint32 `protobuf:"varint,1,opt,name=asset_id,json=assetId,proto3" json:"asset_id,omitempty"` + // The absolute size of the position in base quantums. + Quantums uint64 `protobuf:"varint,2,opt,name=quantums,proto3" json:"quantums,omitempty"` +} + +func (m *SubaccountAssetPosition) Reset() { *m = SubaccountAssetPosition{} } +func (m *SubaccountAssetPosition) String() string { return proto.CompactTextString(m) } +func (*SubaccountAssetPosition) ProtoMessage() {} +func (*SubaccountAssetPosition) Descriptor() ([]byte, []int) { + return fileDescriptor_3365c195b25c5bc0, []int{18} +} +func (m *SubaccountAssetPosition) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *SubaccountAssetPosition) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_SubaccountAssetPosition.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *SubaccountAssetPosition) XXX_Merge(src proto.Message) { + xxx_messageInfo_SubaccountAssetPosition.Merge(m, src) +} +func (m *SubaccountAssetPosition) XXX_Size() int { + return m.Size() +} +func (m *SubaccountAssetPosition) XXX_DiscardUnknown() { + xxx_messageInfo_SubaccountAssetPosition.DiscardUnknown(m) +} + +var xxx_messageInfo_SubaccountAssetPosition proto.InternalMessageInfo + +func (m *SubaccountAssetPosition) GetAssetId() uint32 { + if m != nil { + return m.AssetId + } + return 0 +} + +func (m *SubaccountAssetPosition) GetQuantums() uint64 { + if m != nil { + return m.Quantums + } + return 0 +} + +// StreamSubaccountUpdate provides information on a subaccount update. Used in +// the full node GRPC stream. +type StreamSubaccountUpdate struct { + SubaccountId *types.SubaccountId `protobuf:"bytes,1,opt,name=subaccount_id,json=subaccountId,proto3" json:"subaccount_id,omitempty"` + // updated_perpetual_positions will each be for unique perpetuals. + UpdatedPerpetualPositions []*SubaccountPerpetualPosition `protobuf:"bytes,2,rep,name=updated_perpetual_positions,json=updatedPerpetualPositions,proto3" json:"updated_perpetual_positions,omitempty"` + // updated_asset_positions will each be for unique assets. + UpdatedAssetPositions []*SubaccountAssetPosition `protobuf:"bytes,3,rep,name=updated_asset_positions,json=updatedAssetPositions,proto3" json:"updated_asset_positions,omitempty"` + // Snapshot indicates if the response is from a snapshot of the subaccount. + // All updates should be ignored until snapshot is received. + // If the snapshot is true, then all previous entries should be + // discarded and the subaccount should be resynced. + // For a snapshot subaccount update, the `updated_perpetual_positions` and + // `updated_asset_positions` fields will contain the full state of the + // subaccount. + Snapshot bool `protobuf:"varint,4,opt,name=snapshot,proto3" json:"snapshot,omitempty"` +} + +func (m *StreamSubaccountUpdate) Reset() { *m = StreamSubaccountUpdate{} } +func (m *StreamSubaccountUpdate) String() string { return proto.CompactTextString(m) } +func (*StreamSubaccountUpdate) ProtoMessage() {} +func (*StreamSubaccountUpdate) Descriptor() ([]byte, []int) { + return fileDescriptor_3365c195b25c5bc0, []int{19} +} +func (m *StreamSubaccountUpdate) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *StreamSubaccountUpdate) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_StreamSubaccountUpdate.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *StreamSubaccountUpdate) XXX_Merge(src proto.Message) { + xxx_messageInfo_StreamSubaccountUpdate.Merge(m, src) +} +func (m *StreamSubaccountUpdate) XXX_Size() int { + return m.Size() +} +func (m *StreamSubaccountUpdate) XXX_DiscardUnknown() { + xxx_messageInfo_StreamSubaccountUpdate.DiscardUnknown(m) +} + +var xxx_messageInfo_StreamSubaccountUpdate proto.InternalMessageInfo + +func (m *StreamSubaccountUpdate) GetSubaccountId() *types.SubaccountId { + if m != nil { + return m.SubaccountId + } + return nil +} + +func (m *StreamSubaccountUpdate) GetUpdatedPerpetualPositions() []*SubaccountPerpetualPosition { + if m != nil { + return m.UpdatedPerpetualPositions + } + return nil +} + +func (m *StreamSubaccountUpdate) GetUpdatedAssetPositions() []*SubaccountAssetPosition { + if m != nil { + return m.UpdatedAssetPositions } + return nil +} + +func (m *StreamSubaccountUpdate) GetSnapshot() bool { + if m != nil { + return m.Snapshot + } + return false } // StreamOrderbookUpdate provides information on an orderbook update. Used in @@ -986,7 +1200,7 @@ func (*StreamUpdate) XXX_OneofWrappers() []interface{} { type StreamOrderbookUpdate struct { // Orderbook updates for the clob pair. Can contain order place, removals, // or updates. - Updates []types.OffChainUpdateV1 `protobuf:"bytes,1,rep,name=updates,proto3" json:"updates"` + Updates []types1.OffChainUpdateV1 `protobuf:"bytes,1,rep,name=updates,proto3" json:"updates"` // Snapshot indicates if the response is from a snapshot of the orderbook. // All updates should be ignored until snapshot is recieved. // If the snapshot is true, then all previous entries should be @@ -998,7 +1212,7 @@ func (m *StreamOrderbookUpdate) Reset() { *m = StreamOrderbookUpdate{} } func (m *StreamOrderbookUpdate) String() string { return proto.CompactTextString(m) } func (*StreamOrderbookUpdate) ProtoMessage() {} func (*StreamOrderbookUpdate) Descriptor() ([]byte, []int) { - return fileDescriptor_3365c195b25c5bc0, []int{17} + return fileDescriptor_3365c195b25c5bc0, []int{20} } func (m *StreamOrderbookUpdate) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1027,7 +1241,7 @@ func (m *StreamOrderbookUpdate) XXX_DiscardUnknown() { var xxx_messageInfo_StreamOrderbookUpdate proto.InternalMessageInfo -func (m *StreamOrderbookUpdate) GetUpdates() []types.OffChainUpdateV1 { +func (m *StreamOrderbookUpdate) GetUpdates() []types1.OffChainUpdateV1 { if m != nil { return m.Updates } @@ -1058,7 +1272,7 @@ func (m *StreamOrderbookFill) Reset() { *m = StreamOrderbookFill{} } func (m *StreamOrderbookFill) String() string { return proto.CompactTextString(m) } func (*StreamOrderbookFill) ProtoMessage() {} func (*StreamOrderbookFill) Descriptor() ([]byte, []int) { - return fileDescriptor_3365c195b25c5bc0, []int{18} + return fileDescriptor_3365c195b25c5bc0, []int{21} } func (m *StreamOrderbookFill) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1129,7 +1343,7 @@ func (m *StreamTakerOrder) Reset() { *m = StreamTakerOrder{} } func (m *StreamTakerOrder) String() string { return proto.CompactTextString(m) } func (*StreamTakerOrder) ProtoMessage() {} func (*StreamTakerOrder) Descriptor() ([]byte, []int) { - return fileDescriptor_3365c195b25c5bc0, []int{19} + return fileDescriptor_3365c195b25c5bc0, []int{22} } func (m *StreamTakerOrder) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1232,7 +1446,7 @@ func (m *StreamTakerOrderStatus) Reset() { *m = StreamTakerOrderStatus{} func (m *StreamTakerOrderStatus) String() string { return proto.CompactTextString(m) } func (*StreamTakerOrderStatus) ProtoMessage() {} func (*StreamTakerOrderStatus) Descriptor() ([]byte, []int) { - return fileDescriptor_3365c195b25c5bc0, []int{20} + return fileDescriptor_3365c195b25c5bc0, []int{23} } func (m *StreamTakerOrderStatus) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1301,6 +1515,9 @@ func init() { proto.RegisterType((*StreamOrderbookUpdatesRequest)(nil), "dydxprotocol.clob.StreamOrderbookUpdatesRequest") proto.RegisterType((*StreamOrderbookUpdatesResponse)(nil), "dydxprotocol.clob.StreamOrderbookUpdatesResponse") proto.RegisterType((*StreamUpdate)(nil), "dydxprotocol.clob.StreamUpdate") + proto.RegisterType((*SubaccountPerpetualPosition)(nil), "dydxprotocol.clob.SubaccountPerpetualPosition") + proto.RegisterType((*SubaccountAssetPosition)(nil), "dydxprotocol.clob.SubaccountAssetPosition") + proto.RegisterType((*StreamSubaccountUpdate)(nil), "dydxprotocol.clob.StreamSubaccountUpdate") proto.RegisterType((*StreamOrderbookUpdate)(nil), "dydxprotocol.clob.StreamOrderbookUpdate") proto.RegisterType((*StreamOrderbookFill)(nil), "dydxprotocol.clob.StreamOrderbookFill") proto.RegisterType((*StreamTakerOrder)(nil), "dydxprotocol.clob.StreamTakerOrder") @@ -1310,106 +1527,119 @@ func init() { func init() { proto.RegisterFile("dydxprotocol/clob/query.proto", fileDescriptor_3365c195b25c5bc0) } var fileDescriptor_3365c195b25c5bc0 = []byte{ - // 1578 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x58, 0xcf, 0x4f, 0x1b, 0x47, - 0x14, 0xf6, 0x1a, 0x92, 0xc0, 0x73, 0x7e, 0x90, 0x21, 0x24, 0x8e, 0x21, 0x86, 0x6c, 0x1a, 0x02, - 0xa4, 0xf1, 0x02, 0x89, 0xa2, 0x34, 0x54, 0xa9, 0x80, 0x96, 0x10, 0x29, 0x34, 0x64, 0x43, 0x12, - 0xd4, 0x46, 0x5a, 0x8d, 0xbd, 0xc3, 0xb2, 0x62, 0x77, 0xc7, 0xec, 0xce, 0x5a, 0xa0, 0xaa, 0xaa, - 0xd4, 0x43, 0x2e, 0x6d, 0xa5, 0x4a, 0x3d, 0xf4, 0x50, 0xa9, 0x97, 0x9e, 0x2b, 0xf5, 0xd2, 0x63, - 0xd5, 0xf6, 0x96, 0x63, 0xa4, 0x5e, 0x7a, 0xa8, 0xaa, 0x2a, 0xe9, 0xb9, 0x7f, 0x43, 0xb5, 0x33, - 0xb3, 0xc6, 0x6b, 0xef, 0x1a, 0xc2, 0x05, 0x3c, 0x6f, 0xde, 0x7b, 0xf3, 0xbd, 0xf7, 0xbe, 0x79, - 0xf3, 0x6c, 0xb8, 0x60, 0xee, 0x9a, 0x3b, 0x75, 0x9f, 0x32, 0x5a, 0xa3, 0x8e, 0x56, 0x73, 0x68, - 0x55, 0xdb, 0x0e, 0x89, 0xbf, 0x5b, 0xe1, 0x32, 0x74, 0xba, 0x75, 0xbb, 0x12, 0x6d, 0x97, 0xce, - 0x58, 0xd4, 0xa2, 0x5c, 0xa4, 0x45, 0x9f, 0x84, 0x62, 0x69, 0xc4, 0xa2, 0xd4, 0x72, 0x88, 0x86, - 0xeb, 0xb6, 0x86, 0x3d, 0x8f, 0x32, 0xcc, 0x6c, 0xea, 0x05, 0x72, 0x77, 0xaa, 0x46, 0x03, 0x97, - 0x06, 0x5a, 0x15, 0x07, 0x44, 0xf8, 0xd7, 0x1a, 0x33, 0x55, 0xc2, 0xf0, 0x8c, 0x56, 0xc7, 0x96, - 0xed, 0x71, 0x65, 0xa9, 0xab, 0x75, 0x22, 0xaa, 0x3a, 0xb4, 0xb6, 0x65, 0xf8, 0x98, 0x11, 0xc3, - 0xb1, 0x5d, 0x9b, 0x19, 0x35, 0xea, 0x6d, 0xd8, 0x96, 0x34, 0xb8, 0xd8, 0x69, 0x10, 0xfd, 0x31, - 0xea, 0xd8, 0xf6, 0xa5, 0xca, 0x74, 0xa7, 0x0a, 0xd9, 0x0e, 0x6d, 0xb6, 0x6b, 0x30, 0x9b, 0xf8, - 0x69, 0x4e, 0x53, 0xf2, 0x42, 0x7d, 0x93, 0xc4, 0x0e, 0x47, 0x3b, 0xb7, 0x5d, 0xcc, 0x6a, 0x9b, - 0x24, 0x8e, 0xf8, 0x6a, 0xa7, 0x82, 0x63, 0x6f, 0x87, 0xb6, 0x29, 0xf2, 0x92, 0x3c, 0x6c, 0x38, - 0xc5, 0x1b, 0x69, 0xc8, 0xcd, 0x3b, 0x89, 0x4d, 0xdb, 0x33, 0xc9, 0x0e, 0xf1, 0x35, 0xba, 0xb1, - 0x61, 0xd4, 0x36, 0xb1, 0xed, 0x19, 0x61, 0xdd, 0xc4, 0x8c, 0x04, 0x9d, 0x12, 0x61, 0xaf, 0x4e, - 0xc2, 0xb9, 0x87, 0x51, 0xc6, 0xef, 0x12, 0xb6, 0xe8, 0xd0, 0xea, 0x2a, 0xb6, 0x7d, 0x9d, 0x6c, - 0x87, 0x24, 0x60, 0xe8, 0x24, 0xe4, 0x6d, 0xb3, 0xa8, 0x8c, 0x29, 0x13, 0x27, 0xf4, 0xbc, 0x6d, - 0xaa, 0x4f, 0x61, 0x88, 0xab, 0xee, 0xe9, 0x05, 0x75, 0xea, 0x05, 0x04, 0xdd, 0x81, 0xfe, 0x66, - 0x4a, 0xb9, 0x7e, 0x61, 0x76, 0xb8, 0xd2, 0x41, 0x8d, 0x4a, 0x6c, 0xb7, 0xd0, 0xfb, 0xe2, 0xef, - 0xd1, 0x9c, 0xde, 0x57, 0x93, 0x6b, 0x15, 0x4b, 0x0c, 0xf3, 0x8e, 0xd3, 0x8e, 0x61, 0x09, 0x60, - 0x8f, 0x02, 0xd2, 0xf7, 0x78, 0x45, 0xf0, 0xa5, 0x12, 0xf1, 0xa5, 0x22, 0xf8, 0x28, 0xf9, 0x52, - 0x59, 0xc5, 0x16, 0x91, 0xb6, 0x7a, 0x8b, 0xa5, 0xfa, 0x83, 0x02, 0xc5, 0x04, 0xf8, 0x79, 0xc7, - 0xc9, 0xc2, 0xdf, 0xf3, 0x86, 0xf8, 0xd1, 0xdd, 0x04, 0xc8, 0x3c, 0x07, 0x79, 0x65, 0x5f, 0x90, - 0xe2, 0xf0, 0x04, 0xca, 0xbf, 0x14, 0x18, 0x5d, 0x21, 0x8d, 0x0f, 0xa9, 0x49, 0xd6, 0x68, 0xf4, - 0x77, 0x11, 0x3b, 0xb5, 0xd0, 0xe1, 0x9b, 0x71, 0x46, 0x9e, 0xc1, 0x59, 0x41, 0xf8, 0xba, 0x4f, - 0xeb, 0x34, 0x20, 0xbe, 0x21, 0xa9, 0xd5, 0xcc, 0x4e, 0x27, 0xf2, 0x27, 0xd8, 0x89, 0xa8, 0x45, - 0xfd, 0x15, 0xd2, 0x58, 0x11, 0xda, 0xfa, 0x19, 0xee, 0x65, 0x55, 0x3a, 0x91, 0x52, 0xf4, 0x31, - 0x0c, 0x35, 0x62, 0x65, 0xc3, 0x25, 0x0d, 0xc3, 0x25, 0xcc, 0xb7, 0x6b, 0x41, 0x33, 0xaa, 0x4e, - 0xe7, 0x09, 0xc0, 0x2b, 0x42, 0x5d, 0x1f, 0x6c, 0xb4, 0x1e, 0x29, 0x84, 0xea, 0x7f, 0x0a, 0x8c, - 0x65, 0x87, 0x27, 0x8b, 0x61, 0xc1, 0x31, 0x9f, 0x04, 0xa1, 0xc3, 0x02, 0x59, 0x8a, 0xbb, 0xfb, - 0x9d, 0x99, 0xe2, 0x25, 0x52, 0x98, 0xf7, 0xcc, 0x27, 0xd4, 0x09, 0x5d, 0xb2, 0x4a, 0xfc, 0xa8, - 0x74, 0xb2, 0x6c, 0xb1, 0xf7, 0x12, 0x86, 0xc1, 0x14, 0x2d, 0x34, 0x06, 0xc7, 0x9b, 0x64, 0x30, - 0x9a, 0xfc, 0x87, 0xb8, 0xd8, 0xf7, 0x4c, 0x34, 0x00, 0x3d, 0x2e, 0x69, 0xf0, 0x8c, 0xe4, 0xf5, - 0xe8, 0x23, 0x3a, 0x0b, 0x47, 0x1b, 0xdc, 0x49, 0xb1, 0x67, 0x4c, 0x99, 0xe8, 0xd5, 0xe5, 0x4a, - 0x9d, 0x82, 0x09, 0x4e, 0xba, 0x0f, 0x78, 0x37, 0x59, 0xb3, 0x89, 0x7f, 0x3f, 0xea, 0x25, 0x8b, - 0xfc, 0x76, 0x87, 0x7e, 0x6b, 0x5d, 0xd5, 0xef, 0x14, 0x98, 0x3c, 0x80, 0xb2, 0xcc, 0x92, 0x07, - 0xc5, 0xac, 0x16, 0x25, 0x79, 0xa0, 0xa5, 0xa4, 0xad, 0x9b, 0x6b, 0x99, 0x9e, 0x21, 0x92, 0xa6, - 0xa3, 0x4e, 0xc2, 0x15, 0x0e, 0x6e, 0x21, 0x22, 0x8d, 0x8e, 0x19, 0xc9, 0x0e, 0xe4, 0x5b, 0x45, - 0x46, 0xdd, 0x55, 0x57, 0xc6, 0xb1, 0x05, 0xe7, 0x32, 0xda, 0xb7, 0x0c, 0xa3, 0x92, 0x12, 0x46, - 0x17, 0xc7, 0x32, 0x0a, 0x41, 0xee, 0x36, 0x15, 0x75, 0x1d, 0xce, 0x73, 0x60, 0x8f, 0x18, 0x66, - 0x64, 0x23, 0x74, 0x1e, 0x44, 0x2d, 0x3b, 0xbe, 0x57, 0x73, 0xd0, 0xc7, 0x5b, 0x78, 0x5c, 0xf3, - 0xc2, 0x6c, 0x29, 0xe5, 0x68, 0x6e, 0x72, 0xcf, 0x8c, 0xb9, 0x44, 0xc5, 0x52, 0xfd, 0x59, 0x81, - 0x52, 0x9a, 0x6b, 0x19, 0xe5, 0x3a, 0x9c, 0x12, 0xbe, 0xeb, 0x0e, 0xae, 0x11, 0x97, 0x78, 0x4c, - 0x1e, 0x31, 0x99, 0x72, 0xc4, 0x7d, 0xea, 0x59, 0x6b, 0xc4, 0x77, 0xb9, 0x8b, 0xd5, 0xd8, 0x40, - 0x9e, 0x78, 0x92, 0x26, 0xa4, 0x68, 0x14, 0x0a, 0x1b, 0xb6, 0xe3, 0x18, 0xd8, 0xa5, 0xa1, 0xc7, - 0x38, 0x27, 0x7b, 0x75, 0x88, 0x44, 0xf3, 0x5c, 0x82, 0x46, 0xa0, 0x9f, 0xf9, 0xb6, 0x65, 0x11, - 0x9f, 0x98, 0x9c, 0x9d, 0x7d, 0xfa, 0x9e, 0x40, 0xbd, 0x02, 0x97, 0x39, 0xec, 0xfb, 0x2d, 0x8f, - 0x4f, 0x6a, 0x51, 0x9f, 0x2b, 0x30, 0xbe, 0x9f, 0xa6, 0x0c, 0xf6, 0x19, 0x0c, 0xa6, 0xbc, 0x65, - 0x32, 0xe0, 0xcb, 0x69, 0x01, 0x77, 0xb8, 0x94, 0xc1, 0x22, 0xa7, 0x63, 0x47, 0x9d, 0x87, 0x0b, - 0x8f, 0x98, 0x4f, 0xb0, 0x48, 0x4f, 0x95, 0xd2, 0xad, 0xc7, 0xe2, 0x3d, 0x8b, 0xeb, 0xd8, 0x79, - 0x7f, 0x7b, 0x92, 0xf7, 0x57, 0xc5, 0x50, 0xce, 0x72, 0x21, 0x43, 0x78, 0x0f, 0x8e, 0xc9, 0x57, - 0x52, 0xf6, 0xa0, 0xd1, 0x14, 0xd8, 0xc2, 0x87, 0x30, 0x8d, 0xf9, 0x20, 0xad, 0xd4, 0x17, 0x79, - 0x38, 0xde, 0xba, 0x8f, 0x1e, 0xc3, 0x00, 0x8d, 0x4f, 0x93, 0x2f, 0xb0, 0xcc, 0xc8, 0x44, 0xa6, - 0xeb, 0x36, 0x78, 0xcb, 0x39, 0xfd, 0x14, 0x4d, 0x8a, 0xa2, 0x97, 0x47, 0x10, 0x2b, 0xaa, 0xb8, - 0xec, 0xd1, 0xe3, 0xfb, 0x3b, 0x5c, 0xb2, 0x1d, 0x67, 0x39, 0xa7, 0xf7, 0x73, 0xdb, 0x68, 0x81, - 0x96, 0xa0, 0xc0, 0xf0, 0x16, 0xf1, 0x0d, 0x2e, 0xe2, 0x44, 0x29, 0xcc, 0x5e, 0xca, 0xf4, 0xb4, - 0x16, 0xe9, 0x72, 0x77, 0xcb, 0x39, 0x1d, 0x58, 0x73, 0x85, 0x2e, 0xc2, 0x71, 0x71, 0x9f, 0x37, - 0x89, 0x6d, 0x6d, 0xb2, 0x62, 0x2f, 0xef, 0x9e, 0x05, 0x2e, 0x5b, 0xe6, 0x22, 0x34, 0x0c, 0xfd, - 0x64, 0x87, 0xd4, 0x0c, 0x97, 0x9a, 0xa4, 0x78, 0x84, 0xef, 0xf7, 0x45, 0x82, 0x15, 0x6a, 0x92, - 0x85, 0x01, 0x38, 0x29, 0xb2, 0x63, 0xb8, 0x24, 0x08, 0xb0, 0x45, 0xd4, 0xaf, 0x14, 0x18, 0x4a, - 0xcd, 0x07, 0x5a, 0x6f, 0xaf, 0xd2, 0xad, 0x24, 0x5e, 0x39, 0x0c, 0x55, 0x3a, 0x47, 0x9f, 0x07, - 0x1b, 0x1b, 0x8b, 0x91, 0x40, 0x38, 0x7a, 0x32, 0xd3, 0x56, 0x3e, 0x54, 0x82, 0xbe, 0xc0, 0xc3, - 0xf5, 0x60, 0x93, 0x8a, 0x2b, 0xd5, 0xa7, 0x37, 0xd7, 0xea, 0x8f, 0x0a, 0x0c, 0xa6, 0xa4, 0x13, - 0xcd, 0x01, 0xe7, 0x98, 0x78, 0x8d, 0x65, 0x6d, 0x47, 0x32, 0xa6, 0x08, 0xfe, 0xda, 0xea, 0x7c, - 0xe8, 0xe0, 0x1f, 0xd1, 0x4d, 0x38, 0xca, 0x13, 0x1f, 0xbd, 0xb3, 0x51, 0x24, 0xc5, 0xac, 0xd6, - 0x23, 0x91, 0x4a, 0xed, 0x28, 0xdd, 0x2d, 0xd7, 0x3f, 0x28, 0xf6, 0x8c, 0xf5, 0x4c, 0xf4, 0xea, - 0x85, 0xbd, 0xfb, 0x1f, 0xa8, 0xcf, 0xf3, 0x30, 0xd0, 0x5e, 0x34, 0x34, 0x0d, 0x47, 0x44, 0xa1, - 0x05, 0xce, 0xcc, 0xe3, 0x96, 0x73, 0xba, 0x50, 0x44, 0xeb, 0x70, 0xba, 0xe5, 0x36, 0x4a, 0x9a, - 0xe4, 0x33, 0x9b, 0x98, 0x38, 0xb1, 0xe5, 0x66, 0xc7, 0xee, 0x06, 0x9c, 0x36, 0x19, 0x7a, 0x0a, - 0xa8, 0x85, 0x7a, 0x46, 0xc0, 0x30, 0x0b, 0x03, 0xc9, 0xc0, 0xc9, 0x03, 0x30, 0xf0, 0x11, 0x37, - 0xd0, 0x07, 0x58, 0x9b, 0x64, 0xe1, 0x44, 0x82, 0xd3, 0xea, 0x4f, 0x0a, 0x9c, 0x4d, 0xb7, 0x8d, - 0xd2, 0x98, 0x38, 0x5c, 0xbc, 0xf9, 0x05, 0xda, 0xa2, 0x72, 0x0d, 0x90, 0x4f, 0x5c, 0x6c, 0x7b, - 0xb6, 0x67, 0x19, 0xdb, 0x21, 0xf6, 0x58, 0xe8, 0x06, 0xb2, 0xdf, 0x9e, 0x6e, 0xee, 0x3c, 0x94, - 0x1b, 0xe8, 0x7d, 0x28, 0xd3, 0x3a, 0xb3, 0x5d, 0x3b, 0x60, 0x76, 0x0d, 0x3b, 0xce, 0x2e, 0xbf, - 0xa1, 0xc4, 0xdc, 0x33, 0x15, 0x93, 0xc2, 0x48, 0x52, 0x6b, 0x89, 0x2b, 0xc5, 0x5e, 0x66, 0xbf, - 0x07, 0x38, 0xc2, 0xbb, 0x2e, 0xfa, 0x42, 0x81, 0xbe, 0x78, 0xfe, 0x44, 0x53, 0x29, 0x59, 0xc9, - 0x18, 0xe2, 0x4b, 0x13, 0x59, 0xba, 0xed, 0x53, 0xbc, 0x3a, 0xf9, 0xf9, 0x1f, 0xff, 0x7e, 0x93, - 0xbf, 0x84, 0x2e, 0x6a, 0x5d, 0xbe, 0x31, 0x69, 0x9f, 0xd8, 0xe6, 0xa7, 0xe8, 0x4b, 0x05, 0x0a, - 0x2d, 0x83, 0x74, 0x36, 0xa0, 0xce, 0x89, 0xbe, 0x74, 0x75, 0x3f, 0x40, 0x2d, 0x93, 0xb9, 0xfa, - 0x16, 0xc7, 0x54, 0x46, 0x23, 0xdd, 0x30, 0xa1, 0x5f, 0x15, 0x28, 0x66, 0x4d, 0x84, 0x68, 0xf6, - 0x8d, 0xc6, 0x47, 0x81, 0xf1, 0xfa, 0x21, 0x46, 0x4e, 0xf5, 0x36, 0xc7, 0x7a, 0xe3, 0xb6, 0x32, - 0xa5, 0x6a, 0x5a, 0xea, 0x57, 0x36, 0xc3, 0xa3, 0x26, 0x31, 0x18, 0x15, 0xff, 0x6b, 0x2d, 0x20, - 0x7f, 0x57, 0x60, 0xa4, 0xdb, 0x70, 0x86, 0xe6, 0xb2, 0xb2, 0x76, 0x80, 0xd1, 0xb2, 0xf4, 0xee, - 0xe1, 0x8c, 0x65, 0x5c, 0xe3, 0x3c, 0xae, 0x31, 0x54, 0xd6, 0xba, 0x7e, 0x4d, 0x46, 0xbf, 0x28, - 0x30, 0xdc, 0x65, 0x32, 0x43, 0xb7, 0xb3, 0x50, 0xec, 0x3f, 0x53, 0x96, 0xe6, 0x0e, 0x65, 0x2b, - 0x03, 0xb8, 0xcc, 0x03, 0x18, 0x45, 0x17, 0xba, 0xfe, 0x76, 0x80, 0x7e, 0x53, 0xe0, 0x7c, 0xe6, - 0x74, 0x83, 0x6e, 0x65, 0x21, 0xd8, 0x6f, 0x74, 0x2a, 0xbd, 0x73, 0x08, 0x4b, 0x89, 0xbc, 0xc2, - 0x91, 0x4f, 0xa0, 0x71, 0xed, 0x40, 0xbf, 0x17, 0x20, 0x0f, 0x4e, 0x24, 0x06, 0x50, 0xf4, 0x76, - 0xd6, 0xd9, 0x69, 0x23, 0x70, 0xe9, 0xda, 0x01, 0xb5, 0x25, 0xba, 0x1c, 0xfa, 0x2c, 0xee, 0xa8, - 0xed, 0x93, 0x14, 0x9a, 0x3e, 0xe8, 0x54, 0x13, 0xcf, 0x6d, 0xa5, 0x99, 0x37, 0xb0, 0x10, 0x00, - 0xa6, 0x95, 0x85, 0xd5, 0x17, 0xaf, 0xca, 0xca, 0xcb, 0x57, 0x65, 0xe5, 0x9f, 0x57, 0x65, 0xe5, - 0xeb, 0xd7, 0xe5, 0xdc, 0xcb, 0xd7, 0xe5, 0xdc, 0x9f, 0xaf, 0xcb, 0xb9, 0x8f, 0x6e, 0x5a, 0x36, - 0xdb, 0x0c, 0xab, 0x95, 0x1a, 0x75, 0x93, 0xc9, 0x6b, 0xdc, 0xb8, 0xc6, 0x87, 0x01, 0xad, 0x29, - 0xd9, 0x11, 0x09, 0x65, 0xbb, 0x75, 0x12, 0x54, 0x8f, 0x72, 0xf1, 0xf5, 0xff, 0x03, 0x00, 0x00, - 0xff, 0xff, 0xe3, 0x8b, 0xbe, 0xb6, 0xfa, 0x12, 0x00, 0x00, + // 1778 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x58, 0x41, 0x6f, 0x1c, 0x49, + 0x15, 0x9e, 0x1e, 0x3b, 0xc9, 0xe4, 0x4d, 0x9c, 0x75, 0x2a, 0xeb, 0x64, 0x32, 0x76, 0xc6, 0x4e, + 0x2f, 0x71, 0xec, 0x2c, 0x99, 0x4e, 0xbc, 0xab, 0xd5, 0x92, 0xa0, 0x45, 0x76, 0x20, 0x71, 0x44, + 0xcc, 0xce, 0x76, 0xb2, 0x59, 0x0b, 0x22, 0xb5, 0x6a, 0xba, 0xcb, 0xe3, 0x96, 0xbb, 0xbb, 0xc6, + 0x5d, 0xd5, 0x23, 0x5b, 0x08, 0x21, 0x71, 0xd8, 0x0b, 0x20, 0xad, 0xc4, 0x81, 0x03, 0x12, 0x07, + 0x38, 0x23, 0x71, 0xe1, 0x88, 0x80, 0xdb, 0x1e, 0x23, 0x21, 0x24, 0x0e, 0x08, 0xa1, 0x84, 0x33, + 0xbf, 0x01, 0x75, 0x55, 0xf5, 0x4c, 0xf7, 0x74, 0xf7, 0x8c, 0xe3, 0x8b, 0x3d, 0xf5, 0xea, 0xbd, + 0xef, 0x7d, 0xef, 0xd5, 0xab, 0x57, 0x55, 0x0d, 0xd7, 0x9d, 0x63, 0xe7, 0xa8, 0x1f, 0x52, 0x4e, + 0x6d, 0xea, 0x19, 0xb6, 0x47, 0xbb, 0xc6, 0x61, 0x44, 0xc2, 0xe3, 0xb6, 0x90, 0xa1, 0x4b, 0xe9, + 0xe9, 0x76, 0x3c, 0xdd, 0x7c, 0xb7, 0x47, 0x7b, 0x54, 0x88, 0x8c, 0xf8, 0x97, 0x54, 0x6c, 0x2e, + 0xf5, 0x28, 0xed, 0x79, 0xc4, 0xc0, 0x7d, 0xd7, 0xc0, 0x41, 0x40, 0x39, 0xe6, 0x2e, 0x0d, 0x98, + 0x9a, 0xbd, 0x6d, 0x53, 0xe6, 0x53, 0x66, 0x74, 0x31, 0x23, 0x12, 0xdf, 0x18, 0xdc, 0xeb, 0x12, + 0x8e, 0xef, 0x19, 0x7d, 0xdc, 0x73, 0x03, 0xa1, 0xac, 0x74, 0x8d, 0x3c, 0xa3, 0xae, 0x47, 0xed, + 0x03, 0x2b, 0xc4, 0x9c, 0x58, 0x9e, 0xeb, 0xbb, 0xdc, 0xb2, 0x69, 0xb0, 0xe7, 0xf6, 0x94, 0xc1, + 0x8d, 0xbc, 0x41, 0xfc, 0xc7, 0xea, 0x63, 0x37, 0x54, 0x2a, 0x77, 0xf3, 0x2a, 0xe4, 0x30, 0x72, + 0xf9, 0xb1, 0xc5, 0x5d, 0x12, 0x16, 0x81, 0x16, 0xe4, 0x85, 0x86, 0x0e, 0x49, 0x00, 0x97, 0xf3, + 0xd3, 0x3e, 0xe6, 0xf6, 0x3e, 0x49, 0x22, 0x7e, 0x3f, 0xaf, 0xe0, 0xb9, 0x87, 0x91, 0xeb, 0xc8, + 0xbc, 0x64, 0x9d, 0x2d, 0x16, 0xa0, 0x91, 0x81, 0x9a, 0xfc, 0x24, 0x33, 0xe9, 0x06, 0x0e, 0x39, + 0x22, 0xa1, 0x41, 0xf7, 0xf6, 0x2c, 0x7b, 0x1f, 0xbb, 0x81, 0x15, 0xf5, 0x1d, 0xcc, 0x09, 0xcb, + 0x4b, 0x94, 0xfd, 0x7a, 0xc6, 0x9e, 0x45, 0x5d, 0x6c, 0xdb, 0x34, 0x0a, 0x38, 0x4b, 0xfd, 0x96, + 0xaa, 0xfa, 0x3a, 0x5c, 0xfd, 0x2c, 0x5e, 0x9c, 0xc7, 0x84, 0x3f, 0xf4, 0x68, 0xb7, 0x83, 0xdd, + 0xd0, 0x24, 0x87, 0x11, 0x61, 0x1c, 0x5d, 0x84, 0xaa, 0xeb, 0x34, 0xb4, 0x15, 0x6d, 0x6d, 0xce, + 0xac, 0xba, 0x8e, 0xfe, 0x05, 0x2c, 0x08, 0xd5, 0x91, 0x1e, 0xeb, 0xd3, 0x80, 0x11, 0xf4, 0x09, + 0x9c, 0x1f, 0x66, 0x5f, 0xe8, 0xd7, 0x37, 0x16, 0xdb, 0xb9, 0x2a, 0x6a, 0x27, 0x76, 0x5b, 0xb3, + 0x5f, 0xff, 0x7b, 0xb9, 0x62, 0xd6, 0x6c, 0x35, 0xd6, 0xb1, 0xe2, 0xb0, 0xe9, 0x79, 0xe3, 0x1c, + 0x1e, 0x01, 0x8c, 0xaa, 0x45, 0x61, 0xaf, 0xb6, 0x65, 0x69, 0xb5, 0xe3, 0xd2, 0x6a, 0xcb, 0xd2, + 0x55, 0xa5, 0xd5, 0xee, 0xe0, 0x1e, 0x51, 0xb6, 0x66, 0xca, 0x52, 0xff, 0xbd, 0x06, 0x8d, 0x0c, + 0xf9, 0x4d, 0xcf, 0x2b, 0xe3, 0x3f, 0xf3, 0x96, 0xfc, 0xd1, 0xe3, 0x0c, 0xc9, 0xaa, 0x20, 0x79, + 0x6b, 0x2a, 0x49, 0xe9, 0x3c, 0xc3, 0xf2, 0x5f, 0x1a, 0x2c, 0xef, 0x90, 0xc1, 0x0f, 0xa8, 0x43, + 0x9e, 0xd3, 0xf8, 0xef, 0x43, 0xec, 0xd9, 0x91, 0x27, 0x26, 0x93, 0x8c, 0xbc, 0x84, 0x2b, 0x72, + 0x6f, 0xf4, 0x43, 0xda, 0xa7, 0x8c, 0x84, 0x96, 0xaa, 0xc2, 0x61, 0x76, 0xf2, 0xcc, 0x5f, 0x60, + 0x2f, 0xae, 0x42, 0x1a, 0xee, 0x90, 0xc1, 0x8e, 0xd4, 0x36, 0xdf, 0x15, 0x28, 0x1d, 0x05, 0xa2, + 0xa4, 0xe8, 0x47, 0xb0, 0x30, 0x48, 0x94, 0x2d, 0x9f, 0x0c, 0x2c, 0x9f, 0xf0, 0xd0, 0xb5, 0xd9, + 0x30, 0xaa, 0x3c, 0x78, 0x86, 0xf0, 0x8e, 0x54, 0x37, 0x2f, 0x0f, 0xd2, 0x2e, 0xa5, 0x50, 0xff, + 0x9f, 0x06, 0x2b, 0xe5, 0xe1, 0xa9, 0xc5, 0xe8, 0xc1, 0xb9, 0x90, 0xb0, 0xc8, 0xe3, 0x4c, 0x2d, + 0xc5, 0xe3, 0x69, 0x3e, 0x0b, 0x50, 0x62, 0x85, 0xcd, 0xc0, 0x79, 0x41, 0xbd, 0xc8, 0x27, 0x1d, + 0x12, 0xc6, 0x4b, 0xa7, 0x96, 0x2d, 0x41, 0x6f, 0x62, 0xb8, 0x5c, 0xa0, 0x85, 0x56, 0xe0, 0xc2, + 0xb0, 0x18, 0xac, 0x61, 0xfd, 0x43, 0xb2, 0xd8, 0x4f, 0x1c, 0x34, 0x0f, 0x33, 0x3e, 0x19, 0x88, + 0x8c, 0x54, 0xcd, 0xf8, 0x27, 0xba, 0x02, 0x67, 0x07, 0x02, 0xa4, 0x31, 0xb3, 0xa2, 0xad, 0xcd, + 0x9a, 0x6a, 0xa4, 0xdf, 0x86, 0x35, 0x51, 0x74, 0xdf, 0x13, 0x8d, 0xe7, 0xb9, 0x4b, 0xc2, 0xa7, + 0x71, 0xdb, 0x79, 0x28, 0x1a, 0x41, 0x14, 0xa6, 0xd7, 0x55, 0xff, 0x8d, 0x06, 0xeb, 0x27, 0x50, + 0x56, 0x59, 0x0a, 0xa0, 0x51, 0xd6, 0xcd, 0x54, 0x1d, 0x18, 0x05, 0x69, 0x9b, 0x04, 0xad, 0xd2, + 0xb3, 0x40, 0x8a, 0x74, 0xf4, 0x75, 0xb8, 0x25, 0xc8, 0x6d, 0xc5, 0x45, 0x63, 0x62, 0x4e, 0xca, + 0x03, 0xf9, 0xb5, 0xa6, 0xa2, 0x9e, 0xa8, 0xab, 0xe2, 0x38, 0x80, 0xab, 0x25, 0x9d, 0x5e, 0x85, + 0xd1, 0x2e, 0x08, 0x63, 0x02, 0xb0, 0x8a, 0x42, 0x16, 0xf7, 0x98, 0x8a, 0xbe, 0x0b, 0xd7, 0x04, + 0xb1, 0x67, 0x1c, 0x73, 0xb2, 0x17, 0x79, 0x9f, 0xc6, 0xdd, 0x3d, 0xd9, 0x57, 0x0f, 0xa0, 0x26, + 0xba, 0x7d, 0xb2, 0xe6, 0xf5, 0x8d, 0x66, 0x81, 0x6b, 0x61, 0xf2, 0xc4, 0x49, 0x6a, 0x89, 0xca, + 0xa1, 0xfe, 0x27, 0x0d, 0x9a, 0x45, 0xd0, 0x2a, 0xca, 0x5d, 0x78, 0x47, 0x62, 0xf7, 0x3d, 0x6c, + 0x13, 0x9f, 0x04, 0x5c, 0xb9, 0x58, 0x2f, 0x70, 0xf1, 0x94, 0x06, 0xbd, 0xe7, 0x24, 0xf4, 0x05, + 0x44, 0x27, 0x31, 0x50, 0x1e, 0x2f, 0xd2, 0x8c, 0x14, 0x2d, 0x43, 0x7d, 0xcf, 0xf5, 0x3c, 0x0b, + 0xfb, 0x71, 0x4f, 0x17, 0x35, 0x39, 0x6b, 0x42, 0x2c, 0xda, 0x14, 0x12, 0xb4, 0x04, 0xe7, 0x79, + 0xe8, 0xf6, 0x7a, 0x24, 0x24, 0x8e, 0xa8, 0xce, 0x9a, 0x39, 0x12, 0xe8, 0xb7, 0xe0, 0xa6, 0xa0, + 0xfd, 0x34, 0x75, 0x4e, 0x15, 0x2e, 0xea, 0x97, 0x1a, 0xac, 0x4e, 0xd3, 0x54, 0xc1, 0xbe, 0x84, + 0xcb, 0x05, 0xc7, 0x9e, 0x0a, 0xf8, 0x66, 0x51, 0xc0, 0x39, 0x48, 0x15, 0x2c, 0xf2, 0x72, 0x33, + 0xfa, 0x57, 0x1a, 0x5c, 0x7f, 0xc6, 0x43, 0x82, 0x65, 0x7e, 0xba, 0x94, 0x1e, 0x7c, 0x2e, 0xcf, + 0xbe, 0x64, 0x21, 0xf3, 0x1b, 0x78, 0x66, 0x6c, 0x03, 0xef, 0xc0, 0xc5, 0xd1, 0x39, 0x68, 0xb9, + 0x4e, 0xdc, 0xdd, 0x66, 0xf2, 0xad, 0x33, 0x75, 0x6e, 0xb6, 0x9f, 0x0d, 0x7f, 0x3f, 0x71, 0xcc, + 0x39, 0x96, 0x1a, 0x31, 0x1d, 0x43, 0xab, 0x8c, 0x91, 0x4a, 0xc9, 0x77, 0xe0, 0x9c, 0x3a, 0xa0, + 0x55, 0x4f, 0x5b, 0x2e, 0x48, 0x83, 0xc4, 0x90, 0xa6, 0x49, 0x7d, 0x29, 0x2b, 0xfd, 0x77, 0x33, + 0x70, 0x21, 0x3d, 0x8f, 0x3e, 0x87, 0x79, 0x9a, 0x78, 0x53, 0x87, 0xbf, 0xca, 0xf0, 0x5a, 0x29, + 0xf4, 0x18, 0xbd, 0xed, 0x8a, 0xf9, 0x0e, 0xcd, 0x8a, 0xe2, 0x93, 0x4c, 0x16, 0x6a, 0x5c, 0x41, + 0xaa, 0xe7, 0xaf, 0x4e, 0x07, 0x7c, 0xe4, 0x7a, 0xde, 0x76, 0xc5, 0x3c, 0x2f, 0x6c, 0xe3, 0x01, + 0x7a, 0x04, 0x75, 0x8e, 0x0f, 0x48, 0x68, 0x09, 0x91, 0x28, 0xbc, 0xfa, 0xc6, 0x7b, 0xa5, 0x48, + 0xcf, 0x63, 0x5d, 0x01, 0xb7, 0x5d, 0x31, 0x81, 0x0f, 0x47, 0x68, 0x17, 0x2e, 0xa5, 0x96, 0x4a, + 0x05, 0x3a, 0x5b, 0xba, 0x77, 0x24, 0xda, 0x68, 0xb1, 0x86, 0x91, 0xce, 0xb3, 0x31, 0x19, 0xba, + 0x01, 0x17, 0x64, 0xe7, 0xd9, 0x27, 0x6e, 0x6f, 0x9f, 0x37, 0xce, 0x88, 0x3e, 0x5f, 0x17, 0xb2, + 0x6d, 0x21, 0x42, 0x8b, 0x70, 0x9e, 0x1c, 0x11, 0xdb, 0xf2, 0xa9, 0x43, 0x1a, 0x67, 0xc5, 0x7c, + 0x2d, 0x16, 0xec, 0x50, 0x87, 0x6c, 0xcd, 0xc3, 0x45, 0x49, 0xc7, 0xf2, 0x09, 0x63, 0xb8, 0x47, + 0xf4, 0x97, 0xb0, 0x38, 0xf2, 0xdc, 0x21, 0x61, 0x9f, 0xf0, 0x08, 0x7b, 0x1d, 0xca, 0xdc, 0xb8, + 0x80, 0x63, 0x87, 0xfd, 0x44, 0x38, 0x3a, 0x58, 0xea, 0x43, 0xd9, 0x13, 0x07, 0x35, 0xa1, 0x76, + 0x18, 0xe1, 0x80, 0x47, 0x3e, 0x53, 0x5b, 0x79, 0x38, 0xd6, 0x3b, 0x70, 0x75, 0x84, 0xbe, 0xc9, + 0x18, 0xe1, 0x43, 0xe4, 0x6b, 0x50, 0xc3, 0xb1, 0x60, 0x84, 0x7a, 0x4e, 0x8c, 0xa7, 0x20, 0xfe, + 0xa3, 0x0a, 0x57, 0x8a, 0x13, 0x86, 0xbe, 0x0f, 0x73, 0x99, 0x1d, 0x52, 0x7c, 0xb7, 0x28, 0xdd, + 0x20, 0x17, 0xd2, 0x1b, 0x04, 0x05, 0xb0, 0x28, 0x33, 0xe5, 0x58, 0xa3, 0x04, 0xf4, 0x15, 0xf9, + 0x64, 0xef, 0x15, 0xf5, 0xf9, 0x09, 0xd9, 0x34, 0xaf, 0x29, 0xc8, 0xdc, 0x0c, 0x43, 0x5d, 0xb8, + 0x9a, 0xf8, 0x93, 0x69, 0x19, 0xf9, 0x9a, 0x11, 0xbe, 0x6e, 0x4f, 0xf4, 0x95, 0xc9, 0xad, 0xb9, + 0xa0, 0xa0, 0x32, 0x52, 0x16, 0xe7, 0x95, 0x05, 0xb8, 0xcf, 0xf6, 0x29, 0x17, 0xe5, 0x58, 0x33, + 0x87, 0x63, 0xfd, 0x97, 0x1a, 0x2c, 0x14, 0xee, 0x38, 0xb4, 0x3b, 0xde, 0x07, 0x3e, 0xce, 0x32, + 0x51, 0x37, 0xfd, 0x76, 0xfe, 0x5e, 0xff, 0xe9, 0xde, 0xde, 0xc3, 0x58, 0x20, 0x81, 0x5e, 0xdc, + 0x1b, 0x6b, 0x10, 0x19, 0x3e, 0xd5, 0x31, 0x3e, 0x7f, 0xd0, 0xe0, 0x72, 0xc1, 0x86, 0x45, 0x0f, + 0x40, 0x34, 0x45, 0x79, 0x7f, 0x54, 0x2b, 0xbc, 0x54, 0x72, 0xef, 0x15, 0xf7, 0x43, 0x53, 0x5c, + 0x93, 0xc5, 0x4f, 0xf4, 0x11, 0x9c, 0x15, 0x5b, 0x3b, 0x59, 0xbf, 0x46, 0xd9, 0x61, 0xa9, 0x98, + 0x2a, 0xed, 0x78, 0x17, 0xa4, 0x0e, 0x2c, 0xb9, 0x22, 0xb3, 0x66, 0x7d, 0x74, 0x62, 0x31, 0xfd, + 0xcb, 0x2a, 0xcc, 0x8f, 0xb7, 0x05, 0x74, 0x17, 0xce, 0xc8, 0x56, 0x22, 0x79, 0x96, 0xba, 0xdb, + 0xae, 0x98, 0x52, 0x31, 0x6e, 0x1d, 0xa9, 0xf3, 0x43, 0x35, 0xa2, 0xea, 0x94, 0xd6, 0x91, 0x3a, + 0x8b, 0x12, 0xb8, 0x79, 0x6f, 0x4c, 0x86, 0xbe, 0x00, 0x94, 0x6a, 0x6e, 0x16, 0xe3, 0x98, 0x47, + 0x4c, 0xf5, 0xb8, 0xf5, 0x13, 0xf4, 0xb8, 0x67, 0xc2, 0xc0, 0x9c, 0xe7, 0x63, 0x92, 0xad, 0xb9, + 0x4c, 0xd7, 0xd4, 0xff, 0xa8, 0x25, 0x1b, 0x74, 0xdc, 0x36, 0x4e, 0x63, 0xc6, 0xb9, 0x6a, 0x26, + 0x34, 0xa5, 0x72, 0x07, 0x50, 0x48, 0x7c, 0xec, 0x06, 0x6e, 0xd0, 0xb3, 0xc6, 0x9a, 0xc0, 0xa5, + 0xe1, 0xcc, 0x67, 0x6a, 0x02, 0x7d, 0x17, 0x5a, 0xb4, 0xcf, 0x5d, 0xdf, 0x65, 0xdc, 0xb5, 0xb1, + 0xe7, 0x1d, 0x8b, 0x33, 0x80, 0x38, 0x23, 0x53, 0x79, 0xb7, 0x5d, 0xca, 0x6a, 0x3d, 0x12, 0x4a, + 0x09, 0xca, 0xc6, 0x6f, 0x01, 0xce, 0x88, 0x7b, 0x02, 0xfa, 0xb9, 0x06, 0xb5, 0xe4, 0xc5, 0x84, + 0x8a, 0x76, 0x5c, 0xc9, 0xb3, 0xb3, 0xb9, 0x56, 0xa6, 0x3b, 0xfe, 0xee, 0xd4, 0xd7, 0x7f, 0xf6, + 0xf7, 0xff, 0xfe, 0xaa, 0xfa, 0x1e, 0xba, 0x61, 0x4c, 0xf8, 0x1c, 0x60, 0xfc, 0xd8, 0x75, 0x7e, + 0x82, 0x7e, 0xa1, 0x41, 0x3d, 0xf5, 0xf4, 0x2b, 0x27, 0x94, 0x7f, 0x83, 0x36, 0xdf, 0x9f, 0x46, + 0x28, 0xf5, 0x96, 0xd4, 0xbf, 0x21, 0x38, 0xb5, 0xd0, 0xd2, 0x24, 0x4e, 0xe8, 0x2f, 0x1a, 0x34, + 0xca, 0xde, 0x30, 0x68, 0xe3, 0xad, 0x1e, 0x3c, 0x92, 0xe3, 0x07, 0xa7, 0x78, 0x24, 0xe9, 0xf7, + 0x05, 0xd7, 0x0f, 0xef, 0x6b, 0xb7, 0x75, 0xc3, 0x28, 0xfc, 0x1e, 0x61, 0x05, 0xd4, 0x21, 0x16, + 0xa7, 0xf2, 0xbf, 0x9d, 0x22, 0xf9, 0x37, 0x0d, 0x96, 0x26, 0x3d, 0x27, 0xd0, 0x83, 0xb2, 0xac, + 0x9d, 0xe0, 0x31, 0xd4, 0xfc, 0xf6, 0xe9, 0x8c, 0x55, 0x5c, 0xab, 0x22, 0xae, 0x15, 0xd4, 0x32, + 0x26, 0x7e, 0x03, 0x42, 0x7f, 0xd6, 0x60, 0x71, 0xc2, 0x5b, 0x02, 0xdd, 0x2f, 0x63, 0x31, 0xfd, + 0x15, 0xd4, 0x7c, 0x70, 0x2a, 0x5b, 0x15, 0xc0, 0x4d, 0x11, 0xc0, 0x32, 0xba, 0x3e, 0xf1, 0xc3, + 0x18, 0xfa, 0xab, 0x06, 0xd7, 0x4a, 0xef, 0xe3, 0xe8, 0xe3, 0x32, 0x06, 0xd3, 0x2e, 0xfb, 0xcd, + 0x6f, 0x9d, 0xc2, 0x52, 0x31, 0x6f, 0x0b, 0xe6, 0x6b, 0x68, 0xd5, 0x38, 0xd1, 0xc7, 0x30, 0x14, + 0xc0, 0x5c, 0xe6, 0xc9, 0x84, 0xbe, 0x59, 0xe6, 0xbb, 0xe8, 0xd1, 0xd6, 0xbc, 0x73, 0x42, 0x6d, + 0xc5, 0xae, 0x82, 0x7e, 0x9a, 0x74, 0xd4, 0xf1, 0xbb, 0x3a, 0xba, 0x7b, 0xd2, 0x7b, 0x73, 0xf2, + 0xd0, 0x68, 0xde, 0x7b, 0x0b, 0x0b, 0x49, 0xe0, 0xae, 0xb6, 0xd5, 0xf9, 0xfa, 0x75, 0x4b, 0x7b, + 0xf5, 0xba, 0xa5, 0xfd, 0xe7, 0x75, 0x4b, 0xfb, 0xea, 0x4d, 0xab, 0xf2, 0xea, 0x4d, 0xab, 0xf2, + 0xcf, 0x37, 0xad, 0xca, 0x0f, 0x3f, 0xea, 0xb9, 0x7c, 0x3f, 0xea, 0xb6, 0x6d, 0xea, 0x67, 0x93, + 0x37, 0xf8, 0xf0, 0x8e, 0xb8, 0x0c, 0x18, 0x43, 0xc9, 0x91, 0x4c, 0x28, 0x3f, 0xee, 0x13, 0xd6, + 0x3d, 0x2b, 0xc4, 0x1f, 0xfc, 0x3f, 0x00, 0x00, 0xff, 0xff, 0x1a, 0x19, 0x57, 0xae, 0xd7, 0x15, + 0x00, 0x00, } // Reference imports to suppress errors if they are not otherwise used. @@ -2327,6 +2557,20 @@ func (m *StreamOrderbookUpdatesRequest) MarshalToSizedBuffer(dAtA []byte) (int, _ = i var l int _ = l + if len(m.SubaccountIds) > 0 { + for iNdEx := len(m.SubaccountIds) - 1; iNdEx >= 0; iNdEx-- { + { + size, err := m.SubaccountIds[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintQuery(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x12 + } + } if len(m.ClobPairId) > 0 { dAtA12 := make([]byte, len(m.ClobPairId)*10) var j11 int @@ -2408,12 +2652,12 @@ func (m *StreamUpdate) MarshalToSizedBuffer(dAtA []byte) (int, error) { if m.ExecMode != 0 { i = encodeVarintQuery(dAtA, i, uint64(m.ExecMode)) i-- - dAtA[i] = 0x28 + dAtA[i] = 0x30 } if m.BlockHeight != 0 { i = encodeVarintQuery(dAtA, i, uint64(m.BlockHeight)) i-- - dAtA[i] = 0x20 + dAtA[i] = 0x28 } if m.UpdateMessage != nil { { @@ -2490,7 +2734,28 @@ func (m *StreamUpdate_TakerOrder) MarshalToSizedBuffer(dAtA []byte) (int, error) } return len(dAtA) - i, nil } -func (m *StreamOrderbookUpdate) Marshal() (dAtA []byte, err error) { +func (m *StreamUpdate_SubaccountUpdate) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *StreamUpdate_SubaccountUpdate) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + if m.SubaccountUpdate != nil { + { + size, err := m.SubaccountUpdate.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintQuery(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x22 + } + return len(dAtA) - i, nil +} +func (m *SubaccountPerpetualPosition) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) @@ -2500,44 +2765,63 @@ func (m *StreamOrderbookUpdate) Marshal() (dAtA []byte, err error) { return dAtA[:n], nil } -func (m *StreamOrderbookUpdate) MarshalTo(dAtA []byte) (int, error) { +func (m *SubaccountPerpetualPosition) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } -func (m *StreamOrderbookUpdate) MarshalToSizedBuffer(dAtA []byte) (int, error) { +func (m *SubaccountPerpetualPosition) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l - if m.Snapshot { + if m.Quantums != 0 { + i = encodeVarintQuery(dAtA, i, uint64(m.Quantums)) i-- - if m.Snapshot { - dAtA[i] = 1 - } else { - dAtA[i] = 0 - } + dAtA[i] = 0x10 + } + if m.PerpetualId != 0 { + i = encodeVarintQuery(dAtA, i, uint64(m.PerpetualId)) + i-- + dAtA[i] = 0x8 + } + return len(dAtA) - i, nil +} + +func (m *SubaccountAssetPosition) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *SubaccountAssetPosition) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *SubaccountAssetPosition) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.Quantums != 0 { + i = encodeVarintQuery(dAtA, i, uint64(m.Quantums)) i-- dAtA[i] = 0x10 } - if len(m.Updates) > 0 { - for iNdEx := len(m.Updates) - 1; iNdEx >= 0; iNdEx-- { - { - size, err := m.Updates[iNdEx].MarshalToSizedBuffer(dAtA[:i]) - if err != nil { - return 0, err - } - i -= size - i = encodeVarintQuery(dAtA, i, uint64(size)) - } - i-- - dAtA[i] = 0xa - } + if m.AssetId != 0 { + i = encodeVarintQuery(dAtA, i, uint64(m.AssetId)) + i-- + dAtA[i] = 0x8 } return len(dAtA) - i, nil } -func (m *StreamOrderbookFill) Marshal() (dAtA []byte, err error) { +func (m *StreamSubaccountUpdate) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) @@ -2547,38 +2831,44 @@ func (m *StreamOrderbookFill) Marshal() (dAtA []byte, err error) { return dAtA[:n], nil } -func (m *StreamOrderbookFill) MarshalTo(dAtA []byte) (int, error) { +func (m *StreamSubaccountUpdate) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } -func (m *StreamOrderbookFill) MarshalToSizedBuffer(dAtA []byte) (int, error) { +func (m *StreamSubaccountUpdate) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l - if len(m.FillAmounts) > 0 { - dAtA17 := make([]byte, len(m.FillAmounts)*10) - var j16 int - for _, num := range m.FillAmounts { - for num >= 1<<7 { - dAtA17[j16] = uint8(uint64(num)&0x7f | 0x80) - num >>= 7 - j16++ - } - dAtA17[j16] = uint8(num) - j16++ + if m.Snapshot { + i-- + if m.Snapshot { + dAtA[i] = 1 + } else { + dAtA[i] = 0 } - i -= j16 - copy(dAtA[i:], dAtA17[:j16]) - i = encodeVarintQuery(dAtA, i, uint64(j16)) i-- - dAtA[i] = 0x1a + dAtA[i] = 0x20 } - if len(m.Orders) > 0 { - for iNdEx := len(m.Orders) - 1; iNdEx >= 0; iNdEx-- { + if len(m.UpdatedAssetPositions) > 0 { + for iNdEx := len(m.UpdatedAssetPositions) - 1; iNdEx >= 0; iNdEx-- { { - size, err := m.Orders[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + size, err := m.UpdatedAssetPositions[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintQuery(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x1a + } + } + if len(m.UpdatedPerpetualPositions) > 0 { + for iNdEx := len(m.UpdatedPerpetualPositions) - 1; iNdEx >= 0; iNdEx-- { + { + size, err := m.UpdatedPerpetualPositions[iNdEx].MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } @@ -2589,9 +2879,9 @@ func (m *StreamOrderbookFill) MarshalToSizedBuffer(dAtA []byte) (int, error) { dAtA[i] = 0x12 } } - if m.ClobMatch != nil { + if m.SubaccountId != nil { { - size, err := m.ClobMatch.MarshalToSizedBuffer(dAtA[:i]) + size, err := m.SubaccountId.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } @@ -2604,7 +2894,7 @@ func (m *StreamOrderbookFill) MarshalToSizedBuffer(dAtA []byte) (int, error) { return len(dAtA) - i, nil } -func (m *StreamTakerOrder) Marshal() (dAtA []byte, err error) { +func (m *StreamOrderbookUpdate) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) @@ -2614,27 +2904,141 @@ func (m *StreamTakerOrder) Marshal() (dAtA []byte, err error) { return dAtA[:n], nil } -func (m *StreamTakerOrder) MarshalTo(dAtA []byte) (int, error) { +func (m *StreamOrderbookUpdate) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } -func (m *StreamTakerOrder) MarshalToSizedBuffer(dAtA []byte) (int, error) { +func (m *StreamOrderbookUpdate) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l - if m.TakerOrderStatus != nil { - { - size, err := m.TakerOrderStatus.MarshalToSizedBuffer(dAtA[:i]) - if err != nil { - return 0, err - } - i -= size - i = encodeVarintQuery(dAtA, i, uint64(size)) + if m.Snapshot { + i-- + if m.Snapshot { + dAtA[i] = 1 + } else { + dAtA[i] = 0 } i-- - dAtA[i] = 0x1a + dAtA[i] = 0x10 + } + if len(m.Updates) > 0 { + for iNdEx := len(m.Updates) - 1; iNdEx >= 0; iNdEx-- { + { + size, err := m.Updates[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintQuery(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0xa + } + } + return len(dAtA) - i, nil +} + +func (m *StreamOrderbookFill) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *StreamOrderbookFill) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *StreamOrderbookFill) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if len(m.FillAmounts) > 0 { + dAtA19 := make([]byte, len(m.FillAmounts)*10) + var j18 int + for _, num := range m.FillAmounts { + for num >= 1<<7 { + dAtA19[j18] = uint8(uint64(num)&0x7f | 0x80) + num >>= 7 + j18++ + } + dAtA19[j18] = uint8(num) + j18++ + } + i -= j18 + copy(dAtA[i:], dAtA19[:j18]) + i = encodeVarintQuery(dAtA, i, uint64(j18)) + i-- + dAtA[i] = 0x1a + } + if len(m.Orders) > 0 { + for iNdEx := len(m.Orders) - 1; iNdEx >= 0; iNdEx-- { + { + size, err := m.Orders[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintQuery(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x12 + } + } + if m.ClobMatch != nil { + { + size, err := m.ClobMatch.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintQuery(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func (m *StreamTakerOrder) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *StreamTakerOrder) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *StreamTakerOrder) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.TakerOrderStatus != nil { + { + size, err := m.TakerOrderStatus.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintQuery(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x1a } if m.TakerOrder != nil { { @@ -2945,6 +3349,12 @@ func (m *StreamOrderbookUpdatesRequest) Size() (n int) { } n += 1 + sovQuery(uint64(l)) + l } + if len(m.SubaccountIds) > 0 { + for _, e := range m.SubaccountIds { + l = e.Size() + n += 1 + l + sovQuery(uint64(l)) + } + } return n } @@ -3017,6 +3427,76 @@ func (m *StreamUpdate_TakerOrder) Size() (n int) { } return n } +func (m *StreamUpdate_SubaccountUpdate) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.SubaccountUpdate != nil { + l = m.SubaccountUpdate.Size() + n += 1 + l + sovQuery(uint64(l)) + } + return n +} +func (m *SubaccountPerpetualPosition) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.PerpetualId != 0 { + n += 1 + sovQuery(uint64(m.PerpetualId)) + } + if m.Quantums != 0 { + n += 1 + sovQuery(uint64(m.Quantums)) + } + return n +} + +func (m *SubaccountAssetPosition) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.AssetId != 0 { + n += 1 + sovQuery(uint64(m.AssetId)) + } + if m.Quantums != 0 { + n += 1 + sovQuery(uint64(m.Quantums)) + } + return n +} + +func (m *StreamSubaccountUpdate) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.SubaccountId != nil { + l = m.SubaccountId.Size() + n += 1 + l + sovQuery(uint64(l)) + } + if len(m.UpdatedPerpetualPositions) > 0 { + for _, e := range m.UpdatedPerpetualPositions { + l = e.Size() + n += 1 + l + sovQuery(uint64(l)) + } + } + if len(m.UpdatedAssetPositions) > 0 { + for _, e := range m.UpdatedAssetPositions { + l = e.Size() + n += 1 + l + sovQuery(uint64(l)) + } + } + if m.Snapshot { + n += 2 + } + return n +} + func (m *StreamOrderbookUpdate) Size() (n int) { if m == nil { return 0 @@ -4497,6 +4977,40 @@ func (m *StreamOrderbookUpdatesRequest) Unmarshal(dAtA []byte) error { } else { return fmt.Errorf("proto: wrong wireType = %d for field ClobPairId", wireType) } + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field SubaccountIds", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowQuery + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthQuery + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthQuery + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.SubaccountIds = append(m.SubaccountIds, &types.SubaccountId{}) + if err := m.SubaccountIds[len(m.SubaccountIds)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipQuery(dAtA[iNdEx:]) @@ -4737,6 +5251,41 @@ func (m *StreamUpdate) Unmarshal(dAtA []byte) error { m.UpdateMessage = &StreamUpdate_TakerOrder{v} iNdEx = postIndex case 4: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field SubaccountUpdate", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowQuery + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthQuery + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthQuery + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + v := &StreamSubaccountUpdate{} + if err := v.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + m.UpdateMessage = &StreamUpdate_SubaccountUpdate{v} + iNdEx = postIndex + case 5: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field BlockHeight", wireType) } @@ -4755,7 +5304,7 @@ func (m *StreamUpdate) Unmarshal(dAtA []byte) error { break } } - case 5: + case 6: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field ExecMode", wireType) } @@ -4795,6 +5344,356 @@ func (m *StreamUpdate) Unmarshal(dAtA []byte) error { } return nil } +func (m *SubaccountPerpetualPosition) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowQuery + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: SubaccountPerpetualPosition: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: SubaccountPerpetualPosition: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field PerpetualId", wireType) + } + m.PerpetualId = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowQuery + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.PerpetualId |= uint32(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Quantums", wireType) + } + m.Quantums = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowQuery + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Quantums |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := skipQuery(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthQuery + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *SubaccountAssetPosition) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowQuery + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: SubaccountAssetPosition: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: SubaccountAssetPosition: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field AssetId", wireType) + } + m.AssetId = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowQuery + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.AssetId |= uint32(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Quantums", wireType) + } + m.Quantums = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowQuery + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Quantums |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := skipQuery(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthQuery + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *StreamSubaccountUpdate) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowQuery + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: StreamSubaccountUpdate: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: StreamSubaccountUpdate: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field SubaccountId", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowQuery + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthQuery + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthQuery + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.SubaccountId == nil { + m.SubaccountId = &types.SubaccountId{} + } + if err := m.SubaccountId.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field UpdatedPerpetualPositions", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowQuery + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthQuery + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthQuery + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.UpdatedPerpetualPositions = append(m.UpdatedPerpetualPositions, &SubaccountPerpetualPosition{}) + if err := m.UpdatedPerpetualPositions[len(m.UpdatedPerpetualPositions)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field UpdatedAssetPositions", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowQuery + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthQuery + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthQuery + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.UpdatedAssetPositions = append(m.UpdatedAssetPositions, &SubaccountAssetPosition{}) + if err := m.UpdatedAssetPositions[len(m.UpdatedAssetPositions)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 4: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Snapshot", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowQuery + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.Snapshot = bool(v != 0) + default: + iNdEx = preIndex + skippy, err := skipQuery(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthQuery + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func (m *StreamOrderbookUpdate) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 @@ -4853,7 +5752,7 @@ func (m *StreamOrderbookUpdate) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.Updates = append(m.Updates, types.OffChainUpdateV1{}) + m.Updates = append(m.Updates, types1.OffChainUpdateV1{}) if err := m.Updates[len(m.Updates)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } From 3b06a02dde5e8dc137cddf860926f36139fecc0c Mon Sep 17 00:00:00 2001 From: dydxwill <119354122+dydxwill@users.noreply.github.com> Date: Wed, 7 Aug 2024 10:37:05 -0400 Subject: [PATCH 06/33] Restructure FNS global cache to be list (#2036) --- .../streaming/full_node_streaming_manager.go | 129 ++++++++++++------ 1 file changed, 85 insertions(+), 44 deletions(-) diff --git a/protocol/streaming/full_node_streaming_manager.go b/protocol/streaming/full_node_streaming_manager.go index 1d035ed6b3..4c526a200b 100644 --- a/protocol/streaming/full_node_streaming_manager.go +++ b/protocol/streaming/full_node_streaming_manager.go @@ -29,9 +29,15 @@ type FullNodeStreamingManagerImpl struct { ticker *time.Ticker done chan bool - // map of clob pair id to stream updates. - streamUpdateCache map[uint32][]clobtypes.StreamUpdate - numUpdatesInCache uint32 + // TODO: Consolidate the streamUpdateCache and streamUpdateSubscriptionCache into a single + // struct to avoid the need to maintain two separate slices for the same data. + + // list of stream updates. + streamUpdateCache []clobtypes.StreamUpdate + // list of subscription ids for each stream update. + streamUpdateSubscriptionCache [][]uint32 + // map from clob pair id to subscription ids. + clobPairIdToSubscriptionIdMapping map[uint32][]uint32 maxUpdatesInCache uint32 maxSubscriptionChannelSize uint32 @@ -66,10 +72,11 @@ func NewFullNodeStreamingManager( orderbookSubscriptions: make(map[uint32]*OrderbookSubscription), nextSubscriptionId: 0, - ticker: time.NewTicker(time.Duration(flushIntervalMs) * time.Millisecond), - done: make(chan bool), - streamUpdateCache: make(map[uint32][]clobtypes.StreamUpdate), - numUpdatesInCache: 0, + ticker: time.NewTicker(time.Duration(flushIntervalMs) * time.Millisecond), + done: make(chan bool), + streamUpdateCache: make([]clobtypes.StreamUpdate, 0), + streamUpdateSubscriptionCache: make([][]uint32, 0), + clobPairIdToSubscriptionIdMapping: make(map[uint32][]uint32), maxUpdatesInCache: maxUpdatesInCache, maxSubscriptionChannelSize: maxSubscriptionChannelSize, @@ -101,7 +108,7 @@ func (sm *FullNodeStreamingManagerImpl) Enabled() bool { func (sm *FullNodeStreamingManagerImpl) EmitMetrics() { metrics.SetGauge( metrics.GrpcStreamNumUpdatesBuffered, - float32(sm.numUpdatesInCache), + float32(len(sm.streamUpdateCache)), ) metrics.SetGauge( metrics.GrpcStreamSubscriberCount, @@ -134,6 +141,17 @@ func (sm *FullNodeStreamingManagerImpl) Subscribe( messageSender: messageSender, updatesChannel: make(chan []clobtypes.StreamUpdate, sm.maxSubscriptionChannelSize), } + for _, clobPairId := range clobPairIds { + // if clobPairId exists in the map, append the subscription id to the slice + // otherwise, create a new slice with the subscription id + if _, ok := sm.clobPairIdToSubscriptionIdMapping[clobPairId]; !ok { + sm.clobPairIdToSubscriptionIdMapping[clobPairId] = []uint32{} + } + sm.clobPairIdToSubscriptionIdMapping[clobPairId] = append( + sm.clobPairIdToSubscriptionIdMapping[clobPairId], + sm.nextSubscriptionId, + ) + } sm.logger.Info( fmt.Sprintf( @@ -194,6 +212,22 @@ func (sm *FullNodeStreamingManagerImpl) removeSubscription( } close(subscription.updatesChannel) delete(sm.orderbookSubscriptions, subscriptionIdToRemove) + + // Iterate over the clobPairIdToSubscriptionIdMapping to remove the subscriptionIdToRemove + for pairId, subscriptionIds := range sm.clobPairIdToSubscriptionIdMapping { + for i, id := range subscriptionIds { + if id == subscriptionIdToRemove { + // Remove the subscription ID from the slice + sm.clobPairIdToSubscriptionIdMapping[pairId] = append(subscriptionIds[:i], subscriptionIds[i+1:]...) + break + } + } + // If the list is empty after removal, delete the key from the map + if len(sm.clobPairIdToSubscriptionIdMapping[pairId]) == 0 { + delete(sm.clobPairIdToSubscriptionIdMapping, pairId) + } + } + sm.logger.Info( fmt.Sprintf("Removed streaming subscription id %+v", subscriptionIdToRemove), ) @@ -295,27 +329,28 @@ func (sm *FullNodeStreamingManagerImpl) SendOrderbookUpdates( } // Unmarshal each per-clob pair message to v1 updates. - updatesByClobPairId := make(map[uint32][]clobtypes.StreamUpdate) + streamUpdates := make([]clobtypes.StreamUpdate, 0) + clobPairIds := make([]uint32, 0) for clobPairId, update := range updates { v1updates, err := streaming_util.GetOffchainUpdatesV1(update) if err != nil { panic(err) } - updatesByClobPairId[clobPairId] = []clobtypes.StreamUpdate{ - { - UpdateMessage: &clobtypes.StreamUpdate_OrderbookUpdate{ - OrderbookUpdate: &clobtypes.StreamOrderbookUpdate{ - Updates: v1updates, - Snapshot: false, - }, + streamUpdate := clobtypes.StreamUpdate{ + UpdateMessage: &clobtypes.StreamUpdate_OrderbookUpdate{ + OrderbookUpdate: &clobtypes.StreamOrderbookUpdate{ + Updates: v1updates, + Snapshot: false, }, - BlockHeight: blockHeight, - ExecMode: uint32(execMode), }, + BlockHeight: blockHeight, + ExecMode: uint32(execMode), } + streamUpdates = append(streamUpdates, streamUpdate) + clobPairIds = append(clobPairIds, clobPairId) } - sm.AddUpdatesToCache(updatesByClobPairId, uint32(len(updates))) + sm.AddUpdatesToCache(streamUpdates, clobPairIds, uint32(len(updates))) } // SendOrderbookFillUpdates groups fills by their clob pair ids and @@ -333,7 +368,8 @@ func (sm *FullNodeStreamingManagerImpl) SendOrderbookFillUpdates( ) // Group fills by clob pair id. - updatesByClobPairId := make(map[uint32][]clobtypes.StreamUpdate) + streamUpdates := make([]clobtypes.StreamUpdate, 0) + clobPairIds := make([]uint32, 0) for _, orderbookFill := range orderbookFills { // If this is a deleveraging fill, fetch the clob pair id from the deleveraged // perpetual id. @@ -346,9 +382,6 @@ func (sm *FullNodeStreamingManagerImpl) SendOrderbookFillUpdates( } else { clobPairId = orderbookFill.Orders[0].OrderId.ClobPairId } - if _, ok := updatesByClobPairId[clobPairId]; !ok { - updatesByClobPairId[clobPairId] = []clobtypes.StreamUpdate{} - } streamUpdate := clobtypes.StreamUpdate{ UpdateMessage: &clobtypes.StreamUpdate_OrderFill{ OrderFill: &orderbookFill, @@ -356,14 +389,16 @@ func (sm *FullNodeStreamingManagerImpl) SendOrderbookFillUpdates( BlockHeight: blockHeight, ExecMode: uint32(execMode), } - updatesByClobPairId[clobPairId] = append(updatesByClobPairId[clobPairId], streamUpdate) + streamUpdates = append(streamUpdates, streamUpdate) + clobPairIds = append(clobPairIds, clobPairId) } - sm.AddUpdatesToCache(updatesByClobPairId, uint32(len(orderbookFills))) + sm.AddUpdatesToCache(streamUpdates, clobPairIds, uint32(len(orderbookFills))) } func (sm *FullNodeStreamingManagerImpl) AddUpdatesToCache( - updatesByClobPairId map[uint32][]clobtypes.StreamUpdate, + updates []clobtypes.StreamUpdate, + clobPairIds []uint32, numUpdatesToAdd uint32, ) { sm.Lock() @@ -374,20 +409,22 @@ func (sm *FullNodeStreamingManagerImpl) AddUpdatesToCache( float32(numUpdatesToAdd), ) - for clobPairId, streamUpdates := range updatesByClobPairId { - sm.streamUpdateCache[clobPairId] = append(sm.streamUpdateCache[clobPairId], streamUpdates...) + sm.streamUpdateCache = append(sm.streamUpdateCache, updates...) + for _, clobPairId := range clobPairIds { + sm.streamUpdateSubscriptionCache = append( + sm.streamUpdateSubscriptionCache, + sm.clobPairIdToSubscriptionIdMapping[clobPairId], + ) } - sm.numUpdatesInCache += numUpdatesToAdd // Remove all subscriptions and wipe the buffer if buffer overflows. - if sm.numUpdatesInCache > sm.maxUpdatesInCache { + if len(sm.streamUpdateCache) > int(sm.maxUpdatesInCache) { sm.logger.Error("Streaming buffer full capacity. Dropping messages and all subscriptions. " + "Disconnect all clients and increase buffer size via the grpc-stream-buffer-size flag.") for id := range sm.orderbookSubscriptions { sm.removeSubscription(id) } clear(sm.streamUpdateCache) - sm.numUpdatesInCache = 0 } sm.EmitMetrics() } @@ -398,8 +435,8 @@ func (sm *FullNodeStreamingManagerImpl) FlushStreamUpdates() { sm.FlushStreamUpdatesWithLock() } -// FlushStreamUpdatesWithLock takes in a map of clob pair id to stream updates and emits them to subscribers. -// Note this method requires the lock and assumes that the lock has already been +// FlushStreamUpdatesWithLock takes in a list of stream updates and their corresponding subscription IDs, +// and emits them to subscribers. Note this method requires the lock and assumes that the lock has already been // acquired by the caller. func (sm *FullNodeStreamingManagerImpl) FlushStreamUpdatesWithLock() { defer metrics.ModuleMeasureSince( @@ -408,24 +445,28 @@ func (sm *FullNodeStreamingManagerImpl) FlushStreamUpdatesWithLock() { time.Now(), ) - // Non-blocking send updates through subscriber's buffered channel. - // If the buffer is full, drop the subscription. + // Map to collect updates for each subscription. + subscriptionUpdates := make(map[uint32][]clobtypes.StreamUpdate) idsToRemove := make([]uint32, 0) - for id, subscription := range sm.orderbookSubscriptions { - streamUpdatesForSubscription := make([]clobtypes.StreamUpdate, 0) - for _, clobPairId := range subscription.clobPairIds { - if update, ok := sm.streamUpdateCache[clobPairId]; ok { - streamUpdatesForSubscription = append(streamUpdatesForSubscription, update...) - } + + // Collect updates for each subscription. + for i, update := range sm.streamUpdateCache { + subscriptionIds := sm.streamUpdateSubscriptionCache[i] + for _, id := range subscriptionIds { + subscriptionUpdates[id] = append(subscriptionUpdates[id], update) } + } - if len(streamUpdatesForSubscription) > 0 { + // Non-blocking send updates through subscriber's buffered channel. + // If the buffer is full, drop the subscription. + for id, updates := range subscriptionUpdates { + if subscription, ok := sm.orderbookSubscriptions[id]; ok { metrics.IncrCounter( metrics.GrpcAddToSubscriptionChannelCount, 1, ) select { - case subscription.updatesChannel <- streamUpdatesForSubscription: + case subscription.updatesChannel <- updates: default: idsToRemove = append(idsToRemove, id) } @@ -433,7 +474,7 @@ func (sm *FullNodeStreamingManagerImpl) FlushStreamUpdatesWithLock() { } clear(sm.streamUpdateCache) - sm.numUpdatesInCache = 0 + clear(sm.streamUpdateSubscriptionCache) for _, id := range idsToRemove { sm.logger.Error( From 3ec47bc03168585011633c62eefa35bfc540f4dd Mon Sep 17 00:00:00 2001 From: dydxwill <119354122+dydxwill@users.noreply.github.com> Date: Wed, 7 Aug 2024 14:48:17 -0400 Subject: [PATCH 07/33] Bump grpc stream flag default values (#2051) --- protocol/app/flags/flags.go | 4 ++-- protocol/app/flags/flags_test.go | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/protocol/app/flags/flags.go b/protocol/app/flags/flags.go index bdbe34a514..74d72e46a9 100644 --- a/protocol/app/flags/flags.go +++ b/protocol/app/flags/flags.go @@ -64,8 +64,8 @@ const ( DefaultGrpcStreamingEnabled = false DefaultGrpcStreamingFlushIntervalMs = 50 - DefaultGrpcStreamingMaxBatchSize = 2000 - DefaultGrpcStreamingMaxChannelBufferSize = 2000 + DefaultGrpcStreamingMaxBatchSize = 1000000 + DefaultGrpcStreamingMaxChannelBufferSize = 1000000 DefaultVEOracleEnabled = true DefaultOptimisticExecutionEnabled = false diff --git a/protocol/app/flags/flags_test.go b/protocol/app/flags/flags_test.go index 8260efe313..3aac1ebed7 100644 --- a/protocol/app/flags/flags_test.go +++ b/protocol/app/flags/flags_test.go @@ -80,8 +80,8 @@ func TestValidate(t *testing.T) { GrpcEnable: true, GrpcStreamingEnabled: true, GrpcStreamingFlushIntervalMs: 100, - GrpcStreamingMaxBatchSize: 2000, - GrpcStreamingMaxChannelBufferSize: 2000, + GrpcStreamingMaxBatchSize: 1000000, + GrpcStreamingMaxChannelBufferSize: 1000000, }, }, "success - optimistic execution": { @@ -130,7 +130,7 @@ func TestValidate(t *testing.T) { GrpcEnable: true, GrpcStreamingEnabled: true, GrpcStreamingFlushIntervalMs: 0, - GrpcStreamingMaxBatchSize: 2000, + GrpcStreamingMaxBatchSize: 1000000, }, expectedErr: fmt.Errorf("grpc streaming flush interval must be positive number"), }, @@ -140,7 +140,7 @@ func TestValidate(t *testing.T) { GrpcEnable: true, GrpcStreamingEnabled: true, GrpcStreamingFlushIntervalMs: 100, - GrpcStreamingMaxBatchSize: 2000, + GrpcStreamingMaxBatchSize: 1000000, GrpcStreamingMaxChannelBufferSize: 0, }, expectedErr: fmt.Errorf("grpc streaming channel size must be positive number"), @@ -183,8 +183,8 @@ func TestGetFlagValuesFromOptions(t *testing.T) { expectedGrpcEnable: true, expectedGrpcStreamingEnable: false, expectedGrpcStreamingFlushMs: 50, - expectedGrpcStreamingBatchSize: 2000, - expectedGrpcStreamingMaxChannelBufferSize: 2000, + expectedGrpcStreamingBatchSize: 1000000, + expectedGrpcStreamingMaxChannelBufferSize: 1000000, expectedOptimisticExecutionEnabled: false, }, "Sets values from options": { From acecca38524dc7fa15cfb468f8af1ed0fd550b15 Mon Sep 17 00:00:00 2001 From: Jonathan Fung <121899091+jonfung-dydx@users.noreply.github.com> Date: Thu, 8 Aug 2024 15:21:47 -0400 Subject: [PATCH 08/33] [Full node streaming] emit taker order status at end of matching loop (#2022) --- protocol/mocks/MemClobKeeper.go | 5 ++++ .../streaming/full_node_streaming_manager.go | 28 +++++++++++++++++++ protocol/streaming/noop_streaming_manager.go | 7 +++++ protocol/streaming/types/interface.go | 5 ++++ protocol/testutil/memclob/keeper.go | 6 ++++ protocol/x/clob/keeper/keeper.go | 16 +++++++++-- protocol/x/clob/memclob/memclob.go | 12 ++++++++ .../x/clob/memclob/memclob_grpc_streaming.go | 25 +++++++++++++++++ protocol/x/clob/types/liquidation_order.go | 6 ++++ protocol/x/clob/types/mem_clob_keeper.go | 4 +++ protocol/x/clob/types/order.go | 6 ++++ protocol/x/clob/types/orderbook.go | 3 ++ 12 files changed, 121 insertions(+), 2 deletions(-) diff --git a/protocol/mocks/MemClobKeeper.go b/protocol/mocks/MemClobKeeper.go index 0b8e5d2504..8eacc26019 100644 --- a/protocol/mocks/MemClobKeeper.go +++ b/protocol/mocks/MemClobKeeper.go @@ -416,6 +416,11 @@ func (_m *MemClobKeeper) SendOrderbookUpdates(ctx types.Context, offchainUpdates _m.Called(ctx, offchainUpdates) } +// SendTakerOrderStatus provides a mock function with given fields: ctx, takerOrder +func (_m *MemClobKeeper) SendTakerOrderStatus(ctx types.Context, takerOrder clobtypes.StreamTakerOrder) { + _m.Called(ctx, takerOrder) +} + // SetLongTermOrderPlacement provides a mock function with given fields: ctx, order, blockHeight func (_m *MemClobKeeper) SetLongTermOrderPlacement(ctx types.Context, order clobtypes.Order, blockHeight uint32) { _m.Called(ctx, order, blockHeight) diff --git a/protocol/streaming/full_node_streaming_manager.go b/protocol/streaming/full_node_streaming_manager.go index 4c526a200b..b2f4d24aba 100644 --- a/protocol/streaming/full_node_streaming_manager.go +++ b/protocol/streaming/full_node_streaming_manager.go @@ -396,6 +396,34 @@ func (sm *FullNodeStreamingManagerImpl) SendOrderbookFillUpdates( sm.AddUpdatesToCache(streamUpdates, clobPairIds, uint32(len(orderbookFills))) } +// SendTakerOrderStatus sends out a taker order and its status to the full node streaming service. +func (sm *FullNodeStreamingManagerImpl) SendTakerOrderStatus( + streamTakerOrder clobtypes.StreamTakerOrder, + blockHeight uint32, + execMode sdk.ExecMode, +) { + clobPairId := uint32(0) + if liqOrder := streamTakerOrder.GetLiquidationOrder(); liqOrder != nil { + clobPairId = liqOrder.ClobPairId + } + if takerOrder := streamTakerOrder.GetOrder(); takerOrder != nil { + clobPairId = takerOrder.OrderId.ClobPairId + } + + sm.AddUpdatesToCache( + map[uint32][]clobtypes.StreamUpdate{ + clobPairId: { + { + UpdateMessage: &clobtypes.StreamUpdate_TakerOrder{ + TakerOrder: &streamTakerOrder, + }, + }, + }, + }, + 1, + ) +} + func (sm *FullNodeStreamingManagerImpl) AddUpdatesToCache( updates []clobtypes.StreamUpdate, clobPairIds []uint32, diff --git a/protocol/streaming/noop_streaming_manager.go b/protocol/streaming/noop_streaming_manager.go index 2334142223..749bcf2b67 100644 --- a/protocol/streaming/noop_streaming_manager.go +++ b/protocol/streaming/noop_streaming_manager.go @@ -42,6 +42,13 @@ func (sm *NoopGrpcStreamingManager) SendOrderbookFillUpdates( ) { } +func (sm *NoopGrpcStreamingManager) SendTakerOrderStatus( + takerOrder clobtypes.StreamTakerOrder, + blockHeight uint32, + execMode sdk.ExecMode, +) { +} + func (sm *NoopGrpcStreamingManager) InitializeNewStreams( getOrderbookSnapshot func(clobPairId clobtypes.ClobPairId) *clobtypes.OffchainUpdates, blockHeight uint32, diff --git a/protocol/streaming/types/interface.go b/protocol/streaming/types/interface.go index d357dddc39..66fcf808e5 100644 --- a/protocol/streaming/types/interface.go +++ b/protocol/streaming/types/interface.go @@ -34,6 +34,11 @@ type FullNodeStreamingManager interface { execMode sdk.ExecMode, perpetualIdToClobPairId map[uint32][]clobtypes.ClobPairId, ) + SendTakerOrderStatus( + takerOrder clobtypes.StreamTakerOrder, + blockHeight uint32, + execMode sdk.ExecMode, + ) } type OutgoingMessageSender interface { diff --git a/protocol/testutil/memclob/keeper.go b/protocol/testutil/memclob/keeper.go index ea515ff62d..ce15a95e29 100644 --- a/protocol/testutil/memclob/keeper.go +++ b/protocol/testutil/memclob/keeper.go @@ -512,6 +512,12 @@ func (f *FakeMemClobKeeper) SendOrderbookFillUpdates( ) { } +func (f *FakeMemClobKeeper) SendTakerOrderStatus( + ctx sdk.Context, + takerOrder types.StreamTakerOrder, +) { +} + // Placeholder to satisfy interface implementation of types.MemClobKeeper func (f *FakeMemClobKeeper) AddOrderToOrderbookSubaccountUpdatesCheck( ctx sdk.Context, diff --git a/protocol/x/clob/keeper/keeper.go b/protocol/x/clob/keeper/keeper.go index 92921b9afb..0fbf4535b4 100644 --- a/protocol/x/clob/keeper/keeper.go +++ b/protocol/x/clob/keeper/keeper.go @@ -267,7 +267,7 @@ func (k Keeper) InitializeNewStreams(ctx sdk.Context) { ) } -// SendOrderbookUpdates sends the offchain updates to the gRPC streaming manager. +// SendOrderbookUpdates sends the offchain updates to the Full Node streaming manager. func (k Keeper) SendOrderbookUpdates( ctx sdk.Context, offchainUpdates *types.OffchainUpdates, @@ -283,7 +283,7 @@ func (k Keeper) SendOrderbookUpdates( ) } -// SendOrderbookFillUpdates sends the orderbook fills to the gRPC streaming manager. +// SendOrderbookFillUpdates sends the orderbook fills to the Full Node streaming manager. func (k Keeper) SendOrderbookFillUpdates( ctx sdk.Context, orderbookFills []types.StreamOrderbookFill, @@ -298,3 +298,15 @@ func (k Keeper) SendOrderbookFillUpdates( k.PerpetualIdToClobPairId, ) } + +// SendTakerOrderStatus sends the taker order with its status to the Full Node streaming manager. +func (k Keeper) SendTakerOrderStatus( + ctx sdk.Context, + takerOrder types.StreamTakerOrder, +) { + k.GetFullNodeStreamingManager().SendTakerOrderStatus( + takerOrder, + lib.MustConvertIntegerToUint32(ctx.BlockHeight()), + ctx.ExecMode(), + ) +} diff --git a/protocol/x/clob/memclob/memclob.go b/protocol/x/clob/memclob/memclob.go index 6c9779571f..8054a41e78 100644 --- a/protocol/x/clob/memclob/memclob.go +++ b/protocol/x/clob/memclob/memclob.go @@ -767,6 +767,18 @@ func (m *MemClobPriceTimePriority) matchOrder( order, ) + // If full node streaming is on, emit the taker order and its resulting status. + if m.generateOrderbookUpdates { + streamTakerOrder := m.GenerateStreamTakerOrder( + order, + takerOrderStatus, + ) + m.clobKeeper.SendTakerOrderStatus( + ctx, + streamTakerOrder, + ) + } + // If this is a replacement order, then ensure we remove the existing order from the orderbook. if !order.IsLiquidation() { orderId := order.MustGetOrder().OrderId diff --git a/protocol/x/clob/memclob/memclob_grpc_streaming.go b/protocol/x/clob/memclob/memclob_grpc_streaming.go index d33988b078..ba2f48b236 100644 --- a/protocol/x/clob/memclob/memclob_grpc_streaming.go +++ b/protocol/x/clob/memclob/memclob_grpc_streaming.go @@ -157,3 +157,28 @@ func (m *MemClobPriceTimePriority) GetOrderbookUpdatesForOrderUpdate( } return offchainUpdates } + +// GenerateStreamTakerOrder returns a `StreamTakerOrder` object used in full node +// streaming from a matchableOrder and a taker order status. +func (m *MemClobPriceTimePriority) GenerateStreamTakerOrder( + takerOrder types.MatchableOrder, + takerOrderStatus types.TakerOrderStatus, +) types.StreamTakerOrder { + if takerOrder.IsLiquidation() { + liquidationOrder := takerOrder.MustGetLiquidationOrder() + streamLiquidationOrder := liquidationOrder.ToStreamLiquidationOrder() + return types.StreamTakerOrder{ + TakerOrder: &types.StreamTakerOrder_LiquidationOrder{ + LiquidationOrder: streamLiquidationOrder, + }, + TakerOrderStatus: takerOrderStatus.ToStreamingTakerOrderStatus(), + } + } + order := takerOrder.MustGetOrder() + return types.StreamTakerOrder{ + TakerOrder: &types.StreamTakerOrder_Order{ + Order: &order, + }, + TakerOrderStatus: takerOrderStatus.ToStreamingTakerOrderStatus(), + } +} diff --git a/protocol/x/clob/types/liquidation_order.go b/protocol/x/clob/types/liquidation_order.go index 20aa56fe3d..5804d0f9f0 100644 --- a/protocol/x/clob/types/liquidation_order.go +++ b/protocol/x/clob/types/liquidation_order.go @@ -115,6 +115,12 @@ func (lo *LiquidationOrder) MustGetOrder() Order { panic("MustGetOrder: No underlying order on a LiquidationOrder type.") } +// MustGetLiquidationOrder returns the underlying `LiquidationOrder` type. +// This function is necessary for the `LiquidationOrder` type to implement the `MatchableOrder` interface. +func (lo *LiquidationOrder) MustGetLiquidationOrder() LiquidationOrder { + return *lo +} + // MustGetLiquidatedPerpetualId returns the perpetual ID that this perpetual order is liquidating. // This function is necessary for the `LiquidationOrder` type to implement the `MatchableOrder` interface. func (lo *LiquidationOrder) MustGetLiquidatedPerpetualId() uint32 { diff --git a/protocol/x/clob/types/mem_clob_keeper.go b/protocol/x/clob/types/mem_clob_keeper.go index c198c739cf..18898e79e2 100644 --- a/protocol/x/clob/types/mem_clob_keeper.go +++ b/protocol/x/clob/types/mem_clob_keeper.go @@ -104,6 +104,10 @@ type MemClobKeeper interface { ctx sdk.Context, orderbookFills []StreamOrderbookFill, ) + SendTakerOrderStatus( + ctx sdk.Context, + takerOrder StreamTakerOrder, + ) AddOrderToOrderbookSubaccountUpdatesCheck( ctx sdk.Context, subaccountId satypes.SubaccountId, diff --git a/protocol/x/clob/types/order.go b/protocol/x/clob/types/order.go index 9a4e559391..deac6716e1 100644 --- a/protocol/x/clob/types/order.go +++ b/protocol/x/clob/types/order.go @@ -132,6 +132,12 @@ func (o *Order) MustGetOrder() Order { return *o } +// MustGetLiquidationOrder always panics since Order is not a Liquidation Order. +// This function is necessary for the `Order` type to implement the `MatchableOrder` interface. +func (o *Order) MustGetLiquidationOrder() LiquidationOrder { + panic("MustGetLiquidationOrder: Order is not a liquidation order") +} + // MustGetLiquidatedPerpetualId always panics since there is no underlying perpetual ID for a `Order`. // This function is necessary for the `Order` type to implement the `MatchableOrder` interface. func (o *Order) MustGetLiquidatedPerpetualId() uint32 { diff --git a/protocol/x/clob/types/orderbook.go b/protocol/x/clob/types/orderbook.go index ef4cf65e93..ba6d1f31bc 100644 --- a/protocol/x/clob/types/orderbook.go +++ b/protocol/x/clob/types/orderbook.go @@ -206,6 +206,9 @@ type MatchableOrder interface { // MustGetOrder returns the underlying order if this is not a liquidation order. Panics if called // for a liquidation order. MustGetOrder() Order + // MustGetLiquidationOrder returns the underlying liquidation order if this is not a regular order. + // Panics if called for a regular order. + MustGetLiquidationOrder() LiquidationOrder // MustGetLiquidatedPerpetualId returns the perpetual ID if this is a liquidation order. Panics // if called for a non-liquidation order. MustGetLiquidatedPerpetualId() uint32 From 84746efdd10931e7a001806077d74b0785ddf7ef Mon Sep 17 00:00:00 2001 From: Jonathan Fung <121899091+jonfung-dydx@users.noreply.github.com> Date: Thu, 8 Aug 2024 16:00:38 -0400 Subject: [PATCH 09/33] fix merge conflict and metric emissions (#2065) --- .../streaming/full_node_streaming_manager.go | 23 ++++++++++--------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/protocol/streaming/full_node_streaming_manager.go b/protocol/streaming/full_node_streaming_manager.go index b2f4d24aba..473b611b9c 100644 --- a/protocol/streaming/full_node_streaming_manager.go +++ b/protocol/streaming/full_node_streaming_manager.go @@ -350,7 +350,7 @@ func (sm *FullNodeStreamingManagerImpl) SendOrderbookUpdates( clobPairIds = append(clobPairIds, clobPairId) } - sm.AddUpdatesToCache(streamUpdates, clobPairIds, uint32(len(updates))) + sm.AddUpdatesToCache(streamUpdates, clobPairIds) } // SendOrderbookFillUpdates groups fills by their clob pair ids and @@ -393,7 +393,7 @@ func (sm *FullNodeStreamingManagerImpl) SendOrderbookFillUpdates( clobPairIds = append(clobPairIds, clobPairId) } - sm.AddUpdatesToCache(streamUpdates, clobPairIds, uint32(len(orderbookFills))) + sm.AddUpdatesToCache(streamUpdates, clobPairIds) } // SendTakerOrderStatus sends out a taker order and its status to the full node streaming service. @@ -411,30 +411,31 @@ func (sm *FullNodeStreamingManagerImpl) SendTakerOrderStatus( } sm.AddUpdatesToCache( - map[uint32][]clobtypes.StreamUpdate{ - clobPairId: { - { - UpdateMessage: &clobtypes.StreamUpdate_TakerOrder{ - TakerOrder: &streamTakerOrder, - }, + []clobtypes.StreamUpdate{ + { + UpdateMessage: &clobtypes.StreamUpdate_TakerOrder{ + TakerOrder: &streamTakerOrder, }, + BlockHeight: blockHeight, + ExecMode: uint32(execMode), }, }, - 1, + []uint32{clobPairId}, ) } +// AddUpdatesToCache adds a series of updates to the full node streaming cache. +// Clob pair ids are the clob pair id each update is relevant to. func (sm *FullNodeStreamingManagerImpl) AddUpdatesToCache( updates []clobtypes.StreamUpdate, clobPairIds []uint32, - numUpdatesToAdd uint32, ) { sm.Lock() defer sm.Unlock() metrics.IncrCounter( metrics.GrpcAddUpdateToBufferCount, - float32(numUpdatesToAdd), + float32(len(updates)), ) sm.streamUpdateCache = append(sm.streamUpdateCache, updates...) From c643a298943d200d573cf93b21f8985e5252a11f Mon Sep 17 00:00:00 2001 From: Jonathan Fung <121899091+jonfung-dydx@users.noreply.github.com> Date: Fri, 9 Aug 2024 09:10:35 -0700 Subject: [PATCH 10/33] Full node streaming batch size reset to 2000, properly zero out cache on flush (#2068) --- protocol/app/flags/flags.go | 4 ++-- protocol/app/flags/flags_test.go | 12 ++++++------ protocol/streaming/full_node_streaming_manager.go | 7 ++++--- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/protocol/app/flags/flags.go b/protocol/app/flags/flags.go index 74d72e46a9..bdbe34a514 100644 --- a/protocol/app/flags/flags.go +++ b/protocol/app/flags/flags.go @@ -64,8 +64,8 @@ const ( DefaultGrpcStreamingEnabled = false DefaultGrpcStreamingFlushIntervalMs = 50 - DefaultGrpcStreamingMaxBatchSize = 1000000 - DefaultGrpcStreamingMaxChannelBufferSize = 1000000 + DefaultGrpcStreamingMaxBatchSize = 2000 + DefaultGrpcStreamingMaxChannelBufferSize = 2000 DefaultVEOracleEnabled = true DefaultOptimisticExecutionEnabled = false diff --git a/protocol/app/flags/flags_test.go b/protocol/app/flags/flags_test.go index 3aac1ebed7..8260efe313 100644 --- a/protocol/app/flags/flags_test.go +++ b/protocol/app/flags/flags_test.go @@ -80,8 +80,8 @@ func TestValidate(t *testing.T) { GrpcEnable: true, GrpcStreamingEnabled: true, GrpcStreamingFlushIntervalMs: 100, - GrpcStreamingMaxBatchSize: 1000000, - GrpcStreamingMaxChannelBufferSize: 1000000, + GrpcStreamingMaxBatchSize: 2000, + GrpcStreamingMaxChannelBufferSize: 2000, }, }, "success - optimistic execution": { @@ -130,7 +130,7 @@ func TestValidate(t *testing.T) { GrpcEnable: true, GrpcStreamingEnabled: true, GrpcStreamingFlushIntervalMs: 0, - GrpcStreamingMaxBatchSize: 1000000, + GrpcStreamingMaxBatchSize: 2000, }, expectedErr: fmt.Errorf("grpc streaming flush interval must be positive number"), }, @@ -140,7 +140,7 @@ func TestValidate(t *testing.T) { GrpcEnable: true, GrpcStreamingEnabled: true, GrpcStreamingFlushIntervalMs: 100, - GrpcStreamingMaxBatchSize: 1000000, + GrpcStreamingMaxBatchSize: 2000, GrpcStreamingMaxChannelBufferSize: 0, }, expectedErr: fmt.Errorf("grpc streaming channel size must be positive number"), @@ -183,8 +183,8 @@ func TestGetFlagValuesFromOptions(t *testing.T) { expectedGrpcEnable: true, expectedGrpcStreamingEnable: false, expectedGrpcStreamingFlushMs: 50, - expectedGrpcStreamingBatchSize: 1000000, - expectedGrpcStreamingMaxChannelBufferSize: 1000000, + expectedGrpcStreamingBatchSize: 2000, + expectedGrpcStreamingMaxChannelBufferSize: 2000, expectedOptimisticExecutionEnabled: false, }, "Sets values from options": { diff --git a/protocol/streaming/full_node_streaming_manager.go b/protocol/streaming/full_node_streaming_manager.go index 473b611b9c..fb10dfb3ec 100644 --- a/protocol/streaming/full_node_streaming_manager.go +++ b/protocol/streaming/full_node_streaming_manager.go @@ -453,7 +453,8 @@ func (sm *FullNodeStreamingManagerImpl) AddUpdatesToCache( for id := range sm.orderbookSubscriptions { sm.removeSubscription(id) } - clear(sm.streamUpdateCache) + sm.streamUpdateCache = nil + sm.streamUpdateSubscriptionCache = nil } sm.EmitMetrics() } @@ -502,8 +503,8 @@ func (sm *FullNodeStreamingManagerImpl) FlushStreamUpdatesWithLock() { } } - clear(sm.streamUpdateCache) - clear(sm.streamUpdateSubscriptionCache) + sm.streamUpdateCache = nil + sm.streamUpdateSubscriptionCache = nil for _, id := range idsToRemove { sm.logger.Error( From 0805c931b80fe5f9e62bc319c0053dd25b12e968 Mon Sep 17 00:00:00 2001 From: dydxwill <119354122+dydxwill@users.noreply.github.com> Date: Mon, 12 Aug 2024 14:26:06 -0400 Subject: [PATCH 11/33] FNS subaccount implementation (#2059) --- .../src/codegen/dydxprotocol/bundle.ts | 248 ++-- .../src/codegen/dydxprotocol/clob/query.ts | 284 +---- .../dydxprotocol/subaccounts/streaming.ts | 286 +++++ .../v4-protos/src/codegen/gogoproto/bundle.ts | 4 +- .../v4-protos/src/codegen/google/bundle.ts | 22 +- proto/dydxprotocol/clob/query.proto | 39 +- .../dydxprotocol/subaccounts/streaming.proto | 42 + protocol/app/app.go | 1 + protocol/lib/metrics/metric_keys.go | 2 + .../streaming/full_node_streaming_manager.go | 274 ++++- protocol/streaming/noop_streaming_manager.go | 14 + protocol/streaming/types/interface.go | 9 + protocol/testutil/keeper/subaccounts.go | 2 + .../x/clob/keeper/grpc_stream_orderbook.go | 1 + protocol/x/clob/keeper/keeper.go | 28 + protocol/x/clob/types/query.pb.go | 1051 +++-------------- protocol/x/subaccounts/keeper/keeper.go | 4 + protocol/x/subaccounts/keeper/subaccount.go | 74 ++ protocol/x/subaccounts/types/streaming.pb.go | 900 ++++++++++++++ protocol/x/subaccounts/types/types.go | 4 + 20 files changed, 1869 insertions(+), 1420 deletions(-) create mode 100644 indexer/packages/v4-protos/src/codegen/dydxprotocol/subaccounts/streaming.ts create mode 100644 proto/dydxprotocol/subaccounts/streaming.proto create mode 100644 protocol/x/subaccounts/types/streaming.pb.go diff --git a/indexer/packages/v4-protos/src/codegen/dydxprotocol/bundle.ts b/indexer/packages/v4-protos/src/codegen/dydxprotocol/bundle.ts index 21e5cb3e3c..fe4846372f 100644 --- a/indexer/packages/v4-protos/src/codegen/dydxprotocol/bundle.ts +++ b/indexer/packages/v4-protos/src/codegen/dydxprotocol/bundle.ts @@ -98,71 +98,72 @@ import * as _101 from "./subaccounts/asset_position"; import * as _102 from "./subaccounts/genesis"; import * as _103 from "./subaccounts/perpetual_position"; import * as _104 from "./subaccounts/query"; -import * as _105 from "./subaccounts/subaccount"; -import * as _106 from "./vault/genesis"; -import * as _107 from "./vault/params"; -import * as _108 from "./vault/query"; -import * as _109 from "./vault/share"; -import * as _110 from "./vault/tx"; -import * as _111 from "./vault/vault"; -import * as _112 from "./vest/genesis"; -import * as _113 from "./vest/query"; -import * as _114 from "./vest/tx"; -import * as _115 from "./vest/vest_entry"; -import * as _123 from "./assets/query.lcd"; -import * as _124 from "./blocktime/query.lcd"; -import * as _125 from "./bridge/query.lcd"; -import * as _126 from "./clob/query.lcd"; -import * as _127 from "./delaymsg/query.lcd"; -import * as _128 from "./epochs/query.lcd"; -import * as _129 from "./feetiers/query.lcd"; -import * as _130 from "./perpetuals/query.lcd"; -import * as _131 from "./prices/query.lcd"; -import * as _132 from "./ratelimit/query.lcd"; -import * as _133 from "./revshare/query.lcd"; -import * as _134 from "./rewards/query.lcd"; -import * as _135 from "./stats/query.lcd"; -import * as _136 from "./subaccounts/query.lcd"; -import * as _137 from "./vault/query.lcd"; -import * as _138 from "./vest/query.lcd"; -import * as _139 from "./assets/query.rpc.Query"; -import * as _140 from "./blocktime/query.rpc.Query"; -import * as _141 from "./bridge/query.rpc.Query"; -import * as _142 from "./clob/query.rpc.Query"; -import * as _143 from "./delaymsg/query.rpc.Query"; -import * as _144 from "./epochs/query.rpc.Query"; -import * as _145 from "./feetiers/query.rpc.Query"; -import * as _146 from "./govplus/query.rpc.Query"; -import * as _147 from "./listing/query.rpc.Query"; -import * as _148 from "./perpetuals/query.rpc.Query"; -import * as _149 from "./prices/query.rpc.Query"; -import * as _150 from "./ratelimit/query.rpc.Query"; -import * as _151 from "./revshare/query.rpc.Query"; -import * as _152 from "./rewards/query.rpc.Query"; -import * as _153 from "./sending/query.rpc.Query"; -import * as _154 from "./stats/query.rpc.Query"; -import * as _155 from "./subaccounts/query.rpc.Query"; -import * as _156 from "./vault/query.rpc.Query"; -import * as _157 from "./vest/query.rpc.Query"; -import * as _158 from "./blocktime/tx.rpc.msg"; -import * as _159 from "./bridge/tx.rpc.msg"; -import * as _160 from "./clob/tx.rpc.msg"; -import * as _161 from "./delaymsg/tx.rpc.msg"; -import * as _162 from "./feetiers/tx.rpc.msg"; -import * as _163 from "./govplus/tx.rpc.msg"; -import * as _164 from "./listing/tx.rpc.msg"; -import * as _165 from "./perpetuals/tx.rpc.msg"; -import * as _166 from "./prices/tx.rpc.msg"; -import * as _167 from "./ratelimit/tx.rpc.msg"; -import * as _168 from "./revshare/tx.rpc.msg"; -import * as _169 from "./rewards/tx.rpc.msg"; -import * as _170 from "./sending/tx.rpc.msg"; -import * as _171 from "./stats/tx.rpc.msg"; -import * as _172 from "./vault/tx.rpc.msg"; -import * as _173 from "./vest/tx.rpc.msg"; -import * as _174 from "./lcd"; -import * as _175 from "./rpc.query"; -import * as _176 from "./rpc.tx"; +import * as _105 from "./subaccounts/streaming"; +import * as _106 from "./subaccounts/subaccount"; +import * as _107 from "./vault/genesis"; +import * as _108 from "./vault/params"; +import * as _109 from "./vault/query"; +import * as _110 from "./vault/share"; +import * as _111 from "./vault/tx"; +import * as _112 from "./vault/vault"; +import * as _113 from "./vest/genesis"; +import * as _114 from "./vest/query"; +import * as _115 from "./vest/tx"; +import * as _116 from "./vest/vest_entry"; +import * as _124 from "./assets/query.lcd"; +import * as _125 from "./blocktime/query.lcd"; +import * as _126 from "./bridge/query.lcd"; +import * as _127 from "./clob/query.lcd"; +import * as _128 from "./delaymsg/query.lcd"; +import * as _129 from "./epochs/query.lcd"; +import * as _130 from "./feetiers/query.lcd"; +import * as _131 from "./perpetuals/query.lcd"; +import * as _132 from "./prices/query.lcd"; +import * as _133 from "./ratelimit/query.lcd"; +import * as _134 from "./revshare/query.lcd"; +import * as _135 from "./rewards/query.lcd"; +import * as _136 from "./stats/query.lcd"; +import * as _137 from "./subaccounts/query.lcd"; +import * as _138 from "./vault/query.lcd"; +import * as _139 from "./vest/query.lcd"; +import * as _140 from "./assets/query.rpc.Query"; +import * as _141 from "./blocktime/query.rpc.Query"; +import * as _142 from "./bridge/query.rpc.Query"; +import * as _143 from "./clob/query.rpc.Query"; +import * as _144 from "./delaymsg/query.rpc.Query"; +import * as _145 from "./epochs/query.rpc.Query"; +import * as _146 from "./feetiers/query.rpc.Query"; +import * as _147 from "./govplus/query.rpc.Query"; +import * as _148 from "./listing/query.rpc.Query"; +import * as _149 from "./perpetuals/query.rpc.Query"; +import * as _150 from "./prices/query.rpc.Query"; +import * as _151 from "./ratelimit/query.rpc.Query"; +import * as _152 from "./revshare/query.rpc.Query"; +import * as _153 from "./rewards/query.rpc.Query"; +import * as _154 from "./sending/query.rpc.Query"; +import * as _155 from "./stats/query.rpc.Query"; +import * as _156 from "./subaccounts/query.rpc.Query"; +import * as _157 from "./vault/query.rpc.Query"; +import * as _158 from "./vest/query.rpc.Query"; +import * as _159 from "./blocktime/tx.rpc.msg"; +import * as _160 from "./bridge/tx.rpc.msg"; +import * as _161 from "./clob/tx.rpc.msg"; +import * as _162 from "./delaymsg/tx.rpc.msg"; +import * as _163 from "./feetiers/tx.rpc.msg"; +import * as _164 from "./govplus/tx.rpc.msg"; +import * as _165 from "./listing/tx.rpc.msg"; +import * as _166 from "./perpetuals/tx.rpc.msg"; +import * as _167 from "./prices/tx.rpc.msg"; +import * as _168 from "./ratelimit/tx.rpc.msg"; +import * as _169 from "./revshare/tx.rpc.msg"; +import * as _170 from "./rewards/tx.rpc.msg"; +import * as _171 from "./sending/tx.rpc.msg"; +import * as _172 from "./stats/tx.rpc.msg"; +import * as _173 from "./vault/tx.rpc.msg"; +import * as _174 from "./vest/tx.rpc.msg"; +import * as _175 from "./lcd"; +import * as _176 from "./rpc.query"; +import * as _177 from "./rpc.tx"; export namespace dydxprotocol { export const accountplus = { ..._5, ..._6 @@ -171,17 +172,17 @@ export namespace dydxprotocol { ..._8, ..._9, ..._10, - ..._123, - ..._139 + ..._124, + ..._140 }; export const blocktime = { ..._11, ..._12, ..._13, ..._14, ..._15, - ..._124, - ..._140, - ..._158 + ..._125, + ..._141, + ..._159 }; export const bridge = { ..._16, ..._17, @@ -189,9 +190,9 @@ export namespace dydxprotocol { ..._19, ..._20, ..._21, - ..._125, - ..._141, - ..._159 + ..._126, + ..._142, + ..._160 }; export const clob = { ..._22, ..._23, @@ -207,9 +208,9 @@ export namespace dydxprotocol { ..._33, ..._34, ..._35, - ..._126, - ..._142, - ..._160 + ..._127, + ..._143, + ..._161 }; export namespace daemons { export const bridge = { ..._36 @@ -224,29 +225,29 @@ export namespace dydxprotocol { ..._41, ..._42, ..._43, - ..._127, - ..._143, - ..._161 + ..._128, + ..._144, + ..._162 }; export const epochs = { ..._44, ..._45, ..._46, - ..._128, - ..._144 + ..._129, + ..._145 }; export const feetiers = { ..._47, ..._48, ..._49, ..._50, - ..._129, - ..._145, - ..._162 + ..._130, + ..._146, + ..._163 }; export const govplus = { ..._51, ..._52, ..._53, - ..._146, - ..._163 + ..._147, + ..._164 }; export namespace indexer { export const events = { ..._54 @@ -271,26 +272,26 @@ export namespace dydxprotocol { export const listing = { ..._63, ..._64, ..._65, - ..._147, - ..._164 + ..._148, + ..._165 }; export const perpetuals = { ..._66, ..._67, ..._68, ..._69, ..._70, - ..._130, - ..._148, - ..._165 + ..._131, + ..._149, + ..._166 }; export const prices = { ..._71, ..._72, ..._73, ..._74, ..._75, - ..._131, - ..._149, - ..._166 + ..._132, + ..._150, + ..._167 }; export const ratelimit = { ..._76, ..._77, @@ -298,72 +299,73 @@ export namespace dydxprotocol { ..._79, ..._80, ..._81, - ..._132, - ..._150, - ..._167 + ..._133, + ..._151, + ..._168 }; export const revshare = { ..._82, ..._83, ..._84, ..._85, ..._86, - ..._133, - ..._151, - ..._168 + ..._134, + ..._152, + ..._169 }; export const rewards = { ..._87, ..._88, ..._89, ..._90, ..._91, - ..._134, - ..._152, - ..._169 + ..._135, + ..._153, + ..._170 }; export const sending = { ..._92, ..._93, ..._94, ..._95, - ..._153, - ..._170 + ..._154, + ..._171 }; export const stats = { ..._96, ..._97, ..._98, ..._99, ..._100, - ..._135, - ..._154, - ..._171 + ..._136, + ..._155, + ..._172 }; export const subaccounts = { ..._101, ..._102, ..._103, ..._104, ..._105, - ..._136, - ..._155 + ..._106, + ..._137, + ..._156 }; - export const vault = { ..._106, - ..._107, + export const vault = { ..._107, ..._108, ..._109, ..._110, ..._111, - ..._137, - ..._156, - ..._172 - }; - export const vest = { ..._112, - ..._113, - ..._114, - ..._115, + ..._112, ..._138, ..._157, ..._173 }; - export const ClientFactory = { ..._174, - ..._175, - ..._176 + export const vest = { ..._113, + ..._114, + ..._115, + ..._116, + ..._139, + ..._158, + ..._174 + }; + export const ClientFactory = { ..._175, + ..._176, + ..._177 }; } \ No newline at end of file diff --git a/indexer/packages/v4-protos/src/codegen/dydxprotocol/clob/query.ts b/indexer/packages/v4-protos/src/codegen/dydxprotocol/clob/query.ts index 3eb7ca7df2..27872be567 100644 --- a/indexer/packages/v4-protos/src/codegen/dydxprotocol/clob/query.ts +++ b/indexer/packages/v4-protos/src/codegen/dydxprotocol/clob/query.ts @@ -6,6 +6,7 @@ import { ClobPair, ClobPairSDKType } from "./clob_pair"; import { EquityTierLimitConfiguration, EquityTierLimitConfigurationSDKType } from "./equity_tier_limit_config"; import { BlockRateLimitConfiguration, BlockRateLimitConfigurationSDKType } from "./block_rate_limit_config"; import { LiquidationsConfig, LiquidationsConfigSDKType } from "./liquidations_config"; +import { StreamSubaccountUpdate, StreamSubaccountUpdateSDKType } from "../subaccounts/streaming"; import { OffChainUpdateV1, OffChainUpdateV1SDKType } from "../indexer/off_chain_updates/off_chain_updates"; import { ClobMatch, ClobMatchSDKType } from "./matches"; import * as _m0 from "protobufjs/minimal"; @@ -320,104 +321,6 @@ export interface StreamUpdateSDKType { exec_mode: number; } -/** - * SubaccountPerpetualPosition provides information on a subaccount's updated - * perpetual positions. - */ - -export interface SubaccountPerpetualPosition { - /** The `Id` of the `Perpetual`. */ - perpetualId: number; - /** The size of the position in base quantums. */ - - quantums: Long; -} -/** - * SubaccountPerpetualPosition provides information on a subaccount's updated - * perpetual positions. - */ - -export interface SubaccountPerpetualPositionSDKType { - /** The `Id` of the `Perpetual`. */ - perpetual_id: number; - /** The size of the position in base quantums. */ - - quantums: Long; -} -/** - * SubaccountAssetPosition provides information on a subaccount's updated asset - * positions. - */ - -export interface SubaccountAssetPosition { - /** The `Id` of the `Asset`. */ - assetId: number; - /** The absolute size of the position in base quantums. */ - - quantums: Long; -} -/** - * SubaccountAssetPosition provides information on a subaccount's updated asset - * positions. - */ - -export interface SubaccountAssetPositionSDKType { - /** The `Id` of the `Asset`. */ - asset_id: number; - /** The absolute size of the position in base quantums. */ - - quantums: Long; -} -/** - * StreamSubaccountUpdate provides information on a subaccount update. Used in - * the full node GRPC stream. - */ - -export interface StreamSubaccountUpdate { - subaccountId?: SubaccountId; - /** updated_perpetual_positions will each be for unique perpetuals. */ - - updatedPerpetualPositions: SubaccountPerpetualPosition[]; - /** updated_asset_positions will each be for unique assets. */ - - updatedAssetPositions: SubaccountAssetPosition[]; - /** - * Snapshot indicates if the response is from a snapshot of the subaccount. - * All updates should be ignored until snapshot is received. - * If the snapshot is true, then all previous entries should be - * discarded and the subaccount should be resynced. - * For a snapshot subaccount update, the `updated_perpetual_positions` and - * `updated_asset_positions` fields will contain the full state of the - * subaccount. - */ - - snapshot: boolean; -} -/** - * StreamSubaccountUpdate provides information on a subaccount update. Used in - * the full node GRPC stream. - */ - -export interface StreamSubaccountUpdateSDKType { - subaccount_id?: SubaccountIdSDKType; - /** updated_perpetual_positions will each be for unique perpetuals. */ - - updated_perpetual_positions: SubaccountPerpetualPositionSDKType[]; - /** updated_asset_positions will each be for unique assets. */ - - updated_asset_positions: SubaccountAssetPositionSDKType[]; - /** - * Snapshot indicates if the response is from a snapshot of the subaccount. - * All updates should be ignored until snapshot is received. - * If the snapshot is true, then all previous entries should be - * discarded and the subaccount should be resynced. - * For a snapshot subaccount update, the `updated_perpetual_positions` and - * `updated_asset_positions` fields will contain the full state of the - * subaccount. - */ - - snapshot: boolean; -} /** * StreamOrderbookUpdate provides information on an orderbook update. Used in * the full node GRPC stream. @@ -1495,191 +1398,6 @@ export const StreamUpdate = { }; -function createBaseSubaccountPerpetualPosition(): SubaccountPerpetualPosition { - return { - perpetualId: 0, - quantums: Long.UZERO - }; -} - -export const SubaccountPerpetualPosition = { - encode(message: SubaccountPerpetualPosition, writer: _m0.Writer = _m0.Writer.create()): _m0.Writer { - if (message.perpetualId !== 0) { - writer.uint32(8).uint32(message.perpetualId); - } - - if (!message.quantums.isZero()) { - writer.uint32(16).uint64(message.quantums); - } - - return writer; - }, - - decode(input: _m0.Reader | Uint8Array, length?: number): SubaccountPerpetualPosition { - const reader = input instanceof _m0.Reader ? input : new _m0.Reader(input); - let end = length === undefined ? reader.len : reader.pos + length; - const message = createBaseSubaccountPerpetualPosition(); - - while (reader.pos < end) { - const tag = reader.uint32(); - - switch (tag >>> 3) { - case 1: - message.perpetualId = reader.uint32(); - break; - - case 2: - message.quantums = (reader.uint64() as Long); - break; - - default: - reader.skipType(tag & 7); - break; - } - } - - return message; - }, - - fromPartial(object: DeepPartial): SubaccountPerpetualPosition { - const message = createBaseSubaccountPerpetualPosition(); - message.perpetualId = object.perpetualId ?? 0; - message.quantums = object.quantums !== undefined && object.quantums !== null ? Long.fromValue(object.quantums) : Long.UZERO; - return message; - } - -}; - -function createBaseSubaccountAssetPosition(): SubaccountAssetPosition { - return { - assetId: 0, - quantums: Long.UZERO - }; -} - -export const SubaccountAssetPosition = { - encode(message: SubaccountAssetPosition, writer: _m0.Writer = _m0.Writer.create()): _m0.Writer { - if (message.assetId !== 0) { - writer.uint32(8).uint32(message.assetId); - } - - if (!message.quantums.isZero()) { - writer.uint32(16).uint64(message.quantums); - } - - return writer; - }, - - decode(input: _m0.Reader | Uint8Array, length?: number): SubaccountAssetPosition { - const reader = input instanceof _m0.Reader ? input : new _m0.Reader(input); - let end = length === undefined ? reader.len : reader.pos + length; - const message = createBaseSubaccountAssetPosition(); - - while (reader.pos < end) { - const tag = reader.uint32(); - - switch (tag >>> 3) { - case 1: - message.assetId = reader.uint32(); - break; - - case 2: - message.quantums = (reader.uint64() as Long); - break; - - default: - reader.skipType(tag & 7); - break; - } - } - - return message; - }, - - fromPartial(object: DeepPartial): SubaccountAssetPosition { - const message = createBaseSubaccountAssetPosition(); - message.assetId = object.assetId ?? 0; - message.quantums = object.quantums !== undefined && object.quantums !== null ? Long.fromValue(object.quantums) : Long.UZERO; - return message; - } - -}; - -function createBaseStreamSubaccountUpdate(): StreamSubaccountUpdate { - return { - subaccountId: undefined, - updatedPerpetualPositions: [], - updatedAssetPositions: [], - snapshot: false - }; -} - -export const StreamSubaccountUpdate = { - encode(message: StreamSubaccountUpdate, writer: _m0.Writer = _m0.Writer.create()): _m0.Writer { - if (message.subaccountId !== undefined) { - SubaccountId.encode(message.subaccountId, writer.uint32(10).fork()).ldelim(); - } - - for (const v of message.updatedPerpetualPositions) { - SubaccountPerpetualPosition.encode(v!, writer.uint32(18).fork()).ldelim(); - } - - for (const v of message.updatedAssetPositions) { - SubaccountAssetPosition.encode(v!, writer.uint32(26).fork()).ldelim(); - } - - if (message.snapshot === true) { - writer.uint32(32).bool(message.snapshot); - } - - return writer; - }, - - decode(input: _m0.Reader | Uint8Array, length?: number): StreamSubaccountUpdate { - const reader = input instanceof _m0.Reader ? input : new _m0.Reader(input); - let end = length === undefined ? reader.len : reader.pos + length; - const message = createBaseStreamSubaccountUpdate(); - - while (reader.pos < end) { - const tag = reader.uint32(); - - switch (tag >>> 3) { - case 1: - message.subaccountId = SubaccountId.decode(reader, reader.uint32()); - break; - - case 2: - message.updatedPerpetualPositions.push(SubaccountPerpetualPosition.decode(reader, reader.uint32())); - break; - - case 3: - message.updatedAssetPositions.push(SubaccountAssetPosition.decode(reader, reader.uint32())); - break; - - case 4: - message.snapshot = reader.bool(); - break; - - default: - reader.skipType(tag & 7); - break; - } - } - - return message; - }, - - fromPartial(object: DeepPartial): StreamSubaccountUpdate { - const message = createBaseStreamSubaccountUpdate(); - message.subaccountId = object.subaccountId !== undefined && object.subaccountId !== null ? SubaccountId.fromPartial(object.subaccountId) : undefined; - message.updatedPerpetualPositions = object.updatedPerpetualPositions?.map(e => SubaccountPerpetualPosition.fromPartial(e)) || []; - message.updatedAssetPositions = object.updatedAssetPositions?.map(e => SubaccountAssetPosition.fromPartial(e)) || []; - message.snapshot = object.snapshot ?? false; - return message; - } - -}; - function createBaseStreamOrderbookUpdate(): StreamOrderbookUpdate { return { updates: [], diff --git a/indexer/packages/v4-protos/src/codegen/dydxprotocol/subaccounts/streaming.ts b/indexer/packages/v4-protos/src/codegen/dydxprotocol/subaccounts/streaming.ts new file mode 100644 index 0000000000..fd54ef914b --- /dev/null +++ b/indexer/packages/v4-protos/src/codegen/dydxprotocol/subaccounts/streaming.ts @@ -0,0 +1,286 @@ +import { SubaccountId, SubaccountIdSDKType } from "./subaccount"; +import * as _m0 from "protobufjs/minimal"; +import { DeepPartial, Long } from "../../helpers"; +/** + * StreamSubaccountUpdate provides information on a subaccount update. Used in + * the full node GRPC stream. + */ + +export interface StreamSubaccountUpdate { + subaccountId?: SubaccountId; + /** updated_perpetual_positions will each be for unique perpetuals. */ + + updatedPerpetualPositions: SubaccountPerpetualPosition[]; + /** updated_asset_positions will each be for unique assets. */ + + updatedAssetPositions: SubaccountAssetPosition[]; + /** + * Snapshot indicates if the response is from a snapshot of the subaccount. + * All updates should be ignored until snapshot is received. + * If the snapshot is true, then all previous entries should be + * discarded and the subaccount should be resynced. + * For a snapshot subaccount update, the `updated_perpetual_positions` and + * `updated_asset_positions` fields will contain the full state of the + * subaccount. + */ + + snapshot: boolean; +} +/** + * StreamSubaccountUpdate provides information on a subaccount update. Used in + * the full node GRPC stream. + */ + +export interface StreamSubaccountUpdateSDKType { + subaccount_id?: SubaccountIdSDKType; + /** updated_perpetual_positions will each be for unique perpetuals. */ + + updated_perpetual_positions: SubaccountPerpetualPositionSDKType[]; + /** updated_asset_positions will each be for unique assets. */ + + updated_asset_positions: SubaccountAssetPositionSDKType[]; + /** + * Snapshot indicates if the response is from a snapshot of the subaccount. + * All updates should be ignored until snapshot is received. + * If the snapshot is true, then all previous entries should be + * discarded and the subaccount should be resynced. + * For a snapshot subaccount update, the `updated_perpetual_positions` and + * `updated_asset_positions` fields will contain the full state of the + * subaccount. + */ + + snapshot: boolean; +} +/** + * SubaccountPerpetualPosition provides information on a subaccount's updated + * perpetual positions. + */ + +export interface SubaccountPerpetualPosition { + /** The `Id` of the `Perpetual`. */ + perpetualId: number; + /** The size of the position in base quantums. */ + + quantums: Long; +} +/** + * SubaccountPerpetualPosition provides information on a subaccount's updated + * perpetual positions. + */ + +export interface SubaccountPerpetualPositionSDKType { + /** The `Id` of the `Perpetual`. */ + perpetual_id: number; + /** The size of the position in base quantums. */ + + quantums: Long; +} +/** + * SubaccountAssetPosition provides information on a subaccount's updated asset + * positions. + */ + +export interface SubaccountAssetPosition { + /** The `Id` of the `Asset`. */ + assetId: number; + /** The absolute size of the position in base quantums. */ + + quantums: Long; +} +/** + * SubaccountAssetPosition provides information on a subaccount's updated asset + * positions. + */ + +export interface SubaccountAssetPositionSDKType { + /** The `Id` of the `Asset`. */ + asset_id: number; + /** The absolute size of the position in base quantums. */ + + quantums: Long; +} + +function createBaseStreamSubaccountUpdate(): StreamSubaccountUpdate { + return { + subaccountId: undefined, + updatedPerpetualPositions: [], + updatedAssetPositions: [], + snapshot: false + }; +} + +export const StreamSubaccountUpdate = { + encode(message: StreamSubaccountUpdate, writer: _m0.Writer = _m0.Writer.create()): _m0.Writer { + if (message.subaccountId !== undefined) { + SubaccountId.encode(message.subaccountId, writer.uint32(10).fork()).ldelim(); + } + + for (const v of message.updatedPerpetualPositions) { + SubaccountPerpetualPosition.encode(v!, writer.uint32(18).fork()).ldelim(); + } + + for (const v of message.updatedAssetPositions) { + SubaccountAssetPosition.encode(v!, writer.uint32(26).fork()).ldelim(); + } + + if (message.snapshot === true) { + writer.uint32(32).bool(message.snapshot); + } + + return writer; + }, + + decode(input: _m0.Reader | Uint8Array, length?: number): StreamSubaccountUpdate { + const reader = input instanceof _m0.Reader ? input : new _m0.Reader(input); + let end = length === undefined ? reader.len : reader.pos + length; + const message = createBaseStreamSubaccountUpdate(); + + while (reader.pos < end) { + const tag = reader.uint32(); + + switch (tag >>> 3) { + case 1: + message.subaccountId = SubaccountId.decode(reader, reader.uint32()); + break; + + case 2: + message.updatedPerpetualPositions.push(SubaccountPerpetualPosition.decode(reader, reader.uint32())); + break; + + case 3: + message.updatedAssetPositions.push(SubaccountAssetPosition.decode(reader, reader.uint32())); + break; + + case 4: + message.snapshot = reader.bool(); + break; + + default: + reader.skipType(tag & 7); + break; + } + } + + return message; + }, + + fromPartial(object: DeepPartial): StreamSubaccountUpdate { + const message = createBaseStreamSubaccountUpdate(); + message.subaccountId = object.subaccountId !== undefined && object.subaccountId !== null ? SubaccountId.fromPartial(object.subaccountId) : undefined; + message.updatedPerpetualPositions = object.updatedPerpetualPositions?.map(e => SubaccountPerpetualPosition.fromPartial(e)) || []; + message.updatedAssetPositions = object.updatedAssetPositions?.map(e => SubaccountAssetPosition.fromPartial(e)) || []; + message.snapshot = object.snapshot ?? false; + return message; + } + +}; + +function createBaseSubaccountPerpetualPosition(): SubaccountPerpetualPosition { + return { + perpetualId: 0, + quantums: Long.UZERO + }; +} + +export const SubaccountPerpetualPosition = { + encode(message: SubaccountPerpetualPosition, writer: _m0.Writer = _m0.Writer.create()): _m0.Writer { + if (message.perpetualId !== 0) { + writer.uint32(8).uint32(message.perpetualId); + } + + if (!message.quantums.isZero()) { + writer.uint32(16).uint64(message.quantums); + } + + return writer; + }, + + decode(input: _m0.Reader | Uint8Array, length?: number): SubaccountPerpetualPosition { + const reader = input instanceof _m0.Reader ? input : new _m0.Reader(input); + let end = length === undefined ? reader.len : reader.pos + length; + const message = createBaseSubaccountPerpetualPosition(); + + while (reader.pos < end) { + const tag = reader.uint32(); + + switch (tag >>> 3) { + case 1: + message.perpetualId = reader.uint32(); + break; + + case 2: + message.quantums = (reader.uint64() as Long); + break; + + default: + reader.skipType(tag & 7); + break; + } + } + + return message; + }, + + fromPartial(object: DeepPartial): SubaccountPerpetualPosition { + const message = createBaseSubaccountPerpetualPosition(); + message.perpetualId = object.perpetualId ?? 0; + message.quantums = object.quantums !== undefined && object.quantums !== null ? Long.fromValue(object.quantums) : Long.UZERO; + return message; + } + +}; + +function createBaseSubaccountAssetPosition(): SubaccountAssetPosition { + return { + assetId: 0, + quantums: Long.UZERO + }; +} + +export const SubaccountAssetPosition = { + encode(message: SubaccountAssetPosition, writer: _m0.Writer = _m0.Writer.create()): _m0.Writer { + if (message.assetId !== 0) { + writer.uint32(8).uint32(message.assetId); + } + + if (!message.quantums.isZero()) { + writer.uint32(16).uint64(message.quantums); + } + + return writer; + }, + + decode(input: _m0.Reader | Uint8Array, length?: number): SubaccountAssetPosition { + const reader = input instanceof _m0.Reader ? input : new _m0.Reader(input); + let end = length === undefined ? reader.len : reader.pos + length; + const message = createBaseSubaccountAssetPosition(); + + while (reader.pos < end) { + const tag = reader.uint32(); + + switch (tag >>> 3) { + case 1: + message.assetId = reader.uint32(); + break; + + case 2: + message.quantums = (reader.uint64() as Long); + break; + + default: + reader.skipType(tag & 7); + break; + } + } + + return message; + }, + + fromPartial(object: DeepPartial): SubaccountAssetPosition { + const message = createBaseSubaccountAssetPosition(); + message.assetId = object.assetId ?? 0; + message.quantums = object.quantums !== undefined && object.quantums !== null ? Long.fromValue(object.quantums) : Long.UZERO; + return message; + } + +}; \ No newline at end of file diff --git a/indexer/packages/v4-protos/src/codegen/gogoproto/bundle.ts b/indexer/packages/v4-protos/src/codegen/gogoproto/bundle.ts index 709b81ce7c..cdc090d0d6 100644 --- a/indexer/packages/v4-protos/src/codegen/gogoproto/bundle.ts +++ b/indexer/packages/v4-protos/src/codegen/gogoproto/bundle.ts @@ -1,3 +1,3 @@ -import * as _116 from "./gogo"; -export const gogoproto = { ..._116 +import * as _117 from "./gogo"; +export const gogoproto = { ..._117 }; \ No newline at end of file diff --git a/indexer/packages/v4-protos/src/codegen/google/bundle.ts b/indexer/packages/v4-protos/src/codegen/google/bundle.ts index 2cbfe8f57c..7103f2f637 100644 --- a/indexer/packages/v4-protos/src/codegen/google/bundle.ts +++ b/indexer/packages/v4-protos/src/codegen/google/bundle.ts @@ -1,16 +1,16 @@ -import * as _117 from "./api/annotations"; -import * as _118 from "./api/http"; -import * as _119 from "./protobuf/descriptor"; -import * as _120 from "./protobuf/duration"; -import * as _121 from "./protobuf/timestamp"; -import * as _122 from "./protobuf/any"; +import * as _118 from "./api/annotations"; +import * as _119 from "./api/http"; +import * as _120 from "./protobuf/descriptor"; +import * as _121 from "./protobuf/duration"; +import * as _122 from "./protobuf/timestamp"; +import * as _123 from "./protobuf/any"; export namespace google { - export const api = { ..._117, - ..._118 + export const api = { ..._118, + ..._119 }; - export const protobuf = { ..._119, - ..._120, + export const protobuf = { ..._120, ..._121, - ..._122 + ..._122, + ..._123 }; } \ No newline at end of file diff --git a/proto/dydxprotocol/clob/query.proto b/proto/dydxprotocol/clob/query.proto index 34f9899801..b0342bc3c6 100644 --- a/proto/dydxprotocol/clob/query.proto +++ b/proto/dydxprotocol/clob/query.proto @@ -12,6 +12,7 @@ import "dydxprotocol/clob/matches.proto"; import "dydxprotocol/clob/liquidations_config.proto"; import "dydxprotocol/clob/mev.proto"; import "dydxprotocol/indexer/off_chain_updates/off_chain_updates.proto"; +import "dydxprotocol/subaccounts/streaming.proto"; import "dydxprotocol/subaccounts/subaccount.proto"; option go_package = "github.com/dydxprotocol/v4-chain/protocol/x/clob/types"; @@ -187,7 +188,7 @@ message StreamUpdate { StreamOrderbookUpdate orderbook_update = 1; StreamOrderbookFill order_fill = 2; StreamTakerOrder taker_order = 3; - StreamSubaccountUpdate subaccount_update = 4; + dydxprotocol.subaccounts.StreamSubaccountUpdate subaccount_update = 4; } // Block height of the update. @@ -197,42 +198,6 @@ message StreamUpdate { uint32 exec_mode = 6; } -// SubaccountPerpetualPosition provides information on a subaccount's updated -// perpetual positions. -message SubaccountPerpetualPosition { - // The `Id` of the `Perpetual`. - uint32 perpetual_id = 1; - // The size of the position in base quantums. - uint64 quantums = 2; -} - -// SubaccountAssetPosition provides information on a subaccount's updated asset -// positions. -message SubaccountAssetPosition { - // The `Id` of the `Asset`. - uint32 asset_id = 1; - // The absolute size of the position in base quantums. - uint64 quantums = 2; -} - -// StreamSubaccountUpdate provides information on a subaccount update. Used in -// the full node GRPC stream. -message StreamSubaccountUpdate { - dydxprotocol.subaccounts.SubaccountId subaccount_id = 1; - // updated_perpetual_positions will each be for unique perpetuals. - repeated SubaccountPerpetualPosition updated_perpetual_positions = 2; - // updated_asset_positions will each be for unique assets. - repeated SubaccountAssetPosition updated_asset_positions = 3; - // Snapshot indicates if the response is from a snapshot of the subaccount. - // All updates should be ignored until snapshot is received. - // If the snapshot is true, then all previous entries should be - // discarded and the subaccount should be resynced. - // For a snapshot subaccount update, the `updated_perpetual_positions` and - // `updated_asset_positions` fields will contain the full state of the - // subaccount. - bool snapshot = 4; -} - // StreamOrderbookUpdate provides information on an orderbook update. Used in // the full node GRPC stream. message StreamOrderbookUpdate { diff --git a/proto/dydxprotocol/subaccounts/streaming.proto b/proto/dydxprotocol/subaccounts/streaming.proto new file mode 100644 index 0000000000..13b71ee1ae --- /dev/null +++ b/proto/dydxprotocol/subaccounts/streaming.proto @@ -0,0 +1,42 @@ +syntax = "proto3"; +package dydxprotocol.subaccounts; + +import "dydxprotocol/subaccounts/subaccount.proto"; + +option go_package = "github.com/dydxprotocol/v4-chain/protocol/x/subaccounts/types"; + +// StreamSubaccountUpdate provides information on a subaccount update. Used in +// the full node GRPC stream. +message StreamSubaccountUpdate { + SubaccountId subaccount_id = 1; + // updated_perpetual_positions will each be for unique perpetuals. + repeated SubaccountPerpetualPosition updated_perpetual_positions = 2; + // updated_asset_positions will each be for unique assets. + repeated SubaccountAssetPosition updated_asset_positions = 3; + // Snapshot indicates if the response is from a snapshot of the subaccount. + // All updates should be ignored until snapshot is received. + // If the snapshot is true, then all previous entries should be + // discarded and the subaccount should be resynced. + // For a snapshot subaccount update, the `updated_perpetual_positions` and + // `updated_asset_positions` fields will contain the full state of the + // subaccount. + bool snapshot = 4; +} + +// SubaccountPerpetualPosition provides information on a subaccount's updated +// perpetual positions. +message SubaccountPerpetualPosition { + // The `Id` of the `Perpetual`. + uint32 perpetual_id = 1; + // The size of the position in base quantums. + uint64 quantums = 2; +} + +// SubaccountAssetPosition provides information on a subaccount's updated asset +// positions. +message SubaccountAssetPosition { + // The `Id` of the `Asset`. + uint32 asset_id = 1; + // The absolute size of the position in base quantums. + uint64 quantums = 2; +} diff --git a/protocol/app/app.go b/protocol/app/app.go index 4938eeb0ba..92bef24378 100644 --- a/protocol/app/app.go +++ b/protocol/app/app.go @@ -1061,6 +1061,7 @@ func New( app.BlockTimeKeeper, app.RevShareKeeper, app.IndexerEventManager, + app.FullNodeStreamingManager, ) subaccountsModule := subaccountsmodule.NewAppModule( appCodec, diff --git a/protocol/lib/metrics/metric_keys.go b/protocol/lib/metrics/metric_keys.go index 0c549c2dfa..088e742386 100644 --- a/protocol/lib/metrics/metric_keys.go +++ b/protocol/lib/metrics/metric_keys.go @@ -69,7 +69,9 @@ const ( FullNodeGrpc = "full_node_grpc" GrpcSendOrderbookUpdatesLatency = "grpc_send_orderbook_updates_latency" GrpcSendOrderbookSnapshotLatency = "grpc_send_orderbook_snapshot_latency" + GrpcSendSubaccountSnapshotLatency = "grpc_send_subaccount_snapshot_latency" GrpcSendOrderbookFillsLatency = "grpc_send_orderbook_fills_latency" + GrpcSendSubaccountUpdatesLatency = "grpc_send_subaccount_updates_latency" GrpcAddUpdateToBufferCount = "grpc_add_update_to_buffer_count" GrpcAddToSubscriptionChannelCount = "grpc_add_to_subscription_channel_count" GrpcSendResponseToSubscriberCount = "grpc_send_response_to_subscriber_count" diff --git a/protocol/streaming/full_node_streaming_manager.go b/protocol/streaming/full_node_streaming_manager.go index fb10dfb3ec..c53b76e2c0 100644 --- a/protocol/streaming/full_node_streaming_manager.go +++ b/protocol/streaming/full_node_streaming_manager.go @@ -2,6 +2,7 @@ package streaming import ( "fmt" + satypes "github.com/dydxprotocol/v4-chain/protocol/x/subaccounts/types" "sync" "time" @@ -38,6 +39,8 @@ type FullNodeStreamingManagerImpl struct { streamUpdateSubscriptionCache [][]uint32 // map from clob pair id to subscription ids. clobPairIdToSubscriptionIdMapping map[uint32][]uint32 + // map from subaccount id to subscription ids. + subaccountIdToSubscriptionIdMapping map[satypes.SubaccountId][]uint32 maxUpdatesInCache uint32 maxSubscriptionChannelSize uint32 @@ -53,6 +56,9 @@ type OrderbookSubscription struct { // Clob pair ids to subscribe to. clobPairIds []uint32 + // Subaccount ids to subscribe to. + subaccountIds []satypes.SubaccountId + // Stream messageSender types.OutgoingMessageSender @@ -72,11 +78,12 @@ func NewFullNodeStreamingManager( orderbookSubscriptions: make(map[uint32]*OrderbookSubscription), nextSubscriptionId: 0, - ticker: time.NewTicker(time.Duration(flushIntervalMs) * time.Millisecond), - done: make(chan bool), - streamUpdateCache: make([]clobtypes.StreamUpdate, 0), - streamUpdateSubscriptionCache: make([][]uint32, 0), - clobPairIdToSubscriptionIdMapping: make(map[uint32][]uint32), + ticker: time.NewTicker(time.Duration(flushIntervalMs) * time.Millisecond), + done: make(chan bool), + streamUpdateCache: make([]clobtypes.StreamUpdate, 0), + streamUpdateSubscriptionCache: make([][]uint32, 0), + clobPairIdToSubscriptionIdMapping: make(map[uint32][]uint32), + subaccountIdToSubscriptionIdMapping: make(map[satypes.SubaccountId][]uint32), maxUpdatesInCache: maxUpdatesInCache, maxSubscriptionChannelSize: maxSubscriptionChannelSize, @@ -125,6 +132,7 @@ func (sm *FullNodeStreamingManagerImpl) EmitMetrics() { // Subscribe subscribes to the orderbook updates stream. func (sm *FullNodeStreamingManagerImpl) Subscribe( clobPairIds []uint32, + subaccountIds []*satypes.SubaccountId, messageSender types.OutgoingMessageSender, ) ( err error, @@ -135,9 +143,14 @@ func (sm *FullNodeStreamingManagerImpl) Subscribe( } sm.Lock() + sIds := make([]satypes.SubaccountId, len(subaccountIds)) + for i, subaccountId := range subaccountIds { + sIds[i] = *subaccountId + } subscription := &OrderbookSubscription{ subscriptionId: sm.nextSubscriptionId, clobPairIds: clobPairIds, + subaccountIds: sIds, messageSender: messageSender, updatesChannel: make(chan []clobtypes.StreamUpdate, sm.maxSubscriptionChannelSize), } @@ -152,12 +165,24 @@ func (sm *FullNodeStreamingManagerImpl) Subscribe( sm.nextSubscriptionId, ) } + for _, subaccountId := range sIds { + // if subaccountId exists in the map, append the subscription id to the slice + // otherwise, create a new slice with the subscription id + if _, ok := sm.subaccountIdToSubscriptionIdMapping[subaccountId]; !ok { + sm.subaccountIdToSubscriptionIdMapping[subaccountId] = []uint32{} + } + sm.subaccountIdToSubscriptionIdMapping[subaccountId] = append( + sm.subaccountIdToSubscriptionIdMapping[subaccountId], + sm.nextSubscriptionId, + ) + } sm.logger.Info( fmt.Sprintf( - "New subscription id %+v for clob pair ids: %+v", + "New subscription id %+v for clob pair ids: %+v and subaccount ids: %+v", subscription.subscriptionId, clobPairIds, + subaccountIds, ), ) sm.orderbookSubscriptions[subscription.subscriptionId] = subscription @@ -228,6 +253,21 @@ func (sm *FullNodeStreamingManagerImpl) removeSubscription( } } + // Iterate over the subaccountIdToSubscriptionIdMapping to remove the subscriptionIdToRemove + for subaccountId, subscriptionIds := range sm.subaccountIdToSubscriptionIdMapping { + for i, id := range subscriptionIds { + if id == subscriptionIdToRemove { + // Remove the subscription ID from the slice + sm.subaccountIdToSubscriptionIdMapping[subaccountId] = append(subscriptionIds[:i], subscriptionIds[i+1:]...) + break + } + } + // If the list is empty after removal, delete the key from the map + if len(sm.subaccountIdToSubscriptionIdMapping[subaccountId]) == 0 { + delete(sm.subaccountIdToSubscriptionIdMapping, subaccountId) + } + } + sm.logger.Info( fmt.Sprintf("Removed streaming subscription id %+v", subscriptionIdToRemove), ) @@ -237,74 +277,110 @@ func (sm *FullNodeStreamingManagerImpl) Stop() { sm.done <- true } -// SendSnapshot sends messages to a particular subscriber without buffering. -// Note this method requires the lock and assumes that the lock has already been -// acquired by the caller. -func (sm *FullNodeStreamingManagerImpl) SendSnapshot( +func toOrderbookStreamUpdate( offchainUpdates *clobtypes.OffchainUpdates, - subscriptionId uint32, blockHeight uint32, execMode sdk.ExecMode, -) { - defer metrics.ModuleMeasureSince( - metrics.FullNodeGrpc, - metrics.GrpcSendOrderbookSnapshotLatency, - time.Now(), - ) - +) []clobtypes.StreamUpdate { v1updates, err := streaming_util.GetOffchainUpdatesV1(offchainUpdates) if err != nil { panic(err) } - - removeSubscription := false - if len(v1updates) > 0 { - subscription, ok := sm.orderbookSubscriptions[subscriptionId] - if !ok { - sm.logger.Error( - fmt.Sprintf( - "Streaming subscription id %+v not found. This should not happen.", - subscriptionId, - ), - ) - return - } - streamUpdates := []clobtypes.StreamUpdate{ - { - UpdateMessage: &clobtypes.StreamUpdate_OrderbookUpdate{ - OrderbookUpdate: &clobtypes.StreamOrderbookUpdate{ - Updates: v1updates, - Snapshot: true, - }, + return []clobtypes.StreamUpdate{ + { + UpdateMessage: &clobtypes.StreamUpdate_OrderbookUpdate{ + OrderbookUpdate: &clobtypes.StreamOrderbookUpdate{ + Updates: v1updates, + Snapshot: true, }, - BlockHeight: blockHeight, - ExecMode: uint32(execMode), }, - } - metrics.IncrCounter( - metrics.GrpcAddToSubscriptionChannelCount, - 1, + BlockHeight: blockHeight, + ExecMode: uint32(execMode), + }, + } +} + +func toSubaccountStreamUpdates( + saUpdates []*satypes.StreamSubaccountUpdate, + blockHeight uint32, + execMode sdk.ExecMode, +) []clobtypes.StreamUpdate { + streamUpdates := make([]clobtypes.StreamUpdate, 0) + for _, saUpdate := range saUpdates { + streamUpdates = append(streamUpdates, clobtypes.StreamUpdate{ + UpdateMessage: &clobtypes.StreamUpdate_SubaccountUpdate{ + SubaccountUpdate: saUpdate, + }, + BlockHeight: blockHeight, + ExecMode: uint32(execMode), + }) + } + return streamUpdates +} + +func (sm *FullNodeStreamingManagerImpl) sendStreamUpdates( + subscriptionId uint32, + streamUpdates []clobtypes.StreamUpdate, +) { + removeSubscription := false + subscription, ok := sm.orderbookSubscriptions[subscriptionId] + if !ok { + sm.logger.Error( + fmt.Sprintf( + "Streaming subscription id %+v not found. This should not happen.", + subscriptionId, + ), ) - select { - case subscription.updatesChannel <- streamUpdates: - default: - sm.logger.Error( - fmt.Sprintf( - "Streaming subscription id %+v channel full capacity. Dropping subscription connection.", - subscriptionId, - ), - ) - removeSubscription = true - } + return + } + + select { + case subscription.updatesChannel <- streamUpdates: + default: + sm.logger.Error( + fmt.Sprintf( + "Streaming subscription id %+v channel full capacity. Dropping subscription connection.", + subscriptionId, + ), + ) + removeSubscription = true } - // Clean up subscriptions that have been closed. - // If a Send update has failed for any clob pair id, the whole subscription will be removed. if removeSubscription { sm.removeSubscription(subscriptionId) } } +// SendCombinedSnapshot sends messages to a particular subscriber without buffering. +// Note this method requires the lock and assumes that the lock has already been +// acquired by the caller. +func (sm *FullNodeStreamingManagerImpl) SendCombinedSnapshot( + offchainUpdates *clobtypes.OffchainUpdates, + saUpdates []*satypes.StreamSubaccountUpdate, + subscriptionId uint32, + blockHeight uint32, + execMode sdk.ExecMode, +) { + defer metrics.ModuleMeasureSince( + metrics.FullNodeGrpc, + metrics.GrpcSendOrderbookSnapshotLatency, + time.Now(), + ) + + var streamUpdates []clobtypes.StreamUpdate + streamUpdates = append(streamUpdates, toOrderbookStreamUpdate(offchainUpdates, blockHeight, execMode)...) + streamUpdates = append(streamUpdates, toSubaccountStreamUpdates(saUpdates, blockHeight, execMode)...) + sm.sendStreamUpdates(subscriptionId, streamUpdates) +} + +// TracksSubaccountId checks if a subaccount id is being tracked by the streaming manager. +func (sm *FullNodeStreamingManagerImpl) TracksSubaccountId(subaccountId satypes.SubaccountId) bool { + sm.Lock() + defer sm.Unlock() + _, exists := sm.subaccountIdToSubscriptionIdMapping[subaccountId] + return exists +} + // SendOrderbookUpdates groups updates by their clob pair ids and // sends messages to the subscribers. func (sm *FullNodeStreamingManagerImpl) SendOrderbookUpdates( @@ -350,7 +426,7 @@ func (sm *FullNodeStreamingManagerImpl) SendOrderbookUpdates( clobPairIds = append(clobPairIds, clobPairId) } - sm.AddUpdatesToCache(streamUpdates, clobPairIds) + sm.AddOrderUpdatesToCache(streamUpdates, clobPairIds) } // SendOrderbookFillUpdates groups fills by their clob pair ids and @@ -393,7 +469,7 @@ func (sm *FullNodeStreamingManagerImpl) SendOrderbookFillUpdates( clobPairIds = append(clobPairIds, clobPairId) } - sm.AddUpdatesToCache(streamUpdates, clobPairIds) + sm.AddOrderUpdatesToCache(streamUpdates, clobPairIds) } // SendTakerOrderStatus sends out a taker order and its status to the full node streaming service. @@ -410,7 +486,7 @@ func (sm *FullNodeStreamingManagerImpl) SendTakerOrderStatus( clobPairId = takerOrder.OrderId.ClobPairId } - sm.AddUpdatesToCache( + sm.AddOrderUpdatesToCache( []clobtypes.StreamUpdate{ { UpdateMessage: &clobtypes.StreamUpdate_TakerOrder{ @@ -424,9 +500,41 @@ func (sm *FullNodeStreamingManagerImpl) SendTakerOrderStatus( ) } -// AddUpdatesToCache adds a series of updates to the full node streaming cache. +// SendSubaccountUpdates groups subaccount updates by their subaccount ids and +// sends messages to the subscribers. +// TODO(CT-1117): Aggregate subaccount updates by subaccount id. +func (sm *FullNodeStreamingManagerImpl) SendSubaccountUpdates( + subaccountUpdates []satypes.StreamSubaccountUpdate, + blockHeight uint32, + execMode sdk.ExecMode, +) { + defer metrics.ModuleMeasureSince( + metrics.FullNodeGrpc, + metrics.GrpcSendSubaccountUpdatesLatency, + time.Now(), + ) + + // Group subaccount updates by subaccount id. + streamUpdates := make([]clobtypes.StreamUpdate, 0) + subaccountIds := make([]*satypes.SubaccountId, 0) + for _, subaccountUpdate := range subaccountUpdates { + streamUpdate := clobtypes.StreamUpdate{ + UpdateMessage: &clobtypes.StreamUpdate_SubaccountUpdate{ + SubaccountUpdate: &subaccountUpdate, + }, + BlockHeight: blockHeight, + ExecMode: uint32(execMode), + } + streamUpdates = append(streamUpdates, streamUpdate) + subaccountIds = append(subaccountIds, subaccountUpdate.SubaccountId) + } + + sm.AddSubaccountUpdatesToCache(streamUpdates, subaccountIds) +} + +// AddOrderUpdatesToCache adds a series of updates to the full node streaming cache. // Clob pair ids are the clob pair id each update is relevant to. -func (sm *FullNodeStreamingManagerImpl) AddUpdatesToCache( +func (sm *FullNodeStreamingManagerImpl) AddOrderUpdatesToCache( updates []clobtypes.StreamUpdate, clobPairIds []uint32, ) { @@ -446,6 +554,40 @@ func (sm *FullNodeStreamingManagerImpl) AddUpdatesToCache( ) } + // Remove all subscriptions and wipe the buffer if buffer overflows. + sm.RemoveSubscriptionsAndClearBufferIfFull() + sm.EmitMetrics() +} + +// AddSubaccountUpdatesToCache adds a series of updates to the full node streaming cache. +// Subaccount ids are the subaccount id each update is relevant to. +func (sm *FullNodeStreamingManagerImpl) AddSubaccountUpdatesToCache( + updates []clobtypes.StreamUpdate, + subaccountIds []*satypes.SubaccountId, +) { + sm.Lock() + defer sm.Unlock() + + metrics.IncrCounter( + metrics.GrpcAddUpdateToBufferCount, + float32(len(updates)), + ) + + sm.streamUpdateCache = append(sm.streamUpdateCache, updates...) + for _, subaccountId := range subaccountIds { + sm.streamUpdateSubscriptionCache = append( + sm.streamUpdateSubscriptionCache, + sm.subaccountIdToSubscriptionIdMapping[*subaccountId], + ) + } + sm.RemoveSubscriptionsAndClearBufferIfFull() + sm.EmitMetrics() +} + +// RemoveSubscriptionsAndClearBufferIfFull removes all subscriptions and wipes the buffer if buffer overflows. +// Note this method requires the lock and assumes that the lock has already been +// acquired by the caller. +func (sm *FullNodeStreamingManagerImpl) RemoveSubscriptionsAndClearBufferIfFull() { // Remove all subscriptions and wipe the buffer if buffer overflows. if len(sm.streamUpdateCache) > int(sm.maxUpdatesInCache) { sm.logger.Error("Streaming buffer full capacity. Dropping messages and all subscriptions. " + @@ -456,7 +598,6 @@ func (sm *FullNodeStreamingManagerImpl) AddUpdatesToCache( sm.streamUpdateCache = nil sm.streamUpdateSubscriptionCache = nil } - sm.EmitMetrics() } func (sm *FullNodeStreamingManagerImpl) FlushStreamUpdates() { @@ -521,6 +662,7 @@ func (sm *FullNodeStreamingManagerImpl) FlushStreamUpdatesWithLock() { func (sm *FullNodeStreamingManagerImpl) InitializeNewStreams( getOrderbookSnapshot func(clobPairId clobtypes.ClobPairId) *clobtypes.OffchainUpdates, + getSubaccountSnapshot func(subaccountId satypes.SubaccountId) *satypes.StreamSubaccountUpdate, blockHeight uint32, execMode sdk.ExecMode, ) { @@ -542,8 +684,12 @@ func (sm *FullNodeStreamingManagerImpl) InitializeNewStreams( } allUpdates.Append(updatesByClobPairId[clobPairId]) } + saUpdates := []*satypes.StreamSubaccountUpdate{} + for _, subaccountId := range subscription.subaccountIds { + saUpdates = append(saUpdates, getSubaccountSnapshot(subaccountId)) + } - sm.SendSnapshot(allUpdates, subscriptionId, blockHeight, execMode) + sm.SendCombinedSnapshot(allUpdates, saUpdates, subscriptionId, blockHeight, execMode) }, ) } diff --git a/protocol/streaming/noop_streaming_manager.go b/protocol/streaming/noop_streaming_manager.go index 749bcf2b67..24810fefe2 100644 --- a/protocol/streaming/noop_streaming_manager.go +++ b/protocol/streaming/noop_streaming_manager.go @@ -4,6 +4,7 @@ import ( sdk "github.com/cosmos/cosmos-sdk/types" "github.com/dydxprotocol/v4-chain/protocol/streaming/types" clobtypes "github.com/dydxprotocol/v4-chain/protocol/x/clob/types" + satypes "github.com/dydxprotocol/v4-chain/protocol/x/subaccounts/types" ) var _ types.FullNodeStreamingManager = (*NoopGrpcStreamingManager)(nil) @@ -20,6 +21,7 @@ func (sm *NoopGrpcStreamingManager) Enabled() bool { func (sm *NoopGrpcStreamingManager) Subscribe( _ []uint32, + _ []*satypes.SubaccountId, _ types.OutgoingMessageSender, ) ( err error, @@ -49,8 +51,20 @@ func (sm *NoopGrpcStreamingManager) SendTakerOrderStatus( ) { } +func (sm *NoopGrpcStreamingManager) SendSubaccountUpdates( + subaccountUpdates []satypes.StreamSubaccountUpdate, + blockHeight uint32, + execMode sdk.ExecMode, +) { +} + +func (sm *NoopGrpcStreamingManager) TracksSubaccountId(id satypes.SubaccountId) bool { + return false +} + func (sm *NoopGrpcStreamingManager) InitializeNewStreams( getOrderbookSnapshot func(clobPairId clobtypes.ClobPairId) *clobtypes.OffchainUpdates, + getSubaccountSnapshot func(subaccountId satypes.SubaccountId) *satypes.StreamSubaccountUpdate, blockHeight uint32, execMode sdk.ExecMode, ) { diff --git a/protocol/streaming/types/interface.go b/protocol/streaming/types/interface.go index 66fcf808e5..7930853be6 100644 --- a/protocol/streaming/types/interface.go +++ b/protocol/streaming/types/interface.go @@ -3,6 +3,7 @@ package types import ( sdk "github.com/cosmos/cosmos-sdk/types" clobtypes "github.com/dydxprotocol/v4-chain/protocol/x/clob/types" + satypes "github.com/dydxprotocol/v4-chain/protocol/x/subaccounts/types" ) type FullNodeStreamingManager interface { @@ -12,6 +13,7 @@ type FullNodeStreamingManager interface { // Subscribe to streams Subscribe( clobPairIds []uint32, + subaccountIds []*satypes.SubaccountId, srv OutgoingMessageSender, ) ( err error, @@ -20,6 +22,7 @@ type FullNodeStreamingManager interface { // L3+ Orderbook updates. InitializeNewStreams( getOrderbookSnapshot func(clobPairId clobtypes.ClobPairId) *clobtypes.OffchainUpdates, + getSubaccountSnapshot func(subaccountId satypes.SubaccountId) *satypes.StreamSubaccountUpdate, blockHeight uint32, execMode sdk.ExecMode, ) @@ -39,6 +42,12 @@ type FullNodeStreamingManager interface { blockHeight uint32, execMode sdk.ExecMode, ) + SendSubaccountUpdates( + subaccountUpdates []satypes.StreamSubaccountUpdate, + blockHeight uint32, + execMode sdk.ExecMode, + ) + TracksSubaccountId(id satypes.SubaccountId) bool } type OutgoingMessageSender interface { diff --git a/protocol/testutil/keeper/subaccounts.go b/protocol/testutil/keeper/subaccounts.go index 10a6060d0a..a51a5f1c36 100644 --- a/protocol/testutil/keeper/subaccounts.go +++ b/protocol/testutil/keeper/subaccounts.go @@ -1,6 +1,7 @@ package keeper import ( + "github.com/dydxprotocol/v4-chain/protocol/streaming" "testing" revsharekeeper "github.com/dydxprotocol/v4-chain/protocol/x/revshare/keeper" @@ -128,6 +129,7 @@ func createSubaccountsKeeper( btk, rsk, mockIndexerEventsManager, + streaming.NewNoopGrpcStreamingManager(), ) return k, storeKey diff --git a/protocol/x/clob/keeper/grpc_stream_orderbook.go b/protocol/x/clob/keeper/grpc_stream_orderbook.go index 8e72a8640d..caca5fbfbe 100644 --- a/protocol/x/clob/keeper/grpc_stream_orderbook.go +++ b/protocol/x/clob/keeper/grpc_stream_orderbook.go @@ -10,6 +10,7 @@ func (k Keeper) StreamOrderbookUpdates( ) error { err := k.GetFullNodeStreamingManager().Subscribe( req.GetClobPairId(), + req.GetSubaccountIds(), stream, ) if err != nil { diff --git a/protocol/x/clob/keeper/keeper.go b/protocol/x/clob/keeper/keeper.go index 0fbf4535b4..85e958423e 100644 --- a/protocol/x/clob/keeper/keeper.go +++ b/protocol/x/clob/keeper/keeper.go @@ -3,6 +3,7 @@ package keeper import ( "errors" "fmt" + satypes "github.com/dydxprotocol/v4-chain/protocol/x/subaccounts/types" "sync/atomic" "cosmossdk.io/log" @@ -262,6 +263,33 @@ func (k Keeper) InitializeNewStreams(ctx sdk.Context) { clobPairId, ) }, + func(subaccountId satypes.SubaccountId) *satypes.StreamSubaccountUpdate { + subaccount := k.subaccountsKeeper.GetSubaccount( + ctx, + subaccountId, + ) + assetPositions := make([]*satypes.SubaccountAssetPosition, len(subaccount.AssetPositions)) + for i, ap := range subaccount.AssetPositions { + assetPositions[i] = &satypes.SubaccountAssetPosition{ + AssetId: ap.AssetId, + Quantums: ap.Quantums.BigInt().Uint64(), + } + } + perpetualPositions := make([]*satypes.SubaccountPerpetualPosition, len(subaccount.PerpetualPositions)) + for i, pp := range subaccount.PerpetualPositions { + perpetualPositions[i] = &satypes.SubaccountPerpetualPosition{ + PerpetualId: pp.PerpetualId, + Quantums: pp.Quantums.BigInt().Uint64(), + } + } + + return &satypes.StreamSubaccountUpdate{ + SubaccountId: &subaccountId, + UpdatedAssetPositions: assetPositions, + UpdatedPerpetualPositions: perpetualPositions, + Snapshot: true, + } + }, lib.MustConvertIntegerToUint32(ctx.BlockHeight()), ctx.ExecMode(), ) diff --git a/protocol/x/clob/types/query.pb.go b/protocol/x/clob/types/query.pb.go index 260961811e..d0d00fd50a 100644 --- a/protocol/x/clob/types/query.pb.go +++ b/protocol/x/clob/types/query.pb.go @@ -937,7 +937,7 @@ type StreamUpdate_TakerOrder struct { TakerOrder *StreamTakerOrder `protobuf:"bytes,3,opt,name=taker_order,json=takerOrder,proto3,oneof" json:"taker_order,omitempty"` } type StreamUpdate_SubaccountUpdate struct { - SubaccountUpdate *StreamSubaccountUpdate `protobuf:"bytes,4,opt,name=subaccount_update,json=subaccountUpdate,proto3,oneof" json:"subaccount_update,omitempty"` + SubaccountUpdate *types.StreamSubaccountUpdate `protobuf:"bytes,4,opt,name=subaccount_update,json=subaccountUpdate,proto3,oneof" json:"subaccount_update,omitempty"` } func (*StreamUpdate_OrderbookUpdate) isStreamUpdate_UpdateMessage() {} @@ -973,7 +973,7 @@ func (m *StreamUpdate) GetTakerOrder() *StreamTakerOrder { return nil } -func (m *StreamUpdate) GetSubaccountUpdate() *StreamSubaccountUpdate { +func (m *StreamUpdate) GetSubaccountUpdate() *types.StreamSubaccountUpdate { if x, ok := m.GetUpdateMessage().(*StreamUpdate_SubaccountUpdate); ok { return x.SubaccountUpdate } @@ -1004,197 +1004,6 @@ func (*StreamUpdate) XXX_OneofWrappers() []interface{} { } } -// SubaccountPerpetualPosition provides information on a subaccount's updated -// perpetual positions. -type SubaccountPerpetualPosition struct { - // The `Id` of the `Perpetual`. - PerpetualId uint32 `protobuf:"varint,1,opt,name=perpetual_id,json=perpetualId,proto3" json:"perpetual_id,omitempty"` - // The size of the position in base quantums. - Quantums uint64 `protobuf:"varint,2,opt,name=quantums,proto3" json:"quantums,omitempty"` -} - -func (m *SubaccountPerpetualPosition) Reset() { *m = SubaccountPerpetualPosition{} } -func (m *SubaccountPerpetualPosition) String() string { return proto.CompactTextString(m) } -func (*SubaccountPerpetualPosition) ProtoMessage() {} -func (*SubaccountPerpetualPosition) Descriptor() ([]byte, []int) { - return fileDescriptor_3365c195b25c5bc0, []int{17} -} -func (m *SubaccountPerpetualPosition) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *SubaccountPerpetualPosition) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_SubaccountPerpetualPosition.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalToSizedBuffer(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (m *SubaccountPerpetualPosition) XXX_Merge(src proto.Message) { - xxx_messageInfo_SubaccountPerpetualPosition.Merge(m, src) -} -func (m *SubaccountPerpetualPosition) XXX_Size() int { - return m.Size() -} -func (m *SubaccountPerpetualPosition) XXX_DiscardUnknown() { - xxx_messageInfo_SubaccountPerpetualPosition.DiscardUnknown(m) -} - -var xxx_messageInfo_SubaccountPerpetualPosition proto.InternalMessageInfo - -func (m *SubaccountPerpetualPosition) GetPerpetualId() uint32 { - if m != nil { - return m.PerpetualId - } - return 0 -} - -func (m *SubaccountPerpetualPosition) GetQuantums() uint64 { - if m != nil { - return m.Quantums - } - return 0 -} - -// SubaccountAssetPosition provides information on a subaccount's updated asset -// positions. -type SubaccountAssetPosition struct { - // The `Id` of the `Asset`. - AssetId uint32 `protobuf:"varint,1,opt,name=asset_id,json=assetId,proto3" json:"asset_id,omitempty"` - // The absolute size of the position in base quantums. - Quantums uint64 `protobuf:"varint,2,opt,name=quantums,proto3" json:"quantums,omitempty"` -} - -func (m *SubaccountAssetPosition) Reset() { *m = SubaccountAssetPosition{} } -func (m *SubaccountAssetPosition) String() string { return proto.CompactTextString(m) } -func (*SubaccountAssetPosition) ProtoMessage() {} -func (*SubaccountAssetPosition) Descriptor() ([]byte, []int) { - return fileDescriptor_3365c195b25c5bc0, []int{18} -} -func (m *SubaccountAssetPosition) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *SubaccountAssetPosition) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_SubaccountAssetPosition.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalToSizedBuffer(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (m *SubaccountAssetPosition) XXX_Merge(src proto.Message) { - xxx_messageInfo_SubaccountAssetPosition.Merge(m, src) -} -func (m *SubaccountAssetPosition) XXX_Size() int { - return m.Size() -} -func (m *SubaccountAssetPosition) XXX_DiscardUnknown() { - xxx_messageInfo_SubaccountAssetPosition.DiscardUnknown(m) -} - -var xxx_messageInfo_SubaccountAssetPosition proto.InternalMessageInfo - -func (m *SubaccountAssetPosition) GetAssetId() uint32 { - if m != nil { - return m.AssetId - } - return 0 -} - -func (m *SubaccountAssetPosition) GetQuantums() uint64 { - if m != nil { - return m.Quantums - } - return 0 -} - -// StreamSubaccountUpdate provides information on a subaccount update. Used in -// the full node GRPC stream. -type StreamSubaccountUpdate struct { - SubaccountId *types.SubaccountId `protobuf:"bytes,1,opt,name=subaccount_id,json=subaccountId,proto3" json:"subaccount_id,omitempty"` - // updated_perpetual_positions will each be for unique perpetuals. - UpdatedPerpetualPositions []*SubaccountPerpetualPosition `protobuf:"bytes,2,rep,name=updated_perpetual_positions,json=updatedPerpetualPositions,proto3" json:"updated_perpetual_positions,omitempty"` - // updated_asset_positions will each be for unique assets. - UpdatedAssetPositions []*SubaccountAssetPosition `protobuf:"bytes,3,rep,name=updated_asset_positions,json=updatedAssetPositions,proto3" json:"updated_asset_positions,omitempty"` - // Snapshot indicates if the response is from a snapshot of the subaccount. - // All updates should be ignored until snapshot is received. - // If the snapshot is true, then all previous entries should be - // discarded and the subaccount should be resynced. - // For a snapshot subaccount update, the `updated_perpetual_positions` and - // `updated_asset_positions` fields will contain the full state of the - // subaccount. - Snapshot bool `protobuf:"varint,4,opt,name=snapshot,proto3" json:"snapshot,omitempty"` -} - -func (m *StreamSubaccountUpdate) Reset() { *m = StreamSubaccountUpdate{} } -func (m *StreamSubaccountUpdate) String() string { return proto.CompactTextString(m) } -func (*StreamSubaccountUpdate) ProtoMessage() {} -func (*StreamSubaccountUpdate) Descriptor() ([]byte, []int) { - return fileDescriptor_3365c195b25c5bc0, []int{19} -} -func (m *StreamSubaccountUpdate) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *StreamSubaccountUpdate) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_StreamSubaccountUpdate.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalToSizedBuffer(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (m *StreamSubaccountUpdate) XXX_Merge(src proto.Message) { - xxx_messageInfo_StreamSubaccountUpdate.Merge(m, src) -} -func (m *StreamSubaccountUpdate) XXX_Size() int { - return m.Size() -} -func (m *StreamSubaccountUpdate) XXX_DiscardUnknown() { - xxx_messageInfo_StreamSubaccountUpdate.DiscardUnknown(m) -} - -var xxx_messageInfo_StreamSubaccountUpdate proto.InternalMessageInfo - -func (m *StreamSubaccountUpdate) GetSubaccountId() *types.SubaccountId { - if m != nil { - return m.SubaccountId - } - return nil -} - -func (m *StreamSubaccountUpdate) GetUpdatedPerpetualPositions() []*SubaccountPerpetualPosition { - if m != nil { - return m.UpdatedPerpetualPositions - } - return nil -} - -func (m *StreamSubaccountUpdate) GetUpdatedAssetPositions() []*SubaccountAssetPosition { - if m != nil { - return m.UpdatedAssetPositions - } - return nil -} - -func (m *StreamSubaccountUpdate) GetSnapshot() bool { - if m != nil { - return m.Snapshot - } - return false -} - // StreamOrderbookUpdate provides information on an orderbook update. Used in // the full node GRPC stream. type StreamOrderbookUpdate struct { @@ -1212,7 +1021,7 @@ func (m *StreamOrderbookUpdate) Reset() { *m = StreamOrderbookUpdate{} } func (m *StreamOrderbookUpdate) String() string { return proto.CompactTextString(m) } func (*StreamOrderbookUpdate) ProtoMessage() {} func (*StreamOrderbookUpdate) Descriptor() ([]byte, []int) { - return fileDescriptor_3365c195b25c5bc0, []int{20} + return fileDescriptor_3365c195b25c5bc0, []int{17} } func (m *StreamOrderbookUpdate) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1272,7 +1081,7 @@ func (m *StreamOrderbookFill) Reset() { *m = StreamOrderbookFill{} } func (m *StreamOrderbookFill) String() string { return proto.CompactTextString(m) } func (*StreamOrderbookFill) ProtoMessage() {} func (*StreamOrderbookFill) Descriptor() ([]byte, []int) { - return fileDescriptor_3365c195b25c5bc0, []int{21} + return fileDescriptor_3365c195b25c5bc0, []int{18} } func (m *StreamOrderbookFill) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1343,7 +1152,7 @@ func (m *StreamTakerOrder) Reset() { *m = StreamTakerOrder{} } func (m *StreamTakerOrder) String() string { return proto.CompactTextString(m) } func (*StreamTakerOrder) ProtoMessage() {} func (*StreamTakerOrder) Descriptor() ([]byte, []int) { - return fileDescriptor_3365c195b25c5bc0, []int{22} + return fileDescriptor_3365c195b25c5bc0, []int{19} } func (m *StreamTakerOrder) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1446,7 +1255,7 @@ func (m *StreamTakerOrderStatus) Reset() { *m = StreamTakerOrderStatus{} func (m *StreamTakerOrderStatus) String() string { return proto.CompactTextString(m) } func (*StreamTakerOrderStatus) ProtoMessage() {} func (*StreamTakerOrderStatus) Descriptor() ([]byte, []int) { - return fileDescriptor_3365c195b25c5bc0, []int{23} + return fileDescriptor_3365c195b25c5bc0, []int{20} } func (m *StreamTakerOrderStatus) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1515,9 +1324,6 @@ func init() { proto.RegisterType((*StreamOrderbookUpdatesRequest)(nil), "dydxprotocol.clob.StreamOrderbookUpdatesRequest") proto.RegisterType((*StreamOrderbookUpdatesResponse)(nil), "dydxprotocol.clob.StreamOrderbookUpdatesResponse") proto.RegisterType((*StreamUpdate)(nil), "dydxprotocol.clob.StreamUpdate") - proto.RegisterType((*SubaccountPerpetualPosition)(nil), "dydxprotocol.clob.SubaccountPerpetualPosition") - proto.RegisterType((*SubaccountAssetPosition)(nil), "dydxprotocol.clob.SubaccountAssetPosition") - proto.RegisterType((*StreamSubaccountUpdate)(nil), "dydxprotocol.clob.StreamSubaccountUpdate") proto.RegisterType((*StreamOrderbookUpdate)(nil), "dydxprotocol.clob.StreamOrderbookUpdate") proto.RegisterType((*StreamOrderbookFill)(nil), "dydxprotocol.clob.StreamOrderbookFill") proto.RegisterType((*StreamTakerOrder)(nil), "dydxprotocol.clob.StreamTakerOrder") @@ -1527,119 +1333,111 @@ func init() { func init() { proto.RegisterFile("dydxprotocol/clob/query.proto", fileDescriptor_3365c195b25c5bc0) } var fileDescriptor_3365c195b25c5bc0 = []byte{ - // 1778 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x58, 0x41, 0x6f, 0x1c, 0x49, - 0x15, 0x9e, 0x1e, 0x3b, 0xc9, 0xe4, 0x4d, 0x9c, 0x75, 0x2a, 0xeb, 0x64, 0x32, 0x76, 0xc6, 0x4e, - 0x2f, 0x71, 0xec, 0x2c, 0x99, 0x4e, 0xbc, 0xab, 0xd5, 0x92, 0xa0, 0x45, 0x76, 0x20, 0x71, 0x44, - 0xcc, 0xce, 0x76, 0xb2, 0x59, 0x0b, 0x22, 0xb5, 0x6a, 0xba, 0xcb, 0xe3, 0x96, 0xbb, 0xbb, 0xc6, - 0x5d, 0xd5, 0x23, 0x5b, 0x08, 0x21, 0x71, 0xd8, 0x0b, 0x20, 0xad, 0xc4, 0x81, 0x03, 0x12, 0x07, - 0x38, 0x23, 0x71, 0xe1, 0x88, 0x80, 0xdb, 0x1e, 0x23, 0x21, 0x24, 0x0e, 0x08, 0xa1, 0x84, 0x33, - 0xbf, 0x01, 0x75, 0x55, 0xf5, 0x4c, 0xf7, 0x74, 0xf7, 0x8c, 0xe3, 0x8b, 0x3d, 0xf5, 0xea, 0xbd, - 0xef, 0x7d, 0xef, 0xd5, 0xab, 0x57, 0x55, 0x0d, 0xd7, 0x9d, 0x63, 0xe7, 0xa8, 0x1f, 0x52, 0x4e, - 0x6d, 0xea, 0x19, 0xb6, 0x47, 0xbb, 0xc6, 0x61, 0x44, 0xc2, 0xe3, 0xb6, 0x90, 0xa1, 0x4b, 0xe9, - 0xe9, 0x76, 0x3c, 0xdd, 0x7c, 0xb7, 0x47, 0x7b, 0x54, 0x88, 0x8c, 0xf8, 0x97, 0x54, 0x6c, 0x2e, - 0xf5, 0x28, 0xed, 0x79, 0xc4, 0xc0, 0x7d, 0xd7, 0xc0, 0x41, 0x40, 0x39, 0xe6, 0x2e, 0x0d, 0x98, - 0x9a, 0xbd, 0x6d, 0x53, 0xe6, 0x53, 0x66, 0x74, 0x31, 0x23, 0x12, 0xdf, 0x18, 0xdc, 0xeb, 0x12, - 0x8e, 0xef, 0x19, 0x7d, 0xdc, 0x73, 0x03, 0xa1, 0xac, 0x74, 0x8d, 0x3c, 0xa3, 0xae, 0x47, 0xed, - 0x03, 0x2b, 0xc4, 0x9c, 0x58, 0x9e, 0xeb, 0xbb, 0xdc, 0xb2, 0x69, 0xb0, 0xe7, 0xf6, 0x94, 0xc1, - 0x8d, 0xbc, 0x41, 0xfc, 0xc7, 0xea, 0x63, 0x37, 0x54, 0x2a, 0x77, 0xf3, 0x2a, 0xe4, 0x30, 0x72, - 0xf9, 0xb1, 0xc5, 0x5d, 0x12, 0x16, 0x81, 0x16, 0xe4, 0x85, 0x86, 0x0e, 0x49, 0x00, 0x97, 0xf3, - 0xd3, 0x3e, 0xe6, 0xf6, 0x3e, 0x49, 0x22, 0x7e, 0x3f, 0xaf, 0xe0, 0xb9, 0x87, 0x91, 0xeb, 0xc8, - 0xbc, 0x64, 0x9d, 0x2d, 0x16, 0xa0, 0x91, 0x81, 0x9a, 0xfc, 0x24, 0x33, 0xe9, 0x06, 0x0e, 0x39, - 0x22, 0xa1, 0x41, 0xf7, 0xf6, 0x2c, 0x7b, 0x1f, 0xbb, 0x81, 0x15, 0xf5, 0x1d, 0xcc, 0x09, 0xcb, - 0x4b, 0x94, 0xfd, 0x7a, 0xc6, 0x9e, 0x45, 0x5d, 0x6c, 0xdb, 0x34, 0x0a, 0x38, 0x4b, 0xfd, 0x96, - 0xaa, 0xfa, 0x3a, 0x5c, 0xfd, 0x2c, 0x5e, 0x9c, 0xc7, 0x84, 0x3f, 0xf4, 0x68, 0xb7, 0x83, 0xdd, - 0xd0, 0x24, 0x87, 0x11, 0x61, 0x1c, 0x5d, 0x84, 0xaa, 0xeb, 0x34, 0xb4, 0x15, 0x6d, 0x6d, 0xce, - 0xac, 0xba, 0x8e, 0xfe, 0x05, 0x2c, 0x08, 0xd5, 0x91, 0x1e, 0xeb, 0xd3, 0x80, 0x11, 0xf4, 0x09, - 0x9c, 0x1f, 0x66, 0x5f, 0xe8, 0xd7, 0x37, 0x16, 0xdb, 0xb9, 0x2a, 0x6a, 0x27, 0x76, 0x5b, 0xb3, - 0x5f, 0xff, 0x7b, 0xb9, 0x62, 0xd6, 0x6c, 0x35, 0xd6, 0xb1, 0xe2, 0xb0, 0xe9, 0x79, 0xe3, 0x1c, - 0x1e, 0x01, 0x8c, 0xaa, 0x45, 0x61, 0xaf, 0xb6, 0x65, 0x69, 0xb5, 0xe3, 0xd2, 0x6a, 0xcb, 0xd2, - 0x55, 0xa5, 0xd5, 0xee, 0xe0, 0x1e, 0x51, 0xb6, 0x66, 0xca, 0x52, 0xff, 0xbd, 0x06, 0x8d, 0x0c, - 0xf9, 0x4d, 0xcf, 0x2b, 0xe3, 0x3f, 0xf3, 0x96, 0xfc, 0xd1, 0xe3, 0x0c, 0xc9, 0xaa, 0x20, 0x79, - 0x6b, 0x2a, 0x49, 0xe9, 0x3c, 0xc3, 0xf2, 0x5f, 0x1a, 0x2c, 0xef, 0x90, 0xc1, 0x0f, 0xa8, 0x43, - 0x9e, 0xd3, 0xf8, 0xef, 0x43, 0xec, 0xd9, 0x91, 0x27, 0x26, 0x93, 0x8c, 0xbc, 0x84, 0x2b, 0x72, - 0x6f, 0xf4, 0x43, 0xda, 0xa7, 0x8c, 0x84, 0x96, 0xaa, 0xc2, 0x61, 0x76, 0xf2, 0xcc, 0x5f, 0x60, - 0x2f, 0xae, 0x42, 0x1a, 0xee, 0x90, 0xc1, 0x8e, 0xd4, 0x36, 0xdf, 0x15, 0x28, 0x1d, 0x05, 0xa2, - 0xa4, 0xe8, 0x47, 0xb0, 0x30, 0x48, 0x94, 0x2d, 0x9f, 0x0c, 0x2c, 0x9f, 0xf0, 0xd0, 0xb5, 0xd9, - 0x30, 0xaa, 0x3c, 0x78, 0x86, 0xf0, 0x8e, 0x54, 0x37, 0x2f, 0x0f, 0xd2, 0x2e, 0xa5, 0x50, 0xff, - 0x9f, 0x06, 0x2b, 0xe5, 0xe1, 0xa9, 0xc5, 0xe8, 0xc1, 0xb9, 0x90, 0xb0, 0xc8, 0xe3, 0x4c, 0x2d, - 0xc5, 0xe3, 0x69, 0x3e, 0x0b, 0x50, 0x62, 0x85, 0xcd, 0xc0, 0x79, 0x41, 0xbd, 0xc8, 0x27, 0x1d, - 0x12, 0xc6, 0x4b, 0xa7, 0x96, 0x2d, 0x41, 0x6f, 0x62, 0xb8, 0x5c, 0xa0, 0x85, 0x56, 0xe0, 0xc2, - 0xb0, 0x18, 0xac, 0x61, 0xfd, 0x43, 0xb2, 0xd8, 0x4f, 0x1c, 0x34, 0x0f, 0x33, 0x3e, 0x19, 0x88, - 0x8c, 0x54, 0xcd, 0xf8, 0x27, 0xba, 0x02, 0x67, 0x07, 0x02, 0xa4, 0x31, 0xb3, 0xa2, 0xad, 0xcd, - 0x9a, 0x6a, 0xa4, 0xdf, 0x86, 0x35, 0x51, 0x74, 0xdf, 0x13, 0x8d, 0xe7, 0xb9, 0x4b, 0xc2, 0xa7, - 0x71, 0xdb, 0x79, 0x28, 0x1a, 0x41, 0x14, 0xa6, 0xd7, 0x55, 0xff, 0x8d, 0x06, 0xeb, 0x27, 0x50, - 0x56, 0x59, 0x0a, 0xa0, 0x51, 0xd6, 0xcd, 0x54, 0x1d, 0x18, 0x05, 0x69, 0x9b, 0x04, 0xad, 0xd2, - 0xb3, 0x40, 0x8a, 0x74, 0xf4, 0x75, 0xb8, 0x25, 0xc8, 0x6d, 0xc5, 0x45, 0x63, 0x62, 0x4e, 0xca, - 0x03, 0xf9, 0xb5, 0xa6, 0xa2, 0x9e, 0xa8, 0xab, 0xe2, 0x38, 0x80, 0xab, 0x25, 0x9d, 0x5e, 0x85, - 0xd1, 0x2e, 0x08, 0x63, 0x02, 0xb0, 0x8a, 0x42, 0x16, 0xf7, 0x98, 0x8a, 0xbe, 0x0b, 0xd7, 0x04, - 0xb1, 0x67, 0x1c, 0x73, 0xb2, 0x17, 0x79, 0x9f, 0xc6, 0xdd, 0x3d, 0xd9, 0x57, 0x0f, 0xa0, 0x26, - 0xba, 0x7d, 0xb2, 0xe6, 0xf5, 0x8d, 0x66, 0x81, 0x6b, 0x61, 0xf2, 0xc4, 0x49, 0x6a, 0x89, 0xca, - 0xa1, 0xfe, 0x27, 0x0d, 0x9a, 0x45, 0xd0, 0x2a, 0xca, 0x5d, 0x78, 0x47, 0x62, 0xf7, 0x3d, 0x6c, - 0x13, 0x9f, 0x04, 0x5c, 0xb9, 0x58, 0x2f, 0x70, 0xf1, 0x94, 0x06, 0xbd, 0xe7, 0x24, 0xf4, 0x05, - 0x44, 0x27, 0x31, 0x50, 0x1e, 0x2f, 0xd2, 0x8c, 0x14, 0x2d, 0x43, 0x7d, 0xcf, 0xf5, 0x3c, 0x0b, - 0xfb, 0x71, 0x4f, 0x17, 0x35, 0x39, 0x6b, 0x42, 0x2c, 0xda, 0x14, 0x12, 0xb4, 0x04, 0xe7, 0x79, - 0xe8, 0xf6, 0x7a, 0x24, 0x24, 0x8e, 0xa8, 0xce, 0x9a, 0x39, 0x12, 0xe8, 0xb7, 0xe0, 0xa6, 0xa0, - 0xfd, 0x34, 0x75, 0x4e, 0x15, 0x2e, 0xea, 0x97, 0x1a, 0xac, 0x4e, 0xd3, 0x54, 0xc1, 0xbe, 0x84, - 0xcb, 0x05, 0xc7, 0x9e, 0x0a, 0xf8, 0x66, 0x51, 0xc0, 0x39, 0x48, 0x15, 0x2c, 0xf2, 0x72, 0x33, - 0xfa, 0x57, 0x1a, 0x5c, 0x7f, 0xc6, 0x43, 0x82, 0x65, 0x7e, 0xba, 0x94, 0x1e, 0x7c, 0x2e, 0xcf, - 0xbe, 0x64, 0x21, 0xf3, 0x1b, 0x78, 0x66, 0x6c, 0x03, 0xef, 0xc0, 0xc5, 0xd1, 0x39, 0x68, 0xb9, - 0x4e, 0xdc, 0xdd, 0x66, 0xf2, 0xad, 0x33, 0x75, 0x6e, 0xb6, 0x9f, 0x0d, 0x7f, 0x3f, 0x71, 0xcc, - 0x39, 0x96, 0x1a, 0x31, 0x1d, 0x43, 0xab, 0x8c, 0x91, 0x4a, 0xc9, 0x77, 0xe0, 0x9c, 0x3a, 0xa0, - 0x55, 0x4f, 0x5b, 0x2e, 0x48, 0x83, 0xc4, 0x90, 0xa6, 0x49, 0x7d, 0x29, 0x2b, 0xfd, 0x77, 0x33, - 0x70, 0x21, 0x3d, 0x8f, 0x3e, 0x87, 0x79, 0x9a, 0x78, 0x53, 0x87, 0xbf, 0xca, 0xf0, 0x5a, 0x29, - 0xf4, 0x18, 0xbd, 0xed, 0x8a, 0xf9, 0x0e, 0xcd, 0x8a, 0xe2, 0x93, 0x4c, 0x16, 0x6a, 0x5c, 0x41, - 0xaa, 0xe7, 0xaf, 0x4e, 0x07, 0x7c, 0xe4, 0x7a, 0xde, 0x76, 0xc5, 0x3c, 0x2f, 0x6c, 0xe3, 0x01, - 0x7a, 0x04, 0x75, 0x8e, 0x0f, 0x48, 0x68, 0x09, 0x91, 0x28, 0xbc, 0xfa, 0xc6, 0x7b, 0xa5, 0x48, - 0xcf, 0x63, 0x5d, 0x01, 0xb7, 0x5d, 0x31, 0x81, 0x0f, 0x47, 0x68, 0x17, 0x2e, 0xa5, 0x96, 0x4a, - 0x05, 0x3a, 0x5b, 0xba, 0x77, 0x24, 0xda, 0x68, 0xb1, 0x86, 0x91, 0xce, 0xb3, 0x31, 0x19, 0xba, - 0x01, 0x17, 0x64, 0xe7, 0xd9, 0x27, 0x6e, 0x6f, 0x9f, 0x37, 0xce, 0x88, 0x3e, 0x5f, 0x17, 0xb2, - 0x6d, 0x21, 0x42, 0x8b, 0x70, 0x9e, 0x1c, 0x11, 0xdb, 0xf2, 0xa9, 0x43, 0x1a, 0x67, 0xc5, 0x7c, - 0x2d, 0x16, 0xec, 0x50, 0x87, 0x6c, 0xcd, 0xc3, 0x45, 0x49, 0xc7, 0xf2, 0x09, 0x63, 0xb8, 0x47, - 0xf4, 0x97, 0xb0, 0x38, 0xf2, 0xdc, 0x21, 0x61, 0x9f, 0xf0, 0x08, 0x7b, 0x1d, 0xca, 0xdc, 0xb8, - 0x80, 0x63, 0x87, 0xfd, 0x44, 0x38, 0x3a, 0x58, 0xea, 0x43, 0xd9, 0x13, 0x07, 0x35, 0xa1, 0x76, - 0x18, 0xe1, 0x80, 0x47, 0x3e, 0x53, 0x5b, 0x79, 0x38, 0xd6, 0x3b, 0x70, 0x75, 0x84, 0xbe, 0xc9, - 0x18, 0xe1, 0x43, 0xe4, 0x6b, 0x50, 0xc3, 0xb1, 0x60, 0x84, 0x7a, 0x4e, 0x8c, 0xa7, 0x20, 0xfe, - 0xa3, 0x0a, 0x57, 0x8a, 0x13, 0x86, 0xbe, 0x0f, 0x73, 0x99, 0x1d, 0x52, 0x7c, 0xb7, 0x28, 0xdd, - 0x20, 0x17, 0xd2, 0x1b, 0x04, 0x05, 0xb0, 0x28, 0x33, 0xe5, 0x58, 0xa3, 0x04, 0xf4, 0x15, 0xf9, - 0x64, 0xef, 0x15, 0xf5, 0xf9, 0x09, 0xd9, 0x34, 0xaf, 0x29, 0xc8, 0xdc, 0x0c, 0x43, 0x5d, 0xb8, - 0x9a, 0xf8, 0x93, 0x69, 0x19, 0xf9, 0x9a, 0x11, 0xbe, 0x6e, 0x4f, 0xf4, 0x95, 0xc9, 0xad, 0xb9, - 0xa0, 0xa0, 0x32, 0x52, 0x16, 0xe7, 0x95, 0x05, 0xb8, 0xcf, 0xf6, 0x29, 0x17, 0xe5, 0x58, 0x33, - 0x87, 0x63, 0xfd, 0x97, 0x1a, 0x2c, 0x14, 0xee, 0x38, 0xb4, 0x3b, 0xde, 0x07, 0x3e, 0xce, 0x32, - 0x51, 0x37, 0xfd, 0x76, 0xfe, 0x5e, 0xff, 0xe9, 0xde, 0xde, 0xc3, 0x58, 0x20, 0x81, 0x5e, 0xdc, - 0x1b, 0x6b, 0x10, 0x19, 0x3e, 0xd5, 0x31, 0x3e, 0x7f, 0xd0, 0xe0, 0x72, 0xc1, 0x86, 0x45, 0x0f, - 0x40, 0x34, 0x45, 0x79, 0x7f, 0x54, 0x2b, 0xbc, 0x54, 0x72, 0xef, 0x15, 0xf7, 0x43, 0x53, 0x5c, - 0x93, 0xc5, 0x4f, 0xf4, 0x11, 0x9c, 0x15, 0x5b, 0x3b, 0x59, 0xbf, 0x46, 0xd9, 0x61, 0xa9, 0x98, - 0x2a, 0xed, 0x78, 0x17, 0xa4, 0x0e, 0x2c, 0xb9, 0x22, 0xb3, 0x66, 0x7d, 0x74, 0x62, 0x31, 0xfd, - 0xcb, 0x2a, 0xcc, 0x8f, 0xb7, 0x05, 0x74, 0x17, 0xce, 0xc8, 0x56, 0x22, 0x79, 0x96, 0xba, 0xdb, - 0xae, 0x98, 0x52, 0x31, 0x6e, 0x1d, 0xa9, 0xf3, 0x43, 0x35, 0xa2, 0xea, 0x94, 0xd6, 0x91, 0x3a, - 0x8b, 0x12, 0xb8, 0x79, 0x6f, 0x4c, 0x86, 0xbe, 0x00, 0x94, 0x6a, 0x6e, 0x16, 0xe3, 0x98, 0x47, - 0x4c, 0xf5, 0xb8, 0xf5, 0x13, 0xf4, 0xb8, 0x67, 0xc2, 0xc0, 0x9c, 0xe7, 0x63, 0x92, 0xad, 0xb9, - 0x4c, 0xd7, 0xd4, 0xff, 0xa8, 0x25, 0x1b, 0x74, 0xdc, 0x36, 0x4e, 0x63, 0xc6, 0xb9, 0x6a, 0x26, - 0x34, 0xa5, 0x72, 0x07, 0x50, 0x48, 0x7c, 0xec, 0x06, 0x6e, 0xd0, 0xb3, 0xc6, 0x9a, 0xc0, 0xa5, - 0xe1, 0xcc, 0x67, 0x6a, 0x02, 0x7d, 0x17, 0x5a, 0xb4, 0xcf, 0x5d, 0xdf, 0x65, 0xdc, 0xb5, 0xb1, - 0xe7, 0x1d, 0x8b, 0x33, 0x80, 0x38, 0x23, 0x53, 0x79, 0xb7, 0x5d, 0xca, 0x6a, 0x3d, 0x12, 0x4a, - 0x09, 0xca, 0xc6, 0x6f, 0x01, 0xce, 0x88, 0x7b, 0x02, 0xfa, 0xb9, 0x06, 0xb5, 0xe4, 0xc5, 0x84, - 0x8a, 0x76, 0x5c, 0xc9, 0xb3, 0xb3, 0xb9, 0x56, 0xa6, 0x3b, 0xfe, 0xee, 0xd4, 0xd7, 0x7f, 0xf6, - 0xf7, 0xff, 0xfe, 0xaa, 0xfa, 0x1e, 0xba, 0x61, 0x4c, 0xf8, 0x1c, 0x60, 0xfc, 0xd8, 0x75, 0x7e, - 0x82, 0x7e, 0xa1, 0x41, 0x3d, 0xf5, 0xf4, 0x2b, 0x27, 0x94, 0x7f, 0x83, 0x36, 0xdf, 0x9f, 0x46, - 0x28, 0xf5, 0x96, 0xd4, 0xbf, 0x21, 0x38, 0xb5, 0xd0, 0xd2, 0x24, 0x4e, 0xe8, 0x2f, 0x1a, 0x34, - 0xca, 0xde, 0x30, 0x68, 0xe3, 0xad, 0x1e, 0x3c, 0x92, 0xe3, 0x07, 0xa7, 0x78, 0x24, 0xe9, 0xf7, - 0x05, 0xd7, 0x0f, 0xef, 0x6b, 0xb7, 0x75, 0xc3, 0x28, 0xfc, 0x1e, 0x61, 0x05, 0xd4, 0x21, 0x16, - 0xa7, 0xf2, 0xbf, 0x9d, 0x22, 0xf9, 0x37, 0x0d, 0x96, 0x26, 0x3d, 0x27, 0xd0, 0x83, 0xb2, 0xac, - 0x9d, 0xe0, 0x31, 0xd4, 0xfc, 0xf6, 0xe9, 0x8c, 0x55, 0x5c, 0xab, 0x22, 0xae, 0x15, 0xd4, 0x32, - 0x26, 0x7e, 0x03, 0x42, 0x7f, 0xd6, 0x60, 0x71, 0xc2, 0x5b, 0x02, 0xdd, 0x2f, 0x63, 0x31, 0xfd, - 0x15, 0xd4, 0x7c, 0x70, 0x2a, 0x5b, 0x15, 0xc0, 0x4d, 0x11, 0xc0, 0x32, 0xba, 0x3e, 0xf1, 0xc3, - 0x18, 0xfa, 0xab, 0x06, 0xd7, 0x4a, 0xef, 0xe3, 0xe8, 0xe3, 0x32, 0x06, 0xd3, 0x2e, 0xfb, 0xcd, - 0x6f, 0x9d, 0xc2, 0x52, 0x31, 0x6f, 0x0b, 0xe6, 0x6b, 0x68, 0xd5, 0x38, 0xd1, 0xc7, 0x30, 0x14, - 0xc0, 0x5c, 0xe6, 0xc9, 0x84, 0xbe, 0x59, 0xe6, 0xbb, 0xe8, 0xd1, 0xd6, 0xbc, 0x73, 0x42, 0x6d, - 0xc5, 0xae, 0x82, 0x7e, 0x9a, 0x74, 0xd4, 0xf1, 0xbb, 0x3a, 0xba, 0x7b, 0xd2, 0x7b, 0x73, 0xf2, - 0xd0, 0x68, 0xde, 0x7b, 0x0b, 0x0b, 0x49, 0xe0, 0xae, 0xb6, 0xd5, 0xf9, 0xfa, 0x75, 0x4b, 0x7b, - 0xf5, 0xba, 0xa5, 0xfd, 0xe7, 0x75, 0x4b, 0xfb, 0xea, 0x4d, 0xab, 0xf2, 0xea, 0x4d, 0xab, 0xf2, - 0xcf, 0x37, 0xad, 0xca, 0x0f, 0x3f, 0xea, 0xb9, 0x7c, 0x3f, 0xea, 0xb6, 0x6d, 0xea, 0x67, 0x93, - 0x37, 0xf8, 0xf0, 0x8e, 0xb8, 0x0c, 0x18, 0x43, 0xc9, 0x91, 0x4c, 0x28, 0x3f, 0xee, 0x13, 0xd6, - 0x3d, 0x2b, 0xc4, 0x1f, 0xfc, 0x3f, 0x00, 0x00, 0xff, 0xff, 0x1a, 0x19, 0x57, 0xae, 0xd7, 0x15, - 0x00, 0x00, + // 1656 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x58, 0x41, 0x4f, 0xdc, 0x46, + 0x14, 0x5e, 0xb3, 0x84, 0xc0, 0xdb, 0x40, 0x60, 0x08, 0xc9, 0x66, 0x21, 0x0b, 0x71, 0x1a, 0xb2, + 0x90, 0x66, 0x0d, 0x24, 0x8a, 0xd2, 0x50, 0xa5, 0x02, 0x5a, 0x42, 0xa4, 0xd0, 0x10, 0x43, 0x12, + 0xd4, 0x46, 0xb2, 0xbc, 0xf6, 0xb0, 0x58, 0xd8, 0x9e, 0xc5, 0x1e, 0xaf, 0x40, 0x55, 0x55, 0xa9, + 0x87, 0x5c, 0xda, 0x4a, 0x91, 0x7a, 0xe8, 0xa1, 0x52, 0x2f, 0x3d, 0xf5, 0x50, 0xa9, 0x97, 0x1e, + 0xab, 0xb6, 0xb7, 0x1c, 0x23, 0xf5, 0xd2, 0x43, 0x55, 0x55, 0x49, 0xcf, 0xfd, 0x0d, 0x95, 0x67, + 0xc6, 0x8b, 0x77, 0xd7, 0x5e, 0x08, 0x17, 0xb0, 0xdf, 0xbc, 0xf7, 0xe6, 0x7b, 0xef, 0x7d, 0xf3, + 0xe6, 0x79, 0xe1, 0x82, 0xb9, 0x6f, 0xee, 0xd5, 0x3c, 0x42, 0x89, 0x41, 0x6c, 0xc5, 0xb0, 0x49, + 0x45, 0xd9, 0x0d, 0xb0, 0xb7, 0x5f, 0x66, 0x32, 0x34, 0x14, 0x5f, 0x2e, 0x87, 0xcb, 0x85, 0x33, + 0x55, 0x52, 0x25, 0x4c, 0xa4, 0x84, 0x4f, 0x5c, 0xb1, 0x30, 0x56, 0x25, 0xa4, 0x6a, 0x63, 0x45, + 0xaf, 0x59, 0x8a, 0xee, 0xba, 0x84, 0xea, 0xd4, 0x22, 0xae, 0x2f, 0x56, 0xa7, 0x0d, 0xe2, 0x3b, + 0xc4, 0x57, 0x2a, 0xba, 0x8f, 0xb9, 0x7f, 0xa5, 0x3e, 0x5b, 0xc1, 0x54, 0x9f, 0x55, 0x6a, 0x7a, + 0xd5, 0x72, 0x99, 0xb2, 0xd0, 0x55, 0xda, 0x11, 0x55, 0x6c, 0x62, 0xec, 0x68, 0x9e, 0x4e, 0xb1, + 0x66, 0x5b, 0x8e, 0x45, 0x35, 0x83, 0xb8, 0x5b, 0x56, 0x55, 0x18, 0x5c, 0x6c, 0x37, 0x08, 0xff, + 0x68, 0x35, 0xdd, 0xf2, 0x84, 0xca, 0x4c, 0xbb, 0x0a, 0xde, 0x0d, 0x2c, 0xba, 0xaf, 0x51, 0x0b, + 0x7b, 0x49, 0x4e, 0x13, 0xf2, 0x42, 0x3c, 0x13, 0x47, 0x0e, 0xc7, 0xdb, 0x97, 0x1d, 0x9d, 0x1a, + 0xdb, 0x38, 0x8a, 0xf8, 0x6a, 0xbb, 0x82, 0x6d, 0xed, 0x06, 0x96, 0xc9, 0xf3, 0xd2, 0xbc, 0xd9, + 0x68, 0x82, 0x37, 0x5c, 0x17, 0x8b, 0x77, 0x9a, 0x16, 0x2d, 0xd7, 0xc4, 0x7b, 0xd8, 0x53, 0xc8, + 0xd6, 0x96, 0x66, 0x6c, 0xeb, 0x96, 0xab, 0x05, 0x35, 0x53, 0xa7, 0xd8, 0x6f, 0x97, 0x08, 0xfb, + 0x52, 0x93, 0xbd, 0x1f, 0x54, 0x74, 0xc3, 0x20, 0x81, 0x4b, 0x7d, 0xc5, 0xa7, 0x1e, 0xd6, 0x1d, + 0xcb, 0x8d, 0x60, 0x4c, 0xa5, 0x6b, 0x36, 0x9e, 0xb9, 0xaa, 0x3c, 0x05, 0xe7, 0x1e, 0x86, 0x65, + 0xbc, 0x8b, 0xe9, 0x92, 0x4d, 0x2a, 0x6b, 0xba, 0xe5, 0xa9, 0x78, 0x37, 0xc0, 0x3e, 0x45, 0x03, + 0xd0, 0x65, 0x99, 0x79, 0x69, 0x42, 0x2a, 0xf5, 0xab, 0x5d, 0x96, 0x29, 0x3f, 0x81, 0x11, 0xa6, + 0x7a, 0xa0, 0xe7, 0xd7, 0x88, 0xeb, 0x63, 0x74, 0x07, 0xfa, 0x1a, 0x75, 0x62, 0xfa, 0xb9, 0xb9, + 0xd1, 0x72, 0x1b, 0xdf, 0xca, 0x91, 0xdd, 0x62, 0xf7, 0x8b, 0xbf, 0xc7, 0x33, 0x6a, 0xaf, 0x21, + 0xde, 0x65, 0x5d, 0x60, 0x58, 0xb0, 0xed, 0x56, 0x0c, 0xcb, 0x00, 0x07, 0xbc, 0x12, 0xbe, 0x27, + 0xcb, 0x9c, 0x84, 0xe5, 0x90, 0x84, 0x65, 0x4e, 0x72, 0x41, 0xc2, 0xf2, 0x9a, 0x5e, 0xc5, 0xc2, + 0x56, 0x8d, 0x59, 0xca, 0xdf, 0x4b, 0x90, 0x6f, 0x02, 0xbf, 0x60, 0xdb, 0x69, 0xf8, 0xb3, 0x6f, + 0x88, 0x1f, 0xdd, 0x6d, 0x02, 0xd9, 0xc5, 0x40, 0x5e, 0x39, 0x14, 0x24, 0xdf, 0xbc, 0x09, 0xe5, + 0x5f, 0x12, 0x8c, 0xaf, 0xe2, 0xfa, 0x87, 0xc4, 0xc4, 0x1b, 0x24, 0xfc, 0xbb, 0xa4, 0xdb, 0x46, + 0x60, 0xb3, 0xc5, 0x28, 0x23, 0x4f, 0xe1, 0x2c, 0x3f, 0x45, 0x35, 0x8f, 0xd4, 0x88, 0x8f, 0x3d, + 0x4d, 0xf0, 0xb5, 0x91, 0x9d, 0x76, 0xe4, 0x8f, 0x75, 0x3b, 0xe4, 0x2b, 0xf1, 0x56, 0x71, 0x7d, + 0x95, 0x6b, 0xab, 0x67, 0x98, 0x97, 0x35, 0xe1, 0x44, 0x48, 0xd1, 0xc7, 0x30, 0x52, 0x8f, 0x94, + 0x35, 0x07, 0xd7, 0x35, 0x07, 0x53, 0xcf, 0x32, 0xfc, 0x46, 0x54, 0xed, 0xce, 0x9b, 0x00, 0xaf, + 0x72, 0x75, 0x75, 0xb8, 0x1e, 0xdf, 0x92, 0x0b, 0xe5, 0xff, 0x24, 0x98, 0x48, 0x0f, 0x4f, 0x14, + 0xa3, 0x0a, 0x27, 0x3d, 0xec, 0x07, 0x36, 0xf5, 0x45, 0x29, 0xee, 0x1e, 0xb6, 0x67, 0x82, 0x97, + 0x50, 0x61, 0xc1, 0x35, 0x1f, 0x13, 0x3b, 0x70, 0xf0, 0x1a, 0xf6, 0xc2, 0xd2, 0x89, 0xb2, 0x45, + 0xde, 0x0b, 0x3a, 0x0c, 0x27, 0x68, 0xa1, 0x09, 0x38, 0xd5, 0x20, 0x83, 0xd6, 0xe0, 0x3f, 0x44, + 0xc5, 0xbe, 0x67, 0xa2, 0x41, 0xc8, 0x3a, 0xb8, 0xce, 0x32, 0xd2, 0xa5, 0x86, 0x8f, 0xe8, 0x2c, + 0xf4, 0xd4, 0x99, 0x93, 0x7c, 0x76, 0x42, 0x2a, 0x75, 0xab, 0xe2, 0x4d, 0x9e, 0x86, 0x12, 0x23, + 0xdd, 0x07, 0xac, 0x45, 0x6d, 0x58, 0xd8, 0xbb, 0x1f, 0x36, 0xa8, 0x25, 0xd6, 0x32, 0x02, 0x2f, + 0x5e, 0x57, 0xf9, 0x5b, 0x09, 0xa6, 0x8e, 0xa0, 0x2c, 0xb2, 0xe4, 0x42, 0x3e, 0xad, 0xef, 0x09, + 0x1e, 0x28, 0x09, 0x69, 0xeb, 0xe4, 0x5a, 0xa4, 0x67, 0x04, 0x27, 0xe9, 0xc8, 0x53, 0x70, 0x85, + 0x81, 0x5b, 0x0c, 0x49, 0xa3, 0xea, 0x14, 0xa7, 0x07, 0xf2, 0x8d, 0x24, 0xa2, 0xee, 0xa8, 0x2b, + 0xe2, 0xd8, 0x81, 0x73, 0x29, 0x77, 0x82, 0x08, 0xa3, 0x9c, 0x10, 0x46, 0x07, 0xc7, 0x22, 0x0a, + 0x4e, 0xee, 0x16, 0x15, 0x79, 0x13, 0xce, 0x33, 0x60, 0xeb, 0x54, 0xa7, 0x78, 0x2b, 0xb0, 0x1f, + 0x84, 0xf7, 0x40, 0x74, 0xae, 0xe6, 0xa1, 0x97, 0xdd, 0x0b, 0x51, 0xcd, 0x73, 0x73, 0x85, 0x84, + 0xad, 0x99, 0xc9, 0x3d, 0x33, 0xe2, 0x12, 0xe1, 0xaf, 0xf2, 0xcf, 0x12, 0x14, 0x92, 0x5c, 0x8b, + 0x28, 0x37, 0xe1, 0x34, 0xf7, 0x5d, 0xb3, 0x75, 0x03, 0x3b, 0xd8, 0xa5, 0x62, 0x8b, 0xa9, 0x84, + 0x2d, 0xee, 0x13, 0xb7, 0xba, 0x81, 0x3d, 0x87, 0xb9, 0x58, 0x8b, 0x0c, 0xc4, 0x8e, 0x03, 0xa4, + 0x49, 0x8a, 0xc6, 0x21, 0xb7, 0x65, 0xd9, 0xb6, 0xa6, 0x3b, 0x61, 0x4f, 0x67, 0x9c, 0xec, 0x56, + 0x21, 0x14, 0x2d, 0x30, 0x09, 0x1a, 0x83, 0x3e, 0xea, 0x59, 0xd5, 0x2a, 0xf6, 0xb0, 0xc9, 0xd8, + 0xd9, 0xab, 0x1e, 0x08, 0xe4, 0x2b, 0x70, 0x99, 0xc1, 0xbe, 0x1f, 0xbb, 0xd1, 0x12, 0x8b, 0xfa, + 0x4c, 0x82, 0xc9, 0xc3, 0x34, 0x45, 0xb0, 0x4f, 0x61, 0x38, 0xe1, 0x82, 0x14, 0x01, 0x5f, 0x4e, + 0x0a, 0xb8, 0xcd, 0xa5, 0x08, 0x16, 0xd9, 0x6d, 0x2b, 0xf2, 0x73, 0x09, 0x2e, 0xac, 0xb3, 0xeb, + 0x8e, 0xe5, 0xa7, 0x42, 0xc8, 0xce, 0x23, 0x7e, 0x4b, 0x46, 0x85, 0x6c, 0x3f, 0xc0, 0xd9, 0x96, + 0x03, 0xbc, 0x0a, 0x03, 0x07, 0xf7, 0xa0, 0x66, 0x99, 0x61, 0x77, 0xcb, 0xb6, 0xb7, 0xce, 0xd8, + 0xbd, 0x59, 0x5e, 0x6f, 0x3c, 0xdf, 0x33, 0xd5, 0x7e, 0x3f, 0xf6, 0xe6, 0xcb, 0x3a, 0x14, 0xd3, + 0x10, 0x89, 0x94, 0xbc, 0x07, 0x27, 0xc5, 0x55, 0x2e, 0x7a, 0xda, 0x78, 0x42, 0x1a, 0xb8, 0x0f, + 0x6e, 0x1a, 0xf1, 0x4b, 0x58, 0xc9, 0x3f, 0x64, 0xe1, 0x54, 0x7c, 0x1d, 0x3d, 0x82, 0x41, 0x12, + 0xed, 0x26, 0xc6, 0x04, 0x91, 0xe1, 0x52, 0xaa, 0xeb, 0x16, 0x78, 0x2b, 0x19, 0xf5, 0x34, 0x69, + 0x16, 0x85, 0x37, 0x19, 0x27, 0x6a, 0xc8, 0x20, 0xd1, 0xf3, 0x27, 0x0f, 0x77, 0xb8, 0x6c, 0xd9, + 0xf6, 0x4a, 0x46, 0xed, 0x63, 0xb6, 0xe1, 0x0b, 0x5a, 0x86, 0x1c, 0xd5, 0x77, 0xb0, 0xa7, 0x31, + 0x11, 0x23, 0x5e, 0x6e, 0xee, 0x52, 0xaa, 0xa7, 0x8d, 0x50, 0x97, 0xb9, 0x5b, 0xc9, 0xa8, 0x40, + 0x1b, 0x6f, 0x48, 0x83, 0xa1, 0x58, 0xa9, 0x44, 0xa0, 0xdd, 0xcc, 0xdb, 0x4c, 0x87, 0x6a, 0x31, + 0xa7, 0x07, 0x35, 0x6b, 0x04, 0x3c, 0xe8, 0xb7, 0xc8, 0xd0, 0x45, 0x38, 0xc5, 0x1b, 0xd0, 0x36, + 0xb6, 0xaa, 0xdb, 0x34, 0x7f, 0x82, 0xb5, 0xfb, 0x1c, 0x93, 0xad, 0x30, 0x11, 0x1a, 0x85, 0x3e, + 0xbc, 0x87, 0x0d, 0xcd, 0x21, 0x26, 0xce, 0xf7, 0xb0, 0xf5, 0xde, 0x50, 0xb0, 0x4a, 0x4c, 0xbc, + 0x38, 0x08, 0x03, 0x1c, 0x95, 0xe6, 0x60, 0xdf, 0xd7, 0xab, 0x58, 0xfe, 0x4a, 0x82, 0x91, 0xc4, + 0x84, 0xa3, 0xcd, 0x56, 0x1a, 0xdc, 0x6a, 0x0e, 0x41, 0x8c, 0x84, 0xe5, 0xf6, 0x01, 0xf0, 0xc1, + 0xd6, 0xd6, 0x52, 0x28, 0xe0, 0x8e, 0x1e, 0xcf, 0xb6, 0xf0, 0x03, 0x15, 0xa0, 0xd7, 0x77, 0xf5, + 0x9a, 0xbf, 0x4d, 0x78, 0x0f, 0xe8, 0x55, 0x1b, 0xef, 0xf2, 0x8f, 0x12, 0x0c, 0x27, 0xd4, 0x0b, + 0xcd, 0x03, 0x3b, 0x13, 0x7c, 0x7c, 0x10, 0xe4, 0x19, 0x4b, 0x19, 0x7b, 0xd8, 0x78, 0xa0, 0xb2, + 0x29, 0x89, 0x3d, 0xa2, 0x9b, 0xd0, 0xc3, 0x2a, 0x1b, 0x1d, 0x9d, 0x7c, 0x5a, 0xaf, 0x14, 0x48, + 0x85, 0x76, 0x98, 0xee, 0x58, 0xbf, 0xf2, 0xf3, 0xd9, 0x89, 0x6c, 0xa9, 0x5b, 0xcd, 0x1d, 0x34, + 0x2c, 0x5f, 0x7e, 0xd6, 0x05, 0x83, 0xad, 0xac, 0x40, 0x33, 0x70, 0x82, 0x33, 0x89, 0xe3, 0x4c, + 0xdd, 0x6e, 0x25, 0xa3, 0x72, 0x45, 0xb4, 0x09, 0x43, 0xb1, 0xf6, 0x21, 0x78, 0xd8, 0x95, 0xda, + 0x75, 0xf9, 0x8e, 0xb1, 0x56, 0x14, 0xb9, 0x1b, 0xb4, 0x5b, 0x64, 0xe8, 0x09, 0xa0, 0x18, 0xb7, + 0x35, 0x9f, 0xea, 0x34, 0xf0, 0x05, 0xc5, 0xa7, 0x8e, 0x40, 0xf1, 0x75, 0x66, 0xa0, 0x0e, 0xd2, + 0x16, 0xc9, 0x62, 0x7f, 0xd3, 0xa1, 0x91, 0x7f, 0x92, 0xe0, 0x6c, 0xb2, 0x6d, 0x98, 0xc6, 0xa6, + 0xcd, 0xf9, 0x90, 0x92, 0x23, 0x31, 0x95, 0x6b, 0x80, 0x3c, 0xec, 0xe8, 0x96, 0x6b, 0xb9, 0x55, + 0x6d, 0x37, 0xd0, 0x5d, 0x1a, 0x38, 0xbe, 0xb8, 0x20, 0x86, 0x1a, 0x2b, 0x0f, 0xc5, 0x02, 0x7a, + 0x1f, 0x8a, 0xa4, 0x46, 0x2d, 0xc7, 0xf2, 0xa9, 0x65, 0xe8, 0xb6, 0xbd, 0xcf, 0x5a, 0x00, 0x36, + 0x0f, 0x4c, 0xf9, 0x68, 0x33, 0xd6, 0xac, 0xb5, 0xcc, 0x94, 0x22, 0x2f, 0x73, 0xdf, 0x01, 0x9c, + 0x60, 0xd7, 0x04, 0xfa, 0x42, 0x82, 0xde, 0x68, 0x60, 0x46, 0xd3, 0x09, 0x59, 0x49, 0xf9, 0xea, + 0x28, 0x94, 0xd2, 0x74, 0x5b, 0x3f, 0x3b, 0xe4, 0xa9, 0xcf, 0xff, 0xf8, 0xf7, 0xeb, 0xae, 0x4b, + 0xe8, 0xa2, 0xd2, 0xe1, 0xbb, 0x51, 0xf9, 0xc4, 0x32, 0x3f, 0x45, 0x5f, 0x4a, 0x90, 0x8b, 0x4d, + 0xfe, 0xe9, 0x80, 0xda, 0x3f, 0x41, 0x0a, 0x57, 0x0f, 0x03, 0x14, 0xfb, 0x94, 0x90, 0xdf, 0x62, + 0x98, 0x8a, 0x68, 0xac, 0x13, 0x26, 0xf4, 0xab, 0x04, 0xf9, 0xb4, 0x11, 0x16, 0xcd, 0xbd, 0xd1, + 0xbc, 0xcb, 0x31, 0x5e, 0x3f, 0xc6, 0x8c, 0x2c, 0xdf, 0x66, 0x58, 0x6f, 0xdc, 0x96, 0xa6, 0x65, + 0x45, 0x49, 0xfc, 0x70, 0xd5, 0x5c, 0x62, 0x62, 0x8d, 0x12, 0xfe, 0xdf, 0x88, 0x81, 0xfc, 0x5d, + 0x82, 0xb1, 0x4e, 0xd3, 0x24, 0x9a, 0x4f, 0xcb, 0xda, 0x11, 0x66, 0xe1, 0xc2, 0xbb, 0xc7, 0x33, + 0x16, 0x71, 0x4d, 0xb2, 0xb8, 0x26, 0x50, 0x51, 0xe9, 0xf8, 0x63, 0x01, 0xfa, 0x45, 0x82, 0xd1, + 0x0e, 0xa3, 0x24, 0xba, 0x9d, 0x86, 0xe2, 0xf0, 0x21, 0xb8, 0x30, 0x7f, 0x2c, 0x5b, 0x11, 0xc0, + 0x65, 0x16, 0xc0, 0x38, 0xba, 0xd0, 0xf1, 0x17, 0x14, 0xf4, 0x9b, 0x04, 0xe7, 0x53, 0xc7, 0x31, + 0x74, 0x2b, 0x0d, 0xc1, 0x61, 0xb3, 0x5e, 0xe1, 0x9d, 0x63, 0x58, 0x0a, 0xe4, 0x65, 0x86, 0xbc, + 0x84, 0x26, 0x95, 0x23, 0xfd, 0x6a, 0x82, 0x5c, 0xe8, 0x6f, 0x9a, 0x98, 0xd1, 0xdb, 0x69, 0x7b, + 0x27, 0xcd, 0xec, 0x85, 0x6b, 0x47, 0xd4, 0x16, 0xe8, 0x32, 0xe8, 0xb3, 0xa8, 0xa3, 0xb6, 0x8e, + 0x6a, 0x68, 0xe6, 0xa8, 0x63, 0x53, 0x34, 0x67, 0x16, 0x66, 0xdf, 0xc0, 0x82, 0x03, 0x98, 0x91, + 0x16, 0xd7, 0x5e, 0xbc, 0x2a, 0x4a, 0x2f, 0x5f, 0x15, 0xa5, 0x7f, 0x5e, 0x15, 0xa5, 0xe7, 0xaf, + 0x8b, 0x99, 0x97, 0xaf, 0x8b, 0x99, 0x3f, 0x5f, 0x17, 0x33, 0x1f, 0xdd, 0xac, 0x5a, 0x74, 0x3b, + 0xa8, 0x94, 0x0d, 0xe2, 0x34, 0x27, 0xaf, 0x7e, 0xe3, 0x1a, 0x1b, 0x06, 0x94, 0x86, 0x64, 0x8f, + 0x27, 0x94, 0xee, 0xd7, 0xb0, 0x5f, 0xe9, 0x61, 0xe2, 0xeb, 0xff, 0x07, 0x00, 0x00, 0xff, 0xff, + 0x4d, 0x56, 0xef, 0xe3, 0x00, 0x14, 0x00, 0x00, } // Reference imports to suppress errors if they are not otherwise used. @@ -2755,145 +2553,6 @@ func (m *StreamUpdate_SubaccountUpdate) MarshalToSizedBuffer(dAtA []byte) (int, } return len(dAtA) - i, nil } -func (m *SubaccountPerpetualPosition) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) - if err != nil { - return nil, err - } - return dAtA[:n], nil -} - -func (m *SubaccountPerpetualPosition) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} - -func (m *SubaccountPerpetualPosition) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) - _ = i - var l int - _ = l - if m.Quantums != 0 { - i = encodeVarintQuery(dAtA, i, uint64(m.Quantums)) - i-- - dAtA[i] = 0x10 - } - if m.PerpetualId != 0 { - i = encodeVarintQuery(dAtA, i, uint64(m.PerpetualId)) - i-- - dAtA[i] = 0x8 - } - return len(dAtA) - i, nil -} - -func (m *SubaccountAssetPosition) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) - if err != nil { - return nil, err - } - return dAtA[:n], nil -} - -func (m *SubaccountAssetPosition) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} - -func (m *SubaccountAssetPosition) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) - _ = i - var l int - _ = l - if m.Quantums != 0 { - i = encodeVarintQuery(dAtA, i, uint64(m.Quantums)) - i-- - dAtA[i] = 0x10 - } - if m.AssetId != 0 { - i = encodeVarintQuery(dAtA, i, uint64(m.AssetId)) - i-- - dAtA[i] = 0x8 - } - return len(dAtA) - i, nil -} - -func (m *StreamSubaccountUpdate) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) - if err != nil { - return nil, err - } - return dAtA[:n], nil -} - -func (m *StreamSubaccountUpdate) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} - -func (m *StreamSubaccountUpdate) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) - _ = i - var l int - _ = l - if m.Snapshot { - i-- - if m.Snapshot { - dAtA[i] = 1 - } else { - dAtA[i] = 0 - } - i-- - dAtA[i] = 0x20 - } - if len(m.UpdatedAssetPositions) > 0 { - for iNdEx := len(m.UpdatedAssetPositions) - 1; iNdEx >= 0; iNdEx-- { - { - size, err := m.UpdatedAssetPositions[iNdEx].MarshalToSizedBuffer(dAtA[:i]) - if err != nil { - return 0, err - } - i -= size - i = encodeVarintQuery(dAtA, i, uint64(size)) - } - i-- - dAtA[i] = 0x1a - } - } - if len(m.UpdatedPerpetualPositions) > 0 { - for iNdEx := len(m.UpdatedPerpetualPositions) - 1; iNdEx >= 0; iNdEx-- { - { - size, err := m.UpdatedPerpetualPositions[iNdEx].MarshalToSizedBuffer(dAtA[:i]) - if err != nil { - return 0, err - } - i -= size - i = encodeVarintQuery(dAtA, i, uint64(size)) - } - i-- - dAtA[i] = 0x12 - } - } - if m.SubaccountId != nil { - { - size, err := m.SubaccountId.MarshalToSizedBuffer(dAtA[:i]) - if err != nil { - return 0, err - } - i -= size - i = encodeVarintQuery(dAtA, i, uint64(size)) - } - i-- - dAtA[i] = 0xa - } - return len(dAtA) - i, nil -} - func (m *StreamOrderbookUpdate) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) @@ -2962,20 +2621,20 @@ func (m *StreamOrderbookFill) MarshalToSizedBuffer(dAtA []byte) (int, error) { var l int _ = l if len(m.FillAmounts) > 0 { - dAtA19 := make([]byte, len(m.FillAmounts)*10) - var j18 int + dAtA18 := make([]byte, len(m.FillAmounts)*10) + var j17 int for _, num := range m.FillAmounts { for num >= 1<<7 { - dAtA19[j18] = uint8(uint64(num)&0x7f | 0x80) + dAtA18[j17] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 - j18++ + j17++ } - dAtA19[j18] = uint8(num) - j18++ + dAtA18[j17] = uint8(num) + j17++ } - i -= j18 - copy(dAtA[i:], dAtA19[:j18]) - i = encodeVarintQuery(dAtA, i, uint64(j18)) + i -= j17 + copy(dAtA[i:], dAtA18[:j17]) + i = encodeVarintQuery(dAtA, i, uint64(j17)) i-- dAtA[i] = 0x1a } @@ -3439,119 +3098,61 @@ func (m *StreamUpdate_SubaccountUpdate) Size() (n int) { } return n } -func (m *SubaccountPerpetualPosition) Size() (n int) { +func (m *StreamOrderbookUpdate) Size() (n int) { if m == nil { return 0 } var l int _ = l - if m.PerpetualId != 0 { - n += 1 + sovQuery(uint64(m.PerpetualId)) + if len(m.Updates) > 0 { + for _, e := range m.Updates { + l = e.Size() + n += 1 + l + sovQuery(uint64(l)) + } } - if m.Quantums != 0 { - n += 1 + sovQuery(uint64(m.Quantums)) + if m.Snapshot { + n += 2 } return n } -func (m *SubaccountAssetPosition) Size() (n int) { +func (m *StreamOrderbookFill) Size() (n int) { if m == nil { return 0 } var l int _ = l - if m.AssetId != 0 { - n += 1 + sovQuery(uint64(m.AssetId)) + if m.ClobMatch != nil { + l = m.ClobMatch.Size() + n += 1 + l + sovQuery(uint64(l)) + } + if len(m.Orders) > 0 { + for _, e := range m.Orders { + l = e.Size() + n += 1 + l + sovQuery(uint64(l)) + } } - if m.Quantums != 0 { - n += 1 + sovQuery(uint64(m.Quantums)) + if len(m.FillAmounts) > 0 { + l = 0 + for _, e := range m.FillAmounts { + l += sovQuery(uint64(e)) + } + n += 1 + sovQuery(uint64(l)) + l } return n } -func (m *StreamSubaccountUpdate) Size() (n int) { +func (m *StreamTakerOrder) Size() (n int) { if m == nil { return 0 } var l int _ = l - if m.SubaccountId != nil { - l = m.SubaccountId.Size() - n += 1 + l + sovQuery(uint64(l)) - } - if len(m.UpdatedPerpetualPositions) > 0 { - for _, e := range m.UpdatedPerpetualPositions { - l = e.Size() - n += 1 + l + sovQuery(uint64(l)) - } - } - if len(m.UpdatedAssetPositions) > 0 { - for _, e := range m.UpdatedAssetPositions { - l = e.Size() - n += 1 + l + sovQuery(uint64(l)) - } - } - if m.Snapshot { - n += 2 - } - return n -} - -func (m *StreamOrderbookUpdate) Size() (n int) { - if m == nil { - return 0 - } - var l int - _ = l - if len(m.Updates) > 0 { - for _, e := range m.Updates { - l = e.Size() - n += 1 + l + sovQuery(uint64(l)) - } - } - if m.Snapshot { - n += 2 - } - return n -} - -func (m *StreamOrderbookFill) Size() (n int) { - if m == nil { - return 0 - } - var l int - _ = l - if m.ClobMatch != nil { - l = m.ClobMatch.Size() - n += 1 + l + sovQuery(uint64(l)) - } - if len(m.Orders) > 0 { - for _, e := range m.Orders { - l = e.Size() - n += 1 + l + sovQuery(uint64(l)) - } - } - if len(m.FillAmounts) > 0 { - l = 0 - for _, e := range m.FillAmounts { - l += sovQuery(uint64(e)) - } - n += 1 + sovQuery(uint64(l)) + l - } - return n -} - -func (m *StreamTakerOrder) Size() (n int) { - if m == nil { - return 0 - } - var l int - _ = l - if m.TakerOrder != nil { - n += m.TakerOrder.Size() - } - if m.TakerOrderStatus != nil { - l = m.TakerOrderStatus.Size() + if m.TakerOrder != nil { + n += m.TakerOrder.Size() + } + if m.TakerOrderStatus != nil { + l = m.TakerOrderStatus.Size() n += 1 + l + sovQuery(uint64(l)) } return n @@ -5279,7 +4880,7 @@ func (m *StreamUpdate) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - v := &StreamSubaccountUpdate{} + v := &types.StreamSubaccountUpdate{} if err := v.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } @@ -5344,356 +4945,6 @@ func (m *StreamUpdate) Unmarshal(dAtA []byte) error { } return nil } -func (m *SubaccountPerpetualPosition) Unmarshal(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowQuery - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: SubaccountPerpetualPosition: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: SubaccountPerpetualPosition: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - case 1: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field PerpetualId", wireType) - } - m.PerpetualId = 0 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowQuery - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - m.PerpetualId |= uint32(b&0x7F) << shift - if b < 0x80 { - break - } - } - case 2: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field Quantums", wireType) - } - m.Quantums = 0 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowQuery - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - m.Quantums |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - default: - iNdEx = preIndex - skippy, err := skipQuery(dAtA[iNdEx:]) - if err != nil { - return err - } - if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthQuery - } - if (iNdEx + skippy) > l { - return io.ErrUnexpectedEOF - } - iNdEx += skippy - } - } - - if iNdEx > l { - return io.ErrUnexpectedEOF - } - return nil -} -func (m *SubaccountAssetPosition) Unmarshal(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowQuery - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: SubaccountAssetPosition: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: SubaccountAssetPosition: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - case 1: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field AssetId", wireType) - } - m.AssetId = 0 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowQuery - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - m.AssetId |= uint32(b&0x7F) << shift - if b < 0x80 { - break - } - } - case 2: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field Quantums", wireType) - } - m.Quantums = 0 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowQuery - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - m.Quantums |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - default: - iNdEx = preIndex - skippy, err := skipQuery(dAtA[iNdEx:]) - if err != nil { - return err - } - if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthQuery - } - if (iNdEx + skippy) > l { - return io.ErrUnexpectedEOF - } - iNdEx += skippy - } - } - - if iNdEx > l { - return io.ErrUnexpectedEOF - } - return nil -} -func (m *StreamSubaccountUpdate) Unmarshal(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowQuery - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: StreamSubaccountUpdate: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: StreamSubaccountUpdate: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - case 1: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field SubaccountId", wireType) - } - var msglen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowQuery - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - msglen |= int(b&0x7F) << shift - if b < 0x80 { - break - } - } - if msglen < 0 { - return ErrInvalidLengthQuery - } - postIndex := iNdEx + msglen - if postIndex < 0 { - return ErrInvalidLengthQuery - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - if m.SubaccountId == nil { - m.SubaccountId = &types.SubaccountId{} - } - if err := m.SubaccountId.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { - return err - } - iNdEx = postIndex - case 2: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field UpdatedPerpetualPositions", wireType) - } - var msglen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowQuery - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - msglen |= int(b&0x7F) << shift - if b < 0x80 { - break - } - } - if msglen < 0 { - return ErrInvalidLengthQuery - } - postIndex := iNdEx + msglen - if postIndex < 0 { - return ErrInvalidLengthQuery - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.UpdatedPerpetualPositions = append(m.UpdatedPerpetualPositions, &SubaccountPerpetualPosition{}) - if err := m.UpdatedPerpetualPositions[len(m.UpdatedPerpetualPositions)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { - return err - } - iNdEx = postIndex - case 3: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field UpdatedAssetPositions", wireType) - } - var msglen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowQuery - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - msglen |= int(b&0x7F) << shift - if b < 0x80 { - break - } - } - if msglen < 0 { - return ErrInvalidLengthQuery - } - postIndex := iNdEx + msglen - if postIndex < 0 { - return ErrInvalidLengthQuery - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.UpdatedAssetPositions = append(m.UpdatedAssetPositions, &SubaccountAssetPosition{}) - if err := m.UpdatedAssetPositions[len(m.UpdatedAssetPositions)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { - return err - } - iNdEx = postIndex - case 4: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field Snapshot", wireType) - } - var v int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowQuery - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - v |= int(b&0x7F) << shift - if b < 0x80 { - break - } - } - m.Snapshot = bool(v != 0) - default: - iNdEx = preIndex - skippy, err := skipQuery(dAtA[iNdEx:]) - if err != nil { - return err - } - if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthQuery - } - if (iNdEx + skippy) > l { - return io.ErrUnexpectedEOF - } - iNdEx += skippy - } - } - - if iNdEx > l { - return io.ErrUnexpectedEOF - } - return nil -} func (m *StreamOrderbookUpdate) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 diff --git a/protocol/x/subaccounts/keeper/keeper.go b/protocol/x/subaccounts/keeper/keeper.go index 084f848a80..1d82f9d1ea 100644 --- a/protocol/x/subaccounts/keeper/keeper.go +++ b/protocol/x/subaccounts/keeper/keeper.go @@ -2,6 +2,7 @@ package keeper import ( "fmt" + streamingtypes "github.com/dydxprotocol/v4-chain/protocol/streaming/types" "cosmossdk.io/log" storetypes "cosmossdk.io/store/types" @@ -21,6 +22,7 @@ type ( blocktimeKeeper types.BlocktimeKeeper revShareKeeper types.RevShareKeeper indexerEventManager indexer_manager.IndexerEventManager + streamingManager streamingtypes.FullNodeStreamingManager } ) @@ -33,6 +35,7 @@ func NewKeeper( blocktimeKeeper types.BlocktimeKeeper, revShareKeeper types.RevShareKeeper, indexerEventManager indexer_manager.IndexerEventManager, + streamingManager streamingtypes.FullNodeStreamingManager, ) *Keeper { return &Keeper{ cdc: cdc, @@ -43,6 +46,7 @@ func NewKeeper( blocktimeKeeper: blocktimeKeeper, revShareKeeper: revShareKeeper, indexerEventManager: indexerEventManager, + streamingManager: streamingManager, } } diff --git a/protocol/x/subaccounts/keeper/subaccount.go b/protocol/x/subaccounts/keeper/subaccount.go index 1f14ca1275..d6932b6842 100644 --- a/protocol/x/subaccounts/keeper/subaccount.go +++ b/protocol/x/subaccounts/keeper/subaccount.go @@ -3,6 +3,7 @@ package keeper import ( "errors" "fmt" + streamingtypes "github.com/dydxprotocol/v4-chain/protocol/streaming/types" "math/big" "math/rand" "time" @@ -253,6 +254,47 @@ func (k Keeper) getSettledUpdates( return settledUpdates, subaccountIdToFundingPayments, nil } +func GenerateStreamSubaccountUpdate( + settledUpdate types.SettledUpdate, + fundingPayments map[uint32]dtypes.SerializableInt, +) types.StreamSubaccountUpdate { + // Get updated perpetual positions + updatedPerpetualPositions := salib.GetUpdatedPerpetualPositions( + settledUpdate, + fundingPayments, + ) + // Convert updated perpetual positions to SubaccountPerpetualPosition type + perpetualPositions := make([]*types.SubaccountPerpetualPosition, len(updatedPerpetualPositions)) + for i, pp := range updatedPerpetualPositions { + perpetualPositions[i] = &types.SubaccountPerpetualPosition{ + PerpetualId: pp.PerpetualId, + Quantums: pp.Quantums.BigInt().Uint64(), + } + } + + updatedAssetPositions := salib.GetUpdatedAssetPositions(settledUpdate) + assetPositionsWithQuoteBalance := indexerevents.AddQuoteBalanceFromPerpetualPositions( + updatedPerpetualPositions, + updatedAssetPositions, + ) + + // Convert updated asset positions to SubaccountAssetPosition type + assetPositions := make([]*types.SubaccountAssetPosition, len(assetPositionsWithQuoteBalance)) + for i, ap := range assetPositionsWithQuoteBalance { + assetPositions[i] = &types.SubaccountAssetPosition{ + AssetId: ap.AssetId, + Quantums: ap.Quantums.BigInt().Uint64(), + } + } + + return types.StreamSubaccountUpdate{ + SubaccountId: settledUpdate.SettledSubaccount.Id, + UpdatedAssetPositions: assetPositions, + UpdatedPerpetualPositions: perpetualPositions, + Snapshot: false, + } +} + // UpdateSubaccounts validates and applies all `updates` to the relevant subaccounts as long as this is a // valid state-transition for all subaccounts involved. All `updates` are made atomically, meaning that // all state-changes will either succeed or all will fail. @@ -369,6 +411,19 @@ func (k Keeper) UpdateSubaccounts( ), ) + // if GRPC streaming is on, emit a generated subaccount update to stream. + if streamingManager := k.GetFullNodeStreamingManager(); streamingManager.Enabled() { + if k.GetFullNodeStreamingManager().TracksSubaccountId(*u.SettledSubaccount.Id) { + subaccountUpdate := GenerateStreamSubaccountUpdate(u, fundingPayments) + k.SendSubaccountUpdates( + ctx, + []types.StreamSubaccountUpdate{ + subaccountUpdate, + }, + ) + } + } + // Emit an event indicating a funding payment was paid / received for each settled funding // payment. Note that `fundingPaid` is positive if the subaccount paid funding, // and negative if the subaccount received funding. @@ -738,3 +793,22 @@ func (k Keeper) GetAllRelevantPerpetuals( return perpInfos, nil } + +func (k Keeper) GetFullNodeStreamingManager() streamingtypes.FullNodeStreamingManager { + return k.streamingManager +} + +// SendSubaccountUpdates sends the subaccount updates to the gRPC streaming manager. +func (k Keeper) SendSubaccountUpdates( + ctx sdk.Context, + subaccountUpdates []types.StreamSubaccountUpdate, +) { + if len(subaccountUpdates) == 0 { + return + } + k.GetFullNodeStreamingManager().SendSubaccountUpdates( + subaccountUpdates, + lib.MustConvertIntegerToUint32(ctx.BlockHeight()), + ctx.ExecMode(), + ) +} diff --git a/protocol/x/subaccounts/types/streaming.pb.go b/protocol/x/subaccounts/types/streaming.pb.go new file mode 100644 index 0000000000..2babfbf862 --- /dev/null +++ b/protocol/x/subaccounts/types/streaming.pb.go @@ -0,0 +1,900 @@ +// Code generated by protoc-gen-gogo. DO NOT EDIT. +// source: dydxprotocol/subaccounts/streaming.proto + +package types + +import ( + fmt "fmt" + proto "github.com/cosmos/gogoproto/proto" + io "io" + math "math" + math_bits "math/bits" +) + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package + +// StreamSubaccountUpdate provides information on a subaccount update. Used in +// the full node GRPC stream. +type StreamSubaccountUpdate struct { + SubaccountId *SubaccountId `protobuf:"bytes,1,opt,name=subaccount_id,json=subaccountId,proto3" json:"subaccount_id,omitempty"` + // updated_perpetual_positions will each be for unique perpetuals. + UpdatedPerpetualPositions []*SubaccountPerpetualPosition `protobuf:"bytes,2,rep,name=updated_perpetual_positions,json=updatedPerpetualPositions,proto3" json:"updated_perpetual_positions,omitempty"` + // updated_asset_positions will each be for unique assets. + UpdatedAssetPositions []*SubaccountAssetPosition `protobuf:"bytes,3,rep,name=updated_asset_positions,json=updatedAssetPositions,proto3" json:"updated_asset_positions,omitempty"` + // Snapshot indicates if the response is from a snapshot of the subaccount. + // All updates should be ignored until snapshot is received. + // If the snapshot is true, then all previous entries should be + // discarded and the subaccount should be resynced. + // For a snapshot subaccount update, the `updated_perpetual_positions` and + // `updated_asset_positions` fields will contain the full state of the + // subaccount. + Snapshot bool `protobuf:"varint,4,opt,name=snapshot,proto3" json:"snapshot,omitempty"` +} + +func (m *StreamSubaccountUpdate) Reset() { *m = StreamSubaccountUpdate{} } +func (m *StreamSubaccountUpdate) String() string { return proto.CompactTextString(m) } +func (*StreamSubaccountUpdate) ProtoMessage() {} +func (*StreamSubaccountUpdate) Descriptor() ([]byte, []int) { + return fileDescriptor_e6cf3092946c3c13, []int{0} +} +func (m *StreamSubaccountUpdate) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *StreamSubaccountUpdate) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_StreamSubaccountUpdate.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *StreamSubaccountUpdate) XXX_Merge(src proto.Message) { + xxx_messageInfo_StreamSubaccountUpdate.Merge(m, src) +} +func (m *StreamSubaccountUpdate) XXX_Size() int { + return m.Size() +} +func (m *StreamSubaccountUpdate) XXX_DiscardUnknown() { + xxx_messageInfo_StreamSubaccountUpdate.DiscardUnknown(m) +} + +var xxx_messageInfo_StreamSubaccountUpdate proto.InternalMessageInfo + +func (m *StreamSubaccountUpdate) GetSubaccountId() *SubaccountId { + if m != nil { + return m.SubaccountId + } + return nil +} + +func (m *StreamSubaccountUpdate) GetUpdatedPerpetualPositions() []*SubaccountPerpetualPosition { + if m != nil { + return m.UpdatedPerpetualPositions + } + return nil +} + +func (m *StreamSubaccountUpdate) GetUpdatedAssetPositions() []*SubaccountAssetPosition { + if m != nil { + return m.UpdatedAssetPositions + } + return nil +} + +func (m *StreamSubaccountUpdate) GetSnapshot() bool { + if m != nil { + return m.Snapshot + } + return false +} + +// SubaccountPerpetualPosition provides information on a subaccount's updated +// perpetual positions. +type SubaccountPerpetualPosition struct { + // The `Id` of the `Perpetual`. + PerpetualId uint32 `protobuf:"varint,1,opt,name=perpetual_id,json=perpetualId,proto3" json:"perpetual_id,omitempty"` + // The size of the position in base quantums. + Quantums uint64 `protobuf:"varint,2,opt,name=quantums,proto3" json:"quantums,omitempty"` +} + +func (m *SubaccountPerpetualPosition) Reset() { *m = SubaccountPerpetualPosition{} } +func (m *SubaccountPerpetualPosition) String() string { return proto.CompactTextString(m) } +func (*SubaccountPerpetualPosition) ProtoMessage() {} +func (*SubaccountPerpetualPosition) Descriptor() ([]byte, []int) { + return fileDescriptor_e6cf3092946c3c13, []int{1} +} +func (m *SubaccountPerpetualPosition) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *SubaccountPerpetualPosition) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_SubaccountPerpetualPosition.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *SubaccountPerpetualPosition) XXX_Merge(src proto.Message) { + xxx_messageInfo_SubaccountPerpetualPosition.Merge(m, src) +} +func (m *SubaccountPerpetualPosition) XXX_Size() int { + return m.Size() +} +func (m *SubaccountPerpetualPosition) XXX_DiscardUnknown() { + xxx_messageInfo_SubaccountPerpetualPosition.DiscardUnknown(m) +} + +var xxx_messageInfo_SubaccountPerpetualPosition proto.InternalMessageInfo + +func (m *SubaccountPerpetualPosition) GetPerpetualId() uint32 { + if m != nil { + return m.PerpetualId + } + return 0 +} + +func (m *SubaccountPerpetualPosition) GetQuantums() uint64 { + if m != nil { + return m.Quantums + } + return 0 +} + +// SubaccountAssetPosition provides information on a subaccount's updated asset +// positions. +type SubaccountAssetPosition struct { + // The `Id` of the `Asset`. + AssetId uint32 `protobuf:"varint,1,opt,name=asset_id,json=assetId,proto3" json:"asset_id,omitempty"` + // The absolute size of the position in base quantums. + Quantums uint64 `protobuf:"varint,2,opt,name=quantums,proto3" json:"quantums,omitempty"` +} + +func (m *SubaccountAssetPosition) Reset() { *m = SubaccountAssetPosition{} } +func (m *SubaccountAssetPosition) String() string { return proto.CompactTextString(m) } +func (*SubaccountAssetPosition) ProtoMessage() {} +func (*SubaccountAssetPosition) Descriptor() ([]byte, []int) { + return fileDescriptor_e6cf3092946c3c13, []int{2} +} +func (m *SubaccountAssetPosition) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *SubaccountAssetPosition) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_SubaccountAssetPosition.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *SubaccountAssetPosition) XXX_Merge(src proto.Message) { + xxx_messageInfo_SubaccountAssetPosition.Merge(m, src) +} +func (m *SubaccountAssetPosition) XXX_Size() int { + return m.Size() +} +func (m *SubaccountAssetPosition) XXX_DiscardUnknown() { + xxx_messageInfo_SubaccountAssetPosition.DiscardUnknown(m) +} + +var xxx_messageInfo_SubaccountAssetPosition proto.InternalMessageInfo + +func (m *SubaccountAssetPosition) GetAssetId() uint32 { + if m != nil { + return m.AssetId + } + return 0 +} + +func (m *SubaccountAssetPosition) GetQuantums() uint64 { + if m != nil { + return m.Quantums + } + return 0 +} + +func init() { + proto.RegisterType((*StreamSubaccountUpdate)(nil), "dydxprotocol.subaccounts.StreamSubaccountUpdate") + proto.RegisterType((*SubaccountPerpetualPosition)(nil), "dydxprotocol.subaccounts.SubaccountPerpetualPosition") + proto.RegisterType((*SubaccountAssetPosition)(nil), "dydxprotocol.subaccounts.SubaccountAssetPosition") +} + +func init() { + proto.RegisterFile("dydxprotocol/subaccounts/streaming.proto", fileDescriptor_e6cf3092946c3c13) +} + +var fileDescriptor_e6cf3092946c3c13 = []byte{ + // 353 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x92, 0xc1, 0x6a, 0xea, 0x40, + 0x14, 0x86, 0x1d, 0x95, 0x7b, 0x65, 0xd4, 0xcd, 0xc0, 0xbd, 0x46, 0x85, 0x60, 0x5d, 0x94, 0x74, + 0xd1, 0x84, 0xda, 0x76, 0xd9, 0x45, 0xbb, 0x93, 0x6e, 0x24, 0x52, 0x0a, 0xa5, 0x20, 0x63, 0x26, + 0xe8, 0x80, 0x66, 0xa6, 0x9e, 0x99, 0xa2, 0x6f, 0xd1, 0xc7, 0xea, 0xd2, 0x65, 0x97, 0xc5, 0xbc, + 0x48, 0x31, 0xc6, 0x31, 0x52, 0x14, 0x77, 0xf9, 0xcf, 0x7f, 0xce, 0xf7, 0x67, 0x0e, 0x07, 0x3b, + 0x6c, 0xc1, 0xe6, 0x72, 0x26, 0x94, 0x08, 0xc4, 0xc4, 0x03, 0x3d, 0xa4, 0x41, 0x20, 0x74, 0xa4, + 0xc0, 0x03, 0x35, 0x0b, 0xe9, 0x94, 0x47, 0x23, 0x37, 0xb1, 0x89, 0x95, 0xed, 0x74, 0x33, 0x9d, + 0x8d, 0x8b, 0xc3, 0x0c, 0xf3, 0xbd, 0x81, 0xb4, 0xe3, 0x3c, 0xfe, 0xdf, 0x4f, 0xc0, 0x7d, 0x63, + 0x3d, 0x49, 0x46, 0x55, 0x48, 0x1e, 0x71, 0x75, 0xd7, 0x3e, 0xe0, 0xcc, 0x42, 0x2d, 0xe4, 0x94, + 0x3b, 0xe7, 0xee, 0xa1, 0x5c, 0x77, 0x87, 0xe8, 0x32, 0xbf, 0x02, 0x19, 0x45, 0x34, 0x6e, 0xea, + 0x04, 0xcb, 0x06, 0x32, 0x9c, 0xc9, 0x50, 0x69, 0x3a, 0x19, 0x48, 0x01, 0x5c, 0x71, 0x11, 0x81, + 0x95, 0x6f, 0x15, 0x9c, 0x72, 0xe7, 0xf6, 0x14, 0x74, 0x6f, 0x3b, 0xde, 0x4b, 0xa7, 0xfd, 0x7a, + 0x4a, 0xfe, 0xe5, 0x00, 0xe1, 0xb8, 0xb6, 0x8d, 0xa5, 0x00, 0xa1, 0xca, 0x44, 0x16, 0x92, 0xc8, + 0xab, 0x53, 0x22, 0xef, 0xd7, 0xa3, 0x26, 0xee, 0x5f, 0x4a, 0xdc, 0xab, 0x02, 0x69, 0xe0, 0x12, + 0x44, 0x54, 0xc2, 0x58, 0x28, 0xab, 0xd8, 0x42, 0x4e, 0xc9, 0x37, 0xba, 0xfd, 0x8a, 0x9b, 0x47, + 0x1e, 0x40, 0xce, 0x70, 0x65, 0xb7, 0x94, 0x74, 0xd1, 0x55, 0xbf, 0x6c, 0x6a, 0x5d, 0xb6, 0xa6, + 0xbf, 0x69, 0x1a, 0x29, 0x3d, 0x5d, 0x2f, 0x0b, 0x39, 0x45, 0xdf, 0xe8, 0x76, 0x0f, 0xd7, 0x0e, + 0xfc, 0x2b, 0xa9, 0xe3, 0xd2, 0xe6, 0xdd, 0x86, 0xfa, 0x37, 0xd1, 0xc7, 0x89, 0x0f, 0xcf, 0x9f, + 0x2b, 0x1b, 0x2d, 0x57, 0x36, 0xfa, 0x5e, 0xd9, 0xe8, 0x23, 0xb6, 0x73, 0xcb, 0xd8, 0xce, 0x7d, + 0xc5, 0x76, 0xee, 0xe5, 0x6e, 0xc4, 0xd5, 0x58, 0x0f, 0xdd, 0x40, 0x4c, 0xbd, 0xbd, 0x2b, 0x7b, + 0xbf, 0xb9, 0x0c, 0xc6, 0x94, 0x47, 0x9e, 0xa9, 0xcc, 0xf7, 0x2e, 0x4f, 0x2d, 0x64, 0x08, 0xc3, + 0x3f, 0x89, 0x7b, 0xfd, 0x13, 0x00, 0x00, 0xff, 0xff, 0xe9, 0x34, 0x6c, 0x66, 0xe6, 0x02, 0x00, + 0x00, +} + +func (m *StreamSubaccountUpdate) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *StreamSubaccountUpdate) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *StreamSubaccountUpdate) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.Snapshot { + i-- + if m.Snapshot { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i-- + dAtA[i] = 0x20 + } + if len(m.UpdatedAssetPositions) > 0 { + for iNdEx := len(m.UpdatedAssetPositions) - 1; iNdEx >= 0; iNdEx-- { + { + size, err := m.UpdatedAssetPositions[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintStreaming(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x1a + } + } + if len(m.UpdatedPerpetualPositions) > 0 { + for iNdEx := len(m.UpdatedPerpetualPositions) - 1; iNdEx >= 0; iNdEx-- { + { + size, err := m.UpdatedPerpetualPositions[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintStreaming(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x12 + } + } + if m.SubaccountId != nil { + { + size, err := m.SubaccountId.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintStreaming(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func (m *SubaccountPerpetualPosition) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *SubaccountPerpetualPosition) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *SubaccountPerpetualPosition) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.Quantums != 0 { + i = encodeVarintStreaming(dAtA, i, uint64(m.Quantums)) + i-- + dAtA[i] = 0x10 + } + if m.PerpetualId != 0 { + i = encodeVarintStreaming(dAtA, i, uint64(m.PerpetualId)) + i-- + dAtA[i] = 0x8 + } + return len(dAtA) - i, nil +} + +func (m *SubaccountAssetPosition) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *SubaccountAssetPosition) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *SubaccountAssetPosition) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.Quantums != 0 { + i = encodeVarintStreaming(dAtA, i, uint64(m.Quantums)) + i-- + dAtA[i] = 0x10 + } + if m.AssetId != 0 { + i = encodeVarintStreaming(dAtA, i, uint64(m.AssetId)) + i-- + dAtA[i] = 0x8 + } + return len(dAtA) - i, nil +} + +func encodeVarintStreaming(dAtA []byte, offset int, v uint64) int { + offset -= sovStreaming(v) + base := offset + for v >= 1<<7 { + dAtA[offset] = uint8(v&0x7f | 0x80) + v >>= 7 + offset++ + } + dAtA[offset] = uint8(v) + return base +} +func (m *StreamSubaccountUpdate) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.SubaccountId != nil { + l = m.SubaccountId.Size() + n += 1 + l + sovStreaming(uint64(l)) + } + if len(m.UpdatedPerpetualPositions) > 0 { + for _, e := range m.UpdatedPerpetualPositions { + l = e.Size() + n += 1 + l + sovStreaming(uint64(l)) + } + } + if len(m.UpdatedAssetPositions) > 0 { + for _, e := range m.UpdatedAssetPositions { + l = e.Size() + n += 1 + l + sovStreaming(uint64(l)) + } + } + if m.Snapshot { + n += 2 + } + return n +} + +func (m *SubaccountPerpetualPosition) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.PerpetualId != 0 { + n += 1 + sovStreaming(uint64(m.PerpetualId)) + } + if m.Quantums != 0 { + n += 1 + sovStreaming(uint64(m.Quantums)) + } + return n +} + +func (m *SubaccountAssetPosition) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.AssetId != 0 { + n += 1 + sovStreaming(uint64(m.AssetId)) + } + if m.Quantums != 0 { + n += 1 + sovStreaming(uint64(m.Quantums)) + } + return n +} + +func sovStreaming(x uint64) (n int) { + return (math_bits.Len64(x|1) + 6) / 7 +} +func sozStreaming(x uint64) (n int) { + return sovStreaming(uint64((x << 1) ^ uint64((int64(x) >> 63)))) +} +func (m *StreamSubaccountUpdate) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowStreaming + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: StreamSubaccountUpdate: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: StreamSubaccountUpdate: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field SubaccountId", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowStreaming + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthStreaming + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthStreaming + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.SubaccountId == nil { + m.SubaccountId = &SubaccountId{} + } + if err := m.SubaccountId.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field UpdatedPerpetualPositions", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowStreaming + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthStreaming + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthStreaming + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.UpdatedPerpetualPositions = append(m.UpdatedPerpetualPositions, &SubaccountPerpetualPosition{}) + if err := m.UpdatedPerpetualPositions[len(m.UpdatedPerpetualPositions)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field UpdatedAssetPositions", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowStreaming + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthStreaming + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthStreaming + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.UpdatedAssetPositions = append(m.UpdatedAssetPositions, &SubaccountAssetPosition{}) + if err := m.UpdatedAssetPositions[len(m.UpdatedAssetPositions)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 4: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Snapshot", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowStreaming + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.Snapshot = bool(v != 0) + default: + iNdEx = preIndex + skippy, err := skipStreaming(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthStreaming + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *SubaccountPerpetualPosition) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowStreaming + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: SubaccountPerpetualPosition: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: SubaccountPerpetualPosition: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field PerpetualId", wireType) + } + m.PerpetualId = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowStreaming + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.PerpetualId |= uint32(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Quantums", wireType) + } + m.Quantums = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowStreaming + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Quantums |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := skipStreaming(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthStreaming + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *SubaccountAssetPosition) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowStreaming + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: SubaccountAssetPosition: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: SubaccountAssetPosition: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field AssetId", wireType) + } + m.AssetId = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowStreaming + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.AssetId |= uint32(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Quantums", wireType) + } + m.Quantums = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowStreaming + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Quantums |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := skipStreaming(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthStreaming + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func skipStreaming(dAtA []byte) (n int, err error) { + l := len(dAtA) + iNdEx := 0 + depth := 0 + for iNdEx < l { + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowStreaming + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + wireType := int(wire & 0x7) + switch wireType { + case 0: + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowStreaming + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + iNdEx++ + if dAtA[iNdEx-1] < 0x80 { + break + } + } + case 1: + iNdEx += 8 + case 2: + var length int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowStreaming + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + length |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if length < 0 { + return 0, ErrInvalidLengthStreaming + } + iNdEx += length + case 3: + depth++ + case 4: + if depth == 0 { + return 0, ErrUnexpectedEndOfGroupStreaming + } + depth-- + case 5: + iNdEx += 4 + default: + return 0, fmt.Errorf("proto: illegal wireType %d", wireType) + } + if iNdEx < 0 { + return 0, ErrInvalidLengthStreaming + } + if depth == 0 { + return iNdEx, nil + } + } + return 0, io.ErrUnexpectedEOF +} + +var ( + ErrInvalidLengthStreaming = fmt.Errorf("proto: negative length found during unmarshaling") + ErrIntOverflowStreaming = fmt.Errorf("proto: integer overflow") + ErrUnexpectedEndOfGroupStreaming = fmt.Errorf("proto: unexpected end of group") +) diff --git a/protocol/x/subaccounts/types/types.go b/protocol/x/subaccounts/types/types.go index 0212e23791..dd8bd473bd 100644 --- a/protocol/x/subaccounts/types/types.go +++ b/protocol/x/subaccounts/types/types.go @@ -72,4 +72,8 @@ type SubaccountsKeeper interface { perpetualId uint32, blockHeight uint32, ) error + SendSubaccountUpdates( + ctx sdk.Context, + subaccountUpdates []StreamSubaccountUpdate, + ) } From b95a24d665c98802fdc14d07801c63e970286aa5 Mon Sep 17 00:00:00 2001 From: dydxwill <119354122+dydxwill@users.noreply.github.com> Date: Mon, 12 Aug 2024 17:04:40 -0400 Subject: [PATCH 12/33] [CT-1050] DeliverTx state change reset for subaccount updates (#2063) Co-authored-by: Jonathan Fung Co-authored-by: Jonathan Fung <121899091+jonfung-dydx@users.noreply.github.com> --- .github/workflows/protocol-build-and-push.yml | 181 +----------------- protocol/x/clob/keeper/keeper.go | 24 +-- protocol/x/clob/keeper/process_operations.go | 30 +++ protocol/x/clob/types/expected_keepers.go | 10 + protocol/x/clob/types/message_clob_match.go | 26 +++ protocol/x/subaccounts/keeper/subaccount.go | 28 +++ protocol/x/subaccounts/types/types.go | 4 + 7 files changed, 101 insertions(+), 202 deletions(-) diff --git a/.github/workflows/protocol-build-and-push.yml b/.github/workflows/protocol-build-and-push.yml index fc26dd6e69..938f973e6d 100644 --- a/.github/workflows/protocol-build-and-push.yml +++ b/.github/workflows/protocol-build-and-push.yml @@ -3,47 +3,12 @@ name: Protocol Build & Push Image to AWS ECR on: # yamllint disable-line rule:truthy push: branches: + - 'wl/sa3' - main - 'release/protocol/v[0-9]+.[0-9]+.x' # e.g. release/protocol/v0.1.x - 'release/protocol/v[0-9]+.x' # e.g. release/protocol/v1.x jobs: - build-and-push-dev: - runs-on: ubuntu-latest - defaults: - run: - working-directory: ./protocol - steps: - - name: Checkout - uses: actions/checkout@v3 - with: - fetch-depth: '0' # without this, ignite fails. - - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID_VALIDATOR_DEV }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_VALIDATOR_DEV }} - aws-region: us-east-2 - - - name: Login to Amazon ECR - id: login-ecr - uses: aws-actions/amazon-ecr-login@v1 - - - name: Build, Tag, and Push the Image to Amazon ECR - id: build-image - env: - ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} - ECR_REPOSITORY: dev-validator - run: | - make localnet-build-amd64 - commit_hash=$(git rev-parse --short=7 HEAD) - docker build \ - --platform amd64 \ - -t $ECR_REGISTRY/$ECR_REPOSITORY:$commit_hash \ - -f testing/testnet-dev/Dockerfile . - docker push $ECR_REGISTRY/$ECR_REPOSITORY --all-tags - build-and-push-dev2: runs-on: ubuntu-latest defaults: @@ -79,147 +44,3 @@ jobs: -t $ECR_REGISTRY/$ECR_REPOSITORY:$commit_hash \ -f testing/testnet-dev/Dockerfile . docker push $ECR_REGISTRY/$ECR_REPOSITORY --all-tags - - build-and-push-dev3: - runs-on: ubuntu-latest - defaults: - run: - working-directory: ./protocol - steps: - - name: Checkout - uses: actions/checkout@v3 - with: - fetch-depth: '0' # without this, ignite fails. - - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID_VALIDATOR_DEV3 }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_VALIDATOR_DEV3 }} - aws-region: us-east-2 - - - name: Login to Amazon ECR - id: login-ecr - uses: aws-actions/amazon-ecr-login@v1 - - - name: Build, Tag, and Push the Image to Amazon ECR - id: build-image - env: - ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} - ECR_REPOSITORY: dev3-validator - run: | - make localnet-build-amd64 - commit_hash=$(git rev-parse --short=7 HEAD) - docker build \ - --platform amd64 \ - -t $ECR_REGISTRY/$ECR_REPOSITORY:$commit_hash \ - -f testing/testnet-dev/Dockerfile . - docker push $ECR_REGISTRY/$ECR_REPOSITORY --all-tags - - build-and-push-dev4: - runs-on: ubuntu-latest - defaults: - run: - working-directory: ./protocol - steps: - - name: Checkout - uses: actions/checkout@v3 - with: - fetch-depth: '0' # without this, ignite fails. - - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID_VALIDATOR_DEV4 }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_VALIDATOR_DEV4 }} - aws-region: us-east-2 - - - name: Login to Amazon ECR - id: login-ecr - uses: aws-actions/amazon-ecr-login@v1 - - - name: Build, Tag, and Push the Image to Amazon ECR - id: build-image - env: - ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} - ECR_REPOSITORY: dev4-validator - run: | - make localnet-build-amd64 - commit_hash=$(git rev-parse --short=7 HEAD) - docker build \ - --platform amd64 \ - -t $ECR_REGISTRY/$ECR_REPOSITORY:$commit_hash \ - -f testing/testnet-dev/Dockerfile . - docker push $ECR_REGISTRY/$ECR_REPOSITORY --all-tags - - build-and-push-dev5: - runs-on: ubuntu-latest - defaults: - run: - working-directory: ./protocol - steps: - - name: Checkout - uses: actions/checkout@v3 - with: - fetch-depth: '0' # without this, ignite fails. - - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID_VALIDATOR_DEV5 }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_VALIDATOR_DEV5 }} - aws-region: us-east-2 - - - name: Login to Amazon ECR - id: login-ecr - uses: aws-actions/amazon-ecr-login@v1 - - - name: Build, Tag, and Push the Image to Amazon ECR - id: build-image - env: - ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} - ECR_REPOSITORY: dev5-validator - run: | - make localnet-build-amd64 - commit_hash=$(git rev-parse --short=7 HEAD) - docker build \ - --platform amd64 \ - -t $ECR_REGISTRY/$ECR_REPOSITORY:$commit_hash \ - -f testing/testnet-dev/Dockerfile . - docker push $ECR_REGISTRY/$ECR_REPOSITORY --all-tags - - build-and-push-staging: - runs-on: ubuntu-latest - defaults: - run: - working-directory: ./protocol - steps: - - name: Checkout - uses: actions/checkout@v3 - with: - fetch-depth: '0' # without this, ignite fails. - - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID_VALIDATOR_STAGING }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_VALIDATOR_STAGING }} - aws-region: us-east-2 - - - name: Login to Amazon ECR - id: login-ecr - uses: aws-actions/amazon-ecr-login@v1 - - - name: Build, Tag, and Push the Image to Amazon ECR - id: build-image - env: - ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} - ECR_REPOSITORY: staging-validator - run: | - make localnet-build-amd64 - commit_hash=$(git rev-parse --short=7 HEAD) - docker build \ - --platform amd64 \ - -t $ECR_REGISTRY/$ECR_REPOSITORY:$commit_hash \ - -f testing/testnet-staging/Dockerfile . - docker push $ECR_REGISTRY/$ECR_REPOSITORY --all-tags diff --git a/protocol/x/clob/keeper/keeper.go b/protocol/x/clob/keeper/keeper.go index 85e958423e..3dfa801685 100644 --- a/protocol/x/clob/keeper/keeper.go +++ b/protocol/x/clob/keeper/keeper.go @@ -264,31 +264,11 @@ func (k Keeper) InitializeNewStreams(ctx sdk.Context) { ) }, func(subaccountId satypes.SubaccountId) *satypes.StreamSubaccountUpdate { - subaccount := k.subaccountsKeeper.GetSubaccount( + subaccountUpdate := k.subaccountsKeeper.GetStreamSubaccountUpdate( ctx, subaccountId, ) - assetPositions := make([]*satypes.SubaccountAssetPosition, len(subaccount.AssetPositions)) - for i, ap := range subaccount.AssetPositions { - assetPositions[i] = &satypes.SubaccountAssetPosition{ - AssetId: ap.AssetId, - Quantums: ap.Quantums.BigInt().Uint64(), - } - } - perpetualPositions := make([]*satypes.SubaccountPerpetualPosition, len(subaccount.PerpetualPositions)) - for i, pp := range subaccount.PerpetualPositions { - perpetualPositions[i] = &satypes.SubaccountPerpetualPosition{ - PerpetualId: pp.PerpetualId, - Quantums: pp.Quantums.BigInt().Uint64(), - } - } - - return &satypes.StreamSubaccountUpdate{ - SubaccountId: &subaccountId, - UpdatedAssetPositions: assetPositions, - UpdatedPerpetualPositions: perpetualPositions, - Snapshot: true, - } + return &subaccountUpdate }, lib.MustConvertIntegerToUint32(ctx.BlockHeight()), ctx.ExecMode(), diff --git a/protocol/x/clob/keeper/process_operations.go b/protocol/x/clob/keeper/process_operations.go index 42e75f2034..60e731efdc 100644 --- a/protocol/x/clob/keeper/process_operations.go +++ b/protocol/x/clob/keeper/process_operations.go @@ -38,6 +38,21 @@ func fetchOrdersInvolvedInOpQueue( return orderIdSet } +// fetchSubaccountIdsInvolvedInOpQueue fetches all SubaccountIds involved in an operations +// queue's matches and returns them as a set. +func fetchSubaccountIdsInvolvedInOpQueue( + operations []types.InternalOperation, +) (subaccountIdSet map[satypes.SubaccountId]struct{}) { + subaccountIdSet = make(map[satypes.SubaccountId]struct{}) + for _, operation := range operations { + if clobMatch := operation.GetMatch(); clobMatch != nil { + subaccountIdSetForClobMatch := clobMatch.GetAllSubaccountIds() + subaccountIdSet = lib.MergeMaps(subaccountIdSet, subaccountIdSetForClobMatch) + } + } + return subaccountIdSet +} + // ProcessProposerOperations updates on-chain state given an []OperationRaw operations queue // representing matches that occurred in the previous block. It performs validation on an operations // queue. If all validation passes, the operations queue is written to state. @@ -58,6 +73,10 @@ func (k Keeper) ProcessProposerOperations( } // If grpc streams are on, send absolute fill amounts from local + proposed opqueue to the grpc stream. + // Also send subaccount snapshots for impacted subaccounts. + // An impacted subaccount is defined as: + // - A subaccount that was involved in any match in the local opqueue. + // Only matches generate subaccount updates. // This must be sent out to account for checkState being discarded and deliverState being used. if streamingManager := k.GetFullNodeStreamingManager(); streamingManager.Enabled() { localValidatorOperationsQueue, _ := k.MemClob.GetOperationsToReplay(ctx) @@ -75,6 +94,17 @@ func (k Keeper) ProcessProposerOperations( allUpdates.Append(orderbookUpdate) } k.SendOrderbookUpdates(ctx, allUpdates) + + // send local subaccount snapshots + subaccountIdsToUpdate := fetchSubaccountIdsInvolvedInOpQueue( + localValidatorOperationsQueue, + ) + allSubaccountUpdates := make([]satypes.StreamSubaccountUpdate, 0) + for subaccountId := range subaccountIdsToUpdate { + subaccountUpdate := k.subaccountsKeeper.GetStreamSubaccountUpdate(ctx, subaccountId) + allSubaccountUpdates = append(allSubaccountUpdates, subaccountUpdate) + } + k.subaccountsKeeper.SendSubaccountUpdates(ctx, allSubaccountUpdates) } log.DebugLog(ctx, "Processing operations queue", diff --git a/protocol/x/clob/types/expected_keepers.go b/protocol/x/clob/types/expected_keepers.go index 73db544217..57acdc7664 100644 --- a/protocol/x/clob/types/expected_keepers.go +++ b/protocol/x/clob/types/expected_keepers.go @@ -37,6 +37,12 @@ type SubaccountsKeeper interface { ) ( val satypes.Subaccount, ) + GetStreamSubaccountUpdate( + ctx sdk.Context, + id satypes.SubaccountId, + ) ( + val satypes.StreamSubaccountUpdate, + ) GetAllSubaccount( ctx sdk.Context, ) ( @@ -78,6 +84,10 @@ type SubaccountsKeeper interface { quantums *big.Int, perpetualId uint32, ) error + SendSubaccountUpdates( + ctx sdk.Context, + subaccountUpdates []satypes.StreamSubaccountUpdate, + ) } type AssetsKeeper interface { diff --git a/protocol/x/clob/types/message_clob_match.go b/protocol/x/clob/types/message_clob_match.go index 47593e05ee..c2b2f0a665 100644 --- a/protocol/x/clob/types/message_clob_match.go +++ b/protocol/x/clob/types/message_clob_match.go @@ -1,5 +1,7 @@ package types +import satypes "github.com/dydxprotocol/v4-chain/protocol/x/subaccounts/types" + // NewClobMatchFromMatchOrders creates a `ClobMatch` from the provided `MatchOrders`. func NewClobMatchFromMatchOrders( msgMatchOrders *MatchOrders, @@ -40,3 +42,27 @@ func (clobMatch *ClobMatch) GetAllOrderIds() (orderIds map[OrderId]struct{}) { } return orderIds } + +// GetAllSubaccountIds returns a set of subaccountIds involved in a ClobMatch. +func (clobMatch *ClobMatch) GetAllSubaccountIds() (subaccountIds map[satypes.SubaccountId]struct{}) { + subaccountIds = make(map[satypes.SubaccountId]struct{}) + if matchOrders := clobMatch.GetMatchOrders(); matchOrders != nil { + subaccountIds[matchOrders.GetTakerOrderId().SubaccountId] = struct{}{} + for _, makerFill := range matchOrders.GetFills() { + subaccountIds[makerFill.GetMakerOrderId().SubaccountId] = struct{}{} + } + } + if matchOrders := clobMatch.GetMatchPerpetualLiquidation(); matchOrders != nil { + subaccountIds[matchOrders.GetLiquidated()] = struct{}{} + for _, makerFill := range matchOrders.GetFills() { + subaccountIds[makerFill.GetMakerOrderId().SubaccountId] = struct{}{} + } + } + if matchOrders := clobMatch.GetMatchPerpetualDeleveraging(); matchOrders != nil { + subaccountIds[matchOrders.GetLiquidated()] = struct{}{} + for _, makerFill := range matchOrders.GetFills() { + subaccountIds[makerFill.GetOffsettingSubaccountId()] = struct{}{} + } + } + return subaccountIds +} diff --git a/protocol/x/subaccounts/keeper/subaccount.go b/protocol/x/subaccounts/keeper/subaccount.go index d6932b6842..73df2d2e0d 100644 --- a/protocol/x/subaccounts/keeper/subaccount.go +++ b/protocol/x/subaccounts/keeper/subaccount.go @@ -132,6 +132,34 @@ func (k Keeper) GetSubaccount( return val } +func (k Keeper) GetStreamSubaccountUpdate( + ctx sdk.Context, + id types.SubaccountId, +) (val types.StreamSubaccountUpdate) { + subaccount := k.GetSubaccount(ctx, id) + assetPositions := make([]*types.SubaccountAssetPosition, len(subaccount.AssetPositions)) + for i, ap := range subaccount.AssetPositions { + assetPositions[i] = &types.SubaccountAssetPosition{ + AssetId: ap.AssetId, + Quantums: ap.Quantums.BigInt().Uint64(), + } + } + perpetualPositions := make([]*types.SubaccountPerpetualPosition, len(subaccount.PerpetualPositions)) + for i, pp := range subaccount.PerpetualPositions { + perpetualPositions[i] = &types.SubaccountPerpetualPosition{ + PerpetualId: pp.PerpetualId, + Quantums: pp.Quantums.BigInt().Uint64(), + } + } + + return types.StreamSubaccountUpdate{ + SubaccountId: &id, + UpdatedAssetPositions: assetPositions, + UpdatedPerpetualPositions: perpetualPositions, + Snapshot: true, + } +} + // GetAllSubaccount returns all subaccount. // For more performant searching and iteration, use `ForEachSubaccount`. func (k Keeper) GetAllSubaccount(ctx sdk.Context) (list []types.Subaccount) { diff --git a/protocol/x/subaccounts/types/types.go b/protocol/x/subaccounts/types/types.go index dd8bd473bd..d4e7018ed6 100644 --- a/protocol/x/subaccounts/types/types.go +++ b/protocol/x/subaccounts/types/types.go @@ -62,6 +62,10 @@ type SubaccountsKeeper interface { ctx sdk.Context, id SubaccountId, ) (val Subaccount) + GetStreamSubaccountUpdate( + ctx sdk.Context, + id SubaccountId, + ) (val StreamSubaccountUpdate) LegacyGetNegativeTncSubaccountSeenAtBlock(ctx sdk.Context) (uint32, bool) GetNegativeTncSubaccountSeenAtBlock( ctx sdk.Context, From 87f94f8b2f6e18561c4753c8ce7f5587cbfc71bd Mon Sep 17 00:00:00 2001 From: dydxwill <119354122+dydxwill@users.noreply.github.com> Date: Mon, 12 Aug 2024 17:16:47 -0400 Subject: [PATCH 13/33] Fix snapshot bool (#2078) --- .github/workflows/protocol-build-and-push.yml | 181 +++++++++++++++++- protocol/x/clob/keeper/keeper.go | 1 + protocol/x/clob/keeper/process_operations.go | 2 +- protocol/x/clob/types/expected_keepers.go | 1 + protocol/x/subaccounts/keeper/subaccount.go | 3 +- protocol/x/subaccounts/types/types.go | 1 + 6 files changed, 186 insertions(+), 3 deletions(-) diff --git a/.github/workflows/protocol-build-and-push.yml b/.github/workflows/protocol-build-and-push.yml index 938f973e6d..fc26dd6e69 100644 --- a/.github/workflows/protocol-build-and-push.yml +++ b/.github/workflows/protocol-build-and-push.yml @@ -3,12 +3,47 @@ name: Protocol Build & Push Image to AWS ECR on: # yamllint disable-line rule:truthy push: branches: - - 'wl/sa3' - main - 'release/protocol/v[0-9]+.[0-9]+.x' # e.g. release/protocol/v0.1.x - 'release/protocol/v[0-9]+.x' # e.g. release/protocol/v1.x jobs: + build-and-push-dev: + runs-on: ubuntu-latest + defaults: + run: + working-directory: ./protocol + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: '0' # without this, ignite fails. + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID_VALIDATOR_DEV }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_VALIDATOR_DEV }} + aws-region: us-east-2 + + - name: Login to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v1 + + - name: Build, Tag, and Push the Image to Amazon ECR + id: build-image + env: + ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} + ECR_REPOSITORY: dev-validator + run: | + make localnet-build-amd64 + commit_hash=$(git rev-parse --short=7 HEAD) + docker build \ + --platform amd64 \ + -t $ECR_REGISTRY/$ECR_REPOSITORY:$commit_hash \ + -f testing/testnet-dev/Dockerfile . + docker push $ECR_REGISTRY/$ECR_REPOSITORY --all-tags + build-and-push-dev2: runs-on: ubuntu-latest defaults: @@ -44,3 +79,147 @@ jobs: -t $ECR_REGISTRY/$ECR_REPOSITORY:$commit_hash \ -f testing/testnet-dev/Dockerfile . docker push $ECR_REGISTRY/$ECR_REPOSITORY --all-tags + + build-and-push-dev3: + runs-on: ubuntu-latest + defaults: + run: + working-directory: ./protocol + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: '0' # without this, ignite fails. + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID_VALIDATOR_DEV3 }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_VALIDATOR_DEV3 }} + aws-region: us-east-2 + + - name: Login to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v1 + + - name: Build, Tag, and Push the Image to Amazon ECR + id: build-image + env: + ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} + ECR_REPOSITORY: dev3-validator + run: | + make localnet-build-amd64 + commit_hash=$(git rev-parse --short=7 HEAD) + docker build \ + --platform amd64 \ + -t $ECR_REGISTRY/$ECR_REPOSITORY:$commit_hash \ + -f testing/testnet-dev/Dockerfile . + docker push $ECR_REGISTRY/$ECR_REPOSITORY --all-tags + + build-and-push-dev4: + runs-on: ubuntu-latest + defaults: + run: + working-directory: ./protocol + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: '0' # without this, ignite fails. + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID_VALIDATOR_DEV4 }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_VALIDATOR_DEV4 }} + aws-region: us-east-2 + + - name: Login to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v1 + + - name: Build, Tag, and Push the Image to Amazon ECR + id: build-image + env: + ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} + ECR_REPOSITORY: dev4-validator + run: | + make localnet-build-amd64 + commit_hash=$(git rev-parse --short=7 HEAD) + docker build \ + --platform amd64 \ + -t $ECR_REGISTRY/$ECR_REPOSITORY:$commit_hash \ + -f testing/testnet-dev/Dockerfile . + docker push $ECR_REGISTRY/$ECR_REPOSITORY --all-tags + + build-and-push-dev5: + runs-on: ubuntu-latest + defaults: + run: + working-directory: ./protocol + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: '0' # without this, ignite fails. + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID_VALIDATOR_DEV5 }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_VALIDATOR_DEV5 }} + aws-region: us-east-2 + + - name: Login to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v1 + + - name: Build, Tag, and Push the Image to Amazon ECR + id: build-image + env: + ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} + ECR_REPOSITORY: dev5-validator + run: | + make localnet-build-amd64 + commit_hash=$(git rev-parse --short=7 HEAD) + docker build \ + --platform amd64 \ + -t $ECR_REGISTRY/$ECR_REPOSITORY:$commit_hash \ + -f testing/testnet-dev/Dockerfile . + docker push $ECR_REGISTRY/$ECR_REPOSITORY --all-tags + + build-and-push-staging: + runs-on: ubuntu-latest + defaults: + run: + working-directory: ./protocol + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: '0' # without this, ignite fails. + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID_VALIDATOR_STAGING }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_VALIDATOR_STAGING }} + aws-region: us-east-2 + + - name: Login to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v1 + + - name: Build, Tag, and Push the Image to Amazon ECR + id: build-image + env: + ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} + ECR_REPOSITORY: staging-validator + run: | + make localnet-build-amd64 + commit_hash=$(git rev-parse --short=7 HEAD) + docker build \ + --platform amd64 \ + -t $ECR_REGISTRY/$ECR_REPOSITORY:$commit_hash \ + -f testing/testnet-staging/Dockerfile . + docker push $ECR_REGISTRY/$ECR_REPOSITORY --all-tags diff --git a/protocol/x/clob/keeper/keeper.go b/protocol/x/clob/keeper/keeper.go index 3dfa801685..6817b43469 100644 --- a/protocol/x/clob/keeper/keeper.go +++ b/protocol/x/clob/keeper/keeper.go @@ -267,6 +267,7 @@ func (k Keeper) InitializeNewStreams(ctx sdk.Context) { subaccountUpdate := k.subaccountsKeeper.GetStreamSubaccountUpdate( ctx, subaccountId, + true, ) return &subaccountUpdate }, diff --git a/protocol/x/clob/keeper/process_operations.go b/protocol/x/clob/keeper/process_operations.go index 60e731efdc..2e349ed0a6 100644 --- a/protocol/x/clob/keeper/process_operations.go +++ b/protocol/x/clob/keeper/process_operations.go @@ -101,7 +101,7 @@ func (k Keeper) ProcessProposerOperations( ) allSubaccountUpdates := make([]satypes.StreamSubaccountUpdate, 0) for subaccountId := range subaccountIdsToUpdate { - subaccountUpdate := k.subaccountsKeeper.GetStreamSubaccountUpdate(ctx, subaccountId) + subaccountUpdate := k.subaccountsKeeper.GetStreamSubaccountUpdate(ctx, subaccountId, false) allSubaccountUpdates = append(allSubaccountUpdates, subaccountUpdate) } k.subaccountsKeeper.SendSubaccountUpdates(ctx, allSubaccountUpdates) diff --git a/protocol/x/clob/types/expected_keepers.go b/protocol/x/clob/types/expected_keepers.go index 57acdc7664..3ed63c0592 100644 --- a/protocol/x/clob/types/expected_keepers.go +++ b/protocol/x/clob/types/expected_keepers.go @@ -40,6 +40,7 @@ type SubaccountsKeeper interface { GetStreamSubaccountUpdate( ctx sdk.Context, id satypes.SubaccountId, + snapshot bool, ) ( val satypes.StreamSubaccountUpdate, ) diff --git a/protocol/x/subaccounts/keeper/subaccount.go b/protocol/x/subaccounts/keeper/subaccount.go index 73df2d2e0d..bd7bf619dc 100644 --- a/protocol/x/subaccounts/keeper/subaccount.go +++ b/protocol/x/subaccounts/keeper/subaccount.go @@ -135,6 +135,7 @@ func (k Keeper) GetSubaccount( func (k Keeper) GetStreamSubaccountUpdate( ctx sdk.Context, id types.SubaccountId, + snapshot bool, ) (val types.StreamSubaccountUpdate) { subaccount := k.GetSubaccount(ctx, id) assetPositions := make([]*types.SubaccountAssetPosition, len(subaccount.AssetPositions)) @@ -156,7 +157,7 @@ func (k Keeper) GetStreamSubaccountUpdate( SubaccountId: &id, UpdatedAssetPositions: assetPositions, UpdatedPerpetualPositions: perpetualPositions, - Snapshot: true, + Snapshot: snapshot, } } diff --git a/protocol/x/subaccounts/types/types.go b/protocol/x/subaccounts/types/types.go index d4e7018ed6..3effca3878 100644 --- a/protocol/x/subaccounts/types/types.go +++ b/protocol/x/subaccounts/types/types.go @@ -65,6 +65,7 @@ type SubaccountsKeeper interface { GetStreamSubaccountUpdate( ctx sdk.Context, id SubaccountId, + snapshot bool, ) (val StreamSubaccountUpdate) LegacyGetNegativeTncSubaccountSeenAtBlock(ctx sdk.Context) (uint32, bool) GetNegativeTncSubaccountSeenAtBlock( From 7cfd0f3ca0cfec702bd162c6ec8b1ed26c319a16 Mon Sep 17 00:00:00 2001 From: Jonathan Fung <121899091+jonfung-dydx@users.noreply.github.com> Date: Tue, 13 Aug 2024 13:51:07 -0700 Subject: [PATCH 14/33] Full Node Streaming Recurring snapshots (#2079) --- protocol/app/app.go | 6 ++- protocol/app/flags/flags.go | 20 ++++++++ protocol/app/flags/flags_test.go | 48 ++++++++++++++++--- .../streaming/full_node_streaming_manager.go | 28 +++++++++-- 4 files changed, 92 insertions(+), 10 deletions(-) diff --git a/protocol/app/app.go b/protocol/app/app.go index 92bef24378..de242f3d21 100644 --- a/protocol/app/app.go +++ b/protocol/app/app.go @@ -2020,12 +2020,16 @@ func getFullNodeStreamingManagerFromOptions( logger log.Logger, ) (manager streamingtypes.FullNodeStreamingManager) { if appFlags.GrpcStreamingEnabled { - logger.Info("GRPC streaming is enabled") + logger.Info("GRPC streaming is enabled", log.ModuleKey, "full-node-streaming") + if appFlags.FullNodeStreamingSnapshotInterval > 0 { + logger.Info("Interval snapshots enabled", log.ModuleKey, "full-node-streaming") + } return streaming.NewFullNodeStreamingManager( logger, appFlags.GrpcStreamingFlushIntervalMs, appFlags.GrpcStreamingMaxBatchSize, appFlags.GrpcStreamingMaxChannelBufferSize, + appFlags.FullNodeStreamingSnapshotInterval, ) } return streaming.NewNoopGrpcStreamingManager() diff --git a/protocol/app/flags/flags.go b/protocol/app/flags/flags.go index bdbe34a514..41b13ae826 100644 --- a/protocol/app/flags/flags.go +++ b/protocol/app/flags/flags.go @@ -25,6 +25,7 @@ type Flags struct { GrpcStreamingFlushIntervalMs uint32 GrpcStreamingMaxBatchSize uint32 GrpcStreamingMaxChannelBufferSize uint32 + FullNodeStreamingSnapshotInterval uint32 VEOracleEnabled bool // Slinky Vote Extensions // Optimistic block execution @@ -47,6 +48,7 @@ const ( GrpcStreamingFlushIntervalMs = "grpc-streaming-flush-interval-ms" GrpcStreamingMaxBatchSize = "grpc-streaming-max-batch-size" GrpcStreamingMaxChannelBufferSize = "grpc-streaming-max-channel-buffer-size" + FullNodeStreamingSnapshotInterval = "fns-snapshot-interval" // Slinky VEs enabled VEOracleEnabled = "slinky-vote-extension-oracle-enabled" @@ -66,6 +68,7 @@ const ( DefaultGrpcStreamingFlushIntervalMs = 50 DefaultGrpcStreamingMaxBatchSize = 2000 DefaultGrpcStreamingMaxChannelBufferSize = 2000 + DefaultFullNodeStreamingSnapshotInterval = 0 DefaultVEOracleEnabled = true DefaultOptimisticExecutionEnabled = false @@ -117,6 +120,12 @@ func AddFlagsToCmd(cmd *cobra.Command) { DefaultGrpcStreamingMaxChannelBufferSize, "Maximum per-subscription channel size before grpc streaming cancels a singular subscription", ) + cmd.Flags().Uint32( + FullNodeStreamingSnapshotInterval, + DefaultFullNodeStreamingSnapshotInterval, + "If set to positive number, number of blocks between each periodic snapshot will be sent out. "+ + "Defaults to zero for regular behavior of one initial snapshot.", + ) cmd.Flags().Bool( VEOracleEnabled, DefaultVEOracleEnabled, @@ -155,6 +164,10 @@ func (f *Flags) Validate() error { return fmt.Errorf("grpc streaming channel size must be positive number") } } + if f.FullNodeStreamingSnapshotInterval > 0 && f.FullNodeStreamingSnapshotInterval < 50 { + return fmt.Errorf("full node streaming snapshot interval must be >= 50 blocks or zero") + } + return nil } @@ -178,6 +191,7 @@ func GetFlagValuesFromOptions( GrpcStreamingFlushIntervalMs: DefaultGrpcStreamingFlushIntervalMs, GrpcStreamingMaxBatchSize: DefaultGrpcStreamingMaxBatchSize, GrpcStreamingMaxChannelBufferSize: DefaultGrpcStreamingMaxChannelBufferSize, + FullNodeStreamingSnapshotInterval: DefaultFullNodeStreamingSnapshotInterval, VEOracleEnabled: true, OptimisticExecutionEnabled: DefaultOptimisticExecutionEnabled, @@ -244,6 +258,12 @@ func GetFlagValuesFromOptions( } } + if option := appOpts.Get(FullNodeStreamingSnapshotInterval); option != nil { + if v, err := cast.ToUint32E(option); err == nil { + result.FullNodeStreamingSnapshotInterval = v + } + } + if option := appOpts.Get(VEOracleEnabled); option != nil { if v, err := cast.ToBoolE(option); err == nil { result.VEOracleEnabled = v diff --git a/protocol/app/flags/flags_test.go b/protocol/app/flags/flags_test.go index 8260efe313..84b10ef93e 100644 --- a/protocol/app/flags/flags_test.go +++ b/protocol/app/flags/flags_test.go @@ -38,6 +38,9 @@ func TestAddFlagsToCommand(t *testing.T) { fmt.Sprintf("Has %s flag", flags.GrpcStreamingMaxBatchSize): { flagName: flags.GrpcStreamingMaxBatchSize, }, + fmt.Sprintf("Has %s flag", flags.FullNodeStreamingSnapshotInterval): { + flagName: flags.FullNodeStreamingSnapshotInterval, + }, fmt.Sprintf("Has %s flag", flags.GrpcStreamingMaxChannelBufferSize): { flagName: flags.GrpcStreamingMaxChannelBufferSize, }, @@ -60,12 +63,13 @@ func TestValidate(t *testing.T) { }{ "success (default values)": { flags: flags.Flags{ - NonValidatingFullNode: flags.DefaultNonValidatingFullNode, - DdAgentHost: flags.DefaultDdAgentHost, - DdTraceAgentPort: flags.DefaultDdTraceAgentPort, - GrpcAddress: config.DefaultGRPCAddress, - GrpcEnable: true, - OptimisticExecutionEnabled: false, + NonValidatingFullNode: flags.DefaultNonValidatingFullNode, + DdAgentHost: flags.DefaultDdAgentHost, + DdTraceAgentPort: flags.DefaultDdTraceAgentPort, + GrpcAddress: config.DefaultGRPCAddress, + GrpcEnable: true, + FullNodeStreamingSnapshotInterval: flags.DefaultFullNodeStreamingSnapshotInterval, + OptimisticExecutionEnabled: false, }, }, "success - full node & gRPC disabled": { @@ -145,6 +149,29 @@ func TestValidate(t *testing.T) { }, expectedErr: fmt.Errorf("grpc streaming channel size must be positive number"), }, + "failure - full node streaming enabled with <= 49 snapshot interval": { + flags: flags.Flags{ + NonValidatingFullNode: true, + GrpcEnable: true, + GrpcStreamingEnabled: true, + GrpcStreamingFlushIntervalMs: 100, + GrpcStreamingMaxBatchSize: 2000, + GrpcStreamingMaxChannelBufferSize: 2000, + FullNodeStreamingSnapshotInterval: 49, + }, + expectedErr: fmt.Errorf("full node streaming snapshot interval must be >= 50 blocks or zero"), + }, + "success - full node streaming enabled with 50 snapshot interval": { + flags: flags.Flags{ + NonValidatingFullNode: true, + GrpcEnable: true, + GrpcStreamingEnabled: true, + GrpcStreamingFlushIntervalMs: 100, + GrpcStreamingMaxBatchSize: 2000, + GrpcStreamingMaxChannelBufferSize: 2000, + FullNodeStreamingSnapshotInterval: 50, + }, + }, } for name, tc := range tests { t.Run(name, func(t *testing.T) { @@ -173,6 +200,7 @@ func TestGetFlagValuesFromOptions(t *testing.T) { expectedGrpcStreamingFlushMs uint32 expectedGrpcStreamingBatchSize uint32 expectedGrpcStreamingMaxChannelBufferSize uint32 + expectedFullNodeStreamingSnapshotInterval uint32 expectedOptimisticExecutionEnabled bool }{ "Sets to default if unset": { @@ -185,6 +213,7 @@ func TestGetFlagValuesFromOptions(t *testing.T) { expectedGrpcStreamingFlushMs: 50, expectedGrpcStreamingBatchSize: 2000, expectedGrpcStreamingMaxChannelBufferSize: 2000, + expectedFullNodeStreamingSnapshotInterval: 0, expectedOptimisticExecutionEnabled: false, }, "Sets values from options": { @@ -198,6 +227,7 @@ func TestGetFlagValuesFromOptions(t *testing.T) { flags.GrpcStreamingFlushIntervalMs: uint32(408), flags.GrpcStreamingMaxBatchSize: uint32(650), flags.GrpcStreamingMaxChannelBufferSize: uint32(972), + flags.FullNodeStreamingSnapshotInterval: uint32(123), flags.OptimisticExecutionEnabled: "true", }, expectedNonValidatingFullNodeFlag: true, @@ -209,6 +239,7 @@ func TestGetFlagValuesFromOptions(t *testing.T) { expectedGrpcStreamingFlushMs: 408, expectedGrpcStreamingBatchSize: 650, expectedGrpcStreamingMaxChannelBufferSize: 972, + expectedFullNodeStreamingSnapshotInterval: 123, expectedOptimisticExecutionEnabled: true, }, } @@ -262,6 +293,11 @@ func TestGetFlagValuesFromOptions(t *testing.T) { tc.expectedGrpcStreamingBatchSize, flags.GrpcStreamingMaxBatchSize, ) + require.Equal( + t, + tc.expectedFullNodeStreamingSnapshotInterval, + flags.FullNodeStreamingSnapshotInterval, + ) require.Equal( t, tc.expectedGrpcStreamingMaxChannelBufferSize, diff --git a/protocol/streaming/full_node_streaming_manager.go b/protocol/streaming/full_node_streaming_manager.go index c53b76e2c0..12bc38cfed 100644 --- a/protocol/streaming/full_node_streaming_manager.go +++ b/protocol/streaming/full_node_streaming_manager.go @@ -2,10 +2,11 @@ package streaming import ( "fmt" - satypes "github.com/dydxprotocol/v4-chain/protocol/x/subaccounts/types" "sync" "time" + satypes "github.com/dydxprotocol/v4-chain/protocol/x/subaccounts/types" + "cosmossdk.io/log" sdk "github.com/cosmos/cosmos-sdk/types" "github.com/dydxprotocol/v4-chain/protocol/lib/metrics" @@ -44,6 +45,10 @@ type FullNodeStreamingManagerImpl struct { maxUpdatesInCache uint32 maxSubscriptionChannelSize uint32 + + // Block interval in which snapshot info should be sent out in. + // Defaults to 0, which means only one snapshot will be sent out. + snapshotBlockInterval uint32 } // OrderbookSubscription represents a active subscription to the orderbook updates stream. @@ -51,7 +56,7 @@ type OrderbookSubscription struct { subscriptionId uint32 // Initialize the subscription with orderbook snapshots. - initialize sync.Once + initialize *sync.Once // Clob pair ids to subscribe to. clobPairIds []uint32 @@ -64,6 +69,10 @@ type OrderbookSubscription struct { // Channel to buffer writes before the stream updatesChannel chan []clobtypes.StreamUpdate + + // If interval snapshots are turned on, the next block height at which + // a snapshot should be sent out. + nextSnapshotBlock uint32 } func NewFullNodeStreamingManager( @@ -71,6 +80,7 @@ func NewFullNodeStreamingManager( flushIntervalMs uint32, maxUpdatesInCache uint32, maxSubscriptionChannelSize uint32, + snapshotBlockInterval uint32, ) *FullNodeStreamingManagerImpl { logger = logger.With(log.ModuleKey, "full-node-streaming") fullNodeStreamingManager := &FullNodeStreamingManagerImpl{ @@ -87,6 +97,7 @@ func NewFullNodeStreamingManager( maxUpdatesInCache: maxUpdatesInCache, maxSubscriptionChannelSize: maxSubscriptionChannelSize, + snapshotBlockInterval: snapshotBlockInterval, } // Start the goroutine for pushing order updates through. @@ -149,6 +160,7 @@ func (sm *FullNodeStreamingManagerImpl) Subscribe( } subscription := &OrderbookSubscription{ subscriptionId: sm.nextSubscriptionId, + initialize: &sync.Once{}, clobPairIds: clobPairIds, subaccountIds: sIds, messageSender: messageSender, @@ -674,7 +686,15 @@ func (sm *FullNodeStreamingManagerImpl) InitializeNewStreams( sm.FlushStreamUpdatesWithLock() updatesByClobPairId := make(map[uint32]*clobtypes.OffchainUpdates) + for subscriptionId, subscription := range sm.orderbookSubscriptions { + // If the snapshot block interval is enabled, reset the sync.Once in order to + // re-send snapshots out. + if sm.snapshotBlockInterval > 0 && + blockHeight == subscription.nextSnapshotBlock { + subscription.initialize = &sync.Once{} + } + subscription.initialize.Do( func() { allUpdates := clobtypes.NewOffchainUpdates() @@ -688,8 +708,10 @@ func (sm *FullNodeStreamingManagerImpl) InitializeNewStreams( for _, subaccountId := range subscription.subaccountIds { saUpdates = append(saUpdates, getSubaccountSnapshot(subaccountId)) } - sm.SendCombinedSnapshot(allUpdates, saUpdates, subscriptionId, blockHeight, execMode) + if sm.snapshotBlockInterval != 0 { + subscription.nextSnapshotBlock = blockHeight + sm.snapshotBlockInterval + } }, ) } From 297982ba87d3a6657dc32ed5522ff3fe7a560668 Mon Sep 17 00:00:00 2001 From: dydxwill <119354122+dydxwill@users.noreply.github.com> Date: Wed, 14 Aug 2024 11:51:12 -0400 Subject: [PATCH 15/33] Remove todo (#2087) --- protocol/streaming/full_node_streaming_manager.go | 1 - 1 file changed, 1 deletion(-) diff --git a/protocol/streaming/full_node_streaming_manager.go b/protocol/streaming/full_node_streaming_manager.go index 12bc38cfed..40051d4e11 100644 --- a/protocol/streaming/full_node_streaming_manager.go +++ b/protocol/streaming/full_node_streaming_manager.go @@ -514,7 +514,6 @@ func (sm *FullNodeStreamingManagerImpl) SendTakerOrderStatus( // SendSubaccountUpdates groups subaccount updates by their subaccount ids and // sends messages to the subscribers. -// TODO(CT-1117): Aggregate subaccount updates by subaccount id. func (sm *FullNodeStreamingManagerImpl) SendSubaccountUpdates( subaccountUpdates []satypes.StreamSubaccountUpdate, blockHeight uint32, From 4ca53f96073e534a0ed81db68d2bb8ab5f6ef1fa Mon Sep 17 00:00:00 2001 From: jayy04 <103467857+jayy04@users.noreply.github.com> Date: Wed, 14 Aug 2024 13:47:22 -0400 Subject: [PATCH 16/33] Add websocket support to full node streaming (#1908) --- .github/workflows/protocol-build-and-push.yml | 2 +- protocol/app/app.go | 41 ++++- protocol/app/app_test.go | 1 + protocol/app/flags/flags.go | 45 +++++- protocol/app/flags/flags_test.go | 83 ++++++++++- protocol/docker-compose.yml | 3 + protocol/go.mod | 2 +- .../streaming/full_node_streaming_manager.go | 1 - .../streaming/ws/websocket_message_sender.go | 27 ++++ protocol/streaming/ws/websocket_server.go | 141 ++++++++++++++++++ 10 files changed, 326 insertions(+), 20 deletions(-) create mode 100644 protocol/streaming/ws/websocket_message_sender.go create mode 100644 protocol/streaming/ws/websocket_server.go diff --git a/.github/workflows/protocol-build-and-push.yml b/.github/workflows/protocol-build-and-push.yml index fc26dd6e69..ade5341369 100644 --- a/.github/workflows/protocol-build-and-push.yml +++ b/.github/workflows/protocol-build-and-push.yml @@ -222,4 +222,4 @@ jobs: --platform amd64 \ -t $ECR_REGISTRY/$ECR_REPOSITORY:$commit_hash \ -f testing/testnet-staging/Dockerfile . - docker push $ECR_REGISTRY/$ECR_REPOSITORY --all-tags + docker push $ECR_REGISTRY/$ECR_REPOSITORY --all-tags \ No newline at end of file diff --git a/protocol/app/app.go b/protocol/app/app.go index de242f3d21..35386767e8 100644 --- a/protocol/app/app.go +++ b/protocol/app/app.go @@ -233,6 +233,7 @@ import ( // Full Node Streaming streaming "github.com/dydxprotocol/v4-chain/protocol/streaming" streamingtypes "github.com/dydxprotocol/v4-chain/protocol/streaming/types" + "github.com/dydxprotocol/v4-chain/protocol/streaming/ws" ) var ( @@ -344,7 +345,9 @@ type App struct { IndexerEventManager indexer_manager.IndexerEventManager FullNodeStreamingManager streamingtypes.FullNodeStreamingManager - Server *daemonserver.Server + WebsocketStreamingServer *ws.WebsocketServer + + Server *daemonserver.Server // startDaemons encapsulates the logic that starts all daemons and daemon services. This function contains a // closure of all relevant data structures that are shared with various keepers. Daemon services startup is @@ -492,6 +495,9 @@ func New( if app.FullNodeStreamingManager != nil { app.FullNodeStreamingManager.Stop() } + if app.WebsocketStreamingServer != nil { + app.WebsocketStreamingServer.Shutdown() + } return nil }, ) @@ -752,7 +758,11 @@ func New( indexerFlags.SendOffchainData, ) - app.FullNodeStreamingManager = getFullNodeStreamingManagerFromOptions(appFlags, logger) + app.FullNodeStreamingManager, app.WebsocketStreamingServer = getFullNodeStreamingManagerFromOptions( + appFlags, + appCodec, + logger, + ) timeProvider := &timelib.TimeProviderImpl{} @@ -2017,20 +2027,37 @@ func getIndexerFromOptions( // from the specified options. This function will default to returning a no-op instance. func getFullNodeStreamingManagerFromOptions( appFlags flags.Flags, + cdc codec.Codec, logger log.Logger, -) (manager streamingtypes.FullNodeStreamingManager) { +) (manager streamingtypes.FullNodeStreamingManager, wsServer *ws.WebsocketServer) { + logger = logger.With(log.ModuleKey, "full-node-streaming") if appFlags.GrpcStreamingEnabled { - logger.Info("GRPC streaming is enabled", log.ModuleKey, "full-node-streaming") + logger.Info("Full node streaming is enabled") if appFlags.FullNodeStreamingSnapshotInterval > 0 { - logger.Info("Interval snapshots enabled", log.ModuleKey, "full-node-streaming") + logger.Info("Interval snapshots enabled") } - return streaming.NewFullNodeStreamingManager( + manager := streaming.NewFullNodeStreamingManager( logger, appFlags.GrpcStreamingFlushIntervalMs, appFlags.GrpcStreamingMaxBatchSize, appFlags.GrpcStreamingMaxChannelBufferSize, appFlags.FullNodeStreamingSnapshotInterval, ) + + // Start websocket server. + if appFlags.WebsocketStreamingEnabled { + port := appFlags.WebsocketStreamingPort + logger.Info("Websocket full node streaming is enabled") + wsServer = ws.NewWebsocketServer( + manager, + cdc, + logger, + port, + ) + wsServer.Start() + } + + return manager, wsServer } - return streaming.NewNoopGrpcStreamingManager() + return streaming.NewNoopGrpcStreamingManager(), wsServer } diff --git a/protocol/app/app_test.go b/protocol/app/app_test.go index a91b93ae50..c89eeaac4b 100644 --- a/protocol/app/app_test.go +++ b/protocol/app/app_test.go @@ -111,6 +111,7 @@ func TestAppIsFullyInitialized(t *testing.T) { "BridgeClient", "SlinkyClient", "oraclePrometheusServer", + "WebsocketStreamingServer", // Any default constructed type can be considered initialized if the default is what is // expected. getUninitializedStructFields relies on fields being the non-default and diff --git a/protocol/app/flags/flags.go b/protocol/app/flags/flags.go index 41b13ae826..433aea327d 100644 --- a/protocol/app/flags/flags.go +++ b/protocol/app/flags/flags.go @@ -20,11 +20,13 @@ type Flags struct { GrpcAddress string GrpcEnable bool - // Grpc Streaming + // Full Node Streaming GrpcStreamingEnabled bool GrpcStreamingFlushIntervalMs uint32 GrpcStreamingMaxBatchSize uint32 GrpcStreamingMaxChannelBufferSize uint32 + WebsocketStreamingEnabled bool + WebsocketStreamingPort uint16 FullNodeStreamingSnapshotInterval uint32 VEOracleEnabled bool // Slinky Vote Extensions @@ -48,6 +50,8 @@ const ( GrpcStreamingFlushIntervalMs = "grpc-streaming-flush-interval-ms" GrpcStreamingMaxBatchSize = "grpc-streaming-max-batch-size" GrpcStreamingMaxChannelBufferSize = "grpc-streaming-max-channel-buffer-size" + WebsocketStreamingEnabled = "websocket-streaming-enabled" + WebsocketStreamingPort = "websocket-streaming-port" FullNodeStreamingSnapshotInterval = "fns-snapshot-interval" // Slinky VEs enabled @@ -68,6 +72,8 @@ const ( DefaultGrpcStreamingFlushIntervalMs = 50 DefaultGrpcStreamingMaxBatchSize = 2000 DefaultGrpcStreamingMaxChannelBufferSize = 2000 + DefaultWebsocketStreamingEnabled = false + DefaultWebsocketStreamingPort = 9091 DefaultFullNodeStreamingSnapshotInterval = 0 DefaultVEOracleEnabled = true @@ -126,6 +132,16 @@ func AddFlagsToCmd(cmd *cobra.Command) { "If set to positive number, number of blocks between each periodic snapshot will be sent out. "+ "Defaults to zero for regular behavior of one initial snapshot.", ) + cmd.Flags().Bool( + WebsocketStreamingEnabled, + DefaultWebsocketStreamingEnabled, + "Whether to enable websocket full node streaming for full nodes", + ) + cmd.Flags().Uint16( + WebsocketStreamingPort, + DefaultWebsocketStreamingPort, + "Port for websocket full node streaming connections", + ) cmd.Flags().Bool( VEOracleEnabled, DefaultVEOracleEnabled, @@ -155,15 +171,22 @@ func (f *Flags) Validate() error { return fmt.Errorf("grpc.enable must be set to true - grpc streaming requires gRPC server") } if f.GrpcStreamingMaxBatchSize == 0 { - return fmt.Errorf("grpc streaming batch size must be positive number") + return fmt.Errorf("full node streaming batch size must be positive number") } if f.GrpcStreamingFlushIntervalMs == 0 { - return fmt.Errorf("grpc streaming flush interval must be positive number") + return fmt.Errorf("full node streaming flush interval must be positive number") } if f.GrpcStreamingMaxChannelBufferSize == 0 { - return fmt.Errorf("grpc streaming channel size must be positive number") + return fmt.Errorf("full node streaming channel size must be positive number") } } + + if f.WebsocketStreamingEnabled { + if !f.GrpcStreamingEnabled { + return fmt.Errorf("websocket full node streaming requires grpc streaming to be enabled") + } + } + if f.FullNodeStreamingSnapshotInterval > 0 && f.FullNodeStreamingSnapshotInterval < 50 { return fmt.Errorf("full node streaming snapshot interval must be >= 50 blocks or zero") } @@ -191,6 +214,8 @@ func GetFlagValuesFromOptions( GrpcStreamingFlushIntervalMs: DefaultGrpcStreamingFlushIntervalMs, GrpcStreamingMaxBatchSize: DefaultGrpcStreamingMaxBatchSize, GrpcStreamingMaxChannelBufferSize: DefaultGrpcStreamingMaxChannelBufferSize, + WebsocketStreamingEnabled: DefaultWebsocketStreamingEnabled, + WebsocketStreamingPort: DefaultWebsocketStreamingPort, FullNodeStreamingSnapshotInterval: DefaultFullNodeStreamingSnapshotInterval, VEOracleEnabled: true, @@ -258,6 +283,18 @@ func GetFlagValuesFromOptions( } } + if option := appOpts.Get(WebsocketStreamingEnabled); option != nil { + if v, err := cast.ToBoolE(option); err == nil { + result.WebsocketStreamingEnabled = v + } + } + + if option := appOpts.Get(WebsocketStreamingPort); option != nil { + if v, err := cast.ToUint16E(option); err == nil { + result.WebsocketStreamingPort = v + } + } + if option := appOpts.Get(FullNodeStreamingSnapshotInterval); option != nil { if v, err := cast.ToUint32E(option); err == nil { result.FullNodeStreamingSnapshotInterval = v diff --git a/protocol/app/flags/flags_test.go b/protocol/app/flags/flags_test.go index 84b10ef93e..caa321d651 100644 --- a/protocol/app/flags/flags_test.go +++ b/protocol/app/flags/flags_test.go @@ -5,7 +5,6 @@ import ( "testing" "github.com/cosmos/cosmos-sdk/server/config" - "github.com/dydxprotocol/v4-chain/protocol/app/flags" "github.com/dydxprotocol/v4-chain/protocol/mocks" "github.com/spf13/cobra" @@ -44,6 +43,12 @@ func TestAddFlagsToCommand(t *testing.T) { fmt.Sprintf("Has %s flag", flags.GrpcStreamingMaxChannelBufferSize): { flagName: flags.GrpcStreamingMaxChannelBufferSize, }, + fmt.Sprintf("Has %s flag", flags.WebsocketStreamingEnabled): { + flagName: flags.WebsocketStreamingEnabled, + }, + fmt.Sprintf("Has %s flag", flags.WebsocketStreamingPort): { + flagName: flags.WebsocketStreamingPort, + }, fmt.Sprintf("Has %s flag", flags.OptimisticExecutionEnabled): { flagName: flags.OptimisticExecutionEnabled, }, @@ -86,6 +91,19 @@ func TestValidate(t *testing.T) { GrpcStreamingFlushIntervalMs: 100, GrpcStreamingMaxBatchSize: 2000, GrpcStreamingMaxChannelBufferSize: 2000, + WebsocketStreamingEnabled: false, + }, + }, + "success - both grpc and websocket streaming enabled for validating nodes": { + flags: flags.Flags{ + NonValidatingFullNode: false, + GrpcEnable: true, + GrpcStreamingEnabled: true, + GrpcStreamingFlushIntervalMs: 100, + GrpcStreamingMaxBatchSize: 2000, + GrpcStreamingMaxChannelBufferSize: 2000, + WebsocketStreamingEnabled: true, + WebsocketStreamingPort: 8989, }, }, "success - optimistic execution": { @@ -118,6 +136,30 @@ func TestValidate(t *testing.T) { }, expectedErr: fmt.Errorf("grpc.enable must be set to true - grpc streaming requires gRPC server"), }, + "failure - websocket streaming enabled with gRPC streaming disabled": { + flags: flags.Flags{ + NonValidatingFullNode: true, + GrpcEnable: true, + GrpcStreamingEnabled: false, + WebsocketStreamingEnabled: true, + GrpcStreamingFlushIntervalMs: 100, + GrpcStreamingMaxBatchSize: 10000, + GrpcStreamingMaxChannelBufferSize: 10000, + }, + expectedErr: fmt.Errorf("websocket full node streaming requires grpc streaming to be enabled"), + }, + "success - websocket streaming enabled with gRPC enabled for validating node": { + flags: flags.Flags{ + NonValidatingFullNode: true, + GrpcEnable: true, + WebsocketStreamingEnabled: true, + GrpcStreamingEnabled: true, + GrpcStreamingFlushIntervalMs: 100, + GrpcStreamingMaxBatchSize: 10000, + GrpcStreamingMaxChannelBufferSize: 10000, + WebsocketStreamingPort: 8989, + }, + }, "failure - gRPC streaming enabled with zero batch size": { flags: flags.Flags{ NonValidatingFullNode: true, @@ -126,7 +168,7 @@ func TestValidate(t *testing.T) { GrpcStreamingFlushIntervalMs: 100, GrpcStreamingMaxBatchSize: 0, }, - expectedErr: fmt.Errorf("grpc streaming batch size must be positive number"), + expectedErr: fmt.Errorf("full node streaming batch size must be positive number"), }, "failure - gRPC streaming enabled with zero flush interval ms": { flags: flags.Flags{ @@ -136,7 +178,7 @@ func TestValidate(t *testing.T) { GrpcStreamingFlushIntervalMs: 0, GrpcStreamingMaxBatchSize: 2000, }, - expectedErr: fmt.Errorf("grpc streaming flush interval must be positive number"), + expectedErr: fmt.Errorf("full node streaming flush interval must be positive number"), }, "failure - gRPC streaming enabled with zero channel size ms": { flags: flags.Flags{ @@ -147,7 +189,18 @@ func TestValidate(t *testing.T) { GrpcStreamingMaxBatchSize: 2000, GrpcStreamingMaxChannelBufferSize: 0, }, - expectedErr: fmt.Errorf("grpc streaming channel size must be positive number"), + expectedErr: fmt.Errorf("full node streaming channel size must be positive number"), + }, + "failure - websocket streaming enabled with zero batch size": { + flags: flags.Flags{ + NonValidatingFullNode: true, + GrpcEnable: true, + GrpcStreamingEnabled: true, + GrpcStreamingFlushIntervalMs: 100, + GrpcStreamingMaxBatchSize: 0, + WebsocketStreamingEnabled: true, + }, + expectedErr: fmt.Errorf("full node streaming batch size must be positive number"), }, "failure - full node streaming enabled with <= 49 snapshot interval": { flags: flags.Flags{ @@ -200,6 +253,8 @@ func TestGetFlagValuesFromOptions(t *testing.T) { expectedGrpcStreamingFlushMs uint32 expectedGrpcStreamingBatchSize uint32 expectedGrpcStreamingMaxChannelBufferSize uint32 + expectedWebsocketEnabled bool + expectedWebsocketPort uint16 expectedFullNodeStreamingSnapshotInterval uint32 expectedOptimisticExecutionEnabled bool }{ @@ -213,6 +268,8 @@ func TestGetFlagValuesFromOptions(t *testing.T) { expectedGrpcStreamingFlushMs: 50, expectedGrpcStreamingBatchSize: 2000, expectedGrpcStreamingMaxChannelBufferSize: 2000, + expectedWebsocketEnabled: false, + expectedWebsocketPort: 9091, expectedFullNodeStreamingSnapshotInterval: 0, expectedOptimisticExecutionEnabled: false, }, @@ -222,11 +279,13 @@ func TestGetFlagValuesFromOptions(t *testing.T) { flags.DdAgentHost: "agentHostTest", flags.DdTraceAgentPort: uint16(777), flags.GrpcEnable: false, - flags.GrpcAddress: "localhost:9091", + flags.GrpcAddress: "localhost:1234", flags.GrpcStreamingEnabled: "true", flags.GrpcStreamingFlushIntervalMs: uint32(408), flags.GrpcStreamingMaxBatchSize: uint32(650), flags.GrpcStreamingMaxChannelBufferSize: uint32(972), + flags.WebsocketStreamingEnabled: "true", + flags.WebsocketStreamingPort: 8989, flags.FullNodeStreamingSnapshotInterval: uint32(123), flags.OptimisticExecutionEnabled: "true", }, @@ -234,11 +293,13 @@ func TestGetFlagValuesFromOptions(t *testing.T) { expectedDdAgentHost: "agentHostTest", expectedDdTraceAgentPort: 777, expectedGrpcEnable: false, - expectedGrpcAddress: "localhost:9091", + expectedGrpcAddress: "localhost:1234", expectedGrpcStreamingEnable: true, expectedGrpcStreamingFlushMs: 408, expectedGrpcStreamingBatchSize: 650, expectedGrpcStreamingMaxChannelBufferSize: 972, + expectedWebsocketEnabled: true, + expectedWebsocketPort: 8989, expectedFullNodeStreamingSnapshotInterval: 123, expectedOptimisticExecutionEnabled: true, }, @@ -303,6 +364,16 @@ func TestGetFlagValuesFromOptions(t *testing.T) { tc.expectedGrpcStreamingMaxChannelBufferSize, flags.GrpcStreamingMaxChannelBufferSize, ) + require.Equal( + t, + tc.expectedWebsocketEnabled, + flags.WebsocketStreamingEnabled, + ) + require.Equal( + t, + tc.expectedWebsocketPort, + flags.WebsocketStreamingPort, + ) }) } } diff --git a/protocol/docker-compose.yml b/protocol/docker-compose.yml index 09ab7ca45f..9d65db813e 100644 --- a/protocol/docker-compose.yml +++ b/protocol/docker-compose.yml @@ -21,6 +21,8 @@ services: - "true" - --max-daemon-unhealthy-seconds - "4294967295" # Effectively disable the daemon monitor because bridge daemon is flaky in localnet. + - --grpc-streaming-enabled + - "true" environment: # See https://docs.datadoghq.com/profiler/enabling/go/ for DD_ specific environment variables - DD_ENV=localnet_${USER} @@ -31,6 +33,7 @@ services: ports: - "26657:26657" - "9090:9090" + - "9091:9091" - "1317:1317" dydxprotocold1: diff --git a/protocol/go.mod b/protocol/go.mod index 883eac2346..b2224d6ad0 100644 --- a/protocol/go.mod +++ b/protocol/go.mod @@ -60,6 +60,7 @@ require ( github.com/deckarep/golang-set/v2 v2.6.0 github.com/ethereum/go-ethereum v1.14.7 github.com/go-kit/log v0.2.1 + github.com/gorilla/websocket v1.5.3 github.com/hashicorp/go-metrics v0.5.3 github.com/ory/dockertest/v3 v3.10.0 github.com/pelletier/go-toml v1.9.5 @@ -236,7 +237,6 @@ require ( github.com/gordonklaus/ineffassign v0.1.0 // indirect github.com/gorilla/handlers v1.5.2 // indirect github.com/gorilla/rpc v1.2.0 // indirect - github.com/gorilla/websocket v1.5.3 // indirect github.com/gostaticanalysis/analysisutil v0.7.1 // indirect github.com/gostaticanalysis/comment v1.4.2 // indirect github.com/gostaticanalysis/forcetypeassert v0.1.0 // indirect diff --git a/protocol/streaming/full_node_streaming_manager.go b/protocol/streaming/full_node_streaming_manager.go index 40051d4e11..1f98c5e1ba 100644 --- a/protocol/streaming/full_node_streaming_manager.go +++ b/protocol/streaming/full_node_streaming_manager.go @@ -82,7 +82,6 @@ func NewFullNodeStreamingManager( maxSubscriptionChannelSize uint32, snapshotBlockInterval uint32, ) *FullNodeStreamingManagerImpl { - logger = logger.With(log.ModuleKey, "full-node-streaming") fullNodeStreamingManager := &FullNodeStreamingManagerImpl{ logger: logger, orderbookSubscriptions: make(map[uint32]*OrderbookSubscription), diff --git a/protocol/streaming/ws/websocket_message_sender.go b/protocol/streaming/ws/websocket_message_sender.go new file mode 100644 index 0000000000..7a502b098b --- /dev/null +++ b/protocol/streaming/ws/websocket_message_sender.go @@ -0,0 +1,27 @@ +package ws + +import ( + "github.com/gorilla/websocket" + + "github.com/cosmos/cosmos-sdk/codec" + "github.com/dydxprotocol/v4-chain/protocol/streaming/types" + clobtypes "github.com/dydxprotocol/v4-chain/protocol/x/clob/types" +) + +var _ types.OutgoingMessageSender = (*WebsocketMessageSender)(nil) + +type WebsocketMessageSender struct { + cdc codec.JSONCodec + + conn *websocket.Conn +} + +func (wms *WebsocketMessageSender) Send( + response *clobtypes.StreamOrderbookUpdatesResponse, +) (err error) { + responseJson, err := wms.cdc.MarshalJSON(response) + if err != nil { + return err + } + return wms.conn.WriteMessage(websocket.TextMessage, responseJson) +} diff --git a/protocol/streaming/ws/websocket_server.go b/protocol/streaming/ws/websocket_server.go new file mode 100644 index 0000000000..04202143b0 --- /dev/null +++ b/protocol/streaming/ws/websocket_server.go @@ -0,0 +1,141 @@ +package ws + +import ( + "context" + "fmt" + "net/http" + "strconv" + "strings" + "time" + + "cosmossdk.io/log" + "github.com/cosmos/cosmos-sdk/codec" + "github.com/dydxprotocol/v4-chain/protocol/streaming/types" + satypes "github.com/dydxprotocol/v4-chain/protocol/x/subaccounts/types" + "github.com/gorilla/websocket" +) + +var upgrader = websocket.Upgrader{ + ReadBufferSize: 1024, + WriteBufferSize: 1024, + CheckOrigin: func(r *http.Request) bool { + return true // Allow all connections by default + }, +} + +type WebsocketServer struct { + streamingManager types.FullNodeStreamingManager + cdc codec.JSONCodec + logger log.Logger + port uint16 + server *http.Server +} + +func NewWebsocketServer( + streamingManager types.FullNodeStreamingManager, + cdc codec.JSONCodec, + logger log.Logger, + port uint16, +) *WebsocketServer { + return &WebsocketServer{ + streamingManager: streamingManager, + cdc: cdc, + logger: logger.With(log.ModuleKey, "full-node-streaming"), + port: port, + } +} + +func (ws *WebsocketServer) Handler(w http.ResponseWriter, r *http.Request) { + conn, err := upgrader.Upgrade(w, r, nil) + if err != nil { + ws.logger.Error( + "Error upgrading websocket connection", + "error", err, + ) + return + } + defer conn.Close() + + // Parse clobPairIds from query parameters + clobPairIds, err := parseClobPairIds(r) + if err != nil { + ws.logger.Error( + "Error parsing clobPairIds", + "err", err, + ) + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + websocketMessageSender := &WebsocketMessageSender{ + cdc: ws.cdc, + conn: conn, + } + + ws.logger.Info( + fmt.Sprintf("Received websocket streaming request for clob pair ids: %+v", clobPairIds), + ) + + err = ws.streamingManager.Subscribe( + clobPairIds, + // TODO@(wliu) add subaccount ids + []*satypes.SubaccountId{}, + websocketMessageSender, + ) + if err != nil { + ws.logger.Error( + "Ending handler for websocket connection", + "err", err, + ) + return + } +} + +// parseClobPairIds is a helper function to parse the clobPairIds from the query parameters. +func parseClobPairIds(r *http.Request) ([]uint32, error) { + clobPairIdsParam := r.URL.Query().Get("clobPairIds") + if clobPairIdsParam == "" { + return nil, fmt.Errorf("missing clobPairIds parameter") + } + + idStrs := strings.Split(clobPairIdsParam, ",") + clobPairIds := make([]uint32, len(idStrs)) + for i, idStr := range idStrs { + id, err := strconv.Atoi(idStr) + if err != nil { + return nil, fmt.Errorf("invalid clobPairId: %s", idStr) + } + clobPairIds[i] = uint32(id) + } + + return clobPairIds, nil +} + +// Start the websocket server in a separate goroutine. +func (ws *WebsocketServer) Start() { + go func() { + http.HandleFunc("/ws", ws.Handler) + addr := fmt.Sprintf(":%d", ws.port) + ws.logger.Info("Starting websocket server on address " + addr) + + server := &http.Server{Addr: addr} + ws.server = server + err := server.ListenAndServe() + if err != nil { + ws.logger.Error( + "Http websocket server error", + "err", err, + ) + } + ws.logger.Info("Shutting down websocket server") + }() +} + +func (ws *WebsocketServer) Shutdown() { + shutdownCtx, shutdownRelease := context.WithTimeout(context.Background(), 5*time.Second) + defer shutdownRelease() + err := ws.server.Shutdown(shutdownCtx) + if err != nil { + ws.logger.Error("Failed to shutdown websocket server", "err", err) + } +} From b620982abf07d445dfb4bf66aca5dac583ef276d Mon Sep 17 00:00:00 2001 From: Jonathan Fung <121899091+jonfung-dydx@users.noreply.github.com> Date: Wed, 14 Aug 2024 22:19:44 -0700 Subject: [PATCH 17/33] Full Node Streaming default port 9092 (#2092) --- protocol/app/flags/flags.go | 4 ++-- protocol/app/flags/flags_test.go | 2 +- protocol/docker-compose.yml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/protocol/app/flags/flags.go b/protocol/app/flags/flags.go index 433aea327d..657196ac5c 100644 --- a/protocol/app/flags/flags.go +++ b/protocol/app/flags/flags.go @@ -73,7 +73,7 @@ const ( DefaultGrpcStreamingMaxBatchSize = 2000 DefaultGrpcStreamingMaxChannelBufferSize = 2000 DefaultWebsocketStreamingEnabled = false - DefaultWebsocketStreamingPort = 9091 + DefaultWebsocketStreamingPort = 9092 DefaultFullNodeStreamingSnapshotInterval = 0 DefaultVEOracleEnabled = true @@ -140,7 +140,7 @@ func AddFlagsToCmd(cmd *cobra.Command) { cmd.Flags().Uint16( WebsocketStreamingPort, DefaultWebsocketStreamingPort, - "Port for websocket full node streaming connections", + "Port for websocket full node streaming connections. Defaults to 9092.", ) cmd.Flags().Bool( VEOracleEnabled, diff --git a/protocol/app/flags/flags_test.go b/protocol/app/flags/flags_test.go index caa321d651..e2c390eeb7 100644 --- a/protocol/app/flags/flags_test.go +++ b/protocol/app/flags/flags_test.go @@ -269,7 +269,7 @@ func TestGetFlagValuesFromOptions(t *testing.T) { expectedGrpcStreamingBatchSize: 2000, expectedGrpcStreamingMaxChannelBufferSize: 2000, expectedWebsocketEnabled: false, - expectedWebsocketPort: 9091, + expectedWebsocketPort: 9092, expectedFullNodeStreamingSnapshotInterval: 0, expectedOptimisticExecutionEnabled: false, }, diff --git a/protocol/docker-compose.yml b/protocol/docker-compose.yml index 9d65db813e..fe30ee77f9 100644 --- a/protocol/docker-compose.yml +++ b/protocol/docker-compose.yml @@ -33,7 +33,7 @@ services: ports: - "26657:26657" - "9090:9090" - - "9091:9091" + - "9092:9092" # full node streaming - "1317:1317" dydxprotocold1: From c7b6d09004551328cc8e9f285900c611286fc451 Mon Sep 17 00:00:00 2001 From: dydxwill <119354122+dydxwill@users.noreply.github.com> Date: Thu, 15 Aug 2024 10:26:01 -0400 Subject: [PATCH 18/33] [CT-1103] FNS subaccount WS support (#2088) --- .../streaming/full_node_streaming_manager.go | 2 +- protocol/streaming/ws/websocket_server.go | 48 +++++++++++++++---- 2 files changed, 40 insertions(+), 10 deletions(-) diff --git a/protocol/streaming/full_node_streaming_manager.go b/protocol/streaming/full_node_streaming_manager.go index 1f98c5e1ba..31a82855ac 100644 --- a/protocol/streaming/full_node_streaming_manager.go +++ b/protocol/streaming/full_node_streaming_manager.go @@ -148,7 +148,7 @@ func (sm *FullNodeStreamingManagerImpl) Subscribe( err error, ) { // Perform some basic validation on the request. - if len(clobPairIds) == 0 { + if len(clobPairIds) == 0 && len(subaccountIds) == 0 { return types.ErrInvalidStreamingRequest } diff --git a/protocol/streaming/ws/websocket_server.go b/protocol/streaming/ws/websocket_server.go index 04202143b0..9df21294e4 100644 --- a/protocol/streaming/ws/websocket_server.go +++ b/protocol/streaming/ws/websocket_server.go @@ -66,6 +66,16 @@ func (ws *WebsocketServer) Handler(w http.ResponseWriter, r *http.Request) { http.Error(w, err.Error(), http.StatusBadRequest) return } + // Parse subaccountIds from query parameters + subaccountIds, err := parseSubaccountIds(r) + if err != nil { + ws.logger.Error( + "Error parsing subaccountIds", + "err", err, + ) + http.Error(w, err.Error(), http.StatusBadRequest) + return + } websocketMessageSender := &WebsocketMessageSender{ cdc: ws.cdc, @@ -78,8 +88,7 @@ func (ws *WebsocketServer) Handler(w http.ResponseWriter, r *http.Request) { err = ws.streamingManager.Subscribe( clobPairIds, - // TODO@(wliu) add subaccount ids - []*satypes.SubaccountId{}, + subaccountIds, websocketMessageSender, ) if err != nil { @@ -91,21 +100,42 @@ func (ws *WebsocketServer) Handler(w http.ResponseWriter, r *http.Request) { } } +// parseSubaccountIds is a helper function to parse the subaccountIds from the query parameters. +func parseSubaccountIds(r *http.Request) ([]*satypes.SubaccountId, error) { + subaccountIdsParam := r.URL.Query().Get("subaccountIds") + idStrs := strings.Split(subaccountIdsParam, ",") + subaccountIds := make([]*satypes.SubaccountId, 0) + for _, idStr := range idStrs { + parts := strings.Split(idStr, "/") + if len(parts) != 2 { + return nil, fmt.Errorf("invalid subaccountId format: %s, expected subaccount_id format: owner/number", idStr) + } + + number, err := strconv.Atoi(parts[1]) + if err != nil { + return nil, fmt.Errorf("invalid subaccount number: %s, expected subaccount_id format: owner/number", parts[1]) + } + + subaccountIds = append(subaccountIds, &satypes.SubaccountId{ + Owner: parts[0], + Number: uint32(number), + }) + } + + return subaccountIds, nil +} + // parseClobPairIds is a helper function to parse the clobPairIds from the query parameters. func parseClobPairIds(r *http.Request) ([]uint32, error) { clobPairIdsParam := r.URL.Query().Get("clobPairIds") - if clobPairIdsParam == "" { - return nil, fmt.Errorf("missing clobPairIds parameter") - } - idStrs := strings.Split(clobPairIdsParam, ",") - clobPairIds := make([]uint32, len(idStrs)) - for i, idStr := range idStrs { + clobPairIds := make([]uint32, 0) + for _, idStr := range idStrs { id, err := strconv.Atoi(idStr) if err != nil { return nil, fmt.Errorf("invalid clobPairId: %s", idStr) } - clobPairIds[i] = uint32(id) + clobPairIds = append(clobPairIds, uint32(id)) } return clobPairIds, nil From 73fb9f6fe8eacb7ae62304cfa743a02a9641f47c Mon Sep 17 00:00:00 2001 From: dydxwill <119354122+dydxwill@users.noreply.github.com> Date: Tue, 20 Aug 2024 13:14:37 -0400 Subject: [PATCH 19/33] Support empty params for websocket endpoint (#2111) --- protocol/streaming/ws/websocket_server.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/protocol/streaming/ws/websocket_server.go b/protocol/streaming/ws/websocket_server.go index 9df21294e4..0b66804595 100644 --- a/protocol/streaming/ws/websocket_server.go +++ b/protocol/streaming/ws/websocket_server.go @@ -103,6 +103,9 @@ func (ws *WebsocketServer) Handler(w http.ResponseWriter, r *http.Request) { // parseSubaccountIds is a helper function to parse the subaccountIds from the query parameters. func parseSubaccountIds(r *http.Request) ([]*satypes.SubaccountId, error) { subaccountIdsParam := r.URL.Query().Get("subaccountIds") + if subaccountIdsParam == "" { + return []*satypes.SubaccountId{}, nil + } idStrs := strings.Split(subaccountIdsParam, ",") subaccountIds := make([]*satypes.SubaccountId, 0) for _, idStr := range idStrs { @@ -128,6 +131,9 @@ func parseSubaccountIds(r *http.Request) ([]*satypes.SubaccountId, error) { // parseClobPairIds is a helper function to parse the clobPairIds from the query parameters. func parseClobPairIds(r *http.Request) ([]uint32, error) { clobPairIdsParam := r.URL.Query().Get("clobPairIds") + if clobPairIdsParam == "" { + return []uint32{}, nil + } idStrs := strings.Split(clobPairIdsParam, ",") clobPairIds := make([]uint32, 0) for _, idStr := range idStrs { From a918df9ab306c42f943ca9b489cb1027326d1b85 Mon Sep 17 00:00:00 2001 From: Jonathan Fung <121899091+jonfung-dydx@users.noreply.github.com> Date: Thu, 22 Aug 2024 23:55:32 +0900 Subject: [PATCH 20/33] Full node streaming remove minimum for snapshot interval flag (#2138) --- protocol/app/flags/flags.go | 4 ---- protocol/app/flags/flags_test.go | 16 ++-------------- 2 files changed, 2 insertions(+), 18 deletions(-) diff --git a/protocol/app/flags/flags.go b/protocol/app/flags/flags.go index 657196ac5c..cb1715ba62 100644 --- a/protocol/app/flags/flags.go +++ b/protocol/app/flags/flags.go @@ -187,10 +187,6 @@ func (f *Flags) Validate() error { } } - if f.FullNodeStreamingSnapshotInterval > 0 && f.FullNodeStreamingSnapshotInterval < 50 { - return fmt.Errorf("full node streaming snapshot interval must be >= 50 blocks or zero") - } - return nil } diff --git a/protocol/app/flags/flags_test.go b/protocol/app/flags/flags_test.go index e2c390eeb7..50e3a32c72 100644 --- a/protocol/app/flags/flags_test.go +++ b/protocol/app/flags/flags_test.go @@ -202,7 +202,7 @@ func TestValidate(t *testing.T) { }, expectedErr: fmt.Errorf("full node streaming batch size must be positive number"), }, - "failure - full node streaming enabled with <= 49 snapshot interval": { + "success - full node streaming enabled with 20 snapshot interval": { flags: flags.Flags{ NonValidatingFullNode: true, GrpcEnable: true, @@ -210,19 +210,7 @@ func TestValidate(t *testing.T) { GrpcStreamingFlushIntervalMs: 100, GrpcStreamingMaxBatchSize: 2000, GrpcStreamingMaxChannelBufferSize: 2000, - FullNodeStreamingSnapshotInterval: 49, - }, - expectedErr: fmt.Errorf("full node streaming snapshot interval must be >= 50 blocks or zero"), - }, - "success - full node streaming enabled with 50 snapshot interval": { - flags: flags.Flags{ - NonValidatingFullNode: true, - GrpcEnable: true, - GrpcStreamingEnabled: true, - GrpcStreamingFlushIntervalMs: 100, - GrpcStreamingMaxBatchSize: 2000, - GrpcStreamingMaxChannelBufferSize: 2000, - FullNodeStreamingSnapshotInterval: 50, + FullNodeStreamingSnapshotInterval: 20, }, }, } From 8ed863f2fcf87d7bf3dfa8d1bdfb7ffb57b81912 Mon Sep 17 00:00:00 2001 From: dydxwill <119354122+dydxwill@users.noreply.github.com> Date: Fri, 23 Aug 2024 07:08:02 -0400 Subject: [PATCH 21/33] [CT-968] Fix snapshot validation by including subaccounts from proposed operation queue (#2142) --- protocol/x/clob/keeper/process_operations.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/protocol/x/clob/keeper/process_operations.go b/protocol/x/clob/keeper/process_operations.go index 2e349ed0a6..486dd6ab0e 100644 --- a/protocol/x/clob/keeper/process_operations.go +++ b/protocol/x/clob/keeper/process_operations.go @@ -95,10 +95,14 @@ func (k Keeper) ProcessProposerOperations( } k.SendOrderbookUpdates(ctx, allUpdates) - // send local subaccount snapshots - subaccountIdsToUpdate := fetchSubaccountIdsInvolvedInOpQueue( + subaccountIdsFromProposed := fetchSubaccountIdsInvolvedInOpQueue( + operations, + ) + + subaccountIdsFromLocal := fetchSubaccountIdsInvolvedInOpQueue( localValidatorOperationsQueue, ) + subaccountIdsToUpdate := lib.MergeMaps(subaccountIdsFromLocal, subaccountIdsFromProposed) allSubaccountUpdates := make([]satypes.StreamSubaccountUpdate, 0) for subaccountId := range subaccountIdsToUpdate { subaccountUpdate := k.subaccountsKeeper.GetStreamSubaccountUpdate(ctx, subaccountId, false) From 6a80188364219c95b20e59b1e0343fbe49ce318a Mon Sep 17 00:00:00 2001 From: Teddy Ding Date: Mon, 9 Sep 2024 11:33:24 -0400 Subject: [PATCH 22/33] [CT-1178] Only emit finalized subaccount update (#2181) --- protocol/lib/metrics/metric_keys.go | 26 ++--- protocol/mocks/ClobKeeper.go | 8 +- .../streaming/full_node_streaming_manager.go | 104 ++++++++++++------ protocol/streaming/noop_streaming_manager.go | 10 +- protocol/streaming/types/interface.go | 7 +- protocol/x/clob/abci.go | 16 ++- protocol/x/clob/keeper/keeper.go | 36 ++++-- protocol/x/clob/keeper/process_operations.go | 36 +----- protocol/x/clob/types/clob_keeper.go | 5 +- protocol/x/clob/types/expected_keepers.go | 2 +- protocol/x/subaccounts/keeper/subaccount.go | 16 +-- protocol/x/subaccounts/types/types.go | 2 +- 12 files changed, 160 insertions(+), 108 deletions(-) diff --git a/protocol/lib/metrics/metric_keys.go b/protocol/lib/metrics/metric_keys.go index 088e742386..e38890d8ac 100644 --- a/protocol/lib/metrics/metric_keys.go +++ b/protocol/lib/metrics/metric_keys.go @@ -66,19 +66,19 @@ const ( GateWithdrawalsIfNegativeTncSubaccountSeenLatency = "gate_withdrawals_if_negative_tnc_subaccount_seen_latency" // Full node grpc - FullNodeGrpc = "full_node_grpc" - GrpcSendOrderbookUpdatesLatency = "grpc_send_orderbook_updates_latency" - GrpcSendOrderbookSnapshotLatency = "grpc_send_orderbook_snapshot_latency" - GrpcSendSubaccountSnapshotLatency = "grpc_send_subaccount_snapshot_latency" - GrpcSendOrderbookFillsLatency = "grpc_send_orderbook_fills_latency" - GrpcSendSubaccountUpdatesLatency = "grpc_send_subaccount_updates_latency" - GrpcAddUpdateToBufferCount = "grpc_add_update_to_buffer_count" - GrpcAddToSubscriptionChannelCount = "grpc_add_to_subscription_channel_count" - GrpcSendResponseToSubscriberCount = "grpc_send_response_to_subscriber_count" - GrpcStreamSubscriberCount = "grpc_stream_subscriber_count" - GrpcStreamNumUpdatesBuffered = "grpc_stream_num_updates_buffered" - GrpcFlushUpdatesLatency = "grpc_flush_updates_latency" - GrpcSubscriptionChannelLength = "grpc_subscription_channel_length" + FullNodeGrpc = "full_node_grpc" + GrpcSendOrderbookUpdatesLatency = "grpc_send_orderbook_updates_latency" + GrpcSendOrderbookSnapshotLatency = "grpc_send_orderbook_snapshot_latency" + GrpcSendSubaccountSnapshotLatency = "grpc_send_subaccount_snapshot_latency" + GrpcSendOrderbookFillsLatency = "grpc_send_orderbook_fills_latency" + GrpcSendFinalizedSubaccountUpdatesLatency = "grpc_send_finalized_subaccount_updates_latency" + GrpcAddUpdateToBufferCount = "grpc_add_update_to_buffer_count" + GrpcAddToSubscriptionChannelCount = "grpc_add_to_subscription_channel_count" + GrpcSendResponseToSubscriberCount = "grpc_send_response_to_subscriber_count" + GrpcStreamSubscriberCount = "grpc_stream_subscriber_count" + GrpcStreamNumUpdatesBuffered = "grpc_stream_num_updates_buffered" + GrpcFlushUpdatesLatency = "grpc_flush_updates_latency" + GrpcSubscriptionChannelLength = "grpc_subscription_channel_length" EndBlocker = "end_blocker" EndBlockerLag = "end_blocker_lag" diff --git a/protocol/mocks/ClobKeeper.go b/protocol/mocks/ClobKeeper.go index 5eb830f15d..62ab311a6b 100644 --- a/protocol/mocks/ClobKeeper.go +++ b/protocol/mocks/ClobKeeper.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.43.2. DO NOT EDIT. +// Code generated by mockery v2.44.1. DO NOT EDIT. package mocks @@ -706,9 +706,9 @@ func (_m *ClobKeeper) InitializeEquityTierLimit(ctx types.Context, config clobty return r0 } -// InitializeNewStreams provides a mock function with given fields: ctx -func (_m *ClobKeeper) InitializeNewStreams(ctx types.Context) { - _m.Called(ctx) +// InitializeNewStreams provides a mock function with given fields: ctx, subaccountSnapshots +func (_m *ClobKeeper) InitializeNewStreams(ctx types.Context, subaccountSnapshots map[subaccountstypes.SubaccountId]*subaccountstypes.StreamSubaccountUpdate) { + _m.Called(ctx, subaccountSnapshots) } // IsInitialized provides a mock function with given fields: diff --git a/protocol/streaming/full_node_streaming_manager.go b/protocol/streaming/full_node_streaming_manager.go index 31a82855ac..a6fe37c512 100644 --- a/protocol/streaming/full_node_streaming_manager.go +++ b/protocol/streaming/full_node_streaming_manager.go @@ -3,6 +3,7 @@ package streaming import ( "fmt" "sync" + "sync/atomic" "time" satypes "github.com/dydxprotocol/v4-chain/protocol/x/subaccounts/types" @@ -55,8 +56,8 @@ type FullNodeStreamingManagerImpl struct { type OrderbookSubscription struct { subscriptionId uint32 - // Initialize the subscription with orderbook snapshots. - initialize *sync.Once + // Whether the subscription is initialized with snapshot. + initialized *atomic.Bool // Clob pair ids to subscribe to. clobPairIds []uint32 @@ -75,6 +76,10 @@ type OrderbookSubscription struct { nextSnapshotBlock uint32 } +func (sub *OrderbookSubscription) IsInitialized() bool { + return sub.initialized.Load() +} + func NewFullNodeStreamingManager( logger log.Logger, flushIntervalMs uint32, @@ -159,7 +164,7 @@ func (sm *FullNodeStreamingManagerImpl) Subscribe( } subscription := &OrderbookSubscription{ subscriptionId: sm.nextSubscriptionId, - initialize: &sync.Once{}, + initialized: &atomic.Bool{}, // False by default. clobPairIds: clobPairIds, subaccountIds: sIds, messageSender: messageSender, @@ -511,19 +516,23 @@ func (sm *FullNodeStreamingManagerImpl) SendTakerOrderStatus( ) } -// SendSubaccountUpdates groups subaccount updates by their subaccount ids and +// SendFinalizedSubaccountUpdates groups subaccount updates by their subaccount ids and // sends messages to the subscribers. -func (sm *FullNodeStreamingManagerImpl) SendSubaccountUpdates( +func (sm *FullNodeStreamingManagerImpl) SendFinalizedSubaccountUpdates( subaccountUpdates []satypes.StreamSubaccountUpdate, blockHeight uint32, execMode sdk.ExecMode, ) { defer metrics.ModuleMeasureSince( metrics.FullNodeGrpc, - metrics.GrpcSendSubaccountUpdatesLatency, + metrics.GrpcSendFinalizedSubaccountUpdatesLatency, time.Now(), ) + if execMode != sdk.ExecModeFinalize { + panic("SendFinalizedSubaccountUpdates should only be called in ExecModeFinalize") + } + // Group subaccount updates by subaccount id. streamUpdates := make([]clobtypes.StreamUpdate, 0) subaccountIds := make([]*satypes.SubaccountId, 0) @@ -670,9 +679,33 @@ func (sm *FullNodeStreamingManagerImpl) FlushStreamUpdatesWithLock() { sm.EmitMetrics() } +func (sm *FullNodeStreamingManagerImpl) GetSubaccountSnapshotsForInitStreams( + getSubaccountSnapshot func(subaccountId satypes.SubaccountId) *satypes.StreamSubaccountUpdate, +) map[satypes.SubaccountId]*satypes.StreamSubaccountUpdate { + sm.Lock() + defer sm.Unlock() + + ret := make(map[satypes.SubaccountId]*satypes.StreamSubaccountUpdate) + for _, subscription := range sm.orderbookSubscriptions { + // If the subscription has been initialized, no need to grab the subaccount snapshot. + if alreadyInitialized := subscription.initialized.Load(); alreadyInitialized { + continue + } + + for _, subaccountId := range subscription.subaccountIds { + if _, exists := ret[subaccountId]; exists { + continue + } + + ret[subaccountId] = getSubaccountSnapshot(subaccountId) + } + } + return ret +} + func (sm *FullNodeStreamingManagerImpl) InitializeNewStreams( getOrderbookSnapshot func(clobPairId clobtypes.ClobPairId) *clobtypes.OffchainUpdates, - getSubaccountSnapshot func(subaccountId satypes.SubaccountId) *satypes.StreamSubaccountUpdate, + subaccountSnapshots map[satypes.SubaccountId]*satypes.StreamSubaccountUpdate, blockHeight uint32, execMode sdk.ExecMode, ) { @@ -686,31 +719,40 @@ func (sm *FullNodeStreamingManagerImpl) InitializeNewStreams( updatesByClobPairId := make(map[uint32]*clobtypes.OffchainUpdates) for subscriptionId, subscription := range sm.orderbookSubscriptions { - // If the snapshot block interval is enabled, reset the sync.Once in order to - // re-send snapshots out. - if sm.snapshotBlockInterval > 0 && - blockHeight == subscription.nextSnapshotBlock { - subscription.initialize = &sync.Once{} - } - - subscription.initialize.Do( - func() { - allUpdates := clobtypes.NewOffchainUpdates() - for _, clobPairId := range subscription.clobPairIds { - if _, ok := updatesByClobPairId[clobPairId]; !ok { - updatesByClobPairId[clobPairId] = getOrderbookSnapshot(clobtypes.ClobPairId(clobPairId)) - } - allUpdates.Append(updatesByClobPairId[clobPairId]) - } - saUpdates := []*satypes.StreamSubaccountUpdate{} - for _, subaccountId := range subscription.subaccountIds { - saUpdates = append(saUpdates, getSubaccountSnapshot(subaccountId)) + if alreadyInitialized := subscription.initialized.Swap(true); !alreadyInitialized { + allUpdates := clobtypes.NewOffchainUpdates() + for _, clobPairId := range subscription.clobPairIds { + if _, ok := updatesByClobPairId[clobPairId]; !ok { + updatesByClobPairId[clobPairId] = getOrderbookSnapshot(clobtypes.ClobPairId(clobPairId)) } - sm.SendCombinedSnapshot(allUpdates, saUpdates, subscriptionId, blockHeight, execMode) - if sm.snapshotBlockInterval != 0 { - subscription.nextSnapshotBlock = blockHeight + sm.snapshotBlockInterval + allUpdates.Append(updatesByClobPairId[clobPairId]) + } + + saUpdates := []*satypes.StreamSubaccountUpdate{} + for _, subaccountId := range subscription.subaccountIds { + // The subaccount snapshot may not exist due to the following race condition + // 1. At beginning of PrepareCheckState we get snapshot for all subscribed subaccounts. + // 2. A new subaccount is subscribed to by a new subscription. + // 3. InitializeNewStreams is called. + // Then the new subaccount would not be included in the snapshot. + // We are okay with this behavior. + if saUpdate, ok := subaccountSnapshots[subaccountId]; ok { + saUpdates = append(saUpdates, saUpdate) } - }, - ) + } + + sm.SendCombinedSnapshot(allUpdates, saUpdates, subscriptionId, blockHeight, execMode) + + if sm.snapshotBlockInterval != 0 { + subscription.nextSnapshotBlock = blockHeight + sm.snapshotBlockInterval + } + } + + // If the snapshot block interval is enabled and the next block is a snapshot block, + // reset the `atomic.Bool` so snapshots are sent for the next block. + if sm.snapshotBlockInterval > 0 && + blockHeight+1 == subscription.nextSnapshotBlock { + subscription.initialized = &atomic.Bool{} // False by default. + } } } diff --git a/protocol/streaming/noop_streaming_manager.go b/protocol/streaming/noop_streaming_manager.go index 24810fefe2..6e9c00895d 100644 --- a/protocol/streaming/noop_streaming_manager.go +++ b/protocol/streaming/noop_streaming_manager.go @@ -51,7 +51,7 @@ func (sm *NoopGrpcStreamingManager) SendTakerOrderStatus( ) { } -func (sm *NoopGrpcStreamingManager) SendSubaccountUpdates( +func (sm *NoopGrpcStreamingManager) SendFinalizedSubaccountUpdates( subaccountUpdates []satypes.StreamSubaccountUpdate, blockHeight uint32, execMode sdk.ExecMode, @@ -62,9 +62,15 @@ func (sm *NoopGrpcStreamingManager) TracksSubaccountId(id satypes.SubaccountId) return false } +func (sm *NoopGrpcStreamingManager) GetSubaccountSnapshotsForInitStreams( + getSubaccountSnapshot func(subaccountId satypes.SubaccountId) *satypes.StreamSubaccountUpdate, +) map[satypes.SubaccountId]*satypes.StreamSubaccountUpdate { + return nil +} + func (sm *NoopGrpcStreamingManager) InitializeNewStreams( getOrderbookSnapshot func(clobPairId clobtypes.ClobPairId) *clobtypes.OffchainUpdates, - getSubaccountSnapshot func(subaccountId satypes.SubaccountId) *satypes.StreamSubaccountUpdate, + subaccountSnapshots map[satypes.SubaccountId]*satypes.StreamSubaccountUpdate, blockHeight uint32, execMode sdk.ExecMode, ) { diff --git a/protocol/streaming/types/interface.go b/protocol/streaming/types/interface.go index 7930853be6..e3dff9d94b 100644 --- a/protocol/streaming/types/interface.go +++ b/protocol/streaming/types/interface.go @@ -22,10 +22,13 @@ type FullNodeStreamingManager interface { // L3+ Orderbook updates. InitializeNewStreams( getOrderbookSnapshot func(clobPairId clobtypes.ClobPairId) *clobtypes.OffchainUpdates, - getSubaccountSnapshot func(subaccountId satypes.SubaccountId) *satypes.StreamSubaccountUpdate, + subaccountSnapshots map[satypes.SubaccountId]*satypes.StreamSubaccountUpdate, blockHeight uint32, execMode sdk.ExecMode, ) + GetSubaccountSnapshotsForInitStreams( + getSubaccountSnapshot func(subaccountId satypes.SubaccountId) *satypes.StreamSubaccountUpdate, + ) map[satypes.SubaccountId]*satypes.StreamSubaccountUpdate SendOrderbookUpdates( offchainUpdates *clobtypes.OffchainUpdates, blockHeight uint32, @@ -42,7 +45,7 @@ type FullNodeStreamingManager interface { blockHeight uint32, execMode sdk.ExecMode, ) - SendSubaccountUpdates( + SendFinalizedSubaccountUpdates( subaccountUpdates []satypes.StreamSubaccountUpdate, blockHeight uint32, execMode sdk.ExecMode, diff --git a/protocol/x/clob/abci.go b/protocol/x/clob/abci.go index 641e49e897..96bacb2cc5 100644 --- a/protocol/x/clob/abci.go +++ b/protocol/x/clob/abci.go @@ -14,6 +14,7 @@ import ( "github.com/dydxprotocol/v4-chain/protocol/lib/metrics" "github.com/dydxprotocol/v4-chain/protocol/x/clob/keeper" "github.com/dydxprotocol/v4-chain/protocol/x/clob/types" + satypes "github.com/dydxprotocol/v4-chain/protocol/x/subaccounts/types" ) // PreBlocker executes all ABCI PreBlock logic respective to the clob module. @@ -123,6 +124,15 @@ func PrepareCheckState( log.BlockHeight, ctx.BlockHeight()+1, ) + // We just committed block `h`, preparing `CheckState` of `h+1` + // Before we modify the `CheckState`, we first take the snapshot of + // the subscribed subaccounts at the end of block `h`. This we send finalized state of + // the subaccounts below in `InitializeNewStreams`. + var subaccountSnapshots map[satypes.SubaccountId]*satypes.StreamSubaccountUpdate + if keeper.GetFullNodeStreamingManager().Enabled() { + subaccountSnapshots = keeper.GetSubaccountSnapshotsForInitStreams(ctx) + } + // Prune any rate limiting information that is no longer relevant. keeper.PruneRateLimits(ctx) @@ -239,7 +249,11 @@ func PrepareCheckState( ) // Initialize new streams with orderbook snapshots, if any. - keeper.InitializeNewStreams(ctx) + keeper.InitializeNewStreams( + ctx, + // Use the subaccount snapshot at the top of function to initialize the streams. + subaccountSnapshots, + ) // Set per-orderbook gauges. keeper.MemClob.SetMemclobGauges(ctx) diff --git a/protocol/x/clob/keeper/keeper.go b/protocol/x/clob/keeper/keeper.go index 6817b43469..8e607d6833 100644 --- a/protocol/x/clob/keeper/keeper.go +++ b/protocol/x/clob/keeper/keeper.go @@ -3,9 +3,10 @@ package keeper import ( "errors" "fmt" - satypes "github.com/dydxprotocol/v4-chain/protocol/x/subaccounts/types" "sync/atomic" + satypes "github.com/dydxprotocol/v4-chain/protocol/x/subaccounts/types" + "cosmossdk.io/log" "cosmossdk.io/store/prefix" storetypes "cosmossdk.io/store/types" @@ -251,9 +252,31 @@ func (k *Keeper) SetAnteHandler(anteHandler sdk.AnteHandler) { k.antehandler = anteHandler } +func (k Keeper) GetSubaccountSnapshotsForInitStreams( + ctx sdk.Context, +) ( + subaccountSnapshots map[satypes.SubaccountId]*satypes.StreamSubaccountUpdate, +) { + lib.AssertCheckTxMode(ctx) + + return k.GetFullNodeStreamingManager().GetSubaccountSnapshotsForInitStreams( + func(subaccountId satypes.SubaccountId) *satypes.StreamSubaccountUpdate { + subaccountUpdate := k.subaccountsKeeper.GetStreamSubaccountUpdate( + ctx, + subaccountId, + true, + ) + return &subaccountUpdate + }, + ) +} + // InitializeNewStreams initializes new streams for all uninitialized clob pairs // by sending the corresponding orderbook snapshots. -func (k Keeper) InitializeNewStreams(ctx sdk.Context) { +func (k Keeper) InitializeNewStreams( + ctx sdk.Context, + subaccountSnapshots map[satypes.SubaccountId]*satypes.StreamSubaccountUpdate, +) { streamingManager := k.GetFullNodeStreamingManager() streamingManager.InitializeNewStreams( @@ -263,14 +286,7 @@ func (k Keeper) InitializeNewStreams(ctx sdk.Context) { clobPairId, ) }, - func(subaccountId satypes.SubaccountId) *satypes.StreamSubaccountUpdate { - subaccountUpdate := k.subaccountsKeeper.GetStreamSubaccountUpdate( - ctx, - subaccountId, - true, - ) - return &subaccountUpdate - }, + subaccountSnapshots, lib.MustConvertIntegerToUint32(ctx.BlockHeight()), ctx.ExecMode(), ) diff --git a/protocol/x/clob/keeper/process_operations.go b/protocol/x/clob/keeper/process_operations.go index 486dd6ab0e..d202c2a270 100644 --- a/protocol/x/clob/keeper/process_operations.go +++ b/protocol/x/clob/keeper/process_operations.go @@ -38,21 +38,6 @@ func fetchOrdersInvolvedInOpQueue( return orderIdSet } -// fetchSubaccountIdsInvolvedInOpQueue fetches all SubaccountIds involved in an operations -// queue's matches and returns them as a set. -func fetchSubaccountIdsInvolvedInOpQueue( - operations []types.InternalOperation, -) (subaccountIdSet map[satypes.SubaccountId]struct{}) { - subaccountIdSet = make(map[satypes.SubaccountId]struct{}) - for _, operation := range operations { - if clobMatch := operation.GetMatch(); clobMatch != nil { - subaccountIdSetForClobMatch := clobMatch.GetAllSubaccountIds() - subaccountIdSet = lib.MergeMaps(subaccountIdSet, subaccountIdSetForClobMatch) - } - } - return subaccountIdSet -} - // ProcessProposerOperations updates on-chain state given an []OperationRaw operations queue // representing matches that occurred in the previous block. It performs validation on an operations // queue. If all validation passes, the operations queue is written to state. @@ -73,11 +58,7 @@ func (k Keeper) ProcessProposerOperations( } // If grpc streams are on, send absolute fill amounts from local + proposed opqueue to the grpc stream. - // Also send subaccount snapshots for impacted subaccounts. - // An impacted subaccount is defined as: - // - A subaccount that was involved in any match in the local opqueue. - // Only matches generate subaccount updates. - // This must be sent out to account for checkState being discarded and deliverState being used. + // This effetively reverts the optimitic orderbook updates during CheckTx. if streamingManager := k.GetFullNodeStreamingManager(); streamingManager.Enabled() { localValidatorOperationsQueue, _ := k.MemClob.GetOperationsToReplay(ctx) orderIdsFromProposed := fetchOrdersInvolvedInOpQueue( @@ -94,21 +75,6 @@ func (k Keeper) ProcessProposerOperations( allUpdates.Append(orderbookUpdate) } k.SendOrderbookUpdates(ctx, allUpdates) - - subaccountIdsFromProposed := fetchSubaccountIdsInvolvedInOpQueue( - operations, - ) - - subaccountIdsFromLocal := fetchSubaccountIdsInvolvedInOpQueue( - localValidatorOperationsQueue, - ) - subaccountIdsToUpdate := lib.MergeMaps(subaccountIdsFromLocal, subaccountIdsFromProposed) - allSubaccountUpdates := make([]satypes.StreamSubaccountUpdate, 0) - for subaccountId := range subaccountIdsToUpdate { - subaccountUpdate := k.subaccountsKeeper.GetStreamSubaccountUpdate(ctx, subaccountId, false) - allSubaccountUpdates = append(allSubaccountUpdates, subaccountUpdate) - } - k.subaccountsKeeper.SendSubaccountUpdates(ctx, allSubaccountUpdates) } log.DebugLog(ctx, "Processing operations queue", diff --git a/protocol/x/clob/types/clob_keeper.go b/protocol/x/clob/types/clob_keeper.go index a961046f23..9705439dc4 100644 --- a/protocol/x/clob/types/clob_keeper.go +++ b/protocol/x/clob/types/clob_keeper.go @@ -137,7 +137,10 @@ type ClobKeeper interface { ) error UpdateLiquidationsConfig(ctx sdk.Context, config LiquidationsConfig) error // full node streaming - InitializeNewStreams(ctx sdk.Context) + InitializeNewStreams( + ctx sdk.Context, + subaccountSnapshots map[satypes.SubaccountId]*satypes.StreamSubaccountUpdate, + ) SendOrderbookUpdates( ctx sdk.Context, offchainUpdates *OffchainUpdates, diff --git a/protocol/x/clob/types/expected_keepers.go b/protocol/x/clob/types/expected_keepers.go index 3ed63c0592..b14c4aeacd 100644 --- a/protocol/x/clob/types/expected_keepers.go +++ b/protocol/x/clob/types/expected_keepers.go @@ -85,7 +85,7 @@ type SubaccountsKeeper interface { quantums *big.Int, perpetualId uint32, ) error - SendSubaccountUpdates( + SendFinalizedSubaccountUpdates( ctx sdk.Context, subaccountUpdates []satypes.StreamSubaccountUpdate, ) diff --git a/protocol/x/subaccounts/keeper/subaccount.go b/protocol/x/subaccounts/keeper/subaccount.go index bd7bf619dc..a2fb54aacb 100644 --- a/protocol/x/subaccounts/keeper/subaccount.go +++ b/protocol/x/subaccounts/keeper/subaccount.go @@ -3,11 +3,12 @@ package keeper import ( "errors" "fmt" - streamingtypes "github.com/dydxprotocol/v4-chain/protocol/streaming/types" "math/big" "math/rand" "time" + streamingtypes "github.com/dydxprotocol/v4-chain/protocol/streaming/types" + authtypes "github.com/cosmos/cosmos-sdk/x/auth/types" "github.com/cosmos/gogoproto/proto" @@ -440,11 +441,11 @@ func (k Keeper) UpdateSubaccounts( ), ) - // if GRPC streaming is on, emit a generated subaccount update to stream. - if streamingManager := k.GetFullNodeStreamingManager(); streamingManager.Enabled() { + // If DeliverTx and GRPC streaming is on, emit a generated subaccount update to stream. + if lib.IsDeliverTxMode(ctx) && k.GetFullNodeStreamingManager().Enabled() { if k.GetFullNodeStreamingManager().TracksSubaccountId(*u.SettledSubaccount.Id) { subaccountUpdate := GenerateStreamSubaccountUpdate(u, fundingPayments) - k.SendSubaccountUpdates( + k.SendFinalizedSubaccountUpdates( ctx, []types.StreamSubaccountUpdate{ subaccountUpdate, @@ -827,15 +828,16 @@ func (k Keeper) GetFullNodeStreamingManager() streamingtypes.FullNodeStreamingMa return k.streamingManager } -// SendSubaccountUpdates sends the subaccount updates to the gRPC streaming manager. -func (k Keeper) SendSubaccountUpdates( +// SendFinalizedSubaccountUpdates sends the subaccount updates to the gRPC streaming manager. +func (k Keeper) SendFinalizedSubaccountUpdates( ctx sdk.Context, subaccountUpdates []types.StreamSubaccountUpdate, ) { + lib.AssertDeliverTxMode(ctx) if len(subaccountUpdates) == 0 { return } - k.GetFullNodeStreamingManager().SendSubaccountUpdates( + k.GetFullNodeStreamingManager().SendFinalizedSubaccountUpdates( subaccountUpdates, lib.MustConvertIntegerToUint32(ctx.BlockHeight()), ctx.ExecMode(), diff --git a/protocol/x/subaccounts/types/types.go b/protocol/x/subaccounts/types/types.go index 3effca3878..cbccc9d2b9 100644 --- a/protocol/x/subaccounts/types/types.go +++ b/protocol/x/subaccounts/types/types.go @@ -77,7 +77,7 @@ type SubaccountsKeeper interface { perpetualId uint32, blockHeight uint32, ) error - SendSubaccountUpdates( + SendFinalizedSubaccountUpdates( ctx sdk.Context, subaccountUpdates []StreamSubaccountUpdate, ) From 9558320704bbeb8438b3df30f712d5ca1d4247e7 Mon Sep 17 00:00:00 2001 From: Jonathan Fung <121899091+jonfung-dydx@users.noreply.github.com> Date: Mon, 9 Sep 2024 16:43:44 -0400 Subject: [PATCH 23/33] Full Node Streaming - reorder protobuf single length fields before variable length fields (#2227) --- .../src/codegen/dydxprotocol/clob/query.ts | 106 ++-- proto/dydxprotocol/clob/query.proto | 32 +- protocol/x/clob/types/query.pb.go | 458 +++++++++--------- 3 files changed, 297 insertions(+), 299 deletions(-) diff --git a/indexer/packages/v4-protos/src/codegen/dydxprotocol/clob/query.ts b/indexer/packages/v4-protos/src/codegen/dydxprotocol/clob/query.ts index 27872be567..41ca0c872a 100644 --- a/indexer/packages/v4-protos/src/codegen/dydxprotocol/clob/query.ts +++ b/indexer/packages/v4-protos/src/codegen/dydxprotocol/clob/query.ts @@ -293,16 +293,15 @@ export interface StreamOrderbookUpdatesResponseSDKType { */ export interface StreamUpdate { - orderbookUpdate?: StreamOrderbookUpdate; - orderFill?: StreamOrderbookFill; - takerOrder?: StreamTakerOrder; - subaccountUpdate?: StreamSubaccountUpdate; /** Block height of the update. */ - blockHeight: number; /** Exec mode of the update. */ execMode: number; + orderbookUpdate?: StreamOrderbookUpdate; + orderFill?: StreamOrderbookFill; + takerOrder?: StreamTakerOrder; + subaccountUpdate?: StreamSubaccountUpdate; } /** * StreamUpdate is an update that will be pushed through the @@ -310,16 +309,15 @@ export interface StreamUpdate { */ export interface StreamUpdateSDKType { - orderbook_update?: StreamOrderbookUpdateSDKType; - order_fill?: StreamOrderbookFillSDKType; - taker_order?: StreamTakerOrderSDKType; - subaccount_update?: StreamSubaccountUpdateSDKType; /** Block height of the update. */ - block_height: number; /** Exec mode of the update. */ exec_mode: number; + orderbook_update?: StreamOrderbookUpdateSDKType; + order_fill?: StreamOrderbookFillSDKType; + taker_order?: StreamTakerOrderSDKType; + subaccount_update?: StreamSubaccountUpdateSDKType; } /** * StreamOrderbookUpdate provides information on an orderbook update. Used in @@ -327,19 +325,19 @@ export interface StreamUpdateSDKType { */ export interface StreamOrderbookUpdate { - /** - * Orderbook updates for the clob pair. Can contain order place, removals, - * or updates. - */ - updates: OffChainUpdateV1[]; /** * Snapshot indicates if the response is from a snapshot of the orderbook. * All updates should be ignored until snapshot is recieved. * If the snapshot is true, then all previous entries should be * discarded and the orderbook should be resynced. */ - snapshot: boolean; + /** + * Orderbook updates for the clob pair. Can contain order place, removals, + * or updates. + */ + + updates: OffChainUpdateV1[]; } /** * StreamOrderbookUpdate provides information on an orderbook update. Used in @@ -347,19 +345,19 @@ export interface StreamOrderbookUpdate { */ export interface StreamOrderbookUpdateSDKType { - /** - * Orderbook updates for the clob pair. Can contain order place, removals, - * or updates. - */ - updates: OffChainUpdateV1SDKType[]; /** * Snapshot indicates if the response is from a snapshot of the orderbook. * All updates should be ignored until snapshot is recieved. * If the snapshot is true, then all previous entries should be * discarded and the orderbook should be resynced. */ - snapshot: boolean; + /** + * Orderbook updates for the clob pair. Can contain order place, removals, + * or updates. + */ + + updates: OffChainUpdateV1SDKType[]; } /** * StreamOrderbookFill provides information on an orderbook fill. Used in @@ -1305,39 +1303,39 @@ export const StreamOrderbookUpdatesResponse = { function createBaseStreamUpdate(): StreamUpdate { return { + blockHeight: 0, + execMode: 0, orderbookUpdate: undefined, orderFill: undefined, takerOrder: undefined, - subaccountUpdate: undefined, - blockHeight: 0, - execMode: 0 + subaccountUpdate: undefined }; } export const StreamUpdate = { encode(message: StreamUpdate, writer: _m0.Writer = _m0.Writer.create()): _m0.Writer { - if (message.orderbookUpdate !== undefined) { - StreamOrderbookUpdate.encode(message.orderbookUpdate, writer.uint32(10).fork()).ldelim(); + if (message.blockHeight !== 0) { + writer.uint32(8).uint32(message.blockHeight); } - if (message.orderFill !== undefined) { - StreamOrderbookFill.encode(message.orderFill, writer.uint32(18).fork()).ldelim(); + if (message.execMode !== 0) { + writer.uint32(16).uint32(message.execMode); } - if (message.takerOrder !== undefined) { - StreamTakerOrder.encode(message.takerOrder, writer.uint32(26).fork()).ldelim(); + if (message.orderbookUpdate !== undefined) { + StreamOrderbookUpdate.encode(message.orderbookUpdate, writer.uint32(26).fork()).ldelim(); } - if (message.subaccountUpdate !== undefined) { - StreamSubaccountUpdate.encode(message.subaccountUpdate, writer.uint32(34).fork()).ldelim(); + if (message.orderFill !== undefined) { + StreamOrderbookFill.encode(message.orderFill, writer.uint32(34).fork()).ldelim(); } - if (message.blockHeight !== 0) { - writer.uint32(40).uint32(message.blockHeight); + if (message.takerOrder !== undefined) { + StreamTakerOrder.encode(message.takerOrder, writer.uint32(42).fork()).ldelim(); } - if (message.execMode !== 0) { - writer.uint32(48).uint32(message.execMode); + if (message.subaccountUpdate !== undefined) { + StreamSubaccountUpdate.encode(message.subaccountUpdate, writer.uint32(50).fork()).ldelim(); } return writer; @@ -1353,27 +1351,27 @@ export const StreamUpdate = { switch (tag >>> 3) { case 1: - message.orderbookUpdate = StreamOrderbookUpdate.decode(reader, reader.uint32()); + message.blockHeight = reader.uint32(); break; case 2: - message.orderFill = StreamOrderbookFill.decode(reader, reader.uint32()); + message.execMode = reader.uint32(); break; case 3: - message.takerOrder = StreamTakerOrder.decode(reader, reader.uint32()); + message.orderbookUpdate = StreamOrderbookUpdate.decode(reader, reader.uint32()); break; case 4: - message.subaccountUpdate = StreamSubaccountUpdate.decode(reader, reader.uint32()); + message.orderFill = StreamOrderbookFill.decode(reader, reader.uint32()); break; case 5: - message.blockHeight = reader.uint32(); + message.takerOrder = StreamTakerOrder.decode(reader, reader.uint32()); break; case 6: - message.execMode = reader.uint32(); + message.subaccountUpdate = StreamSubaccountUpdate.decode(reader, reader.uint32()); break; default: @@ -1387,12 +1385,12 @@ export const StreamUpdate = { fromPartial(object: DeepPartial): StreamUpdate { const message = createBaseStreamUpdate(); + message.blockHeight = object.blockHeight ?? 0; + message.execMode = object.execMode ?? 0; message.orderbookUpdate = object.orderbookUpdate !== undefined && object.orderbookUpdate !== null ? StreamOrderbookUpdate.fromPartial(object.orderbookUpdate) : undefined; message.orderFill = object.orderFill !== undefined && object.orderFill !== null ? StreamOrderbookFill.fromPartial(object.orderFill) : undefined; message.takerOrder = object.takerOrder !== undefined && object.takerOrder !== null ? StreamTakerOrder.fromPartial(object.takerOrder) : undefined; message.subaccountUpdate = object.subaccountUpdate !== undefined && object.subaccountUpdate !== null ? StreamSubaccountUpdate.fromPartial(object.subaccountUpdate) : undefined; - message.blockHeight = object.blockHeight ?? 0; - message.execMode = object.execMode ?? 0; return message; } @@ -1400,19 +1398,19 @@ export const StreamUpdate = { function createBaseStreamOrderbookUpdate(): StreamOrderbookUpdate { return { - updates: [], - snapshot: false + snapshot: false, + updates: [] }; } export const StreamOrderbookUpdate = { encode(message: StreamOrderbookUpdate, writer: _m0.Writer = _m0.Writer.create()): _m0.Writer { - for (const v of message.updates) { - OffChainUpdateV1.encode(v!, writer.uint32(10).fork()).ldelim(); + if (message.snapshot === true) { + writer.uint32(8).bool(message.snapshot); } - if (message.snapshot === true) { - writer.uint32(16).bool(message.snapshot); + for (const v of message.updates) { + OffChainUpdateV1.encode(v!, writer.uint32(18).fork()).ldelim(); } return writer; @@ -1428,11 +1426,11 @@ export const StreamOrderbookUpdate = { switch (tag >>> 3) { case 1: - message.updates.push(OffChainUpdateV1.decode(reader, reader.uint32())); + message.snapshot = reader.bool(); break; case 2: - message.snapshot = reader.bool(); + message.updates.push(OffChainUpdateV1.decode(reader, reader.uint32())); break; default: @@ -1446,8 +1444,8 @@ export const StreamOrderbookUpdate = { fromPartial(object: DeepPartial): StreamOrderbookUpdate { const message = createBaseStreamOrderbookUpdate(); - message.updates = object.updates?.map(e => OffChainUpdateV1.fromPartial(e)) || []; message.snapshot = object.snapshot ?? false; + message.updates = object.updates?.map(e => OffChainUpdateV1.fromPartial(e)) || []; return message; } diff --git a/proto/dydxprotocol/clob/query.proto b/proto/dydxprotocol/clob/query.proto index b0342bc3c6..9d47ea4641 100644 --- a/proto/dydxprotocol/clob/query.proto +++ b/proto/dydxprotocol/clob/query.proto @@ -182,35 +182,35 @@ message StreamOrderbookUpdatesResponse { // StreamUpdate is an update that will be pushed through the // GRPC stream. message StreamUpdate { + // Block height of the update. + uint32 block_height = 1; + + // Exec mode of the update. + uint32 exec_mode = 2; + // Contains one of an StreamOrderbookUpdate, // StreamOrderbookFill, StreamTakerOrderStatus. oneof update_message { - StreamOrderbookUpdate orderbook_update = 1; - StreamOrderbookFill order_fill = 2; - StreamTakerOrder taker_order = 3; - dydxprotocol.subaccounts.StreamSubaccountUpdate subaccount_update = 4; + StreamOrderbookUpdate orderbook_update = 3; + StreamOrderbookFill order_fill = 4; + StreamTakerOrder taker_order = 5; + dydxprotocol.subaccounts.StreamSubaccountUpdate subaccount_update = 6; } - - // Block height of the update. - uint32 block_height = 5; - - // Exec mode of the update. - uint32 exec_mode = 6; } // StreamOrderbookUpdate provides information on an orderbook update. Used in // the full node GRPC stream. message StreamOrderbookUpdate { - // Orderbook updates for the clob pair. Can contain order place, removals, - // or updates. - repeated dydxprotocol.indexer.off_chain_updates.OffChainUpdateV1 updates = 1 - [ (gogoproto.nullable) = false ]; - // Snapshot indicates if the response is from a snapshot of the orderbook. // All updates should be ignored until snapshot is recieved. // If the snapshot is true, then all previous entries should be // discarded and the orderbook should be resynced. - bool snapshot = 2; + bool snapshot = 1; + + // Orderbook updates for the clob pair. Can contain order place, removals, + // or updates. + repeated dydxprotocol.indexer.off_chain_updates.OffChainUpdateV1 updates = 2 + [ (gogoproto.nullable) = false ]; } // StreamOrderbookFill provides information on an orderbook fill. Used in diff --git a/protocol/x/clob/types/query.pb.go b/protocol/x/clob/types/query.pb.go index d0d00fd50a..4f4d01af14 100644 --- a/protocol/x/clob/types/query.pb.go +++ b/protocol/x/clob/types/query.pb.go @@ -872,6 +872,10 @@ func (m *StreamOrderbookUpdatesResponse) GetUpdates() []StreamUpdate { // StreamUpdate is an update that will be pushed through the // GRPC stream. type StreamUpdate struct { + // Block height of the update. + BlockHeight uint32 `protobuf:"varint,1,opt,name=block_height,json=blockHeight,proto3" json:"block_height,omitempty"` + // Exec mode of the update. + ExecMode uint32 `protobuf:"varint,2,opt,name=exec_mode,json=execMode,proto3" json:"exec_mode,omitempty"` // Contains one of an StreamOrderbookUpdate, // StreamOrderbookFill, StreamTakerOrderStatus. // @@ -882,10 +886,6 @@ type StreamUpdate struct { // *StreamUpdate_TakerOrder // *StreamUpdate_SubaccountUpdate UpdateMessage isStreamUpdate_UpdateMessage `protobuf_oneof:"update_message"` - // Block height of the update. - BlockHeight uint32 `protobuf:"varint,5,opt,name=block_height,json=blockHeight,proto3" json:"block_height,omitempty"` - // Exec mode of the update. - ExecMode uint32 `protobuf:"varint,6,opt,name=exec_mode,json=execMode,proto3" json:"exec_mode,omitempty"` } func (m *StreamUpdate) Reset() { *m = StreamUpdate{} } @@ -928,16 +928,16 @@ type isStreamUpdate_UpdateMessage interface { } type StreamUpdate_OrderbookUpdate struct { - OrderbookUpdate *StreamOrderbookUpdate `protobuf:"bytes,1,opt,name=orderbook_update,json=orderbookUpdate,proto3,oneof" json:"orderbook_update,omitempty"` + OrderbookUpdate *StreamOrderbookUpdate `protobuf:"bytes,3,opt,name=orderbook_update,json=orderbookUpdate,proto3,oneof" json:"orderbook_update,omitempty"` } type StreamUpdate_OrderFill struct { - OrderFill *StreamOrderbookFill `protobuf:"bytes,2,opt,name=order_fill,json=orderFill,proto3,oneof" json:"order_fill,omitempty"` + OrderFill *StreamOrderbookFill `protobuf:"bytes,4,opt,name=order_fill,json=orderFill,proto3,oneof" json:"order_fill,omitempty"` } type StreamUpdate_TakerOrder struct { - TakerOrder *StreamTakerOrder `protobuf:"bytes,3,opt,name=taker_order,json=takerOrder,proto3,oneof" json:"taker_order,omitempty"` + TakerOrder *StreamTakerOrder `protobuf:"bytes,5,opt,name=taker_order,json=takerOrder,proto3,oneof" json:"taker_order,omitempty"` } type StreamUpdate_SubaccountUpdate struct { - SubaccountUpdate *types.StreamSubaccountUpdate `protobuf:"bytes,4,opt,name=subaccount_update,json=subaccountUpdate,proto3,oneof" json:"subaccount_update,omitempty"` + SubaccountUpdate *types.StreamSubaccountUpdate `protobuf:"bytes,6,opt,name=subaccount_update,json=subaccountUpdate,proto3,oneof" json:"subaccount_update,omitempty"` } func (*StreamUpdate_OrderbookUpdate) isStreamUpdate_UpdateMessage() {} @@ -952,6 +952,20 @@ func (m *StreamUpdate) GetUpdateMessage() isStreamUpdate_UpdateMessage { return nil } +func (m *StreamUpdate) GetBlockHeight() uint32 { + if m != nil { + return m.BlockHeight + } + return 0 +} + +func (m *StreamUpdate) GetExecMode() uint32 { + if m != nil { + return m.ExecMode + } + return 0 +} + func (m *StreamUpdate) GetOrderbookUpdate() *StreamOrderbookUpdate { if x, ok := m.GetUpdateMessage().(*StreamUpdate_OrderbookUpdate); ok { return x.OrderbookUpdate @@ -980,20 +994,6 @@ func (m *StreamUpdate) GetSubaccountUpdate() *types.StreamSubaccountUpdate { return nil } -func (m *StreamUpdate) GetBlockHeight() uint32 { - if m != nil { - return m.BlockHeight - } - return 0 -} - -func (m *StreamUpdate) GetExecMode() uint32 { - if m != nil { - return m.ExecMode - } - return 0 -} - // XXX_OneofWrappers is for the internal use of the proto package. func (*StreamUpdate) XXX_OneofWrappers() []interface{} { return []interface{}{ @@ -1007,14 +1007,14 @@ func (*StreamUpdate) XXX_OneofWrappers() []interface{} { // StreamOrderbookUpdate provides information on an orderbook update. Used in // the full node GRPC stream. type StreamOrderbookUpdate struct { - // Orderbook updates for the clob pair. Can contain order place, removals, - // or updates. - Updates []types1.OffChainUpdateV1 `protobuf:"bytes,1,rep,name=updates,proto3" json:"updates"` // Snapshot indicates if the response is from a snapshot of the orderbook. // All updates should be ignored until snapshot is recieved. // If the snapshot is true, then all previous entries should be // discarded and the orderbook should be resynced. - Snapshot bool `protobuf:"varint,2,opt,name=snapshot,proto3" json:"snapshot,omitempty"` + Snapshot bool `protobuf:"varint,1,opt,name=snapshot,proto3" json:"snapshot,omitempty"` + // Orderbook updates for the clob pair. Can contain order place, removals, + // or updates. + Updates []types1.OffChainUpdateV1 `protobuf:"bytes,2,rep,name=updates,proto3" json:"updates"` } func (m *StreamOrderbookUpdate) Reset() { *m = StreamOrderbookUpdate{} } @@ -1050,18 +1050,18 @@ func (m *StreamOrderbookUpdate) XXX_DiscardUnknown() { var xxx_messageInfo_StreamOrderbookUpdate proto.InternalMessageInfo -func (m *StreamOrderbookUpdate) GetUpdates() []types1.OffChainUpdateV1 { +func (m *StreamOrderbookUpdate) GetSnapshot() bool { if m != nil { - return m.Updates + return m.Snapshot } - return nil + return false } -func (m *StreamOrderbookUpdate) GetSnapshot() bool { +func (m *StreamOrderbookUpdate) GetUpdates() []types1.OffChainUpdateV1 { if m != nil { - return m.Snapshot + return m.Updates } - return false + return nil } // StreamOrderbookFill provides information on an orderbook fill. Used in @@ -1333,111 +1333,111 @@ func init() { func init() { proto.RegisterFile("dydxprotocol/clob/query.proto", fileDescriptor_3365c195b25c5bc0) } var fileDescriptor_3365c195b25c5bc0 = []byte{ - // 1656 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x58, 0x41, 0x4f, 0xdc, 0x46, - 0x14, 0x5e, 0xb3, 0x84, 0xc0, 0xdb, 0x40, 0x60, 0x08, 0xc9, 0x66, 0x21, 0x0b, 0x71, 0x1a, 0xb2, - 0x90, 0x66, 0x0d, 0x24, 0x8a, 0xd2, 0x50, 0xa5, 0x02, 0x5a, 0x42, 0xa4, 0xd0, 0x10, 0x43, 0x12, - 0xd4, 0x46, 0xb2, 0xbc, 0xf6, 0xb0, 0x58, 0xd8, 0x9e, 0xc5, 0x1e, 0xaf, 0x40, 0x55, 0x55, 0xa9, - 0x87, 0x5c, 0xda, 0x4a, 0x91, 0x7a, 0xe8, 0xa1, 0x52, 0x2f, 0x3d, 0xf5, 0x50, 0xa9, 0x97, 0x1e, - 0xab, 0xb6, 0xb7, 0x1c, 0x23, 0xf5, 0xd2, 0x43, 0x55, 0x55, 0x49, 0xcf, 0xfd, 0x0d, 0x95, 0x67, - 0xc6, 0x8b, 0x77, 0xd7, 0x5e, 0x08, 0x17, 0xb0, 0xdf, 0xbc, 0xf7, 0xe6, 0x7b, 0xef, 0x7d, 0xf3, - 0xe6, 0x79, 0xe1, 0x82, 0xb9, 0x6f, 0xee, 0xd5, 0x3c, 0x42, 0x89, 0x41, 0x6c, 0xc5, 0xb0, 0x49, - 0x45, 0xd9, 0x0d, 0xb0, 0xb7, 0x5f, 0x66, 0x32, 0x34, 0x14, 0x5f, 0x2e, 0x87, 0xcb, 0x85, 0x33, - 0x55, 0x52, 0x25, 0x4c, 0xa4, 0x84, 0x4f, 0x5c, 0xb1, 0x30, 0x56, 0x25, 0xa4, 0x6a, 0x63, 0x45, - 0xaf, 0x59, 0x8a, 0xee, 0xba, 0x84, 0xea, 0xd4, 0x22, 0xae, 0x2f, 0x56, 0xa7, 0x0d, 0xe2, 0x3b, - 0xc4, 0x57, 0x2a, 0xba, 0x8f, 0xb9, 0x7f, 0xa5, 0x3e, 0x5b, 0xc1, 0x54, 0x9f, 0x55, 0x6a, 0x7a, - 0xd5, 0x72, 0x99, 0xb2, 0xd0, 0x55, 0xda, 0x11, 0x55, 0x6c, 0x62, 0xec, 0x68, 0x9e, 0x4e, 0xb1, - 0x66, 0x5b, 0x8e, 0x45, 0x35, 0x83, 0xb8, 0x5b, 0x56, 0x55, 0x18, 0x5c, 0x6c, 0x37, 0x08, 0xff, - 0x68, 0x35, 0xdd, 0xf2, 0x84, 0xca, 0x4c, 0xbb, 0x0a, 0xde, 0x0d, 0x2c, 0xba, 0xaf, 0x51, 0x0b, - 0x7b, 0x49, 0x4e, 0x13, 0xf2, 0x42, 0x3c, 0x13, 0x47, 0x0e, 0xc7, 0xdb, 0x97, 0x1d, 0x9d, 0x1a, - 0xdb, 0x38, 0x8a, 0xf8, 0x6a, 0xbb, 0x82, 0x6d, 0xed, 0x06, 0x96, 0xc9, 0xf3, 0xd2, 0xbc, 0xd9, - 0x68, 0x82, 0x37, 0x5c, 0x17, 0x8b, 0x77, 0x9a, 0x16, 0x2d, 0xd7, 0xc4, 0x7b, 0xd8, 0x53, 0xc8, - 0xd6, 0x96, 0x66, 0x6c, 0xeb, 0x96, 0xab, 0x05, 0x35, 0x53, 0xa7, 0xd8, 0x6f, 0x97, 0x08, 0xfb, - 0x52, 0x93, 0xbd, 0x1f, 0x54, 0x74, 0xc3, 0x20, 0x81, 0x4b, 0x7d, 0xc5, 0xa7, 0x1e, 0xd6, 0x1d, - 0xcb, 0x8d, 0x60, 0x4c, 0xa5, 0x6b, 0x36, 0x9e, 0xb9, 0xaa, 0x3c, 0x05, 0xe7, 0x1e, 0x86, 0x65, - 0xbc, 0x8b, 0xe9, 0x92, 0x4d, 0x2a, 0x6b, 0xba, 0xe5, 0xa9, 0x78, 0x37, 0xc0, 0x3e, 0x45, 0x03, - 0xd0, 0x65, 0x99, 0x79, 0x69, 0x42, 0x2a, 0xf5, 0xab, 0x5d, 0x96, 0x29, 0x3f, 0x81, 0x11, 0xa6, - 0x7a, 0xa0, 0xe7, 0xd7, 0x88, 0xeb, 0x63, 0x74, 0x07, 0xfa, 0x1a, 0x75, 0x62, 0xfa, 0xb9, 0xb9, - 0xd1, 0x72, 0x1b, 0xdf, 0xca, 0x91, 0xdd, 0x62, 0xf7, 0x8b, 0xbf, 0xc7, 0x33, 0x6a, 0xaf, 0x21, - 0xde, 0x65, 0x5d, 0x60, 0x58, 0xb0, 0xed, 0x56, 0x0c, 0xcb, 0x00, 0x07, 0xbc, 0x12, 0xbe, 0x27, - 0xcb, 0x9c, 0x84, 0xe5, 0x90, 0x84, 0x65, 0x4e, 0x72, 0x41, 0xc2, 0xf2, 0x9a, 0x5e, 0xc5, 0xc2, - 0x56, 0x8d, 0x59, 0xca, 0xdf, 0x4b, 0x90, 0x6f, 0x02, 0xbf, 0x60, 0xdb, 0x69, 0xf8, 0xb3, 0x6f, - 0x88, 0x1f, 0xdd, 0x6d, 0x02, 0xd9, 0xc5, 0x40, 0x5e, 0x39, 0x14, 0x24, 0xdf, 0xbc, 0x09, 0xe5, - 0x5f, 0x12, 0x8c, 0xaf, 0xe2, 0xfa, 0x87, 0xc4, 0xc4, 0x1b, 0x24, 0xfc, 0xbb, 0xa4, 0xdb, 0x46, - 0x60, 0xb3, 0xc5, 0x28, 0x23, 0x4f, 0xe1, 0x2c, 0x3f, 0x45, 0x35, 0x8f, 0xd4, 0x88, 0x8f, 0x3d, - 0x4d, 0xf0, 0xb5, 0x91, 0x9d, 0x76, 0xe4, 0x8f, 0x75, 0x3b, 0xe4, 0x2b, 0xf1, 0x56, 0x71, 0x7d, - 0x95, 0x6b, 0xab, 0x67, 0x98, 0x97, 0x35, 0xe1, 0x44, 0x48, 0xd1, 0xc7, 0x30, 0x52, 0x8f, 0x94, - 0x35, 0x07, 0xd7, 0x35, 0x07, 0x53, 0xcf, 0x32, 0xfc, 0x46, 0x54, 0xed, 0xce, 0x9b, 0x00, 0xaf, - 0x72, 0x75, 0x75, 0xb8, 0x1e, 0xdf, 0x92, 0x0b, 0xe5, 0xff, 0x24, 0x98, 0x48, 0x0f, 0x4f, 0x14, - 0xa3, 0x0a, 0x27, 0x3d, 0xec, 0x07, 0x36, 0xf5, 0x45, 0x29, 0xee, 0x1e, 0xb6, 0x67, 0x82, 0x97, - 0x50, 0x61, 0xc1, 0x35, 0x1f, 0x13, 0x3b, 0x70, 0xf0, 0x1a, 0xf6, 0xc2, 0xd2, 0x89, 0xb2, 0x45, - 0xde, 0x0b, 0x3a, 0x0c, 0x27, 0x68, 0xa1, 0x09, 0x38, 0xd5, 0x20, 0x83, 0xd6, 0xe0, 0x3f, 0x44, - 0xc5, 0xbe, 0x67, 0xa2, 0x41, 0xc8, 0x3a, 0xb8, 0xce, 0x32, 0xd2, 0xa5, 0x86, 0x8f, 0xe8, 0x2c, - 0xf4, 0xd4, 0x99, 0x93, 0x7c, 0x76, 0x42, 0x2a, 0x75, 0xab, 0xe2, 0x4d, 0x9e, 0x86, 0x12, 0x23, - 0xdd, 0x07, 0xac, 0x45, 0x6d, 0x58, 0xd8, 0xbb, 0x1f, 0x36, 0xa8, 0x25, 0xd6, 0x32, 0x02, 0x2f, - 0x5e, 0x57, 0xf9, 0x5b, 0x09, 0xa6, 0x8e, 0xa0, 0x2c, 0xb2, 0xe4, 0x42, 0x3e, 0xad, 0xef, 0x09, - 0x1e, 0x28, 0x09, 0x69, 0xeb, 0xe4, 0x5a, 0xa4, 0x67, 0x04, 0x27, 0xe9, 0xc8, 0x53, 0x70, 0x85, - 0x81, 0x5b, 0x0c, 0x49, 0xa3, 0xea, 0x14, 0xa7, 0x07, 0xf2, 0x8d, 0x24, 0xa2, 0xee, 0xa8, 0x2b, - 0xe2, 0xd8, 0x81, 0x73, 0x29, 0x77, 0x82, 0x08, 0xa3, 0x9c, 0x10, 0x46, 0x07, 0xc7, 0x22, 0x0a, - 0x4e, 0xee, 0x16, 0x15, 0x79, 0x13, 0xce, 0x33, 0x60, 0xeb, 0x54, 0xa7, 0x78, 0x2b, 0xb0, 0x1f, - 0x84, 0xf7, 0x40, 0x74, 0xae, 0xe6, 0xa1, 0x97, 0xdd, 0x0b, 0x51, 0xcd, 0x73, 0x73, 0x85, 0x84, - 0xad, 0x99, 0xc9, 0x3d, 0x33, 0xe2, 0x12, 0xe1, 0xaf, 0xf2, 0xcf, 0x12, 0x14, 0x92, 0x5c, 0x8b, - 0x28, 0x37, 0xe1, 0x34, 0xf7, 0x5d, 0xb3, 0x75, 0x03, 0x3b, 0xd8, 0xa5, 0x62, 0x8b, 0xa9, 0x84, - 0x2d, 0xee, 0x13, 0xb7, 0xba, 0x81, 0x3d, 0x87, 0xb9, 0x58, 0x8b, 0x0c, 0xc4, 0x8e, 0x03, 0xa4, - 0x49, 0x8a, 0xc6, 0x21, 0xb7, 0x65, 0xd9, 0xb6, 0xa6, 0x3b, 0x61, 0x4f, 0x67, 0x9c, 0xec, 0x56, - 0x21, 0x14, 0x2d, 0x30, 0x09, 0x1a, 0x83, 0x3e, 0xea, 0x59, 0xd5, 0x2a, 0xf6, 0xb0, 0xc9, 0xd8, - 0xd9, 0xab, 0x1e, 0x08, 0xe4, 0x2b, 0x70, 0x99, 0xc1, 0xbe, 0x1f, 0xbb, 0xd1, 0x12, 0x8b, 0xfa, - 0x4c, 0x82, 0xc9, 0xc3, 0x34, 0x45, 0xb0, 0x4f, 0x61, 0x38, 0xe1, 0x82, 0x14, 0x01, 0x5f, 0x4e, - 0x0a, 0xb8, 0xcd, 0xa5, 0x08, 0x16, 0xd9, 0x6d, 0x2b, 0xf2, 0x73, 0x09, 0x2e, 0xac, 0xb3, 0xeb, - 0x8e, 0xe5, 0xa7, 0x42, 0xc8, 0xce, 0x23, 0x7e, 0x4b, 0x46, 0x85, 0x6c, 0x3f, 0xc0, 0xd9, 0x96, - 0x03, 0xbc, 0x0a, 0x03, 0x07, 0xf7, 0xa0, 0x66, 0x99, 0x61, 0x77, 0xcb, 0xb6, 0xb7, 0xce, 0xd8, - 0xbd, 0x59, 0x5e, 0x6f, 0x3c, 0xdf, 0x33, 0xd5, 0x7e, 0x3f, 0xf6, 0xe6, 0xcb, 0x3a, 0x14, 0xd3, - 0x10, 0x89, 0x94, 0xbc, 0x07, 0x27, 0xc5, 0x55, 0x2e, 0x7a, 0xda, 0x78, 0x42, 0x1a, 0xb8, 0x0f, - 0x6e, 0x1a, 0xf1, 0x4b, 0x58, 0xc9, 0x3f, 0x64, 0xe1, 0x54, 0x7c, 0x1d, 0x3d, 0x82, 0x41, 0x12, - 0xed, 0x26, 0xc6, 0x04, 0x91, 0xe1, 0x52, 0xaa, 0xeb, 0x16, 0x78, 0x2b, 0x19, 0xf5, 0x34, 0x69, - 0x16, 0x85, 0x37, 0x19, 0x27, 0x6a, 0xc8, 0x20, 0xd1, 0xf3, 0x27, 0x0f, 0x77, 0xb8, 0x6c, 0xd9, - 0xf6, 0x4a, 0x46, 0xed, 0x63, 0xb6, 0xe1, 0x0b, 0x5a, 0x86, 0x1c, 0xd5, 0x77, 0xb0, 0xa7, 0x31, - 0x11, 0x23, 0x5e, 0x6e, 0xee, 0x52, 0xaa, 0xa7, 0x8d, 0x50, 0x97, 0xb9, 0x5b, 0xc9, 0xa8, 0x40, - 0x1b, 0x6f, 0x48, 0x83, 0xa1, 0x58, 0xa9, 0x44, 0xa0, 0xdd, 0xcc, 0xdb, 0x4c, 0x87, 0x6a, 0x31, - 0xa7, 0x07, 0x35, 0x6b, 0x04, 0x3c, 0xe8, 0xb7, 0xc8, 0xd0, 0x45, 0x38, 0xc5, 0x1b, 0xd0, 0x36, - 0xb6, 0xaa, 0xdb, 0x34, 0x7f, 0x82, 0xb5, 0xfb, 0x1c, 0x93, 0xad, 0x30, 0x11, 0x1a, 0x85, 0x3e, - 0xbc, 0x87, 0x0d, 0xcd, 0x21, 0x26, 0xce, 0xf7, 0xb0, 0xf5, 0xde, 0x50, 0xb0, 0x4a, 0x4c, 0xbc, - 0x38, 0x08, 0x03, 0x1c, 0x95, 0xe6, 0x60, 0xdf, 0xd7, 0xab, 0x58, 0xfe, 0x4a, 0x82, 0x91, 0xc4, - 0x84, 0xa3, 0xcd, 0x56, 0x1a, 0xdc, 0x6a, 0x0e, 0x41, 0x8c, 0x84, 0xe5, 0xf6, 0x01, 0xf0, 0xc1, - 0xd6, 0xd6, 0x52, 0x28, 0xe0, 0x8e, 0x1e, 0xcf, 0xb6, 0xf0, 0x03, 0x15, 0xa0, 0xd7, 0x77, 0xf5, - 0x9a, 0xbf, 0x4d, 0x78, 0x0f, 0xe8, 0x55, 0x1b, 0xef, 0xf2, 0x8f, 0x12, 0x0c, 0x27, 0xd4, 0x0b, - 0xcd, 0x03, 0x3b, 0x13, 0x7c, 0x7c, 0x10, 0xe4, 0x19, 0x4b, 0x19, 0x7b, 0xd8, 0x78, 0xa0, 0xb2, - 0x29, 0x89, 0x3d, 0xa2, 0x9b, 0xd0, 0xc3, 0x2a, 0x1b, 0x1d, 0x9d, 0x7c, 0x5a, 0xaf, 0x14, 0x48, - 0x85, 0x76, 0x98, 0xee, 0x58, 0xbf, 0xf2, 0xf3, 0xd9, 0x89, 0x6c, 0xa9, 0x5b, 0xcd, 0x1d, 0x34, - 0x2c, 0x5f, 0x7e, 0xd6, 0x05, 0x83, 0xad, 0xac, 0x40, 0x33, 0x70, 0x82, 0x33, 0x89, 0xe3, 0x4c, - 0xdd, 0x6e, 0x25, 0xa3, 0x72, 0x45, 0xb4, 0x09, 0x43, 0xb1, 0xf6, 0x21, 0x78, 0xd8, 0x95, 0xda, - 0x75, 0xf9, 0x8e, 0xb1, 0x56, 0x14, 0xb9, 0x1b, 0xb4, 0x5b, 0x64, 0xe8, 0x09, 0xa0, 0x18, 0xb7, - 0x35, 0x9f, 0xea, 0x34, 0xf0, 0x05, 0xc5, 0xa7, 0x8e, 0x40, 0xf1, 0x75, 0x66, 0xa0, 0x0e, 0xd2, - 0x16, 0xc9, 0x62, 0x7f, 0xd3, 0xa1, 0x91, 0x7f, 0x92, 0xe0, 0x6c, 0xb2, 0x6d, 0x98, 0xc6, 0xa6, - 0xcd, 0xf9, 0x90, 0x92, 0x23, 0x31, 0x95, 0x6b, 0x80, 0x3c, 0xec, 0xe8, 0x96, 0x6b, 0xb9, 0x55, - 0x6d, 0x37, 0xd0, 0x5d, 0x1a, 0x38, 0xbe, 0xb8, 0x20, 0x86, 0x1a, 0x2b, 0x0f, 0xc5, 0x02, 0x7a, - 0x1f, 0x8a, 0xa4, 0x46, 0x2d, 0xc7, 0xf2, 0xa9, 0x65, 0xe8, 0xb6, 0xbd, 0xcf, 0x5a, 0x00, 0x36, - 0x0f, 0x4c, 0xf9, 0x68, 0x33, 0xd6, 0xac, 0xb5, 0xcc, 0x94, 0x22, 0x2f, 0x73, 0xdf, 0x01, 0x9c, - 0x60, 0xd7, 0x04, 0xfa, 0x42, 0x82, 0xde, 0x68, 0x60, 0x46, 0xd3, 0x09, 0x59, 0x49, 0xf9, 0xea, - 0x28, 0x94, 0xd2, 0x74, 0x5b, 0x3f, 0x3b, 0xe4, 0xa9, 0xcf, 0xff, 0xf8, 0xf7, 0xeb, 0xae, 0x4b, - 0xe8, 0xa2, 0xd2, 0xe1, 0xbb, 0x51, 0xf9, 0xc4, 0x32, 0x3f, 0x45, 0x5f, 0x4a, 0x90, 0x8b, 0x4d, - 0xfe, 0xe9, 0x80, 0xda, 0x3f, 0x41, 0x0a, 0x57, 0x0f, 0x03, 0x14, 0xfb, 0x94, 0x90, 0xdf, 0x62, - 0x98, 0x8a, 0x68, 0xac, 0x13, 0x26, 0xf4, 0xab, 0x04, 0xf9, 0xb4, 0x11, 0x16, 0xcd, 0xbd, 0xd1, - 0xbc, 0xcb, 0x31, 0x5e, 0x3f, 0xc6, 0x8c, 0x2c, 0xdf, 0x66, 0x58, 0x6f, 0xdc, 0x96, 0xa6, 0x65, - 0x45, 0x49, 0xfc, 0x70, 0xd5, 0x5c, 0x62, 0x62, 0x8d, 0x12, 0xfe, 0xdf, 0x88, 0x81, 0xfc, 0x5d, - 0x82, 0xb1, 0x4e, 0xd3, 0x24, 0x9a, 0x4f, 0xcb, 0xda, 0x11, 0x66, 0xe1, 0xc2, 0xbb, 0xc7, 0x33, - 0x16, 0x71, 0x4d, 0xb2, 0xb8, 0x26, 0x50, 0x51, 0xe9, 0xf8, 0x63, 0x01, 0xfa, 0x45, 0x82, 0xd1, - 0x0e, 0xa3, 0x24, 0xba, 0x9d, 0x86, 0xe2, 0xf0, 0x21, 0xb8, 0x30, 0x7f, 0x2c, 0x5b, 0x11, 0xc0, - 0x65, 0x16, 0xc0, 0x38, 0xba, 0xd0, 0xf1, 0x17, 0x14, 0xf4, 0x9b, 0x04, 0xe7, 0x53, 0xc7, 0x31, - 0x74, 0x2b, 0x0d, 0xc1, 0x61, 0xb3, 0x5e, 0xe1, 0x9d, 0x63, 0x58, 0x0a, 0xe4, 0x65, 0x86, 0xbc, - 0x84, 0x26, 0x95, 0x23, 0xfd, 0x6a, 0x82, 0x5c, 0xe8, 0x6f, 0x9a, 0x98, 0xd1, 0xdb, 0x69, 0x7b, - 0x27, 0xcd, 0xec, 0x85, 0x6b, 0x47, 0xd4, 0x16, 0xe8, 0x32, 0xe8, 0xb3, 0xa8, 0xa3, 0xb6, 0x8e, - 0x6a, 0x68, 0xe6, 0xa8, 0x63, 0x53, 0x34, 0x67, 0x16, 0x66, 0xdf, 0xc0, 0x82, 0x03, 0x98, 0x91, - 0x16, 0xd7, 0x5e, 0xbc, 0x2a, 0x4a, 0x2f, 0x5f, 0x15, 0xa5, 0x7f, 0x5e, 0x15, 0xa5, 0xe7, 0xaf, - 0x8b, 0x99, 0x97, 0xaf, 0x8b, 0x99, 0x3f, 0x5f, 0x17, 0x33, 0x1f, 0xdd, 0xac, 0x5a, 0x74, 0x3b, - 0xa8, 0x94, 0x0d, 0xe2, 0x34, 0x27, 0xaf, 0x7e, 0xe3, 0x1a, 0x1b, 0x06, 0x94, 0x86, 0x64, 0x8f, - 0x27, 0x94, 0xee, 0xd7, 0xb0, 0x5f, 0xe9, 0x61, 0xe2, 0xeb, 0xff, 0x07, 0x00, 0x00, 0xff, 0xff, - 0x4d, 0x56, 0xef, 0xe3, 0x00, 0x14, 0x00, 0x00, + // 1661 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x58, 0xc1, 0x4f, 0xdc, 0x46, + 0x17, 0x5f, 0xb3, 0x84, 0xc0, 0xdb, 0x40, 0x60, 0x08, 0xc9, 0x66, 0x21, 0x0b, 0x71, 0xbe, 0x90, + 0x85, 0x7c, 0x59, 0x03, 0x89, 0xa2, 0x7c, 0xe1, 0x53, 0x3e, 0x01, 0xdf, 0x47, 0x88, 0x14, 0xbe, + 0x10, 0x43, 0x12, 0xd4, 0x46, 0xb2, 0xbc, 0xf6, 0xb0, 0x58, 0xd8, 0x9e, 0xc5, 0x1e, 0xaf, 0x40, + 0x55, 0x55, 0xa9, 0x87, 0x5c, 0xda, 0x4a, 0x91, 0x7a, 0xe8, 0xa1, 0x52, 0x2f, 0x3d, 0xf5, 0x50, + 0xa9, 0x97, 0x1e, 0xab, 0xb6, 0xb7, 0x1c, 0x23, 0xf5, 0xd2, 0x43, 0x55, 0x55, 0x49, 0xcf, 0xfd, + 0x1b, 0x2a, 0xcf, 0x8c, 0x77, 0xbd, 0xbb, 0xf6, 0x42, 0xb8, 0x80, 0xfd, 0xe6, 0xbd, 0x37, 0xbf, + 0xf7, 0xde, 0x6f, 0xde, 0x3c, 0x2f, 0x5c, 0x32, 0x0f, 0xcd, 0x83, 0x9a, 0x47, 0x28, 0x31, 0x88, + 0xad, 0x18, 0x36, 0xa9, 0x28, 0xfb, 0x01, 0xf6, 0x0e, 0xcb, 0x4c, 0x86, 0x46, 0xe2, 0xcb, 0xe5, + 0x70, 0xb9, 0x70, 0xae, 0x4a, 0xaa, 0x84, 0x89, 0x94, 0xf0, 0x89, 0x2b, 0x16, 0x26, 0xaa, 0x84, + 0x54, 0x6d, 0xac, 0xe8, 0x35, 0x4b, 0xd1, 0x5d, 0x97, 0x50, 0x9d, 0x5a, 0xc4, 0xf5, 0xc5, 0xea, + 0xac, 0x41, 0x7c, 0x87, 0xf8, 0x4a, 0x45, 0xf7, 0x31, 0xf7, 0xaf, 0xd4, 0xe7, 0x2b, 0x98, 0xea, + 0xf3, 0x4a, 0x4d, 0xaf, 0x5a, 0x2e, 0x53, 0x16, 0xba, 0x4a, 0x27, 0xa2, 0x8a, 0x4d, 0x8c, 0x3d, + 0xcd, 0xd3, 0x29, 0xd6, 0x6c, 0xcb, 0xb1, 0xa8, 0x66, 0x10, 0x77, 0xc7, 0xaa, 0x0a, 0x83, 0xcb, + 0x9d, 0x06, 0xe1, 0x1f, 0xad, 0xa6, 0x5b, 0x9e, 0x50, 0x99, 0xeb, 0x54, 0xc1, 0xfb, 0x81, 0x45, + 0x0f, 0x35, 0x6a, 0x61, 0x2f, 0xc9, 0x69, 0x42, 0x5e, 0x88, 0x67, 0xe2, 0xc8, 0xe1, 0x64, 0xe7, + 0xb2, 0xa3, 0x53, 0x63, 0x17, 0x47, 0x11, 0x5f, 0xef, 0x54, 0xb0, 0xad, 0xfd, 0xc0, 0x32, 0x79, + 0x5e, 0x5a, 0x37, 0x1b, 0x4f, 0xf0, 0x86, 0xeb, 0x62, 0xf1, 0x5e, 0xcb, 0xa2, 0xe5, 0x9a, 0xf8, + 0x00, 0x7b, 0x0a, 0xd9, 0xd9, 0xd1, 0x8c, 0x5d, 0xdd, 0x72, 0xb5, 0xa0, 0x66, 0xea, 0x14, 0xfb, + 0x9d, 0x12, 0x61, 0x5f, 0x6a, 0xb1, 0xf7, 0x83, 0x8a, 0x6e, 0x18, 0x24, 0x70, 0xa9, 0xaf, 0xf8, + 0xd4, 0xc3, 0xba, 0x63, 0xb9, 0x11, 0x8c, 0x99, 0x74, 0xcd, 0xc6, 0x33, 0x57, 0x95, 0x67, 0xe0, + 0xc2, 0xe3, 0xb0, 0x8c, 0xf7, 0x31, 0x5d, 0xb1, 0x49, 0x65, 0x43, 0xb7, 0x3c, 0x15, 0xef, 0x07, + 0xd8, 0xa7, 0x68, 0x08, 0x7a, 0x2c, 0x33, 0x2f, 0x4d, 0x49, 0xa5, 0x41, 0xb5, 0xc7, 0x32, 0xe5, + 0x67, 0x30, 0xc6, 0x54, 0x9b, 0x7a, 0x7e, 0x8d, 0xb8, 0x3e, 0x46, 0xf7, 0x60, 0xa0, 0x51, 0x27, + 0xa6, 0x9f, 0x5b, 0x18, 0x2f, 0x77, 0xf0, 0xad, 0x1c, 0xd9, 0x2d, 0xf7, 0xbe, 0xfa, 0x7d, 0x32, + 0xa3, 0xf6, 0x1b, 0xe2, 0x5d, 0xd6, 0x05, 0x86, 0x25, 0xdb, 0x6e, 0xc7, 0xb0, 0x0a, 0xd0, 0xe4, + 0x95, 0xf0, 0x3d, 0x5d, 0xe6, 0x24, 0x2c, 0x87, 0x24, 0x2c, 0x73, 0x92, 0x0b, 0x12, 0x96, 0x37, + 0xf4, 0x2a, 0x16, 0xb6, 0x6a, 0xcc, 0x52, 0xfe, 0x5a, 0x82, 0x7c, 0x0b, 0xf8, 0x25, 0xdb, 0x4e, + 0xc3, 0x9f, 0x7d, 0x47, 0xfc, 0xe8, 0x7e, 0x0b, 0xc8, 0x1e, 0x06, 0xf2, 0xda, 0x91, 0x20, 0xf9, + 0xe6, 0x2d, 0x28, 0x7f, 0x93, 0x60, 0x72, 0x1d, 0xd7, 0xff, 0x4f, 0x4c, 0xbc, 0x45, 0xc2, 0xbf, + 0x2b, 0xba, 0x6d, 0x04, 0x36, 0x5b, 0x8c, 0x32, 0xf2, 0x1c, 0xce, 0xf3, 0x53, 0x54, 0xf3, 0x48, + 0x8d, 0xf8, 0xd8, 0xd3, 0x04, 0x5f, 0x1b, 0xd9, 0xe9, 0x44, 0xfe, 0x54, 0xb7, 0x43, 0xbe, 0x12, + 0x6f, 0x1d, 0xd7, 0xd7, 0xb9, 0xb6, 0x7a, 0x8e, 0x79, 0xd9, 0x10, 0x4e, 0x84, 0x14, 0xbd, 0x0f, + 0x63, 0xf5, 0x48, 0x59, 0x73, 0x70, 0x5d, 0x73, 0x30, 0xf5, 0x2c, 0xc3, 0x6f, 0x44, 0xd5, 0xe9, + 0xbc, 0x05, 0xf0, 0x3a, 0x57, 0x57, 0x47, 0xeb, 0xf1, 0x2d, 0xb9, 0x50, 0xfe, 0x4b, 0x82, 0xa9, + 0xf4, 0xf0, 0x44, 0x31, 0xaa, 0x70, 0xda, 0xc3, 0x7e, 0x60, 0x53, 0x5f, 0x94, 0xe2, 0xfe, 0x51, + 0x7b, 0x26, 0x78, 0x09, 0x15, 0x96, 0x5c, 0xf3, 0x29, 0xb1, 0x03, 0x07, 0x6f, 0x60, 0x2f, 0x2c, + 0x9d, 0x28, 0x5b, 0xe4, 0xbd, 0xa0, 0xc3, 0x68, 0x82, 0x16, 0x9a, 0x82, 0x33, 0x0d, 0x32, 0x68, + 0x0d, 0xfe, 0x43, 0x54, 0xec, 0x07, 0x26, 0x1a, 0x86, 0xac, 0x83, 0xeb, 0x2c, 0x23, 0x3d, 0x6a, + 0xf8, 0x88, 0xce, 0x43, 0x5f, 0x9d, 0x39, 0xc9, 0x67, 0xa7, 0xa4, 0x52, 0xaf, 0x2a, 0xde, 0xe4, + 0x59, 0x28, 0x31, 0xd2, 0xfd, 0x8f, 0xb5, 0xa8, 0x2d, 0x0b, 0x7b, 0x0f, 0xc3, 0x06, 0xb5, 0xc2, + 0x5a, 0x46, 0xe0, 0xc5, 0xeb, 0x2a, 0x7f, 0x29, 0xc1, 0xcc, 0x31, 0x94, 0x45, 0x96, 0x5c, 0xc8, + 0xa7, 0xf5, 0x3d, 0xc1, 0x03, 0x25, 0x21, 0x6d, 0xdd, 0x5c, 0x8b, 0xf4, 0x8c, 0xe1, 0x24, 0x1d, + 0x79, 0x06, 0xae, 0x31, 0x70, 0xcb, 0x21, 0x69, 0x54, 0x9d, 0xe2, 0xf4, 0x40, 0xbe, 0x90, 0x44, + 0xd4, 0x5d, 0x75, 0x45, 0x1c, 0x7b, 0x70, 0x21, 0xe5, 0x4e, 0x10, 0x61, 0x94, 0x13, 0xc2, 0xe8, + 0xe2, 0x58, 0x44, 0xc1, 0xc9, 0xdd, 0xa6, 0x22, 0x6f, 0xc3, 0x45, 0x06, 0x6c, 0x93, 0xea, 0x14, + 0xef, 0x04, 0xf6, 0xa3, 0xf0, 0x1e, 0x88, 0xce, 0xd5, 0x22, 0xf4, 0xb3, 0x7b, 0x21, 0xaa, 0x79, + 0x6e, 0xa1, 0x90, 0xb0, 0x35, 0x33, 0x79, 0x60, 0x46, 0x5c, 0x22, 0xfc, 0x55, 0xfe, 0x5e, 0x82, + 0x42, 0x92, 0x6b, 0x11, 0xe5, 0x36, 0x9c, 0xe5, 0xbe, 0x6b, 0xb6, 0x6e, 0x60, 0x07, 0xbb, 0x54, + 0x6c, 0x31, 0x93, 0xb0, 0xc5, 0x43, 0xe2, 0x56, 0xb7, 0xb0, 0xe7, 0x30, 0x17, 0x1b, 0x91, 0x81, + 0xd8, 0x71, 0x88, 0xb4, 0x48, 0xd1, 0x24, 0xe4, 0x76, 0x2c, 0xdb, 0xd6, 0x74, 0x27, 0xec, 0xe9, + 0x8c, 0x93, 0xbd, 0x2a, 0x84, 0xa2, 0x25, 0x26, 0x41, 0x13, 0x30, 0x40, 0x3d, 0xab, 0x5a, 0xc5, + 0x1e, 0x36, 0x19, 0x3b, 0xfb, 0xd5, 0xa6, 0x40, 0xbe, 0x06, 0x57, 0x19, 0xec, 0x87, 0xb1, 0x1b, + 0x2d, 0xb1, 0xa8, 0x2f, 0x24, 0x98, 0x3e, 0x4a, 0x53, 0x04, 0xfb, 0x1c, 0x46, 0x13, 0x2e, 0x48, + 0x11, 0xf0, 0xd5, 0xa4, 0x80, 0x3b, 0x5c, 0x8a, 0x60, 0x91, 0xdd, 0xb1, 0x22, 0xbf, 0x94, 0xe0, + 0xd2, 0x26, 0xbb, 0xee, 0x58, 0x7e, 0x2a, 0x84, 0xec, 0x3d, 0xe1, 0xb7, 0x64, 0x54, 0xc8, 0xce, + 0x03, 0x9c, 0x6d, 0x3b, 0xc0, 0xeb, 0x30, 0xd4, 0xbc, 0x07, 0x35, 0xcb, 0x0c, 0xbb, 0x5b, 0xb6, + 0xb3, 0x75, 0xc6, 0xee, 0xcd, 0xf2, 0x66, 0xe3, 0xf9, 0x81, 0xa9, 0x0e, 0xfa, 0xb1, 0x37, 0x5f, + 0xd6, 0xa1, 0x98, 0x86, 0x48, 0xa4, 0xe4, 0x3f, 0x70, 0x5a, 0x5c, 0xe5, 0xa2, 0xa7, 0x4d, 0x26, + 0xa4, 0x81, 0xfb, 0xe0, 0xa6, 0x11, 0xbf, 0x84, 0x95, 0xfc, 0x4d, 0x16, 0xce, 0xc4, 0xd7, 0xd1, + 0x65, 0x38, 0xc3, 0xcf, 0xcd, 0x2e, 0xb6, 0xaa, 0xbb, 0x54, 0x74, 0xa9, 0x1c, 0x93, 0xad, 0x31, + 0x11, 0x1a, 0x87, 0x01, 0x7c, 0x80, 0x0d, 0xcd, 0x21, 0x26, 0x66, 0xc4, 0x18, 0x54, 0xfb, 0x43, + 0xc1, 0x3a, 0x31, 0x31, 0x7a, 0x02, 0xc3, 0x24, 0x42, 0x2b, 0xc6, 0x0c, 0xc6, 0x8e, 0xdc, 0x42, + 0x29, 0x15, 0x5a, 0x5b, 0x78, 0x6b, 0x19, 0xf5, 0x2c, 0x69, 0x15, 0x85, 0x37, 0x21, 0x27, 0x7a, + 0xc8, 0xc0, 0x7c, 0x6f, 0xea, 0x85, 0xd4, 0xe6, 0x70, 0xd5, 0xb2, 0xed, 0xb5, 0x8c, 0x3a, 0xc0, + 0x6c, 0xc3, 0x17, 0xb4, 0x0a, 0x39, 0xaa, 0xef, 0x61, 0x4f, 0x63, 0xa2, 0xfc, 0x29, 0xe6, 0xe9, + 0x4a, 0xaa, 0xa7, 0xad, 0x50, 0x97, 0xb9, 0x5b, 0xcb, 0xa8, 0x40, 0x1b, 0x6f, 0x48, 0x83, 0x91, + 0x58, 0xa9, 0x45, 0xa0, 0x7d, 0xcc, 0xdb, 0x5c, 0x97, 0x6a, 0x33, 0xa7, 0xcd, 0x9a, 0x37, 0x02, + 0x1e, 0xf6, 0xdb, 0x64, 0xcb, 0xc3, 0x30, 0xc4, 0xbd, 0x6a, 0x0e, 0xf6, 0x7d, 0xbd, 0x8a, 0xe5, + 0xcf, 0x24, 0x18, 0x4b, 0x4c, 0x18, 0x2a, 0x40, 0xbf, 0xef, 0xea, 0x35, 0x7f, 0x97, 0xf0, 0x82, + 0xf5, 0xab, 0x8d, 0x77, 0xb4, 0xdd, 0xa4, 0x08, 0x27, 0xe3, 0x9d, 0x56, 0x78, 0x62, 0x5c, 0x2c, + 0x77, 0x0e, 0x87, 0x8f, 0x76, 0x76, 0x56, 0x42, 0x01, 0xdf, 0xe4, 0xe9, 0x7c, 0x3b, 0x77, 0xbe, + 0x95, 0x60, 0x34, 0x21, 0xdf, 0x68, 0x11, 0xd8, 0x99, 0xe0, 0xe3, 0x83, 0x38, 0x9e, 0x13, 0x29, + 0x63, 0x0f, 0x1b, 0x0f, 0x54, 0x36, 0x25, 0xb1, 0x47, 0x74, 0x1b, 0xfa, 0x58, 0x65, 0x22, 0xb4, + 0xf9, 0xb4, 0x5e, 0x29, 0xd0, 0x08, 0xed, 0x90, 0xb7, 0xb1, 0x7e, 0xe5, 0xe7, 0xb3, 0x53, 0xd9, + 0x52, 0xaf, 0x9a, 0x6b, 0x36, 0x2c, 0x5f, 0x7e, 0xd1, 0x03, 0xc3, 0xed, 0x55, 0x45, 0x73, 0x70, + 0x8a, 0x33, 0x81, 0xe3, 0x4c, 0xdd, 0x6e, 0x2d, 0xa3, 0x72, 0x45, 0xb4, 0x0d, 0x23, 0xb1, 0xf6, + 0x21, 0x78, 0xd4, 0x93, 0xda, 0x75, 0xf9, 0x8e, 0xb1, 0x56, 0x14, 0xb9, 0x1b, 0xb6, 0xdb, 0x64, + 0xe8, 0x19, 0xa0, 0x18, 0x37, 0x35, 0x9f, 0xea, 0x34, 0xf0, 0xc5, 0xe9, 0x99, 0x39, 0x06, 0x45, + 0x37, 0x99, 0x81, 0x3a, 0x4c, 0xdb, 0x24, 0xcb, 0x83, 0x2d, 0xa4, 0x97, 0xbf, 0x93, 0xe0, 0x7c, + 0xb2, 0x6d, 0x98, 0xc6, 0x96, 0xcd, 0xc5, 0xf1, 0x27, 0x31, 0x95, 0x1b, 0x80, 0x3c, 0xec, 0xe8, + 0x96, 0x6b, 0xb9, 0x55, 0x6d, 0x3f, 0xd0, 0x5d, 0x1a, 0x38, 0xbe, 0xb8, 0x20, 0x46, 0x1a, 0x2b, + 0x8f, 0xc5, 0x02, 0xfa, 0x2f, 0x14, 0x49, 0x8d, 0x5a, 0x8e, 0xe5, 0x53, 0xcb, 0xd0, 0x6d, 0xfb, + 0x90, 0x1d, 0x61, 0x6c, 0x36, 0x4d, 0xf9, 0x68, 0x33, 0xd1, 0xaa, 0xb5, 0xca, 0x94, 0x22, 0x2f, + 0x0b, 0x5f, 0x01, 0x9c, 0x62, 0xd7, 0x04, 0xfa, 0x44, 0x82, 0xfe, 0x68, 0x60, 0x46, 0xb3, 0x09, + 0x59, 0x49, 0xf9, 0xea, 0x28, 0x94, 0xd2, 0x74, 0xdb, 0x3f, 0x3b, 0xe4, 0x99, 0x8f, 0x7f, 0xf9, + 0xf3, 0xf3, 0x9e, 0x2b, 0xe8, 0xb2, 0xd2, 0xe5, 0xbb, 0x51, 0xf9, 0xc0, 0x32, 0x3f, 0x44, 0x9f, + 0x4a, 0x90, 0x8b, 0x4d, 0xfe, 0xe9, 0x80, 0x3a, 0x3f, 0x41, 0x0a, 0xd7, 0x8f, 0x02, 0x14, 0xfb, + 0x94, 0x90, 0xff, 0xc1, 0x30, 0x15, 0xd1, 0x44, 0x37, 0x4c, 0xe8, 0x47, 0x09, 0xf2, 0x69, 0x23, + 0x2c, 0x5a, 0x78, 0xa7, 0x79, 0x97, 0x63, 0xbc, 0x79, 0x82, 0x19, 0x59, 0xbe, 0xcb, 0xb0, 0xde, + 0xba, 0x2b, 0xcd, 0xca, 0x8a, 0x92, 0xf8, 0xe1, 0xaa, 0xb9, 0xc4, 0xc4, 0x1a, 0x25, 0xfc, 0xbf, + 0x11, 0x03, 0xf9, 0xb3, 0x04, 0x13, 0xdd, 0xa6, 0x49, 0xb4, 0x98, 0x96, 0xb5, 0x63, 0xcc, 0xc2, + 0x85, 0x7f, 0x9f, 0xcc, 0x58, 0xc4, 0x35, 0xcd, 0xe2, 0x9a, 0x42, 0x45, 0xa5, 0xeb, 0x8f, 0x05, + 0xe8, 0x07, 0x09, 0xc6, 0xbb, 0x8c, 0x92, 0xe8, 0x6e, 0x1a, 0x8a, 0xa3, 0x87, 0xe0, 0xc2, 0xe2, + 0x89, 0x6c, 0x45, 0x00, 0x57, 0x59, 0x00, 0x93, 0xe8, 0x52, 0xd7, 0x5f, 0x50, 0xd0, 0x4f, 0x12, + 0x5c, 0x4c, 0x1d, 0xc7, 0xd0, 0x9d, 0x34, 0x04, 0x47, 0xcd, 0x7a, 0x85, 0x7f, 0x9d, 0xc0, 0x52, + 0x20, 0x2f, 0x33, 0xe4, 0x25, 0x34, 0xad, 0x1c, 0xeb, 0x57, 0x13, 0xe4, 0xc2, 0x60, 0xcb, 0xc4, + 0x8c, 0xfe, 0x99, 0xb6, 0x77, 0xd2, 0xcc, 0x5e, 0xb8, 0x71, 0x4c, 0x6d, 0x81, 0x2e, 0x83, 0x3e, + 0x8a, 0x3a, 0x6a, 0xfb, 0xa8, 0x86, 0xe6, 0x8e, 0x3b, 0xf6, 0x44, 0x73, 0x66, 0x61, 0xfe, 0x1d, + 0x2c, 0x38, 0x80, 0x39, 0x69, 0x79, 0xe3, 0xd5, 0x9b, 0xa2, 0xf4, 0xfa, 0x4d, 0x51, 0xfa, 0xe3, + 0x4d, 0x51, 0x7a, 0xf9, 0xb6, 0x98, 0x79, 0xfd, 0xb6, 0x98, 0xf9, 0xf5, 0x6d, 0x31, 0xf3, 0xde, + 0xed, 0xaa, 0x45, 0x77, 0x83, 0x4a, 0xd9, 0x20, 0x4e, 0x6b, 0xf2, 0xea, 0xb7, 0x6e, 0xb0, 0x0b, + 0x5f, 0x69, 0x48, 0x0e, 0x78, 0x42, 0xe9, 0x61, 0x0d, 0xfb, 0x95, 0x3e, 0x26, 0xbe, 0xf9, 0x77, + 0x00, 0x00, 0x00, 0xff, 0xff, 0xa1, 0x46, 0x6b, 0x53, 0x00, 0x14, 0x00, 0x00, } // Reference imports to suppress errors if they are not otherwise used. @@ -2447,16 +2447,6 @@ func (m *StreamUpdate) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l - if m.ExecMode != 0 { - i = encodeVarintQuery(dAtA, i, uint64(m.ExecMode)) - i-- - dAtA[i] = 0x30 - } - if m.BlockHeight != 0 { - i = encodeVarintQuery(dAtA, i, uint64(m.BlockHeight)) - i-- - dAtA[i] = 0x28 - } if m.UpdateMessage != nil { { size := m.UpdateMessage.Size() @@ -2466,6 +2456,16 @@ func (m *StreamUpdate) MarshalToSizedBuffer(dAtA []byte) (int, error) { } } } + if m.ExecMode != 0 { + i = encodeVarintQuery(dAtA, i, uint64(m.ExecMode)) + i-- + dAtA[i] = 0x10 + } + if m.BlockHeight != 0 { + i = encodeVarintQuery(dAtA, i, uint64(m.BlockHeight)) + i-- + dAtA[i] = 0x8 + } return len(dAtA) - i, nil } @@ -2486,7 +2486,7 @@ func (m *StreamUpdate_OrderbookUpdate) MarshalToSizedBuffer(dAtA []byte) (int, e i = encodeVarintQuery(dAtA, i, uint64(size)) } i-- - dAtA[i] = 0xa + dAtA[i] = 0x1a } return len(dAtA) - i, nil } @@ -2507,7 +2507,7 @@ func (m *StreamUpdate_OrderFill) MarshalToSizedBuffer(dAtA []byte) (int, error) i = encodeVarintQuery(dAtA, i, uint64(size)) } i-- - dAtA[i] = 0x12 + dAtA[i] = 0x22 } return len(dAtA) - i, nil } @@ -2528,7 +2528,7 @@ func (m *StreamUpdate_TakerOrder) MarshalToSizedBuffer(dAtA []byte) (int, error) i = encodeVarintQuery(dAtA, i, uint64(size)) } i-- - dAtA[i] = 0x1a + dAtA[i] = 0x2a } return len(dAtA) - i, nil } @@ -2549,7 +2549,7 @@ func (m *StreamUpdate_SubaccountUpdate) MarshalToSizedBuffer(dAtA []byte) (int, i = encodeVarintQuery(dAtA, i, uint64(size)) } i-- - dAtA[i] = 0x22 + dAtA[i] = 0x32 } return len(dAtA) - i, nil } @@ -2573,16 +2573,6 @@ func (m *StreamOrderbookUpdate) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l - if m.Snapshot { - i-- - if m.Snapshot { - dAtA[i] = 1 - } else { - dAtA[i] = 0 - } - i-- - dAtA[i] = 0x10 - } if len(m.Updates) > 0 { for iNdEx := len(m.Updates) - 1; iNdEx >= 0; iNdEx-- { { @@ -2594,9 +2584,19 @@ func (m *StreamOrderbookUpdate) MarshalToSizedBuffer(dAtA []byte) (int, error) { i = encodeVarintQuery(dAtA, i, uint64(size)) } i-- - dAtA[i] = 0xa + dAtA[i] = 0x12 } } + if m.Snapshot { + i-- + if m.Snapshot { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i-- + dAtA[i] = 0x8 + } return len(dAtA) - i, nil } @@ -3038,15 +3038,15 @@ func (m *StreamUpdate) Size() (n int) { } var l int _ = l - if m.UpdateMessage != nil { - n += m.UpdateMessage.Size() - } if m.BlockHeight != 0 { n += 1 + sovQuery(uint64(m.BlockHeight)) } if m.ExecMode != 0 { n += 1 + sovQuery(uint64(m.ExecMode)) } + if m.UpdateMessage != nil { + n += m.UpdateMessage.Size() + } return n } @@ -3104,15 +3104,15 @@ func (m *StreamOrderbookUpdate) Size() (n int) { } var l int _ = l + if m.Snapshot { + n += 2 + } if len(m.Updates) > 0 { for _, e := range m.Updates { l = e.Size() n += 1 + l + sovQuery(uint64(l)) } } - if m.Snapshot { - n += 2 - } return n } @@ -4747,6 +4747,44 @@ func (m *StreamUpdate) Unmarshal(dAtA []byte) error { } switch fieldNum { case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field BlockHeight", wireType) + } + m.BlockHeight = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowQuery + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.BlockHeight |= uint32(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field ExecMode", wireType) + } + m.ExecMode = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowQuery + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.ExecMode |= uint32(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 3: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field OrderbookUpdate", wireType) } @@ -4781,7 +4819,7 @@ func (m *StreamUpdate) Unmarshal(dAtA []byte) error { } m.UpdateMessage = &StreamUpdate_OrderbookUpdate{v} iNdEx = postIndex - case 2: + case 4: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field OrderFill", wireType) } @@ -4816,7 +4854,7 @@ func (m *StreamUpdate) Unmarshal(dAtA []byte) error { } m.UpdateMessage = &StreamUpdate_OrderFill{v} iNdEx = postIndex - case 3: + case 5: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field TakerOrder", wireType) } @@ -4851,7 +4889,7 @@ func (m *StreamUpdate) Unmarshal(dAtA []byte) error { } m.UpdateMessage = &StreamUpdate_TakerOrder{v} iNdEx = postIndex - case 4: + case 6: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field SubaccountUpdate", wireType) } @@ -4886,44 +4924,6 @@ func (m *StreamUpdate) Unmarshal(dAtA []byte) error { } m.UpdateMessage = &StreamUpdate_SubaccountUpdate{v} iNdEx = postIndex - case 5: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field BlockHeight", wireType) - } - m.BlockHeight = 0 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowQuery - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - m.BlockHeight |= uint32(b&0x7F) << shift - if b < 0x80 { - break - } - } - case 6: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field ExecMode", wireType) - } - m.ExecMode = 0 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowQuery - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - m.ExecMode |= uint32(b&0x7F) << shift - if b < 0x80 { - break - } - } default: iNdEx = preIndex skippy, err := skipQuery(dAtA[iNdEx:]) @@ -4975,6 +4975,26 @@ func (m *StreamOrderbookUpdate) Unmarshal(dAtA []byte) error { } switch fieldNum { case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Snapshot", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowQuery + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.Snapshot = bool(v != 0) + case 2: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Updates", wireType) } @@ -5008,26 +5028,6 @@ func (m *StreamOrderbookUpdate) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex - case 2: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field Snapshot", wireType) - } - var v int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowQuery - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - v |= int(b&0x7F) << shift - if b < 0x80 { - break - } - } - m.Snapshot = bool(v != 0) default: iNdEx = preIndex skippy, err := skipQuery(dAtA[iNdEx:]) From 26c4554d7dfdc17da43ad848867f374f94c86663 Mon Sep 17 00:00:00 2001 From: Teddy Ding Date: Mon, 16 Sep 2024 12:41:19 -0400 Subject: [PATCH 24/33] [OTE-456] FNS x OE: stage `FinalizeBlock` events and emit in `Precommit` (#2253) --- .../src/codegen/dydxprotocol/bundle.ts | 466 ++++++++--------- .../codegen/dydxprotocol/clob/streaming.ts | 71 +++ .../v4-protos/src/codegen/gogoproto/bundle.ts | 4 +- .../v4-protos/src/codegen/google/bundle.ts | 22 +- proto/dydxprotocol/clob/streaming.proto | 16 + protocol/app/app.go | 4 + protocol/lib/metrics/constants.go | 1 + protocol/lib/metrics/metric_keys.go | 29 +- protocol/streaming/constants.go | 13 + .../streaming/full_node_streaming_manager.go | 167 +++++++ protocol/streaming/noop_streaming_manager.go | 25 + protocol/streaming/types/interface.go | 16 + protocol/x/clob/abci.go | 11 + .../clob/keeper/grpc_stream_finalize_block.go | 50 ++ protocol/x/clob/keeper/process_operations.go | 34 +- protocol/x/clob/types/streaming.pb.go | 473 ++++++++++++++++++ protocol/x/subaccounts/keeper/subaccount.go | 6 +- 17 files changed, 1118 insertions(+), 290 deletions(-) create mode 100644 indexer/packages/v4-protos/src/codegen/dydxprotocol/clob/streaming.ts create mode 100644 proto/dydxprotocol/clob/streaming.proto create mode 100644 protocol/streaming/constants.go create mode 100644 protocol/x/clob/keeper/grpc_stream_finalize_block.go create mode 100644 protocol/x/clob/types/streaming.pb.go diff --git a/indexer/packages/v4-protos/src/codegen/dydxprotocol/bundle.ts b/indexer/packages/v4-protos/src/codegen/dydxprotocol/bundle.ts index fe4846372f..135d26b777 100644 --- a/indexer/packages/v4-protos/src/codegen/dydxprotocol/bundle.ts +++ b/indexer/packages/v4-protos/src/codegen/dydxprotocol/bundle.ts @@ -28,142 +28,143 @@ import * as _31 from "./clob/order_removals"; import * as _32 from "./clob/order"; import * as _33 from "./clob/process_proposer_matches_events"; import * as _34 from "./clob/query"; -import * as _35 from "./clob/tx"; -import * as _36 from "./daemons/bridge/bridge"; -import * as _37 from "./daemons/liquidation/liquidation"; -import * as _38 from "./daemons/pricefeed/price_feed"; -import * as _39 from "./delaymsg/block_message_ids"; -import * as _40 from "./delaymsg/delayed_message"; -import * as _41 from "./delaymsg/genesis"; -import * as _42 from "./delaymsg/query"; -import * as _43 from "./delaymsg/tx"; -import * as _44 from "./epochs/epoch_info"; -import * as _45 from "./epochs/genesis"; -import * as _46 from "./epochs/query"; -import * as _47 from "./feetiers/genesis"; -import * as _48 from "./feetiers/params"; -import * as _49 from "./feetiers/query"; -import * as _50 from "./feetiers/tx"; -import * as _51 from "./govplus/genesis"; -import * as _52 from "./govplus/query"; -import * as _53 from "./govplus/tx"; -import * as _54 from "./indexer/events/events"; -import * as _55 from "./indexer/indexer_manager/event"; -import * as _56 from "./indexer/off_chain_updates/off_chain_updates"; -import * as _57 from "./indexer/protocol/v1/clob"; -import * as _58 from "./indexer/protocol/v1/perpetual"; -import * as _59 from "./indexer/protocol/v1/subaccount"; -import * as _60 from "./indexer/redis/redis_order"; -import * as _61 from "./indexer/shared/removal_reason"; -import * as _62 from "./indexer/socks/messages"; -import * as _63 from "./listing/genesis"; -import * as _64 from "./listing/query"; -import * as _65 from "./listing/tx"; -import * as _66 from "./perpetuals/genesis"; -import * as _67 from "./perpetuals/params"; -import * as _68 from "./perpetuals/perpetual"; -import * as _69 from "./perpetuals/query"; -import * as _70 from "./perpetuals/tx"; -import * as _71 from "./prices/genesis"; -import * as _72 from "./prices/market_param"; -import * as _73 from "./prices/market_price"; -import * as _74 from "./prices/query"; -import * as _75 from "./prices/tx"; -import * as _76 from "./ratelimit/capacity"; -import * as _77 from "./ratelimit/genesis"; -import * as _78 from "./ratelimit/limit_params"; -import * as _79 from "./ratelimit/pending_send_packet"; -import * as _80 from "./ratelimit/query"; -import * as _81 from "./ratelimit/tx"; -import * as _82 from "./revshare/genesis"; -import * as _83 from "./revshare/params"; -import * as _84 from "./revshare/query"; -import * as _85 from "./revshare/revshare"; -import * as _86 from "./revshare/tx"; -import * as _87 from "./rewards/genesis"; -import * as _88 from "./rewards/params"; -import * as _89 from "./rewards/query"; -import * as _90 from "./rewards/reward_share"; -import * as _91 from "./rewards/tx"; -import * as _92 from "./sending/genesis"; -import * as _93 from "./sending/query"; -import * as _94 from "./sending/transfer"; -import * as _95 from "./sending/tx"; -import * as _96 from "./stats/genesis"; -import * as _97 from "./stats/params"; -import * as _98 from "./stats/query"; -import * as _99 from "./stats/stats"; -import * as _100 from "./stats/tx"; -import * as _101 from "./subaccounts/asset_position"; -import * as _102 from "./subaccounts/genesis"; -import * as _103 from "./subaccounts/perpetual_position"; -import * as _104 from "./subaccounts/query"; -import * as _105 from "./subaccounts/streaming"; -import * as _106 from "./subaccounts/subaccount"; -import * as _107 from "./vault/genesis"; -import * as _108 from "./vault/params"; -import * as _109 from "./vault/query"; -import * as _110 from "./vault/share"; -import * as _111 from "./vault/tx"; -import * as _112 from "./vault/vault"; -import * as _113 from "./vest/genesis"; -import * as _114 from "./vest/query"; -import * as _115 from "./vest/tx"; -import * as _116 from "./vest/vest_entry"; -import * as _124 from "./assets/query.lcd"; -import * as _125 from "./blocktime/query.lcd"; -import * as _126 from "./bridge/query.lcd"; -import * as _127 from "./clob/query.lcd"; -import * as _128 from "./delaymsg/query.lcd"; -import * as _129 from "./epochs/query.lcd"; -import * as _130 from "./feetiers/query.lcd"; -import * as _131 from "./perpetuals/query.lcd"; -import * as _132 from "./prices/query.lcd"; -import * as _133 from "./ratelimit/query.lcd"; -import * as _134 from "./revshare/query.lcd"; -import * as _135 from "./rewards/query.lcd"; -import * as _136 from "./stats/query.lcd"; -import * as _137 from "./subaccounts/query.lcd"; -import * as _138 from "./vault/query.lcd"; -import * as _139 from "./vest/query.lcd"; -import * as _140 from "./assets/query.rpc.Query"; -import * as _141 from "./blocktime/query.rpc.Query"; -import * as _142 from "./bridge/query.rpc.Query"; -import * as _143 from "./clob/query.rpc.Query"; -import * as _144 from "./delaymsg/query.rpc.Query"; -import * as _145 from "./epochs/query.rpc.Query"; -import * as _146 from "./feetiers/query.rpc.Query"; -import * as _147 from "./govplus/query.rpc.Query"; -import * as _148 from "./listing/query.rpc.Query"; -import * as _149 from "./perpetuals/query.rpc.Query"; -import * as _150 from "./prices/query.rpc.Query"; -import * as _151 from "./ratelimit/query.rpc.Query"; -import * as _152 from "./revshare/query.rpc.Query"; -import * as _153 from "./rewards/query.rpc.Query"; -import * as _154 from "./sending/query.rpc.Query"; -import * as _155 from "./stats/query.rpc.Query"; -import * as _156 from "./subaccounts/query.rpc.Query"; -import * as _157 from "./vault/query.rpc.Query"; -import * as _158 from "./vest/query.rpc.Query"; -import * as _159 from "./blocktime/tx.rpc.msg"; -import * as _160 from "./bridge/tx.rpc.msg"; -import * as _161 from "./clob/tx.rpc.msg"; -import * as _162 from "./delaymsg/tx.rpc.msg"; -import * as _163 from "./feetiers/tx.rpc.msg"; -import * as _164 from "./govplus/tx.rpc.msg"; -import * as _165 from "./listing/tx.rpc.msg"; -import * as _166 from "./perpetuals/tx.rpc.msg"; -import * as _167 from "./prices/tx.rpc.msg"; -import * as _168 from "./ratelimit/tx.rpc.msg"; -import * as _169 from "./revshare/tx.rpc.msg"; -import * as _170 from "./rewards/tx.rpc.msg"; -import * as _171 from "./sending/tx.rpc.msg"; -import * as _172 from "./stats/tx.rpc.msg"; -import * as _173 from "./vault/tx.rpc.msg"; -import * as _174 from "./vest/tx.rpc.msg"; -import * as _175 from "./lcd"; -import * as _176 from "./rpc.query"; -import * as _177 from "./rpc.tx"; +import * as _35 from "./clob/streaming"; +import * as _36 from "./clob/tx"; +import * as _37 from "./daemons/bridge/bridge"; +import * as _38 from "./daemons/liquidation/liquidation"; +import * as _39 from "./daemons/pricefeed/price_feed"; +import * as _40 from "./delaymsg/block_message_ids"; +import * as _41 from "./delaymsg/delayed_message"; +import * as _42 from "./delaymsg/genesis"; +import * as _43 from "./delaymsg/query"; +import * as _44 from "./delaymsg/tx"; +import * as _45 from "./epochs/epoch_info"; +import * as _46 from "./epochs/genesis"; +import * as _47 from "./epochs/query"; +import * as _48 from "./feetiers/genesis"; +import * as _49 from "./feetiers/params"; +import * as _50 from "./feetiers/query"; +import * as _51 from "./feetiers/tx"; +import * as _52 from "./govplus/genesis"; +import * as _53 from "./govplus/query"; +import * as _54 from "./govplus/tx"; +import * as _55 from "./indexer/events/events"; +import * as _56 from "./indexer/indexer_manager/event"; +import * as _57 from "./indexer/off_chain_updates/off_chain_updates"; +import * as _58 from "./indexer/protocol/v1/clob"; +import * as _59 from "./indexer/protocol/v1/perpetual"; +import * as _60 from "./indexer/protocol/v1/subaccount"; +import * as _61 from "./indexer/redis/redis_order"; +import * as _62 from "./indexer/shared/removal_reason"; +import * as _63 from "./indexer/socks/messages"; +import * as _64 from "./listing/genesis"; +import * as _65 from "./listing/query"; +import * as _66 from "./listing/tx"; +import * as _67 from "./perpetuals/genesis"; +import * as _68 from "./perpetuals/params"; +import * as _69 from "./perpetuals/perpetual"; +import * as _70 from "./perpetuals/query"; +import * as _71 from "./perpetuals/tx"; +import * as _72 from "./prices/genesis"; +import * as _73 from "./prices/market_param"; +import * as _74 from "./prices/market_price"; +import * as _75 from "./prices/query"; +import * as _76 from "./prices/tx"; +import * as _77 from "./ratelimit/capacity"; +import * as _78 from "./ratelimit/genesis"; +import * as _79 from "./ratelimit/limit_params"; +import * as _80 from "./ratelimit/pending_send_packet"; +import * as _81 from "./ratelimit/query"; +import * as _82 from "./ratelimit/tx"; +import * as _83 from "./revshare/genesis"; +import * as _84 from "./revshare/params"; +import * as _85 from "./revshare/query"; +import * as _86 from "./revshare/revshare"; +import * as _87 from "./revshare/tx"; +import * as _88 from "./rewards/genesis"; +import * as _89 from "./rewards/params"; +import * as _90 from "./rewards/query"; +import * as _91 from "./rewards/reward_share"; +import * as _92 from "./rewards/tx"; +import * as _93 from "./sending/genesis"; +import * as _94 from "./sending/query"; +import * as _95 from "./sending/transfer"; +import * as _96 from "./sending/tx"; +import * as _97 from "./stats/genesis"; +import * as _98 from "./stats/params"; +import * as _99 from "./stats/query"; +import * as _100 from "./stats/stats"; +import * as _101 from "./stats/tx"; +import * as _102 from "./subaccounts/asset_position"; +import * as _103 from "./subaccounts/genesis"; +import * as _104 from "./subaccounts/perpetual_position"; +import * as _105 from "./subaccounts/query"; +import * as _106 from "./subaccounts/streaming"; +import * as _107 from "./subaccounts/subaccount"; +import * as _108 from "./vault/genesis"; +import * as _109 from "./vault/params"; +import * as _110 from "./vault/query"; +import * as _111 from "./vault/share"; +import * as _112 from "./vault/tx"; +import * as _113 from "./vault/vault"; +import * as _114 from "./vest/genesis"; +import * as _115 from "./vest/query"; +import * as _116 from "./vest/tx"; +import * as _117 from "./vest/vest_entry"; +import * as _125 from "./assets/query.lcd"; +import * as _126 from "./blocktime/query.lcd"; +import * as _127 from "./bridge/query.lcd"; +import * as _128 from "./clob/query.lcd"; +import * as _129 from "./delaymsg/query.lcd"; +import * as _130 from "./epochs/query.lcd"; +import * as _131 from "./feetiers/query.lcd"; +import * as _132 from "./perpetuals/query.lcd"; +import * as _133 from "./prices/query.lcd"; +import * as _134 from "./ratelimit/query.lcd"; +import * as _135 from "./revshare/query.lcd"; +import * as _136 from "./rewards/query.lcd"; +import * as _137 from "./stats/query.lcd"; +import * as _138 from "./subaccounts/query.lcd"; +import * as _139 from "./vault/query.lcd"; +import * as _140 from "./vest/query.lcd"; +import * as _141 from "./assets/query.rpc.Query"; +import * as _142 from "./blocktime/query.rpc.Query"; +import * as _143 from "./bridge/query.rpc.Query"; +import * as _144 from "./clob/query.rpc.Query"; +import * as _145 from "./delaymsg/query.rpc.Query"; +import * as _146 from "./epochs/query.rpc.Query"; +import * as _147 from "./feetiers/query.rpc.Query"; +import * as _148 from "./govplus/query.rpc.Query"; +import * as _149 from "./listing/query.rpc.Query"; +import * as _150 from "./perpetuals/query.rpc.Query"; +import * as _151 from "./prices/query.rpc.Query"; +import * as _152 from "./ratelimit/query.rpc.Query"; +import * as _153 from "./revshare/query.rpc.Query"; +import * as _154 from "./rewards/query.rpc.Query"; +import * as _155 from "./sending/query.rpc.Query"; +import * as _156 from "./stats/query.rpc.Query"; +import * as _157 from "./subaccounts/query.rpc.Query"; +import * as _158 from "./vault/query.rpc.Query"; +import * as _159 from "./vest/query.rpc.Query"; +import * as _160 from "./blocktime/tx.rpc.msg"; +import * as _161 from "./bridge/tx.rpc.msg"; +import * as _162 from "./clob/tx.rpc.msg"; +import * as _163 from "./delaymsg/tx.rpc.msg"; +import * as _164 from "./feetiers/tx.rpc.msg"; +import * as _165 from "./govplus/tx.rpc.msg"; +import * as _166 from "./listing/tx.rpc.msg"; +import * as _167 from "./perpetuals/tx.rpc.msg"; +import * as _168 from "./prices/tx.rpc.msg"; +import * as _169 from "./ratelimit/tx.rpc.msg"; +import * as _170 from "./revshare/tx.rpc.msg"; +import * as _171 from "./rewards/tx.rpc.msg"; +import * as _172 from "./sending/tx.rpc.msg"; +import * as _173 from "./stats/tx.rpc.msg"; +import * as _174 from "./vault/tx.rpc.msg"; +import * as _175 from "./vest/tx.rpc.msg"; +import * as _176 from "./lcd"; +import * as _177 from "./rpc.query"; +import * as _178 from "./rpc.tx"; export namespace dydxprotocol { export const accountplus = { ..._5, ..._6 @@ -172,17 +173,17 @@ export namespace dydxprotocol { ..._8, ..._9, ..._10, - ..._124, - ..._140 + ..._125, + ..._141 }; export const blocktime = { ..._11, ..._12, ..._13, ..._14, ..._15, - ..._125, - ..._141, - ..._159 + ..._126, + ..._142, + ..._160 }; export const bridge = { ..._16, ..._17, @@ -190,9 +191,9 @@ export namespace dydxprotocol { ..._19, ..._20, ..._21, - ..._126, - ..._142, - ..._160 + ..._127, + ..._143, + ..._161 }; export const clob = { ..._22, ..._23, @@ -208,164 +209,165 @@ export namespace dydxprotocol { ..._33, ..._34, ..._35, - ..._127, - ..._143, - ..._161 + ..._36, + ..._128, + ..._144, + ..._162 }; export namespace daemons { - export const bridge = { ..._36 + export const bridge = { ..._37 }; - export const liquidation = { ..._37 + export const liquidation = { ..._38 }; - export const pricefeed = { ..._38 + export const pricefeed = { ..._39 }; } - export const delaymsg = { ..._39, - ..._40, + export const delaymsg = { ..._40, ..._41, ..._42, ..._43, - ..._128, - ..._144, - ..._162 + ..._44, + ..._129, + ..._145, + ..._163 }; - export const epochs = { ..._44, - ..._45, + export const epochs = { ..._45, ..._46, - ..._129, - ..._145 + ..._47, + ..._130, + ..._146 }; - export const feetiers = { ..._47, - ..._48, + export const feetiers = { ..._48, ..._49, ..._50, - ..._130, - ..._146, - ..._163 - }; - export const govplus = { ..._51, - ..._52, - ..._53, + ..._51, + ..._131, ..._147, ..._164 }; + export const govplus = { ..._52, + ..._53, + ..._54, + ..._148, + ..._165 + }; export namespace indexer { - export const events = { ..._54 + export const events = { ..._55 }; - export const indexer_manager = { ..._55 + export const indexer_manager = { ..._56 }; - export const off_chain_updates = { ..._56 + export const off_chain_updates = { ..._57 }; export namespace protocol { - export const v1 = { ..._57, - ..._58, - ..._59 + export const v1 = { ..._58, + ..._59, + ..._60 }; } - export const redis = { ..._60 + export const redis = { ..._61 }; - export const shared = { ..._61 + export const shared = { ..._62 }; - export const socks = { ..._62 + export const socks = { ..._63 }; } - export const listing = { ..._63, - ..._64, + export const listing = { ..._64, ..._65, - ..._148, - ..._165 + ..._66, + ..._149, + ..._166 }; - export const perpetuals = { ..._66, - ..._67, + export const perpetuals = { ..._67, ..._68, ..._69, ..._70, - ..._131, - ..._149, - ..._166 + ..._71, + ..._132, + ..._150, + ..._167 }; - export const prices = { ..._71, - ..._72, + export const prices = { ..._72, ..._73, ..._74, ..._75, - ..._132, - ..._150, - ..._167 + ..._76, + ..._133, + ..._151, + ..._168 }; - export const ratelimit = { ..._76, - ..._77, + export const ratelimit = { ..._77, ..._78, ..._79, ..._80, ..._81, - ..._133, - ..._151, - ..._168 + ..._82, + ..._134, + ..._152, + ..._169 }; - export const revshare = { ..._82, - ..._83, + export const revshare = { ..._83, ..._84, ..._85, ..._86, - ..._134, - ..._152, - ..._169 + ..._87, + ..._135, + ..._153, + ..._170 }; - export const rewards = { ..._87, - ..._88, + export const rewards = { ..._88, ..._89, ..._90, ..._91, - ..._135, - ..._153, - ..._170 + ..._92, + ..._136, + ..._154, + ..._171 }; - export const sending = { ..._92, - ..._93, + export const sending = { ..._93, ..._94, ..._95, - ..._154, - ..._171 + ..._96, + ..._155, + ..._172 }; - export const stats = { ..._96, - ..._97, + export const stats = { ..._97, ..._98, ..._99, ..._100, - ..._136, - ..._155, - ..._172 + ..._101, + ..._137, + ..._156, + ..._173 }; - export const subaccounts = { ..._101, - ..._102, + export const subaccounts = { ..._102, ..._103, ..._104, ..._105, ..._106, - ..._137, - ..._156 + ..._107, + ..._138, + ..._157 }; - export const vault = { ..._107, - ..._108, + export const vault = { ..._108, ..._109, ..._110, ..._111, ..._112, - ..._138, - ..._157, - ..._173 - }; - export const vest = { ..._113, - ..._114, - ..._115, - ..._116, + ..._113, ..._139, ..._158, ..._174 }; - export const ClientFactory = { ..._175, - ..._176, - ..._177 + export const vest = { ..._114, + ..._115, + ..._116, + ..._117, + ..._140, + ..._159, + ..._175 + }; + export const ClientFactory = { ..._176, + ..._177, + ..._178 }; } \ No newline at end of file diff --git a/indexer/packages/v4-protos/src/codegen/dydxprotocol/clob/streaming.ts b/indexer/packages/v4-protos/src/codegen/dydxprotocol/clob/streaming.ts new file mode 100644 index 0000000000..dab8f1e122 --- /dev/null +++ b/indexer/packages/v4-protos/src/codegen/dydxprotocol/clob/streaming.ts @@ -0,0 +1,71 @@ +import { StreamOrderbookFill, StreamOrderbookFillSDKType } from "./query"; +import { StreamSubaccountUpdate, StreamSubaccountUpdateSDKType } from "../subaccounts/streaming"; +import * as _m0 from "protobufjs/minimal"; +import { DeepPartial } from "../../helpers"; +/** StagedFinalizeBlockEvent is an event staged during `FinalizeBlock`. */ + +export interface StagedFinalizeBlockEvent { + orderFill?: StreamOrderbookFill; + subaccountUpdate?: StreamSubaccountUpdate; +} +/** StagedFinalizeBlockEvent is an event staged during `FinalizeBlock`. */ + +export interface StagedFinalizeBlockEventSDKType { + order_fill?: StreamOrderbookFillSDKType; + subaccount_update?: StreamSubaccountUpdateSDKType; +} + +function createBaseStagedFinalizeBlockEvent(): StagedFinalizeBlockEvent { + return { + orderFill: undefined, + subaccountUpdate: undefined + }; +} + +export const StagedFinalizeBlockEvent = { + encode(message: StagedFinalizeBlockEvent, writer: _m0.Writer = _m0.Writer.create()): _m0.Writer { + if (message.orderFill !== undefined) { + StreamOrderbookFill.encode(message.orderFill, writer.uint32(10).fork()).ldelim(); + } + + if (message.subaccountUpdate !== undefined) { + StreamSubaccountUpdate.encode(message.subaccountUpdate, writer.uint32(18).fork()).ldelim(); + } + + return writer; + }, + + decode(input: _m0.Reader | Uint8Array, length?: number): StagedFinalizeBlockEvent { + const reader = input instanceof _m0.Reader ? input : new _m0.Reader(input); + let end = length === undefined ? reader.len : reader.pos + length; + const message = createBaseStagedFinalizeBlockEvent(); + + while (reader.pos < end) { + const tag = reader.uint32(); + + switch (tag >>> 3) { + case 1: + message.orderFill = StreamOrderbookFill.decode(reader, reader.uint32()); + break; + + case 2: + message.subaccountUpdate = StreamSubaccountUpdate.decode(reader, reader.uint32()); + break; + + default: + reader.skipType(tag & 7); + break; + } + } + + return message; + }, + + fromPartial(object: DeepPartial): StagedFinalizeBlockEvent { + const message = createBaseStagedFinalizeBlockEvent(); + message.orderFill = object.orderFill !== undefined && object.orderFill !== null ? StreamOrderbookFill.fromPartial(object.orderFill) : undefined; + message.subaccountUpdate = object.subaccountUpdate !== undefined && object.subaccountUpdate !== null ? StreamSubaccountUpdate.fromPartial(object.subaccountUpdate) : undefined; + return message; + } + +}; \ No newline at end of file diff --git a/indexer/packages/v4-protos/src/codegen/gogoproto/bundle.ts b/indexer/packages/v4-protos/src/codegen/gogoproto/bundle.ts index cdc090d0d6..318577daaa 100644 --- a/indexer/packages/v4-protos/src/codegen/gogoproto/bundle.ts +++ b/indexer/packages/v4-protos/src/codegen/gogoproto/bundle.ts @@ -1,3 +1,3 @@ -import * as _117 from "./gogo"; -export const gogoproto = { ..._117 +import * as _118 from "./gogo"; +export const gogoproto = { ..._118 }; \ No newline at end of file diff --git a/indexer/packages/v4-protos/src/codegen/google/bundle.ts b/indexer/packages/v4-protos/src/codegen/google/bundle.ts index 7103f2f637..75cf041bac 100644 --- a/indexer/packages/v4-protos/src/codegen/google/bundle.ts +++ b/indexer/packages/v4-protos/src/codegen/google/bundle.ts @@ -1,16 +1,16 @@ -import * as _118 from "./api/annotations"; -import * as _119 from "./api/http"; -import * as _120 from "./protobuf/descriptor"; -import * as _121 from "./protobuf/duration"; -import * as _122 from "./protobuf/timestamp"; -import * as _123 from "./protobuf/any"; +import * as _119 from "./api/annotations"; +import * as _120 from "./api/http"; +import * as _121 from "./protobuf/descriptor"; +import * as _122 from "./protobuf/duration"; +import * as _123 from "./protobuf/timestamp"; +import * as _124 from "./protobuf/any"; export namespace google { - export const api = { ..._118, - ..._119 + export const api = { ..._119, + ..._120 }; - export const protobuf = { ..._120, - ..._121, + export const protobuf = { ..._121, ..._122, - ..._123 + ..._123, + ..._124 }; } \ No newline at end of file diff --git a/proto/dydxprotocol/clob/streaming.proto b/proto/dydxprotocol/clob/streaming.proto new file mode 100644 index 0000000000..06c74ffbe1 --- /dev/null +++ b/proto/dydxprotocol/clob/streaming.proto @@ -0,0 +1,16 @@ +syntax = "proto3"; +package dydxprotocol.clob; + +import "dydxprotocol/subaccounts/streaming.proto"; +import "dydxprotocol/clob/query.proto"; + +option go_package = "github.com/dydxprotocol/v4-chain/protocol/x/clob/types"; + +// StagedFinalizeBlockEvent is an event staged during `FinalizeBlock`. +message StagedFinalizeBlockEvent { + // Contains one of StreamOrderbookFill, StreamSubaccountUpdate. + oneof event { + StreamOrderbookFill order_fill = 1; + dydxprotocol.subaccounts.StreamSubaccountUpdate subaccount_update = 2; + } +} diff --git a/protocol/app/app.go b/protocol/app/app.go index 35386767e8..67d12d2711 100644 --- a/protocol/app/app.go +++ b/protocol/app/app.go @@ -467,6 +467,7 @@ func New( statsmoduletypes.TransientStoreKey, rewardsmoduletypes.TransientStoreKey, indexer_manager.TransientStoreKey, + streaming.StreamingManagerTransientStoreKey, perpetualsmoduletypes.TransientStoreKey, ) memKeys := storetypes.NewMemoryStoreKeys(capabilitytypes.MemStoreKey, clobmoduletypes.MemStoreKey) @@ -762,6 +763,7 @@ func New( appFlags, appCodec, logger, + tkeys[streaming.StreamingManagerTransientStoreKey], ) timeProvider := &timelib.TimeProviderImpl{} @@ -2029,6 +2031,7 @@ func getFullNodeStreamingManagerFromOptions( appFlags flags.Flags, cdc codec.Codec, logger log.Logger, + streamingManagerTransientStoreKey storetypes.StoreKey, ) (manager streamingtypes.FullNodeStreamingManager, wsServer *ws.WebsocketServer) { logger = logger.With(log.ModuleKey, "full-node-streaming") if appFlags.GrpcStreamingEnabled { @@ -2042,6 +2045,7 @@ func getFullNodeStreamingManagerFromOptions( appFlags.GrpcStreamingMaxBatchSize, appFlags.GrpcStreamingMaxChannelBufferSize, appFlags.FullNodeStreamingSnapshotInterval, + streamingManagerTransientStoreKey, ) // Start websocket server. diff --git a/protocol/lib/metrics/constants.go b/protocol/lib/metrics/constants.go index 0ac766bf5b..a0f851eb58 100644 --- a/protocol/lib/metrics/constants.go +++ b/protocol/lib/metrics/constants.go @@ -202,6 +202,7 @@ const ( UpdateType = "update_type" ValidateMatches = "validate_matches" ValidateOrder = "validate_order" + StreamBatchUpdatesAfterFinalizeBlock = "stream_batch_updates_after_finalize_block" // MemCLOB. AddedToOrderBook = "added_to_orderbook" diff --git a/protocol/lib/metrics/metric_keys.go b/protocol/lib/metrics/metric_keys.go index e38890d8ac..ac8e8a17d1 100644 --- a/protocol/lib/metrics/metric_keys.go +++ b/protocol/lib/metrics/metric_keys.go @@ -66,19 +66,22 @@ const ( GateWithdrawalsIfNegativeTncSubaccountSeenLatency = "gate_withdrawals_if_negative_tnc_subaccount_seen_latency" // Full node grpc - FullNodeGrpc = "full_node_grpc" - GrpcSendOrderbookUpdatesLatency = "grpc_send_orderbook_updates_latency" - GrpcSendOrderbookSnapshotLatency = "grpc_send_orderbook_snapshot_latency" - GrpcSendSubaccountSnapshotLatency = "grpc_send_subaccount_snapshot_latency" - GrpcSendOrderbookFillsLatency = "grpc_send_orderbook_fills_latency" - GrpcSendFinalizedSubaccountUpdatesLatency = "grpc_send_finalized_subaccount_updates_latency" - GrpcAddUpdateToBufferCount = "grpc_add_update_to_buffer_count" - GrpcAddToSubscriptionChannelCount = "grpc_add_to_subscription_channel_count" - GrpcSendResponseToSubscriberCount = "grpc_send_response_to_subscriber_count" - GrpcStreamSubscriberCount = "grpc_stream_subscriber_count" - GrpcStreamNumUpdatesBuffered = "grpc_stream_num_updates_buffered" - GrpcFlushUpdatesLatency = "grpc_flush_updates_latency" - GrpcSubscriptionChannelLength = "grpc_subscription_channel_length" + FullNodeGrpc = "full_node_grpc" + GrpcSendOrderbookUpdatesLatency = "grpc_send_orderbook_updates_latency" + GrpcSendOrderbookSnapshotLatency = "grpc_send_orderbook_snapshot_latency" + GrpcSendSubaccountSnapshotLatency = "grpc_send_subaccount_snapshot_latency" + GrpcSendOrderbookFillsLatency = "grpc_send_orderbook_fills_latency" + GrpcSendFinalizedSubaccountUpdatesLatency = "grpc_send_finalized_subaccount_updates_latency" + GrpcAddUpdateToBufferCount = "grpc_add_update_to_buffer_count" + GrpcAddToSubscriptionChannelCount = "grpc_add_to_subscription_channel_count" + GrpcSendResponseToSubscriberCount = "grpc_send_response_to_subscriber_count" + GrpcStreamSubscriberCount = "grpc_stream_subscriber_count" + GrpcStreamNumUpdatesBuffered = "grpc_stream_num_updates_buffered" + GrpcFlushUpdatesLatency = "grpc_flush_updates_latency" + GrpcSubscriptionChannelLength = "grpc_subscription_channel_length" + GrpcStagedAllFinalizeBlockUpdatesCount = "grpc_staged_all_finalize_block_updates_count" + GrpcStagedFillFinalizeBlockUpdatesCount = "grpc_staged_finalize_block_fill_updates_count" + GrpcStagedSubaccountFinalizeBlockUpdatesCount = "grpc_staged_finalize_block_subaccount_updates_count" EndBlocker = "end_blocker" EndBlockerLag = "end_blocker_lag" diff --git a/protocol/streaming/constants.go b/protocol/streaming/constants.go new file mode 100644 index 0000000000..3e3a3a6676 --- /dev/null +++ b/protocol/streaming/constants.go @@ -0,0 +1,13 @@ +package streaming + +// Constants for FullNodeStreamingManager. +const ( + // Transient store key for storing staged events. + StreamingManagerTransientStoreKey = "tmp_streaming" + + // Key for storing the count of staged events. + StagedEventsCountKey = "EvtCnt" + + // Key prefix for staged events. + StagedEventsKeyPrefix = "Evt:" +) diff --git a/protocol/streaming/full_node_streaming_manager.go b/protocol/streaming/full_node_streaming_manager.go index a6fe37c512..34b7eaaf36 100644 --- a/protocol/streaming/full_node_streaming_manager.go +++ b/protocol/streaming/full_node_streaming_manager.go @@ -1,15 +1,20 @@ package streaming import ( + "encoding/binary" "fmt" "sync" "sync/atomic" "time" + "github.com/dydxprotocol/v4-chain/protocol/lib" satypes "github.com/dydxprotocol/v4-chain/protocol/x/subaccounts/types" "cosmossdk.io/log" + "cosmossdk.io/store/prefix" + storetypes "cosmossdk.io/store/types" sdk "github.com/cosmos/cosmos-sdk/types" + ante_types "github.com/dydxprotocol/v4-chain/protocol/app/ante/types" "github.com/dydxprotocol/v4-chain/protocol/lib/metrics" "github.com/dydxprotocol/v4-chain/protocol/streaming/types" streaming_util "github.com/dydxprotocol/v4-chain/protocol/streaming/util" @@ -50,6 +55,9 @@ type FullNodeStreamingManagerImpl struct { // Block interval in which snapshot info should be sent out in. // Defaults to 0, which means only one snapshot will be sent out. snapshotBlockInterval uint32 + + // stores the staged FinalizeBlock events for full node streaming. + streamingManagerTransientStoreKey storetypes.StoreKey } // OrderbookSubscription represents a active subscription to the orderbook updates stream. @@ -86,6 +94,7 @@ func NewFullNodeStreamingManager( maxUpdatesInCache uint32, maxSubscriptionChannelSize uint32, snapshotBlockInterval uint32, + streamingManagerTransientStoreKey storetypes.StoreKey, ) *FullNodeStreamingManagerImpl { fullNodeStreamingManager := &FullNodeStreamingManagerImpl{ logger: logger, @@ -102,6 +111,8 @@ func NewFullNodeStreamingManager( maxUpdatesInCache: maxUpdatesInCache, maxSubscriptionChannelSize: maxSubscriptionChannelSize, snapshotBlockInterval: snapshotBlockInterval, + + streamingManagerTransientStoreKey: streamingManagerTransientStoreKey, } // Start the goroutine for pushing order updates through. @@ -367,6 +378,88 @@ func (sm *FullNodeStreamingManagerImpl) sendStreamUpdates( } } +func getStagedEventsCount(store storetypes.KVStore) uint32 { + countsBytes := store.Get([]byte(StagedEventsCountKey)) + if countsBytes == nil { + return 0 + } + return binary.BigEndian.Uint32(countsBytes) +} + +// Stage a subaccount update event in transient store, during `FinalizeBlock`. +func (sm *FullNodeStreamingManagerImpl) StageFinalizeBlockSubaccountUpdate( + ctx sdk.Context, + subaccountUpdate satypes.StreamSubaccountUpdate, +) { + stagedEvent := clobtypes.StagedFinalizeBlockEvent{ + Event: &clobtypes.StagedFinalizeBlockEvent_SubaccountUpdate{ + SubaccountUpdate: &subaccountUpdate, + }, + } + sm.stageFinalizeBlockEvent( + ctx, + clobtypes.Amino.MustMarshal(stagedEvent), + ) +} + +// Stage a fill event in transient store, during `FinalizeBlock`. +// Since `FinalizeBlock` code block can be called more than once with optimistic +// execution (once optimistically and optionally once on the canonical block), +// we need to stage the events in transient store and later emit them +// during `Precommit`. +func (sm *FullNodeStreamingManagerImpl) StageFinalizeBlockFill( + ctx sdk.Context, + fill clobtypes.StreamOrderbookFill, +) { + stagedEvent := clobtypes.StagedFinalizeBlockEvent{ + Event: &clobtypes.StagedFinalizeBlockEvent_OrderFill{ + OrderFill: &fill, + }, + } + sm.stageFinalizeBlockEvent( + ctx, + clobtypes.Amino.MustMarshal(stagedEvent), + ) +} + +func getStagedFinalizeBlockEvents(store storetypes.KVStore) []clobtypes.StagedFinalizeBlockEvent { + count := getStagedEventsCount(store) + events := make([]clobtypes.StagedFinalizeBlockEvent, count) + store = prefix.NewStore(store, []byte(StagedEventsKeyPrefix)) + for i := uint32(0); i < count; i++ { + var event clobtypes.StagedFinalizeBlockEvent + bytes := store.Get(lib.Uint32ToKey(i)) + clobtypes.Amino.MustUnmarshal(bytes, &event) + events[i] = event + } + return events +} + +// Retrieve all events staged during `FinalizeBlock`. +func (sm *FullNodeStreamingManagerImpl) GetStagedFinalizeBlockEvents( + ctx sdk.Context, +) []clobtypes.StagedFinalizeBlockEvent { + noGasCtx := ctx.WithGasMeter(ante_types.NewFreeInfiniteGasMeter()) + store := noGasCtx.TransientStore(sm.streamingManagerTransientStoreKey) + return getStagedFinalizeBlockEvents(store) +} + +func (sm *FullNodeStreamingManagerImpl) stageFinalizeBlockEvent( + ctx sdk.Context, + eventBytes []byte, +) { + noGasCtx := ctx.WithGasMeter(ante_types.NewFreeInfiniteGasMeter()) + store := noGasCtx.TransientStore(sm.streamingManagerTransientStoreKey) + + // Increment events count. + count := getStagedEventsCount(store) + store.Set([]byte(StagedEventsCountKey), lib.Uint32ToKey(count+1)) + + // Store events keyed by index. + store = prefix.NewStore(store, []byte(StagedEventsKeyPrefix)) + store.Set(lib.Uint32ToKey(count), eventBytes) +} + // SendCombinedSnapshot sends messages to a particular subscriber without buffering. // Note this method requires the lock and assumes that the lock has already been // acquired by the caller. @@ -703,6 +796,80 @@ func (sm *FullNodeStreamingManagerImpl) GetSubaccountSnapshotsForInitStreams( return ret } +// Grpc Streaming logic after consensus agrees on a block. +// - Stream all events staged during `FinalizeBlock`. +// - Stream orderbook updates to sync fills in local ops queue. +func (sm *FullNodeStreamingManagerImpl) StreamBatchUpdatesAfterFinalizeBlock( + ctx sdk.Context, + orderBookUpdatesToSyncLocalOpsQueue *clobtypes.OffchainUpdates, + perpetualIdToClobPairId map[uint32][]clobtypes.ClobPairId, +) { + // Flush all pending updates, since we want the onchain updates to arrive in a batch. + sm.FlushStreamUpdates() + + finalizedFills, finalizedSubaccountUpdates := sm.getStagedEventsFromFinalizeBlock(ctx) + + // TODO(CT-1190): Stream below in a single batch. + // Send orderbook updates to sync optimistic orderbook onchain state after FinalizeBlock. + sm.SendOrderbookUpdates( + orderBookUpdatesToSyncLocalOpsQueue, + uint32(ctx.BlockHeight()), + ctx.ExecMode(), + ) + + // Send finalized fills from FinalizeBlock. + sm.SendOrderbookFillUpdates( + finalizedFills, + uint32(ctx.BlockHeight()), + ctx.ExecMode(), + perpetualIdToClobPairId, + ) + + // Send finalized subaccount updates from FinalizeBlock. + sm.SendFinalizedSubaccountUpdates( + finalizedSubaccountUpdates, + uint32(ctx.BlockHeight()), + ctx.ExecMode(), + ) +} + +// getStagedEventsFromFinalizeBlock returns staged events from `FinalizeBlock`. +// It should be called after the consensus agrees on a block (e.g. Precommitter). +func (sm *FullNodeStreamingManagerImpl) getStagedEventsFromFinalizeBlock( + ctx sdk.Context, +) ( + finalizedFills []clobtypes.StreamOrderbookFill, + finalizedSubaccountUpdates []satypes.StreamSubaccountUpdate, +) { + // Get onchain stream events stored in transient store. + stagedEvents := sm.GetStagedFinalizeBlockEvents(ctx) + + metrics.SetGauge( + metrics.GrpcStagedAllFinalizeBlockUpdatesCount, + float32(len(stagedEvents)), + ) + + for _, stagedEvent := range stagedEvents { + switch event := stagedEvent.Event.(type) { + case *clobtypes.StagedFinalizeBlockEvent_OrderFill: + finalizedFills = append(finalizedFills, *event.OrderFill) + case *clobtypes.StagedFinalizeBlockEvent_SubaccountUpdate: + finalizedSubaccountUpdates = append(finalizedSubaccountUpdates, *event.SubaccountUpdate) + } + } + + metrics.SetGauge( + metrics.GrpcStagedSubaccountFinalizeBlockUpdatesCount, + float32(len(finalizedSubaccountUpdates)), + ) + metrics.SetGauge( + metrics.GrpcStagedFillFinalizeBlockUpdatesCount, + float32(len(finalizedFills)), + ) + + return finalizedFills, finalizedSubaccountUpdates +} + func (sm *FullNodeStreamingManagerImpl) InitializeNewStreams( getOrderbookSnapshot func(clobPairId clobtypes.ClobPairId) *clobtypes.OffchainUpdates, subaccountSnapshots map[satypes.SubaccountId]*satypes.StreamSubaccountUpdate, diff --git a/protocol/streaming/noop_streaming_manager.go b/protocol/streaming/noop_streaming_manager.go index 6e9c00895d..4df60bc427 100644 --- a/protocol/streaming/noop_streaming_manager.go +++ b/protocol/streaming/noop_streaming_manager.go @@ -78,3 +78,28 @@ func (sm *NoopGrpcStreamingManager) InitializeNewStreams( func (sm *NoopGrpcStreamingManager) Stop() { } + +func (sm *NoopGrpcStreamingManager) StageFinalizeBlockFill( + ctx sdk.Context, + fill clobtypes.StreamOrderbookFill, +) { +} + +func (sm *NoopGrpcStreamingManager) GetStagedFinalizeBlockEvents( + ctx sdk.Context, +) []clobtypes.StagedFinalizeBlockEvent { + return nil +} + +func (sm *NoopGrpcStreamingManager) StageFinalizeBlockSubaccountUpdate( + ctx sdk.Context, + subaccountUpdate satypes.StreamSubaccountUpdate, +) { +} + +func (sm *NoopGrpcStreamingManager) StreamBatchUpdatesAfterFinalizeBlock( + ctx sdk.Context, + orderBookUpdatesToSyncLocalOpsQueue *clobtypes.OffchainUpdates, + perpetualIdToClobPairId map[uint32][]clobtypes.ClobPairId, +) { +} diff --git a/protocol/streaming/types/interface.go b/protocol/streaming/types/interface.go index e3dff9d94b..0f097d3e75 100644 --- a/protocol/streaming/types/interface.go +++ b/protocol/streaming/types/interface.go @@ -50,7 +50,23 @@ type FullNodeStreamingManager interface { blockHeight uint32, execMode sdk.ExecMode, ) + StageFinalizeBlockFill( + ctx sdk.Context, + fill clobtypes.StreamOrderbookFill, + ) + StageFinalizeBlockSubaccountUpdate( + ctx sdk.Context, + subaccountUpdate satypes.StreamSubaccountUpdate, + ) + GetStagedFinalizeBlockEvents( + ctx sdk.Context, + ) []clobtypes.StagedFinalizeBlockEvent TracksSubaccountId(id satypes.SubaccountId) bool + StreamBatchUpdatesAfterFinalizeBlock( + ctx sdk.Context, + orderBookUpdatesToSyncLocalOpsQueue *clobtypes.OffchainUpdates, + perpetualIdToClobPairId map[uint32][]clobtypes.ClobPairId, + ) } type OutgoingMessageSender interface { diff --git a/protocol/x/clob/abci.go b/protocol/x/clob/abci.go index 96bacb2cc5..9fd5f4a0b5 100644 --- a/protocol/x/clob/abci.go +++ b/protocol/x/clob/abci.go @@ -46,6 +46,17 @@ func BeginBlocker( keeper.ResetAllDeliveredOrderIds(ctx) } +// Precommit executes all ABCI Precommit logic respective to the clob module. +func Precommit( + ctx sdk.Context, + keeper keeper.Keeper, +) { + if streamingManager := keeper.GetFullNodeStreamingManager(); !streamingManager.Enabled() { + return + } + keeper.StreamBatchUpdatesAfterFinalizeBlock(ctx) +} + // EndBlocker executes all ABCI EndBlock logic respective to the clob module. func EndBlocker( ctx sdk.Context, diff --git a/protocol/x/clob/keeper/grpc_stream_finalize_block.go b/protocol/x/clob/keeper/grpc_stream_finalize_block.go new file mode 100644 index 0000000000..1bc23e46c6 --- /dev/null +++ b/protocol/x/clob/keeper/grpc_stream_finalize_block.go @@ -0,0 +1,50 @@ +package keeper + +import ( + "time" + + "github.com/cosmos/cosmos-sdk/telemetry" + sdk "github.com/cosmos/cosmos-sdk/types" + "github.com/dydxprotocol/v4-chain/protocol/lib/metrics" + "github.com/dydxprotocol/v4-chain/protocol/x/clob/types" +) + +// Returns the order updates needed to update the fill amount for the orders +// from local ops queue, according to the latest onchain state (after FinalizeBlock). +// Effectively reverts the optimistic fill amounts removed from the CheckTx to DeliverTx state transition. +func (k Keeper) getUpdatesToSyncLocalOpsQueue( + ctx sdk.Context, +) *types.OffchainUpdates { + localValidatorOperationsQueue, _ := k.MemClob.GetOperationsToReplay(ctx) + fetchOrdersInvolvedInOpQueue(localValidatorOperationsQueue) + orderIdsFromLocal := fetchOrdersInvolvedInOpQueue( + localValidatorOperationsQueue, + ) + allUpdates := types.NewOffchainUpdates() + for orderId := range orderIdsFromLocal { + orderbookUpdate := k.MemClob.GetOrderbookUpdatesForOrderUpdate(ctx, orderId) + allUpdates.Append(orderbookUpdate) + } + return allUpdates +} + +// Grpc Streaming logic after consensus agrees on a block. +// - Stream all events staged during `FinalizeBlock`. +// - Stream orderbook updates to sync fills in local ops queue. +func (k Keeper) StreamBatchUpdatesAfterFinalizeBlock( + ctx sdk.Context, +) { + defer telemetry.MeasureSince( + time.Now(), + types.ModuleName, + metrics.StreamBatchUpdatesAfterFinalizeBlock, + metrics.Latency, + ) + orderBookUpdatesToSyncLocalOpsQueue := k.getUpdatesToSyncLocalOpsQueue(ctx) + + k.GetFullNodeStreamingManager().StreamBatchUpdatesAfterFinalizeBlock( + ctx, + orderBookUpdatesToSyncLocalOpsQueue, + k.PerpetualIdToClobPairId, + ) +} diff --git a/protocol/x/clob/keeper/process_operations.go b/protocol/x/clob/keeper/process_operations.go index d202c2a270..cddb5fb68b 100644 --- a/protocol/x/clob/keeper/process_operations.go +++ b/protocol/x/clob/keeper/process_operations.go @@ -57,26 +57,6 @@ func (k Keeper) ProcessProposerOperations( return errorsmod.Wrapf(types.ErrInvalidMsgProposedOperations, "Error: %+v", err) } - // If grpc streams are on, send absolute fill amounts from local + proposed opqueue to the grpc stream. - // This effetively reverts the optimitic orderbook updates during CheckTx. - if streamingManager := k.GetFullNodeStreamingManager(); streamingManager.Enabled() { - localValidatorOperationsQueue, _ := k.MemClob.GetOperationsToReplay(ctx) - orderIdsFromProposed := fetchOrdersInvolvedInOpQueue( - operations, - ) - orderIdsFromLocal := fetchOrdersInvolvedInOpQueue( - localValidatorOperationsQueue, - ) - orderIdSetToUpdate := lib.MergeMaps(orderIdsFromLocal, orderIdsFromProposed) - - allUpdates := types.NewOffchainUpdates() - for orderId := range orderIdSetToUpdate { - orderbookUpdate := k.MemClob.GetOrderbookUpdatesForOrderUpdate(ctx, orderId) - allUpdates.Append(orderbookUpdate) - } - k.SendOrderbookUpdates(ctx, allUpdates) - } - log.DebugLog(ctx, "Processing operations queue", log.OperationsQueue, types.GetInternalOperationsQueueTextString(operations)) @@ -550,6 +530,7 @@ func (k Keeper) PersistMatchOrdersToState( // if GRPC streaming is on, emit a generated clob match to stream. if streamingManager := k.GetFullNodeStreamingManager(); streamingManager.Enabled() { + // Note: GenerateStreamOrderbookFill doesn't rely on MemClob state. streamOrderbookFill := k.MemClob.GenerateStreamOrderbookFill( ctx, types.ClobMatch{ @@ -560,11 +541,10 @@ func (k Keeper) PersistMatchOrdersToState( &takerOrder, makerOrders, ) - k.SendOrderbookFillUpdates( + + k.GetFullNodeStreamingManager().StageFinalizeBlockFill( ctx, - []types.StreamOrderbookFill{ - streamOrderbookFill, - }, + streamOrderbookFill, ) } @@ -669,11 +649,9 @@ func (k Keeper) PersistMatchLiquidationToState( takerOrder, makerOrders, ) - k.SendOrderbookFillUpdates( + k.GetFullNodeStreamingManager().StageFinalizeBlockFill( ctx, - []types.StreamOrderbookFill{ - streamOrderbookFill, - }, + streamOrderbookFill, ) } return nil diff --git a/protocol/x/clob/types/streaming.pb.go b/protocol/x/clob/types/streaming.pb.go new file mode 100644 index 0000000000..83b8db719a --- /dev/null +++ b/protocol/x/clob/types/streaming.pb.go @@ -0,0 +1,473 @@ +// Code generated by protoc-gen-gogo. DO NOT EDIT. +// source: dydxprotocol/clob/streaming.proto + +package types + +import ( + fmt "fmt" + proto "github.com/cosmos/gogoproto/proto" + types "github.com/dydxprotocol/v4-chain/protocol/x/subaccounts/types" + io "io" + math "math" + math_bits "math/bits" +) + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package + +// StagedFinalizeBlockEvent is an event staged during `FinalizeBlock`. +type StagedFinalizeBlockEvent struct { + // Contains one of StreamOrderbookFill, StreamSubaccountUpdate. + // + // Types that are valid to be assigned to Event: + // *StagedFinalizeBlockEvent_OrderFill + // *StagedFinalizeBlockEvent_SubaccountUpdate + Event isStagedFinalizeBlockEvent_Event `protobuf_oneof:"event"` +} + +func (m *StagedFinalizeBlockEvent) Reset() { *m = StagedFinalizeBlockEvent{} } +func (m *StagedFinalizeBlockEvent) String() string { return proto.CompactTextString(m) } +func (*StagedFinalizeBlockEvent) ProtoMessage() {} +func (*StagedFinalizeBlockEvent) Descriptor() ([]byte, []int) { + return fileDescriptor_cecf6ffcf2554dee, []int{0} +} +func (m *StagedFinalizeBlockEvent) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *StagedFinalizeBlockEvent) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_StagedFinalizeBlockEvent.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *StagedFinalizeBlockEvent) XXX_Merge(src proto.Message) { + xxx_messageInfo_StagedFinalizeBlockEvent.Merge(m, src) +} +func (m *StagedFinalizeBlockEvent) XXX_Size() int { + return m.Size() +} +func (m *StagedFinalizeBlockEvent) XXX_DiscardUnknown() { + xxx_messageInfo_StagedFinalizeBlockEvent.DiscardUnknown(m) +} + +var xxx_messageInfo_StagedFinalizeBlockEvent proto.InternalMessageInfo + +type isStagedFinalizeBlockEvent_Event interface { + isStagedFinalizeBlockEvent_Event() + MarshalTo([]byte) (int, error) + Size() int +} + +type StagedFinalizeBlockEvent_OrderFill struct { + OrderFill *StreamOrderbookFill `protobuf:"bytes,1,opt,name=order_fill,json=orderFill,proto3,oneof" json:"order_fill,omitempty"` +} +type StagedFinalizeBlockEvent_SubaccountUpdate struct { + SubaccountUpdate *types.StreamSubaccountUpdate `protobuf:"bytes,2,opt,name=subaccount_update,json=subaccountUpdate,proto3,oneof" json:"subaccount_update,omitempty"` +} + +func (*StagedFinalizeBlockEvent_OrderFill) isStagedFinalizeBlockEvent_Event() {} +func (*StagedFinalizeBlockEvent_SubaccountUpdate) isStagedFinalizeBlockEvent_Event() {} + +func (m *StagedFinalizeBlockEvent) GetEvent() isStagedFinalizeBlockEvent_Event { + if m != nil { + return m.Event + } + return nil +} + +func (m *StagedFinalizeBlockEvent) GetOrderFill() *StreamOrderbookFill { + if x, ok := m.GetEvent().(*StagedFinalizeBlockEvent_OrderFill); ok { + return x.OrderFill + } + return nil +} + +func (m *StagedFinalizeBlockEvent) GetSubaccountUpdate() *types.StreamSubaccountUpdate { + if x, ok := m.GetEvent().(*StagedFinalizeBlockEvent_SubaccountUpdate); ok { + return x.SubaccountUpdate + } + return nil +} + +// XXX_OneofWrappers is for the internal use of the proto package. +func (*StagedFinalizeBlockEvent) XXX_OneofWrappers() []interface{} { + return []interface{}{ + (*StagedFinalizeBlockEvent_OrderFill)(nil), + (*StagedFinalizeBlockEvent_SubaccountUpdate)(nil), + } +} + +func init() { + proto.RegisterType((*StagedFinalizeBlockEvent)(nil), "dydxprotocol.clob.StagedFinalizeBlockEvent") +} + +func init() { proto.RegisterFile("dydxprotocol/clob/streaming.proto", fileDescriptor_cecf6ffcf2554dee) } + +var fileDescriptor_cecf6ffcf2554dee = []byte{ + // 281 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x52, 0x4c, 0xa9, 0x4c, 0xa9, + 0x28, 0x28, 0xca, 0x2f, 0xc9, 0x4f, 0xce, 0xcf, 0xd1, 0x4f, 0xce, 0xc9, 0x4f, 0xd2, 0x2f, 0x2e, + 0x29, 0x4a, 0x4d, 0xcc, 0xcd, 0xcc, 0x4b, 0xd7, 0x03, 0x8b, 0x0b, 0x09, 0x22, 0x2b, 0xd1, 0x03, + 0x29, 0x91, 0xd2, 0x40, 0xd1, 0x55, 0x5c, 0x9a, 0x94, 0x98, 0x9c, 0x9c, 0x5f, 0x9a, 0x57, 0x52, + 0x8c, 0xae, 0x59, 0x4a, 0x16, 0xd3, 0xfc, 0xc2, 0xd2, 0xd4, 0xa2, 0x4a, 0x88, 0xb4, 0xd2, 0x59, + 0x46, 0x2e, 0x89, 0xe0, 0x92, 0xc4, 0xf4, 0xd4, 0x14, 0xb7, 0xcc, 0xbc, 0xc4, 0x9c, 0xcc, 0xaa, + 0x54, 0xa7, 0x9c, 0xfc, 0xe4, 0x6c, 0xd7, 0xb2, 0xd4, 0xbc, 0x12, 0x21, 0x77, 0x2e, 0xae, 0xfc, + 0xa2, 0x94, 0xd4, 0xa2, 0xf8, 0xb4, 0xcc, 0x9c, 0x1c, 0x09, 0x46, 0x05, 0x46, 0x0d, 0x6e, 0x23, + 0x35, 0x3d, 0x0c, 0xd7, 0xe8, 0x05, 0x83, 0xed, 0xf4, 0x07, 0x29, 0x4d, 0xca, 0xcf, 0xcf, 0x76, + 0xcb, 0xcc, 0xc9, 0xf1, 0x60, 0x08, 0xe2, 0x04, 0xeb, 0x05, 0x71, 0x84, 0xe2, 0xb9, 0x04, 0x11, + 0x6e, 0x8c, 0x2f, 0x2d, 0x48, 0x49, 0x2c, 0x49, 0x95, 0x60, 0x02, 0x9b, 0x67, 0x80, 0x6a, 0x1e, + 0x92, 0x57, 0xa0, 0xc6, 0x06, 0xc3, 0x45, 0x42, 0xc1, 0xfa, 0x3c, 0x18, 0x82, 0x04, 0x8a, 0xd1, + 0xc4, 0x9c, 0xd8, 0xb9, 0x58, 0x53, 0x41, 0x4e, 0x76, 0x0a, 0x38, 0xf1, 0x48, 0x8e, 0xf1, 0xc2, + 0x23, 0x39, 0xc6, 0x07, 0x8f, 0xe4, 0x18, 0x27, 0x3c, 0x96, 0x63, 0xb8, 0xf0, 0x58, 0x8e, 0xe1, + 0xc6, 0x63, 0x39, 0x86, 0x28, 0xb3, 0xf4, 0xcc, 0x92, 0x8c, 0xd2, 0x24, 0xbd, 0xe4, 0xfc, 0x5c, + 0x7d, 0x94, 0x30, 0x29, 0x33, 0xd1, 0x4d, 0xce, 0x48, 0xcc, 0xcc, 0xd3, 0x87, 0x8b, 0x54, 0x40, + 0xc2, 0xa9, 0xa4, 0xb2, 0x20, 0xb5, 0x38, 0x89, 0x0d, 0x2c, 0x6c, 0x0c, 0x08, 0x00, 0x00, 0xff, + 0xff, 0x65, 0x71, 0xd8, 0xa8, 0xa9, 0x01, 0x00, 0x00, +} + +func (m *StagedFinalizeBlockEvent) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *StagedFinalizeBlockEvent) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *StagedFinalizeBlockEvent) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.Event != nil { + { + size := m.Event.Size() + i -= size + if _, err := m.Event.MarshalTo(dAtA[i:]); err != nil { + return 0, err + } + } + } + return len(dAtA) - i, nil +} + +func (m *StagedFinalizeBlockEvent_OrderFill) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *StagedFinalizeBlockEvent_OrderFill) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + if m.OrderFill != nil { + { + size, err := m.OrderFill.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintStreaming(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} +func (m *StagedFinalizeBlockEvent_SubaccountUpdate) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *StagedFinalizeBlockEvent_SubaccountUpdate) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + if m.SubaccountUpdate != nil { + { + size, err := m.SubaccountUpdate.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintStreaming(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x12 + } + return len(dAtA) - i, nil +} +func encodeVarintStreaming(dAtA []byte, offset int, v uint64) int { + offset -= sovStreaming(v) + base := offset + for v >= 1<<7 { + dAtA[offset] = uint8(v&0x7f | 0x80) + v >>= 7 + offset++ + } + dAtA[offset] = uint8(v) + return base +} +func (m *StagedFinalizeBlockEvent) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.Event != nil { + n += m.Event.Size() + } + return n +} + +func (m *StagedFinalizeBlockEvent_OrderFill) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.OrderFill != nil { + l = m.OrderFill.Size() + n += 1 + l + sovStreaming(uint64(l)) + } + return n +} +func (m *StagedFinalizeBlockEvent_SubaccountUpdate) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.SubaccountUpdate != nil { + l = m.SubaccountUpdate.Size() + n += 1 + l + sovStreaming(uint64(l)) + } + return n +} + +func sovStreaming(x uint64) (n int) { + return (math_bits.Len64(x|1) + 6) / 7 +} +func sozStreaming(x uint64) (n int) { + return sovStreaming(uint64((x << 1) ^ uint64((int64(x) >> 63)))) +} +func (m *StagedFinalizeBlockEvent) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowStreaming + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: StagedFinalizeBlockEvent: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: StagedFinalizeBlockEvent: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field OrderFill", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowStreaming + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthStreaming + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthStreaming + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + v := &StreamOrderbookFill{} + if err := v.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + m.Event = &StagedFinalizeBlockEvent_OrderFill{v} + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field SubaccountUpdate", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowStreaming + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthStreaming + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthStreaming + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + v := &types.StreamSubaccountUpdate{} + if err := v.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + m.Event = &StagedFinalizeBlockEvent_SubaccountUpdate{v} + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipStreaming(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthStreaming + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func skipStreaming(dAtA []byte) (n int, err error) { + l := len(dAtA) + iNdEx := 0 + depth := 0 + for iNdEx < l { + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowStreaming + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + wireType := int(wire & 0x7) + switch wireType { + case 0: + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowStreaming + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + iNdEx++ + if dAtA[iNdEx-1] < 0x80 { + break + } + } + case 1: + iNdEx += 8 + case 2: + var length int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowStreaming + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + length |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if length < 0 { + return 0, ErrInvalidLengthStreaming + } + iNdEx += length + case 3: + depth++ + case 4: + if depth == 0 { + return 0, ErrUnexpectedEndOfGroupStreaming + } + depth-- + case 5: + iNdEx += 4 + default: + return 0, fmt.Errorf("proto: illegal wireType %d", wireType) + } + if iNdEx < 0 { + return 0, ErrInvalidLengthStreaming + } + if depth == 0 { + return iNdEx, nil + } + } + return 0, io.ErrUnexpectedEOF +} + +var ( + ErrInvalidLengthStreaming = fmt.Errorf("proto: negative length found during unmarshaling") + ErrIntOverflowStreaming = fmt.Errorf("proto: integer overflow") + ErrUnexpectedEndOfGroupStreaming = fmt.Errorf("proto: unexpected end of group") +) diff --git a/protocol/x/subaccounts/keeper/subaccount.go b/protocol/x/subaccounts/keeper/subaccount.go index a2fb54aacb..5eb650cce0 100644 --- a/protocol/x/subaccounts/keeper/subaccount.go +++ b/protocol/x/subaccounts/keeper/subaccount.go @@ -445,11 +445,9 @@ func (k Keeper) UpdateSubaccounts( if lib.IsDeliverTxMode(ctx) && k.GetFullNodeStreamingManager().Enabled() { if k.GetFullNodeStreamingManager().TracksSubaccountId(*u.SettledSubaccount.Id) { subaccountUpdate := GenerateStreamSubaccountUpdate(u, fundingPayments) - k.SendFinalizedSubaccountUpdates( + k.GetFullNodeStreamingManager().StageFinalizeBlockSubaccountUpdate( ctx, - []types.StreamSubaccountUpdate{ - subaccountUpdate, - }, + subaccountUpdate, ) } } From 78aac64f11c66f830bd4a85270ab1f639ffe8247 Mon Sep 17 00:00:00 2001 From: Teddy Ding Date: Mon, 16 Sep 2024 18:17:33 -0400 Subject: [PATCH 25/33] [CT-1190] Emit `FinalizeBlock` updates in single batch. (#2260) --- .../streaming/full_node_streaming_manager.go | 185 +++++++++++++----- 1 file changed, 141 insertions(+), 44 deletions(-) diff --git a/protocol/streaming/full_node_streaming_manager.go b/protocol/streaming/full_node_streaming_manager.go index 34b7eaaf36..3b3cab68ad 100644 --- a/protocol/streaming/full_node_streaming_manager.go +++ b/protocol/streaming/full_node_streaming_manager.go @@ -490,19 +490,11 @@ func (sm *FullNodeStreamingManagerImpl) TracksSubaccountId(subaccountId satypes. return exists } -// SendOrderbookUpdates groups updates by their clob pair ids and -// sends messages to the subscribers. -func (sm *FullNodeStreamingManagerImpl) SendOrderbookUpdates( +func getStreamUpdatesFromOffchainUpdates( offchainUpdates *clobtypes.OffchainUpdates, blockHeight uint32, execMode sdk.ExecMode, -) { - defer metrics.ModuleMeasureSince( - metrics.FullNodeGrpc, - metrics.GrpcSendOrderbookUpdatesLatency, - time.Now(), - ) - +) (streamUpdates []clobtypes.StreamUpdate, clobPairIds []uint32) { // Group updates by clob pair id. updates := make(map[uint32]*clobtypes.OffchainUpdates) for _, message := range offchainUpdates.Messages { @@ -514,8 +506,8 @@ func (sm *FullNodeStreamingManagerImpl) SendOrderbookUpdates( } // Unmarshal each per-clob pair message to v1 updates. - streamUpdates := make([]clobtypes.StreamUpdate, 0) - clobPairIds := make([]uint32, 0) + streamUpdates = make([]clobtypes.StreamUpdate, 0) + clobPairIds = make([]uint32, 0) for clobPairId, update := range updates { v1updates, err := streaming_util.GetOffchainUpdatesV1(update) if err != nil { @@ -535,26 +527,39 @@ func (sm *FullNodeStreamingManagerImpl) SendOrderbookUpdates( clobPairIds = append(clobPairIds, clobPairId) } - sm.AddOrderUpdatesToCache(streamUpdates, clobPairIds) + return streamUpdates, clobPairIds } -// SendOrderbookFillUpdates groups fills by their clob pair ids and +// SendOrderbookUpdates groups updates by their clob pair ids and // sends messages to the subscribers. -func (sm *FullNodeStreamingManagerImpl) SendOrderbookFillUpdates( - orderbookFills []clobtypes.StreamOrderbookFill, +func (sm *FullNodeStreamingManagerImpl) SendOrderbookUpdates( + offchainUpdates *clobtypes.OffchainUpdates, blockHeight uint32, execMode sdk.ExecMode, - perpetualIdToClobPairId map[uint32][]clobtypes.ClobPairId, ) { defer metrics.ModuleMeasureSince( metrics.FullNodeGrpc, - metrics.GrpcSendOrderbookFillsLatency, + metrics.GrpcSendOrderbookUpdatesLatency, time.Now(), ) + streamUpdates, clobPairIds := getStreamUpdatesFromOffchainUpdates(offchainUpdates, blockHeight, execMode) + + sm.AddOrderUpdatesToCache(streamUpdates, clobPairIds) +} + +func (sm *FullNodeStreamingManagerImpl) getStreamUpdatesForOrderbookFills( + orderbookFills []clobtypes.StreamOrderbookFill, + blockHeight uint32, + execMode sdk.ExecMode, + perpetualIdToClobPairId map[uint32][]clobtypes.ClobPairId, +) ( + streamUpdates []clobtypes.StreamUpdate, + clobPairIds []uint32, +) { // Group fills by clob pair id. - streamUpdates := make([]clobtypes.StreamUpdate, 0) - clobPairIds := make([]uint32, 0) + streamUpdates = make([]clobtypes.StreamUpdate, 0) + clobPairIds = make([]uint32, 0) for _, orderbookFill := range orderbookFills { // If this is a deleveraging fill, fetch the clob pair id from the deleveraged // perpetual id. @@ -577,6 +582,29 @@ func (sm *FullNodeStreamingManagerImpl) SendOrderbookFillUpdates( streamUpdates = append(streamUpdates, streamUpdate) clobPairIds = append(clobPairIds, clobPairId) } + return streamUpdates, clobPairIds +} + +// SendOrderbookFillUpdates groups fills by their clob pair ids and +// sends messages to the subscribers. +func (sm *FullNodeStreamingManagerImpl) SendOrderbookFillUpdates( + orderbookFills []clobtypes.StreamOrderbookFill, + blockHeight uint32, + execMode sdk.ExecMode, + perpetualIdToClobPairId map[uint32][]clobtypes.ClobPairId, +) { + defer metrics.ModuleMeasureSince( + metrics.FullNodeGrpc, + metrics.GrpcSendOrderbookFillsLatency, + time.Now(), + ) + + streamUpdates, clobPairIds := sm.getStreamUpdatesForOrderbookFills( + orderbookFills, + blockHeight, + execMode, + perpetualIdToClobPairId, + ) sm.AddOrderUpdatesToCache(streamUpdates, clobPairIds) } @@ -609,6 +637,31 @@ func (sm *FullNodeStreamingManagerImpl) SendTakerOrderStatus( ) } +func getStreamUpdatesForSubaccountUpdates( + subaccountUpdates []satypes.StreamSubaccountUpdate, + blockHeight uint32, + execMode sdk.ExecMode, +) ( + streamUpdates []clobtypes.StreamUpdate, + subaccountIds []*satypes.SubaccountId, +) { + // Group subaccount updates by subaccount id. + streamUpdates = make([]clobtypes.StreamUpdate, 0) + subaccountIds = make([]*satypes.SubaccountId, 0) + for _, subaccountUpdate := range subaccountUpdates { + streamUpdate := clobtypes.StreamUpdate{ + UpdateMessage: &clobtypes.StreamUpdate_SubaccountUpdate{ + SubaccountUpdate: &subaccountUpdate, + }, + BlockHeight: blockHeight, + ExecMode: uint32(execMode), + } + streamUpdates = append(streamUpdates, streamUpdate) + subaccountIds = append(subaccountIds, subaccountUpdate.SubaccountId) + } + return streamUpdates, subaccountIds +} + // SendFinalizedSubaccountUpdates groups subaccount updates by their subaccount ids and // sends messages to the subscribers. func (sm *FullNodeStreamingManagerImpl) SendFinalizedSubaccountUpdates( @@ -626,20 +679,11 @@ func (sm *FullNodeStreamingManagerImpl) SendFinalizedSubaccountUpdates( panic("SendFinalizedSubaccountUpdates should only be called in ExecModeFinalize") } - // Group subaccount updates by subaccount id. - streamUpdates := make([]clobtypes.StreamUpdate, 0) - subaccountIds := make([]*satypes.SubaccountId, 0) - for _, subaccountUpdate := range subaccountUpdates { - streamUpdate := clobtypes.StreamUpdate{ - UpdateMessage: &clobtypes.StreamUpdate_SubaccountUpdate{ - SubaccountUpdate: &subaccountUpdate, - }, - BlockHeight: blockHeight, - ExecMode: uint32(execMode), - } - streamUpdates = append(streamUpdates, streamUpdate) - subaccountIds = append(subaccountIds, subaccountUpdate.SubaccountId) - } + streamUpdates, subaccountIds := getStreamUpdatesForSubaccountUpdates( + subaccountUpdates, + blockHeight, + execMode, + ) sm.AddSubaccountUpdatesToCache(streamUpdates, subaccountIds) } @@ -796,6 +840,47 @@ func (sm *FullNodeStreamingManagerImpl) GetSubaccountSnapshotsForInitStreams( return ret } +// addBatchUpdatesToCacheWithLock adds batched updates to the cache. +// Used by `StreamBatchUpdatesAfterFinalizeBlock` to batch orderbook, fill +// and subaccount updates in a single stream. +// Note this method requires the lock and assumes that the lock has already been +// acquired by the caller. +func (sm *FullNodeStreamingManagerImpl) addBatchUpdatesToCacheWithLock( + orderbookStreamUpdates []clobtypes.StreamUpdate, + orderbookClobPairIds []uint32, + fillStreamUpdates []clobtypes.StreamUpdate, + fillClobPairIds []uint32, + subaccountStreamUpdates []clobtypes.StreamUpdate, + subaccountIds []*satypes.SubaccountId, +) { + // Add orderbook updates to cache. + sm.streamUpdateCache = append(sm.streamUpdateCache, orderbookStreamUpdates...) + for _, clobPairId := range orderbookClobPairIds { + sm.streamUpdateSubscriptionCache = append( + sm.streamUpdateSubscriptionCache, + sm.clobPairIdToSubscriptionIdMapping[clobPairId], + ) + } + + // Add fill updates to cache. + sm.streamUpdateCache = append(sm.streamUpdateCache, fillStreamUpdates...) + for _, clobPairId := range fillClobPairIds { + sm.streamUpdateSubscriptionCache = append( + sm.streamUpdateSubscriptionCache, + sm.clobPairIdToSubscriptionIdMapping[clobPairId], + ) + } + + // Add subaccount updates to cache. + sm.streamUpdateCache = append(sm.streamUpdateCache, subaccountStreamUpdates...) + for _, subaccountId := range subaccountIds { + sm.streamUpdateSubscriptionCache = append( + sm.streamUpdateSubscriptionCache, + sm.subaccountIdToSubscriptionIdMapping[*subaccountId], + ) + } +} + // Grpc Streaming logic after consensus agrees on a block. // - Stream all events staged during `FinalizeBlock`. // - Stream orderbook updates to sync fills in local ops queue. @@ -804,33 +889,45 @@ func (sm *FullNodeStreamingManagerImpl) StreamBatchUpdatesAfterFinalizeBlock( orderBookUpdatesToSyncLocalOpsQueue *clobtypes.OffchainUpdates, perpetualIdToClobPairId map[uint32][]clobtypes.ClobPairId, ) { - // Flush all pending updates, since we want the onchain updates to arrive in a batch. - sm.FlushStreamUpdates() - finalizedFills, finalizedSubaccountUpdates := sm.getStagedEventsFromFinalizeBlock(ctx) - // TODO(CT-1190): Stream below in a single batch. - // Send orderbook updates to sync optimistic orderbook onchain state after FinalizeBlock. - sm.SendOrderbookUpdates( + orderbookStreamUpdates, orderbookClobPairIds := getStreamUpdatesFromOffchainUpdates( orderBookUpdatesToSyncLocalOpsQueue, uint32(ctx.BlockHeight()), ctx.ExecMode(), ) - // Send finalized fills from FinalizeBlock. - sm.SendOrderbookFillUpdates( + fillStreamUpdates, fillClobPairIds := sm.getStreamUpdatesForOrderbookFills( finalizedFills, uint32(ctx.BlockHeight()), ctx.ExecMode(), perpetualIdToClobPairId, ) - // Send finalized subaccount updates from FinalizeBlock. - sm.SendFinalizedSubaccountUpdates( + subaccountStreamUpdates, subaccountIds := getStreamUpdatesForSubaccountUpdates( finalizedSubaccountUpdates, uint32(ctx.BlockHeight()), ctx.ExecMode(), ) + + sm.Lock() + defer sm.Unlock() + + // Flush all pending updates, since we want the onchain updates to arrive in a batch. + sm.FlushStreamUpdatesWithLock() + + sm.addBatchUpdatesToCacheWithLock( + orderbookStreamUpdates, + orderbookClobPairIds, + fillStreamUpdates, + fillClobPairIds, + subaccountStreamUpdates, + subaccountIds, + ) + + // Emit all stream updates in a single batch. + // Note we still have the lock, which is released right before function returns. + sm.FlushStreamUpdatesWithLock() } // getStagedEventsFromFinalizeBlock returns staged events from `FinalizeBlock`. From 4ecc13370c0ec261ed70f2c4df68573c6b6afcad Mon Sep 17 00:00:00 2001 From: Teddy Ding Date: Thu, 26 Sep 2024 11:43:27 -0400 Subject: [PATCH 26/33] [OTE-823] Fix FNS onchain events staging + retrieval logic (#2318) --- protocol/app/app.go | 1 + protocol/app/flags/flags.go | 4 ---- protocol/app/flags/flags_test.go | 15 +++++++----- .../streaming/full_node_streaming_manager.go | 23 +++++++++++++++---- protocol/x/clob/module.go | 10 ++++++++ 5 files changed, 38 insertions(+), 15 deletions(-) diff --git a/protocol/app/app.go b/protocol/app/app.go index 67d12d2711..58a85af2ba 100644 --- a/protocol/app/app.go +++ b/protocol/app/app.go @@ -2046,6 +2046,7 @@ func getFullNodeStreamingManagerFromOptions( appFlags.GrpcStreamingMaxChannelBufferSize, appFlags.FullNodeStreamingSnapshotInterval, streamingManagerTransientStoreKey, + cdc, ) // Start websocket server. diff --git a/protocol/app/flags/flags.go b/protocol/app/flags/flags.go index cb1715ba62..9a49d4c15c 100644 --- a/protocol/app/flags/flags.go +++ b/protocol/app/flags/flags.go @@ -163,10 +163,6 @@ func (f *Flags) Validate() error { // Grpc streaming if f.GrpcStreamingEnabled { - if f.OptimisticExecutionEnabled { - // TODO(OTE-456): Finish gRPC streaming x OE integration. - return fmt.Errorf("grpc streaming cannot be enabled together with optimistic execution") - } if !f.GrpcEnable { return fmt.Errorf("grpc.enable must be set to true - grpc streaming requires gRPC server") } diff --git a/protocol/app/flags/flags_test.go b/protocol/app/flags/flags_test.go index 50e3a32c72..0ad03872fa 100644 --- a/protocol/app/flags/flags_test.go +++ b/protocol/app/flags/flags_test.go @@ -113,14 +113,17 @@ func TestValidate(t *testing.T) { OptimisticExecutionEnabled: true, }, }, - "failure - optimistic execution cannot be enabled with gRPC streaming": { + "success - optimistic execution canbe enabled with gRPC streaming": { flags: flags.Flags{ - NonValidatingFullNode: false, - GrpcEnable: true, - GrpcStreamingEnabled: true, - OptimisticExecutionEnabled: true, + NonValidatingFullNode: false, + GrpcEnable: true, + GrpcStreamingEnabled: true, + OptimisticExecutionEnabled: true, + GrpcStreamingMaxBatchSize: 2000, + GrpcStreamingFlushIntervalMs: 100, + GrpcStreamingMaxChannelBufferSize: 2000, + WebsocketStreamingPort: 8989, }, - expectedErr: fmt.Errorf("grpc streaming cannot be enabled together with optimistic execution"), }, "failure - gRPC disabled": { flags: flags.Flags{ diff --git a/protocol/streaming/full_node_streaming_manager.go b/protocol/streaming/full_node_streaming_manager.go index 3b3cab68ad..c43a254e78 100644 --- a/protocol/streaming/full_node_streaming_manager.go +++ b/protocol/streaming/full_node_streaming_manager.go @@ -13,6 +13,7 @@ import ( "cosmossdk.io/log" "cosmossdk.io/store/prefix" storetypes "cosmossdk.io/store/types" + "github.com/cosmos/cosmos-sdk/codec" sdk "github.com/cosmos/cosmos-sdk/types" ante_types "github.com/dydxprotocol/v4-chain/protocol/app/ante/types" "github.com/dydxprotocol/v4-chain/protocol/lib/metrics" @@ -27,6 +28,7 @@ var _ types.FullNodeStreamingManager = (*FullNodeStreamingManagerImpl)(nil) type FullNodeStreamingManagerImpl struct { sync.Mutex + cdc codec.BinaryCodec logger log.Logger // orderbookSubscriptions maps subscription IDs to their respective orderbook subscriptions. @@ -95,6 +97,7 @@ func NewFullNodeStreamingManager( maxSubscriptionChannelSize uint32, snapshotBlockInterval uint32, streamingManagerTransientStoreKey storetypes.StoreKey, + cdc codec.BinaryCodec, ) *FullNodeStreamingManagerImpl { fullNodeStreamingManager := &FullNodeStreamingManagerImpl{ logger: logger, @@ -113,6 +116,7 @@ func NewFullNodeStreamingManager( snapshotBlockInterval: snapshotBlockInterval, streamingManagerTransientStoreKey: streamingManagerTransientStoreKey, + cdc: cdc, } // Start the goroutine for pushing order updates through. @@ -391,6 +395,7 @@ func (sm *FullNodeStreamingManagerImpl) StageFinalizeBlockSubaccountUpdate( ctx sdk.Context, subaccountUpdate satypes.StreamSubaccountUpdate, ) { + lib.AssertDeliverTxMode(ctx) stagedEvent := clobtypes.StagedFinalizeBlockEvent{ Event: &clobtypes.StagedFinalizeBlockEvent_SubaccountUpdate{ SubaccountUpdate: &subaccountUpdate, @@ -398,7 +403,7 @@ func (sm *FullNodeStreamingManagerImpl) StageFinalizeBlockSubaccountUpdate( } sm.stageFinalizeBlockEvent( ctx, - clobtypes.Amino.MustMarshal(stagedEvent), + sm.cdc.MustMarshal(&stagedEvent), ) } @@ -411,25 +416,30 @@ func (sm *FullNodeStreamingManagerImpl) StageFinalizeBlockFill( ctx sdk.Context, fill clobtypes.StreamOrderbookFill, ) { + lib.AssertDeliverTxMode(ctx) stagedEvent := clobtypes.StagedFinalizeBlockEvent{ Event: &clobtypes.StagedFinalizeBlockEvent_OrderFill{ OrderFill: &fill, }, } + sm.stageFinalizeBlockEvent( ctx, - clobtypes.Amino.MustMarshal(stagedEvent), + sm.cdc.MustMarshal(&stagedEvent), ) } -func getStagedFinalizeBlockEvents(store storetypes.KVStore) []clobtypes.StagedFinalizeBlockEvent { +func getStagedFinalizeBlockEventsFromStore( + store storetypes.KVStore, + cdc codec.BinaryCodec, +) []clobtypes.StagedFinalizeBlockEvent { count := getStagedEventsCount(store) events := make([]clobtypes.StagedFinalizeBlockEvent, count) store = prefix.NewStore(store, []byte(StagedEventsKeyPrefix)) for i := uint32(0); i < count; i++ { var event clobtypes.StagedFinalizeBlockEvent bytes := store.Get(lib.Uint32ToKey(i)) - clobtypes.Amino.MustUnmarshal(bytes, &event) + cdc.MustUnmarshal(bytes, &event) events[i] = event } return events @@ -441,7 +451,7 @@ func (sm *FullNodeStreamingManagerImpl) GetStagedFinalizeBlockEvents( ) []clobtypes.StagedFinalizeBlockEvent { noGasCtx := ctx.WithGasMeter(ante_types.NewFreeInfiniteGasMeter()) store := noGasCtx.TransientStore(sm.streamingManagerTransientStoreKey) - return getStagedFinalizeBlockEvents(store) + return getStagedFinalizeBlockEventsFromStore(store, sm.cdc) } func (sm *FullNodeStreamingManagerImpl) stageFinalizeBlockEvent( @@ -889,6 +899,9 @@ func (sm *FullNodeStreamingManagerImpl) StreamBatchUpdatesAfterFinalizeBlock( orderBookUpdatesToSyncLocalOpsQueue *clobtypes.OffchainUpdates, perpetualIdToClobPairId map[uint32][]clobtypes.ClobPairId, ) { + // Prevent gas metering from state read. + ctx = ctx.WithGasMeter(ante_types.NewFreeInfiniteGasMeter()) + finalizedFills, finalizedSubaccountUpdates := sm.getStagedEventsFromFinalizeBlock(ctx) orderbookStreamUpdates, orderbookClobPairIds := getStreamUpdatesFromOffchainUpdates( diff --git a/protocol/x/clob/module.go b/protocol/x/clob/module.go index 2b61ed37c7..0e06d7e035 100644 --- a/protocol/x/clob/module.go +++ b/protocol/x/clob/module.go @@ -177,6 +177,16 @@ func (am AppModule) PreBlock(ctx context.Context) (appmodule.ResponsePreBlock, e }, nil } +// BeginBlock executes all ABCI BeginBlock logic respective to the clob module. +func (am AppModule) Precommit(ctx context.Context) error { + defer telemetry.ModuleMeasureSince(am.Name(), time.Now(), telemetry.MetricKeyPrecommiter) + Precommit( + lib.UnwrapSDKContext(ctx, types.ModuleName), + *am.keeper, + ) + return nil +} + // BeginBlock executes all ABCI BeginBlock logic respective to the clob module. func (am AppModule) BeginBlock(ctx context.Context) error { defer telemetry.ModuleMeasureSince(am.Name(), time.Now(), telemetry.MetricKeyBeginBlocker) From b44e09d05838064fc41f9879e89cb4ece571eb15 Mon Sep 17 00:00:00 2001 From: Teddy Ding Date: Thu, 26 Sep 2024 13:47:47 -0400 Subject: [PATCH 27/33] Add OE Abort rate test flag (#2366) --- protocol/app/app.go | 15 ++++++++++++++- protocol/app/flags/flags.go | 28 ++++++++++++++++++++++------ 2 files changed, 36 insertions(+), 7 deletions(-) diff --git a/protocol/app/app.go b/protocol/app/app.go index 58a85af2ba..74470433b2 100644 --- a/protocol/app/app.go +++ b/protocol/app/app.go @@ -3,6 +3,7 @@ package app import ( "context" "encoding/json" + "fmt" "io" "math/big" "net/http" @@ -33,6 +34,7 @@ import ( tmproto "github.com/cometbft/cometbft/proto/tendermint/types" dbm "github.com/cosmos/cosmos-db" "github.com/cosmos/cosmos-sdk/baseapp" + "github.com/cosmos/cosmos-sdk/baseapp/oe" "github.com/cosmos/cosmos-sdk/client" cosmosflags "github.com/cosmos/cosmos-sdk/client/flags" "github.com/cosmos/cosmos-sdk/client/grpc/cmtservice" @@ -410,7 +412,18 @@ func New( // Enable optimistic block execution. if appFlags.OptimisticExecutionEnabled { - baseAppOptions = append(baseAppOptions, baseapp.SetOptimisticExecution()) + logger.Info("optimistic execution is enabled.") + if appFlags.OptimisticExecutionTestAbortRate > 0 { + logger.Warn(fmt.Sprintf( + "Test flag optimistic-execution-test-abort-rate is set: %v\n", + appFlags.OptimisticExecutionTestAbortRate, + )) + } + baseAppOptions = append( + baseAppOptions, + baseapp.SetOptimisticExecution( + oe.WithAbortRate(int(appFlags.OptimisticExecutionTestAbortRate)), + )) } bApp := baseapp.NewBaseApp(appconstants.AppName, logger, db, txConfig.TxDecoder(), baseAppOptions...) diff --git a/protocol/app/flags/flags.go b/protocol/app/flags/flags.go index 9a49d4c15c..aadc805696 100644 --- a/protocol/app/flags/flags.go +++ b/protocol/app/flags/flags.go @@ -31,7 +31,8 @@ type Flags struct { VEOracleEnabled bool // Slinky Vote Extensions // Optimistic block execution - OptimisticExecutionEnabled bool + OptimisticExecutionEnabled bool + OptimisticExecutionTestAbortRate uint16 } // List of CLI flags. @@ -58,7 +59,8 @@ const ( VEOracleEnabled = "slinky-vote-extension-oracle-enabled" // Enable optimistic block execution. - OptimisticExecutionEnabled = "optimistic-execution-enabled" + OptimisticExecutionEnabled = "optimistic-execution-enabled" + OptimisticExecutionTestAbortRate = "optimistic-execution-test-abort-rate" ) // Default values. @@ -76,8 +78,10 @@ const ( DefaultWebsocketStreamingPort = 9092 DefaultFullNodeStreamingSnapshotInterval = 0 - DefaultVEOracleEnabled = true - DefaultOptimisticExecutionEnabled = false + DefaultVEOracleEnabled = true + + DefaultOptimisticExecutionEnabled = false + DefaultOptimisticExecutionTestAbortRate = 0 ) // AddFlagsToCmd adds flags to app initialization. @@ -152,6 +156,11 @@ func AddFlagsToCmd(cmd *cobra.Command) { DefaultOptimisticExecutionEnabled, "Whether to enable optimistic block execution", ) + cmd.Flags().Uint16( + OptimisticExecutionTestAbortRate, + DefaultOptimisticExecutionTestAbortRate, + "[Test only] Abort rate for optimistic execution", + ) } // Validate checks that the flags are valid. @@ -210,8 +219,9 @@ func GetFlagValuesFromOptions( WebsocketStreamingPort: DefaultWebsocketStreamingPort, FullNodeStreamingSnapshotInterval: DefaultFullNodeStreamingSnapshotInterval, - VEOracleEnabled: true, - OptimisticExecutionEnabled: DefaultOptimisticExecutionEnabled, + VEOracleEnabled: true, + OptimisticExecutionEnabled: DefaultOptimisticExecutionEnabled, + OptimisticExecutionTestAbortRate: DefaultOptimisticExecutionTestAbortRate, } // Populate the flags if they exist. @@ -304,5 +314,11 @@ func GetFlagValuesFromOptions( result.OptimisticExecutionEnabled = v } } + + if option := appOpts.Get(OptimisticExecutionTestAbortRate); option != nil { + if v, err := cast.ToUint16E(option); err == nil { + result.OptimisticExecutionTestAbortRate = v + } + } return result } From 202fbb102c56d333ca44cbdd0099babd64238ce2 Mon Sep 17 00:00:00 2001 From: Teddy Ding Date: Thu, 26 Sep 2024 15:00:17 -0400 Subject: [PATCH 28/33] Remove warning on OE as experimental feature --- protocol/app/app.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/protocol/app/app.go b/protocol/app/app.go index 74470433b2..c3afbfcd1c 100644 --- a/protocol/app/app.go +++ b/protocol/app/app.go @@ -395,11 +395,6 @@ func New( if err := appFlags.Validate(); err != nil { panic(err) } - if appFlags.OptimisticExecutionEnabled { - // TODO(OTE-573): Remove warning once OE is fully supported. - logger.Warn("Optimistic execution is enabled. This is a test feature not intended for production use!") - } - initDatadogProfiler(logger, appFlags.DdAgentHost, appFlags.DdTraceAgentPort) encodingConfig := GetEncodingConfig() From e64fdc473a6effaef189d8ddf389106118b3a1f1 Mon Sep 17 00:00:00 2001 From: Teddy Ding Date: Mon, 30 Sep 2024 14:38:39 -0400 Subject: [PATCH 29/33] Internalize logic to stage FinalizeBlock events (#2399) --- .../codegen/dydxprotocol/clob/streaming.ts | 16 +- proto/dydxprotocol/clob/streaming.proto | 1 + protocol/mocks/MemClobKeeper.go | 6 +- .../streaming/full_node_streaming_manager.go | 304 ++++++++++-------- protocol/streaming/noop_streaming_manager.go | 21 +- protocol/streaming/types/interface.go | 19 +- protocol/streaming/util/util.go | 8 +- protocol/testutil/memclob/keeper.go | 4 +- protocol/x/clob/keeper/keeper.go | 22 +- protocol/x/clob/keeper/process_operations.go | 16 +- protocol/x/clob/memclob/memclob.go | 2 +- protocol/x/clob/types/mem_clob_keeper.go | 4 +- protocol/x/clob/types/streaming.pb.go | 100 +++++- protocol/x/subaccounts/keeper/subaccount.go | 2 +- 14 files changed, 317 insertions(+), 208 deletions(-) diff --git a/indexer/packages/v4-protos/src/codegen/dydxprotocol/clob/streaming.ts b/indexer/packages/v4-protos/src/codegen/dydxprotocol/clob/streaming.ts index dab8f1e122..1600c2e39c 100644 --- a/indexer/packages/v4-protos/src/codegen/dydxprotocol/clob/streaming.ts +++ b/indexer/packages/v4-protos/src/codegen/dydxprotocol/clob/streaming.ts @@ -1,4 +1,4 @@ -import { StreamOrderbookFill, StreamOrderbookFillSDKType } from "./query"; +import { StreamOrderbookFill, StreamOrderbookFillSDKType, StreamOrderbookUpdate, StreamOrderbookUpdateSDKType } from "./query"; import { StreamSubaccountUpdate, StreamSubaccountUpdateSDKType } from "../subaccounts/streaming"; import * as _m0 from "protobufjs/minimal"; import { DeepPartial } from "../../helpers"; @@ -7,18 +7,21 @@ import { DeepPartial } from "../../helpers"; export interface StagedFinalizeBlockEvent { orderFill?: StreamOrderbookFill; subaccountUpdate?: StreamSubaccountUpdate; + orderbookUpdate?: StreamOrderbookUpdate; } /** StagedFinalizeBlockEvent is an event staged during `FinalizeBlock`. */ export interface StagedFinalizeBlockEventSDKType { order_fill?: StreamOrderbookFillSDKType; subaccount_update?: StreamSubaccountUpdateSDKType; + orderbook_update?: StreamOrderbookUpdateSDKType; } function createBaseStagedFinalizeBlockEvent(): StagedFinalizeBlockEvent { return { orderFill: undefined, - subaccountUpdate: undefined + subaccountUpdate: undefined, + orderbookUpdate: undefined }; } @@ -32,6 +35,10 @@ export const StagedFinalizeBlockEvent = { StreamSubaccountUpdate.encode(message.subaccountUpdate, writer.uint32(18).fork()).ldelim(); } + if (message.orderbookUpdate !== undefined) { + StreamOrderbookUpdate.encode(message.orderbookUpdate, writer.uint32(26).fork()).ldelim(); + } + return writer; }, @@ -52,6 +59,10 @@ export const StagedFinalizeBlockEvent = { message.subaccountUpdate = StreamSubaccountUpdate.decode(reader, reader.uint32()); break; + case 3: + message.orderbookUpdate = StreamOrderbookUpdate.decode(reader, reader.uint32()); + break; + default: reader.skipType(tag & 7); break; @@ -65,6 +76,7 @@ export const StagedFinalizeBlockEvent = { const message = createBaseStagedFinalizeBlockEvent(); message.orderFill = object.orderFill !== undefined && object.orderFill !== null ? StreamOrderbookFill.fromPartial(object.orderFill) : undefined; message.subaccountUpdate = object.subaccountUpdate !== undefined && object.subaccountUpdate !== null ? StreamSubaccountUpdate.fromPartial(object.subaccountUpdate) : undefined; + message.orderbookUpdate = object.orderbookUpdate !== undefined && object.orderbookUpdate !== null ? StreamOrderbookUpdate.fromPartial(object.orderbookUpdate) : undefined; return message; } diff --git a/proto/dydxprotocol/clob/streaming.proto b/proto/dydxprotocol/clob/streaming.proto index 06c74ffbe1..ae3811134e 100644 --- a/proto/dydxprotocol/clob/streaming.proto +++ b/proto/dydxprotocol/clob/streaming.proto @@ -12,5 +12,6 @@ message StagedFinalizeBlockEvent { oneof event { StreamOrderbookFill order_fill = 1; dydxprotocol.subaccounts.StreamSubaccountUpdate subaccount_update = 2; + StreamOrderbookUpdate orderbook_update = 3; } } diff --git a/protocol/mocks/MemClobKeeper.go b/protocol/mocks/MemClobKeeper.go index 8eacc26019..f3e94f1f5c 100644 --- a/protocol/mocks/MemClobKeeper.go +++ b/protocol/mocks/MemClobKeeper.go @@ -406,9 +406,9 @@ func (_m *MemClobKeeper) ReplayPlaceOrder(ctx types.Context, msg *clobtypes.MsgP return r0, r1, r2, r3 } -// SendOrderbookFillUpdates provides a mock function with given fields: ctx, orderbookFills -func (_m *MemClobKeeper) SendOrderbookFillUpdates(ctx types.Context, orderbookFills []clobtypes.StreamOrderbookFill) { - _m.Called(ctx, orderbookFills) +// SendOrderbookFillUpdate provides a mock function with given fields: ctx, orderbookFills +func (_m *MemClobKeeper) SendOrderbookFillUpdate(ctx types.Context, orderbookFill clobtypes.StreamOrderbookFill) { + _m.Called(ctx, orderbookFill) } // SendOrderbookUpdates provides a mock function with given fields: ctx, offchainUpdates diff --git a/protocol/streaming/full_node_streaming_manager.go b/protocol/streaming/full_node_streaming_manager.go index c43a254e78..85c265f12e 100644 --- a/protocol/streaming/full_node_streaming_manager.go +++ b/protocol/streaming/full_node_streaming_manager.go @@ -20,6 +20,8 @@ import ( "github.com/dydxprotocol/v4-chain/protocol/streaming/types" streaming_util "github.com/dydxprotocol/v4-chain/protocol/streaming/util" clobtypes "github.com/dydxprotocol/v4-chain/protocol/x/clob/types" + + ocutypes "github.com/dydxprotocol/v4-chain/protocol/indexer/off_chain_updates/types" ) var _ types.FullNodeStreamingManager = (*FullNodeStreamingManagerImpl)(nil) @@ -313,10 +315,7 @@ func toOrderbookStreamUpdate( blockHeight uint32, execMode sdk.ExecMode, ) []clobtypes.StreamUpdate { - v1updates, err := streaming_util.GetOffchainUpdatesV1(offchainUpdates) - if err != nil { - panic(err) - } + v1updates := streaming_util.GetOffchainUpdatesV1(offchainUpdates) return []clobtypes.StreamUpdate{ { UpdateMessage: &clobtypes.StreamUpdate_OrderbookUpdate{ @@ -390,39 +389,22 @@ func getStagedEventsCount(store storetypes.KVStore) uint32 { return binary.BigEndian.Uint32(countsBytes) } -// Stage a subaccount update event in transient store, during `FinalizeBlock`. -func (sm *FullNodeStreamingManagerImpl) StageFinalizeBlockSubaccountUpdate( +// Send a subaccount update event. +func (sm *FullNodeStreamingManagerImpl) SendSubaccountUpdate( ctx sdk.Context, subaccountUpdate satypes.StreamSubaccountUpdate, ) { - lib.AssertDeliverTxMode(ctx) - stagedEvent := clobtypes.StagedFinalizeBlockEvent{ - Event: &clobtypes.StagedFinalizeBlockEvent_SubaccountUpdate{ - SubaccountUpdate: &subaccountUpdate, - }, + // If not `DeliverTx`, return since we don't stream optimistic subaccount updates. + if !lib.IsDeliverTxMode(ctx) { + return } - sm.stageFinalizeBlockEvent( - ctx, - sm.cdc.MustMarshal(&stagedEvent), - ) -} -// Stage a fill event in transient store, during `FinalizeBlock`. -// Since `FinalizeBlock` code block can be called more than once with optimistic -// execution (once optimistically and optionally once on the canonical block), -// we need to stage the events in transient store and later emit them -// during `Precommit`. -func (sm *FullNodeStreamingManagerImpl) StageFinalizeBlockFill( - ctx sdk.Context, - fill clobtypes.StreamOrderbookFill, -) { - lib.AssertDeliverTxMode(ctx) + // If `DeliverTx`, updates should be staged to be streamed after consensus finalizes on a block. stagedEvent := clobtypes.StagedFinalizeBlockEvent{ - Event: &clobtypes.StagedFinalizeBlockEvent_OrderFill{ - OrderFill: &fill, + Event: &clobtypes.StagedFinalizeBlockEvent_SubaccountUpdate{ + SubaccountUpdate: &subaccountUpdate, }, } - sm.stageFinalizeBlockEvent( ctx, sm.cdc.MustMarshal(&stagedEvent), @@ -501,29 +483,49 @@ func (sm *FullNodeStreamingManagerImpl) TracksSubaccountId(subaccountId satypes. } func getStreamUpdatesFromOffchainUpdates( - offchainUpdates *clobtypes.OffchainUpdates, + v1updates []ocutypes.OffChainUpdateV1, blockHeight uint32, execMode sdk.ExecMode, ) (streamUpdates []clobtypes.StreamUpdate, clobPairIds []uint32) { // Group updates by clob pair id. - updates := make(map[uint32]*clobtypes.OffchainUpdates) - for _, message := range offchainUpdates.Messages { - clobPairId := message.OrderId.ClobPairId - if _, ok := updates[clobPairId]; !ok { - updates[clobPairId] = clobtypes.NewOffchainUpdates() + clobPairIdToV1Updates := make(map[uint32][]ocutypes.OffChainUpdateV1) + // unique list of clob pair Ids to send updates for. + clobPairIds = make([]uint32, 0) + for _, v1update := range v1updates { + var clobPairId uint32 + switch u := v1update.UpdateMessage.(type) { + case *ocutypes.OffChainUpdateV1_OrderPlace: + clobPairId = u.OrderPlace.Order.OrderId.ClobPairId + case *ocutypes.OffChainUpdateV1_OrderReplace: + clobPairId = u.OrderReplace.OldOrderId.ClobPairId + case *ocutypes.OffChainUpdateV1_OrderRemove: + clobPairId = u.OrderRemove.RemovedOrderId.ClobPairId + case *ocutypes.OffChainUpdateV1_OrderUpdate: + clobPairId = u.OrderUpdate.OrderId.ClobPairId + default: + panic(fmt.Sprintf("Unhandled UpdateMessage type: %v", u)) } - updates[clobPairId].Messages = append(updates[clobPairId].Messages, message) + + if _, ok := clobPairIdToV1Updates[clobPairId]; !ok { + clobPairIdToV1Updates[clobPairId] = []ocutypes.OffChainUpdateV1{} + clobPairIds = append(clobPairIds, clobPairId) + } + clobPairIdToV1Updates[clobPairId] = append(clobPairIdToV1Updates[clobPairId], v1update) } // Unmarshal each per-clob pair message to v1 updates. - streamUpdates = make([]clobtypes.StreamUpdate, 0) - clobPairIds = make([]uint32, 0) - for clobPairId, update := range updates { - v1updates, err := streaming_util.GetOffchainUpdatesV1(update) - if err != nil { - panic(err) + streamUpdates = make([]clobtypes.StreamUpdate, len(clobPairIds)) + + for i, clobPairId := range clobPairIds { + v1updates, exists := clobPairIdToV1Updates[clobPairId] + if !exists { + panic(fmt.Sprintf( + "clob pair id %v not found in clobPairIdToV1Updates: %v", + clobPairId, + clobPairIdToV1Updates, + )) } - streamUpdate := clobtypes.StreamUpdate{ + streamUpdates[i] = clobtypes.StreamUpdate{ UpdateMessage: &clobtypes.StreamUpdate_OrderbookUpdate{ OrderbookUpdate: &clobtypes.StreamOrderbookUpdate{ Updates: v1updates, @@ -533,8 +535,6 @@ func getStreamUpdatesFromOffchainUpdates( BlockHeight: blockHeight, ExecMode: uint32(execMode), } - streamUpdates = append(streamUpdates, streamUpdate) - clobPairIds = append(clobPairIds, clobPairId) } return streamUpdates, clobPairIds @@ -544,18 +544,40 @@ func getStreamUpdatesFromOffchainUpdates( // sends messages to the subscribers. func (sm *FullNodeStreamingManagerImpl) SendOrderbookUpdates( offchainUpdates *clobtypes.OffchainUpdates, - blockHeight uint32, - execMode sdk.ExecMode, + ctx sdk.Context, ) { - defer metrics.ModuleMeasureSince( - metrics.FullNodeGrpc, - metrics.GrpcSendOrderbookUpdatesLatency, - time.Now(), - ) + v1updates := streaming_util.GetOffchainUpdatesV1(offchainUpdates) + + // If not `DeliverTx`, then updates are optimistic. Stream them directly. + if !lib.IsDeliverTxMode(ctx) { + defer metrics.ModuleMeasureSince( + metrics.FullNodeGrpc, + metrics.GrpcSendOrderbookUpdatesLatency, + time.Now(), + ) - streamUpdates, clobPairIds := getStreamUpdatesFromOffchainUpdates(offchainUpdates, blockHeight, execMode) + streamUpdates, clobPairIds := getStreamUpdatesFromOffchainUpdates( + v1updates, + lib.MustConvertIntegerToUint32(ctx.BlockHeight()), + ctx.ExecMode(), + ) + sm.AddOrderUpdatesToCache(streamUpdates, clobPairIds) + return + } - sm.AddOrderUpdatesToCache(streamUpdates, clobPairIds) + // If `DeliverTx`, updates should be staged to be streamed after consensus finalizes on a block. + stagedEvent := clobtypes.StagedFinalizeBlockEvent{ + Event: &clobtypes.StagedFinalizeBlockEvent_OrderbookUpdate{ + OrderbookUpdate: &clobtypes.StreamOrderbookUpdate{ + Updates: v1updates, + Snapshot: false, + }, + }, + } + sm.stageFinalizeBlockEvent( + ctx, + sm.cdc.MustMarshal(&stagedEvent), + ) } func (sm *FullNodeStreamingManagerImpl) getStreamUpdatesForOrderbookFills( @@ -595,36 +617,52 @@ func (sm *FullNodeStreamingManagerImpl) getStreamUpdatesForOrderbookFills( return streamUpdates, clobPairIds } -// SendOrderbookFillUpdates groups fills by their clob pair ids and +// SendOrderbookFillUpdate groups fills by their clob pair ids and // sends messages to the subscribers. -func (sm *FullNodeStreamingManagerImpl) SendOrderbookFillUpdates( - orderbookFills []clobtypes.StreamOrderbookFill, - blockHeight uint32, - execMode sdk.ExecMode, +func (sm *FullNodeStreamingManagerImpl) SendOrderbookFillUpdate( + orderbookFill clobtypes.StreamOrderbookFill, + ctx sdk.Context, perpetualIdToClobPairId map[uint32][]clobtypes.ClobPairId, ) { - defer metrics.ModuleMeasureSince( - metrics.FullNodeGrpc, - metrics.GrpcSendOrderbookFillsLatency, - time.Now(), - ) + // If not `DeliverTx`, then updates are optimistic. Stream them directly. + if !lib.IsDeliverTxMode(ctx) { + defer metrics.ModuleMeasureSince( + metrics.FullNodeGrpc, + metrics.GrpcSendOrderbookFillsLatency, + time.Now(), + ) - streamUpdates, clobPairIds := sm.getStreamUpdatesForOrderbookFills( - orderbookFills, - blockHeight, - execMode, - perpetualIdToClobPairId, - ) + streamUpdates, clobPairIds := sm.getStreamUpdatesForOrderbookFills( + []clobtypes.StreamOrderbookFill{orderbookFill}, + lib.MustConvertIntegerToUint32(ctx.BlockHeight()), + ctx.ExecMode(), + perpetualIdToClobPairId, + ) + sm.AddOrderUpdatesToCache(streamUpdates, clobPairIds) + return + } + + // If `DeliverTx`, updates should be staged to be streamed after consensus finalizes on a block. + stagedEvent := clobtypes.StagedFinalizeBlockEvent{ + Event: &clobtypes.StagedFinalizeBlockEvent_OrderFill{ + OrderFill: &orderbookFill, + }, + } - sm.AddOrderUpdatesToCache(streamUpdates, clobPairIds) + sm.stageFinalizeBlockEvent( + ctx, + sm.cdc.MustMarshal(&stagedEvent), + ) } // SendTakerOrderStatus sends out a taker order and its status to the full node streaming service. func (sm *FullNodeStreamingManagerImpl) SendTakerOrderStatus( streamTakerOrder clobtypes.StreamTakerOrder, - blockHeight uint32, - execMode sdk.ExecMode, + ctx sdk.Context, ) { + // In current design, we never send this during `DeliverTx` (`FinalizeBlock`). + lib.AssertCheckTxMode(ctx) + clobPairId := uint32(0) if liqOrder := streamTakerOrder.GetLiquidationOrder(); liqOrder != nil { clobPairId = liqOrder.ClobPairId @@ -639,8 +677,8 @@ func (sm *FullNodeStreamingManagerImpl) SendTakerOrderStatus( UpdateMessage: &clobtypes.StreamUpdate_TakerOrder{ TakerOrder: &streamTakerOrder, }, - BlockHeight: blockHeight, - ExecMode: uint32(execMode), + BlockHeight: lib.MustConvertIntegerToUint32(ctx.BlockHeight()), + ExecMode: uint32(ctx.ExecMode()), }, }, []uint32{clobPairId}, @@ -712,13 +750,7 @@ func (sm *FullNodeStreamingManagerImpl) AddOrderUpdatesToCache( float32(len(updates)), ) - sm.streamUpdateCache = append(sm.streamUpdateCache, updates...) - for _, clobPairId := range clobPairIds { - sm.streamUpdateSubscriptionCache = append( - sm.streamUpdateSubscriptionCache, - sm.clobPairIdToSubscriptionIdMapping[clobPairId], - ) - } + sm.cacheStreamUpdatesByClobPairWithLock(updates, clobPairIds) // Remove all subscriptions and wipe the buffer if buffer overflows. sm.RemoveSubscriptionsAndClearBufferIfFull() @@ -739,13 +771,8 @@ func (sm *FullNodeStreamingManagerImpl) AddSubaccountUpdatesToCache( float32(len(updates)), ) - sm.streamUpdateCache = append(sm.streamUpdateCache, updates...) - for _, subaccountId := range subaccountIds { - sm.streamUpdateSubscriptionCache = append( - sm.streamUpdateSubscriptionCache, - sm.subaccountIdToSubscriptionIdMapping[*subaccountId], - ) - } + sm.cacheStreamUpdatesBySubaccountWithLock(updates, subaccountIds) + sm.RemoveSubscriptionsAndClearBufferIfFull() sm.EmitMetrics() } @@ -850,38 +877,31 @@ func (sm *FullNodeStreamingManagerImpl) GetSubaccountSnapshotsForInitStreams( return ret } -// addBatchUpdatesToCacheWithLock adds batched updates to the cache. -// Used by `StreamBatchUpdatesAfterFinalizeBlock` to batch orderbook, fill -// and subaccount updates in a single stream. -// Note this method requires the lock and assumes that the lock has already been +// cacheStreamUpdatesByClobPairWithLock adds stream updates to cache, +// and store corresponding clob pair Ids. +// This method requires the lock and assumes that the lock has already been // acquired by the caller. -func (sm *FullNodeStreamingManagerImpl) addBatchUpdatesToCacheWithLock( - orderbookStreamUpdates []clobtypes.StreamUpdate, - orderbookClobPairIds []uint32, - fillStreamUpdates []clobtypes.StreamUpdate, - fillClobPairIds []uint32, - subaccountStreamUpdates []clobtypes.StreamUpdate, - subaccountIds []*satypes.SubaccountId, +func (sm *FullNodeStreamingManagerImpl) cacheStreamUpdatesByClobPairWithLock( + streamUpdates []clobtypes.StreamUpdate, + clobPairIds []uint32, ) { - // Add orderbook updates to cache. - sm.streamUpdateCache = append(sm.streamUpdateCache, orderbookStreamUpdates...) - for _, clobPairId := range orderbookClobPairIds { - sm.streamUpdateSubscriptionCache = append( - sm.streamUpdateSubscriptionCache, - sm.clobPairIdToSubscriptionIdMapping[clobPairId], - ) - } - - // Add fill updates to cache. - sm.streamUpdateCache = append(sm.streamUpdateCache, fillStreamUpdates...) - for _, clobPairId := range fillClobPairIds { + sm.streamUpdateCache = append(sm.streamUpdateCache, streamUpdates...) + for _, clobPairId := range clobPairIds { sm.streamUpdateSubscriptionCache = append( sm.streamUpdateSubscriptionCache, sm.clobPairIdToSubscriptionIdMapping[clobPairId], ) } +} - // Add subaccount updates to cache. +// cacheStreamUpdatesBySubaccountWithLock adds subaccount stream updates to cache, +// and store corresponding subaccount Ids. +// This method requires the lock and assumes that the lock has already been +// acquired by the caller. +func (sm *FullNodeStreamingManagerImpl) cacheStreamUpdatesBySubaccountWithLock( + subaccountStreamUpdates []clobtypes.StreamUpdate, + subaccountIds []*satypes.SubaccountId, +) { sm.streamUpdateCache = append(sm.streamUpdateCache, subaccountStreamUpdates...) for _, subaccountId := range subaccountIds { sm.streamUpdateSubscriptionCache = append( @@ -902,41 +922,50 @@ func (sm *FullNodeStreamingManagerImpl) StreamBatchUpdatesAfterFinalizeBlock( // Prevent gas metering from state read. ctx = ctx.WithGasMeter(ante_types.NewFreeInfiniteGasMeter()) - finalizedFills, finalizedSubaccountUpdates := sm.getStagedEventsFromFinalizeBlock(ctx) + finalizedFills, + finalizedSubaccountUpdates, + finalizedOrderbookUpdates := sm.getStagedEventsFromFinalizeBlock(ctx) + + sm.Lock() + defer sm.Unlock() + + // Flush all pending updates, since we want the onchain updates to arrive in a batch. + sm.FlushStreamUpdatesWithLock() - orderbookStreamUpdates, orderbookClobPairIds := getStreamUpdatesFromOffchainUpdates( - orderBookUpdatesToSyncLocalOpsQueue, - uint32(ctx.BlockHeight()), + // Cache updates to sync local ops queue + sycnLocalUpdates, syncLocalClobPairIds := getStreamUpdatesFromOffchainUpdates( + streaming_util.GetOffchainUpdatesV1(orderBookUpdatesToSyncLocalOpsQueue), + lib.MustConvertIntegerToUint32(ctx.BlockHeight()), ctx.ExecMode(), ) + sm.cacheStreamUpdatesByClobPairWithLock(sycnLocalUpdates, syncLocalClobPairIds) + // Cache updates for finalized fills. fillStreamUpdates, fillClobPairIds := sm.getStreamUpdatesForOrderbookFills( finalizedFills, - uint32(ctx.BlockHeight()), + lib.MustConvertIntegerToUint32(ctx.BlockHeight()), ctx.ExecMode(), perpetualIdToClobPairId, ) + sm.cacheStreamUpdatesByClobPairWithLock(fillStreamUpdates, fillClobPairIds) + + // Cache updates for finalized orderbook updates (e.g. RemoveOrderFillAmount in `EndBlocker`). + for _, finalizedUpdate := range finalizedOrderbookUpdates { + streamUpdates, clobPairIds := getStreamUpdatesFromOffchainUpdates( + finalizedUpdate.Updates, + lib.MustConvertIntegerToUint32(ctx.BlockHeight()), + ctx.ExecMode(), + ) + sm.cacheStreamUpdatesByClobPairWithLock(streamUpdates, clobPairIds) + } + // Finally, cache updates for finalized subaccount updates subaccountStreamUpdates, subaccountIds := getStreamUpdatesForSubaccountUpdates( finalizedSubaccountUpdates, - uint32(ctx.BlockHeight()), + lib.MustConvertIntegerToUint32(ctx.BlockHeight()), ctx.ExecMode(), ) - - sm.Lock() - defer sm.Unlock() - - // Flush all pending updates, since we want the onchain updates to arrive in a batch. - sm.FlushStreamUpdatesWithLock() - - sm.addBatchUpdatesToCacheWithLock( - orderbookStreamUpdates, - orderbookClobPairIds, - fillStreamUpdates, - fillClobPairIds, - subaccountStreamUpdates, - subaccountIds, - ) + sm.cacheStreamUpdatesBySubaccountWithLock(subaccountStreamUpdates, subaccountIds) // Emit all stream updates in a single batch. // Note we still have the lock, which is released right before function returns. @@ -950,6 +979,7 @@ func (sm *FullNodeStreamingManagerImpl) getStagedEventsFromFinalizeBlock( ) ( finalizedFills []clobtypes.StreamOrderbookFill, finalizedSubaccountUpdates []satypes.StreamSubaccountUpdate, + finalizedOrderbookUpdates []clobtypes.StreamOrderbookUpdate, ) { // Get onchain stream events stored in transient store. stagedEvents := sm.GetStagedFinalizeBlockEvents(ctx) @@ -965,6 +995,10 @@ func (sm *FullNodeStreamingManagerImpl) getStagedEventsFromFinalizeBlock( finalizedFills = append(finalizedFills, *event.OrderFill) case *clobtypes.StagedFinalizeBlockEvent_SubaccountUpdate: finalizedSubaccountUpdates = append(finalizedSubaccountUpdates, *event.SubaccountUpdate) + case *clobtypes.StagedFinalizeBlockEvent_OrderbookUpdate: + finalizedOrderbookUpdates = append(finalizedOrderbookUpdates, *event.OrderbookUpdate) + default: + panic(fmt.Sprintf("Unhandled staged event type: %v\n", stagedEvent.Event)) } } @@ -977,7 +1011,7 @@ func (sm *FullNodeStreamingManagerImpl) getStagedEventsFromFinalizeBlock( float32(len(finalizedFills)), ) - return finalizedFills, finalizedSubaccountUpdates + return finalizedFills, finalizedSubaccountUpdates, finalizedOrderbookUpdates } func (sm *FullNodeStreamingManagerImpl) InitializeNewStreams( diff --git a/protocol/streaming/noop_streaming_manager.go b/protocol/streaming/noop_streaming_manager.go index 4df60bc427..9dc7bf6de9 100644 --- a/protocol/streaming/noop_streaming_manager.go +++ b/protocol/streaming/noop_streaming_manager.go @@ -31,23 +31,20 @@ func (sm *NoopGrpcStreamingManager) Subscribe( func (sm *NoopGrpcStreamingManager) SendOrderbookUpdates( updates *clobtypes.OffchainUpdates, - blockHeight uint32, - execMode sdk.ExecMode, + ctx sdk.Context, ) { } -func (sm *NoopGrpcStreamingManager) SendOrderbookFillUpdates( - orderbookFills []clobtypes.StreamOrderbookFill, - blockHeight uint32, - execMode sdk.ExecMode, +func (sm *NoopGrpcStreamingManager) SendOrderbookFillUpdate( + orderbookFill clobtypes.StreamOrderbookFill, + ctx sdk.Context, perpetualIdToClobPairId map[uint32][]clobtypes.ClobPairId, ) { } func (sm *NoopGrpcStreamingManager) SendTakerOrderStatus( takerOrder clobtypes.StreamTakerOrder, - blockHeight uint32, - execMode sdk.ExecMode, + ctx sdk.Context, ) { } @@ -79,19 +76,13 @@ func (sm *NoopGrpcStreamingManager) InitializeNewStreams( func (sm *NoopGrpcStreamingManager) Stop() { } -func (sm *NoopGrpcStreamingManager) StageFinalizeBlockFill( - ctx sdk.Context, - fill clobtypes.StreamOrderbookFill, -) { -} - func (sm *NoopGrpcStreamingManager) GetStagedFinalizeBlockEvents( ctx sdk.Context, ) []clobtypes.StagedFinalizeBlockEvent { return nil } -func (sm *NoopGrpcStreamingManager) StageFinalizeBlockSubaccountUpdate( +func (sm *NoopGrpcStreamingManager) SendSubaccountUpdate( ctx sdk.Context, subaccountUpdate satypes.StreamSubaccountUpdate, ) { diff --git a/protocol/streaming/types/interface.go b/protocol/streaming/types/interface.go index 0f097d3e75..5b42864016 100644 --- a/protocol/streaming/types/interface.go +++ b/protocol/streaming/types/interface.go @@ -31,30 +31,23 @@ type FullNodeStreamingManager interface { ) map[satypes.SubaccountId]*satypes.StreamSubaccountUpdate SendOrderbookUpdates( offchainUpdates *clobtypes.OffchainUpdates, - blockHeight uint32, - execMode sdk.ExecMode, + ctx sdk.Context, ) - SendOrderbookFillUpdates( - orderbookFills []clobtypes.StreamOrderbookFill, - blockHeight uint32, - execMode sdk.ExecMode, + SendOrderbookFillUpdate( + orderbookFill clobtypes.StreamOrderbookFill, + ctx sdk.Context, perpetualIdToClobPairId map[uint32][]clobtypes.ClobPairId, ) SendTakerOrderStatus( takerOrder clobtypes.StreamTakerOrder, - blockHeight uint32, - execMode sdk.ExecMode, + ctx sdk.Context, ) SendFinalizedSubaccountUpdates( subaccountUpdates []satypes.StreamSubaccountUpdate, blockHeight uint32, execMode sdk.ExecMode, ) - StageFinalizeBlockFill( - ctx sdk.Context, - fill clobtypes.StreamOrderbookFill, - ) - StageFinalizeBlockSubaccountUpdate( + SendSubaccountUpdate( ctx sdk.Context, subaccountUpdate satypes.StreamSubaccountUpdate, ) diff --git a/protocol/streaming/util/util.go b/protocol/streaming/util/util.go index 985a29ef33..bbf37e3340 100644 --- a/protocol/streaming/util/util.go +++ b/protocol/streaming/util/util.go @@ -1,21 +1,23 @@ package util import ( + "fmt" + "github.com/cosmos/gogoproto/proto" ocutypes "github.com/dydxprotocol/v4-chain/protocol/indexer/off_chain_updates/types" clobtypes "github.com/dydxprotocol/v4-chain/protocol/x/clob/types" ) // GetOffchainUpdatesV1 unmarshals messages in offchain updates to OffchainUpdateV1. -func GetOffchainUpdatesV1(offchainUpdates *clobtypes.OffchainUpdates) ([]ocutypes.OffChainUpdateV1, error) { +func GetOffchainUpdatesV1(offchainUpdates *clobtypes.OffchainUpdates) []ocutypes.OffChainUpdateV1 { v1updates := make([]ocutypes.OffChainUpdateV1, 0) for _, message := range offchainUpdates.Messages { var update ocutypes.OffChainUpdateV1 err := proto.Unmarshal(message.Message.Value, &update) if err != nil { - return nil, err + panic(fmt.Sprintf("Failed to get OffchainUpdatesV1: %v", err)) } v1updates = append(v1updates, update) } - return v1updates, nil + return v1updates } diff --git a/protocol/testutil/memclob/keeper.go b/protocol/testutil/memclob/keeper.go index ce15a95e29..dea464d086 100644 --- a/protocol/testutil/memclob/keeper.go +++ b/protocol/testutil/memclob/keeper.go @@ -506,9 +506,9 @@ func (f *FakeMemClobKeeper) SendOrderbookUpdates( ) { } -func (f *FakeMemClobKeeper) SendOrderbookFillUpdates( +func (f *FakeMemClobKeeper) SendOrderbookFillUpdate( ctx sdk.Context, - orderbookFills []types.StreamOrderbookFill, + orderbookFill types.StreamOrderbookFill, ) { } diff --git a/protocol/x/clob/keeper/keeper.go b/protocol/x/clob/keeper/keeper.go index 8e607d6833..f1968c3124 100644 --- a/protocol/x/clob/keeper/keeper.go +++ b/protocol/x/clob/keeper/keeper.go @@ -303,23 +303,18 @@ func (k Keeper) SendOrderbookUpdates( k.GetFullNodeStreamingManager().SendOrderbookUpdates( offchainUpdates, - lib.MustConvertIntegerToUint32(ctx.BlockHeight()), - ctx.ExecMode(), + ctx, ) } -// SendOrderbookFillUpdates sends the orderbook fills to the Full Node streaming manager. -func (k Keeper) SendOrderbookFillUpdates( +// SendOrderbookFillUpdate sends the orderbook fills to the Full Node streaming manager. +func (k Keeper) SendOrderbookFillUpdate( ctx sdk.Context, - orderbookFills []types.StreamOrderbookFill, + orderbookFill types.StreamOrderbookFill, ) { - if len(orderbookFills) == 0 { - return - } - k.GetFullNodeStreamingManager().SendOrderbookFillUpdates( - orderbookFills, - lib.MustConvertIntegerToUint32(ctx.BlockHeight()), - ctx.ExecMode(), + k.GetFullNodeStreamingManager().SendOrderbookFillUpdate( + orderbookFill, + ctx, k.PerpetualIdToClobPairId, ) } @@ -331,7 +326,6 @@ func (k Keeper) SendTakerOrderStatus( ) { k.GetFullNodeStreamingManager().SendTakerOrderStatus( takerOrder, - lib.MustConvertIntegerToUint32(ctx.BlockHeight()), - ctx.ExecMode(), + ctx, ) } diff --git a/protocol/x/clob/keeper/process_operations.go b/protocol/x/clob/keeper/process_operations.go index cddb5fb68b..4345fcdbef 100644 --- a/protocol/x/clob/keeper/process_operations.go +++ b/protocol/x/clob/keeper/process_operations.go @@ -542,9 +542,10 @@ func (k Keeper) PersistMatchOrdersToState( makerOrders, ) - k.GetFullNodeStreamingManager().StageFinalizeBlockFill( - ctx, + k.GetFullNodeStreamingManager().SendOrderbookFillUpdate( streamOrderbookFill, + ctx, + k.PerpetualIdToClobPairId, ) } @@ -649,9 +650,10 @@ func (k Keeper) PersistMatchLiquidationToState( takerOrder, makerOrders, ) - k.GetFullNodeStreamingManager().StageFinalizeBlockFill( - ctx, + k.GetFullNodeStreamingManager().SendOrderbookFillUpdate( streamOrderbookFill, + ctx, + k.PerpetualIdToClobPairId, ) } return nil @@ -822,11 +824,9 @@ func (k Keeper) PersistMatchDeleveragingToState( }, }, } - k.SendOrderbookFillUpdates( + k.SendOrderbookFillUpdate( ctx, - []types.StreamOrderbookFill{ - streamOrderbookFill, - }, + streamOrderbookFill, ) } } diff --git a/protocol/x/clob/memclob/memclob.go b/protocol/x/clob/memclob/memclob.go index 8054a41e78..d50b81b6c4 100644 --- a/protocol/x/clob/memclob/memclob.go +++ b/protocol/x/clob/memclob/memclob.go @@ -402,7 +402,7 @@ func (m *MemClobPriceTimePriority) mustUpdateMemclobStateWithMatches( ) clobMatch := internalOperation.GetMatch() orderbookMatchFill := m.GenerateStreamOrderbookFill(ctx, *clobMatch, takerOrder, makerOrders) - m.clobKeeper.SendOrderbookFillUpdates(ctx, []types.StreamOrderbookFill{orderbookMatchFill}) + m.clobKeeper.SendOrderbookFillUpdate(ctx, orderbookMatchFill) } // Build a slice of all subaccounts which had matches this matching loop, and sort them for determinism. diff --git a/protocol/x/clob/types/mem_clob_keeper.go b/protocol/x/clob/types/mem_clob_keeper.go index 18898e79e2..90d0ed382a 100644 --- a/protocol/x/clob/types/mem_clob_keeper.go +++ b/protocol/x/clob/types/mem_clob_keeper.go @@ -100,9 +100,9 @@ type MemClobKeeper interface { ctx sdk.Context, offchainUpdates *OffchainUpdates, ) - SendOrderbookFillUpdates( + SendOrderbookFillUpdate( ctx sdk.Context, - orderbookFills []StreamOrderbookFill, + orderbookFill StreamOrderbookFill, ) SendTakerOrderStatus( ctx sdk.Context, diff --git a/protocol/x/clob/types/streaming.pb.go b/protocol/x/clob/types/streaming.pb.go index 83b8db719a..1f6f552fb3 100644 --- a/protocol/x/clob/types/streaming.pb.go +++ b/protocol/x/clob/types/streaming.pb.go @@ -30,6 +30,7 @@ type StagedFinalizeBlockEvent struct { // Types that are valid to be assigned to Event: // *StagedFinalizeBlockEvent_OrderFill // *StagedFinalizeBlockEvent_SubaccountUpdate + // *StagedFinalizeBlockEvent_OrderbookUpdate Event isStagedFinalizeBlockEvent_Event `protobuf_oneof:"event"` } @@ -78,9 +79,13 @@ type StagedFinalizeBlockEvent_OrderFill struct { type StagedFinalizeBlockEvent_SubaccountUpdate struct { SubaccountUpdate *types.StreamSubaccountUpdate `protobuf:"bytes,2,opt,name=subaccount_update,json=subaccountUpdate,proto3,oneof" json:"subaccount_update,omitempty"` } +type StagedFinalizeBlockEvent_OrderbookUpdate struct { + OrderbookUpdate *StreamOrderbookUpdate `protobuf:"bytes,3,opt,name=orderbook_update,json=orderbookUpdate,proto3,oneof" json:"orderbook_update,omitempty"` +} func (*StagedFinalizeBlockEvent_OrderFill) isStagedFinalizeBlockEvent_Event() {} func (*StagedFinalizeBlockEvent_SubaccountUpdate) isStagedFinalizeBlockEvent_Event() {} +func (*StagedFinalizeBlockEvent_OrderbookUpdate) isStagedFinalizeBlockEvent_Event() {} func (m *StagedFinalizeBlockEvent) GetEvent() isStagedFinalizeBlockEvent_Event { if m != nil { @@ -103,11 +108,19 @@ func (m *StagedFinalizeBlockEvent) GetSubaccountUpdate() *types.StreamSubaccount return nil } +func (m *StagedFinalizeBlockEvent) GetOrderbookUpdate() *StreamOrderbookUpdate { + if x, ok := m.GetEvent().(*StagedFinalizeBlockEvent_OrderbookUpdate); ok { + return x.OrderbookUpdate + } + return nil +} + // XXX_OneofWrappers is for the internal use of the proto package. func (*StagedFinalizeBlockEvent) XXX_OneofWrappers() []interface{} { return []interface{}{ (*StagedFinalizeBlockEvent_OrderFill)(nil), (*StagedFinalizeBlockEvent_SubaccountUpdate)(nil), + (*StagedFinalizeBlockEvent_OrderbookUpdate)(nil), } } @@ -118,25 +131,26 @@ func init() { func init() { proto.RegisterFile("dydxprotocol/clob/streaming.proto", fileDescriptor_cecf6ffcf2554dee) } var fileDescriptor_cecf6ffcf2554dee = []byte{ - // 281 bytes of a gzipped FileDescriptorProto + // 303 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x52, 0x4c, 0xa9, 0x4c, 0xa9, 0x28, 0x28, 0xca, 0x2f, 0xc9, 0x4f, 0xce, 0xcf, 0xd1, 0x4f, 0xce, 0xc9, 0x4f, 0xd2, 0x2f, 0x2e, 0x29, 0x4a, 0x4d, 0xcc, 0xcd, 0xcc, 0x4b, 0xd7, 0x03, 0x8b, 0x0b, 0x09, 0x22, 0x2b, 0xd1, 0x03, 0x29, 0x91, 0xd2, 0x40, 0xd1, 0x55, 0x5c, 0x9a, 0x94, 0x98, 0x9c, 0x9c, 0x5f, 0x9a, 0x57, 0x52, - 0x8c, 0xae, 0x59, 0x4a, 0x16, 0xd3, 0xfc, 0xc2, 0xd2, 0xd4, 0xa2, 0x4a, 0x88, 0xb4, 0xd2, 0x59, - 0x46, 0x2e, 0x89, 0xe0, 0x92, 0xc4, 0xf4, 0xd4, 0x14, 0xb7, 0xcc, 0xbc, 0xc4, 0x9c, 0xcc, 0xaa, + 0x8c, 0xae, 0x59, 0x4a, 0x16, 0xd3, 0xfc, 0xc2, 0xd2, 0xd4, 0xa2, 0x4a, 0x88, 0xb4, 0xd2, 0x12, + 0x26, 0x2e, 0x89, 0xe0, 0x92, 0xc4, 0xf4, 0xd4, 0x14, 0xb7, 0xcc, 0xbc, 0xc4, 0x9c, 0xcc, 0xaa, 0x54, 0xa7, 0x9c, 0xfc, 0xe4, 0x6c, 0xd7, 0xb2, 0xd4, 0xbc, 0x12, 0x21, 0x77, 0x2e, 0xae, 0xfc, 0xa2, 0x94, 0xd4, 0xa2, 0xf8, 0xb4, 0xcc, 0x9c, 0x1c, 0x09, 0x46, 0x05, 0x46, 0x0d, 0x6e, 0x23, 0x35, 0x3d, 0x0c, 0xd7, 0xe8, 0x05, 0x83, 0xed, 0xf4, 0x07, 0x29, 0x4d, 0xca, 0xcf, 0xcf, 0x76, 0xcb, 0xcc, 0xc9, 0xf1, 0x60, 0x08, 0xe2, 0x04, 0xeb, 0x05, 0x71, 0x84, 0xe2, 0xb9, 0x04, 0x11, 0x6e, 0x8c, 0x2f, 0x2d, 0x48, 0x49, 0x2c, 0x49, 0x95, 0x60, 0x02, 0x9b, 0x67, 0x80, 0x6a, 0x1e, 0x92, 0x57, 0xa0, 0xc6, 0x06, 0xc3, 0x45, 0x42, 0xc1, 0xfa, 0x3c, 0x18, 0x82, 0x04, 0x8a, 0xd1, - 0xc4, 0x9c, 0xd8, 0xb9, 0x58, 0x53, 0x41, 0x4e, 0x76, 0x0a, 0x38, 0xf1, 0x48, 0x8e, 0xf1, 0xc2, - 0x23, 0x39, 0xc6, 0x07, 0x8f, 0xe4, 0x18, 0x27, 0x3c, 0x96, 0x63, 0xb8, 0xf0, 0x58, 0x8e, 0xe1, - 0xc6, 0x63, 0x39, 0x86, 0x28, 0xb3, 0xf4, 0xcc, 0x92, 0x8c, 0xd2, 0x24, 0xbd, 0xe4, 0xfc, 0x5c, - 0x7d, 0x94, 0x30, 0x29, 0x33, 0xd1, 0x4d, 0xce, 0x48, 0xcc, 0xcc, 0xd3, 0x87, 0x8b, 0x54, 0x40, - 0xc2, 0xa9, 0xa4, 0xb2, 0x20, 0xb5, 0x38, 0x89, 0x0d, 0x2c, 0x6c, 0x0c, 0x08, 0x00, 0x00, 0xff, - 0xff, 0x65, 0x71, 0xd8, 0xa8, 0xa9, 0x01, 0x00, 0x00, + 0xc4, 0x84, 0x42, 0xb9, 0x04, 0xf2, 0x61, 0xd6, 0xc3, 0xcc, 0x67, 0x06, 0x9b, 0xaf, 0x41, 0xd8, + 0xbd, 0x70, 0x73, 0xf9, 0xf3, 0x51, 0x85, 0x9c, 0xd8, 0xb9, 0x58, 0x53, 0x41, 0x21, 0xe1, 0x14, + 0x70, 0xe2, 0x91, 0x1c, 0xe3, 0x85, 0x47, 0x72, 0x8c, 0x0f, 0x1e, 0xc9, 0x31, 0x4e, 0x78, 0x2c, + 0xc7, 0x70, 0xe1, 0xb1, 0x1c, 0xc3, 0x8d, 0xc7, 0x72, 0x0c, 0x51, 0x66, 0xe9, 0x99, 0x25, 0x19, + 0xa5, 0x49, 0x7a, 0xc9, 0xf9, 0xb9, 0xfa, 0x28, 0x41, 0x5d, 0x66, 0xa2, 0x9b, 0x9c, 0x91, 0x98, + 0x99, 0xa7, 0x0f, 0x17, 0xa9, 0x80, 0x04, 0x7f, 0x49, 0x65, 0x41, 0x6a, 0x71, 0x12, 0x1b, 0x58, + 0xd8, 0x18, 0x10, 0x00, 0x00, 0xff, 0xff, 0xe1, 0xc0, 0xbc, 0x6c, 0x00, 0x02, 0x00, 0x00, } func (m *StagedFinalizeBlockEvent) Marshal() (dAtA []byte, err error) { @@ -213,6 +227,27 @@ func (m *StagedFinalizeBlockEvent_SubaccountUpdate) MarshalToSizedBuffer(dAtA [] } return len(dAtA) - i, nil } +func (m *StagedFinalizeBlockEvent_OrderbookUpdate) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *StagedFinalizeBlockEvent_OrderbookUpdate) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + if m.OrderbookUpdate != nil { + { + size, err := m.OrderbookUpdate.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintStreaming(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x1a + } + return len(dAtA) - i, nil +} func encodeVarintStreaming(dAtA []byte, offset int, v uint64) int { offset -= sovStreaming(v) base := offset @@ -260,6 +295,18 @@ func (m *StagedFinalizeBlockEvent_SubaccountUpdate) Size() (n int) { } return n } +func (m *StagedFinalizeBlockEvent_OrderbookUpdate) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.OrderbookUpdate != nil { + l = m.OrderbookUpdate.Size() + n += 1 + l + sovStreaming(uint64(l)) + } + return n +} func sovStreaming(x uint64) (n int) { return (math_bits.Len64(x|1) + 6) / 7 @@ -366,6 +413,41 @@ func (m *StagedFinalizeBlockEvent) Unmarshal(dAtA []byte) error { } m.Event = &StagedFinalizeBlockEvent_SubaccountUpdate{v} iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field OrderbookUpdate", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowStreaming + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthStreaming + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthStreaming + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + v := &StreamOrderbookUpdate{} + if err := v.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + m.Event = &StagedFinalizeBlockEvent_OrderbookUpdate{v} + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipStreaming(dAtA[iNdEx:]) diff --git a/protocol/x/subaccounts/keeper/subaccount.go b/protocol/x/subaccounts/keeper/subaccount.go index 5eb650cce0..e72ccd60e7 100644 --- a/protocol/x/subaccounts/keeper/subaccount.go +++ b/protocol/x/subaccounts/keeper/subaccount.go @@ -445,7 +445,7 @@ func (k Keeper) UpdateSubaccounts( if lib.IsDeliverTxMode(ctx) && k.GetFullNodeStreamingManager().Enabled() { if k.GetFullNodeStreamingManager().TracksSubaccountId(*u.SettledSubaccount.Id) { subaccountUpdate := GenerateStreamSubaccountUpdate(u, fundingPayments) - k.GetFullNodeStreamingManager().StageFinalizeBlockSubaccountUpdate( + k.GetFullNodeStreamingManager().SendSubaccountUpdate( ctx, subaccountUpdate, ) From b3575927d7c190f3c9720caad51feccaac717c92 Mon Sep 17 00:00:00 2001 From: Teddy Ding Date: Wed, 2 Oct 2024 15:42:21 -0400 Subject: [PATCH 30/33] Move FinalizeBlock event staging logic into a generic EventStager (#2435) --- protocol/finalizeblock/event_stager.go | 83 ++++++++++++++ .../streaming/full_node_streaming_manager.go | 107 +++++------------- protocol/streaming/noop_streaming_manager.go | 7 -- protocol/streaming/types/interface.go | 5 - protocol/x/clob/types/expected_keepers.go | 4 - protocol/x/subaccounts/keeper/subaccount.go | 16 --- protocol/x/subaccounts/types/types.go | 4 - 7 files changed, 113 insertions(+), 113 deletions(-) create mode 100644 protocol/finalizeblock/event_stager.go diff --git a/protocol/finalizeblock/event_stager.go b/protocol/finalizeblock/event_stager.go new file mode 100644 index 0000000000..cb8d7f2fa6 --- /dev/null +++ b/protocol/finalizeblock/event_stager.go @@ -0,0 +1,83 @@ +package finalizeblock + +import ( + "encoding/binary" + + "cosmossdk.io/store/prefix" + storetypes "cosmossdk.io/store/types" + "github.com/cosmos/cosmos-sdk/codec" + sdk "github.com/cosmos/cosmos-sdk/types" + "github.com/cosmos/gogoproto/proto" + ante_types "github.com/dydxprotocol/v4-chain/protocol/app/ante/types" + "github.com/dydxprotocol/v4-chain/protocol/lib" +) + +// EventStager supports staging and retrieval of events (of type T) from FinalizeBlock. +type EventStager[T proto.Message] struct { + transientStoreKey storetypes.StoreKey + cdc codec.BinaryCodec + stagedEventCountKey string + stagedEventKeyPrefix string +} + +// NewEventStager creates a new EventStager. +func NewEventStager[T proto.Message]( + transientStoreKey storetypes.StoreKey, + cdc codec.BinaryCodec, + stagedEventCountKey string, + stagedEventKeyPrefix string, +) EventStager[T] { + return EventStager[T]{ + transientStoreKey: transientStoreKey, + cdc: cdc, + stagedEventCountKey: stagedEventCountKey, + stagedEventKeyPrefix: stagedEventKeyPrefix, + } +} + +// GetStagedFinalizeBlockEvents retrieves all staged events from the store. +func (s EventStager[T]) GetStagedFinalizeBlockEvents( + ctx sdk.Context, + newStagedEvent func() T, +) []T { + noGasCtx := ctx.WithGasMeter(ante_types.NewFreeInfiniteGasMeter()) + store := noGasCtx.TransientStore(s.transientStoreKey) + + count := s.getStagedEventsCount(store) + events := make([]T, count) + store = prefix.NewStore(store, []byte(s.stagedEventKeyPrefix)) + for i := uint32(0); i < count; i++ { + event := newStagedEvent() + bytes := store.Get(lib.Uint32ToKey(i)) + s.cdc.MustUnmarshal(bytes, event) + events[i] = event + } + return events +} + +func (s EventStager[T]) getStagedEventsCount( + store storetypes.KVStore, +) uint32 { + countsBytes := store.Get([]byte(s.stagedEventCountKey)) + if countsBytes == nil { + return 0 + } + return binary.BigEndian.Uint32(countsBytes) +} + +// StageFinalizeBlockEvent stages an event in the transient store. +func (s EventStager[T]) StageFinalizeBlockEvent( + ctx sdk.Context, + stagedEvent T, +) { + noGasCtx := ctx.WithGasMeter(ante_types.NewFreeInfiniteGasMeter()) + store := noGasCtx.TransientStore(s.transientStoreKey) + + // Increment events count. + count := s.getStagedEventsCount(store) + store.Set([]byte(s.stagedEventCountKey), lib.Uint32ToKey(count+1)) + + // Store events keyed by index. + store = prefix.NewStore(store, []byte(s.stagedEventKeyPrefix)) + store.Set(lib.Uint32ToKey(count), s.cdc.MustMarshal(stagedEvent)) +} diff --git a/protocol/streaming/full_node_streaming_manager.go b/protocol/streaming/full_node_streaming_manager.go index 85c265f12e..f63beac347 100644 --- a/protocol/streaming/full_node_streaming_manager.go +++ b/protocol/streaming/full_node_streaming_manager.go @@ -1,7 +1,6 @@ package streaming import ( - "encoding/binary" "fmt" "sync" "sync/atomic" @@ -11,7 +10,6 @@ import ( satypes "github.com/dydxprotocol/v4-chain/protocol/x/subaccounts/types" "cosmossdk.io/log" - "cosmossdk.io/store/prefix" storetypes "cosmossdk.io/store/types" "github.com/cosmos/cosmos-sdk/codec" sdk "github.com/cosmos/cosmos-sdk/types" @@ -22,6 +20,8 @@ import ( clobtypes "github.com/dydxprotocol/v4-chain/protocol/x/clob/types" ocutypes "github.com/dydxprotocol/v4-chain/protocol/indexer/off_chain_updates/types" + + "github.com/dydxprotocol/v4-chain/protocol/finalizeblock" ) var _ types.FullNodeStreamingManager = (*FullNodeStreamingManagerImpl)(nil) @@ -62,6 +62,8 @@ type FullNodeStreamingManagerImpl struct { // stores the staged FinalizeBlock events for full node streaming. streamingManagerTransientStoreKey storetypes.StoreKey + + finalizeBlockStager finalizeblock.EventStager[*clobtypes.StagedFinalizeBlockEvent] } // OrderbookSubscription represents a active subscription to the orderbook updates stream. @@ -119,6 +121,12 @@ func NewFullNodeStreamingManager( streamingManagerTransientStoreKey: streamingManagerTransientStoreKey, cdc: cdc, + finalizeBlockStager: finalizeblock.NewEventStager[*clobtypes.StagedFinalizeBlockEvent]( + streamingManagerTransientStoreKey, + cdc, + StagedEventsCountKey, + StagedEventsKeyPrefix, + ), } // Start the goroutine for pushing order updates through. @@ -381,14 +389,6 @@ func (sm *FullNodeStreamingManagerImpl) sendStreamUpdates( } } -func getStagedEventsCount(store storetypes.KVStore) uint32 { - countsBytes := store.Get([]byte(StagedEventsCountKey)) - if countsBytes == nil { - return 0 - } - return binary.BigEndian.Uint32(countsBytes) -} - // Send a subaccount update event. func (sm *FullNodeStreamingManagerImpl) SendSubaccountUpdate( ctx sdk.Context, @@ -405,51 +405,30 @@ func (sm *FullNodeStreamingManagerImpl) SendSubaccountUpdate( SubaccountUpdate: &subaccountUpdate, }, } - sm.stageFinalizeBlockEvent( + sm.finalizeBlockStager.StageFinalizeBlockEvent( ctx, - sm.cdc.MustMarshal(&stagedEvent), + &stagedEvent, ) } -func getStagedFinalizeBlockEventsFromStore( - store storetypes.KVStore, - cdc codec.BinaryCodec, -) []clobtypes.StagedFinalizeBlockEvent { - count := getStagedEventsCount(store) - events := make([]clobtypes.StagedFinalizeBlockEvent, count) - store = prefix.NewStore(store, []byte(StagedEventsKeyPrefix)) - for i := uint32(0); i < count; i++ { - var event clobtypes.StagedFinalizeBlockEvent - bytes := store.Get(lib.Uint32ToKey(i)) - cdc.MustUnmarshal(bytes, &event) - events[i] = event - } - return events -} - // Retrieve all events staged during `FinalizeBlock`. func (sm *FullNodeStreamingManagerImpl) GetStagedFinalizeBlockEvents( ctx sdk.Context, ) []clobtypes.StagedFinalizeBlockEvent { - noGasCtx := ctx.WithGasMeter(ante_types.NewFreeInfiniteGasMeter()) - store := noGasCtx.TransientStore(sm.streamingManagerTransientStoreKey) - return getStagedFinalizeBlockEventsFromStore(store, sm.cdc) -} - -func (sm *FullNodeStreamingManagerImpl) stageFinalizeBlockEvent( - ctx sdk.Context, - eventBytes []byte, -) { - noGasCtx := ctx.WithGasMeter(ante_types.NewFreeInfiniteGasMeter()) - store := noGasCtx.TransientStore(sm.streamingManagerTransientStoreKey) - - // Increment events count. - count := getStagedEventsCount(store) - store.Set([]byte(StagedEventsCountKey), lib.Uint32ToKey(count+1)) - - // Store events keyed by index. - store = prefix.NewStore(store, []byte(StagedEventsKeyPrefix)) - store.Set(lib.Uint32ToKey(count), eventBytes) + events := sm.finalizeBlockStager.GetStagedFinalizeBlockEvents( + ctx, + func() *clobtypes.StagedFinalizeBlockEvent { + return &clobtypes.StagedFinalizeBlockEvent{} + }, + ) + results := make([]clobtypes.StagedFinalizeBlockEvent, len(events)) + for i, event := range events { + if event == nil { + panic("Got nil event from finalizeBlockStager") + } + results[i] = *event + } + return results } // SendCombinedSnapshot sends messages to a particular subscriber without buffering. @@ -574,9 +553,9 @@ func (sm *FullNodeStreamingManagerImpl) SendOrderbookUpdates( }, }, } - sm.stageFinalizeBlockEvent( + sm.finalizeBlockStager.StageFinalizeBlockEvent( ctx, - sm.cdc.MustMarshal(&stagedEvent), + &stagedEvent, ) } @@ -649,9 +628,9 @@ func (sm *FullNodeStreamingManagerImpl) SendOrderbookFillUpdate( }, } - sm.stageFinalizeBlockEvent( + sm.finalizeBlockStager.StageFinalizeBlockEvent( ctx, - sm.cdc.MustMarshal(&stagedEvent), + &stagedEvent, ) } @@ -710,32 +689,6 @@ func getStreamUpdatesForSubaccountUpdates( return streamUpdates, subaccountIds } -// SendFinalizedSubaccountUpdates groups subaccount updates by their subaccount ids and -// sends messages to the subscribers. -func (sm *FullNodeStreamingManagerImpl) SendFinalizedSubaccountUpdates( - subaccountUpdates []satypes.StreamSubaccountUpdate, - blockHeight uint32, - execMode sdk.ExecMode, -) { - defer metrics.ModuleMeasureSince( - metrics.FullNodeGrpc, - metrics.GrpcSendFinalizedSubaccountUpdatesLatency, - time.Now(), - ) - - if execMode != sdk.ExecModeFinalize { - panic("SendFinalizedSubaccountUpdates should only be called in ExecModeFinalize") - } - - streamUpdates, subaccountIds := getStreamUpdatesForSubaccountUpdates( - subaccountUpdates, - blockHeight, - execMode, - ) - - sm.AddSubaccountUpdatesToCache(streamUpdates, subaccountIds) -} - // AddOrderUpdatesToCache adds a series of updates to the full node streaming cache. // Clob pair ids are the clob pair id each update is relevant to. func (sm *FullNodeStreamingManagerImpl) AddOrderUpdatesToCache( diff --git a/protocol/streaming/noop_streaming_manager.go b/protocol/streaming/noop_streaming_manager.go index 9dc7bf6de9..89250854c4 100644 --- a/protocol/streaming/noop_streaming_manager.go +++ b/protocol/streaming/noop_streaming_manager.go @@ -48,13 +48,6 @@ func (sm *NoopGrpcStreamingManager) SendTakerOrderStatus( ) { } -func (sm *NoopGrpcStreamingManager) SendFinalizedSubaccountUpdates( - subaccountUpdates []satypes.StreamSubaccountUpdate, - blockHeight uint32, - execMode sdk.ExecMode, -) { -} - func (sm *NoopGrpcStreamingManager) TracksSubaccountId(id satypes.SubaccountId) bool { return false } diff --git a/protocol/streaming/types/interface.go b/protocol/streaming/types/interface.go index 5b42864016..33907fc1ec 100644 --- a/protocol/streaming/types/interface.go +++ b/protocol/streaming/types/interface.go @@ -42,11 +42,6 @@ type FullNodeStreamingManager interface { takerOrder clobtypes.StreamTakerOrder, ctx sdk.Context, ) - SendFinalizedSubaccountUpdates( - subaccountUpdates []satypes.StreamSubaccountUpdate, - blockHeight uint32, - execMode sdk.ExecMode, - ) SendSubaccountUpdate( ctx sdk.Context, subaccountUpdate satypes.StreamSubaccountUpdate, diff --git a/protocol/x/clob/types/expected_keepers.go b/protocol/x/clob/types/expected_keepers.go index b14c4aeacd..384d950604 100644 --- a/protocol/x/clob/types/expected_keepers.go +++ b/protocol/x/clob/types/expected_keepers.go @@ -85,10 +85,6 @@ type SubaccountsKeeper interface { quantums *big.Int, perpetualId uint32, ) error - SendFinalizedSubaccountUpdates( - ctx sdk.Context, - subaccountUpdates []satypes.StreamSubaccountUpdate, - ) } type AssetsKeeper interface { diff --git a/protocol/x/subaccounts/keeper/subaccount.go b/protocol/x/subaccounts/keeper/subaccount.go index e72ccd60e7..52d4c05bb3 100644 --- a/protocol/x/subaccounts/keeper/subaccount.go +++ b/protocol/x/subaccounts/keeper/subaccount.go @@ -825,19 +825,3 @@ func (k Keeper) GetAllRelevantPerpetuals( func (k Keeper) GetFullNodeStreamingManager() streamingtypes.FullNodeStreamingManager { return k.streamingManager } - -// SendFinalizedSubaccountUpdates sends the subaccount updates to the gRPC streaming manager. -func (k Keeper) SendFinalizedSubaccountUpdates( - ctx sdk.Context, - subaccountUpdates []types.StreamSubaccountUpdate, -) { - lib.AssertDeliverTxMode(ctx) - if len(subaccountUpdates) == 0 { - return - } - k.GetFullNodeStreamingManager().SendFinalizedSubaccountUpdates( - subaccountUpdates, - lib.MustConvertIntegerToUint32(ctx.BlockHeight()), - ctx.ExecMode(), - ) -} diff --git a/protocol/x/subaccounts/types/types.go b/protocol/x/subaccounts/types/types.go index cbccc9d2b9..3e180d05c3 100644 --- a/protocol/x/subaccounts/types/types.go +++ b/protocol/x/subaccounts/types/types.go @@ -77,8 +77,4 @@ type SubaccountsKeeper interface { perpetualId uint32, blockHeight uint32, ) error - SendFinalizedSubaccountUpdates( - ctx sdk.Context, - subaccountUpdates []StreamSubaccountUpdate, - ) } From ca6870dc7b6076893fe64ed1208867beb833fd0e Mon Sep 17 00:00:00 2001 From: Jonathan Fung <121899091+jonfung-dydx@users.noreply.github.com> Date: Fri, 18 Oct 2024 12:42:05 -0400 Subject: [PATCH 31/33] FNS polish - metrics, max msg size, default flag values (#2517) --- protocol/app/flags/flags.go | 4 +-- protocol/app/flags/flags_test.go | 16 +++++----- protocol/lib/metrics/metric_keys.go | 4 +-- .../streaming/full_node_streaming_manager.go | 31 +++++++++++++++---- protocol/streaming/ws/websocket_server.go | 3 ++ 5 files changed, 40 insertions(+), 18 deletions(-) diff --git a/protocol/app/flags/flags.go b/protocol/app/flags/flags.go index aadc805696..b166f88352 100644 --- a/protocol/app/flags/flags.go +++ b/protocol/app/flags/flags.go @@ -72,8 +72,8 @@ const ( DefaultGrpcStreamingEnabled = false DefaultGrpcStreamingFlushIntervalMs = 50 - DefaultGrpcStreamingMaxBatchSize = 2000 - DefaultGrpcStreamingMaxChannelBufferSize = 2000 + DefaultGrpcStreamingMaxBatchSize = 10000 + DefaultGrpcStreamingMaxChannelBufferSize = 10000 DefaultWebsocketStreamingEnabled = false DefaultWebsocketStreamingPort = 9092 DefaultFullNodeStreamingSnapshotInterval = 0 diff --git a/protocol/app/flags/flags_test.go b/protocol/app/flags/flags_test.go index 0ad03872fa..8b3ed1a8f4 100644 --- a/protocol/app/flags/flags_test.go +++ b/protocol/app/flags/flags_test.go @@ -89,8 +89,8 @@ func TestValidate(t *testing.T) { GrpcEnable: true, GrpcStreamingEnabled: true, GrpcStreamingFlushIntervalMs: 100, - GrpcStreamingMaxBatchSize: 2000, - GrpcStreamingMaxChannelBufferSize: 2000, + GrpcStreamingMaxBatchSize: 10000, + GrpcStreamingMaxChannelBufferSize: 10000, WebsocketStreamingEnabled: false, }, }, @@ -100,8 +100,8 @@ func TestValidate(t *testing.T) { GrpcEnable: true, GrpcStreamingEnabled: true, GrpcStreamingFlushIntervalMs: 100, - GrpcStreamingMaxBatchSize: 2000, - GrpcStreamingMaxChannelBufferSize: 2000, + GrpcStreamingMaxBatchSize: 10000, + GrpcStreamingMaxChannelBufferSize: 10000, WebsocketStreamingEnabled: true, WebsocketStreamingPort: 8989, }, @@ -119,9 +119,9 @@ func TestValidate(t *testing.T) { GrpcEnable: true, GrpcStreamingEnabled: true, OptimisticExecutionEnabled: true, - GrpcStreamingMaxBatchSize: 2000, + GrpcStreamingMaxBatchSize: 10000, GrpcStreamingFlushIntervalMs: 100, - GrpcStreamingMaxChannelBufferSize: 2000, + GrpcStreamingMaxChannelBufferSize: 10000, WebsocketStreamingPort: 8989, }, }, @@ -257,8 +257,8 @@ func TestGetFlagValuesFromOptions(t *testing.T) { expectedGrpcEnable: true, expectedGrpcStreamingEnable: false, expectedGrpcStreamingFlushMs: 50, - expectedGrpcStreamingBatchSize: 2000, - expectedGrpcStreamingMaxChannelBufferSize: 2000, + expectedGrpcStreamingBatchSize: 10000, + expectedGrpcStreamingMaxChannelBufferSize: 10000, expectedWebsocketEnabled: false, expectedWebsocketPort: 9092, expectedFullNodeStreamingSnapshotInterval: 0, diff --git a/protocol/lib/metrics/metric_keys.go b/protocol/lib/metrics/metric_keys.go index ac8e8a17d1..ead248f516 100644 --- a/protocol/lib/metrics/metric_keys.go +++ b/protocol/lib/metrics/metric_keys.go @@ -69,9 +69,8 @@ const ( FullNodeGrpc = "full_node_grpc" GrpcSendOrderbookUpdatesLatency = "grpc_send_orderbook_updates_latency" GrpcSendOrderbookSnapshotLatency = "grpc_send_orderbook_snapshot_latency" - GrpcSendSubaccountSnapshotLatency = "grpc_send_subaccount_snapshot_latency" + GrpcSendSubaccountUpdateCount = "grpc_send_subaccount_update_count" GrpcSendOrderbookFillsLatency = "grpc_send_orderbook_fills_latency" - GrpcSendFinalizedSubaccountUpdatesLatency = "grpc_send_finalized_subaccount_updates_latency" GrpcAddUpdateToBufferCount = "grpc_add_update_to_buffer_count" GrpcAddToSubscriptionChannelCount = "grpc_add_to_subscription_channel_count" GrpcSendResponseToSubscriberCount = "grpc_send_response_to_subscriber_count" @@ -82,6 +81,7 @@ const ( GrpcStagedAllFinalizeBlockUpdatesCount = "grpc_staged_all_finalize_block_updates_count" GrpcStagedFillFinalizeBlockUpdatesCount = "grpc_staged_finalize_block_fill_updates_count" GrpcStagedSubaccountFinalizeBlockUpdatesCount = "grpc_staged_finalize_block_subaccount_updates_count" + SubscriptionId = "subscription_id" EndBlocker = "end_blocker" EndBlockerLag = "end_blocker_lag" diff --git a/protocol/streaming/full_node_streaming_manager.go b/protocol/streaming/full_node_streaming_manager.go index f63beac347..6cf24bd0b5 100644 --- a/protocol/streaming/full_node_streaming_manager.go +++ b/protocol/streaming/full_node_streaming_manager.go @@ -153,7 +153,7 @@ func (sm *FullNodeStreamingManagerImpl) Enabled() bool { } func (sm *FullNodeStreamingManagerImpl) EmitMetrics() { - metrics.SetGauge( + metrics.AddSample( metrics.GrpcStreamNumUpdatesBuffered, float32(len(sm.streamUpdateCache)), ) @@ -162,9 +162,10 @@ func (sm *FullNodeStreamingManagerImpl) EmitMetrics() { float32(len(sm.orderbookSubscriptions)), ) for _, subscription := range sm.orderbookSubscriptions { - metrics.AddSample( + metrics.AddSampleWithLabels( metrics.GrpcSubscriptionChannelLength, float32(len(subscription.updatesChannel)), + metrics.GetLabelForIntValue(metrics.SubscriptionId, int(subscription.subscriptionId)), ) } } @@ -234,9 +235,10 @@ func (sm *FullNodeStreamingManagerImpl) Subscribe( // Use current goroutine to consistently poll subscription channel for updates // to send through stream. for updates := range subscription.updatesChannel { - metrics.IncrCounter( + metrics.IncrCounterWithLabels( metrics.GrpcSendResponseToSubscriberCount, 1, + metrics.GetLabelForIntValue(metrics.SubscriptionId, int(subscription.subscriptionId)), ) err = subscription.messageSender.Send( &clobtypes.StreamOrderbookUpdatesResponse{ @@ -372,9 +374,17 @@ func (sm *FullNodeStreamingManagerImpl) sendStreamUpdates( return } + metrics.IncrCounterWithLabels( + metrics.GrpcAddToSubscriptionChannelCount, + 1, + metrics.GetLabelForIntValue(metrics.SubscriptionId, int(subscriptionId)), + ) + select { case subscription.updatesChannel <- streamUpdates: default: + // Buffer is full. Emit metric and drop subscription. + sm.EmitMetrics() sm.logger.Error( fmt.Sprintf( "Streaming subscription id %+v channel full capacity. Dropping subscription connection.", @@ -399,6 +409,11 @@ func (sm *FullNodeStreamingManagerImpl) SendSubaccountUpdate( return } + metrics.IncrCounter( + metrics.GrpcSendSubaccountUpdateCount, + 1, + ) + // If `DeliverTx`, updates should be staged to be streamed after consensus finalizes on a block. stagedEvent := clobtypes.StagedFinalizeBlockEvent{ Event: &clobtypes.StagedFinalizeBlockEvent_SubaccountUpdate{ @@ -705,9 +720,9 @@ func (sm *FullNodeStreamingManagerImpl) AddOrderUpdatesToCache( sm.cacheStreamUpdatesByClobPairWithLock(updates, clobPairIds) + sm.EmitMetrics() // Remove all subscriptions and wipe the buffer if buffer overflows. sm.RemoveSubscriptionsAndClearBufferIfFull() - sm.EmitMetrics() } // AddSubaccountUpdatesToCache adds a series of updates to the full node streaming cache. @@ -726,8 +741,8 @@ func (sm *FullNodeStreamingManagerImpl) AddSubaccountUpdatesToCache( sm.cacheStreamUpdatesBySubaccountWithLock(updates, subaccountIds) - sm.RemoveSubscriptionsAndClearBufferIfFull() sm.EmitMetrics() + sm.RemoveSubscriptionsAndClearBufferIfFull() } // RemoveSubscriptionsAndClearBufferIfFull removes all subscriptions and wipes the buffer if buffer overflows. @@ -743,6 +758,7 @@ func (sm *FullNodeStreamingManagerImpl) RemoveSubscriptionsAndClearBufferIfFull( } sm.streamUpdateCache = nil sm.streamUpdateSubscriptionCache = nil + sm.EmitMetrics() } } @@ -778,13 +794,16 @@ func (sm *FullNodeStreamingManagerImpl) FlushStreamUpdatesWithLock() { // If the buffer is full, drop the subscription. for id, updates := range subscriptionUpdates { if subscription, ok := sm.orderbookSubscriptions[id]; ok { - metrics.IncrCounter( + metrics.IncrCounterWithLabels( metrics.GrpcAddToSubscriptionChannelCount, 1, + metrics.GetLabelForIntValue(metrics.SubscriptionId, int(id)), ) select { case subscription.updatesChannel <- updates: default: + // Buffer is full. Emit metric and drop subscription. + sm.EmitMetrics() idsToRemove = append(idsToRemove, id) } } diff --git a/protocol/streaming/ws/websocket_server.go b/protocol/streaming/ws/websocket_server.go index 0b66804595..c94e84208e 100644 --- a/protocol/streaming/ws/websocket_server.go +++ b/protocol/streaming/ws/websocket_server.go @@ -56,6 +56,9 @@ func (ws *WebsocketServer) Handler(w http.ResponseWriter, r *http.Request) { } defer conn.Close() + // Set ws max message size to 10 mb. + conn.SetReadLimit(10 * 1024 * 1024) + // Parse clobPairIds from query parameters clobPairIds, err := parseClobPairIds(r) if err != nil { From dd4629a1b2ad18f301bb8bca15f2dda53a164208 Mon Sep 17 00:00:00 2001 From: Jonathan Fung <121899091+jonfung-dydx@users.noreply.github.com> Date: Fri, 18 Oct 2024 16:49:41 -0400 Subject: [PATCH 32/33] reusing subscription ids (#2518) --- .../streaming/full_node_streaming_manager.go | 26 ++++++++++++++----- 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/protocol/streaming/full_node_streaming_manager.go b/protocol/streaming/full_node_streaming_manager.go index 6cf24bd0b5..af0d5671cd 100644 --- a/protocol/streaming/full_node_streaming_manager.go +++ b/protocol/streaming/full_node_streaming_manager.go @@ -35,7 +35,7 @@ type FullNodeStreamingManagerImpl struct { // orderbookSubscriptions maps subscription IDs to their respective orderbook subscriptions. orderbookSubscriptions map[uint32]*OrderbookSubscription - nextSubscriptionId uint32 + activeSubscriptionIds map[uint32]bool // stream will batch and flush out messages every 10 ms. ticker *time.Ticker @@ -106,7 +106,7 @@ func NewFullNodeStreamingManager( fullNodeStreamingManager := &FullNodeStreamingManagerImpl{ logger: logger, orderbookSubscriptions: make(map[uint32]*OrderbookSubscription), - nextSubscriptionId: 0, + activeSubscriptionIds: make(map[uint32]bool), ticker: time.NewTicker(time.Duration(flushIntervalMs) * time.Millisecond), done: make(chan bool), @@ -170,6 +170,16 @@ func (sm *FullNodeStreamingManagerImpl) EmitMetrics() { } } +// getNextAvailableSubscriptionId returns next available subscription id. Assumes the +// lock has been acquired. +func (sm *FullNodeStreamingManagerImpl) getNextAvailableSubscriptionId() uint32 { + id := uint32(0) + for _, inUse := sm.activeSubscriptionIds[id]; inUse; _, inUse = sm.activeSubscriptionIds[id] { + id = id + uint32(1) + } + return id +} + // Subscribe subscribes to the orderbook updates stream. func (sm *FullNodeStreamingManagerImpl) Subscribe( clobPairIds []uint32, @@ -188,8 +198,11 @@ func (sm *FullNodeStreamingManagerImpl) Subscribe( for i, subaccountId := range subaccountIds { sIds[i] = *subaccountId } + + subscriptionId := sm.getNextAvailableSubscriptionId() + subscription := &OrderbookSubscription{ - subscriptionId: sm.nextSubscriptionId, + subscriptionId: subscriptionId, initialized: &atomic.Bool{}, // False by default. clobPairIds: clobPairIds, subaccountIds: sIds, @@ -204,7 +217,7 @@ func (sm *FullNodeStreamingManagerImpl) Subscribe( } sm.clobPairIdToSubscriptionIdMapping[clobPairId] = append( sm.clobPairIdToSubscriptionIdMapping[clobPairId], - sm.nextSubscriptionId, + subscription.subscriptionId, ) } for _, subaccountId := range sIds { @@ -215,7 +228,7 @@ func (sm *FullNodeStreamingManagerImpl) Subscribe( } sm.subaccountIdToSubscriptionIdMapping[subaccountId] = append( sm.subaccountIdToSubscriptionIdMapping[subaccountId], - sm.nextSubscriptionId, + subscription.subscriptionId, ) } @@ -228,7 +241,7 @@ func (sm *FullNodeStreamingManagerImpl) Subscribe( ), ) sm.orderbookSubscriptions[subscription.subscriptionId] = subscription - sm.nextSubscriptionId++ + sm.activeSubscriptionIds[subscription.subscriptionId] = true sm.EmitMetrics() sm.Unlock() @@ -280,6 +293,7 @@ func (sm *FullNodeStreamingManagerImpl) removeSubscription( } close(subscription.updatesChannel) delete(sm.orderbookSubscriptions, subscriptionIdToRemove) + delete(sm.activeSubscriptionIds, subscriptionIdToRemove) // Iterate over the clobPairIdToSubscriptionIdMapping to remove the subscriptionIdToRemove for pairId, subscriptionIds := range sm.clobPairIdToSubscriptionIdMapping { From 3d4c575af18b8411eaf8f527eed1ad358318e508 Mon Sep 17 00:00:00 2001 From: Jonathan Fung <121899091+jonfung-dydx@users.noreply.github.com> Date: Mon, 21 Oct 2024 13:56:53 -0400 Subject: [PATCH 33/33] FNS - bounds check raw subscription payload int vals (#2523) --- protocol/streaming/ws/websocket_server.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/protocol/streaming/ws/websocket_server.go b/protocol/streaming/ws/websocket_server.go index c94e84208e..ba4477d03b 100644 --- a/protocol/streaming/ws/websocket_server.go +++ b/protocol/streaming/ws/websocket_server.go @@ -3,6 +3,7 @@ package ws import ( "context" "fmt" + "math" "net/http" "strconv" "strings" @@ -122,6 +123,10 @@ func parseSubaccountIds(r *http.Request) ([]*satypes.SubaccountId, error) { return nil, fmt.Errorf("invalid subaccount number: %s, expected subaccount_id format: owner/number", parts[1]) } + if number < 0 || number > math.MaxInt32 { + return nil, fmt.Errorf("invalid subaccount number: %s", parts[1]) + } + subaccountIds = append(subaccountIds, &satypes.SubaccountId{ Owner: parts[0], Number: uint32(number), @@ -144,6 +149,9 @@ func parseClobPairIds(r *http.Request) ([]uint32, error) { if err != nil { return nil, fmt.Errorf("invalid clobPairId: %s", idStr) } + if id < 0 || id > math.MaxInt32 { + return nil, fmt.Errorf("invalid clob pair id: %s", idStr) + } clobPairIds = append(clobPairIds, uint32(id)) }