From 4badf3af02f7d13a87ee33910f33d69fb79717d0 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Tue, 13 Jun 2023 15:35:29 +0200 Subject: [PATCH 01/38] chore: simplify debug message broadcasting --- app/client/cli/debug.go | 30 +++++++++--------------------- 1 file changed, 9 insertions(+), 21 deletions(-) diff --git a/app/client/cli/debug.go b/app/client/cli/debug.go index f23a278d7..e0d25e58d 100644 --- a/app/client/cli/debug.go +++ b/app/client/cli/debug.go @@ -76,7 +76,7 @@ func NewDebugCommand() *cobra.Command { } } -func runDebug(cmd *cobra.Command, args []string) (err error) { +func runDebug(cmd *cobra.Command, _ []string) (err error) { for { if selection, err := promptGetInput(); err == nil { handleSelect(cmd, selection) @@ -164,32 +164,17 @@ func handleSelect(cmd *cobra.Command, selection string) { } } -// Broadcast to the entire validator set -func broadcastDebugMessage(cmd *cobra.Command, debugMsg *messaging.DebugMessage) { +// Broadcast to the entire network. +func broadcastDebugMessage(_ *cobra.Command, debugMsg *messaging.DebugMessage) { anyProto, err := anypb.New(debugMsg) if err != nil { logger.Global.Fatal().Err(err).Msg("Failed to create Any proto") } - // TODO(olshansky): Once we implement the cleanup layer in RainTree, we'll be able to use - // broadcast. The reason it cannot be done right now is because this client is not in the - // address book of the actual validator nodes, so `validator1` never receives the message. - // p2pMod.Broadcast(anyProto) - - pstore, err := fetchPeerstore(cmd) - if err != nil { - logger.Global.Fatal().Err(err).Msg("Unable to retrieve the pstore") + // TECHDEBT: prefer to retrieve P2P module from the bus instead. + if err := helpers.P2PMod.Broadcast(anyProto); err != nil { + logger.Global.Error().Err(err).Msg("Failed to broadcast debug message") } - for _, val := range pstore.GetPeerList() { - addr := val.GetAddress() - if err != nil { - logger.Global.Fatal().Err(err).Msg("Failed to convert validator address into pocketCrypto.Address") - } - if err := helpers.P2PMod.Send(addr, anyProto); err != nil { - logger.Global.Error().Err(err).Msg("Failed to send debug message") - } - } - } // Send to just a single (i.e. first) validator in the set @@ -210,6 +195,9 @@ func sendDebugMessage(cmd *cobra.Command, debugMsg *messaging.DebugMessage) { } // if the message needs to be broadcast, it'll be handled by the business logic of the message handler + // + // DISCUSS_THIS_COMMIT: The statement above is false. Using `#Send()` will only + // be unicast with no opportunity for further propagation. validatorAddress = pstore.GetPeerList()[0].GetAddress() if err != nil { logger.Global.Fatal().Err(err).Msg("Failed to convert validator address into pocketCrypto.Address") From 2b83d32e8eb107fd02fbc9809c97a462ff1e49b9 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Fri, 7 Jul 2023 11:16:49 +0200 Subject: [PATCH 02/38] refactor: CLI config parsing --- app/client/cli/cmd.go | 18 ++++-------------- app/client/cli/flags/config.go | 19 +++++++++++++++++++ app/client/cli/helpers/setup.go | 5 +++++ app/client/cli/utils.go | 2 +- 4 files changed, 29 insertions(+), 15 deletions(-) create mode 100644 app/client/cli/flags/config.go diff --git a/app/client/cli/cmd.go b/app/client/cli/cmd.go index 8e3f975bb..160360ac1 100644 --- a/app/client/cli/cmd.go +++ b/app/client/cli/cmd.go @@ -8,7 +8,6 @@ import ( "github.com/spf13/viper" "github.com/pokt-network/pocket/app/client/cli/flags" - "github.com/pokt-network/pocket/runtime/configs" "github.com/pokt-network/pocket/runtime/defaults" ) @@ -17,8 +16,6 @@ const ( flagBindErrFormat = "could not bind flag %q: %v" ) -var cfg *configs.Config - func init() { rootCmd.PersistentFlags().StringVar(&flags.RemoteCLIURL, "remote_cli_url", defaults.DefaultRemoteCLIURL, "takes a remote endpoint in the form of ://: (uses RPC Port)") // ensure that this flag can be overridden by the respective viper-conventional environment variable (i.e. `POCKET_REMOTE_CLI_URL`) @@ -42,17 +39,10 @@ func init() { } var rootCmd = &cobra.Command{ - Use: cliExecutableName, - Short: "Pocket Network Command Line Interface (CLI)", - Long: "The CLI is meant to be an user but also a machine friendly way for interacting with Pocket Network.", - PersistentPreRunE: func(cmd *cobra.Command, args []string) error { - // by this time, the config path should be set - cfg = configs.ParseConfig(flags.ConfigPath) - - // set final `remote_cli_url` value; order of precedence: flag > env var > config > default - flags.RemoteCLIURL = viper.GetString("remote_cli_url") - return nil - }, + Use: cliExecutableName, + Short: "Pocket Network Command Line Interface (CLI)", + Long: "The CLI is meant to be an user but also a machine friendly way for interacting with Pocket Network.", + PersistentPreRunE: flags.ParseConfigAndFlags, } func ExecuteContext(ctx context.Context) error { diff --git a/app/client/cli/flags/config.go b/app/client/cli/flags/config.go new file mode 100644 index 000000000..0de55a7aa --- /dev/null +++ b/app/client/cli/flags/config.go @@ -0,0 +1,19 @@ +package flags + +import ( + "github.com/spf13/cobra" + "github.com/spf13/viper" + + "github.com/pokt-network/pocket/runtime/configs" +) + +var Cfg *configs.Config + +func ParseConfigAndFlags(_ *cobra.Command, _ []string) error { + // by this time, the config path should be set + Cfg = configs.ParseConfig(ConfigPath) + + // set final `remote_cli_url` value; order of precedence: flag > env var > config > default + RemoteCLIURL = viper.GetString("remote_cli_url") + return nil +} diff --git a/app/client/cli/helpers/setup.go b/app/client/cli/helpers/setup.go index 956102a04..350bcbf23 100644 --- a/app/client/cli/helpers/setup.go +++ b/app/client/cli/helpers/setup.go @@ -17,6 +17,11 @@ import ( // P2PDependenciesPreRunE initializes peerstore & current height providers, and a // p2p module which consumes them. Everything is registered to the bus. func P2PDependenciesPreRunE(cmd *cobra.Command, _ []string) error { + if err := flags.ParseConfigAndFlags(nil, nil); err != nil { + return err + } + + // TODO_THIS_COMMIT: figure out what to do with this vvv // TECHDEBT: this is to keep backwards compatibility with localnet flags.ConfigPath = runtime.GetEnv("CONFIG_PATH", "build/config/config.validator1.json") diff --git a/app/client/cli/utils.go b/app/client/cli/utils.go index 2e3e0263e..36a7229e6 100644 --- a/app/client/cli/utils.go +++ b/app/client/cli/utils.go @@ -311,7 +311,7 @@ func attachKeybaseFlagsToSubcommands() []cmdOption { } func keybaseForCLI() (keybase.Keybase, error) { - return keybase.NewKeybase(cfg.Keybase) + return keybase.NewKeybase(flags.Cfg.Keybase) } func unableToConnectToRpc(err error) error { From eac769509dd9ad5ff85019678b6c54fa5ca08a0d Mon Sep 17 00:00:00 2001 From: Bryan White Date: Fri, 7 Jul 2023 11:16:59 +0200 Subject: [PATCH 03/38] refactor: common CLI helpers --- app/client/cli/debug.go | 47 ++----------------------------- app/client/cli/helpers/common.go | 48 ++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 45 deletions(-) diff --git a/app/client/cli/debug.go b/app/client/cli/debug.go index e0d25e58d..93408a368 100644 --- a/app/client/cli/debug.go +++ b/app/client/cli/debug.go @@ -1,8 +1,6 @@ package cli import ( - "errors" - "fmt" "os" "github.com/manifoldco/promptui" @@ -11,10 +9,7 @@ import ( "github.com/pokt-network/pocket/app/client/cli/helpers" "github.com/pokt-network/pocket/logger" - "github.com/pokt-network/pocket/p2p/providers/peerstore_provider" - typesP2P "github.com/pokt-network/pocket/p2p/types" "github.com/pokt-network/pocket/shared/messaging" - "github.com/pokt-network/pocket/shared/modules" ) // TECHDEBT: Lowercase variables / constants that do not need to be exported. @@ -184,7 +179,7 @@ func sendDebugMessage(cmd *cobra.Command, debugMsg *messaging.DebugMessage) { logger.Global.Error().Err(err).Msg("Failed to create Any proto") } - pstore, err := fetchPeerstore(cmd) + pstore, err := helpers.FetchPeerstore(cmd) if err != nil { logger.Global.Fatal().Err(err).Msg("Unable to retrieve the pstore") } @@ -203,46 +198,8 @@ func sendDebugMessage(cmd *cobra.Command, debugMsg *messaging.DebugMessage) { logger.Global.Fatal().Err(err).Msg("Failed to convert validator address into pocketCrypto.Address") } + // TECHDEBT: prefer to retrieve P2P module from the bus instead. if err := helpers.P2PMod.Send(validatorAddress, anyProto); err != nil { logger.Global.Error().Err(err).Msg("Failed to send debug message") } } - -// fetchPeerstore retrieves the providers from the CLI context and uses them to retrieve the address book for the current height -func fetchPeerstore(cmd *cobra.Command) (typesP2P.Peerstore, error) { - bus, ok := helpers.GetValueFromCLIContext[modules.Bus](cmd, helpers.BusCLICtxKey) - if !ok || bus == nil { - return nil, errors.New("retrieving bus from CLI context") - } - // TECHDEBT(#810, #811): use `bus.GetPeerstoreProvider()` after peerstore provider - // is retrievable as a proper submodule - pstoreProvider, err := bus.GetModulesRegistry().GetModule(peerstore_provider.PeerstoreProviderSubmoduleName) - if err != nil { - return nil, errors.New("retrieving peerstore provider") - } - currentHeightProvider := bus.GetCurrentHeightProvider() - - height := currentHeightProvider.CurrentHeight() - pstore, err := pstoreProvider.(peerstore_provider.PeerstoreProvider).GetStakedPeerstoreAtHeight(height) - if err != nil { - return nil, fmt.Errorf("retrieving peerstore at height %d", height) - } - // Inform the client's main P2P that a the blockchain is at a new height so it can, if needed, update its view of the validator set - err = sendConsensusNewHeightEventToP2PModule(height, bus) - if err != nil { - return nil, errors.New("sending consensus new height event") - } - return pstore, nil -} - -// sendConsensusNewHeightEventToP2PModule mimicks the consensus module sending a ConsensusNewHeightEvent to the p2p module -// This is necessary because the debug client is not a validator and has no consensus module but it has to update the peerstore -// depending on the changes in the validator set. -// TODO(#613): Make the debug client mimic a full node. -func sendConsensusNewHeightEventToP2PModule(height uint64, bus modules.Bus) error { - newHeightEvent, err := messaging.PackMessage(&messaging.ConsensusNewHeightEvent{Height: height}) - if err != nil { - logger.Global.Fatal().Err(err).Msg("Failed to pack consensus new height event") - } - return bus.GetP2PModule().HandleEvent(newHeightEvent.Content) -} diff --git a/app/client/cli/helpers/common.go b/app/client/cli/helpers/common.go index b9f6d547b..be9d75250 100644 --- a/app/client/cli/helpers/common.go +++ b/app/client/cli/helpers/common.go @@ -1,7 +1,16 @@ package helpers import ( + "errors" + "fmt" + + "github.com/spf13/cobra" + + "github.com/pokt-network/pocket/logger" + "github.com/pokt-network/pocket/p2p/providers/peerstore_provider" + "github.com/pokt-network/pocket/p2p/types" "github.com/pokt-network/pocket/runtime" + "github.com/pokt-network/pocket/shared/messaging" "github.com/pokt-network/pocket/shared/modules" ) @@ -10,5 +19,44 @@ var ( genesisPath = runtime.GetEnv("GENESIS_PATH", "build/config/genesis.json") // P2PMod is initialized in order to broadcast a message to the local network + // TECHDEBT: prefer to retrieve P2P module from the bus instead. P2PMod modules.P2PModule ) + +// fetchPeerstore retrieves the providers from the CLI context and uses them to retrieve the address book for the current height +func FetchPeerstore(cmd *cobra.Command) (types.Peerstore, error) { + bus, ok := GetValueFromCLIContext[modules.Bus](cmd, BusCLICtxKey) + if !ok || bus == nil { + return nil, errors.New("retrieving bus from CLI context") + } + // TECHDEBT(#810, #811): use `bus.GetPeerstoreProvider()` after peerstore provider + // is retrievable as a proper submodule + pstoreProvider, err := bus.GetModulesRegistry().GetModule(peerstore_provider.PeerstoreProviderSubmoduleName) + if err != nil { + return nil, errors.New("retrieving peerstore provider") + } + currentHeightProvider := bus.GetCurrentHeightProvider() + height := currentHeightProvider.CurrentHeight() + pstore, err := pstoreProvider.(peerstore_provider.PeerstoreProvider).GetStakedPeerstoreAtHeight(height) + if err != nil { + return nil, fmt.Errorf("retrieving peerstore at height %d", height) + } + // Inform the client's main P2P that a the blockchain is at a new height so it can, if needed, update its view of the validator set + err = sendConsensusNewHeightEventToP2PModule(height, bus) + if err != nil { + return nil, errors.New("sending consensus new height event") + } + return pstore, nil +} + +// sendConsensusNewHeightEventToP2PModule mimicks the consensus module sending a ConsensusNewHeightEvent to the p2p module +// This is necessary because the debug client is not a validator and has no consensus module but it has to update the peerstore +// depending on the changes in the validator set. +// TODO(#613): Make the debug client mimic a full node. +func sendConsensusNewHeightEventToP2PModule(height uint64, bus modules.Bus) error { + newHeightEvent, err := messaging.PackMessage(&messaging.ConsensusNewHeightEvent{Height: height}) + if err != nil { + logger.Global.Fatal().Err(err).Msg("Failed to pack consensus new height event") + } + return bus.GetP2PModule().HandleEvent(newHeightEvent.Content) +} From bca000b347e83ce75a9a1694c5ddcf0f564ee72c Mon Sep 17 00:00:00 2001 From: Bryan White Date: Tue, 11 Jul 2023 09:28:28 +0200 Subject: [PATCH 04/38] chore: add `GetBusFromCmd()` CLI helper --- app/client/cli/helpers/common.go | 6 +++--- app/client/cli/helpers/context.go | 14 ++++++++++++++ 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/app/client/cli/helpers/common.go b/app/client/cli/helpers/common.go index be9d75250..b89f79ba9 100644 --- a/app/client/cli/helpers/common.go +++ b/app/client/cli/helpers/common.go @@ -25,9 +25,9 @@ var ( // fetchPeerstore retrieves the providers from the CLI context and uses them to retrieve the address book for the current height func FetchPeerstore(cmd *cobra.Command) (types.Peerstore, error) { - bus, ok := GetValueFromCLIContext[modules.Bus](cmd, BusCLICtxKey) - if !ok || bus == nil { - return nil, errors.New("retrieving bus from CLI context") + bus, err := GetBusFromCmd(cmd) + if err != nil { + return nil, err } // TECHDEBT(#810, #811): use `bus.GetPeerstoreProvider()` after peerstore provider // is retrievable as a proper submodule diff --git a/app/client/cli/helpers/context.go b/app/client/cli/helpers/context.go index f9f3f4549..f1494c5ac 100644 --- a/app/client/cli/helpers/context.go +++ b/app/client/cli/helpers/context.go @@ -2,12 +2,17 @@ package helpers import ( "context" + "fmt" "github.com/spf13/cobra" + + "github.com/pokt-network/pocket/shared/modules" ) const BusCLICtxKey cliContextKey = "bus" +var ErrCxtFromBus = fmt.Errorf("could not get context from bus") + // NOTE: this is required by the linter, otherwise a simple string constant would have been enough type cliContextKey string @@ -19,3 +24,12 @@ func GetValueFromCLIContext[T any](cmd *cobra.Command, key cliContextKey) (T, bo value, ok := cmd.Context().Value(key).(T) return value, ok } + +func GetBusFromCmd(cmd *cobra.Command) (modules.Bus, error) { + bus, ok := GetValueFromCLIContext[modules.Bus](cmd, BusCLICtxKey) + if !ok { + return nil, ErrCxtFromBus + } + + return bus, nil +} From 5e963bef8caa704110163d6ad470fdb0a97a7e56 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Tue, 11 Jul 2023 09:28:32 +0200 Subject: [PATCH 05/38] chore: consistent debug CLI identity --- app/client/cli/helpers/setup.go | 6 +++++- runtime/manager.go | 1 + 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/app/client/cli/helpers/setup.go b/app/client/cli/helpers/setup.go index 350bcbf23..6f5a8af2c 100644 --- a/app/client/cli/helpers/setup.go +++ b/app/client/cli/helpers/setup.go @@ -14,6 +14,10 @@ import ( "github.com/pokt-network/pocket/shared/modules" ) +// TODO_THIS_COMMIT: add godoc comment explaining what this **is** and **is not** +// intended to be used for. +const debugPrivKey = "09fc8ee114e678e665d09179acb9a30060f680df44ba06b51434ee47940a8613be19b2b886e743eb1ff7880968d6ce1a46350315e569243e747a227ee8faec3d" + // P2PDependenciesPreRunE initializes peerstore & current height providers, and a // p2p module which consumes them. Everything is registered to the bus. func P2PDependenciesPreRunE(cmd *cobra.Command, _ []string) error { @@ -28,7 +32,7 @@ func P2PDependenciesPreRunE(cmd *cobra.Command, _ []string) error { runtimeMgr := runtime.NewManagerFromFiles( flags.ConfigPath, genesisPath, runtime.WithClientDebugMode(), - runtime.WithRandomPK(), + runtime.WithPK(debugPrivKey), ) bus := runtimeMgr.GetBus() diff --git a/runtime/manager.go b/runtime/manager.go index 151f2c198..7c182f34f 100644 --- a/runtime/manager.go +++ b/runtime/manager.go @@ -104,6 +104,7 @@ func WithRandomPK() func(*Manager) { return WithPK(privateKey.String()) } +// TECHDEBT(#750): separate conseneus and P2P keys. func WithPK(pk string) func(*Manager) { return func(b *Manager) { if b.config.Consensus == nil { From a883f088f375d44ed70f7c9f81a0e53c398e0ac5 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Thu, 6 Jul 2023 11:56:15 +0200 Subject: [PATCH 06/38] chore: add `enable_peer_discovery_debug_rpc` to P2P config --- charts/pocket/templates/statefulset.yaml | 2 ++ charts/pocket/values.yaml | 1 + runtime/configs/proto/p2p_config.proto | 1 + 3 files changed, 4 insertions(+) diff --git a/charts/pocket/templates/statefulset.yaml b/charts/pocket/templates/statefulset.yaml index 4be68b50b..782d69a94 100644 --- a/charts/pocket/templates/statefulset.yaml +++ b/charts/pocket/templates/statefulset.yaml @@ -100,6 +100,8 @@ spec: valueFrom: fieldRef: fieldPath: status.podIP + - name: POCKET_P2P_ENABLE_PEER_DISCOVERY_DEBUG_RPC + value: "true" livenessProbe: httpGet: path: /v1/health diff --git a/charts/pocket/values.yaml b/charts/pocket/values.yaml index 01d42b52e..afa383c2f 100644 --- a/charts/pocket/values.yaml +++ b/charts/pocket/values.yaml @@ -101,6 +101,7 @@ config: is_empty_connection_type: false private_key: "" # @ignored This value is needed but ignored - use privateKeySecretKeyRef instead max_mempool_count: 100000 + enable_peer_discovery_debug_rpc: false telemetry: enabled: true address: 0.0.0.0:9000 diff --git a/runtime/configs/proto/p2p_config.proto b/runtime/configs/proto/p2p_config.proto index e01cea09a..2661cc773 100644 --- a/runtime/configs/proto/p2p_config.proto +++ b/runtime/configs/proto/p2p_config.proto @@ -14,4 +14,5 @@ message P2PConfig { uint64 max_nonces = 5; // used to limit the number of nonces that can be stored before a FIFO mechanism is used to remove the oldest nonces and make space for the new ones bool is_client_only = 6; string bootstrap_nodes_csv = 7; // string in the format "http://somenode:50832,http://someothernode:50832". Refer to `p2p/module_test.go` for additional details. + bool enable_peer_discovery_debug_rpc = 8; } From 83c360400b8ae7547400b94b0fee2fd85fc9f27b Mon Sep 17 00:00:00 2001 From: Bryan White Date: Thu, 6 Jul 2023 11:48:57 +0200 Subject: [PATCH 07/38] chore: add P2P debug message handling support --- p2p/event_handler.go | 11 +++++++++++ p2p/types/errors.go | 5 +++-- shared/node.go | 7 ++++++- 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/p2p/event_handler.go b/p2p/event_handler.go index b1184e860..60e8afca4 100644 --- a/p2p/event_handler.go +++ b/p2p/event_handler.go @@ -87,9 +87,20 @@ func (m *p2pModule) HandleEvent(event *anypb.Any) error { } } + case messaging.DebugMessageEventType: + debugMessage, ok := evt.(*messaging.DebugMessage) + if !ok { + return fmt.Errorf("unexpected DebugMessage type: %T", evt) + } + + return m.handleDebugMessage(debugMessage) default: return fmt.Errorf("unknown event type: %s", event.MessageName()) } return nil } + +func (m *p2pModule) handleDebugMessage(msg *messaging.DebugMessage) error { + return nil +} diff --git a/p2p/types/errors.go b/p2p/types/errors.go index 8042ad775..3e225c304 100644 --- a/p2p/types/errors.go +++ b/p2p/types/errors.go @@ -6,8 +6,9 @@ import ( ) var ( - ErrUnknownPeer = errors.New("unknown peer") - ErrInvalidNonce = errors.New("invalid nonce") + ErrUnknownPeer = errors.New("unknown peer") + ErrInvalidNonce = errors.New("invalid nonce") + ErrPeerDiscoveryDebugRPCDisabled = errors.New("peer discovery debug RPC disabled") ) func ErrUnknownEventType(msg any) error { diff --git a/shared/node.go b/shared/node.go index f1e842382..b90130c94 100644 --- a/shared/node.go +++ b/shared/node.go @@ -181,7 +181,12 @@ func (node *Node) handleEvent(message *messaging.PocketEnvelope) error { case messaging.TxGossipMessageContentType: return node.GetBus().GetUtilityModule().HandleUtilityMessage(message.Content) case messaging.DebugMessageEventType: - return node.handleDebugMessage(message) + if err := node.GetBus().GetP2PModule().HandleEvent(message.Content); err != nil { + return err + } + if err := node.handleDebugMessage(message); err != nil { + return err + } case messaging.ConsensusNewHeightEventType: err_p2p := node.GetBus().GetP2PModule().HandleEvent(message.Content) err_ibc := node.GetBus().GetIBCModule().HandleEvent(message.Content) From 6ecca53d809f0aa2238aa9b8a41df731b00846d7 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Tue, 6 Jun 2023 09:16:24 +0200 Subject: [PATCH 08/38] feat: add `peer` subcommand --- app/client/cli/cmd.go | 3 +++ app/client/cli/peer/peer.go | 25 +++++++++++++++++++++++++ 2 files changed, 28 insertions(+) create mode 100644 app/client/cli/peer/peer.go diff --git a/app/client/cli/cmd.go b/app/client/cli/cmd.go index 160360ac1..95fc9b70f 100644 --- a/app/client/cli/cmd.go +++ b/app/client/cli/cmd.go @@ -8,6 +8,7 @@ import ( "github.com/spf13/viper" "github.com/pokt-network/pocket/app/client/cli/flags" + "github.com/pokt-network/pocket/app/client/cli/peer" "github.com/pokt-network/pocket/runtime/defaults" ) @@ -36,6 +37,8 @@ func init() { if err := viper.BindPFlag("verbose", rootCmd.PersistentFlags().Lookup("verbose")); err != nil { log.Fatalf(flagBindErrFormat, "verbose", err) } + + rootCmd.AddCommand(peer.PeerCmd) } var rootCmd = &cobra.Command{ diff --git a/app/client/cli/peer/peer.go b/app/client/cli/peer/peer.go new file mode 100644 index 000000000..45abb6da8 --- /dev/null +++ b/app/client/cli/peer/peer.go @@ -0,0 +1,25 @@ +package peer + +import ( + "github.com/spf13/cobra" + + "github.com/pokt-network/pocket/app/client/cli/helpers" +) + +var ( + allFlag, + stakedFlag, + unstakedFlag bool + + PeerCmd = &cobra.Command{ + Use: "peer", + Short: "Manage peers", + PersistentPreRunE: helpers.P2PDependenciesPreRunE, + } +) + +func init() { + PeerCmd.PersistentFlags().BoolVarP(&allFlag, "all", "a", false, "operations apply to both staked & unstaked router peerstores") + PeerCmd.PersistentFlags().BoolVarP(&stakedFlag, "staked", "s", false, "operations only apply to staked router peerstore (i.e. raintree)") + PeerCmd.PersistentFlags().BoolVarP(&unstakedFlag, "unstaked", "u", false, "operations only apply to unstaked router peerstore (i.e. gossipsub)") +} From d570b35c7e9c5067864dca4f9db521c1b67e0f5b Mon Sep 17 00:00:00 2001 From: Bryan White Date: Thu, 6 Jul 2023 11:45:31 +0200 Subject: [PATCH 09/38] chore: add peer `--local` persistent flag --- app/client/cli/peer/peer.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app/client/cli/peer/peer.go b/app/client/cli/peer/peer.go index 45abb6da8..1d9ddb2c8 100644 --- a/app/client/cli/peer/peer.go +++ b/app/client/cli/peer/peer.go @@ -9,7 +9,8 @@ import ( var ( allFlag, stakedFlag, - unstakedFlag bool + unstakedFlag, + localFlag bool PeerCmd = &cobra.Command{ Use: "peer", @@ -22,4 +23,5 @@ func init() { PeerCmd.PersistentFlags().BoolVarP(&allFlag, "all", "a", false, "operations apply to both staked & unstaked router peerstores") PeerCmd.PersistentFlags().BoolVarP(&stakedFlag, "staked", "s", false, "operations only apply to staked router peerstore (i.e. raintree)") PeerCmd.PersistentFlags().BoolVarP(&unstakedFlag, "unstaked", "u", false, "operations only apply to unstaked router peerstore (i.e. gossipsub)") + PeerCmd.PersistentFlags().BoolVarP(&localFlag, "local", "l", false, "operations apply to the local (CLI binary's) P2P module rather than being sent to the --remote_cli_url") } From e952365d9fd696194a76648b5b00d201ccbded5c Mon Sep 17 00:00:00 2001 From: Bryan White Date: Tue, 6 Jun 2023 09:17:34 +0200 Subject: [PATCH 10/38] feat: add `peer list` subcommand --- app/client/cli/peer/list.go | 107 +++++++++++++++++++++ app/client/cli/peer/peer.go | 4 +- p2p/debug.go | 28 ++++++ p2p/debug/list.go | 106 ++++++++++++++++++++ p2p/debug/peers.go | 62 ++++++++++++ p2p/event_handler.go | 5 - p2p/utils/logging.go | 72 +++++++++++++- shared/messaging/proto/debug_message.proto | 1 + 8 files changed, 377 insertions(+), 8 deletions(-) create mode 100644 app/client/cli/peer/list.go create mode 100644 p2p/debug.go create mode 100644 p2p/debug/list.go create mode 100644 p2p/debug/peers.go diff --git a/app/client/cli/peer/list.go b/app/client/cli/peer/list.go new file mode 100644 index 000000000..19b70b7e9 --- /dev/null +++ b/app/client/cli/peer/list.go @@ -0,0 +1,107 @@ +package peer + +import ( + "fmt" + + "github.com/spf13/cobra" + "google.golang.org/protobuf/types/known/anypb" + + "github.com/pokt-network/pocket/app/client/cli/helpers" + "github.com/pokt-network/pocket/logger" + "github.com/pokt-network/pocket/p2p/debug" + "github.com/pokt-network/pocket/shared/messaging" +) + +var ( + listCmd = &cobra.Command{ + Use: "list", + Short: "Print addresses and service URLs of known peers", + RunE: listRunE, + } + + ErrRouterType = fmt.Errorf("must specify one of --staked, --unstaked, or --all") +) + +func init() { + PeerCmd.AddCommand(listCmd) +} + +func listRunE(cmd *cobra.Command, _ []string) error { + var routerType debug.RouterType + + bus, err := helpers.GetBusFromCmd(cmd) + if err != nil { + return err + } + + switch { + case stakedFlag: + if unstakedFlag || allFlag { + return ErrRouterType + } + routerType = debug.StakedRouterType + case unstakedFlag: + if stakedFlag || allFlag { + return ErrRouterType + } + routerType = debug.UnstakedRouterType + case allFlag: + if stakedFlag || unstakedFlag { + return ErrRouterType + } + routerType = debug.AllRouterTypes + default: + return ErrRouterType + } + + debugMsg := &messaging.DebugMessage{ + Action: messaging.DebugMessageAction_DEBUG_P2P_PEER_LIST, + Type: messaging.DebugMessageRoutingType_DEBUG_MESSAGE_TYPE_BROADCAST, + Message: &anypb.Any{ + Value: []byte(routerType), + }, + } + debugMsgAny, err := anypb.New(debugMsg) + if err != nil { + return fmt.Errorf("creating anypb from debug message: %w", err) + } + + if localFlag { + if err := debug.PrintPeerList(bus, routerType); err != nil { + return fmt.Errorf("printing peer list: %w", err) + } + return nil + } + + // TECHDEBT(#810, #811): will need to wait for DHT bootstrapping to complete before + // p2p broadcast can be used with to reach unstaked actors. + // CONSIDERATION: add the peer commands to the interactive CLI as the P2P module + // instance could persist between commands. Other interactive CLI commands which + // rely on unstaked actor router broadcast are working as expected. + + // TECHDEBT(#810, #811): use broadcast instead to reach all peers. + return sendToStakedPeers(cmd, debugMsgAny) +} + +func sendToStakedPeers(cmd *cobra.Command, debugMsgAny *anypb.Any) error { + bus, err := helpers.GetBusFromCmd(cmd) + if err != nil { + return err + } + + pstore, err := helpers.FetchPeerstore(cmd) + if err != nil { + logger.Global.Fatal().Err(err).Msg("Unable to retrieve the pstore") + } + + if pstore.Size() == 0 { + logger.Global.Fatal().Msg("No validators found") + } + + for _, peer := range pstore.GetPeerList() { + if err := bus.GetP2PModule().Send(peer.GetAddress(), debugMsgAny); err != nil { + logger.Global.Error().Err(err).Msg("Failed to send debug message") + } + } + return nil +} diff --git a/app/client/cli/peer/peer.go b/app/client/cli/peer/peer.go index 1d9ddb2c8..60403b7b8 100644 --- a/app/client/cli/peer/peer.go +++ b/app/client/cli/peer/peer.go @@ -20,8 +20,8 @@ var ( ) func init() { - PeerCmd.PersistentFlags().BoolVarP(&allFlag, "all", "a", false, "operations apply to both staked & unstaked router peerstores") + PeerCmd.PersistentFlags().BoolVarP(&allFlag, "all", "a", true, "operations apply to both staked & unstaked router peerstores") PeerCmd.PersistentFlags().BoolVarP(&stakedFlag, "staked", "s", false, "operations only apply to staked router peerstore (i.e. raintree)") PeerCmd.PersistentFlags().BoolVarP(&unstakedFlag, "unstaked", "u", false, "operations only apply to unstaked router peerstore (i.e. gossipsub)") - PeerCmd.PersistentFlags().BoolVarP(&localFlag, "local", "l", false, "operations apply to the local (CLI binary's) P2P module rather than being sent to the --remote_cli_url") + PeerCmd.PersistentFlags().BoolVarP(&localFlag, "local", "l", false, "operations apply to the local (CLI binary's) P2P module instead of being broadcast") } diff --git a/p2p/debug.go b/p2p/debug.go new file mode 100644 index 000000000..a6ed20a8d --- /dev/null +++ b/p2p/debug.go @@ -0,0 +1,28 @@ +package p2p + +import ( + "fmt" + "github.com/pokt-network/pocket/p2p/debug" + typesP2P "github.com/pokt-network/pocket/p2p/types" + "github.com/pokt-network/pocket/shared/messaging" +) + +func (m *p2pModule) handleDebugMessage(msg *messaging.DebugMessage) error { + switch msg.Action { + case messaging.DebugMessageAction_DEBUG_P2P_PEER_LIST: + if !m.cfg.EnablePeerDiscoveryDebugRpc { + return typesP2P.ErrPeerDiscoveryDebugRPCDisabled + } + default: + // This debug message isn't intended for the P2P module, ignore it. + return nil + } + + switch msg.Action { + case messaging.DebugMessageAction_DEBUG_P2P_PEER_LIST: + routerType := debug.RouterType(msg.Message.Value) + return debug.PrintPeerList(m.GetBus(), routerType) + default: + return fmt.Errorf("unsupported P2P debug message action: %s", msg.Action) + } +} diff --git a/p2p/debug/list.go b/p2p/debug/list.go new file mode 100644 index 000000000..8c1d4f39c --- /dev/null +++ b/p2p/debug/list.go @@ -0,0 +1,106 @@ +package debug + +import ( + "fmt" + "github.com/pokt-network/pocket/p2p/providers/peerstore_provider" + "github.com/pokt-network/pocket/p2p/types" + "github.com/pokt-network/pocket/shared/modules" + "os" +) + +func PrintPeerList(bus modules.Bus, routerType RouterType) error { + var ( + peers types.PeerList + pstorePlurality = "" + ) + + // TECHDEBT(#810, #811): use `bus.GetPeerstoreProvider()` after peerstore provider + // is retrievable as a proper submodule. + pstoreProviderModule, err := bus.GetModulesRegistry(). + GetModule(peerstore_provider.PeerstoreProviderSubmoduleName) + if err != nil { + return fmt.Errorf("getting peerstore provider: %w", err) + } + pstoreProvider, ok := pstoreProviderModule.(peerstore_provider.PeerstoreProvider) + if !ok { + return fmt.Errorf("unknown peerstore provider type: %T", pstoreProviderModule) + } + //-- + + switch routerType { + case StakedRouterType: + // TODO_THIS_COMMIT: what about unstaked peers actors? + // if !isStaked ... + pstore, err := pstoreProvider.GetStakedPeerstoreAtCurrentHeight() + if err != nil { + return fmt.Errorf("getting unstaked peerstore: %v", err) + } + + peers = pstore.GetPeerList() + case UnstakedRouterType: + pstore, err := pstoreProvider.GetUnstakedPeerstore() + if err != nil { + return fmt.Errorf("getting unstaked peerstore: %v", err) + } + + peers = pstore.GetPeerList() + case AllRouterTypes: + pstorePlurality = "s" + + // TODO_THIS_COMMIT: what about unstaked peers actors? + // if !isStaked ... + stakedPStore, err := pstoreProvider.GetStakedPeerstoreAtCurrentHeight() + if err != nil { + return fmt.Errorf("getting unstaked peerstore: %v", err) + } + unstakedPStore, err := pstoreProvider.GetUnstakedPeerstore() + if err != nil { + return fmt.Errorf("getting unstaked peerstore: %v", err) + } + + unstakedPeers := unstakedPStore.GetPeerList() + stakedPeers := stakedPStore.GetPeerList() + additionalPeers, _ := unstakedPeers.Delta(stakedPeers) + + // NB: there should never be any "additional" peers. This would represent + // a staked actor who is not participating in background gossip for some + // reason. It's possible that a staked actor node which has restarted + // recently and hasn't yet completed background router bootstrapping may + // result in peers experiencing this state. + if len(additionalPeers) == 0 { + return PrintPeerListTable(unstakedPeers) + } + + allPeers := append(types.PeerList{}, unstakedPeers...) + allPeers = append(allPeers, additionalPeers...) + peers = allPeers + default: + return fmt.Errorf("unsupported router type: %s", routerType) + } + + if err := LogSelfAddress(bus); err != nil { + return fmt.Errorf("printing self address: %w", err) + } + + // NB: Intentionally printing with `fmt` instead of the logger to match + // `utils.PrintPeerListTable` which does not use the logger due to + // incompatibilities with the tabwriter. + // (This doesn't seem to work as expected; i.e. not printing at all in tilt.) + if _, err := fmt.Fprintf( + os.Stdout, + "%s router peerstore%s:\n", + routerType, + pstorePlurality, + ); err != nil { + return fmt.Errorf("printing to stdout: %w", err) + } + + if err := PrintPeerListTable(peers); err != nil { + return fmt.Errorf("printing peer list: %w", err) + } + return nil +} + +func getPeerstoreProvider() (peerstore_provider.PeerstoreProvider, error) { + return nil, nil +} diff --git a/p2p/debug/peers.go b/p2p/debug/peers.go new file mode 100644 index 000000000..56afd9e89 --- /dev/null +++ b/p2p/debug/peers.go @@ -0,0 +1,62 @@ +package debug + +import ( + "fmt" + "os" + + "github.com/pokt-network/pocket/p2p/types" + "github.com/pokt-network/pocket/p2p/utils" + "github.com/pokt-network/pocket/shared/modules" +) + +type RouterType string + +const ( + StakedRouterType RouterType = "staked" + UnstakedRouterType RouterType = "unstaked" + AllRouterTypes RouterType = "all" +) + +var peerListTableHeader = []string{"Peer ID", "Pokt Address", "ServiceURL"} + +func LogSelfAddress(bus modules.Bus) error { + p2pModule := bus.GetP2PModule() + if p2pModule == nil { + return fmt.Errorf("no p2p module found on the bus") + } + + selfAddr, err := p2pModule.GetAddress() + if err != nil { + return fmt.Errorf("getting self address: %w", err) + } + + _, err = fmt.Fprintf(os.Stdout, "self address: %s", selfAddr.String()) + return err +} + +// PrintPeerListTable prints a table of the passed peers to stdout. Header row is defined +// by `peerListTableHeader`. Row printing behavior is defined by `peerListRowConsumerFactory`. +func PrintPeerListTable(peers types.PeerList) error { + return utils.PrintTable(peerListTableHeader, peerListRowConsumerFactory(peers)) +} + +func peerListRowConsumerFactory(peers types.PeerList) utils.RowConsumer { + return func(provideRow utils.RowProvider) error { + for _, peer := range peers { + libp2pAddrInfo, err := utils.Libp2pAddrInfoFromPeer(peer) + if err != nil { + return fmt.Errorf("converting peer to libp2p addr info: %w", err) + } + + err = provideRow( + libp2pAddrInfo.ID.String(), + peer.GetAddress().String(), + peer.GetServiceURL(), + ) + if err != nil { + return err + } + } + return nil + } +} diff --git a/p2p/event_handler.go b/p2p/event_handler.go index 60e8afca4..2c612ab27 100644 --- a/p2p/event_handler.go +++ b/p2p/event_handler.go @@ -2,7 +2,6 @@ package p2p import ( "fmt" - "google.golang.org/protobuf/types/known/anypb" "github.com/pokt-network/pocket/shared/codec" @@ -100,7 +99,3 @@ func (m *p2pModule) HandleEvent(event *anypb.Any) error { return nil } - -func (m *p2pModule) handleDebugMessage(msg *messaging.DebugMessage) error { - return nil -} diff --git a/p2p/utils/logging.go b/p2p/utils/logging.go index ac999c62a..ea9a787c6 100644 --- a/p2p/utils/logging.go +++ b/p2p/utils/logging.go @@ -1,14 +1,27 @@ package utils import ( + "fmt" "net" + "os" + "text/tabwriter" "github.com/libp2p/go-libp2p/core/network" + "github.com/rs/zerolog" + "github.com/pokt-network/pocket/p2p/types" "github.com/pokt-network/pocket/shared/modules" - "github.com/rs/zerolog" ) +// RowProvider is a function which receives a variadic number of "column" values. +// It is intended to be passed to a `RowConsumer` so that the consumer can operate +// on the column values, row-by-row, without having to know how to produce them. +type RowProvider func(columns ...string) error + +// RowConsumer is any function which receives a `RowProvider` in order to consume +// its column values, row-by-row. +type RowConsumer func(RowProvider) error + type scopeCallback func(scope network.ResourceScope) error // LogScopeStatFactory returns a function which prints the given scope stat fields @@ -41,6 +54,63 @@ func LogIncomingMsg(logger *modules.Logger, hostname string, peer types.Peer) { logMessage(logger, msg, hostname, peer) } +// Print table prints a table to stdout. Header row is defined by `header`. Row printing +// behavior is defined by `consumeRows`. Header length SHOULD match row length. +func PrintTable(header []string, consumeRows RowConsumer) error { + w := new(tabwriter.Writer) + w.Init(os.Stdout, 0, 0, 1, ' ', 0) + + // Print header + for _, col := range header { + if _, err := fmt.Fprintf(w, "| %s\t", col); err != nil { + return err + } + } + if _, err := fmt.Fprintln(w, "|"); err != nil { + return err + } + + // Print separator + for _, col := range header { + if _, err := fmt.Fprintf(w, "| "); err != nil { + return err + } + for range col { + if _, err := fmt.Fprintf(w, "-"); err != nil { + return err + } + } + if _, err := fmt.Fprintf(w, "\t"); err != nil { + return err + } + } + if _, err := fmt.Fprintln(w, "|"); err != nil { + return err + } + + // Print rows -- `consumeRows` will call this function once for each row. + if err := consumeRows(func(row ...string) error { + for _, col := range row { + if _, err := fmt.Fprintf(w, "| %s\t", col); err != nil { + return err + } + } + if _, err := fmt.Fprintln(w, "|"); err != nil { + return err + } + return nil + }); err != nil { + return err + } + + // Flush the buffer and print the table + if err := w.Flush(); err != nil { + return err + } + + return nil +} + func logMessage(logger *modules.Logger, msg, hostname string, peer types.Peer) { remoteHostname, _, err := net.SplitHostPort(peer.GetServiceURL()) if err != nil { diff --git a/shared/messaging/proto/debug_message.proto b/shared/messaging/proto/debug_message.proto index 7ce079afa..55a87c695 100644 --- a/shared/messaging/proto/debug_message.proto +++ b/shared/messaging/proto/debug_message.proto @@ -22,6 +22,7 @@ enum DebugMessageAction { DEBUG_PERSISTENCE_CLEAR_STATE = 8; DEBUG_PERSISTENCE_RESET_TO_GENESIS = 9; + DEBUG_P2P_PEER_LIST = 10; } message DebugMessage { From e80843ca0388655acf8e84cb3e445e18c56f43cf Mon Sep 17 00:00:00 2001 From: Bryan White Date: Mon, 5 Jun 2023 10:00:42 +0200 Subject: [PATCH 11/38] chore: implement `PeerstoreProvider#GetUnstakedPeerstore()` --- .../persistence/provider.go | 15 +++++++- .../peerstore_provider/rpc/provider.go | 13 ++++++- p2p/providers/peerstore_provider/unstaked.go | 34 ------------------- 3 files changed, 26 insertions(+), 36 deletions(-) delete mode 100644 p2p/providers/peerstore_provider/unstaked.go diff --git a/p2p/providers/peerstore_provider/persistence/provider.go b/p2p/providers/peerstore_provider/persistence/provider.go index cbd1ec82c..300c49a38 100644 --- a/p2p/providers/peerstore_provider/persistence/provider.go +++ b/p2p/providers/peerstore_provider/persistence/provider.go @@ -1,6 +1,8 @@ package persistence import ( + "fmt" + "github.com/pokt-network/pocket/p2p/providers/peerstore_provider" typesP2P "github.com/pokt-network/pocket/p2p/types" "github.com/pokt-network/pocket/shared/modules" @@ -58,5 +60,16 @@ func (persistencePSP *persistencePeerstoreProvider) GetStakedPeerstoreAtHeight(h // GetStakedPeerstoreAtHeight implements the respective `PeerstoreProvider` interface method. func (persistencePSP *persistencePeerstoreProvider) GetUnstakedPeerstore() (typesP2P.Peerstore, error) { - return peerstore_provider.GetUnstakedPeerstore(persistencePSP.GetBus()) + // TECHDEBT(#810, #811): use `bus.GetUnstakedActorRouter()` once it's available. + unstakedActorRouterMod, err := persistencePSP.GetBus().GetModulesRegistry().GetModule(typesP2P.UnstakedActorRouterSubmoduleName) + if err != nil { + return nil, err + } + + unstakedActorRouter, ok := unstakedActorRouterMod.(typesP2P.Router) + if !ok { + return nil, fmt.Errorf("unexpected unstaked actor router submodule type: %T", unstakedActorRouterMod) + } + + return unstakedActorRouter.GetPeerstore(), nil } diff --git a/p2p/providers/peerstore_provider/rpc/provider.go b/p2p/providers/peerstore_provider/rpc/provider.go index b0da88f69..c36963604 100644 --- a/p2p/providers/peerstore_provider/rpc/provider.go +++ b/p2p/providers/peerstore_provider/rpc/provider.go @@ -92,7 +92,18 @@ func (rpcPSP *rpcPeerstoreProvider) GetStakedPeerstoreAtHeight(height uint64) (t } func (rpcPSP *rpcPeerstoreProvider) GetUnstakedPeerstore() (typesP2P.Peerstore, error) { - return peerstore_provider.GetUnstakedPeerstore(rpcPSP.GetBus()) + // TECHDEBT(#810, #811): use `bus.GetUnstakedActorRouter()` once it's available. + unstakedActorRouterMod, err := rpcPSP.GetBus().GetModulesRegistry().GetModule(typesP2P.UnstakedActorRouterSubmoduleName) + if err != nil { + return nil, err + } + + unstakedActorRouter, ok := unstakedActorRouterMod.(typesP2P.Router) + if !ok { + return nil, fmt.Errorf("unexpected unstaked actor router submodule type: %T", unstakedActorRouterMod) + } + + return unstakedActorRouter.GetPeerstore(), nil } func (rpcPSP *rpcPeerstoreProvider) initRPCClient() { diff --git a/p2p/providers/peerstore_provider/unstaked.go b/p2p/providers/peerstore_provider/unstaked.go deleted file mode 100644 index b107922fe..000000000 --- a/p2p/providers/peerstore_provider/unstaked.go +++ /dev/null @@ -1,34 +0,0 @@ -package peerstore_provider - -import ( - "fmt" - - typesP2P "github.com/pokt-network/pocket/p2p/types" - "github.com/pokt-network/pocket/shared/modules" -) - -// unstakedPeerstoreProvider is an interface which the p2p module supports in -// order to allow access to the unstaked-actor-router's peerstore. -// -// NB: this peerstore includes all actors which participate in P2P (e.g. full -// and light clients but also validators, servicers, etc.). -// -// TECHDEBT(#811): will become unnecessary after `modules.P2PModule#GetUnstakedPeerstore` is added.` -// CONSIDERATION: split `PeerstoreProvider` into `StakedPeerstoreProvider` and `UnstakedPeerstoreProvider`. -// (see: https://github.com/pokt-network/pocket/pull/804#issuecomment-1576531916) -type unstakedPeerstoreProvider interface { - GetUnstakedPeerstore() (typesP2P.Peerstore, error) -} - -func GetUnstakedPeerstore(bus modules.Bus) (typesP2P.Peerstore, error) { - p2pModule := bus.GetP2PModule() - if p2pModule == nil { - return nil, fmt.Errorf("p2p module is not registered to bus and is required") - } - - unstakedPSP, ok := p2pModule.(unstakedPeerstoreProvider) - if !ok { - return nil, fmt.Errorf("p2p module does not implement unstakedPeerstoreProvider") - } - return unstakedPSP.GetUnstakedPeerstore() -} From 8f90e22d3ba1e2614b49f10883b4792a5e66f5fb Mon Sep 17 00:00:00 2001 From: Bryan White Date: Thu, 13 Jul 2023 15:32:07 +0200 Subject: [PATCH 12/38] chore: add `PeerstoreProvider#GetStakedPeerstoreAtCurrentHeight()` --- p2p/providers/peerstore_provider/peerstore_provider.go | 4 ++++ p2p/providers/peerstore_provider/persistence/provider.go | 5 +++++ p2p/providers/peerstore_provider/rpc/provider.go | 5 +++++ 3 files changed, 14 insertions(+) diff --git a/p2p/providers/peerstore_provider/peerstore_provider.go b/p2p/providers/peerstore_provider/peerstore_provider.go index bbf57746a..968c3a2b1 100644 --- a/p2p/providers/peerstore_provider/peerstore_provider.go +++ b/p2p/providers/peerstore_provider/peerstore_provider.go @@ -21,6 +21,10 @@ type PeerstoreProvider interface { // at a given height. These peers communicate via the p2p module's staked actor // router. GetStakedPeerstoreAtHeight(height uint64) (typesP2P.Peerstore, error) + // GetStakedPeerstoreAtCurrentHeight returns a peerstore containing all staked + // peers at the current height. These peers communicate via the p2p module's + // staked actor router. + GetStakedPeerstoreAtCurrentHeight() (typesP2P.Peerstore, error) // GetUnstakedPeerstore returns a peerstore containing all peers which // communicate via the p2p module's unstaked actor router. GetUnstakedPeerstore() (typesP2P.Peerstore, error) diff --git a/p2p/providers/peerstore_provider/persistence/provider.go b/p2p/providers/peerstore_provider/persistence/provider.go index 300c49a38..59496921a 100644 --- a/p2p/providers/peerstore_provider/persistence/provider.go +++ b/p2p/providers/peerstore_provider/persistence/provider.go @@ -58,6 +58,11 @@ func (persistencePSP *persistencePeerstoreProvider) GetStakedPeerstoreAtHeight(h return peerstore_provider.ActorsToPeerstore(persistencePSP, validators) } +func (persistencePSP *persistencePeerstoreProvider) GetStakedPeerstoreAtCurrentHeight() (typesP2P.Peerstore, error) { + currentHeight := persistencePSP.GetBus().GetCurrentHeightProvider().CurrentHeight() + return persistencePSP.GetStakedPeerstoreAtHeight(currentHeight) +} + // GetStakedPeerstoreAtHeight implements the respective `PeerstoreProvider` interface method. func (persistencePSP *persistencePeerstoreProvider) GetUnstakedPeerstore() (typesP2P.Peerstore, error) { // TECHDEBT(#810, #811): use `bus.GetUnstakedActorRouter()` once it's available. diff --git a/p2p/providers/peerstore_provider/rpc/provider.go b/p2p/providers/peerstore_provider/rpc/provider.go index c36963604..f119832e7 100644 --- a/p2p/providers/peerstore_provider/rpc/provider.go +++ b/p2p/providers/peerstore_provider/rpc/provider.go @@ -91,6 +91,11 @@ func (rpcPSP *rpcPeerstoreProvider) GetStakedPeerstoreAtHeight(height uint64) (t return peerstore_provider.ActorsToPeerstore(rpcPSP, coreActors) } +func (rpcPSP *rpcPeerstoreProvider) GetStakedPeerstoreAtCurrentHeight() (typesP2P.Peerstore, error) { + currentHeight := rpcPSP.GetBus().GetCurrentHeightProvider().CurrentHeight() + return rpcPSP.GetStakedPeerstoreAtHeight(currentHeight) +} + func (rpcPSP *rpcPeerstoreProvider) GetUnstakedPeerstore() (typesP2P.Peerstore, error) { // TECHDEBT(#810, #811): use `bus.GetUnstakedActorRouter()` once it's available. unstakedActorRouterMod, err := rpcPSP.GetBus().GetModulesRegistry().GetModule(typesP2P.UnstakedActorRouterSubmoduleName) From 6e691cda0f888cdfea50ef18e49cdd984563fd6a Mon Sep 17 00:00:00 2001 From: Bryan White Date: Tue, 11 Jul 2023 09:28:54 +0200 Subject: [PATCH 13/38] chore: interim bootstrapping changes (pre-#859) --- p2p/background/router.go | 32 +++++++++++++++---------------- p2p/background/router_test.go | 11 +++-------- p2p/bootstrap.go | 36 ++++++++++++++++++++++------------- p2p/event_handler.go | 14 +------------- 4 files changed, 42 insertions(+), 51 deletions(-) diff --git a/p2p/background/router.go b/p2p/background/router.go index 5e6254d20..f32c629f9 100644 --- a/p2p/background/router.go +++ b/p2p/background/router.go @@ -65,7 +65,7 @@ type backgroundRouter struct { subscription *pubsub.Subscription // kadDHT is a kademlia distributed hash table used for routing and peer discovery. kadDHT *dht.IpfsDHT - // TECHDEBT: `pstore` will likely be removed in future refactoring / simplification + // TECHDEBT(#747, #749): `pstore` will likely be removed in future refactoring / simplification // of the `Router` interface. // pstore is the background router's peerstore. Assigned in `backgroundRouter#setupPeerstore()`. pstore typesP2P.Peerstore @@ -250,8 +250,6 @@ func (rtr *backgroundRouter) setupDependencies(ctx context.Context, _ *config.Ba } func (rtr *backgroundRouter) setupPeerstore(ctx context.Context) (err error) { - rtr.logger.Warn().Msg("setting up peerstore...") - // TECHDEBT(#810, #811): use `bus.GetPeerstoreProvider()` after peerstore provider // is retrievable as a proper submodule pstoreProviderModule, err := rtr.GetBus().GetModulesRegistry(). @@ -276,10 +274,7 @@ func (rtr *backgroundRouter) setupPeerstore(ctx context.Context) (err error) { } // TECHDEBT(#859): integrate with `p2pModule#bootstrap()`. - if err := rtr.bootstrap(ctx); err != nil { - return fmt.Errorf("bootstrapping peerstore: %w", err) - } - + rtr.bootstrap(ctx) return nil } @@ -335,33 +330,36 @@ func (rtr *backgroundRouter) setupSubscription() (err error) { } // TECHDEBT(#859): integrate with `p2pModule#bootstrap()`. -func (rtr *backgroundRouter) bootstrap(ctx context.Context) error { +func (rtr *backgroundRouter) bootstrap(ctx context.Context) { // CONSIDERATION: add `GetPeers` method, which returns a map, // to the `PeerstoreProvider` interface to simplify this loop. for _, peer := range rtr.pstore.GetPeerList() { if err := utils.AddPeerToLibp2pHost(rtr.host, peer); err != nil { - return err + rtr.logger.Error().Err(err).Msg("adding peer to libp2p host") + continue } libp2pAddrInfo, err := utils.Libp2pAddrInfoFromPeer(peer) if err != nil { - return fmt.Errorf( - "converting peer info, pokt address: %s: %w", - peer.GetAddress(), - err, - ) + rtr.logger.Error().Err(err).Msg("converting peer info") + continue } // don't attempt to connect to self if rtr.host.ID() == libp2pAddrInfo.ID { - return nil + rtr.logger.Debug().Msg("not bootstrapping against self") + continue } + rtr.logger.Debug().Fields(map[string]any{ + "peer_id": libp2pAddrInfo.ID.String(), + "peer_addr": libp2pAddrInfo.Addrs[0].String(), + }).Msg("connecting to peer") if err := rtr.host.Connect(ctx, libp2pAddrInfo); err != nil { - return fmt.Errorf("connecting to peer: %w", err) + rtr.logger.Error().Err(err).Msg("connecting to bootstrap peer") + continue } } - return nil } // topicValidator is used in conjunction with libp2p-pubsub's notion of "topic diff --git a/p2p/background/router_test.go b/p2p/background/router_test.go index 3c93cf758..14c7143fa 100644 --- a/p2p/background/router_test.go +++ b/p2p/background/router_test.go @@ -363,18 +363,13 @@ func bootstrap(t *testing.T, ctx context.Context, testHosts []libp2pHost.Host) { continue } - p2pAddr, err := multiaddr.NewMultiaddr(fmt.Sprintf("/p2p/%s", bootstrapHost.ID())) - require.NoError(t, err) - addrInfo := libp2pPeer.AddrInfo{ - ID: bootstrapHost.ID(), - Addrs: []multiaddr.Multiaddr{ - bootstrapAddr.Encapsulate(p2pAddr), - }, + ID: bootstrapHost.ID(), + Addrs: []multiaddr.Multiaddr{bootstrapAddr}, } t.Logf("connecting to %s...", addrInfo.ID.String()) - err = h.Connect(ctx, addrInfo) + err := h.Connect(ctx, addrInfo) require.NoError(t, err) } } diff --git a/p2p/bootstrap.go b/p2p/bootstrap.go index 258fb19a2..c74d34ad2 100644 --- a/p2p/bootstrap.go +++ b/p2p/bootstrap.go @@ -10,7 +10,7 @@ import ( "strings" rpcCHP "github.com/pokt-network/pocket/p2p/providers/current_height_provider/rpc" - rpcABP "github.com/pokt-network/pocket/p2p/providers/peerstore_provider/rpc" + rpcPSP "github.com/pokt-network/pocket/p2p/providers/peerstore_provider/rpc" typesP2P "github.com/pokt-network/pocket/p2p/types" "github.com/pokt-network/pocket/rpc" "github.com/pokt-network/pocket/runtime/defaults" @@ -59,9 +59,9 @@ func (m *p2pModule) bootstrap() error { continue } - pstoreProvider, err := rpcABP.Create( + pstoreProvider, err := rpcPSP.Create( m.GetBus(), - rpcABP.WithCustomRPCURL(bootstrapNode), + rpcPSP.WithCustomRPCURL(bootstrapNode), ) if err != nil { return fmt.Errorf("creating RPC peerstore provider: %w", err) @@ -81,20 +81,30 @@ func (m *p2pModule) bootstrap() error { m.logger.Warn().Err(err).Str("endpoint", bootstrapNode).Msg("Error getting address book from bootstrap node") continue } - } - for _, peer := range pstore.GetPeerList() { - m.logger.Debug().Str("address", peer.GetAddress().String()).Msg("Adding peer to router") - if err := m.stakedActorRouter.AddPeer(peer); err != nil { - m.logger.Error().Err(err). - Str("pokt_address", peer.GetAddress().String()). - Msg("adding peer") + for _, peer := range pstore.GetPeerList() { + m.logger.Debug().Str("address", peer.GetAddress().String()).Msg("Adding peer to router") + isStaked, err := m.isStakedActor() + if err != nil { + m.logger.Error().Err(err).Msg("checking if node is staked") + } + if isStaked { + if err := m.stakedActorRouter.AddPeer(peer); err != nil { + m.logger.Error().Err(err). + Str("pokt_address", peer.GetAddress().String()). + Msg("adding peer to staked actor router") + } + } + + if err := m.unstakedActorRouter.AddPeer(peer); err != nil { + m.logger.Error().Err(err). + Str("pokt_address", peer.GetAddress().String()). + Msg("adding peer to unstaked actor router") + } } } - if m.stakedActorRouter.GetPeerstore().Size() == 0 { - return fmt.Errorf("bootstrap failed") - } + // TECHDEBT(#859): determine bootstrapping success/error conditions. return nil } diff --git a/p2p/event_handler.go b/p2p/event_handler.go index 2c612ab27..5dde1c5e1 100644 --- a/p2p/event_handler.go +++ b/p2p/event_handler.go @@ -61,21 +61,9 @@ func (m *p2pModule) HandleEvent(event *anypb.Any) error { m.logger.Debug().Fields(messaging.TransitionEventToMap(stateMachineTransitionEvent)).Msg("Received state machine transition event") if stateMachineTransitionEvent.NewState == string(coreTypes.StateMachineState_P2P_Bootstrapping) { - staked, err := m.isStakedActor() - if err != nil { + if err := m.bootstrap(); err != nil { return err } - if staked { - // TECHDEBT(#859): this will never happen as the peerstore is - // seeded from consensus during P2P module construction. - if m.stakedActorRouter.GetPeerstore().Size() == 0 { - m.logger.Warn().Msg("No peers in peerstore, bootstrapping") - - if err := m.bootstrap(); err != nil { - return err - } - } - } // TECHDEBT(#859): for unstaked actors, unstaked actor (background) // router bootstrapping SHOULD complete before the event below is sent. From 430db08e8b0758f590de057b5ff26a115cf449c4 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Wed, 12 Jul 2023 11:32:28 +0200 Subject: [PATCH 14/38] fix: gofmt --- p2p/event_handler.go | 1 + 1 file changed, 1 insertion(+) diff --git a/p2p/event_handler.go b/p2p/event_handler.go index 5dde1c5e1..21fbfb9bf 100644 --- a/p2p/event_handler.go +++ b/p2p/event_handler.go @@ -2,6 +2,7 @@ package p2p import ( "fmt" + "google.golang.org/protobuf/types/known/anypb" "github.com/pokt-network/pocket/shared/codec" From 440b59a4c779a4545b6d457e02887a05694bd639 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Fri, 7 Jul 2023 11:16:49 +0200 Subject: [PATCH 15/38] chore: ensure flag and config parsing --- app/client/cli/helpers/setup.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/app/client/cli/helpers/setup.go b/app/client/cli/helpers/setup.go index 956102a04..fca506576 100644 --- a/app/client/cli/helpers/setup.go +++ b/app/client/cli/helpers/setup.go @@ -4,6 +4,7 @@ import ( "fmt" "github.com/spf13/cobra" + "github.com/spf13/viper" "github.com/pokt-network/pocket/app/client/cli/flags" "github.com/pokt-network/pocket/logger" @@ -11,14 +12,20 @@ import ( rpcCHP "github.com/pokt-network/pocket/p2p/providers/current_height_provider/rpc" rpcPSP "github.com/pokt-network/pocket/p2p/providers/peerstore_provider/rpc" "github.com/pokt-network/pocket/runtime" + "github.com/pokt-network/pocket/runtime/configs" "github.com/pokt-network/pocket/shared/modules" ) // P2PDependenciesPreRunE initializes peerstore & current height providers, and a // p2p module which consumes them. Everything is registered to the bus. func P2PDependenciesPreRunE(cmd *cobra.Command, _ []string) error { + // TECHDEBT: this is to keep backwards compatibility with localnet flags.ConfigPath = runtime.GetEnv("CONFIG_PATH", "build/config/config.validator1.json") + configs.ParseConfig(flags.ConfigPath) + + // set final `remote_cli_url` value; order of precedence: flag > env var > config > default + flags.RemoteCLIURL = viper.GetString("remote_cli_url") runtimeMgr := runtime.NewManagerFromFiles( flags.ConfigPath, genesisPath, From fcfa837525b144e73f45cba21b793f4acfe50600 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Tue, 11 Jul 2023 09:28:28 +0200 Subject: [PATCH 16/38] chore: add `GetBusFromCmd()` CLI helper --- app/client/cli/helpers/common.go | 6 +++--- app/client/cli/helpers/context.go | 14 ++++++++++++++ 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/app/client/cli/helpers/common.go b/app/client/cli/helpers/common.go index be9d75250..b89f79ba9 100644 --- a/app/client/cli/helpers/common.go +++ b/app/client/cli/helpers/common.go @@ -25,9 +25,9 @@ var ( // fetchPeerstore retrieves the providers from the CLI context and uses them to retrieve the address book for the current height func FetchPeerstore(cmd *cobra.Command) (types.Peerstore, error) { - bus, ok := GetValueFromCLIContext[modules.Bus](cmd, BusCLICtxKey) - if !ok || bus == nil { - return nil, errors.New("retrieving bus from CLI context") + bus, err := GetBusFromCmd(cmd) + if err != nil { + return nil, err } // TECHDEBT(#810, #811): use `bus.GetPeerstoreProvider()` after peerstore provider // is retrievable as a proper submodule diff --git a/app/client/cli/helpers/context.go b/app/client/cli/helpers/context.go index f9f3f4549..f1494c5ac 100644 --- a/app/client/cli/helpers/context.go +++ b/app/client/cli/helpers/context.go @@ -2,12 +2,17 @@ package helpers import ( "context" + "fmt" "github.com/spf13/cobra" + + "github.com/pokt-network/pocket/shared/modules" ) const BusCLICtxKey cliContextKey = "bus" +var ErrCxtFromBus = fmt.Errorf("could not get context from bus") + // NOTE: this is required by the linter, otherwise a simple string constant would have been enough type cliContextKey string @@ -19,3 +24,12 @@ func GetValueFromCLIContext[T any](cmd *cobra.Command, key cliContextKey) (T, bo value, ok := cmd.Context().Value(key).(T) return value, ok } + +func GetBusFromCmd(cmd *cobra.Command) (modules.Bus, error) { + bus, ok := GetValueFromCLIContext[modules.Bus](cmd, BusCLICtxKey) + if !ok { + return nil, ErrCxtFromBus + } + + return bus, nil +} From 04dc0aabcc96f5f66c0a7eade5225a85cd3fab8e Mon Sep 17 00:00:00 2001 From: Bryan White Date: Tue, 11 Jul 2023 09:28:32 +0200 Subject: [PATCH 17/38] chore: consistent debug CLI identity --- app/client/cli/helpers/setup.go | 6 +++++- runtime/manager.go | 1 + 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/app/client/cli/helpers/setup.go b/app/client/cli/helpers/setup.go index fca506576..c91673b8d 100644 --- a/app/client/cli/helpers/setup.go +++ b/app/client/cli/helpers/setup.go @@ -16,6 +16,10 @@ import ( "github.com/pokt-network/pocket/shared/modules" ) +// TODO_THIS_COMMIT: add godoc comment explaining what this **is** and **is not** +// intended to be used for. +const debugPrivKey = "09fc8ee114e678e665d09179acb9a30060f680df44ba06b51434ee47940a8613be19b2b886e743eb1ff7880968d6ce1a46350315e569243e747a227ee8faec3d" + // P2PDependenciesPreRunE initializes peerstore & current height providers, and a // p2p module which consumes them. Everything is registered to the bus. func P2PDependenciesPreRunE(cmd *cobra.Command, _ []string) error { @@ -30,7 +34,7 @@ func P2PDependenciesPreRunE(cmd *cobra.Command, _ []string) error { runtimeMgr := runtime.NewManagerFromFiles( flags.ConfigPath, genesisPath, runtime.WithClientDebugMode(), - runtime.WithRandomPK(), + runtime.WithPK(debugPrivKey), ) bus := runtimeMgr.GetBus() diff --git a/runtime/manager.go b/runtime/manager.go index 151f2c198..7c182f34f 100644 --- a/runtime/manager.go +++ b/runtime/manager.go @@ -104,6 +104,7 @@ func WithRandomPK() func(*Manager) { return WithPK(privateKey.String()) } +// TECHDEBT(#750): separate conseneus and P2P keys. func WithPK(pk string) func(*Manager) { return func(b *Manager) { if b.config.Consensus == nil { From 1bbad386d862fd9c685fc78a77147b01590cb2d2 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Thu, 13 Jul 2023 16:55:50 +0200 Subject: [PATCH 18/38] fixup: add `peer list` subcommand --- app/client/cli/peer/list.go | 5 ++--- app/client/cli/peer/peer.go | 2 +- p2p/debug.go | 1 + p2p/debug/list.go | 39 +++++++++++++++++++++++++++++++------ p2p/debug/peers.go | 31 ----------------------------- 5 files changed, 37 insertions(+), 41 deletions(-) diff --git a/app/client/cli/peer/list.go b/app/client/cli/peer/list.go index 19b70b7e9..197903848 100644 --- a/app/client/cli/peer/list.go +++ b/app/client/cli/peer/list.go @@ -45,13 +45,12 @@ func listRunE(cmd *cobra.Command, _ []string) error { return ErrRouterType } routerType = debug.UnstakedRouterType - case allFlag: + // even if `allFlag` is false, we still want to print all peers + default: if stakedFlag || unstakedFlag { return ErrRouterType } routerType = debug.AllRouterTypes - default: - return ErrRouterType } debugMsg := &messaging.DebugMessage{ diff --git a/app/client/cli/peer/peer.go b/app/client/cli/peer/peer.go index 60403b7b8..725d49a3c 100644 --- a/app/client/cli/peer/peer.go +++ b/app/client/cli/peer/peer.go @@ -20,7 +20,7 @@ var ( ) func init() { - PeerCmd.PersistentFlags().BoolVarP(&allFlag, "all", "a", true, "operations apply to both staked & unstaked router peerstores") + PeerCmd.PersistentFlags().BoolVarP(&allFlag, "all", "a", false, "operations apply to both staked & unstaked router peerstores (default)") PeerCmd.PersistentFlags().BoolVarP(&stakedFlag, "staked", "s", false, "operations only apply to staked router peerstore (i.e. raintree)") PeerCmd.PersistentFlags().BoolVarP(&unstakedFlag, "unstaked", "u", false, "operations only apply to unstaked router peerstore (i.e. gossipsub)") PeerCmd.PersistentFlags().BoolVarP(&localFlag, "local", "l", false, "operations apply to the local (CLI binary's) P2P module instead of being broadcast") diff --git a/p2p/debug.go b/p2p/debug.go index a6ed20a8d..7c352eb5a 100644 --- a/p2p/debug.go +++ b/p2p/debug.go @@ -2,6 +2,7 @@ package p2p import ( "fmt" + "github.com/pokt-network/pocket/p2p/debug" typesP2P "github.com/pokt-network/pocket/p2p/types" "github.com/pokt-network/pocket/shared/messaging" diff --git a/p2p/debug/list.go b/p2p/debug/list.go index 8c1d4f39c..6a752e71b 100644 --- a/p2p/debug/list.go +++ b/p2p/debug/list.go @@ -2,16 +2,20 @@ package debug import ( "fmt" + "os" + "github.com/pokt-network/pocket/p2p/providers/peerstore_provider" "github.com/pokt-network/pocket/p2p/types" + "github.com/pokt-network/pocket/p2p/utils" "github.com/pokt-network/pocket/shared/modules" - "os" ) +var peerListTableHeader = []string{"Peer ID", "Pokt Address", "ServiceURL"} + func PrintPeerList(bus modules.Bus, routerType RouterType) error { var ( peers types.PeerList - pstorePlurality = "" + routerPlurality = "" ) // TECHDEBT(#810, #811): use `bus.GetPeerstoreProvider()` after peerstore provider @@ -45,7 +49,7 @@ func PrintPeerList(bus modules.Bus, routerType RouterType) error { peers = pstore.GetPeerList() case AllRouterTypes: - pstorePlurality = "s" + routerPlurality = "s" // TODO_THIS_COMMIT: what about unstaked peers actors? // if !isStaked ... @@ -90,7 +94,7 @@ func PrintPeerList(bus modules.Bus, routerType RouterType) error { os.Stdout, "%s router peerstore%s:\n", routerType, - pstorePlurality, + routerPlurality, ); err != nil { return fmt.Errorf("printing to stdout: %w", err) } @@ -101,6 +105,29 @@ func PrintPeerList(bus modules.Bus, routerType RouterType) error { return nil } -func getPeerstoreProvider() (peerstore_provider.PeerstoreProvider, error) { - return nil, nil +// PrintPeerListTable prints a table of the passed peers to stdout. Header row is defined +// by `peerListTableHeader`. Row printing behavior is defined by `peerListRowConsumerFactory`. +func PrintPeerListTable(peers types.PeerList) error { + return utils.PrintTable(peerListTableHeader, peerListRowConsumerFactory(peers)) +} + +func peerListRowConsumerFactory(peers types.PeerList) utils.RowConsumer { + return func(provideRow utils.RowProvider) error { + for _, peer := range peers { + libp2pAddrInfo, err := utils.Libp2pAddrInfoFromPeer(peer) + if err != nil { + return fmt.Errorf("converting peer to libp2p addr info: %w", err) + } + + err = provideRow( + libp2pAddrInfo.ID.String(), + peer.GetAddress().String(), + peer.GetServiceURL(), + ) + if err != nil { + return err + } + } + return nil + } } diff --git a/p2p/debug/peers.go b/p2p/debug/peers.go index 56afd9e89..60f206b00 100644 --- a/p2p/debug/peers.go +++ b/p2p/debug/peers.go @@ -4,8 +4,6 @@ import ( "fmt" "os" - "github.com/pokt-network/pocket/p2p/types" - "github.com/pokt-network/pocket/p2p/utils" "github.com/pokt-network/pocket/shared/modules" ) @@ -17,8 +15,6 @@ const ( AllRouterTypes RouterType = "all" ) -var peerListTableHeader = []string{"Peer ID", "Pokt Address", "ServiceURL"} - func LogSelfAddress(bus modules.Bus) error { p2pModule := bus.GetP2PModule() if p2pModule == nil { @@ -33,30 +29,3 @@ func LogSelfAddress(bus modules.Bus) error { _, err = fmt.Fprintf(os.Stdout, "self address: %s", selfAddr.String()) return err } - -// PrintPeerListTable prints a table of the passed peers to stdout. Header row is defined -// by `peerListTableHeader`. Row printing behavior is defined by `peerListRowConsumerFactory`. -func PrintPeerListTable(peers types.PeerList) error { - return utils.PrintTable(peerListTableHeader, peerListRowConsumerFactory(peers)) -} - -func peerListRowConsumerFactory(peers types.PeerList) utils.RowConsumer { - return func(provideRow utils.RowProvider) error { - for _, peer := range peers { - libp2pAddrInfo, err := utils.Libp2pAddrInfoFromPeer(peer) - if err != nil { - return fmt.Errorf("converting peer to libp2p addr info: %w", err) - } - - err = provideRow( - libp2pAddrInfo.ID.String(), - peer.GetAddress().String(), - peer.GetServiceURL(), - ) - if err != nil { - return err - } - } - return nil - } -} From 64abbc02cb4c9fc86408164d16d1839895622bf5 Mon Sep 17 00:00:00 2001 From: github-actions Date: Thu, 13 Jul 2023 15:26:51 +0000 Subject: [PATCH 19/38] add generated helm docs --- charts/pocket/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/charts/pocket/README.md b/charts/pocket/README.md index 35993ddfd..b4aca5b9d 100644 --- a/charts/pocket/README.md +++ b/charts/pocket/README.md @@ -46,6 +46,7 @@ privateKeySecretKeyRef: | config.fisherman.enabled | bool | `false` | | | config.logger.format | string | `"json"` | | | config.logger.level | string | `"debug"` | | +| config.p2p.enable_peer_discovery_debug_rpc | bool | `false` | | | config.p2p.hostname | string | `""` | | | config.p2p.is_empty_connection_type | bool | `false` | | | config.p2p.max_mempool_count | int | `100000` | | From d8b6296035d5e4666b467168d4aa9cbecc4f575b Mon Sep 17 00:00:00 2001 From: harry <53987565+h5law@users.noreply.github.com> Date: Tue, 25 Jul 2023 10:09:44 +0100 Subject: [PATCH 20/38] squash: merge refactor/cli with main --- .github/workflows/golangci-lint.yml | 2 +- .github/workflows/main.yml | 4 +- .golangci.yml | 2 +- Makefile | 9 +- README.md | 26 ++ app/client/cli/cache/session.go | 87 ++++++ app/client/cli/cache/session_test.go | 75 ++++++ app/client/cli/cmd.go | 1 - app/client/cli/debug.go | 39 ++- app/client/cli/docgen/main.go | 3 + app/client/cli/helpers/setup.go | 8 +- app/client/cli/servicer.go | 62 ++++- app/client/cli/servicer_test.go | 88 +++++++ build.mk | 2 +- build/Dockerfile.client | 4 +- build/Dockerfile.debian.dev | 2 +- build/Dockerfile.debian.prod | 2 +- build/Dockerfile.localdev | 2 +- build/deployments/docker-compose.yaml | 2 + build/localnet/manifests/cli-client.yaml | 9 +- build/localnet/manifests/configs.yaml | 6 +- .../pocket/templates/configmap-genesis.yaml | 6 +- docs/development/CODE_REVIEW_GUIDELINES.md | 130 +++++++++ docs/development/GOLANG_UPGRADE.md | 35 +++ docs/development/README.md | 19 +- .../assets/github_line_comment.png | Bin 0 -> 14643 bytes .../assets/line_comment_dialog_start.png | Bin 0 -> 7335 bytes docs/development/assets/submit_review.png | Bin 0 -> 47729 bytes docs/devlog/devlog11.md | 100 +++++++ docs/devlog_agenda.md | 12 - e2e/tests/steps_init_test.go | 11 +- go.mod | 2 +- ibc/docs/ics24.md | 9 + ibc/events/event_manager.go | 58 ++++ ibc/host/submodule.go | 17 +- ibc/ibc_handle_event_test.go | 2 +- ibc/module.go | 2 +- ibc/store/bulk_store_cache.go | 6 +- ibc/store/provable_store.go | 4 +- ibc/store/provable_store_test.go | 14 +- internal/testutil/ibc/mock.go | 2 +- p2p/background/kad_discovery_baseline_test.go | 20 +- p2p/background/router.go | 27 +- p2p/background/router_test.go | 17 ++ p2p/transport_encryption_test.go | 2 + p2p/utils_test.go | 2 + persistence/actor.go | 18 ++ persistence/block.go | 47 ++++ persistence/db.go | 3 + persistence/debug.go | 3 +- persistence/ibc.go | 46 ++++ persistence/module.go | 19 +- persistence/sql/sql.go | 11 - persistence/test/actor_test.go | 147 +++++++++++ persistence/test/benchmark_state_test.go | 2 +- persistence/test/ibc_test.go | 248 ++++++++++++++++-- persistence/trees/module.go | 7 +- persistence/trees/module_test.go | 176 +++++++++++++ persistence/trees/trees.go | 81 ++++-- persistence/trees/trees_test.go | 5 + persistence/types/ibc.go | 39 ++- rpc/utils.go | 5 +- rpc/v1/openapi.yaml | 11 + runtime/bus.go | 10 +- runtime/configs/proto/p2p_config.proto | 2 +- shared/core/types/proto/block.proto | 16 +- shared/core/types/proto/ibc_events.proto | 21 ++ shared/k8s/debug.go | 49 +++- shared/modules/bus_module.go | 1 + shared/modules/doc/README.md | 91 +++++-- shared/modules/ibc_event_module.go | 21 ++ shared/modules/ibc_store_module.go | 4 +- shared/modules/mocks/mocks.go | 5 + shared/modules/persistence_module.go | 8 +- shared/modules/treestore_module.go | 7 +- 75 files changed, 1816 insertions(+), 219 deletions(-) create mode 100644 app/client/cli/cache/session.go create mode 100644 app/client/cli/cache/session_test.go create mode 100644 app/client/cli/servicer_test.go create mode 100644 docs/development/CODE_REVIEW_GUIDELINES.md create mode 100644 docs/development/GOLANG_UPGRADE.md create mode 100644 docs/development/assets/github_line_comment.png create mode 100644 docs/development/assets/line_comment_dialog_start.png create mode 100644 docs/development/assets/submit_review.png create mode 100644 docs/devlog/devlog11.md delete mode 100644 docs/devlog_agenda.md create mode 100644 ibc/events/event_manager.go create mode 100644 persistence/trees/module_test.go create mode 100644 shared/core/types/proto/ibc_events.proto create mode 100644 shared/modules/ibc_event_module.go create mode 100644 shared/modules/mocks/mocks.go diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml index 12355c894..ec4bf4ebb 100644 --- a/.github/workflows/golangci-lint.yml +++ b/.github/workflows/golangci-lint.yml @@ -10,7 +10,7 @@ permissions: env: # Even though we can test against multiple versions, this one is considered a target version. - TARGET_GOLANG_VERSION: "1.19" + TARGET_GOLANG_VERSION: "1.20" PROTOC_VERSION: "3.19.4" jobs: diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 9cfe65d92..24d809706 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -17,7 +17,7 @@ on: env: # Even though we can test against multiple versions, this one is considered a target version. - TARGET_GOLANG_VERSION: "1.19" + TARGET_GOLANG_VERSION: "1.20" PROTOC_VERSION: "3.19.4" jobs: @@ -25,7 +25,7 @@ jobs: runs-on: custom-runner strategy: matrix: - go: ["1.19"] # As we are relying on generics, we can't go lower than 1.18. + go: ["1.20"] # As we are relying on generics, we can't go lower than 1.18. fail-fast: false name: Go ${{ matrix.go }} test steps: diff --git a/.golangci.yml b/.golangci.yml index bf024354e..bf1cb4987 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -54,7 +54,7 @@ linters-settings: failOn: dsl rules: "build/linters/*.go" run: - go: "1.19" + go: "1.20" skip-dirs: - build/linters build-tags: diff --git a/Makefile b/Makefile index 1d1831bb0..003c72b11 100644 --- a/Makefile +++ b/Makefile @@ -43,6 +43,11 @@ kubectl_check: fi; \ } +.PHONY: trigger_ci +trigger_ci: ## Trigger the CI pipeline by submitting an empty commit; See https://github.com/pokt-network/pocket/issues/900 for details + git commit --allow-empty -m "Empty commit" + git push + .PHONY: prompt_user # Internal helper target - prompt the user before continuing prompt_user: @@ -159,7 +164,7 @@ rebuild_client_start: docker_check ## Rebuild and run a client daemon which is o .PHONY: client_connect client_connect: docker_check ## Connect to the running client debugging daemon - docker exec -it client /bin/bash -c "POCKET_P2P_IS_CLIENT_ONLY=true go run -tags=debug app/client/*.go debug --remote_cli_url=http://validator1:50832" + docker exec -it client /bin/bash -c "go run -tags=debug app/client/*.go DebugUI" .PHONY: build_and_watch build_and_watch: ## Continous build Pocket's main entrypoint as files change @@ -525,7 +530,7 @@ localnet_up: ## Starts up a k8s LocalNet with all necessary dependencies (tl;dr .PHONY: localnet_client_debug localnet_client_debug: ## Opens a `client debug` cli to interact with blockchain (e.g. change pacemaker mode, reset to genesis, etc). Though the node binary updates automatiacally on every code change (i.e. hot reloads), if client is already open you need to re-run this command to execute freshly compiled binary. - kubectl exec -it deploy/dev-cli-client --container pocket -- p1 debug --remote_cli_url http://pocket-validators:50832 + kubectl exec -it deploy/dev-cli-client --container pocket -- p1 DebugUI .PHONY: localnet_shell localnet_shell: ## Opens a shell in the pod that has the `client` cli available. The binary updates automatically whenever the code changes (i.e. hot reloads). diff --git a/README.md b/README.md index 5c324ec3a..9cd79b5a6 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,32 @@ The official implementation of the [V1 Pocket Network Protocol Specification](ht \*_Please note that V1 protocol is currently under development and see [pocket-core](https://github.com/pokt-network/pocket-core) for the version that is currently live on mainnet._\* +## Implementation + +Official Golang implementation of the Pocket Network v1 Protocol. + +
+ + + + +
+ +## Overview + +
+ + + + + + + + + + +
+ ## Getting Started --- diff --git a/app/client/cli/cache/session.go b/app/client/cli/cache/session.go new file mode 100644 index 000000000..6412459fa --- /dev/null +++ b/app/client/cli/cache/session.go @@ -0,0 +1,87 @@ +package cache + +// TODO: add a TTL for cached sessions, since we know the sessions' length +import ( + "encoding/json" + "errors" + "fmt" + + "github.com/pokt-network/pocket/persistence/kvstore" + "github.com/pokt-network/pocket/rpc" +) + +var errSessionNotFound = errors.New("session not found in cache") + +// SessionCache defines the set of methods used to interact with the client-side session cache +type SessionCache interface { + Get(appAddr, chain string) (*rpc.Session, error) + Set(session *rpc.Session) error + Stop() error +} + +// sessionCache stores and retrieves sessions for application+relaychain pairs +// +// It uses a key-value store as backing storage +type sessionCache struct { + // store is the local store for cached sessions + store kvstore.KVStore +} + +// NewSessionCache returns a session cache backed by a kvstore using the provided database path. +func NewSessionCache(databasePath string) (SessionCache, error) { + store, err := kvstore.NewKVStore(databasePath) + if err != nil { + return nil, fmt.Errorf("Error initializing key-value store using path %s: %w", databasePath, err) + } + + return &sessionCache{ + store: store, + }, nil +} + +// Get returns the cached session, if found, for an app+chain combination. +// The caller is responsible to verify that the returned session is valid for the current block height. +// Get is NOT safe to use concurrently +// DISCUSS: do we need concurrency here? +func (s *sessionCache) Get(appAddr, chain string) (*rpc.Session, error) { + key := sessionKey(appAddr, chain) + bz, err := s.store.Get(key) + if err != nil { + return nil, fmt.Errorf("error getting session from the store: %s %w", err.Error(), errSessionNotFound) + } + + var session rpc.Session + if err := json.Unmarshal(bz, &session); err != nil { + return nil, fmt.Errorf("error unmarshalling session from store: %w", err) + } + + return &session, nil +} + +// Set stores the provided session in the cache with the key being the app+chain combination. +// For each app+chain combination, a single session will be stored. Subsequent calls to Set will overwrite the entry for the provided app and chain. +// Set is NOT safe to use concurrently +func (s *sessionCache) Set(session *rpc.Session) error { + bz, err := json.Marshal(*session) + if err != nil { + return fmt.Errorf("error marshalling session for app: %s, chain: %s, session height: %d: %w", session.Application.Address, session.Chain, session.SessionHeight, err) + } + + key := sessionKey(session.Application.Address, session.Chain) + if err := s.store.Set(key, bz); err != nil { + return fmt.Errorf("error storing session for app: %s, chain: %s, session height: %d in the cache: %w", session.Application.Address, session.Chain, session.SessionHeight, err) + } + return nil +} + +// Stop call stop on the backing store. No calls should be made to Get or Set after calling Stop. +func (s *sessionCache) Stop() error { + return s.store.Stop() +} + +// sessionKey returns a key to get/set a session, based on application's address and the relay chain. +// +// The height is not used as part of the key, because for each app+chain combination only one session, i.e. the current one, is of interest. +func sessionKey(appAddr, chain string) []byte { + return []byte(fmt.Sprintf("%s-%s", appAddr, chain)) +} diff --git a/app/client/cli/cache/session_test.go b/app/client/cli/cache/session_test.go new file mode 100644 index 000000000..4b5afbaec --- /dev/null +++ b/app/client/cli/cache/session_test.go @@ -0,0 +1,75 @@ +package cache + +import ( + "os" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/pokt-network/pocket/rpc" +) + +func TestGet(t *testing.T) { + const ( + app1 = "app1Addr" + relaychainEth = "ETH-Goerli" + numSessionBlocks = 4 + sessionHeight = 8 + sessionNumber = 2 + ) + + session1 := &rpc.Session{ + Application: rpc.ProtocolActor{ + ActorType: rpc.Application, + Address: "app1Addr", + Chains: []string{relaychainEth}, + }, + Chain: relaychainEth, + NumSessionBlocks: numSessionBlocks, + SessionHeight: sessionHeight, + SessionNumber: sessionNumber, + } + + testCases := []struct { + name string + cacheContents []*rpc.Session + app string + chain string + expected *rpc.Session + expectedErr error + }{ + { + name: "Return cached session", + cacheContents: []*rpc.Session{session1}, + app: app1, + chain: relaychainEth, + expected: session1, + }, + { + name: "Error returned for session not found in cache", + app: "foo", + chain: relaychainEth, + expectedErr: errSessionNotFound, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + dbPath, err := os.MkdirTemp("", "cacheStoragePath") + require.NoError(t, err) + defer os.RemoveAll(dbPath) + + cache, err := NewSessionCache(dbPath) + require.NoError(t, err) + + for _, s := range tc.cacheContents { + err := cache.Set(s) + require.NoError(t, err) + } + + got, err := cache.Get(tc.app, tc.chain) + require.ErrorIs(t, err, tc.expectedErr) + require.EqualValues(t, tc.expected, got) + }) + } +} diff --git a/app/client/cli/cmd.go b/app/client/cli/cmd.go index 8e3f975bb..f9632cd0c 100644 --- a/app/client/cli/cmd.go +++ b/app/client/cli/cmd.go @@ -48,7 +48,6 @@ var rootCmd = &cobra.Command{ PersistentPreRunE: func(cmd *cobra.Command, args []string) error { // by this time, the config path should be set cfg = configs.ParseConfig(flags.ConfigPath) - // set final `remote_cli_url` value; order of precedence: flag > env var > config > default flags.RemoteCLIURL = viper.GetString("remote_cli_url") return nil diff --git a/app/client/cli/debug.go b/app/client/cli/debug.go index 93408a368..0bc7952f0 100644 --- a/app/client/cli/debug.go +++ b/app/client/cli/debug.go @@ -23,29 +23,27 @@ const ( PromptSendBlockRequest string = "BlockRequest (broadcast)" ) -var ( - items = []string{ - PromptPrintNodeState, - PromptTriggerNextView, - PromptTogglePacemakerMode, - PromptResetToGenesis, - PromptShowLatestBlockInStore, - PromptSendMetadataRequest, - PromptSendBlockRequest, - } -) +var items = []string{ + PromptPrintNodeState, + PromptTriggerNextView, + PromptTogglePacemakerMode, + PromptResetToGenesis, + PromptShowLatestBlockInStore, + PromptSendMetadataRequest, + PromptSendBlockRequest, +} func init() { - dbg := NewDebugCommand() - dbg.AddCommand(NewDebugSubCommands()...) - rootCmd.AddCommand(dbg) + dbgUI := newDebugUICommand() + dbgUI.AddCommand(newDebugUISubCommands()...) + rootCmd.AddCommand(dbgUI) } -// NewDebugSubCommands builds out the list of debug subcommands by matching the +// newDebugUISubCommands builds out the list of debug subcommands by matching the // handleSelect dispatch to the appropriate command. // * To add a debug subcommand, you must add it to the `items` array and then // write a function handler to match for it in `handleSelect`. -func NewDebugSubCommands() []*cobra.Command { +func newDebugUISubCommands() []*cobra.Command { commands := make([]*cobra.Command, len(items)) for idx, promptItem := range items { commands[idx] = &cobra.Command{ @@ -60,11 +58,12 @@ func NewDebugSubCommands() []*cobra.Command { return commands } -// NewDebugCommand returns the cobra CLI for the Debug command. -func NewDebugCommand() *cobra.Command { +// newDebugUICommand returns the cobra CLI for the Debug UI interface. +func newDebugUICommand() *cobra.Command { return &cobra.Command{ - Use: "debug", - Short: "Debug utility for rapid development", + Aliases: []string{"dui"}, + Use: "DebugUI", + Short: "Debug selection ui for rapid development", Args: cobra.MaximumNArgs(0), PersistentPreRunE: helpers.P2PDependenciesPreRunE, RunE: runDebug, diff --git a/app/client/cli/docgen/main.go b/app/client/cli/docgen/main.go index b69c7dfd2..6cdcf70c0 100644 --- a/app/client/cli/docgen/main.go +++ b/app/client/cli/docgen/main.go @@ -9,6 +9,9 @@ import ( "github.com/spf13/cobra/doc" ) +// TODO: Document that `Aliases` should be either dromedaryCase, one word lowercase, or just a few lowercase letters. +// TODO: Document that `Use` should also be PascalCase + func main() { workingDir, err := os.Getwd() if err != nil { diff --git a/app/client/cli/helpers/setup.go b/app/client/cli/helpers/setup.go index c91673b8d..8bc5e8ca6 100644 --- a/app/client/cli/helpers/setup.go +++ b/app/client/cli/helpers/setup.go @@ -23,7 +23,6 @@ const debugPrivKey = "09fc8ee114e678e665d09179acb9a30060f680df44ba06b51434ee4794 // P2PDependenciesPreRunE initializes peerstore & current height providers, and a // p2p module which consumes them. Everything is registered to the bus. func P2PDependenciesPreRunE(cmd *cobra.Command, _ []string) error { - // TECHDEBT: this is to keep backwards compatibility with localnet flags.ConfigPath = runtime.GetEnv("CONFIG_PATH", "build/config/config.validator1.json") configs.ParseConfig(flags.ConfigPath) @@ -31,6 +30,13 @@ func P2PDependenciesPreRunE(cmd *cobra.Command, _ []string) error { // set final `remote_cli_url` value; order of precedence: flag > env var > config > default flags.RemoteCLIURL = viper.GetString("remote_cli_url") + // By this time, the config path should be set. + // This is only being called for viper related side effects + // TECHDEBT(#907): refactor and improve how viper is used to parse configs throughout the codebase + _ = configs.ParseConfig(flags.ConfigPath) + // set final `remote_cli_url` value; order of precedence: flag > env var > config > default + flags.RemoteCLIURL = viper.GetString("remote_cli_url") + runtimeMgr := runtime.NewManagerFromFiles( flags.ConfigPath, genesisPath, runtime.WithClientDebugMode(), diff --git a/app/client/cli/servicer.go b/app/client/cli/servicer.go index 5787d2d7e..0ed35dff4 100644 --- a/app/client/cli/servicer.go +++ b/app/client/cli/servicer.go @@ -4,19 +4,39 @@ import ( "context" "encoding/hex" "encoding/json" + "errors" "fmt" "net/http" "github.com/spf13/cobra" + "github.com/pokt-network/pocket/app/client/cli/cache" "github.com/pokt-network/pocket/app/client/cli/flags" + "github.com/pokt-network/pocket/logger" "github.com/pokt-network/pocket/rpc" coreTypes "github.com/pokt-network/pocket/shared/core/types" "github.com/pokt-network/pocket/shared/crypto" ) +// IMPROVE: make this configurable +const sessionCacheDBPath = "/tmp" + +var ( + errNoSessionCache = errors.New("session cache not set up") + errSessionNotFoundInCache = errors.New("session not found in cache") + errNoMatchingSessionInCache = errors.New("no session matching the requested height found in cache") + + sessionCache cache.SessionCache +) + func init() { rootCmd.AddCommand(NewServicerCommand()) + + var err error + sessionCache, err = cache.NewSessionCache(sessionCacheDBPath) + if err != nil { + logger.Global.Warn().Err(err).Msg("failed to initialize session cache") + } } func NewServicerCommand() *cobra.Command { @@ -52,6 +72,12 @@ Will prompt the user for the *application* account passphrase`, Aliases: []string{}, Args: cobra.ExactArgs(4), RunE: func(cmd *cobra.Command, args []string) error { + defer func() { + if err := sessionCache.Stop(); err != nil { + logger.Global.Warn().Err(err).Msg("failed to stop session cache") + } + }() + applicationAddr := args[0] servicerAddr := args[1] chain := args[2] @@ -115,6 +141,25 @@ func validateServicer(session *rpc.Session, servicerAddress string) (*rpc.Protoc return nil, fmt.Errorf("Error getting servicer: address %s does not match any servicers in the session %d", servicerAddress, session.SessionNumber) } +// getSessionFromCache uses the client-side session cache to fetch a session for app+chain combination at the provided height, if one has already been retrieved and cached. +func getSessionFromCache(c cache.SessionCache, appAddress, chain string, height int64) (*rpc.Session, error) { + if c == nil { + return nil, errNoSessionCache + } + + session, err := c.Get(appAddress, chain) + if err != nil { + return nil, fmt.Errorf("%w: %s", errSessionNotFoundInCache, err.Error()) + } + + // verify the cached session matches the provided height + if height >= session.SessionHeight && height < session.SessionHeight+session.NumSessionBlocks { + return session, nil + } + + return nil, errNoMatchingSessionInCache +} + func getCurrentSession(ctx context.Context, appAddress, chain string) (*rpc.Session, error) { // CONSIDERATION: passing 0 as the height value to get the current session seems more optimal than this. currentHeight, err := getCurrentHeight(ctx) @@ -122,6 +167,11 @@ func getCurrentSession(ctx context.Context, appAddress, chain string) (*rpc.Sess return nil, fmt.Errorf("Error getting current session: %w", err) } + session, err := getSessionFromCache(sessionCache, appAddress, chain, currentHeight) + if err == nil { + return session, nil + } + req := rpc.SessionRequest{ AppAddress: appAddress, Chain: chain, @@ -148,7 +198,17 @@ func getCurrentSession(ctx context.Context, appAddress, chain string) (*rpc.Sess return nil, fmt.Errorf("Error getting current session: Unexpected response %v", resp) } - return resp.JSON200, nil + session = resp.JSON200 + if sessionCache == nil { + logger.Global.Warn().Msg("session cache not available: cannot cache the retrieved session") + return session, nil + } + + if err := sessionCache.Set(session); err != nil { + logger.Global.Warn().Err(err).Msg("failed to store session in cache") + } + + return session, nil } // REFACTOR: reuse this function in all the query commands diff --git a/app/client/cli/servicer_test.go b/app/client/cli/servicer_test.go new file mode 100644 index 000000000..cd84a1e87 --- /dev/null +++ b/app/client/cli/servicer_test.go @@ -0,0 +1,88 @@ +package cli + +import ( + "os" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/pokt-network/pocket/app/client/cli/cache" + "github.com/pokt-network/pocket/rpc" +) + +const ( + testRelaychainEth = "ETH-Goerli" + testSessionHeight = 8 + testCurrentHeight = 9 +) + +func TestGetSessionFromCache(t *testing.T) { + const app1Addr = "app1Addr" + + testCases := []struct { + name string + cachedSessions []*rpc.Session + expected *rpc.Session + expectedErr error + }{ + { + name: "cached session is returned", + cachedSessions: []*rpc.Session{testSession(app1Addr, testSessionHeight)}, + expected: testSession(app1Addr, testSessionHeight), + }, + { + name: "nil session cache returns an error", + expectedErr: errNoSessionCache, + }, + { + name: "session not found in cache", + cachedSessions: []*rpc.Session{testSession("foo", testSessionHeight)}, + expectedErr: errSessionNotFoundInCache, + }, + { + name: "cached session does not match the provided height", + cachedSessions: []*rpc.Session{testSession(app1Addr, 9999999)}, + expectedErr: errNoMatchingSessionInCache, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + var c cache.SessionCache + // prepare cache with test session for this unit test + if len(tc.cachedSessions) > 0 { + dbPath, err := os.MkdirTemp("", "cliCacheStoragePath") + require.NoError(t, err) + defer os.RemoveAll(dbPath) + + c, err = cache.NewSessionCache(dbPath) + require.NoError(t, err) + + for _, s := range tc.cachedSessions { + err := c.Set(s) + require.NoError(t, err) + } + } + + got, err := getSessionFromCache(c, app1Addr, testRelaychainEth, testCurrentHeight) + require.ErrorIs(t, err, tc.expectedErr) + require.EqualValues(t, tc.expected, got) + }) + } +} + +func testSession(appAddr string, height int64) *rpc.Session { + const numSessionBlocks = 4 + + return &rpc.Session{ + Application: rpc.ProtocolActor{ + ActorType: rpc.Application, + Address: appAddr, + Chains: []string{testRelaychainEth}, + }, + Chain: testRelaychainEth, + NumSessionBlocks: numSessionBlocks, + SessionHeight: height, + SessionNumber: (height / numSessionBlocks), // assumes numSessionBlocks never changed + } +} diff --git a/build.mk b/build.mk index 5f80c7f32..dbe51ee20 100644 --- a/build.mk +++ b/build.mk @@ -2,7 +2,7 @@ OS = $(shell uname | tr A-Z a-z) GOARCH = $(shell go env GOARCH) ## The expected golang version; crashes if the local env is different -GOLANG_VERSION ?= 1.18 +GOLANG_VERSION ?= 1.20 ## Build variables BUILD_DIR ?= bin diff --git a/build/Dockerfile.client b/build/Dockerfile.client index dfd5ee50a..0999c339b 100644 --- a/build/Dockerfile.client +++ b/build/Dockerfile.client @@ -1,4 +1,4 @@ -ARG GOLANG_IMAGE_VERSION=golang:1.19-alpine3.16 +ARG GOLANG_IMAGE_VERSION=golang:1.20-alpine3.16 FROM ${GOLANG_IMAGE_VERSION} AS builder @@ -14,4 +14,4 @@ RUN apk add --no-cache bash # Hot reloading RUN go install github.com/cespare/reflex@latest -CMD ["/bin/bash"] \ No newline at end of file +CMD ["/bin/bash"] diff --git a/build/Dockerfile.debian.dev b/build/Dockerfile.debian.dev index e4f3d9625..387ef70ca 100644 --- a/build/Dockerfile.debian.dev +++ b/build/Dockerfile.debian.dev @@ -1,7 +1,7 @@ # Purpose of this container image is to ship pocket binary with additional # tools such as dlv, curl, etc. -ARG TARGET_GOLANG_VERSION=1.19 +ARG TARGET_GOLANG_VERSION=1.20 FROM golang:${TARGET_GOLANG_VERSION}-bullseye AS builder diff --git a/build/Dockerfile.debian.prod b/build/Dockerfile.debian.prod index 054fd0518..9e305b3f2 100644 --- a/build/Dockerfile.debian.prod +++ b/build/Dockerfile.debian.prod @@ -1,6 +1,6 @@ # Purpose of this container image is to ship pocket binary with minimal dependencies. -ARG TARGET_GOLANG_VERSION=1.19 +ARG TARGET_GOLANG_VERSION=1.20 FROM golang:${TARGET_GOLANG_VERSION}-bullseye AS builder diff --git a/build/Dockerfile.localdev b/build/Dockerfile.localdev index 7595f8240..b337472cd 100644 --- a/build/Dockerfile.localdev +++ b/build/Dockerfile.localdev @@ -1,4 +1,4 @@ -ARG GOLANG_IMAGE_VERSION=golang:1.19-alpine3.16 +ARG GOLANG_IMAGE_VERSION=golang:1.20-alpine3.16 FROM ${GOLANG_IMAGE_VERSION} AS builder diff --git a/build/deployments/docker-compose.yaml b/build/deployments/docker-compose.yaml index 4c4caab48..366967711 100755 --- a/build/deployments/docker-compose.yaml +++ b/build/deployments/docker-compose.yaml @@ -166,6 +166,8 @@ services: security_opt: - "seccomp:unconfined" environment: + # BUG: The `SERVICER1_SERVICER_ENABLED` env var is not currnetly visible in the `command` above and needs to be investigate + - SERVICER1_SERVICER_ENABLED=true - POCKET_RPC_USE_CORS=true # Uncomment to enable DLV debugging # - DEBUG_PORT=7085 diff --git a/build/localnet/manifests/cli-client.yaml b/build/localnet/manifests/cli-client.yaml index 719668ac6..c173910aa 100644 --- a/build/localnet/manifests/cli-client.yaml +++ b/build/localnet/manifests/cli-client.yaml @@ -38,8 +38,6 @@ spec: memory: "512Mi" cpu: "4" env: - - name: POCKET_P2P_IS_CLIENT_ONLY - value: "true" - name: CONFIG_PATH value: "/var/pocket/config/config.json" - name: GENESIS_PATH @@ -75,9 +73,10 @@ spec: value: validator1 # Any host that is visible and connected to the cluster can be arbitrarily selected as the RPC host - name: POCKET_REMOTE_CLI_URL - value: http://full-node-001-pocket:50832 - # TECHDEBT(#678): debug client requires hostname to participate - # in P2P networking. + # CONSIDERATION: Should we use a validator or full node for this? + value: http://pocket-validators:50832 + # value: http://full-node-001-pocket:50832 + # TECHDEBT(#678): debug client requires hostname to participate in P2P networking. - name: POCKET_P2P_HOSTNAME value: "127.0.0.1" volumeMounts: diff --git a/build/localnet/manifests/configs.yaml b/build/localnet/manifests/configs.yaml index 29bc520df..da8a091e7 100644 --- a/build/localnet/manifests/configs.yaml +++ b/build/localnet/manifests/configs.yaml @@ -1688,7 +1688,7 @@ data: "address": "00104055c00bed7c983a48aac7dc6335d7c607a7", "public_key": "dfe357de55649e6d2ce889acf15eb77e94ab3c5756fe46d3c7538d37f27f115e", "chains": ["0001"], - "service_url": "validator-001-pocket:42069", + "service_url": "http://validator-001-pocket:50832", "staked_amount": "1000000000000", "paused_height": -1, "unstaking_height": -1, @@ -1699,7 +1699,7 @@ data: "address": "001022b138896c4c5466ac86b24a9bbe249905c2", "public_key": "56915c1270bc8d9280a633e0be51647f62388a851318381614877ef2ed84a495", "chains": ["0001"], - "service_url": "servicer-001-pocket:42069", + "service_url": "http://servicer-001-pocket:50832", "staked_amount": "1000000000000", "paused_height": -1, "unstaking_height": -1, @@ -1710,7 +1710,7 @@ data: "address": "00202cd8f828a3818da2d24356984120f1cc3e8e", "public_key": "6435e4d5d1ace32f187ea0cd571dc4fda767638d69dcec3b5a6ac952777d142d", "chains": ["0001"], - "service_url": "servicer-002-pocket:42069", + "service_url": "http://servicer-002-pocket:50832", "staked_amount": "1000000000000", "paused_height": -1, "unstaking_height": -1, diff --git a/charts/pocket/templates/configmap-genesis.yaml b/charts/pocket/templates/configmap-genesis.yaml index 8d0ed0780..29705e497 100644 --- a/charts/pocket/templates/configmap-genesis.yaml +++ b/charts/pocket/templates/configmap-genesis.yaml @@ -1694,7 +1694,7 @@ data: "address": "00104055c00bed7c983a48aac7dc6335d7c607a7", "public_key": "dfe357de55649e6d2ce889acf15eb77e94ab3c5756fe46d3c7538d37f27f115e", "chains": ["0001"], - "service_url": "validator-001-pocket:42069", + "service_url": "http://validator-001-pocket:50832", "staked_amount": "1000000000000", "paused_height": -1, "unstaking_height": -1, @@ -1705,7 +1705,7 @@ data: "address": "001022b138896c4c5466ac86b24a9bbe249905c2", "public_key": "56915c1270bc8d9280a633e0be51647f62388a851318381614877ef2ed84a495", "chains": ["0001"], - "service_url": "servicer-001-pocket:42069", + "service_url": "http://servicer-001-pocket:50832", "staked_amount": "1000000000000", "paused_height": -1, "unstaking_height": -1, @@ -1716,7 +1716,7 @@ data: "address": "00202cd8f828a3818da2d24356984120f1cc3e8e", "public_key": "6435e4d5d1ace32f187ea0cd571dc4fda767638d69dcec3b5a6ac952777d142d", "chains": ["0001"], - "service_url": "servicer-002-pocket:42069", + "service_url": "http://servicer-002-pocket:50832", "staked_amount": "1000000000000", "paused_height": -1, "unstaking_height": -1, diff --git a/docs/development/CODE_REVIEW_GUIDELINES.md b/docs/development/CODE_REVIEW_GUIDELINES.md new file mode 100644 index 000000000..8b8a823d2 --- /dev/null +++ b/docs/development/CODE_REVIEW_GUIDELINES.md @@ -0,0 +1,130 @@ +# Pocket's Code Development & Review Guidelines + +_This document is a living document and will be updated as the team learns and grows._ + +## Table of Contents + +- [Code Quality](#code-quality) +- [Code Reviews](#code-reviews) + - [Code Review Guidelines](#code-review-guidelines) + - [Expectations](#expectations) + - [Best Practices](#best-practices) + - [Smaller PRs](#smaller-prs) + - [Ordering Commits](#ordering-commits) + - [Approving PRs](#approving-prs) + - [Review Comments](#review-comments) + - [Figure 1: Inline Github Comment](#figure-1-inline-github-comment) + - [Figure 2: Line Comment Dialog](#figure-2-line-comment-dialog) + - [Finishing a Review](#finishing-a-review) + - [Figure 3: Submitting A Review](#figure-3-submitting-a-review) + - [Merging](#merging) + +## Code Quality + +_tl;dr Code Quality is an art moreso than a science._ + +`Code Quality` can be a vague concept, as it usually addresses what is more the `art` side (vs. the `science` side) of software development. In this document, we outline a framework to guide that human judgement towards -- collectively -- `better code`. + +There are often several _technically correct_ ways to address a problem -- that is, the correct answer or behavior is produced. +Selecting the "_best_" solution is often a matter of style. Sometimes, the best solution is one that fits the surrounding code in the most cohesive way. + +Terms like `maintainability` or `readability` are used; these address the ability of other contributors to understand and improve the code. Unlike correctness or performance concepts, there's no single metric or mathematical solution that can be optimized to achieve better code quality. Thus, we rely on human judgement. + +For decades, the `IETF` (Internet Engineering Task Force) has used the motto `rough consensus and running code`. This motto encapsulates (`running code`) that developers' core output is still software: if there is no code that runs and produces correct results, we have nothing. It also encapsulates (`rough consensus`) that we may not always precisely agree and that's okay. + +## Code Reviews + +_tl;dr Code Reviews are a necessary evil, and there are no specific guidelines that will university apply everywhere all the time._ + +One tactic often employed to produce `maintainable` or `more-readable` code is a `code review`. These can take many shapes and forms and often have goals beyond simply `code quality``. + +Broadly speaking, code reviews involve developers looking at some proposed new code (or code changes). This is _usually_ developers other than the author (although a `self-review` can also be employed). In many projects, such attention is a scarce commodity: most programmers would rather write code than read someone else's. + +Remember, writing code is the **fun part** but reading code is **work part**, so try to make it as easy for the reviewer as possible. + +### Code Review Guidelines + +All participants must adhere to the overall idea that the review is an attempt to achieve `better`` code. This is a vague statement on purpose. + +Participants must be cautious in their criticism and generous with praise. + +Participants must remember the scarcity of another developer's attention. + +### Expectations + +**Pull Request Authors:** The author is responsible for tracking up-to-date next actions for a Pull Request to progress towards being merged. + +**Reviewers:** Reviewers should prefer engaging in code review over starting new work (i.e. taking planned work items that haven't been started yet). + +**Reviewers (and prospective reviewers)** are encouraged to engage in reviews of codebases outside the projects and technologies they use on a day-to-day basis (but not expected to provide an approving review). + +### Best Practices + +#### Smaller PRs + +Consider if it could be broken into smaller Pull Requests. If it is clear that it can be, summarize your thinking on how in your Review. + +#### Ordering Commits + +If the commits be (re)organized (i.e. reordered and/or amended) such that there is a commit at which the tests are passing prior to the conclusion of the main change, that's a signal that there's likely a logic split which can be made at that point in such a (re)arrangement. + +#### Approving PRs + +Use the following guidelines to evaluate whether a Pull Request should be approved: + +- _Every_ Pull Request should have tests or justification otherwise; esp. bug fixes. +- _Every_ Pull Request should have at least 1 approval from a team member internal or external to the project. Exceptions made by repository maintainer(s), as necessary, on a case-by-case basis. + +### Review Comments + +_tl;dr Use `SOS`, `TECHDEBT`, `TECHDEBT(XXX)`, `NIT` if you think it'll help the author get context on the next steps._ + +When leaving review comments, consider if any of the following characterizations applies and prefix the comment, respectively: + +- `NIT`: Comment is a nitpick +- `TECHDEBT`: Comment should have a TECHDEBT comment w/o a ticket +- `TECHDEBT(XXX)`: Comment should have a TECHDEBT comment but imporant enough that it requires a ticket to track the work in the near future +- `SOS`: Show Stopper. You feel strongly enought that it needs to be addressed now. + +During review, submit feedback using line comments (Fig1); prefer `Add\[ing a\] single comment` over `Start[ing] a review` (Fi2). Once a review has been started, the option to add single comments is removed. Preferring single comments allows feedback to happen even in the event of an interrupted review. + +### Figure 1: Inline Github Comment + +![github_line_comment.png](./assets/github_line_comment.png) + +### Figure 2: Line Comment Dialog + +![line_comment_dialog_start.png](./assets/line_comment_dialog_start.png) + +**Referencing Issues Across Repositories:** When referencing issues from one repository, in another's Issues and Pull Requests, GitHub supports automatic links in markdown using the following format: `/#`. + +### Finishing a Review + +Write a summary of your observations and consider including positive remarks in addition to any constructive criticism. + +If you observe a deviation from these practices or another reason that this change should not be merged, select `request changes` and include a summary of the observation(s), as well as any practice(s) you find them to be in conflict with, in the review body (Fig3). + +If you don't feel comfortable giving approval or requesting changes but want to share a summary or observations of larger patterns in the codebase or the company, select "Comment" and submit your review (C). + +Confirm that all items in the `required checklist` are checked or not applicable. + +If you believe the Pull Request looks good to merge, select "Approve" and submit your review (Fig3). + +### Figure 3: Submitting A Review + +![submit_review.png](./assets/submit_review.png) + +### Merging + +1. Utilize the `Squash & Merge` feature (maintain a clean history) +2. Copy the `Github PR Description` into the commit message (add sufficient detail) + +**Authors are core members or regular external contributors:** + +- Core member needs to approve PR +- Author should merge in the PR themselves (following instructions above) + +**Authors are non-regular external contributors:** + +- Core member needs to approve PR +- Core member can merge PR on behalf of contributor (following instructions above) diff --git a/docs/development/GOLANG_UPGRADE.md b/docs/development/GOLANG_UPGRADE.md new file mode 100644 index 000000000..793971af5 --- /dev/null +++ b/docs/development/GOLANG_UPGRADE.md @@ -0,0 +1,35 @@ +# Checklist to upgrade Go version + +A short guide for carrying out Go version upgrades to Pocket V1 + +## Previous upgrades + +A list of upgrades from the past, which can be used as a reference. + +* 1.20 upgrade: [#910](https://github.com/pokt-network/pocket/pull/910) + +## File Locations + +- [ ] go.mod +- [ ] build.mk +- [ ] Makefile +- [ ] README.md +- [ ] .golangci.yml +- [ ] .github/workflows + - [ ] main.yml + - [ ] golangci-lint.yml +- [ ] build/ + - [ ] Dockerfile.client + - [ ] Dockerfile.debian.dev + - [ ] Dockerfile.debian.prod + - [ ] Dockerfile.localdev +- [ ] docs/development + - [ ] README.md + +## Testing + +- [ ] LocalNet builds and runs locally +- [ ] LocalNet E2E tests pass +- [ ] GitHub Actions CI tests pass +- [ ] Remote network (such as DevNet) is functional and E2E tests pass +- [ ] Update this document with current Pocket Go version \ No newline at end of file diff --git a/docs/development/README.md b/docs/development/README.md index 0aea46ad1..3f62d8a6b 100644 --- a/docs/development/README.md +++ b/docs/development/README.md @@ -17,6 +17,7 @@ Please note that this repository is under very active development and breaking c - [Profiling](#profiling) - [Code Organization](#code-organization) - [Maintaining Documentation](#maintaining-documentation) +- [Code Review Guidelines](#code-review-guidelines) - [Documentation Resources and Implementation](#documentation-resources-and-implementation) - [Your Project Dashboard](#your-project-dashboard) - [Github Labels](#github-labels) @@ -55,7 +56,7 @@ which protoc-go-inject-tag && echo "protoc-go-inject-tag Installed" # protoc-go-inject-tag Installed go version -# go version go1.18.1 darwin/arm64 +# go version go1.20.5 darwin/arm64 mockgen --version # v1.6.0 @@ -85,9 +86,7 @@ Optionally activate changelog pre-commit hook cp .githooks/pre-commit .git/hooks/pre-commit chmod +x .git/hooks/pre-commit ``` - -_Please note that the Github workflow will still prevent this from merging -unless the CHANGELOG is updated._ +_**NOTE**: The pre-commit changelog verification has been disabled during the developement of V1 as of 2023-05-16 to unblock development velocity; see more details [here](https://github.com/pokt-network/pocket/assets/1892194/394fdb09-e388-44aa-820d-e9d5a23578cf). This check is no longer done in the CI and is not recommended for local development either currently._ ### Pocket Network CLI @@ -320,6 +319,18 @@ To keep the Wiki organized, a comment is added at the end of each `.md` file. Fo If you are adding a new `.md` file for documentation please included a similar comment. Use your best judgment for the category and subcategory if its a new directory. Otherwise, copy the comment from a similar file in the directory and choose a relevant filename. +## Code Review Guidelines + +- [Code Quality](./CODE_REVIEW_GUIDELINES.md#code-quality) +- [Code Reviews](./CODE_REVIEW_GUIDELINES.md#code-reviews) +- [Code Review Guidelines](./CODE_REVIEW_GUIDELINES.md#code-review-guidelines) + - [Expectations](./CODE_REVIEW_GUIDELINES.md#expectations) +- [Best Practices](./CODE_REVIEW_GUIDELINES.md#best-practices) + - [Reviewing](./CODE_REVIEW_GUIDELINES.md#reviewing) + - [Starting a Review](./CODE_REVIEW_GUIDELINES.md#starting-a-review) + - [Finishing a Review](./CODE_REVIEW_GUIDELINES.md#finishing-a-review) + - [Merging](./CODE_REVIEW_GUIDELINES.md#merging) + ## Documentation Resources and Implementation ### Your Project Dashboard diff --git a/docs/development/assets/github_line_comment.png b/docs/development/assets/github_line_comment.png new file mode 100644 index 0000000000000000000000000000000000000000..4094aa38e6e14283f1dcace8c1151392986a22cc GIT binary patch literal 14643 zcmbum1#IO&*QS{cV~3fUnVFdxJJ4agVTKMfGcz+YGc$E=k`6O7Z@zyOFn3ma;ICk;c4L#P3t$X6$dfeSYC6D+K5lf~!e85t$*B`G=@I_28y0U8>I zDQ_uoh7l!y;E2XbruwEnKA2gVY*?rQxw2;OfRb6rr@Cd zoB3_q_(;&;Ey7{CM0Kh`Z4jWm;{U(H;hWQkp|?#ae<4FKJXYG5tt$a7@no5WkIMj| zYawEwqko?Q@-7DSIU2Fu|C`XEYa73g9VI;L(++H-aCclRI1xZffD6oov{MP6fF__D z@_;(AN!alpLwDtzR3zVBYd?32VaFP(V`z4OvJj61eJlq}noHsecZ2dMYN6UHEMrzKZUeyP81Y6_B0l*EbppNbpE~I3r!T z+37H*;_VqoO212gd1VdS-0DBFZgJ{P+Mod(efS!++O><}bw2Sd7@dm(Bi06v{Bk5y z-pj8Z2)|x92IyH5mKfHDQ>0H+ikGT(miarmRjVlhZq>1fe@e&|pI$>(d*1^K`3Vcs zlUE9wPMJyLC(;oJZ)f}3qsr3Vw!>z(_Y)&?mh6@zk$s-(pIcGXvqdRB*)F!zC3)0D zA@VHcT2uV)CJ(iKlcm!%sL|LakKXzA*6B`GhsEwzNn%yQhrYarn(#!q2RiuhaoZ1t zXGDWuONH-+pquU%FJL!I6`{Bh|499#qgMY6EZ2Z`LD1w%Rcd+Al0Alt)MC5yi#lIV z5?bEutYFCBbO+Me5Yc{u>po?PZniNa$lJMDHmPsTynPJ$4&!4^(gOeuisuJ9iFRyi zGGjOhb{Qa`K0xKd<_fu!d{TAVz5T-e?}i{$bWeuKR3NnmDkaFsbTnhCS}r^J+4Ho7 za8h(l1YzeQEBipJhO$i}ds`}fe&2o2Zqu=v1HW5hFgdawoC5(|NG5tu!jQgV((+s~ zu&7(Z2HORO!YPYnh1u?}x zfv`6%v|tsVs_@Tc@yW#?Yn0Y##%;5;V9f3n?Mm&eKe~v&XTQ-YVD0b_-MB zTp3vdiVAZZ=V3c14j56f9tAoA`>*QGG@05Q5T0nC)DXANWKjNaB~)V57J#s(~K*a=(hvoxp8@WZ6P;S4Uh6-BIQEMel$H=lXG+E)v;mHGv? z%d2^n31;6u2EPB6Krj9b?wZ}$eNJq?5{H>1Vq=|@GC9ivH^BI0VUx|@wwR|uRi-kq zvP~nd+NZLd(kwCqvRe^~Q?p-|24hy#)@St?*a+2>-HK--J&`Z3@>|=PZhXB~@5CTo z1b&fj^0pYO=HiJva=%G&TdNO0uA(Eg_(XMCobUf+iPNmKxKK{)dGs2rJ;1V$rqNFs z_(fC93W^?qxRKd{milgwwd}VStEp8}fbTvGfUk{k?!v!(OmNf_y19fy zclv+Vy4wiZ$W*vn&cPG}m;;Zj1+XTO2!;eS!T(!M0h;}P$}D&Q&n+J=V8aml)!Qb7 z|37~bt^aw(sBdorWD5?@gUuKH?I89Sii64w*PZB+9!O8r3tL>T2#572@g;2J<(s*l zc6jQf4S<5r4C6hRVvfmalc?Z{%R|!CWxM%PXu4c=C)Z`u=ETe$p3REU*xX;)#W}Zq zpC7YX`%thmR$ZjTB$Be1uFj|K2Tdr3eb(b)Px^Jn`A0vRcd_{aNn#2(0%3dUdsKp$ zZ)|atRV})=1@G=jVU0+c@rpFa>C-u1(=WC1Yy9G>n0xS8s~2kJuG6dCP3KkA!|2>epU5lbf=0EGt`yUS zwR~79{JJ?BSONX`tiNtGuPvt1hy^geBz_EdDCVtP?F!kWPjp zQlxz9c#pLw4W_D`-?gm9`6|=-8^N>TWorUXbH#28Z-^KbQn{+z0nY03G7L<@T(0(H z5i+jy6@QKg+?gu1D`91+PoJ!`f_Gz+u0r-n|A*GcO(QrpF)eXOVJ@BNPT1EDR!=~_ z?}+E-uiwv)sj~1s)X>r~CX+GXz$_V8y6THq_9CdKy}?*0mC0RJY;p#TVD*+yEHmZ+ z_Fh5J6z_=Sk0Xb}^kO=~z%5+mmPWMS_I0;JJU#uC96~gGygrJ`Qf- zX2R7;(cMzpb}!pb=Ne!fWz)Wima}gxfAnfQwee8 zM+ZJA7RO1Gi(aw5%KCQ)-b{H`5*RGkkUvJ|7EVlG*W}r~s~1ECE`G0vlVi%o`s!II zHL7WaIcTY3DSqfy<<5AY`{qA2uV(Mp`7N|6A^Y)j%?^_|s6K&Xh?q`qMe1I=*#E>$ zb0^dgY^V0F_!PZ9v|GM>2DOA73pCzFOunvhhGZMjvQZqCfanVxz1vU^BjvE{}B5A&~F2U1uz=x3UGwk*qQ1^OeNZU&<67ijzWI{2zXI}k$Fcd6ja+I2}ctn+s|XP4006`Y}u zov0J*rK?6Ok4wXA<@EWLuqfDC*&DSWs%PV2^Hnn(3D%6h#GcjnCbKAdeLw76z%f(e zmkvf9lHHHxvI6;CnSv;iJLtR4pN7|^=0DnecW}!6`@ttLxm;P$UG6Y~w9c;<8t9Ee zuzqIIG5BYqLYms;-e2%8QkzoZDwa4Lhl;vCT4+`s#UDM7&p)l@E(~WF9~;QwKSk<# z`5w)lAKqJPWdQ*mPg8SRnnr?pN5|>fB7KW+lS@qYK4chbz9FU z0dG7)Kk?-=;^sENS=&PLArBWs=5*>UA~{w&S9d^ZFmic%(Zgeg+qdFS?x};}9&pf) z+J7jkH@hyzpVg@lXta8FanV;UMm65asG`GX*|soI{m-qT1jjcy@#RSd=O{KuXcNHn z;Kw~MAGBft7<(4fN2da!$OMzERC=4ylj77u36)NY{HmlEO_9&bZ3S)AFiB#&TqQ`w z>^++0*u%PFypiwmWC`$UwEs*5Pw$fQcnalv6JE*oJNk9lJ_~(04JDg9#d^rm=Alh5 zbrxgvGDz%8s2f9N(OUvzMRqS>d%R^2(a1??uA`+or`Wj@Wx`iS|H1uL z5jx+o0i8~>bmL~|WvO`*Qaf*PWZjaOy{S!-I83`VUjeM%Y;o+xU-OpcyzrXe`2_! zus%fJ>+*><>WY7`R#S)0|5pW*pB|GkW81)ti7RMcUPvyfUXOgzfkKB(jp?y0Fy;I(68A)pVG2Z6e+k-A zl(0Q{_e#P4CBMpd3DibsfKp(9G5|7sHjruuU3I;yt z_KvOhyt~5N0tgS!d^}Pk*2RFy1Xfaup_%9%>_2^(?kGzxdZ$Da8SmEB6En*vcb9_c z%BtAbi6Jg{yp$d!V7oa>9ni5^>>76#;n*?>6_XCM>Jq?w8*=1JS=yXgBhO%3x!t+1 z82I8xj-9v+p6JM1GI#$|JK=Dr)_Fdq(B{u~EtPMGA?Be_egnUux{_GyyWkU^nGmfTC#ADYtP& zHplgEZX&jEIhp&iOBi2b^l--JuR~apKR;X)`Nw-@p$xBdv1{~=un6HomRb7twdI8l zc-n-?D&bUsdY%224#lIwKpsqy?{=$}V7|+|v1&OE7)JE()jx>oO-0POBkCQYg(;~C z_eZB*wYN)NDY;zp&clp!CpKB$&e2-~nVYR>7K3{a&F}XrO8&EOgWtWivp4!+XmFy% zmgR$1jt)Y7^{Au0?KYnxOT?LzgK32I*;J<&o-1 zp7*1_6gZaF57PQ$M7q87_^-K(H+FbL6hsbia?!9(K2pkhuO;Y`F~;jfNq$)$@uXiQ zl=Xg>C_QhCjmIQ1nmWhXD!u;%^@sT{k)a;Y zD@Ih-2kig{?izvADr8vtzZ0&V``C%g(9L5eNGVK%CZi^^^*o1a6z?4!N+f`!4}DqadRz3&*4Ow)doye#I3>4EnIcn%{jAeY}{y?8-AY6gec^HH zBLt_Jm#X9_O-jbtIMH$(sct$g?`>Iv+%@`;Y#n za#`ro&FO?JjpCPoF1A!4v}2o3yc`$lP2-Uie?PFDQe}AZ9fPu_ z4&ch$T#n`82n%2DX%wvl1QV!5|Du-B^EFmv2}Xh-HG87^X`YTGK?GW$ssx@!cV16~ zs(SR43`OHo&>EWZVYsYv z-+R_2#1pe6^s37^04axkIP-AwI^*$;kP$N{5R_d$X1meu058Al>c!>`*`usY0G8p? z)76*(q~VhO$Tt)4YJ$iW0*ySLk?Qkad`c5BE!@7&xSOr0pa`6GKiVexibS%>o+fuJvrBg4>?m7FE&;!(`yZ$GHA-0 zq!BN(S)t;RQ3luL&$Qctze)-@_TgnV z4TO@%U^INGBD8#5&f{})$7Xz|rInKHlAw428dv{R^RT_kUVuT*JREVxH+gk6pQ$K! zma&yk`SR#u?u6K$zDgSn*tV|7Tc9xi2?7K9UM4UYyYHuOGX8gukG&FG$@@QTAKP4> z4KJX6ttxC&T_NuIi^E|u5j9~xemHt>)Nuh8yERddW)N!gzrN`|q|R)&H$^kmo+QDB z4xDK#9;cHKZoyQP*|`)x-UI1Vv7knBnp`+4^?d2cQ;VM$#$U#hUMG~l^L`&JqR)R%RU4_|sjm4R6O%Zaz9mlvL8k28StU4imvC)$Hm>E^GOW%z=@~8N^uAmBt>i*{tb~ z$aYM7%OY47C&m*my0=8U)S>7gT)j@D{u*)FHT#)p0e&3Jzh<9Fa&BJUm+Z}nQk3@~ zB(nYR`s|2=IxB}wZy`7K}!fDgXodq1!;7XL;Z!Pv{2ZZ zdEGRN)7UEKe$F=JYyrKguo-dqi`!z8jO?bp%NWHT&mIDUQZYpgF`y&wzrG-08xKdY zt5zqfd#}*v>L?!*{;jhfGo3Vd&9VX|X1dnwMb8_~rk=qkNO>Y(yKy{T@0h8&gw2mh zbQ1Ns>~S~mpo1EsqW7LCPIKD>=Yklzqn562fNRdf0q9#HFibb)3W#7mOfS6~fDZjn zp!1+TASI5hT+1IbluR1SgnRS^O$1ZT9M(-WZEAsxo2jf8KbQeO_0CDcl{gHsy)Z<<)Q~Mk%MD@rk}gW@ z%oBKZG=X-RiGn<2tPBdvzQZa+K!i{_AN-VceKe{*DM#lYje$1s3CK*0oDCbj1qyqE#HawuY#mCxwVpuT&nO~g%hX0kTRwk`b6$XRH~EoM$VId zJcVqZ%8(na=C?Z5#5Tg>{Uy6Ifad8A9!Gg$>FVpri|qc!eY+(xp?LSH63Lk^?N~g!qJ`GtX7m2a8%>vwx zp=5y~kYMc?Kw;${1~6!9R5m1b^bBpm?y8#`!xqHLm(OpV_7^rG)C*?o5gQ;@Qw| zX|?0DowF0F_B{8+?I2c2*M9BOW2nK~@Z^_-^^Vn#nZUDfL9G-i$SX3T@#k}N1>#n| zd7^6@G=h-N7gk_Azt0}abyRohJ4^0o;`su8LK(aTXxozG1vO)+f!NU41p<)rFMk{- zBua;U#QGKH3ICcU;Xh+9wVP)VvdloA9d1krDEXAJ+e+ca(PZMdjSxL?dt7lEq9gE! zq4~+-hjv)xJTShB`Tdj5;P;4Bfw4J04=0mkUQK$ztlJ^QQEiNg9!FW99)yHRA!+<9OStIgns13QHaU{z<}r z&LW_~4r^k-9Dy7z$stDpVd=b9xYH^4T}C}hQSERVJA1&T!}!yO`=HzaJwg=-x}%YxUYr>Atg4+`{oHVb^5#g2~V^58NG9eI0k z=|)3Zr}YY>8+m(NQyz&jMPzdnt-!QHWPY*1_||w&YIS5USc1)EXcDTdJW-w7gbjHK zydjFm>}yCuZ>mqKZm*K$UbHw%4KpS*4J-$0D_MGDSegJ6wyJi3oe@fZeh3iJwkbH6 zXO2}aJPWmNcnv$y5;+pf3+@~{Q`b%!6np%g1>?};%vAA*6)vnE{oRWy0~JVcxleEs z=TyCr;p`e;NX7}HaZqs<6U@I$P~B1yd#RJo<7g2)Zk}a)w^U%RIfx0hn~Hpp4$2xY zhrCRjq)>y6H}MqFk-#0t$Hx&hn6BdBPGw(nXH$hby7P@&5x|GM8;2%ZKR_g&mrn_6 z8?w=PI9QjV&iuP0kxpR3GP#4|9+2RO@rf7az={fVnA$U%f)pqv5ae^>wI!Ch%fPy` z&PpI7rA8Z5sYx?Ul(qBaof6JRVvL18C&&~ynvov}9quhr>l~9Rf#C=iiFX21k}{0Q z40xeu2qw!Xs*SW2%pB8{BBe6~s$0Wtg zXhcFCm1x`GV4p+jna@34n+`ODD>>m_)cX6}CeK382@zbx)J%MEPEST(MwPUFy9i{b zQ;GSdK{Z9tk%ZcYqzxII)OZapEs}7jW2c?qs1L^eWgF*6T7_i*-%oXpmmgW*lJ{G= zBxU1RB9JcArrczTuOQBG^ysj@bsYyxqV?~j9T$$^l-m^r585$aRqft#ytwtFEW2N8>??{LdCLX$d1-m#BWWJWj(bCDsj( zbrzReENeI#BvZ3?r|qAtx);FaL)hRGb)`jFMTyF_r`pz}7|95SW>= zZVLJUh8oet9R2LutNB)a+i)Y}I9qj!V(8Cc?j?-NmVvI?VBXwC$lqD?W$sDik)+h- z3Jx}!WSmVY)-z7xcAuIo@MIno+7alOr0XC9~iBV8w_%$5-W8V~Ufo}AW#{AY66J+^Sy zj9u>oA^H*h^eXEV!n}WW#-{y&D!akGzjWYkhOYP%IJUlG{NiD2080Othtmhzhm>%q zbb1!q=9TYzYTL#4#uBYjYiI_C@Qq-0iUG!Oc+A?cMYC=O*Ao0^Hu6Tc*aPMZwpJ*G zPmKA!nKzA@P@E0+TV^r8P@yb`!N%*bt-}WgNiyNfVOsl!&v)=tx z_mE<2?SArPB-YC--FwSn1d@=9{9(vUVhFtzCQ^&24cDT+)=>KjXZW(WaWmUzu1NEqT$6#fy7wjw22#V-q5z%eHV2To zO1|=!cc~Jxi;=wLAEyC=%G!0a%9WfqZpC(VdQ4#)G%#iTw1gGoSxuR^oF?{QWRzCT zfT>_YU((xRLm|ro?n8vqdJSzc}#c(gXzq6yxPWx z4S9gH#4%mf8O5`V)I$Z)Pf92sgOlZOj(O#WH=?3QPt4boK7=amD2w zEjet4&o`xF-g39m8HR@He1I2k@Fr1H+HNTZ1B0^v-&V?eW$6MZ)zi}1hFgj1gaSv7 z+}<2w==J%0%*0DYXY-NU=aij7u4}X9cH_@*!A}Vj|Mj2tQGx>l%#+;nJ}8s=R))={ zwDM%!8OD^V%A5crv=*4u6k8b9A^U$OIC(`@)fH{I&_|tvoMUVs7=ZvdNQ4XFLa)bb zy80UWKrd4L>@AoFOxZPW=J3Htf#13p#8YF~Ey-209QDA~0!Dk@x}U<21vYSG>ckWa zD9NgqgfbQ9ePcUgPV1jug)1~q&{|e1joruD2 zu1l}u6lu`c%Msy7krTOmNru}^P2oyAcQ=pE$dbH4p5WmqDqxAsD9|+gzcV80rqw^P zvyv9gCj17u-oe^N7`yHG!y}v=sJ`Q&1a~^UmS>I=bN8hE!zPeon zG>uq&WV4$1A~EdkKV8z~{|#lKW~YI-sXE>3PIcV~$mS4^r3&~5HCaf%<-{&j%U+1Q z`7&y4q;%{-Q?y|BdlgdV_3TdkLjZF+M3+%PToVD_%J1$un>c1H9=4;>AODesigDy` z`jIctqS;L**-)@*($o7}4v*@jVJ_xWm=h#vDg6nZ_DYq9n3_I^P0#W`=>3LPcl`^LcTw#KcKYNfE$unMcezu{M7NiS9*==lyqBa0 zlf=~DGLe8Ad#qfy#L$lDd*czHrJFloc{%6z_+R8=tL|z$+1rA(}e-0Xbi`F#* zZ-G0eUDvf$MD#Z}>HAKx%{*gA{wDtesM_~obMx8Ih&fGA{^>X1?8LT3_i!}G{qCyD zlLnr0vXRYQS+YRgj4l0fbTe{};mz4=Oc0q+{#fi;eY(^pIhpny>PXq~hd^_xk)`)M z3{C2i-1L$Wx#|BWu}GwlZoSZToa-mY*(o9|T)^b`c)%GGeuY8wBK({57kFA49jGgz z4^%{dS3}aRw2IeHWq(K-C&4J5cn7S<^a8yrb}|2j8vG&rlp#;3rFh z;7MJC^*#FUTGC>2*X*#V3wy@0^Wg%*9Yoa-YZLjbf=2=m@{UdT1m|aR#axl83tQ-8 z{pxQsxrrzf)7~FZpUlJtFIM+D_X4ULN1Z3*%;(QH23c>jh94`8jxCJ6_@Wydp)Z~W zb_f4=sp&=rCU&Nq^nc)FhY&1}=cF>oa-~e1Xw+YIhsonpA8)STRM@}&x!p7|!zy~# z7{7Q~L;8#w)nW2f2!{_(OeYe8TL-^EgTZ0UMAVSossp&cyN(Od3-`ZN zs@{qmhUI5y?)}B8^F`g495C-F{f1}8Wc6a!CP$bT*W*5z6kQ%$Dr)}+_yolI_!jX{ zUZJ})aMwPjiB(|JwVZt2=^Aq$$UP5jR(?(Ki&RXR)C~XHvjQ&j>57yybSW?F9+X;m z@@J(30DDWpvSL?TeY~=diR*4uKJVo485Q8zp9i7ojq7z!1~p_rB5IC+Y*vjsO;05J zjb*Cltt1Uy^6FB#)s$_{BAWZndwQR%jsW-~hrM}%G4z&z*|F1SY>-!Gr?Pv!yya%> zB_n5>%c&ClP$}#*HxlWT0PHR+f)Eeu=e#e^J(84cMMplOG>O8a`np(`0`R#v*Wa&4 zER}~o(+j1m@fD84y$6L;AKqLS>?pw&4IuHEpSvdZ=3DTSS0`-?JpkGYMMQJS-Kw3g$_UT3R|zup6DTDyt;Z)awFWETsi+IV66GZT~__L$Da7M zfZfK^n#3BFb?kfmw^qhHTTL+BsKHQHAF*In8T7Em7NTQ`683=k5cub_k0l5D1#z>- zVDLJxK=JaM`>fsWhmhpX@(uw^&q>d2^q<%l2PWu7gZ}-<`>l297$#bIgtcsjsCxJ% zvK{M*@p4@JunVy83H5}}5PAnDaJxR^r5m|iPC6Z^2rK;l0+M?_8O+n>m=x1iOY~0X zEi_d;iEv*v)&ZZ8zlZ)?_)Mv?0Pm>`fgi(HO5oJJjC<4Cy4=}iq?7IRfU8XT+BYsE zDk{SNVqY@cHvn)@VN22>dZaaK5g@34Hp%}+_&c$6plC`<0l!P(laJIhmq;hb6TKr= zh=}T!&F^+{TMQP0CHFRLd3p}w^S4m;@<{L&@%URrSx*6lFLHCI0hA4tjjq{hi0p`h zMn`BF82U=HVf)7y9l8qN(6@Vh6-@3f!5H1`ZMwEtH3*LnzO2e$Lrl&L>BOW;GdBL* zCPBNs)fS=ea*t6@_LryXbp*+fh z5AS!6>(=S~Y!CoTjVzEKzkU}QBJhxzppX`{Wo7tBp~eXKFTp-`C`!Z1ux2Z9i~Pq_ z^$H3Deyi$yNHiH*s|d%ezCbH# z^f4Zs2OS&$cuDJ0oD8IXx9lCODFwftFlYo}LK`#3@xEwZB}4PFRH25GjZJ2CB00v9 zmw~R_7e_0k#G7sKXFWo^M%3xJ+rqz|V6uFfMhuhpsqRGuggNbTDg<&yvTSNH-dN&` z<|G{N1=Du}OnKQnVwAI8nH)bE;b&3-)lJ_V(nY6mnd4b4U!v|?T*@30*WXv8VY6{0 z1lNZ>36enstCy^0z1%?iNrnvqvE*=6nOdfVf{sJRl_XanT0$~vFN)jxn$-?J6SjECiTliLLKHEc4F(@&u1;MESE=ot086CEVoNX}Io_@%~4)z57N|NO_20 z#f|yA;CDpt4?ftqA%I9+! zMC8{$Ej^qD>Jea=n|^+`o1Hv$r{F>0@CKJFbF^GDCHr4QL*TtXH}k;3wWV`*7b3mT zSIs;4AJqOx-~GnXth?r8nb7=SWp}_Qo2b)4+y@&`b|86k@AE~o3c&bWQ)W>>q3n}V3;D66%H4^UxW6XW(FeEa#V6q_%;Azic7 zhIBwsz}GXGr$UHe;~8!v_G{vCuNR%fhIBo8V`1bB7AjVcetA1Fm*vh79!FEn zdG=9=Bo0q%DGQldybZx!3Fsb+5_CHMv2~T$;IPPww4)0>axU>-e0$XbBsV` z*Y3OhvLsIkk8zCqQys1*X9yK$AV_Z@NNz}CwyNR@G5Jse z)p(m{XMu!>hGy|zLG2pr+fG(k(^u73bYL|UYk4U1O-?^&5ap;_;a`7ySG{q5OqygA z@;o&f+@*qv@;YXU< zlOTBZF0DY&l!r-9iR&id}j^-0Hr`KB2A)zp4tH& zWo#GvL^$chjK<^2PY_(?EcW`u_SzKcU_Y6fOLvNSkMGO5h333ICLA4y@jcEC&|2c} zMpCS)^F+JqwFoX%VrdY4V>zwoe+OJAo+Da{?P1VcY&XKihNSgQMI z-@tbaB)~3jGZn`ES5_>!xs8R~yIN1I>dtCqb&-ml+|pvuak#HdnZwePIzyUM#zUvK zK+N}`n4_Au(a(P|42IP zOy*X=uhWn=qcDNV^k0(P=-8qqUPuoU&)jYv@|4cHrIHtWBDD5#p!gt)cFeH<%g>8y?z$r#{$>4e%T_QF7ntUT1`m(pDwH7D%H7~MksYa(VUkPj%GaZ z9tW;Kw@~BDjhOapYck$-;)4xOa({8Aww_|!bpNCWV+U_B!-TLGTnz=$(FT0U3-WM> z;#k)@R%m^+63A;tw^HwZ`28p5@j?}PHj1T`Io_^YL2ftdU@>i-VAfPJB{Sa^T(Q9p zX1njPl-lyFaoQc+x<-o$efkdT$=z}1yZe0FmE45f?z^C7yEZKsC7fkp>FfNvBNvnck($ZQs2)km;+LfdRh1|8S3#Fr zR1x@7mO|ORz4hKKC+Duatuh;+G=ng>z1}ej@nH&^_J7}5^`5PQnI6w@DYET``SJn! zk;dKD@*X;+m>9o}9lP*5%GUkVRIRsH){OYlL;3jZr&#k)de!iDB0CiXu%d5|RH^50 zmjx~9%MQ~Sc2H( zpGJ!d@RN~_QvkMd{nG)1!6yVM(%8}-Fj=QOo!0&80?7!D31Fmo${nI4hPW%_%~>1$ zB=gf@f1)imP~jr6^ML64yCfdaK$H!J8p6zSVCma(LxSN@LJKmoBwk*X{RIbOT34d2 z#P+&p(D1u7;%MZluDnh-tn)XK$GqhTxM%j~bI<-H29|Qs`(Wr|OTx|jMigx02YizK zr9vB|G0C8IG$C>n9%p3eS5 zq3)pSU*$U7?JJtWS#GtRpF$@1eCw3dsksYO*7kl`==4Tk>Jda*TtTcx#4zCh04+G} Ap#T5? literal 0 HcmV?d00001 diff --git a/docs/development/assets/line_comment_dialog_start.png b/docs/development/assets/line_comment_dialog_start.png new file mode 100644 index 0000000000000000000000000000000000000000..c6ad317a015ad22f03142ab34ea3ea6888695b35 GIT binary patch literal 7335 zcmb7JRZtwjmLjxDM_?2G;N>Z(`kn|i6*&x45>z-iI1GTiv<4j9JK$Se`~%WkZF47N{ssshp8;AQ-b&yH ztMIozv8Rllr>2XIr;oY2HJq)p3(%U~!_wW_+S$X-#q$)tQvwc-Oc)?7rRAG-yy|C! zw?p~&eBy|sb%hCyDd+c(l5hP9OgX;m87XqR2^n*$n!NlY7fig>Z7j!HBmY23nssfb zkKMHRs?yC;RrubEtM||E@(17RglbGYUSC|}?53xmqVul%?fT`UCs^sHU?WI<97gC% z#QgLTLCV*Y#vbhlRTB088_GbagnwE2IWm<1YJ_?4CnEX2sto})x;Z*Onx z@Bfr7=rSk>I(moJJ;P+^@~L`0F(oC`04URk>dcfpjI*cM77*6h$Zj(-wk0p1Q&N${ zG)q6h_~et0&JZokuw6kWq8pr|50u-oI##ptivk$>+;X_Np)4#fLyszCKy;sE+rZLy z)$w$2DEcV?06@05Rk1G;zowy<**yndP?h&^evwYWr`Elu|7SEH<((V!LtlcL0*(Tm zJI|leGEWQjuyX|Mn+Clli%WR}1Cg5>h+PB6;GbyU;-oXv#DKv)id%z9w0LO;2ZLI` zW0;4be9}q2`(!*&z?@*nz?Y6kmKaaoj*L&*w(uO-+fS3|ofkWJ6^(x7FFrnQkWhj6 zF5a5h)|TDBJ?CvS%38p9si%cCEdFTLD&i;m6QYgPpHT+70Df2%nZ*oPoL~fBVadMq zKuAR7dFvhOXZ@}HeQ5)SmOgt{+8{(ye56IAVRe*O0i;EAfwC3L;UAv?%khaJvYtYy zpQ%Y=NWR4GU~8${yA86GCeh-&R9YoVYZ+wq+5ZAThO9mpOg(K$p?uaan6l$aS0X6x zvjtO^_E}vvivBE(tPJvy;|xLj0jg=T}=Q%J=VgW8%h6&HJ3D-LUj|z3lUOO4s;pQZ^WI z^vK|~-bv4MfQm9qJ|+}*fm${Kg?>{JdgSg zfxloe9&5q1z%|>5ur}koeQElyBwT0BT3g#8Q_4E#`Q$|%?Y;E_lSreJEc`3n)0gE* z>rt|xsG)Jg;I3V4>m?hL8?@iI{pDG&C*=MRCu#E6kMCRB`&*H5+4}!+G5!3F2~X5| z{vO5Aqz9Is$EM2BGSu%2B(0OZ0_(2{H2RveQJ!>YL$kx_2HfplGOwH#qmf>y5xj|p zNjHnp_+U&ps%Ou4y8C1#pWjUX5V_h-78Y%)=rrB!u0PR&T}cFRI$61Qj`N`puNKn^ znEZ_=7*gFro?+la$laLp3~)gcyxeVVba@Ch-I#n*zYne}`a)R4E1Y>pp39Z5=s~i5 zT&=2t=j5tUD1#&MRVdqVEeZ%nn|jQc(_Ks649Q#!37F<$g8Q#AUzprDi4`4ak*|ar zNSG^!-yWL!1YdH!IfZj$L~LeE!Fb|sn5t(7(j2ZvUwvYE7vSTLK^W?-Hxj#~-Ia&P z`tZvm&+Ws_Hx=VBY-JnQ-+N_GMakl0{*!YLK{Cu8b9DJ8MuX_L1Z;;bS$BkMmPS4I zgJT(+EPh-~GjY9MLFv0PKkd7JrA$QID&FF+>{F^3t>b4o34(MRZu*=}+P>-r9aE0m zxR@f<-zAm2m|l9}JHkx^54l=~&|}+cBAZ^yn^wMZz}}N;n7;c>!?djlHOU%L8u($) zrl=pbC_aZX|04XvCJFjK71uuhaGZ5L;9YYtVnv= zp!eoiRIkna1|qRz3JT91WwPxKMN zimhVxYk6bI7&p?>zT{RH=ODqv>fvjf%}`~h?AZi&xlZeg{e-sK@lc7Vp2gZ1T7SLl z-GW<*373xeKeYG$y=H4mKpNj8F`bI1G-C19rmF*3{qvdK3nI+KiFbYZ+v=pS(*qEf zblo}pd_Ioo;$Ft8o-g;D{1mBV&>J{*M5%kZ?D~om8^S~NOfZ#HQ0v0kqGu@1Hzlv+ zPlIBk1{y93(Y&M64{aStYw^OVmd>JoV{&8K;6IidY2#-QcdPCB&AlTNYv&+n5^}d< zn8b`s#~%ap$yYHFav?V!=$9nmI#b{qjClIvg4&Mhv@MNo?vJQs<|l6Iv^B@;b;~A8 zz78k^dKB@!Qa)Ek0_=)b=HCm=?+-XmyX}%2^$QO9|4H)FBN9jloZy7fNbJm#U*4bT zplY`d)X@kb2u3?~xPk13LTp)eIhsbYNN1)w!Y=cDYUYZ&^<#s8?}F`7K-aCwOxtQ(udPd4g|$)Mx-H z61F`mUY=QqyKwhS0H6C)C5{~~jx)TT2xelwC%gNE(mdjo2RAEd%nY%StjD0jbxDa8 zP<)AE7Q9F$Y~4W)n$a%UL*^wu$_#k6C(=sab`A+=6Nt?`63q;+I#>?83ruWVN17TP zNc0c;Gr9jgY$8q!d)S67s94klu8Henrzr5Xp8}Gbd@!X)<2^EQG_AeS{FL?s<-Bn> z+4=O{Y-G6n#QB88j*m90#Pbokh|u;)*n^bB!b(t%xuPIXUSE8h7eQg<+GxJ=l$Nh0 zyD4T_dh1tnDW)o~t(ddbtYhuOGQ$pw9c9BMPW{oOe!pr049t8R3m1{Ety%nnOT0_8 zKG`7Z5Qk^R^R`?Jwe;_tp|xj|6J%8)vEfzMwMbZ1 zKOMf|3B(6o&vEZ$egEFsp%kTnGx=lF7n2Ip2Sw`UxeYVcD(uh+qUpT zs>$0fNIn>HVZNi?!1*C}jFgqT-Qk=U*F;W} zSLyuwBAT+yHYxt%^AYR0z zmpXYjLEqo))ZJOnE17g=`<#*mMADWCA6*^wj}XT4(4SMadLAF>HoTJ{%)iWZ?G_6x zb%?g;8Z-P9^S*pUW6nI^^vf_~MbXTfmo(hRF2eKlCU5dCs#By<(Vz_6MPoad4%#D( zaY$OG6*XsjMWR0<^4h*?_RPyjKm;tPJW?LfO3>9T_Z%XD1U8rvAYUCV(!*D7feb45 zep78!xwFmPf>q9+AbW{|(3ovVej@Fp!cT?@6g$7D`g7z&NS|&ivUWaZnnP~DUcX8E zKv|^)vu3Tq!ju$EDeKW@Wm8$-IUxh{dHWes;NO4W4{2`Y_hG}Ara!Et#TwHrGuM1l zVwj)10vdU8cG9g|Jw_l*Ka2ei&<;<;vV{RW*z1CVVq&z z^gMjuX%%9XpO_@HYbiZBO%2uu(c2o4@vl@KDVl*FA7`}7`wYgZI2e5bXfVkbyO`r+ zJ0hQs=U~_WK4AQ%eIzrSSThY#-;^u=`kTC^{u3pz*FemXhfm$e1E zvWS4BwY^9jj%&G%cLx>_w8pzU2OV>`6_XNB82{c=~RQzluGeHMwF11qYr3l8pGJz!%GqlW7S*F1 z*D52V82|wPD+6|CtY@H+apa2@);2@7Go{kzU>b4YtplfykWMdW2{(D`b-t}4Wn=(% zrL3mw;syLb!P`z^Bs6aO60ITdptV9TD2;B5L^L~owBa1+1@5GBLw>XA;`xOlOxpk4XGh z!glN@ssOBbDGhf9fznK3;eSkk{}K)!c=hG1DZgizF1 zs{LYdVC5TrWR%^UyX)q0*SLIj=}(Ci@cxZr<9s?h+Mp&eyM!WddpVmZJ?|is8M%`l z%>PStd;(`T?MrW;Ws0b3<;W&d)BPAU8vNqWj_VvrHrH#kR|n_vzJgxBuI)p;3RE8I z6)RNCQyOSo=7En)=u$?^S;r>jVlMTXrJ=YqT>tgSlMBE6sAtq^{YN^MvJVLI=V1kQ zcTb>uA6yTAs2=(;qpW`O`OKz`z%Be))TeIiugPw-$$%Wj`Ws_7H=gAk{f*qQ&gyL} z7cUgPqRsGYRlLl*ME?7x)m!!GSiHZLe7uw)eEDc-_cCRsyLgQveZ|L%lj|Ou%xrtN z3l->xJjeTpi%zpbKji8Qoo!{eTsSVkJ@LfW?hm@FSQmjDbn+^nC3XGj+Ji{k3hL%^wwYvN<0UnD)owVL{!CUs>r$KAi)WMmUsh?*dIz6U1N4s=Py?9TNx!u%~FBLK3B_&$it{uubr&uDxo$-Q6@ z0jV%3ikU}j5}o=W5iqE>wQms@6(xS?2#_YVdZH)+Dsrp8Mci2Xe(Sg36nGAU0K8(1 zg@-%!S7y?i`@(d$!UQ?oma=?3-<`m`jQS_4#l>0mdPfX6M;_>;GjGe{z9Mr(5F-#Jh z_PzJgNZl(hFN)BHP_)YQr2sGN%Pvc@01yysJrXz2qzM+T?jRbj>ykozL2MwA@1!D) z78P{=OJ!Mgq(~kRF@qdSNTiKZ=P0Us5j?$daQdC$m`rqh?UYL0D%axj9BL?(91(W! ze}UH#6If#@6H*)uT;XyZIMa3U;k&yF_zj5M?u|i~Yd+d34k?uxr&pY{J3LjG4X=D6 z&^$A1edxR!oMs!EMx23~3447Y|<*?$iREjq86hFo8)h z;uA_oUX#_`ZL?HAnanKYR+5q|Qh1=W4@Zq_QfiAY^4=m+mtkAv-r{5Q;+bXPFS>_coM39^ZJM&%)tLZRL%5A?45!Igj7g^!`(k0| zsQ|cP7>#h1M?Per?#_T72ipUt>b$(J%fb)bX#hTXN<>BW_+wku{)De=3Dq#4rsxn( zQz<9FI9RS?A)O!FN*3lSOCT+Z%Dj8JRVpjnlwC8TAb{KzoyIj+KQ1c`jjz2XupW0R*CE-oGp}=vIo9Fhq z#y@&s%U1<6>h?4M`F~=YJf`2jutqCVs<>dudCo)5ce-6;=Q9UuEJS{x&1`IJd|R#j z!e-c-D4Nq?PokCH`7$5)Cy9>xb{k7<-T{sik`b~#rbz~98B?x{Zcii}D-$S8FeVAZ zth5H2jH~eP|1IM9`qcmm3d&Z6(gI4BBX2NT+>V~;qcN`!v<1IqE$>auUokZKb17pm zqHxlLmM@B2X<&3a_*kiRv#S`3W(z{0@0j<5b8%%8Mc|36))kVrRWx(Q5p1(|vkY;? z%J_>lhl6fb0D;?>trg|4`#D>l9ugIc<}rVPLR*VHywMGR^M5MFeS27epGpF!gCK=I zMRCE36*0kF$2ZwDsg9N{f8}gdrzHw$6}znnOeEymwxwOiD6V|wYxZ`1{ODsUqT9!H zyYlLFi|Y89Zzbf|vRTfTA&(t5 zUMk;lOdEJInnB;&DDvvpztGCQ{4O$hhD2wGXM}eZG?@(oBlxusdROQHO4-liii~94 z+Zx$LU)HVjplIDT#FK>!Uw?3lk>8hx#scY(3wSLsAZ}Mb9^r)UPHbRu-@K zqxm1Xg1y-^2`X%F91Yd|AEqB>iQ7oULoj6+fLz)KU8$_c>4J}>DXJ85Ki)Zsf3<&^ z(_pwM*HnVK{ho>j2cC$;$#e#eX0lozu>{NA%Va`|I-h~Hkm z%=p?UJ>e!mEqV1T-{Mjx^*URh`P=;K5fF)iRIOQyv1X>VBB0$cA%rdpN6eFmU3JuE zprVxQgzs0jwU9l0lH$X!o6bJ2xoTcx<1cKa@*`i!|!gSgy0kzN+V7Ifzf#0;y@YcsjF9{=!gB*gVMrjUqk1uoJCr_Fx%| z<;=``h&K@F-{F8&YKT&cTSj=8?2Go`(nuQZ8XIsj$)5c)x7F(I-Rus!Fs?Ff457zL zu5FbI$?@%R;C3PKjRyNb%6Pnm+?S3XyK0>RK{^~i4`I&SH6yd6NP~`HtlCCju(x(} zqfQ-n>s_MY&M?^nG4i?dYJb~Q6Ryg>yh@DKamS8Qd$xP0*^2!*D?G~NYuDz}w$t|Q z9c*y+Io?^OW)CG0M+-#xuaHtsqTEWkqTlB7qIo;makgJ&+WEav zcax%R7)`{|op~e`dDFf?X?XW0ts842b*%BPSfAFs05|Z8lug2?Gf@s?DeQeDQkoOw z`<5>M(7jG2;c%Ol-oQabd&3j_B=pDRScZ0wEMG8is6w&=!ZEqHyQ$}!HL#L{Np(to_8Bx8N8S`0m@Pzf8vA}K4YoP+V%Vd1Jlk5;js}m zP37wUhlui)IX3|QGBJIC??>FUFP?{~m$OTf-$mCky2pYE?LKtpGVZ1M_Akyr>$?HV z!>A#f_P#OkRAR>HNbnAfLWM@sPCnq}ssIorYb4uND zCR(Ro)q9e$wdE81wms&mQ1Hc3{YdePZA8y{MC^2C33~OGe+2;Ovl7_c*a(*`kkb68Ymc~T${=bO*JVZ}E=~^C_&(Px z27y52HKwYmxroqL!lQP33Pu)50!S7OBAmc*$wY zAzhyw3`jDoPPSZ+^s82y`@mNV8sU0OR4}Vlx8%vZm(-VVP70HJczshE;Q%r!(p8_$e*6b;@N=60 literal 0 HcmV?d00001 diff --git a/docs/development/assets/submit_review.png b/docs/development/assets/submit_review.png new file mode 100644 index 0000000000000000000000000000000000000000..34101a79cdd7bd41adcd22082a1911145e7298cf GIT binary patch literal 47729 zcmeFYbySqm+cr9MgLJpj5+Wg83L@Q|g3{eB-5}i{-GX#?NJ%5zF|?%Q+4y_k^Uqn| zTIamy`}3Q%T+R&7%uRByhP7(u^1Qh~-V7!--P=Y|GI)3*e+&Vyi5=f+I4awiI=UFzn?OEU+gO>rbTGCzF|l^|Y~y$g*DeHs zP(j{Hh^n}zA1t|eDsQbLp3uuq((UQt?fOo<|3DbXdij-FoEsLEspv%6HhSMlrEsxRy^&z|0irTTwyNjVOk zr@z?R_F;4ykImj)`rom32#<}=!Ca$u;TG;#MsQlaa8RVD%4aVAmVzT?8t{PfRy&;8 zo_uEc`E!U=r=YBb%ma%|)Y&>yZvVAs3~FyLWq(=Y8Hwjz_zOzouInm|>jME56_t*u zDTy7YXi|Q}wpP#8y}eT^)jYli`OuJ%5DLAduamZnFED<;adWjCW+x&gHWScO(~COv zz@p*j!P+jM+odFNTyYXoNG^I2OSB%l)(uVI=XFGYu((#Nb&SWH8uts>w$3+|lrV+# zuA6Efc~b;=Nk47S{Kes6_;6Z%Oh#HT_&Rg=uqM&2zIhxkk6UQXsOjGJM;e|^b$w5@ z<8~p@R+y>d?ajRf(#*D2yEliZ1tXp~LVle@T;4cF8|%~iE%hJdoSe84eVSp#B_jh} zMD*dWD8H!d`JlN}|KNhs5QJmn?dVpz(IJu=wKW&-3YwB#MTa1kZ|eb`=fL{rpEqW6=rpgEKshLU5n~op0Qax z9gk1TEevG!I#k*A?6N=oQ)Vxg@B@0gP|jTIC;e7FuDZo6BuvA*P2!WwWB=^$Fs`>* z$GJrfL=bl4OC9jr#%GD;P^YC<4?@9v|J5U0-?Vybi^J`yUwm=ON>AVa+es^vp!JSq zvE}}BE}j@#9WC@13%sa$><}%#O$MhFGLL9XODjXhNya^s2o+IO)6-*<0xw(iO5ci~ zzk+Df#AcV2pjNj2#S(bsNfd-Z956LCq;WT&#$c0;HOpqfvszl&?jLM{<$1l#pUl0- zaj^IU3r)}Uz44nsF~4Xby)QAVZQoz>h^xquJ>2ho(wco^JpU2W(I$LZI`j}0A)%^@ zkBGI4rL9ex&hG=mxg!ucca57YWP(E=ButqyTe&gK^XT_-!Fl&CH8N7wMoMcQzii<< zN^~SC8a=&aMiPHqOb9}_-aET-UC*0wq+~8*5^kHNC@i5HcnJHEqH1ZkpMUW5KHs_e zb0#JxYzmTy(aQ{|RcV!zdqZ>Q{5(nM&S-8^6Nz*_4u@U8p}EnQGeWkU?CkHRKQL0+T7fv5fp^8wywCyjEhf4hn$>U^j=pP8+`m2fZZM3H9Rb1+(=q)ZG0T{ z^2FAqJ2zL-XgmFr?cFA0`Nh!+8yTi)3)p>;yDhrpT4``1y>+(sYfjUSOT2A=TM2;> z2tBgAizI`AgoH*u+2zTMMWw;;uH?yh;bZ6cE>V0u169({GsQ_QcDpX#@&zr;Nt(%D zeB%DM2FH}Qc5hx{;N#;D42|_qI%IapTX?^<^b|nEBJ+LPvLqLI4UOde8p@}=l4>`H z>#b`=nu042gh$Cv(V_#JU-d~}qV>_dQjh2PuN5p`^WnFjJUk#GqB)m40(_=;=olDP zj=yJ-k&l8k<%J@5lX*3?war*&2S+g}j!JM(<`pJ>@qukFkWYb!%n?>mnwXgI;^1R2 zfB1k-{2Cczd(`z-RTmC!YV7Pvd72;WJZj*#@mW_k3yXb3e+lCuUAYf%ke0ur9w#X1 zLGlLda`fqeO2s`1rLzVmW}z@^h74?M{Z+f48SqdgC?dbe&~+of+|YE?{G=hp4pBvW z=NM6fHXrwcGFXRIT*gvhG^5@3=TGT?;GoxLluyw)_92vN;^w&Eb7CUO2Q;+wkT(;X zJH*rPg@TM!D-@X$w2}@fK?vioH@aH(;&pu4*olD!NJ!#~#_4noBeVfbYAMXe1k5%7?5|7xdkYili+LoGfK z|3V1~g`%_6%SAk=46kooX}JzjmDmbV^la$)`O!&VA&2g;8SGzXKz!dvN2&`S?QdW} zc%Aq}b)9b_pexpbI#dbLhI*6JOtiFT$0j(Vs1|2vhE*qm3 zLz@Q%^6?ch1u-`~zw>c#rk784T!wD0Zp=SB)3>ZlX=Exh)y<(U>71$Rm#3<+p_Iy> zFm;Q3N_;6XeW^k65{qg0PTXLV~cj zXj#FmrNgC+R3`Qg%Axi^mHFJkbOG`VP%7MYg;wiVH`kxDUD4>Zb|DNJhETMPMJw5$a7x>Q) zLPFpTld1mwQIv@8Da3_HJq!A;s{{Tgm!MDnu-RnyP^%B_)X!%FVO{;f4$Z-E(uueSPn4Dlnh$rDEyl z*D?$V(s;`+#hg^RhDT%!K0f!0l_S}W7QdIb4WVnXsZ@jib1*f}&w?)T7U>+JZ;rM3uco3t&l|y3Pe*;>8(Z#K=B-dVQwG4E0HkAb34=BW>s?HH zW#21$TpT+VK7vm;qlSYMTOiUzxH*{Cx-NbFs%*#jqPDZ&m~Qf9SV ztmnbdx2mH6H>0l|*)0u6XG^a^WXa z!{08YN6m{ZZ#WlvV9q`d+gYyE9=8p>H>q~M5T~?b!1)BP_rlArlhYkV?mc5M2I5E` z#bnYvE+eT0^KceFX(okr2rpjfVe_TY(zcbYy}c#ROT2w&)@be^A5O!3?2 zswOk#Zw#GZ^YuZrLVHzQCVR&{)5|V8CAzh_32jEz+H-$q%}lQi*LBmSLf3nEKn4*3 zTW@cF_KboAU7M5{B0{fC3XgcrU6;xYQZs4NSJF17*=NNvVPC&asA1vO(r?epNz-7y zU9Xqb);s*LXIuhy)l)Ky6yx6tfNUMQN!AvtGk^VH`>*uk9{!-YdpsZ>LzftTQTL zn5|97S>9FXiX5i^sg%x!jq5xJ*b+frM{Qi$J3E)CA77-^?Qu!*@;UdG?cr)W-8LD3 zz}xD!pFp^O|9wU;Ln)r&yVa~7iyphae64cq&?Yo)_8?v7U=bIlSjK%tEvwFYre-ji z%c{DqG$(GJiaO!_+TNPp@hSLqU=H=F6`!Fcg~6n-cOHfhi<%c-W`WsNImp9*{knhE zZe98glU(Jv#8vw*x|@4Lr^JT_E9MIiqI)&mBRws4{fz(t#y9xb!I&H8DH0G7179|9 zec1y!aO94b#viBP*iSZ6oshMf^4{*hVHc3tu~om~J)ijUSEJeNL%54u6OYtXEBkoj zYhErE>{uaS-QVilDIrup{Os^=?14c;w7en)yUUKSt`{mDKq4+1i}?quy2QrPc1nTz5EMdTfr2gO3}0kS;K%p*_9t zT=#L${$htx!HY1FMK`J_nb%1Zn4!SZQZ|F)=lK1})_!VvES{EoX!y(d)M>ph@Esrz zn)^%Ry?{VWNCB?M@+n3%!a~NHc95mOyn68EnNjTm3$U5pqk*aMM(ceLkG(!y)rV?Y zz`}WQI6$9ALR5g^UM`hz@akn!lxwv|nLp-8ZEkK;y?Hxb{aZ=bSnI=56*q2wVd=vZ$jwCyV9Wv&L1ZW1?9Gm2VuQv+okQednuY4jqWei$~Ccybt=4%-=R}R$mAZrd!Q&(q<2xw$Z@ndmUXaNf#NF4Xa)$tr_zte~VcZ>s-6 z`HRF$FT}8>3k)obckZTia#`pVj9Ua2PlnSq%e{3zB~wAOU%jgSz8SuCQ}HgB zd+}3=gtqpATFK!3TQVi`VWgn?a?RHPT8-}H;NH4<^u0^0`Px@hLa+YVyI90_<}H}_ z2_yf7Y3EU>>6>1f$L;W_YXgA$534$obY0h8)?S#cS0!Mk&=EmxRQcGmyq`Gwc^peNw-Xv$h_Sh5NUn~!2#HW`W)z2%M8Y#xd!$MxYswmZ;q(pjD zsz=?U>1bCEKf;QYa(oQbT5Xh71Ri~9rU#r)X478A)e#RejeoCdF;na>;X&D=Us~>^ zcQl>2U+*=m7tbU;{UgpPo@zXXhTK%7>8ijPEZ}&dX?cQ#r04O5pOUI$rM{OHbxKbB zc8_oIn20afpfEa5o$kate%AtWeBmGnW|L_1HHM^duo z!hFMJd$G=f&G{(wb3V!R?635Fb`6F5UXvD-# zV{#Kpa3Sh2uAXb^?Xgv+&*#wP6ROK#A zWAxWFx;)k=j17Iw9|8qvm<4f)M!j9yjTgw%lLgi;Em3&S0aoiKAFnzEe2lN7hvM|$=Y>+Vt z+h-k5&+~y8)c)edDn`c-{=6jk;Q%}&r>1taC-D>raATU97Vx^P;k0R7l4p8BQ33+b z$LB_&we8e=NAQXFSH_mhb)|Ydg`6m%bB#8_uYOc?D_A1C)Keti&1vxR^R4!kYhU-@ zW*ICm(*I&RKfY*t7;TS9%8IsWyTgizjDSGYD@<0q<%Rp#1J9nV&Uz!wD{jb`Smm08 zPj*7(m0{mnAfiAT;^WapxQs_yg%9euXWZL`e|G0xI#k(T@LkB*9*SCJYqZ-jo62QP zQGbDlxOy~EgDL?#m}Z*36d>4So=DHs!N}0lO+7Agh0uo{ZdBic@Q$jQTW^jy`rjWej6$UIc{*aF9iMqQ^M{& z3eH%Dq=B!nZ+o-w#W5iwg#9a#sN%N{AE8Gahpc3jSqed@)y@iKOR&r11@>2E){Xw- zcE8$M^Fr^I&327uL)y%=D?G6;x4_$Q^;sVgci40~5+nDgURJMi?epT4_7?ZuzV?>Dc3Q(@))oxz;go6ppooqnUow7RYN`_Xr8TST$dQ8sbB4scifAXLIv&t5S|+axaS9V;mhEdwT40 z0cu6Gk>|MsW!Nju%I>~-)U%yY>nh{+$VjdjSP3$|GKxGK3#)v)7d`-7G2mVFyXy=M z++p`8P)e~&te0lpiwkse@mmo z>ee0%8k&Pde_wI8gpQ}c@bK{JPO16!SE=Vubnno{BNpc;naL)MPoLR|afpJ$-RP?jhBeu0wxIp51 zt%A^~?Q*o*{^(6s6CuU2sM@##f~YXh7)ebK5<{lo~6YM!K~`mexm0qoCIQ2S#RGsU4LuJbWOZ|{c4S##SM=k~-93qup|RBxlk&3}3_K7E_ninqH9rS6 z#t{(IiH-jn76KQ{)n_u5Q9C%<-P#=x8X_J$SLHzo3Zo)B$?LikH&6$nMfLV<1R|Wh zmJ?c#zNdH>EuR%p*{&T3&_eg%pyIvY$$3!*N9n`Wxv2c-G$R97WVsjVy?dOY(6rsb z&1$|S@0sGlq?vb2adkJt-u5XiBTY%-UXJC z(Z;FIm~bg7$tnFa3LKqEw#QXJxYU}Zh<)&9ClPYKs@f^a)`eTD8jg$L<<`u!A~m zKjCMrI1(4{LqR?~!cD2Q($y?>a&q!&Y>s~nST{HU8UW@2{3VBS+A}f|hv6WwKF=?a zZpKzV2JC3da?Bak%(}YEW4^&E~)OkYxOju8C|Z ze)R}Qnr;{U2c-90JCwCA(|ot_05?AxRGi4Dxd7E5^D>)eUz=zpZE`*Yxt3Sp@j#%b zuUDKduCqR=WQ+QB16!i!!xLj~d!fZ=fA;5>{D)as?dF0JL7)AZmynM0-wvd%cg~Y~ zQlxeE7xV5WO2r(evT09|j%15~V4K8eXPXcL5&C<_T)sbA>ef)RWV zVwLr&lpnj~t4N^|6RT%+M{|+o{LZr;i&H)DkkySnn0%$sAv5*W{f)@@sUf0^0fjjY zP3rXy{i2GBzH)vS1=o6HA-xs{OK}0gMRw!)=NZ~Z7N1ljpO76djPlbJcb z`pO;`_Z$#b7o|;rrTCWDpi#Ir-#l;n(8M_>L&j6^{#JnDSNn^Sne0BH!t(o#BLKgC zPtqA47L?B^oavW0T^a2%fICBHlgEnOkkHgb?5vbjpKt@?Q~%NHeu=4b z9~s?Je*+0x$H01Pjc;kCa^;DFg)yUX z{x@|dqA8*#{X2JobK9wjzGqN{Zu68!%TZ1B;SzveYkCgCD6z;fBp=o`=T!IKdt12_ zYSm6wvnBHoa?e~K@9Y5Kw9DMZ0V@6LF!Vk&=c3w6B4`me7vsG{Ma8s7PYJ; zXN!TZQcgQFqqh6;Q*iu{`uNCiVjk<|FU_TBWh^X;YHCq{)YFk23#h8%(D87bwF(x< zb8MwCw=gbGDlHU!XtVmmS_>kDsliK>5mET z8>e!Yt|i6K{Ay1STfF}}K-CcFhBWw_=ol9IxkVBo$M#r8;Ig0h+;=-PzP zRc=YFyvX4Zh18;yu2zSK9`H*_Z_R>{aR8OE-3fRmxSK%3$;5JApEVk}OdHhm!cd>!T-`gGFPogh!A6-*l+!4-^3Y&aSuDL=R|G{U@b-0;E={Kpz&@aAlUkf( zVNF{gS6Z38jvf5|dCk+iFs*31Tf$X?L4pRm?szfu?h9-{;N8;V3lM5a7Lf|ap0A22 z;RIn2IQObNhmBWNg|>@}eX*!(L^E_sowavxIQ>od*!|-y^Bw3miRmi^0|1Q}D;2Oi zw*l=O^)+YuJ)-SmG+bl|jq_l`a?wamqdx6=k^aVmLbUast51K*#L6mg^zt%<-y3zK zKLT{zRG6x~&$zGux^~V=a?N!055d{>%dac_G-+sl5Y$d8wZO{LY5c;h{)*j`z5fHC zuxsgku)bS+_=jjaoVZ9%K@GdQy2!j+J-?gWvQAH19%qt72g4(62$5fr;WiBWfI-1 z&C}e|k)P8VO`Zrr=;*;YIYhDj>$aDRZjo+ztpZEH)Pt@!=s8TMkewQV;-GBdn*WFZ zKewzPK+nCgQ4Am)DJg+qYB;2+{R$kC3BngIP(yl+BGm#4k_8#Aw$9Ey zrQw;gL=Y2eQ&jrkih+S9%xJ!sxJMPN4caPY-sw;6g>!ioC9R5<`V%>4)^zfN3&kwV z7lQi)2bK%l=js3YKv>cq^e+FJPv35z*wClP|E4z7|D>7K16Eqmw*M=~?h)syEh70s zi77$aOZ}frB4nP@+2P*`|934B-AS{kNW*WU`W28Iefw5jlE43-?wC6j2c9?$N5Iq^ zQ2|}`Eh`&X#?K&%vXT-pYj0{MrV>C{P;=vnTT)rip#JOaC@v_d>IRnlWQvcE7XzQH zO?cCTh~Z*t&Mp(5m`KfE2~nao9+3I?j5LHwNf3cXaY!Ze>4<>%BLOvDK`fvIYK}*k zl>O}+EVmj#3<;^3mz=*L&@)l`BGb{?6NSC0`L_wXhDw^Mk?ZS55a06J{27y)f`Ij} zO7ZF{T~qO*|8>Pj25w&+R!L1wgs>0NwXC^ZjClA-B6JB|zV|8rev?A0M1)#g(JpkR zzcwiJX9*M02Xtzq{~W*j=THe6qW3*TWfMZVrJ$9is)_;UZ1Zp5-FJyeNWPVnXzd$g z&>_Nc6#723!fD*+LM1SGUcJ6X=H~^c2F}fxmU1rXNv!oh>~6-GqTp(~-23&~+@qns zHFGG7`uAP)Fm*Bt;f!pR9Yc@T$Zs0|-c+DI{a!632*sQO@T34m%(!qpU0W1LRJYv+ z>JUtd(02!?$^#?JzM(pZ$C(sFJoZE~vYXJYqs>htmmcao-A2T~Z+Jkagp7vLm3{Io zM1tnvW`hcoOen~w_5Kyh%VDm%)6*UO?x`vG@GKuxNQ^5r$}JXo??qa@0RusX0dI$zfE*R>FKPS)R6}|ZK_Us` zphe7;ys|EF&yw#d(f8@LU>olDqaAOsWh6GLBI;}f`l{UM)v*BviC!{EGfiS0-KMqE z!3{&;^^mzHW>}ORR)bFEAK$%)SU>8v!+^XLK_P_v(Asi(JL)CXVv7=F#NmE>YqUNm zX<%TGedeK}UC%=x7z|q6AOIq$sH)~R|9~qH;wGr$R=eK#g?+H_2+EW0u{N{zOg`k&M!7yS!aB_=tCd zfV37%4Y&F$pYz*>-0DH#^MDJC0IBC9mE_lBZJY8H&*(Jou!zOxirJ{QVCRV?|MzhvGYWv&q`U)Yr9H&G=MU)ieZ}2jY1VIrN zkfKaSwSR;0Tu%xX@^NQ7H?AC`K!(VFHT#LmF@zc$KQUm1qnSXU<9PT*u8vzuz~{X& z>8DZMr{l@4#DW;iQX;KRj1bF@q{G^l&z;8>mlumYfeqgpcJqZWnEu?WhY)KT>%WEs4 zS~*`AO|g_QURX~FdcExC*CLRPL;l~%uMA%wSeCCHZee209yPdgZ5&I7_*Vb1M~70P zf{G15RR7NVZlFxUP{zd#2tN0VtbGxL1w}G8rqyXn9M8Kdsj=JTcWA3kQC^HrXL%d9Q|`N zBBp9AAih9zQ?;5xzq+wiaq0=WKAVJiC6AaV7w^pHxgv@4a7J~%1gD`5%gDy0sr&N( zZalf1Sj<#0N2l*+m1?yw?`(o-ha{_X1$^@LyNpJg#Uu8W#(3`~QDuiix%&CaNzJY_|&#xxg?a6qQs2MP95JW5V9IR6SPaJ_Q|7KktUPjBrvO8$+qb zQ;2U@bIHYFC~trub}-axOoG*-rmbm8H0|z7v#rbi`07SrpbHE{ZF~EI;s-lV{}ywq zp_sS4AO<=*RnoFD-78RCoVZOjuahEvvGore>uK z%9aZdVXF?lTY0dquEZ7mH51ig3tUHw(IQ$tdIATer z%>A9Lq{*SJqZF)C&8dE0y4t(DY9nM?9`}-XeK?4@XVkt|)k&=X{Xnwh2e!lRLW^!u zN4KWDPIdQ&VozFgazLqkv-O|+i0_-JRZ0k-H%?#BSNAP1*n0yr!>ETGFT2-t6z_0Q zJl)+z6qUXR)8;Cws04&Y3juxC+62Z9D;@3nFKpMQMk?Af#Bb~JIW{&vPk}W2eX;G{ zHhl3EQ3k)um)2Ht=VM{`PzhK{8?4Afp_>7PyB2HCwp+88H-YDY0GV@~(%Vrp;)4uE z;z=%B+}#SiJb3cc*xG_>yMgT^yGU_QR&4)M9BXY+T`KQGr~IPNW7R#j?<4tH0V4h3 z1Ge2Y5qb|Ih>R=#EeKc=z)RG(3rf@yX%$ax503&M!dZz$+)oi;@BNRUP+bqK&o5nE z_F+W&CvtpwYtY0z|)mH-|oIYqV0U2(`L40Z2YccPD8a_sP*2!fT|1N zf$P)RNcPvSC9nuT4W|fUJ9E##UV5I5ZwXE$$jiI9HZ)PuJv{%a%*Odo&F{oclb=}Z z0t7-i;ey9#BL8ASN2K;|Tj%eU>dWTF|IwKh8UJrqp8ZcC|Nlth_J4i|gfE=`vvp5F z{C{1pLNyy!|K758|JSGs2?47s#fs14y;qX)f0eqm^Hhqe z3JZsf^FLEESNSicY={4&q2wybq00H7P9OV}+EmV@j<^g~{%I+X^BwJDLFzxv#G67BlO@GH9@4o z=HpRN)76bktN)>Df{3`gIHqH;b{kT)|6DSxrx7o4Tb5Co zs{N$G+{>PrWq70}5ucD*!M46&SOO94M?ri-_9|#0bWBNLI&dnNn*d^!fu)3zGogD? zMI)uLm{CS{OKS)`7>RrQx<7e$@2B1TUC&`=o<%{5)T zq@tfJBO_Ev`+AA`8X-Wvmio$JU`wgo1OXwCL2W8tGg+KkuR2G40 zgx?x~hnQ+S01OTjlf_TVa%v9CA{He41XYMH!0+s@@ho08gOVpfmCMALn;j30`A7W^ zzXrc*ZI^0Ly6Y;NqE*d_Tvac;DJoGbuXLa%B_ax`Opi4=eMJ+e@A$fx2FRc@iFJoej(4Of)(zo?jR z@2b-N5g^!H%>Tbs%m0s%-lMRB|BsOR)`y=IPesdFsyVZ@xeLKI%yW1rmKucD=toDQg)g{>Y&7gN<3dSF<_Bg8<%dB~|mU14Tr6pG?;-dePWFitqyI%<0s@9y|N9y=HSP z1khp?;PT*Wa;~yTINaLYUTdcG>5Le}{k~p0_H1_rZAkrYifO^xth|+NA^dISBQ8+F zPWy}{3&4o_wYirmE)-PWJ6%h?PV}lT>8%%+n3~~IkAe7}pAVaAtB2Rh03n&URMzqn z6&t&&0SN4I*Snw778jWgTjqR=9*#D}OkD336;aB{SwIa31Bmt-U*lO;_lkrEKrx82 z5Y*3K0u2+;3%5_#xxA#sH3d_81QiC zA9IwvG&>v{Zar^oXVsSlclHdsmhiq?oc;SY(cOOEOK40^`QqRpjJ!;ij?M4wDH=l> z#uxZJHcO)IF+yOi-8(~V-5CdDU~eIiy2Hir^5#nzNM`aQ4P7K@`NM4zM*GIozuAPL z9Kd*c$-{$&dTd-8Qln0k-m+fu8z&kdRG(h>(Cm%@v;vaz)1@&(ACXbd`y&`}sp6!k zhrlG(dZ~>%mP^eA^lAtf(6TugWD_xRZXC%SgLYxfj0$R;!e13Y@a{q&X0Bw(ESPU^Mh&P z=QrG*3ME-`o2bz1hB~Fi#-<>EzJ*K%FD7UMd;>$Al`pljCtqDTTm5yOCIJNJH+`v< z)CwuL?>~NYmYKz^?o2AT9vY9^%{5t2{`rIUk5L%QjFNBD2;#^M;<@&Q^KDFF3=0bf zF$w`PSH}Ze!Mb!z`kdmU{`MB3LHF@Otm_dCUVc8vW#NY6hz6U#L`$v&Y0)09ZBYo= z(xba2b8Q(J@r}eezTCpE5B3*|8GSASD>@ni#QAsYGwzn#z7N-KhjhSao;o=|OWD() zNK4_WKclLu8aT4xOncPB2&($7DTyecJOXF^?dG<4$Yn>5Uhaj#ZKiOH(>+~YUmwsC z_YN0FcTV+4w92Rq+Z`;{Q`R4U?4JLLLmPzNQG_jS$H9n@1*9L4lgoCYz@%XQWjC+b zclG4DSFx;FPFCNH)~lG;A=b++7=Ww*Jj}*vG4A-_Xx%ul?6w={7eASqu<>@obv+&> z^t@sCT#f?hKglTSBad(KrluuDU$onF=YL>iU{&k@LJfb!0K>&1?2)GS)SHx>>-Hfq zxq{#e)wqkEB`BtBUg5a+&MF1tnj05B-T||OY#l&B+mIX&3j}TbeNkU3<>jArBuSMz zfq{*0D6;sly-5EG;9%Dl$Z^l??iKSLd$_H6ww$fu%+1fQHWr$eCUL|LPCr)a*433Y z)=DLV@<%-SXlZrz7lLlnT=&ljSV`k`cC$qyNXOhbt%88@nnBR!2ZWTwACXmxj|zW9R=^jylu#DH?t)Z|gnRgIh_ z5RL{-HsHyjTg=`)yV&BV2g*MbR24(Zo39M_W>+@OqF4MSXhhA-h}$&kV4agH+5T5-etndrP>kp+BOjVx?kI zoe@?ZTL}D-&by1i;KTkXd6`Bl!HckoTfOPSqXT##`Y&6k0X4hM7r!|O+w9-F*AlyJ zpu5rcN#83RG%;FE7L~u(Rf8coiLzTOv>rqnbV`9|h+Iy~-`z+v-Tv6M)YuAvwx+@F z`cea$IOD!}6uy_aR{Dtv3C6&8Plwa{*8uyANg))*wa=5Al!TR-<#zn=#*>9) zO!(}1lXG1^2+d`B^(gI@*W_b@T-t@lqphRqg3*<2+JR3>$u2hs^C-0THJ_ETF5wjo zbV!J>8{*329Wz}NWh6+%Pdhoi&(jHd-Q9>-Js%!0bM?UsY`E%vc0Bq7g?$o=#A4F^bUL4ymtF zXx6Mv*F>dH2VqtV7McTRMil6Ul!+@=!LqfpQx?z_NVR$*2v;s4f8gW{{03*B3!Wlr?F&dyTGdX>@?+jV+RYf zOAg%pe3KAiI}-n!;1X^F17Gi)PF@0Xtb8RDNQ~5Tz!n4`({#@9m{2dyH~$_<~vQdM)$uBxP{|943JkTD*HD;=hO`Zl|fh(Az(5fM6gMKVqq;pj8He`1?GI}K=>PH;y9&rljF)lj^dhP#N3=Hq-r~k7#7Yl#G z(>vXp5+WpsVns2Kc$n%Yj(=)0^!*uZ>R-dppQX-3eCBJd20Ll{%`*!4%xhDH1e@26 zw!_47vMmZP!4z=C5{Z&hR#O?vrMp1I=qLzk>zlhG**?kzt$$$eJ)DrtU5*yb4*SYq zTTg|Y8V7A`L&Zcmg)_|mmk* z(DkKK&~xG{%gB}Hrq=9x($M&wh2j(F)?4je2k+I`SCn8Q&;WDE?`NPw~#J5!E-!O8ZG1 z@vTB}EGKPx?_OIA1X^Gfz@%wA_h7)^WMI zvjj-pKF>PXJ9dm5UMf_~=M_ig`U?^e)Hm<440ClVdHZei(mkh}9LUX))Ba3B<~k+- zqgJ-uF+<)=JlD=+F=N$NHt}}o?(GLV7l+bIOdl_3^*U^6XywuA)#abfsgyH3Lc)EP z(&3Tmz$FhXB@Gpo!q+Z;&Km?@kNo=e%s6*pC!o#AtuT?g^X~N(PupKRZY%9Ns*d(z zX9ldPqs3yGay5sS+=2 zicC!r?}dapSuDQ5lm)Y#dQ{XD#LO24%4D(a{{))_Pd~_!tAYb$H2Gs{wP2AkKofth%M^$b(DZ{G@YgU-b|Y~&DUE6=I0|UI3JnJ z+y#Oe{z-v>Ec*VauFmBOwH-N=-rg7SNu_fQPDp9T4Gb)7>UNy`NkAF|e4OseQ0k}P zzErfcjv}{1xcPiYBXMA0VEtoV(bAk6XIzP33|(K}yl)MNDRF&~eUq)!pt)~+ThR&t z)eCkj>h%ZS_a4*%&#orR{)T_vTpJA>Qs;Gp zrpvK&csz&+d0xpLViW=?m+6?k>&lR=zJKUu{OfNO5}bC63@t|urB2XcH*V{N-T)3- z^Lxj1f61uvQf(5m&-I6y?YCMWpx`G;j+;;ZBt3% zWM*nD{r){;zKIDR%v4XlJDg3!yjF>opfgv2tqDuR_-20yhQ7h(9dD z%a>bsfeZvMwR^9GAAcg*$d_L*!eZdt}|6 zjAH(&7E{@rsa0N(oh{<=sfCrveVd7yg=t34wsPdS2xPZU^`qwpGbt^1o>3ISz28hq z*|}ep)Tzt;U(~&2RFz%#J_;frQX)!+D1w4?Nu!js0@B^xohmKeu<1s+Ltq06Y`VLp zHr>66|3csQod1X48E1_1<(%{38N+9@=5B|DN2sBZe9*QT7n z^?bIH&C4-nb-#I@I}1VKF`s`wJiK=s(TCtM*1K)}4O9vvIr_YfB<`D66ItrCl9G}R z^V}J^o5d7k#kcYC@d_Fm-wb_boJ+cUdbmBdeVr_knB?S276S9qhsm2pj9(-k48cD$ zmQlpe%!hs|9E*vI0gIM-JHS=DspH|5*D85_Gfwt!P&!IZT6+9OGl~7u#@2R4?0uAz6;8{P zJ>hPB7%*(W(;C26?m@~onfSRPF+VAv+wahTid2`0mLhO;4_F&}YNti~=#{ffCkLP6 z>+A1S^eF2)47X{9!&>KMjknz_he!RuJ5!?b1|p*%XePU-{(kpa(&ccMtlF}a3e6pN z6_spvvvYE`vYj1m(fsb27ovecgoNwx*=B{+T!~t3(4O~@r}GEL=6BUOa=c2fo`ASw z^oBq&Ff1siQRA~=otkBKuAvPZuQa$Q6H=%#a3dRc@|K)5M9K~+sb76_hEc872Nf2_upVlhrnLuH?@;eRd=ok@ijI}87?7|>5@vmnl^Bse zWMiS)lw`NH<$A@W49YB8TYtmAAC|>D?k!xi#~aCT{B|zU3eKr23fa}<(MkWvY(MS+ zY=Zrz26Z!&>etZF*#7fN^w)#UuR-v5;X42RIg?mqYGhZ}2>g6;gh{pD=>?MU3#Nm< z-YGI}ukSI6eK$ojjo6s>_Oql)w=41Ay&br$=DPf(;w2z{yd6v{0ixt$cPkJa1fAz!=MfpzvS znfst;_Yea^qF22xhM3Z`#FOy7NE!t=MjcB}%|W9Y{Q;}6Ftr&oU92+#@$L~46$ zgPOh9x*4snV!KJa#e2GYRLfkr?Y-2cEs}GK)A;Pejv@(7H{mF=X_v$BtNWl`%nJdk z$jQ1baPnr^EKxnDKU-=Bhb~{sL02}b6Cn7QRBb3_do6eZH&?tVx@+mKc$)2p-Ap;^ zbrw%C3n9e_QsF3D0clY6Kt$;v z-J6)$LaxCa{W0UA1ydUlO0c_BtBp{4y1O5k6u6S~=V^P1*Kh91DG`r0bD3^zc--vB zB@mI0lxvcLv?R*zGqL%E^}8R6!_CgPTB%p;!Eq%kbKwH8v9Uy!-)NN}hTnN>P>2@C zkvk-AV)HD^AJ^!pM$=-UDV#PMEspDUMj_I z7) zB??r&i1Bki%AW&!n1HIa@#@z|(Nvuu2WhRIg@tdpvi^MHDQgWUe~oc2n2@ zG*v=E0>i1M=6G{lmDlvzYcyT}PAil~p?P82e@_5mMiZS%q)=$NF;Zeyn4TU8ZuH^9 z0vucX_krkLzq2y5m{JON&k7jr6U;L|78c6VH>5^nBgn}|icZ!mxc=*}6UYkn0!xtJ^Rz-Ij z1Dk_+wK?;wNjyhu7@ha{KR9WXXvs-lYVSXrw$;31Hm68T@- zDvi_r*m8n6M%(I^b1f*&j&UI2D7u_~k?42I6`v3zJ-TW#ZZ-K!VHw7EER>m_pTMkbkemN@Nd`e)^o4$QPmvN)HS<^~jCFd}^Jl*Vw zxH;K^Hf!rGmn@JJdXYa_++FL?_j&HHn>+!6G#tvmPx6DTZ2N3)4A{gWB%JpNjbWF& z#d1M|NZeFvqgTkisQy4ZrBrGMY&`{?!=UF<{s<;?{QMNej3gAWoL zbJ}$;|NZgY_|TYO zvyAeN)D~U-s2MA32^PdIFAjF{gfAhuPh{UOiU^)V=kClwXAJ8;((lJ9YKQa$%FA@A(gPZu- zc>1fN_a;sG$)D)Q@<;b2^1oT)W1mwiewD<|QYtV7e<*r?U0d$HCqTo`n;5slvrPF| zHqWZ*d?IFo+jDL9deTpPx=SJN0Q&)GD^ZEt@ao^qn~XIWKQDZ=4SfTfM$Ob_ZJ80v zxGu@7T|25yGV>rPC__{v_fEu4LdGmf1qUF%2^TzYgH_S#&Lxao=n%u1Q zsw1`Rvn(eb_hLF02>xgLK3PTVHQz#^^h^!;OEW$;e9aF!rBPUili z#|o-gxOWukK~M-e|2J9knM&td+Q}}`of8yb3i~B>%eB54d9{PRLUb~VsHGMraG(j}kOsSkzYRf|R9W=0$ zSJIL_W7M2%xJ3dni9u(}4d}^ef7#1T!*@-#W|Z4_y~aK}M_Qrthmtxkd$7S6*HRS- z7whd#9s(-wB?m_=eqR5hw1}23fzY?S+_s>#<>BF>>%oOF(O@%WC?DT3zrnw)ILmcH zvUaSV4Y}z9yRdbXoUSi>Ym@pn_y=@dSLoIVaw2v)VMPj1G2{mi^S%7ULanq_Zu!g^ z0lv<6drX}RJ$;B2%1_E-xanYY!(jgdlH;9naNt>V~kr_Yo<9GUsz!gymQj}R#Uhtja`C$@CjC4_U}I0Q?HLDRRe5R>88dl!pQ71?Ay zY@zU7&5JceBIph{SS)4yUKlb_vh>g7xU_6?^qLAkbg&uIc;xLW;W<$Cbivu?a~W06 z=}ettQ>a>T0n5$>CL!_zOw2JxOw*G3bEwxf8URL&zwmN%>sec)e<>_vL%QjMAOpSb z_u^1&Bv4bWs1(^THin9%ZOsIgQAG*=prXd9}J&ucIcE@a{$uE;M%G zgn~-L87H-BELi_ILA6?RY3A^Qg71cq)!Jre7S>@e&W|Q~bj#g6SXc!`+T1cHTdxW< z;X7P;j|oWR4Vz0A>N2{Pm6-3Pvwu`d3i#*O!XcM3El&RW5@tJa$ndY?V~(8SZ0!Xt zA`%ji>#$bB`zJ1}HV$JO&%3v<$atcnq8@)k0U~+V%@EGMA0vvAtYH8t^=TD^8}1(s%|N9?` z<2-5b)YQP}tQX>8X*!@|DQ;QYzoR;u`d)LO^`e*{&Yq8Sl*x=SEN}PY!Kx@nHc~%? zB8bV&M2`R%cr~@(<+jz$5&w0}kWF%F#YSXO^+1Y31lsmPa356~Ip5wKyP?$ld;Uv9 z3VS;D*pWr|Vpqw+{l4A|&VPPKuZ#+d8YPpKuiSlJWr3w0Ao1*~G7qV%_7Pw}Iy?uCJ{|829H~~AU28TA+Fh}a%p*c>5od7RA}r7Z^WtbmH zFD_6mW}!*;gqffpk0>bF*0GjCj5DlHm+uHO%)}7VEUj(d7cx@fR4#2F=HXk(h}+Pe ztn$PK^eGnkAVQaLH4^@3tsA^i&&cdv%b0mN$o@PU`EDy%`OGY<4_y`GtHP(f?t-&h zcJE-FrFc{+fMers5kRWYLl zz|TNi+mn7;(7z(wzXT+83jy-i)@*ESW2Rg)XB2G>kuYr%>APRE^BjG-lT|}$R!st+ z?60kY1tZFLf$QW8N@;)-1Eu?$=nkBT`pM1FlBY`8Z&)~nX6Kr0HXaK0UZj)?dZFXv z_o43O^bC1$nSbrCNS@_`rd59_oLDNWFcT^*Azz<`asXZS_Uctmk1r~^-C`$3MMWhB zj!8#4bnZgGT`fvsghi2#m{*V43kt!6bSW6`_u}87LHfUI<^cAc+jv4XqU}mSXe^%M z3Gx<;>6#=IaXEPv6_MZwq2y#x`+t6aPtF^4@c7po^~#_l!mN*lV|Dd!*{CyVq4S~0 z=U(HWQ_|e4*RNNHR|`S#y9D3r!_V>u-Jb6Y0gFLDZbbt~I{;#zyKv~8r#}GNPUOD* z^zz^6<&G0xMa881o0!obWCnCDATk=kO<@HZeo2LFFVN1iY+sy_gb$1or^(C9cMpHw zQf)S*P3uokTJ{2xRZGpg-_kLbs+G0en5@a++#7*2mKlvWv9eh&%@4+gS^h4xxYdGH zW@pDdHRrPBJOq_pqLVXIgSGC)(^XhwC6w;`sjKFkwl4(mfIw}!+N!lhVD8haC!cE^ z_d79^ie`EX2jtp*&&f3Ky%yiRm;penrxKjb&a3V zhjL)L-O44{x7YY`W$_dWeFeWIfyQ^*8J_lQQ1=9t=M# z!0d>}@$_Dj^EMR+KMF{yzM()^sWS;_XgWAjF?r7@ipq=cCG***V$V2513f`?>`SV5 z$XJ^*C_tocT)@HD-VQ+@+O!)0*9L`toyurS9Z)RXLSbYQ7I+x!kO zMMV~G-t;m)B6I$)7QmEvJUQdqT^A7gYQ<~qrh6<#lR~m74MJ*eXzA(VaHnalfl{yJ zU~yqCpm7ZhYIvusi`VCKMa)u1Ddz8gx4VLsGY?LzdQa!jT{aRAKoR%c5s~N3wMB9Y znTy`&Al=S-EIbu(uY2xs>fn&NDDoY)`etNM$%dVRYa-k02ra9u9`Yi5yrK=is@dai zNznCiF}wO8n=CM}6uV_(Yu7d~feB!F4?R|36B>DUUV?U;7yH4KA3jK~<>bNi^)1_@ zz;1#X;afKgsM0QYHT{f=bc`DcH9eIs-(5 zqq&LAE-zm_X*}!bNk~evJztlAA`Fr5jt8$gDOQZ+DLG$9cKL5&Zt8DIQ`{cs7r116 zZc&Wn(;qc7ew|f%WhuK(ye|gt%f6()P7a zBrnA28Jqt`AIZ}($u~Ihx4b&udM=Q{vXe^41Uim*9CbBFCmp;NzC6nnPTE)7ft@y? zbK4zaMW>MjBn9KM*aDFYf)~gbf&3#n(J^|enZeNJR?A^;GA^dd!Ar9$Qx<05musof z=n*@i<#AW{FPqlXa@9_MV6y%loIg>~_TJ7@z8| z$Ncn0yCH}$jU*5M0}?n@OY1tYSo3la?4~Of){~L@WHphq*mCV39b^Ky`@GEn6h*$* z{S@gm7oV2W5+c`}tDh8iygM(^aJ79Ae7rHA=zY6TNS&8^u)}o3?Ql$;GE=)|=YVDJ zrQO{I3%~=M=UShIS7LJls+7{et+{*50Y%J}KCmJXP;9KBj+sHCf{>SSsW75Hwe{TwDd6J-ijPW6&f|DFymh@;VON z@-qPJ{io8}s{`O5RCrB53`5PqfC5~J$MLCoLQ>+mOUSfuK$+39Q+ELF@F^CBpd$|W zvMY}!vp2OMW;n<{SD#&S^IB5D!g|tpbfA2NXkI9{nKA<=LHN@r+mxe+*f`J9e=w~X z89A6uJfuZ;2WK=rI(LcET$zxTL`Chf;!uQy)jh>LVZ zx{X1aWdD->={y3~0vGVMUITLR#EWF37=(HFQ} zkE0ck-+FDnk<5uguHAi>ln7zf`zQ$9PaT0f3XYnM50c0_XE!g{kB%}7RqSQTc68+StjM|TZ=z(= zv>Sl=W_=m+Q<8Og)19~_@8 zj}VWTOt+B&2np;T(BRFEfl-TAvygyl+P*O{4cCgXKRKu|`+9!lcW5qj{>xf2CML$p zcJbcRhp0e)jq7@GsTDkoByY+9B~&&MYZMT>hFK+48v5$-W? z|4VQ*ib87I2hf!d#CRG(LHgI@DQ{G(Q$Y!Fp#rX7Qt&Rd9J;$YE4?POI+sa0TU&c) z=c5^QLgMs#d~GGA+zBrOqxo#tk8CN1p``WpjI0m8#3;%{9M0!0D5};Tism;J7xy~@ z`xX-ucP}D2uD@@aj7Bcy)64MVUTKde4&ioCyy6HAmXkIb3F8Ok+Fd(zwPLjNV8%c-hPK~fVWo1Qb)`ak;YRJxoP9S@t9;SFrSllx?g z@i0BVE@4={??6gopk;ns$(U5egJf$Z9+KMH%Z+JDOj3lsSb#q`8ZW3+g!*i=n*-wF z{s%snAj5n5p0dv2gaqsyMP82r9@(&;Fj!~j2ltSXVYY*#F^0U#tXy$wP#2(dF+u&3 zAq8L-WW27q-OgN<_P~LGO1@;<7XkarvwrLcrW&id4}gK0)>VD#jRK4q6kY?4p2<=% zuQxRxyGrTxD<*p5nOzhSt&r!X>5ou9swheilEa2q!!B#8tH)pGMQ2&fRMwUcT|f^b zr1CF_>Kp1GgCkBWn?~EY5Du9A$IILb4b+_ZlkGc=#IH+X{X~B6c_)O9oX0$#j4_P*kT}LR(rIgu%wAr#2iykKCr)V| z)lZxN8oQ-$EYVrZ`ykbjrN|-DnT7 zUn24G>SATIj!`yLRIu*yc|3J)bF6hdTfR7I6a;?OU{h*9XH1AKW;^suU_?Solk~^- z;TX|$4c6*lxq^emOq7^2mTdvB&Z0n*{9}VyqtPyulL4~4GZwhT_4qCp1#iDR+jJYS z+}Kr?v6MYS%pg8pyK(FoJI?G)movNw5SeAu1VIakv9D-B_JlM7CFH4`h}xXbhovP&G`YF8w~VEio~sMk!oN2u^lbL@M{5jU zH8ZKpsJe24%Gn&-7tTA$Nz?xVm`REYUD^23jyr=B6Cc0zMj24WT&G~xN+2j9-x1t4 zj2$*qViTb~38Rt(`nkzzmCq7q&j5}4nV9p5^NV+gse|#kAyNzp9#eoBn|b2>+18N* zy}(7=YXiQFIly>!#qNQeU*vfETX}gd#c~PlEQhCz*)%TTz3=2TX%YEH*Wf5wmWHQ< za;S1yZ{C0ORJfk5VdelR7M=qSRzf~wf)aEb@+Jj-$jdt!otpS#1Yvhz#|~`N@XZ`l z=Iyc=P6n_DYgf?Y65%cBkkA9u*}#1Ii9jfUI6ftlw2U;M>6a{4MCIj2OxNJ{XOe{| zAXJW0aM*gz&4CTfnopBu@qnMZtwUL<-PJ?P;7+Djqcf80r^Au3t{SX#kdXIm`(?BDr8D z3^WIJp!vSQUBv?GHyX@nqLlMDh>-82vx|QHmftf(CTO4PRQzjG!wTemOS+A`#Symi>7mWXA9 zB$I7pn2h_zn%72vNFIFrxIWlpe_^kB`O5q;1wlwq5Q~(u<+JAWh^Q#}am%SLnKePlxZOOqXx9;AMgiS~>EO5iz~-Z16Js zJu0McC`nO3o(O_$ASu3cB>`a*H!cteM90WX(qY>XAYedI@xHxfptJDw%$R74!q_k@Lo3Fb(;6;r3aADqcF@=rC| zQvy>C+KYEhb|#F5NP`71f6DH(HR&&k1s+4^v``xoT}*+H6kvdl}1W_q-ag zH05WDpCo&_L6OojNR|?7O}NwR>ry~`y`xs!GBFbktfS5LifX@R;P9F*2#?*CIYGmi z&b}UF*t`#DAHIQwULYHPT3qkJ;V%r6jY`vQ>0*NfouHw8cGm=(pI695|=n53znJC`eoN3}CU2jJNm57KR)?!A|{qZ5})&H!;BFw)6B+jrk>hlVq ziwrD$eNq^jemZ4Kq4((v6_cp2+I_fh&b6bnTr)R0H91&lj&8)D@=Y>lSm3tF`mFz| zUY8z(E2cW8U6X%&_8awo6+@@19{d?i{9xp9$T1*G1O~Fr^DH>;jfNGy=~K`I&kwMv z!{A_o<^rR+u6D=zck%cD^g0p9vy#ck6VA_>8~OOnKBIw-6gMZwE=nacztkzczJC4s zt%G7$+Vw^vh3ZwTE4xFEcWKF04QR$h_w>dK%6HUevUXekjk}p?dE?;4mIfimj(9lZ z&TB?Sqt>O*E32#ehK3Ri<9`63iiZUHPtphogq)~5mwRn7@{#;xEtA6q*|G1Tx;>8> zPb$7utnNaqeqB8e8^{y;H5+@^cHEgc*|WXVb{o%+-^P_rweeDV2Ko7M-`!5_utF>y z(6t>LpGRK7leFL3oWCWXR!YI580?K7Bdy)wn;##T2~KgxWl6UVZy5g!KP^_N_QHfS ziI`I2c`ylRrSh?uz1phTHUQVEqRNpj`r#SMr31Q|0Ac!C&2(KPZ*)hP4u~C!R4Os> z74W}7E?7Kl1d%xRX2q?v1caZvAw!TVy!WiGpcWi#xzPJxQc|!P!*D$I)AnA0W(crC zJYWca*xq_H@!w%8Nws^c;ZL{9YAXSJlrbjjRcb2ekF);!TRVN?>AW%M9>`K=?fgc) zR%iU%OAo}6hZb0Oq@Ru!Js_b3*>Q%Ycx*tGUjVB8OPa8pLM{+{o03!54Gb@kVa6+~^n)h_ws({G{))y&SuH`e7^y5*E zY)OP^6t|D0UbbCV+af4rlX+kbBU)~gqCbX$N9OOIYV}@S-AJwA0Box3>rgoMd9c(S z707pa=csO_WK+2Zs)*BUU-A&dN{WJHC@f6?C=gnHGeZ91m5!&Kxb~OYkH`5UR;sWF z-l$5*%2IH0Djjc(-x2bB+nE@Dp~_Fjti2NdNYD&I;x+1j*#F@Fh%x^E@I|8efnLzk zHz*{8nuR4YJDXZVQ`6So{sl901n?S&`qzIKfvQb77a(Svmqq|n>pWcT2?`6N0fmuV zTwLJ&5|Wb7!G}TM=;7f(#G-fK$jHde+89>C&v&-eg9L_31>Slb>Ust<{%7$NntXyl z)sX;Vn+<%Nn8WNzO-;?`VF2Qk>$KjhsHjK~@ccQC+z$A^j<_9OX0C}MZ5)L0nQ&!gqUhG=}yV34E7yA}gx8x_TUwcFBZOh1I+P zJe9w$zMj?XocZCyhf|G?NwN@1uS-MiP*TP=gN2o+_nA+BNq&^ z7J3jK>X9rO96wZVtbHZ?ia`YC52&NHgxsaNm0X z;lRyP2p70wbuZC6dH#2q+J4KI>O5yHtwo!K4F<& zRTUB_&=j;jf&S&Qcgc{1zMjRsmL)@a7Eac+XzylmVP(P_jfOB)d${Rv0ujn1$5lGe zeR*we!+NmVo;EDyZP#tnQ7R{szP>(ygD=itX~Wg_;=j~=Bje*?>$Y+gZaZ#jezl9$ z%35IHT?nODYWob^Ji3m~?E|Emy~*M}b9f0qkX;W!EzN;`K7n_S3wplhPU7x%xauuH zL;Wgnp!p_jmHxKKzUki)Iw%|EO?ZHX)o32xmSSOPiGhVRN}r%o?Z$v~jX0qKY|kz> zsonXc?GnD#84bX^ksA3(55|v8k2l7mxpJWoTheOBOFt+mDRmI0d5V;#&RuM0dFz{) zbpFYLeUOwKm*Dc+CK9+_NAbrd?@cow$4iS40wdtqnh~I02+}#L?bQZ?AF6a|J zNK3c%^%cu*fsGZJkf4(1$SA5wbGY=9gyZnp+Yd*I`CqRQ$oue3$Irv)Xy@jFTn8tp z&!}D#$4cgC?rtu9OGzO|0XlyHe;(;2HzVT*jqA79jF?_|BSn zA^+GCzV6KvlgaO}=5f(>h15;G)wDw7I>16gk*R(z6l?WJMW>CNQX|F0FF>C$(U0^R z=3ns()fZ^M_bP_JJN%rfsVW=vDoH5xGS-@2HHpI|nM~7kKP&fyF#`7us5#I@5(!|w z-ya57ZC#h^Fm`lwjII46aN4p;v|2?(brzSG6}B8VNxp!sQf_ojBgEF$^=noGF?X8M z5G40uaqKVLZTMT27^dQ+dDgD zH8syc`*D1H{Pz%akSqQA^8#k$;1EG8_Set{Hix>Q;$LE^7HkdX&$*vQB_;;t8NYyD zA2#zWOig*h1}fndhXWiuts-JZG)zs0JXBPzy@`2=0nB#iA5$Fr`g#+t4hyjY*L!|q zCjn#WurZ2kOqdivtwZCyinNZ?Z4&el61 zC!|wlrfjldVO7>MVo3v)(Wxs(rOnHIweyCv%UM&kx6i=TIo<8`NDt=a9kq$je~t`S zp=nK8Q98H3ySw1?<#)o#u~o}SdS&y}Np`*UMY;uSZ|O>lk#WoEu6`s>_mh^sKEE`h zzMSm31kgv5^&segXN%`853^J6t2L%8pC08BP`JY@aC3$@dlQ@k`XYO$4W!>(PdYC? zyS?622i<=XPp9d7L6NQO^gEPGl#lXi-HTuGc$%Ke$+<$XHoffsc)wrv7tYkCuj2 zFn1rmN&G)FUhAUY{__rBt^b=BS1F%LH@@BBmrV(6EOnJl<;jwDWdv~4YirNW(qN_8 z2|L9a5C92WU7;go>MtdLmb<|H;99ujW@>G1&ZNOB4aP15)X;2a{26$wE%!?wIluS{ z7&0K)hBM6=dCItM2?Ip+s4V#IDo3T77QS37cMdS=ET7ILH#`z`a$*EkydaI7uC;nM z5oSqgHkI4RjK^h6i^N)40mk>ljsKw<%kAs$GTdv;k|yCz9$fs=T#@kpDhHIVy!$9L zf*#kbR%|p=at)C1jjxu%cgw+nP`{7g{6^(Hm^^?lCg=}}>I@AH!SA&nj*)))G3u~4 zyPFZ)Hvp4DT=ZUGCO12*mb8ihl{XU5eG-w}FcfI5|M=(Y%Ytne#+;gg(h@|dt~#rL z0MWgUOL_cv*>R^{c#En7ZvYrlaBXCiOmaJivO8+JV=a$C&VU5u(7$l)z2IBn3wL!6 zU!>x*F097yr5JM8yLt$kBLn(X*Kq8MmL~6Dy{9=6$clO{q~b9!aaKU4ZeVDPPc|6x z&a>eN_PrXw`uh2btWIq@R~{Vi+E#wBxvT|!0?L_k-5D!znib|RBQ{F;!hZ`zS4;?R#}nP!mOXP&tc$Rw$9ct9u~76B?nfwCJPh*A97K zthBtg)Eg0)dhP~P8qOvY)^qI{S+uW#6zFYwqSIVAKzz$U3JJy&fvnzObnphC`S;TQ zFUdh~kmzw*E(j|rapIiTh#YT<^^`5r%>aht;zuT)t!t-Z^vi}KMEkw=>U-u!wPF@ zosR=~)Sb3nz+;Oer2-1bgDM$evNm0j}*RS z1EflbSCJsYZ%CzXkF0EJQwvB%?rtv4TQF%t+v9ueR!u~7Qh>Z^OJ^(I9i1?mplukR zrGKtFZXFXF&$e2OOMKTw5?d4bgQUcAJSgP-AUU+6U`oo7A_Y$K9sHbMIPr>+u@&4K zbdJ~D8fUc1%h8O-8lVLU)jBMZ@g+bA@IF4wvU;a6_o!m&tU$Ae*$K$?LDO-ZQzt$a zw^y<_r_}|@v6Ahe2%6jGm>LABpykPMv4T2JjU|RjKb#|}#{RgC7>KyatEvWTWKev( zM_;Le)9RezaeIBX;63SAM~H%g5|+XvWoL#a!TFCfa4H_7DIxMdBM*^?(8^8F@)MpUV@y^F=6Neku&c+#oJo9e^;H+&Tc`z9_r}DFG`O zvWDEJ(r8EiNm>8y=p66Og#sQAFiP)03XC*a=12(Odk3=xNN-V+ikjN~JSmgDscA+f zCAdM~`yBdvt@?MQdtPCc9?uoFOII-${jWaAw5Io7Bkgp;Ou99e3fuL^BBM~ zE*Dc6BFDREW*p6oYVVTbG7=IK?EoC3WIfrzhA+mlS&mFZ7Ym9zP=KxY>0lx=-e-YB z&MOMIfn{-Qs!U^LP`9X3Yl;qdypGP!2*8*+gva5XMpjSa(jG%nCE6Wa63lJ&g%sYG$5In8(9sYg1r;k_m@|w!RgBn`H_|b{E z>>l__{wA4$rGTV+puSTogy?bW;7tNNXNY^C}k2p|ZZY{YI8~Du@VDl@rL69}>cDSsdmpeacyaoO?OK7V54*)Lb^OHt z)dF;4*c}}bA599Oo$koKmwz~OMwYES5r)K(D%GQ8y=dfwQ`*EF^wrZ{$5{V2V3q;0 z@x4CLG<0j>NA8i)QCJgoPWEuAIH4H!&K<2+IBq#!a|G${`My9;D~_H@EchM3o3`td zuR%o?K4>2+nNv`%RlCF8lA%xQ3b;=(tnvGX%?z4;0BZv@7hqT774%^X(9*^akr)Md%LB_Xi27(py<% zfc6885^K6O7;+Q}8lM`rrF#K)XUlMk26%}(-_S{QyD;ww6aTd1Np@=?;iLF?_z{S3 zqI?J6{d_i@{i^-mchp!+0t!+sOoDB~((3}1PoI7a^F9cC-sF#y;VZ>p6{bR%pJuk0)e2-HfBcm|WBE#}WhEp|iz(S(mfTYXC-F3Lal)79zEPkr=^T;c+%W2oHrB=!0FkFJewWO8c6@?nXggp&Av z0_q}wJ8*j(zx`plalAA1lnad9Y2w=$xfUx@+T!lHKUl5LtC1Wm_8{v8Nn~QIjFD2! z?~Tvd?(kDEZUwX5wFPG@q}av2ust*?mj3Wa8>4c#+;UgBkyel5_3`?r~%h#n@R@gzz0z8mX%7PyxY*WVqj-ykLS&jeLg^}^fTBV|*fZ5dIUS314sj||U zUM4B58`UOiPTxmzEm?q2IT{gwBe198>eG5x$$##7Z8SVQTs~#70w>gAjSV$7kCDt; z|1;;?9wi52H3BICulBEKPeQfE-lUG(vbbJO9Md07P0S?4jItw)DXp23C5s&o4j3V5 zif(BTj)9nE2l>ZP4v+4l@PmWXUlyFv5eZmANd!Yh=avj8h|?CM%5v8kS*wkqr}>SZ zzA26}k}mOPr7u?*D*L)Za?YofIKsckpQzh4j~??^GOS#xJ&nt$8s5n?I3CpNa(>(N zJqGIk5;?a$${IJd-wXiva%b#jz zj6%ZqS(hg!V~rAtr_|4%-&#zFFn{_5)Cy}uk?i4NZR5>c3v!9=xC+K8Yzbvk7OOSH z*v8l$L|9|sM zjjcBkh$$|?rH$Qwg>XI-%1Yp^@oE8UlAtIaxr@stEI3I#iwwzzrH5Lo) zG1_ZKEoH6jp=KdNCI%;zTZs)Ou2YApS^by~G3nn7T)34NLh`E1EJ&0JtEFdNRmI&> zv~+a^@df9s?^P0vD>LI`Sa2dvz4Mcj=9Z(_jI_DHu=HCC!7^%^oR|yEhu(qEZ9K(?G)c^p+0Nkw&Q(KEA3pEq^cGoIaFU*E z`3=@i$TaRBzfd_mn!>BhlZ2%e)1CFzE3QI|qzz9tm$pOV^Uz^(ZL!>p91L^wMhtNt zz`9`IU@hh{?f|Om0m(Qns6CMR%oq?Dc-5qN((0RLJ@mdut~s?Aic$`WNuv{De);#! zB-Xm1_CZ8+I1?Z2pZzvruh3T>B0D<{k-nKM%h7NHJzf?&%kfo;l>o%kz2Qr(#>sH2 z^j!QcH*X{aBCF(tbJ4DV-Jd5~`AS10Ig!;~Ky`{+<;_K}X5+V7f%D`>K1{%vv0JI6 z5!(|0yAd*GX1EnL*Lc`3e(E|hgoiy<4M%D?Bd2B>xVxIS@~u9H6`z}dknzZ<_&)Tv{ktf-X~0T&k3rYNLg(DxjoK8rZj_+MSg= zV*8%sd!(Ua$-PEmv;4l~4W|`GJN^#`rY2xXE2D~lfd|9r4W_ymf&^oQtWt9%&2>V= zUf_jpR&Ji1B=e+v0AJMHtL-+gUjTu3osWQG;!waTgpr5qADt4Z#kw7ZO&REof6G}> zePdu?RPJ#?DVoL1_%!@imD&9L=&sIfznr#o$6TXqaxbyD~4%cQ=J z8A^}-kow@R?*3M@YP;h}3?N<2R~@O9JX@M7uvY8CbbTK)|C&iT1*DQ)r|GX>bNhk)d@tCiaPkH0-1ahAit|K4M5KiO?zoZ& z%!hbx>k=?%_hqLUM@2>kMOWf`ad@oH=b)s_5H^jEf&EQ|;>HHt6fhDJ*w)rRolOVY zBn0)V)qwYzwBU4e+G%4tinlpmw*~rVVxIgo&+^T7vk^WB!;Bu$S%*6x`xOpJjGInT zU|}$3cvxK$u|46@>^TNeDB627ZCFC&*|fHy`yP2Mi)H`&`O8JNhliN^r5i9$XY6bc zst66~9gJ3C(Ok3p%r^2tMA)<4<8iLaz1Tih3E`)!=Q(-oU(^B&WoZu%%SQxE=dEBd z^)J^7ay}w^(vpILsL($X0R1ZbK1cT#>fr+vQkyGD3uT}AWY{FM$(bAM_ z*EPQP!X>nCU#0h+?ab7y2yK7Tx*ia(`&l+mI0(}}=xs#n#p8wIBe zQgmM5+77ND2t94}18UgwwQFbDvoZTB!C5(lQRsFTT_i};sMQ8C9%Th`n7QDqjD#!C z-$cyL@@JC z{}}4PE&fP7lkQ}?*q#?8`w7P*(yU-M_pt-qACw??=?{^Xq2aE#kE&w2z<&pUlxXRX zR?6BC9-yW!rxo(>Rgd6eARP+w;t>~FN$b)nw2&|D6l_6H^OXA1OzAx9B;~ijq*C); zM%`(DX(arr{bd7>YOzugXOicEwVN2t0Q3+?pVE6WWAREpk5*K%^!anUdd-uOWRaE4 zqiNspi&Gdeu>WBAbWahg6nK)t#g$^2C5mqHA}4gupT{SO(TEIow)yc8dqaLy64;zi z9%O2tmS_gQyLQ8K7%z-3{XgIO4dbpUG@fl>F89A-IKoe){|>SV{QvGhdgEEk6B3B@ z^z;IQgO^vjqVJ=k;*pU_i-_E{zm8|6e|@?-5$$6B%bS>WUumA`_Ar+5?{uCkAB`xT z=d`o|rX#tw`-|^DYi^UNikI)+y-VhEjRNsku55~AliLXqp9?d9tqEgJf1Z#Mx$Ono z&G7T{=PPshy8C12Sf^>lFR|a)CDmFOb`JiLzTI$#h;h{qxBp$q`e%gX(%|XrkE>hb zJ$cfWz@oo@a8T~8T&l&71Zo%(*i9c>H=Pg+0QMFY{Rxt+tdSM;2PqOT`se2g7V|$t z8D>&na&ZN;qF6UfDqQq1sMah=68>%S-KI;}WTdw_`UBiBi`_~m>sJv|0Re$iA^qtp zQ=Cy7{bX(ajYgGk*qqy=@wX7b5JaV>zEfE?`Sl_xwyW86>(rMcj7H$* z36fG_>;R3I#6*(&NSOpCkJDBwT-(ENVZp5WaOhSb%X~oHWVY&QG4E_AzMP=cd%fpK zp4Q~O@e9)TbOuwL?QeZ<3-%i;D=J-GuSlzQMq@i<)NOmdyz5PAE}QiNX^u6Gr`(LJ zrs;eQ%0Rd~YH+*Wt=FNP!iH-P5($^3D1vd$b*Im{v!|!hd*N9DH+Ck(e*fbe;oYAB zfkwp}Q+u2@8nyG#fmY%2S(fd2R>z4fIdAtl&F^k^&nVIyCW?v=&=fKrhrk)@<|au5 zE~P-S4?6B+e0+PBscF?WqPV|PQ&W*F=4yPr`A@wK42|~J&f|&QV2^V9lG}Th7s@D) z&Z7s)td?mYV&b-lh<6@`QzB81qmgXv>@#k*?BX}HL&UPk(!a`OA_dGkR zh?|l2=WqoNT48CAwU}a*Vf+ITGic1lH}7vwI*E`v_;?_IgF_Mv_lQ93HRS#BxVgT1 z>veAW@Q?0^%Tl>`XIIys+f=VOR7}G7DeK>B3|Fki0&_ndXnVUlGNEmtdltK$3~zJP4+Ni%%s%kZ~=HhFWnst+jmh;ky}CasSA=)@3{i zx)Tn`eBaz%mJ_5cd0R*KqanDM@NUKGois+-^T3${C#IkKxh*FpB_;UhHcT_HX!{L? z>y-ooUKdR}1!Y51R#x6^t~K#p3ND2r>x^~=tWRa}BAMT63~bjQxt{)@ed~D1&g;Ab zN;G4G(!4_4Pc{M57b$>(hOEP`#9wo{-IFeUOJ8t)W+&?v{LE=^D83uEF6AheDuo8! zA|8NiQmZlPKa2KWanaD2y6o!I>0i14{jKqd9s#rL(cK}FRBt?~dH;N3B~)4aIcp|M zC@Dv_lFg&$>+-m2?~(vma2OX67SgQy0Q2S9jRN*=t*xyVvvmjGj=BMNG&F2zSR*Pfdf1;- z`iZHE5eXR1sS6aw${4iq7S5j=Iur8q)9`J-gYO9tX)PQnt-|oI&IOp{D!CxvplEN!B8ewko$`7J`UORLWcoA_TM{qao#0-n`@V*N?Q};? zcw(HGD_t23(%jyjJ?!{Xi(o*rp)C3u4UQ>1ug^YGY!|zsOxwI$tF50)xa=T$;YO+X zT*sUfdCSeJKkQI0>no!-uH7xZ)F z3%8LPSz~Gs4I=`R?lA3K0Xk6~-#{Q_>QS{WI}Zf~CsJf9ENnn%12P%rkX+KO$@*Lf z*ZhX|RnDReXKD@(4(v4+Wr^ff2AbO2ufx0BJ~PM(x$~sR$%QW^a!MGT@301knDxDv zE&)gDzV)5ly?0N;EybYSwpIA9=WkFm&%0CEhF&z^)y=;+)BTaNlEsV0hA_jy3YSVzw*IXFDd{rR12MGEOf22M@VF7cqOQ_@5f|#S2A$>$(A= zB8RzGY$TCBVI0ZApbpf1fUj+lw40X2=O|_TE;nH{x;%e0ZM)u)}r06 zS;(OD(QL2p%vn^)Kz&zn?e<(Hzi+YE^3%W>?vKz8u>l%e6@4&@DbJ%<8IP#pNeMke zi=DfBdyM5aH*j=yb#Wk=JsJ=6vtI@0XE8#Zc=Ly-$!an%_7%Ozuyn`O2+YC$8)91V zmNeukG%qD7sShyBx9NRDu*08@5{4vl!!5?9p3%8}0msLOa-jnXBf%kIX+HI@Ks(&F z)=Q7C%HVlmHbd0m!|%tlzx|sy0Ua=L-G=L+cIneCdJ2lw!G(>GYsY8%YmKtFf7WSi z;h#TCmgNk29n4&_(1V6v`W)Zk1^OesR9(oa#Um#8O@L2%lFcxc+A|1PP4;~?=cU|V zTXT4wt?PUEI6+3xU?U0YCCPwiJ$$AE$pdmbjlK1*B-zDj zewW^8ndsps?_6m>G;1{3o<~?~A3oG_%^s@Y^!O;B)5!HC;6|o?PYB44=2&V0p4~VS{zn_+a-S zN>Bq18!Xs%KTA7X8NTH>i2J5Ca*GkdP20SvXqy6ESd9l6H2*-eNM{qd9|v zuL23F9MqLfhPqyiC5ciEBwo75N=#Tk;AIU3mH2E>WB#8>#O>5|p?^JG9fv%Z>LVtm zK;Y#-!m(*iODR#-mAfVAwhrDjthMQAggkw8fcT= z9mDlQ08@l{SlkB~bZpfdXyx$4n!$u?xLp~0yf!_=dS<=&MEH!Lf7oZ z!ugd#TJS@xl!>)up*1kbkI`NBx^!ohy0PDA^%Q~FLD_f_4_X6a2nX^irVqgb%;yl` zIzbl7(A3BTQnJn86|v@J_w*|!*CtVctQtT6csOtQ+9Iog*cY9;e}xCgkNkuyidwB2 zn7&_pDMMI6Qp=)LNeF9=o7;lnxp9?l?ds%^H{VKrRQ(D`5b^a+hh6j z+O3`Giv$D&-0=`FffPmGm-hbQr+Dz_dOBYslUJl+Z1OL;TlE4to*c$U5o)9$)vo{l+FNZOoC#5p28KD zL`@6J`G{&5eSgNIWAAAuwc+6)i>W9dvD&@G1EIE5a@i_M-k z%olHP!SV7oz-6pd?~0xszr?+2G1*%So0Swl_u#=Psz1A_^oV*7ps4pFMY<1!LZ1On zg5%dOEcy@lh`4$?D$p8(ftQ!S^=KVzX{E-LX;1_3&>bmt(|@SR z)>5$84)GMAi+L0dVy?43thy<5IMF}D0TeH6GhGsjAczHO#~$a34{L)zW#ah&ht&@N zbjJl_QSd(r*Cz=4oAob!=7*y$t3#p1 z#_hIu?^RYKZBUZAE(M5EZ;H^Eq6H^I(y@GXhVS-Q@eZ~l01<{yOgoP?0G|S9yO+$% z)`lC1z){`$IZMn?o4gK`HK?!83QZpZ(#yz{X<_UQec~%W0#*d$Wy!U${Sd8RUn(jq zaZW0Ye`wTQWcg=rZ|?HKSzFB3j~ z8f!P^u@~h?0jvr3#z|ok{l_=yu?rFzRz(BqocW{(MVgmU>|x-UJz2_@kff3`$|fU)Jw~n$7TzzSZRdn5lQBtR#!QK;AE+H$Ni#P z70{NT`YIkJVckKW(VP60j?@jo{rOcmkVkap%`f%?SXc9OgIYF{QWn^ZAVhuSP7dZn zB@Cf^X_D`=`rg2Q*U1&Of_dz-IJm zMFo9ujV_$&oi|2TfSW+?O*@_yxOsZ=EXSTe67~po>O}9&)h!bgL1O{z_NR4Jv+lAZ zlw5cJOUgcg?*H@|k$eUf=aWe!z>uXW1*#;lK=VfTi1~B|3$UN+lMz_TmwrQF z7j%ZWHhQpZj4{75R#$<8L)9t&Lh{YU(ILsT>!Ux)*}nq+u{<}Q8Rm^KV2kkrO~r5N zopVJq%Zk0q{%QZSr_czfxAnXP11`L4Mk=JWK`;9l&03=}#PMb#eIEJUE(VerC6HuE z^v3}!R<;9(S)t^p7h77Ab!ZIP8%M`wP`(M2>m-z&@)r3V-ij?Gy5RacPTcj5E77Ls zOpV1z1`GC%IjZ6A2hE4TClKzdUCT0jE2# zKBfumifx1^wVyY>(aGK!%arTH^8{IbCL#XtfyVP-d5M9!^m$i90g6Mk?eEfLSZ7#U zy$2Aimkb#%L^>zBEj4>qouRK_o#ktvcopC*MTKg=qaci)j03IK^dvJH#Tiu|*gDopLjwpgbe*D!b|M z0I&PmatS3N9+OI9YssR$=>D-ZI8M!Fjg|s&EhIE&S%eY9Rg(@%eJYghv#-KruX5I1 z=;?Z|O8dkKhaM*W{G6ER+HB4rZUI@uGXV3Lxkyy-=V;%gH{zWM8a35Ggd_R>!mSF* zoMGVY$*JdGTmxDY*B!%g1#A!z-Np+s3v@byP$=Hfh71OE4TRtws5VAJ^d<~Iscs%s zMgs9`cG9f4S23EE_N=cC?1rlhD2ZC6xDz3ORes8Q_`OXj83}kOzgkb{)1#VT0BQr3 z@wMlTDT2G3iwwI*`|aIUIqLgN0C&cM>Npw~4G}?shvwsqAJ)gF%n)oLfC_Hu@6U$N zHD3ITAxf;=|J{1DfSUJ_bRMdA^!$K(Ztv)vEu9!+Y-$Rc0;y;3sjGL=Xc_QP+XJlu zkT)HljHiLn1qZxYC0YfHOpl=~1z~*2OmL0(~*pMK!b8NQU_`K1JEvJu03w zL15=c5AvJ)1%IvrFAriU=%m0?HBB+=sh@BQo5HElJX(+xX-U3!GIKP&ySldfjFc3P znyx1R@HUtiN5x2*UeP!jLOh@eMIp`-K&LDM%O4|t$gAQ9 zrE&_+{(f&Qas;$=mh>#e6`;`>u~$}l`?{u`NqbEqQ6|McJK9Naem8sk#+j3OLIPLA&pFKZ$o^L}WsiWs5? zv!%5$=U(7erY|pNfH)vUPz<0(0y>8vKnE2C!l>NoK@@pEOswVNH$$Ix>4#L;|D-<1wdm=N_e`veT{!Kk5w9sPWb@ZLQ=9>CaIrl_hvAl^my0v?3Wu9 zA<9h5f)swSGhU!}{ls;Aa7_IFMrt5qo8LXaDLgO|EI&$8@S3T<%=50Gb*RDq&*Uiy zr($WxymQM_#HEYFQ;V2;iayajQ1$Eyvu8r_yXd`V`_*nlp^qgR70fFvr$5lv9__YR z?vQG5ew01j82auGC(y!QSVpOu9&`*;;=OWGVzs0(fW0I6R+pltHvDaV@m)@*$lxWU zo;f#*Ja(~9*w0UGBKrjj6@ZgVN-E?c8UeO+G*XbCy?(fcTDXcHa4ho!O4{a>bh7Xc zq9C>uy4RXwRnKbQJ#w=Lsj)~%8etpefx-?Ll4Ruj;m;PB?3qkMzQ z83ZDDF3nG21gJ*xXZi0yzW&)eNON~$?9XHo-}Y9#5YuiJ&aAL60 zb7|IsPhef3|K^+~6xj{pvXE${eTXN^n2_pvJ3)lb;$C%ifB>{#TH2^jw%mP%Byr0S zHA6KX@fqmKSi}{d21Z6Ec%rLmjRq>E?~87q5wmiP5C@A;`$gKq;+BkIUW$re)$z#k z-c#=xaB3V>Lz)A>R#xU3-Q7Al?2-tyamf-Ko~Q`4VuH@*1Mp0_!D62$pn1l32RcxY z5(G<|sbuhw!pzE8Shn?|m3m$-^^9&-R!+>#2MT&n`kjpH)!E3`I+#3#fy|s#+iHOX zI9yR<7uowXKOZpffJir|9*YB!ok?Ho5owtxP-%8YkNanui0e&5mm5U10Z+D@+cU$k zn@a|S6ehAtJ@*~oPyD_Uu{5tPY8d-QRPFlj2?3#&dJq0ES)=$fX3!V~PL4?>1>pXG zIz8n0SAbBUaz07Y)@ABzU1;X&r|@+HJ{?*IW@d{mOg0oPFVJ4-Ze zkjYkHKQ@bZ*B)v)9ImgG&AbD2dv{z(3_#VCRm{VpFKIhYQltnV=owd4?^)g4d;tzW z2g|b%7i8wWeC3C2Kg8HrRq3;wAki!8bs_3Vm+B*ixbTYyx(+Edp7-q}8S3T)vhdH? z#0ORaL1E3?^mPsYQybxQE!Ul5;r^ozt^{AO_F*4?JAo-|hBx{6C zC}VhhZ+?oT>ujA)PWL}WQ1I&h5a`cDl(Glf+tzhTx9${kZi`VsDJbY@|5Dx{%O~Ta zCi*zq5CP0sMPm86INHcsO0_ zG#7c8c;yKj+j_t@&`r{NU9fkvj+dnyFD62Jt0~^GmBqgkG6ZgqecmXdNig=2mQy{d$m61_lo+KH%;i9ay9I!pM2P6#oEV1Bh+`l! z)vq#^+5hzD5xQ&Ffx$mJK*yIBz&L#^%lUEQo)0++IbH}5!d_kKMvq4$j2A$P^R~)L zUDeNmOO5rJ-w?@^wO*gLrkIXdJZrw^Hw-XBAepOLMQg{`x?P9+Bn0Q?;se)Ll`S{n zIF1a^HIjSw zHWx`mrcMk?mKHa?OuZ+2ZGE5D3b0pID}GW`GX-0h2kTe>7CPZ!-O`E8h#VXoy1Tng zfN%#W9MWU_Gw~#T#q}oc_$sJ5+yWrm56#;5ZCQ6SLS`o)KYBF55e{(7msG%A@H&s< z64Q%B44E-zxv&EuLS|62XuR+{P;2C1cv1lbXHc?4_G%=uL9CW^tg87jM>P;$js2%w zfUjTvFF1NXLtpCu;|D`=kA=}_4*$O*1|5IG7z{2ln&acYA3gt{57Mx(N14}^)2M%_ ze^Xwxr|_#$Y%RoriXR+&V`$t!*?AM#SdVTgP{?7Yq|g9Q(dYe^jvkX~?w^GamA6h- zWO#zieacLbUpbAXUJk+JTDMrbINM_2Dm%gjZkZdIZq*(-C_}6?ql^NZZ|N zx0aEr)PBz{`!-F00FB+R-_nIQ@gP7SOt=JFa!BqDy_~@?vima>IyGZ&yF~5y%Ut=j z(LK0vSI6eORAI&K{0~8mH)*a5a2YnWw+1XynL}p8Rhs|Kl1}k03VqJsl?j@kF52d{ zArZL2jgMIc9DW2)P9?`Xuc_SI)b`E}1>#b$ed2EU2X~9y`;bT#+J6d-mAf#!mqvA!ex3%PfIe@*IjF z!|UMQUVX_a4cP*ZPcm{B*U}P(Aeds*0cf}5lT_CH)+X`J)tw5u=8K2$AgTlMOKUM) zip-mS#6E|?^a9Cr?^W404)4|C`?j{E6p?Lj56D@|?ZGZd5Zr47be5x?DZ1T!xi(kA zPqL4Y&uEf^>KE~4KTf~d7Pa|-h&nbHsV;D_bRoG2%uOJ?mX@tTYdU>ETH=an`X#By zleho+qsIRAxn)z=Qu`+6<>5)UjZ4aj1GK|P5^mD;N!@woCe0rEnQdwthOTI+z40MV zYK7-VPLYU`+@ZD&;e}61t0zqUo5<4%3&~AdiQgN4*LJDM4)Gzlo<(j=E&qD5MPMxA z0_MOr+gaF)s(J4=xDj%~Tx)*0Cv}-0@oo%0ZKy>(Zux#zB5cE2aL%xK=0X@|>^rGCc}*5k zIeq(X_vA6{k|qJ?H(eR*Q*(5f5HNMvUZl(_+n6{UK@kFWk7k`bKFCp{OOIysf->WG zy~ z*KTHffkkQLh!du#fop!Oe1MZK}pc| zk?02;Gm6oVJ7_k>QdzMnC1sp)q9Eh3;C>1(n}2WJVrfDA%btPx&lRbAErCVV>88?J z*ZHz+U&^Ie+jHu5U|M>$eG475&R$1GpLp@@-}ROf{)%vUB+`CSnLQ}wrCfUMsBAE_ z_8}jE`270H-1EE#4?qosCJUc7l0;pb#!0#>QN^D$OY*5^ib)1+FX$NXJl@SlkW?zM zRQkrjxN!2H#@(x6Ugp|$kKa@fe}VMt*z6aTvAn#Y`}UIDe|@@($YTQiS zYv<#szq|3iJ=lz-dDq?9M61ICm@BWmrlgw;s6>jjZ2soo6w8mLYtrZXiqvo${{W*A z*bBM(aTwW?WyE@RVi*l^#RW=-O(ikNs=G?E<0Vx|Blph}yUDXa)&9*7v5wBBv?rNU zJ^}5^VM!h&!K-eyBewS1W;B%D+h1E5SF_0yliLDEYO5YTAq9JjIGYyctI;BplK$6J z1XQz)BuPzq9hbH)zHlFZdNkefu^wmCY5(Db*30z%5cn zK>T=5RYCcl?TfRtC*#Ra6KX0hMmZ+0y|9;l#hAbFQ>WCC3~SZ%zW0(s?)S?-Zes}I zw7K7#HfepMNEco+=7qXL*#>O^VOK7MsDIi~u#Qca(=HmWM! zm3I3U%CD^XC2OkiLE%V*`=<$=FQqGYm*$Q6fZd9Wgj+^O#M`?OAx^66#!e+M$@z;0 z4u(1-$oR1{>pF$&lx4()S&juOPWjUzRQSFHTm|o_c7H@5`x>>0bJIO>O#MG2Ehp&fGsSB@gxs+Ya{ zQ0D@_m|(=YsdL@462?jkJ~(UwtiCCel9d};`^Y@Xw6)5o#M!A2+IZt;l{W-gvHN%ox9|<_yIQQL3KxNd0U9lqI==tY z+xrJ=Z_Eb0R<0K8=$Vb`H*IC<>}dK^2(5X7H5(4$d0cTpx2&4dqR^COY`)I!OPr=X zJ*$O-951eY)I(+beHHj~zz`A=X&4^awigW&No$DX5vgIeD6RCndYCCm=qwy)#-^qU zoj(74a_ZOfocdSavfYbF5$@2YX*4p&2v`*|St-kMZCrmf!6V&3e(MLN0|9D*~by8ANq0`X6zjpkkj~jv;QeMt?udUF)8$*oF?I__qU((YQuWoA*^Z&L=4oPf! zdHT_%4T;o#|2MXJwp7=A?JoPEl8Q>FVTpbPib%y5U)KK)3pHowPg?brc3{eSp(6%i literal 0 HcmV?d00001 diff --git a/docs/devlog/devlog11.md b/docs/devlog/devlog11.md new file mode 100644 index 000000000..21524bda7 --- /dev/null +++ b/docs/devlog/devlog11.md @@ -0,0 +1,100 @@ +# Pocket V1 DevLog #11 + +**Date Published**: July 17th, 2023 + +We have kept the goals and details in this document short, but feel free to reach out to @Olshansk in the [core-dev-chat](https://discord.com/channels/553741558869131266/986789914379186226) for additional details, links & resources. + +## Table of Contents + +- [Iteration 20 Goals \& Results](#iteration-20-goals--results) + - [V0](#v0) + - [V1](#v1) + - [P2P - Presentation \& Audio](#p2p---presentation--audio) + - [Savepoints \& Rollbacks - Presentation, Demo \& Audio](#savepoints--rollbacks---presentation-demo--audio) +- [Screenshots](#screenshots) + - [Iteration 20 - Completed](#iteration-20---completed) + - [V0 Results](#v0-results) + - [V1 Results](#v1-results) + - [Iteration 21 - Planned](#iteration-21---planned) +- [Contribute to V1 🧑‍💻](#contribute-to-v1-) + - [Links \& References](#links--references) + +## Iteration 20 Goals & Results + +**Iterate Dates**: July 3rd - July 17th, 2023 + +@red-0ne joined the team, we made a ton of progress on the latest v0 release and we're honing in on a hanful of demos that are coming together! + +```bash +# V1 Repo +git diff b1c64d3ca89b2c284b2b22ff7fdeb333601266c8 --stat +# 152 files changed, 6579 insertions(+), 1072 deletions(-) +``` + +### V0 + +Lots of work from our community members which can be accessed [pokt-network/pocket-core/README](https://github.com/pokt-network/pocket-core/blob/staging/README.md): + +- Snapshots from the `Liquify` team +- TestNet observability from the `Nodefleet` team +- Pocket Prunner from the `c0d3r` team +- Adoption of the RC 0.10.4 release ongoing + +![RC 0.10 adoption](https://github.com/pokt-network/pocket/assets/1892194/5684b877-5a75-46df-9be3-c5967fa5b309) + +![Documentation](https://github.com/pokt-network/pocket/assets/1892194/d80e8a0d-b16f-4880-93dd-6c295831224f) + +### V1 + +Our goal was **to finalize and demo** as much as possible from the [previous iteration](https://github.com/pokt-network/pocket/blob/main/docs/devlog/devlog10.md). + +🟡 Though this was not fully complete, we give ourselves an overall `6.5/10` by: + +1. Reviewing & merged in a lot of code necessary for the demos +2. Doing a couple of internal demos & presentations +3. Aiming to tie the 🪢 this iteration +4. Having one huge demo showcasing all the hard work from the last couple of iterations + +#### P2P - Presentation & Audio + +[Audio](https://drive.google.com/file/d/1Ps6PAkaUnbW8BSV1bmAFAomkwr_YtMdP/view?usp=sharing) + +[![Presentation](https://github.com/pokt-network/pocket/assets/1892194/1cf6ea45-0979-40ab-9923-6a5f254f2fa9)](https://drive.google.com/file/d/1MiiCRxMyrO0T-9nAzUSV9ICX-ySQ7vGZ/view) + +#### Savepoints & Rollbacks - Presentation, Demo & Audio + +[Audio](https://drive.google.com/file/d/1NO6n6iwnvqWgIPVSUNJRJTsRBzri8Oub/view?usp=sharing) + +[![Presentation](https://github.com/pokt-network/pocket/assets/1892194/73cb78e3-0709-4cb2-a5f7-d8efd0a77121)](https://drive.google.com/file/d/1MiiCRxMyrO0T-9nAzUSV9ICX-ySQ7vGZ/view) + +[![Demo Video](https://github.com/pokt-network/pocket-core/assets/1892194/89326008-621e-46db-b0bb-2f51e84c683c)](https://drive.google.com/file/d/1N4G9TPkcxEcYGq99wR8JrDXFk3dMvBGl/view) + +## Screenshots + +Please note that everything that was not `Done` in ` iteration20` is moving over to `iteration21`. + +### Iteration 20 - Completed + +#### V0 Results + +![V0 Completed](https://github.com/pokt-network/pocket/assets/1892194/381cacde-8e9a-4b15-8b69-b8e1f2f3803a) + +#### V1 Results + +![V1 Completed - 1](https://github.com/pokt-network/pocket/assets/1892194/17c90d3d-efcf-40f0-b0fc-793343442524) +![V1 Completed - 2](https://github.com/pokt-network/pocket/assets/1892194/584c28b7-76a6-45b7-b6ff-aa5cd2abc482) + +### Iteration 21 - Planned + +![V1 Planned](https://github.com/pokt-network/pocket/assets/1892194/6d645e0b-3f07-4c58-ba9d-c99fe672fc58) + +## Contribute to V1 🧑‍💻 + +### Links & References + +- [V1 Specifications](https://github.com/pokt-network/pocket-network-protocol) +- [V1 Repo](https://github.com/pokt-network/pocket) +- [V1 Wiki](https://github.com/pokt-network/pocket/wiki) +- [V1 Project Dashboard](https://github.com/pokt-network/pocket/projects?query=is%3Aopen) + + diff --git a/docs/devlog_agenda.md b/docs/devlog_agenda.md deleted file mode 100644 index 3e4d841ca..000000000 --- a/docs/devlog_agenda.md +++ /dev/null @@ -1,12 +0,0 @@ -# Pocket Protocol Iteration Reviews - -Protocol Iteration Reviews are an opportunity to discuss and demo recent developments of the V1 project. Contributors and Community Members are invited to share feedback for and get involved with planned work. Join on our [Discord](https://discord.gg/sae7XfnF?event=1062036308450615316). - -## Agenda - -- Current Iteration Goals Review -- Current Iteration Results Review and Demos -- Upcoming Iteration Issue Candidates Review -- Feedback and Q&A - - diff --git a/e2e/tests/steps_init_test.go b/e2e/tests/steps_init_test.go index f981f5793..ee680cd82 100644 --- a/e2e/tests/steps_init_test.go +++ b/e2e/tests/steps_init_test.go @@ -23,8 +23,8 @@ import ( var e2eLogger = pocketLogger.Global.CreateLoggerForModule("e2e") const ( - // defines the host & port scheme that LocalNet uses for naming validators. - // e.g. validator-001 thru validator-999 + // Each actor is represented e.g. validator-001-pocket:42069 thru validator-999-pocket:42069. + // Defines the host & port scheme that LocalNet uses for naming actors. validatorServiceURLTmpl = "validator-%s-pocket:%d" // validatorA maps to suffix ID 001 and is also used by the cluster-manager // though it has no special permissions. @@ -43,6 +43,7 @@ type rootSuite struct { // clientset is the kubernetes API we acquire from the user's $HOME/.kube/config clientset *kubernetes.Clientset // validator holds command results between runs and reports errors to the test suite + // TECHDEBT: Rename `validator` to something more appropriate validator *validatorPod // validatorA maps to suffix ID 001 of the kube pod that we use as our control agent } @@ -148,9 +149,7 @@ func (s *rootSuite) unstakeValidator() { } // getPrivateKey generates a new keypair from the private hex key that we get from the clientset -func (s *rootSuite) getPrivateKey( - validatorId string, -) cryptoPocket.PrivateKey { +func (s *rootSuite) getPrivateKey(validatorId string) cryptoPocket.PrivateKey { privHexString := s.validatorKeys[validatorId] privateKey, err := cryptoPocket.NewPrivateKey(privHexString) require.NoErrorf(s, err, "failed to extract privkey") @@ -161,6 +160,8 @@ func (s *rootSuite) getPrivateKey( // getClientset uses the default path `$HOME/.kube/config` to build a kubeconfig // and then connects to that cluster and returns a *Clientset or an error func getClientset(t gocuke.TestingT) (*kubernetes.Clientset, error) { + t.Helper() + userHomeDir, err := os.UserHomeDir() if err != nil { return nil, fmt.Errorf("failed to get home dir: %w", err) diff --git a/go.mod b/go.mod index 1be7fb2dd..e9772889b 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/pokt-network/pocket -go 1.18 +go 1.20 // TECHDEBT: remove once upstream PR is merged (see: https://github.com/cosmos/ics23/pull/153) replace github.com/cosmos/ics23/go => github.com/h5law/ics23/go v0.0.0-20230619152251-56d948cafb83 diff --git a/ibc/docs/ics24.md b/ibc/docs/ics24.md index d63f214ac..da56f7ab1 100644 --- a/ibc/docs/ics24.md +++ b/ibc/docs/ics24.md @@ -16,6 +16,7 @@ - [Provable Stores](#provable-stores) - [Bulk Store Cacher](#bulk-store-cacher) - [Caching](#caching) +- [Event Logging System](#event-logging-system) ## Overview @@ -222,6 +223,14 @@ In the event of a node failure, or local changes not being propagated correctly. _TODO: Implement this functionality_ +## Event Logging System + +The `EventLogger` submodule defined in [ibc_event_module.go](../../shared/modules/ibc_event_module.go) implements the Event Logging system defined in the [ICS-24 specification][ics24]. This is used to store and query IBC related events for the relayers to read packet data and timeouts, as only the proofs of these are stored in the chain state. + +Events are `IBCEvent` types defined in [ibc_events.proto](../../shared/core/types/proto/ibc_events.proto). They hold the height at which they were created, a string defining their topic (what type of event it represents) and a series of key-value pairs that represent the data of the event. + +The persistence layer is used for event storage and retrieval. + [ics24]: https://github.com/cosmos/ibc/blob/main/spec/core/ics-024-host-requirements/README.md [ics20]: https://github.com/cosmos/ibc/blob/main/spec/app/ics-020-fungible-token-transfer/README.md [smt]: https://github.com/pokt-network/smt diff --git a/ibc/events/event_manager.go b/ibc/events/event_manager.go new file mode 100644 index 000000000..19e48cb95 --- /dev/null +++ b/ibc/events/event_manager.go @@ -0,0 +1,58 @@ +package events + +import ( + coreTypes "github.com/pokt-network/pocket/shared/core/types" + "github.com/pokt-network/pocket/shared/modules" + "github.com/pokt-network/pocket/shared/modules/base_modules" +) + +var _ modules.EventLogger = &EventManager{} + +type EventManager struct { + base_modules.IntegrableModule + + logger *modules.Logger +} + +func Create(bus modules.Bus, options ...modules.EventLoggerOption) (modules.EventLogger, error) { + return new(EventManager).Create(bus, options...) +} + +func WithLogger(logger *modules.Logger) modules.EventLoggerOption { + return func(m modules.EventLogger) { + if mod, ok := m.(*EventManager); ok { + mod.logger = logger + } + } +} + +func (*EventManager) Create(bus modules.Bus, options ...modules.EventLoggerOption) (modules.EventLogger, error) { + e := &EventManager{} + + for _, option := range options { + option(e) + } + + e.logger.Info().Msg("🪵 Creating Event Logger 🪵") + + bus.RegisterModule(e) + + return e, nil +} + +func (e *EventManager) GetModuleName() string { return modules.EventLoggerModuleName } + +func (e *EventManager) EmitEvent(event *coreTypes.IBCEvent) error { + wCtx := e.GetBus().GetPersistenceModule().NewWriteContext() + defer wCtx.Release() + return wCtx.SetIBCEvent(event) +} + +func (e *EventManager) QueryEvents(topic string, height uint64) ([]*coreTypes.IBCEvent, error) { + rCtx, err := e.GetBus().GetPersistenceModule().NewReadContext(int64(height)) + if err != nil { + return nil, err + } + defer rCtx.Release() + return rCtx.GetIBCEvents(height, topic) +} diff --git a/ibc/host/submodule.go b/ibc/host/submodule.go index b484bcc76..655985b73 100644 --- a/ibc/host/submodule.go +++ b/ibc/host/submodule.go @@ -4,6 +4,7 @@ import ( "errors" "time" + "github.com/pokt-network/pocket/ibc/events" "github.com/pokt-network/pocket/ibc/store" "github.com/pokt-network/pocket/runtime/configs" coreTypes "github.com/pokt-network/pocket/shared/core/types" @@ -19,6 +20,10 @@ type ibcHost struct { cfg *configs.IBCHostConfig logger *modules.Logger storesDir string + + // only a single bulk store cacher and event logger are allowed + bsc modules.BulkStoreCacher + em modules.EventLogger } func Create(bus modules.Bus, config *configs.IBCHostConfig, options ...modules.IBCHostOption) (modules.IBCHostSubmodule, error) { @@ -51,8 +56,10 @@ func (*ibcHost) Create(bus modules.Bus, config *configs.IBCHostConfig, options . option(h) } h.logger.Info().Msg("🛰️ Creating IBC host 🛰️") + bus.RegisterModule(h) - _, err := store.Create(h.GetBus(), + + bsc, err := store.Create(h.GetBus(), h.cfg.BulkStoreCacher, store.WithLogger(h.logger), store.WithStoresDir(h.storesDir), @@ -61,6 +68,14 @@ func (*ibcHost) Create(bus modules.Bus, config *configs.IBCHostConfig, options . if err != nil { return nil, err } + h.bsc = bsc + + em, err := events.Create(h.GetBus(), events.WithLogger(h.logger)) + if err != nil { + return nil, err + } + h.em = em + return h, nil } diff --git a/ibc/ibc_handle_event_test.go b/ibc/ibc_handle_event_test.go index 9d6ed3ebe..422f450f6 100644 --- a/ibc/ibc_handle_event_test.go +++ b/ibc/ibc_handle_event_test.go @@ -81,7 +81,7 @@ func TestHandleEvent_FlushCaches(t *testing.T) { require.NoError(t, cache.Stop()) // flush the cache - err = ibcHost.GetBus().GetBulkStoreCacher().FlushAllEntries() + err = ibcHost.GetBus().GetBulkStoreCacher().FlushCachesToStore() require.NoError(t, err) cache, err = kvstore.NewKVStore(tmpDir) diff --git a/ibc/module.go b/ibc/module.go index 24736e9d4..aae9b4581 100644 --- a/ibc/module.go +++ b/ibc/module.go @@ -95,7 +95,7 @@ func (m *ibcModule) HandleEvent(event *anypb.Any) error { } // Flush all caches to disk for last height bsc := m.GetBus().GetBulkStoreCacher() - if err := bsc.FlushAllEntries(); err != nil { + if err := bsc.FlushCachesToStore(); err != nil { return err } // Prune old cache entries diff --git a/ibc/store/bulk_store_cache.go b/ibc/store/bulk_store_cache.go index 953e9ca3b..0e71de3cf 100644 --- a/ibc/store/bulk_store_cache.go +++ b/ibc/store/bulk_store_cache.go @@ -124,8 +124,8 @@ func (s *bulkStoreCache) GetAllStores() map[string]modules.ProvableStore { return s.ls.stores } -// FlushAllEntries caches all the entries for all stores in the bulkStoreCache -func (s *bulkStoreCache) FlushAllEntries() error { +// FlushdCachesToStore caches all the entries for all stores in the bulkStoreCache +func (s *bulkStoreCache) FlushCachesToStore() error { s.ls.m.Lock() defer s.ls.m.Unlock() s.logger.Info().Msg("🚽 Flushing All Cache Entries to Disk 🚽") @@ -134,7 +134,7 @@ func (s *bulkStoreCache) FlushAllEntries() error { return err } for _, store := range s.ls.stores { - if err := store.FlushEntries(disk); err != nil { + if err := store.FlushCache(disk); err != nil { s.logger.Error().Err(err).Str("store", string(store.GetCommitmentPrefix())).Msg("🚨 Error Flushing Cache 🚨") return err } diff --git a/ibc/store/provable_store.go b/ibc/store/provable_store.go index c3ff8a171..df4612725 100644 --- a/ibc/store/provable_store.go +++ b/ibc/store/provable_store.go @@ -166,8 +166,8 @@ func (p *provableStore) Root() ics23.CommitmentRoot { return root } -// FlushEntries writes all local changes to disk and clears the in-memory cache -func (p *provableStore) FlushEntries(store kvstore.KVStore) error { +// FlushCache writes all local changes to disk and clears the in-memory cache +func (p *provableStore) FlushCache(store kvstore.KVStore) error { p.m.Lock() defer p.m.Unlock() for _, entry := range p.cache { diff --git a/ibc/store/provable_store_test.go b/ibc/store/provable_store_test.go index f5739e359..174d62827 100644 --- a/ibc/store/provable_store_test.go +++ b/ibc/store/provable_store_test.go @@ -148,7 +148,7 @@ func TestProvableStore_GetAndProve(t *testing.T) { } } -func TestProvableStore_FlushEntries(t *testing.T) { +func TestProvableStore_FlushCache(t *testing.T) { provableStore := newTestProvableStore(t) kvs := []struct { key []byte @@ -177,7 +177,7 @@ func TestProvableStore_FlushEntries(t *testing.T) { } } cache := kvstore.NewMemKVStore() - require.NoError(t, provableStore.FlushEntries(cache)) + require.NoError(t, provableStore.FlushCache(cache)) keys, values, err := cache.GetAll([]byte{}, false) require.NoError(t, err) require.Len(t, keys, 3) @@ -221,7 +221,7 @@ func TestProvableStore_PruneCache(t *testing.T) { } } cache := kvstore.NewMemKVStore() - require.NoError(t, provableStore.FlushEntries(cache)) + require.NoError(t, provableStore.FlushCache(cache)) keys, _, err := cache.GetAll([]byte{}, false) require.NoError(t, err) require.Len(t, keys, 3) // 3 entries in cache should be flushed to disk @@ -264,12 +264,12 @@ func TestProvableStore_RestoreCache(t *testing.T) { } cache := kvstore.NewMemKVStore() - require.NoError(t, provableStore.FlushEntries(cache)) + require.NoError(t, provableStore.FlushCache(cache)) keys, values, err := cache.GetAll([]byte{}, false) require.NoError(t, err) require.Len(t, keys, 3) require.NoError(t, cache.ClearAll()) - require.NoError(t, provableStore.FlushEntries(cache)) + require.NoError(t, provableStore.FlushCache(cache)) newKeys, _, err := cache.GetAll([]byte{}, false) require.NoError(t, err) require.Len(t, newKeys, 0) @@ -284,7 +284,7 @@ func TestProvableStore_RestoreCache(t *testing.T) { newKeys, _, err = cache.GetAll([]byte{}, false) require.NoError(t, err) require.Len(t, newKeys, 0) - require.NoError(t, provableStore.FlushEntries(cache)) + require.NoError(t, provableStore.FlushCache(cache)) newKeys, newValues, err := cache.GetAll([]byte{}, false) require.NoError(t, err) require.Len(t, newKeys, 3) @@ -465,7 +465,7 @@ func newTreeStoreMock(t *testing.T, ctrl := gomock.NewController(t) treeStoreMock := mockModules.NewMockTreeStoreModule(ctrl) - treeStoreMock.EXPECT().GetModuleName().Return(modules.TreeStoreModuleName).AnyTimes() + treeStoreMock.EXPECT().GetModuleName().Return(modules.TreeStoreSubmoduleName).AnyTimes() treeStoreMock.EXPECT().SetBus(gomock.Any()).Return().AnyTimes() treeStoreMock.EXPECT().GetBus().Return(bus).AnyTimes() diff --git a/internal/testutil/ibc/mock.go b/internal/testutil/ibc/mock.go index d1bc22728..03ab3faa3 100644 --- a/internal/testutil/ibc/mock.go +++ b/internal/testutil/ibc/mock.go @@ -74,7 +74,7 @@ func baseBulkStoreCacherMock(t gocuke.TestingT, bus modules.Bus) *mockModules.Mo storeMock.EXPECT().AddStore(gomock.Any()).Return(nil).AnyTimes() storeMock.EXPECT().GetStore(gomock.Any()).Return(provableStoreMock, nil).AnyTimes() storeMock.EXPECT().RemoveStore(gomock.Any()).Return(nil).AnyTimes() - storeMock.EXPECT().FlushAllEntries().Return(nil).AnyTimes() + storeMock.EXPECT().FlushCachesToStore().Return(nil).AnyTimes() storeMock.EXPECT().PruneCaches(gomock.Any()).Return(nil).AnyTimes() storeMock.EXPECT().RestoreCaches(gomock.Any()).Return(nil).AnyTimes() diff --git a/p2p/background/kad_discovery_baseline_test.go b/p2p/background/kad_discovery_baseline_test.go index fd352456c..4728979f3 100644 --- a/p2p/background/kad_discovery_baseline_test.go +++ b/p2p/background/kad_discovery_baseline_test.go @@ -22,13 +22,13 @@ const dhtUpdateSleepDuration = time.Millisecond * 500 func TestLibp2pKademliaPeerDiscovery(t *testing.T) { ctx := context.Background() - addr1, host1, _ := setupHostAndDiscovery(t, ctx, defaults.DefaultP2PPort, nil) + addr1, host1, kad1 := setupHostAndDiscovery(t, ctx, defaults.DefaultP2PPort, nil) bootstrapAddr, err := multiaddr.NewMultiaddr(fmt.Sprintf("%s/p2p/%s", addr1, host1.ID().String())) require.NoError(t, err) - addr2, host2, _ := setupHostAndDiscovery(t, ctx, defaults.DefaultP2PPort+1, bootstrapAddr) - addr3, host3, _ := setupHostAndDiscovery(t, ctx, defaults.DefaultP2PPort+2, bootstrapAddr) + addr2, host2, kad2 := setupHostAndDiscovery(t, ctx, defaults.DefaultP2PPort+1, bootstrapAddr) + addr3, host3, kad3 := setupHostAndDiscovery(t, ctx, defaults.DefaultP2PPort+2, bootstrapAddr) expectedPeerIDs := []libp2pPeer.ID{host1.ID(), host2.ID(), host3.ID()} @@ -50,9 +50,21 @@ func TestLibp2pKademliaPeerDiscovery(t *testing.T) { require.ElementsMatchf(t, expectedPeerIDs, host3.Peerstore().Peers(), "host3 peer IDs don't match") // add another peer to network... - addr4, host4, _ := setupHostAndDiscovery(t, ctx, defaults.DefaultP2PPort+3, bootstrapAddr) + addr4, host4, kad4 := setupHostAndDiscovery(t, ctx, defaults.DefaultP2PPort+3, bootstrapAddr) expectedPeerIDs = append(expectedPeerIDs, host4.ID()) + t.Cleanup(func() { + for _, kad := range []*dht.IpfsDHT{kad1, kad2, kad3, kad4} { + err := kad.Close() + require.NoError(t, err) + } + + for _, host := range []libp2pHost.Host{host1, host2, host3, host4} { + err := host.Close() + require.NoError(t, err) + } + }) + // TECHDEBT: consider using `host.ConnManager().Notifee()` to avoid sleeping here time.Sleep(time.Millisecond * 500) diff --git a/p2p/background/router.go b/p2p/background/router.go index 5e6254d20..7899f7817 100644 --- a/p2p/background/router.go +++ b/p2p/background/router.go @@ -5,6 +5,7 @@ package background import ( "context" "fmt" + "time" dht "github.com/libp2p/go-libp2p-kad-dht" pubsub "github.com/libp2p/go-libp2p-pubsub" @@ -32,6 +33,13 @@ var ( _ backgroundRouterFactory = &backgroundRouter{} ) +// TECHDEBT: Make these values configurable +// TECHDEBT: Consider using an exponential backoff instead +const ( + connectMaxRetries = 5 + connectRetryTimeout = time.Second * 2 +) + type backgroundRouterFactory = modules.FactoryWithConfig[typesP2P.Router, *config.BackgroundConfig] // backgroundRouter implements `typesP2P.Router` for use with all P2P participants. @@ -357,13 +365,30 @@ func (rtr *backgroundRouter) bootstrap(ctx context.Context) error { return nil } - if err := rtr.host.Connect(ctx, libp2pAddrInfo); err != nil { + if err := rtr.connectWithRetry(ctx, libp2pAddrInfo); err != nil { return fmt.Errorf("connecting to peer: %w", err) } } return nil } +// connectWithRetry attempts to connect to the given peer, retrying up to connectMaxRetries times +// and waiting connectRetryTimeout between each attempt. +func (rtr *backgroundRouter) connectWithRetry(ctx context.Context, libp2pAddrInfo libp2pPeer.AddrInfo) error { + var err error + for i := 0; i < connectMaxRetries; i++ { + err = rtr.host.Connect(ctx, libp2pAddrInfo) + if err == nil { + return nil + } + + fmt.Printf("Failed to connect (attempt %d), retrying in %v...\n", i+1, connectRetryTimeout) + time.Sleep(connectRetryTimeout) + } + + return fmt.Errorf("failed to connect after %d attempts, last error: %w", 5, err) +} + // topicValidator is used in conjunction with libp2p-pubsub's notion of "topic // validaton". It is used for arbitrary and concurrent pre-propagation validation // of messages. diff --git a/p2p/background/router_test.go b/p2p/background/router_test.go index 3c93cf758..5d5224c86 100644 --- a/p2p/background/router_test.go +++ b/p2p/background/router_test.go @@ -306,6 +306,23 @@ func TestBackgroundRouter_Broadcast(t *testing.T) { // setup notifee/notify BEFORE bootstrapping notifee := &libp2pNetwork.NotifyBundle{ ConnectedF: func(_ libp2pNetwork.Network, _ libp2pNetwork.Conn) { + // TECHDEBT: it's rare but possible that a host will re-connect, + // causing the `bootstrapWaitgroup` to go negative. + // This test should be redesigned using atomic counters or + // something similar to avoid this issue. + defer func() { + if err := recover(); err != nil { + if err.(error).Error() == "sync: negative WaitGroup counter" { + // ignore negative WaitGroup counter error + return + } + // fail the test for anything else; converting the panic into + // test failure allows the test to run with the `-count` flag + // to completion. + t.Fatal(err) + } + }() + t.Logf("connected!") bootstrapWaitgroup.Done() }, diff --git a/p2p/transport_encryption_test.go b/p2p/transport_encryption_test.go index 0d88f7607..7236eaa19 100644 --- a/p2p/transport_encryption_test.go +++ b/p2p/transport_encryption_test.go @@ -1,3 +1,5 @@ +//go:build test + package p2p import ( diff --git a/p2p/utils_test.go b/p2p/utils_test.go index 43222a0bb..bebab237f 100644 --- a/p2p/utils_test.go +++ b/p2p/utils_test.go @@ -1,3 +1,5 @@ +//go:build test + package p2p import ( diff --git a/persistence/actor.go b/persistence/actor.go index de96548ae..e2ff160b9 100644 --- a/persistence/actor.go +++ b/persistence/actor.go @@ -80,6 +80,24 @@ func (p *PostgresContext) GetAllValidators(height int64) (vals []*coreTypes.Acto return } +// GetValidatorSet returns the validator set for a given height +func (p *PostgresContext) GetValidatorSet(height int64) (*coreTypes.ValidatorSet, error) { + validators, err := p.GetAllValidators(height) // sorted by address asc + if err != nil { + return nil, err + } + valSet := new(coreTypes.ValidatorSet) + for _, val := range validators { + validator := &coreTypes.ValidatorIdentity{ + Address: val.GetAddress(), + PubKey: val.GetPublicKey(), + } + valSet.Validators = append(valSet.Validators, validator) + } + // Assumption: Validators are sorted by address based on return value from `p.GetAllValidators` + return valSet, nil +} + func (p *PostgresContext) GetAllServicers(height int64) (sn []*coreTypes.Actor, err error) { ctx, tx := p.getCtxAndTx() rows, err := tx.Query(ctx, types.ServicerActor.GetAllQuery(height)) diff --git a/persistence/block.go b/persistence/block.go index 57ae39f4d..e14d2777b 100644 --- a/persistence/block.go +++ b/persistence/block.go @@ -7,7 +7,9 @@ import ( "github.com/dgraph-io/badger/v3" "github.com/pokt-network/pocket/persistence/types" + "github.com/pokt-network/pocket/shared/codec" coreTypes "github.com/pokt-network/pocket/shared/core/types" + "github.com/pokt-network/pocket/shared/crypto" "google.golang.org/protobuf/types/known/timestamppb" ) @@ -24,6 +26,7 @@ func (p *persistenceModule) TransactionExists(transactionHash string) (bool, err } return false, err } + return true, nil } @@ -85,6 +88,12 @@ func (p *PostgresContext) prepareBlock(proposerAddr, quorumCert []byte) (*coreTy // TECHDEBT: This will lead to different timestamp in each node's block store because `prepareBlock` is called locally. Needs to be revisisted and decided on a proper implementation. timestamp := timestamppb.Now() + // Get the current validator set and next validator set hashes + currSetHash, nextSetHash, err := p.getCurrentAndNextValSetHashes() + if err != nil { + return nil, err + } + // Preapre the block proto blockHeader := &coreTypes.BlockHeader{ Height: uint64(p.Height), @@ -94,6 +103,9 @@ func (p *PostgresContext) prepareBlock(proposerAddr, quorumCert []byte) (*coreTy ProposerAddress: proposerAddr, QuorumCertificate: quorumCert, Timestamp: timestamp, + StateTreeHashes: p.stateTrees.GetTreeHashes(), + ValSetHash: currSetHash, + NextValSetHash: nextSetHash, } block := &coreTypes.Block{ BlockHeader: blockHeader, @@ -117,3 +129,38 @@ func (p *PostgresContext) insertBlock(block *coreTypes.Block) error { _, err := tx.Exec(ctx, types.InsertBlockQuery(blockHeader.Height, blockHeader.StateHash, blockHeader.ProposerAddress, blockHeader.QuorumCertificate)) return err } + +// getValidatorSetHashes returns the present (current p.Height-1) and next (p.Height) validator set hashes +func (p *PostgresContext) getCurrentAndNextValSetHashes() (currentValSetHash, nextValSetHash string, err error) { + // Get the next validator set + nextValSetHash, err = p.hashValidatorSet(p.Height) + if err != nil { + return "", "", err + } + + if p.Height == 0 { + return "", nextValSetHash, nil + } + + // Get the current validator set + currentValSetHash, err = p.hashValidatorSet(p.Height - 1) + if err != nil { + return "", "", err + } + + return currentValSetHash, nextValSetHash, nil +} + +// hashValidatorSet hashes the validator set at the given height +func (p *PostgresContext) hashValidatorSet(height int64) (string, error) { + valSet, err := p.GetValidatorSet(height) + if err != nil { + return "", err + } + valSetBz, err := codec.GetCodec().Marshal(valSet) + if err != nil { + return "", err + } + valSetHash := crypto.SHA3Hash(valSetBz) + return hex.EncodeToString(valSetHash), nil +} diff --git a/persistence/db.go b/persistence/db.go index 27a0fd2c1..2a65e7819 100644 --- a/persistence/db.go +++ b/persistence/db.go @@ -193,5 +193,8 @@ func initialiseIBCTables(ctx context.Context, db *pgxpool.Conn) error { if _, err := db.Exec(ctx, fmt.Sprintf(`%s %s %s %s`, CreateTable, IfNotExists, types.IBCStoreTableName, types.IBCStoreTableSchema)); err != nil { return err } + if _, err := db.Exec(ctx, fmt.Sprintf(`%s %s %s %s`, CreateTable, IfNotExists, types.IBCEventLogTableName, types.IBCEventLogTableSchema)); err != nil { + return err + } return nil } diff --git a/persistence/debug.go b/persistence/debug.go index 38e69e221..eb0577f5e 100644 --- a/persistence/debug.go +++ b/persistence/debug.go @@ -14,7 +14,8 @@ var nonActorClearFunctions = []func() string{ types.ClearAllGovParamsQuery, types.ClearAllGovFlagsQuery, types.ClearAllBlocksQuery, - types.ClearAllIBCQuery, + types.ClearAllIBCStoreQuery, + types.ClearAllIBCEventsQuery, } func (m *persistenceModule) HandleDebugMessage(debugMessage *messaging.DebugMessage) error { diff --git a/persistence/ibc.go b/persistence/ibc.go index 1a2d5880a..fc9affea4 100644 --- a/persistence/ibc.go +++ b/persistence/ibc.go @@ -7,6 +7,7 @@ import ( "github.com/jackc/pgx/v5" pTypes "github.com/pokt-network/pocket/persistence/types" + "github.com/pokt-network/pocket/shared/codec" coreTypes "github.com/pokt-network/pocket/shared/core/types" ) @@ -39,3 +40,48 @@ func (p *PostgresContext) GetIBCStoreEntry(key []byte, height int64) ([]byte, er } return value, nil } + +// SetIBCEvent sets the IBC event at the current height in the persitence DB +func (p *PostgresContext) SetIBCEvent(event *coreTypes.IBCEvent) error { + ctx, tx := p.getCtxAndTx() + typeStr := event.GetTopic() + eventBz, err := codec.GetCodec().Marshal(event) + if err != nil { + return err + } + eventHex := hex.EncodeToString(eventBz) + if _, err := tx.Exec(ctx, pTypes.InsertIBCEventQuery(p.Height, typeStr, eventHex)); err != nil { + return err + } + return nil +} + +// GetIBCEvents returns all the IBC events at the height provided with the matching topic +func (p *PostgresContext) GetIBCEvents(height uint64, topic string) ([]*coreTypes.IBCEvent, error) { + ctx, tx := p.getCtxAndTx() + rows, err := tx.Query(ctx, pTypes.GetIBCEventQuery(height, topic)) + if err != nil { + return nil, err + } + defer rows.Close() + var events []*coreTypes.IBCEvent + for rows.Next() { + var eventHex string + if err := rows.Scan(&eventHex); err != nil { + return nil, err + } + eventBz, err := hex.DecodeString(eventHex) + if err != nil { + return nil, err + } + event := &coreTypes.IBCEvent{} + if err := codec.GetCodec().Unmarshal(eventBz, event); err != nil { + return nil, err + } + events = append(events, event) + } + if err := rows.Err(); err != nil { + return nil, err + } + return events, nil +} diff --git a/persistence/module.go b/persistence/module.go index dec15112b..0d6acd17f 100644 --- a/persistence/module.go +++ b/persistence/module.go @@ -41,10 +41,6 @@ type persistenceModule struct { // IMPORTANT: It doubles as the data store for the transaction tree in state tree set. txIndexer indexer.TxIndexer - // stateTrees manages all of the merkle trees maintained by the - // persistence module that roll up into the state commitment. - stateTrees modules.TreeStoreModule - // Only one write context is allowed at a time writeContext *PostgresContext @@ -103,21 +99,22 @@ func (*persistenceModule) Create(bus modules.Bus, options ...modules.ModuleOptio return nil, err } - treeModule, err := trees.Create( + _, err = trees.Create( bus, trees.WithTreeStoreDirectory(persistenceCfg.TreesStoreDir), trees.WithLogger(m.logger)) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to create TreeStoreModule: %w", err) } m.config = persistenceCfg m.genesisState = genesisState m.networkId = runtimeMgr.GetConfig().NetworkId + // TECHDEBT: fetch blockstore from bus once it's a proper submodule m.blockStore = blockStore + // TECHDEBT: fetch txIndexer from bus m.txIndexer = txIndexer - m.stateTrees = treeModule // TECHDEBT: reconsider if this is the best place to call `populateGenesisState`. Note that // this forces the genesis state to be reloaded on every node startup until state @@ -180,7 +177,7 @@ func (m *persistenceModule) NewRWContext(height int64) (modules.PersistenceRWCon stateHash: "", blockStore: m.blockStore, txIndexer: m.txIndexer, - stateTrees: m.stateTrees, + stateTrees: m.GetBus().GetTreeStore(), networkId: m.networkId, } @@ -212,7 +209,7 @@ func (m *persistenceModule) NewReadContext(height int64) (modules.PersistenceRea stateHash: "", blockStore: m.blockStore, txIndexer: m.txIndexer, - stateTrees: m.stateTrees, + stateTrees: m.GetBus().GetTreeStore(), networkId: m.networkId, }, nil } @@ -238,10 +235,6 @@ func (m *persistenceModule) GetTxIndexer() indexer.TxIndexer { return m.txIndexer } -func (m *persistenceModule) GetTreeStore() modules.TreeStoreModule { - return m.stateTrees -} - func (m *persistenceModule) GetNetworkID() string { return m.networkId } diff --git a/persistence/sql/sql.go b/persistence/sql/sql.go index 80a1c8320..9e99189bb 100644 --- a/persistence/sql/sql.go +++ b/persistence/sql/sql.go @@ -6,7 +6,6 @@ import ( "fmt" "github.com/jackc/pgx/v5" - "github.com/pokt-network/pocket/persistence/indexer" ptypes "github.com/pokt-network/pocket/persistence/types" coreTypes "github.com/pokt-network/pocket/shared/core/types" ) @@ -91,16 +90,6 @@ func GetAccountsUpdated( return accounts, nil } -// GetTransactions takes a transaction indexer and returns the transactions for the current height -func GetTransactions(txi indexer.TxIndexer, height uint64) ([]*coreTypes.IndexedTransaction, error) { - // TECHDEBT(#813): Avoid this cast to int64 - indexedTxs, err := txi.GetByHeight(int64(height), false) - if err != nil { - return nil, fmt.Errorf("failed to get transactions by height: %w", err) - } - return indexedTxs, nil -} - // GetPools returns the pools updated at the given height func GetPools(pgtx pgx.Tx, height uint64) ([]*coreTypes.Account, error) { pools, err := GetAccountsUpdated(pgtx, ptypes.Pool, height) diff --git a/persistence/test/actor_test.go b/persistence/test/actor_test.go index 89ff4afb4..1f605f240 100644 --- a/persistence/test/actor_test.go +++ b/persistence/test/actor_test.go @@ -1,11 +1,14 @@ package test import ( + "encoding/hex" "testing" "github.com/stretchr/testify/require" + "github.com/pokt-network/pocket/shared/codec" coreTypes "github.com/pokt-network/pocket/shared/core/types" + "github.com/pokt-network/pocket/shared/crypto" ) func TestGetAllStakedActors(t *testing.T) { @@ -37,3 +40,147 @@ func TestGetAllStakedActors(t *testing.T) { require.Equal(t, genesisStateNumApplications, actualApplications) require.Equal(t, genesisStateNumFishermen, actualFishermen) } + +func TestPostgresContext_GetValidatorSet(t *testing.T) { + expectedHashes := []string{ + "5831e3e6a8d3beda0adb6027126a4bc3b0181836eebcc45a83dc2970ee9b4468", + "1f6faa8782a4608341fef83ee72c9e9cd3f96e042f2ddfdeebda8d971bb2ac13", + } + + // Ensure genesis next val set hash is correct + db := NewTestPostgresContext(t, 0) + nextValSet, err := db.GetValidatorSet(0) + require.NoError(t, err) + + // ensure validator set is ordered lexicographically + for i, val := range nextValSet.Validators { + if i == 0 { + continue + } + require.True(t, val.Address > nextValSet.Validators[i-1].Address) + } + + nextValSetHash := hashValSet(t, nextValSet) + require.Equal(t, expectedHashes[0], nextValSetHash) + + // Ensure next val set hash for genesis == curr val set hash for height 1 + // and next val set hash remains the same with no changes + currHeight := int64(1) + db.Height = currHeight + + currValSet, err := db.GetValidatorSet(currHeight - 1) + require.NoError(t, err) + + // ensure validator set is ordered lexicographically + for i, val := range currValSet.Validators { + if i == 0 { + continue + } + require.True(t, val.Address > nextValSet.Validators[i-1].Address) + } + + currValSetHash := hashValSet(t, currValSet) + nextValSet, err = db.GetValidatorSet(currHeight) + require.NoError(t, err) + + // ensure validator set is ordered lexicographically + for i, val := range nextValSet.Validators { + if i == 0 { + continue + } + require.True(t, val.Address > nextValSet.Validators[i-1].Address) + } + + nextValSetHash = hashValSet(t, nextValSet) + + require.Equal(t, expectedHashes[0], currValSetHash) + require.Equal(t, expectedHashes[0], nextValSetHash) + + // ensure both hashes remain the same with no changes + currHeight = int64(2) + db.Height = currHeight + + currValSet, err = db.GetValidatorSet(currHeight - 1) + require.NoError(t, err) + + // ensure validator set is ordered lexicographically + for i, val := range currValSet.Validators { + if i == 0 { + continue + } + require.True(t, val.Address > nextValSet.Validators[i-1].Address) + } + + currValSetHash = hashValSet(t, currValSet) + nextValSet, err = db.GetValidatorSet(currHeight) + require.NoError(t, err) + + // ensure validator set is ordered lexicographically + for i, val := range nextValSet.Validators { + if i == 0 { + continue + } + require.True(t, val.Address > nextValSet.Validators[i-1].Address) + } + + nextValSetHash = hashValSet(t, nextValSet) + + require.Equal(t, expectedHashes[0], currValSetHash) + require.Equal(t, expectedHashes[0], nextValSetHash) + + // ensure next val set hash changes when we add a new validator at current + // height but the current val set hash remains the same + currHeight = int64(3) + db.Height = currHeight + + err = db.InsertValidator( + []byte("address"), + []byte("publickey"), + []byte("output"), + false, 0, + "serviceurl", + "1000000000", + 0, + 0, + ) + require.NoError(t, err) + + currValSet, err = db.GetValidatorSet(currHeight - 1) + require.NoError(t, err) + + // ensure validator set is ordered lexicographically + for i, val := range currValSet.Validators { + if i == 0 { + continue + } + require.True(t, val.Address > nextValSet.Validators[i-1].Address) + } + + currValSetHash = hashValSet(t, currValSet) + nextValSet, err = db.GetValidatorSet(currHeight) + require.NoError(t, err) + t.Log(nextValSet.Validators) + + // ensure validator set is ordered lexicographically + for i, val := range currValSet.Validators { + if i == 0 { + continue + } + require.True(t, val.Address > nextValSet.Validators[i-1].Address) + } + + nextValSetHash = hashValSet(t, nextValSet) + + require.Equal(t, expectedHashes[0], currValSetHash) + require.Equal(t, expectedHashes[1], nextValSetHash) +} + +func hashValSet(t *testing.T, valSet *coreTypes.ValidatorSet) string { + t.Helper() + + bz, err := codec.GetCodec().Marshal(valSet) + require.NoError(t, err) + + hash := crypto.SHA3Hash(bz) + return hex.EncodeToString(hash) +} diff --git a/persistence/test/benchmark_state_test.go b/persistence/test/benchmark_state_test.go index 9084faf8a..6f01efaca 100644 --- a/persistence/test/benchmark_state_test.go +++ b/persistence/test/benchmark_state_test.go @@ -118,7 +118,7 @@ MethodLoop: case reflect.Slice: switch arg.Elem().Kind() { case reflect.Uint8: - v = reflect.ValueOf([]uint8{0}) + v = reflect.ValueOf([]uint8{uint8(rand.Intn(2 ^ 8 - 1))}) // needs to be random to stop dupilcate keys case reflect.String: v = reflect.ValueOf([]string{"abc"}) default: diff --git a/persistence/test/ibc_test.go b/persistence/test/ibc_test.go index 8c9a25ac2..2fcf86f4e 100644 --- a/persistence/test/ibc_test.go +++ b/persistence/test/ibc_test.go @@ -1,6 +1,8 @@ package test import ( + "fmt" + "strconv" "testing" coreTypes "github.com/pokt-network/pocket/shared/core/types" @@ -11,39 +13,39 @@ func TestIBC_SetIBCStoreEntry(t *testing.T) { db := NewTestPostgresContext(t, 1) testCases := []struct { - name string - height int64 - key []byte - value []byte - expectedErr string + name string + height int64 + key []byte + value []byte + expectedErrStr *string }{ { - name: "Successfully set key at height 1", - height: 1, - key: []byte("key"), - value: []byte("value"), - expectedErr: "", + name: "Successfully set key at height 1", + height: 1, + key: []byte("key"), + value: []byte("value"), + expectedErrStr: nil, }, { - name: "Successfully set key at height 2", - height: 2, - key: []byte("key"), - value: []byte("value2"), - expectedErr: "", + name: "Successfully set key at height 2", + height: 2, + key: []byte("key"), + value: []byte("value2"), + expectedErrStr: nil, }, { - name: "Successfully set key to nil at height 3", - height: 3, - key: []byte("key"), - value: nil, - expectedErr: "", + name: "Successfully set key to nil at height 3", + height: 3, + key: []byte("key"), + value: nil, + expectedErrStr: nil, }, { - name: "Fails to set an existing key at height 3", - height: 3, - key: []byte("key"), - value: []byte("new value"), - expectedErr: "ERROR: duplicate key value violates unique constraint \"ibc_entries_pkey\" (SQLSTATE 23505)", + name: "Fails to set an existing key at height 3", + height: 3, + key: []byte("key"), + value: []byte("new value"), + expectedErrStr: duplicateError("ibc_entries"), }, } @@ -51,8 +53,8 @@ func TestIBC_SetIBCStoreEntry(t *testing.T) { t.Run(tc.name, func(t *testing.T) { db.Height = tc.height err := db.SetIBCStoreEntry(tc.key, tc.value) - if tc.expectedErr != "" { - require.EqualError(t, err, tc.expectedErr) + if tc.expectedErrStr != nil { + require.EqualError(t, err, *tc.expectedErrStr) } else { require.NoError(t, err) } @@ -120,3 +122,195 @@ func TestIBC_GetIBCStoreEntry(t *testing.T) { }) } } + +type attribute struct { + key []byte + value []byte +} + +var ( + baseAttributeKey = []byte("testKey") + baseAttributeValue = []byte("testValue") +) + +func TestIBCSetEvent(t *testing.T) { + // Setup database + db := NewTestPostgresContext(t, 1) + // Add a single event at height 1 + event := new(coreTypes.IBCEvent) + event.Topic = "test" + event.Height = 1 + event.Attributes = append(event.Attributes, &coreTypes.Attribute{ + Key: baseAttributeKey, + Value: baseAttributeValue, + }) + require.NoError(t, db.SetIBCEvent(event)) + + testCases := []struct { + name string + height uint64 + topic string + attributes []attribute + expectedErrStr *string + }{ + { + name: "Successfully set new event at height 1", + height: 1, + topic: "test", + attributes: []attribute{ + { + key: []byte("key"), + value: []byte("value"), + }, + { + key: []byte("key2"), + value: []byte("value2"), + }, + }, + expectedErrStr: nil, + }, + { + name: "Successfully set new event at height 2", + height: 2, + topic: "test", + attributes: []attribute{ + { + key: []byte("key"), + value: []byte("value"), + }, + { + key: []byte("key2"), + value: []byte("value2"), + }, + }, + expectedErrStr: nil, + }, + { + name: "Successfully set a duplicate event new height", + height: 2, + topic: "test", + attributes: []attribute{ + { + key: []byte("testKey"), + value: []byte("testValue"), + }, + }, + expectedErrStr: nil, + }, + { + name: "Fails to set a duplicate event at height 1", + height: 1, + topic: "test", + attributes: []attribute{ + { + key: baseAttributeKey, + value: baseAttributeValue, + }, + }, + expectedErrStr: duplicateError("ibc_events"), + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + db.Height = int64(tc.height) + event := new(coreTypes.IBCEvent) + event.Topic = tc.topic + event.Height = tc.height + for _, attr := range tc.attributes { + event.Attributes = append(event.Attributes, &coreTypes.Attribute{ + Key: attr.key, + Value: attr.value, + }) + } + err := db.SetIBCEvent(event) + if tc.expectedErrStr != nil { + require.EqualError(t, err, *tc.expectedErrStr) + } else { + require.NoError(t, err) + } + }) + } +} + +func TestGetIBCEvent(t *testing.T) { + // Setup database + db := NewTestPostgresContext(t, 1) + // Add events "testKey0", "testKey1", "testKey2", "testKey3" + // at heights 1, 2, 3, 3 respectively + events := make([]*coreTypes.IBCEvent, 0, 4) + for i := 0; i < 4; i++ { + event := new(coreTypes.IBCEvent) + event.Topic = "test" + event.Height = uint64(i + 1) + if i == 3 { + event.Height = uint64(i) // add a second event at height 3 + } + s := strconv.Itoa(i) + event.Attributes = append(event.Attributes, &coreTypes.Attribute{ + Key: []byte("testKey" + s), + Value: []byte("testValue" + s), + }) + events = append(events, event) + } + for _, event := range events { + db.Height = int64(event.Height) + require.NoError(t, db.SetIBCEvent(event)) + } + + testCases := []struct { + name string + height uint64 + topic string + eventsIndexes []int + expectedLength int + }{ + { + name: "Successfully get events at height 1", + height: 1, + topic: "test", + eventsIndexes: []int{0}, + expectedLength: 1, + }, + { + name: "Successfully get events at height 2", + height: 2, + topic: "test", + eventsIndexes: []int{1}, + expectedLength: 1, + }, + { + name: "Successfully get events at height 3", + height: 3, + topic: "test", + eventsIndexes: []int{2, 3}, + expectedLength: 2, + }, + { + name: "Successfully returns empty array when no events found", + height: 3, + topic: "test2", + eventsIndexes: []int{}, + expectedLength: 0, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + got, err := db.GetIBCEvents(tc.height, tc.topic) + require.NoError(t, err) + require.Len(t, got, tc.expectedLength) + for i, index := range tc.eventsIndexes { + require.Equal(t, events[index].Height, got[i].Height) + require.Equal(t, events[index].Topic, got[i].Topic) + require.Equal(t, events[index].Attributes[0].Key, got[i].Attributes[0].Key) + require.Equal(t, events[index].Attributes[0].Value, got[i].Attributes[0].Value) + } + }) + } +} + +func duplicateError(tableName string) *string { + str := fmt.Sprintf("ERROR: duplicate key value violates unique constraint \"%s_pkey\" (SQLSTATE 23505)", tableName) + return &str +} diff --git a/persistence/trees/module.go b/persistence/trees/module.go index 942ab0423..7da8bf20f 100644 --- a/persistence/trees/module.go +++ b/persistence/trees/module.go @@ -8,6 +8,8 @@ import ( "github.com/pokt-network/smt" ) +var _ modules.TreeStoreModule = &treeStore{} + func (*treeStore) Create(bus modules.Bus, options ...modules.TreeStoreOption) (modules.TreeStoreModule, error) { m := &treeStore{} @@ -41,12 +43,15 @@ func WithLogger(logger *modules.Logger) modules.TreeStoreOption { // saves its data. func WithTreeStoreDirectory(path string) modules.TreeStoreOption { return func(m modules.TreeStoreModule) { - if mod, ok := m.(*treeStore); ok { + mod, ok := m.(*treeStore) + if ok { mod.treeStoreDir = path } } } +func (t *treeStore) GetModuleName() string { return modules.TreeStoreSubmoduleName } + func (t *treeStore) setupTrees() error { if t.treeStoreDir == ":memory:" { return t.setupInMemory() diff --git a/persistence/trees/module_test.go b/persistence/trees/module_test.go new file mode 100644 index 000000000..7c7bc660c --- /dev/null +++ b/persistence/trees/module_test.go @@ -0,0 +1,176 @@ +package trees_test + +import ( + "fmt" + "testing" + + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/pokt-network/pocket/internal/testutil" + "github.com/pokt-network/pocket/p2p/providers/peerstore_provider" + "github.com/pokt-network/pocket/persistence/trees" + "github.com/pokt-network/pocket/runtime" + "github.com/pokt-network/pocket/runtime/genesis" + "github.com/pokt-network/pocket/runtime/test_artifacts" + coreTypes "github.com/pokt-network/pocket/shared/core/types" + cryptoPocket "github.com/pokt-network/pocket/shared/crypto" + "github.com/pokt-network/pocket/shared/modules" + mockModules "github.com/pokt-network/pocket/shared/modules/mocks" +) + +const ( + serviceURLFormat = "node%d.consensus:42069" +) + +func TestTreeStore_Create(t *testing.T) { + ctrl := gomock.NewController(t) + mockRuntimeMgr := mockModules.NewMockRuntimeMgr(ctrl) + mockBus := createMockBus(t, mockRuntimeMgr) + genesisStateMock := createMockGenesisState(nil) + persistenceMock := preparePersistenceMock(t, mockBus, genesisStateMock) + + mockBus.EXPECT(). + GetPersistenceModule(). + Return(persistenceMock). + AnyTimes() + persistenceMock.EXPECT(). + GetBus(). + AnyTimes(). + Return(mockBus) + persistenceMock.EXPECT(). + NewRWContext(int64(0)). + AnyTimes() + persistenceMock.EXPECT(). + GetTxIndexer(). + AnyTimes() + + treemod, err := trees.Create(mockBus, trees.WithTreeStoreDirectory(":memory:")) + assert.NoError(t, err) + + got := treemod.GetBus() + assert.Equal(t, got, mockBus) + + // root hash should be empty for empty tree + root, ns := treemod.GetTree(trees.TransactionsTreeName) + require.Equal(t, root, make([]byte, 32)) + + // nodestore should have no values in it + keys, vals, err := ns.GetAll(nil, false) + require.NoError(t, err) + require.Empty(t, keys, vals) +} + +func TestTreeStore_DebugClearAll(t *testing.T) { + // TODO: Write test case for the DebugClearAll method + t.Skip("TODO: Write test case for DebugClearAll method") +} + +// createMockGenesisState configures and returns a mocked GenesisState +func createMockGenesisState(valKeys []cryptoPocket.PrivateKey) *genesis.GenesisState { + genesisState := new(genesis.GenesisState) + validators := make([]*coreTypes.Actor, len(valKeys)) + for i, valKey := range valKeys { + addr := valKey.Address().String() + mockActor := &coreTypes.Actor{ + ActorType: coreTypes.ActorType_ACTOR_TYPE_VAL, + Address: addr, + PublicKey: valKey.PublicKey().String(), + ServiceUrl: validatorId(i + 1), + StakedAmount: test_artifacts.DefaultStakeAmountString, + PausedHeight: int64(0), + UnstakingHeight: int64(0), + Output: addr, + } + validators[i] = mockActor + } + genesisState.Validators = validators + + return genesisState +} + +// Persistence mock - only needed for validatorMap access +func preparePersistenceMock(t *testing.T, busMock *mockModules.MockBus, genesisState *genesis.GenesisState) *mockModules.MockPersistenceModule { + ctrl := gomock.NewController(t) + + persistenceModuleMock := mockModules.NewMockPersistenceModule(ctrl) + readCtxMock := mockModules.NewMockPersistenceReadContext(ctrl) + + readCtxMock.EXPECT(). + GetAllValidators(gomock.Any()). + Return(genesisState.GetValidators(), nil).AnyTimes() + readCtxMock.EXPECT(). + GetAllStakedActors(gomock.Any()). + DoAndReturn(func(height int64) ([]*coreTypes.Actor, error) { + return testutil.Concatenate[*coreTypes.Actor]( + genesisState.GetValidators(), + genesisState.GetServicers(), + genesisState.GetFishermen(), + genesisState.GetApplications(), + ), nil + }). + AnyTimes() + persistenceModuleMock.EXPECT(). + NewReadContext(gomock.Any()). + Return(readCtxMock, nil). + AnyTimes() + readCtxMock.EXPECT(). + Release(). + AnyTimes() + persistenceModuleMock.EXPECT(). + GetBus(). + Return(busMock). + AnyTimes() + persistenceModuleMock.EXPECT(). + SetBus(busMock). + AnyTimes() + persistenceModuleMock.EXPECT(). + GetModuleName(). + Return(modules.PersistenceModuleName). + AnyTimes() + busMock. + RegisterModule(persistenceModuleMock) + + return persistenceModuleMock +} + +func validatorId(i int) string { + return fmt.Sprintf(serviceURLFormat, i) +} + +// createMockBus returns a mock bus with stubbed out functions for bus registration +func createMockBus(t *testing.T, runtimeMgr modules.RuntimeMgr) *mockModules.MockBus { + t.Helper() + ctrl := gomock.NewController(t) + mockBus := mockModules.NewMockBus(ctrl) + mockModulesRegistry := mockModules.NewMockModulesRegistry(ctrl) + + mockBus.EXPECT(). + GetRuntimeMgr(). + Return(runtimeMgr). + AnyTimes() + mockBus.EXPECT(). + RegisterModule(gomock.Any()). + DoAndReturn(func(m modules.Submodule) { + m.SetBus(mockBus) + }). + AnyTimes() + mockModulesRegistry.EXPECT(). + GetModule(peerstore_provider.PeerstoreProviderSubmoduleName). + Return(nil, runtime.ErrModuleNotRegistered(peerstore_provider.PeerstoreProviderSubmoduleName)). + AnyTimes() + mockModulesRegistry.EXPECT(). + GetModule(modules.CurrentHeightProviderSubmoduleName). + Return(nil, runtime.ErrModuleNotRegistered(modules.CurrentHeightProviderSubmoduleName)). + AnyTimes() + mockBus.EXPECT(). + GetModulesRegistry(). + Return(mockModulesRegistry). + AnyTimes() + mockBus.EXPECT(). + PublishEventToBus(gomock.Any()). + AnyTimes() + + return mockBus +} diff --git a/persistence/trees/trees.go b/persistence/trees/trees.go index 64f210122..0ec61b79e 100644 --- a/persistence/trees/trees.go +++ b/persistence/trees/trees.go @@ -74,23 +74,25 @@ type stateTree struct { var _ modules.TreeStoreModule = &treeStore{} -// treeStore stores a set of merkle trees that -// it manages. It fulfills the modules.TreeStore interface. -// * It is responsible for atomic commit or rollback behavior -// of the underlying trees by utilizing the lazy loading -// functionality provided by the underlying smt library. +// treeStore stores a set of merkle trees that it manages. +// It fulfills the modules.treeStore interface +// * It is responsible for atomic commit or rollback behavior of the underlying +// trees by utilizing the lazy loading functionality of the smt library. +// TECHDEBT(#880): treeStore is exported for testing purposes to avoid import cycle errors. +// Make it private and export a custom struct with a test build tag when necessary. type treeStore struct { base_modules.IntegrableModule - logger *modules.Logger + logger *modules.Logger + treeStoreDir string rootTree *stateTree merkleTrees map[string]*stateTree } -// GetTree returns the name, root hash, and nodeStore for the matching tree tree -// stored in the TreeStore. This enables the caller to import the smt and not -// change the one stored +// GetTree returns the root hash and nodeStore for the matching tree stored in the TreeStore. +// This enables the caller to import the SMT without changing the one stored unless they call +// `Commit()` to write to the nodestore. func (t *treeStore) GetTree(name string) ([]byte, kvstore.KVStore) { if name == RootTreeName { return t.rootTree.tree.Root(), t.rootTree.nodeStore @@ -101,12 +103,26 @@ func (t *treeStore) GetTree(name string) ([]byte, kvstore.KVStore) { return nil, nil } +// GetTreeHashes returns a map of tree names to their root hashes for all +// the trees tracked by the treestore, excluding the root tree +func (t *treeStore) GetTreeHashes() map[string]string { + hashes := make(map[string]string, len(t.merkleTrees)) + for treeName, stateTree := range t.merkleTrees { + hashes[treeName] = hex.EncodeToString(stateTree.tree.Root()) + } + return hashes +} + // Update takes a transaction and a height and updates // all of the trees in the treeStore for that height. func (t *treeStore) Update(pgtx pgx.Tx, height uint64) (string, error) { - txi := t.GetBus().GetPersistenceModule().GetTxIndexer() t.logger.Info().Msgf("🌴 updating state trees at height %d", height) - return t.updateMerkleTrees(pgtx, txi, height) + txi := t.GetBus().GetPersistenceModule().GetTxIndexer() + stateHash, err := t.updateMerkleTrees(pgtx, txi, height) + if err != nil { + return "", fmt.Errorf("failed to update merkle trees: %w", err) + } + return stateHash, nil } // DebugClearAll is used by the debug cli to completely reset all merkle trees. @@ -127,13 +143,10 @@ func (t *treeStore) DebugClearAll() error { return nil } -// GetModuleName implements the respective `TreeStoreModule` interface method. -func (t *treeStore) GetModuleName() string { - return modules.TreeStoreModuleName -} - // updateMerkleTrees updates all of the merkle trees in order defined by `numMerkleTrees` -// * it returns the new state hash capturing the state of all the trees or an error if one occurred +// * It returns the new state hash capturing the state of all the trees or an error if one occurred. +// * This function does not commit state to disk. The caller must manually invoke `Commit` to persist +// changes to disk. func (t *treeStore) updateMerkleTrees(pgtx pgx.Tx, txi indexer.TxIndexer, height uint64) (string, error) { for treeName := range t.merkleTrees { switch treeName { @@ -173,7 +186,7 @@ func (t *treeStore) updateMerkleTrees(pgtx pgx.Tx, txi indexer.TxIndexer, height // Data Merkle Trees case TransactionsTreeName: - indexedTxs, err := sql.GetTransactions(txi, height) + indexedTxs, err := getTransactions(txi, height) if err != nil { return "", fmt.Errorf("failed to get transactions: %w", err) } @@ -210,24 +223,34 @@ func (t *treeStore) updateMerkleTrees(pgtx pgx.Tx, txi indexer.TxIndexer, height } } - if err := t.commit(); err != nil { + if err := t.Commit(); err != nil { return "", fmt.Errorf("failed to commit: %w", err) } return t.getStateHash(), nil } -func (t *treeStore) commit() error { - for treeName, stateTree := range t.merkleTrees { - if err := stateTree.tree.Commit(); err != nil { - return fmt.Errorf("failed to commit %s: %w", treeName, err) +// Commit commits changes in the sub-trees to the root tree and then commits updates for each sub-tree. +func (t *treeStore) Commit() error { + if err := t.rootTree.tree.Commit(); err != nil { + t.logger.Err(err).Msg("TECHDEBT: failed to commit root tree: changes to sub-trees will not be committed - this should be investigated") + return fmt.Errorf("failed to commit root tree: %w", err) + } + + for name, treeStore := range t.merkleTrees { + if err := treeStore.tree.Commit(); err != nil { + t.logger.Err(err).Msgf("TECHDEBT: failed to commit to %s tree: changes will not be saved - this should be investigated", name) + return fmt.Errorf("failed to commit %s: %w", name, err) } } + return nil } func (t *treeStore) getStateHash() string { for _, stateTree := range t.merkleTrees { - if err := t.rootTree.tree.Update([]byte(stateTree.name), stateTree.tree.Root()); err != nil { + key := []byte(stateTree.name) + val := stateTree.tree.Root() + if err := t.rootTree.tree.Update(key, val); err != nil { log.Fatalf("failed to update root tree with %s tree's hash: %v", stateTree.name, err) } } @@ -374,3 +397,13 @@ func (t *treeStore) updateIBCTree(keys, values [][]byte) error { } return nil } + +// getTransactions takes a transaction indexer and returns the transactions for the current height +func getTransactions(txi indexer.TxIndexer, height uint64) ([]*coreTypes.IndexedTransaction, error) { + // TECHDEBT(#813): Avoid this cast to int64 + indexedTxs, err := txi.GetByHeight(int64(height), false) + if err != nil { + return nil, fmt.Errorf("failed to get transactions by height: %w", err) + } + return indexedTxs, nil +} diff --git a/persistence/trees/trees_test.go b/persistence/trees/trees_test.go index 17c502d1a..8acd2a64f 100644 --- a/persistence/trees/trees_test.go +++ b/persistence/trees/trees_test.go @@ -17,3 +17,8 @@ func TestTreeStore_DebugClearAll(t *testing.T) { // TODO: Write test case for the DebugClearAll method t.Skip("TODO: Write test case for DebugClearAll method") } + +// TODO_AFTER(#861): Implement this test with the test suite available in #861 +func TestTreeStore_GetTreeHashes(t *testing.T) { + t.Skip("TODO: Write test case for GetTreeHashes method") // context: https://github.com/pokt-network/pocket/pull/915#discussion_r1267313664 +} diff --git a/persistence/types/ibc.go b/persistence/types/ibc.go index 59af34f1f..a783bcf82 100644 --- a/persistence/types/ibc.go +++ b/persistence/types/ibc.go @@ -13,8 +13,16 @@ const ( value TEXT NOT NULL, PRIMARY KEY (height, key) )` + IBCEventLogTableName = "ibc_events" + IBCEventLogTableSchema = `( + height BIGINT NOT NULL, + topic TEXT NOT NULL, + event TEXT NOT NULL, + PRIMARY KEY (height, topic, event) + )` ) +// InsertIBCStoreEntryQuery returns the query to insert a key/value pair into the ibc_entries table func InsertIBCStoreEntryQuery(height int64, key, value []byte) string { return fmt.Sprintf( `INSERT INTO %s(height, key, value) VALUES(%d, '%s', '%s')`, @@ -25,7 +33,18 @@ func InsertIBCStoreEntryQuery(height int64, key, value []byte) string { ) } -// Return the latest value for the key at the height provided or at the last updated height +// InsertIBCEventQuery returns the query to insert an event into the ibc_events table +func InsertIBCEventQuery(height int64, topic, eventHex string) string { + return fmt.Sprintf( + `INSERT INTO %s(height, topic, event) VALUES(%d, '%s', '%s')`, + IBCEventLogTableName, + height, + topic, + eventHex, + ) +} + +// GetIBCStoreEntryQuery returns the latest value for the key at the height provided or at the last updated height func GetIBCStoreEntryQuery(height int64, key []byte) string { return fmt.Sprintf( `SELECT value FROM %s WHERE height <= %d AND key = '%s' ORDER BY height DESC LIMIT 1`, @@ -35,6 +54,22 @@ func GetIBCStoreEntryQuery(height int64, key []byte) string { ) } -func ClearAllIBCQuery() string { +// GetIBCEventQuery returns the query to get all events for a given height and topic +func GetIBCEventQuery(height uint64, topic string) string { + return fmt.Sprintf( + `SELECT event FROM %s WHERE height = %d AND topic = '%s'`, + IBCEventLogTableName, + height, + topic, + ) +} + +// ClearAllIBCStoreQuery returns the query to clear all entries from the ibc_entries table +func ClearAllIBCStoreQuery() string { return fmt.Sprintf(`DELETE FROM %s`, IBCStoreTableName) } + +// ClearAllIBCEventsQuery returns the query to clear all entries from the ibc_events table +func ClearAllIBCEventsQuery() string { + return fmt.Sprintf(`DELETE FROM %s`, IBCEventLogTableName) +} diff --git a/rpc/utils.go b/rpc/utils.go index f3ce77343..349408bba 100644 --- a/rpc/utils.go +++ b/rpc/utils.go @@ -431,7 +431,10 @@ func (s *rpcServer) blockToRPCBlock(protoBlock *coreTypes.Block) (*Block, error) }, Transactions: qcTxs, }, - Timestamp: protoBlock.BlockHeader.GetTimestamp().AsTime().String(), + Timestamp: protoBlock.BlockHeader.GetTimestamp().AsTime().String(), + StateTreeHashes: BlockHeader_StateTreeHashes{protoBlock.BlockHeader.GetStateTreeHashes()}, + ValSetHash: protoBlock.BlockHeader.GetValSetHash(), + NextValSetHash: protoBlock.BlockHeader.GetNextValSetHash(), }, Transactions: txs, }, nil diff --git a/rpc/v1/openapi.yaml b/rpc/v1/openapi.yaml index eb36f0c76..c068b4238 100644 --- a/rpc/v1/openapi.yaml +++ b/rpc/v1/openapi.yaml @@ -1501,6 +1501,9 @@ components: - proposer_addr - quorum_cert - timestamp + - state_tree_hashes + - val_set_hash + - next_val_set_hash properties: height: type: integer @@ -1517,6 +1520,14 @@ components: $ref: "#/components/schemas/QuorumCertificate" timestamp: type: string + state_tree_hashes: + type: object + additionalProperties: + type: string + val_set_hash: + type: string + next_val_set_hash: + type: string Coin: type: object required: diff --git a/runtime/bus.go b/runtime/bus.go index 9bf4fc3f2..9d7a6e05f 100644 --- a/runtime/bus.go +++ b/runtime/bus.go @@ -71,6 +71,10 @@ func (m *bus) GetPersistenceModule() modules.PersistenceModule { return getModuleFromRegistry[modules.PersistenceModule](m, modules.PersistenceModuleName) } +func (m *bus) GetTreeStoreModule() modules.TreeStoreModule { + return getModuleFromRegistry[modules.TreeStoreModule](m, modules.TreeStoreSubmoduleName) +} + func (m *bus) GetP2PModule() modules.P2PModule { return getModuleFromRegistry[modules.P2PModule](m, modules.P2PModuleName) } @@ -124,7 +128,7 @@ func (m *bus) GetIBCModule() modules.IBCModule { } func (m *bus) GetTreeStore() modules.TreeStoreModule { - return getModuleFromRegistry[modules.TreeStoreModule](m, modules.TreeStoreModuleName) + return getModuleFromRegistry[modules.TreeStoreModule](m, modules.TreeStoreSubmoduleName) } func (m *bus) GetIBCHost() modules.IBCHostSubmodule { @@ -135,6 +139,10 @@ func (m *bus) GetBulkStoreCacher() modules.BulkStoreCacher { return getModuleFromRegistry[modules.BulkStoreCacher](m, modules.BulkStoreCacherModuleName) } +func (m *bus) GetEventLogger() modules.EventLogger { + return getModuleFromRegistry[modules.EventLogger](m, modules.EventLoggerModuleName) +} + func (m *bus) GetCurrentHeightProvider() modules.CurrentHeightProvider { return getModuleFromRegistry[modules.CurrentHeightProvider](m, modules.CurrentHeightProviderSubmoduleName) } diff --git a/runtime/configs/proto/p2p_config.proto b/runtime/configs/proto/p2p_config.proto index e01cea09a..c331176fa 100644 --- a/runtime/configs/proto/p2p_config.proto +++ b/runtime/configs/proto/p2p_config.proto @@ -12,6 +12,6 @@ message P2PConfig { uint32 port = 3; conn.ConnectionType connection_type = 4; uint64 max_nonces = 5; // used to limit the number of nonces that can be stored before a FIFO mechanism is used to remove the oldest nonces and make space for the new ones - bool is_client_only = 6; + bool is_client_only = 6; // TECHDEBT(bryanchriswhite,olshansky): Re-evaluate if this is still needed string bootstrap_nodes_csv = 7; // string in the format "http://somenode:50832,http://someothernode:50832". Refer to `p2p/module_test.go` for additional details. } diff --git a/shared/core/types/proto/block.proto b/shared/core/types/proto/block.proto index b519ffcbe..a0e2ed6c3 100644 --- a/shared/core/types/proto/block.proto +++ b/shared/core/types/proto/block.proto @@ -13,10 +13,24 @@ message BlockHeader { string prevStateHash = 4; // the state committment at this block height-1 bytes proposerAddress = 5; // the address of the proposer of this block; TECHDEBT: Change this to an string bytes quorumCertificate = 6; // the quorum certificate containing signature from 2/3+ validators at this height - google.protobuf.Timestamp timestamp = 7; // CONSIDERATION: Is this needed? + google.protobuf.Timestamp timestamp = 7; // unixnano timestamp of when the block was created + map state_tree_hashes = 8; // map[TreeName]hex(TreeRootHash) + string val_set_hash = 9; // the hash of the current validator set who were able to sign the current block + string next_val_set_hash = 10; // the hash of the next validator set; needed to ensure the validity of staked validators proposing the next block } message Block { core.BlockHeader blockHeader = 1; repeated bytes transactions = 2; } + +message ValidatorSet { + repeated ValidatorIdentity validators = 1; +} + +// DISCUSS(M5): Should we include voting power in this identity? +// Ignoring voting_power/stake for leader election? Is this needed - I dont think so +message ValidatorIdentity { + string address = 1; + string pub_key = 2; +} diff --git a/shared/core/types/proto/ibc_events.proto b/shared/core/types/proto/ibc_events.proto new file mode 100644 index 000000000..15041214a --- /dev/null +++ b/shared/core/types/proto/ibc_events.proto @@ -0,0 +1,21 @@ +syntax = "proto3"; + +package core; + +option go_package = "github.com/pokt-network/pocket/shared/core/types"; + +// Attribute represents a key-value pair in an IBC event +message Attribute { + bytes key = 1; + bytes value = 2; +} + +// IBCEvent are used after a series of insertions/updates/deletions to the IBC store +// they capture the type of changes made, such as creating a new light client, or +// opening a connection. They also capture the height at which the change was made +// and the different key-value pairs that were modified in the attributes field. +message IBCEvent { + string topic = 1; + uint64 height = 2; + repeated Attribute attributes = 3; +} diff --git a/shared/k8s/debug.go b/shared/k8s/debug.go index bfb966e04..a340fcfda 100644 --- a/shared/k8s/debug.go +++ b/shared/k8s/debug.go @@ -16,9 +16,14 @@ import ( ) //nolint:gosec // G101 Not a credential -const privateKeysSecretResourceName = "validators-private-keys" -const kubernetesServiceAccountNamespaceFile = "/var/run/secrets/kubernetes.io/serviceaccount/namespace" -const defaultNamespace = "default" +const ( + privateKeysSecretResourceNameValidators = "validators-private-keys" + privateKeysSecretResourceNameServicers = "servicers-private-keys" + privateKeysSecretResourceNameFishermen = "fishermen-private-keys" + privateKeysSecretResourceNameApplications = "applications-private-keys" + kubernetesServiceAccountNamespaceFile = "/var/run/secrets/kubernetes.io/serviceaccount/namespace" + defaultNamespace = "default" +) var CurrentNamespace = "" @@ -34,20 +39,42 @@ func init() { } // FetchValidatorPrivateKeys returns a map corresponding to the data section of -// the validator private keys k8s secret (yaml), located at `privateKeysSecretResourceName`. +// the validator private keys Kubernetes secret. func FetchValidatorPrivateKeys(clientset *kubernetes.Clientset) (map[string]string, error) { - validatorKeysMap := make(map[string]string) + return fetchPrivateKeys(clientset, privateKeysSecretResourceNameValidators) +} - privateKeysSecret, err := clientset.CoreV1().Secrets(CurrentNamespace).Get(context.TODO(), privateKeysSecretResourceName, metav1.GetOptions{}) +// FetchServicerPrivateKeys returns a map corresponding to the data section of +// the servicer private keys Kubernetes secret. +func FetchServicerPrivateKeys(clientset *kubernetes.Clientset) (map[string]string, error) { + return fetchPrivateKeys(clientset, privateKeysSecretResourceNameServicers) +} + +// FetchFishermanPrivateKeys returns a map corresponding to the data section of +// the fisherman private keys Kubernetes secret. +func FetchFishermanPrivateKeys(clientset *kubernetes.Clientset) (map[string]string, error) { + return fetchPrivateKeys(clientset, privateKeysSecretResourceNameFishermen) +} + +// FetchApplicationPrivateKeys returns a map corresponding to the data section of +// the application private keys Kubernetes secret. +func FetchApplicationPrivateKeys(clientset *kubernetes.Clientset) (map[string]string, error) { + return fetchPrivateKeys(clientset, privateKeysSecretResourceNameApplications) +} + +// fetchPrivateKeys returns a map corresponding to the data section of +// the private keys Kubernetes secret for the specified resource name and actor. +func fetchPrivateKeys(clientset *kubernetes.Clientset, resourceName string) (map[string]string, error) { + privateKeysMap := make(map[string]string) + privateKeysSecret, err := clientset.CoreV1().Secrets(CurrentNamespace).Get(context.TODO(), resourceName, metav1.GetOptions{}) if err != nil { - panic(err) + return nil, err } - for id, privHexString := range privateKeysSecret.Data { - // it's safe to cast []byte to string here - validatorKeysMap[id] = string(privHexString) + // It's safe to cast []byte to string here + privateKeysMap[id] = string(privHexString) } - return validatorKeysMap, nil + return privateKeysMap, nil } func getNamespace() (string, error) { diff --git a/shared/modules/bus_module.go b/shared/modules/bus_module.go index 50b2e23f5..3981f9f8c 100644 --- a/shared/modules/bus_module.go +++ b/shared/modules/bus_module.go @@ -44,4 +44,5 @@ type Bus interface { GetTreeStore() TreeStoreModule GetIBCHost() IBCHostSubmodule GetBulkStoreCacher() BulkStoreCacher + GetEventLogger() EventLogger } diff --git a/shared/modules/doc/README.md b/shared/modules/doc/README.md index 28a2c249b..201856453 100644 --- a/shared/modules/doc/README.md +++ b/shared/modules/doc/README.md @@ -6,32 +6,34 @@ This document outlines how we structured the code by splitting it into modules, - [tl;dr Just show me an example](#tldr-just-show-me-an-example) - [Definitions](#definitions) - - [Requirement Level Keywords](#requirement-level-keywords) - - [Module](#module) - - [Module mock](#module-mock) - - [Shared module interfaces](#shared-module-interfaces) - - [Base module](#base-module) - - [Submodule](#submodule) - - [Shared submodule](#shared-submodule) - - [Class Diagram Legend](#class-diagram-legend) - - [Module, Submodule \& Shared Interfaces](#module-submodule--shared-interfaces) - - [Factory interfaces](#factory-interfaces) + - [Requirement Level Keywords](#requirement-level-keywords) + - [Module](#module) + - [Module mock](#module-mock) + - [Shared module interfaces](#shared-module-interfaces) + - [Base module](#base-module) + - [Submodule](#submodule) + - [Shared submodule](#shared-submodule) + - [Class Diagram Legend](#class-diagram-legend) + - [Module, Submodule \& Shared Interfaces](#module-submodule--shared-interfaces) + - [Factory interfaces](#factory-interfaces) - [Code Organization](#code-organization) -- [(Sub)Modules in detail](#submodules-in-detail) - [Shared (sub)module interfaces](#shared-submodule-interfaces) - [Construction parameters \& non-(sub)module dependencies](#construction-parameters--non-submodule-dependencies) - - [Module creation](#module-creation) - - [Module configs \& options](#module-configs--options) - - [Submodule creation](#submodule-creation) - - [Submodule configs \& options](#submodule-configs--options) - - [Configs](#configs) - - [Options](#options) - - [Comprehensive Submodule Example:](#comprehensive-submodule-example) - - [Interacting \& Registering with the `bus`](#interacting--registering-with-the-bus) - - [Modules Registry](#modules-registry) - - [Modules Registry Example](#modules-registry-example) - - [Start the module](#start-the-module) - - [Add a logger to the module](#add-a-logger-to-the-module) - - [Get the module `bus`](#get-the-module-bus) - - [Stop the module](#stop-the-module) +- [(Sub)Modules in detail](#submodules-in-detail) + - [Shared (sub)module interfaces](#shared-submodule-interfaces) + - [Construction parameters \& non-(sub)module dependencies](#construction-parameters--non-submodule-dependencies) + - [Module creation](#module-creation) + - [Module configs \& options](#module-configs--options) + - [Submodule creation](#submodule-creation) + - [Submodule configs \& options](#submodule-configs--options) + - [Configs](#configs) + - [Options](#options) + - [Comprehensive Submodule Example:](#comprehensive-submodule-example) + - [Interacting \& Registering with the `bus`](#interacting--registering-with-the-bus) + - [Modules Registry](#modules-registry) + - [Modules Registry Example](#modules-registry-example) + - [Start the module](#start-the-module) + - [Add a logger to the module](#add-a-logger-to-the-module) + - [Get the module `bus`](#get-the-module-bus) + - [Stop the module](#stop-the-module) ## tl;dr Just show me an example @@ -420,8 +422,8 @@ What the `bus` does is setting its reference to the module instance and delegati ```golang func (m *bus) RegisterModule(module modules.Module) { - module.SetBus(m) - m.modulesRegistry.RegisterModule(module) + module.SetBus(m) + m.modulesRegistry.RegisterModule(module) } ``` @@ -432,7 +434,40 @@ This is quite **important** because it unlocks a powerful concept **Dependency I This enables the developer to define different implementations of a module and to register the one that is needed at runtime. This is because we can only have one module registered with a unique name and also because, by convention, we keep module names defined as constants. This is useful not only for prototyping but also for different use cases such as the `p1` CLI and the `pocket` binary where different implementations of the same module are necessary due to the fact that the `p1` CLI doesn't have a persistence module but still needs to know what's going on in the network. -Submodules can be registered the same way full Modules can be, by passing the Submodule to the RegisterModule function. Submodules should typically be registered to the bus for dependency injection reasons. +**Registration**: Submodules can be registered the same way full Modules by passing the Submodule to the `RegisterModule` function. Submodules should typically be registered to the bus for dependency injection reasons. Registration should occur _after_ processing the module's options. + +```go +func (*treeStore) Create(bus modules.Bus, options ...modules.TreeStoreOption) (modules.TreeStoreModule, error) { + m := &treeStore{} + + for _, option := range options { + option(m) + } + + bus.RegisterModule(m) + // ... +} +``` + +**IMPORTANT**: Modules and Submodules are all responsible for registering themselves with the Bus. This pattern emerged organically during development and is now considered best practice. + +**Module Access**: Full Modules should not maintain pointer references to other full Modules. Instead, they should call the Bus to get a new module reference whenever needed. + +Submodule interfaces are typically defined in the `shared/modules` package with the rest of the module interfaces in a file named `XXX_submodule.go`, where XXX denotes the name of the Submodule. That same file SHOULD contain the factory function definition for a Submodule where the `Submodule` interface type MUST be embedded. Factory function definitions SHOULD NOT be exported. + +For example, in the TreeStore code below, the `treeStoreFactory` is defined and then embedded in the `TreeStoreModule`, which also embeds the Submodule interface. Typically these factory functions should be kept private at the package level. + +```go +type treeStoreFactory = FactoryWithOptions[TreeStoreModule, TreeStoreOption] + +// TreeStoreModules defines the interface for atomic updates and rollbacks to the internal +// merkle trees that compose the state hash of pocket. +type TreeStoreModule interface { + Submodule + treeStoreFactory + // ... +} +``` ##### Modules Registry Example diff --git a/shared/modules/ibc_event_module.go b/shared/modules/ibc_event_module.go new file mode 100644 index 000000000..56a80f07f --- /dev/null +++ b/shared/modules/ibc_event_module.go @@ -0,0 +1,21 @@ +package modules + +//go:generate mockgen -destination=./mocks/ibc_event_module_mock.go github.com/pokt-network/pocket/shared/modules EventLogger + +import ( + coreTypes "github.com/pokt-network/pocket/shared/core/types" +) + +const EventLoggerModuleName = "event_logger" + +type EventLoggerOption func(EventLogger) + +type eventLoggerFactory = FactoryWithOptions[EventLogger, EventLoggerOption] + +type EventLogger interface { + Submodule + eventLoggerFactory + + EmitEvent(event *coreTypes.IBCEvent) error + QueryEvents(topic string, height uint64) ([]*coreTypes.IBCEvent, error) +} diff --git a/shared/modules/ibc_store_module.go b/shared/modules/ibc_store_module.go index 2a308a66b..fc0196804 100644 --- a/shared/modules/ibc_store_module.go +++ b/shared/modules/ibc_store_module.go @@ -25,7 +25,7 @@ type BulkStoreCacher interface { GetStore(name string) (ProvableStore, error) RemoveStore(name string) error GetAllStores() map[string]ProvableStore - FlushAllEntries() error + FlushCachesToStore() error PruneCaches(height uint64) error RestoreCaches(height uint64) error } @@ -44,7 +44,7 @@ type ProvableStore interface { Delete(key []byte) error GetCommitmentPrefix() coreTypes.CommitmentPrefix Root() ics23.CommitmentRoot - FlushEntries(kvstore.KVStore) error + FlushCache(kvstore.KVStore) error PruneCache(store kvstore.KVStore, height uint64) error RestoreCache(store kvstore.KVStore, height uint64) error } diff --git a/shared/modules/mocks/mocks.go b/shared/modules/mocks/mocks.go new file mode 100644 index 000000000..913944d06 --- /dev/null +++ b/shared/modules/mocks/mocks.go @@ -0,0 +1,5 @@ +package mock_modules + +// IMPORTANT: DO NOT DELETE THIS FILE +// This file is in place to declare the package for dynamically generated mocks +// See the explanation in #905 for details on how removing this can break `make develop_start` diff --git a/shared/modules/persistence_module.go b/shared/modules/persistence_module.go index 55e338fa5..b510d835b 100644 --- a/shared/modules/persistence_module.go +++ b/shared/modules/persistence_module.go @@ -35,9 +35,6 @@ type PersistenceModule interface { GetTxIndexer() indexer.TxIndexer TransactionExists(transactionHash string) (bool, error) - // TreeStore operations - GetTreeStore() TreeStoreModule - // Debugging / development only HandleDebugMessage(*messaging.DebugMessage) error @@ -149,6 +146,8 @@ type PersistenceWriteContext interface { // key-value pairs represent the same key-value pairings in the IBC state tree. This table is // used for data retrieval purposes and to update the state tree from the mempool of IBC transactions. SetIBCStoreEntry(key, value []byte) error + // SetIBCEvent stores an IBC event in the persistence context at the current height + SetIBCEvent(event *coreTypes.IBCEvent) error // Relay Operations RecordRelayService(applicationAddress string, key []byte, relay *coreTypes.Relay, response *coreTypes.RelayResponse) error @@ -221,6 +220,7 @@ type PersistenceReadContext interface { // Validator Queries GetValidator(address []byte, height int64) (*coreTypes.Actor, error) GetAllValidators(height int64) ([]*coreTypes.Actor, error) + GetValidatorSet(height int64) (*coreTypes.ValidatorSet, error) GetValidatorExists(address []byte, height int64) (exists bool, err error) GetValidatorStakeAmount(height int64, address []byte) (string, error) GetValidatorsReadyToUnstake(height int64, status int32) (validators []*moduleTypes.UnstakingActor, err error) @@ -246,6 +246,8 @@ type PersistenceReadContext interface { // IBC Queries // GetIBCStoreEntry returns the value of the key at the given height from the ibc_entries table GetIBCStoreEntry(key []byte, height int64) ([]byte, error) + // GetIBCEvent returns the matching IBC events for any topic at the height provied + GetIBCEvents(height uint64, topic string) ([]*coreTypes.IBCEvent, error) } // PersistenceLocalContext defines the set of operations specific to local persistence. diff --git a/shared/modules/treestore_module.go b/shared/modules/treestore_module.go index a2aedfb8e..117026f05 100644 --- a/shared/modules/treestore_module.go +++ b/shared/modules/treestore_module.go @@ -8,17 +8,18 @@ import ( //go:generate mockgen -destination=./mocks/treestore_module_mock.go github.com/pokt-network/pocket/shared/modules TreeStoreModule const ( - TreeStoreModuleName = "tree_store" + TreeStoreSubmoduleName = "tree_store" ) type TreeStoreOption func(TreeStoreModule) -type TreeStoreFactory = FactoryWithOptions[TreeStoreModule, TreeStoreOption] +type treeStoreFactory = FactoryWithOptions[TreeStoreModule, TreeStoreOption] // TreeStoreModules defines the interface for atomic updates and rollbacks to the internal // merkle trees that compose the state hash of pocket. type TreeStoreModule interface { Submodule + treeStoreFactory // Update returns the new state hash for a given height. // * Height is passed through to the Update function and is used to query the TxIndexer for transactions @@ -32,4 +33,6 @@ type TreeStoreModule interface { DebugClearAll() error // GetTree returns the specified tree's root and nodeStore in order to be imported elsewhere GetTree(name string) ([]byte, kvstore.KVStore) + // GetTreeHashes returns a map of tree names to their root hashes + GetTreeHashes() map[string]string } From 9ecc9e5b04fd8a34465d2a0eeaa3cdcd076ae763 Mon Sep 17 00:00:00 2001 From: harry <53987565+h5law@users.noreply.github.com> Date: Tue, 25 Jul 2023 10:55:04 +0100 Subject: [PATCH 21/38] chore: address review comments --- app/client/cli/debug.go | 19 ++++++++++++------- app/client/cli/helpers/common.go | 18 ++++++------------ app/client/cli/helpers/setup.go | 7 ++++--- runtime/manager.go | 2 +- 4 files changed, 23 insertions(+), 23 deletions(-) diff --git a/app/client/cli/debug.go b/app/client/cli/debug.go index 0bc7952f0..184c07947 100644 --- a/app/client/cli/debug.go +++ b/app/client/cli/debug.go @@ -159,14 +159,17 @@ func handleSelect(cmd *cobra.Command, selection string) { } // Broadcast to the entire network. -func broadcastDebugMessage(_ *cobra.Command, debugMsg *messaging.DebugMessage) { +func broadcastDebugMessage(cmd *cobra.Command, debugMsg *messaging.DebugMessage) { anyProto, err := anypb.New(debugMsg) if err != nil { logger.Global.Fatal().Err(err).Msg("Failed to create Any proto") } - // TECHDEBT: prefer to retrieve P2P module from the bus instead. - if err := helpers.P2PMod.Broadcast(anyProto); err != nil { + bus, err := helpers.GetBusFromCmd(cmd) + if err != nil { + logger.Global.Fatal().Err(err).Msg("Failed to retrieve bus from command") + } + if err := bus.GetP2PModule().Broadcast(anyProto); err != nil { logger.Global.Error().Err(err).Msg("Failed to broadcast debug message") } } @@ -183,7 +186,6 @@ func sendDebugMessage(cmd *cobra.Command, debugMsg *messaging.DebugMessage) { logger.Global.Fatal().Err(err).Msg("Unable to retrieve the pstore") } - var validatorAddress []byte if pstore.Size() == 0 { logger.Global.Fatal().Msg("No validators found") } @@ -192,13 +194,16 @@ func sendDebugMessage(cmd *cobra.Command, debugMsg *messaging.DebugMessage) { // // DISCUSS_THIS_COMMIT: The statement above is false. Using `#Send()` will only // be unicast with no opportunity for further propagation. - validatorAddress = pstore.GetPeerList()[0].GetAddress() + firstStakedActorAddress := pstore.GetPeerList()[0].GetAddress() if err != nil { logger.Global.Fatal().Err(err).Msg("Failed to convert validator address into pocketCrypto.Address") } - // TECHDEBT: prefer to retrieve P2P module from the bus instead. - if err := helpers.P2PMod.Send(validatorAddress, anyProto); err != nil { + bus, err := helpers.GetBusFromCmd(cmd) + if err != nil { + logger.Global.Fatal().Err(err).Msg("Failed to retrieve bus from command") + } + if err := bus.GetP2PModule().Send(firstStakedActorAddress, anyProto); err != nil { logger.Global.Error().Err(err).Msg("Failed to send debug message") } } diff --git a/app/client/cli/helpers/common.go b/app/client/cli/helpers/common.go index b89f79ba9..647d4241d 100644 --- a/app/client/cli/helpers/common.go +++ b/app/client/cli/helpers/common.go @@ -14,22 +14,16 @@ import ( "github.com/pokt-network/pocket/shared/modules" ) -var ( - // TECHDEBT: Accept reading this from `Datadir` and/or as a flag. - genesisPath = runtime.GetEnv("GENESIS_PATH", "build/config/genesis.json") +// TECHDEBT: Accept reading this from `Datadir` and/or as a flag. +var genesisPath = runtime.GetEnv("GENESIS_PATH", "build/config/genesis.json") - // P2PMod is initialized in order to broadcast a message to the local network - // TECHDEBT: prefer to retrieve P2P module from the bus instead. - P2PMod modules.P2PModule -) - -// fetchPeerstore retrieves the providers from the CLI context and uses them to retrieve the address book for the current height +// FetchPeerstore retrieves the providers from the CLI context and uses them to retrieve the address book for the current height func FetchPeerstore(cmd *cobra.Command) (types.Peerstore, error) { bus, err := GetBusFromCmd(cmd) if err != nil { return nil, err } - // TECHDEBT(#810, #811): use `bus.GetPeerstoreProvider()` after peerstore provider + // TECHDEBT(#811): use `bus.GetPeerstoreProvider()` after peerstore provider // is retrievable as a proper submodule pstoreProvider, err := bus.GetModulesRegistry().GetModule(peerstore_provider.PeerstoreProviderSubmoduleName) if err != nil { @@ -42,8 +36,7 @@ func FetchPeerstore(cmd *cobra.Command) (types.Peerstore, error) { return nil, fmt.Errorf("retrieving peerstore at height %d", height) } // Inform the client's main P2P that a the blockchain is at a new height so it can, if needed, update its view of the validator set - err = sendConsensusNewHeightEventToP2PModule(height, bus) - if err != nil { + if err := sendConsensusNewHeightEventToP2PModule(height, bus); err != nil { return nil, errors.New("sending consensus new height event") } return pstore, nil @@ -53,6 +46,7 @@ func FetchPeerstore(cmd *cobra.Command) (types.Peerstore, error) { // This is necessary because the debug client is not a validator and has no consensus module but it has to update the peerstore // depending on the changes in the validator set. // TODO(#613): Make the debug client mimic a full node. +// TECHDEBT: This may no longer be required (https://github.com/pokt-network/pocket/pull/891/files#r1262710098) func sendConsensusNewHeightEventToP2PModule(height uint64, bus modules.Bus) error { newHeightEvent, err := messaging.PackMessage(&messaging.ConsensusNewHeightEvent{Height: height}) if err != nil { diff --git a/app/client/cli/helpers/setup.go b/app/client/cli/helpers/setup.go index 8bc5e8ca6..c466e2ebe 100644 --- a/app/client/cli/helpers/setup.go +++ b/app/client/cli/helpers/setup.go @@ -16,8 +16,9 @@ import ( "github.com/pokt-network/pocket/shared/modules" ) -// TODO_THIS_COMMIT: add godoc comment explaining what this **is** and **is not** -// intended to be used for. +// debugPrivKey is used in the generation of a runtime config to provide a private key to the P2P and Consensus modules +// this is not a private key used for sending transactions, but is used for the purposes of broadcasting messages etc. +// this must be done as the CLI does not take a node configuration file and still requires a Private Key for modules const debugPrivKey = "09fc8ee114e678e665d09179acb9a30060f680df44ba06b51434ee47940a8613be19b2b886e743eb1ff7880968d6ce1a46350315e569243e747a227ee8faec3d" // P2PDependenciesPreRunE initializes peerstore & current height providers, and a @@ -87,7 +88,7 @@ func setupAndStartP2PModule(rm runtime.Manager) { } var ok bool - P2PMod, ok = mod.(modules.P2PModule) + P2PMod, ok := mod.(modules.P2PModule) if !ok { logger.Global.Fatal().Msgf("unexpected P2P module type: %T", mod) } diff --git a/runtime/manager.go b/runtime/manager.go index 7c182f34f..da6209957 100644 --- a/runtime/manager.go +++ b/runtime/manager.go @@ -104,7 +104,7 @@ func WithRandomPK() func(*Manager) { return WithPK(privateKey.String()) } -// TECHDEBT(#750): separate conseneus and P2P keys. +// TECHDEBT(#750): separate consensus and P2P (identity vs communication) keys. func WithPK(pk string) func(*Manager) { return func(b *Manager) { if b.config.Consensus == nil { From ffbc539459151ad6ba2be89f655b7f43ccd8523f Mon Sep 17 00:00:00 2001 From: harry <53987565+h5law@users.noreply.github.com> Date: Tue, 25 Jul 2023 10:57:27 +0100 Subject: [PATCH 22/38] fix merge error --- app/client/cli/debug.go | 27 +-------------------------- 1 file changed, 1 insertion(+), 26 deletions(-) diff --git a/app/client/cli/debug.go b/app/client/cli/debug.go index 751dfd75e..6fa53aaa9 100644 --- a/app/client/cli/debug.go +++ b/app/client/cli/debug.go @@ -23,7 +23,6 @@ const ( PromptSendBlockRequest string = "BlockRequest (broadcast)" ) -<<<<<<< HEAD var items = []string{ PromptPrintNodeState, PromptTriggerNextView, @@ -40,30 +39,6 @@ func init() { rootCmd.AddCommand(dbgUI) } -======= -var ( - items = []string{ - PromptPrintNodeState, - PromptTriggerNextView, - PromptTogglePacemakerMode, - PromptResetToGenesis, - PromptShowLatestBlockInStore, - PromptSendMetadataRequest, - PromptSendBlockRequest, - } -) - -func init() { - dbgUI := newDebugUICommand() - dbgUI.AddCommand(newDebugUISubCommands()...) - rootCmd.AddCommand(dbgUI) - - dbg := newDebugCommand() - dbg.AddCommand(debugCommands()...) - rootCmd.AddCommand(dbg) -} - ->>>>>>> main // newDebugUISubCommands builds out the list of debug subcommands by matching the // handleSelect dispatch to the appropriate command. // * To add a debug subcommand, you must add it to the `items` array and then @@ -74,7 +49,7 @@ func newDebugUISubCommands() []*cobra.Command { commands[idx] = &cobra.Command{ Use: promptItem, PersistentPreRunE: helpers.P2PDependenciesPreRunE, - Run: func(cmd *cobra.Command, args []string) { + Run: func(cmd *cobra.Command, _ []string) { handleSelect(cmd, cmd.Use) }, ValidArgs: items, From 39af37cf891d9bf3b451ef1cd9c73cb8af8fbd46 Mon Sep 17 00:00:00 2001 From: harry <53987565+h5law@users.noreply.github.com> Date: Tue, 25 Jul 2023 12:43:36 +0100 Subject: [PATCH 23/38] revert change --- p2p/background/router.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/p2p/background/router.go b/p2p/background/router.go index 199c977ec..f7ac9c9ff 100644 --- a/p2p/background/router.go +++ b/p2p/background/router.go @@ -363,7 +363,7 @@ func (rtr *backgroundRouter) bootstrap(ctx context.Context) { "peer_id": libp2pAddrInfo.ID.String(), "peer_addr": libp2pAddrInfo.Addrs[0].String(), }).Msg("connecting to peer") - if err := rtr.host.Connect(ctx, libp2pAddrInfo); err != nil { + if err := rtr.connectWithRetry(ctx, libp2pAddrInfo); err != nil { rtr.logger.Error().Err(err).Msg("connecting to bootstrap peer") continue } @@ -380,7 +380,7 @@ func (rtr *backgroundRouter) connectWithRetry(ctx context.Context, libp2pAddrInf return nil } - fmt.Printf("Failed to connect (attempt %d), retrying in %v...\n", i+1, connectRetryTimeout) + rtr.logger.Error().Msgf("Failed to connect (attempt %d), retrying in %v...\n", i+1, connectRetryTimeout) time.Sleep(connectRetryTimeout) } From c22011c5989eadd591bf8f030c2c24ed6a389432 Mon Sep 17 00:00:00 2001 From: harry <53987565+h5law@users.noreply.github.com> Date: Tue, 25 Jul 2023 13:03:02 +0100 Subject: [PATCH 24/38] address comments --- app/client/cli/debug.go | 2 +- app/client/cli/peer/list.go | 25 ++++++++++--------------- p2p/background/router.go | 4 +++- p2p/debug.go | 7 ------- p2p/debug/list.go | 14 ++++++++------ 5 files changed, 22 insertions(+), 30 deletions(-) diff --git a/app/client/cli/debug.go b/app/client/cli/debug.go index 6fa53aaa9..cac5dac09 100644 --- a/app/client/cli/debug.go +++ b/app/client/cli/debug.go @@ -192,7 +192,7 @@ func sendDebugMessage(cmd *cobra.Command, debugMsg *messaging.DebugMessage) { // if the message needs to be broadcast, it'll be handled by the business logic of the message handler // - // DISCUSS_THIS_COMMIT: The statement above is false. Using `#Send()` will only + // DISCUSS_IN_THIS_COMMIT: The statement above is false. Using `#Send()` will only // be unicast with no opportunity for further propagation. firstStakedActorAddress := pstore.GetPeerList()[0].GetAddress() if err != nil { diff --git a/app/client/cli/peer/list.go b/app/client/cli/peer/list.go index 197903848..7c72f7067 100644 --- a/app/client/cli/peer/list.go +++ b/app/client/cli/peer/list.go @@ -14,9 +14,11 @@ import ( var ( listCmd = &cobra.Command{ - Use: "list", - Short: "Print addresses and service URLs of known peers", - RunE: listRunE, + Use: "List", + Short: "List the known peers", + Long: "Prints a table of the Peer ID, Pokt Address and Service URL of the known peers", + Aliases: []string{"list", "ls"}, + RunE: listRunE, } ErrRouterType = fmt.Errorf("must specify one of --staked, --unstaked, or --all") @@ -35,21 +37,14 @@ func listRunE(cmd *cobra.Command, _ []string) error { } switch { - case stakedFlag: - if unstakedFlag || allFlag { - return ErrRouterType - } + case stakedFlag && !unstakedFlag && !allFlag: routerType = debug.StakedRouterType - case unstakedFlag: - if stakedFlag || allFlag { - return ErrRouterType - } + case unstakedFlag && !stakedFlag && !allFlag: routerType = debug.UnstakedRouterType - // even if `allFlag` is false, we still want to print all peers + case stakedFlag || unstakedFlag: + return ErrRouterType + // even if `allFlag` is false, we still want to print all connections default: - if stakedFlag || unstakedFlag { - return ErrRouterType - } routerType = debug.AllRouterTypes } diff --git a/p2p/background/router.go b/p2p/background/router.go index f7ac9c9ff..6d668ec3c 100644 --- a/p2p/background/router.go +++ b/p2p/background/router.go @@ -341,7 +341,8 @@ func (rtr *backgroundRouter) setupSubscription() (err error) { func (rtr *backgroundRouter) bootstrap(ctx context.Context) { // CONSIDERATION: add `GetPeers` method, which returns a map, // to the `PeerstoreProvider` interface to simplify this loop. - for _, peer := range rtr.pstore.GetPeerList() { + peerList := rtr.pstore.GetPeerList() + for _, peer := range peerList { if err := utils.AddPeerToLibp2pHost(rtr.host, peer); err != nil { rtr.logger.Error().Err(err).Msg("adding peer to libp2p host") continue @@ -362,6 +363,7 @@ func (rtr *backgroundRouter) bootstrap(ctx context.Context) { rtr.logger.Debug().Fields(map[string]any{ "peer_id": libp2pAddrInfo.ID.String(), "peer_addr": libp2pAddrInfo.Addrs[0].String(), + "num_peers": len(peerList) - 1, // -1 as includes self }).Msg("connecting to peer") if err := rtr.connectWithRetry(ctx, libp2pAddrInfo); err != nil { rtr.logger.Error().Err(err).Msg("connecting to bootstrap peer") diff --git a/p2p/debug.go b/p2p/debug.go index 7c352eb5a..bf15e1d03 100644 --- a/p2p/debug.go +++ b/p2p/debug.go @@ -14,13 +14,6 @@ func (m *p2pModule) handleDebugMessage(msg *messaging.DebugMessage) error { if !m.cfg.EnablePeerDiscoveryDebugRpc { return typesP2P.ErrPeerDiscoveryDebugRPCDisabled } - default: - // This debug message isn't intended for the P2P module, ignore it. - return nil - } - - switch msg.Action { - case messaging.DebugMessageAction_DEBUG_P2P_PEER_LIST: routerType := debug.RouterType(msg.Message.Value) return debug.PrintPeerList(m.GetBus(), routerType) default: diff --git a/p2p/debug/list.go b/p2p/debug/list.go index 6a752e71b..21e2455d3 100644 --- a/p2p/debug/list.go +++ b/p2p/debug/list.go @@ -12,6 +12,9 @@ import ( var peerListTableHeader = []string{"Peer ID", "Pokt Address", "ServiceURL"} +// PrintPeerList retrieves the correct peer list using the peerstore provider +// on the bus and then passes this list to PrintPeerListTable to print the +// list of peers to os.Stdout as a table func PrintPeerList(bus modules.Bus, routerType RouterType) error { var ( peers types.PeerList @@ -29,11 +32,10 @@ func PrintPeerList(bus modules.Bus, routerType RouterType) error { if !ok { return fmt.Errorf("unknown peerstore provider type: %T", pstoreProviderModule) } - //-- switch routerType { case StakedRouterType: - // TODO_THIS_COMMIT: what about unstaked peers actors? + // TODO_IN_THIS_COMMIT: what about unstaked peers actors? // if !isStaked ... pstore, err := pstoreProvider.GetStakedPeerstoreAtCurrentHeight() if err != nil { @@ -51,7 +53,7 @@ func PrintPeerList(bus modules.Bus, routerType RouterType) error { case AllRouterTypes: routerPlurality = "s" - // TODO_THIS_COMMIT: what about unstaked peers actors? + // TODO_IN_THIS_COMMIT: what about unstaked peers actors? // if !isStaked ... stakedPStore, err := pstoreProvider.GetStakedPeerstoreAtCurrentHeight() if err != nil { @@ -83,7 +85,7 @@ func PrintPeerList(bus modules.Bus, routerType RouterType) error { } if err := LogSelfAddress(bus); err != nil { - return fmt.Errorf("printing self address: %w", err) + return fmt.Errorf("error printing self address: %w", err) } // NB: Intentionally printing with `fmt` instead of the logger to match @@ -96,11 +98,11 @@ func PrintPeerList(bus modules.Bus, routerType RouterType) error { routerType, routerPlurality, ); err != nil { - return fmt.Errorf("printing to stdout: %w", err) + return fmt.Errorf("error printing to stdout: %w", err) } if err := PrintPeerListTable(peers); err != nil { - return fmt.Errorf("printing peer list: %w", err) + return fmt.Errorf("error printing peer list: %w", err) } return nil } From 1fc2bb438c0e37357e3a1ad20eb9b598a6ee8496 Mon Sep 17 00:00:00 2001 From: harry <53987565+h5law@users.noreply.github.com> Date: Tue, 25 Jul 2023 13:57:20 +0100 Subject: [PATCH 25/38] chore: address comments --- app/client/cli/helpers/common.go | 9 ++-- app/client/cli/peer/list.go | 6 +-- p2p/background/router.go | 12 ++--- p2p/debug.go | 2 +- p2p/debug/list.go | 11 ++-- p2p/event_handler.go | 3 +- p2p/module.go | 28 ++-------- .../peerstore_provider/peerstore_provider.go | 17 +++++++ .../persistence/provider.go | 6 ++- .../peerstore_provider/rpc/provider.go | 8 +-- p2p/raintree/peerstore_utils.go | 17 +------ p2p/raintree/router.go | 9 ++-- runtime/configs/proto/p2p_config.proto | 2 +- shared/messaging/proto/debug_message.proto | 51 +++++++++---------- 14 files changed, 77 insertions(+), 104 deletions(-) diff --git a/app/client/cli/helpers/common.go b/app/client/cli/helpers/common.go index 647d4241d..320eb7567 100644 --- a/app/client/cli/helpers/common.go +++ b/app/client/cli/helpers/common.go @@ -23,15 +23,14 @@ func FetchPeerstore(cmd *cobra.Command) (types.Peerstore, error) { if err != nil { return nil, err } - // TECHDEBT(#811): use `bus.GetPeerstoreProvider()` after peerstore provider - // is retrievable as a proper submodule - pstoreProvider, err := bus.GetModulesRegistry().GetModule(peerstore_provider.PeerstoreProviderSubmoduleName) + // TECHDEBT(#811): Remove type casting once Peerstore is available as a submodule + pstoreProvider, err := peerstore_provider.GetPeerstoreProvider(bus) if err != nil { - return nil, errors.New("retrieving peerstore provider") + return nil, err } currentHeightProvider := bus.GetCurrentHeightProvider() height := currentHeightProvider.CurrentHeight() - pstore, err := pstoreProvider.(peerstore_provider.PeerstoreProvider).GetStakedPeerstoreAtHeight(height) + pstore, err := pstoreProvider.GetStakedPeerstoreAtHeight(height) if err != nil { return nil, fmt.Errorf("retrieving peerstore at height %d", height) } diff --git a/app/client/cli/peer/list.go b/app/client/cli/peer/list.go index 7c72f7067..b4639627d 100644 --- a/app/client/cli/peer/list.go +++ b/app/client/cli/peer/list.go @@ -49,7 +49,7 @@ func listRunE(cmd *cobra.Command, _ []string) error { } debugMsg := &messaging.DebugMessage{ - Action: messaging.DebugMessageAction_DEBUG_P2P_PEER_LIST, + Action: messaging.DebugMessageAction_DEBUG_P2P_PRINT_PEER_LIST, Type: messaging.DebugMessageRoutingType_DEBUG_MESSAGE_TYPE_BROADCAST, Message: &anypb.Any{ Value: []byte(routerType), @@ -67,13 +67,13 @@ func listRunE(cmd *cobra.Command, _ []string) error { return nil } - // TECHDEBT(#810, #811): will need to wait for DHT bootstrapping to complete before + // TECHDEBT(#811): will need to wait for DHT bootstrapping to complete before // p2p broadcast can be used with to reach unstaked actors. // CONSIDERATION: add the peer commands to the interactive CLI as the P2P module // instance could persist between commands. Other interactive CLI commands which // rely on unstaked actor router broadcast are working as expected. - // TECHDEBT(#810, #811): use broadcast instead to reach all peers. + // TECHDEBT(#811): use broadcast instead to reach all peers. return sendToStakedPeers(cmd, debugMsgAny) } diff --git a/p2p/background/router.go b/p2p/background/router.go index 6d668ec3c..e573c4484 100644 --- a/p2p/background/router.go +++ b/p2p/background/router.go @@ -17,7 +17,6 @@ import ( "github.com/pokt-network/pocket/logger" "github.com/pokt-network/pocket/p2p/config" "github.com/pokt-network/pocket/p2p/protocol" - "github.com/pokt-network/pocket/p2p/providers" "github.com/pokt-network/pocket/p2p/providers/peerstore_provider" typesP2P "github.com/pokt-network/pocket/p2p/types" "github.com/pokt-network/pocket/p2p/unicast" @@ -258,16 +257,11 @@ func (rtr *backgroundRouter) setupDependencies(ctx context.Context, _ *config.Ba } func (rtr *backgroundRouter) setupPeerstore(ctx context.Context) (err error) { - // TECHDEBT(#810, #811): use `bus.GetPeerstoreProvider()` after peerstore provider + // TECHDEBT(#811): use `bus.GetPeerstoreProvider()` after peerstore provider // is retrievable as a proper submodule - pstoreProviderModule, err := rtr.GetBus().GetModulesRegistry(). - GetModule(peerstore_provider.PeerstoreProviderSubmoduleName) + pstoreProvider, err := peerstore_provider.GetPeerstoreProvider(rtr.GetBus()) if err != nil { - return fmt.Errorf("retrieving peerstore provider: %w", err) - } - pstoreProvider, ok := pstoreProviderModule.(providers.PeerstoreProvider) - if !ok { - return fmt.Errorf("unexpected peerstore provider type: %T", pstoreProviderModule) + return err } rtr.logger.Debug().Msg("setupCurrentHeightProvider") diff --git a/p2p/debug.go b/p2p/debug.go index bf15e1d03..1361de5c4 100644 --- a/p2p/debug.go +++ b/p2p/debug.go @@ -10,7 +10,7 @@ import ( func (m *p2pModule) handleDebugMessage(msg *messaging.DebugMessage) error { switch msg.Action { - case messaging.DebugMessageAction_DEBUG_P2P_PEER_LIST: + case messaging.DebugMessageAction_DEBUG_P2P_PRINT_PEER_LIST: if !m.cfg.EnablePeerDiscoveryDebugRpc { return typesP2P.ErrPeerDiscoveryDebugRPCDisabled } diff --git a/p2p/debug/list.go b/p2p/debug/list.go index 21e2455d3..6d18e25ed 100644 --- a/p2p/debug/list.go +++ b/p2p/debug/list.go @@ -21,16 +21,11 @@ func PrintPeerList(bus modules.Bus, routerType RouterType) error { routerPlurality = "" ) - // TECHDEBT(#810, #811): use `bus.GetPeerstoreProvider()` after peerstore provider + // TECHDEBT(#811): use `bus.GetPeerstoreProvider()` after peerstore provider // is retrievable as a proper submodule. - pstoreProviderModule, err := bus.GetModulesRegistry(). - GetModule(peerstore_provider.PeerstoreProviderSubmoduleName) + pstoreProvider, err := peerstore_provider.GetPeerstoreProvider(bus) if err != nil { - return fmt.Errorf("getting peerstore provider: %w", err) - } - pstoreProvider, ok := pstoreProviderModule.(peerstore_provider.PeerstoreProvider) - if !ok { - return fmt.Errorf("unknown peerstore provider type: %T", pstoreProviderModule) + return err } switch routerType { diff --git a/p2p/event_handler.go b/p2p/event_handler.go index 21fbfb9bf..632adeddc 100644 --- a/p2p/event_handler.go +++ b/p2p/event_handler.go @@ -5,6 +5,7 @@ import ( "google.golang.org/protobuf/types/known/anypb" + "github.com/pokt-network/pocket/p2p/providers/peerstore_provider" "github.com/pokt-network/pocket/shared/codec" coreTypes "github.com/pokt-network/pocket/shared/core/types" "github.com/pokt-network/pocket/shared/messaging" @@ -31,7 +32,7 @@ func (m *p2pModule) HandleEvent(event *anypb.Any) error { } oldPeerList := m.stakedActorRouter.GetPeerstore().GetPeerList() - pstoreProvider, err := m.getPeerstoreProvider() + pstoreProvider, err := peerstore_provider.GetPeerstoreProvider(m.GetBus()) if err != nil { return err } diff --git a/p2p/module.go b/p2p/module.go index 6bb8f479a..b4d7eb86c 100644 --- a/p2p/module.go +++ b/p2p/module.go @@ -15,7 +15,6 @@ import ( "github.com/pokt-network/pocket/logger" "github.com/pokt-network/pocket/p2p/background" "github.com/pokt-network/pocket/p2p/config" - "github.com/pokt-network/pocket/p2p/providers" "github.com/pokt-network/pocket/p2p/providers/peerstore_provider" persPSP "github.com/pokt-network/pocket/p2p/providers/peerstore_provider/persistence" "github.com/pokt-network/pocket/p2p/raintree" @@ -291,12 +290,9 @@ func (m *p2pModule) setupDependencies() error { func (m *p2pModule) setupPeerstoreProvider() error { m.logger.Debug().Msg("setupPeerstoreProvider") - // TECHDEBT(#810, #811): use `bus.GetPeerstoreProvider()` after peerstore provider + // TECHDEBT(#811): use `bus.GetPeerstoreProvider()` after peerstore provider // is retrievable as a proper submodule - pstoreProviderModule, err := m.GetBus(). - GetModulesRegistry(). - GetModule(peerstore_provider.PeerstoreProviderSubmoduleName) - if err != nil { + if _, err := peerstore_provider.GetPeerstoreProvider(m.GetBus()); err != nil { // TECHDEBT: compare against `runtime.ErrModuleNotRegistered(...)`. m.logger.Debug().Msg("creating new persistence peerstore...") // Ensure a peerstore provider exists by creating a `persistencePeerstoreProvider`. @@ -307,9 +303,6 @@ func (m *p2pModule) setupPeerstoreProvider() error { } m.logger.Debug().Msg("loaded peerstore provider...") - if _, ok := pstoreProviderModule.(providers.PeerstoreProvider); !ok { - return fmt.Errorf("unknown peerstore provider type: %T", pstoreProviderModule) - } return nil } @@ -517,7 +510,7 @@ func (m *p2pModule) getMultiaddr() (multiaddr.Multiaddr, error) { } func (m *p2pModule) getStakedPeerstore() (typesP2P.Peerstore, error) { - pstoreProvider, err := m.getPeerstoreProvider() + pstoreProvider, err := peerstore_provider.GetPeerstoreProvider(m.GetBus()) if err != nil { return nil, err } @@ -527,21 +520,6 @@ func (m *p2pModule) getStakedPeerstore() (typesP2P.Peerstore, error) { ) } -// TECHDEBT(#810, #811): replace with `bus.GetPeerstoreProvider()` once available. -func (m *p2pModule) getPeerstoreProvider() (peerstore_provider.PeerstoreProvider, error) { - pstoreProviderModule, err := m.GetBus(). - GetModulesRegistry(). - GetModule(peerstore_provider.PeerstoreProviderSubmoduleName) - if err != nil { - return nil, err - } - pstoreProvider, ok := pstoreProviderModule.(peerstore_provider.PeerstoreProvider) - if !ok { - return nil, fmt.Errorf("peerstore provider not available") - } - return pstoreProvider, nil -} - // isStakedActor returns whether the current node is a staked actor at the current height. // Return an error if a peerstore can't be provided. func (m *p2pModule) isStakedActor() (bool, error) { diff --git a/p2p/providers/peerstore_provider/peerstore_provider.go b/p2p/providers/peerstore_provider/peerstore_provider.go index 968c3a2b1..190625aab 100644 --- a/p2p/providers/peerstore_provider/peerstore_provider.go +++ b/p2p/providers/peerstore_provider/peerstore_provider.go @@ -3,6 +3,8 @@ package peerstore_provider //go:generate mockgen -package=mock_types -destination=../../types/mocks/peerstore_provider_mock.go github.com/pokt-network/pocket/p2p/providers/peerstore_provider PeerstoreProvider import ( + "fmt" + "github.com/pokt-network/pocket/logger" typesP2P "github.com/pokt-network/pocket/p2p/types" coreTypes "github.com/pokt-network/pocket/shared/core/types" @@ -64,3 +66,18 @@ func ActorToPeer(abp PeerstoreProvider, actor *coreTypes.Actor) (typesP2P.Peer, return peer, nil } + +// TECHDEBT(#811): use `bus.GetPeerstoreProvider()` after peerstore provider +// is retrievable as a proper submodule. +func GetPeerstoreProvider(bus modules.Bus) (PeerstoreProvider, error) { + pstoreProviderModule, err := bus.GetModulesRegistry(). + GetModule(PeerstoreProviderSubmoduleName) + if err != nil { + return nil, fmt.Errorf("getting peerstore provider: %w", err) + } + pstoreProvider, ok := pstoreProviderModule.(PeerstoreProvider) + if !ok { + return nil, fmt.Errorf("unknown peerstore provider type: %T", pstoreProviderModule) + } + return pstoreProvider, nil +} diff --git a/p2p/providers/peerstore_provider/persistence/provider.go b/p2p/providers/peerstore_provider/persistence/provider.go index 59496921a..1b98bf0c6 100644 --- a/p2p/providers/peerstore_provider/persistence/provider.go +++ b/p2p/providers/peerstore_provider/persistence/provider.go @@ -14,8 +14,10 @@ var ( _ persistencePStoreProviderFactory = &persistencePeerstoreProvider{} ) -type persistencePStoreProviderOption func(*persistencePeerstoreProvider) -type persistencePStoreProviderFactory = modules.FactoryWithOptions[peerstore_provider.PeerstoreProvider, persistencePStoreProviderOption] +type ( + persistencePStoreProviderOption func(*persistencePeerstoreProvider) + persistencePStoreProviderFactory = modules.FactoryWithOptions[peerstore_provider.PeerstoreProvider, persistencePStoreProviderOption] +) type persistencePeerstoreProvider struct { base_modules.IntegrableModule diff --git a/p2p/providers/peerstore_provider/rpc/provider.go b/p2p/providers/peerstore_provider/rpc/provider.go index f119832e7..56d650e1f 100644 --- a/p2p/providers/peerstore_provider/rpc/provider.go +++ b/p2p/providers/peerstore_provider/rpc/provider.go @@ -21,8 +21,10 @@ var ( _ rpcPeerstoreProviderFactory = &rpcPeerstoreProvider{} ) -type rpcPeerstoreProviderOption func(*rpcPeerstoreProvider) -type rpcPeerstoreProviderFactory = modules.FactoryWithOptions[peerstore_provider.PeerstoreProvider, rpcPeerstoreProviderOption] +type ( + rpcPeerstoreProviderOption func(*rpcPeerstoreProvider) + rpcPeerstoreProviderFactory = modules.FactoryWithOptions[peerstore_provider.PeerstoreProvider, rpcPeerstoreProviderOption] +) type rpcPeerstoreProvider struct { base_modules.IntegrableModule @@ -97,7 +99,7 @@ func (rpcPSP *rpcPeerstoreProvider) GetStakedPeerstoreAtCurrentHeight() (typesP2 } func (rpcPSP *rpcPeerstoreProvider) GetUnstakedPeerstore() (typesP2P.Peerstore, error) { - // TECHDEBT(#810, #811): use `bus.GetUnstakedActorRouter()` once it's available. + // TECHDEBT(#811): use `bus.GetUnstakedActorRouter()` once it's available. unstakedActorRouterMod, err := rpcPSP.GetBus().GetModulesRegistry().GetModule(typesP2P.UnstakedActorRouterSubmoduleName) if err != nil { return nil, err diff --git a/p2p/raintree/peerstore_utils.go b/p2p/raintree/peerstore_utils.go index aab31a5ba..ada801a73 100644 --- a/p2p/raintree/peerstore_utils.go +++ b/p2p/raintree/peerstore_utils.go @@ -21,7 +21,7 @@ func (rtr *rainTreeRouter) getPeerstoreSize(level uint32, height uint64) int { peersView, maxNumLevels := rtr.peersManager.getPeersViewWithLevels() // TECHDEBT(#810, 811): use `bus.GetPeerstoreProvider()` instead once available. - pstoreProvider, err := rtr.getPeerstoreProvider() + pstoreProvider, err := peerstore_provider.GetPeerstoreProvider(rtr.GetBus()) if err != nil { // Should never happen; enforced by a `rtr.getPeerstoreProvider()` call // & error handling in `rtr.broadcastAtLevel()`. @@ -41,21 +41,6 @@ func (rtr *rainTreeRouter) getPeerstoreSize(level uint32, height uint64) int { return int(float64(len(peersView.GetAddrs())) * (shrinkageCoefficient)) } -// TECHDEBT(#810, 811): replace with `bus.GetPeerstoreProvider()` once available. -func (rtr *rainTreeRouter) getPeerstoreProvider() (peerstore_provider.PeerstoreProvider, error) { - pstoreProviderModule, err := rtr.GetBus().GetModulesRegistry(). - GetModule(peerstore_provider.PeerstoreProviderSubmoduleName) - if err != nil { - return nil, err - } - - pstoreProvider, ok := pstoreProviderModule.(peerstore_provider.PeerstoreProvider) - if !ok { - return nil, fmt.Errorf("unexpected peerstore provider module type: %T", pstoreProviderModule) - } - return pstoreProvider, nil -} - // getTargetsAtLevel returns the targets for a given level func (rtr *rainTreeRouter) getTargetsAtLevel(level uint32) []target { height := rtr.GetBus().GetCurrentHeightProvider().CurrentHeight() diff --git a/p2p/raintree/router.go b/p2p/raintree/router.go index 90ff95bc4..3d7b9b899 100644 --- a/p2p/raintree/router.go +++ b/p2p/raintree/router.go @@ -9,6 +9,7 @@ import ( "github.com/pokt-network/pocket/logger" "github.com/pokt-network/pocket/p2p/config" "github.com/pokt-network/pocket/p2p/protocol" + "github.com/pokt-network/pocket/p2p/providers/peerstore_provider" typesP2P "github.com/pokt-network/pocket/p2p/types" "github.com/pokt-network/pocket/p2p/unicast" "github.com/pokt-network/pocket/p2p/utils" @@ -67,7 +68,7 @@ func (*rainTreeRouter) Create( bus.RegisterModule(rtr) currentHeightProvider := bus.GetCurrentHeightProvider() - pstoreProvider, err := rtr.getPeerstoreProvider() + pstoreProvider, err := peerstore_provider.GetPeerstoreProvider(bus) if err != nil { return nil, err } @@ -122,10 +123,10 @@ func (rtr *rainTreeRouter) broadcastAtLevel(data []byte, level uint32) error { return err } - // TECHDEBT(#810, #811): remove once `bus.GetPeerstoreProvider()` is available. - // Pre-handling the error from `rtr.getPeerstoreProvider()` before it is called + // TECHDEBT(#811): remove once `bus.GetPeerstoreProvider()` is available. + // Pre-handling the error from `GetPeerstoreProvider()` before it is called // downstream in a context without an error return value. - if _, err = rtr.getPeerstoreProvider(); err != nil { + if _, err = peerstore_provider.GetPeerstoreProvider(rtr.GetBus()); err != nil { return err } diff --git a/runtime/configs/proto/p2p_config.proto b/runtime/configs/proto/p2p_config.proto index 0aeb90c32..1e01d36fb 100644 --- a/runtime/configs/proto/p2p_config.proto +++ b/runtime/configs/proto/p2p_config.proto @@ -14,5 +14,5 @@ message P2PConfig { uint64 max_nonces = 5; // used to limit the number of nonces that can be stored before a FIFO mechanism is used to remove the oldest nonces and make space for the new ones bool is_client_only = 6; // TECHDEBT(bryanchriswhite,olshansky): Re-evaluate if this is still needed string bootstrap_nodes_csv = 7; // string in the format "http://somenode:50832,http://someothernode:50832". Refer to `p2p/module_test.go` for additional details. - bool enable_peer_discovery_debug_rpc = 8; + bool enable_peer_discovery_debug_rpc = 8; // enables a debug endpoint for various operations related to p2p peer discovery } diff --git a/shared/messaging/proto/debug_message.proto b/shared/messaging/proto/debug_message.proto index 55a87c695..aa6553db5 100644 --- a/shared/messaging/proto/debug_message.proto +++ b/shared/messaging/proto/debug_message.proto @@ -7,36 +7,35 @@ import "google/protobuf/any.proto"; option go_package = "github.com/pokt-network/pocket/shared/messaging"; enum DebugMessageAction { - DEBUG_ACTION_UNKNOWN = 0; - - DEBUG_CONSENSUS_RESET_TO_GENESIS = 1; - DEBUG_CONSENSUS_PRINT_NODE_STATE = 2; - DEBUG_CONSENSUS_TRIGGER_NEXT_VIEW = 3; - DEBUG_CONSENSUS_TOGGLE_PACE_MAKER_MODE = 4; // toggle between manual and automatic - - // TODO: Replace `DEBUG_` with `DEBUG_PERSISTENCE_` below for clarity - DEBUG_CONSENSUS_SEND_METADATA_REQ = 5; - DEBUG_CONSENSUS_SEND_BLOCK_REQ = 6; - - DEBUG_SHOW_LATEST_BLOCK_IN_STORE = 7; - - DEBUG_PERSISTENCE_CLEAR_STATE = 8; - DEBUG_PERSISTENCE_RESET_TO_GENESIS = 9; - DEBUG_P2P_PEER_LIST = 10; + DEBUG_ACTION_UNKNOWN = 0; + + DEBUG_CONSENSUS_RESET_TO_GENESIS = 1; + DEBUG_CONSENSUS_PRINT_NODE_STATE = 2; + DEBUG_CONSENSUS_TRIGGER_NEXT_VIEW = 3; + DEBUG_CONSENSUS_TOGGLE_PACE_MAKER_MODE = 4; // toggle between manual and automatic + + // TODO: Replace `DEBUG_` with `DEBUG_PERSISTENCE_` below for clarity + DEBUG_CONSENSUS_SEND_METADATA_REQ = 5; + DEBUG_CONSENSUS_SEND_BLOCK_REQ = 6; + + DEBUG_SHOW_LATEST_BLOCK_IN_STORE = 7; + + DEBUG_PERSISTENCE_CLEAR_STATE = 8; + DEBUG_PERSISTENCE_RESET_TO_GENESIS = 9; + DEBUG_P2P_PRINT_PEER_LIST = 10; } message DebugMessage { - DebugMessageAction action = 1; - DebugMessageRoutingType type = 2; - google.protobuf.Any message = 3; + DebugMessageAction action = 1; + DebugMessageRoutingType type = 2; + google.protobuf.Any message = 3; } // NB: See https://en.wikipedia.org/wiki/Routing for more info on routing and delivery schemes. enum DebugMessageRoutingType { - DEBUG_MESSAGE_TYPE_UNKNOWN = 0; - - DEBUG_MESSAGE_TYPE_ANYCAST = 1; - DEBUG_MESSAGE_TYPE_MULTICAST = 2; - DEBUG_MESSAGE_TYPE_BROADCAST = 3; - DEBUG_MESSAGE_TYPE_UNICAST = 4; -} \ No newline at end of file + DEBUG_MESSAGE_TYPE_UNKNOWN = 0; + DEBUG_MESSAGE_TYPE_ANYCAST = 1; + DEBUG_MESSAGE_TYPE_MULTICAST = 2; + DEBUG_MESSAGE_TYPE_BROADCAST = 3; + DEBUG_MESSAGE_TYPE_UNICAST = 4; +} From 764e17109f49d4e463b721812669fc6a6b5745e9 Mon Sep 17 00:00:00 2001 From: harry <53987565+h5law@users.noreply.github.com> Date: Tue, 25 Jul 2023 14:30:19 +0100 Subject: [PATCH 26/38] clarify unstaked description --- app/client/cli/peer/peer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/client/cli/peer/peer.go b/app/client/cli/peer/peer.go index 725d49a3c..39fe7b8e2 100644 --- a/app/client/cli/peer/peer.go +++ b/app/client/cli/peer/peer.go @@ -22,6 +22,6 @@ var ( func init() { PeerCmd.PersistentFlags().BoolVarP(&allFlag, "all", "a", false, "operations apply to both staked & unstaked router peerstores (default)") PeerCmd.PersistentFlags().BoolVarP(&stakedFlag, "staked", "s", false, "operations only apply to staked router peerstore (i.e. raintree)") - PeerCmd.PersistentFlags().BoolVarP(&unstakedFlag, "unstaked", "u", false, "operations only apply to unstaked router peerstore (i.e. gossipsub)") + PeerCmd.PersistentFlags().BoolVarP(&unstakedFlag, "unstaked", "u", false, "operations only apply to unstaked (including staked as a subset) router peerstore (i.e. gossipsub)") PeerCmd.PersistentFlags().BoolVarP(&localFlag, "local", "l", false, "operations apply to the local (CLI binary's) P2P module instead of being broadcast") } From 7380260a91f71198120d25164204a0a61b089dcc Mon Sep 17 00:00:00 2001 From: harry <53987565+h5law@users.noreply.github.com> Date: Wed, 26 Jul 2023 17:38:41 +0100 Subject: [PATCH 27/38] chore: up retry attempts and wait time, clarify error messages --- p2p/background/router.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/p2p/background/router.go b/p2p/background/router.go index e573c4484..fcdf2d8dd 100644 --- a/p2p/background/router.go +++ b/p2p/background/router.go @@ -35,8 +35,8 @@ var ( // TECHDEBT: Make these values configurable // TECHDEBT: Consider using an exponential backoff instead const ( - connectMaxRetries = 5 - connectRetryTimeout = time.Second * 2 + connectMaxRetries = 7 + connectRetryTimeout = time.Second * 3 ) type backgroundRouterFactory = modules.FactoryWithConfig[typesP2P.Router, *config.BackgroundConfig] @@ -338,13 +338,13 @@ func (rtr *backgroundRouter) bootstrap(ctx context.Context) { peerList := rtr.pstore.GetPeerList() for _, peer := range peerList { if err := utils.AddPeerToLibp2pHost(rtr.host, peer); err != nil { - rtr.logger.Error().Err(err).Msg("adding peer to libp2p host") + rtr.logger.Error().Err(err).Msg("error adding peer to libp2p host") continue } libp2pAddrInfo, err := utils.Libp2pAddrInfoFromPeer(peer) if err != nil { - rtr.logger.Error().Err(err).Msg("converting peer info") + rtr.logger.Error().Err(err).Msg("error converting peer info") continue } @@ -360,7 +360,7 @@ func (rtr *backgroundRouter) bootstrap(ctx context.Context) { "num_peers": len(peerList) - 1, // -1 as includes self }).Msg("connecting to peer") if err := rtr.connectWithRetry(ctx, libp2pAddrInfo); err != nil { - rtr.logger.Error().Err(err).Msg("connecting to bootstrap peer") + rtr.logger.Error().Err(err).Msg("error connecting to bootstrap peer") continue } } @@ -376,11 +376,11 @@ func (rtr *backgroundRouter) connectWithRetry(ctx context.Context, libp2pAddrInf return nil } - rtr.logger.Error().Msgf("Failed to connect (attempt %d), retrying in %v...\n", i+1, connectRetryTimeout) + rtr.logger.Error().Msgf("failed to connect (attempt %d), retrying in %v...", i+1, connectRetryTimeout) time.Sleep(connectRetryTimeout) } - return fmt.Errorf("failed to connect after %d attempts, last error: %w", 5, err) + return fmt.Errorf("failed to connect after %d attempts, last error: %w", connectMaxRetries, err) } // topicValidator is used in conjunction with libp2p-pubsub's notion of "topic From da62de1e3a46735a0cb955d524bc090954d06dd9 Mon Sep 17 00:00:00 2001 From: Daniel Olshansky Date: Wed, 26 Jul 2023 15:57:21 -0700 Subject: [PATCH 28/38] Fixed case messaging.DebugMessageEventType --- shared/messaging/proto/debug_message.proto | 5 ++--- shared/node.go | 11 +++++------ 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/shared/messaging/proto/debug_message.proto b/shared/messaging/proto/debug_message.proto index aa6553db5..eebaaa11a 100644 --- a/shared/messaging/proto/debug_message.proto +++ b/shared/messaging/proto/debug_message.proto @@ -14,14 +14,13 @@ enum DebugMessageAction { DEBUG_CONSENSUS_TRIGGER_NEXT_VIEW = 3; DEBUG_CONSENSUS_TOGGLE_PACE_MAKER_MODE = 4; // toggle between manual and automatic - // TODO: Replace `DEBUG_` with `DEBUG_PERSISTENCE_` below for clarity DEBUG_CONSENSUS_SEND_METADATA_REQ = 5; DEBUG_CONSENSUS_SEND_BLOCK_REQ = 6; - DEBUG_SHOW_LATEST_BLOCK_IN_STORE = 7; - + DEBUG_SHOW_LATEST_BLOCK_IN_STORE = 7; // TODO: Replace `DEBUG_` with `DEBUG_PERSISTENCE_` DEBUG_PERSISTENCE_CLEAR_STATE = 8; DEBUG_PERSISTENCE_RESET_TO_GENESIS = 9; + DEBUG_P2P_PRINT_PEER_LIST = 10; } diff --git a/shared/node.go b/shared/node.go index b90130c94..df08f06f3 100644 --- a/shared/node.go +++ b/shared/node.go @@ -181,12 +181,7 @@ func (node *Node) handleEvent(message *messaging.PocketEnvelope) error { case messaging.TxGossipMessageContentType: return node.GetBus().GetUtilityModule().HandleUtilityMessage(message.Content) case messaging.DebugMessageEventType: - if err := node.GetBus().GetP2PModule().HandleEvent(message.Content); err != nil { - return err - } - if err := node.handleDebugMessage(message); err != nil { - return err - } + return node.handleDebugMessage(message) case messaging.ConsensusNewHeightEventType: err_p2p := node.GetBus().GetP2PModule().HandleEvent(message.Content) err_ibc := node.GetBus().GetIBCModule().HandleEvent(message.Content) @@ -210,6 +205,7 @@ func (node *Node) handleDebugMessage(message *messaging.PocketEnvelope) error { return err } switch debugMessage.Action { + // Consensus Debug case messaging.DebugMessageAction_DEBUG_CONSENSUS_RESET_TO_GENESIS, messaging.DebugMessageAction_DEBUG_CONSENSUS_PRINT_NODE_STATE, messaging.DebugMessageAction_DEBUG_CONSENSUS_TRIGGER_NEXT_VIEW, @@ -217,6 +213,9 @@ func (node *Node) handleDebugMessage(message *messaging.PocketEnvelope) error { messaging.DebugMessageAction_DEBUG_CONSENSUS_SEND_BLOCK_REQ, messaging.DebugMessageAction_DEBUG_CONSENSUS_SEND_METADATA_REQ: return node.GetBus().GetConsensusModule().HandleDebugMessage(debugMessage) + // P2P Debug + case messaging.DebugMessageAction_DEBUG_P2P_PRINT_PEER_LIST: + return node.GetBus().GetP2PModule().HandleEvent(message.Content) // Persistence Debug case messaging.DebugMessageAction_DEBUG_SHOW_LATEST_BLOCK_IN_STORE: return node.GetBus().GetPersistenceModule().HandleDebugMessage(debugMessage) From 12e22e1dbd826818fa9a1d706c483df1b4fb6acf Mon Sep 17 00:00:00 2001 From: harry <53987565+h5law@users.noreply.github.com> Date: Thu, 27 Jul 2023 11:01:08 +0100 Subject: [PATCH 29/38] clairfy error log messages --- app/client/cli/peer/list.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/app/client/cli/peer/list.go b/app/client/cli/peer/list.go index b4639627d..85a7bb623 100644 --- a/app/client/cli/peer/list.go +++ b/app/client/cli/peer/list.go @@ -57,12 +57,12 @@ func listRunE(cmd *cobra.Command, _ []string) error { } debugMsgAny, err := anypb.New(debugMsg) if err != nil { - return fmt.Errorf("creating anypb from debug message: %w", err) + return fmt.Errorf("error creating anypb from debug message: %w", err) } if localFlag { if err := debug.PrintPeerList(bus, routerType); err != nil { - return fmt.Errorf("printing peer list: %w", err) + return fmt.Errorf("error printing peer list: %w", err) } return nil } @@ -85,16 +85,16 @@ func sendToStakedPeers(cmd *cobra.Command, debugMsgAny *anypb.Any) error { pstore, err := helpers.FetchPeerstore(cmd) if err != nil { - logger.Global.Fatal().Err(err).Msg("Unable to retrieve the pstore") + logger.Global.Fatal().Err(err).Msg("unable to retrieve the pstore") } if pstore.Size() == 0 { - logger.Global.Fatal().Msg("No validators found") + logger.Global.Fatal().Msg("no validators found") } for _, peer := range pstore.GetPeerList() { if err := bus.GetP2PModule().Send(peer.GetAddress(), debugMsgAny); err != nil { - logger.Global.Error().Err(err).Msg("Failed to send debug message") + logger.Global.Error().Err(err).Msg("failed to send debug message") } } return nil From f8f5da54ebd8ebed9c5e4cfb037db5e95ec25fb6 Mon Sep 17 00:00:00 2001 From: harry <53987565+h5law@users.noreply.github.com> Date: Thu, 27 Jul 2023 11:13:12 +0100 Subject: [PATCH 30/38] fix waitgroup recovery in tests --- p2p/background/router_test.go | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/p2p/background/router_test.go b/p2p/background/router_test.go index b99d7e4c1..0c3e2174d 100644 --- a/p2p/background/router_test.go +++ b/p2p/background/router_test.go @@ -312,9 +312,21 @@ func TestBackgroundRouter_Broadcast(t *testing.T) { // something similar to avoid this issue. defer func() { if err := recover(); err != nil { - if err.(error).Error() == "sync: negative WaitGroup counter" { + var ok bool + var er error + var strErr string + er, ok = err.(error) + if !ok { + strErr, ok = err.(string) + if !ok { + t.Fatal(err) + } + } + if er != nil && er.Error() == "sync: negative WaitGroup counter" { // ignore negative WaitGroup counter error return + } else if strErr == "sync: negative WaitGroup counter" { + return } // fail the test for anything else; converting the panic into // test failure allows the test to run with the `-count` flag From 66a134705aa9c6b622aba9a3624c5db9f05e10a2 Mon Sep 17 00:00:00 2001 From: harry <53987565+h5law@users.noreply.github.com> Date: Thu, 27 Jul 2023 12:15:23 +0100 Subject: [PATCH 31/38] chore: move comment line --- p2p/background/router_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/p2p/background/router_test.go b/p2p/background/router_test.go index 0c3e2174d..8bcce4ebf 100644 --- a/p2p/background/router_test.go +++ b/p2p/background/router_test.go @@ -322,8 +322,8 @@ func TestBackgroundRouter_Broadcast(t *testing.T) { t.Fatal(err) } } + // ignore negative WaitGroup counter error if er != nil && er.Error() == "sync: negative WaitGroup counter" { - // ignore negative WaitGroup counter error return } else if strErr == "sync: negative WaitGroup counter" { return From 870805f32720fa18ba2da3583ef0983e194943cb Mon Sep 17 00:00:00 2001 From: harry <53987565+h5law@users.noreply.github.com> Date: Thu, 27 Jul 2023 12:49:34 +0100 Subject: [PATCH 32/38] add NewListCommand function to fit CLI pattern --- app/client/cli/peer/list.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/app/client/cli/peer/list.go b/app/client/cli/peer/list.go index 85a7bb623..e9c8247ed 100644 --- a/app/client/cli/peer/list.go +++ b/app/client/cli/peer/list.go @@ -12,20 +12,20 @@ import ( "github.com/pokt-network/pocket/shared/messaging" ) -var ( - listCmd = &cobra.Command{ +var ErrRouterType = fmt.Errorf("must specify one of --staked, --unstaked, or --all") + +func init() { + PeerCmd.AddCommand(NewListCommand()) +} + +func NewListCommand() *cobra.Command { + return &cobra.Command{ Use: "List", Short: "List the known peers", Long: "Prints a table of the Peer ID, Pokt Address and Service URL of the known peers", Aliases: []string{"list", "ls"}, RunE: listRunE, } - - ErrRouterType = fmt.Errorf("must specify one of --staked, --unstaked, or --all") -) - -func init() { - PeerCmd.AddCommand(listCmd) } func listRunE(cmd *cobra.Command, _ []string) error { From 64ec990ab388159a38b380bc2bc5307283433fae Mon Sep 17 00:00:00 2001 From: harry <53987565+h5law@users.noreply.github.com> Date: Thu, 27 Jul 2023 12:53:04 +0100 Subject: [PATCH 33/38] clarify errors --- p2p/debug/list.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/p2p/debug/list.go b/p2p/debug/list.go index 6d18e25ed..5ba9e8c25 100644 --- a/p2p/debug/list.go +++ b/p2p/debug/list.go @@ -34,14 +34,14 @@ func PrintPeerList(bus modules.Bus, routerType RouterType) error { // if !isStaked ... pstore, err := pstoreProvider.GetStakedPeerstoreAtCurrentHeight() if err != nil { - return fmt.Errorf("getting unstaked peerstore: %v", err) + return fmt.Errorf("error getting staked peerstore: %v", err) } peers = pstore.GetPeerList() case UnstakedRouterType: pstore, err := pstoreProvider.GetUnstakedPeerstore() if err != nil { - return fmt.Errorf("getting unstaked peerstore: %v", err) + return fmt.Errorf("error getting unstaked peerstore: %v", err) } peers = pstore.GetPeerList() @@ -52,11 +52,11 @@ func PrintPeerList(bus modules.Bus, routerType RouterType) error { // if !isStaked ... stakedPStore, err := pstoreProvider.GetStakedPeerstoreAtCurrentHeight() if err != nil { - return fmt.Errorf("getting unstaked peerstore: %v", err) + return fmt.Errorf("error getting staked peerstore: %v", err) } unstakedPStore, err := pstoreProvider.GetUnstakedPeerstore() if err != nil { - return fmt.Errorf("getting unstaked peerstore: %v", err) + return fmt.Errorf("error getting unstaked peerstore: %v", err) } unstakedPeers := unstakedPStore.GetPeerList() @@ -113,7 +113,7 @@ func peerListRowConsumerFactory(peers types.PeerList) utils.RowConsumer { for _, peer := range peers { libp2pAddrInfo, err := utils.Libp2pAddrInfoFromPeer(peer) if err != nil { - return fmt.Errorf("converting peer to libp2p addr info: %w", err) + return fmt.Errorf("error converting peer to libp2p addr info: %w", err) } err = provideRow( From ccec195922e62804b50745ac7e8e770b8f8cd22a Mon Sep 17 00:00:00 2001 From: harry <53987565+h5law@users.noreply.github.com> Date: Thu, 27 Jul 2023 18:32:45 +0100 Subject: [PATCH 34/38] remove unneeded error casting as sync panics a string --- p2p/background/router_test.go | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/p2p/background/router_test.go b/p2p/background/router_test.go index 8bcce4ebf..ef03e6cd7 100644 --- a/p2p/background/router_test.go +++ b/p2p/background/router_test.go @@ -311,21 +311,9 @@ func TestBackgroundRouter_Broadcast(t *testing.T) { // This test should be redesigned using atomic counters or // something similar to avoid this issue. defer func() { - if err := recover(); err != nil { - var ok bool - var er error - var strErr string - er, ok = err.(error) - if !ok { - strErr, ok = err.(string) - if !ok { - t.Fatal(err) - } - } + if rcv := recover(); err != nil { // ignore negative WaitGroup counter error - if er != nil && er.Error() == "sync: negative WaitGroup counter" { - return - } else if strErr == "sync: negative WaitGroup counter" { + if msg, ok := rcv.(string); ok && msg == "sync: negative WaitGroup counter" { return } // fail the test for anything else; converting the panic into From 90385f0fc90b6dea76b1b0bd7f16200cd20761c3 Mon Sep 17 00:00:00 2001 From: harry <53987565+h5law@users.noreply.github.com> Date: Fri, 28 Jul 2023 11:29:24 +0100 Subject: [PATCH 35/38] tidy cli --- app/client/cli/cmd.go | 4 +++- app/client/cli/peer/list.go | 4 ---- app/client/cli/peer/peer.go | 48 +++++++++++++++++++++++++++++-------- 3 files changed, 41 insertions(+), 15 deletions(-) diff --git a/app/client/cli/cmd.go b/app/client/cli/cmd.go index 95fc9b70f..3df36108f 100644 --- a/app/client/cli/cmd.go +++ b/app/client/cli/cmd.go @@ -38,7 +38,9 @@ func init() { log.Fatalf(flagBindErrFormat, "verbose", err) } - rootCmd.AddCommand(peer.PeerCmd) + // Add subdir commands + // DISCUSS: Should we put the peer commands as the other commands are so we dont have to do this? + rootCmd.AddCommand(peer.NewPeerCommand()) } var rootCmd = &cobra.Command{ diff --git a/app/client/cli/peer/list.go b/app/client/cli/peer/list.go index e9c8247ed..b8cbe03eb 100644 --- a/app/client/cli/peer/list.go +++ b/app/client/cli/peer/list.go @@ -14,10 +14,6 @@ import ( var ErrRouterType = fmt.Errorf("must specify one of --staked, --unstaked, or --all") -func init() { - PeerCmd.AddCommand(NewListCommand()) -} - func NewListCommand() *cobra.Command { return &cobra.Command{ Use: "List", diff --git a/app/client/cli/peer/peer.go b/app/client/cli/peer/peer.go index 39fe7b8e2..e5fc01bfe 100644 --- a/app/client/cli/peer/peer.go +++ b/app/client/cli/peer/peer.go @@ -6,22 +6,50 @@ import ( "github.com/pokt-network/pocket/app/client/cli/helpers" ) -var ( - allFlag, +var allFlag, stakedFlag, unstakedFlag, localFlag bool - PeerCmd = &cobra.Command{ - Use: "peer", +func NewPeerCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "Peer", Short: "Manage peers", + Aliases: []string{"peer"}, PersistentPreRunE: helpers.P2PDependenciesPreRunE, } -) -func init() { - PeerCmd.PersistentFlags().BoolVarP(&allFlag, "all", "a", false, "operations apply to both staked & unstaked router peerstores (default)") - PeerCmd.PersistentFlags().BoolVarP(&stakedFlag, "staked", "s", false, "operations only apply to staked router peerstore (i.e. raintree)") - PeerCmd.PersistentFlags().BoolVarP(&unstakedFlag, "unstaked", "u", false, "operations only apply to unstaked (including staked as a subset) router peerstore (i.e. gossipsub)") - PeerCmd.PersistentFlags().BoolVarP(&localFlag, "local", "l", false, "operations apply to the local (CLI binary's) P2P module instead of being broadcast") + cmd.PersistentFlags(). + BoolVarP( + &allFlag, + "all", "a", + false, + "operations apply to both staked & unstaked router peerstores (default)", + ) + cmd.PersistentFlags(). + BoolVarP( + &stakedFlag, + "staked", "s", + false, + "operations only apply to staked router peerstore (i.e. raintree)", + ) + cmd.PersistentFlags(). + BoolVarP( + &unstakedFlag, + "unstaked", "u", + false, + "operations only apply to unstaked (including staked as a subset) router peerstore (i.e. gossipsub)", + ) + cmd.PersistentFlags(). + BoolVarP( + &localFlag, + "local", "l", + false, + "operations apply to the local (CLI binary's) P2P module instead of being broadcast", + ) + + // Add subcommands + cmd.AddCommand(NewListCommand()) + + return cmd } From 6d0d300d60b373d4b4518739d82a282968aaa9ef Mon Sep 17 00:00:00 2001 From: harry <53987565+h5law@users.noreply.github.com> Date: Fri, 28 Jul 2023 11:52:07 +0100 Subject: [PATCH 36/38] remove #810 from comment as merged --- p2p/raintree/peerstore_utils.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/p2p/raintree/peerstore_utils.go b/p2p/raintree/peerstore_utils.go index ada801a73..e0fbf8a77 100644 --- a/p2p/raintree/peerstore_utils.go +++ b/p2p/raintree/peerstore_utils.go @@ -20,7 +20,7 @@ const ( func (rtr *rainTreeRouter) getPeerstoreSize(level uint32, height uint64) int { peersView, maxNumLevels := rtr.peersManager.getPeersViewWithLevels() - // TECHDEBT(#810, 811): use `bus.GetPeerstoreProvider()` instead once available. + // TECHDEBT(#811): use `bus.GetPeerstoreProvider()` instead once available. pstoreProvider, err := peerstore_provider.GetPeerstoreProvider(rtr.GetBus()) if err != nil { // Should never happen; enforced by a `rtr.getPeerstoreProvider()` call From c6488a5b2a280703ba80ab8448cee14d85743aaa Mon Sep 17 00:00:00 2001 From: harry <53987565+h5law@users.noreply.github.com> Date: Fri, 28 Jul 2023 12:11:08 +0100 Subject: [PATCH 37/38] shortern retries --- p2p/background/router.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/p2p/background/router.go b/p2p/background/router.go index fcdf2d8dd..35fd002ce 100644 --- a/p2p/background/router.go +++ b/p2p/background/router.go @@ -35,8 +35,8 @@ var ( // TECHDEBT: Make these values configurable // TECHDEBT: Consider using an exponential backoff instead const ( - connectMaxRetries = 7 - connectRetryTimeout = time.Second * 3 + connectMaxRetries = 5 + connectRetryTimeout = time.Second * 2 ) type backgroundRouterFactory = modules.FactoryWithConfig[typesP2P.Router, *config.BackgroundConfig] From 2317d42c0a6efe1869c7a8273968d3fd369c655c Mon Sep 17 00:00:00 2001 From: harry <53987565+h5law@users.noreply.github.com> Date: Mon, 31 Jul 2023 14:23:30 +0100 Subject: [PATCH 38/38] bug: comment out buggy lines --- charts/pocket/templates/statefulset.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/charts/pocket/templates/statefulset.yaml b/charts/pocket/templates/statefulset.yaml index 782d69a94..39eaa7d9c 100644 --- a/charts/pocket/templates/statefulset.yaml +++ b/charts/pocket/templates/statefulset.yaml @@ -102,10 +102,10 @@ spec: fieldPath: status.podIP - name: POCKET_P2P_ENABLE_PEER_DISCOVERY_DEBUG_RPC value: "true" - livenessProbe: - httpGet: - path: /v1/health - port: rpc +# livenessProbe: +# httpGet: +# path: /v1/health +# port: rpc readinessProbe: httpGet: path: /v1/health