From 4765b3a07425e642b2bd78be70baa31a87343565 Mon Sep 17 00:00:00 2001 From: shakezula Date: Thu, 20 Jul 2023 15:45:56 -0600 Subject: [PATCH] [wip] gets destination backup working for KVStore --- persistence/blockstore/block_store.go | 4 ++ persistence/kvstore/kvstore.go | 30 ++++++++++++ persistence/kvstore/kvstore_test.go | 59 +++++++++++++++++++++++ persistence/module.go | 2 +- persistence/trees/atomic_test.go | 69 ++++++++++++++++++++++++++- persistence/trees/module.go | 7 +++ persistence/trees/module_test.go | 17 +++---- persistence/trees/trees.go | 39 +++++++++++++-- persistence/trees/trees_test.go | 4 -- 9 files changed, 211 insertions(+), 20 deletions(-) diff --git a/persistence/blockstore/block_store.go b/persistence/blockstore/block_store.go index 647d3e634..360804a33 100644 --- a/persistence/blockstore/block_store.go +++ b/persistence/blockstore/block_store.go @@ -93,6 +93,10 @@ func (bs *blockStore) Stop() error { return bs.kv.Stop() } +func (bs *blockStore) Backup(path string) error { + return bs.kv.Backup(path) +} + func (bs *blockStore) Delete(key []byte) error { return bs.kv.Delete(key) } func (bs *blockStore) Exists(key []byte) (bool, error) { return bs.kv.Exists(key) } func (bs *blockStore) GetAll(prefixKey []byte, descending bool) (keys, values [][]byte, err error) { diff --git a/persistence/kvstore/kvstore.go b/persistence/kvstore/kvstore.go index a352e4257..547c72e4f 100644 --- a/persistence/kvstore/kvstore.go +++ b/persistence/kvstore/kvstore.go @@ -4,7 +4,10 @@ package kvstore import ( "errors" + "fmt" "log" + "os" + "path/filepath" badger "github.com/dgraph-io/badger/v3" "github.com/pokt-network/smt" @@ -22,6 +25,8 @@ type KVStore interface { GetAll(prefixKey []byte, descending bool) (keys, values [][]byte, err error) Exists(key []byte) (bool, error) ClearAll() error + + Backup(filepath string) error } const ( @@ -141,6 +146,31 @@ func (store *badgerKVStore) Stop() error { return store.db.Close() } +// Backup creates a backup for the badgerDB at the provided path. +// It creates a file +func (store *badgerKVStore) Backup(backupPath string) error { + // create backup directory if it doesn't exist + if err := os.MkdirAll(filepath.Dir(backupPath), os.ModePerm); err != nil { + return fmt.Errorf("failed to create backup directory: %v", err) + } + + // create the backup file itself + backupFile, err := os.Create(backupPath) + if err != nil { + return fmt.Errorf("failed to create backup file: %v", err) + } + defer backupFile.Close() + + // dump the database to the backup file + // CONSIDER: ??? err := store.db.NewStream().Backup(backupFile, 0) // What's the difference here? + _, err = store.db.Backup(backupFile, 0) + if err != nil { + return err + } + + return nil +} + // PrefixEndBytes returns the end byteslice for a noninclusive range // that would include all byte slices for which the input is the prefix func prefixEndBytes(prefix []byte) []byte { diff --git a/persistence/kvstore/kvstore_test.go b/persistence/kvstore/kvstore_test.go index 402ce4a98..760db8957 100644 --- a/persistence/kvstore/kvstore_test.go +++ b/persistence/kvstore/kvstore_test.go @@ -2,6 +2,10 @@ package kvstore import ( "encoding/hex" + "io" + "io/ioutil" + "os" + "path/filepath" "strings" "testing" @@ -330,6 +334,46 @@ func TestKVStore_ClearAll(t *testing.T) { require.NoError(t, err) } +func TestKVStore_Backup(t *testing.T) { + t.Run("should backup an in-memory database", func(t *testing.T) { + store := NewMemKVStore() + require.NotNil(t, store) + + tmpdir := t.TempDir() + path := filepath.Join(tmpdir, "TestKVStore_Backup_InMemory.bak") + require.NoError(t, store.Backup(path)) + + empty, err := isEmpty(t, tmpdir) + require.NoError(t, err) + require.False(t, empty) + + // assert on individual files + files, err := ioutil.ReadDir(tmpdir) + require.NoError(t, err) + require.Equal(t, len(files), 1) + }) + t.Run("should backup an on-disk store database", func(t *testing.T) { + tmpdir := t.TempDir() + kvpath := filepath.Join(tmpdir, "TestKVStore_Backup_OnDisk_Source.bak") + store, err := NewKVStore(kvpath) + require.NoError(t, err) + require.NotNil(t, store) + + backupDir := t.TempDir() + path := filepath.Join(backupDir, "TestKVStore_Backup_OnDisk_Destination.bak") + require.NoError(t, store.Backup(path)) + + empty, err := isEmpty(t, backupDir) + require.NoError(t, err) + require.False(t, empty) + + // assert on individual files + files, err := ioutil.ReadDir(backupDir) + require.NoError(t, err) + require.Equal(t, len(files), 1) + }) +} + func setupStore(t *testing.T, store KVStore) { t.Helper() err := store.Set([]byte("foo"), []byte("bar")) @@ -337,3 +381,18 @@ func setupStore(t *testing.T, store KVStore) { err = store.Set([]byte("baz"), []byte("bin")) require.NoError(t, err) } + +func isEmpty(t *testing.T, dir string) (bool, error) { + t.Helper() + f, err := os.Open(dir) + if err != nil { + return false, err + } + defer f.Close() + + _, err = f.Readdirnames(1) + if err == io.EOF { + return true, nil + } + return false, err +} diff --git a/persistence/module.go b/persistence/module.go index 0d6acd17f..d16c27a6f 100644 --- a/persistence/module.go +++ b/persistence/module.go @@ -104,7 +104,7 @@ func (*persistenceModule) Create(bus modules.Bus, options ...modules.ModuleOptio trees.WithTreeStoreDirectory(persistenceCfg.TreesStoreDir), trees.WithLogger(m.logger)) if err != nil { - return nil, fmt.Errorf("failed to create TreeStoreModule: %w", err) + return nil, fmt.Errorf("failed to create %s: %w", modules.TreeStoreSubmoduleName, err) } m.config = persistenceCfg diff --git a/persistence/trees/atomic_test.go b/persistence/trees/atomic_test.go index d7a617a76..17eeddc2a 100644 --- a/persistence/trees/atomic_test.go +++ b/persistence/trees/atomic_test.go @@ -3,6 +3,8 @@ package trees import ( "encoding/hex" "fmt" + "io" + "os" "testing" "github.com/pokt-network/pocket/logger" @@ -148,8 +150,6 @@ func TestTreeStore_SaveAndLoad(t *testing.T) { logger: logger.Global.CreateLoggerForModule(modules.TreeStoreSubmoduleName), treeStoreDir: tmpDir, } - // TODO IN THIS COMMIT do we need to start this treestore? - // require.NoError(t, ts2.Start()) // Load sets a tree store to the provided worldstate err = ts2.Load(w) @@ -160,3 +160,68 @@ func TestTreeStore_SaveAndLoad(t *testing.T) { // Assert that hash is unchanged from save and load require.Equal(t, hash1, hash2) } + +func TestTreeStore_SaveBackup(t *testing.T) { + ts := newTestTreeStore(t) + tmpdir := t.TempDir() + // assert that the directory is empty before backup + ok, err := isEmpty(tmpdir) + require.NoError(t, err) + require.True(t, ok) + + // Trigger a backup + require.NoError(t, ts.Backup(tmpdir)) + + // assert that the directory is not empty after Backup has returned + ok, err = isEmpty(tmpdir) + require.NoError(t, err) + require.False(t, ok) +} + +// creates a new tree store with a tmp directory for nodestore persistence +// and then starts the tree store and returns its pointer. +func newTestTreeStore(t *testing.T) *treeStore { + ctrl := gomock.NewController(t) + tmpDir := t.TempDir() + + mockTxIndexer := mock_types.NewMockTxIndexer(ctrl) + mockBus := mockModules.NewMockBus(ctrl) + mockPersistenceMod := mockModules.NewMockPersistenceModule(ctrl) + + mockBus.EXPECT().GetPersistenceModule().AnyTimes().Return(mockPersistenceMod) + mockPersistenceMod.EXPECT().GetTxIndexer().AnyTimes().Return(mockTxIndexer) + + ts := &treeStore{ + logger: &zerolog.Logger{}, + treeStoreDir: tmpDir, + } + require.NoError(t, ts.Start()) + require.NotNil(t, ts.rootTree.tree) + + for _, treeName := range stateTreeNames { + err := ts.merkleTrees[treeName].tree.Update([]byte("foo"), []byte("bar")) + require.NoError(t, err) + } + + err := ts.Commit() + require.NoError(t, err) + + hash1 := ts.getStateHash() + require.NotEmpty(t, hash1) + + return ts +} + +func isEmpty(dir string) (bool, error) { + f, err := os.Open(dir) + if err != nil { + return false, err + } + defer f.Close() + + _, err = f.Readdirnames(1) // Or f.Readdir(1) + if err == io.EOF { + return true, nil + } + return false, err // Either not empty or error, suits both cases +} diff --git a/persistence/trees/module.go b/persistence/trees/module.go index abb3ee306..285070e48 100644 --- a/persistence/trees/module.go +++ b/persistence/trees/module.go @@ -12,6 +12,8 @@ import ( var _ modules.TreeStoreModule = &treeStore{} +// Create returns a TreeStoreSubmodule that has been setup with the provided TreeStoreOptions, started, +// and then registered to the bus. func (*treeStore) Create(bus modules.Bus, options ...modules.TreeStoreOption) (modules.TreeStoreModule, error) { m := &treeStore{} @@ -19,6 +21,10 @@ func (*treeStore) Create(bus modules.Bus, options ...modules.TreeStoreOption) (m option(m) } + if err := m.Start(); err != nil { + return nil, fmt.Errorf("failed to start %s: %w", modules.TreeStoreSubmoduleName, err) + } + bus.RegisterModule(m) return m, nil @@ -71,6 +77,7 @@ func (t *treeStore) Stop() error { func (t *treeStore) GetModuleName() string { return modules.TreeStoreSubmoduleName } +// setupTrees is called by Start and it loads the treestore at the given directory func (t *treeStore) setupTrees() error { if t.treeStoreDir == ":memory:" { return t.setupInMemory() diff --git a/persistence/trees/module_test.go b/persistence/trees/module_test.go index 6cfce7bac..18fe3875c 100644 --- a/persistence/trees/module_test.go +++ b/persistence/trees/module_test.go @@ -48,7 +48,12 @@ func TestTreeStore_Create(t *testing.T) { treemod, err := trees.Create(mockBus, trees.WithTreeStoreDirectory(":memory:")) assert.NoError(t, err) - require.NoError(t, treemod.Start()) + // Create should setup a value for each tree + for _, v := range stateTreeNames { + root, ns := treemod.GetTree(v) + require.NotEmpty(t, root) + require.NotEmpty(t, ns) + } got := treemod.GetBus() assert.Equal(t, got, mockBus) @@ -68,21 +73,13 @@ func TestTreeStore_StartAndStop(t *testing.T) { mockRuntimeMgr := mockModules.NewMockRuntimeMgr(ctrl) mockBus := createMockBus(t, mockRuntimeMgr) + // Create returns a started TreeStoreSubmodule treemod, err := trees.Create( mockBus, trees.WithTreeStoreDirectory(":memory:"), trees.WithLogger(&zerolog.Logger{})) assert.NoError(t, err) - // GetTree should return nil for each tree if Start has not been called - for _, v := range stateTreeNames { - root, ns := treemod.GetTree(v) - require.Empty(t, root) - require.Empty(t, ns) - } - // Should start without error - require.NoError(t, treemod.Start()) - // Should stop without error require.NoError(t, treemod.Stop()) diff --git a/persistence/trees/trees.go b/persistence/trees/trees.go index 9807eada1..75bc354a5 100644 --- a/persistence/trees/trees.go +++ b/persistence/trees/trees.go @@ -18,6 +18,7 @@ import ( "fmt" "hash" "log" + "path/filepath" "github.com/jackc/pgx/v5" "github.com/pokt-network/pocket/persistence/indexer" @@ -29,6 +30,7 @@ import ( "github.com/pokt-network/pocket/shared/modules" "github.com/pokt-network/pocket/shared/modules/base_modules" "github.com/pokt-network/smt" + "go.uber.org/multierr" ) // smtTreeHasher sets the hasher used by the tree SMT trees @@ -320,7 +322,7 @@ func (t *treeStore) Rollback() error { } // Load sets the TreeStore merkle and root trees to the values provided in the worldstate -func (t *treeStore) Load(w *Worldstate) error { +func (t *treeStore) Load(w *worldstate) error { t.merkleTrees = make(map[string]*stateTree) // import root tree @@ -363,16 +365,47 @@ func (t *treeStore) save() (*worldstate, error) { w := &worldstate{ TreeStoreDir: t.treeStoreDir, MerkleRoots: make(map[string][]byte), + MerkleTrees: make(map[string]*stateTree), RootHash: t.rootTree.tree.Root(), + RootTree: t.rootTree, } - for treeName, tree := range t.merkleTrees { - w.MerkleRoots[treeName] = tree.tree.Root() + for treeName := range t.merkleTrees { + root, nodeStore := t.GetTree(treeName) + tree := smt.ImportSparseMerkleTree(nodeStore, smtTreeHasher, root) + w.MerkleTrees[treeName] = &stateTree{ + name: treeName, + tree: tree, + nodeStore: nodeStore, + } + w.MerkleRoots[treeName] = tree.Root() + } + + root, nodeStore := t.GetTree(RootTreeName) + tree := smt.ImportSparseMerkleTree(nodeStore, smtTreeHasher, root) + w.RootTree = &stateTree{ + name: RootTreeName, + tree: tree, + nodeStore: nodeStore, } return w, nil } +// Backup creates a new backup of each tree in the tree store to the provided directory. +// Each tree is backed up in an eponymous file in the provided backupDir. +func (t *treeStore) Backup(backupDir string) error { + errs := []error{} + for _, st := range t.merkleTrees { + treePath := filepath.Join(backupDir, st.name) + if err := st.nodeStore.Backup(treePath); err != nil { + t.logger.Err(err).Msgf("failed to backup %s tree: %+v", st.name, err) + errs = append(errs, err) + } + } + return multierr.Combine(errs...) +} + //////////////////////// // Actor Tree Helpers // //////////////////////// diff --git a/persistence/trees/trees_test.go b/persistence/trees/trees_test.go index 8a196ad4d..14e823d4b 100644 --- a/persistence/trees/trees_test.go +++ b/persistence/trees/trees_test.go @@ -49,8 +49,6 @@ func TestTreeStore_Update(t *testing.T) { pmod := newTestPersistenceModule(t, dbUrl) context := newTestPostgresContext(t, 0, pmod) - require.NoError(t, context.SetSavePoint()) - hash1, err := context.ComputeStateHash() require.NoError(t, err) require.NotEmpty(t, hash1) @@ -59,8 +57,6 @@ func TestTreeStore_Update(t *testing.T) { _, err = createAndInsertDefaultTestApp(t, context) require.NoError(t, err) - require.NoError(t, context.SetSavePoint()) - hash2, err := context.ComputeStateHash() require.NoError(t, err) require.NotEmpty(t, hash2)