Snapshot integration p4

core/state/database.go

@@ -186,9 +186,18 @@ func (db *CachingDB) Reader(stateRoot common.Hash) (Reader, error) {
  // is optional and may be partially useful if it's not fully
  // generated.
  if db.snap != nil {
- sr, err := newStateReader(stateRoot, db.snap)
+ // If standalone state snapshot is available (hash scheme),
+ // then construct the legacy snap reader.
+ snap := db.snap.Snapshot(stateRoot)
+ if snap != nil {
+ readers = append(readers, newStateReader(snap)) // snap reader is optional
+ }
+ } else {
+ // If standalone state snapshot is not available, try to construct
+ // the state reader with database.
+ reader, err := db.triedb.StateReader(stateRoot)
  if err == nil {
- readers = append(readers, sr) // snap reader is optional
+ readers = append(readers, newStateReader(reader)) // state reader is optional
  }
  }
  // Set up the trie reader, which is expected to always be available

core/state/reader.go

@@ -21,13 +21,13 @@ import (
  "maps"
 
  "github.com/ethereum/go-ethereum/common"
- "github.com/ethereum/go-ethereum/core/state/snapshot"
  "github.com/ethereum/go-ethereum/core/types"
  "github.com/ethereum/go-ethereum/crypto"
  "github.com/ethereum/go-ethereum/rlp"
  "github.com/ethereum/go-ethereum/trie"
  "github.com/ethereum/go-ethereum/trie/utils"
  "github.com/ethereum/go-ethereum/triedb"
+ "github.com/ethereum/go-ethereum/triedb/database"
 )
 
 // Reader defines the interface for accessing accounts and storage slots
@@ -52,23 +52,18 @@ type Reader interface {
  Copy() Reader
 }
 
-// stateReader is a wrapper over the state snapshot and implements the Reader
-// interface. It provides an efficient way to access flat state.
+// stateReader wraps a database state reader.
 type stateReader struct {
- snap snapshot.Snapshot
- buff crypto.KeccakState
+ reader database.StateReader
+ buff  crypto.KeccakState
 }
 
-// newStateReader constructs a flat state reader with on the specified state root.
-func newStateReader(root common.Hash, snaps *snapshot.Tree) (*stateReader, error) {
- snap := snaps.Snapshot(root)
- if snap == nil {
- return nil, errors.New("snapshot is not available")
- }
+// newStateReader constructs a state reader with on the given state root.
+func newStateReader(reader database.StateReader) *stateReader {
  return &stateReader{
- snap: snap,
- buff: crypto.NewKeccakState(),
- }, nil
+ reader: reader,
+ buff:  crypto.NewKeccakState(),
+ }
 }
 
 // Account implements Reader, retrieving the account specified by the address.
@@ -78,18 +73,18 @@ func newStateReader(root common.Hash, snaps *snapshot.Tree) (*stateReader, error
 //
 // The returned account might be nil if it's not existent.
 func (r *stateReader) Account(addr common.Address) (*types.StateAccount, error) {
- ret, err := r.snap.Account(crypto.HashData(r.buff, addr.Bytes()))
+ account, err := r.reader.Account(crypto.HashData(r.buff, addr.Bytes()))
  if err != nil {
  return nil, err
  }
- if ret == nil {
+ if account == nil {
  return nil, nil
  }
  acct := &types.StateAccount{
- Nonce: ret.Nonce,
- Balance: ret.Balance,
- CodeHash: ret.CodeHash,
- Root: common.BytesToHash(ret.Root),
+ Nonce: account.Nonce,
+ Balance: account.Balance,
+ CodeHash: account.CodeHash,
+ Root: common.BytesToHash(account.Root),
  }
  if len(acct.CodeHash) == 0 {
  acct.CodeHash = types.EmptyCodeHash.Bytes()
@@ -110,7 +105,7 @@ func (r *stateReader) Account(addr common.Address) (*types.StateAccount, error)
 func (r *stateReader) Storage(addr common.Address, key common.Hash) (common.Hash, error) {
  addrHash := crypto.HashData(r.buff, addr.Bytes())
  slotHash := crypto.HashData(r.buff, key.Bytes())
- ret, err := r.snap.Storage(addrHash, slotHash)
+ ret, err := r.reader.Storage(addrHash, slotHash)
  if err != nil {
  return common.Hash{}, err
  }
@@ -131,8 +126,8 @@ func (r *stateReader) Storage(addr common.Address, key common.Hash) (common.Hash
 // Copy implements Reader, returning a deep-copied snap reader.
 func (r *stateReader) Copy() Reader {
  return &stateReader{
- snap: r.snap,
- buff: crypto.NewKeccakState(),
+ reader: r.reader,
+ buff:  crypto.NewKeccakState(),
  }
 }
 

triedb/database.go

@@ -60,6 +60,10 @@ type backend interface {
  // An error will be returned if the specified state is not available.
  NodeReader(root common.Hash) (database.NodeReader, error)
 
+ // StateReader returns a reader for accessing flat states within the specified
+ // state. An error will be returned if the specified state is not available.
+ StateReader(root common.Hash) (database.StateReader, error)
+
  // Initialized returns an indicator if the state data is already initialized
  // according to the state scheme.
  Initialized(genesisRoot common.Hash) bool
@@ -122,6 +126,13 @@ func (db *Database) NodeReader(blockRoot common.Hash) (database.NodeReader, erro
  return db.backend.NodeReader(blockRoot)
 }
 
+// StateReader returns a reader that allows access to the state data associated
+// with the specified state. An error will be returned if the specified state is
+// not available.
+func (db *Database) StateReader(blockRoot common.Hash) (database.StateReader, error) {
+ return db.backend.StateReader(blockRoot)
+}
+
 // Update performs a state transition by committing dirty nodes contained in the
 // given set in order to update state from the specified parent to the specified
 // root. The held pre-images accumulated up to this point will be flushed in case

triedb/database/database.go

@@ -18,6 +18,7 @@ package database
 
 import (
  "github.com/ethereum/go-ethereum/common"
+ "github.com/ethereum/go-ethereum/core/types"
 )
 
 // NodeReader wraps the Node method of a backing trie reader.
@@ -37,3 +38,31 @@ type NodeDatabase interface {
  // An error will be returned if the specified state is not available.
  NodeReader(stateRoot common.Hash) (NodeReader, error)
 }
+
+// StateReader wraps the Account and Storage method of a backing state reader.
+type StateReader interface {
+ // Account directly retrieves the account associated with a particular hash in
+ // the slim data format. An error will be returned if the read operation exits
+ // abnormally. Specifically, if the layer is already stale.
+ //
+ // Note:
+ // - the returned account object is safe to modify
+ // - no error will be returned if the requested account is not found in database
+ Account(hash common.Hash) (*types.SlimAccount, error)
+
+ // Storage directly retrieves the storage data associated with a particular hash,
+ // within a particular account. An error will be returned if the read operation
+ // exits abnormally.
+ //
+ // Note:
+ // - the returned storage data is not a copy, please don't modify it
+ // - no error will be returned if the requested slot is not found in database
+ Storage(accountHash, storageHash common.Hash) ([]byte, error)
+}
+
+// StateDatabase warps the methods of a backing state store.
+type StateDatabase interface {
+ // StateReader returns a state reader associated with the specific state.
+ // An error will be returned if the specified state is not available.
+ StateReader(stateRoot common.Hash) (StateReader, error)
+}

triedb/hashdb/database.go

@@ -635,3 +635,9 @@ func (reader *reader) Node(owner common.Hash, path []byte, hash common.Hash) ([]
  blob, _ := reader.db.node(hash)
  return blob, nil
 }
+
+// StateReader returns a reader that allows access to the state data associated
+// with the specified state.
+func (db *Database) StateReader(root common.Hash) (database.StateReader, error) {
+ return nil, errors.New("not implemented")
+}

triedb/pathdb/buffer.go

@@ -33,40 +33,56 @@ import (
 // must be checked before diving into disk (since it basically is not yet written
 // data).
 type buffer struct {
- layers uint64 // The number of diff layers aggregated inside
- limit uint64 // The maximum memory allowance in bytes
- nodes *nodeSet // Aggregated trie node set
+ layers uint64 // The number of diff layers aggregated inside
+ limit uint64 // The maximum memory allowance in bytes
+ nodes *nodeSet // Aggregated trie node set
+ states *stateSet // Aggregated state set
 }
 
 // newBuffer initializes the buffer with the provided states and trie nodes.
-func newBuffer(limit int, nodes *nodeSet, layers uint64) *buffer {
+func newBuffer(limit int, nodes *nodeSet, states *stateSet, layers uint64) *buffer {
  // Don't panic for lazy users if any provided set is nil
  if nodes == nil {
  nodes = newNodeSet(nil)
  }
+ if states == nil {
+ states = newStates(nil, nil, nil)
+ }
  return &buffer{
  layers: layers,
  limit: uint64(limit),
  nodes: nodes,
+ states: states,
  }
 }
 
+// account retrieves the account blob with account address hash.
+func (b *buffer) account(hash common.Hash) ([]byte, bool) {
+ return b.states.account(hash)
+}
+
+// storage retrieves the storage slot with account address hash and slot key.
+func (b *buffer) storage(addrHash common.Hash, storageHash common.Hash) ([]byte, bool) {
+ return b.states.storage(addrHash, storageHash)
+}
+
 // node retrieves the trie node with node path and its trie identifier.
 func (b *buffer) node(owner common.Hash, path []byte) (*trienode.Node, bool) {
  return b.nodes.node(owner, path)
 }
 
 // commit merges the provided states and trie nodes into the buffer.
-func (b *buffer) commit(nodes *nodeSet) *buffer {
+func (b *buffer) commit(nodes *nodeSet, states *stateSet) *buffer {
  b.layers++
  b.nodes.merge(nodes)
+ b.states.merge(states)
  return b
 }
 
 // revert is the reverse operation of commit. It also merges the provided states
 // and trie nodes into the buffer. The key difference is that the provided state
 // set should reverse the changes made by the most recent state transition.
-func (b *buffer) revert(db ethdb.KeyValueReader, nodes map[common.Hash]map[string]*trienode.Node) error {
+func (b *buffer) revert(db ethdb.KeyValueReader, nodes map[common.Hash]map[string]*trienode.Node, accounts map[common.Hash][]byte, storages map[common.Hash]map[common.Hash][]byte) error {
  // Short circuit if no embedded state transition to revert
  if b.layers == 0 {
  return errStateUnrecoverable
@@ -79,13 +95,15 @@ func (b *buffer) revert(db ethdb.KeyValueReader, nodes map[common.Hash]map[strin
  return nil
  }
  b.nodes.revert(db, nodes)
+ b.states.revert(accounts, storages)
  return nil
 }
 
 // reset cleans up the disk cache.
 func (b *buffer) reset() {
  b.layers = 0
  b.nodes.reset()
+ b.states.reset()
 }
 
 // empty returns an indicator if buffer is empty.
@@ -101,7 +119,7 @@ func (b *buffer) full() bool {
 
 // size returns the approximate memory size of the held content.
 func (b *buffer) size() uint64 {
- return b.nodes.size
+ return b.states.size + b.nodes.size
 }
 
 // flush persists the in-memory dirty trie node into the disk if the configured

triedb/pathdb/database.go

@@ -68,6 +68,24 @@ type layer interface {
  // - no error will be returned if the requested node is not found in database.
  node(owner common.Hash, path []byte, depth int) ([]byte, common.Hash, *nodeLoc, error)
 
+ // account directly retrieves the account RLP associated with a particular
+ // hash in the slim data format. An error will be returned if the read
+ // operation exits abnormally. Specifically, if the layer is already stale.
+ //
+ // Note:
+ // - the returned account is not a copy, please don't modify it.
+ // - no error will be returned if the requested account is not found in database.
+ account(hash common.Hash, depth int) ([]byte, error)
+
+ // storage directly retrieves the storage data associated with a particular hash,
+ // within a particular account. An error will be returned if the read operation
+ // exits abnormally. Specifically, if the layer is already stale.
+ //
+ // Note:
+ // - the returned storage data is not a copy, please don't modify it.
+ // - no error will be returned if the requested slot is not found in database.
+ storage(accountHash, storageHash common.Hash, depth int) ([]byte, error)
+
  // rootHash returns the root hash for which this layer was made.
  rootHash() common.Hash
 
@@ -130,17 +148,18 @@ var Defaults = &Config{
 // ReadOnly is the config in order to open database in read only mode.
 var ReadOnly = &Config{ReadOnly: true}
 
-// Database is a multiple-layered structure for maintaining in-memory trie nodes.
-// It consists of one persistent base layer backed by a key-value store, on top
-// of which arbitrarily many in-memory diff layers are stacked. The memory diffs
-// can form a tree with branching, but the disk layer is singleton and common to
-// all. If a reorg goes deeper than the disk layer, a batch of reverse diffs can
-// be applied to rollback. The deepest reorg that can be handled depends on the
-// amount of state histories tracked in the disk.
+// Database is a multiple-layered structure for maintaining in-memory states
+// along with its dirty trie nodes. It consists of one persistent base layer
+// backed by a key-value store, on top of which arbitrarily many in-memory diff
+// layers are stacked. The memory diffs can form a tree with branching, but the
+// disk layer is singleton and common to all. If a reorg goes deeper than the
+// disk layer, a batch of reverse diffs can be applied to rollback. The deepest
+// reorg that can be handled depends on the amount of state histories tracked
+// in the disk.
 //
 // At most one readable and writable database can be opened at the same time in
-// the whole system which ensures that only one database writer can operate disk
-// state. Unexpected open operations can cause the system to panic.
+// the whole system which ensures that only one database writer can operate the
+// persistent state. Unexpected open operations can cause the system to panic.
 type Database struct {
  // readOnly is the flag whether the mutation is allowed to be applied.
  // It will be set automatically when the database is journaled during
@@ -358,7 +377,7 @@ func (db *Database) Enable(root common.Hash) error {
  }
  // Re-construct a new disk layer backed by persistent state
  // with **empty clean cache and node buffer**.
- db.tree.reset(newDiskLayer(root, 0, db, nil, newBuffer(db.config.WriteBufferSize, nil, 0)))
+ db.tree.reset(newDiskLayer(root, 0, db, nil, newBuffer(db.config.WriteBufferSize, nil, nil, 0)))
 
  // Re-enable the database as the final step.
  db.waitSync = false
@@ -536,3 +555,21 @@ func (db *Database) StorageHistory(address common.Address, slot common.Hash, sta
 func (db *Database) HistoryRange() (uint64, uint64, error) {
  return historyRange(db.freezer)
 }
+
+// AccountIterator creates a new account iterator for the specified root hash and
+// seeks to a starting account hash.
+func (db *Database) AccountIterator(root common.Hash, seek common.Hash) (AccountIterator, error) {
+ //if gen := db.tree.bottom().generator; gen != nil && !gen.completed() {
+ // return nil, errNotConstructed
+ //}
+ return newFastAccountIterator(db, root, seek)
+}
+
+// StorageIterator creates a new storage iterator for the specified root hash and
+// account. The iterator will be move to the specific start position.
+func (db *Database) StorageIterator(root common.Hash, account common.Hash, seek common.Hash) (StorageIterator, error) {
+ //if gen := db.tree.bottom().generator; gen != nil && !gen.completed() {
+ // return nil, errNotConstructed
+ //}
+ return newFastStorageIterator(db, root, account, seek)
+}

triedb/pathdb/database_test.go

@@ -309,7 +309,18 @@ func (t *tester) generate(parent common.Hash) (common.Hash, *trienode.MergedNode
  delete(t.storages, addrHash)
  }
  }
- return root, ctx.nodes, NewStateSetWithOrigin(ctx.accountOrigin, ctx.storageOrigin)
+ var (
+ accounts = make(map[common.Hash][]byte)
+ destructs = make(map[common.Hash]struct{})
+ )
+ for addrHash, data := range ctx.accounts {
+ if len(data) == 0 {
+ destructs[addrHash] = struct{}{}
+ } else {
+ accounts[addrHash] = data
+ }
+ }
+ return root, ctx.nodes, NewStateSetWithOrigin(destructs, accounts, ctx.storages, ctx.accountOrigin, ctx.storageOrigin)
 }
 
 // lastHash returns the latest root hash, or empty if nothing is cached.