diff --git a/src/Nethermind/Nethermind.Blockchain.Test/Proofs/ReceiptTrieTests.cs b/src/Nethermind/Nethermind.Blockchain.Test/Proofs/ReceiptTrieTests.cs index ec52da837d3..212622a735a 100644 --- a/src/Nethermind/Nethermind.Blockchain.Test/Proofs/ReceiptTrieTests.cs +++ b/src/Nethermind/Nethermind.Blockchain.Test/Proofs/ReceiptTrieTests.cs @@ -21,16 +21,16 @@ public class ReceiptTrieTests public void Can_calculate_root_no_eip_658() { TxReceipt receipt = Build.A.Receipt.WithAllFieldsFilled.TestObject; - ReceiptTrie receiptTrie = new(MainnetSpecProvider.Instance.GetSpec((1, null)), new[] { receipt }); - Assert.That(receiptTrie.RootHash.ToString(), Is.EqualTo("0xe51a2d9f986d68628990c9d65e45c36128ec7bb697bd426b0bb4d18a3f3321be")); + Keccak rootHash = ReceiptTrie.CalculateRoot(MainnetSpecProvider.Instance.GetSpec((1, null)), new[] { receipt }); + Assert.That(rootHash.ToString(), Is.EqualTo("0xe51a2d9f986d68628990c9d65e45c36128ec7bb697bd426b0bb4d18a3f3321be")); } [Test, Timeout(Timeout.MaxTestTime)] public void Can_calculate_root() { TxReceipt receipt = Build.A.Receipt.WithAllFieldsFilled.TestObject; - ReceiptTrie receiptTrie = new(MainnetSpecProvider.Instance.GetSpec((MainnetSpecProvider.MuirGlacierBlockNumber, null)), new[] { receipt }); - Assert.That(receiptTrie.RootHash.ToString(), Is.EqualTo("0x2e6d89c5b539e72409f2e587730643986c2ef33db5e817a4223aa1bb996476d5")); + Keccak rootHash = ReceiptTrie.CalculateRoot(MainnetSpecProvider.Instance.GetSpec((MainnetSpecProvider.MuirGlacierBlockNumber, null)), new[] { receipt }); + Assert.That(rootHash.ToString(), Is.EqualTo("0x2e6d89c5b539e72409f2e587730643986c2ef33db5e817a4223aa1bb996476d5")); } [Test, Timeout(Timeout.MaxTestTime)] diff --git a/src/Nethermind/Nethermind.Blockchain.Test/Proofs/TxTrieTests.cs b/src/Nethermind/Nethermind.Blockchain.Test/Proofs/TxTrieTests.cs index fe25d8075ef..25b0788b6a5 100644 --- a/src/Nethermind/Nethermind.Blockchain.Test/Proofs/TxTrieTests.cs +++ b/src/Nethermind/Nethermind.Blockchain.Test/Proofs/TxTrieTests.cs @@ -29,15 +29,15 @@ public TxTrieTests(bool useEip2718) public void Can_calculate_root() { Block block = Build.A.Block.WithTransactions(Build.A.Transaction.TestObject).TestObject; - TxTrie txTrie = new(block.Transactions); + Keccak rootHash = TxTrie.CalculateRoot(block.Transactions); if (_releaseSpec == Berlin.Instance) { - Assert.That(txTrie.RootHash.ToString(), Is.EqualTo("0x29cc403075ed3d1d6af940d577125cc378ee5a26f7746cbaf87f1cf4a38258b5")); + Assert.That(rootHash.ToString(), Is.EqualTo("0x29cc403075ed3d1d6af940d577125cc378ee5a26f7746cbaf87f1cf4a38258b5")); } else { - Assert.That(txTrie.RootHash.ToString(), Is.EqualTo("0x29cc403075ed3d1d6af940d577125cc378ee5a26f7746cbaf87f1cf4a38258b5")); + Assert.That(rootHash.ToString(), Is.EqualTo("0x29cc403075ed3d1d6af940d577125cc378ee5a26f7746cbaf87f1cf4a38258b5")); } } diff --git a/src/Nethermind/Nethermind.Blockchain/FullPruning/CopyTreeVisitor.cs b/src/Nethermind/Nethermind.Blockchain/FullPruning/CopyTreeVisitor.cs index 3e929ff2ea8..6a80ba86ffa 100644 --- a/src/Nethermind/Nethermind.Blockchain/FullPruning/CopyTreeVisitor.cs +++ b/src/Nethermind/Nethermind.Blockchain/FullPruning/CopyTreeVisitor.cs @@ -5,6 +5,7 @@ using System.Diagnostics; using System.Threading; using Nethermind.Core; +using Nethermind.Core.Buffers; using Nethermind.Core.Crypto; using Nethermind.Db.FullPruning; using Nethermind.Logging; @@ -74,7 +75,7 @@ private void PersistNode(TrieNode node) if (node.Keccak is not null) { // simple copy of nodes RLP - _pruningContext.Set(node.Keccak.Bytes, node.FullRlp, _writeFlags); + _pruningContext.Set(node.Keccak.Bytes, node.FullRlp.ToArray(), _writeFlags); Interlocked.Increment(ref _persistedNodes); // log message every 1 mln nodes diff --git a/src/Nethermind/Nethermind.Blockchain/Receipts/ReceiptsExtensions.cs b/src/Nethermind/Nethermind.Blockchain/Receipts/ReceiptsExtensions.cs index d1be4df28e7..17ea3273b95 100644 --- a/src/Nethermind/Nethermind.Blockchain/Receipts/ReceiptsExtensions.cs +++ b/src/Nethermind/Nethermind.Blockchain/Receipts/ReceiptsExtensions.cs @@ -6,6 +6,7 @@ using Nethermind.Core.Crypto; using Nethermind.Core.Specs; using Nethermind.State.Proofs; +using Nethermind.Trie; namespace Nethermind.Blockchain.Receipts { @@ -29,7 +30,7 @@ Keccak SkipStateAndStatusReceiptsRoot() txReceipts.SetSkipStateAndStatusInRlp(true); try { - return new ReceiptTrie(receiptSpec, txReceipts).RootHash; + return ReceiptTrie.CalculateRoot(receiptSpec, txReceipts); } finally { @@ -37,7 +38,7 @@ Keccak SkipStateAndStatusReceiptsRoot() } } - Keccak receiptsRoot = new ReceiptTrie(receiptSpec, txReceipts).RootHash; + Keccak receiptsRoot = ReceiptTrie.CalculateRoot(receiptSpec, txReceipts); if (!receiptSpec.ValidateReceipts && receiptsRoot != suggestedRoot) { var skipStateAndStatusReceiptsRoot = SkipStateAndStatusReceiptsRoot(); diff --git a/src/Nethermind/Nethermind.Core.Test/Buffers/CappedArrayTests.cs b/src/Nethermind/Nethermind.Core.Test/Buffers/CappedArrayTests.cs new file mode 100644 index 00000000000..41aec9761e8 --- /dev/null +++ b/src/Nethermind/Nethermind.Core.Test/Buffers/CappedArrayTests.cs @@ -0,0 +1,54 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.Linq; +using FluentAssertions; +using Nethermind.Core.Buffers; +using NUnit.Framework; + +namespace Nethermind.Core.Test.Buffers; + +public class CappedArrayTests +{ + [Test] + public void WhenGivenNullArray_IsNull_ShouldReturnTrue() + { + CappedArray array = new(null); + array.IsNull.Should().BeTrue(); + } + + [Test] + public void WhenGivenNullArray_AsSpan_ShouldReturnEmpty() + { + CappedArray array = new(null); + array.AsSpan().IsEmpty.Should().BeTrue(); + array.AsSpan().Length.Should().Be(0); + array.Length.Should().Be(0); + } + + [Test] + public void WhenGivenArray_AndLengthIsSame_ToArray_ShouldReturnSameArray() + { + int[] baseArray = Enumerable.Range(0, 10).ToArray(); + CappedArray array = new(baseArray); + array.IsUncapped.Should().BeTrue(); + array.IsNull.Should().BeFalse(); + array.IsNotNull.Should().BeTrue(); + array.Length.Should().Be(10); + array.ToArray().Should().BeSameAs(baseArray); + } + + [Test] + public void WhenGivenArray_AndLengthIsLess_ToArray_ShouldBeCapped() + { + int[] baseArray = Enumerable.Range(0, 10).ToArray(); + CappedArray array = new(baseArray, 5); + array.IsUncapped.Should().BeFalse(); + array.IsNull.Should().BeFalse(); + array.IsNotNull.Should().BeTrue(); + array.Length.Should().Be(5); + array.ToArray().Should().BeEquivalentTo(baseArray[..5]); + array.AsSpan().Length.Should().Be(5); + array.AsSpan().ToArray().Should().BeEquivalentTo(baseArray[..5]); + } +} diff --git a/src/Nethermind/Nethermind.Core/Buffers/CappedArray.cs b/src/Nethermind/Nethermind.Core/Buffers/CappedArray.cs new file mode 100644 index 00000000000..ae665b44c9a --- /dev/null +++ b/src/Nethermind/Nethermind.Core/Buffers/CappedArray.cs @@ -0,0 +1,69 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using Nethermind.Core.Crypto; +using Nethermind.Core.Extensions; + +namespace Nethermind.Core.Buffers; + +/// +/// Basically like ArraySegment, but only contain length, which reduces it size from 16byte to 12byte. Useful for +/// polling memory where memory pool usually can't return exactly the same size of data. To conserve space, The +/// underlying array can be null and this struct is meant to be non nullable, checking the `IsNull` property to check +/// if it represent null. +/// +public struct CappedArray +{ + private T[]? _array = null; + private int _length = 0; + + public CappedArray(T[]? array, int length) + { + _array = array; + _length = length; + } + + public CappedArray(T[]? array) + { + if (array != null) + { + _array = array; + _length = array.Length; + } + } + + public static implicit operator ReadOnlySpan(CappedArray array) + { + return array.AsSpan(); + } + + public static implicit operator CappedArray(T[]? array) + { + if (array == null) return new CappedArray(null); + return new CappedArray(array); + } + + public int Length + { + get => _length; + set => _length = value; + } + + public T[]? Array => _array; + public bool IsUncapped => _length == _array?.Length; + public bool IsNull => _array is null; + public bool IsNotNull => _array is not null; + + public Span AsSpan() + { + return _array.AsSpan()[..Length]; + } + + public T[]? ToArray() + { + if (_array is null) return null; + if (_length == _array?.Length) return _array; + return AsSpan().ToArray(); + } +} diff --git a/src/Nethermind/Nethermind.Core/Buffers/ICappedArrayPool.cs b/src/Nethermind/Nethermind.Core/Buffers/ICappedArrayPool.cs new file mode 100644 index 00000000000..7a238b65699 --- /dev/null +++ b/src/Nethermind/Nethermind.Core/Buffers/ICappedArrayPool.cs @@ -0,0 +1,25 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +namespace Nethermind.Core.Buffers; + +public interface ICappedArrayPool +{ + CappedArray Rent(int size); + + void Return(CappedArray buffer); +} + +public static class BufferPoolExtensions +{ + public static CappedArray SafeRentBuffer(this ICappedArrayPool? pool, int size) + { + if (pool == null) return new CappedArray(new byte[size]); + return pool.Rent(size); + } + + public static void SafeReturnBuffer(this ICappedArrayPool? pool, CappedArray buffer) + { + pool?.Return(buffer); + } +} diff --git a/src/Nethermind/Nethermind.Core/Extensions/SpanExtensions.cs b/src/Nethermind/Nethermind.Core/Extensions/SpanExtensions.cs index 48daf3033f6..e58a4f035d2 100644 --- a/src/Nethermind/Nethermind.Core/Extensions/SpanExtensions.cs +++ b/src/Nethermind/Nethermind.Core/Extensions/SpanExtensions.cs @@ -55,6 +55,7 @@ private static string ToHexViaLookup(ReadOnlySpan bytes, bool withZeroX, b { return ToHexStringWithEip55Checksum(bytes, withZeroX, skipLeadingZeros); } + if (bytes.Length == 0) return ""; int leadingZeros = skipLeadingZeros ? Bytes.CountLeadingZeros(bytes) : 0; int length = bytes.Length * 2 + (withZeroX ? 2 : 0) - leadingZeros; diff --git a/src/Nethermind/Nethermind.Serialization.Rlp/ByteArrayExtensions.cs b/src/Nethermind/Nethermind.Serialization.Rlp/ByteArrayExtensions.cs index 4c39add0a90..aff78b367c5 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp/ByteArrayExtensions.cs +++ b/src/Nethermind/Nethermind.Serialization.Rlp/ByteArrayExtensions.cs @@ -2,6 +2,7 @@ // SPDX-License-Identifier: LGPL-3.0-only using System; +using Nethermind.Core.Buffers; namespace Nethermind.Serialization.Rlp { @@ -12,6 +13,11 @@ public static RlpStream AsRlpStream(this byte[]? bytes) return new(bytes ?? Array.Empty()); } + public static RlpStream AsRlpStream(this CappedArray bytes) + { + return new(bytes.Array ?? Array.Empty()); + } + public static Rlp.ValueDecoderContext AsRlpValueContext(this byte[]? bytes) { return new(bytes ?? Array.Empty()); diff --git a/src/Nethermind/Nethermind.Serialization.Rlp/Rlp.cs b/src/Nethermind/Nethermind.Serialization.Rlp/Rlp.cs index 093003f5e48..b8dc2c3bebb 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp/Rlp.cs +++ b/src/Nethermind/Nethermind.Serialization.Rlp/Rlp.cs @@ -479,7 +479,7 @@ public static Rlp Encode(Keccak? keccak) return new Rlp(result); } - public static int StartSequence(byte[] buffer, int position, int sequenceLength) + public static int StartSequence(Span buffer, int position, int sequenceLength) { byte prefix; int beforeLength = position + 1; diff --git a/src/Nethermind/Nethermind.Serialization.Rlp/RlpDecoderExtensions.cs b/src/Nethermind/Nethermind.Serialization.Rlp/RlpDecoderExtensions.cs index 0a0cf494604..83edad25ef0 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp/RlpDecoderExtensions.cs +++ b/src/Nethermind/Nethermind.Serialization.Rlp/RlpDecoderExtensions.cs @@ -3,6 +3,7 @@ using System.Collections.Generic; using DotNetty.Buffers; +using Nethermind.Core.Buffers; namespace Nethermind.Serialization.Rlp { @@ -92,6 +93,22 @@ public static NettyRlpStream EncodeToNewNettyStream(this IRlpStreamDecoder return rlpStream; } + public static CappedArray EncodeToCappedArray(this IRlpStreamDecoder decoder, T? item, RlpBehaviors rlpBehaviors = RlpBehaviors.None, ICappedArrayPool? bufferPool = null) + { + int size = decoder.GetLength(item, rlpBehaviors); + CappedArray buffer = bufferPool.SafeRentBuffer(size); + decoder.Encode(buffer.AsRlpStream(), item, rlpBehaviors); + + return buffer; + } + + public static CappedArray EncodeToCappedArray(this int item, ICappedArrayPool? bufferPool = null) + { + CappedArray buffer = bufferPool.SafeRentBuffer(Rlp.LengthOf(item)); + buffer.AsRlpStream().Encode(item); + return buffer; + } + public static Rlp Encode(this IRlpObjectDecoder decoder, IReadOnlyCollection? items, RlpBehaviors behaviors = RlpBehaviors.None) { if (items is null) diff --git a/src/Nethermind/Nethermind.State.Test/NodeTests.cs b/src/Nethermind/Nethermind.State.Test/NodeTests.cs index 2d2e855068e..c0148a241c2 100644 --- a/src/Nethermind/Nethermind.State.Test/NodeTests.cs +++ b/src/Nethermind/Nethermind.State.Test/NodeTests.cs @@ -1,6 +1,7 @@ // SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only +using Nethermind.Core.Buffers; using Nethermind.Core.Test.Builders; using Nethermind.Db; using Nethermind.Logging; @@ -133,11 +134,11 @@ public void Child_and_value_store_encode() private static ITrieNodeResolver BuildATreeFromNode(TrieNode node) { TrieNode.AllowBranchValues = true; - byte[] rlp = node.RlpEncode(null); + CappedArray rlp = node.RlpEncode(null); node.ResolveKey(null, true); MemDb memDb = new(); - memDb[node.Keccak.Bytes] = rlp; + memDb[node.Keccak.Bytes] = rlp.ToArray(); // ITrieNodeResolver tree = new PatriciaTree(memDb, node.Keccak, false, true); return new TrieStore(memDb, NullLogManager.Instance); diff --git a/src/Nethermind/Nethermind.State.Test/Proofs/AccountProofCollectorTests.cs b/src/Nethermind/Nethermind.State.Test/Proofs/AccountProofCollectorTests.cs index 741fa610f66..f0b67a02798 100644 --- a/src/Nethermind/Nethermind.State.Test/Proofs/AccountProofCollectorTests.cs +++ b/src/Nethermind/Nethermind.State.Test/Proofs/AccountProofCollectorTests.cs @@ -6,6 +6,7 @@ using System.Linq; using FluentAssertions; using Nethermind.Core; +using Nethermind.Core.Buffers; using Nethermind.Core.Crypto; using Nethermind.Core.Extensions; using Nethermind.Core.Test.Builders; @@ -746,7 +747,7 @@ public void Chaotic_test() } } - node.Value.Should().BeEquivalentTo(new byte[] { 1 }); + node.Value.ToArray().Should().BeEquivalentTo(new byte[] { 1 }); } } } diff --git a/src/Nethermind/Nethermind.State/Proofs/AccountProofCollector.cs b/src/Nethermind/Nethermind.State/Proofs/AccountProofCollector.cs index e493a541764..510be44f649 100644 --- a/src/Nethermind/Nethermind.State/Proofs/AccountProofCollector.cs +++ b/src/Nethermind/Nethermind.State/Proofs/AccountProofCollector.cs @@ -5,6 +5,7 @@ using System.Collections.Generic; using System.Linq; using Nethermind.Core; +using Nethermind.Core.Buffers; using Nethermind.Core.Crypto; using Nethermind.Int256; using Nethermind.Serialization.Rlp; @@ -241,13 +242,13 @@ private void AddProofItem(TrieNode node, TrieVisitContext trieVisitContext) { foreach (int storageIndex in value.StorageIndices) { - _storageProofItems[storageIndex].Add(node.FullRlp); + _storageProofItems[storageIndex].Add(node.FullRlp.ToArray()); } } } else { - _accountProofItems.Add(node.FullRlp); + _accountProofItems.Add(node.FullRlp.ToArray()); } } @@ -282,13 +283,13 @@ public void VisitLeaf(TrieNode node, TrieVisitContext trieVisitContext, byte[] v bool isPathMatched = IsPathMatched(node, thisStoragePath); if (isPathMatched) { - _accountProof.StorageProofs[storageIndex].Value = new RlpStream(node.Value).DecodeByteArray(); + _accountProof.StorageProofs[storageIndex].Value = new RlpStream(node.Value.ToArray()).DecodeByteArray(); } } } else { - Account account = _accountDecoder.Decode(new RlpStream(node.Value)); + Account account = _accountDecoder.Decode(new RlpStream(node.Value.ToArray())); bool isPathMatched = IsPathMatched(node, _fullAccountPath); if (isPathMatched) { diff --git a/src/Nethermind/Nethermind.State/Proofs/PatriciaTrieT.cs b/src/Nethermind/Nethermind.State/Proofs/PatriciaTrieT.cs index 5287eeeb44c..371535e9a96 100644 --- a/src/Nethermind/Nethermind.State/Proofs/PatriciaTrieT.cs +++ b/src/Nethermind/Nethermind.State/Proofs/PatriciaTrieT.cs @@ -4,6 +4,7 @@ using System; using System.Collections.Generic; using System.Linq; +using Nethermind.Core.Buffers; using Nethermind.Db; using Nethermind.Logging; using Nethermind.Serialization.Rlp; @@ -23,8 +24,8 @@ public abstract class PatriciaTrie : PatriciaTree /// true to maintain an in-memory database for proof computation; /// otherwise, false. /// - public PatriciaTrie(IEnumerable? list, bool canBuildProof) - : base(canBuildProof ? new MemDb() : NullDb.Instance, EmptyTreeHash, false, false, NullLogManager.Instance) + public PatriciaTrie(IEnumerable? list, bool canBuildProof, ICappedArrayPool? bufferPool = null) + : base(canBuildProof ? new MemDb() : NullDb.Instance, EmptyTreeHash, false, false, NullLogManager.Instance, bufferPool: bufferPool) { CanBuildProof = canBuildProof; diff --git a/src/Nethermind/Nethermind.State/Proofs/ProofCollector.cs b/src/Nethermind/Nethermind.State/Proofs/ProofCollector.cs index 416cd37f0c1..cb1cc0076c4 100644 --- a/src/Nethermind/Nethermind.State/Proofs/ProofCollector.cs +++ b/src/Nethermind/Nethermind.State/Proofs/ProofCollector.cs @@ -3,6 +3,7 @@ using System; using System.Collections.Generic; +using Nethermind.Core.Buffers; using Nethermind.Core.Crypto; using Nethermind.Trie; @@ -65,7 +66,7 @@ public void VisitExtension(TrieNode node, TrieVisitContext trieVisitContext) protected virtual void AddProofBits(TrieNode node) { - _proofBits.Add(node.FullRlp); + _proofBits.Add(node.FullRlp.ToArray()); } public void VisitLeaf(TrieNode node, TrieVisitContext trieVisitContext, byte[] value) diff --git a/src/Nethermind/Nethermind.State/Proofs/ProofVerifier.cs b/src/Nethermind/Nethermind.State/Proofs/ProofVerifier.cs index 88855bc7ab6..c55a6cf728d 100644 --- a/src/Nethermind/Nethermind.State/Proofs/ProofVerifier.cs +++ b/src/Nethermind/Nethermind.State/Proofs/ProofVerifier.cs @@ -3,6 +3,7 @@ using System.IO; using System.Linq; +using Nethermind.Core.Buffers; using Nethermind.Core.Crypto; using Nethermind.Serialization.Rlp; using Nethermind.Trie; @@ -44,7 +45,7 @@ public static class ProofVerifier TrieNode trieNode = new(NodeType.Unknown, proof.Last()); trieNode.ResolveNode(null); - return trieNode.Value; + return trieNode.Value.ToArray(); } } } diff --git a/src/Nethermind/Nethermind.State/Proofs/ReceiptTrie.cs b/src/Nethermind/Nethermind.State/Proofs/ReceiptTrie.cs index 14296d471b0..a605bc7fd9a 100644 --- a/src/Nethermind/Nethermind.State/Proofs/ReceiptTrie.cs +++ b/src/Nethermind/Nethermind.State/Proofs/ReceiptTrie.cs @@ -5,9 +5,14 @@ using System.Collections.Generic; using System.Linq; using Nethermind.Core; +using Nethermind.Core.Buffers; +using Nethermind.Core.Crypto; +using Nethermind.Core.Extensions; using Nethermind.Core.Specs; using Nethermind.Serialization.Rlp; using Nethermind.State.Trie; +using Nethermind.Trie; +using Nethermind.Trie.Pruning; namespace Nethermind.State.Proofs; @@ -20,8 +25,8 @@ public class ReceiptTrie : PatriciaTrie /// /// The transaction receipts to build the trie of. - public ReceiptTrie(IReceiptSpec spec, IEnumerable receipts, bool canBuildProof = false) - : base(null, canBuildProof) + public ReceiptTrie(IReceiptSpec spec, IEnumerable receipts, bool canBuildProof = false, ICappedArrayPool? bufferPool = null) + : base(null, canBuildProof, bufferPool: bufferPool) { ArgumentNullException.ThrowIfNull(spec); ArgumentNullException.ThrowIfNull(receipts); @@ -35,15 +40,25 @@ public ReceiptTrie(IReceiptSpec spec, IEnumerable receipts, bool canB private void Initialize(IEnumerable receipts, IReceiptSpec spec) { - var behavior = (spec.IsEip658Enabled ? RlpBehaviors.Eip658Receipts : RlpBehaviors.None) - | RlpBehaviors.SkipTypedWrapping; - var key = 0; + RlpBehaviors behavior = (spec.IsEip658Enabled ? RlpBehaviors.Eip658Receipts : RlpBehaviors.None) + | RlpBehaviors.SkipTypedWrapping; + int key = 0; - foreach (var receipt in receipts) + foreach (TxReceipt? receipt in receipts) { - Set(Rlp.Encode(key++).Bytes, _decoder.EncodeNew(receipt, behavior)); + CappedArray buffer = _decoder.EncodeToCappedArray(receipt, behavior, bufferPool: _bufferPool); + CappedArray keyBuffer = (key++).EncodeToCappedArray(bufferPool: _bufferPool); + Set(keyBuffer.AsSpan(), buffer); } } protected override void Initialize(IEnumerable list) => throw new NotSupportedException(); + + public static Keccak CalculateRoot(IReceiptSpec receiptSpec, IList txReceipts) + { + TrackingCappedArrayPool cappedArrayPool = new(txReceipts.Count * 4); + Keccak receiptsRoot = new ReceiptTrie(receiptSpec, txReceipts, bufferPool: cappedArrayPool).RootHash; + cappedArrayPool.ReturnAll(); + return receiptsRoot; + } } diff --git a/src/Nethermind/Nethermind.State/Proofs/TxTrie.cs b/src/Nethermind/Nethermind.State/Proofs/TxTrie.cs index 6ed6b271323..5aad9928a33 100644 --- a/src/Nethermind/Nethermind.State/Proofs/TxTrie.cs +++ b/src/Nethermind/Nethermind.State/Proofs/TxTrie.cs @@ -4,8 +4,12 @@ using System; using System.Collections.Generic; using Nethermind.Core; +using Nethermind.Core.Buffers; +using Nethermind.Core.Crypto; using Nethermind.Serialization.Rlp; using Nethermind.State.Trie; +using Nethermind.Trie; +using Nethermind.Trie.Pruning; namespace Nethermind.State.Proofs; @@ -18,8 +22,8 @@ public class TxTrie : PatriciaTrie /// /// The transactions to build the trie of. - public TxTrie(IEnumerable transactions, bool canBuildProof = false) - : base(transactions, canBuildProof) => ArgumentNullException.ThrowIfNull(transactions); + public TxTrie(IEnumerable transactions, bool canBuildProof = false, ICappedArrayPool? bufferPool = null) + : base(transactions, canBuildProof, bufferPool: bufferPool) => ArgumentNullException.ThrowIfNull(transactions); protected override void Initialize(IEnumerable list) { @@ -30,8 +34,18 @@ protected override void Initialize(IEnumerable list) // a temporary trie would be a trie that exists to create a state root only and then be disposed of foreach (Transaction? transaction in list) { - Rlp transactionRlp = _txDecoder.Encode(transaction, RlpBehaviors.SkipTypedWrapping); - Set(Rlp.Encode(key++).Bytes, transactionRlp.Bytes); + CappedArray buffer = _txDecoder.EncodeToCappedArray(transaction, RlpBehaviors.SkipTypedWrapping, + bufferPool: _bufferPool); + CappedArray keyBuffer = (key++).EncodeToCappedArray(bufferPool: _bufferPool); + Set(keyBuffer.AsSpan(), buffer); } } + + public static Keccak CalculateRoot(IList transactions) + { + TrackingCappedArrayPool cappedArray = new TrackingCappedArrayPool(transactions.Count * 4); + Keccak rootHash = new TxTrie(transactions, false, bufferPool: cappedArray).RootHash; + cappedArray.ReturnAll(); + return rootHash; + } } diff --git a/src/Nethermind/Nethermind.State/StateTree.cs b/src/Nethermind/Nethermind.State/StateTree.cs index 5a666b68946..b149070ead8 100644 --- a/src/Nethermind/Nethermind.State/StateTree.cs +++ b/src/Nethermind/Nethermind.State/StateTree.cs @@ -3,6 +3,7 @@ using System.Diagnostics; using Nethermind.Core; +using Nethermind.Core.Buffers; using Nethermind.Core.Crypto; using Nethermind.Db; using Nethermind.Logging; @@ -19,8 +20,8 @@ public class StateTree : PatriciaTree private static readonly Rlp EmptyAccountRlp = Rlp.Encode(Account.TotallyEmpty); [DebuggerStepThrough] - public StateTree() - : base(new MemDb(), Keccak.EmptyTreeHash, true, true, NullLogManager.Instance) + public StateTree(ICappedArrayPool? bufferPool = null) + : base(new MemDb(), Keccak.EmptyTreeHash, true, true, NullLogManager.Instance, bufferPool: bufferPool) { TrieType = TrieType.State; } diff --git a/src/Nethermind/Nethermind.Synchronization/FastBlocks/BodiesSyncFeed.cs b/src/Nethermind/Nethermind.Synchronization/FastBlocks/BodiesSyncFeed.cs index 5cd63446ac2..5e441723d73 100644 --- a/src/Nethermind/Nethermind.Synchronization/FastBlocks/BodiesSyncFeed.cs +++ b/src/Nethermind/Nethermind.Synchronization/FastBlocks/BodiesSyncFeed.cs @@ -7,6 +7,7 @@ using Nethermind.Blockchain; using Nethermind.Blockchain.Synchronization; using Nethermind.Core; +using Nethermind.Core.Crypto; using Nethermind.Core.Specs; using Nethermind.Logging; using Nethermind.State.Proofs; @@ -160,7 +161,8 @@ public override SyncResponseHandlingResult HandleResponse(BodiesSyncBatch? batch private bool TryPrepareBlock(BlockInfo blockInfo, BlockBody blockBody, out Block? block) { BlockHeader header = _blockTree.FindHeader(blockInfo.BlockHash); - bool txRootIsValid = new TxTrie(blockBody.Transactions).RootHash == header.TxRoot; + Keccak rootHash = TxTrie.CalculateRoot(blockBody.Transactions); + bool txRootIsValid = rootHash == header.TxRoot; bool unclesHashIsValid = UnclesHash.Calculate(blockBody.Uncles) == header.UnclesHash; if (txRootIsValid && unclesHashIsValid) { diff --git a/src/Nethermind/Nethermind.Synchronization/FastSync/TreeSync.cs b/src/Nethermind/Nethermind.Synchronization/FastSync/TreeSync.cs index 414f40abd41..f0e765ba6bb 100644 --- a/src/Nethermind/Nethermind.Synchronization/FastSync/TreeSync.cs +++ b/src/Nethermind/Nethermind.Synchronization/FastSync/TreeSync.cs @@ -10,6 +10,7 @@ using System.Threading.Tasks; using Nethermind.Blockchain; using Nethermind.Core; +using Nethermind.Core.Buffers; using Nethermind.Core.Caching; using Nethermind.Core.Crypto; using Nethermind.Core.Extensions; @@ -870,7 +871,7 @@ private void HandleTrieNode(StateSyncItem currentStateSyncItem, byte[] currentRe { _pendingItems.MaxStateLevel = 64; DependentItem dependentItem = new(currentStateSyncItem, currentResponseItem, 2, true); - (Keccak codeHash, Keccak storageRoot) = AccountDecoder.DecodeHashesOnly(new RlpStream(trieNode.Value)); + (Keccak codeHash, Keccak storageRoot) = AccountDecoder.DecodeHashesOnly(new RlpStream(trieNode.Value.ToArray())); if (codeHash != Keccak.OfAnEmptyString) { // prepare a branch without the code DB diff --git a/src/Nethermind/Nethermind.Trie.Test/TrackingCappedArrayPoolTests.cs b/src/Nethermind/Nethermind.Trie.Test/TrackingCappedArrayPoolTests.cs new file mode 100644 index 00000000000..3465d66f98f --- /dev/null +++ b/src/Nethermind/Nethermind.Trie.Test/TrackingCappedArrayPoolTests.cs @@ -0,0 +1,35 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.Buffers; +using Nethermind.Core.Buffers; +using NSubstitute; +using NUnit.Framework; + +namespace Nethermind.Trie.Test; + +public class TrackingCappedArrayPoolTests +{ + [Test] + public void Test_Pooling() + { + ArrayPool? arrayPool = Substitute.For>(); + arrayPool + .Rent(Arg.Any()) + .Returns(info => new byte[(int)info[0]]); + TrackingCappedArrayPool? pool = new(0, arrayPool); + + pool.Rent(1); + pool.Rent(1); + pool.Rent(1); + CappedArray sample = pool.Rent(1); + + arrayPool.Received(4).Rent(1); + + pool.Return(sample); + arrayPool.Received(0).Return(Arg.Any()); + + pool.ReturnAll(); + arrayPool.Received(4).Return(Arg.Any()); + } +} diff --git a/src/Nethermind/Nethermind.Trie.Test/TrieNodeTests.cs b/src/Nethermind/Nethermind.Trie.Test/TrieNodeTests.cs index ba7e85baf28..756f2e7548a 100644 --- a/src/Nethermind/Nethermind.Trie.Test/TrieNodeTests.cs +++ b/src/Nethermind/Nethermind.Trie.Test/TrieNodeTests.cs @@ -6,6 +6,7 @@ using System.Threading.Tasks; using FluentAssertions; using Nethermind.Core; +using Nethermind.Core.Buffers; using Nethermind.Core.Crypto; using Nethermind.Core.Extensions; using Nethermind.Core.Test.Builders; @@ -152,7 +153,7 @@ public void Can_check_if_child_is_null_on_a_branch() trieNode.SetChild(j, ctx.TiniestLeaf); } - byte[] rlp = trieNode.RlpEncode(NullTrieNodeResolver.Instance); + CappedArray rlp = trieNode.RlpEncode(NullTrieNodeResolver.Instance); TrieNode restoredNode = new(NodeType.Branch, rlp); for (int childIndex = 0; childIndex < 16; childIndex++) @@ -178,14 +179,14 @@ public void Can_encode_decode_tiny_branch() TrieNode trieNode = new(NodeType.Branch); trieNode.SetChild(11, ctx.TiniestLeaf); - byte[] rlp = trieNode.RlpEncode(NullTrieNodeResolver.Instance); + CappedArray rlp = trieNode.RlpEncode(NullTrieNodeResolver.Instance); TrieNode decoded = new(NodeType.Unknown, rlp); decoded.ResolveNode(NullTrieNodeResolver.Instance); TrieNode decodedTiniest = decoded.GetChild(NullTrieNodeResolver.Instance, 11); decodedTiniest.ResolveNode(NullTrieNodeResolver.Instance); - Assert.That(decodedTiniest.Value, Is.EqualTo(ctx.TiniestLeaf.Value), "value"); + Assert.That(decodedTiniest.Value.ToArray(), Is.EqualTo(ctx.TiniestLeaf.Value.ToArray()), "value"); Assert.That(HexPrefix.ToBytes(decodedTiniest.Key!, true), Is.EqualTo(HexPrefix.ToBytes(ctx.TiniestLeaf.Key!, true)), "key"); } @@ -196,7 +197,7 @@ public void Can_encode_decode_heavy_branch() TrieNode trieNode = new(NodeType.Branch); trieNode.SetChild(11, ctx.HeavyLeaf); - byte[] rlp = trieNode.RlpEncode(NullTrieNodeResolver.Instance); + CappedArray rlp = trieNode.RlpEncode(NullTrieNodeResolver.Instance); TrieNode decoded = new(NodeType.Unknown, rlp); decoded.ResolveNode(NullTrieNodeResolver.Instance); @@ -213,14 +214,14 @@ public void Can_encode_decode_tiny_extension() trieNode.Key = new byte[] { 5 }; trieNode.SetChild(0, ctx.TiniestLeaf); - byte[] rlp = trieNode.RlpEncode(NullTrieNodeResolver.Instance); + CappedArray rlp = trieNode.RlpEncode(NullTrieNodeResolver.Instance); TrieNode decoded = new(NodeType.Unknown, rlp); decoded.ResolveNode(NullTrieNodeResolver.Instance); TrieNode? decodedTiniest = decoded.GetChild(NullTrieNodeResolver.Instance, 0); decodedTiniest?.ResolveNode(NullTrieNodeResolver.Instance); - Assert.That(decodedTiniest.Value, Is.EqualTo(ctx.TiniestLeaf.Value), "value"); + Assert.That(decodedTiniest.Value.ToArray(), Is.EqualTo(ctx.TiniestLeaf.Value.ToArray()), "value"); Assert.That(HexPrefix.ToBytes(decodedTiniest.Key!, true), Is.EqualTo(HexPrefix.ToBytes(ctx.TiniestLeaf.Key!, true)), "key"); } @@ -233,7 +234,7 @@ public void Can_encode_decode_heavy_extension() trieNode.Key = new byte[] { 5 }; trieNode.SetChild(0, ctx.HeavyLeaf); - byte[] rlp = trieNode.RlpEncode(NullTrieNodeResolver.Instance); + CappedArray rlp = trieNode.RlpEncode(NullTrieNodeResolver.Instance); TrieNode decoded = new(NodeType.Unknown, rlp); decoded.ResolveNode(NullTrieNodeResolver.Instance); @@ -261,7 +262,7 @@ public void Get_child_hash_works_on_hashed_child_of_a_branch() Context ctx = new(); TrieNode trieNode = new(NodeType.Branch); trieNode[11] = ctx.HeavyLeaf; - byte[] rlp = trieNode.RlpEncode(NullTrieNodeResolver.Instance); + CappedArray rlp = trieNode.RlpEncode(NullTrieNodeResolver.Instance); TrieNode decoded = new(NodeType.Branch, rlp); Keccak getResult = decoded.GetChildHash(11); @@ -275,7 +276,7 @@ public void Get_child_hash_works_on_inlined_child_of_a_branch() TrieNode trieNode = new(NodeType.Branch); trieNode[11] = ctx.TiniestLeaf; - byte[] rlp = trieNode.RlpEncode(NullTrieNodeResolver.Instance); + CappedArray rlp = trieNode.RlpEncode(NullTrieNodeResolver.Instance); TrieNode decoded = new(NodeType.Branch, rlp); Keccak getResult = decoded.GetChildHash(11); @@ -289,7 +290,7 @@ public void Get_child_hash_works_on_hashed_child_of_an_extension() TrieNode trieNode = new(NodeType.Extension); trieNode[0] = ctx.HeavyLeaf; trieNode.Key = new byte[] { 5 }; - byte[] rlp = trieNode.RlpEncode(NullTrieNodeResolver.Instance); + CappedArray rlp = trieNode.RlpEncode(NullTrieNodeResolver.Instance); TrieNode decoded = new(NodeType.Extension, rlp); Keccak getResult = decoded.GetChildHash(0); @@ -303,7 +304,7 @@ public void Get_child_hash_works_on_inlined_child_of_an_extension() TrieNode trieNode = new(NodeType.Extension); trieNode[0] = ctx.TiniestLeaf; trieNode.Key = new byte[] { 5 }; - byte[] rlp = trieNode.RlpEncode(NullTrieNodeResolver.Instance); + CappedArray rlp = trieNode.RlpEncode(NullTrieNodeResolver.Instance); TrieNode decoded = new(NodeType.Extension, rlp); Keccak getResult = decoded.GetChildHash(0); @@ -346,7 +347,7 @@ public void Leaf_with_simple_account_can_accept_visitors() node.Accept(visitor, NullTrieNodeResolver.Instance, context); - visitor.Received().VisitLeaf(node, context, node.Value); + visitor.Received().VisitLeaf(node, context, node.Value.ToArray()); } [Test] @@ -360,7 +361,7 @@ public void Leaf_with_contract_without_storage_and_empty_code_can_accept_visitor node.Accept(visitor, NullTrieNodeResolver.Instance, context); - visitor.Received().VisitLeaf(node, context, node.Value); + visitor.Received().VisitLeaf(node, context, node.Value.ToArray()); } [Test] @@ -376,7 +377,7 @@ public void Leaf_with_contract_without_storage_and_with_code_can_accept_visitors node.Accept(visitor, NullTrieNodeResolver.Instance, context); - visitor.Received().VisitLeaf(node, context, node.Value); + visitor.Received().VisitLeaf(node, context, node.Value.ToArray()); } [Test] @@ -392,7 +393,7 @@ public void Leaf_with_contract_with_storage_and_without_code_can_accept_visitors node.Accept(visitor, NullTrieNodeResolver.Instance, context); - visitor.Received().VisitLeaf(node, context, node.Value); + visitor.Received().VisitLeaf(node, context, node.Value.ToArray()); } [Test] @@ -408,7 +409,7 @@ public void Extension_with_leaf_can_be_visited() node.Accept(visitor, NullTrieNodeResolver.Instance, context); visitor.Received().VisitExtension(node, context); - visitor.Received().VisitLeaf(ctx.AccountLeaf, context, ctx.AccountLeaf.Value); + visitor.Received().VisitLeaf(ctx.AccountLeaf, context, ctx.AccountLeaf.Value.ToArray()); } [Test] @@ -428,7 +429,7 @@ public void Branch_with_children_can_be_visited() node.Accept(visitor, NullTrieNodeResolver.Instance, context); visitor.Received().VisitBranch(node, context); - visitor.Received(16).VisitLeaf(ctx.AccountLeaf, context, ctx.AccountLeaf.Value); + visitor.Received(16).VisitLeaf(ctx.AccountLeaf, context, ctx.AccountLeaf.Value.ToArray()); } [Test] @@ -515,7 +516,7 @@ public void Can_encode_branch_with_unresolved_children() node.SetChild(i, randomTrieNode); } - byte[] rlp = node.RlpEncode(NullTrieNodeResolver.Instance); + CappedArray rlp = node.RlpEncode(NullTrieNodeResolver.Instance); TrieNode restoredNode = new(NodeType.Branch, rlp); @@ -912,7 +913,7 @@ public void Rlp_is_cloned_when_cloning() trieNode.SetChild(1, leaf1); trieNode.SetChild(2, leaf2); trieNode.ResolveKey(trieStore, true); - byte[] rlp = trieNode.FullRlp; + CappedArray rlp = trieNode.FullRlp; TrieNode restoredBranch = new(NodeType.Branch, rlp); @@ -920,7 +921,7 @@ public void Rlp_is_cloned_when_cloning() var restoredLeaf1 = clone.GetChild(trieStore, 1); restoredLeaf1.Should().NotBeNull(); restoredLeaf1.ResolveNode(trieStore); - restoredLeaf1.Value.Should().BeEquivalentTo(leaf1.Value); + restoredLeaf1.Value.ToArray().Should().BeEquivalentTo(leaf1.Value.ToArray()); } [Test] @@ -935,7 +936,7 @@ public void Can_parallel_read_unresolved_children() node.SetChild(i, randomTrieNode); } - byte[] rlp = node.RlpEncode(NullTrieNodeResolver.Instance); + CappedArray rlp = node.RlpEncode(NullTrieNodeResolver.Instance); TrieNode restoredNode = new(NodeType.Branch, rlp); diff --git a/src/Nethermind/Nethermind.Trie/BatchedTrieVisitor.cs b/src/Nethermind/Nethermind.Trie/BatchedTrieVisitor.cs index 9587b61c3cc..1cc3b330d38 100644 --- a/src/Nethermind/Nethermind.Trie/BatchedTrieVisitor.cs +++ b/src/Nethermind/Nethermind.Trie/BatchedTrieVisitor.cs @@ -249,7 +249,7 @@ void QueueNextNodes(ArrayPoolList<(TrieNode, SmallTrieVisitContext)> batchResult for (int i = batchResult.Count - 1; i >= 0; i--) { (TrieNode trieNode, SmallTrieVisitContext ctx) = batchResult[i]; - if (trieNode.NodeType == NodeType.Unknown && trieNode.FullRlp != null) + if (trieNode.NodeType == NodeType.Unknown && trieNode.FullRlp.IsNotNull) { // Inline node. Seems rare, so its fine to create new list for this. Does not have a keccak // to queue, so we'll just process it inline. @@ -295,7 +295,7 @@ private void BatchedThread() SmallTrieVisitContext ctx = currentBatch[i].Item2; - if (cur.FullRlp != null) continue; + if (cur.FullRlp.IsNotNull) continue; if (cur.Keccak is null) throw new TrieException($"Unable to resolve node without Keccak. ctx: {ctx.Level}, {ctx.ExpectAccounts}, {ctx.IsStorage}, {ctx.BranchChildIndex}"); resolveOrdering.Add(i); @@ -337,7 +337,7 @@ private void BatchedThread() (TrieNode nodeToResolve, SmallTrieVisitContext ctx) = currentBatch[i]; nextToProcesses.Clear(); - if (nodeToResolve.FullRlp == null) + if (nodeToResolve.FullRlp.IsNull) { // Still need to decrement counter QueueNextNodes(nextToProcesses); diff --git a/src/Nethermind/Nethermind.Trie/PatriciaTree.cs b/src/Nethermind/Nethermind.Trie/PatriciaTree.cs index 2362a5dbd97..ed4ed344fde 100644 --- a/src/Nethermind/Nethermind.Trie/PatriciaTree.cs +++ b/src/Nethermind/Nethermind.Trie/PatriciaTree.cs @@ -12,6 +12,7 @@ using System.Runtime.CompilerServices; using System.Threading.Tasks; using Nethermind.Core; +using Nethermind.Core.Buffers; using Nethermind.Core.Crypto; using Nethermind.Core.Extensions; using Nethermind.Logging; @@ -46,6 +47,7 @@ public class PatriciaTree private readonly ConcurrentQueue? _currentCommit; public ITrieStore TrieStore { get; } + public ICappedArrayPool? _bufferPool; private readonly bool _parallelBranches; @@ -83,8 +85,8 @@ public PatriciaTree(IKeyValueStoreWithBatching keyValueStore) { } - public PatriciaTree(ITrieStore trieStore, ILogManager logManager) - : this(trieStore, EmptyTreeHash, false, true, logManager) + public PatriciaTree(ITrieStore trieStore, ILogManager logManager, ICappedArrayPool? bufferPool = null) + : this(trieStore, EmptyTreeHash, false, true, logManager, bufferPool: bufferPool) { } @@ -93,13 +95,15 @@ public PatriciaTree( Keccak rootHash, bool parallelBranches, bool allowCommits, - ILogManager logManager) + ILogManager logManager, + ICappedArrayPool? bufferPool = null) : this( new TrieStore(keyValueStore, logManager), rootHash, parallelBranches, allowCommits, - logManager) + logManager, + bufferPool: bufferPool) { } @@ -108,7 +112,8 @@ public PatriciaTree( Keccak rootHash, bool parallelBranches, bool allowCommits, - ILogManager? logManager) + ILogManager? logManager, + ICappedArrayPool? bufferPool = null) { _logger = logManager?.GetClassLogger() ?? throw new ArgumentNullException(nameof(logManager)); TrieStore = trieStore ?? throw new ArgumentNullException(nameof(trieStore)); @@ -124,6 +129,8 @@ public PatriciaTree( _currentCommit = new ConcurrentQueue(); _commitExceptions = new ConcurrentQueue(); } + + _bufferPool = bufferPool; } public void Commit(long blockNumber, bool skipRoot = false, WriteFlags writeFlags = WriteFlags.None) @@ -149,7 +156,7 @@ public void Commit(long blockNumber, bool skipRoot = false, WriteFlags writeFlag } // reset objects - RootRef!.ResolveKey(TrieStore, true); + RootRef!.ResolveKey(TrieStore, true, bufferPool: _bufferPool); SetRootHash(RootRef.Keccak!, true); } @@ -265,10 +272,10 @@ private void Commit(NodeCommitInfo nodeCommitInfo, bool skipSelf = false) } } - node.ResolveKey(TrieStore, nodeCommitInfo.IsRoot); + node.ResolveKey(TrieStore, nodeCommitInfo.IsRoot, bufferPool: _bufferPool); node.Seal(); - if (node.FullRlp?.Length >= 32) + if (node.FullRlp.Length >= 32) { if (!skipSelf) { @@ -283,7 +290,7 @@ private void Commit(NodeCommitInfo nodeCommitInfo, bool skipSelf = false) public void UpdateRootHash() { - RootRef?.ResolveKey(TrieStore, true); + RootRef?.ResolveKey(TrieStore, true, bufferPool: _bufferPool); SetRootHash(RootRef?.Keccak ?? EmptyTreeHash, false); } @@ -316,7 +323,7 @@ private void SetRootHash(Keccak? value, bool resetObjects) try { Nibbles.BytesToNibbleBytes(rawKey, nibbles); - return Run(nibbles, nibblesCount, Array.Empty(), false, startRootHash: rootHash); + return Run(nibbles, nibblesCount, new CappedArray(Array.Empty()), false, startRootHash: rootHash).ToArray(); } finally { @@ -352,9 +359,16 @@ private static void EnhanceException(ReadOnlySpan rawKey, ValueKeccak root [SkipLocalsInit] [DebuggerStepThrough] public virtual void Set(ReadOnlySpan rawKey, byte[] value) + { + Set(rawKey, new CappedArray(value)); + } + + [SkipLocalsInit] + [DebuggerStepThrough] + public virtual void Set(ReadOnlySpan rawKey, CappedArray value) { if (_logger.IsTrace) - _logger.Trace($"{(value.Length == 0 ? $"Deleting {rawKey.ToHexString()}" : $"Setting {rawKey.ToHexString()} = {value.ToHexString()}")}"); + _logger.Trace($"{(value.Length == 0 ? $"Deleting {rawKey.ToHexString()}" : $"Setting {rawKey.ToHexString()} = {value.AsSpan().ToHexString()}")}"); int nibblesCount = 2 * rawKey.Length; byte[] array = null; @@ -380,10 +394,10 @@ public void Set(ReadOnlySpan rawKey, Rlp? value) Set(rawKey, value is null ? Array.Empty() : value.Bytes); } - private byte[]? Run( + private CappedArray Run( Span updatePath, int nibblesCount, - byte[]? updateValue, + CappedArray updateValue, bool isUpdate, bool ignoreMissingDelete = true, Keccak? startRootHash = null) @@ -409,7 +423,7 @@ public void Set(ReadOnlySpan rawKey, Rlp? value) _nodeStack.Clear(); } - byte[]? result; + CappedArray result; if (startRootHash is not null) { if (_logger.IsTrace) _logger.Trace($"Starting from {startRootHash} - {traverseContext.ToString()}"); @@ -422,7 +436,7 @@ public void Set(ReadOnlySpan rawKey, Rlp? value) bool trieIsEmpty = RootRef is null; if (trieIsEmpty) { - if (traverseContext.UpdateValue is not null) + if (traverseContext.UpdateValue.IsNotNull) { if (_logger.IsTrace) _logger.Trace($"Setting new leaf node with value {traverseContext.UpdateValue}"); byte[] key = updatePath[..nibblesCount].ToArray(); @@ -455,7 +469,7 @@ private void ResolveNode(TrieNode node, in TraverseContext traverseContext) } } - private byte[]? TraverseNode(TrieNode node, in TraverseContext traverseContext) + private CappedArray TraverseNode(TrieNode node, in TraverseContext traverseContext) { if (_logger.IsTrace) _logger.Trace( @@ -694,7 +708,7 @@ L L - - - - - - - - - - - - - - */ RootRef = nextNode; } - private byte[]? TraverseBranch(TrieNode node, in TraverseContext traverseContext) + private CappedArray TraverseBranch(TrieNode node, in TraverseContext traverseContext) { if (traverseContext.RemainingUpdatePathLength == 0) { @@ -708,7 +722,7 @@ which is not possible within the Ethereum protocol which has keys of the same le if (traverseContext.IsDelete) { - if (node.Value is null) + if (node.Value.IsNull) { return null; } @@ -766,7 +780,7 @@ which is not possible within the Ethereum protocol which has keys of the same le return TraverseNext(in traverseContext, 1, nextNode); } - private byte[]? TraverseLeaf(TrieNode node, in TraverseContext traverseContext) + private CappedArray TraverseLeaf(TrieNode node, in TraverseContext traverseContext) { if (node.Key is null) { @@ -787,8 +801,8 @@ which is not possible within the Ethereum protocol which has keys of the same le longerPath = remaining; } - byte[] shorterPathValue; - byte[] longerPathValue; + CappedArray shorterPathValue; + CappedArray longerPathValue; if (Bytes.AreEqual(shorterPath, node.Key)) { @@ -869,7 +883,7 @@ which is not possible within the Ethereum protocol which has keys of the same le return traverseContext.UpdateValue; } - private byte[]? TraverseExtension(TrieNode node, in TraverseContext traverseContext) + private CappedArray TraverseExtension(TrieNode node, in TraverseContext traverseContext) { if (node.Key is null) { @@ -956,7 +970,7 @@ TrieNode secondExtension return traverseContext.UpdateValue; } - private byte[] TraverseNext(in TraverseContext traverseContext, int extensionLength, TrieNode next) + private CappedArray TraverseNext(in TraverseContext traverseContext, int extensionLength, TrieNode next) { // Move large struct creation out of flow so doesn't force additional stack space // in calling method even if not used @@ -978,11 +992,11 @@ private static int FindCommonPrefixLength(ReadOnlySpan shorterPath, ReadOn private readonly ref struct TraverseContext { - public byte[]? UpdateValue { get; } + public CappedArray UpdateValue { get; } public ReadOnlySpan UpdatePath { get; } public bool IsUpdate { get; } public bool IsRead => !IsUpdate; - public bool IsDelete => IsUpdate && UpdateValue is null; + public bool IsDelete => IsUpdate && UpdateValue.IsNull; public bool IgnoreMissingDelete { get; } public int CurrentIndex { get; } public int RemainingUpdatePathLength => UpdatePath.Length - CurrentIndex; @@ -1005,14 +1019,14 @@ public TraverseContext(scoped in TraverseContext context, int index) public TraverseContext( Span updatePath, - byte[]? updateValue, + CappedArray updateValue, bool isUpdate, bool ignoreMissingDelete = true) { UpdatePath = updatePath; - if (updateValue is not null && updateValue.Length == 0) + if (updateValue.IsNotNull && updateValue.Length == 0) { - updateValue = null; + updateValue = new CappedArray(null); } UpdateValue = updateValue; diff --git a/src/Nethermind/Nethermind.Trie/Pruning/TrieStore.cs b/src/Nethermind/Nethermind.Trie/Pruning/TrieStore.cs index 1b248d17cad..66c5cd18d88 100644 --- a/src/Nethermind/Nethermind.Trie/Pruning/TrieStore.cs +++ b/src/Nethermind/Nethermind.Trie/Pruning/TrieStore.cs @@ -8,10 +8,12 @@ using System.Threading; using System.Threading.Tasks; using Nethermind.Core; +using Nethermind.Core.Buffers; using Nethermind.Core.Collections; using Nethermind.Core.Crypto; using Nethermind.Core.Extensions; using Nethermind.Logging; +using Nethermind.Serialization.Rlp; namespace Nethermind.Trie.Pruning { @@ -61,7 +63,7 @@ public TrieNode FromCachedRlpOrUnknown(Keccak hash) // ReSharper disable once ConditionIsAlwaysTrueOrFalse if (_objectsCache.TryGetValue(hash, out TrieNode? trieNode)) { - if (trieNode!.FullRlp is null) + if (trieNode!.FullRlp.IsNull) { // // this happens in SyncProgressResolver // throw new InvalidAsynchronousStateException("Read only trie store is trying to read a transient node."); @@ -652,14 +654,14 @@ private void Persist(TrieNode currentNode, long blockNumber, WriteFlags writeFla // to prevent it from being removed from cache and also want to have it persisted. if (_logger.IsTrace) _logger.Trace($"Persisting {nameof(TrieNode)} {currentNode} in snapshot {blockNumber}."); - _currentBatch.Set(currentNode.Keccak.Bytes, currentNode.FullRlp, writeFlags); + _currentBatch.Set(currentNode.Keccak.Bytes, currentNode.FullRlp.ToArray(), writeFlags); currentNode.IsPersisted = true; currentNode.LastSeen = Math.Max(blockNumber, currentNode.LastSeen ?? 0); PersistedNodesCount++; } else { - Debug.Assert(currentNode.FullRlp is not null && currentNode.FullRlp.Length < 32, + Debug.Assert(currentNode.FullRlp.IsNotNull && currentNode.FullRlp.Length < 32, "We only expect persistence call without Keccak for the nodes that are kept inside the parent RLP (less than 32 bytes)."); } } @@ -795,7 +797,7 @@ void PersistNode(TrieNode n) Keccak? hash = n.Keccak; if (hash is not null) { - store[hash.Bytes] = n.FullRlp; + store[hash.Bytes] = n.FullRlp.ToArray(); int persistedNodesCount = Interlocked.Increment(ref persistedNodes); if (_logger.IsInfo && persistedNodesCount % million == 0) { @@ -824,8 +826,8 @@ void PersistNode(TrieNode n) && _dirtyNodes.AllNodes.TryGetValue(new ValueKeccak(key), out TrieNode? trieNode) && trieNode is not null && trieNode.NodeType != NodeType.Unknown - && trieNode.FullRlp is not null - ? trieNode.FullRlp + && trieNode.FullRlp.IsNotNull + ? trieNode.FullRlp.ToArray() : _currentBatch?.Get(key, flags) ?? _keyValueStore.Get(key, flags); } diff --git a/src/Nethermind/Nethermind.Trie/TrackingCappedArrayPool.cs b/src/Nethermind/Nethermind.Trie/TrackingCappedArrayPool.cs new file mode 100644 index 00000000000..a26edd82ad7 --- /dev/null +++ b/src/Nethermind/Nethermind.Trie/TrackingCappedArrayPool.cs @@ -0,0 +1,56 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using System.Buffers; +using System.Collections.Generic; +using Nethermind.Core.Buffers; +using Nethermind.Trie.Pruning; + +namespace Nethermind.Trie; + +/// +/// Track every rented CappedArray and return them all at once +/// +public class TrackingCappedArrayPool : ICappedArrayPool +{ + private List> _rentedBuffers; + private ArrayPool _arrayPool; + + public TrackingCappedArrayPool() : this(0) + { + } + + public TrackingCappedArrayPool(int initialCapacity, ArrayPool arrayPool = null) + { + _rentedBuffers = new List>(initialCapacity); + _arrayPool = arrayPool ?? ArrayPool.Shared; + } + + public CappedArray Rent(int size) + { + if (size == 0) + { + return new CappedArray(Array.Empty()); + } + + CappedArray rented = new CappedArray(_arrayPool.Rent(size), size); + rented.AsSpan().Fill(0); + _rentedBuffers.Add(rented); + return rented; + } + + public void Return(CappedArray buffer) + { + } + + public void ReturnAll() + { + foreach (CappedArray rentedBuffer in _rentedBuffers) + { + if (rentedBuffer.IsNotNull && rentedBuffer.Array.Length != 0) + _arrayPool.Return(rentedBuffer.Array); + } + _rentedBuffers.Clear(); + } +} diff --git a/src/Nethermind/Nethermind.Trie/TreeDumper.cs b/src/Nethermind/Nethermind.Trie/TreeDumper.cs index 9ad9642e994..a7e020944fc 100644 --- a/src/Nethermind/Nethermind.Trie/TreeDumper.cs +++ b/src/Nethermind/Nethermind.Trie/TreeDumper.cs @@ -88,7 +88,7 @@ public override string ToString() private string? KeccakOrRlpStringOfNode(TrieNode node) { - return node.Keccak != null ? node.Keccak!.Bytes.ToHexString() : node.FullRlp?.ToHexString(); + return node.Keccak != null ? node.Keccak!.Bytes.ToHexString() : node.FullRlp.AsSpan().ToHexString(); } } } diff --git a/src/Nethermind/Nethermind.Trie/TrieNode.Decoder.cs b/src/Nethermind/Nethermind.Trie/TrieNode.Decoder.cs index 19b77aba03c..4676830589e 100644 --- a/src/Nethermind/Nethermind.Trie/TrieNode.Decoder.cs +++ b/src/Nethermind/Nethermind.Trie/TrieNode.Decoder.cs @@ -4,7 +4,9 @@ using System; using System.Buffers; using System.Diagnostics; +using System.Linq; using System.Runtime.CompilerServices; +using Nethermind.Core.Buffers; using Nethermind.Core.Crypto; using Nethermind.Serialization.Rlp; using Nethermind.Trie.Pruning; @@ -21,7 +23,7 @@ public partial class TrieNode private class TrieNodeDecoder { - public byte[] Encode(ITrieNodeResolver tree, TrieNode? item) + public CappedArray Encode(ITrieNodeResolver tree, TrieNode? item, ICappedArrayPool? bufferPool) { Metrics.TreeNodeRlpEncodings++; @@ -32,15 +34,15 @@ public byte[] Encode(ITrieNodeResolver tree, TrieNode? item) return item.NodeType switch { - NodeType.Branch => RlpEncodeBranch(tree, item), - NodeType.Extension => EncodeExtension(tree, item), - NodeType.Leaf => EncodeLeaf(item), + NodeType.Branch => RlpEncodeBranch(tree, item, bufferPool), + NodeType.Extension => EncodeExtension(tree, item, bufferPool), + NodeType.Leaf => EncodeLeaf(item, bufferPool), _ => throw new TrieException($"An attempt was made to encode a trie node of type {item.NodeType}") }; } [SkipLocalsInit] - private static byte[] EncodeExtension(ITrieNodeResolver tree, TrieNode item) + private static CappedArray EncodeExtension(ITrieNodeResolver tree, TrieNode item, ICappedArrayPool? bufferPool) { Debug.Assert(item.NodeType == NodeType.Extension, $"Node passed to {nameof(EncodeExtension)} is {item.NodeType}"); @@ -63,11 +65,13 @@ private static byte[] EncodeExtension(ITrieNodeResolver tree, TrieNode item) Debug.Assert(nodeRef is not null, "Extension child is null when encoding."); - nodeRef.ResolveKey(tree, false); + nodeRef.ResolveKey(tree, false, bufferPool: bufferPool); int contentLength = Rlp.LengthOf(keyBytes) + (nodeRef.Keccak is null ? nodeRef.FullRlp.Length : Rlp.LengthOfKeccakRlp); int totalLength = Rlp.LengthOfSequence(contentLength); - RlpStream rlpStream = new(totalLength); + + CappedArray data = bufferPool.SafeRentBuffer(totalLength); + RlpStream rlpStream = data.AsRlpStream(); rlpStream.StartSequence(contentLength); rlpStream.Encode(keyBytes); if (rentedBuffer is not null) @@ -82,18 +86,18 @@ private static byte[] EncodeExtension(ITrieNodeResolver tree, TrieNode item) // so E - - - - - - - - - - - - - - - // so | // so | - rlpStream.Write(nodeRef.FullRlp); + rlpStream.Write(nodeRef.FullRlp.AsSpan()); } else { rlpStream.Encode(nodeRef.Keccak); } - return rlpStream.Data; + return data; } [SkipLocalsInit] - private static byte[] EncodeLeaf(TrieNode node) + private static CappedArray EncodeLeaf(TrieNode node, ICappedArrayPool? pool) { if (node.Key is null) { @@ -111,42 +115,44 @@ private static byte[] EncodeLeaf(TrieNode node) : rentedBuffer)[..hexLength]; HexPrefix.CopyToSpan(hexPrefix, isLeaf: true, keyBytes); - int contentLength = Rlp.LengthOf(keyBytes) + Rlp.LengthOf(node.Value); + int contentLength = Rlp.LengthOf(keyBytes) + Rlp.LengthOf(node.Value.AsSpan()); int totalLength = Rlp.LengthOfSequence(contentLength); - RlpStream rlpStream = new(totalLength); + + CappedArray data = pool.SafeRentBuffer(totalLength); + RlpStream rlpStream = data.AsRlpStream(); rlpStream.StartSequence(contentLength); rlpStream.Encode(keyBytes); if (rentedBuffer is not null) { ArrayPool.Shared.Return(rentedBuffer); } - rlpStream.Encode(node.Value); - return rlpStream.Data; + rlpStream.Encode(node.Value.AsSpan()); + return data; } - private static byte[] RlpEncodeBranch(ITrieNodeResolver tree, TrieNode item) + private static CappedArray RlpEncodeBranch(ITrieNodeResolver tree, TrieNode item, ICappedArrayPool? pool) { - int valueRlpLength = AllowBranchValues ? Rlp.LengthOf(item.Value) : 1; - int contentLength = valueRlpLength + GetChildrenRlpLength(tree, item); + int valueRlpLength = AllowBranchValues ? Rlp.LengthOf(item.Value.AsSpan()) : 1; + int contentLength = valueRlpLength + GetChildrenRlpLength(tree, item, pool); int sequenceLength = Rlp.LengthOfSequence(contentLength); - byte[] result = new byte[sequenceLength]; + CappedArray result = pool.SafeRentBuffer(sequenceLength); Span resultSpan = result.AsSpan(); - int position = Rlp.StartSequence(result, 0, contentLength); - WriteChildrenRlp(tree, item, resultSpan.Slice(position, contentLength - valueRlpLength)); + int position = Rlp.StartSequence(resultSpan, 0, contentLength); + WriteChildrenRlp(tree, item, resultSpan.Slice(position, contentLength - valueRlpLength), pool); position = sequenceLength - valueRlpLength; if (AllowBranchValues) { - Rlp.Encode(result, position, item.Value); + Rlp.Encode(resultSpan, position, item.Value); } else { - result[position] = 128; + result.AsSpan()[position] = 128; } return result; } - private static int GetChildrenRlpLength(ITrieNodeResolver tree, TrieNode item) + private static int GetChildrenRlpLength(ITrieNodeResolver tree, TrieNode item, ICappedArrayPool? bufferPool) { int totalLength = 0; item.InitData(); @@ -171,8 +177,8 @@ private static int GetChildrenRlpLength(ITrieNodeResolver tree, TrieNode item) else { TrieNode childNode = (TrieNode)item._data[i]; - childNode!.ResolveKey(tree, false); - totalLength += childNode.Keccak is null ? childNode.FullRlp!.Length : Rlp.LengthOfKeccakRlp; + childNode!.ResolveKey(tree, false, bufferPool: bufferPool); + totalLength += childNode.Keccak is null ? childNode.FullRlp.Length : Rlp.LengthOfKeccakRlp; } } @@ -182,7 +188,7 @@ private static int GetChildrenRlpLength(ITrieNodeResolver tree, TrieNode item) return totalLength; } - private static void WriteChildrenRlp(ITrieNodeResolver tree, TrieNode item, Span destination) + private static void WriteChildrenRlp(ITrieNodeResolver tree, TrieNode item, Span destination, ICappedArrayPool? bufferPool) { int position = 0; RlpStream rlpStream = item._rlpStream; @@ -212,10 +218,10 @@ private static void WriteChildrenRlp(ITrieNodeResolver tree, TrieNode item, Span else { TrieNode childNode = (TrieNode)item._data[i]; - childNode!.ResolveKey(tree, false); + childNode!.ResolveKey(tree, false, bufferPool: bufferPool); if (childNode.Keccak is null) { - Span fullRlp = childNode.FullRlp.AsSpan(); + Span fullRlp = childNode.FullRlp!.AsSpan(); fullRlp.CopyTo(destination.Slice(position, fullRlp.Length)); position += fullRlp.Length; } diff --git a/src/Nethermind/Nethermind.Trie/TrieNode.Visitor.cs b/src/Nethermind/Nethermind.Trie/TrieNode.Visitor.cs index b0452aa22b5..2fd7fcc6a0c 100644 --- a/src/Nethermind/Nethermind.Trie/TrieNode.Visitor.cs +++ b/src/Nethermind/Nethermind.Trie/TrieNode.Visitor.cs @@ -6,6 +6,7 @@ using System.Runtime.CompilerServices; using System.Threading.Tasks; using Nethermind.Core; +using Nethermind.Core.Buffers; using Nethermind.Serialization.Rlp; using Nethermind.Trie.Pruning; @@ -84,7 +85,7 @@ internal void AcceptResolvedNode(ITreeVisitor visitor, ITrieNodeResolver nodeRes case NodeType.Leaf: { - visitor.VisitLeaf(this, trieVisitContext.ToVisitContext(), Value); + visitor.VisitLeaf(this, trieVisitContext.ToVisitContext(), Value.ToArray()); if (!trieVisitContext.IsStorage && trieVisitContext.ExpectAccounts) // can combine these conditions { @@ -245,7 +246,7 @@ void VisitMultiThread(ITreeVisitor treeVisitor, ITrieNodeResolver trieNodeResolv case NodeType.Leaf: { - visitor.VisitLeaf(this, trieVisitContext, Value); + visitor.VisitLeaf(this, trieVisitContext, Value.ToArray()); trieVisitContext.AddVisited(); if (!trieVisitContext.IsStorage && trieVisitContext.ExpectAccounts) // can combine these conditions { diff --git a/src/Nethermind/Nethermind.Trie/TrieNode.cs b/src/Nethermind/Nethermind.Trie/TrieNode.cs index 2714abe2dd0..92616812300 100644 --- a/src/Nethermind/Nethermind.Trie/TrieNode.cs +++ b/src/Nethermind/Nethermind.Trie/TrieNode.cs @@ -6,6 +6,7 @@ using System.Runtime.CompilerServices; using System.Threading; using Nethermind.Core; +using Nethermind.Core.Buffers; using Nethermind.Core.Crypto; using Nethermind.Core.Extensions; using Nethermind.Logging; @@ -53,7 +54,7 @@ public partial class TrieNode public Keccak? Keccak { get; internal set; } - public byte[]? FullRlp { get; internal set; } + public CappedArray FullRlp { get; internal set; } public NodeType NodeType { get; private set; } @@ -85,36 +86,58 @@ internal set /// /// Highly optimized /// - public byte[]? Value + public CappedArray Value { get { InitData(); + object? obj; + if (IsLeaf) { - return (byte[])_data![1]; + obj = _data![1]; + + if (obj is null) + { + return new CappedArray(null); + } + + if (obj is byte[] asBytes) + { + return new CappedArray(asBytes); + } + + return (CappedArray)obj; } if (!AllowBranchValues) { // branches that we use for state will never have value set as all the keys are equal length - return Array.Empty(); + return new CappedArray(Array.Empty()); } - if (_data![BranchesCount] is null) + obj = _data![BranchesCount]; + if (obj is null) { if (_rlpStream is null) { _data[BranchesCount] = Array.Empty(); + return new CappedArray(Array.Empty()); } else { SeekChild(BranchesCount); - _data![BranchesCount] = _rlpStream!.DecodeByteArray(); + byte[]? bArr = _rlpStream!.DecodeByteArray(); + _data![BranchesCount] = bArr; + return new CappedArray(bArr); } } - return (byte[])_data[BranchesCount]; + if (obj is byte[] asBytes2) + { + return new CappedArray(asBytes2); + } + return (CappedArray)obj; } set @@ -133,6 +156,19 @@ public byte[]? Value throw new TrieException("Optimized Patricia Trie does not support setting values on branches."); } + if (value.IsNull) + { + _data![IsLeaf ? 1 : BranchesCount] = null; + return; + } + + if (value.IsUncapped) + { + // Store array directly if possible to reduce memory + _data![IsLeaf ? 1 : BranchesCount] = value.Array; + return; + } + _data![IsLeaf ? 1 : BranchesCount] = value; } } @@ -157,7 +193,7 @@ public bool IsValidWithOneNodeLess if (AllowBranchValues) { - nonEmptyNodes += (Value?.Length ?? 0) > 0 ? 1 : 0; + nonEmptyNodes += Value.Length > 0 ? 1 : 0; } return nonEmptyNodes > 2; @@ -180,7 +216,7 @@ public TrieNode(NodeType nodeType, Keccak keccak) } } - public TrieNode(NodeType nodeType, byte[] rlp, bool isDirty = false) + public TrieNode(NodeType nodeType, CappedArray rlp, bool isDirty = false) { NodeType = nodeType; FullRlp = rlp; @@ -189,12 +225,16 @@ public TrieNode(NodeType nodeType, byte[] rlp, bool isDirty = false) _rlpStream = rlp.AsRlpStream(); } + public TrieNode(NodeType nodeType, byte[]? rlp, bool isDirty = false) : this(nodeType, new CappedArray(rlp), isDirty) + { + } + public TrieNode(NodeType nodeType, Keccak keccak, ReadOnlySpan rlp) - : this(nodeType, keccak, rlp.ToArray()) + : this(nodeType, keccak, new CappedArray(rlp.ToArray())) { } - public TrieNode(NodeType nodeType, Keccak keccak, byte[] rlp) + public TrieNode(NodeType nodeType, Keccak keccak, CappedArray rlp) : this(nodeType, rlp) { Keccak = keccak; @@ -208,10 +248,10 @@ public override string ToString() { #if DEBUG return - $"[{NodeType}({FullRlp?.Length}){(FullRlp is not null && FullRlp?.Length < 32 ? $"{FullRlp.ToHexString()}" : "")}" + + $"[{NodeType}({FullRlp.Length}){(FullRlp.IsNotNull && FullRlp.Length < 32 ? $"{FullRlp.AsSpan().ToHexString()}" : "")}" + $"|{Id}|{Keccak}|{LastSeen}|D:{IsDirty}|S:{IsSealed}|P:{IsPersisted}|"; #else - return $"[{NodeType}({FullRlp?.Length})|{Keccak?.ToShortString()}|{LastSeen}|D:{IsDirty}|S:{IsSealed}|P:{IsPersisted}|"; + return $"[{NodeType}({FullRlp.Length})|{Keccak?.ToShortString()}|{LastSeen}|D:{IsDirty}|S:{IsSealed}|P:{IsPersisted}|"; #endif } @@ -231,13 +271,13 @@ public void Seal() /// /// Highly optimized /// - public void ResolveNode(ITrieNodeResolver tree, ReadFlags readFlags = ReadFlags.None) + public void ResolveNode(ITrieNodeResolver tree, ReadFlags readFlags = ReadFlags.None, ICappedArrayPool? bufferPool = null) { try { if (NodeType == NodeType.Unknown) { - if (FullRlp is null) + if (FullRlp.IsNull) { if (Keccak is null) { @@ -247,7 +287,7 @@ public void ResolveNode(ITrieNodeResolver tree, ReadFlags readFlags = ReadFlags. FullRlp = tree.LoadRlp(Keccak, readFlags); IsPersisted = true; - if (FullRlp is null) + if (FullRlp.IsNull) { throw new TrieException($"Trie returned a NULL RLP for node {Keccak}"); } @@ -287,7 +327,11 @@ public void ResolveNode(ITrieNodeResolver tree, ReadFlags readFlags = ReadFlags. { NodeType = NodeType.Leaf; Key = key; - Value = _rlpStream.DecodeByteArray(); + + ReadOnlySpan valueSpan = _rlpStream.DecodeByteArraySpan(); + CappedArray buffer = bufferPool.SafeRentBuffer(valueSpan.Length); + valueSpan.CopyTo(buffer.AsSpan()); + Value = buffer; } else { @@ -299,7 +343,7 @@ public void ResolveNode(ITrieNodeResolver tree, ReadFlags readFlags = ReadFlags. } else { - throw new TrieNodeException($"Unexpected number of items = {numberOfItems} when decoding a node from RLP ({FullRlp?.ToHexString()})", Keccak ?? Keccak.Zero); + throw new TrieNodeException($"Unexpected number of items = {numberOfItems} when decoding a node from RLP ({FullRlp.AsSpan().ToHexString()})", Keccak ?? Keccak.Zero); } } catch (RlpException rlpException) @@ -308,7 +352,7 @@ public void ResolveNode(ITrieNodeResolver tree, ReadFlags readFlags = ReadFlags. } } - public void ResolveKey(ITrieNodeResolver tree, bool isRoot) + public void ResolveKey(ITrieNodeResolver tree, bool isRoot, ICappedArrayPool? bufferPool = null) { if (Keccak is not null) { @@ -317,10 +361,10 @@ public void ResolveKey(ITrieNodeResolver tree, bool isRoot) return; } - Keccak = GenerateKey(tree, isRoot); + Keccak = GenerateKey(tree, isRoot, bufferPool); } - public Keccak? GenerateKey(ITrieNodeResolver tree, bool isRoot) + public Keccak? GenerateKey(ITrieNodeResolver tree, bool isRoot, ICappedArrayPool? bufferPool = null) { Keccak? keccak = Keccak; if (keccak is not null) @@ -328,9 +372,14 @@ public void ResolveKey(ITrieNodeResolver tree, bool isRoot) return keccak; } - if (FullRlp is null || IsDirty) + if (FullRlp.IsNull || IsDirty) { - FullRlp = RlpEncode(tree); + CappedArray oldRlp = FullRlp; + FullRlp = RlpEncode(tree, bufferPool); + if (oldRlp.IsNotNull) + { + bufferPool.SafeReturnBuffer(oldRlp); + } _rlpStream = FullRlp.AsRlpStream(); } @@ -340,7 +389,7 @@ public void ResolveKey(ITrieNodeResolver tree, bool isRoot) if (FullRlp.Length >= 32 || isRoot) { Metrics.TreeNodeHashCalculations++; - return Keccak.Compute(FullRlp); + return Keccak.Compute(FullRlp.AsSpan()); } return null; @@ -369,9 +418,9 @@ public bool TryResolveStorageRootHash(ITrieNodeResolver resolver, out Keccak? st return false; } - internal byte[] RlpEncode(ITrieNodeResolver tree) + internal CappedArray RlpEncode(ITrieNodeResolver tree, ICappedArrayPool? bufferPool = null) { - byte[] rlp = _nodeDecoder.Encode(tree, this); + CappedArray rlp = _nodeDecoder.Encode(tree, this, bufferPool); // just included here to improve the class reading // after some analysis I believe that any non-test Ethereum cases of a trie ever have nodes with RLP shorter than 32 bytes // if (rlp.Bytes.Length < 32) @@ -495,8 +544,8 @@ public TrieNode? this[int i] { // we expect this to happen as a Trie traversal error (please see the stack trace above) // we need to investigate this case when it happens again - bool isKeccakCalculated = Keccak is not null && FullRlp is not null; - bool isKeccakCorrect = isKeccakCalculated && Keccak == Keccak.Compute(FullRlp); + bool isKeccakCalculated = Keccak is not null && FullRlp.IsNotNull; + bool isKeccakCorrect = isKeccakCalculated && Keccak == Keccak.Compute(FullRlp.AsSpan()); throw new TrieException($"Unexpected type found at position {childIndex} of {this} with {nameof(_data)} of length {_data?.Length}. Expected a {nameof(TrieNode)} or {nameof(Keccak)} but found {childOrRef?.GetType()} with a value of {childOrRef}. Keccak calculated? : {isKeccakCalculated}; Keccak correct? : {isKeccakCorrect}"); } @@ -543,10 +592,10 @@ Keccak is null : MemorySizes.RefSize + Keccak.MemorySize; long fullRlpSize = MemorySizes.RefSize + - (FullRlp is null ? 0 : MemorySizes.Align(FullRlp.Length + MemorySizes.ArrayOverhead)); + (FullRlp.IsNull ? 0 : MemorySizes.Align(FullRlp.Array.Length + MemorySizes.ArrayOverhead)); long rlpStreamSize = MemorySizes.RefSize + (_rlpStream?.MemorySize ?? 0) - - (FullRlp is null ? 0 : MemorySizes.Align(FullRlp.Length + MemorySizes.ArrayOverhead)); + - (FullRlp.IsNull ? 0 : MemorySizes.Align(FullRlp.Array.Length + MemorySizes.ArrayOverhead)); long dataSize = MemorySizes.RefSize + (_data is null @@ -574,6 +623,11 @@ Keccak is null dataSize += MemorySizes.ArrayOverhead + array.Length; } + if (_data![i] is CappedArray cappedArray) + { + dataSize += MemorySizes.ArrayOverhead + (cappedArray.Array?.Length ?? 0) + MemorySizes.SmallObjectOverhead; + } + if (recursive) { if (_data![i] is TrieNode node) @@ -613,7 +667,7 @@ public TrieNode Clone() } } - if (FullRlp is not null) + if (FullRlp.IsNotNull) { trieNode.FullRlp = FullRlp; trieNode._rlpStream = FullRlp.AsRlpStream(); @@ -622,14 +676,14 @@ public TrieNode Clone() return trieNode; } - public TrieNode CloneWithChangedValue(byte[]? changedValue) + public TrieNode CloneWithChangedValue(CappedArray changedValue) { TrieNode trieNode = Clone(); trieNode.Value = changedValue; return trieNode; } - public TrieNode CloneWithChangedKeyAndValue(byte[] key, byte[]? changedValue) + public TrieNode CloneWithChangedKeyAndValue(byte[] key, CappedArray changedValue) { TrieNode trieNode = Clone(); trieNode.Key = key; @@ -762,7 +816,7 @@ private bool TryResolveStorageRoot(ITrieNodeResolver resolver, out TrieNode? sto { hasStorage = true; } - else if (Value?.Length > 64) // if not a storage leaf + else if (Value.Length > 64) // if not a storage leaf { Keccak storageRootKey = _accountDecoder.DecodeStorageRootOnly(Value.AsRlpStream()); if (storageRootKey != Keccak.EmptyTreeHash) diff --git a/src/Nethermind/Nethermind.Trie/TrieNodeFactory.cs b/src/Nethermind/Nethermind.Trie/TrieNodeFactory.cs index f297c616cdb..7d9fbc72d44 100644 --- a/src/Nethermind/Nethermind.Trie/TrieNodeFactory.cs +++ b/src/Nethermind/Nethermind.Trie/TrieNodeFactory.cs @@ -1,6 +1,8 @@ // SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only +using Nethermind.Core.Buffers; + namespace Nethermind.Trie { internal static class TrieNodeFactory @@ -11,7 +13,7 @@ public static TrieNode CreateBranch() return node; } - public static TrieNode CreateLeaf(byte[] path, byte[]? value) + public static TrieNode CreateLeaf(byte[] path, CappedArray value) { TrieNode node = new(NodeType.Leaf); node.Key = path; diff --git a/src/Nethermind/Nethermind.Trie/TrieStatsCollector.cs b/src/Nethermind/Nethermind.Trie/TrieStatsCollector.cs index 0ed1c52f121..3158f68ee9a 100644 --- a/src/Nethermind/Nethermind.Trie/TrieStatsCollector.cs +++ b/src/Nethermind/Nethermind.Trie/TrieStatsCollector.cs @@ -51,12 +51,12 @@ public void VisitBranch(TrieNode node, TrieVisitContext trieVisitContext) { if (trieVisitContext.IsStorage) { - Interlocked.Add(ref Stats._storageSize, node.FullRlp?.Length ?? 0); + Interlocked.Add(ref Stats._storageSize, node.FullRlp.Length); Interlocked.Increment(ref Stats._storageBranchCount); } else { - Interlocked.Add(ref Stats._stateSize, node.FullRlp?.Length ?? 0); + Interlocked.Add(ref Stats._stateSize, node.FullRlp.Length); Interlocked.Increment(ref Stats._stateBranchCount); } @@ -67,12 +67,12 @@ public void VisitExtension(TrieNode node, TrieVisitContext trieVisitContext) { if (trieVisitContext.IsStorage) { - Interlocked.Add(ref Stats._storageSize, node.FullRlp?.Length ?? 0); + Interlocked.Add(ref Stats._storageSize, node.FullRlp.Length); Interlocked.Increment(ref Stats._storageExtensionCount); } else { - Interlocked.Add(ref Stats._stateSize, node.FullRlp?.Length ?? 0); + Interlocked.Add(ref Stats._stateSize, node.FullRlp.Length); Interlocked.Increment(ref Stats._stateExtensionCount); } @@ -89,12 +89,12 @@ public void VisitLeaf(TrieNode node, TrieVisitContext trieVisitContext, byte[] v if (trieVisitContext.IsStorage) { - Interlocked.Add(ref Stats._storageSize, node.FullRlp?.Length ?? 0); + Interlocked.Add(ref Stats._storageSize, node.FullRlp.Length); Interlocked.Increment(ref Stats._storageLeafCount); } else { - Interlocked.Add(ref Stats._stateSize, node.FullRlp?.Length ?? 0); + Interlocked.Add(ref Stats._stateSize, node.FullRlp.Length); Interlocked.Increment(ref Stats._accountCount); }