diff --git a/Cargo.toml b/Cargo.toml index 17e25e98..76a014d2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,5 +10,6 @@ members = [ "visualize", "path", "grovedbg-types", - "grovedb-version" + "grovedb-version", + "grovedb-epoch-based-storage-flags" ] diff --git a/costs/Cargo.toml b/costs/Cargo.toml index 4ee1a55b..47f72e3f 100644 --- a/costs/Cargo.toml +++ b/costs/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "grovedb-costs" -version = "1.0.0" +version = "2.0.0" edition = "2021" license = "MIT" description = "Costs extension crate for GroveDB" diff --git a/grovedb-epoch-based-storage-flags/Cargo.toml b/grovedb-epoch-based-storage-flags/Cargo.toml new file mode 100644 index 00000000..8a48f131 --- /dev/null +++ b/grovedb-epoch-based-storage-flags/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "grovedb-epoch-based-storage-flags" +authors = ["Samuel Westrich "] +description = "Epoch based storage flags for GroveDB" +version = "2.0.0" +edition = "2021" +license = "MIT" +repository = "https://github.com/dashpay/grovedb" + +[dependencies] +thiserror = { version = "1.0.63" } +grovedb-costs = { version = "2.0.0", path = "../costs" } +intmap = { version = "2.0.0", features = ["serde"]} +integer-encoding = { version = "4.0.0" } +hex = { version = "0.4.3" } \ No newline at end of file diff --git a/grovedb-epoch-based-storage-flags/src/error.rs b/grovedb-epoch-based-storage-flags/src/error.rs new file mode 100644 index 00000000..006c918d --- /dev/null +++ b/grovedb-epoch-based-storage-flags/src/error.rs @@ -0,0 +1,45 @@ +/// Storage flag errors +#[derive(Debug, thiserror::Error)] +pub enum StorageFlagsError { + /// Error + #[error("deserialize unknown storage flags type error: {0}")] + DeserializeUnknownStorageFlagsType(String), + /// Error + #[error("storage flags wrong size error: {0}")] + StorageFlagsWrongSize(String), + /// Error + #[error("removing at epoch with no associated storage error: {0}")] + RemovingAtEpochWithNoAssociatedStorage(String), + /// Error + #[error("storage flags overflow error: {0}")] + StorageFlagsOverflow(String), + /// Error + #[error("removing flags error: {0}")] + RemovingFlagsError(String), + /// Error + #[error("merging storage flags from different owners error: {0}")] + MergingStorageFlagsFromDifferentOwners(String), + /// Error + #[error("merging storage flags with different base epoch: {0}")] + MergingStorageFlagsWithDifferentBaseEpoch(String), +} + +impl StorageFlagsError { + /// Gets a mutable reference to the inner string of the error variant + pub(crate) fn get_mut_info(&mut self) -> &mut String { + match self { + StorageFlagsError::DeserializeUnknownStorageFlagsType(ref mut msg) + | StorageFlagsError::StorageFlagsWrongSize(ref mut msg) + | StorageFlagsError::RemovingAtEpochWithNoAssociatedStorage(ref mut msg) + | StorageFlagsError::StorageFlagsOverflow(ref mut msg) + | StorageFlagsError::RemovingFlagsError(ref mut msg) + | StorageFlagsError::MergingStorageFlagsFromDifferentOwners(ref mut msg) + | StorageFlagsError::MergingStorageFlagsWithDifferentBaseEpoch(ref mut msg) => msg, + } + } + + /// adds info to the storage flags error + pub(crate) fn add_info(&mut self, info: &str) { + self.get_mut_info().push_str(format!(": {}", info).as_str()); + } +} diff --git a/grovedb-epoch-based-storage-flags/src/lib.rs b/grovedb-epoch-based-storage-flags/src/lib.rs new file mode 100644 index 00000000..add8765b --- /dev/null +++ b/grovedb-epoch-based-storage-flags/src/lib.rs @@ -0,0 +1,1588 @@ +//! Flags + +pub mod error; +mod split_removal_bytes; +mod update_element_flags; + +use crate::{ + error::StorageFlagsError, + StorageFlags::{MultiEpoch, MultiEpochOwned, SingleEpoch, SingleEpochOwned}, +}; + +const DEFAULT_HASH_SIZE_U32: u32 = 32; + +/// Optional meta-data to be stored per element +pub type ElementFlags = Vec; + +use std::{borrow::Cow, cmp::Ordering, collections::BTreeMap, fmt}; + +use grovedb_costs::storage_cost::removal::{ + StorageRemovalPerEpochByIdentifier, StorageRemovedBytes, + StorageRemovedBytes::{NoStorageRemoval, SectionedStorageRemoval}, +}; +use integer_encoding::VarInt; +use intmap::IntMap; + +type EpochIndex = u16; + +type BaseEpoch = EpochIndex; + +type BytesAddedInEpoch = u32; + +type OwnerId = [u8; 32]; + +/// The size of single epoch flags +pub const SINGLE_EPOCH_FLAGS_SIZE: u32 = 3; + +/// The minimum size of the non-base flags +pub const MINIMUM_NON_BASE_FLAGS_SIZE: u32 = 3; + +/// Storage flags +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum StorageFlags { + /// Single epoch + /// represented as byte 0 + SingleEpoch(BaseEpoch), + + /// Multi epoch + /// represented as byte 1 + MultiEpoch(BaseEpoch, BTreeMap), + + /// Single epoch owned + /// represented as byte 2 + SingleEpochOwned(BaseEpoch, OwnerId), + + /// Multi epoch owned + /// represented as byte 3 + MultiEpochOwned(BaseEpoch, BTreeMap, OwnerId), +} + +impl fmt::Display for StorageFlags { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + StorageFlags::SingleEpoch(base_epoch) => { + write!(f, "SingleEpoch(BaseEpoch: {})", base_epoch) + } + StorageFlags::MultiEpoch(base_epoch, epochs) => { + write!(f, "MultiEpoch(BaseEpoch: {}, Epochs: ", base_epoch)?; + for (index, bytes) in epochs { + write!(f, "[EpochIndex: {}, BytesAdded: {}] ", index, bytes)?; + } + write!(f, ")") + } + StorageFlags::SingleEpochOwned(base_epoch, owner_id) => { + write!( + f, + "SingleEpochOwned(BaseEpoch: {}, OwnerId: {})", + base_epoch, + hex::encode(owner_id) + ) + } + StorageFlags::MultiEpochOwned(base_epoch, epochs, owner_id) => { + write!(f, "MultiEpochOwned(BaseEpoch: {}, Epochs: ", base_epoch)?; + for (index, bytes) in epochs { + write!(f, "[EpochIndex: {}, BytesAdded: {}] ", index, bytes)?; + } + write!(f, ", OwnerId: {})", hex::encode(owner_id)) + } + } + } +} + +/// MergingOwnersStrategy decides which owner to keep during a merge +#[derive(Clone, Copy, Debug, PartialEq, Eq, Default)] +pub enum MergingOwnersStrategy { + #[default] + /// Raise an issue that owners of nodes are different + RaiseIssue, + /// Use the original owner id + UseOurs, + /// Use the new owner id + UseTheirs, +} + +impl StorageFlags { + /// Create new single epoch storage flags + pub fn new_single_epoch(epoch: BaseEpoch, maybe_owner_id: Option) -> Self { + match maybe_owner_id { + None => SingleEpoch(epoch), + Some(owner_id) => SingleEpochOwned(epoch, owner_id), + } + } + + /// Sets the owner id if we have owned storage flags + pub fn set_owner_id(&mut self, owner_id: OwnerId) { + match self { + SingleEpochOwned(_, previous_owner_id) | MultiEpochOwned(_, _, previous_owner_id) => { + *previous_owner_id = owner_id; + } + _ => {} + } + } + + fn combine_owner_id<'a>( + &'a self, + rhs: &'a Self, + merging_owners_strategy: MergingOwnersStrategy, + ) -> Result, StorageFlagsError> { + if let Some(our_owner_id) = self.owner_id() { + if let Some(other_owner_id) = rhs.owner_id() { + if our_owner_id != other_owner_id { + match merging_owners_strategy { + MergingOwnersStrategy::RaiseIssue => { + Err(StorageFlagsError::MergingStorageFlagsFromDifferentOwners( + "can not merge from different owners".to_string(), + )) + } + MergingOwnersStrategy::UseOurs => Ok(Some(our_owner_id)), + MergingOwnersStrategy::UseTheirs => Ok(Some(other_owner_id)), + } + } else { + Ok(Some(our_owner_id)) + } + } else { + Ok(Some(our_owner_id)) + } + } else if let Some(other_owner_id) = rhs.owner_id() { + Ok(Some(other_owner_id)) + } else { + Ok(None) + } + } + + fn combine_non_base_epoch_bytes( + &self, + rhs: &Self, + ) -> Option> { + if let Some(our_epoch_index_map) = self.epoch_index_map() { + if let Some(other_epoch_index_map) = rhs.epoch_index_map() { + let mut combined_index_map = our_epoch_index_map.clone(); + other_epoch_index_map + .iter() + .for_each(|(epoch_index, bytes_added)| { + // Simply insert the value from rhs, overwriting any existing value + combined_index_map.insert(*epoch_index, *bytes_added); + }); + // println!( + // " >combine_non_base_epoch_bytes: self:{:?} & rhs:{:?} -> {:?}", + // our_epoch_index_map, other_epoch_index_map, combined_index_map + // ); + Some(combined_index_map) + } else { + Some(our_epoch_index_map.clone()) + } + } else { + rhs.epoch_index_map().cloned() + } + } + + fn combine_same_base_epoch( + &self, + rhs: Self, + merging_owners_strategy: MergingOwnersStrategy, + ) -> Result { + let base_epoch = *self.base_epoch(); + let owner_id = self.combine_owner_id(&rhs, merging_owners_strategy)?; + let other_epoch_bytes = self.combine_non_base_epoch_bytes(&rhs); + + match (owner_id, other_epoch_bytes) { + (None, None) => Ok(SingleEpoch(base_epoch)), + (Some(owner_id), None) => Ok(SingleEpochOwned(base_epoch, *owner_id)), + (None, Some(other_epoch_bytes)) => Ok(MultiEpoch(base_epoch, other_epoch_bytes)), + (Some(owner_id), Some(other_epoch_bytes)) => { + Ok(MultiEpochOwned(base_epoch, other_epoch_bytes, *owner_id)) + } + } + } + + fn combine_with_higher_base_epoch( + &self, + rhs: Self, + added_bytes: u32, + merging_owners_strategy: MergingOwnersStrategy, + ) -> Result { + let base_epoch = *self.base_epoch(); + let epoch_with_adding_bytes = rhs.base_epoch(); + let owner_id = self.combine_owner_id(&rhs, merging_owners_strategy)?; + let mut other_epoch_bytes = self.combine_non_base_epoch_bytes(&rhs).unwrap_or_default(); + let original_value = other_epoch_bytes.remove(epoch_with_adding_bytes); + match original_value { + None => other_epoch_bytes.insert(*epoch_with_adding_bytes, added_bytes), + Some(original_bytes) => { + other_epoch_bytes.insert(*epoch_with_adding_bytes, original_bytes + added_bytes) + } + }; + // println!( + // " >combine_with_higher_base_epoch added_bytes:{} self:{:?} & + // rhs:{:?} -> {:?}", added_bytes, + // self.epoch_index_map(), + // rhs.epoch_index_map(), + // other_epoch_bytes + // ); + + match owner_id { + None => Ok(MultiEpoch(base_epoch, other_epoch_bytes)), + Some(owner_id) => Ok(MultiEpochOwned(base_epoch, other_epoch_bytes, *owner_id)), + } + } + + fn combine_with_higher_base_epoch_remove_bytes( + self, + rhs: Self, + removed_bytes: &StorageRemovedBytes, + merging_owners_strategy: MergingOwnersStrategy, + ) -> Result { + if matches!(&self, &SingleEpoch(_) | &SingleEpochOwned(..)) { + return Ok(self); + } + let base_epoch = *self.base_epoch(); + let owner_id = self.combine_owner_id(&rhs, merging_owners_strategy)?; + let mut other_epoch_bytes = self.combine_non_base_epoch_bytes(&rhs).unwrap_or_default(); + if let SectionedStorageRemoval(sectioned_bytes_by_identifier) = removed_bytes { + if sectioned_bytes_by_identifier.len() > 1 { + return Err(StorageFlagsError::MergingStorageFlagsFromDifferentOwners( + "can not remove bytes when there is no epoch".to_string(), + )); + } + let identifier = owner_id.copied().unwrap_or_default(); + let sectioned_bytes = sectioned_bytes_by_identifier.get(&identifier).ok_or( + StorageFlagsError::MergingStorageFlagsFromDifferentOwners( + "can not remove bytes when there is no epoch".to_string(), + ), + )?; + let mut keys_to_remove = Vec::new(); // To store the keys that need to be removed + + sectioned_bytes + .iter() + .try_for_each(|(epoch, removed_bytes)| { + if *epoch == base_epoch as u64 { + return Ok::<(), StorageFlagsError>(()); + } + let bytes_added_in_epoch = other_epoch_bytes.get_mut(&(*epoch as u16)).ok_or( + StorageFlagsError::RemovingAtEpochWithNoAssociatedStorage(format!( + "can not remove bytes when there is no epoch number [{}]", + *epoch + )), + )?; + + let desired_bytes_in_epoch = bytes_added_in_epoch + .checked_sub(*removed_bytes) + .ok_or(StorageFlagsError::StorageFlagsOverflow( + "can't remove more bytes than exist at that epoch".to_string(), + ))?; + + if desired_bytes_in_epoch <= MINIMUM_NON_BASE_FLAGS_SIZE { + // Collect the key to remove later + keys_to_remove.push(*epoch as u16); + } else { + *bytes_added_in_epoch = desired_bytes_in_epoch; + } + + Ok::<(), StorageFlagsError>(()) + })?; + + // Now remove the keys after the iteration + for key in keys_to_remove { + other_epoch_bytes.remove(&key); + } + } + // println!( + // " >combine_with_higher_base_epoch_remove_bytes: self:{:?} & + // rhs:{:?} -> {:?}", self.epoch_index_map(), + // rhs.epoch_index_map(), + // other_epoch_bytes + // ); + + if other_epoch_bytes.is_empty() { + match owner_id { + None => Ok(SingleEpoch(base_epoch)), + Some(owner_id) => Ok(SingleEpochOwned(base_epoch, *owner_id)), + } + } else { + match owner_id { + None => Ok(MultiEpoch(base_epoch, other_epoch_bytes)), + Some(owner_id) => Ok(MultiEpochOwned(base_epoch, other_epoch_bytes, *owner_id)), + } + } + } + + /// Optional combine added bytes + pub fn optional_combine_added_bytes( + ours: Option, + theirs: Self, + added_bytes: u32, + merging_owners_strategy: MergingOwnersStrategy, + ) -> Result { + match ours { + None => Ok(theirs), + Some(ours) => { + Ok(ours.combine_added_bytes(theirs, added_bytes, merging_owners_strategy)?) + } + } + } + + /// Optional combine removed bytes + pub fn optional_combine_removed_bytes( + ours: Option, + theirs: Self, + removed_bytes: &StorageRemovedBytes, + merging_owners_strategy: MergingOwnersStrategy, + ) -> Result { + match ours { + None => Ok(theirs), + Some(ours) => { + Ok(ours.combine_removed_bytes(theirs, removed_bytes, merging_owners_strategy)?) + } + } + } + + /// Combine added bytes + pub fn combine_added_bytes( + self, + rhs: Self, + added_bytes: u32, + merging_owners_strategy: MergingOwnersStrategy, + ) -> Result { + match self.base_epoch().cmp(rhs.base_epoch()) { + Ordering::Equal => self.combine_same_base_epoch(rhs, merging_owners_strategy), + Ordering::Less => { + self.combine_with_higher_base_epoch(rhs, added_bytes, merging_owners_strategy) + } + Ordering::Greater => Err( + StorageFlagsError::MergingStorageFlagsWithDifferentBaseEpoch( + "can not merge with new item in older base epoch".to_string(), + ), + ), + } + } + + /// Combine removed bytes + pub fn combine_removed_bytes( + self, + rhs: Self, + removed_bytes: &StorageRemovedBytes, + merging_owners_strategy: MergingOwnersStrategy, + ) -> Result { + match self.base_epoch().cmp(rhs.base_epoch()) { + Ordering::Equal => self.combine_same_base_epoch(rhs, merging_owners_strategy), + Ordering::Less => self.combine_with_higher_base_epoch_remove_bytes( + rhs, + removed_bytes, + merging_owners_strategy, + ), + Ordering::Greater => Err( + StorageFlagsError::MergingStorageFlagsWithDifferentBaseEpoch( + "can not merge with new item in older base epoch".to_string(), + ), + ), + } + } + + /// Returns base epoch + pub fn base_epoch(&self) -> &BaseEpoch { + match self { + SingleEpoch(base_epoch) + | MultiEpoch(base_epoch, _) + | SingleEpochOwned(base_epoch, _) + | MultiEpochOwned(base_epoch, ..) => base_epoch, + } + } + + /// Returns owner id + pub fn owner_id(&self) -> Option<&OwnerId> { + match self { + SingleEpochOwned(_, owner_id) | MultiEpochOwned(_, _, owner_id) => Some(owner_id), + _ => None, + } + } + + /// Returns epoch index map + pub fn epoch_index_map(&self) -> Option<&BTreeMap> { + match self { + MultiEpoch(_, epoch_int_map) | MultiEpochOwned(_, epoch_int_map, _) => { + Some(epoch_int_map) + } + _ => None, + } + } + + /// Returns optional default storage flags + pub fn optional_default() -> Option { + None + } + + /// Returns default optional storage flag as ref + pub fn optional_default_as_ref() -> Option<&'static Self> { + None + } + + /// Returns default optional storage flag as ref + pub fn optional_default_as_cow() -> Option> { + None + } + + /// Returns type byte + pub fn type_byte(&self) -> u8 { + match self { + SingleEpoch(_) => 0, + MultiEpoch(..) => 1, + SingleEpochOwned(..) => 2, + MultiEpochOwned(..) => 3, + } + } + + fn append_to_vec_base_epoch(&self, buffer: &mut Vec) { + match self { + SingleEpoch(base_epoch) + | MultiEpoch(base_epoch, ..) + | SingleEpochOwned(base_epoch, ..) + | MultiEpochOwned(base_epoch, ..) => buffer.extend(base_epoch.to_be_bytes()), + } + } + + fn maybe_append_to_vec_epoch_map(&self, buffer: &mut Vec) { + match self { + MultiEpoch(_, epoch_map) | MultiEpochOwned(_, epoch_map, _) => { + if epoch_map.is_empty() { + panic!("this should not be empty"); + } + epoch_map.iter().for_each(|(epoch_index, bytes_added)| { + buffer.extend(epoch_index.to_be_bytes()); + buffer.extend(bytes_added.encode_var_vec()); + }) + } + _ => {} + } + } + + fn maybe_epoch_map_size(&self) -> u32 { + let mut size = 0; + match self { + MultiEpoch(_, epoch_map) | MultiEpochOwned(_, epoch_map, _) => { + epoch_map.iter().for_each(|(_epoch_index, bytes_added)| { + size += 2; + size += bytes_added.encode_var_vec().len() as u32; + }) + } + _ => {} + } + size + } + + fn maybe_append_to_vec_owner_id(&self, buffer: &mut Vec) { + match self { + SingleEpochOwned(_, owner_id) | MultiEpochOwned(_, _, owner_id) => { + buffer.extend(owner_id); + } + _ => {} + } + } + + fn maybe_owner_id_size(&self) -> u32 { + match self { + SingleEpochOwned(..) | MultiEpochOwned(..) => DEFAULT_HASH_SIZE_U32, + _ => 0, + } + } + + /// ApproximateSize + pub fn approximate_size( + has_owner_id: bool, + approximate_changes_and_bytes_count: Option<(u16, u8)>, + ) -> u32 { + let mut size = 3; // 1 for type byte, 2 for epoch number + if has_owner_id { + size += DEFAULT_HASH_SIZE_U32; + } + if let Some((approximate_change_count, bytes_changed_required_size)) = + approximate_changes_and_bytes_count + { + size += (approximate_change_count as u32) * (2 + bytes_changed_required_size as u32) + } + size + } + + /// Serialize storage flags + pub fn serialize(&self) -> Vec { + let mut buffer = vec![self.type_byte()]; + self.maybe_append_to_vec_owner_id(&mut buffer); + self.append_to_vec_base_epoch(&mut buffer); + self.maybe_append_to_vec_epoch_map(&mut buffer); + buffer + } + + /// Serialize storage flags + pub fn serialized_size(&self) -> u32 { + let mut buffer_len = 3; // for type byte and base epoch + buffer_len += self.maybe_owner_id_size(); + buffer_len += self.maybe_epoch_map_size(); + buffer_len + } + + /// Deserialize single epoch storage flags from bytes + pub fn deserialize_single_epoch(data: &[u8]) -> Result { + if data.len() != 3 { + Err(StorageFlagsError::StorageFlagsWrongSize( + "single epoch must be 3 bytes total".to_string(), + )) + } else { + let epoch = u16::from_be_bytes(data[1..3].try_into().map_err(|_| { + StorageFlagsError::StorageFlagsWrongSize( + "single epoch must be 3 bytes total".to_string(), + ) + })?); + Ok(SingleEpoch(epoch)) + } + } + + /// Deserialize multi epoch storage flags from bytes + pub fn deserialize_multi_epoch(data: &[u8]) -> Result { + let len = data.len(); + if len < 6 { + Err(StorageFlagsError::StorageFlagsWrongSize( + "multi epoch must be at least 6 bytes total".to_string(), + )) + } else { + let base_epoch = u16::from_be_bytes(data[1..3].try_into().map_err(|_| { + StorageFlagsError::StorageFlagsWrongSize( + "multi epoch must have enough bytes for the base epoch".to_string(), + ) + })?); + let mut offset = 3; + let mut bytes_per_epoch: BTreeMap = BTreeMap::default(); + while offset + 2 < len { + // 2 for epoch size + let epoch_index = + u16::from_be_bytes(data[offset..offset + 2].try_into().map_err(|_| { + StorageFlagsError::StorageFlagsWrongSize( + "multi epoch must have enough bytes epoch indexes".to_string(), + ) + })?); + offset += 2; + let (bytes_at_epoch, bytes_used) = u32::decode_var(&data[offset..]).ok_or( + StorageFlagsError::StorageFlagsWrongSize( + "multi epoch must have enough bytes for the amount of bytes used" + .to_string(), + ), + )?; + offset += bytes_used; + bytes_per_epoch.insert(epoch_index, bytes_at_epoch); + } + Ok(MultiEpoch(base_epoch, bytes_per_epoch)) + } + } + + /// Deserialize single epoch owned storage flags from bytes + pub fn deserialize_single_epoch_owned(data: &[u8]) -> Result { + if data.len() != 35 { + Err(StorageFlagsError::StorageFlagsWrongSize( + "single epoch owned must be 35 bytes total".to_string(), + )) + } else { + let owner_id: OwnerId = data[1..33].try_into().map_err(|_| { + StorageFlagsError::StorageFlagsWrongSize( + "single epoch owned must be 35 bytes total for owner id".to_string(), + ) + })?; + let epoch = u16::from_be_bytes(data[33..35].try_into().map_err(|_| { + StorageFlagsError::StorageFlagsWrongSize( + "single epoch owned must be 35 bytes total for epoch".to_string(), + ) + })?); + Ok(SingleEpochOwned(epoch, owner_id)) + } + } + + /// Deserialize multi epoch owned storage flags from bytes + pub fn deserialize_multi_epoch_owned(data: &[u8]) -> Result { + let len = data.len(); + if len < 38 { + Err(StorageFlagsError::StorageFlagsWrongSize( + "multi epoch owned must be at least 38 bytes total".to_string(), + )) + } else { + let owner_id: OwnerId = data[1..33].try_into().map_err(|_| { + StorageFlagsError::StorageFlagsWrongSize( + "multi epoch owned must be 38 bytes total for owner id".to_string(), + ) + })?; + let base_epoch = u16::from_be_bytes(data[33..35].try_into().map_err(|_| { + StorageFlagsError::StorageFlagsWrongSize( + "multi epoch must have enough bytes for the base epoch".to_string(), + ) + })?); + let mut offset = 35; + let mut bytes_per_epoch: BTreeMap = BTreeMap::default(); + while offset + 2 < len { + // 2 for epoch size + let epoch_index = + u16::from_be_bytes(data[offset..offset + 2].try_into().map_err(|_| { + StorageFlagsError::StorageFlagsWrongSize( + "multi epoch must have enough bytes epoch indexes".to_string(), + ) + })?); + offset += 2; + let (bytes_at_epoch, bytes_used) = u32::decode_var(&data[offset..]).ok_or( + StorageFlagsError::StorageFlagsWrongSize( + "multi epoch must have enough bytes for the amount of bytes used" + .to_string(), + ), + )?; + offset += bytes_used; + bytes_per_epoch.insert(epoch_index, bytes_at_epoch); + } + Ok(MultiEpochOwned(base_epoch, bytes_per_epoch, owner_id)) + } + } + + /// Deserialize storage flags from bytes + pub fn deserialize(data: &[u8]) -> Result, StorageFlagsError> { + let first_byte = data.first(); + match first_byte { + None => Ok(None), + Some(first_byte) => match *first_byte { + 0 => Ok(Some(Self::deserialize_single_epoch(data)?)), + 1 => Ok(Some(Self::deserialize_multi_epoch(data)?)), + 2 => Ok(Some(Self::deserialize_single_epoch_owned(data)?)), + 3 => Ok(Some(Self::deserialize_multi_epoch_owned(data)?)), + _ => Err(StorageFlagsError::DeserializeUnknownStorageFlagsType( + "unknown storage flags serialization".to_string(), + )), + }, + } + } + + /// Creates storage flags from a slice. + pub fn from_slice(data: &[u8]) -> Result, StorageFlagsError> { + Self::deserialize(data) + } + + /// Creates storage flags from element flags. + pub fn from_element_flags_ref(data: &ElementFlags) -> Result, StorageFlagsError> { + Self::from_slice(data.as_slice()) + } + + /// Create Storage flags from optional element flags ref + pub fn map_some_element_flags_ref( + data: &Option, + ) -> Result, StorageFlagsError> { + match data { + None => Ok(None), + Some(data) => Self::from_slice(data.as_slice()), + } + } + + /// Create Storage flags from optional element flags ref + pub fn map_cow_some_element_flags_ref( + data: &Option, + ) -> Result>, StorageFlagsError> { + match data { + None => Ok(None), + Some(data) => Self::from_slice(data.as_slice()).map(|option| option.map(Cow::Owned)), + } + } + + /// Map to owned optional element flags + pub fn map_owned_to_element_flags(maybe_storage_flags: Option) -> ElementFlags { + maybe_storage_flags + .map(|storage_flags| storage_flags.serialize()) + .unwrap_or_default() + } + + /// Map to optional element flags + pub fn map_to_some_element_flags(maybe_storage_flags: Option<&Self>) -> Option { + maybe_storage_flags.map(|storage_flags| storage_flags.serialize()) + } + + /// Map to optional element flags + pub fn map_cow_to_some_element_flags( + maybe_storage_flags: Option>, + ) -> Option { + maybe_storage_flags.map(|storage_flags| storage_flags.serialize()) + } + + /// Map to optional element flags + pub fn map_borrowed_cow_to_some_element_flags( + maybe_storage_flags: &Option>, + ) -> Option { + maybe_storage_flags + .as_ref() + .map(|storage_flags| storage_flags.serialize()) + } + + /// Creates optional element flags + pub fn to_some_element_flags(&self) -> Option { + Some(self.serialize()) + } + + /// Creates element flags. + pub fn to_element_flags(&self) -> ElementFlags { + self.serialize() + } + + /// split_storage_removed_bytes removes bytes as LIFO + pub fn split_storage_removed_bytes( + &self, + removed_key_bytes: u32, + removed_value_bytes: u32, + ) -> (StorageRemovedBytes, StorageRemovedBytes) { + fn single_storage_removal( + removed_bytes: u32, + base_epoch: &BaseEpoch, + owner_id: Option<&OwnerId>, + ) -> StorageRemovedBytes { + if removed_bytes == 0 { + return NoStorageRemoval; + } + let bytes_left = removed_bytes; + let mut sectioned_storage_removal: IntMap = IntMap::default(); + if bytes_left > 0 { + // We need to take some from the base epoch + sectioned_storage_removal.insert(*base_epoch as u64, removed_bytes); + } + let mut sectioned_storage_removal_by_identifier: StorageRemovalPerEpochByIdentifier = + BTreeMap::new(); + if let Some(owner_id) = owner_id { + sectioned_storage_removal_by_identifier + .insert(*owner_id, sectioned_storage_removal); + } else { + let default = [0u8; 32]; + sectioned_storage_removal_by_identifier.insert(default, sectioned_storage_removal); + } + SectionedStorageRemoval(sectioned_storage_removal_by_identifier) + } + + fn sectioned_storage_removal( + removed_bytes: u32, + base_epoch: &BaseEpoch, + other_epoch_bytes: &BTreeMap, + owner_id: Option<&OwnerId>, + ) -> StorageRemovedBytes { + if removed_bytes == 0 { + return NoStorageRemoval; + } + let mut bytes_left = removed_bytes; + let mut rev_iter = other_epoch_bytes.iter().rev(); + let mut sectioned_storage_removal: IntMap = IntMap::default(); + + while bytes_left > 0 { + if let Some((epoch_index, bytes_in_epoch)) = rev_iter.next() { + if *bytes_in_epoch <= bytes_left + MINIMUM_NON_BASE_FLAGS_SIZE { + sectioned_storage_removal.insert( + *epoch_index as u64, + *bytes_in_epoch - MINIMUM_NON_BASE_FLAGS_SIZE, + ); + bytes_left -= *bytes_in_epoch - MINIMUM_NON_BASE_FLAGS_SIZE; + } else { + // Correctly take only the required bytes_left from this epoch + sectioned_storage_removal.insert(*epoch_index as u64, bytes_left); + bytes_left = 0; // All required bytes have been removed, stop processing + break; // Exit the loop as there's no need to process + // further epochs + } + } else { + break; + } + } + + if bytes_left > 0 { + // If there are still bytes left, take them from the base epoch + sectioned_storage_removal.insert(*base_epoch as u64, bytes_left); + } + + let mut sectioned_storage_removal_by_identifier: StorageRemovalPerEpochByIdentifier = + BTreeMap::new(); + + if let Some(owner_id) = owner_id { + sectioned_storage_removal_by_identifier + .insert(*owner_id, sectioned_storage_removal); + } else { + let default = [0u8; 32]; + sectioned_storage_removal_by_identifier.insert(default, sectioned_storage_removal); + } + + SectionedStorageRemoval(sectioned_storage_removal_by_identifier) + } + + // If key bytes are being removed, it implies a delete; thus, we should remove + // all relevant storage bytes + let key_storage_removal = if removed_key_bytes > 0 { + match self { + // For any variant, always take the key's removed bytes from the base epoch + SingleEpoch(base_epoch) | MultiEpoch(base_epoch, _) => { + single_storage_removal(removed_key_bytes, base_epoch, None) + } + SingleEpochOwned(base_epoch, owner_id) + | MultiEpochOwned(base_epoch, _, owner_id) => { + single_storage_removal(removed_key_bytes, base_epoch, Some(owner_id)) + } + } + } else { + StorageRemovedBytes::default() + }; + + // For normal logic, we only need to process the value-related bytes. + let value_storage_removal = match self { + SingleEpoch(base_epoch) => { + single_storage_removal(removed_value_bytes, base_epoch, None) + } + SingleEpochOwned(base_epoch, owner_id) => { + single_storage_removal(removed_value_bytes, base_epoch, Some(owner_id)) + } + MultiEpoch(base_epoch, other_epoch_bytes) => { + sectioned_storage_removal(removed_value_bytes, base_epoch, other_epoch_bytes, None) + } + MultiEpochOwned(base_epoch, other_epoch_bytes, owner_id) => sectioned_storage_removal( + removed_value_bytes, + base_epoch, + other_epoch_bytes, + Some(owner_id), + ), + }; + + // For key removal, simply return the empty removal since it's an update does + // not modify the key. + (key_storage_removal, value_storage_removal) + } + + /// Wrap Storage Flags into optional owned cow + pub fn into_optional_cow<'a>(self) -> Option> { + Some(Cow::Owned(self)) + } +} + +#[cfg(test)] +mod storage_flags_tests { + use std::collections::BTreeMap; + + use grovedb_costs::storage_cost::removal::{ + StorageRemovalPerEpochByIdentifier, StorageRemovedBytes, + }; + use intmap::IntMap; + + use crate::{ + BaseEpoch, BytesAddedInEpoch, MergingOwnersStrategy, OwnerId, StorageFlags, + MINIMUM_NON_BASE_FLAGS_SIZE, + }; + #[test] + fn test_storage_flags_combine() { + { + // Same SingleEpoch - AdditionBytes + let common_base_index: BaseEpoch = 1; + let left_flag = StorageFlags::new_single_epoch(common_base_index, None); + let right_flag = StorageFlags::new_single_epoch(common_base_index, None); + + let added_bytes: BytesAddedInEpoch = 10; + let combined_flag = left_flag.clone().combine_added_bytes( + right_flag.clone(), + added_bytes, + MergingOwnersStrategy::UseOurs, + ); + // println!( + // "{:?} & {:?} added_bytes:{} --> {:?}\n", + // left_flag, right_flag, added_bytes, combined_flag + // ); + } + // { + // Same SingleEpoch - RemovedBytes + // let common_base_index: BaseEpoch = 1; + // let left_flag = StorageFlags::new_single_epoch(common_base_index, None); + // let right_flag = StorageFlags::new_single_epoch(common_base_index, None); + // + // let removed_bytes = StorageRemovedBytes::BasicStorageRemoval(10); + // let combined_flag = + // left_flag.clone().combine_removed_bytes(right_flag.clone(), &removed_bytes, + // MergingOwnersStrategy::UseOurs); println!("{:?} & {:?} + // removed_bytes:{:?} --> {:?}\n", left_flag, right_flag, removed_bytes, + // combined_flag); } + { + // Different-Higher SingleEpoch - AdditionBytes + let left_base_index: BaseEpoch = 1; + let right_base_index: BaseEpoch = 2; + let left_flag = StorageFlags::new_single_epoch(left_base_index, None); + let right_flag = StorageFlags::new_single_epoch(right_base_index, None); + + let added_bytes: BytesAddedInEpoch = 10; + let combined_flag = left_flag.clone().combine_added_bytes( + right_flag.clone(), + added_bytes, + MergingOwnersStrategy::UseOurs, + ); + // println!( + // "{:?} & {:?} added_bytes:{} --> {:?}\n", + // left_flag, right_flag, added_bytes, combined_flag + // ); + } + { + // Different-Lesser SingleEpoch - AdditionBytes + let left_base_index: BaseEpoch = 2; + let right_base_index: BaseEpoch = 1; + let left_flag = StorageFlags::new_single_epoch(left_base_index, None); + let right_flag = StorageFlags::new_single_epoch(right_base_index, None); + + let added_bytes: BytesAddedInEpoch = 10; + let combined_flag = left_flag.clone().combine_added_bytes( + right_flag.clone(), + added_bytes, + MergingOwnersStrategy::UseOurs, + ); + // println!( + // "{:?} & {:?} added_bytes:{} --> {:?}\n", + // left_flag, right_flag, added_bytes, combined_flag + // ); + } + { + // SingleEpoch-MultiEpoch same BaseEpoch - AdditionBytes + let common_base_index: BaseEpoch = 1; + let left_flag = StorageFlags::new_single_epoch(common_base_index, None); + let right_flag = StorageFlags::MultiEpoch( + common_base_index, + [(common_base_index + 1, 5)].iter().cloned().collect(), + ); + + let added_bytes: BytesAddedInEpoch = 10; + let combined_flag = left_flag.clone().combine_added_bytes( + right_flag.clone(), + added_bytes, + MergingOwnersStrategy::UseOurs, + ); + // println!( + // "{:?} & {:?} added_bytes:{} --> {:?}\n", + // left_flag, right_flag, added_bytes, combined_flag + // ); + } + { + // SingleEpoch-MultiEpoch higher BaseEpoch - AdditionBytes + let left_base_index: BaseEpoch = 1; + let right_base_index: BaseEpoch = 2; + let left_flag = StorageFlags::new_single_epoch(left_base_index, None); + let right_flag = StorageFlags::MultiEpoch( + right_base_index, + [(right_base_index + 1, 5)].iter().cloned().collect(), + ); + + let added_bytes: BytesAddedInEpoch = 10; + let combined_flag = left_flag.clone().combine_added_bytes( + right_flag.clone(), + added_bytes, + MergingOwnersStrategy::UseOurs, + ); + // println!( + // "{:?} & {:?} added_bytes:{} --> {:?}\n", + // left_flag, right_flag, added_bytes, combined_flag + // ); + } + // { + // SingleEpoch-MultiEpoch same BaseEpoch - RemovedBytes (positive difference) + // let common_base_index: BaseEpoch = 1; + // let left_flag = StorageFlags::new_single_epoch(common_base_index, None); + // let right_flag = StorageFlags::MultiEpoch(common_base_index, + // [(common_base_index + 1, 10)].iter().cloned().collect()); + // + // let removed_bytes = StorageRemovedBytes::BasicStorageRemoval(3); + // let combined_flag = + // left_flag.clone().combine_removed_bytes(right_flag.clone(), &removed_bytes, + // MergingOwnersStrategy::UseOurs); println!("{:?} & {:?} + // removed_bytes:{:?} --> {:?}\n", left_flag, right_flag, &removed_bytes, + // combined_flag); } + // { + // SingleEpoch-MultiEpoch same BaseEpoch - RemovedBytes (negative difference) + // let common_base_index: BaseEpoch = 1; + // let left_flag = StorageFlags::new_single_epoch(common_base_index, None); + // let right_flag = StorageFlags::MultiEpoch(common_base_index, + // [(common_base_index + 1, 10)].iter().cloned().collect()); + // + // let removed_bytes = StorageRemovedBytes::BasicStorageRemoval(13); + // let combined_flag = + // left_flag.clone().combine_removed_bytes(right_flag.clone(), &removed_bytes, + // MergingOwnersStrategy::UseOurs); println!("{:?} & {:?} + // removed_bytes:{:?} --> {:?}\n", left_flag, right_flag, &removed_bytes, + // combined_flag); } + // { + // SingleEpoch-MultiEpoch higher BaseEpoch - RemovedBytes (positive difference) + // let left_base_index: BaseEpoch = 1; + // let right_base_index: BaseEpoch = 2; + // let left_flag = StorageFlags::new_single_epoch(left_base_index, None); + // let right_flag = StorageFlags::MultiEpoch(right_base_index, + // [(right_base_index + 1, 10)].iter().cloned().collect()); + // + // let removed_bytes = StorageRemovedBytes::BasicStorageRemoval(3); + // let combined_flag = + // left_flag.clone().combine_removed_bytes(right_flag.clone(), &removed_bytes, + // MergingOwnersStrategy::UseOurs); println!("{:?} & {:?} + // removed_bytes:{:?} --> {:?}\n", left_flag, right_flag, &removed_bytes, + // combined_flag); } + // { + // SingleEpoch-MultiEpoch higher BaseEpoch - RemovedBytes (negative difference) + // let left_base_index: BaseEpoch = 1; + // let right_base_index: BaseEpoch = 2; + // let left_flag = StorageFlags::new_single_epoch(left_base_index, None); + // let right_flag = StorageFlags::MultiEpoch(right_base_index, + // [(right_base_index + 1, 5)].iter().cloned().collect()); + // + // let removed_bytes = StorageRemovedBytes::BasicStorageRemoval(7); + // let combined_flag = + // left_flag.clone().combine_removed_bytes(right_flag.clone(), &removed_bytes, + // MergingOwnersStrategy::UseOurs); println!("{:?} & {:?} + // removed_bytes:{:?} --> {:?}\n", left_flag, right_flag, &removed_bytes, + // combined_flag); } + { + // MultiEpochs same BaseEpoch - AdditionBytes #1 + let common_base_index: BaseEpoch = 1; + let left_flag = StorageFlags::MultiEpoch( + common_base_index, + [(common_base_index + 1, 7)].iter().cloned().collect(), + ); + let right_flag = StorageFlags::MultiEpoch( + common_base_index, + [(common_base_index + 1, 5)].iter().cloned().collect(), + ); + + let added_bytes: BytesAddedInEpoch = 10; + let combined_flag = left_flag.clone().combine_added_bytes( + right_flag.clone(), + added_bytes, + MergingOwnersStrategy::UseOurs, + ); + println!( + "{:?} & {:?} added_bytes:{} --> {:?}\n", + left_flag, right_flag, added_bytes, combined_flag + ); + } + { + // MultiEpochs same BaseEpoch - AdditionBytes #2 + let common_base_index: BaseEpoch = 1; + let left_flag = StorageFlags::MultiEpoch( + common_base_index, + [(common_base_index + 1, 7)].iter().cloned().collect(), + ); + let right_flag = StorageFlags::MultiEpoch( + common_base_index, + [(common_base_index + 2, 5)].iter().cloned().collect(), + ); + + let added_bytes: BytesAddedInEpoch = 10; + let combined_flag = left_flag.clone().combine_added_bytes( + right_flag.clone(), + added_bytes, + MergingOwnersStrategy::UseOurs, + ); + println!( + "{:?} & {:?} added_bytes:{} --> {:?}\n", + left_flag, right_flag, added_bytes, combined_flag + ); + } + { + // MultiEpochs same BaseEpoch - AdditionBytes #3 + let common_base_index: BaseEpoch = 1; + let left_flag = StorageFlags::MultiEpoch( + common_base_index, + [(common_base_index + 1, 7)].iter().cloned().collect(), + ); + let right_flag = StorageFlags::MultiEpoch( + common_base_index, + [(common_base_index + 1, 3), (common_base_index + 2, 5)] + .iter() + .cloned() + .collect(), + ); + + let added_bytes: BytesAddedInEpoch = 10; + let combined_flag = left_flag.clone().combine_added_bytes( + right_flag.clone(), + added_bytes, + MergingOwnersStrategy::UseOurs, + ); + println!( + "{:?} & {:?} added_bytes:{} --> {:?}\n", + left_flag, right_flag, added_bytes, combined_flag + ); + } + { + // MultiEpochs higher BaseEpoch - AdditionBytes #1 + let common_base_index: BaseEpoch = 1; + let left_flag = StorageFlags::MultiEpoch( + common_base_index, + [(common_base_index + 1, 7)].iter().cloned().collect(), + ); + let right_flag = StorageFlags::MultiEpoch( + common_base_index + 1, + [(common_base_index + 1, 5)].iter().cloned().collect(), + ); + + let added_bytes: BytesAddedInEpoch = 10; + let combined_flag = left_flag.clone().combine_added_bytes( + right_flag.clone(), + added_bytes, + MergingOwnersStrategy::UseOurs, + ); + println!( + "{:?} & {:?} added_bytes:{} --> {:?}\n", + left_flag, right_flag, added_bytes, combined_flag + ); + } + { + // MultiEpochs higher BaseEpoch - AdditionBytes #2 + let common_base_index: BaseEpoch = 1; + let left_flag = StorageFlags::MultiEpoch( + common_base_index, + [(common_base_index + 1, 7)].iter().cloned().collect(), + ); + let right_flag = StorageFlags::MultiEpoch( + common_base_index + 1, + [(common_base_index + 2, 5)].iter().cloned().collect(), + ); + + let added_bytes: BytesAddedInEpoch = 10; + let combined_flag = left_flag.clone().combine_added_bytes( + right_flag.clone(), + added_bytes, + MergingOwnersStrategy::UseOurs, + ); + println!( + "{:?} & {:?} added_bytes:{} --> {:?}\n", + left_flag, right_flag, added_bytes, combined_flag + ); + } + { + // MultiEpochs higher BaseEpoch - AdditionBytes #3 + let common_base_index: BaseEpoch = 1; + let left_flag = StorageFlags::MultiEpoch( + common_base_index, + [(common_base_index + 1, 7)].iter().cloned().collect(), + ); + let right_flag = StorageFlags::MultiEpoch( + common_base_index + 1, + [(common_base_index + 2, 3), (common_base_index + 3, 5)] + .iter() + .cloned() + .collect(), + ); + + let added_bytes: BytesAddedInEpoch = 10; + let combined_flag = left_flag.clone().combine_added_bytes( + right_flag.clone(), + added_bytes, + MergingOwnersStrategy::UseOurs, + ); + println!( + "{:?} & {:?} added_bytes:{} --> {:?}\n", + left_flag, right_flag, added_bytes, combined_flag + ); + } + } + + fn create_epoch_map(epoch: u16, bytes: u32) -> BTreeMap { + let mut map = BTreeMap::new(); + map.insert(epoch, bytes); + map + } + + fn default_owner_id() -> OwnerId { + [0u8; 32] + } + + fn single_epoch_removed_bytes_map( + owner_id: [u8; 32], + epoch_index: u64, + bytes_removed: u32, + ) -> StorageRemovalPerEpochByIdentifier { + let mut removed_bytes = StorageRemovalPerEpochByIdentifier::default(); + let mut removed_bytes_for_identity = IntMap::new(); + removed_bytes_for_identity.insert(epoch_index, bytes_removed); + removed_bytes.insert(owner_id, removed_bytes_for_identity); + removed_bytes + } + + fn multi_epoch_removed_bytes_map( + owner_id: [u8; 32], + removed_bytes_per_epoch: IntMap, + ) -> StorageRemovalPerEpochByIdentifier { + let mut removed_bytes = StorageRemovalPerEpochByIdentifier::default(); + removed_bytes.insert(owner_id, removed_bytes_per_epoch); + removed_bytes + } + + #[test] + fn test_combine_that_would_remove_the_epoch_completely() { + let owner_id = [2; 32]; + let left_base_index: BaseEpoch = 0; + let right_base_index: BaseEpoch = 1; + let other_epochs = create_epoch_map(1, 5); + let left_flag = StorageFlags::MultiEpochOwned(left_base_index, other_epochs, owner_id); + let right_flag = StorageFlags::new_single_epoch(right_base_index, Some(owner_id)); + + let removed_bytes = single_epoch_removed_bytes_map(owner_id, 1, 3); + let combined_flag = left_flag + .clone() + .combine_removed_bytes( + right_flag.clone(), + &StorageRemovedBytes::SectionedStorageRemoval(removed_bytes.clone()), + MergingOwnersStrategy::UseOurs, + ) + .expect("expected to combine flags"); + + assert_eq!( + combined_flag, + StorageFlags::SingleEpochOwned(left_base_index, owner_id) + ); + println!( + "{:?} & {:?} removed:{:?} --> {:?}\n", + left_flag, right_flag, removed_bytes, combined_flag + ); + } + + #[test] + fn test_combine_that_would_remove_the_epoch_completely_with_many_entries() { + let owner_id = [2; 32]; + let left_base_index: BaseEpoch = 0; + let right_base_index: BaseEpoch = 1; + let mut other_epochs = BTreeMap::new(); + let mut removed_bytes = IntMap::new(); + for i in 1..200 { + other_epochs.insert(i, MINIMUM_NON_BASE_FLAGS_SIZE + 1); + removed_bytes.insert(i as u64, 1); // anything between 1 and + // MINIMUM_NON_BASE_FLAGS_SIZE + + // 1 would be the same + } + + let left_flag = StorageFlags::MultiEpochOwned(left_base_index, other_epochs, owner_id); + let right_flag = StorageFlags::new_single_epoch(right_base_index, Some(owner_id)); + + let removed_bytes = multi_epoch_removed_bytes_map(owner_id, removed_bytes); + let combined_flag = left_flag + .clone() + .combine_removed_bytes( + right_flag.clone(), + &StorageRemovedBytes::SectionedStorageRemoval(removed_bytes.clone()), + MergingOwnersStrategy::UseOurs, + ) + .expect("expected to combine flags"); + + assert_eq!( + combined_flag, + StorageFlags::SingleEpochOwned(left_base_index, owner_id) + ); + println!( + "{:?} & {:?} removed:{:?} --> {:?}\n", + left_flag, right_flag, removed_bytes, combined_flag + ); + } + + /// Tests the case when using SingleEpoch flags, ensuring that the correct + /// storage removal is calculated. + #[test] + fn test_single_epoch_removal() { + let flags = StorageFlags::SingleEpoch(5); + let (key_removal, value_removal) = flags.split_storage_removed_bytes(100, 200); + + assert_eq!( + key_removal, + StorageRemovedBytes::SectionedStorageRemoval({ + let mut map = BTreeMap::new(); + map.insert(default_owner_id(), IntMap::from_iter([(5u64, 100)])); + map + }) + ); + assert_eq!( + value_removal, + StorageRemovedBytes::SectionedStorageRemoval({ + let mut map = BTreeMap::new(); + map.insert(default_owner_id(), IntMap::from_iter([(5u64, 200)])); + map + }) + ); + } + + /// Tests SingleEpochOwned flags where the removal is done under an OwnerId + #[test] + fn test_single_epoch_owned_removal() { + let owner_id = [1u8; 32]; + let flags = StorageFlags::SingleEpochOwned(5, owner_id); + let (key_removal, value_removal) = flags.split_storage_removed_bytes(50, 150); + + assert_eq!( + key_removal, + StorageRemovedBytes::SectionedStorageRemoval({ + let mut map = BTreeMap::new(); + map.insert(owner_id, IntMap::from_iter([(5u64, 50)])); + map + }) + ); + assert_eq!( + value_removal, + StorageRemovedBytes::SectionedStorageRemoval({ + let mut map = BTreeMap::new(); + map.insert(owner_id, IntMap::from_iter([(5u64, 150)])); + map + }) + ); + } + + /// Tests the case where multiple epochs are used and the total removal + /// doesn’t exceed the extra epoch bytes + #[test] + fn test_multi_epoch_removal_no_remaining_base() { + let mut other_epochs = create_epoch_map(6, 100); + other_epochs.insert(7, 200); + + let flags = StorageFlags::MultiEpoch(5, other_epochs); + let (_key_removal, value_removal) = flags.split_storage_removed_bytes(0, 250); + + assert_eq!( + value_removal, + StorageRemovedBytes::SectionedStorageRemoval({ + let mut map = BTreeMap::new(); + map.insert( + default_owner_id(), + IntMap::from_iter([(7u64, 197), (6u64, 53)]), + ); + map + }) + ); + } + + /// Similar to the previous test, but this time the base epoch is also used + /// due to insufficient bytes in the extra epochs + #[test] + fn test_multi_epoch_removal_with_remaining_base() { + let mut other_epochs = create_epoch_map(6, 100); + other_epochs.insert(7, 50); + + let flags = StorageFlags::MultiEpoch(5, other_epochs); + let (key_removal, value_removal) = flags.split_storage_removed_bytes(250, 250); + + assert_eq!( + key_removal, + StorageRemovedBytes::SectionedStorageRemoval({ + let mut map = BTreeMap::new(); + map.insert(default_owner_id(), IntMap::from_iter([(5u64, 250)])); + map + }) + ); + assert_eq!( + value_removal, + StorageRemovedBytes::SectionedStorageRemoval({ + let mut map = BTreeMap::new(); + map.insert( + default_owner_id(), + IntMap::from_iter([(7u64, 47), (6u64, 97), (5u64, 106)]), + ); + map + }) + ); + } + + /// Same as last test but for owned flags with OwnerId + #[test] + fn test_multi_epoch_owned_removal_with_remaining_base() { + let owner_id = [2u8; 32]; + let mut other_epochs = create_epoch_map(6, 100); + other_epochs.insert(7, 50); + + let flags = StorageFlags::MultiEpochOwned(5, other_epochs, owner_id); + let (key_removal, value_removal) = flags.split_storage_removed_bytes(250, 250); + + assert_eq!( + key_removal, + StorageRemovedBytes::SectionedStorageRemoval({ + let mut map = BTreeMap::new(); + map.insert(owner_id, IntMap::from_iter([(5u64, 250)])); + map + }) + ); + assert_eq!( + value_removal, + StorageRemovedBytes::SectionedStorageRemoval({ + let mut map = BTreeMap::new(); + map.insert( + owner_id, + IntMap::from_iter([(7u64, 47), (6u64, 97), (5u64, 106)]), + ); + map + }) + ); + } + + /// Tests the function when zero bytes are to be removed, expecting an empty + /// removal result + #[test] + fn test_single_epoch_removal_zero_bytes() { + let flags = StorageFlags::SingleEpoch(5); + let (key_removal, value_removal) = flags.split_storage_removed_bytes(0, 0); + + assert_eq!(key_removal, StorageRemovedBytes::NoStorageRemoval); + assert_eq!(value_removal, StorageRemovedBytes::NoStorageRemoval); + } + + /// Tests the removal of only part of the bytes using SingleEpochOwned + #[test] + fn test_single_epoch_owned_removal_partial_bytes() { + let owner_id = [3u8; 32]; + let flags = StorageFlags::SingleEpochOwned(5, owner_id); + let (key_removal, value_removal) = flags.split_storage_removed_bytes(100, 50); + + assert_eq!( + key_removal, + StorageRemovedBytes::SectionedStorageRemoval({ + let mut map = BTreeMap::new(); + map.insert(owner_id, IntMap::from_iter([(5u64, 100)])); + map + }) + ); + assert_eq!( + value_removal, + StorageRemovedBytes::SectionedStorageRemoval({ + let mut map = BTreeMap::new(); + map.insert(owner_id, IntMap::from_iter([(5u64, 50)])); + map + }) + ); + } + + /// Ensures that the function correctly handles when there are more bytes to + /// be removed than are available in the epoch map, requiring the base epoch + /// to be used + #[test] + fn test_multi_epoch_removal_excess_bytes() { + let mut other_epochs = create_epoch_map(6, 100); + other_epochs.insert(7, 200); + + let flags = StorageFlags::MultiEpoch(5, other_epochs); + let (key_removal, value_removal) = flags.split_storage_removed_bytes(400, 300); + + assert_eq!( + key_removal, + StorageRemovedBytes::SectionedStorageRemoval({ + let mut map = BTreeMap::new(); + map.insert(default_owner_id(), IntMap::from_iter([(5u64, 400)])); + map + }) + ); + assert_eq!( + value_removal, + StorageRemovedBytes::SectionedStorageRemoval({ + let mut map = BTreeMap::new(); + map.insert( + default_owner_id(), + IntMap::from_iter([(7u64, 197), (6u64, 97), (5u64, 6)]), + ); + map + }) + ); + } + + /// Similar to the previous test, but for owned flags with OwnerId + #[test] + fn test_multi_epoch_owned_removal_excess_bytes() { + let owner_id = [4u8; 32]; + let mut other_epochs = create_epoch_map(6, 100); + other_epochs.insert(7, 200); + + let flags = StorageFlags::MultiEpochOwned(5, other_epochs, owner_id); + let (key_removal, value_removal) = flags.split_storage_removed_bytes(450, 350); + + assert_eq!( + key_removal, + StorageRemovedBytes::SectionedStorageRemoval({ + let mut map = BTreeMap::new(); + map.insert(owner_id, IntMap::from_iter([(5u64, 450)])); + map + }) + ); + assert_eq!( + value_removal, + StorageRemovedBytes::SectionedStorageRemoval({ + let mut map = BTreeMap::new(); + map.insert( + owner_id, + IntMap::from_iter([(7u64, 197), (6u64, 97), (5u64, 56)]), + ); + map + }) + ); + } + + #[test] + /// This test verifies the `split_storage_removed_bytes` function when all + /// required bytes are taken from non-base epochs during the removal + /// process. + /// + /// The scenario: + /// - The test initializes a `StorageFlags::MultiEpochOwned` with a + /// `BaseEpoch` of 5. + /// - Two additional epochs, 6 and 7, are provided with 300 and 400 bytes + /// respectively. + /// - The function is then called to remove 700 bytes from the value, while + /// no bytes are removed from the key. + /// + /// The expected behavior: + /// - For key removal: No bytes should be removed since the key removal + /// request is zero. + /// - For value removal: It should consume all 400 bytes from epoch 7 (LIFO + /// order) and the remaining 300 bytes from epoch 6. + fn test_multi_epoch_owned_removal_all_bytes_taken_from_non_base_epoch() { + // Define the owner ID as a 32-byte array filled with 5s. + let owner_id = [5u8; 32]; + + // Create a map for additional epochs with 300 bytes in epoch 6. + let mut other_epochs = create_epoch_map(6, 300); + + // Insert 400 bytes for epoch 7 into the map. + other_epochs.insert(7, 400); + + // Initialize the `StorageFlags::MultiEpochOwned` with base epoch 5, additional + // epochs, and the owner ID. + let flags = StorageFlags::MultiEpochOwned(5, other_epochs, owner_id); + + // Call the function to split the storage removal bytes, expecting to remove 700 + // bytes from the value. + let (key_removal, value_removal) = flags.split_storage_removed_bytes(0, 700); + + // Verify that no bytes are removed from the key. + assert_eq!(key_removal, StorageRemovedBytes::NoStorageRemoval); + + // Verify that 700 bytes are removed from the value, consuming 400 bytes from + // epoch 7 and 300 bytes from epoch 6. + assert_eq!( + value_removal, + StorageRemovedBytes::SectionedStorageRemoval({ + let mut map = BTreeMap::new(); + map.insert( + owner_id, + IntMap::from_iter([(5u64, 6), (6u64, 297), (7u64, 397)]), + ); + map + }) + ); + } + + #[test] + fn test_multi_epoch_removal_remaining_base_epoch() { + let mut other_epochs = create_epoch_map(6, 300); + other_epochs.insert(7, 100); + + let flags = StorageFlags::MultiEpoch(5, other_epochs); + let (key_removal, value_removal) = flags.split_storage_removed_bytes(400, 500); + + assert_eq!( + key_removal, + StorageRemovedBytes::SectionedStorageRemoval({ + let mut map = BTreeMap::new(); + map.insert(default_owner_id(), IntMap::from_iter([(5u64, 400)])); + map + }) + ); + assert_eq!( + value_removal, + StorageRemovedBytes::SectionedStorageRemoval({ + let mut map = BTreeMap::new(); + map.insert( + default_owner_id(), + IntMap::from_iter([(7u64, 97), (6u64, 297), (5u64, 106)]), + ); + map + }) + ); + } +} diff --git a/grovedb-epoch-based-storage-flags/src/split_removal_bytes.rs b/grovedb-epoch-based-storage-flags/src/split_removal_bytes.rs new file mode 100644 index 00000000..f9fb351f --- /dev/null +++ b/grovedb-epoch-based-storage-flags/src/split_removal_bytes.rs @@ -0,0 +1,30 @@ +use grovedb_costs::storage_cost::removal::{ + StorageRemovedBytes, StorageRemovedBytes::BasicStorageRemoval, +}; + +use crate::{error::StorageFlagsError, ElementFlags, StorageFlags}; + +impl StorageFlags { + pub fn split_removal_bytes( + flags: &mut ElementFlags, + removed_key_bytes: u32, + removed_value_bytes: u32, + ) -> Result<(StorageRemovedBytes, StorageRemovedBytes), StorageFlagsError> { + let maybe_storage_flags = + StorageFlags::from_element_flags_ref(flags).map_err(|mut e| { + e.add_info("drive did not understand flags of item being updated"); + e + })?; + // if we removed key bytes then we removed the entire value + match maybe_storage_flags { + None => Ok(( + BasicStorageRemoval(removed_key_bytes), + BasicStorageRemoval(removed_value_bytes), + )), + Some(storage_flags) => { + Ok(storage_flags + .split_storage_removed_bytes(removed_key_bytes, removed_value_bytes)) + } + } + } +} diff --git a/grovedb-epoch-based-storage-flags/src/update_element_flags.rs b/grovedb-epoch-based-storage-flags/src/update_element_flags.rs new file mode 100644 index 00000000..932979b1 --- /dev/null +++ b/grovedb-epoch-based-storage-flags/src/update_element_flags.rs @@ -0,0 +1,138 @@ +use grovedb_costs::storage_cost::{transition::OperationStorageTransitionType, StorageCost}; + +use crate::{error::StorageFlagsError, ElementFlags, MergingOwnersStrategy, StorageFlags}; + +impl StorageFlags { + pub fn update_element_flags( + cost: &StorageCost, + old_flags: Option, + new_flags: &mut ElementFlags, + ) -> Result { + // if there were no flags before then the new flags are used + let Some(old_flags) = old_flags else { + return Ok(false); + }; + + // This could be none only because the old element didn't exist + // If they were empty we get an error + let maybe_old_storage_flags = + StorageFlags::from_element_flags_ref(&old_flags).map_err(|mut e| { + e.add_info("drive did not understand flags of old item being updated"); + e + })?; + let new_storage_flags = StorageFlags::from_element_flags_ref(new_flags) + .map_err(|mut e| { + e.add_info("drive did not understand updated item flag information"); + e + })? + .ok_or(StorageFlagsError::RemovingFlagsError( + "removing flags from an item with flags is not allowed".to_string(), + ))?; + let binding = maybe_old_storage_flags.clone().unwrap(); + let old_epoch_index_map = binding.epoch_index_map(); + let new_epoch_index_map = new_storage_flags.epoch_index_map(); + if old_epoch_index_map.is_some() || new_epoch_index_map.is_some() { + // println!("> old:{:?} new:{:?}", old_epoch_index_map, + // new_epoch_index_map); + } + + match &cost.transition_type() { + OperationStorageTransitionType::OperationUpdateBiggerSize => { + // In the case that the owners do not match up this means that there has been a + // transfer of ownership of the underlying document, the value + // held is transferred to the new owner + // println!(">---------------------combine_added_bytes:{}", cost.added_bytes); + // println!(">---------------------apply_batch_with_add_costs old_flags:{:?} + // new_flags:{:?}", maybe_old_storage_flags, new_storage_flags); + let combined_storage_flags = StorageFlags::optional_combine_added_bytes( + maybe_old_storage_flags.clone(), + new_storage_flags.clone(), + cost.added_bytes, + MergingOwnersStrategy::UseTheirs, + ) + .map_err(|mut e| { + e.add_info("drive could not combine storage flags (new flags were bigger)"); + e + })?; + // println!( + // ">added_bytes:{} old:{} new:{} --> combined:{}", + // cost.added_bytes, + // if maybe_old_storage_flags.is_some() { + // maybe_old_storage_flags.as_ref().unwrap().to_string() + // } else { + // "None".to_string() + // }, + // new_storage_flags, + // combined_storage_flags + // ); + // if combined_storage_flags.epoch_index_map().is_some() { + // //println!(" --------> bigger_combined_flags:{:?}", + // combined_storage_flags.epoch_index_map()); } + let combined_flags = combined_storage_flags.to_element_flags(); + // it's possible they got bigger in the same epoch + if combined_flags == *new_flags { + // they are the same there was no update + Ok(false) + } else { + *new_flags = combined_flags; + Ok(true) + } + } + OperationStorageTransitionType::OperationUpdateSmallerSize => { + // println!( + // ">removing_bytes:{:?} old:{} new:{}", + // cost.removed_bytes, + // if maybe_old_storage_flags.is_some() { + // maybe_old_storage_flags.as_ref().unwrap().to_string() + // } else { + // "None".to_string() + // }, + // new_storage_flags, + // ); + // In the case that the owners do not match up this means that there has been a + // transfer of ownership of the underlying document, the value + // held is transferred to the new owner + let combined_storage_flags = StorageFlags::optional_combine_removed_bytes( + maybe_old_storage_flags.clone(), + new_storage_flags.clone(), + &cost.removed_bytes, + MergingOwnersStrategy::UseTheirs, + ) + .map_err(|mut e| { + e.add_info("drive could not combine storage flags (new flags were smaller)"); + e + })?; + // println!( + // ">removed_bytes:{:?} old:{:?} new:{:?} --> combined:{:?}", + // cost.removed_bytes, + // maybe_old_storage_flags, + // new_storage_flags, + // combined_storage_flags + // ); + if combined_storage_flags.epoch_index_map().is_some() { + // println!(" --------> smaller_combined_flags:{:?}", + // combined_storage_flags.epoch_index_map()); + } + let combined_flags = combined_storage_flags.to_element_flags(); + // it's possible they got bigger in the same epoch + if combined_flags == *new_flags { + // they are the same there was no update + Ok(false) + } else { + *new_flags = combined_flags; + Ok(true) + } + } + OperationStorageTransitionType::OperationUpdateSameSize => { + if let Some(old_storage_flags) = maybe_old_storage_flags { + // if there were old storage flags we should just keep them + *new_flags = old_storage_flags.to_element_flags(); + Ok(true) + } else { + Ok(false) + } + } + _ => Ok(false), + } + } +} diff --git a/grovedb-version/Cargo.toml b/grovedb-version/Cargo.toml index 9802a0cf..b466b662 100644 --- a/grovedb-version/Cargo.toml +++ b/grovedb-version/Cargo.toml @@ -2,9 +2,10 @@ name = "grovedb-version" authors = ["Samuel Westrich "] description = "Versioning library for Platform" -version = "1.0.0" +version = "2.0.0" edition = "2021" license = "MIT" +repository = "https://github.com/dashpay/grovedb" [dependencies] thiserror = "1.0.59" diff --git a/grovedb/Cargo.toml b/grovedb/Cargo.toml index 73cd34f4..57eada80 100644 --- a/grovedb/Cargo.toml +++ b/grovedb/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "grovedb" description = "Fully featured database using balanced hierarchical authenticated data structures" -version = "1.0.0" +version = "2.0.0" authors = ["Samuel Westrich ", "Wisdom Ogwu "] edition = "2021" license = "MIT" @@ -11,35 +11,36 @@ readme = "../README.md" documentation = "https://docs.rs/grovedb" [dependencies] -grovedb-merk = { version = "1.0.0", path = "../merk", optional = true, default-features = false } +grovedb-merk = { version = "2.0.0", path = "../merk", optional = true, default-features = false } thiserror = { version = "1.0.59", optional = true } tempfile = { version = "3.10.1", optional = true } bincode = { version = "2.0.0-rc.3" } -grovedb-storage = { version = "1.0.0", path = "../storage", optional = true } -grovedb-visualize = { version = "1.0.0", path = "../visualize", optional = true } +grovedb-storage = { version = "2.0.0", path = "../storage", optional = true } +grovedb-visualize = { version = "2.0.0", path = "../visualize", optional = true } hex = "0.4.3" itertools = { version = "0.12.1", optional = true } derive_more = "0.99.18" integer-encoding = { version = "4.0.0", optional = true } -grovedb-costs = { version = "1.0.0", path = "../costs", optional = true } +grovedb-costs = { version = "2.0.0", path = "../costs" , optional = true } nohash-hasher = { version = "0.2.0", optional = true } indexmap = "2.2.6" intmap = { version = "2.0.0", optional = true } -grovedb-path = { version = "1.0.0", path = "../path" } -grovedbg-types = { version = "1.0.0", path = "../grovedbg-types", optional = true } +grovedb-path = { version = "2.0.0", path = "../path" } +grovedbg-types = { version = "2.0.0", path = "../grovedbg-types", optional = true } tokio = { version = "1.37.0", features = ["rt-multi-thread", "net"], optional = true } axum = { version = "0.7.5", features = ["macros"], optional = true } tower-http = { version = "0.5.2", features = ["fs"], optional = true } blake3 = "1.4.0" bitvec = "1" zip-extensions = { version ="0.6.2", optional = true } -grovedb-version = { path = "../grovedb-version", version = "1.0.0" } +grovedb-version = { path = "../grovedb-version", version = "2.0.0" } [dev-dependencies] rand = "0.8.5" criterion = "0.5.1" hex = "0.4.3" pretty_assertions = "1.4.0" +grovedb-epoch-based-storage-flags = { version = "2.0.0", path = "../grovedb-epoch-based-storage-flags" } [[bench]] name = "insertion_benchmark" @@ -58,7 +59,7 @@ full = [ "integer-encoding", "grovedb-costs", "nohash-hasher", - "intmap" + "intmap", ] visualize = [ "grovedb-visualize", diff --git a/grovedb/build.rs b/grovedb/build.rs index 83a38900..66e72d15 100644 --- a/grovedb/build.rs +++ b/grovedb/build.rs @@ -6,14 +6,17 @@ fn main() { use sha2::{digest::FixedOutput, Digest, Sha256}; const GROVEDBG_SHA256: [u8; 32] = - hex!("c6636f10b43c703128b621a7c4b94139a1a7d0a603e26fca1771734a7994bb7c"); - const GROVEDBG_VERSION: &str = "v1.0.0-rc.5"; + hex!("ea7d9258973aa765eaf5064451fc83efa22e0ce6eaf2938507e2703571364e35"); + const GROVEDBG_VERSION: &str = "v1.0.0-rc.6"; let out_dir = PathBuf::from(&env::var_os("OUT_DIR").unwrap()); let grovedbg_zip_path = out_dir.join("grovedbg.zip"); if !grovedbg_zip_path.exists() { - let response = reqwest::blocking::get(format!("https://github.com/dashpay/grovedbg/releases/download/{GROVEDBG_VERSION}/grovedbg-{GROVEDBG_VERSION}.zip")) + let response = reqwest::blocking::get(format!( + "https://github.com/dashpay/grovedbg/releases/download/\ +{GROVEDBG_VERSION}/grovedbg-{GROVEDBG_VERSION}.zip" + )) .expect("can't download GroveDBG artifact"); let mut grovedbg_zip = File::create(&grovedbg_zip_path).unwrap(); diff --git a/grovedb/src/batch/batch_structure.rs b/grovedb/src/batch/batch_structure.rs index eb00f1e8..9cfe03db 100644 --- a/grovedb/src/batch/batch_structure.rs +++ b/grovedb/src/batch/batch_structure.rs @@ -16,12 +16,12 @@ use nohash_hasher::IntMap; #[cfg(feature = "full")] use crate::{ - batch::{key_info::KeyInfo, GroveDbOp, KeyInfoPath, Op, TreeCache}, + batch::{key_info::KeyInfo, GroveOp, KeyInfoPath, QualifiedGroveDbOp, TreeCache}, Element, ElementFlags, Error, }; #[cfg(feature = "full")] -pub type OpsByPath = BTreeMap>; +pub type OpsByPath = BTreeMap>; /// Level, path, key, op #[cfg(feature = "full")] pub type OpsByLevelPath = IntMap; @@ -32,7 +32,7 @@ pub(super) struct BatchStructure { /// Operations by level path pub(super) ops_by_level_paths: OpsByLevelPath, /// This is for references - pub(super) ops_by_qualified_paths: BTreeMap>, Op>, + pub(super) ops_by_qualified_paths: BTreeMap>, GroveOp>, /// Merk trees /// Very important: the type of run mode we are in is contained in this /// cache @@ -84,7 +84,7 @@ where { /// Create batch structure from a list of ops. Returns CostResult. pub(super) fn from_ops( - ops: Vec, + ops: Vec, update_element_flags_function: F, split_remove_bytes_function: SR, merk_tree_cache: C, @@ -101,7 +101,7 @@ where /// Create batch structure from a list of ops. Returns CostResult. pub(super) fn continue_from_ops( previous_ops: Option, - ops: Vec, + ops: Vec, update_element_flags_function: F, split_remove_bytes_function: SR, mut merk_tree_cache: C, @@ -112,7 +112,7 @@ where let mut current_last_level: u32 = 0; // qualified paths meaning path + key - let mut ops_by_qualified_paths: BTreeMap>, Op> = BTreeMap::new(); + let mut ops_by_qualified_paths: BTreeMap>, GroveOp> = BTreeMap::new(); for op in ops.into_iter() { let mut path = op.path.clone(); @@ -120,7 +120,10 @@ where ops_by_qualified_paths.insert(path.to_path_consume(), op.op.clone()); let op_cost = OperationCost::default(); let op_result = match &op.op { - Op::Insert { element } | Op::Replace { element } | Op::Patch { element, .. } => { + GroveOp::InsertOnly { element } + | GroveOp::InsertOrReplace { element } + | GroveOp::Replace { element } + | GroveOp::Patch { element, .. } => { if let Element::Tree(..) = element { cost_return_on_error!(&mut cost, merk_tree_cache.insert(&op, false)); } else if let Element::SumTree(..) = element { @@ -128,10 +131,11 @@ where } Ok(()) } - Op::RefreshReference { .. } | Op::Delete | Op::DeleteTree | Op::DeleteSumTree => { - Ok(()) - } - Op::ReplaceTreeRootKey { .. } | Op::InsertTreeWithRootHash { .. } => { + GroveOp::RefreshReference { .. } + | GroveOp::Delete + | GroveOp::DeleteTree + | GroveOp::DeleteSumTree => Ok(()), + GroveOp::ReplaceTreeRootKey { .. } | GroveOp::InsertTreeWithRootHash { .. } => { Err(Error::InvalidBatchOperation( "replace and insert tree hash are internal operations only", )) @@ -146,14 +150,14 @@ where if let Some(ops_on_path) = ops_on_level.get_mut(&op.path) { ops_on_path.insert(op.key, op.op); } else { - let mut ops_on_path: BTreeMap = BTreeMap::new(); + let mut ops_on_path: BTreeMap = BTreeMap::new(); ops_on_path.insert(op.key, op.op); ops_on_level.insert(op.path.clone(), ops_on_path); } } else { - let mut ops_on_path: BTreeMap = BTreeMap::new(); + let mut ops_on_path: BTreeMap = BTreeMap::new(); ops_on_path.insert(op.key, op.op); - let mut ops_on_level: BTreeMap> = + let mut ops_on_level: BTreeMap> = BTreeMap::new(); ops_on_level.insert(op.path, ops_on_path); ops_by_level_paths.insert(level, ops_on_level); diff --git a/grovedb/src/batch/estimated_costs/average_case_costs.rs b/grovedb/src/batch/estimated_costs/average_case_costs.rs index 2f50186b..ef64b0f4 100644 --- a/grovedb/src/batch/estimated_costs/average_case_costs.rs +++ b/grovedb/src/batch/estimated_costs/average_case_costs.rs @@ -26,14 +26,14 @@ use crate::Element; #[cfg(feature = "full")] use crate::{ batch::{ - key_info::KeyInfo, mode::BatchRunMode, BatchApplyOptions, GroveDbOp, KeyInfoPath, Op, - TreeCache, + key_info::KeyInfo, mode::BatchRunMode, BatchApplyOptions, GroveOp, KeyInfoPath, + QualifiedGroveDbOp, TreeCache, }, Error, GroveDb, }; #[cfg(feature = "full")] -impl Op { +impl GroveOp { /// Get the estimated average case cost of the op. Calls a lower level /// function to calculate the estimate based on the type of op. Returns /// CostResult. @@ -53,14 +53,14 @@ impl Op { } }; match self { - Op::ReplaceTreeRootKey { sum, .. } => GroveDb::average_case_merk_replace_tree( + GroveOp::ReplaceTreeRootKey { sum, .. } => GroveDb::average_case_merk_replace_tree( key, layer_element_estimates, sum.is_some(), propagate, grove_version, ), - Op::InsertTreeWithRootHash { flags, sum, .. } => { + GroveOp::InsertTreeWithRootHash { flags, sum, .. } => { GroveDb::average_case_merk_insert_tree( key, flags, @@ -70,14 +70,16 @@ impl Op { grove_version, ) } - Op::Insert { element } => GroveDb::average_case_merk_insert_element( - key, - element, - in_tree_using_sums, - propagate_if_input(), - grove_version, - ), - Op::RefreshReference { + GroveOp::InsertOrReplace { element } | GroveOp::InsertOnly { element } => { + GroveDb::average_case_merk_insert_element( + key, + element, + in_tree_using_sums, + propagate_if_input(), + grove_version, + ) + } + GroveOp::RefreshReference { reference_path_type, max_reference_hop, flags, @@ -93,14 +95,14 @@ impl Op { propagate_if_input(), grove_version, ), - Op::Replace { element } => GroveDb::average_case_merk_replace_element( + GroveOp::Replace { element } => GroveDb::average_case_merk_replace_element( key, element, in_tree_using_sums, propagate_if_input(), grove_version, ), - Op::Patch { + GroveOp::Patch { element, change_in_bytes, } => GroveDb::average_case_merk_patch_element( @@ -111,20 +113,20 @@ impl Op { propagate_if_input(), grove_version, ), - Op::Delete => GroveDb::average_case_merk_delete_element( + GroveOp::Delete => GroveDb::average_case_merk_delete_element( key, layer_element_estimates, propagate, grove_version, ), - Op::DeleteTree => GroveDb::average_case_merk_delete_tree( + GroveOp::DeleteTree => GroveDb::average_case_merk_delete_tree( key, false, layer_element_estimates, propagate, grove_version, ), - Op::DeleteSumTree => GroveDb::average_case_merk_delete_tree( + GroveOp::DeleteSumTree => GroveDb::average_case_merk_delete_tree( key, true, layer_element_estimates, @@ -165,7 +167,7 @@ impl fmt::Debug for AverageCaseTreeCacheKnownPaths { #[cfg(feature = "full")] impl TreeCache for AverageCaseTreeCacheKnownPaths { - fn insert(&mut self, op: &GroveDbOp, is_sum_tree: bool) -> CostResult<(), Error> { + fn insert(&mut self, op: &QualifiedGroveDbOp, is_sum_tree: bool) -> CostResult<(), Error> { let mut average_case_cost = OperationCost::default(); let mut inserted_path = op.path.clone(); inserted_path.push(op.key.clone()); @@ -184,8 +186,8 @@ impl TreeCache for AverageCaseTreeCacheKnownPaths { fn execute_ops_on_path( &mut self, path: &KeyInfoPath, - ops_at_path_by_key: BTreeMap, - _ops_by_qualified_paths: &BTreeMap>, Op>, + ops_at_path_by_key: BTreeMap, + _ops_by_qualified_paths: &BTreeMap>, GroveOp>, _batch_apply_options: &BatchApplyOptions, _flags_update: &mut G, _split_removal_bytes: &mut SR, @@ -309,7 +311,7 @@ mod tests { use crate::{ batch::{ estimated_costs::EstimatedCostsType::AverageCaseCostsType, key_info::KeyInfo, - GroveDbOp, KeyInfoPath, + KeyInfoPath, QualifiedGroveDbOp, }, tests::{common::EMPTY_PATH, make_empty_grovedb}, Element, GroveDb, @@ -321,7 +323,7 @@ mod tests { let db = make_empty_grovedb(); let tx = db.start_transaction(); - let ops = vec![GroveDbOp::insert_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![], b"key1".to_vec(), Element::empty_tree(), @@ -390,7 +392,7 @@ mod tests { let db = make_empty_grovedb(); let tx = db.start_transaction(); - let ops = vec![GroveDbOp::insert_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![], b"key1".to_vec(), Element::empty_tree_with_flags(Some(b"cat".to_vec())), @@ -457,7 +459,7 @@ mod tests { let db = make_empty_grovedb(); let tx = db.start_transaction(); - let ops = vec![GroveDbOp::insert_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![], b"key1".to_vec(), Element::new_item(b"cat".to_vec()), @@ -530,7 +532,7 @@ mod tests { .unwrap() .expect("successful root tree leaf insert"); - let ops = vec![GroveDbOp::insert_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![], b"key1".to_vec(), Element::empty_tree(), @@ -616,7 +618,7 @@ mod tests { .unwrap() .expect("successful root tree leaf insert"); - let ops = vec![GroveDbOp::insert_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![b"0".to_vec()], b"key1".to_vec(), Element::empty_tree(), @@ -695,7 +697,7 @@ mod tests { #[test] fn test_batch_root_one_sum_item_replace_op_average_case_costs() { let grove_version = GroveVersion::latest(); - let ops = vec![GroveDbOp::replace_op( + let ops = vec![QualifiedGroveDbOp::replace_op( vec![vec![7]], hex::decode("46447a3b4c8939fd4cf8b610ba7da3d3f6b52b39ab2549bf91503b9b07814055") .unwrap(), @@ -774,7 +776,7 @@ mod tests { .unwrap() .expect("successful root tree leaf insert"); - let ops = vec![GroveDbOp::insert_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![], b"key1".to_vec(), Element::empty_tree(), diff --git a/grovedb/src/batch/estimated_costs/worst_case_costs.rs b/grovedb/src/batch/estimated_costs/worst_case_costs.rs index 1b1d42e7..9bf9a808 100644 --- a/grovedb/src/batch/estimated_costs/worst_case_costs.rs +++ b/grovedb/src/batch/estimated_costs/worst_case_costs.rs @@ -25,14 +25,14 @@ use crate::Element; #[cfg(feature = "full")] use crate::{ batch::{ - key_info::KeyInfo, mode::BatchRunMode, BatchApplyOptions, GroveDbOp, KeyInfoPath, Op, - TreeCache, + key_info::KeyInfo, mode::BatchRunMode, BatchApplyOptions, GroveOp, KeyInfoPath, + QualifiedGroveDbOp, TreeCache, }, Error, GroveDb, }; #[cfg(feature = "full")] -impl Op { +impl GroveOp { fn worst_case_cost( &self, key: &KeyInfo, @@ -49,7 +49,7 @@ impl Op { } }; match self { - Op::ReplaceTreeRootKey { sum, .. } => GroveDb::worst_case_merk_replace_tree( + GroveOp::ReplaceTreeRootKey { sum, .. } => GroveDb::worst_case_merk_replace_tree( key, sum.is_some(), is_in_parent_sum_tree, @@ -57,22 +57,26 @@ impl Op { propagate, grove_version, ), - Op::InsertTreeWithRootHash { flags, sum, .. } => GroveDb::worst_case_merk_insert_tree( - key, - flags, - sum.is_some(), - is_in_parent_sum_tree, - propagate_if_input(), - grove_version, - ), - Op::Insert { element } => GroveDb::worst_case_merk_insert_element( - key, - element, - is_in_parent_sum_tree, - propagate_if_input(), - grove_version, - ), - Op::RefreshReference { + GroveOp::InsertTreeWithRootHash { flags, sum, .. } => { + GroveDb::worst_case_merk_insert_tree( + key, + flags, + sum.is_some(), + is_in_parent_sum_tree, + propagate_if_input(), + grove_version, + ) + } + GroveOp::InsertOrReplace { element } | GroveOp::InsertOnly { element } => { + GroveDb::worst_case_merk_insert_element( + key, + element, + is_in_parent_sum_tree, + propagate_if_input(), + grove_version, + ) + } + GroveOp::RefreshReference { reference_path_type, max_reference_hop, flags, @@ -88,14 +92,14 @@ impl Op { propagate_if_input(), grove_version, ), - Op::Replace { element } => GroveDb::worst_case_merk_replace_element( + GroveOp::Replace { element } => GroveDb::worst_case_merk_replace_element( key, element, is_in_parent_sum_tree, propagate_if_input(), grove_version, ), - Op::Patch { + GroveOp::Patch { element, change_in_bytes: _, } => GroveDb::worst_case_merk_replace_element( @@ -105,20 +109,20 @@ impl Op { propagate_if_input(), grove_version, ), - Op::Delete => GroveDb::worst_case_merk_delete_element( + GroveOp::Delete => GroveDb::worst_case_merk_delete_element( key, worst_case_layer_element_estimates, propagate, grove_version, ), - Op::DeleteTree => GroveDb::worst_case_merk_delete_tree( + GroveOp::DeleteTree => GroveDb::worst_case_merk_delete_tree( key, false, worst_case_layer_element_estimates, propagate, grove_version, ), - Op::DeleteSumTree => GroveDb::worst_case_merk_delete_tree( + GroveOp::DeleteSumTree => GroveDb::worst_case_merk_delete_tree( key, true, worst_case_layer_element_estimates, @@ -159,7 +163,7 @@ impl fmt::Debug for WorstCaseTreeCacheKnownPaths { #[cfg(feature = "full")] impl TreeCache for WorstCaseTreeCacheKnownPaths { - fn insert(&mut self, op: &GroveDbOp, _is_sum_tree: bool) -> CostResult<(), Error> { + fn insert(&mut self, op: &QualifiedGroveDbOp, _is_sum_tree: bool) -> CostResult<(), Error> { let mut worst_case_cost = OperationCost::default(); let mut inserted_path = op.path.clone(); inserted_path.push(op.key.clone()); @@ -178,8 +182,8 @@ impl TreeCache for WorstCaseTreeCacheKnownPaths { fn execute_ops_on_path( &mut self, path: &KeyInfoPath, - ops_at_path_by_key: BTreeMap, - _ops_by_qualified_paths: &BTreeMap>, Op>, + ops_at_path_by_key: BTreeMap, + _ops_by_qualified_paths: &BTreeMap>, GroveOp>, _batch_apply_options: &BatchApplyOptions, _flags_update: &mut G, _split_removal_bytes: &mut SR, @@ -273,8 +277,8 @@ mod tests { use crate::{ batch::{ - estimated_costs::EstimatedCostsType::WorstCaseCostsType, key_info::KeyInfo, GroveDbOp, - KeyInfoPath, + estimated_costs::EstimatedCostsType::WorstCaseCostsType, key_info::KeyInfo, + KeyInfoPath, QualifiedGroveDbOp, }, tests::{common::EMPTY_PATH, make_empty_grovedb}, Element, GroveDb, @@ -286,7 +290,7 @@ mod tests { let db = make_empty_grovedb(); let tx = db.start_transaction(); - let ops = vec![GroveDbOp::insert_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![], b"key1".to_vec(), Element::empty_tree(), @@ -341,7 +345,7 @@ mod tests { let db = make_empty_grovedb(); let tx = db.start_transaction(); - let ops = vec![GroveDbOp::insert_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![], b"key1".to_vec(), Element::empty_tree_with_flags(Some(b"cat".to_vec())), @@ -396,7 +400,7 @@ mod tests { let db = make_empty_grovedb(); let tx = db.start_transaction(); - let ops = vec![GroveDbOp::insert_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![], b"key1".to_vec(), Element::new_item(b"cat".to_vec()), @@ -462,7 +466,7 @@ mod tests { .unwrap() .expect("successful root tree leaf insert"); - let ops = vec![GroveDbOp::insert_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![], b"key1".to_vec(), Element::empty_tree(), @@ -528,7 +532,7 @@ mod tests { .unwrap() .expect("successful root tree leaf insert"); - let ops = vec![GroveDbOp::insert_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![b"0".to_vec()], b"key1".to_vec(), Element::empty_tree(), @@ -592,7 +596,7 @@ mod tests { .unwrap() .expect("successful root tree leaf insert"); - let ops = vec![GroveDbOp::insert_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![], b"key1".to_vec(), Element::empty_tree(), diff --git a/grovedb/src/batch/just_in_time_cost_tests.rs b/grovedb/src/batch/just_in_time_cost_tests.rs index e1fddf5c..9c45680b 100644 --- a/grovedb/src/batch/just_in_time_cost_tests.rs +++ b/grovedb/src/batch/just_in_time_cost_tests.rs @@ -3,17 +3,119 @@ #[cfg(feature = "full")] mod tests { - use std::option::Option::None; + use std::{collections::BTreeMap, option::Option::None}; + use grovedb_costs::{ + storage_cost::removal::{StorageRemovalPerEpochByIdentifier, StorageRemovedBytes}, + OperationCost, + }; + use grovedb_epoch_based_storage_flags::StorageFlags; use grovedb_version::version::GroveVersion; + use intmap::IntMap; use crate::{ - batch::GroveDbOp, - reference_path::ReferencePathType::UpstreamFromElementHeightReference, - tests::{common::EMPTY_PATH, make_empty_grovedb}, - Element, + batch::QualifiedGroveDbOp, + reference_path::{ + ReferencePathType, ReferencePathType::UpstreamFromElementHeightReference, + }, + tests::{common::EMPTY_PATH, make_empty_grovedb, TempGroveDb}, + Element, Error, Transaction, }; + fn single_epoch_removed_bytes_map( + owner_id: [u8; 32], + epoch_index: u64, + bytes_removed: u32, + ) -> StorageRemovalPerEpochByIdentifier { + let mut removed_bytes = StorageRemovalPerEpochByIdentifier::default(); + let mut removed_bytes_for_identity = IntMap::new(); + removed_bytes_for_identity.insert(epoch_index, bytes_removed); + removed_bytes.insert(owner_id, removed_bytes_for_identity); + removed_bytes + } + + fn apply_batch( + grove_db: &TempGroveDb, + ops: Vec, + tx: &Transaction, + grove_version: &GroveVersion, + ) -> OperationCost { + grove_db + .apply_batch_with_element_flags_update( + ops, + None, + |cost, old_flags, new_flags| { + StorageFlags::update_element_flags(cost, old_flags, new_flags) + .map_err(|e| Error::JustInTimeElementFlagsClientError(e.to_string())) + }, + |flags, removed_key_bytes, removed_value_bytes| { + StorageFlags::split_removal_bytes(flags, removed_key_bytes, removed_value_bytes) + .map_err(|e| Error::SplitRemovalBytesClientError(e.to_string())) + }, + Some(&tx), + grove_version, + ) + .cost_as_result() + .expect("expected to not error") + } + + fn expect_storage_flags( + grove_db: &TempGroveDb, + tx: &Transaction, + expected_storage_flags: StorageFlags, + grove_version: &GroveVersion, + ) { + let element = grove_db + .get( + [b"tree".as_slice()].as_ref(), + b"key1", + Some(tx), + grove_version, + ) + .unwrap() + .expect("expected element"); + let storage_flags = StorageFlags::from_element_flags_ref( + element.get_flags().as_ref().expect("expected flags"), + ) + .expect("expected to get storage flags") + .expect("expected storage flags"); + assert_eq!(storage_flags, expected_storage_flags); + } + + fn verify_references(grove_db: &TempGroveDb, tx: &Transaction) { + let issues = grove_db + .visualize_verify_grovedb(Some(&tx), true, false, &Default::default()) + .unwrap(); + assert_eq!( + issues.len(), + 0, + "reference issue: {}", + issues + .iter() + .map(|(hash, (a, b, c))| format!("{}: {} {} {}", hash, a, b, c)) + .collect::>() + .join(" | ") + ); + } + + fn create_epoch_map(epoch: u16, bytes: u32) -> BTreeMap { + let mut map = BTreeMap::new(); + map.insert(epoch, bytes); + map + } + + fn create_two_epoch_map( + first_epoch: u16, + first_epoch_bytes: u32, + second_epoch: u16, + second_epoch_bytes: u32, + ) -> BTreeMap { + let mut map = BTreeMap::new(); + map.insert(first_epoch, first_epoch_bytes); + map.insert(second_epoch, second_epoch_bytes); + map + } + #[test] fn test_partial_costs_with_no_new_operations_are_same_as_apply_batch() { let grove_version = GroveVersion::latest(); @@ -41,17 +143,17 @@ mod tests { .cost_as_result() .expect("expected to insert successfully"); let ops = vec![ - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![b"documents".to_vec()], b"key2".to_vec(), Element::new_item_with_flags(b"pizza".to_vec(), Some([0, 1].to_vec())), ), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![b"documents".to_vec()], b"key3".to_vec(), Element::empty_tree(), ), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![b"documents".to_vec(), b"key3".to_vec()], b"key4".to_vec(), Element::new_reference(UpstreamFromElementHeightReference( @@ -175,17 +277,17 @@ mod tests { .cost_as_result() .expect("expected to insert successfully"); let ops = vec![ - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![b"documents".to_vec()], b"key2".to_vec(), Element::new_item_with_flags(b"pizza".to_vec(), Some([0, 1].to_vec())), ), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![b"documents".to_vec()], b"key3".to_vec(), Element::empty_tree(), ), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![b"documents".to_vec(), b"key3".to_vec()], b"key4".to_vec(), Element::new_reference(UpstreamFromElementHeightReference( @@ -247,7 +349,7 @@ mod tests { .get(&0) .expect("expected to have root path"); assert_eq!(ops_by_root_path.len(), 1); - let new_ops = vec![GroveDbOp::insert_op( + let new_ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![b"balances".to_vec()], b"person".to_vec(), Element::new_sum_item_with_flags(1000, Some([0, 1].to_vec())), @@ -311,4 +413,1082 @@ mod tests { assert_ne!(apply_root_hash, apply_partial_root_hash); } + + #[test] + fn test_one_update_bigger_item_same_epoch_with_reference() { + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + let tx = db.start_transaction(); + + let owner_id = [1; 32]; + db.insert( + EMPTY_PATH, + b"tree", + Element::empty_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to insert tree"); + + db.insert( + EMPTY_PATH, + b"refs", + Element::empty_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to insert tree"); + + db.insert( + [b"tree".as_slice()].as_ref(), + b"key1", + Element::new_item_with_flags( + b"value1".to_vec(), + Some(StorageFlags::new_single_epoch(0, Some(owner_id)).to_element_flags()), + ), + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to insert item"); + + // We are adding 2 bytes + let ops = vec![ + QualifiedGroveDbOp::insert_or_replace_op( + vec![b"tree".to_vec()], + b"key1".to_vec(), + Element::new_item_with_flags( + b"value100".to_vec(), + Some(StorageFlags::new_single_epoch(0, Some(owner_id)).to_element_flags()), + ), + ), + QualifiedGroveDbOp::insert_only_op( + vec![b"refs".to_vec()], + b"ref_key".to_vec(), + Element::new_reference_with_hops( + ReferencePathType::AbsolutePathReference(vec![ + b"tree".to_vec(), + b"key1".to_vec(), + ]), + Some(1), + ), + ), + ]; + + apply_batch(&db, ops, &tx, grove_version); + + expect_storage_flags( + &db, + &tx, + StorageFlags::new_single_epoch(0, Some(owner_id)), + grove_version, + ); + + verify_references(&db, &tx); + } + + #[test] + fn test_one_update_bigger_item_different_epoch_with_reference() { + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + + let owner_id = [1; 32]; + + db.insert( + EMPTY_PATH, + b"tree", + Element::empty_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to insert tree"); + + db.insert( + EMPTY_PATH, + b"refs", + Element::empty_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to insert tree"); + + db.insert( + [b"tree".as_slice()].as_ref(), + b"key1", + Element::new_item_with_flags( + b"value1".to_vec(), + Some(StorageFlags::new_single_epoch(0, Some(owner_id)).to_element_flags()), + ), + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to insert item"); + + let base_item = b"value1".to_vec(); + + for n in 1..150 { + let tx = db.start_transaction(); + let mut item = base_item.clone(); + item.extend(std::iter::repeat(0).take(n)); + // We are adding n bytes + let ops = vec![ + QualifiedGroveDbOp::insert_or_replace_op( + vec![b"tree".to_vec()], + b"key1".to_vec(), + Element::new_item_with_flags( + item, // value100 if n was 2 + Some(StorageFlags::new_single_epoch(1, Some(owner_id)).to_element_flags()), + ), + ), + QualifiedGroveDbOp::insert_only_op( + vec![b"refs".to_vec()], + b"ref_key".to_vec(), + Element::new_reference_with_hops( + ReferencePathType::AbsolutePathReference(vec![ + b"tree".to_vec(), + b"key1".to_vec(), + ]), + None, + ), + ), + ]; + + apply_batch(&db, ops, &tx, grove_version); + + let expected_added_bytes = if n < 15 { + n as u32 + 3 + } else if n < 124 { + n as u32 + 4 // the varint requires an extra byte + } else { + n as u32 + 5 // the varint requires an extra byte + }; + expect_storage_flags( + &db, + &tx, + StorageFlags::MultiEpochOwned( + 0, + create_epoch_map(1, expected_added_bytes), + owner_id, + ), + grove_version, + ); + + verify_references(&db, &tx); + } + } + + #[test] + fn test_one_update_bigger_item_different_base_epoch_with_bytes_in_last_epoch_with_reference() { + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + + let owner_id = [1; 32]; + + db.insert( + EMPTY_PATH, + b"tree", + Element::empty_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to insert tree"); + + db.insert( + EMPTY_PATH, + b"refs", + Element::empty_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to insert tree"); + + db.insert( + [b"tree".as_slice()].as_ref(), + b"key1", + Element::new_item_with_flags( + b"value1".to_vec(), + Some( + StorageFlags::MultiEpochOwned(0, create_epoch_map(1, 4), owner_id) + .to_element_flags(), + ), + ), + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to insert item"); + + let base_item = b"value1".to_vec(); + + for n in 1..150 { + let tx = db.start_transaction(); + let mut item = base_item.clone(); + item.extend(std::iter::repeat(0).take(n)); + // We are adding n bytes + let ops = vec![ + QualifiedGroveDbOp::insert_or_replace_op( + vec![b"tree".to_vec()], + b"key1".to_vec(), + Element::new_item_with_flags( + item, // value100 if n was 2 + Some(StorageFlags::new_single_epoch(1, Some(owner_id)).to_element_flags()), + ), + ), + QualifiedGroveDbOp::insert_only_op( + vec![b"refs".to_vec()], + b"ref_key".to_vec(), + Element::new_reference_with_hops( + ReferencePathType::AbsolutePathReference(vec![ + b"tree".to_vec(), + b"key1".to_vec(), + ]), + None, + ), + ), + ]; + + apply_batch(&db, ops, &tx, grove_version); + + let expected_added_bytes = if n < 15 { + n as u32 + 4 + } else if n < 123 { + n as u32 + 5 // the varint requires an extra byte + } else { + n as u32 + 6 // the varint requires an extra byte + }; + expect_storage_flags( + &db, + &tx, + StorageFlags::MultiEpochOwned( + 0, + create_epoch_map(1, expected_added_bytes), + owner_id, + ), + grove_version, + ); + + verify_references(&db, &tx); + } + } + + #[test] + fn test_one_update_bigger_item_different_base_epoch_with_bytes_in_future_epoch_with_reference() + { + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + + let owner_id = [1; 32]; + + db.insert( + EMPTY_PATH, + b"tree", + Element::empty_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to insert tree"); + + db.insert( + EMPTY_PATH, + b"refs", + Element::empty_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to insert tree"); + + db.insert( + [b"tree".as_slice()].as_ref(), + b"key1", + Element::new_item_with_flags( + b"value1".to_vec(), + Some( + StorageFlags::MultiEpochOwned(0, create_epoch_map(1, 4), owner_id) + .to_element_flags(), + ), + ), + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to insert item"); + + let base_item = b"value1".to_vec(); + + for n in 1..150 { + let tx = db.start_transaction(); + let mut item = base_item.clone(); + item.extend(std::iter::repeat(0).take(n)); + // We are adding n bytes + let ops = vec![ + QualifiedGroveDbOp::insert_or_replace_op( + vec![b"tree".to_vec()], + b"key1".to_vec(), + Element::new_item_with_flags( + item, // value100 if n was 2 + Some(StorageFlags::new_single_epoch(2, Some(owner_id)).to_element_flags()), + ), + ), + QualifiedGroveDbOp::insert_only_op( + vec![b"refs".to_vec()], + b"ref_key".to_vec(), + Element::new_reference_with_hops( + ReferencePathType::AbsolutePathReference(vec![ + b"tree".to_vec(), + b"key1".to_vec(), + ]), + None, + ), + ), + ]; + + apply_batch(&db, ops, &tx, grove_version); + + let expected_added_bytes = if n < 12 { + n as u32 + 3 + } else if n < 124 { + n as u32 + 4 // the varint requires an extra byte + } else { + n as u32 + 5 // the varint requires an extra byte + }; + expect_storage_flags( + &db, + &tx, + StorageFlags::MultiEpochOwned( + 0, + create_two_epoch_map(1, 4, 2, expected_added_bytes), + owner_id, + ), + grove_version, + ); + + verify_references(&db, &tx); + } + } + + #[test] + fn test_one_update_smaller_item_same_base_epoch_with_reference() { + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + + let owner_id = [1; 32]; + + db.insert( + EMPTY_PATH, + b"tree", + Element::empty_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to insert tree"); + + db.insert( + EMPTY_PATH, + b"refs", + Element::empty_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to insert tree"); + + let base_item = b"value1".to_vec(); + let mut original_item = base_item.clone(); + original_item.extend(std::iter::repeat(0).take(150)); + + db.insert( + [b"tree".as_slice()].as_ref(), + b"key1", + Element::new_item_with_flags( + original_item, + Some(StorageFlags::new_single_epoch(0, Some(owner_id)).to_element_flags()), + ), + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to insert item"); + + let to = 150usize; + + for n in (0..to).rev() { + let tx = db.start_transaction(); + let mut item = base_item.clone(); + item.extend(std::iter::repeat(0).take(n)); + // We are adding n bytes + let ops = vec![ + QualifiedGroveDbOp::insert_or_replace_op( + vec![b"tree".to_vec()], + b"key1".to_vec(), + Element::new_item_with_flags( + item, // value1 if n was 1 + Some(StorageFlags::new_single_epoch(0, Some(owner_id)).to_element_flags()), + ), + ), + QualifiedGroveDbOp::insert_only_op( + vec![b"refs".to_vec()], + b"ref_key".to_vec(), + Element::new_reference_with_hops( + ReferencePathType::AbsolutePathReference(vec![ + b"tree".to_vec(), + b"key1".to_vec(), + ]), + None, + ), + ), + ]; + + let removed_bytes = if n > 17 { + to as u32 - n as u32 + } else { + to as u32 - n as u32 + 1 // we remove an extra byte + }; + + let storage_removed_bytes = apply_batch(&db, ops, &tx, grove_version) + .storage_cost + .removed_bytes; + + let expected_storage_removed_bytes = + single_epoch_removed_bytes_map(owner_id, 0, removed_bytes); + + assert_eq!( + storage_removed_bytes, + StorageRemovedBytes::SectionedStorageRemoval(expected_storage_removed_bytes) + ); + + expect_storage_flags( + &db, + &tx, + StorageFlags::SingleEpochOwned(0, owner_id), + grove_version, + ); + + verify_references(&db, &tx); + } + } + + #[test] + fn test_one_update_smaller_item_different_base_epoch_with_reference() { + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + + let owner_id = [1; 32]; + + db.insert( + EMPTY_PATH, + b"tree", + Element::empty_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to insert tree"); + + db.insert( + EMPTY_PATH, + b"refs", + Element::empty_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to insert tree"); + + let base_item = b"value1".to_vec(); + let mut original_item = base_item.clone(); + original_item.extend(std::iter::repeat(0).take(150)); + + db.insert( + [b"tree".as_slice()].as_ref(), + b"key1", + Element::new_item_with_flags( + original_item, + Some(StorageFlags::new_single_epoch(0, Some(owner_id)).to_element_flags()), + ), + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to insert item"); + + for n in 0..150 { + let tx = db.start_transaction(); + let mut item = base_item.clone(); + item.extend(std::iter::repeat(0).take(n)); + // We are adding n bytes + let ops = vec![ + QualifiedGroveDbOp::insert_or_replace_op( + vec![b"tree".to_vec()], + b"key1".to_vec(), + Element::new_item_with_flags( + item, // value1 if n was 1 + Some(StorageFlags::new_single_epoch(1, Some(owner_id)).to_element_flags()), + ), + ), + QualifiedGroveDbOp::insert_only_op( + vec![b"refs".to_vec()], + b"ref_key".to_vec(), + Element::new_reference_with_hops( + ReferencePathType::AbsolutePathReference(vec![ + b"tree".to_vec(), + b"key1".to_vec(), + ]), + None, + ), + ), + ]; + + apply_batch(&db, ops, &tx, grove_version); + + expect_storage_flags( + &db, + &tx, + StorageFlags::SingleEpochOwned(0, owner_id), + grove_version, + ); + + verify_references(&db, &tx); + } + } + + #[test] + fn test_one_update_smaller_item_different_base_epoch_with_previous_flags() { + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + let tx = db.start_transaction(); + + let owner_id = [1; 32]; + + db.insert( + EMPTY_PATH, + b"tree", + Element::empty_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to insert tree"); + + db.insert( + EMPTY_PATH, + b"refs", + Element::empty_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to insert tree"); + + db.insert( + [b"tree".as_slice()].as_ref(), + b"key1", + Element::new_item_with_flags( + b"value1500".to_vec(), // the 1500 is 4 bytes + Some( + StorageFlags::MultiEpochOwned(0, create_epoch_map(1, 7), owner_id) + .to_element_flags(), + ), + ), + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to insert item"); + + // We are removing 2 bytes + let ops = vec![ + QualifiedGroveDbOp::insert_or_replace_op( + vec![b"tree".to_vec()], + b"key1".to_vec(), + Element::new_item_with_flags( + b"value15".to_vec(), + Some(StorageFlags::new_single_epoch(1, Some(owner_id)).to_element_flags()), + ), + ), + QualifiedGroveDbOp::insert_only_op( + vec![b"refs".to_vec()], + b"ref_key".to_vec(), + Element::new_reference_with_hops( + ReferencePathType::AbsolutePathReference(vec![ + b"tree".to_vec(), + b"key1".to_vec(), + ]), + None, + ), + ), + ]; + + apply_batch(&db, ops, &tx, grove_version); + + expect_storage_flags( + &db, + &tx, + StorageFlags::MultiEpochOwned(0, create_epoch_map(1, 5), owner_id), + grove_version, + ); + + verify_references(&db, &tx); + } + + #[test] + fn test_one_update_smaller_item_different_base_epoch_with_previous_flags_all_multi_epoch() { + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + let tx = db.start_transaction(); + + let owner_id = [1; 32]; + + db.insert( + EMPTY_PATH, + b"tree", + Element::empty_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to insert tree"); + + db.insert( + EMPTY_PATH, + b"refs", + Element::empty_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to insert tree"); + + db.insert( + [b"tree".as_slice()].as_ref(), + b"key1", + Element::new_item_with_flags( + b"value1500".to_vec(), // the 1500 is 4 bytes + Some( + StorageFlags::MultiEpochOwned(0, create_epoch_map(1, 7), owner_id) + .to_element_flags(), + ), + ), + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to insert item"); + + // We are removing 2 bytes + let ops = vec![ + QualifiedGroveDbOp::insert_or_replace_op( + vec![b"tree".to_vec()], + b"key1".to_vec(), + Element::new_item_with_flags( + b"value".to_vec(), + Some(StorageFlags::new_single_epoch(1, Some(owner_id)).to_element_flags()), + ), + ), + QualifiedGroveDbOp::insert_only_op( + vec![b"refs".to_vec()], + b"ref_key".to_vec(), + Element::new_reference_with_hops( + ReferencePathType::AbsolutePathReference(vec![ + b"tree".to_vec(), + b"key1".to_vec(), + ]), + None, + ), + ), + ]; + + apply_batch(&db, ops, &tx, grove_version); + + expect_storage_flags( + &db, + &tx, + StorageFlags::SingleEpochOwned(0, owner_id), + grove_version, + ); + + verify_references(&db, &tx); + } + + #[test] + fn test_one_update_bigger_sum_item_same_epoch_with_reference() { + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + let tx = db.start_transaction(); + + let owner_id = [1; 32]; + db.insert( + EMPTY_PATH, + b"tree", + Element::empty_sum_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to insert tree"); + + db.insert( + EMPTY_PATH, + b"refs", + Element::empty_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to insert tree"); + + db.insert( + [b"tree".as_slice()].as_ref(), + b"key1", + Element::new_sum_item_with_flags( + 1, + Some(StorageFlags::new_single_epoch(0, Some(owner_id)).to_element_flags()), + ), + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to insert item"); + + // We are adding 2 bytes + let ops = vec![ + QualifiedGroveDbOp::insert_or_replace_op( + vec![b"tree".to_vec()], + b"key1".to_vec(), + Element::new_sum_item_with_flags( + 100000000000, + Some(StorageFlags::new_single_epoch(0, Some(owner_id)).to_element_flags()), + ), + ), + QualifiedGroveDbOp::insert_only_op( + vec![b"refs".to_vec()], + b"ref_key".to_vec(), + Element::new_reference_with_hops( + ReferencePathType::AbsolutePathReference(vec![ + b"tree".to_vec(), + b"key1".to_vec(), + ]), + Some(1), + ), + ), + ]; + + apply_batch(&db, ops, &tx, grove_version); + + expect_storage_flags( + &db, + &tx, + StorageFlags::new_single_epoch(0, Some(owner_id)), + grove_version, + ); + + verify_references(&db, &tx); + } + + #[test] + fn test_one_update_bigger_sum_item_different_epoch_with_reference() { + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + + let owner_id = [1; 32]; + + db.insert( + EMPTY_PATH, + b"tree", + Element::empty_sum_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to insert tree"); + + db.insert( + EMPTY_PATH, + b"refs", + Element::empty_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to insert tree"); + + db.insert( + [b"tree".as_slice()].as_ref(), + b"key1", + Element::new_sum_item_with_flags( + 1, + Some(StorageFlags::new_single_epoch(0, Some(owner_id)).to_element_flags()), + ), + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to insert item"); + + let tx = db.start_transaction(); + // We are adding n bytes + let ops = vec![ + QualifiedGroveDbOp::insert_or_replace_op( + vec![b"tree".to_vec()], + b"key1".to_vec(), + Element::new_sum_item_with_flags( + 10000000000, + Some(StorageFlags::new_single_epoch(1, Some(owner_id)).to_element_flags()), + ), + ), + QualifiedGroveDbOp::insert_only_op( + vec![b"refs".to_vec()], + b"ref_key".to_vec(), + Element::new_reference_with_hops( + ReferencePathType::AbsolutePathReference(vec![ + b"tree".to_vec(), + b"key1".to_vec(), + ]), + None, + ), + ), + ]; + + apply_batch(&db, ops, &tx, grove_version); + + expect_storage_flags( + &db, + &tx, + StorageFlags::SingleEpochOwned(0, owner_id), // no change + grove_version, + ); + + verify_references(&db, &tx); + } + + #[test] + fn test_one_update_bigger_item_add_flags() { + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + + let owner_id = [1; 32]; + + db.insert( + EMPTY_PATH, + b"tree", + Element::empty_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to insert tree"); + + db.insert( + EMPTY_PATH, + b"refs", + Element::empty_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to insert tree"); + + db.insert( + [b"tree".as_slice()].as_ref(), + b"key1", + Element::new_item_with_flags(b"value1".to_vec(), None), + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to insert item"); + + let base_item = b"value1".to_vec(); + + for n in 1..150 { + let tx = db.start_transaction(); + let mut item = base_item.clone(); + item.extend(std::iter::repeat(0).take(n)); + // We are adding n bytes + let ops = vec![ + QualifiedGroveDbOp::insert_or_replace_op( + vec![b"tree".to_vec()], + b"key1".to_vec(), + Element::new_item_with_flags( + item, // value100 if n was 2 + Some(StorageFlags::new_single_epoch(1, Some(owner_id)).to_element_flags()), + ), + ), + QualifiedGroveDbOp::insert_only_op( + vec![b"refs".to_vec()], + b"ref_key".to_vec(), + Element::new_reference_with_hops( + ReferencePathType::AbsolutePathReference(vec![ + b"tree".to_vec(), + b"key1".to_vec(), + ]), + None, + ), + ), + ]; + + apply_batch(&db, ops, &tx, grove_version); + + let expected_added_bytes = if n < 15 { + n as u32 + 3 + } else if n < 124 { + n as u32 + 4 // the varint requires an extra byte + } else { + n as u32 + 5 // the varint requires an extra byte + }; + expect_storage_flags( + &db, + &tx, + StorageFlags::SingleEpochOwned(1, owner_id), + grove_version, + ); + + verify_references(&db, &tx); + } + } + #[test] + fn test_one_update_smaller_item_add_flags() { + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + + let owner_id = [1; 32]; + + db.insert( + EMPTY_PATH, + b"tree", + Element::empty_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to insert tree"); + + db.insert( + EMPTY_PATH, + b"refs", + Element::empty_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to insert tree"); + + let base_item = b"value1".to_vec(); + let mut original_item = base_item.clone(); + original_item.extend(std::iter::repeat(0).take(150)); + + db.insert( + [b"tree".as_slice()].as_ref(), + b"key1", + Element::new_item_with_flags(original_item, None), + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to insert item"); + + let to = 150usize; + + for n in (0..to).rev() { + let tx = db.start_transaction(); + let mut item = base_item.clone(); + item.extend(std::iter::repeat(0).take(n)); + // We are adding n bytes + let ops = vec![ + QualifiedGroveDbOp::insert_or_replace_op( + vec![b"tree".to_vec()], + b"key1".to_vec(), + Element::new_item_with_flags( + item, // value1 if n was 1 + Some(StorageFlags::new_single_epoch(0, Some(owner_id)).to_element_flags()), + ), + ), + QualifiedGroveDbOp::insert_only_op( + vec![b"refs".to_vec()], + b"ref_key".to_vec(), + Element::new_reference_with_hops( + ReferencePathType::AbsolutePathReference(vec![ + b"tree".to_vec(), + b"key1".to_vec(), + ]), + None, + ), + ), + ]; + + let storage_removed_bytes = apply_batch(&db, ops, &tx, grove_version) + .storage_cost + .removed_bytes; + + if n > 113 { + assert_eq!(storage_removed_bytes, StorageRemovedBytes::NoStorageRemoval); + } else if n > 17 { + let removed_bytes = 114 - n as u32; + assert_eq!( + storage_removed_bytes, + StorageRemovedBytes::BasicStorageRemoval(removed_bytes) + ); + } else { + let removed_bytes = 114 - n as u32 + 1; // because of varint + assert_eq!( + storage_removed_bytes, + StorageRemovedBytes::BasicStorageRemoval(removed_bytes) + ); + }; + + expect_storage_flags( + &db, + &tx, + StorageFlags::SingleEpochOwned(0, owner_id), + grove_version, + ); + + verify_references(&db, &tx); + } + } } diff --git a/grovedb/src/batch/just_in_time_reference_update.rs b/grovedb/src/batch/just_in_time_reference_update.rs new file mode 100644 index 00000000..f4385b89 --- /dev/null +++ b/grovedb/src/batch/just_in_time_reference_update.rs @@ -0,0 +1,178 @@ +use std::borrow::Cow; + +use grovedb_costs::{ + cost_return_on_error_no_add, + storage_cost::{ + removal::{StorageRemovedBytes, StorageRemovedBytes::BasicStorageRemoval}, + StorageCost, + }, + CostResult, CostsExt, OperationCost, +}; +use grovedb_merk::{ + tree::{kv::KV, value_hash, TreeNode}, + CryptoHash, Merk, +}; +use grovedb_storage::StorageContext; +use grovedb_version::version::GroveVersion; + +use crate::{ + batch::{MerkError, TreeCacheMerkByPath}, + Element, ElementFlags, Error, +}; + +impl<'db, S, F> TreeCacheMerkByPath +where + F: FnMut(&[Vec], bool) -> CostResult, Error>, + S: StorageContext<'db>, +{ + pub(crate) fn process_old_element_flags( + key: &[u8], + serialized: &[u8], + new_element: &mut Element, + old_element: Element, + old_serialized_element: &[u8], + is_in_sum_tree: bool, + flags_update: &mut G, + split_removal_bytes: &mut SR, + grove_version: &GroveVersion, + ) -> CostResult + where + G: FnMut(&StorageCost, Option, &mut ElementFlags) -> Result, + SR: FnMut( + &mut ElementFlags, + u32, + u32, + ) -> Result<(StorageRemovedBytes, StorageRemovedBytes), Error>, + { + let mut cost = OperationCost::default(); + if old_element.is_sum_item() { + return if new_element.is_sum_item() { + let maybe_old_flags = old_element.get_flags_owned(); + if maybe_old_flags.is_some() { + let mut updated_new_element_with_old_flags = new_element.clone(); + updated_new_element_with_old_flags.set_flags(maybe_old_flags.clone()); + // There are no storage flags, we can just hash new element + let new_serialized_bytes = cost_return_on_error_no_add!( + &cost, + updated_new_element_with_old_flags.serialize(grove_version) + ); + let val_hash = value_hash(&new_serialized_bytes).unwrap_add_cost(&mut cost); + Ok(val_hash).wrap_with_cost(cost) + } else { + let val_hash = value_hash(&serialized).unwrap_add_cost(&mut cost); + Ok(val_hash).wrap_with_cost(cost) + } + } else { + Err(Error::NotSupported( + "going from a sum item to a not sum item is not supported".to_string(), + )) + .wrap_with_cost(cost) + }; + } else if new_element.is_sum_item() { + return Err(Error::NotSupported( + "going from an item to a sum item is not supported".to_string(), + )) + .wrap_with_cost(cost); + } + let mut maybe_old_flags = old_element.get_flags_owned(); + + let old_storage_cost = KV::node_value_byte_cost_size( + key.len() as u32, + old_serialized_element.len() as u32, + is_in_sum_tree, + ); + + let original_new_element = new_element.clone(); + + let mut serialization_to_use = Cow::Borrowed(serialized); + + let mut new_storage_cost = if maybe_old_flags.is_some() { + // we need to get the new storage_cost as if it had the same storage flags as + // before + let mut updated_new_element_with_old_flags = original_new_element.clone(); + updated_new_element_with_old_flags.set_flags(maybe_old_flags.clone()); + + let serialized_with_old_flags = cost_return_on_error_no_add!( + &cost, + updated_new_element_with_old_flags.serialize(grove_version) + ); + KV::node_value_byte_cost_size( + key.len() as u32, + serialized_with_old_flags.len() as u32, + is_in_sum_tree, + ) + } else { + KV::node_value_byte_cost_size(key.len() as u32, serialized.len() as u32, is_in_sum_tree) + }; + + let mut i = 0; + + loop { + // Calculate storage costs + let mut storage_costs = + TreeNode::storage_cost_for_update(new_storage_cost, old_storage_cost); + + if let Some(old_element_flags) = maybe_old_flags.as_mut() { + if let BasicStorageRemoval(removed_bytes) = storage_costs.removed_bytes { + let (_, value_removed_bytes) = cost_return_on_error_no_add!( + &cost, + split_removal_bytes(old_element_flags, 0, removed_bytes) + ); + storage_costs.removed_bytes = value_removed_bytes; + } + } + + let mut new_element_cloned = original_new_element.clone(); + + let changed = cost_return_on_error_no_add!( + &cost, + (flags_update)( + &storage_costs, + maybe_old_flags.clone(), + new_element_cloned.get_flags_mut().as_mut().unwrap() + ) + .map_err(|e| match e { + Error::JustInTimeElementFlagsClientError(_) => { + MerkError::ClientCorruptionError(e.to_string()).into() + } + _ => MerkError::ClientCorruptionError("non client error".to_string(),).into(), + }) + ); + if !changed { + // There are no storage flags, we can just hash new element + + let val_hash = value_hash(&serialization_to_use).unwrap_add_cost(&mut cost); + return Ok(val_hash).wrap_with_cost(cost); + } else { + // There are no storage flags, we can just hash new element + let new_serialized_bytes = cost_return_on_error_no_add!( + &cost, + new_element_cloned.serialize(grove_version) + ); + + new_storage_cost = KV::node_value_byte_cost_size( + key.len() as u32, + new_serialized_bytes.len() as u32, + is_in_sum_tree, + ); + + if serialization_to_use == new_serialized_bytes { + // it hasn't actually changed, let's do the value hash of it + let val_hash = value_hash(&serialization_to_use).unwrap_add_cost(&mut cost); + return Ok(val_hash).wrap_with_cost(cost); + } + + serialization_to_use = Cow::Owned(new_serialized_bytes); + } + + // Prevent potential infinite loop + if i > 8 { + return Err(Error::CyclicError( + "updated value based on costs too many times in reference", + )) + .wrap_with_cost(cost); + } + i += 1; + } + } +} diff --git a/grovedb/src/batch/mod.rs b/grovedb/src/batch/mod.rs index 7f9c119e..2767ab2b 100644 --- a/grovedb/src/batch/mod.rs +++ b/grovedb/src/batch/mod.rs @@ -13,6 +13,7 @@ mod multi_insert_cost_tests; #[cfg(test)] mod just_in_time_cost_tests; +pub mod just_in_time_reference_update; mod options; #[cfg(test)] mod single_deletion_cost_tests; @@ -51,7 +52,7 @@ use grovedb_merk::{ kv::ValueDefinedCostType::{LayeredValueDefinedCost, SpecializedValueDefinedCost}, value_hash, NULL_HASH, }, - CryptoHash, Error as MerkError, Merk, MerkType, RootHashKeyAndSum, + CryptoHash, Error as MerkError, Merk, MerkType, Op, RootHashKeyAndSum, TreeFeatureType::{BasicMerkNode, SummedMerkNode}, }; use grovedb_path::SubtreePath; @@ -74,7 +75,7 @@ use crate::batch::estimated_costs::EstimatedCostsType; use crate::{ batch::{batch_structure::BatchStructure, mode::BatchRunMode}, element::{MaxReferenceHop, SUM_ITEM_COST_SIZE, SUM_TREE_COST_SIZE, TREE_COST_SIZE}, - operations::get::MAX_REFERENCE_HOPS, + operations::{get::MAX_REFERENCE_HOPS, proof::util::hex_to_ascii}, reference_path::{ path_from_reference_path_type, path_from_reference_qualified_path_type, ReferencePathType, }, @@ -83,7 +84,7 @@ use crate::{ /// Operations #[derive(Debug, PartialEq, Eq, Hash, Clone)] -pub enum Op { +pub enum GroveOp { /// Replace tree root key ReplaceTreeRootKey { /// Hash @@ -93,8 +94,13 @@ pub enum Op { /// Sum sum: Option, }, - /// Insert - Insert { + /// Inserts an element that is known to not yet exist + InsertOnly { + /// Element + element: Element, + }, + /// Inserts or Replaces an element + InsertOrReplace { /// Element element: Element, }, @@ -141,29 +147,30 @@ pub enum Op { DeleteSumTree, } -impl Op { +impl GroveOp { fn to_u8(&self) -> u8 { match self { - Op::DeleteTree => 0, - Op::DeleteSumTree => 1, - Op::Delete => 2, - Op::InsertTreeWithRootHash { .. } => 3, - Op::ReplaceTreeRootKey { .. } => 4, - Op::RefreshReference { .. } => 5, - Op::Replace { .. } => 6, - Op::Patch { .. } => 7, - Op::Insert { .. } => 8, + GroveOp::DeleteTree => 0, + GroveOp::DeleteSumTree => 1, + GroveOp::Delete => 2, + GroveOp::InsertTreeWithRootHash { .. } => 3, + GroveOp::ReplaceTreeRootKey { .. } => 4, + GroveOp::RefreshReference { .. } => 5, + GroveOp::Replace { .. } => 6, + GroveOp::Patch { .. } => 7, + GroveOp::InsertOrReplace { .. } => 8, + GroveOp::InsertOnly { .. } => 9, } } } -impl PartialOrd for Op { +impl PartialOrd for GroveOp { fn partial_cmp(&self, other: &Self) -> Option { Some(self.cmp(other)) } } -impl Ord for Op { +impl Ord for GroveOp { fn cmp(&self, other: &Self) -> Ordering { self.to_u8().cmp(&other.to_u8()) } @@ -336,16 +343,16 @@ impl KeyInfoPath { /// Batch operation #[derive(Clone, PartialEq, Eq)] -pub struct GroveDbOp { +pub struct QualifiedGroveDbOp { /// Path to a subtree - subject to an operation pub path: KeyInfoPath, /// Key of an element in the subtree pub key: KeyInfo, /// Operation to perform on the key - pub op: Op, + pub op: GroveOp, } -impl fmt::Debug for GroveDbOp { +impl fmt::Debug for QualifiedGroveDbOp { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { let mut path_out = Vec::new(); let path_drawer = Drawer::new(&mut path_out); @@ -355,10 +362,11 @@ impl fmt::Debug for GroveDbOp { self.key.visualize(key_drawer).unwrap(); let op_dbg = match &self.op { - Op::Insert { element } => format!("Insert {:?}", element), - Op::Replace { element } => format!("Replace {:?}", element), - Op::Patch { element, .. } => format!("Patch {:?}", element), - Op::RefreshReference { + GroveOp::InsertOrReplace { element } => format!("Insert Or Replace {:?}", element), + GroveOp::InsertOnly { element } => format!("Insert {:?}", element), + GroveOp::Replace { element } => format!("Replace {:?}", element), + GroveOp::Patch { element, .. } => format!("Patch {:?}", element), + GroveOp::RefreshReference { reference_path_type, max_reference_hop, trust_refresh_reference, @@ -369,11 +377,11 @@ impl fmt::Debug for GroveDbOp { reference_path_type, max_reference_hop, trust_refresh_reference ) } - Op::Delete => "Delete".to_string(), - Op::DeleteTree => "Delete Tree".to_string(), - Op::DeleteSumTree => "Delete Sum Tree".to_string(), - Op::ReplaceTreeRootKey { .. } => "Replace Tree Hash and Root Key".to_string(), - Op::InsertTreeWithRootHash { .. } => "Insert Tree Hash and Root Key".to_string(), + GroveOp::Delete => "Delete".to_string(), + GroveOp::DeleteTree => "Delete Tree".to_string(), + GroveOp::DeleteSumTree => "Delete Sum Tree".to_string(), + GroveOp::ReplaceTreeRootKey { .. } => "Replace Tree Hash and Root Key".to_string(), + GroveOp::InsertTreeWithRootHash { .. } => "Insert Tree Hash and Root Key".to_string(), }; f.debug_struct("GroveDbOp") @@ -384,14 +392,24 @@ impl fmt::Debug for GroveDbOp { } } -impl GroveDbOp { +impl QualifiedGroveDbOp { + /// An insert op using a known owned path and known key + pub fn insert_only_op(path: Vec>, key: Vec, element: Element) -> Self { + let path = KeyInfoPath::from_known_owned_path(path); + Self { + path, + key: KnownKey(key), + op: GroveOp::InsertOnly { element }, + } + } + /// An insert op using a known owned path and known key - pub fn insert_op(path: Vec>, key: Vec, element: Element) -> Self { + pub fn insert_or_replace_op(path: Vec>, key: Vec, element: Element) -> Self { let path = KeyInfoPath::from_known_owned_path(path); Self { path, key: KnownKey(key), - op: Op::Insert { element }, + op: GroveOp::InsertOrReplace { element }, } } @@ -400,7 +418,7 @@ impl GroveDbOp { Self { path, key, - op: Op::Insert { element }, + op: GroveOp::InsertOrReplace { element }, } } @@ -410,7 +428,7 @@ impl GroveDbOp { Self { path, key: KnownKey(key), - op: Op::Replace { element }, + op: GroveOp::Replace { element }, } } @@ -419,7 +437,7 @@ impl GroveDbOp { Self { path, key, - op: Op::Replace { element }, + op: GroveOp::Replace { element }, } } @@ -434,7 +452,7 @@ impl GroveDbOp { Self { path, key: KnownKey(key), - op: Op::Patch { + op: GroveOp::Patch { element, change_in_bytes, }, @@ -451,7 +469,7 @@ impl GroveDbOp { Self { path, key, - op: Op::Patch { + op: GroveOp::Patch { element, change_in_bytes, }, @@ -471,7 +489,7 @@ impl GroveDbOp { Self { path, key: KnownKey(key), - op: Op::RefreshReference { + op: GroveOp::RefreshReference { reference_path_type, max_reference_hop, flags, @@ -486,7 +504,7 @@ impl GroveDbOp { Self { path, key: KnownKey(key), - op: Op::Delete, + op: GroveOp::Delete, } } @@ -497,9 +515,9 @@ impl GroveDbOp { path, key: KnownKey(key), op: if is_sum_tree { - Op::DeleteSumTree + GroveOp::DeleteSumTree } else { - Op::DeleteTree + GroveOp::DeleteTree }, } } @@ -509,7 +527,7 @@ impl GroveDbOp { Self { path, key, - op: Op::Delete, + op: GroveOp::Delete, } } @@ -519,15 +537,17 @@ impl GroveDbOp { path, key, op: if is_sum_tree { - Op::DeleteSumTree + GroveOp::DeleteSumTree } else { - Op::DeleteTree + GroveOp::DeleteTree }, } } /// Verify consistency of operations - pub fn verify_consistency_of_operations(ops: &[GroveDbOp]) -> GroveDbOpConsistencyResults { + pub fn verify_consistency_of_operations( + ops: &[QualifiedGroveDbOp], + ) -> GroveDbOpConsistencyResults { let ops_len = ops.len(); // operations should not have any duplicates let mut repeated_ops = vec![]; @@ -564,7 +584,7 @@ impl GroveDbOp { None } }) - .collect::>(); + .collect::>(); if !doubled_ops.is_empty() { doubled_ops.push(op.op.clone()); same_path_key_ops.push((op.path.clone(), op.key.clone(), doubled_ops)); @@ -574,21 +594,23 @@ impl GroveDbOp { let inserts = ops .iter() .filter_map(|current_op| match current_op.op { - Op::Insert { .. } | Op::Replace { .. } => Some(current_op.clone()), + GroveOp::InsertOrReplace { .. } | GroveOp::Replace { .. } => { + Some(current_op.clone()) + } _ => None, }) - .collect::>(); + .collect::>(); let deletes = ops .iter() .filter_map(|current_op| { - if let Op::Delete = current_op.op { + if let GroveOp::Delete = current_op.op { Some(current_op.clone()) } else { None } }) - .collect::>(); + .collect::>(); let mut insert_ops_below_deleted_ops = vec![]; @@ -610,7 +632,7 @@ impl GroveDbOp { None } }) - .collect::>(); + .collect::>(); if !inserts_with_deleted_ops_above.is_empty() { insert_ops_below_deleted_ops .push((deleted_op.clone(), inserts_with_deleted_ops_above)); @@ -628,10 +650,13 @@ impl GroveDbOp { /// Results of a consistency check on an operation batch #[derive(Debug)] pub struct GroveDbOpConsistencyResults { - repeated_ops: Vec<(GroveDbOp, u16)>, // the u16 is count - same_path_key_ops: Vec<(KeyInfoPath, KeyInfo, Vec)>, - insert_ops_below_deleted_ops: Vec<(GroveDbOp, Vec)>, /* the deleted op first, - * then inserts under */ + /// Repeated Ops, the second u16 element represents the count + repeated_ops: Vec<(QualifiedGroveDbOp, u16)>, + /// The same path key ops + same_path_key_ops: Vec<(KeyInfoPath, KeyInfo, Vec)>, + /// This shows issues when we delete a tree but insert under the deleted + /// tree Deleted ops are first, with inserts under them in a tree + insert_ops_below_deleted_ops: Vec<(QualifiedGroveDbOp, Vec)>, } impl GroveDbOpConsistencyResults { @@ -656,7 +681,7 @@ impl fmt::Debug for TreeCacheMerkByPath { } trait TreeCache { - fn insert(&mut self, op: &GroveDbOp, is_sum_tree: bool) -> CostResult<(), Error>; + fn insert(&mut self, op: &QualifiedGroveDbOp, is_sum_tree: bool) -> CostResult<(), Error>; fn get_batch_run_mode(&self) -> BatchRunMode; @@ -664,8 +689,8 @@ trait TreeCache { fn execute_ops_on_path( &mut self, path: &KeyInfoPath, - ops_at_path_by_key: BTreeMap, - ops_by_qualified_paths: &BTreeMap>, Op>, + ops_at_path_by_key: BTreeMap, + ops_by_qualified_paths: &BTreeMap>, GroveOp>, batch_apply_options: &BatchApplyOptions, flags_update: &mut G, split_removal_bytes: &mut SR, @@ -725,22 +750,34 @@ where /// deserializing the referenced element. /// * `Error::InvalidBatchOperation`: If the referenced element points to a /// tree being updated. - fn process_reference<'a>( + fn process_reference<'a, G, SR>( &'a mut self, qualified_path: &[Vec], - ops_by_qualified_paths: &'a BTreeMap>, Op>, + ops_by_qualified_paths: &'a BTreeMap>, GroveOp>, recursions_allowed: u8, intermediate_reference_info: Option<&'a ReferencePathType>, + flags_update: &mut G, + split_removal_bytes: &mut SR, grove_version: &GroveVersion, - ) -> CostResult { + ) -> CostResult + where + G: FnMut(&StorageCost, Option, &mut ElementFlags) -> Result, + SR: FnMut( + &mut ElementFlags, + u32, + u32, + ) -> Result<(StorageRemovedBytes, StorageRemovedBytes), Error>, + { let mut cost = OperationCost::default(); let (key, reference_path) = qualified_path.split_last().unwrap(); // already checked - let reference_merk_wrapped = self - .merks - .remove(reference_path) - .map(|x| Ok(x).wrap_with_cost(Default::default())) - .unwrap_or_else(|| (self.get_merk_fn)(reference_path, false)); - let merk = cost_return_on_error!(&mut cost, reference_merk_wrapped); + + let merk = match self.merks.entry(reference_path.to_vec()) { + HashMapEntry::Occupied(o) => o.into_mut(), + HashMapEntry::Vacant(v) => v.insert(cost_return_on_error!( + &mut cost, + (self.get_merk_fn)(reference_path, false) + )), + }; // Here the element being referenced doesn't change in the same batch // and the max hop count is 1, meaning it should point directly to the base @@ -786,6 +823,8 @@ where path.as_slice(), ops_by_qualified_paths, recursions_allowed - 1, + flags_update, + split_removal_bytes, grove_version, ) } else { @@ -793,33 +832,82 @@ where // but the hop count is greater than 1, we can't just take the value hash from // the referenced element as an element further down in the chain might still // change in the batch. - let referenced_element = cost_return_on_error!( + self.process_reference_with_hop_count_greater_than_one( + key, + reference_path, + qualified_path, + ops_by_qualified_paths, + recursions_allowed, + flags_update, + split_removal_bytes, + grove_version, + ) + } + } + + /// Retrieves and deserializes the referenced element from the Merk tree. + /// + /// This function is responsible for fetching the referenced element using + /// the provided key and reference path, deserializing it into an + /// `Element`. It handles potential errors that can occur during these + /// operations, such as missing references or corrupted data. + /// + /// # Arguments + /// + /// * `key` - The key associated with the referenced element within the Merk + /// tree. + /// * `reference_path` - The path to the referenced element, used to locate + /// it in the Merk tree. + /// * `grove_version` - The current version of the GroveDB being used for + /// serialization and deserialization operations. + /// + /// # Returns + /// + /// * `Ok((Element, Vec, bool))` - Returns the deserialized `Element` + /// and the serialized counterpart if the retrieval and deserialization + /// are successful, wrapped in the associated cost. Also returns if the + /// merk of the element is a sum tree as a bool. + /// * `Err(Error)` - Returns an error if any issue occurs during the + /// retrieval or deserialization of the referenced element. + /// + /// # Errors + /// + /// This function may return the following errors: + /// + /// * `Error::MissingReference` - If the referenced element is missing from + /// the Merk tree. + /// * `Error::CorruptedData` - If the referenced element cannot be + /// deserialized due to corrupted data. + fn get_and_deserialize_referenced_element<'a>( + &'a mut self, + key: &[u8], + reference_path: &[Vec], + grove_version: &GroveVersion, + ) -> CostResult, bool)>, Error> { + let mut cost = OperationCost::default(); + + let merk = match self.merks.entry(reference_path.to_vec()) { + HashMapEntry::Occupied(o) => o.into_mut(), + HashMapEntry::Vacant(v) => v.insert(cost_return_on_error!( &mut cost, - merk.get( - key.as_ref(), - true, - Some(Element::value_defined_cost_for_serialized_value), - grove_version - ) - .map_err(|e| Error::CorruptedData(e.to_string())) - ); + (self.get_merk_fn)(reference_path, false) + )), + }; - let referenced_element = cost_return_on_error_no_add!( - &cost, - referenced_element.ok_or({ - let reference_string = reference_path - .iter() - .map(hex::encode) - .collect::>() - .join("/"); - Error::MissingReference(format!( - "reference to path:`{}` key:`{}` in batch is missing", - reference_string, - hex::encode(key) - )) - }) - ); + let referenced_element = cost_return_on_error!( + &mut cost, + merk.get( + key.as_ref(), + true, + Some(Element::value_defined_cost_for_serialized_value), + grove_version + ) + .map_err(|e| Error::CorruptedData(e.to_string())) + ); + + let is_sum_tree = merk.is_sum_tree; + if let Some(referenced_element) = referenced_element { let element = cost_return_on_error_no_add!( &cost, Element::deserialize(referenced_element.as_slice(), grove_version).map_err(|_| { @@ -827,30 +915,122 @@ where }) ); - match element { - Element::Item(..) | Element::SumItem(..) => { - let serialized = - cost_return_on_error_no_add!(&cost, element.serialize(grove_version)); - let val_hash = value_hash(&serialized).unwrap_add_cost(&mut cost); - Ok(val_hash).wrap_with_cost(cost) - } - Element::Reference(path, ..) => { - let path = cost_return_on_error_no_add!( - &cost, - path_from_reference_qualified_path_type(path, qualified_path) - ); - self.follow_reference_get_value_hash( - path.as_slice(), - ops_by_qualified_paths, - recursions_allowed - 1, - grove_version, - ) - } - Element::Tree(..) | Element::SumTree(..) => Err(Error::InvalidBatchOperation( - "references can not point to trees being updated", - )) - .wrap_with_cost(cost), + Ok(Some((element, referenced_element, is_sum_tree))).wrap_with_cost(cost) + } else { + Ok(None).wrap_with_cost(cost) + } + } + + /// Processes a reference with a hop count greater than one, handling the + /// retrieval and further processing of the referenced element. + /// + /// This function is used when the hop count is greater than 1, meaning that + /// the reference points to another element that may also be a reference. + /// It handles retrieving the referenced element, deserializing it, and + /// determining the appropriate action based on the type of the element. + /// + /// # Arguments + /// + /// * `key` - The key corresponding to the referenced element in the current + /// Merk tree. + /// * `reference_path` - The path to the referenced element within the + /// current batch of operations. + /// * `qualified_path` - The fully qualified path to the reference, already + /// validated as a valid path. + /// * `ops_by_qualified_paths` - A map of qualified paths to their + /// corresponding batch operations. Used to track and manage updates + /// within the batch. + /// * `recursions_allowed` - The maximum allowed hop count to reach the + /// final target element. Each recursive call reduces this by one. + /// * `flags_update` - A mutable closure that handles updating element flags + /// during the processing of the reference. + /// * `split_removal_bytes` - A mutable closure that handles splitting and + /// managing the removal of bytes during the processing of the reference. + /// * `grove_version` - The current version of the GroveDB being used for + /// serialization and deserialization operations. + /// + /// # Returns + /// + /// * `Ok(CryptoHash)` - Returns the crypto hash of the referenced element + /// if successful, wrapped in the associated cost. + /// * `Err(Error)` - Returns an error if there is an issue with the + /// operation, such as a missing reference, corrupted data, or an invalid + /// batch operation. + /// + /// # Errors + /// + /// This function will return `Err(Error)` if any issues are encountered + /// during the processing of the reference. Possible errors include: + /// + /// * `Error::MissingReference` - If a direct or indirect reference to the + /// target element is missing in the batch. + /// * `Error::CorruptedData` - If there is an issue while retrieving or + /// deserializing the referenced element. + /// * `Error::InvalidBatchOperation` - If the referenced element points to a + /// tree being updated, which is not allowed. + fn process_reference_with_hop_count_greater_than_one<'a, G, SR>( + &'a mut self, + key: &[u8], + reference_path: &[Vec], + qualified_path: &[Vec], + ops_by_qualified_paths: &'a BTreeMap>, GroveOp>, + recursions_allowed: u8, + flags_update: &mut G, + split_removal_bytes: &mut SR, + grove_version: &GroveVersion, + ) -> CostResult + where + G: FnMut(&StorageCost, Option, &mut ElementFlags) -> Result, + SR: FnMut( + &mut ElementFlags, + u32, + u32, + ) -> Result<(StorageRemovedBytes, StorageRemovedBytes), Error>, + { + let mut cost = OperationCost::default(); + + let Some((element, ..)) = cost_return_on_error!( + &mut cost, + self.get_and_deserialize_referenced_element(key, reference_path, grove_version) + ) else { + let reference_string = reference_path + .iter() + .map(hex::encode) + .collect::>() + .join("/"); + return Err(Error::MissingReference(format!( + "reference to path:`{}` key:`{}` in batch is missing", + reference_string, + hex::encode(key) + ))) + .wrap_with_cost(cost); + }; + + match element { + Element::Item(..) | Element::SumItem(..) => { + let serialized = + cost_return_on_error_no_add!(&cost, element.serialize(grove_version)); + let val_hash = value_hash(&serialized).unwrap_add_cost(&mut cost); + Ok(val_hash).wrap_with_cost(cost) } + Element::Reference(path, ..) => { + let path = cost_return_on_error_no_add!( + &cost, + path_from_reference_qualified_path_type(path, qualified_path) + ); + self.follow_reference_get_value_hash( + path.as_slice(), + ops_by_qualified_paths, + recursions_allowed - 1, + flags_update, + split_removal_bytes, + grove_version, + ) + } + Element::Tree(..) | Element::SumTree(..) => Err(Error::InvalidBatchOperation( + "references can not point to trees being updated", + )) + .wrap_with_cost(cost), } } @@ -864,35 +1044,89 @@ where /// insert ref_3 and another operation to change something in the /// reference chain in the same batch. /// All these has to be taken into account. - fn follow_reference_get_value_hash<'a>( + fn follow_reference_get_value_hash<'a, G, SR>( &'a mut self, qualified_path: &[Vec], - ops_by_qualified_paths: &'a BTreeMap>, Op>, + ops_by_qualified_paths: &'a BTreeMap>, GroveOp>, recursions_allowed: u8, + flags_update: &mut G, + split_removal_bytes: &mut SR, grove_version: &GroveVersion, - ) -> CostResult { + ) -> CostResult + where + G: FnMut(&StorageCost, Option, &mut ElementFlags) -> Result, + SR: FnMut( + &mut ElementFlags, + u32, + u32, + ) -> Result<(StorageRemovedBytes, StorageRemovedBytes), Error>, + { let mut cost = OperationCost::default(); if recursions_allowed == 0 { return Err(Error::ReferenceLimit).wrap_with_cost(cost); } // If the element being referenced changes in the same batch // we need to set the value_hash based on the new change and not the old state. + + // However the operation might either be merged or unmerged, if it is unmerged + // we need to merge it with the state first if let Some(op) = ops_by_qualified_paths.get(qualified_path) { // the path is being modified, inserted or deleted in the batch of operations match op { - Op::ReplaceTreeRootKey { .. } | Op::InsertTreeWithRootHash { .. } => Err( + GroveOp::ReplaceTreeRootKey { .. } | GroveOp::InsertTreeWithRootHash { .. } => Err( Error::InvalidBatchOperation("references can not point to trees being updated"), ) .wrap_with_cost(cost), - Op::Insert { element } | Op::Replace { element } | Op::Patch { element, .. } => { + GroveOp::InsertOrReplace { element } + | GroveOp::Replace { element } + | GroveOp::Patch { element, .. } => { match element { Element::Item(..) | Element::SumItem(..) => { let serialized = cost_return_on_error_no_add!( &cost, element.serialize(grove_version) ); - let val_hash = value_hash(&serialized).unwrap_add_cost(&mut cost); - Ok(val_hash).wrap_with_cost(cost) + if element.get_flags().is_none() { + // There are no storage flags, we can just hash new element + let val_hash = value_hash(&serialized).unwrap_add_cost(&mut cost); + Ok(val_hash).wrap_with_cost(cost) + } else { + let mut new_element = element.clone(); + + // it can be unmerged, let's get the value on disk + let (key, reference_path) = qualified_path.split_last().unwrap(); + let serialized_element_result = cost_return_on_error!( + &mut cost, + self.get_and_deserialize_referenced_element( + key, + reference_path, + grove_version + ) + ); + if let Some((old_element, old_serialized_element, is_in_sum_tree)) = + serialized_element_result + { + let value_hash = cost_return_on_error!( + &mut cost, + Self::process_old_element_flags( + key, + &serialized, + &mut new_element, + old_element, + &old_serialized_element, + is_in_sum_tree, + flags_update, + split_removal_bytes, + grove_version, + ) + ); + Ok(value_hash).wrap_with_cost(cost) + } else { + let value_hash = + value_hash(&serialized).unwrap_add_cost(&mut cost); + Ok(value_hash).wrap_with_cost(cost) + } + } } Element::Reference(path, ..) => { let path = cost_return_on_error_no_add!( @@ -906,6 +1140,8 @@ where path.as_slice(), ops_by_qualified_paths, recursions_allowed - 1, + flags_update, + split_removal_bytes, grove_version, ) } @@ -917,7 +1153,33 @@ where } } } - Op::RefreshReference { + GroveOp::InsertOnly { element } => match element { + Element::Item(..) | Element::SumItem(..) => { + let serialized = + cost_return_on_error_no_add!(&cost, element.serialize(grove_version)); + let val_hash = value_hash(&serialized).unwrap_add_cost(&mut cost); + Ok(val_hash).wrap_with_cost(cost) + } + Element::Reference(path, ..) => { + let path = cost_return_on_error_no_add!( + &cost, + path_from_reference_qualified_path_type(path.clone(), qualified_path) + ); + self.follow_reference_get_value_hash( + path.as_slice(), + ops_by_qualified_paths, + recursions_allowed - 1, + flags_update, + split_removal_bytes, + grove_version, + ) + } + Element::Tree(..) | Element::SumTree(..) => Err(Error::InvalidBatchOperation( + "references can not point to trees being updated", + )) + .wrap_with_cost(cost), + }, + GroveOp::RefreshReference { reference_path_type, trust_refresh_reference, .. @@ -933,10 +1195,12 @@ where ops_by_qualified_paths, recursions_allowed, reference_info, + flags_update, + split_removal_bytes, grove_version, ) } - Op::Delete | Op::DeleteTree | Op::DeleteSumTree => { + GroveOp::Delete | GroveOp::DeleteTree | GroveOp::DeleteSumTree => { Err(Error::InvalidBatchOperation( "references can not point to something currently being deleted", )) @@ -949,6 +1213,8 @@ where ops_by_qualified_paths, recursions_allowed, None, + flags_update, + split_removal_bytes, grove_version, ) } @@ -966,7 +1232,7 @@ where F: FnMut(&[Vec], bool) -> CostResult, Error>, S: StorageContext<'db>, { - fn insert(&mut self, op: &GroveDbOp, is_sum_tree: bool) -> CostResult<(), Error> { + fn insert(&mut self, op: &QualifiedGroveDbOp, is_sum_tree: bool) -> CostResult<(), Error> { let mut cost = OperationCost::default(); let mut inserted_path = op.path.to_path(); @@ -988,12 +1254,15 @@ where ) -> CostResult<(), Error> { let mut cost = OperationCost::default(); let base_path = vec![]; - let merk_wrapped = self - .merks - .remove(&base_path) - .map(|x| Ok(x).wrap_with_cost(Default::default())) - .unwrap_or_else(|| (self.get_merk_fn)(&[], false)); - let mut merk = cost_return_on_error!(&mut cost, merk_wrapped); + + let merk = match self.merks.entry(base_path.clone()) { + HashMapEntry::Occupied(o) => o.into_mut(), + HashMapEntry::Vacant(v) => v.insert(cost_return_on_error!( + &mut cost, + (self.get_merk_fn)(&base_path, false) + )), + }; + merk.set_base_root_key(root_key) .add_cost(cost) .map_err(|_| Error::InternalError("unable to set base root key".to_string())) @@ -1002,8 +1271,8 @@ where fn execute_ops_on_path( &mut self, path: &KeyInfoPath, - ops_at_path_by_key: BTreeMap, - ops_by_qualified_paths: &BTreeMap>, Op>, + ops_at_path_by_key: BTreeMap, + ops_by_qualified_paths: &BTreeMap>, GroveOp>, batch_apply_options: &BatchApplyOptions, flags_update: &mut G, split_removal_bytes: &mut SR, @@ -1014,121 +1283,130 @@ where let p = path.to_path(); let path = &p; - let merk_wrapped = self - .merks - .remove(path) - .map(|x| Ok(x).wrap_with_cost(Default::default())) - .unwrap_or_else(|| (self.get_merk_fn)(path, false)); - let mut merk = cost_return_on_error!(&mut cost, merk_wrapped); - let is_sum_tree = merk.is_sum_tree; + // This also populates Merk trees cache + let is_sum_tree = { + let merk = match self.merks.entry(path.to_vec()) { + HashMapEntry::Occupied(o) => o.into_mut(), + HashMapEntry::Vacant(v) => v.insert(cost_return_on_error!( + &mut cost, + (self.get_merk_fn)(path, false) + )), + }; + merk.is_sum_tree + }; - let mut batch_operations: Vec<(Vec, _)> = vec![]; + let mut batch_operations: Vec<(Vec, Op)> = vec![]; for (key_info, op) in ops_at_path_by_key.into_iter() { match op { - Op::Insert { element } | Op::Replace { element } | Op::Patch { element, .. } => { - match &element { - Element::Reference(path_reference, element_max_reference_hop, _) => { - let merk_feature_type = cost_return_on_error!( - &mut cost, - element - .get_feature_type(is_sum_tree) - .wrap_with_cost(OperationCost::default()) - ); - let path_reference = cost_return_on_error!( - &mut cost, - path_from_reference_path_type( - path_reference.clone(), - path, - Some(key_info.as_slice()) - ) + GroveOp::InsertOnly { element } + | GroveOp::InsertOrReplace { element } + | GroveOp::Replace { element } + | GroveOp::Patch { element, .. } => match &element { + Element::Reference(path_reference, element_max_reference_hop, _) => { + let merk_feature_type = cost_return_on_error!( + &mut cost, + element + .get_feature_type(is_sum_tree) .wrap_with_cost(OperationCost::default()) - ); - if path_reference.is_empty() { - return Err(Error::InvalidBatchOperation( - "attempting to insert an empty reference", - )) - .wrap_with_cost(cost); - } + ); + let path_reference = cost_return_on_error!( + &mut cost, + path_from_reference_path_type( + path_reference.clone(), + path, + Some(key_info.as_slice()) + ) + .wrap_with_cost(OperationCost::default()) + ); + if path_reference.is_empty() { + return Err(Error::InvalidBatchOperation( + "attempting to insert an empty reference", + )) + .wrap_with_cost(cost); + } - let referenced_element_value_hash = cost_return_on_error!( - &mut cost, - self.follow_reference_get_value_hash( - path_reference.as_slice(), - ops_by_qualified_paths, - element_max_reference_hop.unwrap_or(MAX_REFERENCE_HOPS as u8), - grove_version, - ) - ); + let referenced_element_value_hash = cost_return_on_error!( + &mut cost, + self.follow_reference_get_value_hash( + path_reference.as_slice(), + ops_by_qualified_paths, + element_max_reference_hop.unwrap_or(MAX_REFERENCE_HOPS as u8), + flags_update, + split_removal_bytes, + grove_version, + ) + ); - cost_return_on_error!( + cost_return_on_error!( + &mut cost, + element.insert_reference_into_batch_operations( + key_info.get_key_clone(), + referenced_element_value_hash, + &mut batch_operations, + merk_feature_type, + grove_version, + ) + ); + } + Element::Tree(..) | Element::SumTree(..) => { + let merk_feature_type = cost_return_on_error!( + &mut cost, + element + .get_feature_type(is_sum_tree) + .wrap_with_cost(OperationCost::default()) + ); + cost_return_on_error!( + &mut cost, + element.insert_subtree_into_batch_operations( + key_info.get_key_clone(), + NULL_HASH, + false, + &mut batch_operations, + merk_feature_type, + grove_version, + ) + ); + } + Element::Item(..) | Element::SumItem(..) => { + let merk_feature_type = cost_return_on_error!( + &mut cost, + element + .get_feature_type(is_sum_tree) + .wrap_with_cost(OperationCost::default()) + ); + if batch_apply_options.validate_insertion_does_not_override { + let merk = self.merks.get_mut(path).expect("the Merk is cached"); + + let inserted = cost_return_on_error!( &mut cost, - element.insert_reference_into_batch_operations( - key_info.get_key_clone(), - referenced_element_value_hash, + element.insert_if_not_exists_into_batch_operations( + merk, + key_info.get_key(), &mut batch_operations, merk_feature_type, grove_version, ) ); - } - Element::Tree(..) | Element::SumTree(..) => { - let merk_feature_type = cost_return_on_error!( - &mut cost, - element - .get_feature_type(is_sum_tree) - .wrap_with_cost(OperationCost::default()) - ); + if !inserted { + return Err(Error::InvalidBatchOperation( + "attempting to overwrite a tree", + )) + .wrap_with_cost(cost); + } + } else { cost_return_on_error!( &mut cost, - element.insert_subtree_into_batch_operations( - key_info.get_key_clone(), - NULL_HASH, - false, + element.insert_into_batch_operations( + key_info.get_key(), &mut batch_operations, merk_feature_type, grove_version, ) ); } - Element::Item(..) | Element::SumItem(..) => { - let merk_feature_type = cost_return_on_error!( - &mut cost, - element - .get_feature_type(is_sum_tree) - .wrap_with_cost(OperationCost::default()) - ); - if batch_apply_options.validate_insertion_does_not_override { - let inserted = cost_return_on_error!( - &mut cost, - element.insert_if_not_exists_into_batch_operations( - &mut merk, - key_info.get_key(), - &mut batch_operations, - merk_feature_type, - grove_version, - ) - ); - if !inserted { - return Err(Error::InvalidBatchOperation( - "attempting to overwrite a tree", - )) - .wrap_with_cost(cost); - } - } else { - cost_return_on_error!( - &mut cost, - element.insert_into_batch_operations( - key_info.get_key(), - &mut batch_operations, - merk_feature_type, - grove_version, - ) - ); - } - } } - } - Op::RefreshReference { + }, + GroveOp::RefreshReference { reference_path_type, max_reference_hop, flags, @@ -1140,6 +1418,7 @@ where let element = if trust_refresh_reference { Element::Reference(reference_path_type, max_reference_hop, flags) } else { + let merk = self.merks.get(path).expect("the Merk is cached"); let value = cost_return_on_error!( &mut cost, merk.get( @@ -1199,6 +1478,8 @@ where path_reference.as_slice(), ops_by_qualified_paths, max_reference_hop.unwrap_or(MAX_REFERENCE_HOPS as u8), + flags_update, + split_removal_bytes, grove_version ) ); @@ -1214,7 +1495,7 @@ where ) ); } - Op::Delete => { + GroveOp::Delete => { cost_return_on_error!( &mut cost, Element::delete_into_batch_operations( @@ -1227,7 +1508,7 @@ where ) ); } - Op::DeleteTree => { + GroveOp::DeleteTree => { cost_return_on_error!( &mut cost, Element::delete_into_batch_operations( @@ -1239,7 +1520,7 @@ where ) ); } - Op::DeleteSumTree => { + GroveOp::DeleteSumTree => { cost_return_on_error!( &mut cost, Element::delete_into_batch_operations( @@ -1251,11 +1532,12 @@ where ) ); } - Op::ReplaceTreeRootKey { + GroveOp::ReplaceTreeRootKey { hash, root_key, sum, } => { + let merk = self.merks.get(path).expect("the Merk is cached"); cost_return_on_error!( &mut cost, GroveDb::update_tree_item_preserve_flag_into_batch_operations( @@ -1269,7 +1551,7 @@ where ) ); } - Op::InsertTreeWithRootHash { + GroveOp::InsertTreeWithRootHash { hash, root_key, flags, @@ -1298,9 +1580,12 @@ where } } } + + let merk = self.merks.get_mut(path).expect("the Merk is cached"); + cost_return_on_error!( &mut cost, - merk.apply_unchecked::<_, Vec, _, _, _, _>( + merk.apply_unchecked::<_, Vec, _, _, _, _, _>( &batch_operations, &[], Some(batch_apply_options.as_merk_options()), @@ -1309,6 +1594,23 @@ where .map_err(|e| MerkError::ClientCorruptionError(e.to_string())) }, Some(&Element::value_defined_cost_for_serialized_value), + &|old_value, new_value| { + let old_element = Element::deserialize(old_value.as_slice(), grove_version) + .map_err(|e| MerkError::ClientCorruptionError(e.to_string()))?; + let maybe_old_flags = old_element.get_flags_owned(); + if maybe_old_flags.is_some() { + let mut new_element = + Element::deserialize(new_value.as_slice(), grove_version) + .map_err(|e| MerkError::ClientCorruptionError(e.to_string()))?; + new_element.set_flags(maybe_old_flags); + new_element + .serialize(grove_version) + .map(Some) + .map_err(|e| MerkError::ClientCorruptionError(e.to_string())) + } else { + Ok(None) + } + }, &mut |storage_costs, old_value, new_value| { // todo: change the flags without full deserialization let old_element = Element::deserialize(old_value.as_slice(), grove_version) @@ -1391,8 +1693,7 @@ where .root_hash_key_and_sum() .add_cost(cost) .map_err(Error::MerkError); - // We need to reinsert the merk - self.merks.insert(path.clone(), merk); + r } @@ -1501,16 +1802,19 @@ impl GroveDb { { match ops_on_path.entry(key.clone()) { Entry::Vacant(vacant_entry) => { - vacant_entry.insert(Op::ReplaceTreeRootKey { - hash: root_hash, - root_key: calculated_root_key, - sum: sum_value, - }); + vacant_entry.insert( + GroveOp::ReplaceTreeRootKey { + hash: root_hash, + root_key: calculated_root_key, + sum: sum_value, + } + .into(), + ); } Entry::Occupied(occupied_entry) => { let mutable_occupied_entry = occupied_entry.into_mut(); match mutable_occupied_entry { - Op::ReplaceTreeRootKey { + GroveOp::ReplaceTreeRootKey { hash, root_key, sum, @@ -1519,33 +1823,36 @@ impl GroveDb { *root_key = calculated_root_key; *sum = sum_value; } - Op::InsertTreeWithRootHash { .. } => { + GroveOp::InsertTreeWithRootHash { .. } => { return Err(Error::CorruptedCodeExecution( "we can not do this operation twice", )) .wrap_with_cost(cost); } - Op::Insert { element } - | Op::Replace { element } - | Op::Patch { element, .. } => { + GroveOp::InsertOrReplace { element } + | GroveOp::InsertOnly { element } + | GroveOp::Replace { element } + | GroveOp::Patch { element, .. } => { if let Element::Tree(_, flags) = element { *mutable_occupied_entry = - Op::InsertTreeWithRootHash { + GroveOp::InsertTreeWithRootHash { hash: root_hash, root_key: calculated_root_key, flags: flags.clone(), sum: None, - }; + } + .into(); } else if let Element::SumTree(.., flags) = element { *mutable_occupied_entry = - Op::InsertTreeWithRootHash { + GroveOp::InsertTreeWithRootHash { hash: root_hash, root_key: calculated_root_key, flags: flags.clone(), sum: sum_value, - }; + } + .into(); } else { return Err(Error::InvalidBatchOperation( "insertion of element under a non tree", @@ -1553,14 +1860,16 @@ impl GroveDb { .wrap_with_cost(cost); } } - Op::RefreshReference { .. } => { + GroveOp::RefreshReference { .. } => { return Err(Error::InvalidBatchOperation( "insertion of element under a refreshed \ reference", )) .wrap_with_cost(cost); } - Op::Delete | Op::DeleteTree | Op::DeleteSumTree => { + GroveOp::Delete + | GroveOp::DeleteTree + | GroveOp::DeleteSumTree => { if calculated_root_key.is_some() { return Err(Error::InvalidBatchOperation( "modification of tree when it will be \ @@ -1573,10 +1882,11 @@ impl GroveDb { } } } else { - let mut ops_on_path: BTreeMap = BTreeMap::new(); + let mut ops_on_path: BTreeMap = + BTreeMap::new(); ops_on_path.insert( key.clone(), - Op::ReplaceTreeRootKey { + GroveOp::ReplaceTreeRootKey { hash: root_hash, root_key: calculated_root_key, sum: sum_value, @@ -1585,17 +1895,20 @@ impl GroveDb { ops_at_level_above.insert(parent_path, ops_on_path); } } else { - let mut ops_on_path: BTreeMap = BTreeMap::new(); + let mut ops_on_path: BTreeMap = BTreeMap::new(); ops_on_path.insert( key.clone(), - Op::ReplaceTreeRootKey { + GroveOp::ReplaceTreeRootKey { hash: root_hash, root_key: calculated_root_key, sum: sum_value, - }, + } + .into(), ); - let mut ops_on_level: BTreeMap> = - BTreeMap::new(); + let mut ops_on_level: BTreeMap< + KeyInfoPath, + BTreeMap, + > = BTreeMap::new(); ops_on_level.insert(KeyInfoPath(parent_path.to_vec()), ops_on_path); ops_by_level_paths.insert(current_level - 1, ops_on_level); } @@ -1617,7 +1930,7 @@ impl GroveDb { /// Then return the list of leftover operations fn apply_body<'db, S: StorageContext<'db>>( &self, - ops: Vec, + ops: Vec, batch_apply_options: Option, update_element_flags_function: impl FnMut( &StorageCost, @@ -1662,7 +1975,7 @@ impl GroveDb { fn continue_partial_apply_body<'db, S: StorageContext<'db>>( &self, previous_leftover_operations: Option, - additional_ops: Vec, + additional_ops: Vec, batch_apply_options: Option, update_element_flags_function: impl FnMut( &StorageCost, @@ -1708,7 +2021,7 @@ impl GroveDb { /// Applies operations on GroveDB without batching pub fn apply_operations_without_batching( &self, - ops: Vec, + ops: Vec, options: Option, transaction: TransactionArg, grove_version: &GroveVersion, @@ -1723,7 +2036,7 @@ impl GroveDb { let mut cost = OperationCost::default(); for op in ops.into_iter() { match op.op { - Op::Insert { element } | Op::Replace { element } => { + GroveOp::InsertOrReplace { element } | GroveOp::Replace { element } => { // TODO: paths in batches is something to think about let path_slices: Vec<&[u8]> = op.path.iterator().map(|p| p.as_slice()).collect(); @@ -1739,7 +2052,7 @@ impl GroveDb { ) ); } - Op::Delete => { + GroveOp::Delete => { let path_slices: Vec<&[u8]> = op.path.iterator().map(|p| p.as_slice()).collect(); cost_return_on_error!( @@ -1762,7 +2075,7 @@ impl GroveDb { /// Applies batch on GroveDB pub fn apply_batch( &self, - ops: Vec, + ops: Vec, batch_apply_options: Option, transaction: TransactionArg, grove_version: &GroveVersion, @@ -1789,12 +2102,12 @@ impl GroveDb { /// Applies batch on GroveDB pub fn apply_partial_batch( &self, - ops: Vec, + ops: Vec, batch_apply_options: Option, cost_based_add_on_operations: impl FnMut( &OperationCost, &Option, - ) -> Result, Error>, + ) -> Result, Error>, transaction: TransactionArg, grove_version: &GroveVersion, ) -> CostResult<(), Error> { @@ -1858,8 +2171,14 @@ impl GroveDb { Element::get_from_storage(&parent_storage, parent_key, grove_version).map_err( |_| { Error::InvalidPath(format!( - "could not get key for parent of subtree for batch at path {}", - parent_path.to_vec().into_iter().map(hex::encode).join("/") + "could not get key for parent of subtree for batch at path [{}] \ + for key {}", + parent_path + .to_vec() + .into_iter() + .map(|v| hex_to_ascii(&v)) + .join("/"), + hex_to_ascii(parent_key) )) } ) @@ -1969,7 +2288,7 @@ impl GroveDb { /// Applies batch of operations on GroveDB pub fn apply_batch_with_element_flags_update( &self, - ops: Vec, + ops: Vec, batch_apply_options: Option, update_element_flags_function: impl FnMut( &StorageCost, @@ -2009,7 +2328,7 @@ impl GroveDb { .unwrap_or(true); if check_batch_operation_consistency { - let consistency_result = GroveDbOp::verify_consistency_of_operations(&ops); + let consistency_result = QualifiedGroveDbOp::verify_consistency_of_operations(&ops); if !consistency_result.is_empty() { return Err(Error::InvalidBatchOperation( "batch operations fail consistency checks", @@ -2060,6 +2379,21 @@ impl GroveDb { .commit_multi_context_batch(storage_batch, Some(tx)) .map_err(|e| e.into()) ); + + // Keep this commented for easy debugging in the future. + // let issues = self + // .visualize_verify_grovedb(Some(tx), true, + // &Default::default()) .unwrap(); + // if issues.len() > 0 { + // println!( + // "tx_issues: {}", + // issues + // .iter() + // .map(|(hash, (a, b, c))| format!("{}: {} {} {}", + // hash, a, b, c)) .collect::>() + // .join(" | ") + // ); + // } } else { cost_return_on_error!( &mut cost, @@ -2087,6 +2421,21 @@ impl GroveDb { .commit_multi_context_batch(storage_batch, None) .map_err(|e| e.into()) ); + + // Keep this commented for easy debugging in the future. + // let issues = self + // .visualize_verify_grovedb(None, true, &Default::default()) + // .unwrap(); + // if issues.len() > 0 { + // println!( + // "non_tx_issues: {}", + // issues + // .iter() + // .map(|(hash, (a, b, c))| format!("{}: {} {} {}", + // hash, a, b, c)) .collect::>() + // .join(" | ") + // ); + // } } Ok(()).wrap_with_cost(cost) } @@ -2097,7 +2446,7 @@ impl GroveDb { /// If it is not set we default to pausing at the root tree pub fn apply_partial_batch_with_element_flags_update( &self, - ops: Vec, + ops: Vec, batch_apply_options: Option, mut update_element_flags_function: impl FnMut( &StorageCost, @@ -2115,7 +2464,7 @@ impl GroveDb { mut add_on_operations: impl FnMut( &OperationCost, &Option, - ) -> Result, Error>, + ) -> Result, Error>, transaction: TransactionArg, grove_version: &GroveVersion, ) -> CostResult<(), Error> { @@ -2145,7 +2494,7 @@ impl GroveDb { !batch_apply_options.disable_operation_consistency_check; if check_batch_operation_consistency { - let consistency_result = GroveDbOp::verify_consistency_of_operations(&ops); + let consistency_result = QualifiedGroveDbOp::verify_consistency_of_operations(&ops); if !consistency_result.is_empty() { return Err(Error::InvalidBatchOperation( "batch operations fail consistency checks", @@ -2346,7 +2695,7 @@ impl GroveDb { /// ops pub fn estimated_case_operations_for_batch( estimated_costs_type: EstimatedCostsType, - ops: Vec, + ops: Vec, batch_apply_options: Option, update_element_flags_function: impl FnMut( &StorageCost, @@ -2447,28 +2796,32 @@ mod tests { let element = Element::new_item(b"ayy".to_vec()); let element2 = Element::new_item(b"ayy2".to_vec()); let ops = vec![ - GroveDbOp::insert_op(vec![], b"key1".to_vec(), Element::empty_tree()), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( + vec![], + b"key1".to_vec(), + Element::empty_tree(), + ), + QualifiedGroveDbOp::insert_or_replace_op( vec![b"key1".to_vec(), b"key2".to_vec(), b"key3".to_vec()], b"key4".to_vec(), element.clone(), ), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![b"key1".to_vec(), b"key2".to_vec()], b"key3".to_vec(), Element::empty_tree(), ), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![b"key1".to_vec()], b"key2".to_vec(), Element::empty_tree(), ), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec()], b"key1".to_vec(), Element::empty_tree(), ), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec(), b"key1".to_vec()], b"key2".to_vec(), element2.clone(), @@ -2528,8 +2881,16 @@ mod tests { // No two operations should be the same let ops = vec![ - GroveDbOp::insert_op(vec![b"a".to_vec()], b"b".to_vec(), Element::empty_tree()), - GroveDbOp::insert_op(vec![b"a".to_vec()], b"b".to_vec(), Element::empty_tree()), + QualifiedGroveDbOp::insert_or_replace_op( + vec![b"a".to_vec()], + b"b".to_vec(), + Element::empty_tree(), + ), + QualifiedGroveDbOp::insert_or_replace_op( + vec![b"a".to_vec()], + b"b".to_vec(), + Element::empty_tree(), + ), ]; assert!(matches!( db.apply_batch(ops, None, None, grove_version).unwrap(), @@ -2540,12 +2901,16 @@ mod tests { // Can't perform 2 or more operations on the same node let ops = vec![ - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![b"a".to_vec()], b"b".to_vec(), Element::new_item(vec![1]), ), - GroveDbOp::insert_op(vec![b"a".to_vec()], b"b".to_vec(), Element::empty_tree()), + QualifiedGroveDbOp::insert_or_replace_op( + vec![b"a".to_vec()], + b"b".to_vec(), + Element::empty_tree(), + ), ]; assert!(matches!( db.apply_batch(ops, None, None, grove_version).unwrap(), @@ -2556,12 +2921,12 @@ mod tests { // Can't insert under a deleted path let ops = vec![ - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec()], b"b".to_vec(), Element::new_item(vec![1]), ), - GroveDbOp::delete_op(vec![], TEST_LEAF.to_vec()), + QualifiedGroveDbOp::delete_op(vec![], TEST_LEAF.to_vec()), ]; assert!(matches!( db.apply_batch(ops, None, None, grove_version).unwrap(), @@ -2572,12 +2937,12 @@ mod tests { // Should allow invalid operations pass when disable option is set to true let ops = vec![ - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec()], b"b".to_vec(), Element::empty_tree(), ), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec()], b"b".to_vec(), Element::empty_tree(), @@ -2622,28 +2987,32 @@ mod tests { let element = Element::new_item(b"ayy".to_vec()); let element2 = Element::new_item(b"ayy2".to_vec()); let ops = vec![ - GroveDbOp::insert_op(vec![], b"key1".to_vec(), Element::empty_tree()), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( + vec![], + b"key1".to_vec(), + Element::empty_tree(), + ), + QualifiedGroveDbOp::insert_or_replace_op( vec![b"key1".to_vec()], b"key2".to_vec(), Element::empty_tree(), ), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![b"key1".to_vec(), b"key2".to_vec()], b"key3".to_vec(), Element::empty_tree(), ), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![b"key1".to_vec(), b"key2".to_vec(), b"key3".to_vec()], b"key4".to_vec(), element.clone(), ), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec()], b"key1".to_vec(), Element::empty_tree(), ), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec(), b"key1".to_vec()], b"key2".to_vec(), element2.clone(), @@ -2721,28 +3090,28 @@ mod tests { let tx = db.start_transaction(); // let's start by inserting a tree structure let ops = vec![ - GroveDbOp::insert_op(vec![], b"1".to_vec(), Element::empty_tree()), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op(vec![], b"1".to_vec(), Element::empty_tree()), + QualifiedGroveDbOp::insert_or_replace_op( vec![b"1".to_vec()], b"my_contract".to_vec(), Element::empty_tree(), ), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![b"1".to_vec(), b"my_contract".to_vec()], b"0".to_vec(), Element::new_item(b"this is the contract".to_vec()), ), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![b"1".to_vec(), b"my_contract".to_vec()], b"1".to_vec(), Element::empty_tree(), ), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![b"1".to_vec(), b"my_contract".to_vec(), b"1".to_vec()], b"person".to_vec(), Element::empty_tree(), ), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![ b"1".to_vec(), b"my_contract".to_vec(), @@ -2752,7 +3121,7 @@ mod tests { b"0".to_vec(), Element::empty_tree(), ), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![ b"1".to_vec(), b"my_contract".to_vec(), @@ -2781,7 +3150,7 @@ mod tests { // now let's add an item let ops = vec![ - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![ b"1".to_vec(), b"my_contract".to_vec(), @@ -2795,7 +3164,7 @@ mod tests { some_element_flags.clone(), ), ), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![ b"1".to_vec(), b"my_contract".to_vec(), @@ -2806,7 +3175,7 @@ mod tests { b"my apples are safe".to_vec(), Element::empty_tree_with_flags(some_element_flags.clone()), ), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![ b"1".to_vec(), b"my_contract".to_vec(), @@ -2818,7 +3187,7 @@ mod tests { b"0".to_vec(), Element::empty_tree_with_flags(some_element_flags.clone()), ), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![ b"1".to_vec(), b"my_contract".to_vec(), @@ -2855,7 +3224,7 @@ mod tests { // now let's add an item let ops = vec![ - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![ b"1".to_vec(), b"my_contract".to_vec(), @@ -2866,7 +3235,7 @@ mod tests { b"wisdom".to_vec(), Element::new_item_with_flags(b"Wisdom Ogwu".to_vec(), some_element_flags.clone()), ), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![ b"1".to_vec(), b"my_contract".to_vec(), @@ -2877,7 +3246,7 @@ mod tests { b"canteloupe!".to_vec(), Element::empty_tree_with_flags(some_element_flags.clone()), ), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![ b"1".to_vec(), b"my_contract".to_vec(), @@ -2889,7 +3258,7 @@ mod tests { b"0".to_vec(), Element::empty_tree_with_flags(some_element_flags.clone()), ), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![ b"1".to_vec(), b"my_contract".to_vec(), @@ -2930,60 +3299,60 @@ mod tests { .expect("successful batch apply"); } - fn grove_db_ops_for_contract_insert() -> Vec { + fn grove_db_ops_for_contract_insert() -> Vec { let mut grove_db_ops = vec![]; - grove_db_ops.push(GroveDbOp::insert_op( + grove_db_ops.push(QualifiedGroveDbOp::insert_or_replace_op( vec![], b"contract".to_vec(), Element::empty_tree(), )); - grove_db_ops.push(GroveDbOp::insert_op( + grove_db_ops.push(QualifiedGroveDbOp::insert_or_replace_op( vec![b"contract".to_vec()], [0u8].to_vec(), Element::new_item(b"serialized_contract".to_vec()), )); - grove_db_ops.push(GroveDbOp::insert_op( + grove_db_ops.push(QualifiedGroveDbOp::insert_or_replace_op( vec![b"contract".to_vec()], [1u8].to_vec(), Element::empty_tree(), )); - grove_db_ops.push(GroveDbOp::insert_op( + grove_db_ops.push(QualifiedGroveDbOp::insert_or_replace_op( vec![b"contract".to_vec(), [1u8].to_vec()], b"domain".to_vec(), Element::empty_tree(), )); - grove_db_ops.push(GroveDbOp::insert_op( + grove_db_ops.push(QualifiedGroveDbOp::insert_or_replace_op( vec![b"contract".to_vec(), [1u8].to_vec(), b"domain".to_vec()], [0u8].to_vec(), Element::empty_tree(), )); - grove_db_ops.push(GroveDbOp::insert_op( + grove_db_ops.push(QualifiedGroveDbOp::insert_or_replace_op( vec![b"contract".to_vec(), [1u8].to_vec(), b"domain".to_vec()], b"normalized_domain_label".to_vec(), Element::empty_tree(), )); - grove_db_ops.push(GroveDbOp::insert_op( + grove_db_ops.push(QualifiedGroveDbOp::insert_or_replace_op( vec![b"contract".to_vec(), [1u8].to_vec(), b"domain".to_vec()], b"unique_records".to_vec(), Element::empty_tree(), )); - grove_db_ops.push(GroveDbOp::insert_op( + grove_db_ops.push(QualifiedGroveDbOp::insert_or_replace_op( vec![b"contract".to_vec(), [1u8].to_vec(), b"domain".to_vec()], b"alias_records".to_vec(), Element::empty_tree(), )); - grove_db_ops.push(GroveDbOp::insert_op( + grove_db_ops.push(QualifiedGroveDbOp::insert_or_replace_op( vec![b"contract".to_vec(), [1u8].to_vec()], b"preorder".to_vec(), Element::empty_tree(), )); - grove_db_ops.push(GroveDbOp::insert_op( + grove_db_ops.push(QualifiedGroveDbOp::insert_or_replace_op( vec![b"contract".to_vec(), [1u8].to_vec(), b"preorder".to_vec()], [0u8].to_vec(), Element::empty_tree(), )); - grove_db_ops.push(GroveDbOp::insert_op( + grove_db_ops.push(QualifiedGroveDbOp::insert_or_replace_op( vec![b"contract".to_vec(), [1u8].to_vec(), b"preorder".to_vec()], b"salted_domain".to_vec(), Element::empty_tree(), @@ -2992,10 +3361,10 @@ mod tests { grove_db_ops } - fn grove_db_ops_for_contract_document_insert() -> Vec { + fn grove_db_ops_for_contract_document_insert() -> Vec { let mut grove_db_ops = vec![]; - grove_db_ops.push(GroveDbOp::insert_op( + grove_db_ops.push(QualifiedGroveDbOp::insert_or_replace_op( vec![ b"contract".to_vec(), [1u8].to_vec(), @@ -3006,7 +3375,7 @@ mod tests { Element::new_item(b"serialized_domain".to_vec()), )); - grove_db_ops.push(GroveDbOp::insert_op( + grove_db_ops.push(QualifiedGroveDbOp::insert_or_replace_op( vec![ b"contract".to_vec(), [1u8].to_vec(), @@ -3017,7 +3386,7 @@ mod tests { Element::empty_tree(), )); - grove_db_ops.push(GroveDbOp::insert_op( + grove_db_ops.push(QualifiedGroveDbOp::insert_or_replace_op( vec![ b"contract".to_vec(), [1u8].to_vec(), @@ -3029,7 +3398,7 @@ mod tests { Element::empty_tree(), )); - grove_db_ops.push(GroveDbOp::insert_op( + grove_db_ops.push(QualifiedGroveDbOp::insert_or_replace_op( vec![ b"contract".to_vec(), [1u8].to_vec(), @@ -3042,7 +3411,7 @@ mod tests { Element::empty_tree(), )); - grove_db_ops.push(GroveDbOp::insert_op( + grove_db_ops.push(QualifiedGroveDbOp::insert_or_replace_op( vec![ b"contract".to_vec(), [1u8].to_vec(), @@ -3165,13 +3534,17 @@ mod tests { let db = make_test_grovedb(grove_version); let element = Element::new_item(b"ayy".to_vec()); let ops = vec![ - GroveDbOp::insert_op(vec![], b"key1".to_vec(), Element::empty_tree()), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( + vec![], + b"key1".to_vec(), + Element::empty_tree(), + ), + QualifiedGroveDbOp::insert_or_replace_op( vec![b"key1".to_vec(), b"key2".to_vec(), b"key3".to_vec()], b"key4".to_vec(), element, ), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![b"key1".to_vec()], b"key2".to_vec(), Element::empty_tree(), @@ -3193,23 +3566,27 @@ mod tests { let db = make_test_grovedb(grove_version); let element = Element::new_item(b"ayy".to_vec()); let ops = vec![ - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec()], b"key1".to_vec(), Element::empty_tree(), ), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec(), b"key1".to_vec()], b"key2".to_vec(), element.clone(), ), - GroveDbOp::insert_op(vec![], b"key1".to_vec(), Element::empty_tree()), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( + vec![], + b"key1".to_vec(), + Element::empty_tree(), + ), + QualifiedGroveDbOp::insert_or_replace_op( vec![b"key1".to_vec(), b"key2".to_vec(), b"key3".to_vec()], b"key4".to_vec(), element, ), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![b"key1".to_vec()], b"key2".to_vec(), Element::empty_tree(), @@ -3257,17 +3634,17 @@ mod tests { .expect("cannot insert a subtree"); let ops = vec![ - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![b"key1".to_vec(), b"key2".to_vec(), b"key3".to_vec()], b"key4".to_vec(), element, ), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![b"key1".to_vec(), b"key2".to_vec()], b"key3".to_vec(), Element::empty_tree(), ), - GroveDbOp::delete_op(vec![b"key1".to_vec()], b"key2".to_vec()), + QualifiedGroveDbOp::delete_op(vec![b"key1".to_vec()], b"key2".to_vec()), ]; assert!(db .apply_batch(ops, None, None, grove_version) @@ -3281,23 +3658,27 @@ mod tests { let db = make_test_grovedb(grove_version); let element = Element::new_item(b"ayy".to_vec()); let ops = vec![ - GroveDbOp::insert_op(vec![], b"key1".to_vec(), Element::empty_tree()), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( + vec![], + b"key1".to_vec(), + Element::empty_tree(), + ), + QualifiedGroveDbOp::insert_or_replace_op( vec![b"key1".to_vec(), b"key2".to_vec(), b"key3".to_vec()], b"key4".to_vec(), element, ), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![b"key1".to_vec(), b"key2".to_vec()], b"key3".to_vec(), Element::empty_tree(), ), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![b"key1".to_vec()], b"key2".to_vec(), Element::empty_tree(), ), - GroveDbOp::delete_op(vec![b"key1".to_vec()], b"key2".to_vec()), + QualifiedGroveDbOp::delete_op(vec![b"key1".to_vec()], b"key2".to_vec()), ]; db.apply_batch(ops, None, None, grove_version) .unwrap() @@ -3340,7 +3721,7 @@ mod tests { .expect("cannot insert value"); // Insertion into scalar is invalid - let ops = vec![GroveDbOp::insert_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec(), b"invalid".to_vec()], b"key1".to_vec(), element.clone(), @@ -3351,7 +3732,7 @@ mod tests { .is_err()); // Insertion into a tree is correct - let ops = vec![GroveDbOp::insert_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec(), b"valid".to_vec()], b"key1".to_vec(), element.clone(), @@ -3396,8 +3777,8 @@ mod tests { // TEST_LEAF can not be overwritten let ops = vec![ - GroveDbOp::insert_op(vec![], TEST_LEAF.to_vec(), element2), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op(vec![], TEST_LEAF.to_vec(), element2), + QualifiedGroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec(), b"key_subtree".to_vec()], b"key1".to_vec(), Element::empty_tree(), @@ -3423,8 +3804,8 @@ mod tests { // TEST_LEAF will be deleted so you can not insert underneath it let ops = vec![ - GroveDbOp::delete_op(vec![], TEST_LEAF.to_vec()), - GroveDbOp::insert_op( + QualifiedGroveDbOp::delete_op(vec![], TEST_LEAF.to_vec()), + QualifiedGroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec()], b"key1".to_vec(), Element::empty_tree(), @@ -3439,8 +3820,8 @@ mod tests { // We are testing with the batch apply option // validate_tree_insertion_does_not_override set to true let ops = vec![ - GroveDbOp::delete_op(vec![], TEST_LEAF.to_vec()), - GroveDbOp::insert_op( + QualifiedGroveDbOp::delete_op(vec![], TEST_LEAF.to_vec()), + QualifiedGroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec()], b"key1".to_vec(), Element::empty_tree(), @@ -3470,12 +3851,12 @@ mod tests { let grove_version = GroveVersion::latest(); let db = make_test_grovedb(grove_version); let ops = vec![ - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![], TEST_LEAF.to_vec(), Element::new_item(b"ayy".to_vec()), ), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec()], b"key1".to_vec(), Element::empty_tree(), @@ -3526,7 +3907,7 @@ mod tests { ) .unwrap() .expect("cannot insert an item"); - let ops = vec![GroveDbOp::insert_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec()], b"key1".to_vec(), Element::new_item(b"ayy2".to_vec()), @@ -3590,23 +3971,27 @@ mod tests { let element2 = Element::new_item(b"ayy2".to_vec()); let ops = vec![ - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![b"key1".to_vec(), b"key2".to_vec(), b"key3".to_vec()], b"key4".to_vec(), element.clone(), ), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![b"key1".to_vec(), b"key2".to_vec()], b"key3".to_vec(), Element::empty_tree(), ), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![b"key1".to_vec()], b"key2".to_vec(), Element::empty_tree(), ), - GroveDbOp::insert_op(vec![TEST_LEAF.to_vec()], b"key".to_vec(), element2.clone()), - GroveDbOp::delete_op(vec![ANOTHER_TEST_LEAF.to_vec()], b"key1".to_vec()), + QualifiedGroveDbOp::insert_or_replace_op( + vec![TEST_LEAF.to_vec()], + b"key".to_vec(), + element2.clone(), + ), + QualifiedGroveDbOp::delete_op(vec![ANOTHER_TEST_LEAF.to_vec()], b"key1".to_vec()), ]; db.apply_batch(ops, None, None, grove_version) .unwrap() @@ -3692,7 +4077,7 @@ mod tests { } let element = Element::new_item(b"ayy".to_vec()); - let batch = vec![GroveDbOp::insert_op( + let batch = vec![QualifiedGroveDbOp::insert_or_replace_op( acc_path.clone(), b"key".to_vec(), element.clone(), @@ -3701,7 +4086,11 @@ mod tests { .unwrap() .expect("cannot apply batch"); - let batch = vec![GroveDbOp::insert_op(acc_path, b"key".to_vec(), element)]; + let batch = vec![QualifiedGroveDbOp::insert_or_replace_op( + acc_path, + b"key".to_vec(), + element, + )]; db.apply_batch(batch, None, None, grove_version) .unwrap() .expect("cannot apply same batch twice"); @@ -3730,7 +4119,7 @@ mod tests { let root_hash = db.root_hash(None, grove_version).unwrap().unwrap(); let element = Element::new_item(b"ayy".to_vec()); - let batch = vec![GroveDbOp::insert_op( + let batch = vec![QualifiedGroveDbOp::insert_or_replace_op( acc_path.clone(), b"key".to_vec(), element, @@ -3750,7 +4139,7 @@ mod tests { let grove_version = GroveVersion::latest(); // insert reference that points to non-existent item let db = make_test_grovedb(grove_version); - let batch = vec![GroveDbOp::insert_op( + let batch = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec()], b"key1".to_vec(), Element::new_reference(ReferencePathType::AbsolutePathReference(vec![ @@ -3767,7 +4156,7 @@ mod tests { let db = make_test_grovedb(grove_version); let elem = Element::new_item(b"ayy".to_vec()); let batch = vec![ - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec()], b"key1".to_vec(), Element::new_reference(ReferencePathType::AbsolutePathReference(vec![ @@ -3775,7 +4164,7 @@ mod tests { b"invalid_path".to_vec(), ])), ), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec()], b"invalid_path".to_vec(), elem.clone(), @@ -3808,7 +4197,7 @@ mod tests { let db = make_test_grovedb(grove_version); let elem = Element::new_item(b"ayy".to_vec()); let batch = vec![ - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec()], b"key2".to_vec(), Element::new_reference_with_hops( @@ -3819,7 +4208,7 @@ mod tests { Some(1), ), ), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec()], b"key1".to_vec(), Element::new_reference_with_hops( @@ -3830,7 +4219,11 @@ mod tests { Some(1), ), ), - GroveDbOp::insert_op(vec![TEST_LEAF.to_vec()], b"invalid_path".to_vec(), elem), + QualifiedGroveDbOp::insert_or_replace_op( + vec![TEST_LEAF.to_vec()], + b"invalid_path".to_vec(), + elem, + ), ]; assert!(matches!( db.apply_batch(batch, None, None, grove_version).unwrap(), diff --git a/grovedb/src/batch/multi_insert_cost_tests.rs b/grovedb/src/batch/multi_insert_cost_tests.rs index ad171d6d..666de224 100644 --- a/grovedb/src/batch/multi_insert_cost_tests.rs +++ b/grovedb/src/batch/multi_insert_cost_tests.rs @@ -5,13 +5,18 @@ mod tests { use std::{ops::Add, option::Option::None}; use grovedb_costs::{ - storage_cost::{removal::StorageRemovedBytes::NoStorageRemoval, StorageCost}, + storage_cost::{ + removal::StorageRemovedBytes::{BasicStorageRemoval, NoStorageRemoval}, + transition::OperationStorageTransitionType, + StorageCost, + }, OperationCost, }; use grovedb_version::version::GroveVersion; + use integer_encoding::VarInt; use crate::{ - batch::GroveDbOp, + batch::QualifiedGroveDbOp, reference_path::ReferencePathType::{SiblingReference, UpstreamFromElementHeightReference}, tests::{common::EMPTY_PATH, make_empty_grovedb}, Element, @@ -46,8 +51,16 @@ mod tests { let non_batch_cost = non_batch_cost_1.add(non_batch_cost_2); tx.rollback().expect("expected to rollback"); let ops = vec![ - GroveDbOp::insert_op(vec![], b"key1".to_vec(), Element::empty_tree()), - GroveDbOp::insert_op(vec![], b"key2".to_vec(), Element::empty_tree()), + QualifiedGroveDbOp::insert_or_replace_op( + vec![], + b"key1".to_vec(), + Element::empty_tree(), + ), + QualifiedGroveDbOp::insert_or_replace_op( + vec![], + b"key2".to_vec(), + Element::empty_tree(), + ), ]; let cost = db.apply_batch(ops, None, Some(&tx), grove_version).cost; assert_eq!( @@ -97,13 +110,17 @@ mod tests { let non_batch_cost = non_batch_cost_1.add(non_batch_cost_2).add(non_batch_cost_3); tx.rollback().expect("expected to rollback"); let ops = vec![ - GroveDbOp::insert_op(vec![], b"key1".to_vec(), Element::empty_tree()), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( + vec![], + b"key1".to_vec(), + Element::empty_tree(), + ), + QualifiedGroveDbOp::insert_or_replace_op( vec![], b"key2".to_vec(), Element::new_item_with_flags(b"pizza".to_vec(), Some([0, 1].to_vec())), ), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![], b"key3".to_vec(), Element::new_reference(SiblingReference(b"key2".to_vec())), @@ -173,18 +190,22 @@ mod tests { .add(non_batch_cost_4); tx.rollback().expect("expected to rollback"); let ops = vec![ - GroveDbOp::insert_op(vec![], b"key1".to_vec(), Element::empty_tree()), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( + vec![], + b"key1".to_vec(), + Element::empty_tree(), + ), + QualifiedGroveDbOp::insert_or_replace_op( vec![b"key1".to_vec()], b"key2".to_vec(), Element::new_item_with_flags(b"pizza".to_vec(), Some([0, 1].to_vec())), ), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![b"key1".to_vec()], b"key3".to_vec(), Element::empty_tree(), ), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![b"key1".to_vec(), b"key3".to_vec()], b"key4".to_vec(), Element::new_reference(UpstreamFromElementHeightReference( @@ -212,8 +233,16 @@ mod tests { let tx = db.start_transaction(); let ops = vec![ - GroveDbOp::insert_op(vec![], b"key1".to_vec(), Element::empty_tree()), - GroveDbOp::insert_op(vec![], b"key2".to_vec(), Element::empty_tree()), + QualifiedGroveDbOp::insert_or_replace_op( + vec![], + b"key1".to_vec(), + Element::empty_tree(), + ), + QualifiedGroveDbOp::insert_or_replace_op( + vec![], + b"key2".to_vec(), + Element::empty_tree(), + ), ]; let cost_result = db.apply_batch(ops, None, Some(&tx), grove_version); cost_result.value.expect("expected to execute batch"); @@ -268,8 +297,12 @@ mod tests { let tx = db.start_transaction(); let ops = vec![ - GroveDbOp::insert_op(vec![], b"key1".to_vec(), Element::empty_tree()), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( + vec![], + b"key1".to_vec(), + Element::empty_tree(), + ), + QualifiedGroveDbOp::insert_or_replace_op( vec![b"key1".to_vec()], b"key2".to_vec(), Element::empty_tree(), @@ -322,4 +355,116 @@ mod tests { } ); } + + #[test] + fn test_batch_insert_item_in_also_inserted_sub_tree_with_reference() { + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + let tx = db.start_transaction(); + db.insert( + EMPTY_PATH, + b"tree", + Element::empty_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to insert tree"); + + let ops = vec![ + QualifiedGroveDbOp::insert_or_replace_op( + vec![b"tree".to_vec()], + b"tree2".to_vec(), + Element::empty_tree(), + ), + QualifiedGroveDbOp::insert_or_replace_op( + vec![b"tree".to_vec(), b"tree2".to_vec()], + b"key1".to_vec(), + Element::new_item_with_flags(b"value".to_vec(), Some(vec![0, 1])), + ), + QualifiedGroveDbOp::insert_only_op( + vec![b"tree".to_vec(), b"tree2".to_vec()], + b"keyref".to_vec(), + Element::new_reference_with_flags( + SiblingReference(b"key1".to_vec()), + Some(vec![0, 1]), + ), + ), + ]; + + let cost = db + .apply_batch_with_element_flags_update( + ops, + None, + |cost, old_flags, new_flags| match cost.transition_type() { + OperationStorageTransitionType::OperationUpdateBiggerSize => { + if new_flags[0] == 0 { + new_flags[0] = 1; + let new_flags_epoch = new_flags[1]; + new_flags[1] = old_flags.unwrap()[1]; + new_flags.push(new_flags_epoch); + new_flags.extend(cost.added_bytes.encode_var_vec()); + // first pass will be vec![1u8, 0, 1, 2], second pass will be vec![1u8, + // 0, 1, 4] + assert!( + new_flags == &vec![1u8, 0, 1, 2] + || new_flags == &vec![1u8, 0, 1, 4] + ); + Ok(true) + } else { + assert_eq!(new_flags[0], 1); + Ok(false) + } + } + OperationStorageTransitionType::OperationUpdateSmallerSize => { + new_flags.extend(vec![1, 2]); + Ok(true) + } + _ => Ok(false), + }, + |_flags, removed_key_bytes, removed_value_bytes| { + Ok(( + BasicStorageRemoval(removed_key_bytes), + BasicStorageRemoval(removed_value_bytes), + )) + }, + Some(&tx), + grove_version, + ) + .cost_as_result() + .expect("expect no error"); + + // Hash node calls + + // Seek Count + + assert_eq!( + cost, + OperationCost { + seek_count: 8, // todo: verify this + storage_cost: StorageCost { + added_bytes: 430, + replaced_bytes: 78, // todo: verify this + removed_bytes: NoStorageRemoval + }, + storage_loaded_bytes: 152, // todo: verify this + hash_node_calls: 22, // todo: verify this + } + ); + + let issues = db + .visualize_verify_grovedb(Some(&tx), true, false, &Default::default()) + .unwrap(); + assert_eq!( + issues.len(), + 0, + "reference issue: {}", + issues + .iter() + .map(|(hash, (a, b, c))| format!("{}: {} {} {}", hash, a, b, c)) + .collect::>() + .join(" | ") + ); + } } diff --git a/grovedb/src/batch/single_deletion_cost_tests.rs b/grovedb/src/batch/single_deletion_cost_tests.rs index fac9682f..e2b3d9a9 100644 --- a/grovedb/src/batch/single_deletion_cost_tests.rs +++ b/grovedb/src/batch/single_deletion_cost_tests.rs @@ -11,7 +11,7 @@ mod tests { use intmap::IntMap; use crate::{ - batch::GroveDbOp, + batch::QualifiedGroveDbOp, tests::{common::EMPTY_PATH, make_empty_grovedb}, Element, }; @@ -72,7 +72,11 @@ mod tests { ); tx.rollback().expect("expected to rollback"); - let ops = vec![GroveDbOp::delete_tree_op(vec![], b"key1".to_vec(), false)]; + let ops = vec![QualifiedGroveDbOp::delete_tree_op( + vec![], + b"key1".to_vec(), + false, + )]; let batch_cost = db .apply_batch(ops, None, Some(&tx), grove_version) .cost_as_result() @@ -137,7 +141,7 @@ mod tests { ); tx.rollback().expect("expected to rollback"); - let ops = vec![GroveDbOp::delete_op(vec![], b"key1".to_vec())]; + let ops = vec![QualifiedGroveDbOp::delete_op(vec![], b"key1".to_vec())]; let batch_cost = db .apply_batch(ops, None, Some(&tx), grove_version) .cost_as_result() @@ -212,7 +216,11 @@ mod tests { .cost_as_result() .expect("expected to insert successfully"); - let ops = vec![GroveDbOp::delete_tree_op(vec![], b"key1".to_vec(), false)]; + let ops = vec![QualifiedGroveDbOp::delete_tree_op( + vec![], + b"key1".to_vec(), + false, + )]; let batch_cost = db .apply_batch(ops, None, None, grove_version) .cost_as_result() @@ -288,7 +296,7 @@ mod tests { .cost_as_result() .expect("expected to insert successfully"); - let ops = vec![GroveDbOp::delete_op(vec![], b"key1".to_vec())]; + let ops = vec![QualifiedGroveDbOp::delete_op(vec![], b"key1".to_vec())]; let batch_cost = db .apply_batch(ops, None, None, grove_version) .cost_as_result() @@ -357,7 +365,11 @@ mod tests { ); tx.rollback().expect("expected to rollback"); - let ops = vec![GroveDbOp::delete_tree_op(vec![], b"key1".to_vec(), false)]; + let ops = vec![QualifiedGroveDbOp::delete_tree_op( + vec![], + b"key1".to_vec(), + false, + )]; let batch_cost = db .apply_batch(ops, None, Some(&tx), grove_version) .cost_as_result() @@ -452,7 +464,11 @@ mod tests { )); tx.rollback().expect("expected to rollback"); - let ops = vec![GroveDbOp::delete_tree_op(vec![], b"key1".to_vec(), false)]; + let ops = vec![QualifiedGroveDbOp::delete_tree_op( + vec![], + b"key1".to_vec(), + false, + )]; let batch_cost = db .apply_batch_with_element_flags_update( ops, @@ -543,7 +559,7 @@ mod tests { ); tx.rollback().expect("expected to rollback"); - let ops = vec![GroveDbOp::delete_op(vec![], b"key1".to_vec())]; + let ops = vec![QualifiedGroveDbOp::delete_op(vec![], b"key1".to_vec())]; let batch_cost = db .apply_batch(ops, None, Some(&tx), grove_version) .cost_as_result() @@ -623,7 +639,11 @@ mod tests { .cost_as_result() .expect("expected to insert successfully"); - let ops = vec![GroveDbOp::delete_tree_op(vec![], b"key1".to_vec(), false)]; + let ops = vec![QualifiedGroveDbOp::delete_tree_op( + vec![], + b"key1".to_vec(), + false, + )]; let batch_cost = db .apply_batch(ops, None, None, grove_version) .cost_as_result() @@ -699,7 +719,7 @@ mod tests { .cost_as_result() .expect("expected to insert successfully"); - let ops = vec![GroveDbOp::delete_op(vec![], b"key1".to_vec())]; + let ops = vec![QualifiedGroveDbOp::delete_op(vec![], b"key1".to_vec())]; let batch_cost = db .apply_batch(ops, None, None, grove_version) .cost_as_result() diff --git a/grovedb/src/batch/single_insert_cost_tests.rs b/grovedb/src/batch/single_insert_cost_tests.rs index c025fb27..d86a2bf2 100644 --- a/grovedb/src/batch/single_insert_cost_tests.rs +++ b/grovedb/src/batch/single_insert_cost_tests.rs @@ -20,7 +20,8 @@ mod tests { use intmap::IntMap; use crate::{ - batch::GroveDbOp, + batch::QualifiedGroveDbOp, + reference_path::ReferencePathType::SiblingReference, tests::{common::EMPTY_PATH, make_empty_grovedb}, Element, }; @@ -42,7 +43,7 @@ mod tests { ) .cost; tx.rollback().expect("expected to rollback"); - let ops = vec![GroveDbOp::insert_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![], b"key1".to_vec(), Element::empty_tree(), @@ -57,7 +58,7 @@ mod tests { let db = make_empty_grovedb(); let tx = db.start_transaction(); - let ops = vec![GroveDbOp::insert_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![], b"key1".to_vec(), Element::empty_tree(), @@ -122,7 +123,7 @@ mod tests { let db = make_empty_grovedb(); let tx = db.start_transaction(); - let ops = vec![GroveDbOp::insert_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![], b"key1".to_vec(), Element::new_item(b"cat".to_vec()), @@ -200,7 +201,7 @@ mod tests { assert_eq!(cost.storage_cost.added_bytes, 143); - let ops = vec![GroveDbOp::insert_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![], b"key1".to_vec(), Element::empty_tree(), @@ -293,7 +294,7 @@ mod tests { .unwrap() .expect("successful root tree leaf insert"); - let ops = vec![GroveDbOp::insert_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![], b"key1".to_vec(), Element::empty_tree(), @@ -375,7 +376,7 @@ mod tests { .unwrap() .expect("successful root tree leaf insert"); - let ops = vec![GroveDbOp::insert_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![b"0".to_vec()], b"key1".to_vec(), Element::empty_tree(), @@ -453,7 +454,7 @@ mod tests { let db = make_empty_grovedb(); let tx = db.start_transaction(); - let ops = vec![GroveDbOp::insert_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![], b"key1".to_vec(), Element::new_item([0u8; 59].to_vec()), @@ -516,7 +517,7 @@ mod tests { let db = make_empty_grovedb(); let tx = db.start_transaction(); - let ops = vec![GroveDbOp::insert_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![], b"key1".to_vec(), Element::new_item([0u8; 60].to_vec()), @@ -573,6 +574,136 @@ mod tests { ); } + #[test] + fn test_batch_root_one_insert_with_flags_cost_right_below_value_required_cost_of_2() { + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + let tx = db.start_transaction(); + + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( + vec![], + b"key1".to_vec(), + Element::new_item_with_flags([0u8; 56].to_vec(), Some(vec![0, 0])), + )]; + let cost_result = db.apply_batch(ops, None, Some(&tx), grove_version); + cost_result.value.expect("expected to execute batch"); + let cost = cost_result.cost; + // Explanation for 243 storage_written_bytes + + // Key -> 37 bytes + // 32 bytes for the key prefix + // 4 bytes for the key + // 1 byte for key_size (required space for 36) + + // Value -> 128 + // 1 for the flag options + // 1 for flags size + // 2 for flag bytes + // 1 for the enum type + // 1 for the value size + // 56 bytes + // 32 for node hash + // 32 for value hash + // 1 for basic merk + // 1 byte for the value_size (required space for 127) + + // Parent Hook -> 40 + // Key Bytes 4 + // Hash Size 32 + // Key Length 1 + // Child Heights 2 + // Sum 1 + // Total 37 + 128 + 40 = 205 + + // Hash node calls + // 2 for the node hash + // 1 for the value hash + // 1 kv_digest_to_kv_hash + + // Seek Count + // 1 to load from root tree + // 1 to insert + // 1 to update root tree + assert_eq!( + cost, + OperationCost { + seek_count: 3, + storage_cost: StorageCost { + added_bytes: 205, + replaced_bytes: 0, + removed_bytes: NoStorageRemoval, + }, + storage_loaded_bytes: 0, + hash_node_calls: 4, + } + ); + } + + #[test] + fn test_batch_root_one_insert_with_flags_cost_right_above_value_required_cost_of_2() { + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + let tx = db.start_transaction(); + + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( + vec![], + b"key1".to_vec(), + Element::new_item_with_flags([0u8; 57].to_vec(), Some(vec![0, 0])), + )]; + let cost_result = db.apply_batch(ops, None, Some(&tx), grove_version); + cost_result.value.expect("expected to execute batch"); + let cost = cost_result.cost; + // Explanation for 243 storage_written_bytes + + // Key -> 37 bytes + // 32 bytes for the key prefix + // 4 bytes for the key + // 1 byte for key_size (required space for 36) + + // Value -> 130 + // 1 for the flag option + // 1 for flags size + // 2 for flag bytes + // 1 for the enum type + // 1 for the value size + // 60 bytes + // 32 for node hash + // 32 for value hash + // 1 for basic merk + // 2 byte for the value_size (required space for 128) + + // Parent Hook -> 40 + // Key Bytes 4 + // Hash Size 32 + // Key Length 1 + // Child Heights 2 + // Sum 1 + // Total 37 + 130 + 40 = 207 + + // Hash node calls + // 2 for the node hash + // 1 for the value hash (just under) + // 1 kv_digest_to_kv_hash + + // Seek Count + // 1 to load from root tree + // 1 to insert + // 1 to update root tree + assert_eq!( + cost, + OperationCost { + seek_count: 3, + storage_cost: StorageCost { + added_bytes: 207, + replaced_bytes: 0, + removed_bytes: NoStorageRemoval, + }, + storage_loaded_bytes: 0, + hash_node_calls: 4, + } + ); + } + #[test] fn test_batch_root_one_update_item_bigger_cost_no_flags() { let grove_version = GroveVersion::latest(); @@ -601,7 +732,7 @@ mod tests { .expect("expected to insert item"); // We are adding 2 bytes - let ops = vec![GroveDbOp::insert_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![b"tree".to_vec()], b"key1".to_vec(), Element::new_item_with_flags(b"value100".to_vec(), Some(vec![1])), @@ -667,7 +798,7 @@ mod tests { .expect("expected to insert item"); // We are adding 2 bytes - let ops = vec![GroveDbOp::insert_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![b"tree".to_vec()], b"key1".to_vec(), Element::new_item_with_flags(b"value100".to_vec(), Some(vec![0, 1])), @@ -685,7 +816,12 @@ mod tests { new_flags[1] = old_flags.unwrap()[1]; new_flags.push(new_flags_epoch); new_flags.extend(cost.added_bytes.encode_var_vec()); - assert_eq!(new_flags, &vec![1u8, 0, 1, 2]); + // first pass will be vec![1u8, 0, 1, 2], second pass will be vec![1u8, + // 0, 1, 4] + assert!( + new_flags == &vec![1u8, 0, 1, 2] + || new_flags == &vec![1u8, 0, 1, 4] + ); Ok(true) } else { assert_eq!(new_flags[0], 1); @@ -728,6 +864,253 @@ mod tests { ); } + #[test] + fn test_batch_root_one_update_item_bigger_cost_with_refresh_reference() { + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + let tx = db.start_transaction(); + db.insert( + EMPTY_PATH, + b"tree", + Element::empty_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to insert tree"); + + db.insert( + [b"tree".as_slice()].as_ref(), + b"key1", + Element::new_item_with_flags(b"value1".to_vec(), Some(vec![0, 0])), + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to insert item"); + + db.insert( + [b"tree".as_slice()].as_ref(), + b"keyref", + Element::new_reference_with_flags(SiblingReference(b"key1".to_vec()), Some(vec![0, 0])), + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to insert item"); + + // We are adding 2 bytes + let ops = vec![ + QualifiedGroveDbOp::insert_or_replace_op( + vec![b"tree".to_vec()], + b"key1".to_vec(), + Element::new_item_with_flags(b"value100".to_vec(), Some(vec![0, 1])), + ), + QualifiedGroveDbOp::replace_op( + vec![b"tree".to_vec()], + b"keyref".to_vec(), + Element::new_reference_with_flags( + SiblingReference(b"key1".to_vec()), + Some(vec![0, 1]), + ), + ), + ]; + + let cost = db + .apply_batch_with_element_flags_update( + ops, + None, + |cost, old_flags, new_flags| match cost.transition_type() { + OperationStorageTransitionType::OperationUpdateBiggerSize => { + if new_flags[0] == 0 { + new_flags[0] = 1; + let new_flags_epoch = new_flags[1]; + new_flags[1] = old_flags.unwrap()[1]; + new_flags.push(new_flags_epoch); + new_flags.extend(cost.added_bytes.encode_var_vec()); + // first pass will be vec![1u8, 0, 1, 2], second pass will be vec![1u8, + // 0, 1, 4] + assert!( + new_flags == &vec![1u8, 0, 1, 2] + || new_flags == &vec![1u8, 0, 1, 4] + ); + Ok(true) + } else { + assert_eq!(new_flags[0], 1); + Ok(false) + } + } + OperationStorageTransitionType::OperationUpdateSmallerSize => { + new_flags.extend(vec![1, 2]); + Ok(true) + } + _ => Ok(false), + }, + |_flags, removed_key_bytes, removed_value_bytes| { + Ok(( + BasicStorageRemoval(removed_key_bytes), + BasicStorageRemoval(removed_value_bytes), + )) + }, + Some(&tx), + grove_version, + ) + .cost; + + // Hash node calls + + // Seek Count + + assert_eq!( + cost, + OperationCost { + seek_count: 9, // todo: verify this + storage_cost: StorageCost { + added_bytes: 4, + replaced_bytes: 316, // todo: verify this + removed_bytes: NoStorageRemoval + }, + storage_loaded_bytes: 357, // todo: verify this + hash_node_calls: 16, // todo: verify this + } + ); + + let issues = db + .visualize_verify_grovedb(Some(&tx), true, false, &Default::default()) + .unwrap(); + assert_eq!( + issues.len(), + 0, + "reference issue: {}", + issues + .iter() + .map(|(hash, (a, b, c))| format!("{}: {} {} {}", hash, a, b, c)) + .collect::>() + .join(" | ") + ); + } + + #[test] + fn test_batch_root_one_update_item_bigger_cost_with_insert_reference() { + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + let tx = db.start_transaction(); + db.insert( + EMPTY_PATH, + b"tree", + Element::empty_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to insert tree"); + + db.insert( + [b"tree".as_slice()].as_ref(), + b"key1", + Element::new_item_with_flags(b"value1".to_vec(), Some(vec![0, 0])), + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to insert item"); + + // We are adding 2 bytes + let ops = vec![ + QualifiedGroveDbOp::insert_or_replace_op( + vec![b"tree".to_vec()], + b"key1".to_vec(), + Element::new_item_with_flags(b"value100".to_vec(), Some(vec![0, 1])), + ), + QualifiedGroveDbOp::insert_only_op( + vec![b"tree".to_vec()], + b"keyref".to_vec(), + Element::new_reference_with_flags( + SiblingReference(b"key1".to_vec()), + Some(vec![0, 1]), + ), + ), + ]; + + let cost = db + .apply_batch_with_element_flags_update( + ops, + None, + |cost, old_flags, new_flags| match cost.transition_type() { + OperationStorageTransitionType::OperationUpdateBiggerSize => { + if new_flags[0] == 0 { + new_flags[0] = 1; + let new_flags_epoch = new_flags[1]; + new_flags[1] = old_flags.unwrap()[1]; + new_flags.push(new_flags_epoch); + new_flags.extend(cost.added_bytes.encode_var_vec()); + // first pass will be vec![1u8, 0, 1, 2], second pass will be vec![1u8, + // 0, 1, 4] + assert!( + new_flags == &vec![1u8, 0, 1, 2] + || new_flags == &vec![1u8, 0, 1, 4] + ); + Ok(true) + } else { + assert_eq!(new_flags[0], 1); + Ok(false) + } + } + OperationStorageTransitionType::OperationUpdateSmallerSize => { + new_flags.extend(vec![1, 2]); + Ok(true) + } + _ => Ok(false), + }, + |_flags, removed_key_bytes, removed_value_bytes| { + Ok(( + BasicStorageRemoval(removed_key_bytes), + BasicStorageRemoval(removed_value_bytes), + )) + }, + Some(&tx), + grove_version, + ) + .cost; + + // Hash node calls + + // Seek Count + + assert_eq!( + cost, + OperationCost { + seek_count: 8, // todo: verify this + storage_cost: StorageCost { + added_bytes: 163, + replaced_bytes: 196, // todo: verify this + removed_bytes: NoStorageRemoval + }, + storage_loaded_bytes: 236, // todo: verify this + hash_node_calls: 16, // todo: verify this + } + ); + + let issues = db + .visualize_verify_grovedb(Some(&tx), true, false, &Default::default()) + .unwrap(); + assert_eq!( + issues.len(), + 0, + "reference issue: {}", + issues + .iter() + .map(|(hash, (a, b, c))| format!("{}: {} {} {}", hash, a, b, c)) + .collect::>() + .join(" | ") + ); + } + #[test] fn test_batch_root_one_update_item_smaller_cost_no_flags() { let grove_version = GroveVersion::latest(); @@ -756,7 +1139,7 @@ mod tests { .expect("expected to insert item"); // We are adding 2 bytes - let ops = vec![GroveDbOp::insert_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![b"tree".to_vec()], b"key1".to_vec(), Element::new_item_with_flags(b"value".to_vec(), Some(vec![1])), @@ -821,7 +1204,7 @@ mod tests { .expect("expected to insert item"); // We are adding 2 bytes - let ops = vec![GroveDbOp::insert_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![b"tree".to_vec()], b"key1".to_vec(), Element::new_item_with_flags(b"value".to_vec(), Some(vec![0, 1])), @@ -839,7 +1222,12 @@ mod tests { new_flags[1] = old_flags.unwrap()[1]; new_flags.push(new_flags_epoch); new_flags.extend(cost.added_bytes.encode_var_vec()); - assert_eq!(new_flags, &vec![1u8, 0, 1, 2]); + // first pass will be vec![1u8, 0, 1, 2], second pass will be vec![1u8, + // 0, 1, 4] + assert!( + new_flags == &vec![1u8, 0, 1, 2] + || new_flags == &vec![1u8, 0, 1, 4] + ); Ok(true) } else { assert_eq!(new_flags[0], 1); @@ -883,6 +1271,136 @@ mod tests { ); } + #[test] + fn test_batch_root_one_update_item_smaller_cost_with_refresh_reference() { + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + let tx = db.start_transaction(); + db.insert( + EMPTY_PATH, + b"tree", + Element::empty_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to insert tree"); + + db.insert( + [b"tree".as_slice()].as_ref(), + b"key1", + Element::new_item_with_flags(b"value1".to_vec(), Some(vec![0, 0])), + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to insert item"); + + db.insert( + [b"tree".as_slice()].as_ref(), + b"keyref", + Element::new_reference_with_flags(SiblingReference(b"key1".to_vec()), Some(vec![0, 0])), + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to insert item"); + + // We are adding 2 bytes + let ops = vec![ + QualifiedGroveDbOp::insert_or_replace_op( + vec![b"tree".to_vec()], + b"key1".to_vec(), + Element::new_item_with_flags(b"value".to_vec(), Some(vec![0, 1])), + ), + QualifiedGroveDbOp::replace_op( + vec![b"tree".to_vec()], + b"keyref".to_vec(), + Element::new_reference_with_flags( + SiblingReference(b"key1".to_vec()), + Some(vec![0, 1]), + ), + ), + ]; + + let cost = db + .apply_batch_with_element_flags_update( + ops, + None, + |cost, old_flags, new_flags| match cost.transition_type() { + OperationStorageTransitionType::OperationUpdateBiggerSize => { + if new_flags[0] == 0 { + new_flags[0] = 1; + let new_flags_epoch = new_flags[1]; + new_flags[1] = old_flags.unwrap()[1]; + new_flags.push(new_flags_epoch); + new_flags.extend(cost.added_bytes.encode_var_vec()); + // first pass will be vec![1u8, 0, 1, 2], second pass will be vec![1u8, + // 0, 1, 4] + assert!( + new_flags == &vec![1u8, 0, 1, 2] + || new_flags == &vec![1u8, 0, 1, 4] + ); + Ok(true) + } else { + assert_eq!(new_flags[0], 1); + Ok(false) + } + } + OperationStorageTransitionType::OperationUpdateSmallerSize => Ok(true), + _ => Ok(false), + }, + |_flags, _removed_key_bytes, removed_value_bytes| { + let mut removed_bytes = StorageRemovalPerEpochByIdentifier::default(); + // we are removing 1 byte from epoch 0 for an identity + let mut removed_bytes_for_identity = IntMap::new(); + removed_bytes_for_identity.insert(0, removed_value_bytes); + removed_bytes.insert(Identifier::default(), removed_bytes_for_identity); + Ok((NoStorageRemoval, SectionedStorageRemoval(removed_bytes))) + }, + Some(&tx), + grove_version, + ) + .cost; + + let mut removed_bytes = StorageRemovalPerEpochByIdentifier::default(); + // we are removing 1 byte from epoch 0 for an identity + let mut removed_bytes_for_identity = IntMap::new(); + removed_bytes_for_identity.insert(0, 1); + removed_bytes.insert(Identifier::default(), removed_bytes_for_identity); + + assert_eq!( + cost, + OperationCost { + seek_count: 9, // todo: verify this + storage_cost: StorageCost { + added_bytes: 0, + replaced_bytes: 315, // todo: verify this + removed_bytes: SectionedStorageRemoval(removed_bytes) + }, + storage_loaded_bytes: 357, // todo: verify this + hash_node_calls: 16, // todo: verify this + } + ); + + let issues = db + .visualize_verify_grovedb(Some(&tx), true, false, &Default::default()) + .unwrap(); + assert_eq!( + issues.len(), + 0, + "reference issue: {}", + issues + .iter() + .map(|(hash, (a, b, c))| format!("{}: {} {} {}", hash, a, b, c)) + .collect::>() + .join(" | ") + ); + } + #[test] fn test_batch_root_one_update_tree_bigger_flags_cost() { let grove_version = GroveVersion::latest(); @@ -911,7 +1429,7 @@ mod tests { .expect("expected to insert item"); // We are adding 1 byte to the flags - let ops = vec![GroveDbOp::insert_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![b"tree".to_vec()], b"key1".to_vec(), Element::new_tree_with_flags(None, Some(vec![0, 1, 1])), @@ -930,7 +1448,10 @@ mod tests { new_flags[1] = old_flags[1]; new_flags.push(new_flags_epoch); new_flags.extend(cost.added_bytes.encode_var_vec()); - assert_eq!(new_flags, &vec![1u8, 0, 1, 1, 1]); + assert!( + new_flags == &vec![1u8, 0, 1, 1, 1] + || new_flags == &vec![1u8, 0, 1, 1, 3] + ); Ok(true) } else { assert_eq!(new_flags[0], 1); @@ -965,4 +1486,209 @@ mod tests { } ); } + + #[test] + fn test_batch_root_one_update_cost_right_above_value_required_cost_of_2_with_refresh_reference() + { + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + let tx = db.start_transaction(); + + let ops = vec![ + QualifiedGroveDbOp::insert_or_replace_op( + vec![], + b"key1".to_vec(), + Element::new_item_with_flags([0u8; 56].to_vec(), Some(vec![0, 0])), + ), + QualifiedGroveDbOp::insert_or_replace_op( + vec![], + b"keyref".to_vec(), + Element::new_reference(SiblingReference(b"key1".to_vec())), + ), + ]; + + let cost_result = db.apply_batch(ops, None, Some(&tx), grove_version); + cost_result.value.expect("expected to execute batch"); + // We are adding 2 bytes + let ops = vec![ + QualifiedGroveDbOp::insert_or_replace_op( + vec![], + b"key1".to_vec(), + Element::new_item_with_flags([0u8; 57].to_vec(), Some(vec![0, 1])), + ), + QualifiedGroveDbOp::replace_op( + vec![], + b"keyref".to_vec(), + Element::new_reference(SiblingReference(b"key1".to_vec())), + ), + ]; + + let cost = db + .apply_batch_with_element_flags_update( + ops, + None, + |cost, old_flags, new_flags| match cost.transition_type() { + OperationStorageTransitionType::OperationUpdateBiggerSize => { + if new_flags[0] == 0 { + new_flags[0] = 1; + let new_flags_epoch = new_flags[1]; + new_flags[1] = old_flags.unwrap()[1]; + new_flags.push(new_flags_epoch); + new_flags.extend(cost.added_bytes.encode_var_vec()); + // first pass will be vec![1u8, 0, 1, 2], second pass will be vec![1u8, + // 0, 1, 4] + assert!( + new_flags == &vec![1u8, 0, 1, 2] + || new_flags == &vec![1u8, 0, 1, 4] + ); + Ok(true) + } else { + assert_eq!(new_flags[0], 1); + Ok(false) + } + } + OperationStorageTransitionType::OperationUpdateSmallerSize => { + new_flags.extend(vec![1, 2]); + Ok(true) + } + _ => Ok(false), + }, + |_flags, removed_key_bytes, removed_value_bytes| { + Ok(( + BasicStorageRemoval(removed_key_bytes), + BasicStorageRemoval(removed_value_bytes), + )) + }, + Some(&tx), + grove_version, + ) + .cost; + + assert_eq!( + cost, + OperationCost { + seek_count: 7, // todo: verify this + storage_cost: StorageCost { + added_bytes: 4, + replaced_bytes: 285, // todo: verify this + removed_bytes: NoStorageRemoval + }, + storage_loaded_bytes: 380, // todo: verify this + hash_node_calls: 12, // todo: verify this + } + ); + + let issues = db + .visualize_verify_grovedb(Some(&tx), true, false, &Default::default()) + .unwrap(); + assert_eq!( + issues.len(), + 0, + "reference issue: {}", + issues + .iter() + .map(|(hash, (a, b, c))| format!("{}: {} {} {}", hash, a, b, c)) + .collect::>() + .join(" | ") + ); + } + + #[test] + fn test_batch_root_one_update_cost_right_above_value_required_cost_of_2_with_insert_reference() + { + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + let tx = db.start_transaction(); + + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( + vec![], + b"key1".to_vec(), + Element::new_item_with_flags([0u8; 56].to_vec(), Some(vec![0, 0])), + )]; + + let cost_result = db.apply_batch(ops, None, Some(&tx), grove_version); + cost_result.value.expect("expected to execute batch"); + // We are adding 2 bytes + let ops = vec![ + QualifiedGroveDbOp::insert_or_replace_op( + vec![], + b"key1".to_vec(), + Element::new_item_with_flags([0u8; 57].to_vec(), Some(vec![0, 1])), + ), + QualifiedGroveDbOp::insert_only_op( + vec![], + b"keyref".to_vec(), + Element::new_reference(SiblingReference(b"key1".to_vec())), + ), + ]; + + let cost = db + .apply_batch_with_element_flags_update( + ops, + None, + |cost, old_flags, new_flags| match cost.transition_type() { + OperationStorageTransitionType::OperationUpdateBiggerSize => { + if new_flags[0] == 0 { + new_flags[0] = 1; + let new_flags_epoch = new_flags[1]; + new_flags[1] = old_flags.unwrap()[1]; + new_flags.push(new_flags_epoch); + new_flags.extend(cost.added_bytes.encode_var_vec()); + // first pass will be vec![1u8, 0, 1, 2], second pass will be vec![1u8, + // 0, 1, 4] + assert!( + new_flags == &vec![1u8, 0, 1, 2] + || new_flags == &vec![1u8, 0, 1, 4] + ); + Ok(true) + } else { + assert_eq!(new_flags[0], 1); + Ok(false) + } + } + OperationStorageTransitionType::OperationUpdateSmallerSize => { + new_flags.extend(vec![1, 2]); + Ok(true) + } + _ => Ok(false), + }, + |_flags, removed_key_bytes, removed_value_bytes| { + Ok(( + BasicStorageRemoval(removed_key_bytes), + BasicStorageRemoval(removed_value_bytes), + )) + }, + Some(&tx), + grove_version, + ) + .cost; + + assert_eq!( + cost, + OperationCost { + seek_count: 5, // todo: verify this + storage_cost: StorageCost { + added_bytes: 160, + replaced_bytes: 168, // todo: verify this + removed_bytes: NoStorageRemoval + }, + storage_loaded_bytes: 133, // todo: verify this + hash_node_calls: 12, // todo: verify this + } + ); + + let issues = db + .visualize_verify_grovedb(Some(&tx), true, false, &Default::default()) + .unwrap(); + assert_eq!( + issues.len(), + 0, + "reference issue: {}", + issues + .iter() + .map(|(hash, (a, b, c))| format!("{}: {} {} {}", hash, a, b, c)) + .collect::>() + .join(" | ") + ); + } } diff --git a/grovedb/src/batch/single_sum_item_deletion_cost_tests.rs b/grovedb/src/batch/single_sum_item_deletion_cost_tests.rs index bf5637d0..c0be9a08 100644 --- a/grovedb/src/batch/single_sum_item_deletion_cost_tests.rs +++ b/grovedb/src/batch/single_sum_item_deletion_cost_tests.rs @@ -5,7 +5,7 @@ mod tests { use grovedb_version::version::GroveVersion; use crate::{ - batch::GroveDbOp, + batch::QualifiedGroveDbOp, tests::{common::EMPTY_PATH, make_empty_grovedb}, Element, }; @@ -43,7 +43,11 @@ mod tests { ); tx.rollback().expect("expected to rollback"); - let ops = vec![GroveDbOp::delete_tree_op(vec![], b"key1".to_vec(), false)]; + let ops = vec![QualifiedGroveDbOp::delete_tree_op( + vec![], + b"key1".to_vec(), + false, + )]; let batch_cost = db .apply_batch(ops, None, Some(&tx), grove_version) .cost_as_result() @@ -101,7 +105,7 @@ mod tests { ); tx.rollback().expect("expected to rollback"); - let ops = vec![GroveDbOp::delete_op( + let ops = vec![QualifiedGroveDbOp::delete_op( vec![b"sum_tree".to_vec()], b"key1".to_vec(), )]; @@ -146,7 +150,11 @@ mod tests { ); tx.rollback().expect("expected to rollback"); - let ops = vec![GroveDbOp::delete_tree_op(vec![], b"key1".to_vec(), false)]; + let ops = vec![QualifiedGroveDbOp::delete_tree_op( + vec![], + b"key1".to_vec(), + false, + )]; let batch_cost = db .apply_batch(ops, None, Some(&tx), grove_version) .cost_as_result() diff --git a/grovedb/src/batch/single_sum_item_insert_cost_tests.rs b/grovedb/src/batch/single_sum_item_insert_cost_tests.rs index 0ba3da44..d58e7327 100644 --- a/grovedb/src/batch/single_sum_item_insert_cost_tests.rs +++ b/grovedb/src/batch/single_sum_item_insert_cost_tests.rs @@ -9,7 +9,7 @@ mod tests { use grovedb_version::version::GroveVersion; use crate::{ - batch::GroveDbOp, + batch::QualifiedGroveDbOp, tests::{common::EMPTY_PATH, make_empty_grovedb}, Element, }; @@ -42,7 +42,7 @@ mod tests { ) .cost; tx.rollback().expect("expected to rollback"); - let ops = vec![GroveDbOp::insert_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![b"sum_tree".to_vec()], b"key1".to_vec(), Element::new_sum_item(150), @@ -57,7 +57,7 @@ mod tests { let db = make_empty_grovedb(); let tx = db.start_transaction(); - let ops = vec![GroveDbOp::insert_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![], b"key1".to_vec(), Element::empty_sum_tree(), @@ -134,7 +134,7 @@ mod tests { .unwrap() .expect("successful root tree leaf insert"); - let ops = vec![GroveDbOp::insert_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![], b"key1".to_vec(), Element::empty_sum_tree(), @@ -220,7 +220,7 @@ mod tests { .unwrap() .expect("successful root tree leaf insert"); - let ops = vec![GroveDbOp::insert_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![], b"key1".to_vec(), Element::empty_sum_tree(), @@ -306,7 +306,7 @@ mod tests { .unwrap() .expect("successful root tree leaf insert"); - let ops = vec![GroveDbOp::insert_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![b"0".to_vec()], b"key1".to_vec(), Element::empty_sum_tree(), @@ -396,7 +396,7 @@ mod tests { .unwrap() .expect("successful root tree leaf insert"); - let ops = vec![GroveDbOp::insert_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![b"0".to_vec()], b"key1".to_vec(), Element::empty_sum_tree(), @@ -487,7 +487,7 @@ mod tests { .unwrap() .expect("expected to insert sum tree"); - let ops = vec![GroveDbOp::insert_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![b"sum_tree".to_vec()], b"key1".to_vec(), Element::new_sum_item_with_flags(15, Some([0; 42].to_vec())), @@ -562,7 +562,7 @@ mod tests { .unwrap() .expect("expected to insert sum tree"); - let ops = vec![GroveDbOp::insert_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![b"sum_tree".to_vec()], b"key1".to_vec(), Element::new_sum_item_with_flags(15, Some([0; 43].to_vec())), @@ -648,7 +648,7 @@ mod tests { .expect("expected to insert item"); // We are adding 2 bytes - let ops = vec![GroveDbOp::insert_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![b"tree".to_vec()], b"key1".to_vec(), Element::new_sum_item_with_flags(100000, None), @@ -714,7 +714,7 @@ mod tests { .expect("expected to insert item"); // We are adding 2 bytes - let ops = vec![GroveDbOp::insert_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![b"tree".to_vec()], b"key1".to_vec(), Element::new_sum_item_with_flags(100000, Some(vec![1])), @@ -780,7 +780,7 @@ mod tests { .expect("expected to insert item"); // We are adding 2 bytes - let ops = vec![GroveDbOp::insert_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![b"tree".to_vec()], b"key1".to_vec(), Element::new_sum_item_with_flags(5, None), @@ -846,7 +846,7 @@ mod tests { .expect("expected to insert item"); // We are adding 2 bytes - let ops = vec![GroveDbOp::insert_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![b"tree".to_vec()], b"key1".to_vec(), Element::new_sum_item_with_flags(5, Some(vec![1])), diff --git a/grovedb/src/debugger.rs b/grovedb/src/debugger.rs index e3556acd..da4657ec 100644 --- a/grovedb/src/debugger.rs +++ b/grovedb/src/debugger.rs @@ -23,12 +23,12 @@ use tower_http::services::ServeDir; use crate::{ operations::proof::{GroveDBProof, LayerProof, ProveOptions}, + query_result_type::{QueryResultElement, QueryResultElements, QueryResultType}, reference_path::ReferencePathType, GroveDb, }; -const GROVEDBG_ZIP: [u8; include_bytes!(concat!(env!("OUT_DIR"), "/grovedbg.zip")).len()] = - *include_bytes!(concat!(env!("OUT_DIR"), "/grovedbg.zip")); +const GROVEDBG_ZIP: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/grovedbg.zip")); pub(super) fn start_visualizer(grovedb: Weak, addr: A) where @@ -40,7 +40,7 @@ where let grovedbg_zip = grovedbg_tmp.path().join("grovedbg.zip"); let grovedbg_www = grovedbg_tmp.path().join("grovedbg_www"); - fs::write(&grovedbg_zip, &GROVEDBG_ZIP).expect("cannot crate grovedbg.zip"); + fs::write(&grovedbg_zip, GROVEDBG_ZIP).expect("cannot crate grovedbg.zip"); zip_extensions::read::zip_extract(&grovedbg_zip, &grovedbg_www) .expect("cannot extract grovedbg contents"); @@ -48,7 +48,8 @@ where let app = Router::new() .route("/fetch_node", post(fetch_node)) .route("/fetch_root_node", post(fetch_root_node)) - .route("/execute_path_query", post(execute_path_query)) + .route("/prove_path_query", post(prove_path_query)) + .route("/fetch_with_path_query", post(fetch_with_path_query)) .fallback_service(ServeDir::new(grovedbg_www)) .with_state((shutdown_send, grovedb)); @@ -136,7 +137,7 @@ async fn fetch_root_node( } } -async fn execute_path_query( +async fn prove_path_query( State((shutdown, grovedb)): State<(Sender<()>, Weak)>, Json(json_path_query): Json, ) -> Result, AppError> { @@ -153,6 +154,66 @@ async fn execute_path_query( Ok(Json(proof_to_grovedbg(grovedb_proof)?)) } +async fn fetch_with_path_query( + State((shutdown, grovedb)): State<(Sender<()>, Weak)>, + Json(json_path_query): Json, +) -> Result>, AppError> { + let Some(db) = grovedb.upgrade() else { + shutdown.send(()).await.ok(); + return Err(AppError::Closed); + }; + + let path_query = path_query_to_grovedb(json_path_query); + + let grovedb_query_result = db + .query_raw( + &path_query, + false, + true, + false, + QueryResultType::QueryPathKeyElementTrioResultType, + None, + GroveVersion::latest(), + ) + .unwrap()? + .0; + Ok(Json(query_result_to_grovedbg(&db, grovedb_query_result)?)) +} + +fn query_result_to_grovedbg( + db: &GroveDb, + query_result: QueryResultElements, +) -> Result, crate::Error> { + let mut result = Vec::new(); + + let mut last_merk: Option<(Vec>, grovedb_merk::Merk<_>)> = None; + + for qr in query_result.elements.into_iter() { + if let QueryResultElement::PathKeyElementTrioResultItem((path, key, _)) = qr { + let merk: &grovedb_merk::Merk<_> = match &mut last_merk { + Some((last_merk_path, last_merk)) if last_merk_path == &path => last_merk, + _ => { + last_merk = Some(( + path.clone(), + db.open_non_transactional_merk_at_path( + path.as_slice().into(), + None, + GroveVersion::latest(), + ) + .unwrap()?, + )); + &last_merk.as_ref().unwrap().1 + } + }; + + if let Some(node) = merk.get_node_dbg(&key)? { + result.push(node_to_update(path, node)?); + } + } + } + Ok(result) +} + fn proof_to_grovedbg(proof: GroveDBProof) -> Result { match proof { GroveDBProof::V0(p) => Ok(grovedbg_types::Proof { @@ -334,19 +395,21 @@ fn element_to_grovedbg(element: crate::Element) -> grovedbg_types::Element { ReferencePathType::AbsolutePathReference(path), _, element_flags, - ) => grovedbg_types::Element::AbsolutePathReference { + ) => grovedbg_types::Element::Reference(grovedbg_types::Reference::AbsolutePathReference { path, element_flags, - }, + }), crate::Element::Reference( ReferencePathType::UpstreamRootHeightReference(n_keep, path_append), _, element_flags, - ) => grovedbg_types::Element::UpstreamRootHeightReference { - n_keep: n_keep.into(), - path_append, - element_flags, - }, + ) => grovedbg_types::Element::Reference( + grovedbg_types::Reference::UpstreamRootHeightReference { + n_keep: n_keep.into(), + path_append, + element_flags, + }, + ), crate::Element::Reference( ReferencePathType::UpstreamRootHeightWithParentPathAdditionReference( n_keep, @@ -354,44 +417,50 @@ fn element_to_grovedbg(element: crate::Element) -> grovedbg_types::Element { ), _, element_flags, - ) => grovedbg_types::Element::UpstreamRootHeightWithParentPathAdditionReference { - n_keep: n_keep.into(), - path_append, - element_flags, - }, + ) => grovedbg_types::Element::Reference( + grovedbg_types::Reference::UpstreamRootHeightWithParentPathAdditionReference { + n_keep: n_keep.into(), + path_append, + element_flags, + }, + ), crate::Element::Reference( ReferencePathType::UpstreamFromElementHeightReference(n_remove, path_append), _, element_flags, - ) => grovedbg_types::Element::UpstreamFromElementHeightReference { - n_remove: n_remove.into(), - path_append, - element_flags, - }, + ) => grovedbg_types::Element::Reference( + grovedbg_types::Reference::UpstreamFromElementHeightReference { + n_remove: n_remove.into(), + path_append, + element_flags, + }, + ), crate::Element::Reference( ReferencePathType::CousinReference(swap_parent), _, element_flags, - ) => grovedbg_types::Element::CousinReference { + ) => grovedbg_types::Element::Reference(grovedbg_types::Reference::CousinReference { swap_parent, element_flags, - }, + }), crate::Element::Reference( ReferencePathType::RemovedCousinReference(swap_parent), _, element_flags, - ) => grovedbg_types::Element::RemovedCousinReference { - swap_parent, - element_flags, - }, + ) => { + grovedbg_types::Element::Reference(grovedbg_types::Reference::RemovedCousinReference { + swap_parent, + element_flags, + }) + } crate::Element::Reference( ReferencePathType::SiblingReference(sibling_key), _, element_flags, - ) => grovedbg_types::Element::SiblingReference { + ) => grovedbg_types::Element::Reference(grovedbg_types::Reference::SiblingReference { sibling_key, element_flags, - }, + }), crate::Element::SumItem(value, element_flags) => grovedbg_types::Element::SumItem { value, element_flags, @@ -410,7 +479,9 @@ fn node_to_update( key, value, left_child, + left_merk_hash, right_child, + right_merk_hash, value_hash, kv_digest_hash, feature_type, @@ -426,7 +497,9 @@ fn node_to_update( key, element, left_child, + left_merk_hash, right_child, + right_merk_hash, feature_type: match feature_type { TreeFeatureType::BasicMerkNode => grovedbg_types::TreeFeatureType::BasicMerkNode, TreeFeatureType::SummedMerkNode(x) => { diff --git a/grovedb/src/element/delete.rs b/grovedb/src/element/delete.rs index 9c0879a7..ced24e27 100644 --- a/grovedb/src/element/delete.rs +++ b/grovedb/src/element/delete.rs @@ -95,6 +95,7 @@ impl Element { .map_err(|e| MerkError::ClientCorruptionError(e.to_string())) }, Some(&Element::value_defined_cost_for_serialized_value), + &|_, _| Ok(None), &mut |_costs, _old_value, _value| Ok((false, None)), sectioned_removal, grove_version, diff --git a/grovedb/src/element/get.rs b/grovedb/src/element/get.rs index 1fda91dd..6ec625e7 100644 --- a/grovedb/src/element/get.rs +++ b/grovedb/src/element/get.rs @@ -173,11 +173,7 @@ impl Element { }); let value_len = cost_size + flags_len; cost.storage_loaded_bytes = - KV::specialized_value_byte_cost_size_for_key_and_value_lengths( - key_ref.len() as u32, - value_len, - false, - ) + KV::node_value_byte_cost_size(key_ref.len() as u32, value_len, false) } Some(Element::Tree(_, flags)) | Some(Element::SumTree(_, _, flags)) => { let tree_cost_size = if element.as_ref().unwrap().is_sum_tree() { diff --git a/grovedb/src/element/helpers.rs b/grovedb/src/element/helpers.rs index 2d2db076..5b3662df 100644 --- a/grovedb/src/element/helpers.rs +++ b/grovedb/src/element/helpers.rs @@ -12,6 +12,7 @@ use grovedb_merk::{ TreeFeatureType, TreeFeatureType::{BasicMerkNode, SummedMerkNode}, }; +#[cfg(feature = "full")] use grovedb_version::{check_grovedb_v0, error::GroveVersionError, version::GroveVersion}; #[cfg(feature = "full")] use integer_encoding::VarInt; @@ -189,6 +190,18 @@ impl Element { } } + #[cfg(feature = "full")] + /// Sets the optional flag stored in an element + pub fn set_flags(&mut self, new_flags: Option) { + match self { + Element::Tree(_, flags) + | Element::Item(_, flags) + | Element::Reference(_, _, flags) + | Element::SumTree(.., flags) + | Element::SumItem(_, flags) => *flags = new_flags, + } + } + #[cfg(feature = "full")] /// Get the required item space pub fn required_item_space( @@ -285,17 +298,9 @@ impl Element { }); let value_len = SUM_ITEM_COST_SIZE + flags_len; let key_len = key.len() as u32; - KV::specialized_value_byte_cost_size_for_key_and_value_lengths( - key_len, - value_len, - is_sum_node, - ) + KV::node_value_byte_cost_size(key_len, value_len, is_sum_node) } - _ => KV::specialized_value_byte_cost_size_for_key_and_value_lengths( - key.len() as u32, - value.len() as u32, - is_sum_node, - ), + _ => KV::node_value_byte_cost_size(key.len() as u32, value.len() as u32, is_sum_node), }; Ok(cost) } diff --git a/grovedb/src/element/mod.rs b/grovedb/src/element/mod.rs index ad2540e3..b1667ffb 100644 --- a/grovedb/src/element/mod.rs +++ b/grovedb/src/element/mod.rs @@ -34,6 +34,8 @@ use grovedb_visualize::visualize_to_vec; use crate::operations::proof::util::hex_to_ascii; #[cfg(any(feature = "full", feature = "verify"))] use crate::reference_path::ReferencePathType; +#[cfg(feature = "full")] +use crate::OperationCost; #[cfg(any(feature = "full", feature = "verify"))] /// Optional meta-data to be stored per element @@ -153,6 +155,15 @@ impl Element { Element::SumTree(..) => "sum tree", } } + + #[cfg(feature = "full")] + pub(crate) fn value_hash( + &self, + grove_version: &grovedb_version::version::GroveVersion, + ) -> grovedb_costs::CostResult { + let bytes = grovedb_costs::cost_return_on_error_default!(self.serialize(grove_version)); + crate::value_hash(&bytes).map(Result::Ok) + } } #[cfg(any(feature = "full", feature = "visualize"))] diff --git a/grovedb/src/element/query.rs b/grovedb/src/element/query.rs index 39c0494c..7226db55 100644 --- a/grovedb/src/element/query.rs +++ b/grovedb/src/element/query.rs @@ -12,18 +12,21 @@ use grovedb_costs::{ use grovedb_merk::proofs::query::query_item::QueryItem; #[cfg(feature = "full")] use grovedb_merk::proofs::query::SubqueryBranch; -#[cfg(any(feature = "full", feature = "verify"))] +#[cfg(feature = "full")] use grovedb_merk::proofs::Query; #[cfg(feature = "full")] use grovedb_path::SubtreePath; #[cfg(feature = "full")] use grovedb_storage::{rocksdb_storage::RocksDbStorage, RawIterator, StorageContext}; +#[cfg(feature = "full")] use grovedb_version::{ check_grovedb_v0, check_grovedb_v0_with_cost, error::GroveVersionError, version::GroveVersion, }; #[cfg(feature = "full")] use crate::operations::proof::util::hex_to_ascii; +#[cfg(any(feature = "full", feature = "verify"))] +use crate::Element; #[cfg(feature = "full")] use crate::{ element::helpers::raw_decode, @@ -37,8 +40,8 @@ use crate::{ util::{merk_optional_tx, merk_optional_tx_internal_error, storage_context_optional_tx}, Error, PathQuery, TransactionArg, }; -#[cfg(any(feature = "full", feature = "verify"))] -use crate::{query_result_type::Path, Element, SizedQuery}; +#[cfg(feature = "full")] +use crate::{query_result_type::Path, SizedQuery}; #[cfg(any(feature = "full", feature = "verify"))] #[derive(Copy, Clone, Debug)] @@ -681,7 +684,7 @@ impl Element { Ok(()).wrap_with_cost(cost) } - #[cfg(any(feature = "full", feature = "verify"))] + #[cfg(feature = "full")] /// Takes a sized query and a key and returns subquery key and subquery as /// tuple fn subquery_paths_and_value_for_sized_query( diff --git a/grovedb/src/error.rs b/grovedb/src/error.rs index 0f6cd5d1..92343935 100644 --- a/grovedb/src/error.rs +++ b/grovedb/src/error.rs @@ -152,6 +152,10 @@ pub enum Error { #[error(transparent)] /// Version error VersionError(grovedb_version::error::GroveVersionError), + + #[error("cyclic error")] + /// Cyclic reference + CyclicError(&'static str), } impl From for Error { diff --git a/grovedb/src/lib.rs b/grovedb/src/lib.rs index 012785ca..57f68d33 100644 --- a/grovedb/src/lib.rs +++ b/grovedb/src/lib.rs @@ -199,11 +199,14 @@ use grovedb_storage::{ }; #[cfg(feature = "full")] use grovedb_storage::{Storage, StorageContext}; +#[cfg(feature = "full")] use grovedb_version::version::GroveVersion; #[cfg(feature = "full")] use grovedb_visualize::DebugByteVectors; #[cfg(any(feature = "full", feature = "verify"))] pub use query::{PathQuery, SizedQuery}; +#[cfg(feature = "full")] +use reference_path::path_from_reference_path_type; #[cfg(feature = "grovedbg")] use tokio::net::ToSocketAddrs; @@ -212,6 +215,8 @@ use crate::element::helpers::raw_decode; #[cfg(any(feature = "full", feature = "verify"))] pub use crate::error::Error; #[cfg(feature = "full")] +use crate::operations::proof::util::hex_to_ascii; +#[cfg(feature = "full")] use crate::util::{root_merk_optional_tx, storage_context_optional_tx}; #[cfg(feature = "full")] use crate::Error::MerkError; @@ -898,10 +903,13 @@ impl GroveDb { /// Method to visualize hash mismatch after verification pub fn visualize_verify_grovedb( &self, + transaction: TransactionArg, + verify_references: bool, + allow_cache: bool, grove_version: &GroveVersion, ) -> Result, Error> { Ok(self - .verify_grovedb(None, grove_version)? + .verify_grovedb(transaction, verify_references, allow_cache, grove_version)? .iter() .map(|(path, (root_hash, expected, actual))| { ( @@ -924,6 +932,8 @@ impl GroveDb { pub fn verify_grovedb( &self, transaction: TransactionArg, + verify_references: bool, + allow_cache: bool, grove_version: &GroveVersion, ) -> Result>, (CryptoHash, CryptoHash, CryptoHash)>, Error> { if let Some(transaction) = transaction { @@ -940,13 +950,22 @@ impl GroveDb { &SubtreePath::empty(), None, transaction, + verify_references, + allow_cache, grove_version, ) } else { let root_merk = self .open_non_transactional_merk_at_path(SubtreePath::empty(), None, grove_version) .unwrap()?; - self.verify_merk_and_submerks(root_merk, &SubtreePath::empty(), None, grove_version) + self.verify_merk_and_submerks( + root_merk, + &SubtreePath::empty(), + None, + verify_references, + allow_cache, + grove_version, + ) } } @@ -957,72 +976,138 @@ impl GroveDb { merk: Merk, path: &SubtreePath, batch: Option<&'db StorageBatch>, + verify_references: bool, + allow_cache: bool, grove_version: &GroveVersion, ) -> Result>, (CryptoHash, CryptoHash, CryptoHash)>, Error> { let mut all_query = Query::new(); all_query.insert_all(); - let _in_sum_tree = merk.is_sum_tree; let mut issues = HashMap::new(); let mut element_iterator = KVIterator::new(merk.storage.raw_iter(), &all_query).unwrap(); while let Some((key, element_value)) = element_iterator.next_kv().unwrap() { let element = raw_decode(&element_value, grove_version)?; - if element.is_any_tree() { - let (kv_value, element_value_hash) = merk - .get_value_and_value_hash( - &key, + match element { + Element::SumTree(..) | Element::Tree(..) => { + let (kv_value, element_value_hash) = merk + .get_value_and_value_hash( + &key, + allow_cache, + None::<&fn(&[u8], &GroveVersion) -> Option>, + grove_version, + ) + .unwrap() + .map_err(MerkError)? + .ok_or(Error::CorruptedData(format!( + "expected merk to contain value at key {} for {}", + hex_to_ascii(&key), + element.type_str() + )))?; + let new_path = path.derive_owned_with_child(key); + let new_path_ref = SubtreePath::from(&new_path); + + let inner_merk = self + .open_non_transactional_merk_at_path( + new_path_ref.clone(), + batch, + grove_version, + ) + .unwrap()?; + let root_hash = inner_merk.root_hash().unwrap(); + + let actual_value_hash = value_hash(&kv_value).unwrap(); + let combined_value_hash = combine_hash(&actual_value_hash, &root_hash).unwrap(); + + if combined_value_hash != element_value_hash { + issues.insert( + new_path.to_vec(), + (root_hash, combined_value_hash, element_value_hash), + ); + } + issues.extend(self.verify_merk_and_submerks( + inner_merk, + &new_path_ref, + batch, + verify_references, true, - None::<&fn(&[u8], &GroveVersion) -> Option>, grove_version, - ) - .unwrap() - .map_err(MerkError)? - .ok_or(Error::CorruptedData( - "expected merk to contain value at key".to_string(), - ))?; - let new_path = path.derive_owned_with_child(key); - let new_path_ref = SubtreePath::from(&new_path); - - let inner_merk = self - .open_non_transactional_merk_at_path(new_path_ref.clone(), batch, grove_version) - .unwrap()?; - let root_hash = inner_merk.root_hash().unwrap(); - - let actual_value_hash = value_hash(&kv_value).unwrap(); - let combined_value_hash = combine_hash(&actual_value_hash, &root_hash).unwrap(); - - if combined_value_hash != element_value_hash { - issues.insert( - new_path.to_vec(), - (root_hash, combined_value_hash, element_value_hash), - ); + )?); } - issues.extend(self.verify_merk_and_submerks( - inner_merk, - &new_path_ref, - batch, - grove_version, - )?); - } else if element.is_any_item() { - let (kv_value, element_value_hash) = merk - .get_value_and_value_hash( - &key, - true, - None::<&fn(&[u8], &GroveVersion) -> Option>, - grove_version, - ) - .unwrap() - .map_err(MerkError)? - .ok_or(Error::CorruptedData( - "expected merk to contain value at key".to_string(), - ))?; - let actual_value_hash = value_hash(&kv_value).unwrap(); - if actual_value_hash != element_value_hash { - issues.insert( - path.derive_owned_with_child(key).to_vec(), - (actual_value_hash, element_value_hash, actual_value_hash), - ); + Element::Item(..) | Element::SumItem(..) => { + let (kv_value, element_value_hash) = merk + .get_value_and_value_hash( + &key, + allow_cache, + None::<&fn(&[u8], &GroveVersion) -> Option>, + grove_version, + ) + .unwrap() + .map_err(MerkError)? + .ok_or(Error::CorruptedData(format!( + "expected merk to contain value at key {} for {}", + hex_to_ascii(&key), + element.type_str() + )))?; + let actual_value_hash = value_hash(&kv_value).unwrap(); + if actual_value_hash != element_value_hash { + issues.insert( + path.derive_owned_with_child(key).to_vec(), + (actual_value_hash, element_value_hash, actual_value_hash), + ); + } + } + Element::Reference(ref reference_path, ..) => { + // Skip this whole check if we don't `verify_references` + if !verify_references { + continue; + } + + // Merk we're checking: + let (kv_value, element_value_hash) = merk + .get_value_and_value_hash( + &key, + allow_cache, + None::<&fn(&[u8], &GroveVersion) -> Option>, + grove_version, + ) + .unwrap() + .map_err(MerkError)? + .ok_or(Error::CorruptedData(format!( + "expected merk to contain value at key {} for reference", + hex_to_ascii(&key) + )))?; + + let referenced_value_hash = { + let full_path = path_from_reference_path_type( + reference_path.clone(), + &path.to_vec(), + Some(&key), + )?; + let item = self + .follow_reference( + (full_path.as_slice()).into(), + allow_cache, + None, + grove_version, + ) + .unwrap()?; + item.value_hash(grove_version).unwrap()? + }; + + // Take the current item (reference) hash and combine it with referenced value's + // hash + + let self_actual_value_hash = value_hash(&kv_value).unwrap(); + let combined_value_hash = + combine_hash(&self_actual_value_hash, &referenced_value_hash).unwrap(); + + if combined_value_hash != element_value_hash { + issues.insert( + path.derive_owned_with_child(key).to_vec(), + (combined_value_hash, element_value_hash, combined_value_hash), + ); + } } } } @@ -1035,78 +1120,139 @@ impl GroveDb { path: &SubtreePath, batch: Option<&'db StorageBatch>, transaction: &Transaction, + verify_references: bool, + allow_cache: bool, grove_version: &GroveVersion, ) -> Result>, (CryptoHash, CryptoHash, CryptoHash)>, Error> { let mut all_query = Query::new(); all_query.insert_all(); - let _in_sum_tree = merk.is_sum_tree; let mut issues = HashMap::new(); let mut element_iterator = KVIterator::new(merk.storage.raw_iter(), &all_query).unwrap(); while let Some((key, element_value)) = element_iterator.next_kv().unwrap() { let element = raw_decode(&element_value, grove_version)?; - if element.is_any_tree() { - let (kv_value, element_value_hash) = merk - .get_value_and_value_hash( - &key, - true, - None::<&fn(&[u8], &GroveVersion) -> Option>, - grove_version, - ) - .unwrap() - .map_err(MerkError)? - .ok_or(Error::CorruptedData( - "expected merk to contain value at key".to_string(), - ))?; - let new_path = path.derive_owned_with_child(key); - let new_path_ref = SubtreePath::from(&new_path); - - let inner_merk = self - .open_transactional_merk_at_path( - new_path_ref.clone(), - transaction, - batch, - grove_version, - ) - .unwrap()?; - let root_hash = inner_merk.root_hash().unwrap(); + match element { + Element::SumTree(..) | Element::Tree(..) => { + let (kv_value, element_value_hash) = merk + .get_value_and_value_hash( + &key, + allow_cache, + None::<&fn(&[u8], &GroveVersion) -> Option>, + grove_version, + ) + .unwrap() + .map_err(MerkError)? + .ok_or(Error::CorruptedData(format!( + "expected merk to contain value at key {} for {}", + hex_to_ascii(&key), + element.type_str() + )))?; + let new_path = path.derive_owned_with_child(key); + let new_path_ref = SubtreePath::from(&new_path); - let actual_value_hash = value_hash(&kv_value).unwrap(); - let combined_value_hash = combine_hash(&actual_value_hash, &root_hash).unwrap(); + let inner_merk = self + .open_transactional_merk_at_path( + new_path_ref.clone(), + transaction, + batch, + grove_version, + ) + .unwrap()?; + let root_hash = inner_merk.root_hash().unwrap(); - if combined_value_hash != element_value_hash { - issues.insert( - new_path.to_vec(), - (root_hash, combined_value_hash, element_value_hash), - ); - } - issues.extend(self.verify_merk_and_submerks_in_transaction( - inner_merk, - &new_path_ref, - batch, - transaction, - grove_version, - )?); - } else if element.is_any_item() { - let (kv_value, element_value_hash) = merk - .get_value_and_value_hash( - &key, + let actual_value_hash = value_hash(&kv_value).unwrap(); + let combined_value_hash = combine_hash(&actual_value_hash, &root_hash).unwrap(); + + if combined_value_hash != element_value_hash { + issues.insert( + new_path.to_vec(), + (root_hash, combined_value_hash, element_value_hash), + ); + } + issues.extend(self.verify_merk_and_submerks_in_transaction( + inner_merk, + &new_path_ref, + batch, + transaction, + verify_references, true, - None::<&fn(&[u8], &GroveVersion) -> Option>, grove_version, - ) - .unwrap() - .map_err(MerkError)? - .ok_or(Error::CorruptedData( - "expected merk to contain value at key".to_string(), - ))?; - let actual_value_hash = value_hash(&kv_value).unwrap(); - if actual_value_hash != element_value_hash { - issues.insert( - path.derive_owned_with_child(key).to_vec(), - (actual_value_hash, element_value_hash, actual_value_hash), - ); + )?); + } + Element::Item(..) | Element::SumItem(..) => { + let (kv_value, element_value_hash) = merk + .get_value_and_value_hash( + &key, + allow_cache, + None::<&fn(&[u8], &GroveVersion) -> Option>, + grove_version, + ) + .unwrap() + .map_err(MerkError)? + .ok_or(Error::CorruptedData(format!( + "expected merk to contain value at key {} for {}", + hex_to_ascii(&key), + element.type_str() + )))?; + let actual_value_hash = value_hash(&kv_value).unwrap(); + if actual_value_hash != element_value_hash { + issues.insert( + path.derive_owned_with_child(key).to_vec(), + (actual_value_hash, element_value_hash, actual_value_hash), + ); + } + } + Element::Reference(ref reference_path, ..) => { + // Skip this whole check if we don't `verify_references` + if !verify_references { + continue; + } + + // Merk we're checking: + let (kv_value, element_value_hash) = merk + .get_value_and_value_hash( + &key, + allow_cache, + None::<&fn(&[u8], &GroveVersion) -> Option>, + grove_version, + ) + .unwrap() + .map_err(MerkError)? + .ok_or(Error::CorruptedData(format!( + "expected merk to contain value at key {} for reference", + hex_to_ascii(&key) + )))?; + + let referenced_value_hash = { + let full_path = path_from_reference_path_type( + reference_path.clone(), + &path.to_vec(), + Some(&key), + )?; + let item = self + .follow_reference( + (full_path.as_slice()).into(), + allow_cache, + Some(transaction), + grove_version, + ) + .unwrap()?; + item.value_hash(grove_version).unwrap()? + }; + + // Take the current item (reference) hash and combine it with referenced value's + // hash + let self_actual_value_hash = value_hash(&kv_value).unwrap(); + let combined_value_hash = + combine_hash(&self_actual_value_hash, &referenced_value_hash).unwrap(); + + if combined_value_hash != element_value_hash { + issues.insert( + path.derive_owned_with_child(key).to_vec(), + (combined_value_hash, element_value_hash, combined_value_hash), + ); + } } } } diff --git a/grovedb/src/operations/delete/average_case.rs b/grovedb/src/operations/delete/average_case.rs index 3ed1abd1..986f2b90 100644 --- a/grovedb/src/operations/delete/average_case.rs +++ b/grovedb/src/operations/delete/average_case.rs @@ -17,7 +17,7 @@ use grovedb_version::{ use intmap::IntMap; use crate::{ - batch::{key_info::KeyInfo, GroveDbOp, KeyInfoPath}, + batch::{key_info::KeyInfo, KeyInfoPath, QualifiedGroveDbOp}, Error, GroveDb, }; @@ -34,7 +34,7 @@ impl GroveDb { validate: bool, estimated_layer_info: IntMap, grove_version: &GroveVersion, - ) -> CostResult, Error> { + ) -> CostResult, Error> { check_grovedb_v0_with_cost!( "average_case_delete_operations_for_delete_up_tree_while_empty", grove_version @@ -145,7 +145,7 @@ impl GroveDb { except_keys_count: u16, estimated_key_element_size: EstimatedKeyAndElementSize, grove_version: &GroveVersion, - ) -> CostResult { + ) -> CostResult { check_grovedb_v0_with_cost!( "average_case_delete_operation_for_delete", grove_version @@ -188,6 +188,10 @@ impl GroveDb { estimated_key_element_size.0 + HASH_LENGTH_U32, ); - Ok(GroveDbOp::delete_estimated_op(path.clone(), key.clone())).wrap_with_cost(cost) + Ok(QualifiedGroveDbOp::delete_estimated_op( + path.clone(), + key.clone(), + )) + .wrap_with_cost(cost) } } diff --git a/grovedb/src/operations/delete/delete_up_tree.rs b/grovedb/src/operations/delete/delete_up_tree.rs index dd331b69..7ecfce83 100644 --- a/grovedb/src/operations/delete/delete_up_tree.rs +++ b/grovedb/src/operations/delete/delete_up_tree.rs @@ -11,7 +11,7 @@ use grovedb_version::{ }; use crate::{ - batch::GroveDbOp, operations::delete::DeleteOptions, ElementFlags, Error, GroveDb, + batch::QualifiedGroveDbOp, operations::delete::DeleteOptions, ElementFlags, Error, GroveDb, TransactionArg, }; @@ -122,7 +122,7 @@ impl GroveDb { .delete_up_tree_while_empty_with_sectional_storage ); let mut cost = OperationCost::default(); - let mut batch_operations: Vec = Vec::new(); + let mut batch_operations: Vec = Vec::new(); let maybe_ops = cost_return_on_error!( &mut cost, @@ -170,10 +170,10 @@ impl GroveDb { key: &[u8], options: &DeleteUpTreeOptions, is_known_to_be_subtree_with_sum: Option<(bool, bool)>, - mut current_batch_operations: Vec, + mut current_batch_operations: Vec, transaction: TransactionArg, grove_version: &GroveVersion, - ) -> CostResult, Error> { + ) -> CostResult, Error> { check_grovedb_v0_with_cost!( "delete", grove_version @@ -202,10 +202,10 @@ impl GroveDb { key: &[u8], options: &DeleteUpTreeOptions, is_known_to_be_subtree_with_sum: Option<(bool, bool)>, - current_batch_operations: &mut Vec, + current_batch_operations: &mut Vec, transaction: TransactionArg, grove_version: &GroveVersion, - ) -> CostResult>, Error> { + ) -> CostResult>, Error> { check_grovedb_v0_with_cost!( "delete", grove_version diff --git a/grovedb/src/operations/delete/mod.rs b/grovedb/src/operations/delete/mod.rs index 31d96b85..9244c60b 100644 --- a/grovedb/src/operations/delete/mod.rs +++ b/grovedb/src/operations/delete/mod.rs @@ -33,7 +33,7 @@ use grovedb_version::{ #[cfg(feature = "full")] use crate::{ - batch::{GroveDbOp, Op}, + batch::{GroveOp, QualifiedGroveDbOp}, util::storage_context_with_parent_optional_tx, Element, ElementFlags, Error, GroveDb, Transaction, TransactionArg, }; @@ -512,10 +512,10 @@ impl GroveDb { key: &[u8], options: &DeleteOptions, is_known_to_be_subtree_with_sum: Option<(bool, bool)>, - current_batch_operations: &[GroveDbOp], + current_batch_operations: &[QualifiedGroveDbOp], transaction: TransactionArg, grove_version: &GroveVersion, - ) -> CostResult, Error> { + ) -> CostResult, Error> { check_grovedb_v0_with_cost!( "delete_operation_for_delete_internal", grove_version @@ -565,7 +565,7 @@ impl GroveDb { let batch_deleted_keys = current_batch_operations .iter() .filter_map(|op| match op.op { - Op::Delete | Op::DeleteTree | Op::DeleteSumTree => { + GroveOp::Delete | GroveOp::DeleteTree | GroveOp::DeleteSumTree => { // todo: to_path clones (best to figure out how to compare without // cloning) if op.path.to_path() == subtree_merk_path_vec { @@ -595,7 +595,7 @@ impl GroveDb { // If there is any current batch operation that is inserting something in this // tree then it is not empty either is_empty &= !current_batch_operations.iter().any(|op| match op.op { - Op::Delete | Op::DeleteTree | Op::DeleteSumTree => false, + GroveOp::Delete | GroveOp::DeleteTree | GroveOp::DeleteSumTree => false, // todo: fix for to_path (it clones) _ => op.path.to_path() == subtree_merk_path_vec, }); @@ -610,7 +610,7 @@ impl GroveDb { Ok(None) } } else if is_empty { - Ok(Some(GroveDbOp::delete_tree_op( + Ok(Some(QualifiedGroveDbOp::delete_tree_op( path.to_vec(), key.to_vec(), is_subtree_with_sum, @@ -622,7 +622,11 @@ impl GroveDb { }; result.wrap_with_cost(cost) } else { - Ok(Some(GroveDbOp::delete_op(path.to_vec(), key.to_vec()))).wrap_with_cost(cost) + Ok(Some(QualifiedGroveDbOp::delete_op( + path.to_vec(), + key.to_vec(), + ))) + .wrap_with_cost(cost) } } } diff --git a/grovedb/src/operations/delete/worst_case.rs b/grovedb/src/operations/delete/worst_case.rs index b2a50bb2..8533cde5 100644 --- a/grovedb/src/operations/delete/worst_case.rs +++ b/grovedb/src/operations/delete/worst_case.rs @@ -13,7 +13,7 @@ use grovedb_version::{ use intmap::IntMap; use crate::{ - batch::{key_info::KeyInfo, GroveDbOp, KeyInfoPath}, + batch::{key_info::KeyInfo, KeyInfoPath, QualifiedGroveDbOp}, element::SUM_TREE_COST_SIZE, Error, GroveDb, }; @@ -29,7 +29,7 @@ impl GroveDb { intermediate_tree_info: IntMap<(bool, u32)>, max_element_size: u32, grove_version: &GroveVersion, - ) -> CostResult, Error> { + ) -> CostResult, Error> { check_grovedb_v0_with_cost!( "delete", grove_version @@ -127,7 +127,7 @@ impl GroveDb { except_keys_count: u16, max_element_size: u32, grove_version: &GroveVersion, - ) -> CostResult { + ) -> CostResult { check_grovedb_v0_with_cost!( "worst_case_delete_operation_for_delete", grove_version @@ -165,6 +165,10 @@ impl GroveDb { // in the worst case this is a tree add_worst_case_cost_for_is_empty_tree_except(&mut cost, except_keys_count); - Ok(GroveDbOp::delete_estimated_op(path.clone(), key.clone())).wrap_with_cost(cost) + Ok(QualifiedGroveDbOp::delete_estimated_op( + path.clone(), + key.clone(), + )) + .wrap_with_cost(cost) } } diff --git a/grovedb/src/operations/insert/mod.rs b/grovedb/src/operations/insert/mod.rs index b7e00fc1..af7629ae 100644 --- a/grovedb/src/operations/insert/mod.rs +++ b/grovedb/src/operations/insert/mod.rs @@ -293,44 +293,18 @@ impl GroveDb { .wrap_with_cost(OperationCost::default()) ); - let (referenced_key, referenced_path) = reference_path.split_last().unwrap(); - let subtree_for_reference = cost_return_on_error!( + let referenced_item = cost_return_on_error!( &mut cost, - self.open_transactional_merk_at_path( - referenced_path.into(), - transaction, - Some(batch), - grove_version, - ) - ); - - let referenced_element_value_hash_opt = cost_return_on_error!( - &mut cost, - Element::get_value_hash( - &subtree_for_reference, - referenced_key, - true, + self.follow_reference( + reference_path.as_slice().into(), + false, + Some(transaction), grove_version ) ); - let referenced_element_value_hash = cost_return_on_error!( - &mut cost, - referenced_element_value_hash_opt - .ok_or({ - let reference_string = reference_path - .iter() - .map(hex::encode) - .collect::>() - .join("/"); - Error::MissingReference(format!( - "reference {}/{} can not be found", - reference_string, - hex::encode(key) - )) - }) - .wrap_with_cost(OperationCost::default()) - ); + let referenced_element_value_hash = + cost_return_on_error!(&mut cost, referenced_item.value_hash(grove_version)); cost_return_on_error!( &mut cost, @@ -452,46 +426,18 @@ impl GroveDb { path_from_reference_path_type(reference_path.clone(), path, Some(key)) .wrap_with_cost(OperationCost::default()) ); - - // TODO unwrap? - let (referenced_key, referenced_path) = reference_path.split_last().unwrap(); - let subtree_for_reference = cost_return_on_error!( + let referenced_item = cost_return_on_error!( &mut cost, - self.open_non_transactional_merk_at_path( - referenced_path.into(), - Some(batch), - grove_version - ) - ); - - // when there is no transaction, we don't want to use caching - let referenced_element_value_hash_opt = cost_return_on_error!( - &mut cost, - Element::get_value_hash( - &subtree_for_reference, - referenced_key, + self.follow_reference( + reference_path.as_slice().into(), false, + None, grove_version ) ); - let referenced_element_value_hash = cost_return_on_error!( - &mut cost, - referenced_element_value_hash_opt - .ok_or({ - let reference_string = reference_path - .iter() - .map(hex::encode) - .collect::>() - .join("/"); - Error::MissingReference(format!( - "reference {}/{} can not be found", - reference_string, - hex::encode(key) - )) - }) - .wrap_with_cost(OperationCost::default()) - ); + let referenced_element_value_hash = + cost_return_on_error!(&mut cost, referenced_item.value_hash(grove_version)); cost_return_on_error!( &mut cost, diff --git a/grovedb/src/operations/proof/mod.rs b/grovedb/src/operations/proof/mod.rs index 88243d59..c90e176b 100644 --- a/grovedb/src/operations/proof/mod.rs +++ b/grovedb/src/operations/proof/mod.rs @@ -9,9 +9,20 @@ use std::{collections::BTreeMap, fmt}; use bincode::{Decode, Encode}; use derive_more::From; -use grovedb_merk::proofs::{query::Key, Decoder, Node, Op}; +use grovedb_merk::{ + proofs::{ + query::{Key, VerifyOptions}, + Decoder, Node, Op, + }, + CryptoHash, +}; +use grovedb_version::version::GroveVersion; -use crate::operations::proof::util::{element_hex_to_ascii, hex_to_ascii}; +use crate::{ + operations::proof::util::{element_hex_to_ascii, hex_to_ascii, ProvedPathKeyValues}, + query_result_type::PathKeyOptionalElementTrio, + Error, GroveDb, PathQuery, +}; #[derive(Debug, Clone, Copy, Encode, Decode)] pub struct ProveOptions { @@ -58,6 +69,114 @@ pub enum GroveDBProof { V0(GroveDBProofV0), } +impl GroveDBProof { + /// Verifies a query with options using the proof and returns the root hash + /// and the query result. + pub fn verify_with_options( + &self, + query: &PathQuery, + options: VerifyOptions, + grove_version: &GroveVersion, + ) -> Result<(CryptoHash, Vec), Error> { + GroveDb::verify_proof_internal(self, query, options, grove_version) + } + + /// Verifies a raw query using the proof and returns the root hash and the + /// query result. + pub fn verify_raw( + &self, + query: &PathQuery, + grove_version: &GroveVersion, + ) -> Result<(CryptoHash, ProvedPathKeyValues), Error> { + GroveDb::verify_proof_raw_internal( + self, + query, + VerifyOptions { + absence_proofs_for_non_existing_searched_keys: false, + verify_proof_succinctness: false, + include_empty_trees_in_result: true, + }, + grove_version, + ) + } + + /// Verifies a query using the proof and returns the root hash and the query + /// result. + pub fn verify( + &self, + query: &PathQuery, + grove_version: &GroveVersion, + ) -> Result<(CryptoHash, Vec), Error> { + GroveDb::verify_proof_internal( + self, + query, + VerifyOptions { + absence_proofs_for_non_existing_searched_keys: false, + verify_proof_succinctness: true, + include_empty_trees_in_result: false, + }, + grove_version, + ) + } + + /// Verifies a query with an absence proof and returns the root hash and the + /// query result. + pub fn verify_with_absence_proof( + &self, + query: &PathQuery, + grove_version: &GroveVersion, + ) -> Result<(CryptoHash, Vec), Error> { + GroveDb::verify_proof_internal( + self, + query, + VerifyOptions { + absence_proofs_for_non_existing_searched_keys: true, + verify_proof_succinctness: true, + include_empty_trees_in_result: false, + }, + grove_version, + ) + } + + /// Verifies a subset query using the proof and returns the root hash and + /// the query result. + pub fn verify_subset( + &self, + query: &PathQuery, + grove_version: &GroveVersion, + ) -> Result<(CryptoHash, Vec), Error> { + GroveDb::verify_proof_internal( + self, + query, + VerifyOptions { + absence_proofs_for_non_existing_searched_keys: false, + verify_proof_succinctness: false, + include_empty_trees_in_result: false, + }, + grove_version, + ) + } + + /// Verifies a subset query with an absence proof using the proof and + /// returns the root hash and the query result. + pub fn verify_subset_with_absence_proof( + &self, + query: &PathQuery, + grove_version: &GroveVersion, + ) -> Result<(CryptoHash, Vec), Error> { + GroveDb::verify_proof_internal( + self, + query, + VerifyOptions { + absence_proofs_for_non_existing_searched_keys: true, + verify_proof_succinctness: false, + include_empty_trees_in_result: false, + }, + grove_version, + ) + } +} + #[derive(Encode, Decode)] pub struct GroveDBProofV0 { pub root_layer: LayerProof, diff --git a/grovedb/src/operations/proof/util.rs b/grovedb/src/operations/proof/util.rs index d6b34ecc..a33954d1 100644 --- a/grovedb/src/operations/proof/util.rs +++ b/grovedb/src/operations/proof/util.rs @@ -306,7 +306,7 @@ pub fn path_hex_to_ascii(path: &Path) -> String { } pub fn path_as_slices_hex_to_ascii(path: &[&[u8]]) -> String { - path.into_iter() + path.iter() .map(|e| hex_to_ascii(e)) .collect::>() .join("/") diff --git a/grovedb/src/operations/proof/verify.rs b/grovedb/src/operations/proof/verify.rs index 31cd9ae5..1ac09c8b 100644 --- a/grovedb/src/operations/proof/verify.rs +++ b/grovedb/src/operations/proof/verify.rs @@ -102,7 +102,7 @@ impl GroveDb { Ok((root_hash, result)) } - fn verify_proof_internal( + pub(crate) fn verify_proof_internal( proof: &GroveDBProof, query: &PathQuery, options: VerifyOptions, @@ -192,7 +192,7 @@ impl GroveDb { Ok((root_hash, result)) } - fn verify_proof_raw_internal( + pub(crate) fn verify_proof_raw_internal( proof: &GroveDBProof, query: &PathQuery, options: VerifyOptions, @@ -511,23 +511,49 @@ impl GroveDb { Self::verify_subset_query(proof, first_query, grove_version)?; results.push(elements); - // we should iterate over each chained path queries + // Process the chained path queries + Self::process_chained_path_queries( + proof, + last_root_hash, + chained_path_queries, + grove_version, + &mut results, + )?; + + Ok((last_root_hash, results)) + } + + /// Processes each chained path query and verifies it. + pub(in crate::operations::proof) fn process_chained_path_queries( + proof: &[u8], + last_root_hash: CryptoHash, + chained_path_queries: Vec, + grove_version: &GroveVersion, + results: &mut Vec>, + ) -> Result<(), Error> + where + C: Fn(Vec) -> Option, + { for path_query_generator in chained_path_queries { let new_path_query = path_query_generator(results[results.len() - 1].clone()).ok_or( Error::InvalidInput("one of the path query generators returns no path query"), )?; + let (new_root_hash, new_elements) = Self::verify_subset_query(proof, &new_path_query, grove_version)?; + if new_root_hash != last_root_hash { return Err(Error::InvalidProof(format!( - "root hash for different path queries do no match, first is {}, this one is {}", + "root hash for different path queries do not match, first is {}, this one is \ + {}", hex::encode(last_root_hash), hex::encode(new_root_hash) ))); } + results.push(new_elements); } - Ok((last_root_hash, results)) + Ok(()) } } diff --git a/grovedb/src/query/mod.rs b/grovedb/src/query/mod.rs index f140bb05..a1443836 100644 --- a/grovedb/src/query/mod.rs +++ b/grovedb/src/query/mod.rs @@ -6,6 +6,7 @@ use std::{ fmt, }; +use bincode::{Decode, Encode}; #[cfg(any(feature = "full", feature = "verify"))] use grovedb_merk::proofs::query::query_item::QueryItem; use grovedb_merk::proofs::query::{Key, SubqueryBranch}; @@ -21,14 +22,13 @@ use crate::query_result_type::PathKey; use crate::Error; #[cfg(any(feature = "full", feature = "verify"))] -#[derive(Debug, Clone)] +#[derive(Debug, Clone, PartialEq, Encode, Decode)] /// Path query /// /// Represents a path to a specific GroveDB tree and a corresponding query to /// apply to the given tree. pub struct PathQuery { /// Path - // TODO: Make generic over path type pub path: Vec>, /// Query pub query: SizedQuery, @@ -49,7 +49,7 @@ impl fmt::Display for PathQuery { } #[cfg(any(feature = "full", feature = "verify"))] -#[derive(Debug, Clone)] +#[derive(Debug, Clone, PartialEq, Encode, Decode)] /// Holds a query to apply to a tree and an optional limit/offset value. /// Limit and offset values affect the size of the result set. pub struct SizedQuery { @@ -134,6 +134,14 @@ impl PathQuery { Self { path, query } } + /// The max depth of the query, this is the maximum layers we could get back + /// from grovedb + /// If the max depth can not be calculated we get None + /// This would occur if the recursion level was too high + pub fn max_depth(&self) -> Option { + self.query.query.max_depth() + } + /// Gets the path of all terminal keys pub fn terminal_keys( &self, @@ -577,6 +585,7 @@ impl<'a> SinglePathSubquery<'a> { mod tests { use std::{borrow::Cow, ops::RangeFull}; + use bincode::{config::standard, decode_from_slice, encode_to_vec}; use grovedb_merk::proofs::{ query::{query_item::QueryItem, SubqueryBranch}, Query, @@ -1634,6 +1643,8 @@ mod tests { }, }; + assert_eq!(path_query.max_depth(), Some(4)); + { let path = vec![]; let first = path_query @@ -1710,4 +1721,299 @@ mod tests { ); } } + + #[test] + fn test_max_depth_limit() { + /// Creates a `Query` with nested `SubqueryBranch` up to the specified + /// depth non-recursively. + fn create_non_recursive_query(subquery_depth: usize) -> Query { + let mut root_query = Query::new_range_full(); + let mut current_query = &mut root_query; + + for _ in 0..subquery_depth { + let new_query = Query::new_range_full(); + current_query.default_subquery_branch = SubqueryBranch { + subquery_path: None, + subquery: Some(Box::new(new_query)), + }; + current_query = current_query + .default_subquery_branch + .subquery + .as_mut() + .unwrap(); + } + + root_query + } + + let query = create_non_recursive_query(100); + + assert_eq!(query.max_depth(), Some(101)); + + let query = create_non_recursive_query(500); + + assert_eq!(query.max_depth(), None); + } + + #[test] + fn test_simple_path_query_serialization() { + let path_query = PathQuery { + path: vec![b"root".to_vec(), b"subtree".to_vec()], + query: SizedQuery { + query: Query { + items: vec![QueryItem::Key(b"key1".to_vec())], + default_subquery_branch: SubqueryBranch::default(), + conditional_subquery_branches: None, + left_to_right: true, + }, + limit: None, + offset: None, + }, + }; + + let encoded = encode_to_vec(&path_query, standard()).unwrap(); + let decoded: PathQuery = decode_from_slice(&encoded, standard()).unwrap().0; + + assert_eq!(path_query, decoded); + } + + #[test] + fn test_range_query_serialization() { + let path_query = PathQuery { + path: vec![b"root".to_vec()], + query: SizedQuery { + query: Query { + items: vec![QueryItem::Range(b"a".to_vec()..b"z".to_vec())], + default_subquery_branch: SubqueryBranch::default(), + conditional_subquery_branches: None, + left_to_right: false, + }, + limit: Some(10), + offset: Some(2), + }, + }; + + let encoded = encode_to_vec(&path_query, standard()).unwrap(); + let decoded: PathQuery = decode_from_slice(&encoded, standard()).unwrap().0; + + assert_eq!(path_query, decoded); + } + + #[test] + fn test_range_inclusive_query_serialization() { + let path_query = PathQuery { + path: vec![b"root".to_vec()], + query: SizedQuery { + query: Query { + items: vec![QueryItem::RangeInclusive(b"a".to_vec()..=b"z".to_vec())], + default_subquery_branch: SubqueryBranch::default(), + conditional_subquery_branches: None, + left_to_right: true, + }, + limit: Some(5), + offset: None, + }, + }; + + let encoded = encode_to_vec(&path_query, standard()).unwrap(); + let decoded: PathQuery = decode_from_slice(&encoded, standard()).unwrap().0; + + assert_eq!(path_query, decoded); + } + + #[test] + fn test_conditional_subquery_serialization() { + let mut conditional_branches = IndexMap::new(); + conditional_branches.insert( + QueryItem::Key(b"key1".to_vec()), + SubqueryBranch { + subquery_path: Some(vec![b"conditional_path".to_vec()]), + subquery: Some(Box::new(Query::default())), + }, + ); + + let path_query = PathQuery { + path: vec![b"root".to_vec()], + query: SizedQuery { + query: Query { + items: vec![QueryItem::Key(b"key1".to_vec())], + default_subquery_branch: SubqueryBranch::default(), + conditional_subquery_branches: Some(conditional_branches), + left_to_right: true, + }, + limit: None, + offset: None, + }, + }; + + let encoded = encode_to_vec(&path_query, standard()).unwrap(); + let decoded: PathQuery = decode_from_slice(&encoded, standard()).unwrap().0; + + assert_eq!(path_query, decoded); + } + + #[test] + fn test_empty_path_query_serialization() { + let path_query = PathQuery { + path: vec![], + query: SizedQuery { + query: Query::default(), + limit: None, + offset: None, + }, + }; + + let encoded = encode_to_vec(&path_query, standard()).unwrap(); + let decoded: PathQuery = decode_from_slice(&encoded, standard()).unwrap().0; + + assert_eq!(path_query, decoded); + } + + #[test] + fn test_path_query_with_multiple_keys() { + let path_query = PathQuery { + path: vec![b"root".to_vec()], + query: SizedQuery { + query: Query { + items: vec![ + QueryItem::Key(b"key1".to_vec()), + QueryItem::Key(b"key2".to_vec()), + QueryItem::Key(b"key3".to_vec()), + ], + default_subquery_branch: SubqueryBranch::default(), + conditional_subquery_branches: None, + left_to_right: true, + }, + limit: None, + offset: None, + }, + }; + + let encoded = encode_to_vec(&path_query, standard()).unwrap(); + let decoded: PathQuery = decode_from_slice(&encoded, standard()).unwrap().0; + + assert_eq!(path_query, decoded); + } + + #[test] + fn test_path_query_with_full_range() { + let path_query = PathQuery { + path: vec![b"root".to_vec()], + query: SizedQuery { + query: Query { + items: vec![QueryItem::RangeFull(RangeFull)], + default_subquery_branch: SubqueryBranch::default(), + conditional_subquery_branches: None, + left_to_right: false, + }, + limit: Some(100), + offset: Some(10), + }, + }; + + let encoded = encode_to_vec(&path_query, standard()).unwrap(); + let decoded: PathQuery = decode_from_slice(&encoded, standard()).unwrap().0; + + assert_eq!(path_query, decoded); + } + + #[test] + fn test_path_query_with_complex_conditions() { + let mut conditional_branches = IndexMap::new(); + conditional_branches.insert( + QueryItem::Key(b"key1".to_vec()), + SubqueryBranch { + subquery_path: Some(vec![b"conditional_path1".to_vec()]), + subquery: Some(Box::new(Query { + items: vec![QueryItem::Range(b"a".to_vec()..b"m".to_vec())], + default_subquery_branch: SubqueryBranch::default(), + conditional_subquery_branches: None, + left_to_right: true, + })), + }, + ); + conditional_branches.insert( + QueryItem::Range(b"n".to_vec()..b"z".to_vec()), + SubqueryBranch { + subquery_path: Some(vec![b"conditional_path2".to_vec()]), + subquery: Some(Box::new(Query { + items: vec![QueryItem::Key(b"key2".to_vec())], + default_subquery_branch: SubqueryBranch::default(), + conditional_subquery_branches: None, + left_to_right: false, + })), + }, + ); + + let path_query = PathQuery { + path: vec![b"root".to_vec()], + query: SizedQuery { + query: Query { + items: vec![QueryItem::Key(b"key3".to_vec())], + default_subquery_branch: SubqueryBranch::default(), + conditional_subquery_branches: Some(conditional_branches), + left_to_right: true, + }, + limit: Some(50), + offset: Some(5), + }, + }; + + let encoded = encode_to_vec(&path_query, standard()).unwrap(); + let decoded: PathQuery = decode_from_slice(&encoded, standard()).unwrap().0; + + assert_eq!(path_query, decoded); + } + + #[test] + fn test_path_query_with_subquery_path() { + let path_query = PathQuery { + path: vec![b"root".to_vec()], + query: SizedQuery { + query: Query { + items: vec![QueryItem::Key(b"key1".to_vec())], + default_subquery_branch: SubqueryBranch { + subquery_path: Some(vec![b"subtree_path".to_vec()]), + subquery: Some(Box::new(Query { + items: vec![QueryItem::Key(b"key2".to_vec())], + default_subquery_branch: SubqueryBranch::default(), + conditional_subquery_branches: None, + left_to_right: true, + })), + }, + conditional_subquery_branches: None, + left_to_right: true, + }, + limit: None, + offset: None, + }, + }; + + let encoded = encode_to_vec(&path_query, standard()).unwrap(); + let decoded: PathQuery = decode_from_slice(&encoded, standard()).unwrap().0; + + assert_eq!(path_query, decoded); + } + + #[test] + fn test_path_query_with_empty_query_items() { + let path_query = PathQuery { + path: vec![b"root".to_vec()], + query: SizedQuery { + query: Query { + items: vec![], // No items in the query + default_subquery_branch: SubqueryBranch::default(), + conditional_subquery_branches: None, + left_to_right: true, + }, + limit: Some(20), + offset: None, + }, + }; + + let encoded = encode_to_vec(&path_query, standard()).unwrap(); + let decoded: PathQuery = decode_from_slice(&encoded, standard()).unwrap().0; + + assert_eq!(path_query, decoded); + } } diff --git a/grovedb/src/query_result_type.rs b/grovedb/src/query_result_type.rs index 02035238..1850544e 100644 --- a/grovedb/src/query_result_type.rs +++ b/grovedb/src/query_result_type.rs @@ -283,7 +283,7 @@ impl QueryResultElements { result_item { if let Some(last) = path.pop() { - map.entry(last).or_insert_with(Vec::new).push(key); + map.entry(last).or_default().push(key); } } } @@ -333,7 +333,7 @@ impl QueryResultElements { result_item { if let Some(last) = path.pop() { - map.entry(last).or_insert_with(Vec::new).push(element); + map.entry(last).or_default().push(element); } } } diff --git a/grovedb/src/tests/mod.rs b/grovedb/src/tests/mod.rs index 226a047b..41263669 100644 --- a/grovedb/src/tests/mod.rs +++ b/grovedb/src/tests/mod.rs @@ -758,6 +758,9 @@ pub fn make_deep_tree_with_sum_trees(grove_version: &GroveVersion) -> TempGroveD } mod tests { + use batch::QualifiedGroveDbOp; + use grovedb_merk::proofs::query::SubqueryBranch; + use super::*; #[test] @@ -1187,7 +1190,12 @@ mod tests { ) .unwrap(); - assert!(matches!(result, Err(Error::MissingReference(_)))); + dbg!(&result); + + assert!(matches!( + result, + Err(Error::CorruptedReferencePathKeyNotFound(_)) + )); } #[test] @@ -1226,24 +1234,15 @@ mod tests { } // Add one more reference - db.insert( - [TEST_LEAF].as_ref(), - &keygen(MAX_REFERENCE_HOPS + 1), - Element::new_reference(ReferencePathType::AbsolutePathReference(vec![ - TEST_LEAF.to_vec(), - keygen(MAX_REFERENCE_HOPS), - ])), - None, - None, - grove_version, - ) - .unwrap() - .expect("expected insert"); - let result = db - .get( + .insert( [TEST_LEAF].as_ref(), &keygen(MAX_REFERENCE_HOPS + 1), + Element::new_reference(ReferencePathType::AbsolutePathReference(vec![ + TEST_LEAF.to_vec(), + keygen(MAX_REFERENCE_HOPS), + ])), + None, None, grove_version, ) @@ -3914,4 +3913,215 @@ mod tests { Err(Error::PathParentLayerNotFound(..)) )); } + + #[test] + fn test_grovedb_verify_corrupted_reference() { + // This test is dedicated to a case when references are out of sync, but + // `verify_grovedb` must detect this case as any other inconsistency + + let grove_version = GroveVersion::latest(); + let db = make_deep_tree(grove_version); + + // Insert a reference + db.insert( + &[TEST_LEAF, b"innertree"], + b"ref", + Element::Reference( + ReferencePathType::AbsolutePathReference(vec![ + ANOTHER_TEST_LEAF.to_vec(), + b"innertree2".to_vec(), + b"key3".to_vec(), + ]), + None, + None, + ), + None, + None, + grove_version, + ) + .unwrap() + .unwrap(); + + // Ensure we can prove and verify the inserted reference + let query = PathQuery { + path: vec![TEST_LEAF.to_vec(), b"innertree".to_vec()], + query: SizedQuery { + query: Query { + items: vec![QueryItem::Key(b"ref".to_vec())], + default_subquery_branch: SubqueryBranch { + subquery_path: None, + subquery: None, + }, + conditional_subquery_branches: None, + left_to_right: true, + }, + limit: None, + offset: None, + }, + }; + let proof = db + .prove_query(&query, None, grove_version) + .unwrap() + .unwrap(); + + let (hash, _) = GroveDb::verify_query(&proof, &query, grove_version).unwrap(); + assert_eq!(hash, db.root_hash(None, grove_version).unwrap().unwrap()); + + // Update referenced value to break things + db.insert( + &[ANOTHER_TEST_LEAF.to_vec(), b"innertree2".to_vec()], + b"key3", + Element::Item(b"idk something else i guess?".to_vec(), None), + None, + None, + grove_version, + ) + .unwrap() + .unwrap(); + + let proof = db + .prove_query(&query, None, grove_version) + .unwrap() + .unwrap(); + + assert!(matches!( + GroveDb::verify_query(&proof, &query, grove_version), + Err(_) + )); + + // `verify_grovedb` must identify issues + assert!( + db.verify_grovedb(None, true, false, grove_version) + .unwrap() + .len() + > 0 + ); + } + + #[test] + fn test_verify_corrupted_long_reference() { + let grove_version = GroveVersion::latest(); + let db = make_test_grovedb(grove_version); + + db.insert( + &[TEST_LEAF], + b"value", + Element::new_item(b"hello".to_vec()), + None, + None, + grove_version, + ) + .unwrap() + .unwrap(); + + db.insert( + &[TEST_LEAF], + b"refc", + Element::new_reference(ReferencePathType::SiblingReference(b"value".to_vec())), + None, + None, + grove_version, + ) + .unwrap() + .unwrap(); + + db.insert( + &[TEST_LEAF], + b"refb", + Element::new_reference(ReferencePathType::SiblingReference(b"refc".to_vec())), + None, + None, + grove_version, + ) + .unwrap() + .unwrap(); + + db.insert( + &[TEST_LEAF], + b"refa", + Element::new_reference(ReferencePathType::SiblingReference(b"refb".to_vec())), + None, + None, + grove_version, + ) + .unwrap() + .unwrap(); + + assert!(db + .verify_grovedb(None, true, false, grove_version) + .unwrap() + .is_empty()); + + // Breaking things there: + db.insert( + &[TEST_LEAF], + b"value", + Element::new_item(b"not hello >:(".to_vec()), + None, + None, + grove_version, + ) + .unwrap() + .unwrap(); + + assert!(!db + .verify_grovedb(None, true, false, grove_version) + .unwrap() + .is_empty()); + } + + #[test] + fn test_verify_corrupted_long_reference_batch() { + let grove_version = GroveVersion::latest(); + let db = make_test_grovedb(grove_version); + + let ops = vec![ + QualifiedGroveDbOp::insert_or_replace_op( + vec![TEST_LEAF.to_vec()], + b"value".to_vec(), + Element::new_item(b"hello".to_vec()), + ), + QualifiedGroveDbOp::insert_or_replace_op( + vec![TEST_LEAF.to_vec()], + b"refc".to_vec(), + Element::new_reference(ReferencePathType::SiblingReference(b"value".to_vec())), + ), + QualifiedGroveDbOp::insert_or_replace_op( + vec![TEST_LEAF.to_vec()], + b"refb".to_vec(), + Element::new_reference(ReferencePathType::SiblingReference(b"refc".to_vec())), + ), + QualifiedGroveDbOp::insert_or_replace_op( + vec![TEST_LEAF.to_vec()], + b"refa".to_vec(), + Element::new_reference(ReferencePathType::SiblingReference(b"refb".to_vec())), + ), + ]; + + db.apply_batch(ops, None, None, grove_version) + .unwrap() + .unwrap(); + + assert!(db + .verify_grovedb(None, true, false, grove_version) + .unwrap() + .is_empty()); + + // Breaking things there: + db.insert( + &[TEST_LEAF], + b"value", + Element::new_item(b"not hello >:(".to_vec()), + None, + None, + grove_version, + ) + .unwrap() + .unwrap(); + + assert!(!db + .verify_grovedb(None, true, false, grove_version) + .unwrap() + .is_empty()); + } } diff --git a/grovedb/src/tests/query_tests.rs b/grovedb/src/tests/query_tests.rs index 48c358c6..85294b95 100644 --- a/grovedb/src/tests/query_tests.rs +++ b/grovedb/src/tests/query_tests.rs @@ -1,21 +1,28 @@ mod tests { //! Query tests - use grovedb_merk::proofs::{query::QueryItem, Query}; + use std::ops::RangeFull; + + use grovedb_merk::proofs::{ + query::{QueryItem, SubqueryBranch}, + Query, + }; use grovedb_version::version::GroveVersion; + use indexmap::IndexMap; use rand::random; use tempfile::TempDir; use crate::{ - batch::GroveDbOp, + batch::QualifiedGroveDbOp, + operations::proof::GroveDBProof, query_result_type::{ PathKeyOptionalElementTrio, QueryResultElement::PathKeyElementTrioResultItem, QueryResultElements, QueryResultType, }, reference_path::ReferencePathType, tests::{ - common::compare_result_sets, make_deep_tree, make_test_grovedb, TempGroveDb, - ANOTHER_TEST_LEAF, TEST_LEAF, + common::compare_result_sets, make_deep_tree, make_empty_grovedb, make_test_grovedb, + TempGroveDb, ANOTHER_TEST_LEAF, TEST_LEAF, }, Element, GroveDb, PathQuery, SizedQuery, }; @@ -422,6 +429,288 @@ mod tests { .expect("successful subtree insert"); } + fn populate_tree_create_two_by_two_hierarchy(db: &TempGroveDb) { + // The structure is the following + // ----------------------------------------------------------> + // A ---------------- B ---------------------C + // / \ / \ / \ + // A1 A2 B1 B2 C1 C2 + let grove_version = GroveVersion::latest(); + + let mut ops = Vec::new(); + + // Loop from A to M + for c in b'A'..=b'M' { + let node = vec![c]; + let child1 = vec![c, b'1']; + let child2 = vec![c, b'2']; + + // Insert the parent node as a tree + ops.push(QualifiedGroveDbOp::insert_or_replace_op( + vec![], + node.clone(), + Element::new_tree(None), + )); + + // Insert two children as items with their corresponding values + ops.push(QualifiedGroveDbOp::insert_or_replace_op( + vec![node.clone()], + child1.clone(), + Element::new_item(vec![1]), // A1, B1, etc. has a value of 1 + )); + ops.push(QualifiedGroveDbOp::insert_or_replace_op( + vec![node.clone()], + child2.clone(), + Element::new_item(vec![2]), // A2, B2, etc. has a value of 2 + )); + } + + // Apply the batch of operations to the database + let _ = db + .apply_batch(ops, None, None, grove_version) + .cost_as_result() + .expect("expected to create test data"); + } + + fn populate_tree_create_two_by_two_hierarchy_with_intermediate_value(db: &TempGroveDb) { + // The structure is the following + // ----------------------------------------------------------> + // A ---------------- B ---------------------C + // | | | + // 0 0 0 + // / \ / \ / \ + // A1 A2 B1 B2 C1 C2 + let grove_version = GroveVersion::latest(); + + let mut ops = Vec::new(); + + // Loop from A to M + for c in b'A'..=b'M' { + let node = vec![c]; + let intermediate = vec![0]; + let child1 = vec![c, b'1']; + let child2 = vec![c, b'2']; + + // Insert the parent node as a tree + ops.push(QualifiedGroveDbOp::insert_or_replace_op( + vec![], + node.clone(), + Element::new_tree(None), + )); + + ops.push(QualifiedGroveDbOp::insert_or_replace_op( + vec![node.clone()], + intermediate.clone(), + Element::new_tree(None), + )); + + // Insert two children as items with their corresponding values + ops.push(QualifiedGroveDbOp::insert_or_replace_op( + vec![node.clone(), intermediate.clone()], + child1.clone(), + Element::new_item(vec![1]), // A1, B1, etc. has a value of 1 + )); + ops.push(QualifiedGroveDbOp::insert_or_replace_op( + vec![node.clone(), intermediate.clone()], + child2.clone(), + Element::new_item(vec![2]), // A2, B2, etc. has a value of 2 + )); + } + + // Apply the batch of operations to the database + let _ = db + .apply_batch(ops, None, None, grove_version) + .cost_as_result() + .expect("expected to create test data"); + } + + fn populate_tree_create_two_by_two_reference_hierarchy_with_intermediate_value( + db: &TempGroveDb, + ) { + // The structure is the following + // ----------------------------------------------------------> + // a ------------------------b------------------------c + // / \ / \ / \ + // 0 1 0 1 0 1 + // / \ / / \ / / \ / + // A1 A2 2 B1 B2 2 C1 C2 2 + // / \ / \ / \ + // refA1 refA2 refB1 refB2 refC1 refC2 + let grove_version = GroveVersion::latest(); + + let mut ops = Vec::new(); + + // Loop from A to M + for c in b'a'..=b'm' { + let node = vec![c]; + let intermediate = vec![0]; + let intermediate_ref_1 = vec![1]; + let intermediate_ref_2 = vec![2]; + let child1 = vec![c.to_ascii_uppercase(), b'1']; + let child2 = vec![c.to_ascii_uppercase(), b'2']; + let child1ref = vec![b'r', b'e', b'f', c.to_ascii_uppercase(), b'1']; + let child2ref = vec![b'r', b'e', b'f', c.to_ascii_uppercase(), b'2']; + + // Insert the parent node as a tree + ops.push(QualifiedGroveDbOp::insert_or_replace_op( + vec![], + node.clone(), + Element::new_tree(None), + )); + + ops.push(QualifiedGroveDbOp::insert_or_replace_op( + vec![node.clone()], + intermediate.clone(), + Element::new_tree(None), + )); + ops.push(QualifiedGroveDbOp::insert_or_replace_op( + vec![node.clone()], + intermediate_ref_1.clone(), + Element::new_tree(None), + )); + ops.push(QualifiedGroveDbOp::insert_or_replace_op( + vec![node.clone(), intermediate_ref_1.clone()], + intermediate_ref_2.clone(), + Element::new_tree(None), + )); + + // Insert two children as items with their corresponding values + ops.push(QualifiedGroveDbOp::insert_or_replace_op( + vec![node.clone(), intermediate.clone()], + child1.clone(), + Element::new_item(vec![1]), // A1, B1, etc. has a value of 1 + )); + ops.push(QualifiedGroveDbOp::insert_or_replace_op( + vec![node.clone(), intermediate.clone()], + child2.clone(), + Element::new_item(vec![2]), // A2, B2, etc. has a value of 2 + )); + + // Insert the references + ops.push(QualifiedGroveDbOp::insert_or_replace_op( + vec![ + node.clone(), + intermediate_ref_1.clone(), + intermediate_ref_2.clone(), + ], + child1ref.clone(), + Element::new_reference(ReferencePathType::UpstreamRootHeightReference( + 1, + vec![intermediate.clone(), child1.clone()], + )), // refA1 + )); + ops.push(QualifiedGroveDbOp::insert_or_replace_op( + vec![ + node.clone(), + intermediate_ref_1.clone(), + intermediate_ref_2.clone(), + ], + child2ref.clone(), + Element::new_reference(ReferencePathType::UpstreamRootHeightReference( + 1, + vec![intermediate.clone(), child2.clone()], + )), // refA2 + )); + } + + // Apply the batch of operations to the database + let _ = db + .apply_batch(ops, None, None, grove_version) + .cost_as_result() + .expect("expected to create test data"); + } + + fn populate_tree_create_two_by_two_reference_higher_up_hierarchy_with_intermediate_value( + db: &TempGroveDb, + ) { + // The structure is the following + // ----------------------------------------------------------> + // 0 -------------------------------- 1 + // / \ / | \ + // A1 .. C2 a ------------------------b------------------------c + // | | | + // 0 0 0 + // / \ / \ / \ + // refA1 refA2 refB1 refB2 refC1 refC2 + + let grove_version = GroveVersion::latest(); + + let mut ops = Vec::new(); + + let top_holder = vec![0]; + let top_ref = vec![1]; + + ops.push(QualifiedGroveDbOp::insert_or_replace_op( + vec![], + top_holder.clone(), + Element::empty_tree(), + )); + ops.push(QualifiedGroveDbOp::insert_or_replace_op( + vec![], + top_ref.clone(), + Element::empty_tree(), + )); + + // Loop from A to M + for c in b'a'..=b'm' { + let node = vec![c]; + let intermediate = vec![0]; + let child1 = vec![c.to_ascii_uppercase(), b'1']; + let child2 = vec![c.to_ascii_uppercase(), b'2']; + let child1ref = vec![b'r', b'e', b'f', c.to_ascii_uppercase(), b'1']; + let child2ref = vec![b'r', b'e', b'f', c.to_ascii_uppercase(), b'2']; + + // Insert the parent node as a tree + ops.push(QualifiedGroveDbOp::insert_or_replace_op( + vec![top_ref.clone()], + node.clone(), + Element::new_tree(None), + )); + + ops.push(QualifiedGroveDbOp::insert_or_replace_op( + vec![top_ref.clone(), node.clone()], + intermediate.clone(), + Element::new_tree(None), + )); + + // Insert two children as items with their corresponding values + ops.push(QualifiedGroveDbOp::insert_or_replace_op( + vec![top_holder.clone()], + child1.clone(), + Element::new_item(vec![1]), // A1, B1, etc. has a value of 1 + )); + ops.push(QualifiedGroveDbOp::insert_or_replace_op( + vec![top_holder.clone()], + child2.clone(), + Element::new_item(vec![2]), // A2, B2, etc. has a value of 2 + )); + + // Insert the references + ops.push(QualifiedGroveDbOp::insert_or_replace_op( + vec![top_ref.clone(), node.clone(), intermediate.clone()], + child1ref.clone(), + Element::new_reference(ReferencePathType::AbsolutePathReference(vec![ + top_holder.clone(), + child1.clone(), + ])), // refA1 + )); + ops.push(QualifiedGroveDbOp::insert_or_replace_op( + vec![top_ref.clone(), node.clone(), intermediate.clone()], + child2ref.clone(), + Element::new_reference(ReferencePathType::AbsolutePathReference(vec![ + top_holder.clone(), + child2.clone(), + ])), // refA2 + )); + } + + // Apply the batch of operations to the database + let _ = db + .apply_batch(ops, None, None, grove_version) + .cost_as_result() + .expect("expected to create test data"); + } + #[test] fn test_get_correct_order() { let grove_version = GroveVersion::latest(); @@ -1595,33 +1884,37 @@ mod tests { 77, 252, 86, 99, 107, 197, 226, 188, 54, 239, 64, 17, 37, ]; - let batch = vec![GroveDbOp::insert_op(vec![], vec![1], Element::empty_tree())]; + let batch = vec![QualifiedGroveDbOp::insert_or_replace_op( + vec![], + vec![1], + Element::empty_tree(), + )]; db.apply_batch(batch, None, None, grove_version) .unwrap() .expect("should apply batch"); let batch = vec![ - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![vec![1]], tree_name_slice.to_vec(), Element::empty_tree(), ), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![vec![1], tree_name_slice.to_vec()], b"\0".to_vec(), Element::empty_tree(), ), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![vec![1], tree_name_slice.to_vec()], vec![1], Element::empty_tree(), ), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![vec![1], tree_name_slice.to_vec(), vec![1]], b"person".to_vec(), Element::empty_tree(), ), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![ vec![1], tree_name_slice.to_vec(), @@ -1631,7 +1924,7 @@ mod tests { b"\0".to_vec(), Element::empty_tree(), ), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![ vec![1], tree_name_slice.to_vec(), @@ -1647,7 +1940,7 @@ mod tests { .expect("should apply batch"); let batch = vec![ - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![ vec![1], tree_name_slice.to_vec(), @@ -1658,7 +1951,7 @@ mod tests { b"person_id_1".to_vec(), Element::new_item(vec![50]), ), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![ vec![1], tree_name_slice.to_vec(), @@ -1669,7 +1962,7 @@ mod tests { b"cammi".to_vec(), Element::empty_tree(), ), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![ vec![1], tree_name_slice.to_vec(), @@ -1681,7 +1974,7 @@ mod tests { b"\0".to_vec(), Element::empty_tree(), ), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![ vec![1], tree_name_slice.to_vec(), @@ -3116,4 +3409,1496 @@ mod tests { grovedb.root_hash(None, grove_version).unwrap().unwrap() ); } + + #[test] + fn test_path_query_items_with_subquery_and_limit_2_asc_from_start() { + // The structure is the following + // ----------------------------------------------------------> + // A ---------------- B ---------------------C + // / \ / \ / \ + // A1 A2 B1 B2 C1 C2 + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + populate_tree_create_two_by_two_hierarchy(&db); + + // Constructing the PathQuery + let path_query = PathQuery { + path: vec![], + query: SizedQuery { + query: Query { + items: vec![QueryItem::RangeFull(RangeFull)], + default_subquery_branch: SubqueryBranch { + subquery_path: None, + subquery: Some(Query::new_range_full().into()), + }, + left_to_right: true, + conditional_subquery_branches: None, + }, + limit: Some(2), + offset: None, + }, + }; + + let (elements, _) = db + .query_item_value(&path_query, true, true, true, None, grove_version) + .unwrap() + .expect("expected successful get_path_query"); + + assert_eq!(elements.len(), 2); + + let proof = db + .prove_query(&path_query, None, grove_version) + .value + .expect("expected successful get_path_query"); + + let (_, result_set) = GroveDb::verify_query(proof.as_slice(), &path_query, grove_version) + .expect("should verify proof"); + assert_eq!(result_set.len(), 2); + } + + #[test] + fn test_path_query_items_with_subquery_and_limit_2_desc_from_start() { + // The structure is the following + // ----------------------------------------------------------> + // A ---------------- B ---------------------C + // / \ / \ / \ + // A1 A2 B1 B2 C1 C2 + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + populate_tree_create_two_by_two_hierarchy(&db); + + // Constructing the PathQuery + let path_query = PathQuery { + path: vec![], + query: SizedQuery { + query: Query { + items: vec![QueryItem::RangeToInclusive(..=b"A".to_vec())], + default_subquery_branch: SubqueryBranch { + subquery_path: None, + subquery: Some(Query::new_range_full().into()), + }, + left_to_right: false, + conditional_subquery_branches: None, + }, + limit: Some(2), + offset: None, + }, + }; + + let (elements, _) = db + .query_item_value(&path_query, true, true, true, None, grove_version) + .unwrap() + .expect("expected successful get_path_query"); + + assert_eq!(elements.len(), 2); + + let proof = db + .prove_query(&path_query, None, grove_version) + .value + .expect("expected successful get_path_query"); + + let (_, result_set) = GroveDb::verify_query(proof.as_slice(), &path_query, grove_version) + .expect("should verify proof"); + assert_eq!(result_set.len(), 2); + } + + #[test] + fn test_path_query_items_with_subquery_and_limit_2_asc_in_middle() { + // The structure is the following + // ----------------------------------------------------------> + // A ---------------- B ---------------------C + // / \ / \ / \ + // A1 A2 B1 B2 C1 C2 + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + populate_tree_create_two_by_two_hierarchy(&db); + + // Constructing the PathQuery + let path_query = PathQuery { + path: vec![], + query: SizedQuery { + query: Query { + items: vec![QueryItem::RangeFrom(b"B".to_vec()..)], + default_subquery_branch: SubqueryBranch { + subquery_path: None, + subquery: Some(Query::new_range_full().into()), + }, + left_to_right: true, + conditional_subquery_branches: None, + }, + limit: Some(2), + offset: None, + }, + }; + + let (elements, _) = db + .query_item_value(&path_query, true, true, true, None, grove_version) + .unwrap() + .expect("expected successful get_path_query"); + + assert_eq!(elements.len(), 2); + + let proof = db + .prove_query(&path_query, None, grove_version) + .value + .expect("expected successful get_path_query"); + + let (_, result_set) = GroveDb::verify_query(proof.as_slice(), &path_query, grove_version) + .expect("should verify proof"); + assert_eq!(result_set.len(), 2); + } + + #[test] + fn test_path_query_items_with_subquery_and_limit_2_desc_in_middle() { + // The structure is the following + // ----------------------------------------------------------> + // A ---------------- B ---------------------C + // / \ / \ / \ + // A1 A2 B1 B2 C1 C2 + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + populate_tree_create_two_by_two_hierarchy(&db); + + // Constructing the PathQuery + let path_query = PathQuery { + path: vec![], + query: SizedQuery { + query: Query { + items: vec![QueryItem::RangeToInclusive(..=b"B".to_vec())], + default_subquery_branch: SubqueryBranch { + subquery_path: None, + subquery: Some(Query::new_range_full().into()), + }, + left_to_right: false, + conditional_subquery_branches: None, + }, + limit: Some(2), + offset: None, + }, + }; + + let (elements, _) = db + .query_item_value(&path_query, true, true, true, None, grove_version) + .unwrap() + .expect("expected successful get_path_query"); + + assert_eq!(elements.len(), 2); + + let proof = db + .prove_query(&path_query, None, grove_version) + .value + .expect("expected successful get_path_query"); + + let (_, result_set) = GroveDb::verify_query(proof.as_slice(), &path_query, grove_version) + .expect("should verify proof"); + assert_eq!(result_set.len(), 2); + } + + #[test] + fn test_path_query_items_with_subquery_and_limit_2_asc_at_end() { + // The structure is the following + // ----------------------------------------------------------> + // A ---------------- B ---------------------C + // / \ / \ / \ + // A1 A2 B1 B2 C1 C2 + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + populate_tree_create_two_by_two_hierarchy(&db); + + // Constructing the PathQuery + let path_query = PathQuery { + path: vec![], + query: SizedQuery { + query: Query { + items: vec![QueryItem::RangeFrom(b"M".to_vec()..)], + default_subquery_branch: SubqueryBranch { + subquery_path: None, + subquery: Some(Query::new_range_full().into()), + }, + left_to_right: true, + conditional_subquery_branches: None, + }, + limit: Some(2), + offset: None, + }, + }; + + let (elements, _) = db + .query_item_value(&path_query, true, true, true, None, grove_version) + .unwrap() + .expect("expected successful get_path_query"); + + assert_eq!(elements.len(), 2); + + let proof = db + .prove_query(&path_query, None, grove_version) + .value + .expect("expected successful get_path_query"); + + let (_, result_set) = GroveDb::verify_query(proof.as_slice(), &path_query, grove_version) + .expect("should verify proof"); + assert_eq!(result_set.len(), 2); + } + + #[test] + fn test_path_query_items_with_subquery_and_limit_2_desc_at_end() { + // The structure is the following + // ----------------------------------------------------------> + // A ---------------- B ---------------------C + // / \ / \ / \ + // A1 A2 B1 B2 C1 C2 + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + populate_tree_create_two_by_two_hierarchy(&db); + + // Constructing the PathQuery + let path_query = PathQuery { + path: vec![], + query: SizedQuery { + query: Query { + items: vec![QueryItem::RangeToInclusive(..=b"M".to_vec())], + default_subquery_branch: SubqueryBranch { + subquery_path: None, + subquery: Some(Query::new_range_full().into()), + }, + left_to_right: false, + conditional_subquery_branches: None, + }, + limit: Some(2), + offset: None, + }, + }; + + let (elements, _) = db + .query_item_value(&path_query, true, true, true, None, grove_version) + .unwrap() + .expect("expected successful get_path_query"); + + assert_eq!(elements.len(), 2); + + let proof = db + .prove_query(&path_query, None, grove_version) + .value + .expect("expected successful get_path_query"); + + let (_, result_set) = GroveDb::verify_query(proof.as_slice(), &path_query, grove_version) + .expect("should verify proof"); + assert_eq!(result_set.len(), 2); + } + + #[test] + fn test_path_query_items_with_intermediate_path_limit_2_asc_from_start() { + // The structure is the following + // ----------------------------------------------------------> + // A ---------------- B ---------------------C + // | | | + // 0 0 0 + // / \ / \ / \ + // A1 A2 B1 B2 C1 C2 + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + populate_tree_create_two_by_two_hierarchy_with_intermediate_value(&db); + + // Constructing the PathQuery + let path_query = PathQuery { + path: vec![], + query: SizedQuery { + query: Query { + items: vec![QueryItem::RangeFull(RangeFull)], + default_subquery_branch: SubqueryBranch { + subquery_path: Some(vec![vec![0]]), + subquery: Some(Query::new_range_full().into()), + }, + left_to_right: true, + conditional_subquery_branches: None, + }, + limit: Some(2), + offset: None, + }, + }; + + let (elements, _) = db + .query_item_value(&path_query, true, true, true, None, grove_version) + .unwrap() + .expect("expected successful get_path_query"); + + assert_eq!(elements.len(), 2); + + let proof = db + .prove_query(&path_query, None, grove_version) + .value + .expect("expected successful get_path_query"); + + let (_, result_set) = GroveDb::verify_query(proof.as_slice(), &path_query, grove_version) + .expect("should verify proof"); + assert_eq!(result_set.len(), 2); + } + + #[test] + fn test_path_query_items_with_intermediate_path_limit_2_desc_from_start() { + // The structure is the following + // ----------------------------------------------------------> + // A ---------------- B ---------------------C + // | | | + // 0 0 0 + // / \ / \ / \ + // A1 A2 B1 B2 C1 C2 + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + populate_tree_create_two_by_two_hierarchy_with_intermediate_value(&db); + + // Constructing the PathQuery + let path_query = PathQuery { + path: vec![], + query: SizedQuery { + query: Query { + items: vec![QueryItem::RangeToInclusive(..=b"A".to_vec())], + default_subquery_branch: SubqueryBranch { + subquery_path: Some(vec![vec![0]]), + subquery: Some(Query::new_range_full().into()), + }, + left_to_right: false, + conditional_subquery_branches: None, + }, + limit: Some(2), + offset: None, + }, + }; + + let (elements, _) = db + .query_item_value(&path_query, true, true, true, None, grove_version) + .unwrap() + .expect("expected successful get_path_query"); + + assert_eq!(elements.len(), 2); + + let proof = db + .prove_query(&path_query, None, grove_version) + .value + .expect("expected successful get_path_query"); + + let (_, result_set) = GroveDb::verify_query(proof.as_slice(), &path_query, grove_version) + .expect("should verify proof"); + assert_eq!(result_set.len(), 2); + } + + #[test] + fn test_path_query_items_with_intermediate_path_limit_2_asc_in_middle() { + // The structure is the following + // ----------------------------------------------------------> + // A ---------------- B ---------------------C + // | | | + // 0 0 0 + // / \ / \ / \ + // A1 A2 B1 B2 C1 C2 + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + populate_tree_create_two_by_two_hierarchy_with_intermediate_value(&db); + + // Constructing the PathQuery + let path_query = PathQuery { + path: vec![], + query: SizedQuery { + query: Query { + items: vec![QueryItem::RangeFrom(b"B".to_vec()..)], + default_subquery_branch: SubqueryBranch { + subquery_path: Some(vec![vec![0]]), + subquery: Some(Query::new_range_full().into()), + }, + left_to_right: true, + conditional_subquery_branches: None, + }, + limit: Some(2), + offset: None, + }, + }; + + let (elements, _) = db + .query_item_value(&path_query, true, true, true, None, grove_version) + .unwrap() + .expect("expected successful get_path_query"); + + assert_eq!(elements.len(), 2); + + let proof = db + .prove_query(&path_query, None, grove_version) + .value + .expect("expected successful get_path_query"); + + let (_, result_set) = GroveDb::verify_query(proof.as_slice(), &path_query, grove_version) + .expect("should verify proof"); + assert_eq!(result_set.len(), 2); + } + + #[test] + fn test_path_query_items_with_intermediate_path_limit_2_desc_in_middle() { + // The structure is the following + // ----------------------------------------------------------> + // A ---------------- B ---------------------C + // | | | + // 0 0 0 + // / \ / \ / \ + // A1 A2 B1 B2 C1 C2 + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + populate_tree_create_two_by_two_hierarchy_with_intermediate_value(&db); + + // Constructing the PathQuery + let path_query = PathQuery { + path: vec![], + query: SizedQuery { + query: Query { + items: vec![QueryItem::RangeToInclusive(..=b"B".to_vec())], + default_subquery_branch: SubqueryBranch { + subquery_path: Some(vec![vec![0]]), + subquery: Some(Query::new_range_full().into()), + }, + left_to_right: false, + conditional_subquery_branches: None, + }, + limit: Some(2), + offset: None, + }, + }; + + let (elements, _) = db + .query_item_value(&path_query, true, true, true, None, grove_version) + .unwrap() + .expect("expected successful get_path_query"); + + assert_eq!(elements.len(), 2); + + let proof = db + .prove_query(&path_query, None, grove_version) + .value + .expect("expected successful get_path_query"); + + let (_, result_set) = GroveDb::verify_query(proof.as_slice(), &path_query, grove_version) + .expect("should verify proof"); + assert_eq!(result_set.len(), 2); + } + + #[test] + fn test_path_query_items_with_intermediate_path_limit_2_asc_not_included_in_middle() { + // The structure is the following + // ----------------------------------------------------------> + // A ---------------- B ---------------------C + // | | | + // 0 0 0 + // / \ / \ / \ + // A1 A2 B1 B2 C1 C2 + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + populate_tree_create_two_by_two_hierarchy_with_intermediate_value(&db); + + // Constructing the PathQuery + let path_query = PathQuery { + path: vec![], + query: SizedQuery { + query: Query { + items: vec![QueryItem::RangeTo(..b"F".to_vec())], + default_subquery_branch: SubqueryBranch { + subquery_path: Some(vec![vec![0]]), + subquery: Some(Query::new_range_full().into()), + }, + left_to_right: true, + conditional_subquery_branches: None, + }, + limit: Some(2), + offset: None, + }, + }; + + let (elements, _) = db + .query_item_value(&path_query, true, true, true, None, grove_version) + .unwrap() + .expect("expected successful get_path_query"); + + assert_eq!(elements.len(), 2); + + let proof = db + .prove_query(&path_query, None, grove_version) + .value + .expect("expected successful get_path_query"); + + let (_, result_set) = GroveDb::verify_query(proof.as_slice(), &path_query, grove_version) + .expect("should verify proof"); + assert_eq!(result_set.len(), 2); + } + + #[test] + fn test_path_query_items_with_intermediate_path_limit_2_desc_not_included_in_middle() { + // The structure is the following + // ----------------------------------------------------------> + // A ---------------- B ---------------------C + // | | | + // 0 0 0 + // / \ / \ / \ + // A1 A2 B1 B2 C1 C2 + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + populate_tree_create_two_by_two_hierarchy_with_intermediate_value(&db); + + // Constructing the PathQuery + let path_query = PathQuery { + path: vec![], + query: SizedQuery { + query: Query { + items: vec![QueryItem::RangeTo(..b"F".to_vec())], + default_subquery_branch: SubqueryBranch { + subquery_path: Some(vec![vec![0]]), + subquery: Some(Query::new_range_full().into()), + }, + left_to_right: false, + conditional_subquery_branches: None, + }, + limit: Some(2), + offset: None, + }, + }; + + let (elements, _) = db + .query_item_value(&path_query, true, true, true, None, grove_version) + .unwrap() + .expect("expected successful get_path_query"); + + assert_eq!(elements.len(), 2); + + let proof = db + .prove_query(&path_query, None, grove_version) + .value + .expect("expected successful get_path_query"); + + let (_, result_set) = GroveDb::verify_query(proof.as_slice(), &path_query, grove_version) + .expect("should verify proof"); + assert_eq!(result_set.len(), 2); + } + + #[test] + fn test_path_query_items_with_intermediate_path_limit_2_asc_at_end() { + // The structure is the following + // ----------------------------------------------------------> + // A ---------------- B ---------------------C + // | | | + // 0 0 0 + // / \ / \ / \ + // A1 A2 B1 B2 C1 C2 + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + populate_tree_create_two_by_two_hierarchy_with_intermediate_value(&db); + + // Constructing the PathQuery + let path_query = PathQuery { + path: vec![], + query: SizedQuery { + query: Query { + items: vec![QueryItem::RangeFrom(b"M".to_vec()..)], + default_subquery_branch: SubqueryBranch { + subquery_path: Some(vec![vec![0]]), + subquery: Some(Query::new_range_full().into()), + }, + left_to_right: true, + conditional_subquery_branches: None, + }, + limit: Some(2), + offset: None, + }, + }; + + let (elements, _) = db + .query_item_value(&path_query, true, true, true, None, grove_version) + .unwrap() + .expect("expected successful get_path_query"); + + assert_eq!(elements.len(), 2); + + let proof = db + .prove_query(&path_query, None, grove_version) + .value + .expect("expected successful get_path_query"); + + let (_, result_set) = GroveDb::verify_query(proof.as_slice(), &path_query, grove_version) + .expect("should verify proof"); + assert_eq!(result_set.len(), 2); + } + + #[test] + fn test_path_query_items_with_intermediate_path_limit_2_desc_at_end() { + // The structure is the following + // ----------------------------------------------------------> + // A ---------------- B ---------------------C + // | | | + // 0 0 0 + // / \ / \ / \ + // A1 A2 B1 B2 C1 C2 + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + populate_tree_create_two_by_two_hierarchy_with_intermediate_value(&db); + + // Constructing the PathQuery + let path_query = PathQuery { + path: vec![], + query: SizedQuery { + query: Query { + items: vec![QueryItem::RangeToInclusive(..=b"M".to_vec())], + default_subquery_branch: SubqueryBranch { + subquery_path: Some(vec![vec![0]]), + subquery: Some(Query::new_range_full().into()), + }, + left_to_right: false, + conditional_subquery_branches: None, + }, + limit: Some(2), + offset: None, + }, + }; + + let (elements, _) = db + .query_item_value(&path_query, true, true, true, None, grove_version) + .unwrap() + .expect("expected successful get_path_query"); + + assert_eq!(elements.len(), 2); + + let proof = db + .prove_query(&path_query, None, grove_version) + .value + .expect("expected successful get_path_query"); + + let (_, result_set) = GroveDb::verify_query(proof.as_slice(), &path_query, grove_version) + .expect("should verify proof"); + assert_eq!(result_set.len(), 2); + } + + #[test] + fn test_path_query_items_with_reference_limit_2_asc_from_start() { + // The structure is the following + // ----------------------------------------------------------> + // a ------------------------b------------------------c + // / \ / \ / \ + // 0 1 0 1 0 1 + // / \ / / \ / / \ / + // A1 A2 2 B1 B2 2 C1 C2 2 + // / \ / \ / \ + // refA1 refA2 refB1 refB2 refC1 refC2 + + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + populate_tree_create_two_by_two_reference_hierarchy_with_intermediate_value(&db); + + // Constructing the PathQuery + let path_query = PathQuery { + path: vec![], + query: SizedQuery { + query: Query { + items: vec![QueryItem::RangeFull(RangeFull)], + default_subquery_branch: SubqueryBranch { + subquery_path: Some(vec![vec![1], vec![2]]), + subquery: Some(Query::new_range_full().into()), + }, + left_to_right: true, + conditional_subquery_branches: None, + }, + limit: Some(2), + offset: None, + }, + }; + + let (elements, _) = db + .query_item_value(&path_query, true, true, true, None, grove_version) + .unwrap() + .expect("expected successful get_path_query"); + + assert_eq!(elements.len(), 2); + + let proof = db + .prove_query(&path_query, None, grove_version) + .value + .expect("expected successful get_path_query"); + + let (_, result_set) = GroveDb::verify_query(proof.as_slice(), &path_query, grove_version) + .expect("should verify proof"); + assert_eq!(result_set.len(), 2); + } + + #[test] + fn test_path_query_items_with_reference_limit_2_desc_from_start() { + // The structure is the following + // ----------------------------------------------------------> + // a ------------------------b------------------------c + // / \ / \ / \ + // 0 1 0 1 0 1 + // / \ / / \ / / \ / + // A1 A2 2 B1 B2 2 C1 C2 2 + // / \ / \ / \ + // refA1 refA2 refB1 refB2 refC1 refC2 + + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + populate_tree_create_two_by_two_reference_hierarchy_with_intermediate_value(&db); + + // Constructing the PathQuery + let path_query = PathQuery { + path: vec![], + query: SizedQuery { + query: Query { + items: vec![QueryItem::RangeToInclusive(..=b"a".to_vec())], + default_subquery_branch: SubqueryBranch { + subquery_path: Some(vec![vec![1], vec![2]]), + subquery: Some(Query::new_range_full().into()), + }, + left_to_right: false, + conditional_subquery_branches: None, + }, + limit: Some(2), + offset: None, + }, + }; + + let (elements, _) = db + .query_item_value(&path_query, true, true, true, None, grove_version) + .unwrap() + .expect("expected successful get_path_query"); + + assert_eq!(elements.len(), 2); + + let proof = db + .prove_query(&path_query, None, grove_version) + .value + .expect("expected successful get_path_query"); + + let (_, result_set) = GroveDb::verify_query(proof.as_slice(), &path_query, grove_version) + .expect("should verify proof"); + assert_eq!(result_set.len(), 2); + } + + #[test] + fn test_path_query_items_with_reference_limit_2_asc_in_middle() { + // The structure is the following + // ----------------------------------------------------------> + // a ------------------------b------------------------c + // / \ / \ / \ + // 0 1 0 1 0 1 + // / \ / / \ / / \ / + // A1 A2 2 B1 B2 2 C1 C2 2 + // / \ / \ / \ + // refA1 refA2 refB1 refB2 refC1 refC2 + + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + populate_tree_create_two_by_two_reference_hierarchy_with_intermediate_value(&db); + + // Constructing the PathQuery + let path_query = PathQuery { + path: vec![], + query: SizedQuery { + query: Query { + items: vec![QueryItem::RangeFrom(b"b".to_vec()..)], + default_subquery_branch: SubqueryBranch { + subquery_path: Some(vec![vec![1], vec![2]]), + subquery: Some(Query::new_range_full().into()), + }, + left_to_right: true, + conditional_subquery_branches: None, + }, + limit: Some(2), + offset: None, + }, + }; + + let (elements, _) = db + .query_item_value(&path_query, true, true, true, None, grove_version) + .unwrap() + .expect("expected successful get_path_query"); + + assert_eq!(elements.len(), 2); + + let proof = db + .prove_query(&path_query, None, grove_version) + .value + .expect("expected successful get_path_query"); + + let (_, result_set) = GroveDb::verify_query(proof.as_slice(), &path_query, grove_version) + .expect("should verify proof"); + assert_eq!(result_set.len(), 2); + } + + #[test] + fn test_path_query_items_with_reference_limit_2_desc_in_middle() { + // The structure is the following + // ----------------------------------------------------------> + // a ------------------------b------------------------c + // / \ / \ / \ + // 0 1 0 1 0 1 + // / \ / / \ / / \ / + // A1 A2 2 B1 B2 2 C1 C2 2 + // / \ / \ / \ + // refA1 refA2 refB1 refB2 refC1 refC2 + + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + populate_tree_create_two_by_two_reference_hierarchy_with_intermediate_value(&db); + + // Constructing the PathQuery + let path_query = PathQuery { + path: vec![], + query: SizedQuery { + query: Query { + items: vec![QueryItem::RangeToInclusive(..=b"b".to_vec())], + default_subquery_branch: SubqueryBranch { + subquery_path: Some(vec![vec![1], vec![2]]), + subquery: Some(Query::new_range_full().into()), + }, + left_to_right: false, + conditional_subquery_branches: None, + }, + limit: Some(2), + offset: None, + }, + }; + + let (elements, _) = db + .query_item_value(&path_query, true, true, true, None, grove_version) + .unwrap() + .expect("expected successful get_path_query"); + + assert_eq!(elements.len(), 2); + + let proof = db + .prove_query(&path_query, None, grove_version) + .value + .expect("expected successful get_path_query"); + + let (_, result_set) = GroveDb::verify_query(proof.as_slice(), &path_query, grove_version) + .expect("should verify proof"); + assert_eq!(result_set.len(), 2); + } + + #[test] + fn test_path_query_items_with_reference_limit_2_asc_not_included_in_middle() { + // The structure is the following + // ----------------------------------------------------------> + // a ------------------------b------------------------c + // / \ / \ / \ + // 0 1 0 1 0 1 + // / \ / / \ / / \ / + // A1 A2 2 B1 B2 2 C1 C2 2 + // / \ / \ / \ + // refA1 refA2 refB1 refB2 refC1 refC2 + + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + populate_tree_create_two_by_two_reference_hierarchy_with_intermediate_value(&db); + + // Constructing the PathQuery + let path_query = PathQuery { + path: vec![], + query: SizedQuery { + query: Query { + items: vec![QueryItem::RangeTo(..b"f".to_vec())], + default_subquery_branch: SubqueryBranch { + subquery_path: Some(vec![vec![1], vec![2]]), + subquery: Some(Query::new_range_full().into()), + }, + left_to_right: true, + conditional_subquery_branches: None, + }, + limit: Some(2), + offset: None, + }, + }; + + let (elements, _) = db + .query_item_value(&path_query, true, true, true, None, grove_version) + .unwrap() + .expect("expected successful get_path_query"); + + assert_eq!(elements.len(), 2); + + let proof = db + .prove_query(&path_query, None, grove_version) + .value + .expect("expected successful get_path_query"); + + let (_, result_set) = GroveDb::verify_query(proof.as_slice(), &path_query, grove_version) + .expect("should verify proof"); + assert_eq!(result_set.len(), 2); + } + + #[test] + fn test_path_query_items_with_reference_limit_2_desc_not_included_in_middle() { + // The structure is the following + // ----------------------------------------------------------> + // a ------------------------b------------------------c + // / \ / \ / \ + // 0 1 0 1 0 1 + // / \ / / \ / / \ / + // A1 A2 2 B1 B2 2 C1 C2 2 + // / \ / \ / \ + // refA1 refA2 refB1 refB2 refC1 refC2 + + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + populate_tree_create_two_by_two_reference_hierarchy_with_intermediate_value(&db); + + // Constructing the PathQuery + let path_query = PathQuery { + path: vec![], + query: SizedQuery { + query: Query { + items: vec![QueryItem::RangeTo(..b"f".to_vec())], + default_subquery_branch: SubqueryBranch { + subquery_path: Some(vec![vec![1], vec![2]]), + subquery: Some(Query::new_range_full().into()), + }, + left_to_right: false, + conditional_subquery_branches: None, + }, + limit: Some(2), + offset: None, + }, + }; + + let (elements, _) = db + .query_item_value(&path_query, true, true, true, None, grove_version) + .unwrap() + .expect("expected successful get_path_query"); + + assert_eq!(elements.len(), 2); + + let proof = db + .prove_query(&path_query, None, grove_version) + .value + .expect("expected successful get_path_query"); + + let (_, result_set) = GroveDb::verify_query(proof.as_slice(), &path_query, grove_version) + .expect("should verify proof"); + assert_eq!(result_set.len(), 2); + } + + #[test] + fn test_path_query_items_with_reference_limit_2_asc_at_end() { + // The structure is the following + // ----------------------------------------------------------> + // a ------------------------b------------------------c + // / \ / \ / \ + // 0 1 0 1 0 1 + // / \ / / \ / / \ / + // A1 A2 2 B1 B2 2 C1 C2 2 + // / \ / \ / \ + // refA1 refA2 refB1 refB2 refC1 refC2 + + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + populate_tree_create_two_by_two_reference_hierarchy_with_intermediate_value(&db); + + // Constructing the PathQuery + let path_query = PathQuery { + path: vec![], + query: SizedQuery { + query: Query { + items: vec![QueryItem::RangeFrom(b"m".to_vec()..)], + default_subquery_branch: SubqueryBranch { + subquery_path: Some(vec![vec![1], vec![2]]), + subquery: Some(Query::new_range_full().into()), + }, + left_to_right: true, + conditional_subquery_branches: None, + }, + limit: Some(2), + offset: None, + }, + }; + + let (elements, _) = db + .query_item_value(&path_query, true, true, true, None, grove_version) + .unwrap() + .expect("expected successful get_path_query"); + + assert_eq!(elements.len(), 2); + + let proof = db + .prove_query(&path_query, None, grove_version) + .value + .expect("expected successful get_path_query"); + + let (_, result_set) = GroveDb::verify_query(proof.as_slice(), &path_query, grove_version) + .expect("should verify proof"); + assert_eq!(result_set.len(), 2); + } + + #[test] + fn test_path_query_items_with_reference_limit_2_desc_at_end() { + // The structure is the following + // ----------------------------------------------------------> + // a ------------------------b------------------------c + // / \ / \ / \ + // 0 1 0 1 0 1 + // / \ / / \ / / \ / + // A1 A2 2 B1 B2 2 C1 C2 2 + // / \ / \ / \ + // refA1 refA2 refB1 refB2 refC1 refC2 + + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + populate_tree_create_two_by_two_reference_hierarchy_with_intermediate_value(&db); + + // Constructing the PathQuery + let path_query = PathQuery { + path: vec![], + query: SizedQuery { + query: Query { + items: vec![QueryItem::RangeToInclusive(..=b"m".to_vec())], + default_subquery_branch: SubqueryBranch { + subquery_path: Some(vec![vec![1], vec![2]]), + subquery: Some(Query::new_range_full().into()), + }, + left_to_right: false, + conditional_subquery_branches: None, + }, + limit: Some(2), + offset: None, + }, + }; + + let (elements, _) = db + .query_item_value(&path_query, true, true, true, None, grove_version) + .unwrap() + .expect("expected successful get_path_query"); + + assert_eq!(elements.len(), 2); + + let proof = db + .prove_query(&path_query, None, grove_version) + .value + .expect("expected successful get_path_query"); + + let (_, result_set) = GroveDb::verify_query(proof.as_slice(), &path_query, grove_version) + .expect("should verify proof"); + assert_eq!(result_set.len(), 2); + } + + #[test] + fn test_path_query_items_held_in_top_tree_with_refs_limit_2_asc_from_start() { + // The structure is the following + // ----------------------------------------------------------> + // 0 -------------------------------- 1 + // / \ / | \ + // A1 .. C2 a ------------------------b------------------------c + // | | | + // 0 0 0 + // / \ / \ / \ + // refA1 refA2 refB1 refB2 refC1 refC2 + + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + populate_tree_create_two_by_two_reference_higher_up_hierarchy_with_intermediate_value(&db); + + // Constructing the PathQuery + let path_query = PathQuery { + path: vec![vec![1]], + query: SizedQuery { + query: Query { + items: vec![QueryItem::RangeFull(RangeFull)], + default_subquery_branch: SubqueryBranch { + subquery_path: Some(vec![vec![0]]), + subquery: Some(Query::new_range_full().into()), + }, + left_to_right: true, + conditional_subquery_branches: None, + }, + limit: Some(2), + offset: None, + }, + }; + + let (elements, _) = db + .query_item_value(&path_query, true, true, true, None, grove_version) + .unwrap() + .expect("expected successful get_path_query"); + + assert_eq!(elements.len(), 2); + + let proof = db + .prove_query(&path_query, None, grove_version) + .value + .expect("expected successful get_path_query"); + + let (_, result_set) = GroveDb::verify_query(proof.as_slice(), &path_query, grove_version) + .expect("should verify proof"); + assert_eq!(result_set.len(), 2); + } + + #[test] + fn test_path_query_items_held_in_top_tree_with_refs_limit_2_desc_from_start() { + // The structure is the following + // ----------------------------------------------------------> + // 0 -------------------------------- 1 + // / \ / | \ + // A1 .. C2 a ------------------------b------------------------c + // | | | + // 0 0 0 + // / \ / \ / \ + // refA1 refA2 refB1 refB2 refC1 refC2 + + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + populate_tree_create_two_by_two_reference_higher_up_hierarchy_with_intermediate_value(&db); + + // Constructing the PathQuery + let path_query = PathQuery { + path: vec![vec![1]], + query: SizedQuery { + query: Query { + items: vec![QueryItem::RangeToInclusive(..=b"a".to_vec())], + default_subquery_branch: SubqueryBranch { + subquery_path: Some(vec![vec![0]]), + subquery: Some(Query::new_range_full().into()), + }, + left_to_right: false, + conditional_subquery_branches: None, + }, + limit: Some(2), + offset: None, + }, + }; + + let (elements, _) = db + .query_item_value(&path_query, true, true, true, None, grove_version) + .unwrap() + .expect("expected successful get_path_query"); + + assert_eq!(elements.len(), 2); + + let proof = db + .prove_query(&path_query, None, grove_version) + .value + .expect("expected successful get_path_query"); + + let (_, result_set) = GroveDb::verify_query(proof.as_slice(), &path_query, grove_version) + .expect("should verify proof"); + assert_eq!(result_set.len(), 2); + } + + #[test] + fn test_path_query_items_held_in_top_tree_with_refs_limit_2_asc_in_middle() { + // The structure is the following + // ----------------------------------------------------------> + // 0 -------------------------------- 1 + // / \ / | \ + // A1 .. C2 a ------------------------b------------------------c + // | | | + // 0 0 0 + // / \ / \ / \ + // refA1 refA2 refB1 refB2 refC1 refC2 + + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + populate_tree_create_two_by_two_reference_higher_up_hierarchy_with_intermediate_value(&db); + + // Constructing the PathQuery + let path_query = PathQuery { + path: vec![vec![1]], + query: SizedQuery { + query: Query { + items: vec![QueryItem::RangeFrom(b"b".to_vec()..)], + default_subquery_branch: SubqueryBranch { + subquery_path: Some(vec![vec![0]]), + subquery: Some(Query::new_range_full().into()), + }, + left_to_right: true, + conditional_subquery_branches: None, + }, + limit: Some(2), + offset: None, + }, + }; + + let (elements, _) = db + .query_item_value(&path_query, true, true, true, None, grove_version) + .unwrap() + .expect("expected successful get_path_query"); + + assert_eq!(elements.len(), 2); + + let proof = db + .prove_query(&path_query, None, grove_version) + .value + .expect("expected successful get_path_query"); + + let (_, result_set) = GroveDb::verify_query(proof.as_slice(), &path_query, grove_version) + .expect("should verify proof"); + assert_eq!(result_set.len(), 2); + } + + #[test] + fn test_path_query_items_held_in_top_tree_with_refs_limit_2_desc_in_middle() { + // The structure is the following + // ----------------------------------------------------------> + // 0 -------------------------------- 1 + // / \ / | \ + // A1 .. C2 a ------------------------b------------------------c + // | | | + // 0 0 0 + // / \ / \ / \ + // refA1 refA2 refB1 refB2 refC1 refC2 + + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + populate_tree_create_two_by_two_reference_higher_up_hierarchy_with_intermediate_value(&db); + + // Constructing the PathQuery + let path_query = PathQuery { + path: vec![vec![1]], + query: SizedQuery { + query: Query { + items: vec![QueryItem::RangeToInclusive(..=b"b".to_vec())], + default_subquery_branch: SubqueryBranch { + subquery_path: Some(vec![vec![0]]), + subquery: Some(Query::new_range_full().into()), + }, + left_to_right: false, + conditional_subquery_branches: None, + }, + limit: Some(2), + offset: None, + }, + }; + + let (elements, _) = db + .query_item_value(&path_query, true, true, true, None, grove_version) + .unwrap() + .expect("expected successful get_path_query"); + + assert_eq!(elements.len(), 2); + + let proof = db + .prove_query(&path_query, None, grove_version) + .value + .expect("expected successful get_path_query"); + + let (_, result_set) = GroveDb::verify_query(proof.as_slice(), &path_query, grove_version) + .expect("should verify proof"); + assert_eq!(result_set.len(), 2); + } + + #[test] + fn test_path_query_items_held_in_top_tree_with_refs_limit_2_asc_not_included_in_middle() { + // The structure is the following + // ----------------------------------------------------------> + // 0 -------------------------------- 1 + // / \ / | \ + // A1 .. C2 a ------------------------b------------------------c + // | | | + // 0 0 0 + // / \ / \ / \ + // refA1 refA2 refB1 refB2 refC1 refC2 + + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + populate_tree_create_two_by_two_reference_higher_up_hierarchy_with_intermediate_value(&db); + + // Constructing the PathQuery + let path_query = PathQuery { + path: vec![vec![1]], + query: SizedQuery { + query: Query { + items: vec![QueryItem::RangeTo(..b"f".to_vec())], + default_subquery_branch: SubqueryBranch { + subquery_path: Some(vec![vec![0]]), + subquery: Some(Query::new_range_full().into()), + }, + left_to_right: true, + conditional_subquery_branches: None, + }, + limit: Some(2), + offset: None, + }, + }; + + let (elements, _) = db + .query_item_value(&path_query, true, true, true, None, grove_version) + .unwrap() + .expect("expected successful get_path_query"); + + assert_eq!(elements.len(), 2); + + let proof = db + .prove_query(&path_query, None, grove_version) + .value + .expect("expected successful get_path_query"); + + let (_, result_set) = GroveDb::verify_query(proof.as_slice(), &path_query, grove_version) + .expect("should verify proof"); + assert_eq!(result_set.len(), 2); + } + + #[test] + fn test_path_query_items_held_in_top_tree_with_refs_limit_2_desc_not_included_in_middle() { + // The structure is the following + // ----------------------------------------------------------> + // 0 -------------------------------- 1 + // / \ / | \ + // A1 .. C2 a ------------------------b------------------------c + // | | | + // 0 0 0 + // / \ / \ / \ + // refA1 refA2 refB1 refB2 refC1 refC2 + + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + populate_tree_create_two_by_two_reference_higher_up_hierarchy_with_intermediate_value(&db); + + // Constructing the PathQuery + let path_query = PathQuery { + path: vec![vec![1]], + query: SizedQuery { + query: Query { + items: vec![QueryItem::RangeTo(..b"f".to_vec())], + default_subquery_branch: SubqueryBranch { + subquery_path: Some(vec![vec![0]]), + subquery: Some(Query::new_range_full().into()), + }, + left_to_right: false, + conditional_subquery_branches: Some(IndexMap::from([( + QueryItem::Key(vec![]), + SubqueryBranch { + subquery_path: Some(vec![vec![0]]), + subquery: Some(Query::new_range_full().into()), + }, + )])), + }, + limit: Some(2), + offset: None, + }, + }; + + let (elements, _) = db + .query_item_value(&path_query, true, true, true, None, grove_version) + .unwrap() + .expect("expected successful get_path_query"); + + assert_eq!(elements.len(), 2); + + let proof = db + .prove_query(&path_query, None, grove_version) + .value + .expect("expected successful get_path_query"); + + let config = bincode::config::standard() + .with_big_endian() + .with_no_limit(); + + let gproof: GroveDBProof = bincode::decode_from_slice(&proof, config) + .expect("expected no error") + .0; + + println!("{}", gproof); + + let (_, result_set) = GroveDb::verify_query(proof.as_slice(), &path_query, grove_version) + .expect("should verify proof"); + assert_eq!(result_set.len(), 2); + } + + #[test] + fn test_path_query_items_held_in_top_tree_with_refs_limit_2_asc_at_end() { + // The structure is the following + // ----------------------------------------------------------> + // 0 -------------------------------- 1 + // / \ / | \ + // A1 .. C2 a ------------------------b------------------------c + // | | | + // 0 0 0 + // / \ / \ / \ + // refA1 refA2 refB1 refB2 refC1 refC2 + + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + populate_tree_create_two_by_two_reference_higher_up_hierarchy_with_intermediate_value(&db); + + // Constructing the PathQuery + let path_query = PathQuery { + path: vec![vec![1]], + query: SizedQuery { + query: Query { + items: vec![QueryItem::RangeFrom(b"m".to_vec()..)], + default_subquery_branch: SubqueryBranch { + subquery_path: Some(vec![vec![0]]), + subquery: Some(Query::new_range_full().into()), + }, + left_to_right: true, + conditional_subquery_branches: None, + }, + limit: Some(2), + offset: None, + }, + }; + + let (elements, _) = db + .query_item_value(&path_query, true, true, true, None, grove_version) + .unwrap() + .expect("expected successful get_path_query"); + + assert_eq!(elements.len(), 2); + + let proof = db + .prove_query(&path_query, None, grove_version) + .value + .expect("expected successful get_path_query"); + + let (_, result_set) = GroveDb::verify_query(proof.as_slice(), &path_query, grove_version) + .expect("should verify proof"); + assert_eq!(result_set.len(), 2); + } + + #[test] + fn test_path_query_items_held_in_top_tree_with_refs_limit_2_desc_at_end() { + // The structure is the following + // ----------------------------------------------------------> + // 0 -------------------------------- 1 + // / \ / | \ + // A1 .. C2 a ------------------------b------------------------c + // | | | + // 0 0 0 + // / \ / \ / \ + // refA1 refA2 refB1 refB2 refC1 refC2 + + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + populate_tree_create_two_by_two_reference_higher_up_hierarchy_with_intermediate_value(&db); + + // Constructing the PathQuery + let path_query = PathQuery { + path: vec![vec![1]], + query: SizedQuery { + query: Query { + items: vec![QueryItem::RangeToInclusive(..=b"m".to_vec())], + default_subquery_branch: SubqueryBranch { + subquery_path: Some(vec![vec![0]]), + subquery: Some(Query::new_range_full().into()), + }, + left_to_right: false, + conditional_subquery_branches: None, + }, + limit: Some(2), + offset: None, + }, + }; + + let (elements, _) = db + .query_item_value(&path_query, true, true, true, None, grove_version) + .unwrap() + .expect("expected successful get_path_query"); + + assert_eq!(elements.len(), 2); + + let proof = db + .prove_query(&path_query, None, grove_version) + .value + .expect("expected successful get_path_query"); + + let (_, result_set) = GroveDb::verify_query(proof.as_slice(), &path_query, grove_version) + .expect("should verify proof"); + assert_eq!(result_set.len(), 2); + } } diff --git a/grovedb/src/tests/sum_tree_tests.rs b/grovedb/src/tests/sum_tree_tests.rs index 92df7d73..b255f653 100644 --- a/grovedb/src/tests/sum_tree_tests.rs +++ b/grovedb/src/tests/sum_tree_tests.rs @@ -9,7 +9,7 @@ use grovedb_storage::StorageBatch; use grovedb_version::version::GroveVersion; use crate::{ - batch::GroveDbOp, + batch::QualifiedGroveDbOp, reference_path::ReferencePathType, tests::{make_test_grovedb, TEST_LEAF}, Element, Error, GroveDb, PathQuery, @@ -779,17 +779,17 @@ fn test_sum_tree_with_batches() { let grove_version = GroveVersion::latest(); let db = make_test_grovedb(grove_version); let ops = vec![ - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec()], b"key1".to_vec(), Element::empty_sum_tree(), ), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec(), b"key1".to_vec()], b"a".to_vec(), Element::new_item(vec![214]), ), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec(), b"key1".to_vec()], b"b".to_vec(), Element::new_sum_item(10), @@ -835,7 +835,7 @@ fn test_sum_tree_with_batches() { )); // Create new batch to use existing tree - let ops = vec![GroveDbOp::insert_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec(), b"key1".to_vec()], b"c".to_vec(), Element::new_sum_item(10), @@ -871,42 +871,42 @@ fn test_sum_tree_with_batches() { // Add a new sum tree with its own sum items, should affect sum of original // tree let ops = vec![ - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec(), b"key1".to_vec()], b"d".to_vec(), Element::empty_sum_tree(), ), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec(), b"key1".to_vec(), b"d".to_vec()], b"first".to_vec(), Element::new_sum_item(4), ), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec(), b"key1".to_vec(), b"d".to_vec()], b"second".to_vec(), Element::new_item(vec![4]), ), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec(), b"key1".to_vec()], b"e".to_vec(), Element::empty_sum_tree(), ), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec(), b"key1".to_vec(), b"e".to_vec()], b"first".to_vec(), Element::new_sum_item(12), ), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec(), b"key1".to_vec(), b"e".to_vec()], b"second".to_vec(), Element::new_item(vec![4]), ), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec(), b"key1".to_vec(), b"e".to_vec()], b"third".to_vec(), Element::empty_sum_tree(), ), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![ TEST_LEAF.to_vec(), b"key1".to_vec(), @@ -916,7 +916,7 @@ fn test_sum_tree_with_batches() { b"a".to_vec(), Element::new_sum_item(5), ), - GroveDbOp::insert_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![ TEST_LEAF.to_vec(), b"key1".to_vec(), diff --git a/grovedbg-types/Cargo.toml b/grovedbg-types/Cargo.toml index f1ab09d6..391def86 100644 --- a/grovedbg-types/Cargo.toml +++ b/grovedbg-types/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "grovedbg-types" -version = "1.0.0" +version = "2.0.0" edition = "2021" description = "Common type definitions for data exchange over GroveDBG protocol" authors = ["Evgeny Fomin "] diff --git a/grovedbg-types/src/lib.rs b/grovedbg-types/src/lib.rs index 0019db09..f1cc53bd 100644 --- a/grovedbg-types/src/lib.rs +++ b/grovedbg-types/src/lib.rs @@ -25,7 +25,11 @@ pub struct NodeUpdate { #[serde_as(as = "Option")] pub left_child: Option, #[serde_as(as = "Option")] + pub left_merk_hash: Option, + #[serde_as(as = "Option")] pub right_child: Option, + #[serde_as(as = "Option")] + pub right_merk_hash: Option, #[serde_as(as = "Vec")] pub path: Path, #[serde_as(as = "Base64")] @@ -40,31 +44,7 @@ pub struct NodeUpdate { #[serde_as] #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -pub enum Element { - Subtree { - #[serde_as(as = "Option")] - root_key: Option, - #[serde_as(as = "Option")] - element_flags: Option>, - }, - Sumtree { - #[serde_as(as = "Option")] - root_key: Option, - sum: i64, - #[serde_as(as = "Option")] - element_flags: Option>, - }, - Item { - #[serde_as(as = "Base64")] - value: Vec, - #[serde_as(as = "Option")] - element_flags: Option>, - }, - SumItem { - value: i64, - #[serde_as(as = "Option")] - element_flags: Option>, - }, +pub enum Reference { AbsolutePathReference { #[serde_as(as = "Vec")] path: Path, @@ -112,6 +92,36 @@ pub enum Element { }, } +#[serde_as] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub enum Element { + Subtree { + #[serde_as(as = "Option")] + root_key: Option, + #[serde_as(as = "Option")] + element_flags: Option>, + }, + Sumtree { + #[serde_as(as = "Option")] + root_key: Option, + sum: i64, + #[serde_as(as = "Option")] + element_flags: Option>, + }, + Item { + #[serde_as(as = "Base64")] + value: Vec, + #[serde_as(as = "Option")] + element_flags: Option>, + }, + SumItem { + value: i64, + #[serde_as(as = "Option")] + element_flags: Option>, + }, + Reference(Reference), +} + #[serde_as] #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] pub struct PathQuery { diff --git a/merk/Cargo.toml b/merk/Cargo.toml index 6fcfadd2..2af6432f 100644 --- a/merk/Cargo.toml +++ b/merk/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "grovedb-merk" description = "Merkle key/value store adapted for GroveDB" -version = "1.0.0" +version = "2.0.0" authors = ["Samuel Westrich ", "Wisdom Ogwu ", "Matt Bell "] edition = "2021" license = "MIT" @@ -12,15 +12,16 @@ documentation = "https://docs.rs/grovedb-merk" [dependencies] thiserror = "1.0.58" -grovedb-storage = { version = "1.0.0", path = "../storage", optional = true } +bincode = { version = "2.0.0-rc.3" } +grovedb-storage = { version = "2.0.0", path = "../storage", optional = true } failure = "0.1.8" integer-encoding = "4.0.0" indexmap = "2.2.6" -grovedb-costs = { version = "1.0.0", path = "../costs" } -grovedb-visualize = { version = "1.0.0", path = "../visualize" } -grovedb-path = { version = "1.0.0", path = "../path" } +grovedb-costs = { version = "2.0.0" , path = "../costs" } +grovedb-visualize = { version = "2.0.0", path = "../visualize" } +grovedb-path = { version = "2.0.0", path = "../path" } hex = "0.4.3" -grovedb-version = { version = "1.0.0", path = "../grovedb-version" } +grovedb-version = { version = "2.0.0", path = "../grovedb-version" } [dependencies.time] version = "0.3.34" diff --git a/merk/benches/merk.rs b/merk/benches/merk.rs index e2d55219..62bc6ebb 100644 --- a/merk/benches/merk.rs +++ b/merk/benches/merk.rs @@ -38,10 +38,12 @@ use grovedb_merk::{ }; use grovedb_path::SubtreePath; use grovedb_storage::{rocksdb_storage::test_utils::TempStorage, Storage}; +use grovedb_version::version::GroveVersion; use rand::prelude::*; /// 1 million gets in 2k batches pub fn get(c: &mut Criterion) { + let grove_version = GroveVersion::latest(); let initial_size = 1_000_000; let batch_size = 2_000; let num_batches = initial_size / batch_size; @@ -95,6 +97,7 @@ pub fn get(c: &mut Criterion) { /// 1 million sequential inserts in 2k batches pub fn insert_1m_2k_seq(c: &mut Criterion) { + let grove_version = GroveVersion::latest(); let initial_size = 1_000_000; let batch_size = 2_000; @@ -136,6 +139,7 @@ pub fn insert_1m_2k_seq(c: &mut Criterion) { /// 1 million random inserts in 2k batches pub fn insert_1m_2k_rand(c: &mut Criterion) { + let grove_version = GroveVersion::latest(); let initial_size = 1_000_000; let batch_size = 2_000; @@ -177,6 +181,7 @@ pub fn insert_1m_2k_rand(c: &mut Criterion) { /// 1 million sequential updates in 2k batches pub fn update_1m_2k_seq(c: &mut Criterion) { + let grove_version = GroveVersion::latest(); let initial_size = 1_000_000; let batch_size = 2_000; @@ -237,6 +242,7 @@ pub fn update_1m_2k_seq(c: &mut Criterion) { /// 1 million random updates in 2k batches pub fn update_1m_2k_rand(c: &mut Criterion) { + let grove_version = GroveVersion::latest(); let initial_size = 1_000_000; let batch_size = 2_000; @@ -297,6 +303,7 @@ pub fn update_1m_2k_rand(c: &mut Criterion) { /// 1 million random deletes in 2k batches pub fn delete_1m_2k_rand(c: &mut Criterion) { + let grove_version = GroveVersion::latest(); let initial_size = 1_000_000; let batch_size = 2_000; @@ -382,6 +389,7 @@ pub fn delete_1m_2k_rand(c: &mut Criterion) { /// 1 million random proofs in 2k batches pub fn prove_1m_2k_rand(c: &mut Criterion) { + let grove_version = GroveVersion::latest(); let initial_size = 1_000_000; let batch_size = 2_000; @@ -434,6 +442,7 @@ pub fn prove_1m_2k_rand(c: &mut Criterion) { /// Build 1 million trunk chunks in 2k batches, random pub fn build_trunk_chunk_1m_2k_rand(c: &mut Criterion) { + let grove_version = GroveVersion::latest(); let initial_size = 1_000_000; let batch_size = 2_000; @@ -477,6 +486,7 @@ pub fn build_trunk_chunk_1m_2k_rand(c: &mut Criterion) { /// Chunk producer random 1 million pub fn chunkproducer_rand_1m_1_rand(c: &mut Criterion) { + let grove_version = GroveVersion::latest(); let initial_size = 1_000_000; let batch_size = 2_000; @@ -518,6 +528,7 @@ pub fn chunkproducer_rand_1m_1_rand(c: &mut Criterion) { /// Chunk iter 1 million pub fn chunk_iter_1m_1(c: &mut Criterion) { + let grove_version = GroveVersion::latest(); let initial_size = 1_000_000; let batch_size = 2_000; @@ -565,6 +576,7 @@ pub fn chunk_iter_1m_1(c: &mut Criterion) { /// Restore merk of size 500 pub fn restore_500_1(c: &mut Criterion) { + let grove_version = GroveVersion::latest(); let merk_size = 500; let mut merk = TempMerk::new(grove_version); diff --git a/merk/src/debugger.rs b/merk/src/debugger.rs index c806351d..5dc4710a 100644 --- a/merk/src/debugger.rs +++ b/merk/src/debugger.rs @@ -15,7 +15,9 @@ impl<'a, S: StorageContext<'a>> Merk { key: tree.inner.key_as_slice().to_owned(), value: tree.inner.value_as_slice().to_owned(), left_child: tree.link(true).map(|link| link.key().to_owned()), + left_merk_hash: tree.link(true).map(|link| *link.hash()), right_child: tree.link(false).map(|link| link.key().to_owned()), + right_merk_hash: tree.link(false).map(|link| *link.hash()), value_hash: *tree.inner.kv.value_hash(), kv_digest_hash: *tree.inner.kv.hash(), feature_type: tree.inner.kv.feature_type(), @@ -34,7 +36,9 @@ impl<'a, S: StorageContext<'a>> Merk { key: tree.inner.key_as_slice().to_owned(), value: tree.inner.value_as_slice().to_owned(), left_child: tree.link(true).map(|link| link.key().to_owned()), + left_merk_hash: tree.link(true).map(|link| *link.hash()), right_child: tree.link(false).map(|link| link.key().to_owned()), + right_merk_hash: tree.link(false).map(|link| *link.hash()), value_hash: *tree.inner.kv.value_hash(), kv_digest_hash: *tree.inner.kv.hash(), feature_type: tree.inner.kv.feature_type(), @@ -48,7 +52,9 @@ pub struct NodeDbg { pub key: Vec, pub value: Vec, pub left_child: Option>, + pub left_merk_hash: Option<[u8; 32]>, pub right_child: Option>, + pub right_merk_hash: Option<[u8; 32]>, pub value_hash: CryptoHash, pub kv_digest_hash: CryptoHash, pub feature_type: TreeFeatureType, diff --git a/merk/src/merk/apply.rs b/merk/src/merk/apply.rs index 84b4cb9a..9c5c9ec9 100644 --- a/merk/src/merk/apply.rs +++ b/merk/src/merk/apply.rs @@ -77,6 +77,7 @@ where )) }, None::<&fn(&[u8], &GroveVersion) -> Option>, + &|_old_value, _value| Ok(None), &mut |_costs, _old_value, _value| Ok((false, None)), &mut |_a, key_bytes_to_remove, value_bytes_to_remove| { Ok(( @@ -140,6 +141,7 @@ where options, old_specialized_cost, value_defined_cost_fn, + &|_, _| Ok(None), &mut |_costs, _old_value, _value| Ok((false, None)), &mut |_a, key_bytes_to_remove, value_bytes_to_remove| { Ok(( @@ -169,6 +171,7 @@ where /// None, /// &|k, v| Ok(0), /// None::<&fn(&[u8], &GroveVersion) -> Option>, + /// &|k, v| Ok(None), /// &mut |s, v, o| Ok((false, None)), /// &mut |s, k, v| Ok((NoStorageRemoval, NoStorageRemoval)), /// grove_version, @@ -193,6 +196,7 @@ where /// None, /// &|k, v| Ok(0), /// None::<&fn(&[u8], &GroveVersion) -> Option>, + /// &|k, v| Ok(None), /// &mut |s, v, o| Ok((false, None)), /// &mut |s, k, v| Ok((NoStorageRemoval, NoStorageRemoval)), /// grove_version, @@ -207,6 +211,10 @@ where value_defined_cost_fn: Option< &impl Fn(&[u8], &GroveVersion) -> Option, >, + get_temp_new_value_with_old_flags: &impl Fn( + &Vec, + &Vec, + ) -> Result>, Error>, update_tree_value_based_on_costs: &mut impl FnMut( &StorageCost, &Vec, @@ -254,6 +262,7 @@ where options, old_specialized_cost, value_defined_cost_fn, + get_temp_new_value_with_old_flags, update_tree_value_based_on_costs, section_removal_bytes, grove_version, @@ -278,6 +287,7 @@ where /// None, /// &|k, v| Ok(0), /// None::<&fn(&[u8], &GroveVersion) -> Option>, + /// &|k, v| Ok(None), /// &mut |s, o, v| Ok((false, None)), /// &mut |s, k, v| Ok((NoStorageRemoval, NoStorageRemoval)), /// grove_version, @@ -295,25 +305,27 @@ where /// // deletes key [4,5,6] /// (vec![4, 5, 6], Op::Delete), /// ]; - /// unsafe { store.apply_unchecked::<_, Vec<_>, _, _, _, _>( /// /// /// /// /// ////// + /// unsafe { store.apply_unchecked::<_, Vec<_>, _, _, _, _, _>( /// batch, /// &[], /// None, /// &|k, v| Ok(0), /// None::<&fn(&[u8], &GroveVersion) -> Option>, + /// &|o, v| Ok(None), /// &mut |s, o, v| Ok((false, None)), /// &mut |s, k, v| Ok((NoStorageRemoval, NoStorageRemoval)), /// grove_version, /// ).unwrap().expect(""); /// } /// ``` - pub fn apply_unchecked( + pub fn apply_unchecked( &mut self, batch: &MerkBatch, aux: &AuxMerkBatch, options: Option, old_specialized_cost: &C, value_defined_cost_fn: Option<&V>, + get_temp_new_value_with_old_flags: &T, update_tree_value_based_on_costs: &mut U, section_removal_bytes: &mut R, grove_version: &GroveVersion, @@ -323,6 +335,7 @@ where KA: AsRef<[u8]>, C: Fn(&Vec, &Vec) -> Result, V: Fn(&[u8], &GroveVersion) -> Option, + T: Fn(&Vec, &Vec) -> Result>, Error>, U: FnMut( &StorageCost, &Vec, @@ -342,6 +355,7 @@ where self.source(), old_specialized_cost, value_defined_cost_fn, + get_temp_new_value_with_old_flags, update_tree_value_based_on_costs, section_removal_bytes, grove_version, diff --git a/merk/src/proofs/query/mod.rs b/merk/src/proofs/query/mod.rs index 669940cc..6f1e506a 100644 --- a/merk/src/proofs/query/mod.rs +++ b/merk/src/proofs/query/mod.rs @@ -18,8 +18,15 @@ mod verify; use std::cmp::Ordering; use std::{collections::HashSet, fmt, ops::RangeFull}; +#[cfg(any(feature = "full", feature = "verify"))] +use bincode::{ + enc::write::Writer, + error::{DecodeError, EncodeError}, + BorrowDecode, Decode, Encode, +}; #[cfg(feature = "full")] use grovedb_costs::{cost_return_on_error, CostContext, CostResult, CostsExt, OperationCost}; +#[cfg(feature = "full")] use grovedb_version::version::GroveVersion; #[cfg(any(feature = "full", feature = "verify"))] use indexmap::IndexMap; @@ -61,7 +68,7 @@ pub type Key = Vec; pub type PathKey = (Path, Key); #[cfg(any(feature = "full", feature = "verify"))] -#[derive(Debug, Default, Clone, PartialEq)] +#[derive(Debug, Default, Clone, PartialEq, Encode, Decode)] /// Subquery branch pub struct SubqueryBranch { /// Subquery path @@ -70,6 +77,36 @@ pub struct SubqueryBranch { pub subquery: Option>, } +impl SubqueryBranch { + /// Returns the depth of the subquery branch + /// This depth is how many GroveDB layers down we could query at maximum + #[inline] + pub fn max_depth(&self) -> Option { + self.max_depth_internal(u8::MAX) + } + + /// Returns the depth of the subquery branch + /// This depth is how many GroveDB layers down we could query at maximum + #[inline] + fn max_depth_internal(&self, recursion_limit: u8) -> Option { + if recursion_limit == 0 { + return None; + } + let subquery_path_depth = self.subquery_path.as_ref().map_or(Some(0), |path| { + let path_len = path.len(); + if path_len > u16::MAX as usize { + None + } else { + Some(path_len as u16) + } + })?; + let subquery_depth = self.subquery.as_ref().map_or(Some(0), |query| { + query.max_depth_internal(recursion_limit - 1) + })?; + subquery_path_depth.checked_add(subquery_depth) + } +} + #[cfg(any(feature = "full", feature = "verify"))] /// `Query` represents one or more keys or ranges of keys, which can be used to /// resolve a proof which will include all the requested values. @@ -85,6 +122,112 @@ pub struct Query { pub left_to_right: bool, } +#[cfg(any(feature = "full", feature = "verify"))] +impl Encode for Query { + fn encode(&self, encoder: &mut E) -> Result<(), EncodeError> { + // Encode the items vector + self.items.encode(encoder)?; + + // Encode the default subquery branch + self.default_subquery_branch.encode(encoder)?; + + // Encode the conditional subquery branches + match &self.conditional_subquery_branches { + Some(conditional_subquery_branches) => { + encoder.writer().write(&[1])?; // Write a flag indicating presence of data + // Encode the length of the map + (conditional_subquery_branches.len() as u64).encode(encoder)?; + // Encode each key-value pair in the IndexMap + for (key, value) in conditional_subquery_branches { + key.encode(encoder)?; + value.encode(encoder)?; + } + } + None => { + encoder.writer().write(&[0])?; // Write a flag indicating + // absence of data + } + } + + // Encode the left_to_right boolean + self.left_to_right.encode(encoder)?; + + Ok(()) + } +} + +#[cfg(any(feature = "full", feature = "verify"))] +impl Decode for Query { + fn decode(decoder: &mut D) -> Result { + // Decode the items vector + let items = Vec::::decode(decoder)?; + + // Decode the default subquery branch + let default_subquery_branch = SubqueryBranch::decode(decoder)?; + + // Decode the conditional subquery branches + let conditional_subquery_branches = if u8::decode(decoder)? == 1 { + let len = u64::decode(decoder)? as usize; + let mut map = IndexMap::with_capacity(len); + for _ in 0..len { + let key = QueryItem::decode(decoder)?; + let value = SubqueryBranch::decode(decoder)?; + map.insert(key, value); + } + Some(map) + } else { + None + }; + + // Decode the left_to_right boolean + let left_to_right = bool::decode(decoder)?; + + Ok(Query { + items, + default_subquery_branch, + conditional_subquery_branches, + left_to_right, + }) + } +} + +#[cfg(any(feature = "full", feature = "verify"))] +impl<'de> BorrowDecode<'de> for Query { + fn borrow_decode>( + decoder: &mut D, + ) -> Result { + // Borrow-decode the items vector + let items = Vec::::borrow_decode(decoder)?; + + // Borrow-decode the default subquery branch + let default_subquery_branch = SubqueryBranch::borrow_decode(decoder)?; + + // Borrow-decode the conditional subquery branches + let conditional_subquery_branches = if u8::borrow_decode(decoder)? == 1 { + let len = u64::borrow_decode(decoder)? as usize; + let mut map = IndexMap::with_capacity(len); + for _ in 0..len { + let key = QueryItem::borrow_decode(decoder)?; + let value = SubqueryBranch::borrow_decode(decoder)?; + map.insert(key, value); + } + Some(map) + } else { + None + }; + + // Borrow-decode the left_to_right boolean + let left_to_right = bool::borrow_decode(decoder)?; + + Ok(Query { + items, + default_subquery_branch, + conditional_subquery_branches, + left_to_right, + }) + } +} + #[cfg(any(feature = "full", feature = "verify"))] impl fmt::Display for SubqueryBranch { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { @@ -476,6 +619,33 @@ impl Query { // checks if all searched for items are keys self.items.iter().all(|a| a.is_key()) } + + /// Returns the depth of the subquery branch + /// This depth is how many GroveDB layers down we could query at maximum + pub fn max_depth(&self) -> Option { + self.max_depth_internal(u8::MAX) + } + + /// Returns the depth of the subquery branch + /// This depth is how many GroveDB layers down we could query at maximum + pub(crate) fn max_depth_internal(&self, recursion_limit: u8) -> Option { + let default_subquery_branch_depth = self + .default_subquery_branch + .max_depth_internal(recursion_limit)?; + let conditional_subquery_branches_max_depth = self + .conditional_subquery_branches + .as_ref() + .map_or(Some(0), |condition_subqueries| { + condition_subqueries + .values() + .try_fold(0, |max_depth, conditional_subquery_branch| { + conditional_subquery_branch + .max_depth_internal(recursion_limit) + .map(|depth| max_depth.max(depth)) + }) + })?; + 1u16.checked_add(default_subquery_branch_depth.max(conditional_subquery_branches_max_depth)) + } } #[cfg(feature = "full")] diff --git a/merk/src/proofs/query/query_item/mod.rs b/merk/src/proofs/query/query_item/mod.rs index 7c81a27e..2209f583 100644 --- a/merk/src/proofs/query/query_item/mod.rs +++ b/merk/src/proofs/query/query_item/mod.rs @@ -10,6 +10,7 @@ use std::{ ops::{Range, RangeFrom, RangeFull, RangeInclusive, RangeTo, RangeToInclusive}, }; +use bincode::{enc::write::Writer, error::DecodeError, BorrowDecode, Decode, Encode}; #[cfg(feature = "full")] use grovedb_costs::{CostContext, CostsExt, OperationCost}; #[cfg(feature = "full")] @@ -35,6 +36,175 @@ pub enum QueryItem { RangeAfterToInclusive(RangeInclusive>), } +#[cfg(any(feature = "full", feature = "verify"))] +impl Encode for QueryItem { + fn encode( + &self, + encoder: &mut E, + ) -> Result<(), bincode::error::EncodeError> { + match self { + QueryItem::Key(key) => { + encoder.writer().write(&[0])?; + key.encode(encoder) + } + QueryItem::Range(range) => { + encoder.writer().write(&[1])?; + range.start.encode(encoder)?; + range.end.encode(encoder) + } + QueryItem::RangeInclusive(range) => { + encoder.writer().write(&[2])?; + range.start().encode(encoder)?; + range.end().encode(encoder) + } + QueryItem::RangeFull(_) => { + encoder.writer().write(&[3])?; + Ok(()) + } + QueryItem::RangeFrom(range) => { + encoder.writer().write(&[4])?; + range.start.encode(encoder) + } + QueryItem::RangeTo(range) => { + encoder.writer().write(&[5])?; + range.end.encode(encoder) + } + QueryItem::RangeToInclusive(range) => { + encoder.writer().write(&[6])?; + range.end.encode(encoder) + } + QueryItem::RangeAfter(range) => { + encoder.writer().write(&[7])?; + range.start.encode(encoder) + } + QueryItem::RangeAfterTo(range) => { + encoder.writer().write(&[8])?; + range.start.encode(encoder)?; + range.end.encode(encoder) + } + QueryItem::RangeAfterToInclusive(range) => { + encoder.writer().write(&[9])?; + range.start().encode(encoder)?; + range.end().encode(encoder) + } + } + } +} + +#[cfg(any(feature = "full", feature = "verify"))] +impl Decode for QueryItem { + fn decode(decoder: &mut D) -> Result { + let variant_id = u8::decode(decoder)?; + + match variant_id { + 0 => { + let key = Vec::::decode(decoder)?; + Ok(QueryItem::Key(key)) + } + 1 => { + let start = Vec::::decode(decoder)?; + let end = Vec::::decode(decoder)?; + Ok(QueryItem::Range(start..end)) + } + 2 => { + let start = Vec::::decode(decoder)?; + let end = Vec::::decode(decoder)?; + Ok(QueryItem::RangeInclusive(start..=end)) + } + 3 => Ok(QueryItem::RangeFull(RangeFull)), + 4 => { + let start = Vec::::decode(decoder)?; + Ok(QueryItem::RangeFrom(start..)) + } + 5 => { + let end = Vec::::decode(decoder)?; + Ok(QueryItem::RangeTo(..end)) + } + 6 => { + let end = Vec::::decode(decoder)?; + Ok(QueryItem::RangeToInclusive(..=end)) + } + 7 => { + let start = Vec::::decode(decoder)?; + Ok(QueryItem::RangeAfter(start..)) + } + 8 => { + let start = Vec::::decode(decoder)?; + let end = Vec::::decode(decoder)?; + Ok(QueryItem::RangeAfterTo(start..end)) + } + 9 => { + let start = Vec::::decode(decoder)?; + let end = Vec::::decode(decoder)?; + Ok(QueryItem::RangeAfterToInclusive(start..=end)) + } + _ => Err(DecodeError::UnexpectedVariant { + type_name: "QueryItem", + allowed: &bincode::error::AllowedEnumVariants::Range { min: 0, max: 9 }, + found: variant_id as u32, + }), + } + } +} + +#[cfg(any(feature = "full", feature = "verify"))] +impl<'de> BorrowDecode<'de> for QueryItem { + fn borrow_decode>( + decoder: &mut D, + ) -> Result { + let variant_id = u8::decode(decoder)?; + + match variant_id { + 0 => { + let key = Vec::::borrow_decode(decoder)?; + Ok(QueryItem::Key(key)) + } + 1 => { + let start = Vec::::borrow_decode(decoder)?; + let end = Vec::::borrow_decode(decoder)?; + Ok(QueryItem::Range(start..end)) + } + 2 => { + let start = Vec::::borrow_decode(decoder)?; + let end = Vec::::borrow_decode(decoder)?; + Ok(QueryItem::RangeInclusive(start..=end)) + } + 3 => Ok(QueryItem::RangeFull(RangeFull)), + 4 => { + let start = Vec::::borrow_decode(decoder)?; + Ok(QueryItem::RangeFrom(start..)) + } + 5 => { + let end = Vec::::borrow_decode(decoder)?; + Ok(QueryItem::RangeTo(..end)) + } + 6 => { + let end = Vec::::borrow_decode(decoder)?; + Ok(QueryItem::RangeToInclusive(..=end)) + } + 7 => { + let start = Vec::::borrow_decode(decoder)?; + Ok(QueryItem::RangeAfter(start..)) + } + 8 => { + let start = Vec::::borrow_decode(decoder)?; + let end = Vec::::borrow_decode(decoder)?; + Ok(QueryItem::RangeAfterTo(start..end)) + } + 9 => { + let start = Vec::::borrow_decode(decoder)?; + let end = Vec::::borrow_decode(decoder)?; + Ok(QueryItem::RangeAfterToInclusive(start..=end)) + } + _ => Err(DecodeError::UnexpectedVariant { + type_name: "QueryItem", + allowed: &bincode::error::AllowedEnumVariants::Range { min: 0, max: 9 }, + found: variant_id as u32, + }), + } + } +} + #[cfg(any(feature = "full", feature = "verify"))] impl fmt::Display for QueryItem { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { diff --git a/merk/src/proofs/query/verify.rs b/merk/src/proofs/query/verify.rs index e1d56511..726145cf 100644 --- a/merk/src/proofs/query/verify.rs +++ b/merk/src/proofs/query/verify.rs @@ -164,7 +164,7 @@ impl Query { Some(Node::KVValueHash(..)) => {} // cannot verify lower bound - we have an abridged - // tree so we cannot tell what the preceding key was + // tree, so we cannot tell what the preceding key was Some(_) => { return Err(Error::InvalidProofError( "Cannot verify lower bound of queried range".to_string(), diff --git a/merk/src/test_utils/mod.rs b/merk/src/test_utils/mod.rs index 397ad13a..45beda4f 100644 --- a/merk/src/test_utils/mod.rs +++ b/merk/src/test_utils/mod.rs @@ -94,6 +94,7 @@ pub fn apply_memonly_unchecked( )) }, None::<&fn(&[u8], &GroveVersion) -> Option>, + &|_, _| Ok(None), &mut |_, _, _| Ok((false, None)), &mut |_flags, key_bytes_to_remove, value_bytes_to_remove| { Ok(( @@ -153,6 +154,7 @@ pub fn apply_to_memonly( )) }, None::<&fn(&[u8], &GroveVersion) -> Option>, + &|_, _| Ok(None), &mut |_, _, _| Ok((false, None)), &mut |_flags, key_bytes_to_remove, value_bytes_to_remove| { Ok(( diff --git a/merk/src/tree/just_in_time_value_update.rs b/merk/src/tree/just_in_time_value_update.rs index 20861ec4..fd202670 100644 --- a/merk/src/tree/just_in_time_value_update.rs +++ b/merk/src/tree/just_in_time_value_update.rs @@ -13,6 +13,10 @@ impl TreeNode { pub(in crate::tree) fn just_in_time_tree_node_value_update( &mut self, old_specialized_cost: &impl Fn(&Vec, &Vec) -> Result, + get_temp_new_value_with_old_flags: &impl Fn( + &Vec, + &Vec, + ) -> Result>, Error>, update_tree_value_based_on_costs: &mut impl FnMut( &StorageCost, &Vec, @@ -30,14 +34,37 @@ impl TreeNode { Error, >, ) -> Result<(), Error> { - let (mut current_tree_plus_hook_size, mut storage_costs) = - self.kv_with_parent_hook_size_and_storage_cost(old_specialized_cost)?; let mut i = 0; if let Some(old_value) = self.old_value.clone() { // At this point the tree value can be updated based on client requirements // For example to store the costs + // todo: clean up clones + let original_new_value = self.value_ref().clone(); + + let new_value_with_old_flags = if self.inner.kv.value_defined_cost.is_none() { + // for items + get_temp_new_value_with_old_flags(&old_value, &original_new_value)? + } else { + // don't do this for sum items or trees + None + }; + + let (mut current_tree_plus_hook_size, mut storage_costs) = self + .kv_with_parent_hook_size_and_storage_cost_change_for_value( + old_specialized_cost, + new_value_with_old_flags, + )?; + loop { + if let BasicStorageRemoval(removed_bytes) = + storage_costs.value_storage_cost.removed_bytes + { + let (_, value_removed_bytes) = + section_removal_bytes(&old_value, 0, removed_bytes)?; + storage_costs.value_storage_cost.removed_bytes = value_removed_bytes; + } + let (flags_changed, value_defined_cost) = update_tree_value_based_on_costs( &storage_costs.value_storage_cost, &old_value, @@ -52,10 +79,13 @@ impl TreeNode { if after_update_tree_plus_hook_size == current_tree_plus_hook_size { break; } + // we are calling this with merged flags that are were put in through value mut + // ref let new_size_and_storage_costs = self.kv_with_parent_hook_size_and_storage_cost(old_specialized_cost)?; current_tree_plus_hook_size = new_size_and_storage_costs.0; storage_costs = new_size_and_storage_costs.1; + self.set_value(original_new_value.clone()) } if i > MAX_UPDATE_VALUE_BASED_ON_COSTS_TIMES { return Err(Error::CyclicError( @@ -71,11 +101,14 @@ impl TreeNode { let (_, value_removed_bytes) = section_removal_bytes(&old_value, 0, removed_bytes)?; storage_costs.value_storage_cost.removed_bytes = value_removed_bytes; } + self.known_storage_cost = Some(storage_costs); + } else { + let (_, storage_costs) = + self.kv_with_parent_hook_size_and_storage_cost(old_specialized_cost)?; + self.known_storage_cost = Some(storage_costs); } - // Update old tree size after generating value storage_cost cost self.old_value = Some(self.value_ref().clone()); - self.known_storage_cost = Some(storage_costs); Ok(()) } diff --git a/merk/src/tree/kv.rs b/merk/src/tree/kv.rs index 4fd3f7f7..f4a0e224 100644 --- a/merk/src/tree/kv.rs +++ b/merk/src/tree/kv.rs @@ -356,25 +356,6 @@ impl KV { node_value_size + parent_to_child_cost } - /// Get the costs for the node, this has the parent to child hooks - #[inline] - pub fn specialized_value_byte_cost_size_for_key_and_value_lengths( - not_prefixed_key_len: u32, - inner_value_len: u32, - is_sum_node: bool, - ) -> u32 { - // Sum trees are either 1 or 9 bytes. While they might be more or less on disk, - // costs can not take advantage of the varint aspect of the feature. - let feature_len = if is_sum_node { 9 } else { 1 }; - // Each node stores the key and value, and the node hash and the value hash - let node_value_size = inner_value_len + feature_len + HASH_LENGTH_U32_X2; - let node_value_size = node_value_size + node_value_size.required_space() as u32; - // The node will be a child of another node which stores it's key and hash - // That will be added during propagation - let parent_to_child_cost = Link::encoded_link_size(not_prefixed_key_len, is_sum_node); - node_value_size + parent_to_child_cost - } - /// Get the costs for the value with known value_len and non prefixed key /// len sizes, this has the parent to child hooks #[inline] @@ -452,11 +433,7 @@ impl KV { let key_len = self.key.len() as u32; let is_sum_node = self.feature_type.is_sum_feature(); - Self::specialized_value_byte_cost_size_for_key_and_value_lengths( - key_len, - value_cost, - is_sum_node, - ) + Self::node_value_byte_cost_size(key_len, value_cost, is_sum_node) } /// Costs based on predefined types (Trees, SumTrees, SumItems) that behave diff --git a/merk/src/tree/mod.rs b/merk/src/tree/mod.rs index 6b2710b6..91eebf52 100644 --- a/merk/src/tree/mod.rs +++ b/merk/src/tree/mod.rs @@ -42,6 +42,7 @@ use grovedb_costs::{ }, CostContext, CostResult, CostsExt, OperationCost, }; +#[cfg(feature = "full")] use grovedb_version::version::GroveVersion; #[cfg(any(feature = "full", feature = "verify"))] pub use hash::{ @@ -63,13 +64,14 @@ pub use tree_feature_type::TreeFeatureType; #[cfg(feature = "full")] pub use walk::{Fetch, RefWalker, Walker}; +#[cfg(feature = "full")] +use crate::tree::hash::HASH_LENGTH_X2; #[cfg(feature = "full")] use crate::tree::kv::ValueDefinedCostType; #[cfg(feature = "full")] use crate::tree::kv::ValueDefinedCostType::{LayeredValueDefinedCost, SpecializedValueDefinedCost}; #[cfg(feature = "full")] use crate::{error::Error, Error::Overflow}; - // TODO: remove need for `TreeInner`, and just use `Box` receiver for // relevant methods @@ -158,16 +160,7 @@ impl TreeNode { self.inner.kv.feature_type.is_sum_feature() } - /// Compare current value byte cost with old cost and return - /// current value byte cost with updated `KeyValueStorageCost` - pub fn kv_with_parent_hook_size_and_storage_cost_from_old_cost( - &self, - current_value_byte_cost: u32, - old_cost: u32, - ) -> Result<(u32, KeyValueStorageCost), Error> { - let key_storage_cost = StorageCost { - ..Default::default() - }; + pub fn storage_cost_for_update(current_value_byte_cost: u32, old_cost: u32) -> StorageCost { let mut value_storage_cost = StorageCost { ..Default::default() }; @@ -190,6 +183,20 @@ impl TreeNode { value_storage_cost.added_bytes += current_value_byte_cost - old_cost; } } + value_storage_cost + } + + /// Compare current value byte cost with old cost and return + /// current value byte cost with updated `KeyValueStorageCost` + pub fn kv_with_parent_hook_size_and_storage_cost_from_old_cost( + &self, + current_value_byte_cost: u32, + old_cost: u32, + ) -> Result<(u32, KeyValueStorageCost), Error> { + let key_storage_cost = StorageCost { + ..Default::default() + }; + let value_storage_cost = Self::storage_cost_for_update(current_value_byte_cost, old_cost); let key_value_storage_cost = KeyValueStorageCost { key_storage_cost, // the key storage cost is added later @@ -222,6 +229,45 @@ impl TreeNode { ) } + /// The point of this function is to get the cost change when we create a + /// temp value that's a partial merger between the old value and the new + /// value. Basically it is the new value with the old values flags + /// For example if we had an old value "Sam" with 40 bytes of flags + /// and a new value "Samuel" with 2 bytes of flags, the cost is probably + /// going to go up, As when we merge we will have Samuel with at least + /// 40 bytes of flags/ + pub fn kv_with_parent_hook_size_and_storage_cost_change_for_value( + &self, + old_tree_cost: &impl Fn(&Vec, &Vec) -> Result, + value: Option>, + ) -> Result<(u32, KeyValueStorageCost), Error> { + let current_value_byte_cost = if let Some(value_cost) = &self.inner.kv.value_defined_cost { + self.inner.kv.predefined_value_byte_cost_size(value_cost) + } else if let Some(value) = value { + let key_len = self.inner.kv.key.len() as u32; + let value_len = + HASH_LENGTH_X2 + value.len() + self.inner.kv.feature_type.encoding_cost(); + KV::value_byte_cost_size_for_key_and_value_lengths( + key_len, + value_len as u32, + self.inner.kv.feature_type.is_sum_feature(), + ) + } else { + self.inner.kv.value_byte_cost_size() + }; + + let old_cost = if let Some(old_value) = self.old_value.as_ref() { + old_tree_cost(self.key_as_ref(), old_value) + } else { + Ok(0) // there was no old value, hence old cost would be 0 + }?; + + self.kv_with_parent_hook_size_and_storage_cost_from_old_cost( + current_value_byte_cost, + old_cost, + ) + } + /// Creates a new `Tree` with the given key, value and value hash, and no /// children. /// @@ -330,6 +376,11 @@ impl TreeNode { self.inner.kv.key = key; } + /// Set value of Tree + pub fn set_value(&mut self, value: Vec) { + self.inner.kv.value = value; + } + /// Consumes the tree and returns its root node's key, without having to /// clone or allocate. #[inline] @@ -634,6 +685,10 @@ impl TreeNode { value: Vec, feature_type: TreeFeatureType, old_specialized_cost: &impl Fn(&Vec, &Vec) -> Result, + get_temp_new_value_with_old_flags: &impl Fn( + &Vec, + &Vec, + ) -> Result>, Error>, update_tree_value_based_on_costs: &mut impl FnMut( &StorageCost, &Vec, @@ -664,6 +719,7 @@ impl TreeNode { &cost, self.just_in_time_tree_node_value_update( old_specialized_cost, + get_temp_new_value_with_old_flags, update_tree_value_based_on_costs, section_removal_bytes ) @@ -683,6 +739,10 @@ impl TreeNode { value_fixed_cost: u32, feature_type: TreeFeatureType, old_specialized_cost: &impl Fn(&Vec, &Vec) -> Result, + get_temp_new_value_with_old_flags: &impl Fn( + &Vec, + &Vec, + ) -> Result>, Error>, update_tree_value_based_on_costs: &mut impl FnMut( &StorageCost, &Vec, @@ -715,6 +775,7 @@ impl TreeNode { &cost, self.just_in_time_tree_node_value_update( old_specialized_cost, + get_temp_new_value_with_old_flags, update_tree_value_based_on_costs, section_removal_bytes ) @@ -734,6 +795,10 @@ impl TreeNode { value_hash: CryptoHash, feature_type: TreeFeatureType, old_specialized_cost: &impl Fn(&Vec, &Vec) -> Result, + get_temp_new_value_with_old_flags: &impl Fn( + &Vec, + &Vec, + ) -> Result>, Error>, update_tree_value_based_on_costs: &mut impl FnMut( &StorageCost, &Vec, @@ -764,6 +829,7 @@ impl TreeNode { &cost, self.just_in_time_tree_node_value_update( old_specialized_cost, + get_temp_new_value_with_old_flags, update_tree_value_based_on_costs, section_removal_bytes ) @@ -788,6 +854,10 @@ impl TreeNode { value_cost: u32, feature_type: TreeFeatureType, old_specialized_cost: &impl Fn(&Vec, &Vec) -> Result, + get_temp_new_value_with_old_flags: &impl Fn( + &Vec, + &Vec, + ) -> Result>, Error>, update_tree_value_based_on_costs: &mut impl FnMut( &StorageCost, &Vec, @@ -821,6 +891,7 @@ impl TreeNode { &cost, self.just_in_time_tree_node_value_update( old_specialized_cost, + get_temp_new_value_with_old_flags, update_tree_value_based_on_costs, section_removal_bytes ) diff --git a/merk/src/tree/ops.rs b/merk/src/tree/ops.rs index 156c7e60..3e10b2c8 100644 --- a/merk/src/tree/ops.rs +++ b/merk/src/tree/ops.rs @@ -145,12 +145,13 @@ where /// not require a non-empty tree. /// /// Keys in batch must be sorted and unique. - pub fn apply_to, C, V, U, R>( + pub fn apply_to, C, V, T, U, R>( maybe_tree: Option, batch: &MerkBatch, source: S, old_tree_cost: &C, value_defined_cost_fn: Option<&V>, + get_temp_new_value_with_old_flags: &T, update_tree_value_based_on_costs: &mut U, section_removal_bytes: &mut R, grove_version: &GroveVersion, @@ -158,6 +159,7 @@ where where C: Fn(&Vec, &Vec) -> Result, V: Fn(&[u8], &GroveVersion) -> Option, + T: Fn(&Vec, &Vec) -> Result>, Error>, U: FnMut( &StorageCost, &Vec, @@ -185,6 +187,7 @@ where source, old_tree_cost, value_defined_cost_fn, + get_temp_new_value_with_old_flags, update_tree_value_based_on_costs, section_removal_bytes, grove_version, @@ -212,6 +215,7 @@ where batch, old_tree_cost, value_defined_cost_fn, + get_temp_new_value_with_old_flags, update_tree_value_based_on_costs, section_removal_bytes, grove_version @@ -228,11 +232,12 @@ where /// Builds a `Tree` from a batch of operations. /// /// Keys in batch must be sorted and unique. - fn build, C, V, U, R>( + fn build, C, V, T, U, R>( batch: &MerkBatch, source: S, old_tree_cost: &C, value_defined_cost_fn: Option<&V>, + get_temp_new_value_with_old_flags: &T, update_tree_value_based_on_costs: &mut U, section_removal_bytes: &mut R, grove_version: &GroveVersion, @@ -240,6 +245,7 @@ where where C: Fn(&Vec, &Vec) -> Result, V: Fn(&[u8], &GroveVersion) -> Option, + T: Fn(&Vec, &Vec) -> Result>, Error>, U: FnMut( &StorageCost, &Vec, @@ -267,6 +273,7 @@ where source.clone(), old_tree_cost, value_defined_cost_fn, + get_temp_new_value_with_old_flags, update_tree_value_based_on_costs, section_removal_bytes, grove_version @@ -281,6 +288,7 @@ where right_batch, old_tree_cost, value_defined_cost_fn, + get_temp_new_value_with_old_flags, update_tree_value_based_on_costs, section_removal_bytes, grove_version @@ -295,6 +303,7 @@ where source.clone(), old_tree_cost, value_defined_cost_fn, + get_temp_new_value_with_old_flags, update_tree_value_based_on_costs, section_removal_bytes, grove_version @@ -367,6 +376,7 @@ where ), old_tree_cost, value_defined_cost_fn, + get_temp_new_value_with_old_flags, update_tree_value_based_on_costs, section_removal_bytes, grove_version, @@ -377,8 +387,7 @@ where .wrap_with_cost(cost) } - #[allow(dead_code)] - fn apply_sorted_without_costs>( + pub(crate) fn apply_sorted_without_costs>( self, batch: &MerkBatch, grove_version: &GroveVersion, @@ -387,6 +396,7 @@ where batch, &|_, _| Ok(0), None::<&fn(&[u8], &GroveVersion) -> Option>, + &|_, _| Ok(None), &mut |_, _, _| Ok((false, None)), &mut |_flags, key_bytes_to_remove, value_bytes_to_remove| { Ok(( @@ -402,11 +412,12 @@ where /// `Walker::apply`_to, but requires a populated tree. /// /// Keys in batch must be sorted and unique. - fn apply_sorted, C, V, U, R>( + fn apply_sorted, C, V, T, U, R>( self, batch: &MerkBatch, old_specialized_cost: &C, value_defined_cost_fn: Option<&V>, + get_temp_new_value_with_old_flags: &T, update_tree_value_based_on_costs: &mut U, section_removal_bytes: &mut R, grove_version: &GroveVersion, @@ -414,6 +425,7 @@ where where C: Fn(&Vec, &Vec) -> Result, V: Fn(&[u8], &GroveVersion) -> Option, + T: Fn(&Vec, &Vec) -> Result>, Error>, U: FnMut( &StorageCost, &Vec, @@ -441,6 +453,7 @@ where value.to_vec(), feature_type.to_owned(), old_specialized_cost, + get_temp_new_value_with_old_flags, update_tree_value_based_on_costs, section_removal_bytes, ) @@ -455,6 +468,7 @@ where *value_cost, feature_type.to_owned(), old_specialized_cost, + get_temp_new_value_with_old_flags, update_tree_value_based_on_costs, section_removal_bytes ) @@ -468,6 +482,7 @@ where referenced_value.to_owned(), feature_type.to_owned(), old_specialized_cost, + get_temp_new_value_with_old_flags, update_tree_value_based_on_costs, section_removal_bytes, ) @@ -483,6 +498,7 @@ where *value_cost, feature_type.to_owned(), old_specialized_cost, + get_temp_new_value_with_old_flags, update_tree_value_based_on_costs, section_removal_bytes, ) @@ -565,6 +581,7 @@ where source.clone(), old_specialized_cost, value_defined_cost_fn, + get_temp_new_value_with_old_flags, update_tree_value_based_on_costs, section_removal_bytes, grove_version, @@ -591,6 +608,7 @@ where &batch[..index], old_specialized_cost, value_defined_cost_fn, + get_temp_new_value_with_old_flags, update_tree_value_based_on_costs, section_removal_bytes, grove_version @@ -623,6 +641,7 @@ where source.clone(), old_specialized_cost, value_defined_cost_fn, + get_temp_new_value_with_old_flags, update_tree_value_based_on_costs, section_removal_bytes, grove_version, @@ -649,6 +668,7 @@ where &batch[index + 1..], old_specialized_cost, value_defined_cost_fn, + get_temp_new_value_with_old_flags, update_tree_value_based_on_costs, section_removal_bytes, grove_version @@ -697,6 +717,7 @@ where KeyUpdates::new(new_keys, updated_keys, LinkedList::default(), None), old_specialized_cost, value_defined_cost_fn, + get_temp_new_value_with_old_flags, update_tree_value_based_on_costs, section_removal_bytes, grove_version, @@ -709,7 +730,7 @@ where /// /// This recursion executes serially in the same thread, but in the future /// will be dispatched to workers in other threads. - fn recurse, C, V, U, R>( + fn recurse, C, V, T, U, R>( self, batch: &MerkBatch, mid: usize, @@ -717,12 +738,14 @@ where mut key_updates: KeyUpdates, old_tree_cost: &C, value_defined_cost_fn: Option<&V>, + get_temp_new_value_with_old_flags: &T, update_tree_value_based_on_costs: &mut U, section_removal_bytes: &mut R, grove_version: &GroveVersion, ) -> CostResult<(Option, KeyUpdates), Error> where C: Fn(&Vec, &Vec) -> Result, + T: Fn(&Vec, &Vec) -> Result>, Error>, U: FnMut( &StorageCost, &Vec, @@ -755,6 +778,7 @@ where source, old_tree_cost, value_defined_cost_fn, + get_temp_new_value_with_old_flags, update_tree_value_based_on_costs, section_removal_bytes, grove_version, @@ -791,6 +815,7 @@ where source, old_tree_cost, value_defined_cost_fn, + get_temp_new_value_with_old_flags, update_tree_value_based_on_costs, section_removal_bytes, grove_version, @@ -1189,12 +1214,13 @@ mod test { #[test] fn apply_empty_none() { let grove_version = GroveVersion::latest(); - let (maybe_tree, key_updates) = Walker::::apply_to::, _, _, _, _>( + let (maybe_tree, key_updates) = Walker::::apply_to::, _, _, _, _, _>( None, &[], PanicSource {}, &|_, _| Ok(0), None::<&fn(&[u8], &GroveVersion) -> Option>, + &|_, _| Ok(None), &mut |_, _, _| Ok((false, None)), &mut |_flags, key_bytes_to_remove, value_bytes_to_remove| { Ok(( @@ -1221,6 +1247,7 @@ mod test { PanicSource {}, &|_, _| Ok(0), None::<&fn(&[u8], &GroveVersion) -> Option>, + &|_, _| Ok(None), &mut |_, _, _| Ok((false, None)), &mut |_flags, key_bytes_to_remove, value_bytes_to_remove| { Ok(( @@ -1250,6 +1277,7 @@ mod test { PanicSource {}, &|_, _| Ok(0), None::<&fn(&[u8], &GroveVersion) -> Option>, + &|_, _| Ok(None), &mut |_, _, _| Ok((false, None)), &mut |_flags, key_bytes_to_remove, value_bytes_to_remove| { Ok(( @@ -1275,6 +1303,7 @@ mod test { PanicSource {}, &|_, _| Ok(0), None::<&fn(&[u8], &GroveVersion) -> Option>, + &|_, _| Ok(None), &mut |_, _, _| Ok((false, None)), &mut |_flags, key_bytes_to_remove, value_bytes_to_remove| { Ok(( @@ -1307,6 +1336,7 @@ mod test { PanicSource {}, &|_, _| Ok(0), None::<&fn(&[u8], &GroveVersion) -> Option>, + &|_, _| Ok(None), &mut |_, _, _| Ok((false, None)), &mut |_flags, key_bytes_to_remove, value_bytes_to_remove| { Ok(( @@ -1333,6 +1363,7 @@ mod test { PanicSource {}, &|_, _| Ok(0), None::<&fn(&[u8], &GroveVersion) -> Option>, + &|_, _| Ok(None), &mut |_, _, _| Ok((false, None)), &mut |_flags, key_bytes_to_remove, value_bytes_to_remove| { Ok(( diff --git a/merk/src/tree/walk/mod.rs b/merk/src/tree/walk/mod.rs index a84a1d4c..4b67bb60 100644 --- a/merk/src/tree/walk/mod.rs +++ b/merk/src/tree/walk/mod.rs @@ -207,6 +207,10 @@ where value: Vec, feature_type: TreeFeatureType, old_specialized_cost: &impl Fn(&Vec, &Vec) -> Result, + get_temp_new_value_with_old_flags: &impl Fn( + &Vec, + &Vec, + ) -> Result>, Error>, update_tree_value_based_on_costs: &mut impl FnMut( &StorageCost, &Vec, @@ -232,6 +236,7 @@ where value, feature_type, old_specialized_cost, + get_temp_new_value_with_old_flags, update_tree_value_based_on_costs, section_removal_bytes ) @@ -247,6 +252,10 @@ where value_fixed_cost: u32, feature_type: TreeFeatureType, old_specialized_cost: &impl Fn(&Vec, &Vec) -> Result, + get_temp_new_value_with_old_flags: &impl Fn( + &Vec, + &Vec, + ) -> Result>, Error>, update_tree_value_based_on_costs: &mut impl FnMut( &StorageCost, &Vec, @@ -273,6 +282,7 @@ where value_fixed_cost, feature_type, old_specialized_cost, + get_temp_new_value_with_old_flags, update_tree_value_based_on_costs, section_removal_bytes ) @@ -288,6 +298,10 @@ where value_hash: CryptoHash, feature_type: TreeFeatureType, old_specialized_cost: &impl Fn(&Vec, &Vec) -> Result, + get_temp_new_value_with_old_flags: &impl Fn( + &Vec, + &Vec, + ) -> Result>, Error>, update_tree_value_based_on_costs: &mut impl FnMut( &StorageCost, &Vec, @@ -314,6 +328,7 @@ where value_hash, feature_type, old_specialized_cost, + get_temp_new_value_with_old_flags, update_tree_value_based_on_costs, section_removal_bytes ) @@ -330,6 +345,10 @@ where value_fixed_cost: u32, feature_type: TreeFeatureType, old_specialized_cost: &impl Fn(&Vec, &Vec) -> Result, + get_temp_new_value_with_old_flags: &impl Fn( + &Vec, + &Vec, + ) -> Result>, Error>, update_tree_value_based_on_costs: &mut impl FnMut( &StorageCost, &Vec, @@ -357,6 +376,7 @@ where value_fixed_cost, feature_type, old_specialized_cost, + get_temp_new_value_with_old_flags, update_tree_value_based_on_costs, section_removal_bytes ) diff --git a/node-grove/Cargo.toml b/node-grove/Cargo.toml index 4a4f5281..150f0b50 100644 --- a/node-grove/Cargo.toml +++ b/node-grove/Cargo.toml @@ -11,7 +11,7 @@ crate-type = ["cdylib"] [dependencies] grovedb = { path = "../grovedb", features = ["full", "estimated_costs"] } -grovedb-version = { path = "../grovedb-version", version = "1.0.0" } +grovedb-version = { path = "../grovedb-version", version = "2.0.0" } [dependencies.neon] version = "0.10.1" diff --git a/path/Cargo.toml b/path/Cargo.toml index 111bc474..e899a598 100644 --- a/path/Cargo.toml +++ b/path/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "grovedb-path" -version = "1.0.0" +version = "2.0.0" edition = "2021" license = "MIT" description = "Path extension crate for GroveDB" diff --git a/storage/Cargo.toml b/storage/Cargo.toml index 7db2b599..e6309790 100644 --- a/storage/Cargo.toml +++ b/storage/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "grovedb-storage" -version = "1.0.0" +version = "2.0.0" edition = "2021" license = "MIT" description = "Storage extension crate for GroveDB" @@ -14,13 +14,13 @@ num_cpus = { version = "1.16.0", optional = true } tempfile = { version = "3.10.1", optional = true } blake3 = { version = "1.5.1", optional = true } integer-encoding = { version = "4.0.0", optional = true } -grovedb-visualize = { version = "1.0.0", path = "../visualize" } +grovedb-visualize = { version = "2.0.0", path = "../visualize" } strum = { version = "0.26.2", features = ["derive"] } -grovedb-costs = { version = "1.0.0", path = "../costs" } +grovedb-costs = { version = "2.0.0", path = "../costs" } thiserror = "1.0.59" rocksdb = { version = "0.22.0", optional = true } hex = "0.4.3" -grovedb-path = { version = "1.0.0", path = "../path" } +grovedb-path = { version = "2.0.0", path = "../path" } [features] rocksdb_storage = ["rocksdb", "num_cpus", "lazy_static", "tempfile", "blake3", "integer-encoding"] diff --git a/storage/src/rocksdb_storage/storage_context/batch.rs b/storage/src/rocksdb_storage/storage_context/batch.rs index 8cb9987e..bcf58372 100644 --- a/storage/src/rocksdb_storage/storage_context/batch.rs +++ b/storage/src/rocksdb_storage/storage_context/batch.rs @@ -1,31 +1,3 @@ -// MIT LICENSE -// -// Copyright (c) 2021 Dash Core Group -// -// Permission is hereby granted, free of charge, to any -// person obtaining a copy of this software and associated -// documentation files (the "Software"), to deal in the -// Software without restriction, including without -// limitation the rights to use, copy, modify, merge, -// publish, distribute, sublicense, and/or sell copies of -// the Software, and to permit persons to whom the Software -// is furnished to do so, subject to the following -// conditions: -// -// The above copyright notice and this permission notice -// shall be included in all copies or substantial portions -// of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF -// ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED -// TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A -// PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -// SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY -// CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR -// IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -// DEALINGS IN THE SOFTWARE. - //! Prefixed storage batch implementation for RocksDB backend. use grovedb_costs::{ diff --git a/tutorials/Cargo.toml b/tutorials/Cargo.toml index 409a1c64..12f1a87e 100644 --- a/tutorials/Cargo.toml +++ b/tutorials/Cargo.toml @@ -12,6 +12,7 @@ grovedb = { path = "../grovedb" } grovedb-merk = { path = "../merk" } grovedb-storage = { path = "../storage" } grovedb-visualize = { path = "../visualize" } +grovedb-version = { path = "../grovedb-version" } grovedb-path = { path = "../path" } rand = "0.8.5" hex = "0.4" diff --git a/tutorials/src/bin/delete.rs b/tutorials/src/bin/delete.rs index 063243fd..0655c722 100644 --- a/tutorials/src/bin/delete.rs +++ b/tutorials/src/bin/delete.rs @@ -1,7 +1,9 @@ use grovedb::{Element, GroveDb}; +use grovedb_version::version::GroveVersion; fn main() { let root_path: &[&[u8]] = &[]; + let grove_version = GroveVersion::latest(); // Specify a path and open GroveDB at the path as db let path = String::from("../tutorial-storage"); diff --git a/tutorials/src/bin/insert.rs b/tutorials/src/bin/insert.rs index 3d9f9b2a..3f478281 100644 --- a/tutorials/src/bin/insert.rs +++ b/tutorials/src/bin/insert.rs @@ -1,10 +1,13 @@ use grovedb::{Element, GroveDb}; +use grovedb_version::version::GroveVersion; fn main() { // Specify a path and open GroveDB at the path as db let path = String::from("../tutorial-storage"); let db = GroveDb::open(path).unwrap(); + let grove_version = GroveVersion::latest(); + // Define key-values for insertion let key1 = b"hello"; let val1 = b"world"; diff --git a/tutorials/src/bin/proofs.rs b/tutorials/src/bin/proofs.rs index 02596919..3499a635 100644 --- a/tutorials/src/bin/proofs.rs +++ b/tutorials/src/bin/proofs.rs @@ -1,4 +1,5 @@ use grovedb::{operations::insert::InsertOptions, Element, GroveDb, PathQuery, Query}; +use grovedb_version::version::GroveVersion; const KEY1: &[u8] = b"key1"; const KEY2: &[u8] = b"key2"; @@ -16,6 +17,9 @@ fn main() { let path = String::from("../tutorial-storage"); // Open GroveDB as db let db = GroveDb::open(path).unwrap(); + + let grove_version = GroveVersion::latest(); + // Populate GroveDB with values. This function is defined below. populate(&db); // Define the path to the subtree we want to query. @@ -28,7 +32,7 @@ fn main() { let path_query = PathQuery::new_unsized(path, query.clone()); // Execute the query and collect the result items in "elements". let (_elements, _) = db - .query_item_value(&path_query, true, false, true, None) + .query_item_value(&path_query, true, false, true, None, grove_version) .unwrap() .expect("expected successful get_path_query"); @@ -50,6 +54,8 @@ fn main() { fn populate(db: &GroveDb) { let root_path: &[&[u8]] = &[]; + let grove_version = GroveVersion::latest(); + // Put an empty subtree into the root tree nodes at KEY1. // Call this SUBTREE1. db.insert(root_path, KEY1, Element::empty_tree(), INSERT_OPTIONS, None, grove_version) diff --git a/tutorials/src/bin/query-complex.rs b/tutorials/src/bin/query-complex.rs index b4fb78cf..fa72b085 100644 --- a/tutorials/src/bin/query-complex.rs +++ b/tutorials/src/bin/query-complex.rs @@ -2,6 +2,7 @@ use grovedb::{ operations::insert::InsertOptions, Element, GroveDb, PathQuery, Query, QueryItem, SizedQuery, }; use rand::Rng; +use grovedb_version::version::GroveVersion; const KEY1: &[u8] = b"key1"; const KEY2: &[u8] = b"key2"; @@ -19,6 +20,8 @@ fn main() { // Specify the path where the GroveDB instance exists. let path = String::from("../tutorial-storage"); + let grove_version = GroveVersion::latest(); + // Open GroveDB at the path. let db = GroveDb::open(path).unwrap(); @@ -66,7 +69,7 @@ fn main() { // Execute the path query and collect the result items in "elements". let (elements, _) = db - .query_item_value(&path_query, true, false, true, None) + .query_item_value(&path_query, true, false, true, None, grove_version) .unwrap() .expect("expected successful get_path_query"); @@ -76,6 +79,9 @@ fn main() { fn populate(db: &GroveDb) { let root_path: &[&[u8]] = &[]; + + let grove_version = GroveVersion::latest(); + // Put an empty subtree into the root tree nodes at KEY1. // Call this SUBTREE1. db.insert(root_path, KEY1, Element::empty_tree(), INSERT_OPTIONS, None, grove_version) diff --git a/tutorials/src/bin/query-simple.rs b/tutorials/src/bin/query-simple.rs index ab888873..87d01ca6 100644 --- a/tutorials/src/bin/query-simple.rs +++ b/tutorials/src/bin/query-simple.rs @@ -1,4 +1,5 @@ use grovedb::{operations::insert::InsertOptions, Element, GroveDb, PathQuery, Query}; +use grovedb_version::version::GroveVersion; const KEY1: &[u8] = b"key1"; const KEY2: &[u8] = b"key2"; @@ -15,6 +16,8 @@ fn main() { // Specify the path where the GroveDB instance exists. let path = String::from("../tutorial-storage"); + let grove_version = GroveVersion::latest(); + // Open GroveDB at the path. let db = GroveDb::open(path).unwrap(); @@ -36,7 +39,7 @@ fn main() { // Execute the query and collect the result items in "elements". let (elements, _) = db - .query_item_value(&path_query, true, false, true,None) + .query_item_value(&path_query, true, false, true,None, &grove_version) .unwrap() .expect("expected successful get_path_query"); @@ -46,6 +49,9 @@ fn main() { fn populate(db: &GroveDb) { let root_path: &[&[u8]] = &[]; + + let grove_version = GroveVersion::latest(); + // Put an empty subtree into the root tree nodes at KEY1. // Call this SUBTREE1. db.insert(root_path, KEY1, Element::empty_tree(), INSERT_OPTIONS, None, grove_version) diff --git a/tutorials/src/bin/replication.rs b/tutorials/src/bin/replication.rs index 5ed6ab5b..ceeec2f2 100644 --- a/tutorials/src/bin/replication.rs +++ b/tutorials/src/bin/replication.rs @@ -6,6 +6,7 @@ use rand::{distributions::Alphanumeric, Rng, }; use grovedb::element::SumValue; use grovedb::replication::CURRENT_STATE_SYNC_VERSION; use grovedb::replication::MultiStateSyncInfo; +use grovedb_version::version::GroveVersion; const MAIN_ΚΕΥ: &[u8] = b"key_main"; const MAIN_ΚΕΥ_EMPTY: &[u8] = b"key_main_empty"; @@ -25,47 +26,47 @@ const INSERT_OPTIONS: Option = Some(InsertOptions { base_root_storage_is_free: true, }); -fn populate_db(grovedb_path: String) -> GroveDb { +fn populate_db(grovedb_path: String, grove_version: &GroveVersion) -> GroveDb { let db = GroveDb::open(grovedb_path).unwrap(); - insert_empty_tree_db(&db, ROOT_PATH, MAIN_ΚΕΥ); - insert_empty_tree_db(&db, ROOT_PATH, MAIN_ΚΕΥ_EMPTY); - insert_empty_tree_db(&db, &[MAIN_ΚΕΥ], KEY_INT_0); - insert_empty_tree_db(&db, &[MAIN_ΚΕΥ], KEY_INT_1); - insert_empty_tree_db(&db, &[MAIN_ΚΕΥ], KEY_INT_2); + insert_empty_tree_db(&db, ROOT_PATH, MAIN_ΚΕΥ, &grove_version); + insert_empty_tree_db(&db, ROOT_PATH, MAIN_ΚΕΥ_EMPTY, &grove_version); + insert_empty_tree_db(&db, &[MAIN_ΚΕΥ], KEY_INT_0, &grove_version); + insert_empty_tree_db(&db, &[MAIN_ΚΕΥ], KEY_INT_1, &grove_version); + insert_empty_tree_db(&db, &[MAIN_ΚΕΥ], KEY_INT_2, &grove_version); let tx = db.start_transaction(); let batch_size = 50; for i in 0..=5 { - insert_range_values_db(&db, &[MAIN_ΚΕΥ, KEY_INT_0], i * batch_size, i * batch_size + batch_size - 1, &tx); + insert_range_values_db(&db, &[MAIN_ΚΕΥ, KEY_INT_0], i * batch_size, i * batch_size + batch_size - 1, &tx, &grove_version); } let _ = db.commit_transaction(tx); let tx = db.start_transaction(); let batch_size = 50; for i in 0..=5 { - insert_range_values_db(&db, &[MAIN_ΚΕΥ, KEY_INT_1], i * batch_size, i * batch_size + batch_size - 1, &tx); + insert_range_values_db(&db, &[MAIN_ΚΕΥ, KEY_INT_1], i * batch_size, i * batch_size + batch_size - 1, &tx, &grove_version); } let _ = db.commit_transaction(tx); let tx = db.start_transaction(); let batch_size = 50; for i in 0..=5 { - insert_range_values_db(&db, &[MAIN_ΚΕΥ, KEY_INT_2], i * batch_size, i * batch_size + batch_size - 1, &tx); + insert_range_values_db(&db, &[MAIN_ΚΕΥ, KEY_INT_2], i * batch_size, i * batch_size + batch_size - 1, &tx, &grove_version); } let _ = db.commit_transaction(tx); - insert_empty_tree_db(&db, &[MAIN_ΚΕΥ], KEY_INT_REF_0); + insert_empty_tree_db(&db, &[MAIN_ΚΕΥ], KEY_INT_REF_0, &grove_version); let tx_2 = db.start_transaction(); - insert_range_ref_double_values_db(&db, &[MAIN_ΚΕΥ, KEY_INT_REF_0], KEY_INT_0, 1, 50, &tx_2); + insert_range_ref_double_values_db(&db, &[MAIN_ΚΕΥ, KEY_INT_REF_0], KEY_INT_0, 1, 50, &tx_2, &grove_version); let _ = db.commit_transaction(tx_2); - insert_empty_sum_tree_db(&db, &[MAIN_ΚΕΥ], KEY_INT_A); + insert_empty_sum_tree_db(&db, &[MAIN_ΚΕΥ], KEY_INT_A, &grove_version); let tx_3 = db.start_transaction(); - insert_range_values_db(&db, &[MAIN_ΚΕΥ, KEY_INT_A], 1, 500, &tx_3); - insert_sum_element_db(&db, &[MAIN_ΚΕΥ, KEY_INT_A], 501, 550, &tx_3); + insert_range_values_db(&db, &[MAIN_ΚΕΥ, KEY_INT_A], 1, 500, &tx_3, &grove_version); + insert_sum_element_db(&db, &[MAIN_ΚΕΥ, KEY_INT_A], 501, 550, &tx_3, &grove_version); let _ = db.commit_transaction(tx_3); db } @@ -76,8 +77,9 @@ fn create_empty_db(grovedb_path: String) -> GroveDb { } fn main() { + let grove_version = GroveVersion::latest(); let path_source = generate_random_path("../tutorial-storage/", "/db_0", 24); - let db_source = populate_db(path_source.clone()); + let db_source = populate_db(path_source.clone(), &grove_version); let checkpoint_dir = path_source + "/checkpoint"; let path_checkpoint = Path::new(checkpoint_dir.as_str()); @@ -103,11 +105,11 @@ fn main() { println!("\n######### db_checkpoint_0 -> db_destination state sync"); let state_info = MultiStateSyncInfo::default(); let tx = db_destination.start_transaction(); - sync_db_demo(&db_checkpoint_0, &db_destination, state_info, &tx).unwrap(); + sync_db_demo(&db_checkpoint_0, &db_destination, state_info, &tx, &grove_version).unwrap(); db_destination.commit_transaction(tx).unwrap().expect("expected to commit transaction"); println!("\n######### verify db_destination"); - let incorrect_hashes = db_destination.verify_grovedb(None, grove_version).unwrap(); + let incorrect_hashes = db_destination.verify_grovedb(None, true, false, grove_version).unwrap(); if incorrect_hashes.len() > 0 { println!("DB verification failed!"); } @@ -126,21 +128,21 @@ fn main() { let query_path = &[MAIN_ΚΕΥ, KEY_INT_0]; let query_key = (20487u32).to_be_bytes().to_vec(); println!("\n######## Query on db_checkpoint_0:"); - query_db(&db_checkpoint_0, query_path, query_key.clone()); + query_db(&db_checkpoint_0, query_path, query_key.clone(), &grove_version); println!("\n######## Query on db_destination:"); - query_db(&db_destination, query_path, query_key.clone()); + query_db(&db_destination, query_path, query_key.clone(), &grove_version); return; } -fn insert_empty_tree_db(db: &GroveDb, path: &[&[u8]], key: &[u8]) +fn insert_empty_tree_db(db: &GroveDb, path: &[&[u8]], key: &[u8], grove_version: &GroveVersion) { db.insert(path, key, Element::empty_tree(), INSERT_OPTIONS, None, grove_version) .unwrap() .expect("successfully inserted tree"); } -fn insert_range_values_db(db: &GroveDb, path: &[&[u8]], min_i: u32, max_i: u32, transaction: &Transaction) +fn insert_range_values_db(db: &GroveDb, path: &[&[u8]], min_i: u32, max_i: u32, transaction: &Transaction, grove_version: &GroveVersion) { for i in min_i..=max_i { let i_vec = i.to_be_bytes().to_vec(); @@ -157,7 +159,7 @@ fn insert_range_values_db(db: &GroveDb, path: &[&[u8]], min_i: u32, max_i: u32, } } -fn insert_range_ref_double_values_db(db: &GroveDb, path: &[&[u8]], ref_key: &[u8], min_i: u32, max_i: u32, transaction: &Transaction) +fn insert_range_ref_double_values_db(db: &GroveDb, path: &[&[u8]], ref_key: &[u8], min_i: u32, max_i: u32, transaction: &Transaction, grove_version: &GroveVersion) { for i in min_i..=max_i { let i_vec = i.to_be_bytes().to_vec(); @@ -180,13 +182,13 @@ fn insert_range_ref_double_values_db(db: &GroveDb, path: &[&[u8]], ref_key: &[u8 } } -fn insert_empty_sum_tree_db(db: &GroveDb, path: &[&[u8]], key: &[u8]) +fn insert_empty_sum_tree_db(db: &GroveDb, path: &[&[u8]], key: &[u8], grove_version: &GroveVersion) { db.insert(path, key, Element::empty_sum_tree(), INSERT_OPTIONS, None, grove_version) .unwrap() .expect("successfully inserted tree"); } -fn insert_sum_element_db(db: &GroveDb, path: &[&[u8]], min_i: u32, max_i: u32, transaction: &Transaction) +fn insert_sum_element_db(db: &GroveDb, path: &[&[u8]], min_i: u32, max_i: u32, transaction: &Transaction, grove_version: &GroveVersion) { for i in min_i..=max_i { //let value : u32 = i; @@ -214,7 +216,7 @@ fn generate_random_path(prefix: &str, suffix: &str, len: usize) -> String { format!("{}{}{}", prefix, random_string, suffix) } -fn query_db(db: &GroveDb, path: &[&[u8]], key: Vec) { +fn query_db(db: &GroveDb, path: &[&[u8]], key: Vec, grove_version: &GroveVersion) { let path_vec: Vec> = path.iter() .map(|&slice| slice.to_vec()) .collect(); @@ -225,7 +227,7 @@ fn query_db(db: &GroveDb, path: &[&[u8]], key: Vec) { let path_query = PathQuery::new_unsized(path_vec, query.clone()); let (elements, _) = db - .query_item_value(&path_query, true, false, true, None) + .query_item_value(&path_query, true, false, true, None, grove_version) .unwrap() .expect("expected successful get_path_query"); for e in elements.into_iter() { @@ -246,6 +248,7 @@ fn sync_db_demo( target_db: &GroveDb, state_sync_info: MultiStateSyncInfo, target_tx: &Transaction, + grove_version: &GroveVersion, ) -> Result<(), grovedb::Error> { let app_hash = source_db.root_hash(None, grove_version).value.unwrap(); let mut state_sync_info = target_db.start_snapshot_syncing(state_sync_info, app_hash, target_tx, CURRENT_STATE_SYNC_VERSION, grove_version)?; diff --git a/visualize/Cargo.toml b/visualize/Cargo.toml index 821bf443..9a502088 100644 --- a/visualize/Cargo.toml +++ b/visualize/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "grovedb-visualize" -version = "1.0.0" +version = "2.0.0" edition = "2021" license = "MIT" description = "Debug prints extension crate for GroveDB"