From 7765aa9fb7db347639c9764968261f59abb88993 Mon Sep 17 00:00:00 2001 From: QuantumExplorer Date: Mon, 15 Jul 2024 22:21:00 +0700 Subject: [PATCH 1/7] fix: forgot a ) char in a display method (#319) * set grovedbg-types version (#316) * update grovedbg-types manifest (#317) * fix: forgot a ) char in a display method --------- Co-authored-by: fominok --- grovedb/Cargo.toml | 2 +- grovedb/src/element/mod.rs | 4 ++-- grovedbg-types/Cargo.toml | 6 +++++- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/grovedb/Cargo.toml b/grovedb/Cargo.toml index 57a27479..b8601b15 100644 --- a/grovedb/Cargo.toml +++ b/grovedb/Cargo.toml @@ -26,7 +26,7 @@ nohash-hasher = { version = "0.2.0", optional = true } indexmap = { version = "2.2.6"} intmap = { version = "2.0.0", optional = true } grovedb-path = { version = "1.0.0-rc.2", path = "../path" } -grovedbg-types = { path = "../grovedbg-types", optional = true } +grovedbg-types = { version = "1.0.0-rc.2", path = "../grovedbg-types", optional = true } tokio = { version = "1.37.0", features = ["rt-multi-thread", "net"], optional = true } axum = { version = "0.7.5", features = ["macros"], optional = true } tower-http = { version = "0.5.2", features = ["fs"], optional = true } diff --git a/grovedb/src/element/mod.rs b/grovedb/src/element/mod.rs index a6add9e6..ad2540e3 100644 --- a/grovedb/src/element/mod.rs +++ b/grovedb/src/element/mod.rs @@ -121,7 +121,7 @@ impl fmt::Display for Element { Element::SumItem(sum_value, flags) => { write!( f, - "SumItem({}{}", + "SumItem({}{})", sum_value, flags .as_ref() @@ -131,7 +131,7 @@ impl fmt::Display for Element { Element::SumTree(root_key, sum_value, flags) => { write!( f, - "SumTree({}, {}{}", + "SumTree({}, {}{})", root_key.as_ref().map_or("None".to_string(), hex::encode), sum_value, flags diff --git a/grovedbg-types/Cargo.toml b/grovedbg-types/Cargo.toml index 300eb5c7..a3a2878e 100644 --- a/grovedbg-types/Cargo.toml +++ b/grovedbg-types/Cargo.toml @@ -1,7 +1,11 @@ [package] name = "grovedbg-types" -version = "0.1.0" +version = "1.0.0-rc.2" edition = "2021" +description = "Common type definitions for data exchange over GroveDBG protocol" +authors = ["Evgeny Fomin "] +license = "MIT" +repository = "https://github.com/dashpay/grovedb" [dependencies] serde = { version = "1.0.201", features = ["derive"] } From ab8024675b8268eedb4f25af402d2fab278417f6 Mon Sep 17 00:00:00 2001 From: fominok Date: Mon, 15 Jul 2024 17:23:35 +0200 Subject: [PATCH 2/7] feat: use prebuilt grovedbg (#318) * use prebuilt grovedbg --- README.md | 4 ---- grovedb/Cargo.toml | 4 +++- grovedb/build.rs | 49 ++++++++++++++++++++++------------------------ 3 files changed, 26 insertions(+), 31 deletions(-) diff --git a/README.md b/README.md index 4d66f7f9..d55137b1 100644 --- a/README.md +++ b/README.md @@ -237,10 +237,6 @@ From here we can build: There is a work in progress implementation of a debugger layer for GroveDB. To use this library with these capabilities enabled one needs to set a dependency with `grovedbg` feature. -At build time this requires two environment dependencies: -1. `wasm32-unknown-unknown` Rust toolchain; -2. [trunk](https://trunkrs.dev/) utility. - Then, to launch visualizer tool to observe the database structure inside of your browser on a port, let's say 10000, the following snippet should do: diff --git a/grovedb/Cargo.toml b/grovedb/Cargo.toml index b8601b15..0b871cf2 100644 --- a/grovedb/Cargo.toml +++ b/grovedb/Cargo.toml @@ -82,4 +82,6 @@ grovedbg = [ ] [build-dependencies] -zip-extensions = "0.6.2" +hex-literal = "0.4.1" +reqwest = { version = "0.12.5", features = ["blocking"] } +sha2 = "0.10.8" diff --git a/grovedb/build.rs b/grovedb/build.rs index cbdc11ac..d7b4b7b9 100644 --- a/grovedb/build.rs +++ b/grovedb/build.rs @@ -1,36 +1,33 @@ #[cfg(feature = "grovedbg")] fn main() { - use std::{ - env, - path::PathBuf, - process::{Command, ExitStatus, Output}, - }; + use std::{env, fs::File, io::Cursor, path::PathBuf}; + + use hex_literal::hex; + use sha2::{digest::FixedOutput, Digest, Sha256}; + + const GROVEDBG_SHA256: [u8; 32] = + hex!("206d865949e7f12b843c342cea3b172d4c78d4eb754b80d272164d59f62a2276"); + const GROVEDBG_VERSION: &str = "v1.0.0-rc.3"; let out_dir = PathBuf::from(&env::var_os("OUT_DIR").unwrap()); + let grovedbg_zip_path = out_dir.join("grovedbg.zip"); + + if !grovedbg_zip_path.exists() { + let response = reqwest::blocking::get(format!("https://github.com/dashpay/grovedbg/releases/download/{GROVEDBG_VERSION}/grovedbg-{GROVEDBG_VERSION}.zip")) + .expect("can't download GroveDBG artifact"); - let Output { - status, - stdout, - stderr, - } = Command::new("trunk") - .arg("build") - .arg("--release") - .arg("--dist") - .arg(&out_dir) - .arg("grovedbg/index.html") - .output() - .expect("cannot start trunk process"); - - if !status.success() { - let stdout_msg = String::from_utf8_lossy(&stdout); - let stderr_msg = String::from_utf8_lossy(&stderr); - let bindgen_version = env::var_os("TRUNK_TOOLS_WASM_BINDGEN").unwrap_or_default(); - panic!("Error running `trunk build --release`\nbindgen version:{bindgen_version:?}\n{stdout_msg}\n{stderr_msg}"); + let mut grovedbg_zip = File::create(&grovedbg_zip_path).unwrap(); + let mut content = Cursor::new(response.bytes().unwrap()); + std::io::copy(&mut content, &mut grovedbg_zip).unwrap(); } - let zip_file = out_dir.join("grovedbg.zip"); - zip_extensions::write::zip_create_from_directory(&zip_file, &out_dir) - .expect("can't create a grovedbg zip archive"); + let mut grovedbg_zip = File::open(&grovedbg_zip_path).unwrap(); + + let mut sha256 = Sha256::new(); + std::io::copy(&mut grovedbg_zip, &mut sha256).unwrap(); + let hash = sha256.finalize_fixed(); + + assert_eq!(hash.as_slice(), GROVEDBG_SHA256); } #[cfg(not(feature = "grovedbg"))] From 4d14ae374d58a1b04a4dc63b1435e1781ccff1d1 Mon Sep 17 00:00:00 2001 From: fominok Date: Mon, 15 Jul 2024 20:35:01 +0200 Subject: [PATCH 3/7] remove path from grovedb-version dep (#320) --- grovedb/Cargo.toml | 2 +- merk/Cargo.toml | 2 +- node-grove/Cargo.toml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/grovedb/Cargo.toml b/grovedb/Cargo.toml index 0b871cf2..efedc420 100644 --- a/grovedb/Cargo.toml +++ b/grovedb/Cargo.toml @@ -33,7 +33,7 @@ tower-http = { version = "0.5.2", features = ["fs"], optional = true } blake3 = "1.4.0" bitvec = "1" zip-extensions = { version ="0.6.2", optional = true } -grovedb-version = { version = "1.0.0-rc.2", path = "../grovedb-version" } +grovedb-version = "1.0.0-rc.2" [dev-dependencies] rand = "0.8.5" diff --git a/merk/Cargo.toml b/merk/Cargo.toml index 4364a564..a41cc8f7 100644 --- a/merk/Cargo.toml +++ b/merk/Cargo.toml @@ -20,7 +20,7 @@ grovedb-costs = { version = "1.0.0-rc.2", path = "../costs" } grovedb-visualize = { version = "1.0.0-rc.2", path = "../visualize" } grovedb-path = { version = "1.0.0-rc.2", path = "../path" } hex = { version = "0.4.3" } -grovedb-version = { version = "1.0.0-rc.2", path = "../grovedb-version" } +grovedb-version = "1.0.0-rc.2" [dependencies.time] version = "0.3.34" diff --git a/node-grove/Cargo.toml b/node-grove/Cargo.toml index 7656ea7d..17e5ee09 100644 --- a/node-grove/Cargo.toml +++ b/node-grove/Cargo.toml @@ -11,7 +11,7 @@ crate-type = ["cdylib"] [dependencies] grovedb = { path = "../grovedb", features = ["full", "estimated_costs"] } -grovedb-version = { path = "../grovedb-version" } +grovedb-version = "1.0.0-rc.2" [dependencies.neon] version = "0.10.1" From 96dd04e8269e718d24c83917fe70edebd126f2e6 Mon Sep 17 00:00:00 2001 From: fominok Date: Fri, 26 Jul 2024 10:53:20 +0200 Subject: [PATCH 4/7] feat: queries proofs grovedbg update (#321) --- grovedb/src/debugger.rs | 323 ++++++++++++++++++++--- grovedb/src/operations/proof/generate.rs | 2 +- grovedbg-types/Cargo.toml | 1 + grovedbg-types/src/lib.rs | 178 +++++++++++++ merk/src/debugger.rs | 11 +- merk/src/tree/kv.rs | 4 + 6 files changed, 480 insertions(+), 39 deletions(-) diff --git a/grovedb/src/debugger.rs b/grovedb/src/debugger.rs index de76c6df..e3556acd 100644 --- a/grovedb/src/debugger.rs +++ b/grovedb/src/debugger.rs @@ -1,19 +1,31 @@ //! GroveDB debugging support module. -use std::{fs, net::Ipv4Addr, sync::Weak}; +use std::{collections::BTreeMap, fs, sync::Weak}; use axum::{extract::State, http::StatusCode, response::IntoResponse, routing::post, Json, Router}; -use grovedb_merk::debugger::NodeDbg; +use grovedb_merk::{ + debugger::NodeDbg, + proofs::{Decoder, Node, Op}, + TreeFeatureType, +}; use grovedb_path::SubtreePath; use grovedb_version::version::GroveVersion; -use grovedbg_types::{NodeFetchRequest, NodeUpdate, Path}; +use grovedbg_types::{ + MerkProofNode, MerkProofOp, NodeFetchRequest, NodeUpdate, Path, PathQuery, Query, QueryItem, + SizedQuery, SubqueryBranch, +}; +use indexmap::IndexMap; use tokio::{ net::ToSocketAddrs, sync::mpsc::{self, Sender}, }; use tower_http::services::ServeDir; -use crate::{reference_path::ReferencePathType, GroveDb}; +use crate::{ + operations::proof::{GroveDBProof, LayerProof, ProveOptions}, + reference_path::ReferencePathType, + GroveDb, +}; const GROVEDBG_ZIP: [u8; include_bytes!(concat!(env!("OUT_DIR"), "/grovedbg.zip")).len()] = *include_bytes!(concat!(env!("OUT_DIR"), "/grovedbg.zip")); @@ -36,6 +48,7 @@ where let app = Router::new() .route("/fetch_node", post(fetch_node)) .route("/fetch_root_node", post(fetch_root_node)) + .route("/execute_path_query", post(execute_path_query)) .fallback_service(ServeDir::new(grovedbg_www)) .with_state((shutdown_send, grovedb)); @@ -123,62 +136,290 @@ async fn fetch_root_node( } } -fn node_to_update( - path: Path, - NodeDbg { - key, - value, - left_child, - right_child, - }: NodeDbg, -) -> Result { - // todo: GroveVersion::latest() to actual version - let grovedb_element = crate::Element::deserialize(&value, GroveVersion::latest())?; +async fn execute_path_query( + State((shutdown, grovedb)): State<(Sender<()>, Weak)>, + Json(json_path_query): Json, +) -> Result, AppError> { + let Some(db) = grovedb.upgrade() else { + shutdown.send(()).await.ok(); + return Err(AppError::Closed); + }; + + let path_query = path_query_to_grovedb(json_path_query); + + let grovedb_proof = db + .prove_internal(&path_query, None, GroveVersion::latest()) + .unwrap()?; + Ok(Json(proof_to_grovedbg(grovedb_proof)?)) +} + +fn proof_to_grovedbg(proof: GroveDBProof) -> Result { + match proof { + GroveDBProof::V0(p) => Ok(grovedbg_types::Proof { + root_layer: proof_layer_to_grovedbg(p.root_layer)?, + prove_options: prove_options_to_grovedbg(p.prove_options), + }), + } +} + +fn proof_layer_to_grovedbg( + proof_layer: LayerProof, +) -> Result { + Ok(grovedbg_types::ProofLayer { + merk_proof: merk_proof_to_grovedbg(&proof_layer.merk_proof)?, + lower_layers: proof_layer + .lower_layers + .into_iter() + .map(|(k, v)| proof_layer_to_grovedbg(v).map(|layer| (k, layer))) + .collect::, grovedbg_types::ProofLayer>, crate::Error>>()?, + }) +} + +fn merk_proof_to_grovedbg(merk_proof: &[u8]) -> Result, crate::Error> { + let decoder = Decoder::new(merk_proof); + decoder + .map(|op_result| { + op_result + .map_err(crate::Error::MerkError) + .and_then(merk_proof_op_to_grovedbg) + }) + .collect::, _>>() +} +fn merk_proof_op_to_grovedbg(op: Op) -> Result { + Ok(match op { + Op::Push(node) => MerkProofOp::Push(merk_proof_node_to_grovedbg(node)?), + Op::PushInverted(node) => MerkProofOp::PushInverted(merk_proof_node_to_grovedbg(node)?), + Op::Parent => MerkProofOp::Parent, + Op::Child => MerkProofOp::Child, + Op::ParentInverted => MerkProofOp::ParentInverted, + Op::ChildInverted => MerkProofOp::ChildInverted, + }) +} + +fn merk_proof_node_to_grovedbg(node: Node) -> Result { + Ok(match node { + Node::Hash(hash) => MerkProofNode::Hash(hash), + Node::KVHash(hash) => MerkProofNode::KVHash(hash), + Node::KVDigest(key, hash) => MerkProofNode::KVDigest(key, hash), + Node::KV(key, value) => { + let element = crate::Element::deserialize(&value, GroveVersion::latest())?; + MerkProofNode::KV(key, element_to_grovedbg(element)) + } + Node::KVValueHash(key, value, hash) => { + let element = crate::Element::deserialize(&value, GroveVersion::latest())?; + MerkProofNode::KVValueHash(key, element_to_grovedbg(element), hash) + } + Node::KVValueHashFeatureType(key, value, hash, TreeFeatureType::BasicMerkNode) => { + let element = crate::Element::deserialize(&value, GroveVersion::latest())?; + MerkProofNode::KVValueHashFeatureType( + key, + element_to_grovedbg(element), + hash, + grovedbg_types::TreeFeatureType::BasicMerkNode, + ) + } + Node::KVValueHashFeatureType(key, value, hash, TreeFeatureType::SummedMerkNode(sum)) => { + let element = crate::Element::deserialize(&value, GroveVersion::latest())?; + MerkProofNode::KVValueHashFeatureType( + key, + element_to_grovedbg(element), + hash, + grovedbg_types::TreeFeatureType::SummedMerkNode(sum), + ) + } + Node::KVRefValueHash(key, value, hash) => { + let element = crate::Element::deserialize(&value, GroveVersion::latest())?; + MerkProofNode::KVRefValueHash(key, element_to_grovedbg(element), hash) + } + }) +} + +fn prove_options_to_grovedbg(options: ProveOptions) -> grovedbg_types::ProveOptions { + grovedbg_types::ProveOptions { + decrease_limit_on_empty_sub_query_result: options.decrease_limit_on_empty_sub_query_result, + } +} + +fn path_query_to_grovedb(query: PathQuery) -> crate::PathQuery { + let PathQuery { + path, + query: + SizedQuery { + limit, + offset, + query: inner_query, + }, + } = query; + + crate::PathQuery { + path, + query: crate::SizedQuery { + query: query_to_grovedb(inner_query), + limit, + offset, + }, + } +} - let element = match grovedb_element { - crate::Element::Item(value, ..) => grovedbg_types::Element::Item { value }, - crate::Element::Tree(root_key, ..) => grovedbg_types::Element::Subtree { root_key }, - crate::Element::Reference(ReferencePathType::AbsolutePathReference(path), ..) => { - grovedbg_types::Element::AbsolutePathReference { path } +fn query_to_grovedb(query: Query) -> crate::Query { + crate::Query { + items: query.items.into_iter().map(query_item_to_grovedb).collect(), + default_subquery_branch: subquery_branch_to_grovedb(query.default_subquery_branch), + conditional_subquery_branches: conditional_subquery_branches_to_grovedb( + query.conditional_subquery_branches, + ), + left_to_right: query.left_to_right, + } +} + +fn conditional_subquery_branches_to_grovedb( + conditional_subquery_branches: Vec<(QueryItem, SubqueryBranch)>, +) -> Option> { + if conditional_subquery_branches.is_empty() { + None + } else { + Some( + conditional_subquery_branches + .into_iter() + .map(|(item, branch)| { + ( + query_item_to_grovedb(item), + subquery_branch_to_grovedb(branch), + ) + }) + .collect(), + ) + } +} + +fn subquery_branch_to_grovedb( + subquery_branch: SubqueryBranch, +) -> grovedb_merk::proofs::query::SubqueryBranch { + grovedb_merk::proofs::query::SubqueryBranch { + subquery_path: subquery_branch.subquery_path, + subquery: subquery_branch + .subquery + .map(|q| Box::new(query_to_grovedb(*q))), + } +} + +fn query_item_to_grovedb(item: QueryItem) -> crate::QueryItem { + match item { + QueryItem::Key(x) => crate::QueryItem::Key(x), + QueryItem::Range { start, end } => crate::QueryItem::Range(start..end), + QueryItem::RangeInclusive { start, end } => crate::QueryItem::RangeInclusive(start..=end), + QueryItem::RangeFull => crate::QueryItem::RangeFull(..), + QueryItem::RangeFrom(x) => crate::QueryItem::RangeFrom(x..), + QueryItem::RangeTo(x) => crate::QueryItem::RangeTo(..x), + QueryItem::RangeToInclusive(x) => crate::QueryItem::RangeToInclusive(..=x), + QueryItem::RangeAfter(x) => crate::QueryItem::RangeAfter(x..), + QueryItem::RangeAfterTo { after, to } => crate::QueryItem::RangeAfterTo(after..to), + QueryItem::RangeAfterToInclusive { after, to } => { + crate::QueryItem::RangeAfterToInclusive(after..=to) } + } +} + +fn element_to_grovedbg(element: crate::Element) -> grovedbg_types::Element { + match element { + crate::Element::Item(value, element_flags) => grovedbg_types::Element::Item { + value, + element_flags, + }, + crate::Element::Tree(root_key, element_flags) => grovedbg_types::Element::Subtree { + root_key, + element_flags, + }, + crate::Element::Reference( + ReferencePathType::AbsolutePathReference(path), + _, + element_flags, + ) => grovedbg_types::Element::AbsolutePathReference { + path, + element_flags, + }, crate::Element::Reference( ReferencePathType::UpstreamRootHeightReference(n_keep, path_append), - .., + _, + element_flags, ) => grovedbg_types::Element::UpstreamRootHeightReference { n_keep: n_keep.into(), path_append, + element_flags, }, crate::Element::Reference( ReferencePathType::UpstreamRootHeightWithParentPathAdditionReference( n_keep, path_append, ), - .., + _, + element_flags, ) => grovedbg_types::Element::UpstreamRootHeightWithParentPathAdditionReference { n_keep: n_keep.into(), path_append, + element_flags, }, crate::Element::Reference( ReferencePathType::UpstreamFromElementHeightReference(n_remove, path_append), - .., + _, + element_flags, ) => grovedbg_types::Element::UpstreamFromElementHeightReference { n_remove: n_remove.into(), path_append, + element_flags, }, - crate::Element::Reference(ReferencePathType::CousinReference(swap_parent), ..) => { - grovedbg_types::Element::CousinReference { swap_parent } - } - crate::Element::Reference(ReferencePathType::RemovedCousinReference(swap_parent), ..) => { - grovedbg_types::Element::RemovedCousinReference { swap_parent } - } - crate::Element::Reference(ReferencePathType::SiblingReference(sibling_key), ..) => { - grovedbg_types::Element::SiblingReference { sibling_key } - } - crate::Element::SumItem(value, _) => grovedbg_types::Element::SumItem { value }, - crate::Element::SumTree(root_key, sum, _) => { - grovedbg_types::Element::Sumtree { root_key, sum } - } - }; + crate::Element::Reference( + ReferencePathType::CousinReference(swap_parent), + _, + element_flags, + ) => grovedbg_types::Element::CousinReference { + swap_parent, + element_flags, + }, + crate::Element::Reference( + ReferencePathType::RemovedCousinReference(swap_parent), + _, + element_flags, + ) => grovedbg_types::Element::RemovedCousinReference { + swap_parent, + element_flags, + }, + crate::Element::Reference( + ReferencePathType::SiblingReference(sibling_key), + _, + element_flags, + ) => grovedbg_types::Element::SiblingReference { + sibling_key, + element_flags, + }, + crate::Element::SumItem(value, element_flags) => grovedbg_types::Element::SumItem { + value, + element_flags, + }, + crate::Element::SumTree(root_key, sum, element_flags) => grovedbg_types::Element::Sumtree { + root_key, + sum, + element_flags, + }, + } +} + +fn node_to_update( + path: Path, + NodeDbg { + key, + value, + left_child, + right_child, + value_hash, + kv_digest_hash, + feature_type, + }: NodeDbg, +) -> Result { + // todo: GroveVersion::latest() to actual version + let grovedb_element = crate::Element::deserialize(&value, GroveVersion::latest())?; + + let element = element_to_grovedbg(grovedb_element); Ok(NodeUpdate { path, @@ -186,5 +427,13 @@ fn node_to_update( element, left_child, right_child, + feature_type: match feature_type { + TreeFeatureType::BasicMerkNode => grovedbg_types::TreeFeatureType::BasicMerkNode, + TreeFeatureType::SummedMerkNode(x) => { + grovedbg_types::TreeFeatureType::SummedMerkNode(x) + } + }, + value_hash, + kv_digest_hash, }) } diff --git a/grovedb/src/operations/proof/generate.rs b/grovedb/src/operations/proof/generate.rs index a5297eaf..6e814f67 100644 --- a/grovedb/src/operations/proof/generate.rs +++ b/grovedb/src/operations/proof/generate.rs @@ -97,7 +97,7 @@ impl GroveDb { } /// Generates a proof - fn prove_internal( + pub(crate) fn prove_internal( &self, path_query: &PathQuery, prove_options: Option, diff --git a/grovedbg-types/Cargo.toml b/grovedbg-types/Cargo.toml index a3a2878e..52beb675 100644 --- a/grovedbg-types/Cargo.toml +++ b/grovedbg-types/Cargo.toml @@ -9,3 +9,4 @@ repository = "https://github.com/dashpay/grovedb" [dependencies] serde = { version = "1.0.201", features = ["derive"] } +serde_with = { version = "3.9.0", features = ["base64"] } diff --git a/grovedbg-types/src/lib.rs b/grovedbg-types/src/lib.rs index dacc4255..0019db09 100644 --- a/grovedbg-types/src/lib.rs +++ b/grovedbg-types/src/lib.rs @@ -1,64 +1,242 @@ +use std::collections::BTreeMap; + use serde::{Deserialize, Serialize}; +use serde_with::{base64::Base64, serde_as}; pub type Key = Vec; pub type Path = Vec; pub type PathSegment = Vec; +#[serde_as] #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] pub struct NodeFetchRequest { + #[serde_as(as = "Vec")] pub path: Path, + #[serde_as(as = "Base64")] pub key: Key, } #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] pub struct RootFetchRequest; +#[serde_as] #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] pub struct NodeUpdate { + #[serde_as(as = "Option")] pub left_child: Option, + #[serde_as(as = "Option")] pub right_child: Option, + #[serde_as(as = "Vec")] pub path: Path, + #[serde_as(as = "Base64")] pub key: Key, pub element: Element, + pub feature_type: TreeFeatureType, + #[serde_as(as = "Base64")] + pub value_hash: CryptoHash, + #[serde_as(as = "Base64")] + pub kv_digest_hash: CryptoHash, } +#[serde_as] #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] pub enum Element { Subtree { + #[serde_as(as = "Option")] root_key: Option, + #[serde_as(as = "Option")] + element_flags: Option>, }, Sumtree { + #[serde_as(as = "Option")] root_key: Option, sum: i64, + #[serde_as(as = "Option")] + element_flags: Option>, }, Item { + #[serde_as(as = "Base64")] value: Vec, + #[serde_as(as = "Option")] + element_flags: Option>, }, SumItem { value: i64, + #[serde_as(as = "Option")] + element_flags: Option>, }, AbsolutePathReference { + #[serde_as(as = "Vec")] path: Path, + #[serde_as(as = "Option")] + element_flags: Option>, }, UpstreamRootHeightReference { n_keep: u32, + #[serde_as(as = "Vec")] path_append: Vec, + #[serde_as(as = "Option")] + element_flags: Option>, }, UpstreamRootHeightWithParentPathAdditionReference { n_keep: u32, + #[serde_as(as = "Vec")] path_append: Vec, + #[serde_as(as = "Option")] + element_flags: Option>, }, UpstreamFromElementHeightReference { n_remove: u32, + #[serde_as(as = "Vec")] path_append: Vec, + #[serde_as(as = "Option")] + element_flags: Option>, }, CousinReference { + #[serde_as(as = "Base64")] swap_parent: PathSegment, + #[serde_as(as = "Option")] + element_flags: Option>, }, RemovedCousinReference { + #[serde_as(as = "Vec")] swap_parent: Vec, + #[serde_as(as = "Option")] + element_flags: Option>, }, SiblingReference { + #[serde_as(as = "Base64")] sibling_key: Key, + #[serde_as(as = "Option")] + element_flags: Option>, + }, +} + +#[serde_as] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct PathQuery { + #[serde_as(as = "Vec")] + pub path: Path, + pub query: SizedQuery, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct SizedQuery { + pub query: Query, + pub limit: Option, + pub offset: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct Query { + pub items: Vec, + pub default_subquery_branch: SubqueryBranch, + pub conditional_subquery_branches: Vec<(QueryItem, SubqueryBranch)>, + pub left_to_right: bool, +} + +#[serde_as] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub enum QueryItem { + Key(#[serde_as(as = "Base64")] Vec), + Range { + #[serde_as(as = "Base64")] + start: Key, + #[serde_as(as = "Base64")] + end: Key, }, + RangeInclusive { + #[serde_as(as = "Base64")] + start: Key, + #[serde_as(as = "Base64")] + end: Key, + }, + RangeFull, + RangeFrom(#[serde_as(as = "Base64")] Key), + RangeTo(#[serde_as(as = "Base64")] Key), + RangeToInclusive(#[serde_as(as = "Base64")] Key), + RangeAfter(#[serde_as(as = "Base64")] Key), + RangeAfterTo { + #[serde_as(as = "Base64")] + after: Key, + #[serde_as(as = "Base64")] + to: Key, + }, + RangeAfterToInclusive { + #[serde_as(as = "Base64")] + after: Key, + #[serde_as(as = "Base64")] + to: Key, + }, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct SubqueryBranch { + pub subquery_path: Option>, + pub subquery: Option>, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct Proof { + pub root_layer: ProofLayer, + pub prove_options: ProveOptions, +} + +#[serde_as] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct ProofLayer { + pub merk_proof: Vec, + #[serde_as(as = "BTreeMap")] + pub lower_layers: BTreeMap, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub enum MerkProofOp { + Push(MerkProofNode), + PushInverted(MerkProofNode), + Parent, + Child, + ParentInverted, + ChildInverted, +} + +pub type CryptoHash = [u8; 32]; + +#[serde_as] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub enum MerkProofNode { + Hash(#[serde_as(as = "Base64")] CryptoHash), + KVHash(#[serde_as(as = "Base64")] CryptoHash), + KVDigest( + #[serde_as(as = "Base64")] Key, + #[serde_as(as = "Base64")] CryptoHash, + ), + KV(#[serde_as(as = "Base64")] Key, Element), + KVValueHash( + #[serde_as(as = "Base64")] Key, + Element, + #[serde_as(as = "Base64")] CryptoHash, + ), + KVValueHashFeatureType( + #[serde_as(as = "Base64")] Key, + Element, + #[serde_as(as = "Base64")] CryptoHash, + TreeFeatureType, + ), + KVRefValueHash( + #[serde_as(as = "Base64")] Key, + Element, + #[serde_as(as = "Base64")] CryptoHash, + ), +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub enum TreeFeatureType { + BasicMerkNode, + SummedMerkNode(i64), +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct ProveOptions { + pub decrease_limit_on_empty_sub_query_result: bool, } diff --git a/merk/src/debugger.rs b/merk/src/debugger.rs index c5d322c0..c806351d 100644 --- a/merk/src/debugger.rs +++ b/merk/src/debugger.rs @@ -4,7 +4,7 @@ use grovedb_costs::CostsExt; use grovedb_storage::StorageContext; use grovedb_version::version::GroveVersion; -use crate::{tree::kv::ValueDefinedCostType, Error, Merk}; +use crate::{tree::kv::ValueDefinedCostType, CryptoHash, Error, Merk, TreeFeatureType}; impl<'a, S: StorageContext<'a>> Merk { pub fn get_node_dbg(&self, key: &[u8]) -> Result, Error> { @@ -16,6 +16,9 @@ impl<'a, S: StorageContext<'a>> Merk { value: tree.inner.value_as_slice().to_owned(), left_child: tree.link(true).map(|link| link.key().to_owned()), right_child: tree.link(false).map(|link| link.key().to_owned()), + value_hash: *tree.inner.kv.value_hash(), + kv_digest_hash: *tree.inner.kv.hash(), + feature_type: tree.inner.kv.feature_type(), } .wrap_with_cost(Default::default()) }, @@ -32,6 +35,9 @@ impl<'a, S: StorageContext<'a>> Merk { value: tree.inner.value_as_slice().to_owned(), left_child: tree.link(true).map(|link| link.key().to_owned()), right_child: tree.link(false).map(|link| link.key().to_owned()), + value_hash: *tree.inner.kv.value_hash(), + kv_digest_hash: *tree.inner.kv.hash(), + feature_type: tree.inner.kv.feature_type(), }) })) } @@ -43,4 +49,7 @@ pub struct NodeDbg { pub value: Vec, pub left_child: Option>, pub right_child: Option>, + pub value_hash: CryptoHash, + pub kv_digest_hash: CryptoHash, + pub feature_type: TreeFeatureType, } diff --git a/merk/src/tree/kv.rs b/merk/src/tree/kv.rs index b10733fc..4fd3f7f7 100644 --- a/merk/src/tree/kv.rs +++ b/merk/src/tree/kv.rs @@ -252,6 +252,10 @@ impl KV { &self.hash } + pub fn feature_type(&self) -> TreeFeatureType { + self.feature_type + } + /// Consumes the `KV` and returns its key without allocating or cloning. #[inline] pub fn take_key(self) -> Vec { From f50c03aa69905cb3a1340e270fdc062e405499c9 Mon Sep 17 00:00:00 2001 From: fominok Date: Fri, 26 Jul 2024 12:16:29 +0200 Subject: [PATCH 5/7] bump grovedbg version (#322) --- .gitmodules | 3 --- grovedb/build.rs | 4 ++-- grovedb/grovedbg | 1 - 3 files changed, 2 insertions(+), 6 deletions(-) delete mode 100644 .gitmodules delete mode 160000 grovedb/grovedbg diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index 2b7c58dd..00000000 --- a/.gitmodules +++ /dev/null @@ -1,3 +0,0 @@ -[submodule "grovedb/grovedbg"] - path = grovedb/grovedbg - url = https://github.com/dashpay/grovedbg diff --git a/grovedb/build.rs b/grovedb/build.rs index d7b4b7b9..ad1ea64f 100644 --- a/grovedb/build.rs +++ b/grovedb/build.rs @@ -6,8 +6,8 @@ fn main() { use sha2::{digest::FixedOutput, Digest, Sha256}; const GROVEDBG_SHA256: [u8; 32] = - hex!("206d865949e7f12b843c342cea3b172d4c78d4eb754b80d272164d59f62a2276"); - const GROVEDBG_VERSION: &str = "v1.0.0-rc.3"; + hex!("d1152b9a0257e02625efc223a98a9f2b9a2aa7ade133a7c32fee1e5f7d3a607e"); + const GROVEDBG_VERSION: &str = "v1.0.0-rc.4"; let out_dir = PathBuf::from(&env::var_os("OUT_DIR").unwrap()); let grovedbg_zip_path = out_dir.join("grovedbg.zip"); diff --git a/grovedb/grovedbg b/grovedb/grovedbg deleted file mode 160000 index 954be745..00000000 --- a/grovedb/grovedbg +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 954be74510d3c3bb79a7e622e55af66aae5c6ad4 From 0e7ca34fcb9dfcfd5643c853c97c37f9699f3362 Mon Sep 17 00:00:00 2001 From: fominok Date: Sat, 27 Jul 2024 14:00:08 +0200 Subject: [PATCH 6/7] bump grovedbg version (#323) --- grovedb/build.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/grovedb/build.rs b/grovedb/build.rs index ad1ea64f..83a38900 100644 --- a/grovedb/build.rs +++ b/grovedb/build.rs @@ -6,8 +6,8 @@ fn main() { use sha2::{digest::FixedOutput, Digest, Sha256}; const GROVEDBG_SHA256: [u8; 32] = - hex!("d1152b9a0257e02625efc223a98a9f2b9a2aa7ade133a7c32fee1e5f7d3a607e"); - const GROVEDBG_VERSION: &str = "v1.0.0-rc.4"; + hex!("c6636f10b43c703128b621a7c4b94139a1a7d0a603e26fca1771734a7994bb7c"); + const GROVEDBG_VERSION: &str = "v1.0.0-rc.5"; let out_dir = PathBuf::from(&env::var_os("OUT_DIR").unwrap()); let grovedbg_zip_path = out_dir.join("grovedbg.zip"); From f028ddba30485ca3c70ecf22ab1caed93671774c Mon Sep 17 00:00:00 2001 From: fominok Date: Sun, 28 Jul 2024 09:52:01 +0200 Subject: [PATCH 7/7] update versions to 1.0.0 (#324) --- costs/Cargo.toml | 2 +- grovedb-version/Cargo.toml | 6 +++--- grovedb/Cargo.toml | 22 +++++++++++----------- grovedbg-types/Cargo.toml | 2 +- merk/Cargo.toml | 14 +++++++------- node-grove/Cargo.toml | 2 +- path/Cargo.toml | 2 +- storage/Cargo.toml | 8 ++++---- visualize/Cargo.toml | 6 ++---- 9 files changed, 31 insertions(+), 33 deletions(-) diff --git a/costs/Cargo.toml b/costs/Cargo.toml index 8178f839..4ee1a55b 100644 --- a/costs/Cargo.toml +++ b/costs/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "grovedb-costs" -version = "1.0.0-rc.2" +version = "1.0.0" edition = "2021" license = "MIT" description = "Costs extension crate for GroveDB" diff --git a/grovedb-version/Cargo.toml b/grovedb-version/Cargo.toml index 06189c44..9802a0cf 100644 --- a/grovedb-version/Cargo.toml +++ b/grovedb-version/Cargo.toml @@ -2,13 +2,13 @@ name = "grovedb-version" authors = ["Samuel Westrich "] description = "Versioning library for Platform" -version = "1.0.0-rc.2" +version = "1.0.0" edition = "2021" license = "MIT" [dependencies] -thiserror = { version = "1.0.59" } -versioned-feature-core = { git = "https://github.com/dashpay/versioned-feature-core", version = "1.0.0" } +thiserror = "1.0.59" +versioned-feature-core = "1.0.0" [features] mock-versions = [] diff --git a/grovedb/Cargo.toml b/grovedb/Cargo.toml index efedc420..73cd34f4 100644 --- a/grovedb/Cargo.toml +++ b/grovedb/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "grovedb" description = "Fully featured database using balanced hierarchical authenticated data structures" -version = "1.0.0-rc.2" +version = "1.0.0" authors = ["Samuel Westrich ", "Wisdom Ogwu "] edition = "2021" license = "MIT" @@ -11,29 +11,29 @@ readme = "../README.md" documentation = "https://docs.rs/grovedb" [dependencies] -grovedb-merk = { version = "1.0.0-rc.2", path = "../merk", optional = true, default-features = false } +grovedb-merk = { version = "1.0.0", path = "../merk", optional = true, default-features = false } thiserror = { version = "1.0.59", optional = true } tempfile = { version = "3.10.1", optional = true } bincode = { version = "2.0.0-rc.3" } -grovedb-storage = { version = "1.0.0-rc.2", path = "../storage", optional = true } -grovedb-visualize = { version = "1.0.0-rc.2", path = "../visualize", optional = true } -hex = { version = "0.4.3"} +grovedb-storage = { version = "1.0.0", path = "../storage", optional = true } +grovedb-visualize = { version = "1.0.0", path = "../visualize", optional = true } +hex = "0.4.3" itertools = { version = "0.12.1", optional = true } -derive_more = { version = "0.99.18" } +derive_more = "0.99.18" integer-encoding = { version = "4.0.0", optional = true } -grovedb-costs = { version = "1.0.0-rc.2", path = "../costs", optional = true } +grovedb-costs = { version = "1.0.0", path = "../costs", optional = true } nohash-hasher = { version = "0.2.0", optional = true } -indexmap = { version = "2.2.6"} +indexmap = "2.2.6" intmap = { version = "2.0.0", optional = true } -grovedb-path = { version = "1.0.0-rc.2", path = "../path" } -grovedbg-types = { version = "1.0.0-rc.2", path = "../grovedbg-types", optional = true } +grovedb-path = { version = "1.0.0", path = "../path" } +grovedbg-types = { version = "1.0.0", path = "../grovedbg-types", optional = true } tokio = { version = "1.37.0", features = ["rt-multi-thread", "net"], optional = true } axum = { version = "0.7.5", features = ["macros"], optional = true } tower-http = { version = "0.5.2", features = ["fs"], optional = true } blake3 = "1.4.0" bitvec = "1" zip-extensions = { version ="0.6.2", optional = true } -grovedb-version = "1.0.0-rc.2" +grovedb-version = { path = "../grovedb-version", version = "1.0.0" } [dev-dependencies] rand = "0.8.5" diff --git a/grovedbg-types/Cargo.toml b/grovedbg-types/Cargo.toml index 52beb675..f1ab09d6 100644 --- a/grovedbg-types/Cargo.toml +++ b/grovedbg-types/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "grovedbg-types" -version = "1.0.0-rc.2" +version = "1.0.0" edition = "2021" description = "Common type definitions for data exchange over GroveDBG protocol" authors = ["Evgeny Fomin "] diff --git a/merk/Cargo.toml b/merk/Cargo.toml index a41cc8f7..6fcfadd2 100644 --- a/merk/Cargo.toml +++ b/merk/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "grovedb-merk" description = "Merkle key/value store adapted for GroveDB" -version = "1.0.0-rc.2" +version = "1.0.0" authors = ["Samuel Westrich ", "Wisdom Ogwu ", "Matt Bell "] edition = "2021" license = "MIT" @@ -12,15 +12,15 @@ documentation = "https://docs.rs/grovedb-merk" [dependencies] thiserror = "1.0.58" -grovedb-storage = { version = "1.0.0-rc.2", path = "../storage", optional = true } +grovedb-storage = { version = "1.0.0", path = "../storage", optional = true } failure = "0.1.8" integer-encoding = "4.0.0" indexmap = "2.2.6" -grovedb-costs = { version = "1.0.0-rc.2", path = "../costs" } -grovedb-visualize = { version = "1.0.0-rc.2", path = "../visualize" } -grovedb-path = { version = "1.0.0-rc.2", path = "../path" } -hex = { version = "0.4.3" } -grovedb-version = "1.0.0-rc.2" +grovedb-costs = { version = "1.0.0", path = "../costs" } +grovedb-visualize = { version = "1.0.0", path = "../visualize" } +grovedb-path = { version = "1.0.0", path = "../path" } +hex = "0.4.3" +grovedb-version = { version = "1.0.0", path = "../grovedb-version" } [dependencies.time] version = "0.3.34" diff --git a/node-grove/Cargo.toml b/node-grove/Cargo.toml index 17e5ee09..4a4f5281 100644 --- a/node-grove/Cargo.toml +++ b/node-grove/Cargo.toml @@ -11,7 +11,7 @@ crate-type = ["cdylib"] [dependencies] grovedb = { path = "../grovedb", features = ["full", "estimated_costs"] } -grovedb-version = "1.0.0-rc.2" +grovedb-version = { path = "../grovedb-version", version = "1.0.0" } [dependencies.neon] version = "0.10.1" diff --git a/path/Cargo.toml b/path/Cargo.toml index bae12675..111bc474 100644 --- a/path/Cargo.toml +++ b/path/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "grovedb-path" -version = "1.0.0-rc.2" +version = "1.0.0" edition = "2021" license = "MIT" description = "Path extension crate for GroveDB" diff --git a/storage/Cargo.toml b/storage/Cargo.toml index 7d7030bf..7db2b599 100644 --- a/storage/Cargo.toml +++ b/storage/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "grovedb-storage" -version = "1.0.0-rc.2" +version = "1.0.0" edition = "2021" license = "MIT" description = "Storage extension crate for GroveDB" @@ -14,13 +14,13 @@ num_cpus = { version = "1.16.0", optional = true } tempfile = { version = "3.10.1", optional = true } blake3 = { version = "1.5.1", optional = true } integer-encoding = { version = "4.0.0", optional = true } -grovedb-visualize = { version = "1.0.0-rc.2", path = "../visualize" } +grovedb-visualize = { version = "1.0.0", path = "../visualize" } strum = { version = "0.26.2", features = ["derive"] } -grovedb-costs = { version = "1.0.0-rc.2", path = "../costs" } +grovedb-costs = { version = "1.0.0", path = "../costs" } thiserror = "1.0.59" rocksdb = { version = "0.22.0", optional = true } hex = "0.4.3" -grovedb-path = { version = "1.0.0-rc.2", path = "../path" } +grovedb-path = { version = "1.0.0", path = "../path" } [features] rocksdb_storage = ["rocksdb", "num_cpus", "lazy_static", "tempfile", "blake3", "integer-encoding"] diff --git a/visualize/Cargo.toml b/visualize/Cargo.toml index ac93bff0..821bf443 100644 --- a/visualize/Cargo.toml +++ b/visualize/Cargo.toml @@ -1,15 +1,13 @@ [package] name = "grovedb-visualize" -version = "1.0.0-rc.2" +version = "1.0.0" edition = "2021" license = "MIT" -description = "Visualizer extension crate for GroveDB" +description = "Debug prints extension crate for GroveDB" homepage = "https://www.grovedb.org/" documentation = "https://docs.rs/grovedb-visualize" repository = "https://github.com/dashpay/grovedb" -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - [dependencies] hex = "0.4.3" itertools = "0.12.1"