Adjust the API to read better, and improve some of the top code heira…

…rchy (#148)

Cargo.toml

@@ -28,6 +28,7 @@ bench = false
 [dependencies]
 
 primitive-types = { version = "0.12.1", features = ["serde"] } # H256 & U256 (I think parity uses this so maybe we just use that crate instead)
+thiserror = "1.0"
 
 # crypto
 rand = "0.8.5"
@@ -40,7 +41,6 @@ bulletproofs = "4.0.0"
 curve25519-dalek-ng = "4.1.1"
 
 # concurrency
-thiserror = "1.0"
 displaydoc = "0.2"
 rayon = "1.7.0"
 dashmap = { version = "5.5.3", features = ["serde"] }
@@ -57,9 +57,10 @@ patharg = "0.3.0"
 
 # files & serialization
 serde = { version = "1.0.188", features = ["derive"] }
+serde_with = "3.4.0"
+serde_bytes = "0.11.12"
 toml = "0.8.2"
 csv = "1.3.0"
-serde_bytes = "0.11.12"
 bincode = "1.3.3"
 chrono = "0.4.31"
 derive_builder = "0.12.0"

README.md

@@ -66,18 +66,18 @@ The CLI offers 3 main operations: tree building, proof generation & proof verifi
 #### Tree building
 
 Building a tree can be done:
-- from a config file (see tree_config_example.toml)
+- from a config file (see dapol_config_example.toml)
 - from CLI arguments
 - by deserializing an already-built tree
 
 Build a tree using config file (full log verbosity):
 ```bash
-./target/release/dapol -vvv build-tree config-file ./examples/tree_config_example.toml
+./target/release/dapol -vvv build-tree config-file ./examples/dapol_config_example.toml
 ```
 
 Add serialization:
 ```bash
-./target/release/dapol -vvv build-tree config-file ./examples/tree_config_example.toml --serialize .
+./target/release/dapol -vvv build-tree config-file ./examples/dapol_config_example.toml --serialize .
 ```
 
 Deserialize a tree from a file:
@@ -87,7 +87,7 @@ Deserialize a tree from a file:
 
 Generate proofs (proofs will live in the `./inclusion_proofs/` directory):
 ```bash
-./target/release/dapol -vvv build-tree config-file ./examples/tree_config_example.toml --gen-proofs ./examples/entities_example.csv
+./target/release/dapol -vvv build-tree config-file ./examples/dapol_config_example.toml --gen-proofs ./examples/entities_example.csv
 ```
 
 Build a tree using cli args as apposed to a config file:

benches/criterion_benches.rs

@@ -8,14 +8,14 @@
 //! long (see large_input_benches.rs).
 
 use std::path::Path;
+use std::str::FromStr;
 
 use criterion::measurement::Measurement;
 use criterion::{criterion_group, criterion_main};
 use criterion::{BenchmarkId, Criterion, SamplingMode};
 use statistical::*;
 
-use dapol::accumulators::{NdmSmt, NdmSmtConfigBuilder};
-use dapol::{Accumulator, InclusionProof};
+use dapol::{DapolConfigBuilder, DapolTree, InclusionProof, Secret};
 
 mod inputs;
 use inputs::{max_thread_counts_greater_than, num_entities_in_range, tree_heights_in_range};
@@ -43,6 +43,8 @@ pub fn bench_build_tree<T: Measurement>(c: &mut Criterion<T>) {
     let epoch = jemalloc_ctl::epoch::mib().unwrap();
     let allocated = jemalloc_ctl::stats::allocated::mib().unwrap();
 
+    let master_secret = Secret::from_str("secret").unwrap();
+
     dapol::initialize_machine_parallelism();
     dapol::utils::activate_logging(*LOG_VERBOSITY);
 
@@ -103,7 +105,7 @@ pub fn bench_build_tree<T: Measurement>(c: &mut Criterion<T>) {
                 // Tree build.
 
                 let mut memory_readings = vec![];
-                let mut ndm_smt = Option::<NdmSmt>::None;
+                let mut dapol_tree = Option::<DapolTree>::None;
 
                 group.bench_with_input(
                     BenchmarkId::new(
@@ -119,17 +121,20 @@ pub fn bench_build_tree<T: Measurement>(c: &mut Criterion<T>) {
                     |bench, tup| {
                         bench.iter(|| {
                             // this is necessary for the memory readings to work
-                            ndm_smt = None;
+                            dapol_tree = None;
 
                             epoch.advance().unwrap();
                             let before = allocated.read().unwrap();
 
-                            ndm_smt = Some(
-                                NdmSmtConfigBuilder::default()
+                            dapol_tree = Some(
+                                DapolConfigBuilder::default()
+                                    .accumulator_type(dapol::AccumulatorType::NdmSmt)
                                     .height(tup.0)
                                     .max_thread_count(tup.1)
                                     .num_random_entities(tup.2)
+                                    .master_secret(master_secret.clone())
                                     .build()
+                                    .expect("Unable to build DapolConfig")
                                     .parse()
                                     .expect("Unable to parse NdmSmtConfig"),
                             );
@@ -160,8 +165,8 @@ pub fn bench_build_tree<T: Measurement>(c: &mut Criterion<T>) {
                 let src_dir = env!("CARGO_MANIFEST_DIR");
                 let target_dir = Path::new(&src_dir).join("target");
                 let dir = target_dir.join("serialized_trees");
-                let path = Accumulator::parse_accumulator_serialization_path(dir).unwrap();
-                let acc = Accumulator::NdmSmt(ndm_smt.expect("Tree should have been built"));
+                let path = DapolTree::parse_serialization_path(dir).unwrap();
+                let tree = dapol_tree.expect("Tree should have been built");
 
                 group.bench_function(
                     BenchmarkId::new(
@@ -174,7 +179,7 @@ pub fn bench_build_tree<T: Measurement>(c: &mut Criterion<T>) {
                         ),
                     ),
                     |bench| {
-                        bench.iter(|| acc.serialize(path.clone()).unwrap());
+                        bench.iter(|| tree.serialize(path.clone()).unwrap());
                     },
                 );
 
@@ -196,6 +201,11 @@ pub fn bench_build_tree<T: Measurement>(c: &mut Criterion<T>) {
 pub fn bench_generate_proof<T: Measurement>(c: &mut Criterion<T>) {
     let mut group = c.benchmark_group("proofs");
 
+    let master_secret = Secret::from_str("secret").unwrap();
+
+    dapol::initialize_machine_parallelism();
+    dapol::utils::activate_logging(*LOG_VERBOSITY);
+
     for h in tree_heights_in_range(*MIN_HEIGHT, *MAX_HEIGHT).into_iter() {
         for n in num_entities_in_range(*MIN_ENTITIES, *MAX_ENTITIES).into_iter() {
             {
@@ -238,15 +248,19 @@ pub fn bench_generate_proof<T: Measurement>(c: &mut Criterion<T>) {
                 continue;
             }
 
-            let ndm_smt = NdmSmtConfigBuilder::default()
+            let dapol_tree = DapolConfigBuilder::default()
+                .accumulator_type(dapol::AccumulatorType::NdmSmt)
+                .master_secret(master_secret.clone())
                 .height(h)
                 .num_random_entities(n)
                 .build()
+                .expect("Unable to build DapolConfig")
                 .parse()
                 .expect("Unable to parse NdmSmtConfig");
 
-            let entity_id = ndm_smt
+            let entity_id = dapol_tree
                 .entity_mapping()
+                .unwrap()
                 .keys()
                 .next()
                 .expect("Tree should have at least 1 entity");
@@ -261,7 +275,7 @@ pub fn bench_generate_proof<T: Measurement>(c: &mut Criterion<T>) {
                 |bench| {
                     bench.iter(|| {
                         proof = Some(
-                            ndm_smt
+                            dapol_tree
                                 .generate_inclusion_proof(entity_id)
                                 .expect("Proof should have been generated successfully"),
                         );
@@ -296,6 +310,11 @@ pub fn bench_generate_proof<T: Measurement>(c: &mut Criterion<T>) {
 pub fn bench_verify_proof<T: Measurement>(c: &mut Criterion<T>) {
     let mut group = c.benchmark_group("proofs");
 
+    let master_secret = Secret::from_str("secret").unwrap();
+
+    dapol::initialize_machine_parallelism();
+    dapol::utils::activate_logging(*LOG_VERBOSITY);
+
     for h in tree_heights_in_range(*MIN_HEIGHT, *MAX_HEIGHT).into_iter() {
         for n in num_entities_in_range(*MIN_ENTITIES, *MAX_ENTITIES).into_iter() {
             {
@@ -338,22 +357,26 @@ pub fn bench_verify_proof<T: Measurement>(c: &mut Criterion<T>) {
                 continue;
             }
 
-            let ndm_smt = NdmSmtConfigBuilder::default()
+            let dapol_tree = DapolConfigBuilder::default()
+                .accumulator_type(dapol::AccumulatorType::NdmSmt)
+                .master_secret(master_secret.clone())
                 .height(h)
                 .num_random_entities(n)
                 .build()
+                .expect("Unable to build DapolConfig")
                 .parse()
                 .expect("Unable to parse NdmSmtConfig");
 
-            let root_hash = ndm_smt.root_hash();
+            let root_hash = dapol_tree.root_hash();
 
-            let entity_id = ndm_smt
+            let entity_id = dapol_tree
                 .entity_mapping()
+                .unwrap()
                 .keys()
                 .next()
                 .expect("Tree should have at least 1 entity");
 
-            let proof = ndm_smt
+            let proof = dapol_tree
                 .generate_inclusion_proof(entity_id)
                 .expect("Proof should have been generated successfully");
 

benches/manual_benches.rs

@@ -6,11 +6,12 @@
 //! unfortunately, but this is the trade-off.
 
 use std::path::Path;
+use std::str::FromStr;
 use std::time::Instant;
 
 use statistical::*;
 
-use dapol::accumulators::{Accumulator, NdmSmt, NdmSmtConfigBuilder};
+use dapol::{DapolConfigBuilder, DapolTree, Secret};
 
 mod inputs;
 use inputs::{max_thread_counts_greater_than, num_entities_in_range, tree_heights_in_range};
@@ -36,6 +37,8 @@ fn main() {
 
     let total_mem = system_total_memory_mb();
 
+    let master_secret = Secret::from_str("secret").unwrap();
+
     dapol::initialize_machine_parallelism();
     dapol::utils::activate_logging(*LOG_VERBOSITY);
 
@@ -100,14 +103,14 @@ fn main() {
                 // ==============================================================
                 // Tree build.
 
-                let mut ndm_smt = Option::<NdmSmt>::None;
+                let mut dapol_tree = Option::<DapolTree>::None;
                 let mut memory_readings = vec![];
                 let mut timings = vec![];
 
                 // Do 3 readings (Criterion does 10 minimum).
                 for _i in 0..3 {
                     // this is necessary for the memory readings to work
-                    ndm_smt = None;
+                    dapol_tree = None;
 
                     println!(
                         "building tree i {} time {}",
@@ -119,14 +122,17 @@ fn main() {
                     let mem_before = allocated.read().unwrap();
                     let time_start = Instant::now();
 
-                    ndm_smt = Some(
-                        NdmSmtConfigBuilder::default()
+                    dapol_tree = Some(
+                        DapolConfigBuilder::default()
+                            .accumulator_type(dapol::AccumulatorType::NdmSmt)
                             .height(h)
                             .max_thread_count(t)
+                            .master_secret(master_secret.clone())
                             .num_random_entities(n)
                             .build()
+                            .expect("Unable to build DapolConfig")
                             .parse()
-                            .expect("Unable to parse NdmSmtConfig"),
+                            .expect("Unable to parse DapolConfig"),
                     );
 
                     let tree_build_time = time_start.elapsed();
@@ -154,12 +160,13 @@ fn main() {
                 let src_dir = env!("CARGO_MANIFEST_DIR");
                 let target_dir = Path::new(&src_dir).join("target");
                 let dir = target_dir.join("serialized_trees");
-                let path = Accumulator::parse_accumulator_serialization_path(dir).unwrap();
-                let acc =
-                    Accumulator::NdmSmt(ndm_smt.expect("NDM SMT should have been set in loop"));
+                let path = DapolTree::parse_serialization_path(dir).unwrap();
 
                 let time_start = Instant::now();
-                acc.serialize(path.clone()).unwrap();
+                dapol_tree
+                    .expect("DapolTree should have been set in loop")
+                    .serialize(path.clone())
+                    .unwrap();
                 let serialization_time = time_start.elapsed();
 
                 let file_size = std::fs::metadata(path)

benches/utils.rs

@@ -30,13 +30,13 @@ pub fn bytes_to_string(num_bytes: usize) -> String {
     if n < kb {
         format!("{} bytes", num_bytes)
     } else if n < mb {
-        format!("{} kB", n / kb)
+        format!("{:.3} kB", n as f64 / kb as f64)
     } else if n < gb {
-        format!("{:.2} MB", n as f64 / mb as f64)
+        format!("{:.3} MB", n as f64 / mb as f64)
     } else if n < tb {
-        format!("{:.2} GB", n as f64 / gb as f64)
+        format!("{:.3} GB", n as f64 / gb as f64)
     } else {
-        format!("{:.2} TB", n as f64 / tb as f64)
+        format!("{:.3} TB", n as f64 / tb as f64)
     }
 }
 

examples/dapol_config_example.toml

@@ -0,0 +1,66 @@
+# There are various different accumulator types (e.g. NDM-SMT).
+#
+# This value must be set.
+accumulator_type = "ndm-smt"
+
+# This is a public value that is used to aid the KDF when generating secret
+# blinding factors for the Pedersen commitments.
+#
+# If it is not set then it will be randomly generated.
+salt_b = "salt_b"
+
+# This is a public value that is used to aid the KDF when generating secret
+# salt values, which are in turn used in the hash function when generating
+# node hashes.
+#
+# If it is not set then it will be randomly generated.
+salt_s = "salt_s"
+
+# Height of the tree.
+#
+# If not set the default height will be used:
+# `dapol::Height::default()`.
+height = 16
+
+# This is a public value representing the maximum amount that any single
+# entity's liability can be, and is used in the range proofs:
+# $[0, 2^{\text{height}} \times \text{max_liability}]$
+#
+# If not set then the default value will be used:
+# `2.pow(dapol::DEFAULT_RANGE_PROOF_UPPER_BOUND_BIT_LENGTH)`.
+max_liability = 10_000_000
+
+# Max number of threads to be spawned for multi-threading algorithms.
+#
+# If not set the max parallelism of the underlying machine will be used.
+max_thread_count = 8
+
+# Can be a file or directory (default file name given in this case)
+#
+# If not set then no serialization is done.
+serialization_path = "./tree.dapoltree"
+
+# At least one of file_path or generate_random must be present.
+#
+# If both are given then file_path is preferred and generate_random is ignored.
+[entities]
+
+# Path to a file containing a list of entity IDs and their liabilities.
+file_path = "./entities_example.csv"
+
+# Generate the given number of entities, with random IDs & liabilities.
+# This is useful for testing.
+num_random_entities= 100
+
+# At least on of file_path or master_secret must be present.
+# The master secret is known only to the tree generator and is used to
+# generate all other secret values required by the tree.
+#
+# If both are given then file_path is preferred and master_secret is ignored.
+[secrets]
+
+# Path to a file containing a list of entity IDs and their liabilities.
+file_path = "./dapol_secrets_example.toml"
+
+# String value of the master secret.
+master_secret = "master_secret"

examples/dapol_secrets_example.toml

@@ -0,0 +1 @@
+master_secret = "master_secret"