diff --git a/Cargo.lock b/Cargo.lock index 9258f031..46c31c8c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -46,6 +46,7 @@ name = "bridgetree" version = "0.5.0" dependencies = [ "incrementalmerkletree", + "incrementalmerkletree-testing", "proptest", ] @@ -111,7 +112,7 @@ checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7" [[package]] name = "incrementalmerkletree" -version = "0.6.0" +version = "0.7.0" dependencies = [ "either", "proptest", @@ -120,6 +121,14 @@ dependencies = [ "rand_core", ] +[[package]] +name = "incrementalmerkletree-testing" +version = "0.1.0" +dependencies = [ + "incrementalmerkletree", + "proptest", +] + [[package]] name = "instant" version = "0.1.12" @@ -324,6 +333,7 @@ dependencies = [ "bitflags 2.4.1", "either", "incrementalmerkletree", + "incrementalmerkletree-testing", "proptest", "tempfile", "tracing", diff --git a/Cargo.toml b/Cargo.toml index 98ebc88c..b398bb3e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,23 @@ [workspace] members = [ "incrementalmerkletree", + "incrementalmerkletree-testing", "bridgetree", "shardtree", ] + +[workspace.package] +edition = "2021" +rust-version = "1.64" +repository = "https://github.com/zcash/incrementalmerkletree" +homepage = "https://github.com/zcash/incrementalmerkletree" +license = "MIT OR Apache-2.0" +categories = ["algorithms", "data-structures"] + +[workspace.dependencies] +# Intra-workspace dependencies +incrementalmerkletree = { version = "0.7", path = "incrementalmerkletree" } +incrementalmerkletree-testing = { version = "0.1", path = "incrementalmerkletree-testing" } + +# Testing +proptest = "1" diff --git a/bridgetree/CHANGELOG.md b/bridgetree/CHANGELOG.md index 5f8a7202..4c7d5820 100644 --- a/bridgetree/CHANGELOG.md +++ b/bridgetree/CHANGELOG.md @@ -7,6 +7,9 @@ and this project adheres to Rust's notion of ## [Unreleased] +### Changed +- MSRV is now 1.64 + ## [0.5.0] - 2024-08-12 ### Changed diff --git a/bridgetree/Cargo.toml b/bridgetree/Cargo.toml index 3fb02b33..409e093e 100644 --- a/bridgetree/Cargo.toml +++ b/bridgetree/Cargo.toml @@ -1,25 +1,26 @@ [package] name = "bridgetree" +description = "A space-efficient Merkle tree designed for linear appends with witnessing of marked leaves, checkpointing & state restoration." version = "0.5.0" authors = [ "Kris Nuttycombe ", "Sean Bowe ", ] -edition = "2021" -license = "MIT OR Apache-2.0" -description = "A space-efficient Merkle tree designed for linear appends with witnessing of marked leaves, checkpointing & state restoration." -homepage = "https://github.com/zcash/incrementalmerkletree" -repository = "https://github.com/zcash/incrementalmerkletree" -categories = ["algorithms", "data-structures"] -rust-version = "1.60" +edition.workspace = true +license.workspace = true +homepage.workspace = true +repository.workspace = true +categories.workspace = true +rust-version.workspace = true [dependencies] -incrementalmerkletree = { version = "0.6", path = "../incrementalmerkletree" } -proptest = { version = "1.0.0", optional = true } +incrementalmerkletree.workspace = true +proptest = { workspace = true, optional = true } [dev-dependencies] -incrementalmerkletree = { version = "0.6", path = "../incrementalmerkletree", features = ["test-dependencies"] } -proptest = "1.0.0" +incrementalmerkletree = { workspace = true, features = ["test-dependencies"] } +incrementalmerkletree-testing.workspace = true +proptest.workspace = true [features] test-dependencies = ["proptest"] diff --git a/bridgetree/src/lib.rs b/bridgetree/src/lib.rs index 72300826..4a2bac29 100644 --- a/bridgetree/src/lib.rs +++ b/bridgetree/src/lib.rs @@ -986,13 +986,11 @@ mod tests { use std::fmt::Debug; use super::*; - use incrementalmerkletree::{ - testing::{ - self, apply_operation, arb_operation, check_checkpoint_rewind, check_operations, - check_remove_mark, check_rewind_remove_mark, check_root_hashes, check_witnesses, - complete_tree::CompleteTree, CombinedTree, SipHashable, - }, - Hashable, + use incrementalmerkletree::Hashable; + use incrementalmerkletree_testing::{ + self as testing, apply_operation, arb_operation, check_checkpoint_rewind, check_operations, + check_remove_mark, check_rewind_remove_mark, check_root_hashes, check_witnesses, + complete_tree::CompleteTree, CombinedTree, SipHashable, }; impl testing::Tree diff --git a/incrementalmerkletree-testing/CHANGELOG.md b/incrementalmerkletree-testing/CHANGELOG.md new file mode 100644 index 00000000..3cac1577 --- /dev/null +++ b/incrementalmerkletree-testing/CHANGELOG.md @@ -0,0 +1,11 @@ +# Changelog +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to Rust's notion of +[Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## Unreleased + +## [0.1.0] - 2024-09-25 +Initial release. diff --git a/incrementalmerkletree-testing/Cargo.toml b/incrementalmerkletree-testing/Cargo.toml new file mode 100644 index 00000000..4484b040 --- /dev/null +++ b/incrementalmerkletree-testing/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "incrementalmerkletree-testing" +description = "Common types, interfaces, and utilities for testing Merkle tree data structures" +version = "0.1.0" +authors = [ + "Kris Nuttycombe ", + "Sean Bowe ", +] +edition.workspace = true +license.workspace = true +homepage.workspace = true +repository.workspace = true +categories.workspace = true +rust-version.workspace = true + +[package.metadata.docs.rs] +all-features = true +rustdoc-args = ["--cfg", "docsrs"] + +[dependencies] +incrementalmerkletree = { workspace = true, features = ["test-dependencies"] } + +proptest.workspace = true diff --git a/incrementalmerkletree-testing/LICENSE-APACHE b/incrementalmerkletree-testing/LICENSE-APACHE new file mode 100644 index 00000000..1e5006dc --- /dev/null +++ b/incrementalmerkletree-testing/LICENSE-APACHE @@ -0,0 +1,202 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + diff --git a/incrementalmerkletree-testing/LICENSE-MIT b/incrementalmerkletree-testing/LICENSE-MIT new file mode 100644 index 00000000..94ac1a7c --- /dev/null +++ b/incrementalmerkletree-testing/LICENSE-MIT @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2015-2021 The Electric Coin Company + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/incrementalmerkletree-testing/README.md b/incrementalmerkletree-testing/README.md new file mode 100644 index 00000000..9f6ec89e --- /dev/null +++ b/incrementalmerkletree-testing/README.md @@ -0,0 +1,6 @@ +incrementalmerkletree-testing +============================= + +This crate contains common testing infrastructure used in the implementation of +the `bridgetree` and `shardtree` crates. + diff --git a/incrementalmerkletree/src/testing/complete_tree.rs b/incrementalmerkletree-testing/src/complete_tree.rs similarity index 97% rename from incrementalmerkletree/src/testing/complete_tree.rs rename to incrementalmerkletree-testing/src/complete_tree.rs index 79c189d3..2e4e2f7f 100644 --- a/incrementalmerkletree/src/testing/complete_tree.rs +++ b/incrementalmerkletree-testing/src/complete_tree.rs @@ -2,8 +2,8 @@ use std::cmp::min; use std::collections::{BTreeMap, BTreeSet}; -use crate::Marking; -use crate::{testing::Tree, Hashable, Level, Position, Retention}; +use crate::Tree; +use incrementalmerkletree::{Hashable, Level, Marking, Position, Retention}; const MAX_COMPLETE_SIZE_ERROR: &str = "Positions of a `CompleteTree` must fit into the platform word size, because larger complete trees are not representable."; @@ -320,12 +320,10 @@ mod tests { use super::CompleteTree; use crate::{ - testing::{ - check_append, check_checkpoint_rewind, check_rewind_remove_mark, check_root_hashes, - check_witnesses, compute_root_from_witness, SipHashable, Tree, - }, - Hashable, Level, Position, Retention, + check_append, check_checkpoint_rewind, check_rewind_remove_mark, check_root_hashes, + check_witnesses, compute_root_from_witness, SipHashable, Tree, }; + use incrementalmerkletree::{Hashable, Level, Position, Retention}; #[test] fn correct_empty_root() { @@ -384,7 +382,8 @@ mod tests { #[test] fn correct_witness() { - use crate::{testing::Tree, Retention}; + use crate::Tree; + use incrementalmerkletree::Retention; const DEPTH: u8 = 3; let values = (0..(1 << DEPTH)).map(SipHashable); diff --git a/incrementalmerkletree-testing/src/lib.rs b/incrementalmerkletree-testing/src/lib.rs new file mode 100644 index 00000000..2641aa2c --- /dev/null +++ b/incrementalmerkletree-testing/src/lib.rs @@ -0,0 +1,1307 @@ +//! Traits and types used to permit comparison testing between tree implementations. + +use core::fmt::Debug; +use core::marker::PhantomData; +use proptest::prelude::*; +use std::collections::BTreeSet; + +use incrementalmerkletree::{Hashable, Level, Marking, Position, Retention}; + +pub mod complete_tree; + +// +// Traits used to permit comparison testing between tree implementations. +// + +/// A Merkle tree that supports incremental appends, marking of +/// leaf nodes for construction of witnesses, checkpoints and rollbacks. +pub trait Tree { + /// Returns the depth of the tree. + fn depth(&self) -> u8; + + /// Appends a new value to the tree at the next available slot. + /// Returns true if successful and false if the tree would exceed + /// the maximum allowed depth. + fn append(&mut self, value: H, retention: Retention) -> bool; + + /// Returns the most recently appended leaf value. + fn current_position(&self) -> Option; + + /// Returns the leaf at the specified position if the tree can produce + /// a witness for it. + fn get_marked_leaf(&self, position: Position) -> Option; + + /// Return a set of all the positions for which we have marked. + fn marked_positions(&self) -> BTreeSet; + + /// Obtains the root of the Merkle tree at the specified checkpoint depth + /// by hashing against empty nodes up to the maximum height of the tree. + /// Returns `None` if there are not enough checkpoints available to reach the + /// requested checkpoint depth. + fn root(&self, checkpoint_depth: usize) -> Option; + + /// Obtains a witness for the value at the specified leaf position, as of the tree state at the + /// given checkpoint depth. Returns `None` if there is no witness information for the requested + /// position or if no checkpoint is available at the specified depth. + fn witness(&self, position: Position, checkpoint_depth: usize) -> Option>; + + /// Marks the value at the specified position as a value we're no longer + /// interested in maintaining a mark for. Returns true if successful and + /// false if we were already not maintaining a mark at this position. + fn remove_mark(&mut self, position: Position) -> bool; + + /// Creates a new checkpoint for the current tree state. + /// + /// It is valid to have multiple checkpoints for the same tree state, and each `rewind` call + /// will remove a single checkpoint. Returns `false` if the checkpoint identifier provided is + /// less than or equal to the maximum checkpoint identifier observed. + fn checkpoint(&mut self, id: C) -> bool; + + /// Rewinds the tree state to the previous checkpoint, and then removes that checkpoint record. + /// + /// If there are multiple checkpoints at a given tree state, the tree state will not be altered + /// until all checkpoints at that tree state have been removed using `rewind`. This function + /// will return false and leave the tree unmodified if no checkpoints exist. + fn rewind(&mut self) -> bool; +} + +// +// Types and utilities for shared example tests. +// + +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub struct SipHashable(pub u64); + +impl Hashable for SipHashable { + fn empty_leaf() -> Self { + SipHashable(0) + } + + fn combine(_level: Level, a: &Self, b: &Self) -> Self { + #![allow(deprecated)] + use std::hash::{Hasher, SipHasher}; + + let mut hasher = SipHasher::new(); + hasher.write_u64(a.0); + hasher.write_u64(b.0); + SipHashable(hasher.finish()) + } +} + +// +// Operations +// + +#[derive(Clone, Debug)] +pub enum Operation { + Append(A, Retention), + CurrentPosition, + MarkedLeaf(Position), + MarkedPositions, + Unmark(Position), + Checkpoint(C), + Rewind, + Witness(Position, usize), + GarbageCollect, +} + +use Operation::*; + +pub fn append_str(x: &str, retention: Retention) -> Operation { + Operation::Append(x.to_string(), retention) +} + +pub fn unmark(pos: u64) -> Operation { + Operation::Unmark(Position::from(pos)) +} + +pub fn witness(pos: u64, depth: usize) -> Operation { + Operation::Witness(Position::from(pos), depth) +} + +impl Operation { + pub fn apply>(&self, tree: &mut T) -> Option<(Position, Vec)> { + match self { + Append(a, r) => { + assert!(tree.append(a.clone(), r.clone()), "append failed"); + None + } + CurrentPosition => None, + MarkedLeaf(_) => None, + MarkedPositions => None, + Unmark(p) => { + assert!(tree.remove_mark(*p), "remove mark failed"); + None + } + Checkpoint(id) => { + tree.checkpoint(id.clone()); + None + } + Rewind => { + assert!(tree.rewind(), "rewind failed"); + None + } + Witness(p, d) => tree.witness(*p, *d).map(|xs| (*p, xs)), + GarbageCollect => None, + } + } + + pub fn apply_all>( + ops: &[Operation], + tree: &mut T, + ) -> Option<(Position, Vec)> { + let mut result = None; + for op in ops { + result = op.apply(tree); + } + result + } + + pub fn map_checkpoint_id D>(&self, f: F) -> Operation { + match self { + Append(a, r) => Append(a.clone(), r.map(f)), + CurrentPosition => CurrentPosition, + MarkedLeaf(l) => MarkedLeaf(*l), + MarkedPositions => MarkedPositions, + Unmark(p) => Unmark(*p), + Checkpoint(id) => Checkpoint(f(id)), + Rewind => Rewind, + Witness(p, d) => Witness(*p, *d), + GarbageCollect => GarbageCollect, + } + } +} + +/// Returns a strategy for creating a uniformly-distributed [`Marking`] +/// value. +pub fn arb_marking() -> impl Strategy { + prop_oneof![ + Just(Marking::Marked), + Just(Marking::Reference), + Just(Marking::None) + ] +} + +/// Returns a strategy for creating a uniformly-distributed [`Retention`] +/// value. +pub fn arb_retention() -> impl Strategy> { + prop_oneof![ + Just(Retention::Ephemeral), + arb_marking().prop_map(|marking| Retention::Checkpoint { id: (), marking }), + Just(Retention::Marked), + ] +} + +pub fn arb_operation( + item_gen: G, + pos_gen: impl Strategy + Clone, +) -> impl Strategy> +where + G::Value: Clone + 'static, +{ + prop_oneof![ + (item_gen, arb_retention()).prop_map(|(i, r)| Operation::Append(i, r)), + prop_oneof![ + Just(Operation::CurrentPosition), + Just(Operation::MarkedPositions), + ], + Just(Operation::GarbageCollect), + pos_gen.clone().prop_map(Operation::MarkedLeaf), + pos_gen.clone().prop_map(Operation::Unmark), + Just(Operation::Checkpoint(())), + Just(Operation::Rewind), + pos_gen.prop_flat_map(|i| (0usize..10).prop_map(move |depth| Operation::Witness(i, depth))), + ] +} + +pub fn apply_operation>(tree: &mut T, op: Operation) { + match op { + Append(value, r) => { + tree.append(value, r); + } + Unmark(position) => { + tree.remove_mark(position); + } + Checkpoint(id) => { + tree.checkpoint(id); + } + Rewind => { + tree.rewind(); + } + CurrentPosition => {} + Witness(_, _) => {} + MarkedLeaf(_) => {} + MarkedPositions => {} + GarbageCollect => {} + } +} + +pub fn check_operations>( + mut tree: T, + ops: &[Operation], +) -> Result<(), TestCaseError> { + let mut tree_size = 0; + let mut tree_values: Vec = vec![]; + // the number of leaves in the tree at the time that a checkpoint is made + let mut tree_checkpoints: Vec = vec![]; + + for op in ops { + prop_assert_eq!(tree_size, tree_values.len()); + match op { + Append(value, r) => { + if tree.append(value.clone(), r.clone()) { + prop_assert!(tree_size < (1 << tree.depth())); + tree_size += 1; + tree_values.push(value.clone()); + if r.is_checkpoint() { + tree_checkpoints.push(tree_size); + } + } else { + prop_assert_eq!( + tree_size, + tree.current_position() + .map_or(0, |p| usize::try_from(p).unwrap() + 1) + ); + } + } + CurrentPosition => { + if let Some(pos) = tree.current_position() { + prop_assert!(tree_size > 0); + prop_assert_eq!(tree_size - 1, pos.try_into().unwrap()); + } + } + MarkedLeaf(position) => { + if tree.get_marked_leaf(*position).is_some() { + prop_assert!(::try_from(*position).unwrap() < tree_size); + } + } + Unmark(position) => { + tree.remove_mark(*position); + } + MarkedPositions => {} + Checkpoint(id) => { + tree_checkpoints.push(tree_size); + tree.checkpoint(id.clone()); + } + Rewind => { + if tree.rewind() { + prop_assert!(!tree_checkpoints.is_empty()); + let checkpointed_tree_size = tree_checkpoints.pop().unwrap(); + tree_values.truncate(checkpointed_tree_size); + tree_size = checkpointed_tree_size; + } + } + Witness(position, depth) => { + if let Some(path) = tree.witness(*position, *depth) { + let value: H = tree_values[::try_from(*position).unwrap()].clone(); + let tree_root = tree.root(*depth); + + if tree_checkpoints.len() >= *depth { + let mut extended_tree_values = tree_values.clone(); + if *depth > 0 { + // prune the tree back to the checkpointed size. + if let Some(checkpointed_tree_size) = + tree_checkpoints.get(tree_checkpoints.len() - depth) + { + extended_tree_values.truncate(*checkpointed_tree_size); + } + } + + // compute the root + let expected_root = + complete_tree::root::(&extended_tree_values, tree.depth()); + prop_assert_eq!(&tree_root.unwrap(), &expected_root); + + prop_assert_eq!( + &compute_root_from_witness(value, *position, &path), + &expected_root + ); + } + } + } + GarbageCollect => {} + } + } + + Ok(()) +} + +pub fn compute_root_from_witness(value: H, position: Position, path: &[H]) -> H { + let mut cur = value; + let mut lvl = 0.into(); + for (i, v) in path + .iter() + .enumerate() + .map(|(i, v)| (((::from(position) >> i) & 1) == 1, v)) + { + if i { + cur = H::combine(lvl, v, &cur); + } else { + cur = H::combine(lvl, &cur, v); + } + lvl = lvl + 1; + } + cur +} + +// +// Types and utilities for cross-verification property tests +// + +#[derive(Clone, Debug)] +pub struct CombinedTree, E: Tree> { + inefficient: I, + efficient: E, + _phantom_h: PhantomData, + _phantom_c: PhantomData, +} + +impl, E: Tree> CombinedTree { + pub fn new(inefficient: I, efficient: E) -> Self { + assert_eq!(inefficient.depth(), efficient.depth()); + CombinedTree { + inefficient, + efficient, + _phantom_h: PhantomData, + _phantom_c: PhantomData, + } + } +} + +impl, E: Tree> Tree + for CombinedTree +{ + fn depth(&self) -> u8 { + self.inefficient.depth() + } + + fn append(&mut self, value: H, retention: Retention) -> bool { + let a = self.inefficient.append(value.clone(), retention.clone()); + let b = self.efficient.append(value, retention); + assert_eq!(a, b); + a + } + + fn root(&self, checkpoint_depth: usize) -> Option { + let a = self.inefficient.root(checkpoint_depth); + let b = self.efficient.root(checkpoint_depth); + assert_eq!(a, b); + a + } + + fn current_position(&self) -> Option { + let a = self.inefficient.current_position(); + let b = self.efficient.current_position(); + assert_eq!(a, b); + a + } + + fn get_marked_leaf(&self, position: Position) -> Option { + let a = self.inefficient.get_marked_leaf(position); + let b = self.efficient.get_marked_leaf(position); + assert_eq!(a, b); + a + } + + fn marked_positions(&self) -> BTreeSet { + let a = self.inefficient.marked_positions(); + let b = self.efficient.marked_positions(); + assert_eq!(a, b); + a + } + + fn witness(&self, position: Position, checkpoint_depth: usize) -> Option> { + let a = self.inefficient.witness(position, checkpoint_depth); + let b = self.efficient.witness(position, checkpoint_depth); + assert_eq!(a, b); + a + } + + fn remove_mark(&mut self, position: Position) -> bool { + let a = self.inefficient.remove_mark(position); + let b = self.efficient.remove_mark(position); + assert_eq!(a, b); + a + } + + fn checkpoint(&mut self, id: C) -> bool { + let a = self.inefficient.checkpoint(id.clone()); + let b = self.efficient.checkpoint(id); + assert_eq!(a, b); + a + } + + fn rewind(&mut self) -> bool { + let a = self.inefficient.rewind(); + let b = self.efficient.rewind(); + assert_eq!(a, b); + a + } +} + +// +// Shared example tests +// + +pub trait TestHashable: Hashable + Ord + Clone + Debug { + fn from_u64(value: u64) -> Self; + + fn combine_all(depth: u8, values: &[u64]) -> Self { + let values: Vec = values.iter().map(|v| Self::from_u64(*v)).collect(); + complete_tree::root(&values, depth) + } +} + +impl TestHashable for String { + fn from_u64(value: u64) -> Self { + ('a'..) + .nth( + value + .try_into() + .expect("we do not use test value indices larger than usize::MAX"), + ) + .expect("we do not choose test value indices larger than the iterable character range") + .to_string() + } +} + +pub trait TestCheckpoint: Ord + Clone + Debug { + fn from_u64(value: u64) -> Self; +} + +impl TestCheckpoint for usize { + fn from_u64(value: u64) -> Self { + value + .try_into() + .expect("we do not use test checkpoint identifiers greater than usize::MAX") + } +} + +trait TestTree { + fn assert_root(&self, checkpoint_depth: usize, values: &[u64]); + + fn assert_append(&mut self, value: u64, retention: Retention); + + fn assert_checkpoint(&mut self, value: u64); +} + +impl> TestTree for T { + fn assert_root(&self, checkpoint_depth: usize, values: &[u64]) { + assert_eq!( + self.root(checkpoint_depth).unwrap(), + H::combine_all(self.depth(), values) + ); + } + + fn assert_append(&mut self, value: u64, retention: Retention) { + assert!( + self.append(H::from_u64(value), retention.map(|id| C::from_u64(*id))), + "append failed for value {}", + value + ); + } + + fn assert_checkpoint(&mut self, value: u64) { + assert!( + self.checkpoint(C::from_u64(value)), + "checkpoint failed for value {}", + value + ); + } +} + +/// This checks basic append and root computation functionality +pub fn check_root_hashes, F: Fn(usize) -> T>( + new_tree: F, +) { + use Retention::*; + + { + let mut tree = new_tree(100); + tree.assert_root(0, &[]); + tree.assert_append(0, Ephemeral); + tree.assert_root(0, &[0]); + tree.assert_append(1, Ephemeral); + tree.assert_root(0, &[0, 1]); + tree.assert_append(2, Ephemeral); + tree.assert_root(0, &[0, 1, 2]); + } + + { + let mut t = new_tree(100); + t.assert_append( + 0, + Retention::Checkpoint { + id: 1, + marking: Marking::Marked, + }, + ); + for _ in 0..3 { + t.assert_append(0, Ephemeral); + } + t.assert_root(0, &[0, 0, 0, 0]); + } +} + +/// This test expects a depth-4 tree and verifies that the tree reports itself as full after 2^4 +/// appends. +pub fn check_append, F: Fn(usize) -> T>( + new_tree: F, +) { + use Retention::*; + + { + let mut tree = new_tree(100); + assert_eq!(tree.depth(), 4); + + // 16 appends should succeed + for i in 0..16 { + tree.assert_append(i, Ephemeral); + assert_eq!(tree.current_position(), Some(Position::from(i))); + } + + // 17th append should fail + assert!(!tree.append(H::from_u64(16), Ephemeral)); + } + + { + // The following checks a condition on state restoration in the case that an append fails. + // We want to ensure that a failed append does not cause a loss of information. + let ops = (0..17) + .map(|i| Append(H::from_u64(i), Ephemeral)) + .collect::>(); + let tree = new_tree(100); + check_operations(tree, &ops).unwrap(); + } +} + +pub fn check_witnesses, F: Fn(usize) -> T>( + new_tree: F, +) { + use Retention::*; + + { + let mut tree = new_tree(100); + tree.assert_append(0, Ephemeral); + tree.assert_append(1, Marked); + assert_eq!(tree.witness(Position::from(0), 0), None); + } + + { + let mut tree = new_tree(100); + tree.assert_append(0, Marked); + assert_eq!( + tree.witness(Position::from(0), 0), + Some(vec![ + H::empty_root(0.into()), + H::empty_root(1.into()), + H::empty_root(2.into()), + H::empty_root(3.into()) + ]) + ); + + tree.assert_append(1, Ephemeral); + assert_eq!( + tree.witness(0.into(), 0), + Some(vec![ + H::from_u64(1), + H::empty_root(1.into()), + H::empty_root(2.into()), + H::empty_root(3.into()) + ]) + ); + + tree.assert_append(2, Marked); + assert_eq!( + tree.witness(Position::from(2), 0), + Some(vec![ + H::empty_root(0.into()), + H::combine_all(1, &[0, 1]), + H::empty_root(2.into()), + H::empty_root(3.into()) + ]) + ); + + tree.assert_append(3, Ephemeral); + assert_eq!( + tree.witness(Position::from(2), 0), + Some(vec![ + H::from_u64(3), + H::combine_all(1, &[0, 1]), + H::empty_root(2.into()), + H::empty_root(3.into()) + ]) + ); + + tree.assert_append(4, Ephemeral); + assert_eq!( + tree.witness(Position::from(2), 0), + Some(vec![ + H::from_u64(3), + H::combine_all(1, &[0, 1]), + H::combine_all(2, &[4]), + H::empty_root(3.into()) + ]) + ); + } + + { + let mut tree = new_tree(100); + tree.assert_append(0, Marked); + for i in 1..6 { + tree.assert_append(i, Ephemeral); + } + tree.assert_append(6, Marked); + tree.assert_append(7, Ephemeral); + + assert_eq!( + tree.witness(0.into(), 0), + Some(vec![ + H::from_u64(1), + H::combine_all(1, &[2, 3]), + H::combine_all(2, &[4, 5, 6, 7]), + H::empty_root(3.into()) + ]) + ); + } + + { + let mut tree = new_tree(100); + tree.assert_append(0, Marked); + tree.assert_append(1, Ephemeral); + tree.assert_append(2, Ephemeral); + tree.assert_append(3, Marked); + tree.assert_append(4, Marked); + tree.assert_append(5, Marked); + tree.assert_append(6, Ephemeral); + + assert_eq!( + tree.witness(Position::from(5), 0), + Some(vec![ + H::from_u64(4), + H::combine_all(1, &[6]), + H::combine_all(2, &[0, 1, 2, 3]), + H::empty_root(3.into()) + ]) + ); + } + + { + let mut tree = new_tree(100); + for i in 0..10 { + tree.assert_append(i, Ephemeral); + } + tree.assert_append(10, Marked); + tree.assert_append(11, Ephemeral); + + assert_eq!( + tree.witness(Position::from(10), 0), + Some(vec![ + H::from_u64(11), + H::combine_all(1, &[8, 9]), + H::empty_root(2.into()), + H::combine_all(3, &[0, 1, 2, 3, 4, 5, 6, 7]) + ]) + ); + } + + { + let mut tree = new_tree(100); + tree.assert_append( + 0, + Checkpoint { + id: 1, + marking: Marking::Marked, + }, + ); + assert!(tree.rewind()); + for i in 1..4 { + tree.assert_append(i, Ephemeral); + } + tree.assert_append(4, Marked); + for i in 5..8 { + tree.assert_append(i, Ephemeral); + } + assert_eq!( + tree.witness(0.into(), 0), + Some(vec![ + H::from_u64(1), + H::combine_all(1, &[2, 3]), + H::combine_all(2, &[4, 5, 6, 7]), + H::empty_root(3.into()), + ]) + ); + } + + { + let mut tree = new_tree(100); + tree.assert_append(0, Ephemeral); + tree.assert_append(1, Ephemeral); + tree.assert_append(2, Marked); + tree.assert_append(3, Ephemeral); + tree.assert_append(4, Ephemeral); + tree.assert_append(5, Ephemeral); + tree.assert_append( + 6, + Checkpoint { + id: 1, + marking: Marking::Marked, + }, + ); + tree.assert_append(7, Ephemeral); + assert!(tree.rewind()); + assert_eq!( + tree.witness(Position::from(2), 0), + Some(vec![ + H::from_u64(3), + H::combine_all(1, &[0, 1]), + H::combine_all(2, &[4, 5, 6]), + H::empty_root(3.into()) + ]) + ); + } + + { + let mut tree = new_tree(100); + for i in 0..12 { + tree.assert_append(i, Ephemeral); + } + tree.assert_append(12, Marked); + tree.assert_append(13, Marked); + tree.assert_append(14, Ephemeral); + tree.assert_append(15, Ephemeral); + + assert_eq!( + tree.witness(Position::from(12), 0), + Some(vec![ + H::from_u64(13), + H::combine_all(1, &[14, 15]), + H::combine_all(2, &[8, 9, 10, 11]), + H::combine_all(3, &[0, 1, 2, 3, 4, 5, 6, 7]), + ]) + ); + } + + { + let ops = (0..=11) + .map(|i| Append(H::from_u64(i), Marked)) + .chain(Some(Append(H::from_u64(12), Ephemeral))) + .chain(Some(Append(H::from_u64(13), Ephemeral))) + .chain(Some(Witness(11u64.into(), 0))) + .collect::>(); + + let mut tree = new_tree(100); + assert_eq!( + Operation::apply_all(&ops, &mut tree), + Some(( + Position::from(11), + vec![ + H::from_u64(10), + H::combine_all(1, &[8, 9]), + H::combine_all(2, &[12, 13]), + H::combine_all(3, &[0, 1, 2, 3, 4, 5, 6, 7]), + ] + )) + ); + } + + { + let ops = vec![ + Append(H::from_u64(0), Ephemeral), + Append(H::from_u64(1), Ephemeral), + Append(H::from_u64(2), Ephemeral), + Append( + H::from_u64(3), + Checkpoint { + id: C::from_u64(1), + marking: Marking::Marked, + }, + ), + Append(H::from_u64(4), Marked), + Operation::Checkpoint(C::from_u64(2)), + Append( + H::from_u64(5), + Checkpoint { + id: C::from_u64(3), + marking: Marking::None, + }, + ), + Append( + H::from_u64(6), + Checkpoint { + id: C::from_u64(4), + marking: Marking::None, + }, + ), + Append( + H::from_u64(7), + Checkpoint { + id: C::from_u64(5), + marking: Marking::None, + }, + ), + Witness(3u64.into(), 5), + ]; + let mut tree = new_tree(100); + assert_eq!( + Operation::apply_all(&ops, &mut tree), + Some(( + Position::from(3), + vec![ + H::from_u64(2), + H::combine_all(1, &[0, 1]), + H::combine_all(2, &[]), + H::combine_all(3, &[]), + ] + )) + ); + } + + { + let ops = vec![ + Append(H::from_u64(0), Ephemeral), + Append(H::from_u64(0), Ephemeral), + Append(H::from_u64(0), Ephemeral), + Append( + H::from_u64(0), + Checkpoint { + id: C::from_u64(1), + marking: Marking::Marked, + }, + ), + Append(H::from_u64(0), Ephemeral), + Append(H::from_u64(0), Ephemeral), + Append(H::from_u64(0), Ephemeral), + Append( + H::from_u64(0), + Checkpoint { + id: C::from_u64(2), + marking: Marking::None, + }, + ), + Append(H::from_u64(0), Ephemeral), + Append(H::from_u64(0), Ephemeral), + Witness(Position::from(3), 1), + ]; + let mut tree = new_tree(100); + assert_eq!( + Operation::apply_all(&ops, &mut tree), + Some(( + Position::from(3), + vec![ + H::from_u64(0), + H::combine_all(1, &[0, 0]), + H::combine_all(2, &[0, 0, 0, 0]), + H::combine_all(3, &[]), + ] + )) + ); + } + + { + let ops = vec![ + Append(H::from_u64(0), Marked), + Append(H::from_u64(0), Ephemeral), + Append(H::from_u64(0), Ephemeral), + Append(H::from_u64(0), Ephemeral), + Append(H::from_u64(0), Ephemeral), + Append(H::from_u64(0), Ephemeral), + Append(H::from_u64(0), Ephemeral), + Operation::Checkpoint(C::from_u64(1)), + Append(H::from_u64(0), Marked), + Operation::Checkpoint(C::from_u64(2)), + Operation::Checkpoint(C::from_u64(3)), + Append( + H::from_u64(0), + Checkpoint { + id: C::from_u64(4), + marking: Marking::None, + }, + ), + Rewind, + Rewind, + Witness(Position::from(7), 2), + ]; + let mut tree = new_tree(100); + assert_eq!(Operation::apply_all(&ops, &mut tree), None); + } + + { + let ops = vec![ + Append(H::from_u64(0), Marked), + Append(H::from_u64(0), Ephemeral), + Append( + H::from_u64(0), + Checkpoint { + id: C::from_u64(1), + marking: Marking::Marked, + }, + ), + Append( + H::from_u64(0), + Checkpoint { + id: C::from_u64(4), + marking: Marking::None, + }, + ), + Witness(Position::from(2), 2), + ]; + let mut tree = new_tree(100); + assert_eq!( + Operation::apply_all(&ops, &mut tree), + Some(( + Position::from(2), + vec![ + H::empty_leaf(), + H::combine_all(1, &[0, 0]), + H::combine_all(2, &[]), + H::combine_all(3, &[]), + ] + )) + ); + } +} + +pub fn check_checkpoint_rewind, F: Fn(usize) -> T>( + new_tree: F, +) { + let mut t = new_tree(100); + assert!(!t.rewind()); + + let mut t = new_tree(100); + t.assert_checkpoint(1); + assert!(t.rewind()); + + let mut t = new_tree(100); + t.append("a".to_string(), Retention::Ephemeral); + t.assert_checkpoint(1); + t.append("b".to_string(), Retention::Marked); + assert!(t.rewind()); + assert_eq!(Some(Position::from(0)), t.current_position()); + + let mut t = new_tree(100); + t.append("a".to_string(), Retention::Marked); + t.assert_checkpoint(1); + assert!(t.rewind()); + + let mut t = new_tree(100); + t.append("a".to_string(), Retention::Marked); + t.assert_checkpoint(1); + t.append("a".to_string(), Retention::Ephemeral); + assert!(t.rewind()); + assert_eq!(Some(Position::from(0)), t.current_position()); + + let mut t = new_tree(100); + t.append("a".to_string(), Retention::Ephemeral); + t.assert_checkpoint(1); + t.assert_checkpoint(2); + assert!(t.rewind()); + t.append("b".to_string(), Retention::Ephemeral); + assert!(t.rewind()); + t.append("b".to_string(), Retention::Ephemeral); + assert_eq!(t.root(0).unwrap(), "ab______________"); +} + +pub fn check_remove_mark, F: Fn(usize) -> T>(new_tree: F) { + let samples = vec![ + vec![ + append_str("a", Retention::Ephemeral), + append_str( + "a", + Retention::Checkpoint { + id: C::from_u64(1), + marking: Marking::Marked, + }, + ), + witness(1, 1), + ], + vec![ + append_str("a", Retention::Ephemeral), + append_str("a", Retention::Ephemeral), + append_str("a", Retention::Ephemeral), + append_str("a", Retention::Marked), + Checkpoint(C::from_u64(1)), + unmark(3), + witness(3, 0), + ], + ]; + + for (i, sample) in samples.iter().enumerate() { + let result = check_operations(new_tree(100), sample); + assert!( + matches!(result, Ok(())), + "Reference/Test mismatch at index {}: {:?}", + i, + result + ); + } +} + +pub fn check_rewind_remove_mark, F: Fn(usize) -> T>( + new_tree: F, +) { + // rewinding doesn't remove a mark + let mut tree = new_tree(100); + tree.append("e".to_string(), Retention::Marked); + tree.assert_checkpoint(1); + assert!(tree.rewind()); + assert!(tree.remove_mark(0u64.into())); + + // use a maximum number of checkpoints of 1 + let mut tree = new_tree(1); + assert!(tree.append("e".to_string(), Retention::Marked)); + tree.assert_checkpoint(1); + assert!(tree.marked_positions().contains(&0u64.into())); + assert!(tree.append("f".to_string(), Retention::Ephemeral)); + // simulate a spend of `e` at `f` + assert!(tree.remove_mark(0u64.into())); + // even though the mark has been staged for removal, it's not gone yet + assert!(tree.marked_positions().contains(&0u64.into())); + tree.assert_checkpoint(2); + // the newest checkpoint will have caused the oldest to roll off, and + // so the forgotten node will be unmarked + assert!(!tree.marked_positions().contains(&0u64.into())); + assert!(!tree.remove_mark(0u64.into())); + + // The following check_operations tests cover errors where the + // test framework itself previously did not correctly handle + // chain state restoration. + + let samples = vec![ + vec![ + append_str("x", Retention::Marked), + Checkpoint(C::from_u64(1)), + Rewind, + unmark(0), + ], + vec![ + append_str("d", Retention::Marked), + Checkpoint(C::from_u64(1)), + unmark(0), + Rewind, + unmark(0), + ], + vec![ + append_str("o", Retention::Marked), + Checkpoint(C::from_u64(1)), + Checkpoint(C::from_u64(2)), + unmark(0), + Rewind, + Rewind, + ], + vec![ + append_str("s", Retention::Marked), + append_str("m", Retention::Ephemeral), + Checkpoint(C::from_u64(1)), + unmark(0), + Rewind, + unmark(0), + unmark(0), + ], + vec![ + append_str("a", Retention::Marked), + Checkpoint(C::from_u64(1)), + Rewind, + append_str("a", Retention::Marked), + ], + ]; + + for (i, sample) in samples.iter().enumerate() { + let result = check_operations(new_tree(100), sample); + assert!( + matches!(result, Ok(())), + "Reference/Test mismatch at index {}: {:?}", + i, + result + ); + } +} + +pub fn check_witness_consistency, F: Fn(usize) -> T>( + new_tree: F, +) { + let samples = vec![ + // Reduced examples + vec![ + append_str("a", Retention::Ephemeral), + append_str("b", Retention::Marked), + Checkpoint(C::from_u64(1)), + witness(0, 1), + ], + vec![ + append_str("c", Retention::Ephemeral), + append_str("d", Retention::Marked), + Checkpoint(C::from_u64(1)), + witness(1, 1), + ], + vec![ + append_str("e", Retention::Marked), + Checkpoint(C::from_u64(1)), + append_str("f", Retention::Ephemeral), + witness(0, 1), + ], + vec![ + append_str("g", Retention::Marked), + Checkpoint(C::from_u64(1)), + unmark(0), + append_str("h", Retention::Ephemeral), + witness(0, 0), + ], + vec![ + append_str("i", Retention::Marked), + Checkpoint(C::from_u64(1)), + unmark(0), + append_str("j", Retention::Ephemeral), + witness(0, 0), + ], + vec![ + append_str("i", Retention::Marked), + append_str("j", Retention::Ephemeral), + Checkpoint(C::from_u64(1)), + append_str("k", Retention::Ephemeral), + witness(0, 1), + ], + vec![ + append_str("l", Retention::Marked), + Checkpoint(C::from_u64(1)), + Checkpoint(C::from_u64(2)), + append_str("m", Retention::Ephemeral), + Checkpoint(C::from_u64(3)), + witness(0, 2), + ], + vec![ + Checkpoint(C::from_u64(1)), + append_str("n", Retention::Marked), + witness(0, 1), + ], + vec![ + append_str("a", Retention::Marked), + Checkpoint(C::from_u64(1)), + unmark(0), + Checkpoint(C::from_u64(2)), + append_str("b", Retention::Ephemeral), + witness(0, 1), + ], + vec![ + append_str("a", Retention::Marked), + append_str("b", Retention::Ephemeral), + unmark(0), + Checkpoint(C::from_u64(1)), + witness(0, 0), + ], + vec![ + append_str("a", Retention::Marked), + Checkpoint(C::from_u64(1)), + unmark(0), + Checkpoint(C::from_u64(2)), + Rewind, + append_str("b", Retention::Ephemeral), + witness(0, 0), + ], + vec![ + append_str("a", Retention::Marked), + Checkpoint(C::from_u64(1)), + Checkpoint(C::from_u64(2)), + Rewind, + append_str("a", Retention::Ephemeral), + unmark(0), + witness(0, 1), + ], + // Unreduced examples + vec![ + append_str("o", Retention::Ephemeral), + append_str("p", Retention::Marked), + append_str("q", Retention::Ephemeral), + Checkpoint(C::from_u64(1)), + unmark(1), + witness(1, 1), + ], + vec![ + append_str("r", Retention::Ephemeral), + append_str("s", Retention::Ephemeral), + append_str("t", Retention::Marked), + Checkpoint(C::from_u64(1)), + unmark(2), + Checkpoint(C::from_u64(2)), + witness(2, 2), + ], + vec![ + append_str("u", Retention::Marked), + append_str("v", Retention::Ephemeral), + append_str("w", Retention::Ephemeral), + Checkpoint(C::from_u64(1)), + unmark(0), + append_str("x", Retention::Ephemeral), + Checkpoint(C::from_u64(2)), + Checkpoint(C::from_u64(3)), + witness(0, 3), + ], + ]; + + for (i, sample) in samples.iter().enumerate() { + let result = check_operations(new_tree(100), sample); + assert!( + matches!(result, Ok(())), + "Reference/Test mismatch at index {}: {:?}", + i, + result + ); + } +} + +#[cfg(test)] +pub(crate) mod tests { + use crate::{compute_root_from_witness, SipHashable}; + use incrementalmerkletree::{Hashable, Level, Position}; + + #[test] + fn test_compute_root_from_witness() { + let expected = SipHashable::combine( + ::from(2), + &SipHashable::combine( + Level::from(1), + &SipHashable::combine(0.into(), &SipHashable(0), &SipHashable(1)), + &SipHashable::combine(0.into(), &SipHashable(2), &SipHashable(3)), + ), + &SipHashable::combine( + Level::from(1), + &SipHashable::combine(0.into(), &SipHashable(4), &SipHashable(5)), + &SipHashable::combine(0.into(), &SipHashable(6), &SipHashable(7)), + ), + ); + + assert_eq!( + compute_root_from_witness::( + SipHashable(0), + 0.into(), + &[ + SipHashable(1), + SipHashable::combine(0.into(), &SipHashable(2), &SipHashable(3)), + SipHashable::combine( + Level::from(1), + &SipHashable::combine(0.into(), &SipHashable(4), &SipHashable(5)), + &SipHashable::combine(0.into(), &SipHashable(6), &SipHashable(7)) + ) + ] + ), + expected + ); + + assert_eq!( + compute_root_from_witness( + SipHashable(4), + Position::from(4), + &[ + SipHashable(5), + SipHashable::combine(0.into(), &SipHashable(6), &SipHashable(7)), + SipHashable::combine( + Level::from(1), + &SipHashable::combine(0.into(), &SipHashable(0), &SipHashable(1)), + &SipHashable::combine(0.into(), &SipHashable(2), &SipHashable(3)) + ) + ] + ), + expected + ); + } +} diff --git a/incrementalmerkletree/CHANGELOG.md b/incrementalmerkletree/CHANGELOG.md index efbacf25..a9839f69 100644 --- a/incrementalmerkletree/CHANGELOG.md +++ b/incrementalmerkletree/CHANGELOG.md @@ -7,6 +7,15 @@ and this project adheres to Rust's notion of ## Unreleased +## [0.7.0] - 2024-09-25 + +### Changed +- MSRV is now 1.64 + +### Removed +- `incrementalmerkletree::testing::{Tree, complete_tree}` and related types and functions + have been removed. Use the `incrementalmerkletree-testing` crate instead. + ## [0.6.0] - 2024-08-12 ### Added diff --git a/incrementalmerkletree/Cargo.toml b/incrementalmerkletree/Cargo.toml index c35c7768..9fa103d5 100644 --- a/incrementalmerkletree/Cargo.toml +++ b/incrementalmerkletree/Cargo.toml @@ -1,17 +1,17 @@ [package] name = "incrementalmerkletree" description = "Common types, interfaces, and utilities for Merkle tree data structures" -version = "0.6.0" +version = "0.7.0" authors = [ "Sean Bowe ", "Kris Nuttycombe ", ] -edition = "2021" -license = "MIT OR Apache-2.0" -homepage = "https://github.com/zcash/incrementalmerkletree" -repository = "https://github.com/zcash/incrementalmerkletree" -categories = ["algorithms", "data-structures"] -rust-version = "1.60" +edition.workspace = true +license.workspace = true +homepage.workspace = true +repository.workspace = true +categories.workspace = true +rust-version.workspace = true [package.metadata.docs.rs] all-features = true @@ -19,12 +19,12 @@ rustdoc-args = ["--cfg", "docsrs"] [dependencies] either = "1.8" -proptest = { version = "1.0.0", optional = true } +proptest = { workspace = true, optional = true } rand = { version = "0.8", optional = true } rand_core = { version = "0.6", optional = true } [dev-dependencies] -proptest = "1.0.0" +proptest.workspace = true rand = "0.8" rand_core = "0.6" rand_chacha = "0.3" diff --git a/incrementalmerkletree/src/frontier.rs b/incrementalmerkletree/src/frontier.rs index 901ab608..76095af4 100644 --- a/incrementalmerkletree/src/frontier.rs +++ b/incrementalmerkletree/src/frontier.rs @@ -561,7 +561,6 @@ impl CommitmentTree { left: Some(left), right, parents: (1u8..DEPTH) - .into_iter() .map(|i| { if u64::from(f.position()) & (1 << i) == 0 { None diff --git a/incrementalmerkletree/src/testing.rs b/incrementalmerkletree/src/testing.rs index 0a724d26..a2bbee99 100644 --- a/incrementalmerkletree/src/testing.rs +++ b/incrementalmerkletree/src/testing.rs @@ -1,17 +1,4 @@ -//! Traits and types used to permit comparison testing between tree implementations. - -use core::fmt::Debug; -use core::marker::PhantomData; -use proptest::prelude::*; -use std::collections::BTreeSet; - -use crate::{Hashable, Level, Marking, Position, Retention}; - -pub mod complete_tree; - -// -// Traits used to permit comparison testing between tree implementations. -// +use crate::{Hashable, Level}; /// A possibly-empty incremental Merkle frontier. pub trait Frontier { @@ -26,81 +13,6 @@ pub trait Frontier { fn root(&self) -> H; } -/// A Merkle tree that supports incremental appends, marking of -/// leaf nodes for construction of witnesses, checkpoints and rollbacks. -pub trait Tree { - /// Returns the depth of the tree. - fn depth(&self) -> u8; - - /// Appends a new value to the tree at the next available slot. - /// Returns true if successful and false if the tree would exceed - /// the maximum allowed depth. - fn append(&mut self, value: H, retention: Retention) -> bool; - - /// Returns the most recently appended leaf value. - fn current_position(&self) -> Option; - - /// Returns the leaf at the specified position if the tree can produce - /// a witness for it. - fn get_marked_leaf(&self, position: Position) -> Option; - - /// Return a set of all the positions for which we have marked. - fn marked_positions(&self) -> BTreeSet; - - /// Obtains the root of the Merkle tree at the specified checkpoint depth - /// by hashing against empty nodes up to the maximum height of the tree. - /// Returns `None` if there are not enough checkpoints available to reach the - /// requested checkpoint depth. - fn root(&self, checkpoint_depth: usize) -> Option; - - /// Obtains a witness for the value at the specified leaf position, as of the tree state at the - /// given checkpoint depth. Returns `None` if there is no witness information for the requested - /// position or if no checkpoint is available at the specified depth. - fn witness(&self, position: Position, checkpoint_depth: usize) -> Option>; - - /// Marks the value at the specified position as a value we're no longer - /// interested in maintaining a mark for. Returns true if successful and - /// false if we were already not maintaining a mark at this position. - fn remove_mark(&mut self, position: Position) -> bool; - - /// Creates a new checkpoint for the current tree state. - /// - /// It is valid to have multiple checkpoints for the same tree state, and each `rewind` call - /// will remove a single checkpoint. Returns `false` if the checkpoint identifier provided is - /// less than or equal to the maximum checkpoint identifier observed. - fn checkpoint(&mut self, id: C) -> bool; - - /// Rewinds the tree state to the previous checkpoint, and then removes that checkpoint record. - /// - /// If there are multiple checkpoints at a given tree state, the tree state will not be altered - /// until all checkpoints at that tree state have been removed using `rewind`. This function - /// will return false and leave the tree unmodified if no checkpoints exist. - fn rewind(&mut self) -> bool; -} - -// -// Types and utilities for shared example tests. -// - -#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] -pub struct SipHashable(pub u64); - -impl Hashable for SipHashable { - fn empty_leaf() -> Self { - SipHashable(0) - } - - fn combine(_level: Level, a: &Self, b: &Self) -> Self { - #![allow(deprecated)] - use std::hash::{Hasher, SipHasher}; - - let mut hasher = SipHasher::new(); - hasher.write_u64(a.0); - hasher.write_u64(b.0); - SipHashable(hasher.finish()) - } -} - impl Hashable for String { fn empty_leaf() -> Self { "_".to_string() @@ -123,1223 +35,3 @@ impl Hashable for Option { } } } - -// -// Operations -// - -#[derive(Clone, Debug)] -pub enum Operation { - Append(A, Retention), - CurrentPosition, - MarkedLeaf(Position), - MarkedPositions, - Unmark(Position), - Checkpoint(C), - Rewind, - Witness(Position, usize), - GarbageCollect, -} - -use Operation::*; - -pub fn append_str(x: &str, retention: Retention) -> Operation { - Operation::Append(x.to_string(), retention) -} - -pub fn unmark(pos: u64) -> Operation { - Operation::Unmark(Position::from(pos)) -} - -pub fn witness(pos: u64, depth: usize) -> Operation { - Operation::Witness(Position::from(pos), depth) -} - -impl Operation { - pub fn apply>(&self, tree: &mut T) -> Option<(Position, Vec)> { - match self { - Append(a, r) => { - assert!(tree.append(a.clone(), r.clone()), "append failed"); - None - } - CurrentPosition => None, - MarkedLeaf(_) => None, - MarkedPositions => None, - Unmark(p) => { - assert!(tree.remove_mark(*p), "remove mark failed"); - None - } - Checkpoint(id) => { - tree.checkpoint(id.clone()); - None - } - Rewind => { - assert!(tree.rewind(), "rewind failed"); - None - } - Witness(p, d) => tree.witness(*p, *d).map(|xs| (*p, xs)), - GarbageCollect => None, - } - } - - pub fn apply_all>( - ops: &[Operation], - tree: &mut T, - ) -> Option<(Position, Vec)> { - let mut result = None; - for op in ops { - result = op.apply(tree); - } - result - } - - pub fn map_checkpoint_id D>(&self, f: F) -> Operation { - match self { - Append(a, r) => Append(a.clone(), r.map(f)), - CurrentPosition => CurrentPosition, - MarkedLeaf(l) => MarkedLeaf(*l), - MarkedPositions => MarkedPositions, - Unmark(p) => Unmark(*p), - Checkpoint(id) => Checkpoint(f(id)), - Rewind => Rewind, - Witness(p, d) => Witness(*p, *d), - GarbageCollect => GarbageCollect, - } - } -} - -/// Returns a strategy for creating a uniformly-distributed [`Marking`] -/// value. -pub fn arb_marking() -> impl Strategy { - prop_oneof![ - Just(Marking::Marked), - Just(Marking::Reference), - Just(Marking::None) - ] -} - -/// Returns a strategy for creating a uniformly-distributed [`Retention`] -/// value. -pub fn arb_retention() -> impl Strategy> { - prop_oneof![ - Just(Retention::Ephemeral), - arb_marking().prop_map(|marking| Retention::Checkpoint { id: (), marking }), - Just(Retention::Marked), - ] -} - -pub fn arb_operation( - item_gen: G, - pos_gen: impl Strategy + Clone, -) -> impl Strategy> -where - G::Value: Clone + 'static, -{ - prop_oneof![ - (item_gen, arb_retention()).prop_map(|(i, r)| Operation::Append(i, r)), - prop_oneof![ - Just(Operation::CurrentPosition), - Just(Operation::MarkedPositions), - ], - Just(Operation::GarbageCollect), - pos_gen.clone().prop_map(Operation::MarkedLeaf), - pos_gen.clone().prop_map(Operation::Unmark), - Just(Operation::Checkpoint(())), - Just(Operation::Rewind), - pos_gen.prop_flat_map(|i| (0usize..10).prop_map(move |depth| Operation::Witness(i, depth))), - ] -} - -pub fn apply_operation>(tree: &mut T, op: Operation) { - match op { - Append(value, r) => { - tree.append(value, r); - } - Unmark(position) => { - tree.remove_mark(position); - } - Checkpoint(id) => { - tree.checkpoint(id); - } - Rewind => { - tree.rewind(); - } - CurrentPosition => {} - Witness(_, _) => {} - MarkedLeaf(_) => {} - MarkedPositions => {} - GarbageCollect => {} - } -} - -pub fn check_operations>( - mut tree: T, - ops: &[Operation], -) -> Result<(), TestCaseError> { - let mut tree_size = 0; - let mut tree_values: Vec = vec![]; - // the number of leaves in the tree at the time that a checkpoint is made - let mut tree_checkpoints: Vec = vec![]; - - for op in ops { - prop_assert_eq!(tree_size, tree_values.len()); - match op { - Append(value, r) => { - if tree.append(value.clone(), r.clone()) { - prop_assert!(tree_size < (1 << tree.depth())); - tree_size += 1; - tree_values.push(value.clone()); - if r.is_checkpoint() { - tree_checkpoints.push(tree_size); - } - } else { - prop_assert_eq!( - tree_size, - tree.current_position() - .map_or(0, |p| usize::try_from(p).unwrap() + 1) - ); - } - } - CurrentPosition => { - if let Some(pos) = tree.current_position() { - prop_assert!(tree_size > 0); - prop_assert_eq!(tree_size - 1, pos.try_into().unwrap()); - } - } - MarkedLeaf(position) => { - if tree.get_marked_leaf(*position).is_some() { - prop_assert!(::try_from(*position).unwrap() < tree_size); - } - } - Unmark(position) => { - tree.remove_mark(*position); - } - MarkedPositions => {} - Checkpoint(id) => { - tree_checkpoints.push(tree_size); - tree.checkpoint(id.clone()); - } - Rewind => { - if tree.rewind() { - prop_assert!(!tree_checkpoints.is_empty()); - let checkpointed_tree_size = tree_checkpoints.pop().unwrap(); - tree_values.truncate(checkpointed_tree_size); - tree_size = checkpointed_tree_size; - } - } - Witness(position, depth) => { - if let Some(path) = tree.witness(*position, *depth) { - let value: H = tree_values[::try_from(*position).unwrap()].clone(); - let tree_root = tree.root(*depth); - - if tree_checkpoints.len() >= *depth { - let mut extended_tree_values = tree_values.clone(); - if *depth > 0 { - // prune the tree back to the checkpointed size. - if let Some(checkpointed_tree_size) = - tree_checkpoints.get(tree_checkpoints.len() - depth) - { - extended_tree_values.truncate(*checkpointed_tree_size); - } - } - - // compute the root - let expected_root = - complete_tree::root::(&extended_tree_values, tree.depth()); - prop_assert_eq!(&tree_root.unwrap(), &expected_root); - - prop_assert_eq!( - &compute_root_from_witness(value, *position, &path), - &expected_root - ); - } - } - } - GarbageCollect => {} - } - } - - Ok(()) -} - -pub fn compute_root_from_witness(value: H, position: Position, path: &[H]) -> H { - let mut cur = value; - let mut lvl = 0.into(); - for (i, v) in path - .iter() - .enumerate() - .map(|(i, v)| (((::from(position) >> i) & 1) == 1, v)) - { - if i { - cur = H::combine(lvl, v, &cur); - } else { - cur = H::combine(lvl, &cur, v); - } - lvl = lvl + 1; - } - cur -} - -// -// Types and utilities for cross-verification property tests -// - -#[derive(Clone, Debug)] -pub struct CombinedTree, E: Tree> { - inefficient: I, - efficient: E, - _phantom_h: PhantomData, - _phantom_c: PhantomData, -} - -impl, E: Tree> CombinedTree { - pub fn new(inefficient: I, efficient: E) -> Self { - assert_eq!(inefficient.depth(), efficient.depth()); - CombinedTree { - inefficient, - efficient, - _phantom_h: PhantomData, - _phantom_c: PhantomData, - } - } -} - -impl, E: Tree> Tree - for CombinedTree -{ - fn depth(&self) -> u8 { - self.inefficient.depth() - } - - fn append(&mut self, value: H, retention: Retention) -> bool { - let a = self.inefficient.append(value.clone(), retention.clone()); - let b = self.efficient.append(value, retention); - assert_eq!(a, b); - a - } - - fn root(&self, checkpoint_depth: usize) -> Option { - let a = self.inefficient.root(checkpoint_depth); - let b = self.efficient.root(checkpoint_depth); - assert_eq!(a, b); - a - } - - fn current_position(&self) -> Option { - let a = self.inefficient.current_position(); - let b = self.efficient.current_position(); - assert_eq!(a, b); - a - } - - fn get_marked_leaf(&self, position: Position) -> Option { - let a = self.inefficient.get_marked_leaf(position); - let b = self.efficient.get_marked_leaf(position); - assert_eq!(a, b); - a - } - - fn marked_positions(&self) -> BTreeSet { - let a = self.inefficient.marked_positions(); - let b = self.efficient.marked_positions(); - assert_eq!(a, b); - a - } - - fn witness(&self, position: Position, checkpoint_depth: usize) -> Option> { - let a = self.inefficient.witness(position, checkpoint_depth); - let b = self.efficient.witness(position, checkpoint_depth); - assert_eq!(a, b); - a - } - - fn remove_mark(&mut self, position: Position) -> bool { - let a = self.inefficient.remove_mark(position); - let b = self.efficient.remove_mark(position); - assert_eq!(a, b); - a - } - - fn checkpoint(&mut self, id: C) -> bool { - let a = self.inefficient.checkpoint(id.clone()); - let b = self.efficient.checkpoint(id); - assert_eq!(a, b); - a - } - - fn rewind(&mut self) -> bool { - let a = self.inefficient.rewind(); - let b = self.efficient.rewind(); - assert_eq!(a, b); - a - } -} - -// -// Shared example tests -// - -pub trait TestHashable: Hashable + Ord + Clone + Debug { - fn from_u64(value: u64) -> Self; - - fn combine_all(depth: u8, values: &[u64]) -> Self { - let values: Vec = values.iter().map(|v| Self::from_u64(*v)).collect(); - complete_tree::root(&values, depth) - } -} - -impl TestHashable for String { - fn from_u64(value: u64) -> Self { - ('a'..) - .nth( - value - .try_into() - .expect("we do not use test value indices larger than usize::MAX"), - ) - .expect("we do not choose test value indices larger than the iterable character range") - .to_string() - } -} - -pub trait TestCheckpoint: Ord + Clone + Debug { - fn from_u64(value: u64) -> Self; -} - -impl TestCheckpoint for usize { - fn from_u64(value: u64) -> Self { - value - .try_into() - .expect("we do not use test checkpoint identifiers greater than usize::MAX") - } -} - -trait TestTree { - fn assert_root(&self, checkpoint_depth: usize, values: &[u64]); - - fn assert_append(&mut self, value: u64, retention: Retention); - - fn assert_checkpoint(&mut self, value: u64); -} - -impl> TestTree for T { - fn assert_root(&self, checkpoint_depth: usize, values: &[u64]) { - assert_eq!( - self.root(checkpoint_depth).unwrap(), - H::combine_all(self.depth(), values) - ); - } - - fn assert_append(&mut self, value: u64, retention: Retention) { - assert!( - self.append(H::from_u64(value), retention.map(|id| C::from_u64(*id))), - "append failed for value {}", - value - ); - } - - fn assert_checkpoint(&mut self, value: u64) { - assert!( - self.checkpoint(C::from_u64(value)), - "checkpoint failed for value {}", - value - ); - } -} - -/// This checks basic append and root computation functionality -pub fn check_root_hashes, F: Fn(usize) -> T>( - new_tree: F, -) { - use Retention::*; - - { - let mut tree = new_tree(100); - tree.assert_root(0, &[]); - tree.assert_append(0, Ephemeral); - tree.assert_root(0, &[0]); - tree.assert_append(1, Ephemeral); - tree.assert_root(0, &[0, 1]); - tree.assert_append(2, Ephemeral); - tree.assert_root(0, &[0, 1, 2]); - } - - { - let mut t = new_tree(100); - t.assert_append( - 0, - Retention::Checkpoint { - id: 1, - marking: Marking::Marked, - }, - ); - for _ in 0..3 { - t.assert_append(0, Ephemeral); - } - t.assert_root(0, &[0, 0, 0, 0]); - } -} - -/// This test expects a depth-4 tree and verifies that the tree reports itself as full after 2^4 -/// appends. -pub fn check_append, F: Fn(usize) -> T>( - new_tree: F, -) { - use Retention::*; - - { - let mut tree = new_tree(100); - assert_eq!(tree.depth(), 4); - - // 16 appends should succeed - for i in 0..16 { - tree.assert_append(i, Ephemeral); - assert_eq!(tree.current_position(), Some(Position::from(i))); - } - - // 17th append should fail - assert!(!tree.append(H::from_u64(16), Ephemeral)); - } - - { - // The following checks a condition on state restoration in the case that an append fails. - // We want to ensure that a failed append does not cause a loss of information. - let ops = (0..17) - .map(|i| Append(H::from_u64(i), Ephemeral)) - .collect::>(); - let tree = new_tree(100); - check_operations(tree, &ops).unwrap(); - } -} - -pub fn check_witnesses, F: Fn(usize) -> T>( - new_tree: F, -) { - use Retention::*; - - { - let mut tree = new_tree(100); - tree.assert_append(0, Ephemeral); - tree.assert_append(1, Marked); - assert_eq!(tree.witness(Position::from(0), 0), None); - } - - { - let mut tree = new_tree(100); - tree.assert_append(0, Marked); - assert_eq!( - tree.witness(Position::from(0), 0), - Some(vec![ - H::empty_root(0.into()), - H::empty_root(1.into()), - H::empty_root(2.into()), - H::empty_root(3.into()) - ]) - ); - - tree.assert_append(1, Ephemeral); - assert_eq!( - tree.witness(0.into(), 0), - Some(vec![ - H::from_u64(1), - H::empty_root(1.into()), - H::empty_root(2.into()), - H::empty_root(3.into()) - ]) - ); - - tree.assert_append(2, Marked); - assert_eq!( - tree.witness(Position::from(2), 0), - Some(vec![ - H::empty_root(0.into()), - H::combine_all(1, &[0, 1]), - H::empty_root(2.into()), - H::empty_root(3.into()) - ]) - ); - - tree.assert_append(3, Ephemeral); - assert_eq!( - tree.witness(Position::from(2), 0), - Some(vec![ - H::from_u64(3), - H::combine_all(1, &[0, 1]), - H::empty_root(2.into()), - H::empty_root(3.into()) - ]) - ); - - tree.assert_append(4, Ephemeral); - assert_eq!( - tree.witness(Position::from(2), 0), - Some(vec![ - H::from_u64(3), - H::combine_all(1, &[0, 1]), - H::combine_all(2, &[4]), - H::empty_root(3.into()) - ]) - ); - } - - { - let mut tree = new_tree(100); - tree.assert_append(0, Marked); - for i in 1..6 { - tree.assert_append(i, Ephemeral); - } - tree.assert_append(6, Marked); - tree.assert_append(7, Ephemeral); - - assert_eq!( - tree.witness(0.into(), 0), - Some(vec![ - H::from_u64(1), - H::combine_all(1, &[2, 3]), - H::combine_all(2, &[4, 5, 6, 7]), - H::empty_root(3.into()) - ]) - ); - } - - { - let mut tree = new_tree(100); - tree.assert_append(0, Marked); - tree.assert_append(1, Ephemeral); - tree.assert_append(2, Ephemeral); - tree.assert_append(3, Marked); - tree.assert_append(4, Marked); - tree.assert_append(5, Marked); - tree.assert_append(6, Ephemeral); - - assert_eq!( - tree.witness(Position::from(5), 0), - Some(vec![ - H::from_u64(4), - H::combine_all(1, &[6]), - H::combine_all(2, &[0, 1, 2, 3]), - H::empty_root(3.into()) - ]) - ); - } - - { - let mut tree = new_tree(100); - for i in 0..10 { - tree.assert_append(i, Ephemeral); - } - tree.assert_append(10, Marked); - tree.assert_append(11, Ephemeral); - - assert_eq!( - tree.witness(Position::from(10), 0), - Some(vec![ - H::from_u64(11), - H::combine_all(1, &[8, 9]), - H::empty_root(2.into()), - H::combine_all(3, &[0, 1, 2, 3, 4, 5, 6, 7]) - ]) - ); - } - - { - let mut tree = new_tree(100); - tree.assert_append( - 0, - Checkpoint { - id: 1, - marking: Marking::Marked, - }, - ); - assert!(tree.rewind()); - for i in 1..4 { - tree.assert_append(i, Ephemeral); - } - tree.assert_append(4, Marked); - for i in 5..8 { - tree.assert_append(i, Ephemeral); - } - assert_eq!( - tree.witness(0.into(), 0), - Some(vec![ - H::from_u64(1), - H::combine_all(1, &[2, 3]), - H::combine_all(2, &[4, 5, 6, 7]), - H::empty_root(3.into()), - ]) - ); - } - - { - let mut tree = new_tree(100); - tree.assert_append(0, Ephemeral); - tree.assert_append(1, Ephemeral); - tree.assert_append(2, Marked); - tree.assert_append(3, Ephemeral); - tree.assert_append(4, Ephemeral); - tree.assert_append(5, Ephemeral); - tree.assert_append( - 6, - Checkpoint { - id: 1, - marking: Marking::Marked, - }, - ); - tree.assert_append(7, Ephemeral); - assert!(tree.rewind()); - assert_eq!( - tree.witness(Position::from(2), 0), - Some(vec![ - H::from_u64(3), - H::combine_all(1, &[0, 1]), - H::combine_all(2, &[4, 5, 6]), - H::empty_root(3.into()) - ]) - ); - } - - { - let mut tree = new_tree(100); - for i in 0..12 { - tree.assert_append(i, Ephemeral); - } - tree.assert_append(12, Marked); - tree.assert_append(13, Marked); - tree.assert_append(14, Ephemeral); - tree.assert_append(15, Ephemeral); - - assert_eq!( - tree.witness(Position::from(12), 0), - Some(vec![ - H::from_u64(13), - H::combine_all(1, &[14, 15]), - H::combine_all(2, &[8, 9, 10, 11]), - H::combine_all(3, &[0, 1, 2, 3, 4, 5, 6, 7]), - ]) - ); - } - - { - let ops = (0..=11) - .map(|i| Append(H::from_u64(i), Marked)) - .chain(Some(Append(H::from_u64(12), Ephemeral))) - .chain(Some(Append(H::from_u64(13), Ephemeral))) - .chain(Some(Witness(11u64.into(), 0))) - .collect::>(); - - let mut tree = new_tree(100); - assert_eq!( - Operation::apply_all(&ops, &mut tree), - Some(( - Position::from(11), - vec![ - H::from_u64(10), - H::combine_all(1, &[8, 9]), - H::combine_all(2, &[12, 13]), - H::combine_all(3, &[0, 1, 2, 3, 4, 5, 6, 7]), - ] - )) - ); - } - - { - let ops = vec![ - Append(H::from_u64(0), Ephemeral), - Append(H::from_u64(1), Ephemeral), - Append(H::from_u64(2), Ephemeral), - Append( - H::from_u64(3), - Checkpoint { - id: C::from_u64(1), - marking: Marking::Marked, - }, - ), - Append(H::from_u64(4), Marked), - Operation::Checkpoint(C::from_u64(2)), - Append( - H::from_u64(5), - Checkpoint { - id: C::from_u64(3), - marking: Marking::None, - }, - ), - Append( - H::from_u64(6), - Checkpoint { - id: C::from_u64(4), - marking: Marking::None, - }, - ), - Append( - H::from_u64(7), - Checkpoint { - id: C::from_u64(5), - marking: Marking::None, - }, - ), - Witness(3u64.into(), 5), - ]; - let mut tree = new_tree(100); - assert_eq!( - Operation::apply_all(&ops, &mut tree), - Some(( - Position::from(3), - vec![ - H::from_u64(2), - H::combine_all(1, &[0, 1]), - H::combine_all(2, &[]), - H::combine_all(3, &[]), - ] - )) - ); - } - - { - let ops = vec![ - Append(H::from_u64(0), Ephemeral), - Append(H::from_u64(0), Ephemeral), - Append(H::from_u64(0), Ephemeral), - Append( - H::from_u64(0), - Checkpoint { - id: C::from_u64(1), - marking: Marking::Marked, - }, - ), - Append(H::from_u64(0), Ephemeral), - Append(H::from_u64(0), Ephemeral), - Append(H::from_u64(0), Ephemeral), - Append( - H::from_u64(0), - Checkpoint { - id: C::from_u64(2), - marking: Marking::None, - }, - ), - Append(H::from_u64(0), Ephemeral), - Append(H::from_u64(0), Ephemeral), - Witness(Position(3), 1), - ]; - let mut tree = new_tree(100); - assert_eq!( - Operation::apply_all(&ops, &mut tree), - Some(( - Position::from(3), - vec![ - H::from_u64(0), - H::combine_all(1, &[0, 0]), - H::combine_all(2, &[0, 0, 0, 0]), - H::combine_all(3, &[]), - ] - )) - ); - } - - { - let ops = vec![ - Append(H::from_u64(0), Marked), - Append(H::from_u64(0), Ephemeral), - Append(H::from_u64(0), Ephemeral), - Append(H::from_u64(0), Ephemeral), - Append(H::from_u64(0), Ephemeral), - Append(H::from_u64(0), Ephemeral), - Append(H::from_u64(0), Ephemeral), - Operation::Checkpoint(C::from_u64(1)), - Append(H::from_u64(0), Marked), - Operation::Checkpoint(C::from_u64(2)), - Operation::Checkpoint(C::from_u64(3)), - Append( - H::from_u64(0), - Checkpoint { - id: C::from_u64(4), - marking: Marking::None, - }, - ), - Rewind, - Rewind, - Witness(Position(7), 2), - ]; - let mut tree = new_tree(100); - assert_eq!(Operation::apply_all(&ops, &mut tree), None); - } - - { - let ops = vec![ - Append(H::from_u64(0), Marked), - Append(H::from_u64(0), Ephemeral), - Append( - H::from_u64(0), - Checkpoint { - id: C::from_u64(1), - marking: Marking::Marked, - }, - ), - Append( - H::from_u64(0), - Checkpoint { - id: C::from_u64(4), - marking: Marking::None, - }, - ), - Witness(Position(2), 2), - ]; - let mut tree = new_tree(100); - assert_eq!( - Operation::apply_all(&ops, &mut tree), - Some(( - Position::from(2), - vec![ - H::empty_leaf(), - H::combine_all(1, &[0, 0]), - H::combine_all(2, &[]), - H::combine_all(3, &[]), - ] - )) - ); - } -} - -pub fn check_checkpoint_rewind, F: Fn(usize) -> T>( - new_tree: F, -) { - let mut t = new_tree(100); - assert!(!t.rewind()); - - let mut t = new_tree(100); - t.assert_checkpoint(1); - assert!(t.rewind()); - - let mut t = new_tree(100); - t.append("a".to_string(), Retention::Ephemeral); - t.assert_checkpoint(1); - t.append("b".to_string(), Retention::Marked); - assert!(t.rewind()); - assert_eq!(Some(Position::from(0)), t.current_position()); - - let mut t = new_tree(100); - t.append("a".to_string(), Retention::Marked); - t.assert_checkpoint(1); - assert!(t.rewind()); - - let mut t = new_tree(100); - t.append("a".to_string(), Retention::Marked); - t.assert_checkpoint(1); - t.append("a".to_string(), Retention::Ephemeral); - assert!(t.rewind()); - assert_eq!(Some(Position::from(0)), t.current_position()); - - let mut t = new_tree(100); - t.append("a".to_string(), Retention::Ephemeral); - t.assert_checkpoint(1); - t.assert_checkpoint(2); - assert!(t.rewind()); - t.append("b".to_string(), Retention::Ephemeral); - assert!(t.rewind()); - t.append("b".to_string(), Retention::Ephemeral); - assert_eq!(t.root(0).unwrap(), "ab______________"); -} - -pub fn check_remove_mark, F: Fn(usize) -> T>(new_tree: F) { - let samples = vec![ - vec![ - append_str("a", Retention::Ephemeral), - append_str( - "a", - Retention::Checkpoint { - id: C::from_u64(1), - marking: Marking::Marked, - }, - ), - witness(1, 1), - ], - vec![ - append_str("a", Retention::Ephemeral), - append_str("a", Retention::Ephemeral), - append_str("a", Retention::Ephemeral), - append_str("a", Retention::Marked), - Checkpoint(C::from_u64(1)), - unmark(3), - witness(3, 0), - ], - ]; - - for (i, sample) in samples.iter().enumerate() { - let result = check_operations(new_tree(100), sample); - assert!( - matches!(result, Ok(())), - "Reference/Test mismatch at index {}: {:?}", - i, - result - ); - } -} - -pub fn check_rewind_remove_mark, F: Fn(usize) -> T>( - new_tree: F, -) { - // rewinding doesn't remove a mark - let mut tree = new_tree(100); - tree.append("e".to_string(), Retention::Marked); - tree.assert_checkpoint(1); - assert!(tree.rewind()); - assert!(tree.remove_mark(0u64.into())); - - // use a maximum number of checkpoints of 1 - let mut tree = new_tree(1); - assert!(tree.append("e".to_string(), Retention::Marked)); - tree.assert_checkpoint(1); - assert!(tree.marked_positions().contains(&0u64.into())); - assert!(tree.append("f".to_string(), Retention::Ephemeral)); - // simulate a spend of `e` at `f` - assert!(tree.remove_mark(0u64.into())); - // even though the mark has been staged for removal, it's not gone yet - assert!(tree.marked_positions().contains(&0u64.into())); - tree.assert_checkpoint(2); - // the newest checkpoint will have caused the oldest to roll off, and - // so the forgotten node will be unmarked - assert!(!tree.marked_positions().contains(&0u64.into())); - assert!(!tree.remove_mark(0u64.into())); - - // The following check_operations tests cover errors where the - // test framework itself previously did not correctly handle - // chain state restoration. - - let samples = vec![ - vec![ - append_str("x", Retention::Marked), - Checkpoint(C::from_u64(1)), - Rewind, - unmark(0), - ], - vec![ - append_str("d", Retention::Marked), - Checkpoint(C::from_u64(1)), - unmark(0), - Rewind, - unmark(0), - ], - vec![ - append_str("o", Retention::Marked), - Checkpoint(C::from_u64(1)), - Checkpoint(C::from_u64(2)), - unmark(0), - Rewind, - Rewind, - ], - vec![ - append_str("s", Retention::Marked), - append_str("m", Retention::Ephemeral), - Checkpoint(C::from_u64(1)), - unmark(0), - Rewind, - unmark(0), - unmark(0), - ], - vec![ - append_str("a", Retention::Marked), - Checkpoint(C::from_u64(1)), - Rewind, - append_str("a", Retention::Marked), - ], - ]; - - for (i, sample) in samples.iter().enumerate() { - let result = check_operations(new_tree(100), sample); - assert!( - matches!(result, Ok(())), - "Reference/Test mismatch at index {}: {:?}", - i, - result - ); - } -} - -pub fn check_witness_consistency, F: Fn(usize) -> T>( - new_tree: F, -) { - let samples = vec![ - // Reduced examples - vec![ - append_str("a", Retention::Ephemeral), - append_str("b", Retention::Marked), - Checkpoint(C::from_u64(1)), - witness(0, 1), - ], - vec![ - append_str("c", Retention::Ephemeral), - append_str("d", Retention::Marked), - Checkpoint(C::from_u64(1)), - witness(1, 1), - ], - vec![ - append_str("e", Retention::Marked), - Checkpoint(C::from_u64(1)), - append_str("f", Retention::Ephemeral), - witness(0, 1), - ], - vec![ - append_str("g", Retention::Marked), - Checkpoint(C::from_u64(1)), - unmark(0), - append_str("h", Retention::Ephemeral), - witness(0, 0), - ], - vec![ - append_str("i", Retention::Marked), - Checkpoint(C::from_u64(1)), - unmark(0), - append_str("j", Retention::Ephemeral), - witness(0, 0), - ], - vec![ - append_str("i", Retention::Marked), - append_str("j", Retention::Ephemeral), - Checkpoint(C::from_u64(1)), - append_str("k", Retention::Ephemeral), - witness(0, 1), - ], - vec![ - append_str("l", Retention::Marked), - Checkpoint(C::from_u64(1)), - Checkpoint(C::from_u64(2)), - append_str("m", Retention::Ephemeral), - Checkpoint(C::from_u64(3)), - witness(0, 2), - ], - vec![ - Checkpoint(C::from_u64(1)), - append_str("n", Retention::Marked), - witness(0, 1), - ], - vec![ - append_str("a", Retention::Marked), - Checkpoint(C::from_u64(1)), - unmark(0), - Checkpoint(C::from_u64(2)), - append_str("b", Retention::Ephemeral), - witness(0, 1), - ], - vec![ - append_str("a", Retention::Marked), - append_str("b", Retention::Ephemeral), - unmark(0), - Checkpoint(C::from_u64(1)), - witness(0, 0), - ], - vec![ - append_str("a", Retention::Marked), - Checkpoint(C::from_u64(1)), - unmark(0), - Checkpoint(C::from_u64(2)), - Rewind, - append_str("b", Retention::Ephemeral), - witness(0, 0), - ], - vec![ - append_str("a", Retention::Marked), - Checkpoint(C::from_u64(1)), - Checkpoint(C::from_u64(2)), - Rewind, - append_str("a", Retention::Ephemeral), - unmark(0), - witness(0, 1), - ], - // Unreduced examples - vec![ - append_str("o", Retention::Ephemeral), - append_str("p", Retention::Marked), - append_str("q", Retention::Ephemeral), - Checkpoint(C::from_u64(1)), - unmark(1), - witness(1, 1), - ], - vec![ - append_str("r", Retention::Ephemeral), - append_str("s", Retention::Ephemeral), - append_str("t", Retention::Marked), - Checkpoint(C::from_u64(1)), - unmark(2), - Checkpoint(C::from_u64(2)), - witness(2, 2), - ], - vec![ - append_str("u", Retention::Marked), - append_str("v", Retention::Ephemeral), - append_str("w", Retention::Ephemeral), - Checkpoint(C::from_u64(1)), - unmark(0), - append_str("x", Retention::Ephemeral), - Checkpoint(C::from_u64(2)), - Checkpoint(C::from_u64(3)), - witness(0, 3), - ], - ]; - - for (i, sample) in samples.iter().enumerate() { - let result = check_operations(new_tree(100), sample); - assert!( - matches!(result, Ok(())), - "Reference/Test mismatch at index {}: {:?}", - i, - result - ); - } -} - -#[cfg(test)] -pub(crate) mod tests { - use crate::{ - testing::{compute_root_from_witness, SipHashable}, - Hashable, Level, Position, - }; - - #[test] - fn test_compute_root_from_witness() { - let expected = SipHashable::combine( - ::from(2), - &SipHashable::combine( - Level::from(1), - &SipHashable::combine(0.into(), &SipHashable(0), &SipHashable(1)), - &SipHashable::combine(0.into(), &SipHashable(2), &SipHashable(3)), - ), - &SipHashable::combine( - Level::from(1), - &SipHashable::combine(0.into(), &SipHashable(4), &SipHashable(5)), - &SipHashable::combine(0.into(), &SipHashable(6), &SipHashable(7)), - ), - ); - - assert_eq!( - compute_root_from_witness::( - SipHashable(0), - 0.into(), - &[ - SipHashable(1), - SipHashable::combine(0.into(), &SipHashable(2), &SipHashable(3)), - SipHashable::combine( - Level::from(1), - &SipHashable::combine(0.into(), &SipHashable(4), &SipHashable(5)), - &SipHashable::combine(0.into(), &SipHashable(6), &SipHashable(7)) - ) - ] - ), - expected - ); - - assert_eq!( - compute_root_from_witness( - SipHashable(4), - Position::from(4), - &[ - SipHashable(5), - SipHashable::combine(0.into(), &SipHashable(6), &SipHashable(7)), - SipHashable::combine( - Level::from(1), - &SipHashable::combine(0.into(), &SipHashable(0), &SipHashable(1)), - &SipHashable::combine(0.into(), &SipHashable(2), &SipHashable(3)) - ) - ] - ), - expected - ); - } -} diff --git a/rust-toolchain.toml b/rust-toolchain.toml index e06e5ca5..b2593066 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,3 +1,3 @@ [toolchain] -channel = "1.60.0" +channel = "1.64.0" components = [ "clippy", "rustfmt" ] diff --git a/shardtree/CHANGELOG.md b/shardtree/CHANGELOG.md index 93309010..83720899 100644 --- a/shardtree/CHANGELOG.md +++ b/shardtree/CHANGELOG.md @@ -7,6 +7,9 @@ and this project adheres to Rust's notion of ## Unreleased +### Changed +- MSRV is now 1.64 + ### Added - `shardtree::store::ShardStore::for_each_checkpoint` diff --git a/shardtree/Cargo.toml b/shardtree/Cargo.toml index a56cb969..238fce57 100644 --- a/shardtree/Cargo.toml +++ b/shardtree/Cargo.toml @@ -1,16 +1,16 @@ [package] name = "shardtree" +description = "A space-efficient Merkle tree with witnessing of marked leaves, checkpointing & state restoration." version = "0.4.0" authors = [ "Kris Nuttycombe ", ] -edition = "2021" -rust-version = "1.60" -license = "MIT OR Apache-2.0" -description = "A space-efficient Merkle tree with witnessing of marked leaves, checkpointing & state restoration." -homepage = "https://github.com/zcash/incrementalmerkletree" -repository = "https://github.com/zcash/incrementalmerkletree" -categories = ["algorithms", "data-structures"] +edition.workspace = true +rust-version.workspace = true +license.workspace = true +homepage.workspace = true +repository.workspace = true +categories.workspace = true [package.metadata.docs.rs] all-features = true @@ -20,14 +20,16 @@ rustdoc-args = ["--cfg", "docsrs"] assert_matches = { version = "1.5", optional = true } bitflags = "2" either = "1.8" -incrementalmerkletree = { version = "0.6", path = "../incrementalmerkletree" } -proptest = { version = "1.0.0", optional = true } +incrementalmerkletree.workspace = true +proptest = { workspace = true, optional = true } +incrementalmerkletree-testing = { workspace = true, optional = true } tracing = "0.1" [dev-dependencies] assert_matches = "1.5" -incrementalmerkletree = { version = "0.6", path = "../incrementalmerkletree", features = ["test-dependencies"] } -proptest = "1.0.0" +incrementalmerkletree = { workspace = true, features = ["test-dependencies"] } +incrementalmerkletree-testing.workspace = true +proptest.workspace = true [features] # The legacy-api feature guards types and functions that are useful for diff --git a/shardtree/src/lib.rs b/shardtree/src/lib.rs index 2ad58f8c..bd572788 100644 --- a/shardtree/src/lib.rs +++ b/shardtree/src/lib.rs @@ -1309,14 +1309,13 @@ mod tests { use incrementalmerkletree::{ frontier::{Frontier, NonEmptyFrontier}, - testing::{ - arb_operation, check_append, check_checkpoint_rewind, check_operations, - check_remove_mark, check_rewind_remove_mark, check_root_hashes, - check_witness_consistency, check_witnesses, complete_tree::CompleteTree, CombinedTree, - SipHashable, - }, Address, Hashable, Level, Marking, MerklePath, Position, Retention, }; + use incrementalmerkletree_testing::{ + arb_operation, check_append, check_checkpoint_rewind, check_operations, check_remove_mark, + check_rewind_remove_mark, check_root_hashes, check_witness_consistency, check_witnesses, + complete_tree::CompleteTree, CombinedTree, SipHashable, + }; use crate::{ error::{QueryError, ShardTreeError}, @@ -1462,9 +1461,7 @@ mod tests { // boundary. tree.batch_insert( frontier_end + 1, - ('g'..='j') - .into_iter() - .map(|c| (c.to_string(), Retention::Ephemeral)), + ('g'..='j').map(|c| (c.to_string(), Retention::Ephemeral)), ) .unwrap(); @@ -1476,9 +1473,7 @@ mod tests { // Insert nodes that require the pruned nodes for witnessing tree.batch_insert( frontier_end - 1, - ('e'..='f') - .into_iter() - .map(|c| (c.to_string(), Retention::Marked)), + ('e'..='f').map(|c| (c.to_string(), Retention::Marked)), ) .unwrap(); diff --git a/shardtree/src/store/caching.rs b/shardtree/src/store/caching.rs index 90780946..f00a6a01 100644 --- a/shardtree/src/store/caching.rs +++ b/shardtree/src/store/caching.rs @@ -220,12 +220,9 @@ where #[cfg(test)] mod tests { - use incrementalmerkletree::{ - testing::{ - append_str, check_operations, unmark, witness, CombinedTree, Operation, TestHashable, - Tree, - }, - Hashable, Marking, Position, Retention, + use incrementalmerkletree::{Hashable, Marking, Position, Retention}; + use incrementalmerkletree_testing::{ + append_str, check_operations, unmark, witness, CombinedTree, Operation, TestHashable, Tree, }; use super::CachingShardStore; diff --git a/shardtree/src/testing.rs b/shardtree/src/testing.rs index 6a56d0f0..7ff03ff9 100644 --- a/shardtree/src/testing.rs +++ b/shardtree/src/testing.rs @@ -6,7 +6,8 @@ use proptest::collection::vec; use proptest::prelude::*; use proptest::sample::select; -use incrementalmerkletree::{testing, Hashable}; +use incrementalmerkletree::Hashable; +use incrementalmerkletree_testing as testing; use super::*; use crate::store::{memory::MemoryShardStore, ShardStore}; @@ -323,9 +324,7 @@ pub fn check_shardtree_insertion< assert_matches!( tree.batch_insert( Position::from(4), - ('e'..'k') - .into_iter() - .map(|c| (c.to_string(), Retention::Ephemeral)) + ('e'..'k').map(|c| (c.to_string(), Retention::Ephemeral)) ), Ok(_) ); @@ -379,7 +378,7 @@ pub fn check_witness_with_pruned_subtrees< // simulate discovery of a note tree.batch_insert( Position::from(24), - ('a'..='h').into_iter().map(|c| { + ('a'..='h').map(|c| { ( c.to_string(), match c {