Merge pull request #105 from nuttycom/limitable_pruning

shardtree: Add `Reference` retention to make it possible to limit pruning.

incrementalmerkletree/CHANGELOG.md

@@ -7,6 +7,15 @@ and this project adheres to Rust's notion of
 
 ## Unreleased
 
+### Added
+- `incrementalmerkletree::Marking`
+
+### Changed
+- `incrementalmerkletree::Retention`
+  - Has added variant `Retention::Reference`
+  - `Retention::Checkpoint::is_marked` has been replaced by `Retention::Checkpoint::marking`
+    to permit checkpoints with `Reference` retention to be represented.
+
 ## [0.5.1] - 2024-03-25
 
 ### Added

incrementalmerkletree/src/lib.rs

@@ -62,35 +62,73 @@ pub mod witness;
 #[cfg_attr(docsrs, doc(cfg(feature = "test-dependencies")))]
 pub mod testing;
 
+/// An enumeration of the additional marking states that can be applied
+/// to leaves with [`Retention::Checkpoint`] retention.
+#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
+pub enum Marking {
+    /// A checkpoint with `Marked` marking will have `Marked` retention after `Checkpoint`
+    /// retention is removed.
+    Marked,
+    /// A checkpoint with `Reference` marking will have `Reference` retention after `Checkpoint`
+    /// retention is removed.
+    Reference,
+    /// A checkpoint with `None` marking will have `Ephemeral` retention after `Checkpoint`
+    /// retention is removed.
+    None,
+}
+
 /// A type for metadata that is used to determine when and how a leaf can be pruned from a tree.
 #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
 pub enum Retention<C> {
+    /// A leaf with `Ephemeral` retention will be pruned whenever its sibling is also a leaf with
+    /// `Ephemeral` retention.
     Ephemeral,
-    Checkpoint { id: C, is_marked: bool },
+    /// A leaf with `Checkpoint` retention will have its position retained in the tree
+    /// during pruning, but its value may be pruned (by merging with its sibling). If
+    /// `Checkpoint` retention is removed from the leaf, then the retention for the leaf will
+    /// become either `Ephemeral`, `Marked`, or `Reference` depending upon the value of the
+    /// `marking` field.
+    Checkpoint { id: C, marking: Marking },
+    /// A leaf with `Marked` retention will be retained in the tree during pruning. `Marked`
+    /// retention may only be explicitly removed.
     Marked,
+    /// A leaf with `Reference` retention will be retained in the tree during pruning. `Reference`
+    /// retention is removed whenever the associated leaf is overwritten with a tree node having
+    /// `Ephemeral`, `Checkpoint`, or `Marked` retention.
+    Reference,
 }
 
 impl<C> Retention<C> {
+    /// Returns whether the associated node has [`Retention::Checkpoint`] retention.
     pub fn is_checkpoint(&self) -> bool {
         matches!(self, Retention::Checkpoint { .. })
     }
 
+    /// Returns whether the associated node has [`Retention::Marked`] retention
+    /// or [`Retention::Checkpoint`] retention with [`Marking::Marked`] marking.
     pub fn is_marked(&self) -> bool {
-        match self {
-            Retention::Ephemeral => false,
-            Retention::Checkpoint { is_marked, .. } => *is_marked,
-            Retention::Marked => true,
-        }
+        matches!(
+            self,
+            Retention::Marked
+                | Retention::Checkpoint {
+                    marking: Marking::Marked,
+                    ..
+                }
+        )
     }
 
+    /// Applies the provided function to the checkpoint identifier, if any, and returns a new
+    /// `Retention` value having the same structure with the resulting value used as the checkpoint
+    /// identifier.
     pub fn map<'a, D, F: Fn(&'a C) -> D>(&'a self, f: F) -> Retention<D> {
         match self {
             Retention::Ephemeral => Retention::Ephemeral,
-            Retention::Checkpoint { id, is_marked } => Retention::Checkpoint {
+            Retention::Checkpoint { id, marking } => Retention::Checkpoint {
                 id: f(id),
-                is_marked: *is_marked,
+                marking: *marking,
             },
             Retention::Marked => Retention::Marked,
+            Retention::Reference => Retention::Reference,
         }
     }
 }

incrementalmerkletree/src/testing.rs

@@ -5,7 +5,7 @@ use core::marker::PhantomData;
 use proptest::prelude::*;
 use std::collections::BTreeSet;
 
-use crate::{Hashable, Level, Position, Retention};
+use crate::{Hashable, Level, Marking, Position, Retention};
 
 pub mod complete_tree;
 
@@ -208,10 +208,22 @@ impl<H: Hashable + Clone, C: Clone> Operation<H, C> {
     }
 }
 
+/// Returns a strategy for creating a uniformly-distributed [`Marking`]
+/// value.
+pub fn arb_marking() -> impl Strategy<Value = Marking> {
+    prop_oneof![
+        Just(Marking::Marked),
+        Just(Marking::Reference),
+        Just(Marking::None)
+    ]
+}
+
+/// Returns a strategy for creating a uniformly-distributed [`Retention`]
+/// value.
 pub fn arb_retention() -> impl Strategy<Value = Retention<()>> {
     prop_oneof![
         Just(Retention::Ephemeral),
-        any::<bool>().prop_map(|is_marked| Retention::Checkpoint { id: (), is_marked }),
+        arb_marking().prop_map(|marking| Retention::Checkpoint { id: (), marking }),
         Just(Retention::Marked),
     ]
 }
@@ -557,7 +569,7 @@ pub fn check_root_hashes<H: TestHashable, C: TestCheckpoint, T: Tree<H, C>, F: F
             0,
             Retention::Checkpoint {
                 id: 1,
-                is_marked: true,
+                marking: Marking::Marked,
             },
         );
         for _ in 0..3 {
@@ -735,7 +747,7 @@ pub fn check_witnesses<H: TestHashable, C: TestCheckpoint, T: Tree<H, C>, F: Fn(
             0,
             Checkpoint {
                 id: 1,
-                is_marked: true,
+                marking: Marking::Marked,
             },
         );
         assert!(tree.rewind());
@@ -769,7 +781,7 @@ pub fn check_witnesses<H: TestHashable, C: TestCheckpoint, T: Tree<H, C>, F: Fn(
             6,
             Checkpoint {
                 id: 1,
-                is_marked: true,
+                marking: Marking::Marked,
             },
         );
         tree.assert_append(7, Ephemeral);
@@ -838,7 +850,7 @@ pub fn check_witnesses<H: TestHashable, C: TestCheckpoint, T: Tree<H, C>, F: Fn(
                 H::from_u64(3),
                 Checkpoint {
                     id: C::from_u64(1),
-                    is_marked: true,
+                    marking: Marking::Marked,
                 },
             ),
             Append(H::from_u64(4), Marked),
@@ -847,21 +859,21 @@ pub fn check_witnesses<H: TestHashable, C: TestCheckpoint, T: Tree<H, C>, F: Fn(
                 H::from_u64(5),
                 Checkpoint {
                     id: C::from_u64(3),
-                    is_marked: false,
+                    marking: Marking::None,
                 },
             ),
             Append(
                 H::from_u64(6),
                 Checkpoint {
                     id: C::from_u64(4),
-                    is_marked: false,
+                    marking: Marking::None,
                 },
             ),
             Append(
                 H::from_u64(7),
                 Checkpoint {
                     id: C::from_u64(5),
-                    is_marked: false,
+                    marking: Marking::None,
                 },
             ),
             Witness(3u64.into(), 5),
@@ -890,7 +902,7 @@ pub fn check_witnesses<H: TestHashable, C: TestCheckpoint, T: Tree<H, C>, F: Fn(
                 H::from_u64(0),
                 Checkpoint {
                     id: C::from_u64(1),
-                    is_marked: true,
+                    marking: Marking::Marked,
                 },
             ),
             Append(H::from_u64(0), Ephemeral),
@@ -900,7 +912,7 @@ pub fn check_witnesses<H: TestHashable, C: TestCheckpoint, T: Tree<H, C>, F: Fn(
                 H::from_u64(0),
                 Checkpoint {
                     id: C::from_u64(2),
-                    is_marked: false,
+                    marking: Marking::None,
                 },
             ),
             Append(H::from_u64(0), Ephemeral),
@@ -939,7 +951,7 @@ pub fn check_witnesses<H: TestHashable, C: TestCheckpoint, T: Tree<H, C>, F: Fn(
                 H::from_u64(0),
                 Checkpoint {
                     id: C::from_u64(4),
-                    is_marked: false,
+                    marking: Marking::None,
                 },
             ),
             Rewind,
@@ -958,14 +970,14 @@ pub fn check_witnesses<H: TestHashable, C: TestCheckpoint, T: Tree<H, C>, F: Fn(
                 H::from_u64(0),
                 Checkpoint {
                     id: C::from_u64(1),
-                    is_marked: true,
+                    marking: Marking::Marked,
                 },
             ),
             Append(
                 H::from_u64(0),
                 Checkpoint {
                     id: C::from_u64(4),
-                    is_marked: false,
+                    marking: Marking::None,
                 },
             ),
             Witness(Position(2), 2),
@@ -1034,7 +1046,7 @@ pub fn check_remove_mark<C: TestCheckpoint, T: Tree<String, C>, F: Fn(usize) ->
                 "a",
                 Retention::Checkpoint {
                     id: C::from_u64(1),
-                    is_marked: true,
+                    marking: Marking::Marked,
                 },
             ),
             witness(1, 1),

incrementalmerkletree/src/testing/complete_tree.rs

@@ -2,6 +2,7 @@
 use std::cmp::min;
 use std::collections::{BTreeMap, BTreeSet};
 
+use crate::Marking;
 use crate::{testing::Tree, Hashable, Level, Position, Retention};
 
 const MAX_COMPLETE_SIZE_ERROR: &str = "Positions of a `CompleteTree` must fit into the platform word size, because larger complete trees are not representable.";
@@ -104,11 +105,11 @@ impl<H: Hashable, C: Clone + Ord + core::fmt::Debug, const DEPTH: u8> CompleteTr
                 append(&mut self.leaves, value, DEPTH)?;
                 self.mark();
             }
-            Retention::Checkpoint { id, is_marked } => {
+            Retention::Checkpoint { id, marking } => {
                 let latest_checkpoint = self.checkpoints.keys().rev().next();
                 if Some(&id) > latest_checkpoint {
                     append(&mut self.leaves, value, DEPTH)?;
-                    if is_marked {
+                    if marking == Marking::Marked {
                         self.mark();
                     }
                     self.checkpoint(id, self.current_position());
@@ -119,7 +120,7 @@ impl<H: Hashable, C: Clone + Ord + core::fmt::Debug, const DEPTH: u8> CompleteTr
                     });
                 }
             }
-            Retention::Ephemeral => {
+            Retention::Ephemeral | Retention::Reference => {
                 append(&mut self.leaves, value, DEPTH)?;
             }
         }

shardtree/CHANGELOG.md

@@ -7,6 +7,24 @@ and this project adheres to Rust's notion of
 
 ## Unreleased
 
+### Added
+- `shardtree::tree::Tree::{is_leaf, map, try_map, empty_pruned}`
+- `shardtree::tree::LocatedTree::{map, try_map}`
+- `shardtree::prunable::PrunableTree::{has_computable_root}`
+
+### Changed
+- `shardtree::tree::Node` has additional variant `Node::Pruned`.
+
+### Removed
+- `shardtree::tree::Tree::is_complete` as it is no longer well-defined in the
+  presence of `Pruned` nodes. Use `PrunableTree::has_computable_root` to
+  determine whether it is possible to compute the root of a tree.
+
+### Fixed
+- Fixes an error that could occur if an inserted `Frontier` node was
+  interpreted as a node that had actually had its value observed as though it
+  had been inserted using the ordinary tree insertion methods.
+
 ## [0.3.1] - 2024-04-03
 
 ### Fixed

shardtree/src/legacy.rs

@@ -1,6 +1,8 @@
 use std::fmt;
 
-use incrementalmerkletree::{witness::IncrementalWitness, Address, Hashable, Level, Retention};
+use incrementalmerkletree::{
+    witness::IncrementalWitness, Address, Hashable, Level, Marking, Retention,
+};
 
 use crate::{
     store::ShardStore, InsertionError, LocatedPrunableTree, LocatedTree, PrunableTree,
@@ -198,7 +200,7 @@ impl<H: Hashable + Clone + PartialEq> LocatedPrunableTree<H> {
                     c.ommers_iter().cloned(),
                     &Retention::Checkpoint {
                         id: checkpoint_id,
-                        is_marked: false,
+                        marking: Marking::None,
                     },
                     self.root_addr.level(),
                 )