Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

chore: merge BoundedStorable into Storable #94

Merged
merged 17 commits into from
Aug 17, 2023
8 changes: 4 additions & 4 deletions benchmark-canisters/src/btreemap.rs
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
use crate::{count_instructions, Random};
use ic_cdk_macros::query;
use ic_stable_structures::{storable::Blob, BTreeMap, BoundedStorable, DefaultMemoryImpl};
use ic_stable_structures::{storable::Blob, BTreeMap, DefaultMemoryImpl, Storable};
use tiny_rng::{Rand, Rng};

#[query]
Expand Down Expand Up @@ -216,7 +216,7 @@ fn insert_blob_helper<const K: usize, const V: usize>() -> u64 {
}

// Profiles inserting a large number of random blobs into a btreemap.
fn insert_helper<K: Clone + Ord + BoundedStorable + Random, V: BoundedStorable + Random>() -> u64 {
fn insert_helper<K: Clone + Ord + Storable + Random, V: Storable + Random>() -> u64 {
let mut btree: BTreeMap<K, V, _> = BTreeMap::new(DefaultMemoryImpl::default());
let num_keys = 10_000;
let mut rng = Rng::from_seed(0);
Expand All @@ -241,7 +241,7 @@ fn get_blob_helper<const K: usize, const V: usize>() -> u64 {
get_helper::<Blob<K>, Blob<V>>()
}

fn get_helper<K: Clone + Ord + BoundedStorable + Random, V: BoundedStorable + Random>() -> u64 {
fn get_helper<K: Clone + Ord + Storable + Random, V: Storable + Random>() -> u64 {
let mut btree: BTreeMap<K, V, _> = BTreeMap::new(DefaultMemoryImpl::default());
let num_keys = 10_000;
let mut rng = Rng::from_seed(0);
Expand Down Expand Up @@ -271,7 +271,7 @@ fn remove_blob_helper<const K: usize, const V: usize>() -> u64 {
remove_helper::<Blob<K>, Blob<V>>()
}

fn remove_helper<K: Clone + Ord + BoundedStorable + Random, V: BoundedStorable + Random>() -> u64 {
fn remove_helper<K: Clone + Ord + Storable + Random, V: Storable + Random>() -> u64 {
let mut btree: BTreeMap<K, V, _> = BTreeMap::new(DefaultMemoryImpl::default());
let num_keys = 10_000;
let mut rng = Rng::from_seed(0);
Expand Down
12 changes: 10 additions & 2 deletions benchmark-canisters/src/main.rs
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
use ic_stable_structures::{storable::Blob, BoundedStorable};
use ic_stable_structures::storable::{Blob, Bound, Storable};
use tiny_rng::{Rand, Rng};

mod btreemap;
Expand All @@ -12,13 +12,21 @@ pub(crate) fn count_instructions<R>(f: impl FnOnce() -> R) -> u64 {
ic_cdk::api::performance_counter(0) - start
}

const fn max_size<A: Storable>() -> u32 {
if let Bound::Bounded { max_size, .. } = A::BOUND {
max_size
} else {
panic!("Cannot get max size of unbounded type.");
}
}

trait Random {
fn random(rng: &mut Rng) -> Self;
}

impl<const K: usize> Random for Blob<K> {
fn random(rng: &mut Rng) -> Self {
let size = rng.rand_u32() % Blob::<K>::MAX_SIZE;
let size = rng.rand_u32() % max_size::<Blob<K>>();
Blob::try_from(
rng.iter(Rand::rand_u8)
.take(size as usize)
Expand Down
6 changes: 3 additions & 3 deletions benchmark-canisters/src/vec.rs
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
use crate::{count_instructions, Random};
use ic_cdk_macros::query;
use ic_stable_structures::storable::Blob;
use ic_stable_structures::{BoundedStorable, DefaultMemoryImpl, StableVec};
use ic_stable_structures::{DefaultMemoryImpl, StableVec, Storable};
use tiny_rng::{Rand, Rng};

#[query]
Expand Down Expand Up @@ -78,7 +78,7 @@ fn vec_insert_blob<const N: usize>() -> u64 {
vec_insert::<Blob<N>>()
}

fn vec_insert<T: BoundedStorable + Random>() -> u64 {
fn vec_insert<T: Storable + Random>() -> u64 {
let num_items = 10_000;
let svec: StableVec<T, _> = StableVec::new(DefaultMemoryImpl::default()).unwrap();

Expand All @@ -100,7 +100,7 @@ fn vec_get_blob<const N: usize>() -> u64 {
vec_get::<Blob<N>>()
}

fn vec_get<T: BoundedStorable + Random>() -> u64 {
fn vec_get<T: Storable + Random>() -> u64 {
let num_items = 10_000;
let svec: StableVec<T, _> = StableVec::new(DefaultMemoryImpl::default()).unwrap();

Expand Down
14 changes: 8 additions & 6 deletions examples/src/custom_types_example/src/lib.rs
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
use candid::{CandidType, Decode, Deserialize, Encode};
use ic_stable_structures::memory_manager::{MemoryId, MemoryManager, VirtualMemory};
use ic_stable_structures::{BoundedStorable, DefaultMemoryImpl, StableBTreeMap, Storable};
use ic_stable_structures::{
storable::Bound, DefaultMemoryImpl, StableBTreeMap, Storable,
};
use std::{borrow::Cow, cell::RefCell};

type Memory = VirtualMemory<DefaultMemoryImpl>;
Expand All @@ -14,7 +16,7 @@ struct UserProfile {
}

// For a type to be used in a `StableBTreeMap`, it needs to implement the `Storable`
// and `BoundedStorable` traits, which specify how the type can be serialized/deserialized.
// trait, which specifies how the type can be serialized/deserialized.
//
// In this example, we're using candid to serialize/deserialize the struct, but you
// can use anything as long as you're maintaining backward-compatibility. The
Expand All @@ -31,11 +33,11 @@ impl Storable for UserProfile {
fn from_bytes(bytes: std::borrow::Cow<[u8]>) -> Self {
Decode!(bytes.as_ref(), Self).unwrap()
}
}

impl BoundedStorable for UserProfile {
const MAX_SIZE: u32 = MAX_VALUE_SIZE;
const IS_FIXED_SIZE: bool = false;
const BOUND: Bound = Bound::Bounded {
max_size: MAX_VALUE_SIZE,
is_fixed_size: false,
};
}

thread_local! {
Expand Down
18 changes: 9 additions & 9 deletions examples/src/vecs_and_strings/src/lib.rs
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
use ic_stable_structures::memory_manager::{MemoryId, MemoryManager, VirtualMemory};
use ic_stable_structures::{BoundedStorable, DefaultMemoryImpl, StableBTreeMap, Storable};
use ic_stable_structures::{storable::Bound, DefaultMemoryImpl, StableBTreeMap, Storable};
use std::cell::RefCell;

type Memory = VirtualMemory<DefaultMemoryImpl>;
Expand All @@ -25,11 +25,11 @@ impl Storable for UserName {
fn from_bytes(bytes: std::borrow::Cow<[u8]>) -> Self {
Self(String::from_bytes(bytes))
}
}

impl BoundedStorable for UserName {
const MAX_SIZE: u32 = MAX_USER_NAME_SIZE;
const IS_FIXED_SIZE: bool = false;
const BOUND: Bound = Bound::Bounded {
max_size: MAX_USER_NAME_SIZE,
is_fixed_size: false,
};
}

struct UserData(Vec<u8>);
Expand All @@ -43,11 +43,11 @@ impl Storable for UserData {
fn from_bytes(bytes: std::borrow::Cow<[u8]>) -> Self {
Self(<Vec<u8>>::from_bytes(bytes))
}
}

impl BoundedStorable for UserData {
const MAX_SIZE: u32 = MAX_USER_DATA_SIZE;
const IS_FIXED_SIZE: bool = false;
const BOUND: Bound = Bound::Bounded {
max_size: MAX_USER_DATA_SIZE,
is_fixed_size: false,
};
}

thread_local! {
Expand Down
69 changes: 36 additions & 33 deletions src/base_vec.rs
Original file line number Diff line number Diff line change
Expand Up @@ -28,15 +28,12 @@
//! ```
//!
//! The `SLOT_SIZE` constant depends on the item type. If the item
//! type sets the `BoundedStorable::IS_FIXED_SIZE` flag, the
//! `SLOT_SIZE` is equal to `BoundedStorable::MAX_SIZE`. Otherwise,
//! the `SLOT_SIZE` is `BoundedStorable::MAX_SIZE` plus the number of
//! bytes required to represent integers up to
//! `BoundedStorable::MAX_SIZE`.
use crate::storable::bytes_to_store_size;
//! type is fixed in size, the `SLOT_SIZE` is equal to the max size.
//! Otherwise, the `SLOT_SIZE` is the max size plus the number of
//! bytes required to represent integers up to that max size.
use crate::storable::{bounds, bytes_to_store_size};
use crate::{
read_u32, read_u64, safe_write, write_u32, write_u64, Address, BoundedStorable, GrowFailed,
Memory,
read_u32, read_u64, safe_write, write_u32, write_u64, Address, GrowFailed, Memory, Storable,
};
use std::borrow::{Borrow, Cow};
use std::fmt;
Expand Down Expand Up @@ -66,8 +63,8 @@ pub enum InitError {
/// memory layout.
IncompatibleVersion(u8),
/// The vector type is not compatible with the current vector
/// layout: MAX_SIZE and/or IS_FIXED_SIZE differ from the original
/// initialization parameters.
/// layout: the type's bounds differ from the original initialization
/// parameters.
IncompatibleElementType,
/// Failed to allocate memory for the vector.
OutOfMemory,
Expand All @@ -85,32 +82,34 @@ impl fmt::Display for InitError {
"unsupported layout version {version}; supported version numbers are 1..={LAYOUT_VERSION}"
),
Self::IncompatibleElementType =>
write!(fmt, "either MAX_SIZE or IS_FIXED_SIZE of the element type do not match the persisted vector attributes"),
write!(fmt, "the bounds (either max_size or is_fixed_size) of the element type do not match the persisted vector attributes"),
Self::OutOfMemory => write!(fmt, "failed to allocate memory for vector metadata"),
}
}
}

impl std::error::Error for InitError {}

pub struct BaseVec<T: BoundedStorable, M: Memory> {
pub struct BaseVec<T: Storable, M: Memory> {
memory: M,
_marker: PhantomData<T>,
}

impl<T: BoundedStorable, M: Memory> BaseVec<T, M> {
impl<T: Storable, M: Memory> BaseVec<T, M> {
/// Creates a new empty vector in the specified memory,
/// overwriting any data structures the memory might have
/// contained previously.
///
/// Complexity: O(1)
pub fn new(memory: M, magic: [u8; 3]) -> Result<Self, GrowFailed> {
let t_bounds = bounds::<T>();

let header = HeaderV1 {
magic,
version: LAYOUT_VERSION,
len: 0,
max_size: T::MAX_SIZE,
is_fixed_size: T::IS_FIXED_SIZE,
max_size: t_bounds.max_size,
is_fixed_size: t_bounds.is_fixed_size,
};
Self::write_header(&header, &memory)?;
Ok(Self {
Expand Down Expand Up @@ -139,7 +138,8 @@ impl<T: BoundedStorable, M: Memory> BaseVec<T, M> {
if header.version != LAYOUT_VERSION {
return Err(InitError::IncompatibleVersion(header.version));
}
if header.max_size != T::MAX_SIZE || header.is_fixed_size != T::IS_FIXED_SIZE {
let t_bounds = bounds::<T>();
if header.max_size != t_bounds.max_size || header.is_fixed_size != t_bounds.is_fixed_size {
return Err(InitError::IncompatibleElementType);
}

Expand Down Expand Up @@ -170,7 +170,7 @@ impl<T: BoundedStorable, M: Memory> BaseVec<T, M> {

/// Sets the item at the specified index to the specified value.
///
/// Complexity: O(T::MAX_SIZE)
/// Complexity: O(max_size(T))
///
/// PRECONDITION: index < self.len()
pub fn set(&self, index: u64, item: &T) {
Expand All @@ -186,7 +186,7 @@ impl<T: BoundedStorable, M: Memory> BaseVec<T, M> {

/// Returns the item at the specified index.
///
/// Complexity: O(T::MAX_SIZE)
/// Complexity: O(max_size(T))
pub fn get(&self, index: u64) -> Option<T> {
if index < self.len() {
Some(self.read_entry(index))
Expand All @@ -197,7 +197,7 @@ impl<T: BoundedStorable, M: Memory> BaseVec<T, M> {

/// Adds a new item at the end of the vector.
///
/// Complexity: O(T::MAX_SIZE)
/// Complexity: O(max_size(T))
pub fn push(&self, item: &T) -> Result<(), GrowFailed> {
let index = self.len();
let offset = DATA_OFFSET + slot_size::<T>() as u64 * index;
Expand All @@ -212,7 +212,7 @@ impl<T: BoundedStorable, M: Memory> BaseVec<T, M> {

/// Removes the item at the end of the vector.
///
/// Complexity: O(T::MAX_SIZE)
/// Complexity: O(max_size(T))
pub fn pop(&self) -> Option<T> {
let len = self.len();
if len == 0 {
Expand Down Expand Up @@ -253,14 +253,15 @@ impl<T: BoundedStorable, M: Memory> BaseVec<T, M> {

/// Writes the size of the item at the specified offset.
fn write_entry_size(&self, offset: u64, size: u32) -> Result<u64, GrowFailed> {
debug_assert!(size <= T::MAX_SIZE);
let t_bounds = bounds::<T>();
debug_assert!(size <= t_bounds.max_size);

if T::IS_FIXED_SIZE {
if t_bounds.is_fixed_size {
Ok(offset)
} else if T::MAX_SIZE <= u8::MAX as u32 {
} else if t_bounds.max_size <= u8::MAX as u32 {
safe_write(&self.memory, offset, &[size as u8; 1])?;
Ok(offset + 1)
} else if T::MAX_SIZE <= u16::MAX as u32 {
} else if t_bounds.max_size <= u16::MAX as u32 {
safe_write(&self.memory, offset, &(size as u16).to_le_bytes())?;
Ok(offset + 2)
} else {
Expand All @@ -271,13 +272,14 @@ impl<T: BoundedStorable, M: Memory> BaseVec<T, M> {

/// Reads the size of the entry at the specified offset.
fn read_entry_size(&self, offset: u64) -> (u64, usize) {
if T::IS_FIXED_SIZE {
(offset, T::MAX_SIZE as usize)
} else if T::MAX_SIZE <= u8::MAX as u32 {
let t_bounds = bounds::<T>();
if t_bounds.is_fixed_size {
(offset, t_bounds.max_size as usize)
} else if t_bounds.max_size <= u8::MAX as u32 {
let mut size = [0u8; 1];
self.memory.read(offset, &mut size);
(offset + 1, size[0] as usize)
} else if T::MAX_SIZE <= u16::MAX as u32 {
} else if t_bounds.max_size <= u16::MAX as u32 {
let mut size = [0u8; 2];
self.memory.read(offset, &mut size);
(offset + 2, u16::from_le_bytes(size) as usize)
Expand Down Expand Up @@ -326,19 +328,20 @@ impl<T: BoundedStorable, M: Memory> BaseVec<T, M> {
}
}

impl<T: BoundedStorable + fmt::Debug, M: Memory> fmt::Debug for BaseVec<T, M> {
impl<T: Storable + fmt::Debug, M: Memory> fmt::Debug for BaseVec<T, M> {
fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result {
self.to_vec().fmt(fmt)
}
}

fn slot_size<T: BoundedStorable>() -> u32 {
T::MAX_SIZE + bytes_to_store_size::<T>()
fn slot_size<T: Storable>() -> u32 {
let t_bounds = bounds::<T>();
t_bounds.max_size + bytes_to_store_size(&t_bounds)
}

pub struct Iter<'a, T, M>
where
T: BoundedStorable,
T: Storable,
M: Memory,
{
vec: &'a BaseVec<T, M>,
Expand All @@ -348,7 +351,7 @@ where

impl<T, M> Iterator for Iter<'_, T, M>
where
T: BoundedStorable,
T: Storable,
M: Memory,
{
type Item = T;
Expand Down
Loading
Loading