From c095f4a0188db2dbe8d48b94562a01b4151474f0 Mon Sep 17 00:00:00 2001 From: Dragoljub Duric Date: Tue, 27 Jun 2023 11:29:56 +0000 Subject: [PATCH 01/70] . --- src/memory_manager.rs | 70 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/src/memory_manager.rs b/src/memory_manager.rs index c7d5fedd..66567d1d 100644 --- a/src/memory_manager.rs +++ b/src/memory_manager.rs @@ -127,6 +127,70 @@ const HEADER_RESERVED_BYTES: usize = 32; /// ... /// -------------------------------------------------- <- Page ((MAX_NUM_BUCKETS - 1) * N + 1) /// Bucket MAX_NUM_BUCKETS ↕ N pages +/// +/// # V2 layout +/// +/// ```text +/// -------------------------------------------------- <- Address 0 +/// Magic "MGR" ↕ 3 bytes +/// -------------------------------------------------- +/// Layout version ↕ 1 byte +/// -------------------------------------------------- +/// Number of allocated buckets ↕ 2 bytes +/// -------------------------------------------------- +/// Bucket size (in pages) = N ↕ 2 bytes +/// -------------------------------------------------- +/// Reserved space ↕ 32 bytes +/// -------------------------------------------------- +/// Size of memory 0 (in buckets) = k0 ↕ 8 bytes +/// -------------------------------------------------- +/// Bucket 1 belonging to memory 0 ↕ 1 byte +/// -------------------------------------------------- +/// Bucket 2 belonging to memory 0 ↕ 1 byte +/// -------------------------------------------------- +/// ... +/// -------------------------------------------------- +/// Bucket k0 belonging to memory 0 ↕ 1 byte +/// -------------------------------------------------- +/// Size of memory 1 (in buckets) = k1 ↕ 8 bytes +/// -------------------------------------------------- +/// Bucket 1 belonging to memory 1 ↕ 1 byte +/// -------------------------------------------------- +/// Bucket 2 belonging to memory 1 ↕ 1 byte +/// -------------------------------------------------- +/// ... +/// -------------------------------------------------- +/// Bucket k1 belonging to memory 1 ↕ 1 byte +/// -------------------------------------------------- +/// ... +/// -------------------------------------------------- +/// Size of memory 254 (in buckets) = k254 ↕ 8 bytes +/// -------------------------------------------------- +/// Bucket 1 belonging to memory 254 ↕ 1 byte +/// -------------------------------------------------- +/// Bucket 2 belonging to memory 254 ↕ 1 byte +/// -------------------------------------------------- +/// ... +/// -------------------------------------------------- +/// Bucket k254 belonging to memory 254 ↕ 1 byte +/// -------------------------------------------------- <- Bucket allocations +/// Bucket 1 ↕ 1 byte (1 byte indicating which memory owns it) +/// -------------------------------------------------- +/// Bucket 2 ↕ 1 byte +/// -------------------------------------------------- +/// ... +/// -------------------------------------------------- +/// Bucket `MAX_NUM_BUCKETS` ↕ 1 byte +/// -------------------------------------------------- +/// Unallocated space ↕ 30'688 bytes +/// -------------------------------------------------- <- Buckets (Page 1) +/// Bucket 1 ↕ N pages +/// -------------------------------------------------- <- Page N + 1 +/// Bucket 2 ↕ N pages +/// -------------------------------------------------- +/// ... +/// -------------------------------------------------- <- Page ((MAX_NUM_BUCKETS - 1) * N + 1) +/// Bucket MAX_NUM_BUCKETS ↕ N pages /// ``` pub struct MemoryManager { inner: Rc>>, @@ -155,6 +219,10 @@ impl MemoryManager { memory_manager: self.inner.clone(), } } + + pub fn free(&mut self, id: MemoryId) { + self.inner.borrow_mut().free(id); + } } #[repr(C, packed)] @@ -419,6 +487,8 @@ impl MemoryManagerInner { // Ceiling division. (num_pages + self.bucket_size_in_pages as u64 - 1) / self.bucket_size_in_pages as u64 } + + fn free(&mut self, id: MemoryId) {} } struct Segment { From c175203be42366c08b2df5dbfb5d50ae7fab902e Mon Sep 17 00:00:00 2001 From: Dragoljub Duric Date: Tue, 27 Jun 2023 11:36:08 +0000 Subject: [PATCH 02/70] . --- src/memory_manager.rs | 36 +++++++++++++++--------------------- 1 file changed, 15 insertions(+), 21 deletions(-) diff --git a/src/memory_manager.rs b/src/memory_manager.rs index 66567d1d..77d5f0ee 100644 --- a/src/memory_manager.rs +++ b/src/memory_manager.rs @@ -142,45 +142,39 @@ const HEADER_RESERVED_BYTES: usize = 32; /// -------------------------------------------------- /// Reserved space ↕ 32 bytes /// -------------------------------------------------- -/// Size of memory 0 (in buckets) = k0 ↕ 8 bytes +/// Size of memory 0 (in buckets) = k0 ↕ 8 bytes /// -------------------------------------------------- -/// Bucket 1 belonging to memory 0 ↕ 1 byte -/// -------------------------------------------------- -/// Bucket 2 belonging to memory 0 ↕ 1 byte +/// Size of memory 1 (in buckets) = k1 ↕ 8 bytes /// -------------------------------------------------- /// ... /// -------------------------------------------------- -/// Bucket k0 belonging to memory 0 ↕ 1 byte -/// -------------------------------------------------- -/// Size of memory 1 (in buckets) = k1 ↕ 8 bytes -/// -------------------------------------------------- -/// Bucket 1 belonging to memory 1 ↕ 1 byte +/// Size of memory 254 (in buckets) = k254 ↕ 8 bytes +/// -------------------------------------------------- <- IDs of buckets belonging to different memories +/// Bucket 1 ID belonging to memory 0 ↕ 15 bits 15 bits required since the bucket ID \in [1, 16384] /// -------------------------------------------------- -/// Bucket 2 belonging to memory 1 ↕ 1 byte +/// Bucket 2 ID belonging to memory 0 ↕ 15 bits /// -------------------------------------------------- /// ... /// -------------------------------------------------- -/// Bucket k1 belonging to memory 1 ↕ 1 byte +/// Bucket k0 ID belonging to memory 0 ↕ 15 bits /// -------------------------------------------------- -/// ... +/// Bucket 1 ID belonging to memory 1 ↕ 15 bits /// -------------------------------------------------- -/// Size of memory 254 (in buckets) = k254 ↕ 8 bytes +/// Bucket 2 ID belonging to memory 1 ↕ 15 bits /// -------------------------------------------------- -/// Bucket 1 belonging to memory 254 ↕ 1 byte +/// ... /// -------------------------------------------------- -/// Bucket 2 belonging to memory 254 ↕ 1 byte +/// Bucket k1 ID belonging to memory 1 ↕ 15 bits /// -------------------------------------------------- /// ... +/// --------------------------------------------------- +/// Bucket 1 ID belonging to memory 254 ↕ 15 bits /// -------------------------------------------------- -/// Bucket k254 belonging to memory 254 ↕ 1 byte -/// -------------------------------------------------- <- Bucket allocations -/// Bucket 1 ↕ 1 byte (1 byte indicating which memory owns it) -/// -------------------------------------------------- -/// Bucket 2 ↕ 1 byte +/// Bucket 2 ID belonging to memory 254 ↕ 15 bits /// -------------------------------------------------- /// ... /// -------------------------------------------------- -/// Bucket `MAX_NUM_BUCKETS` ↕ 1 byte +/// Bucket k254 ID belonging to memory 254 ↕ 15 bits /// -------------------------------------------------- /// Unallocated space ↕ 30'688 bytes /// -------------------------------------------------- <- Buckets (Page 1) From 648c787c572829346f10e18ba1a4206c5d511e51 Mon Sep 17 00:00:00 2001 From: Dragoljub Duric Date: Tue, 27 Jun 2023 11:44:38 +0000 Subject: [PATCH 03/70] . --- src/memory_manager.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/memory_manager.rs b/src/memory_manager.rs index 77d5f0ee..1d0c1eb9 100644 --- a/src/memory_manager.rs +++ b/src/memory_manager.rs @@ -102,13 +102,13 @@ const HEADER_RESERVED_BYTES: usize = 32; /// -------------------------------------------------- /// Reserved space ↕ 32 bytes /// -------------------------------------------------- -/// Size of memory 0 (in pages) ↕ 8 bytes +/// Size of memory 0 (in pages) ↕ 1 bytes /// -------------------------------------------------- -/// Size of memory 1 (in pages) ↕ 8 bytes +/// Size of memory 1 (in pages) ↕ 1 bytes /// -------------------------------------------------- /// ... /// -------------------------------------------------- -/// Size of memory 254 (in pages) ↕ 8 bytes +/// Size of memory 254 (in pages) ↕ 1 bytes /// -------------------------------------------------- <- Bucket allocations /// Bucket 1 ↕ 1 byte (1 byte indicating which memory owns it) /// -------------------------------------------------- From 76458d485dde57be35737989177449f646354226 Mon Sep 17 00:00:00 2001 From: Dragoljub Duric Date: Tue, 27 Jun 2023 11:45:20 +0000 Subject: [PATCH 04/70] . --- src/memory_manager.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/memory_manager.rs b/src/memory_manager.rs index 1d0c1eb9..e3be250c 100644 --- a/src/memory_manager.rs +++ b/src/memory_manager.rs @@ -102,13 +102,13 @@ const HEADER_RESERVED_BYTES: usize = 32; /// -------------------------------------------------- /// Reserved space ↕ 32 bytes /// -------------------------------------------------- -/// Size of memory 0 (in pages) ↕ 1 bytes +/// Size of memory 0 (in pages) ↕ 8 bytes /// -------------------------------------------------- -/// Size of memory 1 (in pages) ↕ 1 bytes +/// Size of memory 1 (in pages) ↕ 8 bytes /// -------------------------------------------------- /// ... /// -------------------------------------------------- -/// Size of memory 254 (in pages) ↕ 1 bytes +/// Size of memory 254 (in pages) ↕ 8 bytes /// -------------------------------------------------- <- Bucket allocations /// Bucket 1 ↕ 1 byte (1 byte indicating which memory owns it) /// -------------------------------------------------- @@ -142,13 +142,13 @@ const HEADER_RESERVED_BYTES: usize = 32; /// -------------------------------------------------- /// Reserved space ↕ 32 bytes /// -------------------------------------------------- -/// Size of memory 0 (in buckets) = k0 ↕ 8 bytes +/// Size of memory 0 (in buckets) = k0 ↕ 1 bytes /// -------------------------------------------------- -/// Size of memory 1 (in buckets) = k1 ↕ 8 bytes +/// Size of memory 1 (in buckets) = k1 ↕ 1 bytes /// -------------------------------------------------- /// ... /// -------------------------------------------------- -/// Size of memory 254 (in buckets) = k254 ↕ 8 bytes +/// Size of memory 254 (in buckets) = k254 ↕ 1 bytes /// -------------------------------------------------- <- IDs of buckets belonging to different memories /// Bucket 1 ID belonging to memory 0 ↕ 15 bits 15 bits required since the bucket ID \in [1, 16384] /// -------------------------------------------------- From 8ac4b4f410c24cc5c16aa4632c1e83f45180c65d Mon Sep 17 00:00:00 2001 From: Dragoljub Duric Date: Tue, 27 Jun 2023 12:20:54 +0000 Subject: [PATCH 05/70] . --- src/memory_manager.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/memory_manager.rs b/src/memory_manager.rs index e3be250c..9d1c93a6 100644 --- a/src/memory_manager.rs +++ b/src/memory_manager.rs @@ -142,15 +142,15 @@ const HEADER_RESERVED_BYTES: usize = 32; /// -------------------------------------------------- /// Reserved space ↕ 32 bytes /// -------------------------------------------------- -/// Size of memory 0 (in buckets) = k0 ↕ 1 bytes +/// Size of memory 0 (in buckets) = k0 ↕ 2 bytes /// -------------------------------------------------- -/// Size of memory 1 (in buckets) = k1 ↕ 1 bytes +/// Size of memory 1 (in buckets) = k1 ↕ 2 bytes /// -------------------------------------------------- /// ... /// -------------------------------------------------- -/// Size of memory 254 (in buckets) = k254 ↕ 1 bytes +/// Size of memory 254 (in buckets) = k254 ↕ 2 bytes /// -------------------------------------------------- <- IDs of buckets belonging to different memories -/// Bucket 1 ID belonging to memory 0 ↕ 15 bits 15 bits required since the bucket ID \in [1, 16384] +/// Bucket 1 ID belonging to memory 0 ↕ 15 bits /// -------------------------------------------------- /// Bucket 2 ID belonging to memory 0 ↕ 15 bits /// -------------------------------------------------- @@ -176,7 +176,7 @@ const HEADER_RESERVED_BYTES: usize = 32; /// -------------------------------------------------- /// Bucket k254 ID belonging to memory 254 ↕ 15 bits /// -------------------------------------------------- -/// Unallocated space ↕ 30'688 bytes +/// Unallocated space ↕ 4'566 bytes /// -------------------------------------------------- <- Buckets (Page 1) /// Bucket 1 ↕ N pages /// -------------------------------------------------- <- Page N + 1 From c19e7900aa90005a29133bca01585ffbd7653233 Mon Sep 17 00:00:00 2001 From: Dragoljub Duric Date: Tue, 27 Jun 2023 12:32:49 +0000 Subject: [PATCH 06/70] . --- src/memory_manager.rs | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/memory_manager.rs b/src/memory_manager.rs index 9d1c93a6..48a2307d 100644 --- a/src/memory_manager.rs +++ b/src/memory_manager.rs @@ -132,24 +132,24 @@ const HEADER_RESERVED_BYTES: usize = 32; /// /// ```text /// -------------------------------------------------- <- Address 0 -/// Magic "MGR" ↕ 3 bytes +/// Magic "MGR" ↕ 3 bytes /// -------------------------------------------------- -/// Layout version ↕ 1 byte +/// Layout version ↕ 1 byte /// -------------------------------------------------- -/// Number of allocated buckets ↕ 2 bytes +/// Number of allocated buckets ↕ 2 bytes /// -------------------------------------------------- -/// Bucket size (in pages) = N ↕ 2 bytes +/// Bucket size (in pages) = N ↕ 2 bytes /// -------------------------------------------------- -/// Reserved space ↕ 32 bytes +/// Reserved space ↕ 32 bytes /// -------------------------------------------------- -/// Size of memory 0 (in buckets) = k0 ↕ 2 bytes +/// Size of memory 0 (in buckets) = k0 ↕ 2 bytes /// -------------------------------------------------- -/// Size of memory 1 (in buckets) = k1 ↕ 2 bytes +/// Size of memory 1 (in buckets) = k1 ↕ 2 bytes /// -------------------------------------------------- /// ... /// -------------------------------------------------- /// Size of memory 254 (in buckets) = k254 ↕ 2 bytes -/// -------------------------------------------------- <- IDs of buckets belonging to different memories +/// -------------------------------------------------- <- IDs of buckets /// Bucket 1 ID belonging to memory 0 ↕ 15 bits /// -------------------------------------------------- /// Bucket 2 ID belonging to memory 0 ↕ 15 bits @@ -176,15 +176,15 @@ const HEADER_RESERVED_BYTES: usize = 32; /// -------------------------------------------------- /// Bucket k254 ID belonging to memory 254 ↕ 15 bits /// -------------------------------------------------- -/// Unallocated space ↕ 4'566 bytes +/// Unallocated space ↕ 3'546 bytes /// -------------------------------------------------- <- Buckets (Page 1) -/// Bucket 1 ↕ N pages +/// Bucket 1 ↕ N pages /// -------------------------------------------------- <- Page N + 1 -/// Bucket 2 ↕ N pages +/// Bucket 2 ↕ N pages /// -------------------------------------------------- /// ... /// -------------------------------------------------- <- Page ((MAX_NUM_BUCKETS - 1) * N + 1) -/// Bucket MAX_NUM_BUCKETS ↕ N pages +/// Bucket MAX_NUM_BUCKETS ↕ N pages /// ``` pub struct MemoryManager { inner: Rc>>, From 6062cbbcf56cab5abe79192136a5388fe60b090b Mon Sep 17 00:00:00 2001 From: Dragoljub Duric Date: Wed, 28 Jun 2023 11:44:30 +0000 Subject: [PATCH 07/70] . --- src/memory_manager.rs | 146 +++++++++++++++++++++++++++++++++--------- 1 file changed, 114 insertions(+), 32 deletions(-) diff --git a/src/memory_manager.rs b/src/memory_manager.rs index 48a2307d..5819abaa 100644 --- a/src/memory_manager.rs +++ b/src/memory_manager.rs @@ -43,15 +43,16 @@ use crate::{ read_struct, types::{Address, Bytes}, - write, write_struct, Memory, WASM_PAGE_SIZE, + vec, write, write_struct, Memory, WASM_PAGE_SIZE, }; -use std::cell::RefCell; -use std::cmp::min; use std::collections::BTreeMap; use std::rc::Rc; +use std::{cell::RefCell, collections::BTreeSet}; +use std::{cmp::min, collections::LinkedList}; const MAGIC: &[u8; 3] = b"MGR"; -const LAYOUT_VERSION: u8 = 1; +const LAYOUT_VERSION_V1: u8 = 1; +const LAYOUT_VERSION_V2: u8 = 2; // The maximum number of memories that can be created. const MAX_NUM_MEMORIES: u8 = 255; @@ -235,7 +236,7 @@ struct Header { _reserved: [u8; HEADER_RESERVED_BYTES], // The size of each individual memory that can be created by the memory manager. - memory_sizes_in_pages: [u64; MAX_NUM_MEMORIES as usize], + memory_sizes_in_buckets: [u16; MAX_NUM_MEMORIES as usize], } impl Header { @@ -277,13 +278,80 @@ struct MemoryManagerInner { bucket_size_in_pages: u16, - // An array storing the size (in pages) of each of the managed memories. - memory_sizes_in_pages: [u64; MAX_NUM_MEMORIES as usize], + // An array storing the size (in buckets) of each of the managed memories. + memory_sizes_in_buckets: [u16; MAX_NUM_MEMORIES as usize], // A map mapping each managed memory to the bucket ids that are allocated to it. memory_buckets: BTreeMap>, + + // A list of the all unallocated buckets. + unallocated_buckets: LinkedList, +} + +/* +#[derive(Clone)] +struct MemoryManagerInnerV2 { + memory: M, + + // The number of buckets that have been allocated. + allocated_buckets: u16, + + bucket_size_in_pages: u16, + + // An array storing the size (in pages) of each of the managed memories. + memory_sizes_in_buckets: [u64; MAX_NUM_MEMORIES as usize], + + // A map mapping each managed memory to the bucket ids that are allocated to it. + memory_buckets: BTreeMap>, + + unallocated_buckets: BTreeSet, } +impl MemoryManagerInnerV2 { + fn init(memory: M, bucket_size_in_pages: u16) -> Self { + if memory.size() == 0 { + // Memory is empty. Create a new map. + return Self::new(memory, bucket_size_in_pages); + } + } + + fn new(memory: M, bucket_size_in_pages: u16) -> Self { + let mem_mgr = Self { + memory, + allocated_buckets: 0, + memory_sizes_in_buckets: [0; MAX_NUM_MEMORIES as usize], + memory_buckets: BTreeMap::new(), + bucket_size_in_pages, + unallocated_buckets: BTreeSet::new(), + }; + + mem_mgr.save_header(); + + // Mark all the buckets as unallocated. + write( + &mem_mgr.memory, + bucket_allocations_address(BucketId(0)).get(), + &[UNALLOCATED_BUCKET_MARKER; MAX_NUM_BUCKETS as usize], + ); + + mem_mgr + } + + fn save_header(&self) { + let header = Header { + magic: *MAGIC, + version: LAYOUT_VERSION_V2, + num_allocated_buckets: self.allocated_buckets, + bucket_size_in_pages: self.bucket_size_in_pages, + _reserved: [0; HEADER_RESERVED_BYTES], + memory_sizes_in_pages: self.memory_sizes_in_buckets * self.bucket_size_in_pages, + }; + + write_struct(&header, Address::from(0), &self.memory); + } +} +*/ + impl MemoryManagerInner { fn init(memory: M, bucket_size_in_pages: u16) -> Self { if memory.size() == 0 { @@ -294,6 +362,7 @@ impl MemoryManagerInner { // Check if the magic in the memory corresponds to this object. let mut dst = vec![0; 3]; memory.read(0, &mut dst); + //here if dst != MAGIC { // No memory manager found. Create a new instance. MemoryManagerInner::new(memory, bucket_size_in_pages) @@ -307,9 +376,10 @@ impl MemoryManagerInner { let mem_mgr = Self { memory, allocated_buckets: 0, - memory_sizes_in_pages: [0; MAX_NUM_MEMORIES as usize], + memory_sizes_in_buckets: [0; MAX_NUM_MEMORIES as usize], memory_buckets: BTreeMap::new(), bucket_size_in_pages, + unallocated_buckets: LinkedList::new(), }; mem_mgr.save_header(); @@ -328,38 +398,50 @@ impl MemoryManagerInner { // Read the header from memory. let header: Header = read_struct(Address::from(0), &memory); assert_eq!(&header.magic, MAGIC, "Bad magic."); - assert_eq!(header.version, LAYOUT_VERSION, "Unsupported version."); - - let mut buckets = vec![0; MAX_NUM_BUCKETS as usize]; - memory.read(bucket_allocations_address(BucketId(0)).get(), &mut buckets); - - let mut memory_buckets = BTreeMap::new(); - for (bucket_idx, memory) in buckets.into_iter().enumerate() { - if memory != UNALLOCATED_BUCKET_MARKER { - memory_buckets - .entry(MemoryId(memory)) - .or_insert_with(Vec::new) - .push(BucketId(bucket_idx as u16)); + match header.version { + LAYOUT_VERSION_V1 => { + let mut buckets = vec![0; MAX_NUM_BUCKETS as usize]; + memory.read(bucket_allocations_address(BucketId(0)).get(), &mut buckets); + let unallocated: BTreeSet = + BTreeSet::from((0..header.num_allocated_buckets).collect()); + let mut memory_buckets = BTreeMap::new(); + for (bucket_idx, memory) in buckets.into_iter().enumerate() { + if memory != UNALLOCATED_BUCKET_MARKER { + memory_buckets + .entry(MemoryId(memory)) + .or_insert_with(Vec::new) + .push(BucketId(bucket_idx as u16)); + unallocated.remove(&(bucket_idx as u16)); + } + } + + let unallocated_buckets = LinkedList::new(); + for i in unallocated.iter() { + unallocated_buckets.push_back(BucketId(*i)); + } + + Self { + memory, + allocated_buckets: header.num_allocated_buckets, + bucket_size_in_pages: header.bucket_size_in_pages, + memory_sizes_in_buckets: header.memory_sizes_in_buckets, + memory_buckets, + unallocated_buckets, + } } - } - - Self { - memory, - allocated_buckets: header.num_allocated_buckets, - bucket_size_in_pages: header.bucket_size_in_pages, - memory_sizes_in_pages: header.memory_sizes_in_pages, - memory_buckets, + LAYOUT_VERSION_V2 => {} + _ => panic!("Unsupported version."), } } fn save_header(&self) { let header = Header { magic: *MAGIC, - version: LAYOUT_VERSION, + version: LAYOUT_VERSION_V2, num_allocated_buckets: self.allocated_buckets, bucket_size_in_pages: self.bucket_size_in_pages, _reserved: [0; HEADER_RESERVED_BYTES], - memory_sizes_in_pages: self.memory_sizes_in_pages, + memory_sizes_in_buckets: self.memory_sizes_in_buckets, }; write_struct(&header, Address::from(0), &self.memory); @@ -367,7 +449,7 @@ impl MemoryManagerInner { // Returns the size of a memory (in pages). fn memory_size(&self, id: MemoryId) -> u64 { - self.memory_sizes_in_pages[id.0 as usize] + self.memory_sizes_in_buckets[id.0 as usize] as u64 * self.bucket_size_in_pages as u64 } // Grows the memory with the given id by the given number of pages. @@ -415,7 +497,7 @@ impl MemoryManagerInner { } // Update the memory with the new size. - self.memory_sizes_in_pages[id.0 as usize] = new_size; + self.memory_sizes_in_buckets[id.0 as usize] = required_buckets as u16; // Update the header and return the old size. self.save_header(); From 786372d998c4d9222193207964cef87cb0448492 Mon Sep 17 00:00:00 2001 From: Dragoljub Duric Date: Wed, 28 Jun 2023 12:33:29 +0000 Subject: [PATCH 08/70] remove comment --- src/memory_manager.rs | 64 ------------------------------------------- 1 file changed, 64 deletions(-) diff --git a/src/memory_manager.rs b/src/memory_manager.rs index 5819abaa..e91ae5f6 100644 --- a/src/memory_manager.rs +++ b/src/memory_manager.rs @@ -288,70 +288,6 @@ struct MemoryManagerInner { unallocated_buckets: LinkedList, } -/* -#[derive(Clone)] -struct MemoryManagerInnerV2 { - memory: M, - - // The number of buckets that have been allocated. - allocated_buckets: u16, - - bucket_size_in_pages: u16, - - // An array storing the size (in pages) of each of the managed memories. - memory_sizes_in_buckets: [u64; MAX_NUM_MEMORIES as usize], - - // A map mapping each managed memory to the bucket ids that are allocated to it. - memory_buckets: BTreeMap>, - - unallocated_buckets: BTreeSet, -} - -impl MemoryManagerInnerV2 { - fn init(memory: M, bucket_size_in_pages: u16) -> Self { - if memory.size() == 0 { - // Memory is empty. Create a new map. - return Self::new(memory, bucket_size_in_pages); - } - } - - fn new(memory: M, bucket_size_in_pages: u16) -> Self { - let mem_mgr = Self { - memory, - allocated_buckets: 0, - memory_sizes_in_buckets: [0; MAX_NUM_MEMORIES as usize], - memory_buckets: BTreeMap::new(), - bucket_size_in_pages, - unallocated_buckets: BTreeSet::new(), - }; - - mem_mgr.save_header(); - - // Mark all the buckets as unallocated. - write( - &mem_mgr.memory, - bucket_allocations_address(BucketId(0)).get(), - &[UNALLOCATED_BUCKET_MARKER; MAX_NUM_BUCKETS as usize], - ); - - mem_mgr - } - - fn save_header(&self) { - let header = Header { - magic: *MAGIC, - version: LAYOUT_VERSION_V2, - num_allocated_buckets: self.allocated_buckets, - bucket_size_in_pages: self.bucket_size_in_pages, - _reserved: [0; HEADER_RESERVED_BYTES], - memory_sizes_in_pages: self.memory_sizes_in_buckets * self.bucket_size_in_pages, - }; - - write_struct(&header, Address::from(0), &self.memory); - } -} -*/ - impl MemoryManagerInner { fn init(memory: M, bucket_size_in_pages: u16) -> Self { if memory.size() == 0 { From 49c0eee5408a1b8ef79ec64e72c69a5462437837 Mon Sep 17 00:00:00 2001 From: Dragoljub Duric Date: Thu, 6 Jul 2023 09:39:16 +0000 Subject: [PATCH 09/70] Add u16 <-> 15 bits compression --- Cargo.lock | 1 + Cargo.toml | 3 +++ src/lib.rs | 3 +-- src/memory_manager.rs | 58 +++++++++++++++++++++++++++++++++++++++---- 4 files changed, 58 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a6146a27..0db895ca 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -561,6 +561,7 @@ dependencies = [ name = "ic-stable-structures" version = "0.5.4" dependencies = [ + "bit-vec", "criterion", "ic-cdk", "ic-cdk-macros", diff --git a/Cargo.toml b/Cargo.toml index 0f7bd949..3e20bb77 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,6 +11,9 @@ keywords = ["internet-computer", "dfinity", "stable-structures"] include = ["src", "Cargo.toml", "LICENSE", "README.md"] repository = "https://github.com/dfinity/stable-structures" +[dependencies] +bit-vec = "0.6" + [dev-dependencies] criterion = "0.4.0" ic-cdk = "0.6.8" diff --git a/src/lib.rs b/src/lib.rs index 53ab7725..b57027b6 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -95,8 +95,7 @@ impl Display for GrowFailed { write!( f, "Failed to grow memory: current size={}, delta={}", - self.current_size, - self.delta + self.current_size, self.delta ) } } diff --git a/src/memory_manager.rs b/src/memory_manager.rs index e91ae5f6..0a262b0c 100644 --- a/src/memory_manager.rs +++ b/src/memory_manager.rs @@ -40,6 +40,8 @@ //! memory_1.read(0, &mut bytes); //! assert_eq!(bytes, vec![4, 5, 6]); //! ``` +use bit_vec::BitVec; + use crate::{ read_struct, types::{Address, Bytes}, @@ -240,6 +242,9 @@ struct Header { } impl Header { + fn size_V1() -> Bytes { + Bytes::new(3 + 1 + 2 + 2 + 32 + 255 * 8) + } fn size() -> Bytes { Bytes::new(core::mem::size_of::() as u64) } @@ -323,7 +328,7 @@ impl MemoryManagerInner { // Mark all the buckets as unallocated. write( &mem_mgr.memory, - bucket_allocations_address(BucketId(0)).get(), + bucket_allocations_address_V1(BucketId(0)).get(), &[UNALLOCATED_BUCKET_MARKER; MAX_NUM_BUCKETS as usize], ); @@ -337,7 +342,10 @@ impl MemoryManagerInner { match header.version { LAYOUT_VERSION_V1 => { let mut buckets = vec![0; MAX_NUM_BUCKETS as usize]; - memory.read(bucket_allocations_address(BucketId(0)).get(), &mut buckets); + memory.read( + bucket_allocations_address_V1(BucketId(0)).get(), + &mut buckets, + ); let unallocated: BTreeSet = BTreeSet::from((0..header.num_allocated_buckets).collect()); let mut memory_buckets = BTreeMap::new(); @@ -365,7 +373,17 @@ impl MemoryManagerInner { unallocated_buckets, } } - LAYOUT_VERSION_V2 => {} + LAYOUT_VERSION_V2 => { + let mut buckets = vec![0; MAX_NUM_BUCKETS as usize]; + memory.read( + bucket_allocations_address_V2(BucketId(0)).get(), + &mut buckets, + ); + + let unallocated: BTreeSet = + BTreeSet::from((0..header.num_allocated_buckets).collect()); + let mut memory_buckets = BTreeMap::new(); + } _ => panic!("Unsupported version."), } } @@ -414,7 +432,7 @@ impl MemoryManagerInner { // Write in stable store that this bucket belongs to the memory with the provided `id`. write( &self.memory, - bucket_allocations_address(new_bucket_id).get(), + bucket_allocations_address_V1(new_bucket_id).get(), &[id.0], ); @@ -503,6 +521,32 @@ impl MemoryManagerInner { fn free(&mut self, id: MemoryId) {} } +fn bucket_indexes_to_bytes(input: &[u16]) -> Vec { + let mut bit_vec = BitVec::new(); + for bucket_ind in input { + let mut bit_vec_temp = BitVec::from_bytes(&bucket_ind.to_be_bytes()).split_off(1); + bit_vec.append(&mut bit_vec_temp); + } + bit_vec.to_bytes() +} + +fn bytes_to_bucket_indexes(input: &[u8]) -> Vec { + const BUCKET_IND_LEN: usize = 15; + let mut vec_u16 = vec![]; + let bit_vec = BitVec::from_bytes(input); + for i in 0..bit_vec.len() / BUCKET_IND_LEN { + let mut bucket_ind: u16 = 0; + for j in 0..BUCKET_IND_LEN { + let next_bit = BUCKET_IND_LEN * i + j; + bucket_ind = bucket_ind << 1; + if bit_vec.get(next_bit) == Some(true) { + bucket_ind = bucket_ind | 1; + } + } + vec_u16.push(bucket_ind); + } + vec_u16 +} struct Segment { address: Address, length: Bytes, @@ -604,7 +648,11 @@ impl MemoryId { #[derive(Clone, Copy, Debug, PartialEq)] struct BucketId(u16); -fn bucket_allocations_address(id: BucketId) -> Address { +fn bucket_allocations_address_V1(id: BucketId) -> Address { + Address::from(0) + Header::size_V1() + Bytes::from(id.0) +} + +fn bucket_allocations_address_V2(id: BucketId) -> Address { Address::from(0) + Header::size() + Bytes::from(id.0) } From f3175fdc0d9eaf572b7c76bec452c85550cdb070 Mon Sep 17 00:00:00 2001 From: Dragoljub Duric Date: Thu, 6 Jul 2023 11:27:44 +0000 Subject: [PATCH 10/70] . --- src/memory_manager.rs | 47 ++++++++++++++++++++++++++++++++++--------- 1 file changed, 38 insertions(+), 9 deletions(-) diff --git a/src/memory_manager.rs b/src/memory_manager.rs index 0a262b0c..cb5794ed 100644 --- a/src/memory_manager.rs +++ b/src/memory_manager.rs @@ -222,6 +222,21 @@ impl MemoryManager { } } +#[repr(C, packed)] +struct HeaderV1 { + magic: [u8; 3], + version: u8, + // The number of buckets allocated by the memory manager. + num_allocated_buckets: u16, + // The size of a bucket in Wasm pages. + bucket_size_in_pages: u16, + // Reserved bytes for future extensions + _reserved: [u8; HEADER_RESERVED_BYTES], + + // The size of each individual memory that can be created by the memory manager. + memory_sizes_in_pages: [u64; MAX_NUM_MEMORIES as usize], +} + #[repr(C, packed)] struct Header { magic: [u8; 3], @@ -241,10 +256,13 @@ struct Header { memory_sizes_in_buckets: [u16; MAX_NUM_MEMORIES as usize], } -impl Header { - fn size_V1() -> Bytes { - Bytes::new(3 + 1 + 2 + 2 + 32 + 255 * 8) +impl HeaderV1 { + fn size() -> Bytes { + Bytes::new(core::mem::size_of::() as u64) } +} + +impl Header { fn size() -> Bytes { Bytes::new(core::mem::size_of::() as u64) } @@ -337,7 +355,7 @@ impl MemoryManagerInner { fn load(memory: M) -> Self { // Read the header from memory. - let header: Header = read_struct(Address::from(0), &memory); + let header: HeaderV1 = read_struct(Address::from(0), &memory); assert_eq!(&header.magic, MAGIC, "Bad magic."); match header.version { LAYOUT_VERSION_V1 => { @@ -346,8 +364,8 @@ impl MemoryManagerInner { bucket_allocations_address_V1(BucketId(0)).get(), &mut buckets, ); - let unallocated: BTreeSet = - BTreeSet::from((0..header.num_allocated_buckets).collect()); + let all_buckets: Vec = (0..header.num_allocated_buckets).collect(); + let unallocated: BTreeSet = BTreeSet::from(all_buckets); let mut memory_buckets = BTreeMap::new(); for (bucket_idx, memory) in buckets.into_iter().enumerate() { if memory != UNALLOCATED_BUCKET_MARKER { @@ -364,17 +382,28 @@ impl MemoryManagerInner { unallocated_buckets.push_back(BucketId(*i)); } + let memory_sizes_in_buckets: [u16; 255]; + let mut i = 0; + for memory_size in header.memory_sizes_in_pages { + let size_in_buckets = (memory_size + header.bucket_size_in_pages as u64 - 1) + / header.bucket_size_in_pages as u64; + memory_sizes_in_buckets[i] = size_in_buckets as u16; + i += 1; + } + Self { memory, allocated_buckets: header.num_allocated_buckets, bucket_size_in_pages: header.bucket_size_in_pages, - memory_sizes_in_buckets: header.memory_sizes_in_buckets, + memory_sizes_in_buckets, memory_buckets, unallocated_buckets, } } LAYOUT_VERSION_V2 => { - let mut buckets = vec![0; MAX_NUM_BUCKETS as usize]; + let header: Header = read_struct(Address::from(0), &memory); + let size_of_buckets_ind_in_bytes = (MAX_NUM_BUCKETS * 15 + 15) / 16; + let mut buckets = vec![0; size_of_buckets_ind_in_bytes as usize]; memory.read( bucket_allocations_address_V2(BucketId(0)).get(), &mut buckets, @@ -649,7 +678,7 @@ impl MemoryId { struct BucketId(u16); fn bucket_allocations_address_V1(id: BucketId) -> Address { - Address::from(0) + Header::size_V1() + Bytes::from(id.0) + Address::from(0) + HeaderV1::size() + Bytes::from(id.0) } fn bucket_allocations_address_V2(id: BucketId) -> Address { From 4244659e902d799e603f7fafbef53c836d89a32d Mon Sep 17 00:00:00 2001 From: Dragoljub Duric Date: Thu, 6 Jul 2023 14:52:55 +0000 Subject: [PATCH 11/70] read header V2 --- src/memory_manager.rs | 41 +++++++++++++++++++++++++++++++++++------ 1 file changed, 35 insertions(+), 6 deletions(-) diff --git a/src/memory_manager.rs b/src/memory_manager.rs index cb5794ed..918be14a 100644 --- a/src/memory_manager.rs +++ b/src/memory_manager.rs @@ -364,8 +364,7 @@ impl MemoryManagerInner { bucket_allocations_address_V1(BucketId(0)).get(), &mut buckets, ); - let all_buckets: Vec = (0..header.num_allocated_buckets).collect(); - let unallocated: BTreeSet = BTreeSet::from(all_buckets); + let mut unallocated: BTreeSet = (0..header.num_allocated_buckets).collect(); let mut memory_buckets = BTreeMap::new(); for (bucket_idx, memory) in buckets.into_iter().enumerate() { if memory != UNALLOCATED_BUCKET_MARKER { @@ -377,12 +376,12 @@ impl MemoryManagerInner { } } - let unallocated_buckets = LinkedList::new(); + let mut unallocated_buckets = LinkedList::new(); for i in unallocated.iter() { unallocated_buckets.push_back(BucketId(*i)); } - let memory_sizes_in_buckets: [u16; 255]; + let mut memory_sizes_in_buckets = [0; 255]; let mut i = 0; for memory_size in header.memory_sizes_in_pages { let size_in_buckets = (memory_size + header.bucket_size_in_pages as u64 - 1) @@ -409,9 +408,39 @@ impl MemoryManagerInner { &mut buckets, ); - let unallocated: BTreeSet = - BTreeSet::from((0..header.num_allocated_buckets).collect()); + let buckets_decompressed = bytes_to_bucket_indexes(&buckets); + + let mut unallocated: BTreeSet = (0..header.num_allocated_buckets).collect(); let mut memory_buckets = BTreeMap::new(); + + let mut j = 0; + for (memory, memory_size) in header.memory_sizes_in_buckets.into_iter().enumerate() + { + let mut vec_buckets = vec![]; + for _ in 0..memory_size { + let bucket = BucketId(buckets_decompressed[j]); + vec_buckets.push(bucket); + unallocated.remove(&bucket.0); + j += 1; + } + memory_buckets + .entry(MemoryId(memory as u8)) + .or_insert(vec_buckets); + } + + let mut unallocated_buckets = LinkedList::new(); + for i in unallocated.iter() { + unallocated_buckets.push_back(BucketId(*i)); + } + + Self { + memory, + allocated_buckets: header.num_allocated_buckets, + bucket_size_in_pages: header.bucket_size_in_pages, + memory_sizes_in_buckets: header.memory_sizes_in_buckets, + memory_buckets, + unallocated_buckets, + } } _ => panic!("Unsupported version."), } From 7144cdd494f53c1dfa62598df29668f8c9e5d812 Mon Sep 17 00:00:00 2001 From: Dragoljub Duric Date: Tue, 11 Jul 2023 09:39:30 +0000 Subject: [PATCH 12/70] revert changes --- src/lib.rs | 1 + src/memory_manager.rs | 96 +++++++++++-------------------------------- 2 files changed, 25 insertions(+), 72 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index b57027b6..cb33dc68 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -39,6 +39,7 @@ const WASM_PAGE_SIZE: u64 = 65536; /// The maximum number of stable memory pages a canister can address. pub const MAX_PAGES: u64 = u64::MAX / WASM_PAGE_SIZE; +// Add free method pub trait Memory { /// Returns the current size of the stable memory in WebAssembly /// pages. (One WebAssembly page is 64Ki bytes.) diff --git a/src/memory_manager.rs b/src/memory_manager.rs index 918be14a..a15a2728 100644 --- a/src/memory_manager.rs +++ b/src/memory_manager.rs @@ -45,7 +45,7 @@ use bit_vec::BitVec; use crate::{ read_struct, types::{Address, Bytes}, - vec, write, write_struct, Memory, WASM_PAGE_SIZE, + write, write_struct, Memory, WASM_PAGE_SIZE, }; use std::collections::BTreeMap; use std::rc::Rc; @@ -145,13 +145,13 @@ const HEADER_RESERVED_BYTES: usize = 32; /// -------------------------------------------------- /// Reserved space ↕ 32 bytes /// -------------------------------------------------- -/// Size of memory 0 (in buckets) = k0 ↕ 2 bytes +/// Size of memory 0 (in pages) ↕ 8 bytes /// -------------------------------------------------- -/// Size of memory 1 (in buckets) = k1 ↕ 2 bytes +/// Size of memory 1 (in pages) ↕ 8 bytes /// -------------------------------------------------- /// ... /// -------------------------------------------------- -/// Size of memory 254 (in buckets) = k254 ↕ 2 bytes +/// Size of memory 254 (in pages) ↕ 8 bytes /// -------------------------------------------------- <- IDs of buckets /// Bucket 1 ID belonging to memory 0 ↕ 15 bits /// -------------------------------------------------- @@ -179,7 +179,7 @@ const HEADER_RESERVED_BYTES: usize = 32; /// -------------------------------------------------- /// Bucket k254 ID belonging to memory 254 ↕ 15 bits /// -------------------------------------------------- -/// Unallocated space ↕ 3'546 bytes +/// Unallocated space ↕ 2'016 bytes /// -------------------------------------------------- <- Buckets (Page 1) /// Bucket 1 ↕ N pages /// -------------------------------------------------- <- Page N + 1 @@ -222,44 +222,19 @@ impl MemoryManager { } } -#[repr(C, packed)] -struct HeaderV1 { - magic: [u8; 3], - version: u8, - // The number of buckets allocated by the memory manager. - num_allocated_buckets: u16, - // The size of a bucket in Wasm pages. - bucket_size_in_pages: u16, - // Reserved bytes for future extensions - _reserved: [u8; HEADER_RESERVED_BYTES], - - // The size of each individual memory that can be created by the memory manager. - memory_sizes_in_pages: [u64; MAX_NUM_MEMORIES as usize], -} - #[repr(C, packed)] struct Header { magic: [u8; 3], - version: u8, - // The number of buckets allocated by the memory manager. num_allocated_buckets: u16, - // The size of a bucket in Wasm pages. bucket_size_in_pages: u16, - // Reserved bytes for future extensions _reserved: [u8; HEADER_RESERVED_BYTES], // The size of each individual memory that can be created by the memory manager. - memory_sizes_in_buckets: [u16; MAX_NUM_MEMORIES as usize], -} - -impl HeaderV1 { - fn size() -> Bytes { - Bytes::new(core::mem::size_of::() as u64) - } + memory_sizes_in_pages: [u64; MAX_NUM_MEMORIES as usize], } impl Header { @@ -301,8 +276,8 @@ struct MemoryManagerInner { bucket_size_in_pages: u16, - // An array storing the size (in buckets) of each of the managed memories. - memory_sizes_in_buckets: [u16; MAX_NUM_MEMORIES as usize], + // An array storing the size (in pages) of each of the managed memories. + memory_sizes_in_pages: [u64; MAX_NUM_MEMORIES as usize], // A map mapping each managed memory to the bucket ids that are allocated to it. memory_buckets: BTreeMap>, @@ -321,7 +296,6 @@ impl MemoryManagerInner { // Check if the magic in the memory corresponds to this object. let mut dst = vec![0; 3]; memory.read(0, &mut dst); - //here if dst != MAGIC { // No memory manager found. Create a new instance. MemoryManagerInner::new(memory, bucket_size_in_pages) @@ -335,7 +309,7 @@ impl MemoryManagerInner { let mem_mgr = Self { memory, allocated_buckets: 0, - memory_sizes_in_buckets: [0; MAX_NUM_MEMORIES as usize], + memory_sizes_in_pages: [0; MAX_NUM_MEMORIES as usize], memory_buckets: BTreeMap::new(), bucket_size_in_pages, unallocated_buckets: LinkedList::new(), @@ -343,27 +317,17 @@ impl MemoryManagerInner { mem_mgr.save_header(); - // Mark all the buckets as unallocated. - write( - &mem_mgr.memory, - bucket_allocations_address_V1(BucketId(0)).get(), - &[UNALLOCATED_BUCKET_MARKER; MAX_NUM_BUCKETS as usize], - ); - mem_mgr } fn load(memory: M) -> Self { // Read the header from memory. - let header: HeaderV1 = read_struct(Address::from(0), &memory); + let header: Header = read_struct(Address::from(0), &memory); assert_eq!(&header.magic, MAGIC, "Bad magic."); match header.version { LAYOUT_VERSION_V1 => { let mut buckets = vec![0; MAX_NUM_BUCKETS as usize]; - memory.read( - bucket_allocations_address_V1(BucketId(0)).get(), - &mut buckets, - ); + memory.read(bucket_allocations_address(BucketId(0)).get(), &mut buckets); let mut unallocated: BTreeSet = (0..header.num_allocated_buckets).collect(); let mut memory_buckets = BTreeMap::new(); for (bucket_idx, memory) in buckets.into_iter().enumerate() { @@ -381,20 +345,11 @@ impl MemoryManagerInner { unallocated_buckets.push_back(BucketId(*i)); } - let mut memory_sizes_in_buckets = [0; 255]; - let mut i = 0; - for memory_size in header.memory_sizes_in_pages { - let size_in_buckets = (memory_size + header.bucket_size_in_pages as u64 - 1) - / header.bucket_size_in_pages as u64; - memory_sizes_in_buckets[i] = size_in_buckets as u16; - i += 1; - } - Self { memory, allocated_buckets: header.num_allocated_buckets, bucket_size_in_pages: header.bucket_size_in_pages, - memory_sizes_in_buckets, + memory_sizes_in_pages: header.memory_sizes_in_pages, memory_buckets, unallocated_buckets, } @@ -403,10 +358,7 @@ impl MemoryManagerInner { let header: Header = read_struct(Address::from(0), &memory); let size_of_buckets_ind_in_bytes = (MAX_NUM_BUCKETS * 15 + 15) / 16; let mut buckets = vec![0; size_of_buckets_ind_in_bytes as usize]; - memory.read( - bucket_allocations_address_V2(BucketId(0)).get(), - &mut buckets, - ); + memory.read(bucket_allocations_address(BucketId(0)).get(), &mut buckets); let buckets_decompressed = bytes_to_bucket_indexes(&buckets); @@ -414,8 +366,12 @@ impl MemoryManagerInner { let mut memory_buckets = BTreeMap::new(); let mut j = 0; - for (memory, memory_size) in header.memory_sizes_in_buckets.into_iter().enumerate() + for (memory, memory_size_in_pages) in + header.memory_sizes_in_pages.into_iter().enumerate() { + let memory_size = (memory_size_in_pages + header.bucket_size_in_pages as u64 + - 1) + / header.bucket_size_in_pages as u64; let mut vec_buckets = vec![]; for _ in 0..memory_size { let bucket = BucketId(buckets_decompressed[j]); @@ -437,7 +393,7 @@ impl MemoryManagerInner { memory, allocated_buckets: header.num_allocated_buckets, bucket_size_in_pages: header.bucket_size_in_pages, - memory_sizes_in_buckets: header.memory_sizes_in_buckets, + memory_sizes_in_pages: header.memory_sizes_in_pages, memory_buckets, unallocated_buckets, } @@ -453,7 +409,7 @@ impl MemoryManagerInner { num_allocated_buckets: self.allocated_buckets, bucket_size_in_pages: self.bucket_size_in_pages, _reserved: [0; HEADER_RESERVED_BYTES], - memory_sizes_in_buckets: self.memory_sizes_in_buckets, + memory_sizes_in_pages: self.memory_sizes_in_pages, }; write_struct(&header, Address::from(0), &self.memory); @@ -461,7 +417,7 @@ impl MemoryManagerInner { // Returns the size of a memory (in pages). fn memory_size(&self, id: MemoryId) -> u64 { - self.memory_sizes_in_buckets[id.0 as usize] as u64 * self.bucket_size_in_pages as u64 + self.memory_sizes_in_pages[id.0 as usize] } // Grows the memory with the given id by the given number of pages. @@ -490,7 +446,7 @@ impl MemoryManagerInner { // Write in stable store that this bucket belongs to the memory with the provided `id`. write( &self.memory, - bucket_allocations_address_V1(new_bucket_id).get(), + bucket_allocations_address(new_bucket_id).get(), &[id.0], ); @@ -509,7 +465,7 @@ impl MemoryManagerInner { } // Update the memory with the new size. - self.memory_sizes_in_buckets[id.0 as usize] = required_buckets as u16; + self.memory_sizes_in_pages[id.0 as usize] = new_size; // Update the header and return the old size. self.save_header(); @@ -706,11 +662,7 @@ impl MemoryId { #[derive(Clone, Copy, Debug, PartialEq)] struct BucketId(u16); -fn bucket_allocations_address_V1(id: BucketId) -> Address { - Address::from(0) + HeaderV1::size() + Bytes::from(id.0) -} - -fn bucket_allocations_address_V2(id: BucketId) -> Address { +fn bucket_allocations_address(id: BucketId) -> Address { Address::from(0) + Header::size() + Bytes::from(id.0) } From 4c47311ca54477b95ed6c9a654b30f92f7c310c3 Mon Sep 17 00:00:00 2001 From: Dragoljub Duric Date: Tue, 11 Jul 2023 09:40:50 +0000 Subject: [PATCH 13/70] format --- src/memory_manager.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/memory_manager.rs b/src/memory_manager.rs index a15a2728..5e6138d4 100644 --- a/src/memory_manager.rs +++ b/src/memory_manager.rs @@ -225,11 +225,15 @@ impl MemoryManager { #[repr(C, packed)] struct Header { magic: [u8; 3], + version: u8, + // The number of buckets allocated by the memory manager. num_allocated_buckets: u16, + // The size of a bucket in Wasm pages. bucket_size_in_pages: u16, + // Reserved bytes for future extensions _reserved: [u8; HEADER_RESERVED_BYTES], From 8827389bf8e2ce5ef8c84eab88b2ddede0cdd1a8 Mon Sep 17 00:00:00 2001 From: Dragoljub Duric Date: Tue, 11 Jul 2023 11:08:14 +0000 Subject: [PATCH 14/70] . --- src/memory_manager.rs | 29 ++++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/src/memory_manager.rs b/src/memory_manager.rs index 5e6138d4..e93fe4f2 100644 --- a/src/memory_manager.rs +++ b/src/memory_manager.rs @@ -424,6 +424,12 @@ impl MemoryManagerInner { self.memory_sizes_in_pages[id.0 as usize] } + // Returns the size of a memory (in buckets). + fn memory_size_in_buckets(&self, id: MemoryId) -> u64 { + (self.memory_size(id) + self.bucket_size_in_pages as u64 - 1) + / self.bucket_size_in_pages as u64 + } + // Grows the memory with the given id by the given number of pages. fn grow(&mut self, id: MemoryId, pages: u64) -> i64 { // Compute how many additional buckets are needed. @@ -440,7 +446,10 @@ impl MemoryManagerInner { // Allocate new buckets as needed. for _ in 0..new_buckets_needed { - let new_bucket_id = BucketId(self.allocated_buckets); + let new_bucket_id = match self.unallocated_buckets.pop_front() { + Some(t) => t, + None => panic!("{id:?}: grow failed"), + }; self.memory_buckets .entry(id) @@ -450,8 +459,8 @@ impl MemoryManagerInner { // Write in stable store that this bucket belongs to the memory with the provided `id`. write( &self.memory, - bucket_allocations_address(new_bucket_id).get(), - &[id.0], + bucket_allocations_address(BucketId(0)).get(), + self.get_bucket_ids_in_bytes().as_ref(), ); self.allocated_buckets += 1; @@ -476,6 +485,15 @@ impl MemoryManagerInner { old_size as i64 } + fn get_bucket_ids_in_bytes(&self) -> Vec { + let mut vector_bytes = vec![]; + for memory in self.memory_buckets.iter() { + let mut req = bucket_indexes_to_bytes(memory.1); + vector_bytes.append(&mut req); + } + vector_bytes + } + fn write(&self, id: MemoryId, offset: u64, src: &[u8]) { if (offset + src.len() as u64) > self.memory_size(id) * WASM_PAGE_SIZE { panic!("{id:?}: write out of bounds"); @@ -539,9 +557,10 @@ impl MemoryManagerInner { fn free(&mut self, id: MemoryId) {} } -fn bucket_indexes_to_bytes(input: &[u16]) -> Vec { +fn bucket_indexes_to_bytes(input: &[BucketId]) -> Vec { let mut bit_vec = BitVec::new(); - for bucket_ind in input { + for bucket in input { + let bucket_ind = bucket.0; let mut bit_vec_temp = BitVec::from_bytes(&bucket_ind.to_be_bytes()).split_off(1); bit_vec.append(&mut bit_vec_temp); } From c7629332a0f084cf4d7be79e3131e8289b88c7d3 Mon Sep 17 00:00:00 2001 From: Dragoljub Duric Date: Tue, 11 Jul 2023 11:37:00 +0000 Subject: [PATCH 15/70] . --- src/memory_manager.rs | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/memory_manager.rs b/src/memory_manager.rs index e93fe4f2..ed0a8534 100644 --- a/src/memory_manager.rs +++ b/src/memory_manager.rs @@ -554,7 +554,19 @@ impl MemoryManagerInner { (num_pages + self.bucket_size_in_pages as u64 - 1) / self.bucket_size_in_pages as u64 } - fn free(&mut self, id: MemoryId) {} + fn free(&mut self, id: MemoryId) { + self.memory_sizes_in_pages[id.0 as usize] = 0; + let buckets = self.memory_buckets.remove(&id); + match buckets { + Some(vec_buckets) => { + for bucket in vec_buckets { + self.unallocated_buckets.push_back(bucket); + self.allocated_buckets -= 1; + } + } + None => (), + }; + } } fn bucket_indexes_to_bytes(input: &[BucketId]) -> Vec { From 0b187f8cfafe24fa9f4215901a07e8e7c54e2da9 Mon Sep 17 00:00:00 2001 From: Dragoljub Duric Date: Tue, 11 Jul 2023 11:46:55 +0000 Subject: [PATCH 16/70] . --- src/memory_manager.rs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/memory_manager.rs b/src/memory_manager.rs index ed0a8534..23960cc4 100644 --- a/src/memory_manager.rs +++ b/src/memory_manager.rs @@ -290,6 +290,14 @@ struct MemoryManagerInner { unallocated_buckets: LinkedList, } +fn get_all_buckets() -> LinkedList { + let mut list = LinkedList::new(); + for i in 0..MAX_NUM_BUCKETS { + list.push_back(BucketId(i as u16)); + } + list +} + impl MemoryManagerInner { fn init(memory: M, bucket_size_in_pages: u16) -> Self { if memory.size() == 0 { @@ -316,7 +324,7 @@ impl MemoryManagerInner { memory_sizes_in_pages: [0; MAX_NUM_MEMORIES as usize], memory_buckets: BTreeMap::new(), bucket_size_in_pages, - unallocated_buckets: LinkedList::new(), + unallocated_buckets: get_all_buckets(), }; mem_mgr.save_header(); From a4923bbefea6efc800f8c0ff672dfbdac5b432d2 Mon Sep 17 00:00:00 2001 From: Dragoljub Duric Date: Tue, 11 Jul 2023 12:38:14 +0000 Subject: [PATCH 17/70] . --- src/lib.rs | 1 - src/memory_manager.rs | 40 +++++++++++++++++----------------------- 2 files changed, 17 insertions(+), 24 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index cb33dc68..b57027b6 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -39,7 +39,6 @@ const WASM_PAGE_SIZE: u64 = 65536; /// The maximum number of stable memory pages a canister can address. pub const MAX_PAGES: u64 = u64::MAX / WASM_PAGE_SIZE; -// Add free method pub trait Memory { /// Returns the current size of the stable memory in WebAssembly /// pages. (One WebAssembly page is 64Ki bytes.) diff --git a/src/memory_manager.rs b/src/memory_manager.rs index 23960cc4..601eb6e9 100644 --- a/src/memory_manager.rs +++ b/src/memory_manager.rs @@ -340,7 +340,7 @@ impl MemoryManagerInner { LAYOUT_VERSION_V1 => { let mut buckets = vec![0; MAX_NUM_BUCKETS as usize]; memory.read(bucket_allocations_address(BucketId(0)).get(), &mut buckets); - let mut unallocated: BTreeSet = (0..header.num_allocated_buckets).collect(); + let mut unallocated: BTreeSet = (0..MAX_NUM_BUCKETS as u16).collect(); let mut memory_buckets = BTreeMap::new(); for (bucket_idx, memory) in buckets.into_iter().enumerate() { if memory != UNALLOCATED_BUCKET_MARKER { @@ -374,7 +374,7 @@ impl MemoryManagerInner { let buckets_decompressed = bytes_to_bucket_indexes(&buckets); - let mut unallocated: BTreeSet = (0..header.num_allocated_buckets).collect(); + let mut unallocated: BTreeSet = (0..MAX_NUM_BUCKETS as u16).collect(); let mut memory_buckets = BTreeMap::new(); let mut j = 0; @@ -464,13 +464,6 @@ impl MemoryManagerInner { .or_insert_with(Vec::new) .push(new_bucket_id); - // Write in stable store that this bucket belongs to the memory with the provided `id`. - write( - &self.memory, - bucket_allocations_address(BucketId(0)).get(), - self.get_bucket_ids_in_bytes().as_ref(), - ); - self.allocated_buckets += 1; } @@ -490,16 +483,26 @@ impl MemoryManagerInner { // Update the header and return the old size. self.save_header(); + + // Write in stable store that this bucket belongs to the memory with the provided `id`. + write( + &self.memory, + bucket_allocations_address(BucketId(0)).get(), + self.get_bucket_ids_in_bytes().as_ref(), + ); old_size as i64 } fn get_bucket_ids_in_bytes(&self) -> Vec { - let mut vector_bytes = vec![]; + let mut bit_vec = BitVec::new(); for memory in self.memory_buckets.iter() { - let mut req = bucket_indexes_to_bytes(memory.1); - vector_bytes.append(&mut req); + for bucket in memory.1 { + let bucket_ind = bucket.0; + let mut bit_vec_temp = BitVec::from_bytes(&bucket_ind.to_be_bytes()).split_off(1); + bit_vec.append(&mut bit_vec_temp); + } } - vector_bytes + bit_vec.to_bytes() } fn write(&self, id: MemoryId, offset: u64, src: &[u8]) { @@ -518,6 +521,7 @@ impl MemoryManagerInner { } } + //here fn read(&self, id: MemoryId, offset: u64, dst: &mut [u8]) { if (offset + dst.len() as u64) > self.memory_size(id) * WASM_PAGE_SIZE { panic!("{id:?}: read out of bounds"); @@ -577,16 +581,6 @@ impl MemoryManagerInner { } } -fn bucket_indexes_to_bytes(input: &[BucketId]) -> Vec { - let mut bit_vec = BitVec::new(); - for bucket in input { - let bucket_ind = bucket.0; - let mut bit_vec_temp = BitVec::from_bytes(&bucket_ind.to_be_bytes()).split_off(1); - bit_vec.append(&mut bit_vec_temp); - } - bit_vec.to_bytes() -} - fn bytes_to_bucket_indexes(input: &[u8]) -> Vec { const BUCKET_IND_LEN: usize = 15; let mut vec_u16 = vec![]; From 7bdb67b46167e394c564fbf6f1d9980c41b5ffc0 Mon Sep 17 00:00:00 2001 From: Dragoljub Duric Date: Tue, 11 Jul 2023 12:41:34 +0000 Subject: [PATCH 18/70] . --- src/memory_manager.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/memory_manager.rs b/src/memory_manager.rs index 601eb6e9..4d49465b 100644 --- a/src/memory_manager.rs +++ b/src/memory_manager.rs @@ -521,7 +521,6 @@ impl MemoryManagerInner { } } - //here fn read(&self, id: MemoryId, offset: u64, dst: &mut [u8]) { if (offset + dst.len() as u64) > self.memory_size(id) * WASM_PAGE_SIZE { panic!("{id:?}: read out of bounds"); From 211f1140e2bb18f944a0770ad9f3242ceaf74d96 Mon Sep 17 00:00:00 2001 From: Dragoljub Duric Date: Tue, 11 Jul 2023 12:43:59 +0000 Subject: [PATCH 19/70] . --- src/memory_manager.rs | 23 +++++++---------------- 1 file changed, 7 insertions(+), 16 deletions(-) diff --git a/src/memory_manager.rs b/src/memory_manager.rs index 4d49465b..37bd4546 100644 --- a/src/memory_manager.rs +++ b/src/memory_manager.rs @@ -432,12 +432,6 @@ impl MemoryManagerInner { self.memory_sizes_in_pages[id.0 as usize] } - // Returns the size of a memory (in buckets). - fn memory_size_in_buckets(&self, id: MemoryId) -> u64 { - (self.memory_size(id) + self.bucket_size_in_pages as u64 - 1) - / self.bucket_size_in_pages as u64 - } - // Grows the memory with the given id by the given number of pages. fn grow(&mut self, id: MemoryId, pages: u64) -> i64 { // Compute how many additional buckets are needed. @@ -568,15 +562,12 @@ impl MemoryManagerInner { fn free(&mut self, id: MemoryId) { self.memory_sizes_in_pages[id.0 as usize] = 0; let buckets = self.memory_buckets.remove(&id); - match buckets { - Some(vec_buckets) => { - for bucket in vec_buckets { - self.unallocated_buckets.push_back(bucket); - self.allocated_buckets -= 1; - } + if let Some(vec_buckets) = buckets { + for bucket in vec_buckets { + self.unallocated_buckets.push_back(bucket); + self.allocated_buckets -= 1; } - None => (), - }; + } } } @@ -588,9 +579,9 @@ fn bytes_to_bucket_indexes(input: &[u8]) -> Vec { let mut bucket_ind: u16 = 0; for j in 0..BUCKET_IND_LEN { let next_bit = BUCKET_IND_LEN * i + j; - bucket_ind = bucket_ind << 1; + bucket_ind <<= 1; if bit_vec.get(next_bit) == Some(true) { - bucket_ind = bucket_ind | 1; + bucket_ind |= 1; } } vec_u16.push(bucket_ind); From 608862c65732c77f0e2b150d0fada49bf2b44dc8 Mon Sep 17 00:00:00 2001 From: Dragoljub Duric Date: Tue, 11 Jul 2023 12:45:51 +0000 Subject: [PATCH 20/70] . --- src/memory_manager.rs | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/src/memory_manager.rs b/src/memory_manager.rs index 37bd4546..403fcd0f 100644 --- a/src/memory_manager.rs +++ b/src/memory_manager.rs @@ -153,32 +153,26 @@ const HEADER_RESERVED_BYTES: usize = 32; /// -------------------------------------------------- /// Size of memory 254 (in pages) ↕ 8 bytes /// -------------------------------------------------- <- IDs of buckets -/// Bucket 1 ID belonging to memory 0 ↕ 15 bits +/// Bucket 1 belonging to memory 0 ↕ 15 bits /// -------------------------------------------------- -/// Bucket 2 ID belonging to memory 0 ↕ 15 bits +/// Bucket 2 belonging to memory 0 ↕ 15 bits /// -------------------------------------------------- /// ... /// -------------------------------------------------- -/// Bucket k0 ID belonging to memory 0 ↕ 15 bits +/// Bucket 1 belonging to memory 1 ↕ 15 bits /// -------------------------------------------------- -/// Bucket 1 ID belonging to memory 1 ↕ 15 bits -/// -------------------------------------------------- -/// Bucket 2 ID belonging to memory 1 ↕ 15 bits +/// Bucket 2 belonging to memory 1 ↕ 15 bits /// -------------------------------------------------- /// ... /// -------------------------------------------------- -/// Bucket k1 ID belonging to memory 1 ↕ 15 bits -/// -------------------------------------------------- /// ... /// --------------------------------------------------- -/// Bucket 1 ID belonging to memory 254 ↕ 15 bits +/// Bucket 1 belonging to memory 254 ↕ 15 bits /// -------------------------------------------------- -/// Bucket 2 ID belonging to memory 254 ↕ 15 bits +/// Bucket 2 belonging to memory 254 ↕ 15 bits /// -------------------------------------------------- /// ... /// -------------------------------------------------- -/// Bucket k254 ID belonging to memory 254 ↕ 15 bits -/// -------------------------------------------------- /// Unallocated space ↕ 2'016 bytes /// -------------------------------------------------- <- Buckets (Page 1) /// Bucket 1 ↕ N pages From e95f2a792d29f18f5162201b52ee54cc433a3515 Mon Sep 17 00:00:00 2001 From: Dragoljub Duric Date: Tue, 11 Jul 2023 12:47:07 +0000 Subject: [PATCH 21/70] . --- src/memory_manager.rs | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/src/memory_manager.rs b/src/memory_manager.rs index 403fcd0f..6c02c344 100644 --- a/src/memory_manager.rs +++ b/src/memory_manager.rs @@ -284,14 +284,6 @@ struct MemoryManagerInner { unallocated_buckets: LinkedList, } -fn get_all_buckets() -> LinkedList { - let mut list = LinkedList::new(); - for i in 0..MAX_NUM_BUCKETS { - list.push_back(BucketId(i as u16)); - } - list -} - impl MemoryManagerInner { fn init(memory: M, bucket_size_in_pages: u16) -> Self { if memory.size() == 0 { @@ -318,7 +310,13 @@ impl MemoryManagerInner { memory_sizes_in_pages: [0; MAX_NUM_MEMORIES as usize], memory_buckets: BTreeMap::new(), bucket_size_in_pages, - unallocated_buckets: get_all_buckets(), + unallocated_buckets: { + let mut list = LinkedList::new(); + for i in 0..MAX_NUM_BUCKETS { + list.push_back(BucketId(i as u16)); + } + list + }, }; mem_mgr.save_header(); From 71552f9bc9fc7ff06b39e7fce0249e22a17fa207 Mon Sep 17 00:00:00 2001 From: Dragoljub Duric Date: Tue, 11 Jul 2023 12:53:12 +0000 Subject: [PATCH 22/70] refactor --- src/memory_manager.rs | 146 +++++++++++++++++++++--------------------- 1 file changed, 73 insertions(+), 73 deletions(-) diff --git a/src/memory_manager.rs b/src/memory_manager.rs index 6c02c344..d680571f 100644 --- a/src/memory_manager.rs +++ b/src/memory_manager.rs @@ -324,84 +324,84 @@ impl MemoryManagerInner { mem_mgr } - fn load(memory: M) -> Self { - // Read the header from memory. - let header: Header = read_struct(Address::from(0), &memory); - assert_eq!(&header.magic, MAGIC, "Bad magic."); - match header.version { - LAYOUT_VERSION_V1 => { - let mut buckets = vec![0; MAX_NUM_BUCKETS as usize]; - memory.read(bucket_allocations_address(BucketId(0)).get(), &mut buckets); - let mut unallocated: BTreeSet = (0..MAX_NUM_BUCKETS as u16).collect(); - let mut memory_buckets = BTreeMap::new(); - for (bucket_idx, memory) in buckets.into_iter().enumerate() { - if memory != UNALLOCATED_BUCKET_MARKER { - memory_buckets - .entry(MemoryId(memory)) - .or_insert_with(Vec::new) - .push(BucketId(bucket_idx as u16)); - unallocated.remove(&(bucket_idx as u16)); - } - } + fn load_layout_v1(memory: M, header: Header) -> Self { + let mut buckets = vec![0; MAX_NUM_BUCKETS as usize]; + memory.read(bucket_allocations_address(BucketId(0)).get(), &mut buckets); + let mut unallocated: BTreeSet = (0..MAX_NUM_BUCKETS as u16).collect(); + let mut memory_buckets = BTreeMap::new(); + for (bucket_idx, memory) in buckets.into_iter().enumerate() { + if memory != UNALLOCATED_BUCKET_MARKER { + memory_buckets + .entry(MemoryId(memory)) + .or_insert_with(Vec::new) + .push(BucketId(bucket_idx as u16)); + unallocated.remove(&(bucket_idx as u16)); + } + } - let mut unallocated_buckets = LinkedList::new(); - for i in unallocated.iter() { - unallocated_buckets.push_back(BucketId(*i)); - } + let mut unallocated_buckets = LinkedList::new(); + for i in unallocated.iter() { + unallocated_buckets.push_back(BucketId(*i)); + } - Self { - memory, - allocated_buckets: header.num_allocated_buckets, - bucket_size_in_pages: header.bucket_size_in_pages, - memory_sizes_in_pages: header.memory_sizes_in_pages, - memory_buckets, - unallocated_buckets, - } - } - LAYOUT_VERSION_V2 => { - let header: Header = read_struct(Address::from(0), &memory); - let size_of_buckets_ind_in_bytes = (MAX_NUM_BUCKETS * 15 + 15) / 16; - let mut buckets = vec![0; size_of_buckets_ind_in_bytes as usize]; - memory.read(bucket_allocations_address(BucketId(0)).get(), &mut buckets); - - let buckets_decompressed = bytes_to_bucket_indexes(&buckets); - - let mut unallocated: BTreeSet = (0..MAX_NUM_BUCKETS as u16).collect(); - let mut memory_buckets = BTreeMap::new(); - - let mut j = 0; - for (memory, memory_size_in_pages) in - header.memory_sizes_in_pages.into_iter().enumerate() - { - let memory_size = (memory_size_in_pages + header.bucket_size_in_pages as u64 - - 1) - / header.bucket_size_in_pages as u64; - let mut vec_buckets = vec![]; - for _ in 0..memory_size { - let bucket = BucketId(buckets_decompressed[j]); - vec_buckets.push(bucket); - unallocated.remove(&bucket.0); - j += 1; - } - memory_buckets - .entry(MemoryId(memory as u8)) - .or_insert(vec_buckets); - } + Self { + memory, + allocated_buckets: header.num_allocated_buckets, + bucket_size_in_pages: header.bucket_size_in_pages, + memory_sizes_in_pages: header.memory_sizes_in_pages, + memory_buckets, + unallocated_buckets, + } + } - let mut unallocated_buckets = LinkedList::new(); - for i in unallocated.iter() { - unallocated_buckets.push_back(BucketId(*i)); - } + fn load_layout_v2(memory: M, header: Header) -> Self { + let size_of_buckets_ind_in_bytes = (MAX_NUM_BUCKETS * 15 + 15) / 16; + let mut buckets = vec![0; size_of_buckets_ind_in_bytes as usize]; + memory.read(bucket_allocations_address(BucketId(0)).get(), &mut buckets); - Self { - memory, - allocated_buckets: header.num_allocated_buckets, - bucket_size_in_pages: header.bucket_size_in_pages, - memory_sizes_in_pages: header.memory_sizes_in_pages, - memory_buckets, - unallocated_buckets, - } + let buckets_decompressed = bytes_to_bucket_indexes(&buckets); + + let mut unallocated: BTreeSet = (0..MAX_NUM_BUCKETS as u16).collect(); + let mut memory_buckets = BTreeMap::new(); + + let mut j = 0; + for (memory, memory_size_in_pages) in header.memory_sizes_in_pages.into_iter().enumerate() { + let memory_size = (memory_size_in_pages + header.bucket_size_in_pages as u64 - 1) + / header.bucket_size_in_pages as u64; + let mut vec_buckets = vec![]; + for _ in 0..memory_size { + let bucket = BucketId(buckets_decompressed[j]); + vec_buckets.push(bucket); + unallocated.remove(&bucket.0); + j += 1; } + memory_buckets + .entry(MemoryId(memory as u8)) + .or_insert(vec_buckets); + } + + let mut unallocated_buckets = LinkedList::new(); + for i in unallocated.iter() { + unallocated_buckets.push_back(BucketId(*i)); + } + + Self { + memory, + allocated_buckets: header.num_allocated_buckets, + bucket_size_in_pages: header.bucket_size_in_pages, + memory_sizes_in_pages: header.memory_sizes_in_pages, + memory_buckets, + unallocated_buckets, + } + } + + fn load(memory: M) -> Self { + // Read the header from memory. + let header: Header = read_struct(Address::from(0), &memory); + assert_eq!(&header.magic, MAGIC, "Bad magic."); + match header.version { + LAYOUT_VERSION_V1 => MemoryManagerInner::load_layout_v1(memory, header), + LAYOUT_VERSION_V2 => MemoryManagerInner::load_layout_v2(memory, header), _ => panic!("Unsupported version."), } } From 70390cde3c499976da1319361092d181d7a2cdcd Mon Sep 17 00:00:00 2001 From: Dragoljub Duric Date: Tue, 11 Jul 2023 12:57:40 +0000 Subject: [PATCH 23/70] refactor --- src/memory_manager.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/memory_manager.rs b/src/memory_manager.rs index d680571f..75c3073e 100644 --- a/src/memory_manager.rs +++ b/src/memory_manager.rs @@ -370,7 +370,7 @@ impl MemoryManagerInner { / header.bucket_size_in_pages as u64; let mut vec_buckets = vec![]; for _ in 0..memory_size { - let bucket = BucketId(buckets_decompressed[j]); + let bucket = buckets_decompressed[j]; vec_buckets.push(bucket); unallocated.remove(&bucket.0); j += 1; @@ -563,22 +563,22 @@ impl MemoryManagerInner { } } -fn bytes_to_bucket_indexes(input: &[u8]) -> Vec { +fn bytes_to_bucket_indexes(input: &[u8]) -> Vec { const BUCKET_IND_LEN: usize = 15; - let mut vec_u16 = vec![]; + let mut bucket_ids = vec![]; let bit_vec = BitVec::from_bytes(input); for i in 0..bit_vec.len() / BUCKET_IND_LEN { - let mut bucket_ind: u16 = 0; + let mut bucket: u16 = 0; for j in 0..BUCKET_IND_LEN { let next_bit = BUCKET_IND_LEN * i + j; - bucket_ind <<= 1; + bucket <<= 1; if bit_vec.get(next_bit) == Some(true) { - bucket_ind |= 1; + bucket |= 1; } } - vec_u16.push(bucket_ind); + bucket_ids.push(BucketId(bucket)); } - vec_u16 + bucket_ids } struct Segment { address: Address, From 08eec331c9420922b0b35fa4fca2cd24b9b1cc06 Mon Sep 17 00:00:00 2001 From: Dragoljub Duric Date: Tue, 11 Jul 2023 13:07:48 +0000 Subject: [PATCH 24/70] refactor --- src/memory_manager.rs | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/memory_manager.rs b/src/memory_manager.rs index 75c3073e..d37908c0 100644 --- a/src/memory_manager.rs +++ b/src/memory_manager.rs @@ -75,6 +75,9 @@ const BUCKETS_OFFSET_IN_BYTES: u64 = BUCKETS_OFFSET_IN_PAGES * WASM_PAGE_SIZE; // Reserved bytes in the header for future extensions. const HEADER_RESERVED_BYTES: usize = 32; +// Size of the bucked id in the memory. +const BUCKET_ID_LEN: usize = 15; + /// A memory manager simulates multiple memories within a single memory. /// /// The memory manager can return up to 255 unique instances of [`VirtualMemory`], and each can be @@ -355,8 +358,10 @@ impl MemoryManagerInner { } fn load_layout_v2(memory: M, header: Header) -> Self { - let size_of_buckets_ind_in_bytes = (MAX_NUM_BUCKETS * 15 + 15) / 16; - let mut buckets = vec![0; size_of_buckets_ind_in_bytes as usize]; + const BYTE_SIZE: usize = 8; + let size_of_buckets_ind_in_bytes = + (MAX_NUM_BUCKETS as usize * BUCKET_ID_LEN + (BYTE_SIZE - 1)) / BYTE_SIZE; + let mut buckets = vec![0; size_of_buckets_ind_in_bytes]; memory.read(bucket_allocations_address(BucketId(0)).get(), &mut buckets); let buckets_decompressed = bytes_to_bucket_indexes(&buckets); @@ -564,13 +569,12 @@ impl MemoryManagerInner { } fn bytes_to_bucket_indexes(input: &[u8]) -> Vec { - const BUCKET_IND_LEN: usize = 15; let mut bucket_ids = vec![]; let bit_vec = BitVec::from_bytes(input); - for i in 0..bit_vec.len() / BUCKET_IND_LEN { + for bucket_id in 0..bit_vec.len() / BUCKET_ID_LEN { let mut bucket: u16 = 0; - for j in 0..BUCKET_IND_LEN { - let next_bit = BUCKET_IND_LEN * i + j; + for bucket_id_bit in 0..BUCKET_ID_LEN { + let next_bit = BUCKET_ID_LEN * bucket_id + bucket_id_bit; bucket <<= 1; if bit_vec.get(next_bit) == Some(true) { bucket |= 1; @@ -580,6 +584,7 @@ fn bytes_to_bucket_indexes(input: &[u8]) -> Vec { } bucket_ids } + struct Segment { address: Address, length: Bytes, From e4a32493c2aacf8e6c09e92ad95002760645a8b7 Mon Sep 17 00:00:00 2001 From: Dragoljub Duric Date: Tue, 11 Jul 2023 13:10:05 +0000 Subject: [PATCH 25/70] refactor --- src/memory_manager.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/memory_manager.rs b/src/memory_manager.rs index d37908c0..215d6cc6 100644 --- a/src/memory_manager.rs +++ b/src/memory_manager.rs @@ -571,16 +571,16 @@ impl MemoryManagerInner { fn bytes_to_bucket_indexes(input: &[u8]) -> Vec { let mut bucket_ids = vec![]; let bit_vec = BitVec::from_bytes(input); - for bucket_id in 0..bit_vec.len() / BUCKET_ID_LEN { - let mut bucket: u16 = 0; + for bucket_order_number in 0..bit_vec.len() / BUCKET_ID_LEN { + let mut bucket_id: u16 = 0; for bucket_id_bit in 0..BUCKET_ID_LEN { - let next_bit = BUCKET_ID_LEN * bucket_id + bucket_id_bit; - bucket <<= 1; + let next_bit = BUCKET_ID_LEN * bucket_order_number + bucket_id_bit; + bucket_id <<= 1; if bit_vec.get(next_bit) == Some(true) { - bucket |= 1; + bucket_id |= 1; } } - bucket_ids.push(BucketId(bucket)); + bucket_ids.push(BucketId(bucket_id)); } bucket_ids } From d2d5e299013a5d93ad0f9f9c31834becd3fdc71b Mon Sep 17 00:00:00 2001 From: Dragoljub Duric Date: Wed, 12 Jul 2023 11:28:02 +0000 Subject: [PATCH 26/70] add test --- src/memory_manager.rs | 54 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 53 insertions(+), 1 deletion(-) diff --git a/src/memory_manager.rs b/src/memory_manager.rs index 215d6cc6..3a7382fc 100644 --- a/src/memory_manager.rs +++ b/src/memory_manager.rs @@ -561,7 +561,7 @@ impl MemoryManagerInner { let buckets = self.memory_buckets.remove(&id); if let Some(vec_buckets) = buckets { for bucket in vec_buckets { - self.unallocated_buckets.push_back(bucket); + self.unallocated_buckets.push_front(bucket); self.allocated_buckets -= 1; } } @@ -1017,4 +1017,56 @@ mod test { memory_1.read(0, &mut buf); assert_eq!(buf, vec![2; 1000]); } + + #[test] + fn free_memory_works() { + let mem = make_memory(); + let initial_size = BUCKET_SIZE_IN_PAGES * 2; + // Grow the memory manually before passing it into the memory manager. + mem.grow(initial_size); + + let mut mem_mgr = MemoryManager::init(mem.clone()); + let memory_0 = mem_mgr.get(MemoryId(0)); + let memory_1 = mem_mgr.get(MemoryId(1)); + + // Grow the memory by 1 page. + assert_eq!(memory_0.grow(1), 0); + assert_eq!(mem.size(), initial_size); + + // Grow the memory by 1 page. + assert_eq!(memory_1.grow(1), 0); + assert_eq!(mem.size(), initial_size + 1); + + // Grow the memory by BUCKET_SIZE_IN_PAGES more pages, which will cause the underlying + // allocation to increase. + assert_eq!(memory_0.grow(BUCKET_SIZE_IN_PAGES), 1); + assert_eq!(mem.size(), 1 + BUCKET_SIZE_IN_PAGES * 3); + assert_eq!(memory_1.grow(BUCKET_SIZE_IN_PAGES), 1); + assert_eq!(mem.size(), 1 + BUCKET_SIZE_IN_PAGES * 4); + assert_eq!(mem_mgr.get(MemoryId(1)).size(), 1 + BUCKET_SIZE_IN_PAGES); + + // Free Memory ID 1. + mem_mgr.free(MemoryId(1)); + assert_eq!(mem_mgr.get(MemoryId(1)).size(), 0); + assert_eq!(mem.size(), 1 + BUCKET_SIZE_IN_PAGES * 4); + + let memory_2 = mem_mgr.get(MemoryId(2)); + // When growing memory_2, mem.size() should stay the same since + // MemoryManager should use the memory that is freed above. + assert_eq!(memory_2.grow(1), 0); + assert_eq!(mem_mgr.get(MemoryId(2)).size(), 1); + assert_eq!(mem.size(), 1 + BUCKET_SIZE_IN_PAGES * 4); + assert_eq!(memory_2.grow(BUCKET_SIZE_IN_PAGES), 1); + assert_eq!(mem_mgr.get(MemoryId(2)).size(), 1 + BUCKET_SIZE_IN_PAGES); + assert_eq!(mem.size(), 1 + BUCKET_SIZE_IN_PAGES * 4); + + // When trying to grow memory_2 again, we need more pages, + // because we have already used all that is left from Memory ID 1. + assert_ne!(memory_2.grow(BUCKET_SIZE_IN_PAGES), 0); + assert_eq!( + mem_mgr.get(MemoryId(2)).size(), + 1 + 2 * BUCKET_SIZE_IN_PAGES + ); + assert_eq!(mem.size(), 1 + BUCKET_SIZE_IN_PAGES * 5); + } } From 16542845d5794bca5d0cab94266414be499e896b Mon Sep 17 00:00:00 2001 From: Dragoljub Duric Date: Wed, 12 Jul 2023 11:44:54 +0000 Subject: [PATCH 27/70] add comments --- src/memory_manager.rs | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/src/memory_manager.rs b/src/memory_manager.rs index 3a7382fc..af7d7442 100644 --- a/src/memory_manager.rs +++ b/src/memory_manager.rs @@ -366,18 +366,19 @@ impl MemoryManagerInner { let buckets_decompressed = bytes_to_bucket_indexes(&buckets); - let mut unallocated: BTreeSet = (0..MAX_NUM_BUCKETS as u16).collect(); + let mut unallocated_buckets_set: BTreeSet = (0..MAX_NUM_BUCKETS as u16).collect(); let mut memory_buckets = BTreeMap::new(); let mut j = 0; for (memory, memory_size_in_pages) in header.memory_sizes_in_pages.into_iter().enumerate() { - let memory_size = (memory_size_in_pages + header.bucket_size_in_pages as u64 - 1) - / header.bucket_size_in_pages as u64; + let memory_size_in_buckets = + (memory_size_in_pages + header.bucket_size_in_pages as u64 - 1) + / header.bucket_size_in_pages as u64; let mut vec_buckets = vec![]; - for _ in 0..memory_size { + for _ in 0..memory_size_in_buckets { let bucket = buckets_decompressed[j]; vec_buckets.push(bucket); - unallocated.remove(&bucket.0); + unallocated_buckets_set.remove(&bucket.0); j += 1; } memory_buckets @@ -386,7 +387,7 @@ impl MemoryManagerInner { } let mut unallocated_buckets = LinkedList::new(); - for i in unallocated.iter() { + for i in unallocated_buckets_set.iter() { unallocated_buckets.push_back(BucketId(*i)); } @@ -489,6 +490,9 @@ impl MemoryManagerInner { for memory in self.memory_buckets.iter() { for bucket in memory.1 { let bucket_ind = bucket.0; + // Splits bit_vec returning the slice [1, .., bit_vec.size() - 1]. + // This is precisely what we need since the BucketId can be represented + // using only 15 bits, instead of 16. let mut bit_vec_temp = BitVec::from_bytes(&bucket_ind.to_be_bytes()).split_off(1); bit_vec.append(&mut bit_vec_temp); } From 92d61adbff74e46b9e3ed01668afe1efb568fba9 Mon Sep 17 00:00:00 2001 From: Dragoljub Djuric Date: Wed, 12 Jul 2023 13:49:32 +0200 Subject: [PATCH 28/70] Update lib.rs --- src/lib.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/lib.rs b/src/lib.rs index b57027b6..6e495852 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -95,7 +95,8 @@ impl Display for GrowFailed { write!( f, "Failed to grow memory: current size={}, delta={}", - self.current_size, self.delta + self.current_size, + self.delta ) } } From 20176d2667428544562e938e16f8a5cb40dc5bb6 Mon Sep 17 00:00:00 2001 From: Dragoljub Djuric Date: Wed, 12 Jul 2023 13:50:00 +0200 Subject: [PATCH 29/70] Update lib.rs --- src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib.rs b/src/lib.rs index 6e495852..53ab7725 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -95,7 +95,7 @@ impl Display for GrowFailed { write!( f, "Failed to grow memory: current size={}, delta={}", - self.current_size, + self.current_size, self.delta ) } From 95195f89d7fffc5a142082f3a62310a47cdc8d90 Mon Sep 17 00:00:00 2001 From: Dragoljub Duric Date: Wed, 12 Jul 2023 13:10:49 +0000 Subject: [PATCH 30/70] update Cargo.lock --- Cargo.lock | 400 +++++++++++++++++++++++++---------------------------- 1 file changed, 192 insertions(+), 208 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0db895ca..31a7c55f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4,9 +4,9 @@ version = 3 [[package]] name = "aho-corasick" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67fc08ce920c31afb70f013dcce1bfc3a3195de6a228474e45e1f145b36f8d04" +checksum = "43f6cb1bf222025340178f382c426f13757b2960e89779dfcb319c32542a5a41" dependencies = [ "memchr", ] @@ -115,6 +115,12 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" +[[package]] +name = "bitflags" +version = "2.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "630be753d4e58660abd17930c71b647fe46c27ea6b63cc59e1e3851406972e42" + [[package]] name = "block-buffer" version = "0.10.4" @@ -228,9 +234,9 @@ version = "3.2.25" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ea181bf566f71cb9a5d17a59e1871af638180a18fb0035c92ae62b705207123" dependencies = [ - "bitflags", + "bitflags 1.3.2", "clap_lex", - "indexmap", + "indexmap 1.9.3", "textwrap", ] @@ -255,9 +261,9 @@ dependencies = [ [[package]] name = "cpufeatures" -version = "0.2.7" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e4c1eaa2012c47becbbad2ab175484c2a84d1185b566fb2cc5b8707343dfe58" +checksum = "a17b76ff3a4162b0b27f354a0c87015ddad39d35f9c0c36607a3bdd175dde1f1" dependencies = [ "libc", ] @@ -330,9 +336,9 @@ dependencies = [ [[package]] name = "crossbeam-epoch" -version = "0.9.14" +version = "0.9.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46bd5f3f85273295a9d14aedfb86f6aadbff6d8f5295c4a9edb08e819dcf5695" +checksum = "ae211234986c545741a7dc064309f67ee1e5ad243d0e48335adc0484d960bcc7" dependencies = [ "autocfg", "cfg-if", @@ -343,9 +349,9 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.15" +version = "0.8.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c063cd8cc95f5c377ed0d4b49a4b21f632396ff690e8470c29b3359b346984b" +checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294" dependencies = [ "cfg-if", ] @@ -424,6 +430,12 @@ dependencies = [ "log", ] +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + [[package]] name = "errno" version = "0.3.1" @@ -432,7 +444,7 @@ checksum = "4bcfec3a70f97c962c307b2d2c56e358cf1d00b558d74262b5f929ee8cc7e73a" dependencies = [ "errno-dragonfly", "libc", - "windows-sys 0.48.0", + "windows-sys", ] [[package]] @@ -478,9 +490,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.9" +version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c85e1d9ab2eadba7e5040d4e09cbd6d072b76a557ad64e797c2cb9d4da21d7e4" +checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427" dependencies = [ "cfg-if", "libc", @@ -500,28 +512,25 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" [[package]] -name = "hermit-abi" -version = "0.1.19" +name = "hashbrown" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" -dependencies = [ - "libc", -] +checksum = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a" [[package]] name = "hermit-abi" -version = "0.2.6" +version = "0.1.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee512640fe35acbfb4bb779db6f0d80704c2cacfa2e39b601ef3e3f47d1ae4c7" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" dependencies = [ "libc", ] [[package]] name = "hermit-abi" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fed44880c466736ef9a5c5b5facefb5ed0785676d0c02d612db14e54f0d84286" +checksum = "443144c8cdadd93ebf52ddb4056d257f5b52c04d3c804e657d19eb73fc33668b" [[package]] name = "hex" @@ -584,7 +593,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" dependencies = [ "autocfg", - "hashbrown", + "hashbrown 0.12.3", +] + +[[package]] +name = "indexmap" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5477fe2230a79769d8dc68e0eabf5437907c0457a5614a9e8dddb67f65eb65d" +dependencies = [ + "equivalent", + "hashbrown 0.14.0", ] [[package]] @@ -602,21 +621,20 @@ version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2" dependencies = [ - "hermit-abi 0.3.1", + "hermit-abi 0.3.2", "libc", - "windows-sys 0.48.0", + "windows-sys", ] [[package]] name = "is-terminal" -version = "0.4.7" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adcf93614601c8129ddf72e2d5633df827ba6551541c6d8c59520a371475be1f" +checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b" dependencies = [ - "hermit-abi 0.3.1", - "io-lifetimes", - "rustix", - "windows-sys 0.48.0", + "hermit-abi 0.3.2", + "rustix 0.38.4", + "windows-sys", ] [[package]] @@ -630,15 +648,15 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.6" +version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "453ad9f582a441959e5f0d088b02ce04cfe8d51a8eaf077f12ac6d3e94164ca6" +checksum = "62b02a5381cc465bd3041d84623d0fa3b66738b52b8e2fc3bab8ad63ab032f4a" [[package]] name = "js-sys" -version = "0.3.63" +version = "0.3.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f37a4a5928311ac501dee68b3c7613a1037d0edb30c8e5427bd832d55d1b790" +checksum = "c5f195fe497f702db0f318b07fdd68edb16955aed830df8363d837542f8f935a" dependencies = [ "wasm-bindgen", ] @@ -688,9 +706,9 @@ checksum = "884e2677b40cc8c339eaefcb701c32ef1fd2493d71118dc0ca4b6a736c93bd67" [[package]] name = "libc" -version = "0.2.144" +version = "0.2.147" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b00cc1c228a6782d0f076e7b232802e0c5689d41bb5df366f2a6b6621cfdfe1" +checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3" [[package]] name = "libm" @@ -704,11 +722,17 @@ version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" +[[package]] +name = "linux-raw-sys" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09fc20d2ca12cb9f044c93e3bd6d32d523e6e2ec3db4f7b2939cd99026ecd3f0" + [[package]] name = "lock_api" -version = "0.4.9" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "435011366fe56583b16cf956f9df0095b405b82d76425bc8981c0e22e60ec4df" +checksum = "c1cc9717a20b1bb222f333e6a92fd32f7d8a18ddc5a3191a11af45dcbf4dcd16" dependencies = [ "autocfg", "scopeguard", @@ -716,9 +740,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.18" +version = "0.4.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "518ef76f2f87365916b142844c16d8fefd85039bc5699050210a7778ee1cd1de" +checksum = "b06a4cde4c0f271a446782e3eff8de789548ce57dbc8eca9292c27f4a42004b4" [[package]] name = "logos" @@ -757,9 +781,9 @@ checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" [[package]] name = "memoffset" -version = "0.8.0" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d61c719bcfbcf5d62b3a09efa6088de8c54bc0bfcd3ea7ae39fcc186108b8de1" +checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c" dependencies = [ "autocfg", ] @@ -804,11 +828,11 @@ dependencies = [ [[package]] name = "num_cpus" -version = "1.15.0" +version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fac9e2da13b5eb447a6ce3d392f23a29d8694bff781bf03a16cd9ac8697593b" +checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" dependencies = [ - "hermit-abi 0.2.6", + "hermit-abi 0.3.2", "libc", ] @@ -835,9 +859,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.17.2" +version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9670a07f94779e00908f3e686eab508878ebb390ba6e604d3a284c00e8d0487b" +checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" [[package]] name = "oorandom" @@ -847,9 +871,9 @@ checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575" [[package]] name = "os_str_bytes" -version = "6.5.0" +version = "6.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ceedf44fb00f2d1984b0bc98102627ce622e083e49a5bacdb3e514fa4238e267" +checksum = "4d5d9eb14b174ee9aa2ef96dc2b94637a2d4b6e7cb873c7e171f0c20c6cf3eac" [[package]] name = "parking_lot" @@ -863,22 +887,22 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.7" +version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9069cbb9f99e3a5083476ccb29ceb1de18b9118cafa53e90c9551235de2b9521" +checksum = "93f00c865fe7cabf650081affecd3871070f26767e7b2070a3ffae14c654b447" dependencies = [ "cfg-if", "libc", - "redox_syscall 0.2.16", + "redox_syscall 0.3.5", "smallvec", - "windows-sys 0.45.0", + "windows-targets", ] [[package]] name = "paste" -version = "1.0.12" +version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f746c4065a8fa3fe23974dd82f15431cc8d40779821001404d10d2e79ca7d79" +checksum = "b4b27ab7be369122c218afc2079489cdcb4b517c0a3fc386ff11e1fedfcc2b35" [[package]] name = "petgraph" @@ -887,7 +911,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4dd7d28ee937e54fe3080c91faa1c3a46c06de6252988a7f4592ba2310ef22a4" dependencies = [ "fixedbitset", - "indexmap", + "indexmap 1.9.3", ] [[package]] @@ -901,9 +925,9 @@ dependencies = [ [[package]] name = "plotters" -version = "0.3.4" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2538b639e642295546c50fcd545198c9d64ee2a38620a628724a3b266d5fbf97" +checksum = "d2c224ba00d7cadd4d5c660deaf2098e5e80e07846537c51f9cfa4be50c1fd45" dependencies = [ "num-traits", "plotters-backend", @@ -914,15 +938,15 @@ dependencies = [ [[package]] name = "plotters-backend" -version = "0.3.4" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "193228616381fecdc1224c62e96946dfbc73ff4384fba576e052ff8c1bea8142" +checksum = "9e76628b4d3a7581389a35d5b6e2139607ad7c75b17aed325f210aa91f4a9609" [[package]] name = "plotters-svg" -version = "0.3.3" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9a81d2759aae1dae668f783c308bc5c8ebd191ff4184aaa1b37f65a6ae5a56f" +checksum = "38f6d39893cca0701371e3c27294f09797214b86f1fb951b89ade8ec04e2abab" dependencies = [ "plotters-backend", ] @@ -961,9 +985,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.59" +version = "1.0.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6aeca18b86b413c660b781aa319e4e2648a3e6f9eadc9b47e9038e6fe9f3451b" +checksum = "78803b62cbf1f46fde80d7c0e803111524b9877184cfe7c3033659490ac7a7da" dependencies = [ "unicode-ident", ] @@ -975,7 +999,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4e35c06b98bf36aba164cc17cb25f7e232f5c4aeea73baa14b8a9f0d92dbfa65" dependencies = [ "bit-set", - "bitflags", + "bitflags 1.3.2", "byteorder", "lazy_static", "num-traits", @@ -996,9 +1020,9 @@ checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" [[package]] name = "quote" -version = "1.0.28" +version = "1.0.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b9ab9c7eadfd8df19006f1cf1a4aed13540ed5cbc047010ece5826e10825488" +checksum = "573015e8ab27661678357f27dc26460738fd2b6c86e46f386fde94cb5d913105" dependencies = [ "proc-macro2", ] @@ -1070,7 +1094,7 @@ version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" dependencies = [ - "bitflags", + "bitflags 1.3.2", ] [[package]] @@ -1079,7 +1103,7 @@ version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" dependencies = [ - "bitflags", + "bitflags 1.3.2", ] [[package]] @@ -1095,13 +1119,25 @@ dependencies = [ [[package]] name = "regex" -version = "1.8.3" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2eae68fc220f7cf2532e4494aded17545fce192d59cd996e0fe7887f4ceb575" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax 0.7.4", +] + +[[package]] +name = "regex-automata" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81ca098a9821bd52d6b24fd8b10bd081f47d39c22778cafaa75a2857a62c6390" +checksum = "83d3daa6976cffb758ec878f108ba0e062a45b2d6ca3a2cca965338855476caf" dependencies = [ "aho-corasick", "memchr", - "regex-syntax 0.7.2", + "regex-syntax 0.7.4", ] [[package]] @@ -1112,29 +1148,42 @@ checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" [[package]] name = "regex-syntax" -version = "0.7.2" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "436b050e76ed2903236f032a59761c1eb99e1b0aead2c257922771dab1fc8c78" +checksum = "e5ea92a5b6195c6ef2a0295ea818b312502c6fc94dde986c5553242e18fd4ce2" [[package]] name = "rustix" -version = "0.37.19" +version = "0.37.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acf8729d8542766f1b2cf77eb034d52f40d375bb8b615d0b147089946e16613d" +checksum = "4d69718bf81c6127a49dc64e44a742e8bb9213c0ff8869a22c308f84c1d4ab06" dependencies = [ - "bitflags", + "bitflags 1.3.2", "errno", "io-lifetimes", "libc", - "linux-raw-sys", - "windows-sys 0.48.0", + "linux-raw-sys 0.3.8", + "windows-sys", +] + +[[package]] +name = "rustix" +version = "0.38.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a962918ea88d644592894bc6dc55acc6c0956488adcebbfb6e273506b7fd6e5" +dependencies = [ + "bitflags 2.3.3", + "errno", + "libc", + "linux-raw-sys 0.4.3", + "windows-sys", ] [[package]] name = "rustversion" -version = "1.0.12" +version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f3208ce4d8448b3f3e7d168a73f5e0c43a61e32930de3bceeccedb388b6bf06" +checksum = "dc31bd9b61a32c31f9650d18add92aa83a49ba979c143eefd27fe7177b05bd5f" [[package]] name = "rusty-fork" @@ -1150,9 +1199,9 @@ dependencies = [ [[package]] name = "ryu" -version = "1.0.13" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f91339c0467de62360649f8d3e185ca8de4224ff281f66000de5eb2a77a79041" +checksum = "fe232bdf6be8c8de797b22184ee71118d63780ea42ac85b61d1baa6d3b782ae9" [[package]] name = "same-file" @@ -1171,38 +1220,38 @@ checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" [[package]] name = "serde" -version = "1.0.163" +version = "1.0.171" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2113ab51b87a539ae008b5c6c02dc020ffa39afd2d83cffcb3f4eb2722cebec2" +checksum = "30e27d1e4fd7659406c492fd6cfaf2066ba8773de45ca75e855590f856dc34a9" dependencies = [ "serde_derive", ] [[package]] name = "serde_bytes" -version = "0.11.9" +version = "0.11.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "416bda436f9aab92e02c8e10d49a15ddd339cea90b6e340fe51ed97abb548294" +checksum = "5a16be4fe5320ade08736447e3198294a5ea9a6d44dde6f35f0a5e06859c427a" dependencies = [ "serde", ] [[package]] name = "serde_derive" -version = "1.0.163" +version = "1.0.171" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c805777e3930c8883389c602315a24224bcc738b63905ef87cd1420353ea93e" +checksum = "389894603bd18c46fa56231694f8d827779c0951a667087194cf9de94ed24682" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.25", ] [[package]] name = "serde_json" -version = "1.0.96" +version = "1.0.102" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "057d394a50403bcac12672b2b18fb387ab6d289d957dab67dd201875391e52f1" +checksum = "b5062a995d481b2308b6064e9af76011f2921c35f97b0468811ed9f6cd91dfed" dependencies = [ "itoa", "ryu", @@ -1222,9 +1271,9 @@ dependencies = [ [[package]] name = "sha2" -version = "0.10.6" +version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82e6b795fe2e3b1e845bafcb27aa35405c4d47cdfc92af5fc8d3002f76cebdc0" +checksum = "479fb9d862239e610720565ca91403019f2f00410f1864c5aa7479b950a76ed8" dependencies = [ "cfg-if", "cpufeatures", @@ -1239,9 +1288,9 @@ checksum = "7bd3e3206899af3f8b12af284fafc038cc1dc2b41d1b89dd17297221c5d225de" [[package]] name = "smallvec" -version = "1.10.0" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" +checksum = "62bb4feee49fdd9f707ef802e22365a35de4b7b299de4763d44bfea899442ff9" [[package]] name = "string_cache" @@ -1269,9 +1318,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.18" +version = "2.0.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32d41677bcbe24c20c52e7c70b0d8db04134c5d1066bf98662e2871ad200ea3e" +checksum = "15e3fc8c0c74267e2df136e5e5fb656a464158aa57624053375eb9c8c6e25ae2" dependencies = [ "proc-macro2", "quote", @@ -1280,15 +1329,16 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.5.0" +version = "3.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9fbec84f381d5795b08656e4912bec604d162bff9291d6189a78f4c8ab87998" +checksum = "31c0432476357e58790aaa47a8efb0c5138f137343f3b5f23bd36a27e3b0a6d6" dependencies = [ + "autocfg", "cfg-if", "fastrand", "redox_syscall 0.3.5", - "rustix", - "windows-sys 0.45.0", + "rustix 0.37.23", + "windows-sys", ] [[package]] @@ -1319,22 +1369,22 @@ checksum = "222a222a5bfe1bba4a77b45ec488a741b3cb8872e5e499451fd7d0129c9c7c3d" [[package]] name = "thiserror" -version = "1.0.40" +version = "1.0.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "978c9a314bd8dc99be594bc3c175faaa9794be04a5a5e153caba6915336cebac" +checksum = "a35fc5b8971143ca348fa6df4f024d4d55264f3468c71ad1c2f365b0a4d58c42" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.40" +version = "1.0.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f" +checksum = "463fe12d7993d3b327787537ce8dd4dfa058de32fc2b195ef3cde03dc4771e8f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.25", ] [[package]] @@ -1364,17 +1414,17 @@ dependencies = [ [[package]] name = "toml_datetime" -version = "0.6.2" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a76a9312f5ba4c2dec6b9161fdf25d87ad8a09256ccea5a556fef03c706a10f" +checksum = "7cda73e2f1397b1262d6dfdcef8aafae14d1de7748d66822d3bfeeb6d03e5e4b" [[package]] name = "toml_edit" -version = "0.19.10" +version = "0.19.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2380d56e8670370eee6566b0bfd4265f65b3f432e8c6d85623f728d4fa31f739" +checksum = "c500344a19072298cd05a7224b3c0c629348b78692bf48466c5238656e315a78" dependencies = [ - "indexmap", + "indexmap 2.0.0", "toml_datetime", "winnow", ] @@ -1399,9 +1449,9 @@ checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" [[package]] name = "unicode-ident" -version = "1.0.9" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b15811caf2415fb889178633e7724bad2509101cde276048e013b9def5e51fa0" +checksum = "22049a19f4a68748a168c0fc439f9516686aa045927ff767eca0a85101fb6e73" [[package]] name = "unicode-width" @@ -1448,9 +1498,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.86" +version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5bba0e8cb82ba49ff4e229459ff22a191bbe9a1cb3a341610c9c33efc27ddf73" +checksum = "7706a72ab36d8cb1f80ffbf0e071533974a60d0a308d01a5d0375bf60499a342" dependencies = [ "cfg-if", "wasm-bindgen-macro", @@ -1458,24 +1508,24 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.86" +version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19b04bc93f9d6bdee709f6bd2118f57dd6679cf1176a1af464fca3ab0d66d8fb" +checksum = "5ef2b6d3c510e9625e5fe6f509ab07d66a760f0885d858736483c32ed7809abd" dependencies = [ "bumpalo", "log", "once_cell", "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.25", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-macro" -version = "0.2.86" +version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14d6b024f1a526bb0234f52840389927257beb670610081360e5a03c5df9c258" +checksum = "dee495e55982a3bd48105a7b947fd2a9b4a8ae3010041b9e0faab3f9cd028f1d" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -1483,28 +1533,28 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.86" +version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e128beba882dd1eb6200e1dc92ae6c5dbaa4311aa7bb211ca035779e5efc39f8" +checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.25", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.86" +version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed9d5b4305409d1fc9482fee2d7f9bcbf24b3972bf59817ef757e23982242a93" +checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1" [[package]] name = "web-sys" -version = "0.3.63" +version = "0.3.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3bdd9ef4e984da1187bf8110c5cf5b845fbc87a23602cdf912386a76fcd3a7c2" +checksum = "9b85cbef8c220a6abc02aefd892dfc0fc23afb1c6a426316ec33253a3877249b" dependencies = [ "js-sys", "wasm-bindgen", @@ -1541,132 +1591,66 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" -[[package]] -name = "windows-sys" -version = "0.45.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" -dependencies = [ - "windows-targets 0.42.2", -] - [[package]] name = "windows-sys" version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" dependencies = [ - "windows-targets 0.48.0", -] - -[[package]] -name = "windows-targets" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" -dependencies = [ - "windows_aarch64_gnullvm 0.42.2", - "windows_aarch64_msvc 0.42.2", - "windows_i686_gnu 0.42.2", - "windows_i686_msvc 0.42.2", - "windows_x86_64_gnu 0.42.2", - "windows_x86_64_gnullvm 0.42.2", - "windows_x86_64_msvc 0.42.2", + "windows-targets", ] [[package]] name = "windows-targets" -version = "0.48.0" +version = "0.48.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b1eb6f0cd7c80c79759c929114ef071b87354ce476d9d94271031c0497adfd5" +checksum = "05d4b17490f70499f20b9e791dcf6a299785ce8af4d709018206dc5b4953e95f" dependencies = [ - "windows_aarch64_gnullvm 0.48.0", - "windows_aarch64_msvc 0.48.0", - "windows_i686_gnu 0.48.0", - "windows_i686_msvc 0.48.0", - "windows_x86_64_gnu 0.48.0", - "windows_x86_64_gnullvm 0.48.0", - "windows_x86_64_msvc 0.48.0", + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", ] -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" - [[package]] name = "windows_aarch64_gnullvm" version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc" -[[package]] -name = "windows_aarch64_msvc" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" - [[package]] name = "windows_aarch64_msvc" version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3" -[[package]] -name = "windows_i686_gnu" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" - [[package]] name = "windows_i686_gnu" version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241" -[[package]] -name = "windows_i686_msvc" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" - [[package]] name = "windows_i686_msvc" version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00" -[[package]] -name = "windows_x86_64_gnu" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" - [[package]] name = "windows_x86_64_gnu" version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1" -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" - [[package]] name = "windows_x86_64_gnullvm" version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953" -[[package]] -name = "windows_x86_64_msvc" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" - [[package]] name = "windows_x86_64_msvc" version = "0.48.0" @@ -1675,9 +1659,9 @@ checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" [[package]] name = "winnow" -version = "0.4.6" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61de7bac303dc551fe038e2b3cef0f571087a47571ea6e79a87692ac99b99699" +checksum = "81a2094c43cc94775293eaa0e499fbc30048a6d824ac82c0351a8c0bf9112529" dependencies = [ "memchr", ] From e60af2d0c69cb9f79a7d7cc1e1f661e586d616e7 Mon Sep 17 00:00:00 2001 From: Dragoljub Duric Date: Wed, 12 Jul 2023 13:14:52 +0000 Subject: [PATCH 31/70] Revert "update Cargo.lock" This reverts commit 95195f89d7fffc5a142082f3a62310a47cdc8d90. --- Cargo.lock | 400 ++++++++++++++++++++++++++++------------------------- 1 file changed, 208 insertions(+), 192 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 31a7c55f..0db895ca 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4,9 +4,9 @@ version = 3 [[package]] name = "aho-corasick" -version = "1.0.2" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43f6cb1bf222025340178f382c426f13757b2960e89779dfcb319c32542a5a41" +checksum = "67fc08ce920c31afb70f013dcce1bfc3a3195de6a228474e45e1f145b36f8d04" dependencies = [ "memchr", ] @@ -115,12 +115,6 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" -[[package]] -name = "bitflags" -version = "2.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "630be753d4e58660abd17930c71b647fe46c27ea6b63cc59e1e3851406972e42" - [[package]] name = "block-buffer" version = "0.10.4" @@ -234,9 +228,9 @@ version = "3.2.25" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ea181bf566f71cb9a5d17a59e1871af638180a18fb0035c92ae62b705207123" dependencies = [ - "bitflags 1.3.2", + "bitflags", "clap_lex", - "indexmap 1.9.3", + "indexmap", "textwrap", ] @@ -261,9 +255,9 @@ dependencies = [ [[package]] name = "cpufeatures" -version = "0.2.9" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a17b76ff3a4162b0b27f354a0c87015ddad39d35f9c0c36607a3bdd175dde1f1" +checksum = "3e4c1eaa2012c47becbbad2ab175484c2a84d1185b566fb2cc5b8707343dfe58" dependencies = [ "libc", ] @@ -336,9 +330,9 @@ dependencies = [ [[package]] name = "crossbeam-epoch" -version = "0.9.15" +version = "0.9.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae211234986c545741a7dc064309f67ee1e5ad243d0e48335adc0484d960bcc7" +checksum = "46bd5f3f85273295a9d14aedfb86f6aadbff6d8f5295c4a9edb08e819dcf5695" dependencies = [ "autocfg", "cfg-if", @@ -349,9 +343,9 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.16" +version = "0.8.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294" +checksum = "3c063cd8cc95f5c377ed0d4b49a4b21f632396ff690e8470c29b3359b346984b" dependencies = [ "cfg-if", ] @@ -430,12 +424,6 @@ dependencies = [ "log", ] -[[package]] -name = "equivalent" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" - [[package]] name = "errno" version = "0.3.1" @@ -444,7 +432,7 @@ checksum = "4bcfec3a70f97c962c307b2d2c56e358cf1d00b558d74262b5f929ee8cc7e73a" dependencies = [ "errno-dragonfly", "libc", - "windows-sys", + "windows-sys 0.48.0", ] [[package]] @@ -490,9 +478,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.10" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427" +checksum = "c85e1d9ab2eadba7e5040d4e09cbd6d072b76a557ad64e797c2cb9d4da21d7e4" dependencies = [ "cfg-if", "libc", @@ -512,25 +500,28 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" [[package]] -name = "hashbrown" -version = "0.14.0" +name = "hermit-abi" +version = "0.1.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", +] [[package]] name = "hermit-abi" -version = "0.1.19" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +checksum = "ee512640fe35acbfb4bb779db6f0d80704c2cacfa2e39b601ef3e3f47d1ae4c7" dependencies = [ "libc", ] [[package]] name = "hermit-abi" -version = "0.3.2" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "443144c8cdadd93ebf52ddb4056d257f5b52c04d3c804e657d19eb73fc33668b" +checksum = "fed44880c466736ef9a5c5b5facefb5ed0785676d0c02d612db14e54f0d84286" [[package]] name = "hex" @@ -593,17 +584,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" dependencies = [ "autocfg", - "hashbrown 0.12.3", -] - -[[package]] -name = "indexmap" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5477fe2230a79769d8dc68e0eabf5437907c0457a5614a9e8dddb67f65eb65d" -dependencies = [ - "equivalent", - "hashbrown 0.14.0", + "hashbrown", ] [[package]] @@ -621,20 +602,21 @@ version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2" dependencies = [ - "hermit-abi 0.3.2", + "hermit-abi 0.3.1", "libc", - "windows-sys", + "windows-sys 0.48.0", ] [[package]] name = "is-terminal" -version = "0.4.9" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b" +checksum = "adcf93614601c8129ddf72e2d5633df827ba6551541c6d8c59520a371475be1f" dependencies = [ - "hermit-abi 0.3.2", - "rustix 0.38.4", - "windows-sys", + "hermit-abi 0.3.1", + "io-lifetimes", + "rustix", + "windows-sys 0.48.0", ] [[package]] @@ -648,15 +630,15 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.8" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62b02a5381cc465bd3041d84623d0fa3b66738b52b8e2fc3bab8ad63ab032f4a" +checksum = "453ad9f582a441959e5f0d088b02ce04cfe8d51a8eaf077f12ac6d3e94164ca6" [[package]] name = "js-sys" -version = "0.3.64" +version = "0.3.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5f195fe497f702db0f318b07fdd68edb16955aed830df8363d837542f8f935a" +checksum = "2f37a4a5928311ac501dee68b3c7613a1037d0edb30c8e5427bd832d55d1b790" dependencies = [ "wasm-bindgen", ] @@ -706,9 +688,9 @@ checksum = "884e2677b40cc8c339eaefcb701c32ef1fd2493d71118dc0ca4b6a736c93bd67" [[package]] name = "libc" -version = "0.2.147" +version = "0.2.144" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3" +checksum = "2b00cc1c228a6782d0f076e7b232802e0c5689d41bb5df366f2a6b6621cfdfe1" [[package]] name = "libm" @@ -722,17 +704,11 @@ version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" -[[package]] -name = "linux-raw-sys" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09fc20d2ca12cb9f044c93e3bd6d32d523e6e2ec3db4f7b2939cd99026ecd3f0" - [[package]] name = "lock_api" -version = "0.4.10" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1cc9717a20b1bb222f333e6a92fd32f7d8a18ddc5a3191a11af45dcbf4dcd16" +checksum = "435011366fe56583b16cf956f9df0095b405b82d76425bc8981c0e22e60ec4df" dependencies = [ "autocfg", "scopeguard", @@ -740,9 +716,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.19" +version = "0.4.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b06a4cde4c0f271a446782e3eff8de789548ce57dbc8eca9292c27f4a42004b4" +checksum = "518ef76f2f87365916b142844c16d8fefd85039bc5699050210a7778ee1cd1de" [[package]] name = "logos" @@ -781,9 +757,9 @@ checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" [[package]] name = "memoffset" -version = "0.9.0" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c" +checksum = "d61c719bcfbcf5d62b3a09efa6088de8c54bc0bfcd3ea7ae39fcc186108b8de1" dependencies = [ "autocfg", ] @@ -828,11 +804,11 @@ dependencies = [ [[package]] name = "num_cpus" -version = "1.16.0" +version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" +checksum = "0fac9e2da13b5eb447a6ce3d392f23a29d8694bff781bf03a16cd9ac8697593b" dependencies = [ - "hermit-abi 0.3.2", + "hermit-abi 0.2.6", "libc", ] @@ -859,9 +835,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.18.0" +version = "1.17.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" +checksum = "9670a07f94779e00908f3e686eab508878ebb390ba6e604d3a284c00e8d0487b" [[package]] name = "oorandom" @@ -871,9 +847,9 @@ checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575" [[package]] name = "os_str_bytes" -version = "6.5.1" +version = "6.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d5d9eb14b174ee9aa2ef96dc2b94637a2d4b6e7cb873c7e171f0c20c6cf3eac" +checksum = "ceedf44fb00f2d1984b0bc98102627ce622e083e49a5bacdb3e514fa4238e267" [[package]] name = "parking_lot" @@ -887,22 +863,22 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.8" +version = "0.9.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93f00c865fe7cabf650081affecd3871070f26767e7b2070a3ffae14c654b447" +checksum = "9069cbb9f99e3a5083476ccb29ceb1de18b9118cafa53e90c9551235de2b9521" dependencies = [ "cfg-if", "libc", - "redox_syscall 0.3.5", + "redox_syscall 0.2.16", "smallvec", - "windows-targets", + "windows-sys 0.45.0", ] [[package]] name = "paste" -version = "1.0.13" +version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4b27ab7be369122c218afc2079489cdcb4b517c0a3fc386ff11e1fedfcc2b35" +checksum = "9f746c4065a8fa3fe23974dd82f15431cc8d40779821001404d10d2e79ca7d79" [[package]] name = "petgraph" @@ -911,7 +887,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4dd7d28ee937e54fe3080c91faa1c3a46c06de6252988a7f4592ba2310ef22a4" dependencies = [ "fixedbitset", - "indexmap 1.9.3", + "indexmap", ] [[package]] @@ -925,9 +901,9 @@ dependencies = [ [[package]] name = "plotters" -version = "0.3.5" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2c224ba00d7cadd4d5c660deaf2098e5e80e07846537c51f9cfa4be50c1fd45" +checksum = "2538b639e642295546c50fcd545198c9d64ee2a38620a628724a3b266d5fbf97" dependencies = [ "num-traits", "plotters-backend", @@ -938,15 +914,15 @@ dependencies = [ [[package]] name = "plotters-backend" -version = "0.3.5" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e76628b4d3a7581389a35d5b6e2139607ad7c75b17aed325f210aa91f4a9609" +checksum = "193228616381fecdc1224c62e96946dfbc73ff4384fba576e052ff8c1bea8142" [[package]] name = "plotters-svg" -version = "0.3.5" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38f6d39893cca0701371e3c27294f09797214b86f1fb951b89ade8ec04e2abab" +checksum = "f9a81d2759aae1dae668f783c308bc5c8ebd191ff4184aaa1b37f65a6ae5a56f" dependencies = [ "plotters-backend", ] @@ -985,9 +961,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.64" +version = "1.0.59" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78803b62cbf1f46fde80d7c0e803111524b9877184cfe7c3033659490ac7a7da" +checksum = "6aeca18b86b413c660b781aa319e4e2648a3e6f9eadc9b47e9038e6fe9f3451b" dependencies = [ "unicode-ident", ] @@ -999,7 +975,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4e35c06b98bf36aba164cc17cb25f7e232f5c4aeea73baa14b8a9f0d92dbfa65" dependencies = [ "bit-set", - "bitflags 1.3.2", + "bitflags", "byteorder", "lazy_static", "num-traits", @@ -1020,9 +996,9 @@ checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" [[package]] name = "quote" -version = "1.0.29" +version = "1.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "573015e8ab27661678357f27dc26460738fd2b6c86e46f386fde94cb5d913105" +checksum = "1b9ab9c7eadfd8df19006f1cf1a4aed13540ed5cbc047010ece5826e10825488" dependencies = [ "proc-macro2", ] @@ -1094,7 +1070,7 @@ version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" dependencies = [ - "bitflags 1.3.2", + "bitflags", ] [[package]] @@ -1103,7 +1079,7 @@ version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" dependencies = [ - "bitflags 1.3.2", + "bitflags", ] [[package]] @@ -1119,25 +1095,13 @@ dependencies = [ [[package]] name = "regex" -version = "1.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2eae68fc220f7cf2532e4494aded17545fce192d59cd996e0fe7887f4ceb575" -dependencies = [ - "aho-corasick", - "memchr", - "regex-automata", - "regex-syntax 0.7.4", -] - -[[package]] -name = "regex-automata" -version = "0.3.2" +version = "1.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83d3daa6976cffb758ec878f108ba0e062a45b2d6ca3a2cca965338855476caf" +checksum = "81ca098a9821bd52d6b24fd8b10bd081f47d39c22778cafaa75a2857a62c6390" dependencies = [ "aho-corasick", "memchr", - "regex-syntax 0.7.4", + "regex-syntax 0.7.2", ] [[package]] @@ -1148,42 +1112,29 @@ checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" [[package]] name = "regex-syntax" -version = "0.7.4" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5ea92a5b6195c6ef2a0295ea818b312502c6fc94dde986c5553242e18fd4ce2" +checksum = "436b050e76ed2903236f032a59761c1eb99e1b0aead2c257922771dab1fc8c78" [[package]] name = "rustix" -version = "0.37.23" +version = "0.37.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d69718bf81c6127a49dc64e44a742e8bb9213c0ff8869a22c308f84c1d4ab06" +checksum = "acf8729d8542766f1b2cf77eb034d52f40d375bb8b615d0b147089946e16613d" dependencies = [ - "bitflags 1.3.2", + "bitflags", "errno", "io-lifetimes", "libc", - "linux-raw-sys 0.3.8", - "windows-sys", -] - -[[package]] -name = "rustix" -version = "0.38.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a962918ea88d644592894bc6dc55acc6c0956488adcebbfb6e273506b7fd6e5" -dependencies = [ - "bitflags 2.3.3", - "errno", - "libc", - "linux-raw-sys 0.4.3", - "windows-sys", + "linux-raw-sys", + "windows-sys 0.48.0", ] [[package]] name = "rustversion" -version = "1.0.13" +version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc31bd9b61a32c31f9650d18add92aa83a49ba979c143eefd27fe7177b05bd5f" +checksum = "4f3208ce4d8448b3f3e7d168a73f5e0c43a61e32930de3bceeccedb388b6bf06" [[package]] name = "rusty-fork" @@ -1199,9 +1150,9 @@ dependencies = [ [[package]] name = "ryu" -version = "1.0.14" +version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe232bdf6be8c8de797b22184ee71118d63780ea42ac85b61d1baa6d3b782ae9" +checksum = "f91339c0467de62360649f8d3e185ca8de4224ff281f66000de5eb2a77a79041" [[package]] name = "same-file" @@ -1220,38 +1171,38 @@ checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" [[package]] name = "serde" -version = "1.0.171" +version = "1.0.163" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30e27d1e4fd7659406c492fd6cfaf2066ba8773de45ca75e855590f856dc34a9" +checksum = "2113ab51b87a539ae008b5c6c02dc020ffa39afd2d83cffcb3f4eb2722cebec2" dependencies = [ "serde_derive", ] [[package]] name = "serde_bytes" -version = "0.11.11" +version = "0.11.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a16be4fe5320ade08736447e3198294a5ea9a6d44dde6f35f0a5e06859c427a" +checksum = "416bda436f9aab92e02c8e10d49a15ddd339cea90b6e340fe51ed97abb548294" dependencies = [ "serde", ] [[package]] name = "serde_derive" -version = "1.0.171" +version = "1.0.163" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "389894603bd18c46fa56231694f8d827779c0951a667087194cf9de94ed24682" +checksum = "8c805777e3930c8883389c602315a24224bcc738b63905ef87cd1420353ea93e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.25", + "syn 2.0.18", ] [[package]] name = "serde_json" -version = "1.0.102" +version = "1.0.96" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5062a995d481b2308b6064e9af76011f2921c35f97b0468811ed9f6cd91dfed" +checksum = "057d394a50403bcac12672b2b18fb387ab6d289d957dab67dd201875391e52f1" dependencies = [ "itoa", "ryu", @@ -1271,9 +1222,9 @@ dependencies = [ [[package]] name = "sha2" -version = "0.10.7" +version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "479fb9d862239e610720565ca91403019f2f00410f1864c5aa7479b950a76ed8" +checksum = "82e6b795fe2e3b1e845bafcb27aa35405c4d47cdfc92af5fc8d3002f76cebdc0" dependencies = [ "cfg-if", "cpufeatures", @@ -1288,9 +1239,9 @@ checksum = "7bd3e3206899af3f8b12af284fafc038cc1dc2b41d1b89dd17297221c5d225de" [[package]] name = "smallvec" -version = "1.11.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62bb4feee49fdd9f707ef802e22365a35de4b7b299de4763d44bfea899442ff9" +checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" [[package]] name = "string_cache" @@ -1318,9 +1269,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.25" +version = "2.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15e3fc8c0c74267e2df136e5e5fb656a464158aa57624053375eb9c8c6e25ae2" +checksum = "32d41677bcbe24c20c52e7c70b0d8db04134c5d1066bf98662e2871ad200ea3e" dependencies = [ "proc-macro2", "quote", @@ -1329,16 +1280,15 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.6.0" +version = "3.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31c0432476357e58790aaa47a8efb0c5138f137343f3b5f23bd36a27e3b0a6d6" +checksum = "b9fbec84f381d5795b08656e4912bec604d162bff9291d6189a78f4c8ab87998" dependencies = [ - "autocfg", "cfg-if", "fastrand", "redox_syscall 0.3.5", - "rustix 0.37.23", - "windows-sys", + "rustix", + "windows-sys 0.45.0", ] [[package]] @@ -1369,22 +1319,22 @@ checksum = "222a222a5bfe1bba4a77b45ec488a741b3cb8872e5e499451fd7d0129c9c7c3d" [[package]] name = "thiserror" -version = "1.0.43" +version = "1.0.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a35fc5b8971143ca348fa6df4f024d4d55264f3468c71ad1c2f365b0a4d58c42" +checksum = "978c9a314bd8dc99be594bc3c175faaa9794be04a5a5e153caba6915336cebac" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.43" +version = "1.0.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "463fe12d7993d3b327787537ce8dd4dfa058de32fc2b195ef3cde03dc4771e8f" +checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.25", + "syn 2.0.18", ] [[package]] @@ -1414,17 +1364,17 @@ dependencies = [ [[package]] name = "toml_datetime" -version = "0.6.3" +version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cda73e2f1397b1262d6dfdcef8aafae14d1de7748d66822d3bfeeb6d03e5e4b" +checksum = "5a76a9312f5ba4c2dec6b9161fdf25d87ad8a09256ccea5a556fef03c706a10f" [[package]] name = "toml_edit" -version = "0.19.12" +version = "0.19.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c500344a19072298cd05a7224b3c0c629348b78692bf48466c5238656e315a78" +checksum = "2380d56e8670370eee6566b0bfd4265f65b3f432e8c6d85623f728d4fa31f739" dependencies = [ - "indexmap 2.0.0", + "indexmap", "toml_datetime", "winnow", ] @@ -1449,9 +1399,9 @@ checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" [[package]] name = "unicode-ident" -version = "1.0.10" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22049a19f4a68748a168c0fc439f9516686aa045927ff767eca0a85101fb6e73" +checksum = "b15811caf2415fb889178633e7724bad2509101cde276048e013b9def5e51fa0" [[package]] name = "unicode-width" @@ -1498,9 +1448,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.87" +version = "0.2.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7706a72ab36d8cb1f80ffbf0e071533974a60d0a308d01a5d0375bf60499a342" +checksum = "5bba0e8cb82ba49ff4e229459ff22a191bbe9a1cb3a341610c9c33efc27ddf73" dependencies = [ "cfg-if", "wasm-bindgen-macro", @@ -1508,24 +1458,24 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.87" +version = "0.2.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ef2b6d3c510e9625e5fe6f509ab07d66a760f0885d858736483c32ed7809abd" +checksum = "19b04bc93f9d6bdee709f6bd2118f57dd6679cf1176a1af464fca3ab0d66d8fb" dependencies = [ "bumpalo", "log", "once_cell", "proc-macro2", "quote", - "syn 2.0.25", + "syn 2.0.18", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-macro" -version = "0.2.87" +version = "0.2.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dee495e55982a3bd48105a7b947fd2a9b4a8ae3010041b9e0faab3f9cd028f1d" +checksum = "14d6b024f1a526bb0234f52840389927257beb670610081360e5a03c5df9c258" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -1533,28 +1483,28 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.87" +version = "0.2.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" +checksum = "e128beba882dd1eb6200e1dc92ae6c5dbaa4311aa7bb211ca035779e5efc39f8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.25", + "syn 2.0.18", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.87" +version = "0.2.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1" +checksum = "ed9d5b4305409d1fc9482fee2d7f9bcbf24b3972bf59817ef757e23982242a93" [[package]] name = "web-sys" -version = "0.3.64" +version = "0.3.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b85cbef8c220a6abc02aefd892dfc0fc23afb1c6a426316ec33253a3877249b" +checksum = "3bdd9ef4e984da1187bf8110c5cf5b845fbc87a23602cdf912386a76fcd3a7c2" dependencies = [ "js-sys", "wasm-bindgen", @@ -1591,66 +1541,132 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows-sys" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +dependencies = [ + "windows-targets 0.42.2", +] + [[package]] name = "windows-sys" version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" dependencies = [ - "windows-targets", + "windows-targets 0.48.0", +] + +[[package]] +name = "windows-targets" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" +dependencies = [ + "windows_aarch64_gnullvm 0.42.2", + "windows_aarch64_msvc 0.42.2", + "windows_i686_gnu 0.42.2", + "windows_i686_msvc 0.42.2", + "windows_x86_64_gnu 0.42.2", + "windows_x86_64_gnullvm 0.42.2", + "windows_x86_64_msvc 0.42.2", ] [[package]] name = "windows-targets" -version = "0.48.1" +version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05d4b17490f70499f20b9e791dcf6a299785ce8af4d709018206dc5b4953e95f" +checksum = "7b1eb6f0cd7c80c79759c929114ef071b87354ce476d9d94271031c0497adfd5" dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "windows_aarch64_gnullvm 0.48.0", + "windows_aarch64_msvc 0.48.0", + "windows_i686_gnu 0.48.0", + "windows_i686_msvc 0.48.0", + "windows_x86_64_gnu 0.48.0", + "windows_x86_64_gnullvm 0.48.0", + "windows_x86_64_msvc 0.48.0", ] +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" + [[package]] name = "windows_aarch64_gnullvm" version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc" +[[package]] +name = "windows_aarch64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" + [[package]] name = "windows_aarch64_msvc" version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3" +[[package]] +name = "windows_i686_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" + [[package]] name = "windows_i686_gnu" version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241" +[[package]] +name = "windows_i686_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" + [[package]] name = "windows_i686_msvc" version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00" +[[package]] +name = "windows_x86_64_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" + [[package]] name = "windows_x86_64_gnu" version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" + [[package]] name = "windows_x86_64_gnullvm" version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953" +[[package]] +name = "windows_x86_64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" + [[package]] name = "windows_x86_64_msvc" version = "0.48.0" @@ -1659,9 +1675,9 @@ checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" [[package]] name = "winnow" -version = "0.4.9" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81a2094c43cc94775293eaa0e499fbc30048a6d824ac82c0351a8c0bf9112529" +checksum = "61de7bac303dc551fe038e2b3cef0f571087a47571ea6e79a87692ac99b99699" dependencies = [ "memchr", ] From 283ff2a8636fe751fec005f44857964dd45e4baa Mon Sep 17 00:00:00 2001 From: Dragoljub Duric Date: Wed, 12 Jul 2023 13:21:37 +0000 Subject: [PATCH 32/70] update Cargo.lock --- examples/Cargo.lock | 3 +++ 1 file changed, 3 insertions(+) diff --git a/examples/Cargo.lock b/examples/Cargo.lock index 27039e82..f59cf0e8 100644 --- a/examples/Cargo.lock +++ b/examples/Cargo.lock @@ -415,6 +415,9 @@ dependencies = [ [[package]] name = "ic-stable-structures" version = "0.5.4" +dependencies = [ + "bit-vec", +] [[package]] name = "ic0" From 1b739fb86ce96e73596ee6e06b1c754e971cf7a5 Mon Sep 17 00:00:00 2001 From: Dragoljub Duric Date: Wed, 12 Jul 2023 13:29:30 +0000 Subject: [PATCH 33/70] update Cargo.lock --- examples/Cargo.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/Cargo.lock b/examples/Cargo.lock index e3a23352..ee15b482 100644 --- a/examples/Cargo.lock +++ b/examples/Cargo.lock @@ -414,7 +414,7 @@ dependencies = [ [[package]] name = "ic-stable-structures" -version = "0.5.4" +version = "0.5.6" dependencies = [ "bit-vec", ] From 9c04791bf0eddea5aa71affb79721f5cd89fda9f Mon Sep 17 00:00:00 2001 From: Dragoljub Duric Date: Thu, 13 Jul 2023 10:52:43 +0000 Subject: [PATCH 34/70] . --- src/memory_manager.rs | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/memory_manager.rs b/src/memory_manager.rs index af7d7442..6f9a7fbf 100644 --- a/src/memory_manager.rs +++ b/src/memory_manager.rs @@ -473,7 +473,7 @@ impl MemoryManagerInner { // Update the memory with the new size. self.memory_sizes_in_pages[id.0 as usize] = new_size; - // Update the header and return the old size. + // Update the header. self.save_header(); // Write in stable store that this bucket belongs to the memory with the provided `id`. @@ -482,6 +482,8 @@ impl MemoryManagerInner { bucket_allocations_address(BucketId(0)).get(), self.get_bucket_ids_in_bytes().as_ref(), ); + + // Return the old size. old_size as i64 } @@ -569,6 +571,15 @@ impl MemoryManagerInner { self.allocated_buckets -= 1; } } + // Update the header. + self.save_header(); + + // Write in stable store that no bucket belongs to the memory with the provided `id`. + write( + &self.memory, + bucket_allocations_address(BucketId(0)).get(), + self.get_bucket_ids_in_bytes().as_ref(), + ); } } From 876ad99267356ecbb3e78ece6fa48fd4ecaaa89f Mon Sep 17 00:00:00 2001 From: Dragoljub Duric Date: Thu, 13 Jul 2023 12:36:24 +0000 Subject: [PATCH 35/70] add new test --- src/memory_manager.rs | 44 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/src/memory_manager.rs b/src/memory_manager.rs index 6f9a7fbf..e9973a7a 100644 --- a/src/memory_manager.rs +++ b/src/memory_manager.rs @@ -1084,4 +1084,48 @@ mod test { ); assert_eq!(mem.size(), 1 + BUCKET_SIZE_IN_PAGES * 5); } + + #[test] + #[should_panic = "MemoryId(1): read out of bounds"] + fn can_be_reinitialized_from_memory_after_freeing() { + let mem = make_memory(); + let mem_mgr = MemoryManager::init(mem.clone()); + let memory_0 = mem_mgr.get(MemoryId(0)); + let memory_1 = mem_mgr.get(MemoryId(1)); + let memory_2 = mem_mgr.get(MemoryId(2)); + + assert_eq!(memory_0.grow(1), 0); + assert_eq!(memory_1.grow(1), 0); + assert_eq!(memory_2.grow(1), 0); + + memory_0.write(0, &[1, 2, 3]); + memory_1.write(0, &[4, 5, 6]); + memory_2.write(0, &[7, 8, 9]); + + let mut mem_mgr = MemoryManager::init(mem); + let memory_0 = mem_mgr.get(MemoryId(0)); + let memory_1 = mem_mgr.get(MemoryId(1)); + let memory_2 = mem_mgr.get(MemoryId(2)); + + let mut bytes = vec![0; 3]; + memory_0.read(0, &mut bytes); + assert_eq!(bytes, vec![1, 2, 3]); + + memory_1.read(0, &mut bytes); + assert_eq!(bytes, vec![4, 5, 6]); + + memory_2.read(0, &mut bytes); + assert_eq!(bytes, vec![7, 8, 9]); + + mem_mgr.free(MemoryId(1)); + + memory_0.read(0, &mut bytes); + assert_eq!(bytes, vec![1, 2, 3]); + + memory_2.read(0, &mut bytes); + assert_eq!(bytes, vec![7, 8, 9]); + + assert_eq!(memory_1.size(), 0); + memory_1.read(0, &mut bytes); + } } From c25901e08603e2ff34a8b7a63b9720b9836a988c Mon Sep 17 00:00:00 2001 From: Dragoljub Duric Date: Thu, 13 Jul 2023 12:42:33 +0000 Subject: [PATCH 36/70] add comments to the test --- src/memory_manager.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/memory_manager.rs b/src/memory_manager.rs index e9973a7a..fcc35433 100644 --- a/src/memory_manager.rs +++ b/src/memory_manager.rs @@ -1108,6 +1108,7 @@ mod test { let memory_2 = mem_mgr.get(MemoryId(2)); let mut bytes = vec![0; 3]; + // Check that data is correctly reinitialized. memory_0.read(0, &mut bytes); assert_eq!(bytes, vec![1, 2, 3]); @@ -1117,14 +1118,17 @@ mod test { memory_2.read(0, &mut bytes); assert_eq!(bytes, vec![7, 8, 9]); + // Free MemoryId 1. mem_mgr.free(MemoryId(1)); + // Check that data of MemoryId 0 and 2 is correctly reinitialized. memory_0.read(0, &mut bytes); assert_eq!(bytes, vec![1, 2, 3]); memory_2.read(0, &mut bytes); assert_eq!(bytes, vec![7, 8, 9]); + // Check that date of MemoryId 1 is freed. assert_eq!(memory_1.size(), 0); memory_1.read(0, &mut bytes); } From ba4875d21405ad179537a0874b56adcb39a5d615 Mon Sep 17 00:00:00 2001 From: Dragoljub Djuric Date: Mon, 17 Jul 2023 10:32:08 +0200 Subject: [PATCH 37/70] Update src/memory_manager.rs Co-authored-by: Islam El-Ashi --- src/memory_manager.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/memory_manager.rs b/src/memory_manager.rs index fcc35433..7bf061a1 100644 --- a/src/memory_manager.rs +++ b/src/memory_manager.rs @@ -1087,7 +1087,7 @@ mod test { #[test] #[should_panic = "MemoryId(1): read out of bounds"] - fn can_be_reinitialized_from_memory_after_freeing() { + fn reading_freed_memory_panics() { let mem = make_memory(); let mem_mgr = MemoryManager::init(mem.clone()); let memory_0 = mem_mgr.get(MemoryId(0)); From 25010ad587300a8620b698588d495ecdc79fb3a7 Mon Sep 17 00:00:00 2001 From: Dragoljub Duric Date: Mon, 17 Jul 2023 08:35:48 +0000 Subject: [PATCH 38/70] rename constants --- src/memory_manager.rs | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/src/memory_manager.rs b/src/memory_manager.rs index 7bf061a1..e3a51828 100644 --- a/src/memory_manager.rs +++ b/src/memory_manager.rs @@ -75,8 +75,8 @@ const BUCKETS_OFFSET_IN_BYTES: u64 = BUCKETS_OFFSET_IN_PAGES * WASM_PAGE_SIZE; // Reserved bytes in the header for future extensions. const HEADER_RESERVED_BYTES: usize = 32; -// Size of the bucked id in the memory. -const BUCKET_ID_LEN: usize = 15; +// Size of the bucked ID in the memory. +const BUCKET_ID_LEN_IN_BITS: usize = 15; /// A memory manager simulates multiple memories within a single memory. /// @@ -358,10 +358,11 @@ impl MemoryManagerInner { } fn load_layout_v2(memory: M, header: Header) -> Self { - const BYTE_SIZE: usize = 8; - let size_of_buckets_ind_in_bytes = - (MAX_NUM_BUCKETS as usize * BUCKET_ID_LEN + (BYTE_SIZE - 1)) / BYTE_SIZE; - let mut buckets = vec![0; size_of_buckets_ind_in_bytes]; + const BYTE_SIZE_IN_BITS: usize = 8; + const BUCKETS_INDEX_SIZE_IN_BYTES: usize = + (MAX_NUM_BUCKETS as usize * BUCKET_ID_LEN_IN_BITS + (BYTE_SIZE_IN_BITS - 1)) + / BYTE_SIZE_IN_BITS; + let mut buckets = vec![0; BUCKETS_INDEX_SIZE_IN_BYTES]; memory.read(bucket_allocations_address(BucketId(0)).get(), &mut buckets); let buckets_decompressed = bytes_to_bucket_indexes(&buckets); @@ -586,10 +587,10 @@ impl MemoryManagerInner { fn bytes_to_bucket_indexes(input: &[u8]) -> Vec { let mut bucket_ids = vec![]; let bit_vec = BitVec::from_bytes(input); - for bucket_order_number in 0..bit_vec.len() / BUCKET_ID_LEN { + for bucket_order_number in 0..bit_vec.len() / BUCKET_ID_LEN_IN_BITS { let mut bucket_id: u16 = 0; - for bucket_id_bit in 0..BUCKET_ID_LEN { - let next_bit = BUCKET_ID_LEN * bucket_order_number + bucket_id_bit; + for bucket_id_bit in 0..BUCKET_ID_LEN_IN_BITS { + let next_bit = BUCKET_ID_LEN_IN_BITS * bucket_order_number + bucket_id_bit; bucket_id <<= 1; if bit_vec.get(next_bit) == Some(true) { bucket_id |= 1; From d75b2541e612e3cbe96dc66c939a24bc568279ff Mon Sep 17 00:00:00 2001 From: Dragoljub Duric Date: Mon, 17 Jul 2023 11:32:22 +0000 Subject: [PATCH 39/70] add tests --- src/memory_manager.rs | 68 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/src/memory_manager.rs b/src/memory_manager.rs index e3a51828..45cf2b6d 100644 --- a/src/memory_manager.rs +++ b/src/memory_manager.rs @@ -1133,4 +1133,72 @@ mod test { assert_eq!(memory_1.size(), 0); memory_1.read(0, &mut bytes); } + + #[test] + fn freeing_already_free_memory() { + let mem = make_memory(); + let mut mem_mgr = MemoryManager::init(mem.clone()); + let memory_0 = mem_mgr.get(MemoryId(0)); + + assert_eq!(memory_0.grow(1), 0); + + assert_eq!(mem_mgr.get(MemoryId(0)).size(), 1); + + mem_mgr.free(MemoryId(0)); + + assert_eq!(mem_mgr.get(MemoryId(0)).size(), 0); + + mem_mgr.free(MemoryId(0)); + + assert_eq!(mem_mgr.get(MemoryId(0)).size(), 0); + } + + #[test] + fn grow_memory_after_freeing_it() { + let mem = make_memory(); + let mut mem_mgr = MemoryManager::init(mem.clone()); + let memory_0 = mem_mgr.get(MemoryId(0)); + + // grow and write to memory + assert_eq!(memory_0.grow(1), 0); + memory_0.write(0, &vec![7, 1, 5]); + + assert_eq!(mem_mgr.get(MemoryId(0)).size(), 1); + + let mut bytes = vec![0; 3]; + + // read from memory + memory_0.read(0, &mut bytes); + assert_eq!(bytes, vec![7, 1, 5]); + + // free memory + mem_mgr.free(MemoryId(0)); + + assert_eq!(mem_mgr.get(MemoryId(0)).size(), 0); + + // grow memory + assert_eq!(memory_0.grow(1), 0); + + assert_eq!(mem_mgr.get(MemoryId(0)).size(), 1); + + // check that old bucket is reassign to the memory + memory_0.read(0, &mut bytes); + assert_eq!(bytes, vec![7, 1, 5]); + + // try growing once more + assert_eq!(memory_0.grow(1), 1); + assert_eq!(mem_mgr.get(MemoryId(0)).size(), 2); + } + + #[test] + fn free_memory_that_was_not_used() { + let mem = make_memory(); + let mut mem_mgr = MemoryManager::init(mem.clone()); + let memory_0 = mem_mgr.get(MemoryId(0)); + assert_eq!(memory_0.grow(1), 0); + + mem_mgr.free(MemoryId(5)); + + + } } From 0f0bf51f6bc6ab93869b579234bfd0ba36eb006a Mon Sep 17 00:00:00 2001 From: Dragoljub Duric Date: Mon, 17 Jul 2023 11:41:01 +0000 Subject: [PATCH 40/70] refactor test --- src/memory_manager.rs | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/src/memory_manager.rs b/src/memory_manager.rs index 45cf2b6d..54ac49a6 100644 --- a/src/memory_manager.rs +++ b/src/memory_manager.rs @@ -1037,21 +1037,17 @@ mod test { #[test] fn free_memory_works() { let mem = make_memory(); - let initial_size = BUCKET_SIZE_IN_PAGES * 2; - // Grow the memory manually before passing it into the memory manager. - mem.grow(initial_size); - let mut mem_mgr = MemoryManager::init(mem.clone()); let memory_0 = mem_mgr.get(MemoryId(0)); let memory_1 = mem_mgr.get(MemoryId(1)); // Grow the memory by 1 page. assert_eq!(memory_0.grow(1), 0); - assert_eq!(mem.size(), initial_size); + assert_eq!(mem.size(), BUCKET_SIZE_IN_PAGES + 1); // Grow the memory by 1 page. assert_eq!(memory_1.grow(1), 0); - assert_eq!(mem.size(), initial_size + 1); + assert_eq!(mem.size(), 2 * BUCKET_SIZE_IN_PAGES + 1); // Grow the memory by BUCKET_SIZE_IN_PAGES more pages, which will cause the underlying // allocation to increase. @@ -1198,7 +1194,5 @@ mod test { assert_eq!(memory_0.grow(1), 0); mem_mgr.free(MemoryId(5)); - - } } From 961f1d50d1d7dec000f24a6bc85820e88312bdff Mon Sep 17 00:00:00 2001 From: Dragoljub Duric Date: Mon, 17 Jul 2023 12:03:41 +0000 Subject: [PATCH 41/70] refactor test --- src/memory_manager.rs | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/src/memory_manager.rs b/src/memory_manager.rs index 54ac49a6..25d9ac49 100644 --- a/src/memory_manager.rs +++ b/src/memory_manager.rs @@ -1132,8 +1132,7 @@ mod test { #[test] fn freeing_already_free_memory() { - let mem = make_memory(); - let mut mem_mgr = MemoryManager::init(mem.clone()); + let mut mem_mgr = MemoryManager::init(make_memory()); let memory_0 = mem_mgr.get(MemoryId(0)); assert_eq!(memory_0.grow(1), 0); @@ -1151,13 +1150,12 @@ mod test { #[test] fn grow_memory_after_freeing_it() { - let mem = make_memory(); - let mut mem_mgr = MemoryManager::init(mem.clone()); + let mut mem_mgr = MemoryManager::init(make_memory()); let memory_0 = mem_mgr.get(MemoryId(0)); // grow and write to memory assert_eq!(memory_0.grow(1), 0); - memory_0.write(0, &vec![7, 1, 5]); + memory_0.write(0, &[7, 1, 5]); assert_eq!(mem_mgr.get(MemoryId(0)).size(), 1); @@ -1165,7 +1163,7 @@ mod test { // read from memory memory_0.read(0, &mut bytes); - assert_eq!(bytes, vec![7, 1, 5]); + assert_eq!(bytes, &[7, 1, 5]); // free memory mem_mgr.free(MemoryId(0)); @@ -1179,7 +1177,7 @@ mod test { // check that old bucket is reassign to the memory memory_0.read(0, &mut bytes); - assert_eq!(bytes, vec![7, 1, 5]); + assert_eq!(bytes, &[7, 1, 5]); // try growing once more assert_eq!(memory_0.grow(1), 1); @@ -1188,8 +1186,7 @@ mod test { #[test] fn free_memory_that_was_not_used() { - let mem = make_memory(); - let mut mem_mgr = MemoryManager::init(mem.clone()); + let mut mem_mgr = MemoryManager::init(make_memory()); let memory_0 = mem_mgr.get(MemoryId(0)); assert_eq!(memory_0.grow(1), 0); From 168a755a6f62d75e341d6d487914125be952496b Mon Sep 17 00:00:00 2001 From: Dragoljub Duric Date: Mon, 17 Jul 2023 12:28:17 +0000 Subject: [PATCH 42/70] change unallocated_buckets to freed_buckets --- src/memory_manager.rs | 93 ++++++++++++++++++++++++++++++++----------- 1 file changed, 69 insertions(+), 24 deletions(-) diff --git a/src/memory_manager.rs b/src/memory_manager.rs index 25d9ac49..4c1078be 100644 --- a/src/memory_manager.rs +++ b/src/memory_manager.rs @@ -283,8 +283,11 @@ struct MemoryManagerInner { // A map mapping each managed memory to the bucket ids that are allocated to it. memory_buckets: BTreeMap>, - // A list of the all unallocated buckets. - unallocated_buckets: LinkedList, + // A list of freed buckets. + freed_buckets: LinkedList, + + // Bucket ID of the first bucket that was not used. + first_unused_bucket: BucketId, } impl MemoryManagerInner { @@ -313,13 +316,8 @@ impl MemoryManagerInner { memory_sizes_in_pages: [0; MAX_NUM_MEMORIES as usize], memory_buckets: BTreeMap::new(), bucket_size_in_pages, - unallocated_buckets: { - let mut list = LinkedList::new(); - for i in 0..MAX_NUM_BUCKETS { - list.push_back(BucketId(i as u16)); - } - list - }, + freed_buckets: LinkedList::new(), + first_unused_bucket: BucketId(0), }; mem_mgr.save_header(); @@ -330,21 +328,39 @@ impl MemoryManagerInner { fn load_layout_v1(memory: M, header: Header) -> Self { let mut buckets = vec![0; MAX_NUM_BUCKETS as usize]; memory.read(bucket_allocations_address(BucketId(0)).get(), &mut buckets); - let mut unallocated: BTreeSet = (0..MAX_NUM_BUCKETS as u16).collect(); + let mut unallocated_buckets: BTreeSet = (0..MAX_NUM_BUCKETS as u16).collect(); let mut memory_buckets = BTreeMap::new(); + let mut last_occupied_bucket: i32 = -1; for (bucket_idx, memory) in buckets.into_iter().enumerate() { if memory != UNALLOCATED_BUCKET_MARKER { + last_occupied_bucket = std::cmp::max(last_occupied_bucket, bucket_idx as i32); memory_buckets .entry(MemoryId(memory)) .or_insert_with(Vec::new) .push(BucketId(bucket_idx as u16)); - unallocated.remove(&(bucket_idx as u16)); + unallocated_buckets.remove(&(bucket_idx as u16)); } } - let mut unallocated_buckets = LinkedList::new(); - for i in unallocated.iter() { - unallocated_buckets.push_back(BucketId(*i)); + if last_occupied_bucket == -1 { + return Self { + memory, + allocated_buckets: header.num_allocated_buckets, + bucket_size_in_pages: header.bucket_size_in_pages, + memory_sizes_in_pages: header.memory_sizes_in_pages, + memory_buckets, + freed_buckets: LinkedList::new(), + first_unused_bucket: BucketId(0), + }; + } + + let mut freed_buckets = LinkedList::new(); + let first_unused_bucket = BucketId((last_occupied_bucket + 1) as u16); + + for i in unallocated_buckets.iter() { + if *i < first_unused_bucket.0 { + freed_buckets.push_back(BucketId(*i)); + } } Self { @@ -353,7 +369,8 @@ impl MemoryManagerInner { bucket_size_in_pages: header.bucket_size_in_pages, memory_sizes_in_pages: header.memory_sizes_in_pages, memory_buckets, - unallocated_buckets, + freed_buckets, + first_unused_bucket, } } @@ -367,9 +384,11 @@ impl MemoryManagerInner { let buckets_decompressed = bytes_to_bucket_indexes(&buckets); - let mut unallocated_buckets_set: BTreeSet = (0..MAX_NUM_BUCKETS as u16).collect(); + let mut unallocated_buckets: BTreeSet = (0..MAX_NUM_BUCKETS as u16).collect(); let mut memory_buckets = BTreeMap::new(); + let mut last_occupied_bucket: i32 = -1; + let mut j = 0; for (memory, memory_size_in_pages) in header.memory_sizes_in_pages.into_iter().enumerate() { let memory_size_in_buckets = @@ -378,8 +397,9 @@ impl MemoryManagerInner { let mut vec_buckets = vec![]; for _ in 0..memory_size_in_buckets { let bucket = buckets_decompressed[j]; + last_occupied_bucket = std::cmp::max(bucket.0 as i32, last_occupied_bucket); vec_buckets.push(bucket); - unallocated_buckets_set.remove(&bucket.0); + unallocated_buckets.remove(&bucket.0); j += 1; } memory_buckets @@ -387,9 +407,25 @@ impl MemoryManagerInner { .or_insert(vec_buckets); } - let mut unallocated_buckets = LinkedList::new(); - for i in unallocated_buckets_set.iter() { - unallocated_buckets.push_back(BucketId(*i)); + if last_occupied_bucket == -1 { + return Self { + memory, + allocated_buckets: header.num_allocated_buckets, + bucket_size_in_pages: header.bucket_size_in_pages, + memory_sizes_in_pages: header.memory_sizes_in_pages, + memory_buckets, + freed_buckets: LinkedList::new(), + first_unused_bucket: BucketId(0), + }; + } + + let first_unused_bucket = BucketId((last_occupied_bucket + 1) as u16); + + let mut freed_buckets = LinkedList::new(); + for i in unallocated_buckets.iter() { + if *i < first_unused_bucket.0 { + freed_buckets.push_back(BucketId(*i)); + } } Self { @@ -398,7 +434,8 @@ impl MemoryManagerInner { bucket_size_in_pages: header.bucket_size_in_pages, memory_sizes_in_pages: header.memory_sizes_in_pages, memory_buckets, - unallocated_buckets, + freed_buckets, + first_unused_bucket, } } @@ -447,9 +484,17 @@ impl MemoryManagerInner { // Allocate new buckets as needed. for _ in 0..new_buckets_needed { - let new_bucket_id = match self.unallocated_buckets.pop_front() { + let new_bucket_id = match self.freed_buckets.pop_front() { Some(t) => t, - None => panic!("{id:?}: grow failed"), + None => { + if self.first_unused_bucket.0 != MAX_NUM_BUCKETS as u16 { + let new_id = self.first_unused_bucket; + self.first_unused_bucket = BucketId(self.first_unused_bucket.0 + 1); + new_id + } else { + panic!("{id:?}: grow failed"); + } + } }; self.memory_buckets @@ -568,7 +613,7 @@ impl MemoryManagerInner { let buckets = self.memory_buckets.remove(&id); if let Some(vec_buckets) = buckets { for bucket in vec_buckets { - self.unallocated_buckets.push_front(bucket); + self.freed_buckets.push_front(bucket); self.allocated_buckets -= 1; } } From b365c46dc458ea9b5bd651e363ec6b7db11e18fb Mon Sep 17 00:00:00 2001 From: Dragoljub Duric Date: Mon, 17 Jul 2023 13:41:10 +0000 Subject: [PATCH 43/70] add test V1 -> V2 --- src/memory_manager.rs | 245 ++++++++++++++++++++++++++++++++++++++---- 1 file changed, 226 insertions(+), 19 deletions(-) diff --git a/src/memory_manager.rs b/src/memory_manager.rs index 4c1078be..214e8bd2 100644 --- a/src/memory_manager.rs +++ b/src/memory_manager.rs @@ -206,6 +206,16 @@ impl MemoryManager { } } + #[cfg(test)] + pub fn init_v1(memory: M) -> Self { + Self { + inner: Rc::new(RefCell::new(MemoryManagerInner::init_v1( + memory, + BUCKET_SIZE_IN_PAGES as u16, + ))), + } + } + /// Returns the memory associated with the given ID. pub fn get(&self, id: MemoryId) -> VirtualMemory { VirtualMemory { @@ -268,6 +278,23 @@ impl Memory for VirtualMemory { } } +impl VirtualMemory { + #[cfg(test)] + fn grow_v1(&self, pages: u64) -> i64 { + self.memory_manager.borrow_mut().grow_v1(self.id, pages) + } + + #[cfg(test)] + fn read_v1(&self, offset: u64, dst: &mut [u8]) { + self.memory_manager.borrow().read_v1(self.id, offset, dst) + } + + #[cfg(test)] + fn write_v1(&self, offset: u64, src: &[u8]) { + self.memory_manager.borrow().write_v1(self.id, offset, src) + } +} + #[derive(Clone)] struct MemoryManagerInner { memory: M, @@ -325,6 +352,49 @@ impl MemoryManagerInner { mem_mgr } + #[cfg(test)] + fn init_v1(memory: M, bucket_size_in_pages: u16) -> Self { + if memory.size() == 0 { + // Memory is empty. Create a new map. + return Self::new(memory, bucket_size_in_pages); + } + + // Check if the magic in the memory corresponds to this object. + let mut dst = vec![0; 3]; + memory.read(0, &mut dst); + if dst != MAGIC { + // No memory manager found. Create a new instance. + MemoryManagerInner::new_v1(memory, bucket_size_in_pages) + } else { + // The memory already contains a memory manager. Load it. + MemoryManagerInner::load(memory) + } + } + + #[cfg(test)] + fn new_v1(memory: M, bucket_size_in_pages: u16) -> Self { + let mem_mgr = Self { + memory, + allocated_buckets: 0, + memory_sizes_in_pages: [0; MAX_NUM_MEMORIES as usize], + memory_buckets: BTreeMap::new(), + bucket_size_in_pages, + freed_buckets: LinkedList::new(), + first_unused_bucket: BucketId(0), + }; + + mem_mgr.save_header_v1(); + + // Mark all the buckets as unallocated. + write( + &mem_mgr.memory, + bucket_allocations_address(BucketId(0)).get(), + &[UNALLOCATED_BUCKET_MARKER; MAX_NUM_BUCKETS as usize], + ); + + mem_mgr + } + fn load_layout_v1(memory: M, header: Header) -> Self { let mut buckets = vec![0; MAX_NUM_BUCKETS as usize]; memory.read(bucket_allocations_address(BucketId(0)).get(), &mut buckets); @@ -407,8 +477,8 @@ impl MemoryManagerInner { .or_insert(vec_buckets); } - if last_occupied_bucket == -1 { - return Self { + let mem_mgr = if last_occupied_bucket == -1 { + Self { memory, allocated_buckets: header.num_allocated_buckets, bucket_size_in_pages: header.bucket_size_in_pages, @@ -416,27 +486,39 @@ impl MemoryManagerInner { memory_buckets, freed_buckets: LinkedList::new(), first_unused_bucket: BucketId(0), - }; - } + } + } else { + let first_unused_bucket = BucketId((last_occupied_bucket + 1) as u16); - let first_unused_bucket = BucketId((last_occupied_bucket + 1) as u16); + let mut freed_buckets = LinkedList::new(); + for i in unallocated_buckets.iter() { + if *i < first_unused_bucket.0 { + freed_buckets.push_back(BucketId(*i)); + } + } - let mut freed_buckets = LinkedList::new(); - for i in unallocated_buckets.iter() { - if *i < first_unused_bucket.0 { - freed_buckets.push_back(BucketId(*i)); + Self { + memory, + allocated_buckets: header.num_allocated_buckets, + bucket_size_in_pages: header.bucket_size_in_pages, + memory_sizes_in_pages: header.memory_sizes_in_pages, + memory_buckets, + freed_buckets, + first_unused_bucket, } - } + }; - Self { - memory, - allocated_buckets: header.num_allocated_buckets, - bucket_size_in_pages: header.bucket_size_in_pages, - memory_sizes_in_pages: header.memory_sizes_in_pages, - memory_buckets, - freed_buckets, - first_unused_bucket, - } + // Update the header. + mem_mgr.save_header(); + + // Rewrite the stable store to be layout V2. + write( + &mem_mgr.memory, + bucket_allocations_address(BucketId(0)).get(), + mem_mgr.get_bucket_ids_in_bytes().as_ref(), + ); + + mem_mgr } fn load(memory: M) -> Self { @@ -627,6 +709,106 @@ impl MemoryManagerInner { self.get_bucket_ids_in_bytes().as_ref(), ); } + + #[cfg(test)] + fn save_header_v1(&self) { + let header = Header { + magic: *MAGIC, + version: LAYOUT_VERSION_V1, + num_allocated_buckets: self.allocated_buckets, + bucket_size_in_pages: self.bucket_size_in_pages, + _reserved: [0; HEADER_RESERVED_BYTES], + memory_sizes_in_pages: self.memory_sizes_in_pages, + }; + + write_struct(&header, Address::from(0), &self.memory); + } + + #[cfg(test)] + fn grow_v1(&mut self, id: MemoryId, pages: u64) -> i64 { + // Compute how many additional buckets are needed. + let old_size = self.memory_size(id); + let new_size = old_size + pages; + let current_buckets = self.num_buckets_needed(old_size); + let required_buckets = self.num_buckets_needed(new_size); + let new_buckets_needed = required_buckets - current_buckets; + + if new_buckets_needed + self.allocated_buckets as u64 > MAX_NUM_BUCKETS { + // Exceeded the memory that can be managed. + return -1; + } + + // Allocate new buckets as needed. + for _ in 0..new_buckets_needed { + let new_bucket_id = BucketId(self.allocated_buckets); + + self.memory_buckets + .entry(id) + .or_insert_with(Vec::new) + .push(new_bucket_id); + + // Write in stable store that this bucket belongs to the memory with the provided `id`. + write( + &self.memory, + bucket_allocations_address(new_bucket_id).get(), + &[id.0], + ); + + self.allocated_buckets += 1; + } + + // Grow the underlying memory if necessary. + let pages_needed = BUCKETS_OFFSET_IN_PAGES + + self.bucket_size_in_pages as u64 * self.allocated_buckets as u64; + if pages_needed > self.memory.size() { + let additional_pages_needed = pages_needed - self.memory.size(); + let prev_pages = self.memory.grow(additional_pages_needed); + if prev_pages == -1 { + panic!("{id:?}: grow failed"); + } + } + + // Update the memory with the new size. + self.memory_sizes_in_pages[id.0 as usize] = new_size; + + // Update the header and return the old size. + self.save_header(); + old_size as i64 + } + + #[cfg(test)] + fn write_v1(&self, id: MemoryId, offset: u64, src: &[u8]) { + if (offset + src.len() as u64) > self.memory_size(id) * WASM_PAGE_SIZE { + panic!("{id:?}: write out of bounds"); + } + + let mut bytes_written = 0; + for Segment { address, length } in self.bucket_iter(id, offset, src.len()) { + self.memory.write( + address.get(), + &src[bytes_written as usize..(bytes_written + length.get()) as usize], + ); + + bytes_written += length.get(); + } + } + + #[cfg(test)] + fn read_v1(&self, id: MemoryId, offset: u64, dst: &mut [u8]) { + if (offset + dst.len() as u64) > self.memory_size(id) * WASM_PAGE_SIZE { + panic!("{id:?}: read out of bounds"); + } + + let mut bytes_read = 0; + for Segment { address, length } in self.bucket_iter(id, offset, dst.len()) { + self.memory.read( + address.get(), + &mut dst[bytes_read as usize..(bytes_read + length.get()) as usize], + ); + + bytes_read += length.get(); + } + } } fn bytes_to_bucket_indexes(input: &[u8]) -> Vec { @@ -1237,4 +1419,29 @@ mod test { mem_mgr.free(MemoryId(5)); } + + #[test] + fn upgrade_from_v1_to_v2() { + // Create a memory with layout V1. + let mem = make_memory(); + let mem_mgr = MemoryManager::init_v1(mem.clone()); + let memory = mem_mgr.get(MemoryId(0)); + assert_eq!(memory.grow_v1(1), 0); + assert_eq!(memory.size(), 1); + + memory.write_v1(0, &[8, 1, 9]); + + let mut bytes = vec![0; 3]; + memory.read_v1(0, &mut bytes); + assert_eq!(bytes, vec![8, 1, 9]); + + // Upgrade to layout V2. + + let mem_mgr_v2 = MemoryManager::init(mem); + let memory = mem_mgr_v2.get(MemoryId(0)); + let mut bytes = vec![0; 3]; + assert_eq!(memory.size(), 1); + memory.read(0, &mut bytes); + assert_eq!(bytes, vec![8, 1, 9]); + } } From 2f506a0d6bb70469005c242aa5cdf02c57682f8b Mon Sep 17 00:00:00 2001 From: Dragoljub Duric Date: Mon, 17 Jul 2023 13:43:38 +0000 Subject: [PATCH 44/70] change order of freed buckets --- src/memory_manager.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/memory_manager.rs b/src/memory_manager.rs index 214e8bd2..aa32bbfc 100644 --- a/src/memory_manager.rs +++ b/src/memory_manager.rs @@ -695,7 +695,7 @@ impl MemoryManagerInner { let buckets = self.memory_buckets.remove(&id); if let Some(vec_buckets) = buckets { for bucket in vec_buckets { - self.freed_buckets.push_front(bucket); + self.freed_buckets.push_back(bucket); self.allocated_buckets -= 1; } } From cfa231922b15f182b10edca14d3249308766fb70 Mon Sep 17 00:00:00 2001 From: Dragoljub Duric Date: Mon, 17 Jul 2023 13:59:45 +0000 Subject: [PATCH 45/70] add method bucket_indexes_offset --- src/memory_manager.rs | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/memory_manager.rs b/src/memory_manager.rs index aa32bbfc..07c57ce3 100644 --- a/src/memory_manager.rs +++ b/src/memory_manager.rs @@ -450,7 +450,7 @@ impl MemoryManagerInner { (MAX_NUM_BUCKETS as usize * BUCKET_ID_LEN_IN_BITS + (BYTE_SIZE_IN_BITS - 1)) / BYTE_SIZE_IN_BITS; let mut buckets = vec![0; BUCKETS_INDEX_SIZE_IN_BYTES]; - memory.read(bucket_allocations_address(BucketId(0)).get(), &mut buckets); + memory.read(bucket_indexes_offset().get(), &mut buckets); let buckets_decompressed = bytes_to_bucket_indexes(&buckets); @@ -514,7 +514,7 @@ impl MemoryManagerInner { // Rewrite the stable store to be layout V2. write( &mem_mgr.memory, - bucket_allocations_address(BucketId(0)).get(), + bucket_indexes_offset().get(), mem_mgr.get_bucket_ids_in_bytes().as_ref(), ); @@ -607,7 +607,7 @@ impl MemoryManagerInner { // Write in stable store that this bucket belongs to the memory with the provided `id`. write( &self.memory, - bucket_allocations_address(BucketId(0)).get(), + bucket_indexes_offset().get(), self.get_bucket_ids_in_bytes().as_ref(), ); @@ -705,7 +705,7 @@ impl MemoryManagerInner { // Write in stable store that no bucket belongs to the memory with the provided `id`. write( &self.memory, - bucket_allocations_address(BucketId(0)).get(), + bucket_indexes_offset().get(), self.get_bucket_ids_in_bytes().as_ref(), ); } @@ -933,6 +933,10 @@ fn bucket_allocations_address(id: BucketId) -> Address { Address::from(0) + Header::size() + Bytes::from(id.0) } +fn bucket_indexes_offset() -> Address { + Address::from(0) + Header::size() +} + #[cfg(test)] mod test { use super::*; From d5120b276b59d8db70bc46e4dabfef5b736a4bcf Mon Sep 17 00:00:00 2001 From: Dragoljub Djuric Date: Tue, 18 Jul 2023 11:11:13 +0200 Subject: [PATCH 46/70] Update src/memory_manager.rs Co-authored-by: Islam El-Ashi --- src/memory_manager.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/memory_manager.rs b/src/memory_manager.rs index 07c57ce3..1334e0e7 100644 --- a/src/memory_manager.rs +++ b/src/memory_manager.rs @@ -75,7 +75,7 @@ const BUCKETS_OFFSET_IN_BYTES: u64 = BUCKETS_OFFSET_IN_PAGES * WASM_PAGE_SIZE; // Reserved bytes in the header for future extensions. const HEADER_RESERVED_BYTES: usize = 32; -// Size of the bucked ID in the memory. +// Size of the bucket ID in the header. const BUCKET_ID_LEN_IN_BITS: usize = 15; /// A memory manager simulates multiple memories within a single memory. From 8a229fb141ece569f18c02395cf61be695c5b26e Mon Sep 17 00:00:00 2001 From: Dragoljub Duric Date: Tue, 18 Jul 2023 09:15:14 +0000 Subject: [PATCH 47/70] refactor load_layout_v1 --- src/memory_manager.rs | 29 ++--------------------------- 1 file changed, 2 insertions(+), 27 deletions(-) diff --git a/src/memory_manager.rs b/src/memory_manager.rs index 1334e0e7..c52b018c 100644 --- a/src/memory_manager.rs +++ b/src/memory_manager.rs @@ -398,38 +398,13 @@ impl MemoryManagerInner { fn load_layout_v1(memory: M, header: Header) -> Self { let mut buckets = vec![0; MAX_NUM_BUCKETS as usize]; memory.read(bucket_allocations_address(BucketId(0)).get(), &mut buckets); - let mut unallocated_buckets: BTreeSet = (0..MAX_NUM_BUCKETS as u16).collect(); let mut memory_buckets = BTreeMap::new(); - let mut last_occupied_bucket: i32 = -1; for (bucket_idx, memory) in buckets.into_iter().enumerate() { if memory != UNALLOCATED_BUCKET_MARKER { - last_occupied_bucket = std::cmp::max(last_occupied_bucket, bucket_idx as i32); memory_buckets .entry(MemoryId(memory)) .or_insert_with(Vec::new) .push(BucketId(bucket_idx as u16)); - unallocated_buckets.remove(&(bucket_idx as u16)); - } - } - - if last_occupied_bucket == -1 { - return Self { - memory, - allocated_buckets: header.num_allocated_buckets, - bucket_size_in_pages: header.bucket_size_in_pages, - memory_sizes_in_pages: header.memory_sizes_in_pages, - memory_buckets, - freed_buckets: LinkedList::new(), - first_unused_bucket: BucketId(0), - }; - } - - let mut freed_buckets = LinkedList::new(); - let first_unused_bucket = BucketId((last_occupied_bucket + 1) as u16); - - for i in unallocated_buckets.iter() { - if *i < first_unused_bucket.0 { - freed_buckets.push_back(BucketId(*i)); } } @@ -439,8 +414,8 @@ impl MemoryManagerInner { bucket_size_in_pages: header.bucket_size_in_pages, memory_sizes_in_pages: header.memory_sizes_in_pages, memory_buckets, - freed_buckets, - first_unused_bucket, + freed_buckets: LinkedList::new(), + first_unused_bucket: BucketId(header.num_allocated_buckets), } } From 4a03b51e673e75ad62f17a99f4f386d627429d0a Mon Sep 17 00:00:00 2001 From: Dragoljub Duric Date: Tue, 18 Jul 2023 09:38:31 +0000 Subject: [PATCH 48/70] . --- src/memory_manager.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/memory_manager.rs b/src/memory_manager.rs index c52b018c..871a05fc 100644 --- a/src/memory_manager.rs +++ b/src/memory_manager.rs @@ -399,6 +399,7 @@ impl MemoryManagerInner { let mut buckets = vec![0; MAX_NUM_BUCKETS as usize]; memory.read(bucket_allocations_address(BucketId(0)).get(), &mut buckets); let mut memory_buckets = BTreeMap::new(); + for (bucket_idx, memory) in buckets.into_iter().enumerate() { if memory != UNALLOCATED_BUCKET_MARKER { memory_buckets From 2eff3ae0b6f18a70294e822d5fa3052716e77110 Mon Sep 17 00:00:00 2001 From: Dragoljub Duric Date: Thu, 20 Jul 2023 11:52:52 +0000 Subject: [PATCH 49/70] make freed_buckets BTreeSet --- src/memory_manager.rs | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/memory_manager.rs b/src/memory_manager.rs index 871a05fc..e8b58ff1 100644 --- a/src/memory_manager.rs +++ b/src/memory_manager.rs @@ -47,10 +47,10 @@ use crate::{ types::{Address, Bytes}, write, write_struct, Memory, WASM_PAGE_SIZE, }; +use std::cmp::min; use std::collections::BTreeMap; use std::rc::Rc; use std::{cell::RefCell, collections::BTreeSet}; -use std::{cmp::min, collections::LinkedList}; const MAGIC: &[u8; 3] = b"MGR"; const LAYOUT_VERSION_V1: u8 = 1; @@ -311,7 +311,7 @@ struct MemoryManagerInner { memory_buckets: BTreeMap>, // A list of freed buckets. - freed_buckets: LinkedList, + freed_buckets: BTreeSet, // Bucket ID of the first bucket that was not used. first_unused_bucket: BucketId, @@ -343,7 +343,7 @@ impl MemoryManagerInner { memory_sizes_in_pages: [0; MAX_NUM_MEMORIES as usize], memory_buckets: BTreeMap::new(), bucket_size_in_pages, - freed_buckets: LinkedList::new(), + freed_buckets: BTreeSet::new(), first_unused_bucket: BucketId(0), }; @@ -379,7 +379,7 @@ impl MemoryManagerInner { memory_sizes_in_pages: [0; MAX_NUM_MEMORIES as usize], memory_buckets: BTreeMap::new(), bucket_size_in_pages, - freed_buckets: LinkedList::new(), + freed_buckets: BTreeSet::new(), first_unused_bucket: BucketId(0), }; @@ -415,7 +415,7 @@ impl MemoryManagerInner { bucket_size_in_pages: header.bucket_size_in_pages, memory_sizes_in_pages: header.memory_sizes_in_pages, memory_buckets, - freed_buckets: LinkedList::new(), + freed_buckets: BTreeSet::new(), first_unused_bucket: BucketId(header.num_allocated_buckets), } } @@ -460,16 +460,16 @@ impl MemoryManagerInner { bucket_size_in_pages: header.bucket_size_in_pages, memory_sizes_in_pages: header.memory_sizes_in_pages, memory_buckets, - freed_buckets: LinkedList::new(), + freed_buckets: BTreeSet::new(), first_unused_bucket: BucketId(0), } } else { let first_unused_bucket = BucketId((last_occupied_bucket + 1) as u16); - let mut freed_buckets = LinkedList::new(); + let mut freed_buckets = BTreeSet::new(); for i in unallocated_buckets.iter() { if *i < first_unused_bucket.0 { - freed_buckets.push_back(BucketId(*i)); + freed_buckets.insert(BucketId(*i)); } } @@ -542,7 +542,7 @@ impl MemoryManagerInner { // Allocate new buckets as needed. for _ in 0..new_buckets_needed { - let new_bucket_id = match self.freed_buckets.pop_front() { + let new_bucket_id = match self.freed_buckets.pop_first() { Some(t) => t, None => { if self.first_unused_bucket.0 != MAX_NUM_BUCKETS as u16 { @@ -671,7 +671,7 @@ impl MemoryManagerInner { let buckets = self.memory_buckets.remove(&id); if let Some(vec_buckets) = buckets { for bucket in vec_buckets { - self.freed_buckets.push_back(bucket); + self.freed_buckets.insert(bucket); self.allocated_buckets -= 1; } } @@ -902,7 +902,7 @@ impl MemoryId { } // Referring to a bucket. -#[derive(Clone, Copy, Debug, PartialEq)] +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] struct BucketId(u16); fn bucket_allocations_address(id: BucketId) -> Address { From b7f4180aa69c9f09a3c2e5edfe4b598ed3d008f9 Mon Sep 17 00:00:00 2001 From: Dragoljub Duric Date: Thu, 20 Jul 2023 11:59:36 +0000 Subject: [PATCH 50/70] change initialization of freed_buckets --- src/memory_manager.rs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/memory_manager.rs b/src/memory_manager.rs index e8b58ff1..6e832944 100644 --- a/src/memory_manager.rs +++ b/src/memory_manager.rs @@ -430,7 +430,6 @@ impl MemoryManagerInner { let buckets_decompressed = bytes_to_bucket_indexes(&buckets); - let mut unallocated_buckets: BTreeSet = (0..MAX_NUM_BUCKETS as u16).collect(); let mut memory_buckets = BTreeMap::new(); let mut last_occupied_bucket: i32 = -1; @@ -445,7 +444,6 @@ impl MemoryManagerInner { let bucket = buckets_decompressed[j]; last_occupied_bucket = std::cmp::max(bucket.0 as i32, last_occupied_bucket); vec_buckets.push(bucket); - unallocated_buckets.remove(&bucket.0); j += 1; } memory_buckets @@ -465,12 +463,14 @@ impl MemoryManagerInner { } } else { let first_unused_bucket = BucketId((last_occupied_bucket + 1) as u16); - let mut freed_buckets = BTreeSet::new(); - for i in unallocated_buckets.iter() { - if *i < first_unused_bucket.0 { - freed_buckets.insert(BucketId(*i)); - } + + for id in 0..last_occupied_bucket as u16 { + freed_buckets.insert(BucketId(id)); + } + + for id in buckets_decompressed.iter() { + freed_buckets.remove(id); } Self { From f8cf339f0ff3c86ad0b25f1a0e4e4c158ae965b1 Mon Sep 17 00:00:00 2001 From: Dragoljub Duric Date: Thu, 20 Jul 2023 12:30:07 +0000 Subject: [PATCH 51/70] remove field first_unallocated_bucket --- src/memory_manager.rs | 24 +++++++----------------- 1 file changed, 7 insertions(+), 17 deletions(-) diff --git a/src/memory_manager.rs b/src/memory_manager.rs index 6e832944..caab522b 100644 --- a/src/memory_manager.rs +++ b/src/memory_manager.rs @@ -312,9 +312,6 @@ struct MemoryManagerInner { // A list of freed buckets. freed_buckets: BTreeSet, - - // Bucket ID of the first bucket that was not used. - first_unused_bucket: BucketId, } impl MemoryManagerInner { @@ -344,7 +341,6 @@ impl MemoryManagerInner { memory_buckets: BTreeMap::new(), bucket_size_in_pages, freed_buckets: BTreeSet::new(), - first_unused_bucket: BucketId(0), }; mem_mgr.save_header(); @@ -380,7 +376,6 @@ impl MemoryManagerInner { memory_buckets: BTreeMap::new(), bucket_size_in_pages, freed_buckets: BTreeSet::new(), - first_unused_bucket: BucketId(0), }; mem_mgr.save_header_v1(); @@ -416,7 +411,6 @@ impl MemoryManagerInner { memory_sizes_in_pages: header.memory_sizes_in_pages, memory_buckets, freed_buckets: BTreeSet::new(), - first_unused_bucket: BucketId(header.num_allocated_buckets), } } @@ -459,10 +453,8 @@ impl MemoryManagerInner { memory_sizes_in_pages: header.memory_sizes_in_pages, memory_buckets, freed_buckets: BTreeSet::new(), - first_unused_bucket: BucketId(0), } } else { - let first_unused_bucket = BucketId((last_occupied_bucket + 1) as u16); let mut freed_buckets = BTreeSet::new(); for id in 0..last_occupied_bucket as u16 { @@ -480,7 +472,6 @@ impl MemoryManagerInner { memory_sizes_in_pages: header.memory_sizes_in_pages, memory_buckets, freed_buckets, - first_unused_bucket, } }; @@ -535,7 +526,9 @@ impl MemoryManagerInner { let required_buckets = self.num_buckets_needed(new_size); let new_buckets_needed = required_buckets - current_buckets; - if new_buckets_needed + self.allocated_buckets as u64 > MAX_NUM_BUCKETS { + if new_buckets_needed + self.allocated_buckets as u64 - self.freed_buckets.len() as u64 + > MAX_NUM_BUCKETS + { // Exceeded the memory that can be managed. return -1; } @@ -545,10 +538,10 @@ impl MemoryManagerInner { let new_bucket_id = match self.freed_buckets.pop_first() { Some(t) => t, None => { - if self.first_unused_bucket.0 != MAX_NUM_BUCKETS as u16 { - let new_id = self.first_unused_bucket; - self.first_unused_bucket = BucketId(self.first_unused_bucket.0 + 1); - new_id + if self.allocated_buckets != MAX_NUM_BUCKETS as u16 { + let new_id = self.allocated_buckets; + self.allocated_buckets += 1; + BucketId(new_id) } else { panic!("{id:?}: grow failed"); } @@ -559,8 +552,6 @@ impl MemoryManagerInner { .entry(id) .or_insert_with(Vec::new) .push(new_bucket_id); - - self.allocated_buckets += 1; } // Grow the underlying memory if necessary. @@ -672,7 +663,6 @@ impl MemoryManagerInner { if let Some(vec_buckets) = buckets { for bucket in vec_buckets { self.freed_buckets.insert(bucket); - self.allocated_buckets -= 1; } } // Update the header. From 1674408921e32f4d66efd105fff486ffcd1a4836 Mon Sep 17 00:00:00 2001 From: Dragoljub Duric Date: Thu, 20 Jul 2023 12:38:12 +0000 Subject: [PATCH 52/70] reverse order of allocating freed buckets --- src/memory_manager.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/memory_manager.rs b/src/memory_manager.rs index caab522b..0bc1d366 100644 --- a/src/memory_manager.rs +++ b/src/memory_manager.rs @@ -535,7 +535,7 @@ impl MemoryManagerInner { // Allocate new buckets as needed. for _ in 0..new_buckets_needed { - let new_bucket_id = match self.freed_buckets.pop_first() { + let new_bucket_id = match self.freed_buckets.pop_last() { Some(t) => t, None => { if self.allocated_buckets != MAX_NUM_BUCKETS as u16 { From 1a4b7483ef0ba76d2a5e812886453cbc8d541d3e Mon Sep 17 00:00:00 2001 From: Dragoljub Duric Date: Thu, 20 Jul 2023 14:31:54 +0000 Subject: [PATCH 53/70] add proptest --- src/memory_manager.rs | 55 +++++++++++++++++++++++++++++-------------- 1 file changed, 37 insertions(+), 18 deletions(-) diff --git a/src/memory_manager.rs b/src/memory_manager.rs index 0bc1d366..4d06359f 100644 --- a/src/memory_manager.rs +++ b/src/memory_manager.rs @@ -207,15 +207,20 @@ impl MemoryManager { } #[cfg(test)] - pub fn init_v1(memory: M) -> Self { + pub fn init_with_bucket_size_v1(memory: M, bucket_size_in_pages: u16) -> Self { Self { inner: Rc::new(RefCell::new(MemoryManagerInner::init_v1( memory, - BUCKET_SIZE_IN_PAGES as u16, + bucket_size_in_pages, ))), } } + #[cfg(test)] + pub fn init_v1(memory: M) -> Self { + Self::init_with_bucket_size_v1(memory, BUCKET_SIZE_IN_PAGES as u16) + } + /// Returns the memory associated with the given ID. pub fn get(&self, id: MemoryId) -> VirtualMemory { VirtualMemory { @@ -1392,26 +1397,40 @@ mod test { #[test] fn upgrade_from_v1_to_v2() { - // Create a memory with layout V1. let mem = make_memory(); - let mem_mgr = MemoryManager::init_v1(mem.clone()); - let memory = mem_mgr.get(MemoryId(0)); - assert_eq!(memory.grow_v1(1), 0); - assert_eq!(memory.size(), 1); + let mem_mgr = MemoryManager::init_with_bucket_size_v1(mem.clone(), 1); // very small bucket size. - memory.write_v1(0, &[8, 1, 9]); + let memories_v1: Vec<_> = (0..MAX_NUM_MEMORIES) + .map(|id| mem_mgr.get(MemoryId(id))) + .collect(); - let mut bytes = vec![0; 3]; - memory.read_v1(0, &mut bytes); - assert_eq!(bytes, vec![8, 1, 9]); + proptest!(|( + num_memories in 0..255usize, + data in proptest::collection::vec(0..u8::MAX, 0..2*WASM_PAGE_SIZE as usize), + offset in 0..10*WASM_PAGE_SIZE + )| { + for memory_v1 in memories_v1.iter().take(num_memories) { + // Write a random blob into the memory, growing the memory as it needs to. + write(memory_v1, offset, &data); - // Upgrade to layout V2. + // Verify the blob can be read back. + let mut bytes = vec![0; data.len()]; + memory_v1.read_v1(offset, &mut bytes); + assert_eq!(bytes, data); + } - let mem_mgr_v2 = MemoryManager::init(mem); - let memory = mem_mgr_v2.get(MemoryId(0)); - let mut bytes = vec![0; 3]; - assert_eq!(memory.size(), 1); - memory.read(0, &mut bytes); - assert_eq!(bytes, vec![8, 1, 9]); + // Load as v2 + let mem_mgr_v2 = MemoryManager::init_with_bucket_size(mem.clone(), 1); + let memories_v2: Vec<_> = (0..MAX_NUM_MEMORIES) + .map(|id| mem_mgr_v2.get(MemoryId(id))) + .collect(); + + for memory_v2 in memories_v2.iter().take(num_memories) { + // Verify the blob can be read back. + let mut bytes = vec![0; data.len()]; + memory_v2.read(offset, &mut bytes); + assert_eq!(bytes, data); + } + }); } } From 4e3f4ae6cd7613b36bc28376cff6fc6681b3e16f Mon Sep 17 00:00:00 2001 From: Dragoljub Duric Date: Thu, 20 Jul 2023 14:33:52 +0000 Subject: [PATCH 54/70] remove unnecessary methods --- src/memory_manager.rs | 108 ------------------------------------------ 1 file changed, 108 deletions(-) diff --git a/src/memory_manager.rs b/src/memory_manager.rs index 4d06359f..eab1a80d 100644 --- a/src/memory_manager.rs +++ b/src/memory_manager.rs @@ -283,23 +283,6 @@ impl Memory for VirtualMemory { } } -impl VirtualMemory { - #[cfg(test)] - fn grow_v1(&self, pages: u64) -> i64 { - self.memory_manager.borrow_mut().grow_v1(self.id, pages) - } - - #[cfg(test)] - fn read_v1(&self, offset: u64, dst: &mut [u8]) { - self.memory_manager.borrow().read_v1(self.id, offset, dst) - } - - #[cfg(test)] - fn write_v1(&self, offset: u64, src: &[u8]) { - self.memory_manager.borrow().write_v1(self.id, offset, src) - } -} - #[derive(Clone)] struct MemoryManagerInner { memory: M, @@ -694,92 +677,6 @@ impl MemoryManagerInner { write_struct(&header, Address::from(0), &self.memory); } - - #[cfg(test)] - fn grow_v1(&mut self, id: MemoryId, pages: u64) -> i64 { - // Compute how many additional buckets are needed. - let old_size = self.memory_size(id); - let new_size = old_size + pages; - let current_buckets = self.num_buckets_needed(old_size); - let required_buckets = self.num_buckets_needed(new_size); - let new_buckets_needed = required_buckets - current_buckets; - - if new_buckets_needed + self.allocated_buckets as u64 > MAX_NUM_BUCKETS { - // Exceeded the memory that can be managed. - return -1; - } - - // Allocate new buckets as needed. - for _ in 0..new_buckets_needed { - let new_bucket_id = BucketId(self.allocated_buckets); - - self.memory_buckets - .entry(id) - .or_insert_with(Vec::new) - .push(new_bucket_id); - - // Write in stable store that this bucket belongs to the memory with the provided `id`. - write( - &self.memory, - bucket_allocations_address(new_bucket_id).get(), - &[id.0], - ); - - self.allocated_buckets += 1; - } - - // Grow the underlying memory if necessary. - let pages_needed = BUCKETS_OFFSET_IN_PAGES - + self.bucket_size_in_pages as u64 * self.allocated_buckets as u64; - if pages_needed > self.memory.size() { - let additional_pages_needed = pages_needed - self.memory.size(); - let prev_pages = self.memory.grow(additional_pages_needed); - if prev_pages == -1 { - panic!("{id:?}: grow failed"); - } - } - - // Update the memory with the new size. - self.memory_sizes_in_pages[id.0 as usize] = new_size; - - // Update the header and return the old size. - self.save_header(); - old_size as i64 - } - - #[cfg(test)] - fn write_v1(&self, id: MemoryId, offset: u64, src: &[u8]) { - if (offset + src.len() as u64) > self.memory_size(id) * WASM_PAGE_SIZE { - panic!("{id:?}: write out of bounds"); - } - - let mut bytes_written = 0; - for Segment { address, length } in self.bucket_iter(id, offset, src.len()) { - self.memory.write( - address.get(), - &src[bytes_written as usize..(bytes_written + length.get()) as usize], - ); - - bytes_written += length.get(); - } - } - - #[cfg(test)] - fn read_v1(&self, id: MemoryId, offset: u64, dst: &mut [u8]) { - if (offset + dst.len() as u64) > self.memory_size(id) * WASM_PAGE_SIZE { - panic!("{id:?}: read out of bounds"); - } - - let mut bytes_read = 0; - for Segment { address, length } in self.bucket_iter(id, offset, dst.len()) { - self.memory.read( - address.get(), - &mut dst[bytes_read as usize..(bytes_read + length.get()) as usize], - ); - - bytes_read += length.get(); - } - } } fn bytes_to_bucket_indexes(input: &[u8]) -> Vec { @@ -1412,11 +1309,6 @@ mod test { for memory_v1 in memories_v1.iter().take(num_memories) { // Write a random blob into the memory, growing the memory as it needs to. write(memory_v1, offset, &data); - - // Verify the blob can be read back. - let mut bytes = vec![0; data.len()]; - memory_v1.read_v1(offset, &mut bytes); - assert_eq!(bytes, data); } // Load as v2 From 1bda82083662fc77baa63c394323ed35eb684762 Mon Sep 17 00:00:00 2001 From: Dragoljub Duric Date: Thu, 20 Jul 2023 14:35:06 +0000 Subject: [PATCH 55/70] remove unnecessary methods --- src/memory_manager.rs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/memory_manager.rs b/src/memory_manager.rs index eab1a80d..b5816f04 100644 --- a/src/memory_manager.rs +++ b/src/memory_manager.rs @@ -216,11 +216,6 @@ impl MemoryManager { } } - #[cfg(test)] - pub fn init_v1(memory: M) -> Self { - Self::init_with_bucket_size_v1(memory, BUCKET_SIZE_IN_PAGES as u16) - } - /// Returns the memory associated with the given ID. pub fn get(&self, id: MemoryId) -> VirtualMemory { VirtualMemory { From de116b78d6257b22ff30408769887f5f7932c0f4 Mon Sep 17 00:00:00 2001 From: Dragoljub Djuric Date: Thu, 20 Jul 2023 16:38:05 +0200 Subject: [PATCH 56/70] Update memory_manager.rs update comments --- src/memory_manager.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/memory_manager.rs b/src/memory_manager.rs index b5816f04..447bcfaf 100644 --- a/src/memory_manager.rs +++ b/src/memory_manager.rs @@ -1290,6 +1290,7 @@ mod test { #[test] fn upgrade_from_v1_to_v2() { let mem = make_memory(); + // Initialize with layout v1. let mem_mgr = MemoryManager::init_with_bucket_size_v1(mem.clone(), 1); // very small bucket size. let memories_v1: Vec<_> = (0..MAX_NUM_MEMORIES) @@ -1306,7 +1307,7 @@ mod test { write(memory_v1, offset, &data); } - // Load as v2 + // Load layout v1 and convert it to layout v2. let mem_mgr_v2 = MemoryManager::init_with_bucket_size(mem.clone(), 1); let memories_v2: Vec<_> = (0..MAX_NUM_MEMORIES) .map(|id| mem_mgr_v2.get(MemoryId(id))) From bcb238b202bc28d5b6ea34631eba9cfd2f7dca20 Mon Sep 17 00:00:00 2001 From: Dragoljub Duric Date: Tue, 25 Jul 2023 14:52:55 +0000 Subject: [PATCH 57/70] add test for freed_bucket_allocation order --- src/memory_manager.rs | 45 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/src/memory_manager.rs b/src/memory_manager.rs index 447bcfaf..408f4202 100644 --- a/src/memory_manager.rs +++ b/src/memory_manager.rs @@ -518,7 +518,7 @@ impl MemoryManagerInner { // Allocate new buckets as needed. for _ in 0..new_buckets_needed { - let new_bucket_id = match self.freed_buckets.pop_last() { + let new_bucket_id = match self.freed_buckets.pop_first() { Some(t) => t, None => { if self.allocated_buckets != MAX_NUM_BUCKETS as u16 { @@ -1278,6 +1278,49 @@ mod test { assert_eq!(mem_mgr.get(MemoryId(0)).size(), 2); } + #[test] + fn test_freed_buckets_assignment_order() { + let mut mem_mgr = MemoryManager::init(make_memory()); + let memory_a: VirtualMemory>>> = mem_mgr.get(MemoryId(0)); + let memory_b: VirtualMemory>>> = mem_mgr.get(MemoryId(1)); + + // grow and write to memory + assert_eq!(memory_a.grow(1), 0); + assert_eq!(memory_b.grow(1), 0); + memory_a.write(0, &[7, 1, 5]); + memory_b.write(0, &[9, 4, 8]); + + assert_eq!(mem_mgr.get(MemoryId(0)).size(), 1); + assert_eq!(mem_mgr.get(MemoryId(1)).size(), 1); + + let mut bytes = vec![0; 3]; + + // free memory + mem_mgr.free(MemoryId(0)); + mem_mgr.free(MemoryId(1)); + + assert_eq!(mem_mgr.get(MemoryId(0)).size(), 0); + assert_eq!(mem_mgr.get(MemoryId(1)).size(), 0); + + let memory_c: VirtualMemory>>> = mem_mgr.get(MemoryId(2)); + let memory_d: VirtualMemory>>> = mem_mgr.get(MemoryId(3)); + + // grow memory + assert_eq!(memory_c.grow(1), 0); + assert_eq!(memory_d.grow(1), 0); + + assert_eq!(mem_mgr.get(MemoryId(2)).size(), 1); + assert_eq!(mem_mgr.get(MemoryId(3)).size(), 1); + + // check that old bucket is reassign to the memory + memory_c.read(0, &mut bytes); + assert_eq!(bytes, &[7, 1, 5]); + + // check that old bucket is reassign to the memory + memory_d.read(0, &mut bytes); + assert_eq!(bytes, &[9, 4, 8]); + } + #[test] fn free_memory_that_was_not_used() { let mut mem_mgr = MemoryManager::init(make_memory()); From f6af8e210c6786e19c1441823728a3ea98c93a04 Mon Sep 17 00:00:00 2001 From: Dragoljub Duric Date: Thu, 27 Jul 2023 10:03:16 +0000 Subject: [PATCH 58/70] add test freed_memories_are_tracked and fix the error with loading --- src/memory_manager.rs | 60 ++++++++++++++++++++++++++++++++----------- 1 file changed, 45 insertions(+), 15 deletions(-) diff --git a/src/memory_manager.rs b/src/memory_manager.rs index 408f4202..eeeccad5 100644 --- a/src/memory_manager.rs +++ b/src/memory_manager.rs @@ -398,11 +398,22 @@ impl MemoryManagerInner { } fn load_layout_v2(memory: M, header: Header) -> Self { + let mut memory_size_in_buckets = vec![]; + let mut number_of_used_buckets = 0; + + for memory_size_in_pages in header.memory_sizes_in_pages.into_iter() { + let size_in_buckets = (memory_size_in_pages + header.bucket_size_in_pages as u64 - 1) + / header.bucket_size_in_pages as u64; + memory_size_in_buckets.push(size_in_buckets); + number_of_used_buckets += size_in_buckets; + } + const BYTE_SIZE_IN_BITS: usize = 8; - const BUCKETS_INDEX_SIZE_IN_BYTES: usize = - (MAX_NUM_BUCKETS as usize * BUCKET_ID_LEN_IN_BITS + (BYTE_SIZE_IN_BITS - 1)) + let buckets_index_size_in_bytes: usize = + (number_of_used_buckets as usize * BUCKET_ID_LEN_IN_BITS + (BYTE_SIZE_IN_BITS - 1)) / BYTE_SIZE_IN_BITS; - let mut buckets = vec![0; BUCKETS_INDEX_SIZE_IN_BYTES]; + + let mut buckets = vec![0; buckets_index_size_in_bytes]; memory.read(bucket_indexes_offset().get(), &mut buckets); let buckets_decompressed = bytes_to_bucket_indexes(&buckets); @@ -412,12 +423,10 @@ impl MemoryManagerInner { let mut last_occupied_bucket: i32 = -1; let mut j = 0; - for (memory, memory_size_in_pages) in header.memory_sizes_in_pages.into_iter().enumerate() { - let memory_size_in_buckets = - (memory_size_in_pages + header.bucket_size_in_pages as u64 - 1) - / header.bucket_size_in_pages as u64; + + for (memory, size_in_buckets) in memory_size_in_buckets.into_iter().enumerate() { let mut vec_buckets = vec![]; - for _ in 0..memory_size_in_buckets { + for _ in 0..size_in_buckets { let bucket = buckets_decompressed[j]; last_occupied_bucket = std::cmp::max(bucket.0 as i32, last_occupied_bucket); vec_buckets.push(bucket); @@ -521,13 +530,9 @@ impl MemoryManagerInner { let new_bucket_id = match self.freed_buckets.pop_first() { Some(t) => t, None => { - if self.allocated_buckets != MAX_NUM_BUCKETS as u16 { - let new_id = self.allocated_buckets; - self.allocated_buckets += 1; - BucketId(new_id) - } else { - panic!("{id:?}: grow failed"); - } + let new_id = self.allocated_buckets; + self.allocated_buckets += 1; + BucketId(new_id) } }; @@ -1330,6 +1335,31 @@ mod test { mem_mgr.free(MemoryId(5)); } + #[test] + fn freed_memories_are_tracked() { + let mem = make_memory(); + let mut mem_mgr = MemoryManager::init(mem.clone()); + mem_mgr.get(MemoryId(0)).grow(1); + mem_mgr.get(MemoryId(1)).grow(1); + mem_mgr.get(MemoryId(2)).grow(1); + mem_mgr.get(MemoryId(3)).grow(1); + mem_mgr.get(MemoryId(4)).grow(1); + + mem_mgr.free(MemoryId(0)); + mem_mgr.free(MemoryId(2)); + mem_mgr.free(MemoryId(4)); + + let mem_mgr = MemoryManager::init(mem); + // Only Memory 0 and 2 buckets should be counted as freed_buckets. + // The bucket belonging to Memory 4 should not be counted as + // freed because it has the biggest Bucket ID of all allocated + // buckets hence it should become part of unallocated buckets. + assert_eq!( + mem_mgr.inner.borrow().freed_buckets, + maplit::btreeset! { BucketId(0), BucketId(2) } + ); + } + #[test] fn upgrade_from_v1_to_v2() { let mem = make_memory(); From eb285e594d57e55d3930a77ebd49b9b263ecbd57 Mon Sep 17 00:00:00 2001 From: Dragoljub Duric Date: Wed, 6 Sep 2023 14:50:13 +0000 Subject: [PATCH 59/70] update cargo.lock files --- Cargo.lock | 364 ++++++++++++++++---------------------------- examples/Cargo.lock | 137 +++++++++-------- 2 files changed, 208 insertions(+), 293 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 590049bd..f9f251df 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -37,7 +37,7 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5ca11d4be1bab0c8bc8734a9aa7bf4ee8316d462a08c6ac5052f888fef5b494b" dependencies = [ - "windows-sys 0.48.0", + "windows-sys", ] [[package]] @@ -47,14 +47,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "58f54d10c6dfa51283a066ceab3ec1ab78d13fae00aa49243a45e4571fb79dfd" dependencies = [ "anstyle", - "windows-sys 0.48.0", + "windows-sys", ] [[package]] name = "anyhow" -version = "1.0.71" +version = "1.0.75" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c7d0618f0e0b7e8ff11427422b64564d5fb0be1940354bfe2e0529b18a9d9b8" +checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6" [[package]] name = "arrayvec" @@ -126,6 +126,12 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" +[[package]] +name = "bitflags" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4682ae6287fcf752ecaabbfcc7b6f9b72aa33933dc23a554d853aea8eea8635" + [[package]] name = "block-buffer" version = "0.10.4" @@ -143,9 +149,9 @@ checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" [[package]] name = "candid" -version = "0.9.5" +version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "762aa04e3a889d47a1773b74ee3b13438a9bc895954fff79ebf7f308c3744a6c" +checksum = "88f6eec0ae850e006ef0fe306f362884d370624094ec55a6a26de18b251774be" dependencies = [ "anyhow", "binread", @@ -170,21 +176,24 @@ dependencies = [ [[package]] name = "candid_derive" -version = "0.6.2" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "810b3bd60244f282090652ffc7c30a9d23892e72dfe443e46ee55569044f7dd5" +checksum = "158403ea38fab5904ae47a5d67eb7047650a91681407f5ccbcbcabc4f4ffb489" dependencies = [ "lazy_static", "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.31", ] [[package]] name = "cc" -version = "1.0.79" +version = "1.0.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" +checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" +dependencies = [ + "libc", +] [[package]] name = "cfg-if" @@ -194,20 +203,19 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "clap" -version = "4.4.1" +version = "4.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c8d502cbaec4595d2e7d5f61e318f05417bd2b66fdc3809498f0d3fdf0bea27" +checksum = "6a13b88d2c62ff462f88e4a121f17a82c1af05693a2f192b5c38d14de73c19f6" dependencies = [ "clap_builder", "clap_derive", - "once_cell", ] [[package]] name = "clap_builder" -version = "4.4.1" +version = "4.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5891c7bc0edb3e1c2204fc5e94009affabeb1821c9e5fdc3959536c5c0bb984d" +checksum = "2bb9faaa7c2ef94b2743a21f5a29e6f0010dff4caa69ac8e9d6cf8b6fa74da08" dependencies = [ "anstream", "anstyle", @@ -217,14 +225,14 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.4.0" +version = "4.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9fd1a5729c4548118d7d70ff234a44868d00489a4b6597b0b020918a0e91a1a" +checksum = "0862016ff20d69b84ef8247369fabf5c008a7417002411897d40ee1f4532b873" dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.31", ] [[package]] @@ -257,14 +265,14 @@ checksum = "2674ec482fbc38012cf31e6c42ba0177b431a0cb6f15fe40efa5aab1bda516f6" dependencies = [ "is-terminal", "lazy_static", - "windows-sys 0.48.0", + "windows-sys", ] [[package]] name = "cpufeatures" -version = "0.2.7" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e4c1eaa2012c47becbbad2ab175484c2a84d1185b566fb2cc5b8707343dfe58" +checksum = "a17b76ff3a4162b0b27f354a0c87015ddad39d35f9c0c36607a3bdd175dde1f1" dependencies = [ "libc", ] @@ -306,9 +314,9 @@ dependencies = [ [[package]] name = "either" -version = "1.8.1" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91" +checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" [[package]] name = "equivalent" @@ -318,13 +326,13 @@ checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] name = "errno" -version = "0.3.1" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bcfec3a70f97c962c307b2d2c56e358cf1d00b558d74262b5f929ee8cc7e73a" +checksum = "136526188508e25c6fef639d7927dfb3e0e3084488bf202267829cf7fc23dbdd" dependencies = [ "errno-dragonfly", "libc", - "windows-sys 0.48.0", + "windows-sys", ] [[package]] @@ -339,12 +347,9 @@ dependencies = [ [[package]] name = "fastrand" -version = "1.9.0" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e51093e27b0797c359783294ca4f0a911c270184cb10f85783b118614a1501be" -dependencies = [ - "instant", -] +checksum = "6999dc1837253364c2ebb0704ba97994bd874e8f195d665c50b7548f6ea92764" [[package]] name = "fnv" @@ -364,21 +369,15 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.9" +version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c85e1d9ab2eadba7e5040d4e09cbd6d072b76a557ad64e797c2cb9d4da21d7e4" +checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427" dependencies = [ "cfg-if", "libc", "wasi", ] -[[package]] -name = "hashbrown" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" - [[package]] name = "hashbrown" version = "0.14.0" @@ -393,9 +392,9 @@ checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" [[package]] name = "hermit-abi" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fed44880c466736ef9a5c5b5facefb5ed0785676d0c02d612db14e54f0d84286" +checksum = "443144c8cdadd93ebf52ddb4056d257f5b52c04d3c804e657d19eb73fc33668b" [[package]] name = "hex" @@ -438,7 +437,6 @@ dependencies = [ "candid", "clap", "colored", - "criterion", "hex", "ic-cdk", "ic-cdk-macros", @@ -458,16 +456,6 @@ version = "0.18.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "576c539151d4769fb4d1a0c25c4108dd18facd04c5695b02cf2d226ab4e43aa5" -[[package]] -name = "indexmap" -version = "1.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" -dependencies = [ - "autocfg", - "hashbrown 0.12.3", -] - [[package]] name = "indexmap" version = "2.0.0" @@ -475,46 +463,25 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d5477fe2230a79769d8dc68e0eabf5437907c0457a5614a9e8dddb67f65eb65d" dependencies = [ "equivalent", - "hashbrown 0.14.0", -] - -[[package]] -name = "instant" -version = "0.1.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" -dependencies = [ - "cfg-if", -] - -[[package]] -name = "io-lifetimes" -version = "1.0.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2" -dependencies = [ - "hermit-abi", - "libc", - "windows-sys 0.48.0", + "hashbrown", ] [[package]] name = "is-terminal" -version = "0.4.7" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adcf93614601c8129ddf72e2d5633df827ba6551541c6d8c59520a371475be1f" +checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b" dependencies = [ "hermit-abi", - "io-lifetimes", "rustix", - "windows-sys 0.48.0", + "windows-sys", ] [[package]] name = "itoa" -version = "1.0.6" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "453ad9f582a441959e5f0d088b02ce04cfe8d51a8eaf077f12ac6d3e94164ca6" +checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" [[package]] name = "lazy_static" @@ -530,9 +497,9 @@ checksum = "884e2677b40cc8c339eaefcb701c32ef1fd2493d71118dc0ca4b6a736c93bd67" [[package]] name = "libc" -version = "0.2.144" +version = "0.2.147" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b00cc1c228a6782d0f076e7b232802e0c5689d41bb5df366f2a6b6621cfdfe1" +checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3" [[package]] name = "libm" @@ -542,9 +509,9 @@ checksum = "f7012b1bbb0719e1097c47611d3898568c546d597c2e74d66f6087edd5233ff4" [[package]] name = "linux-raw-sys" -version = "0.3.8" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" +checksum = "57bcfdad1b858c2db7c38303a6d2ad4dfaf5eb53dfeb0910128b2c26d6158503" [[package]] name = "maplit" @@ -554,15 +521,15 @@ checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d" [[package]] name = "memchr" -version = "2.5.0" +version = "2.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" +checksum = "8f232d6ef707e1956a43342693d2a31e72989554d58299d7a88738cc95b0d35c" [[package]] name = "num-bigint" -version = "0.4.3" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f93ab6289c7b344a8a9f60f88d80aa20032336fe78da341afc91c8a2341fc75f" +checksum = "608e7659b5c3d7cba262d894801b9ec9d00de989e8a82bd4bef91d08da45cdc0" dependencies = [ "autocfg", "num-integer", @@ -582,9 +549,9 @@ dependencies = [ [[package]] name = "num-traits" -version = "0.2.15" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" +checksum = "f30b0abd723be7e2ffca1272140fac1a2f084c77ec3e123c192b66af1ee9e6c2" dependencies = [ "autocfg", "libm", @@ -608,20 +575,20 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.31", ] [[package]] name = "once_cell" -version = "1.17.2" +version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9670a07f94779e00908f3e686eab508878ebb390ba6e604d3a284c00e8d0487b" +checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" [[package]] name = "paste" -version = "1.0.12" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f746c4065a8fa3fe23974dd82f15431cc8d40779821001404d10d2e79ca7d79" +checksum = "de3145af08024dea9fa9914f381a17b8fc6034dfb00f3a84013f7ff43f29ed4c" [[package]] name = "ppv-lite86" @@ -652,9 +619,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.59" +version = "1.0.66" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6aeca18b86b413c660b781aa319e4e2648a3e6f9eadc9b47e9038e6fe9f3451b" +checksum = "18fb31db3f9bddb2ea821cde30a9f70117e3f119938b5ee630b7403aa6e2ead9" dependencies = [ "unicode-ident", ] @@ -673,7 +640,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4e35c06b98bf36aba164cc17cb25f7e232f5c4aeea73baa14b8a9f0d92dbfa65" dependencies = [ "bit-set", - "bitflags", + "bitflags 1.3.2", "byteorder", "lazy_static", "num-traits", @@ -703,9 +670,9 @@ checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" [[package]] name = "quote" -version = "1.0.28" +version = "1.0.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b9ab9c7eadfd8df19006f1cf1a4aed13540ed5cbc047010ece5826e10825488" +checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" dependencies = [ "proc-macro2", ] @@ -755,7 +722,7 @@ version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" dependencies = [ - "bitflags", + "bitflags 1.3.2", ] [[package]] @@ -766,23 +733,22 @@ checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" [[package]] name = "rustix" -version = "0.37.19" +version = "0.38.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acf8729d8542766f1b2cf77eb034d52f40d375bb8b615d0b147089946e16613d" +checksum = "c0c3dde1fc030af041adc40e79c0e7fbcf431dd24870053d187d7c66e4b87453" dependencies = [ - "bitflags", + "bitflags 2.4.0", "errno", - "io-lifetimes", "libc", "linux-raw-sys", - "windows-sys 0.48.0", + "windows-sys", ] [[package]] name = "rustversion" -version = "1.0.12" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f3208ce4d8448b3f3e7d168a73f5e0c43a61e32930de3bceeccedb388b6bf06" +checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4" [[package]] name = "rusty-fork" @@ -798,37 +764,37 @@ dependencies = [ [[package]] name = "ryu" -version = "1.0.13" +version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f91339c0467de62360649f8d3e185ca8de4224ff281f66000de5eb2a77a79041" +checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" [[package]] name = "serde" -version = "1.0.163" +version = "1.0.188" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2113ab51b87a539ae008b5c6c02dc020ffa39afd2d83cffcb3f4eb2722cebec2" +checksum = "cf9e0fcba69a370eed61bcf2b728575f726b50b55cba78064753d708ddc7549e" dependencies = [ "serde_derive", ] [[package]] name = "serde_bytes" -version = "0.11.9" +version = "0.11.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "416bda436f9aab92e02c8e10d49a15ddd339cea90b6e340fe51ed97abb548294" +checksum = "ab33ec92f677585af6d88c65593ae2375adde54efdbf16d597f2cbc7a6d368ff" dependencies = [ "serde", ] [[package]] name = "serde_derive" -version = "1.0.163" +version = "1.0.188" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c805777e3930c8883389c602315a24224bcc738b63905ef87cd1420353ea93e" +checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.31", ] [[package]] @@ -848,7 +814,7 @@ version = "0.9.25" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1a49e178e4452f45cb61d0cd8cebc1b0fafd3e41929e996cef79aa3aca91f574" dependencies = [ - "indexmap 2.0.0", + "indexmap", "itoa", "ryu", "serde", @@ -857,9 +823,9 @@ dependencies = [ [[package]] name = "sha2" -version = "0.10.6" +version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82e6b795fe2e3b1e845bafcb27aa35405c4d47cdfc92af5fc8d3002f76cebdc0" +checksum = "479fb9d862239e610720565ca91403019f2f00410f1864c5aa7479b950a76ed8" dependencies = [ "cfg-if", "cpufeatures", @@ -894,7 +860,7 @@ dependencies = [ "proc-macro2", "quote", "structmeta-derive", - "syn 2.0.18", + "syn 2.0.31", ] [[package]] @@ -905,7 +871,7 @@ checksum = "a60bcaff7397072dca0017d1db428e30d5002e00b6847703e2e42005c95fbe00" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.31", ] [[package]] @@ -921,9 +887,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.18" +version = "2.0.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32d41677bcbe24c20c52e7c70b0d8db04134c5d1066bf98662e2871ad200ea3e" +checksum = "718fa2415bcb8d8bd775917a1bf12a7931b6dfa890753378538118181e0cb398" dependencies = [ "proc-macro2", "quote", @@ -932,15 +898,15 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.5.0" +version = "3.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9fbec84f381d5795b08656e4912bec604d162bff9291d6189a78f4c8ab87998" +checksum = "cb94d2f3cc536af71caac6b6fcebf65860b347e7ce0cc9ebe8f70d3e521054ef" dependencies = [ "cfg-if", "fastrand", "redox_syscall", "rustix", - "windows-sys 0.45.0", + "windows-sys", ] [[package]] @@ -961,27 +927,27 @@ dependencies = [ "proc-macro2", "quote", "structmeta", - "syn 2.0.18", + "syn 2.0.31", ] [[package]] name = "thiserror" -version = "1.0.40" +version = "1.0.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "978c9a314bd8dc99be594bc3c175faaa9794be04a5a5e153caba6915336cebac" +checksum = "9d6d7a740b8a666a7e828dd00da9c0dc290dff53154ea77ac109281de90589b7" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.40" +version = "1.0.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f" +checksum = "49922ecae66cc8a249b77e68d1d0623c1b2c514f0060c27cdc68bd62a1219d35" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.31", ] [[package]] @@ -992,17 +958,17 @@ checksum = "d63628286617a0c67ee3c4ef92ccf62044e3813ed7376bb82a4586d2ff7974dd" [[package]] name = "toml_datetime" -version = "0.6.2" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a76a9312f5ba4c2dec6b9161fdf25d87ad8a09256ccea5a556fef03c706a10f" +checksum = "7cda73e2f1397b1262d6dfdcef8aafae14d1de7748d66822d3bfeeb6d03e5e4b" [[package]] name = "toml_edit" -version = "0.19.10" +version = "0.19.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2380d56e8670370eee6566b0bfd4265f65b3f432e8c6d85623f728d4fa31f739" +checksum = "f8123f27e969974a3dfba720fdb560be359f57b44302d280ba72e76a74480e8a" dependencies = [ - "indexmap 1.9.3", + "indexmap", "toml_datetime", "winnow", ] @@ -1027,9 +993,9 @@ checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" [[package]] name = "unicode-ident" -version = "1.0.9" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b15811caf2415fb889178633e7724bad2509101cde276048e013b9def5e51fa0" +checksum = "301abaae475aa91687eb82514b328ab47a211a533026cb25fc3e519b86adfc3c" [[package]] name = "unicode-segmentation" @@ -1107,143 +1073,77 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" -[[package]] -name = "windows-sys" -version = "0.45.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" -dependencies = [ - "windows-targets 0.42.2", -] - [[package]] name = "windows-sys" version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" dependencies = [ - "windows-targets 0.48.0", + "windows-targets", ] [[package]] name = "windows-targets" -version = "0.42.2" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" dependencies = [ - "windows_aarch64_gnullvm 0.42.2", - "windows_aarch64_msvc 0.42.2", - "windows_i686_gnu 0.42.2", - "windows_i686_msvc 0.42.2", - "windows_x86_64_gnu 0.42.2", - "windows_x86_64_gnullvm 0.42.2", - "windows_x86_64_msvc 0.42.2", + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", ] -[[package]] -name = "windows-targets" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b1eb6f0cd7c80c79759c929114ef071b87354ce476d9d94271031c0497adfd5" -dependencies = [ - "windows_aarch64_gnullvm 0.48.0", - "windows_aarch64_msvc 0.48.0", - "windows_i686_gnu 0.48.0", - "windows_i686_msvc 0.48.0", - "windows_x86_64_gnu 0.48.0", - "windows_x86_64_gnullvm 0.48.0", - "windows_x86_64_msvc 0.48.0", -] - -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" - [[package]] name = "windows_aarch64_gnullvm" -version = "0.48.0" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" [[package]] name = "windows_aarch64_msvc" -version = "0.42.2" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" - -[[package]] -name = "windows_aarch64_msvc" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" [[package]] name = "windows_i686_gnu" -version = "0.42.2" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" - -[[package]] -name = "windows_i686_gnu" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241" - -[[package]] -name = "windows_i686_msvc" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" [[package]] name = "windows_i686_msvc" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00" - -[[package]] -name = "windows_x86_64_gnu" -version = "0.42.2" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" [[package]] name = "windows_x86_64_gnu" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1" - -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.42.2" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" [[package]] name = "windows_x86_64_gnullvm" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953" - -[[package]] -name = "windows_x86_64_msvc" -version = "0.42.2" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" [[package]] name = "windows_x86_64_msvc" -version = "0.48.0" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" [[package]] name = "winnow" -version = "0.4.6" +version = "0.5.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61de7bac303dc551fe038e2b3cef0f571087a47571ea6e79a87692ac99b99699" +checksum = "7c2e3184b9c4e92ad5167ca73039d0c42476302ab603e2fec4487511f38ccefc" dependencies = [ "memchr", ] diff --git a/examples/Cargo.lock b/examples/Cargo.lock index f82a4288..e79e45b7 100644 --- a/examples/Cargo.lock +++ b/examples/Cargo.lock @@ -4,9 +4,9 @@ version = 3 [[package]] name = "anyhow" -version = "1.0.71" +version = "1.0.75" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c7d0618f0e0b7e8ff11427422b64564d5fb0be1940354bfe2e0529b18a9d9b8" +checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6" [[package]] name = "arrayvec" @@ -53,6 +53,12 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "bit-vec" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb" + [[package]] name = "block-buffer" version = "0.10.4" @@ -70,9 +76,9 @@ checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" [[package]] name = "candid" -version = "0.9.5" +version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "762aa04e3a889d47a1773b74ee3b13438a9bc895954fff79ebf7f308c3744a6c" +checksum = "88f6eec0ae850e006ef0fe306f362884d370624094ec55a6a26de18b251774be" dependencies = [ "anyhow", "binread", @@ -97,21 +103,24 @@ dependencies = [ [[package]] name = "candid_derive" -version = "0.6.2" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "810b3bd60244f282090652ffc7c30a9d23892e72dfe443e46ee55569044f7dd5" +checksum = "158403ea38fab5904ae47a5d67eb7047650a91681407f5ccbcbcabc4f4ffb489" dependencies = [ "lazy_static", "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.31", ] [[package]] name = "cc" -version = "1.0.79" +version = "1.0.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" +checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" +dependencies = [ + "libc", +] [[package]] name = "cfg-if" @@ -158,9 +167,9 @@ dependencies = [ [[package]] name = "cpufeatures" -version = "0.2.7" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e4c1eaa2012c47becbbad2ab175484c2a84d1185b566fb2cc5b8707343dfe58" +checksum = "a17b76ff3a4162b0b27f354a0c87015ddad39d35f9c0c36607a3bdd175dde1f1" dependencies = [ "libc", ] @@ -213,9 +222,15 @@ dependencies = [ [[package]] name = "either" -version = "1.8.1" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91" +checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" + +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] name = "generic-array" @@ -235,9 +250,9 @@ checksum = "eabb4a44450da02c90444cf74558da904edde8fb4e9035a9a6a4e15445af0bd7" [[package]] name = "hashbrown" -version = "0.12.3" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" +checksum = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a" [[package]] name = "hex" @@ -287,11 +302,11 @@ checksum = "576c539151d4769fb4d1a0c25c4108dd18facd04c5695b02cf2d226ab4e43aa5" [[package]] name = "indexmap" -version = "1.9.3" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +checksum = "d5477fe2230a79769d8dc68e0eabf5437907c0457a5614a9e8dddb67f65eb65d" dependencies = [ - "autocfg", + "equivalent", "hashbrown", ] @@ -309,21 +324,21 @@ checksum = "884e2677b40cc8c339eaefcb701c32ef1fd2493d71118dc0ca4b6a736c93bd67" [[package]] name = "libc" -version = "0.2.144" +version = "0.2.147" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b00cc1c228a6782d0f076e7b232802e0c5689d41bb5df366f2a6b6621cfdfe1" +checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3" [[package]] name = "memchr" -version = "2.5.0" +version = "2.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" +checksum = "8f232d6ef707e1956a43342693d2a31e72989554d58299d7a88738cc95b0d35c" [[package]] name = "num-bigint" -version = "0.4.3" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f93ab6289c7b344a8a9f60f88d80aa20032336fe78da341afc91c8a2341fc75f" +checksum = "608e7659b5c3d7cba262d894801b9ec9d00de989e8a82bd4bef91d08da45cdc0" dependencies = [ "autocfg", "num-integer", @@ -343,9 +358,9 @@ dependencies = [ [[package]] name = "num-traits" -version = "0.2.15" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" +checksum = "f30b0abd723be7e2ffca1272140fac1a2f084c77ec3e123c192b66af1ee9e6c2" dependencies = [ "autocfg", ] @@ -368,20 +383,20 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.31", ] [[package]] name = "once_cell" -version = "1.17.2" +version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9670a07f94779e00908f3e686eab508878ebb390ba6e604d3a284c00e8d0487b" +checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" [[package]] name = "paste" -version = "1.0.12" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f746c4065a8fa3fe23974dd82f15431cc8d40779821001404d10d2e79ca7d79" +checksum = "de3145af08024dea9fa9914f381a17b8fc6034dfb00f3a84013f7ff43f29ed4c" [[package]] name = "pretty" @@ -406,9 +421,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.59" +version = "1.0.66" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6aeca18b86b413c660b781aa319e4e2648a3e6f9eadc9b47e9038e6fe9f3451b" +checksum = "18fb31db3f9bddb2ea821cde30a9f70117e3f119938b5ee630b7403aa6e2ead9" dependencies = [ "unicode-ident", ] @@ -436,46 +451,46 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.28" +version = "1.0.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b9ab9c7eadfd8df19006f1cf1a4aed13540ed5cbc047010ece5826e10825488" +checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" dependencies = [ "proc-macro2", ] [[package]] name = "rustversion" -version = "1.0.12" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f3208ce4d8448b3f3e7d168a73f5e0c43a61e32930de3bceeccedb388b6bf06" +checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4" [[package]] name = "serde" -version = "1.0.163" +version = "1.0.188" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2113ab51b87a539ae008b5c6c02dc020ffa39afd2d83cffcb3f4eb2722cebec2" +checksum = "cf9e0fcba69a370eed61bcf2b728575f726b50b55cba78064753d708ddc7549e" dependencies = [ "serde_derive", ] [[package]] name = "serde_bytes" -version = "0.11.9" +version = "0.11.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "416bda436f9aab92e02c8e10d49a15ddd339cea90b6e340fe51ed97abb548294" +checksum = "ab33ec92f677585af6d88c65593ae2375adde54efdbf16d597f2cbc7a6d368ff" dependencies = [ "serde", ] [[package]] name = "serde_derive" -version = "1.0.163" +version = "1.0.188" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c805777e3930c8883389c602315a24224bcc738b63905ef87cd1420353ea93e" +checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.31", ] [[package]] @@ -491,9 +506,9 @@ dependencies = [ [[package]] name = "sha2" -version = "0.10.6" +version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82e6b795fe2e3b1e845bafcb27aa35405c4d47cdfc92af5fc8d3002f76cebdc0" +checksum = "479fb9d862239e610720565ca91403019f2f00410f1864c5aa7479b950a76ed8" dependencies = [ "cfg-if", "cpufeatures", @@ -526,9 +541,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.18" +version = "2.0.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32d41677bcbe24c20c52e7c70b0d8db04134c5d1066bf98662e2871ad200ea3e" +checksum = "718fa2415bcb8d8bd775917a1bf12a7931b6dfa890753378538118181e0cb398" dependencies = [ "proc-macro2", "quote", @@ -557,35 +572,35 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.40" +version = "1.0.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "978c9a314bd8dc99be594bc3c175faaa9794be04a5a5e153caba6915336cebac" +checksum = "9d6d7a740b8a666a7e828dd00da9c0dc290dff53154ea77ac109281de90589b7" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.40" +version = "1.0.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f" +checksum = "49922ecae66cc8a249b77e68d1d0623c1b2c514f0060c27cdc68bd62a1219d35" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.31", ] [[package]] name = "toml_datetime" -version = "0.6.2" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a76a9312f5ba4c2dec6b9161fdf25d87ad8a09256ccea5a556fef03c706a10f" +checksum = "7cda73e2f1397b1262d6dfdcef8aafae14d1de7748d66822d3bfeeb6d03e5e4b" [[package]] name = "toml_edit" -version = "0.19.10" +version = "0.19.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2380d56e8670370eee6566b0bfd4265f65b3f432e8c6d85623f728d4fa31f739" +checksum = "f8123f27e969974a3dfba720fdb560be359f57b44302d280ba72e76a74480e8a" dependencies = [ "indexmap", "toml_datetime", @@ -606,9 +621,9 @@ checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba" [[package]] name = "unicode-ident" -version = "1.0.9" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b15811caf2415fb889178633e7724bad2509101cde276048e013b9def5e51fa0" +checksum = "301abaae475aa91687eb82514b328ab47a211a533026cb25fc3e519b86adfc3c" [[package]] name = "unicode-segmentation" @@ -671,9 +686,9 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "winnow" -version = "0.4.6" +version = "0.5.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61de7bac303dc551fe038e2b3cef0f571087a47571ea6e79a87692ac99b99699" +checksum = "7c2e3184b9c4e92ad5167ca73039d0c42476302ab603e2fec4487511f38ccefc" dependencies = [ "memchr", ] From 9b5d5b7348925830f59f2d5bc0cfb7c76d97f686 Mon Sep 17 00:00:00 2001 From: Dragoljub Djuric Date: Thu, 21 Sep 2023 15:21:45 +0200 Subject: [PATCH 60/70] Update src/memory_manager.rs Co-authored-by: Islam El-Ashi --- src/memory_manager.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/memory_manager.rs b/src/memory_manager.rs index eeeccad5..7e4f1f3f 100644 --- a/src/memory_manager.rs +++ b/src/memory_manager.rs @@ -293,7 +293,7 @@ struct MemoryManagerInner { // A map mapping each managed memory to the bucket ids that are allocated to it. memory_buckets: BTreeMap>, - // A list of freed buckets. + // Tracks the buckets that were freed to be reused in future calls to `grow`. freed_buckets: BTreeSet, } From 2d1077129026935c7c299b3e6883a0617f063b1c Mon Sep 17 00:00:00 2001 From: Dragoljub Djuric Date: Thu, 21 Sep 2023 15:22:33 +0200 Subject: [PATCH 61/70] Update src/memory_manager.rs Co-authored-by: Islam El-Ashi --- src/memory_manager.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/memory_manager.rs b/src/memory_manager.rs index 7e4f1f3f..0d0b084c 100644 --- a/src/memory_manager.rs +++ b/src/memory_manager.rs @@ -224,6 +224,9 @@ impl MemoryManager { } } + /// Frees the specified memory. + /// Note that the underlying physical memory doesn't shrink, but the space previously + /// occupied by the given memory will be reused. pub fn free(&mut self, id: MemoryId) { self.inner.borrow_mut().free(id); } From 653dba8e608d2b3db43c17564ec8456b6484a4bf Mon Sep 17 00:00:00 2001 From: Dragoljub Djuric Date: Wed, 25 Oct 2023 16:26:52 +0200 Subject: [PATCH 62/70] Update src/memory_manager.rs Co-authored-by: Islam El-Ashi --- src/memory_manager.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/memory_manager.rs b/src/memory_manager.rs index 0d0b084c..65cf2633 100644 --- a/src/memory_manager.rs +++ b/src/memory_manager.rs @@ -297,6 +297,7 @@ struct MemoryManagerInner { memory_buckets: BTreeMap>, // Tracks the buckets that were freed to be reused in future calls to `grow`. + // NOTE: A BTreeSet is used so that bucket IDs are maintained in sorted order. freed_buckets: BTreeSet, } From a55af9e040dd29fac2a2a91719a390e060f48897 Mon Sep 17 00:00:00 2001 From: Dragoljub Duric Date: Wed, 25 Oct 2023 14:28:25 +0000 Subject: [PATCH 63/70] rename variable --- src/memory_manager.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/memory_manager.rs b/src/memory_manager.rs index 65cf2633..51fe9472 100644 --- a/src/memory_manager.rs +++ b/src/memory_manager.rs @@ -426,15 +426,15 @@ impl MemoryManagerInner { let mut last_occupied_bucket: i32 = -1; - let mut j = 0; + let mut bucket_idx: usize = 0; for (memory, size_in_buckets) in memory_size_in_buckets.into_iter().enumerate() { let mut vec_buckets = vec![]; for _ in 0..size_in_buckets { - let bucket = buckets_decompressed[j]; + let bucket = buckets_decompressed[bucket_idx]; last_occupied_bucket = std::cmp::max(bucket.0 as i32, last_occupied_bucket); vec_buckets.push(bucket); - j += 1; + bucket_idx += 1; } memory_buckets .entry(MemoryId(memory as u8)) From 2a75e5048cb6d9b8bbf0029e4502f4393e2d794c Mon Sep 17 00:00:00 2001 From: Dragoljub Duric Date: Thu, 26 Oct 2023 11:33:49 +0000 Subject: [PATCH 64/70] . --- src/memory_manager.rs | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/src/memory_manager.rs b/src/memory_manager.rs index 51fe9472..a7ef71dc 100644 --- a/src/memory_manager.rs +++ b/src/memory_manager.rs @@ -441,7 +441,7 @@ impl MemoryManagerInner { .or_insert(vec_buckets); } - let mem_mgr = if last_occupied_bucket == -1 { + if last_occupied_bucket == -1 { Self { memory, allocated_buckets: header.num_allocated_buckets, @@ -469,19 +469,7 @@ impl MemoryManagerInner { memory_buckets, freed_buckets, } - }; - - // Update the header. - mem_mgr.save_header(); - - // Rewrite the stable store to be layout V2. - write( - &mem_mgr.memory, - bucket_indexes_offset().get(), - mem_mgr.get_bucket_ids_in_bytes().as_ref(), - ); - - mem_mgr + } } fn load(memory: M) -> Self { From 335077d0ab4773c12e2caf84d850e93e5f43418c Mon Sep 17 00:00:00 2001 From: Dragoljub Duric Date: Thu, 26 Oct 2023 11:39:16 +0000 Subject: [PATCH 65/70] Cargo.lock --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 08c6f4c5..157d683b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -733,9 +733,9 @@ checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" [[package]] name = "rustix" -version = "0.37.25" +version = "0.38.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4eb579851244c2c03e7c24f501c3432bed80b8f720af1d6e5b0e0f01555a035" +checksum = "bdf14a7a466ce88b5eac3da815b53aefc208ce7e74d1c263aabb04d88c4abeb1" dependencies = [ "bitflags 2.4.0", "errno", From ac21584e90de0023c0f0aafd03c0b15ce22df028 Mon Sep 17 00:00:00 2001 From: Dragoljub Djuric Date: Mon, 30 Oct 2023 15:52:25 +0100 Subject: [PATCH 66/70] Update src/memory_manager.rs Co-authored-by: Islam El-Ashi --- src/memory_manager.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/memory_manager.rs b/src/memory_manager.rs index d518471f..71d77dd1 100644 --- a/src/memory_manager.rs +++ b/src/memory_manager.rs @@ -406,8 +406,7 @@ impl MemoryManagerInner { let mut number_of_used_buckets = 0; for memory_size_in_pages in header.memory_sizes_in_pages.into_iter() { - let size_in_buckets = (memory_size_in_pages + header.bucket_size_in_pages as u64 - 1) - / header.bucket_size_in_pages as u64; + let size_in_buckets = memory_size_in_pages.div_ceil(header.bucket_size_in_pages as u64); memory_size_in_buckets.push(size_in_buckets); number_of_used_buckets += size_in_buckets; } From 735fff6c7e392268dae40c84eb83ffee0eeb62ce Mon Sep 17 00:00:00 2001 From: Dragoljub Duric Date: Mon, 30 Oct 2023 14:54:34 +0000 Subject: [PATCH 67/70] use div_ceil --- src/memory_manager.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/memory_manager.rs b/src/memory_manager.rs index 71d77dd1..fbbb93c2 100644 --- a/src/memory_manager.rs +++ b/src/memory_manager.rs @@ -413,8 +413,7 @@ impl MemoryManagerInner { const BYTE_SIZE_IN_BITS: usize = 8; let buckets_index_size_in_bytes: usize = - (number_of_used_buckets as usize * BUCKET_ID_LEN_IN_BITS + (BYTE_SIZE_IN_BITS - 1)) - / BYTE_SIZE_IN_BITS; + (number_of_used_buckets as usize * BUCKET_ID_LEN_IN_BITS).div_ceil(BYTE_SIZE_IN_BITS); let mut buckets = vec![0; buckets_index_size_in_bytes]; memory.read(bucket_indexes_offset().get(), &mut buckets); From 5178d71d97fb5849e507e99acf1c3e284cb22663 Mon Sep 17 00:00:00 2001 From: Dragoljub Duric Date: Mon, 30 Oct 2023 15:40:13 +0000 Subject: [PATCH 68/70] refactor --- src/memory_manager.rs | 42 ++++++++++++++---------------------------- 1 file changed, 14 insertions(+), 28 deletions(-) diff --git a/src/memory_manager.rs b/src/memory_manager.rs index fbbb93c2..edeff331 100644 --- a/src/memory_manager.rs +++ b/src/memory_manager.rs @@ -422,7 +422,8 @@ impl MemoryManagerInner { let mut memory_buckets = BTreeMap::new(); - let mut last_occupied_bucket: i32 = -1; + // The last bucket that's accessed. + let mut max_bucket_id: u16 = 0; let mut bucket_idx: usize = 0; @@ -430,7 +431,7 @@ impl MemoryManagerInner { let mut vec_buckets = vec![]; for _ in 0..size_in_buckets { let bucket = buckets_decompressed[bucket_idx]; - last_occupied_bucket = std::cmp::max(bucket.0 as i32, last_occupied_bucket); + max_bucket_id = std::cmp::max(bucket.0, max_bucket_id); vec_buckets.push(bucket); bucket_idx += 1; } @@ -439,34 +440,19 @@ impl MemoryManagerInner { .or_insert(vec_buckets); } - if last_occupied_bucket == -1 { - Self { - memory, - allocated_buckets: header.num_allocated_buckets, - bucket_size_in_pages: header.bucket_size_in_pages, - memory_sizes_in_pages: header.memory_sizes_in_pages, - memory_buckets, - freed_buckets: BTreeSet::new(), - } - } else { - let mut freed_buckets = BTreeSet::new(); - - for id in 0..last_occupied_bucket as u16 { - freed_buckets.insert(BucketId(id)); - } + let mut freed_buckets: BTreeSet = (0..max_bucket_id).map(BucketId).collect(); - for id in buckets_decompressed.iter() { - freed_buckets.remove(id); - } + for id in buckets_decompressed.iter() { + freed_buckets.remove(id); + } - Self { - memory, - allocated_buckets: header.num_allocated_buckets, - bucket_size_in_pages: header.bucket_size_in_pages, - memory_sizes_in_pages: header.memory_sizes_in_pages, - memory_buckets, - freed_buckets, - } + Self { + memory, + allocated_buckets: header.num_allocated_buckets, + bucket_size_in_pages: header.bucket_size_in_pages, + memory_sizes_in_pages: header.memory_sizes_in_pages, + memory_buckets, + freed_buckets, } } From 6783305859372efe8ce2144bb6432f326b11635d Mon Sep 17 00:00:00 2001 From: Dragoljub Duric Date: Mon, 30 Oct 2023 15:46:08 +0000 Subject: [PATCH 69/70] refactor loading of bucket indices --- src/memory_manager.rs | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/src/memory_manager.rs b/src/memory_manager.rs index edeff331..31d792af 100644 --- a/src/memory_manager.rs +++ b/src/memory_manager.rs @@ -411,14 +411,18 @@ impl MemoryManagerInner { number_of_used_buckets += size_in_buckets; } - const BYTE_SIZE_IN_BITS: usize = 8; - let buckets_index_size_in_bytes: usize = - (number_of_used_buckets as usize * BUCKET_ID_LEN_IN_BITS).div_ceil(BYTE_SIZE_IN_BITS); + // Load the buckets. + let buckets = { + const BYTE_SIZE_IN_BITS: usize = 8; + let buckets_index_size_in_bytes: usize = (number_of_used_buckets as usize + * BUCKET_ID_LEN_IN_BITS) + .div_ceil(BYTE_SIZE_IN_BITS); - let mut buckets = vec![0; buckets_index_size_in_bytes]; - memory.read(bucket_indexes_offset().get(), &mut buckets); + let mut buckets = vec![0; buckets_index_size_in_bytes]; + memory.read(bucket_indexes_offset().get(), &mut buckets); - let buckets_decompressed = bytes_to_bucket_indexes(&buckets); + bytes_to_bucket_indexes(&buckets) + }; let mut memory_buckets = BTreeMap::new(); @@ -430,7 +434,7 @@ impl MemoryManagerInner { for (memory, size_in_buckets) in memory_size_in_buckets.into_iter().enumerate() { let mut vec_buckets = vec![]; for _ in 0..size_in_buckets { - let bucket = buckets_decompressed[bucket_idx]; + let bucket = buckets[bucket_idx]; max_bucket_id = std::cmp::max(bucket.0, max_bucket_id); vec_buckets.push(bucket); bucket_idx += 1; @@ -442,7 +446,7 @@ impl MemoryManagerInner { let mut freed_buckets: BTreeSet = (0..max_bucket_id).map(BucketId).collect(); - for id in buckets_decompressed.iter() { + for id in buckets.iter() { freed_buckets.remove(id); } From 1ee03a16167fee332d628ea2c1073203e57e2c99 Mon Sep 17 00:00:00 2001 From: Dragoljub Duric Date: Mon, 30 Oct 2023 16:11:46 +0000 Subject: [PATCH 70/70] Add comments to load_layout_v2 --- src/memory_manager.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/memory_manager.rs b/src/memory_manager.rs index 31d792af..b611509e 100644 --- a/src/memory_manager.rs +++ b/src/memory_manager.rs @@ -405,6 +405,7 @@ impl MemoryManagerInner { let mut memory_size_in_buckets = vec![]; let mut number_of_used_buckets = 0; + // Translate memory sizes expressed in pages to sizes expressed in buckets. for memory_size_in_pages in header.memory_sizes_in_pages.into_iter() { let size_in_buckets = memory_size_in_pages.div_ceil(header.bucket_size_in_pages as u64); memory_size_in_buckets.push(size_in_buckets); @@ -424,6 +425,7 @@ impl MemoryManagerInner { bytes_to_bucket_indexes(&buckets) }; + // Map of all memories with their assigned buckets. let mut memory_buckets = BTreeMap::new(); // The last bucket that's accessed. @@ -431,6 +433,7 @@ impl MemoryManagerInner { let mut bucket_idx: usize = 0; + // Assign buckets to the memories they are part of. for (memory, size_in_buckets) in memory_size_in_buckets.into_iter().enumerate() { let mut vec_buckets = vec![]; for _ in 0..size_in_buckets { @@ -444,6 +447,7 @@ impl MemoryManagerInner { .or_insert(vec_buckets); } + // Set of all buckets with ID smaller than 'max_bucket_id' which were allocated and freed. let mut freed_buckets: BTreeSet = (0..max_bucket_id).map(BucketId).collect(); for id in buckets.iter() {