feat: bump katana (#100)

* wip bump katana * bump deps * replace memdb with cached state and dyn database * add multi-thread + remove tx failure case * renaming * update based on comments
kkrt-labs · Sep 19, 2023 · 574efea · 574efea
1 parent e0fceb5
commit 574efea
Show file tree

Hide file tree

Showing 11 changed files with 1,218 additions and 1,067 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -21,20 +21,20 @@ license = "MIT"
 
 [workspace.dependencies]
 # Eth deps
-ef-tests = { git = "https://github.com/paradigmxyz/reth.git", rev = "fb710e5", features = ["ef-tests"] }
-reth-primitives = { git = "https://github.com/paradigmxyz/reth.git", rev = "fb710e5" }
+ef-tests = { git = "https://github.com/paradigmxyz/reth.git", tag = "v0.1.0-alpha.7", features = ["ef-tests"] }
+reth-primitives = { git = "https://github.com/paradigmxyz/reth.git", tag = "v0.1.0-alpha.7" }
 revm-primitives = "1.1"
-reth-rlp = { git = "https://github.com/paradigmxyz/reth.git", rev = "fb710e5" }
+reth-rlp = { git = "https://github.com/paradigmxyz/reth.git", tag = "v0.1.0-alpha.7" }
 
 # Kakarot deps
-hive-utils = { git = "https://github.com/kkrt-labs/kakarot-rpc.git" }
 kakarot-rpc-core = { git = "https://github.com/kkrt-labs/kakarot-rpc.git" }
+kakarot-test-utils = { git = "https://github.com/kkrt-labs/kakarot-rpc.git" }
 
 # Starknet deps
-katana-core = { git = 'https://github.com/dojoengine/dojo', rev = "7893eed" }
-dojo-test-utils = { git = 'https://github.com/dojoengine/dojo', rev = "7893eed" }    
+katana-core = { git = 'https://github.com/dojoengine/dojo', rev = "b924dac" }
+dojo-test-utils = { git = 'https://github.com/dojoengine/dojo', rev = "b924dac" }
 
-starknet = "0.4.0"
+starknet = "0.6.0"
 starknet_api = { git = "https://github.com/starkware-libs/starknet-api", rev = "ecc9b6946ef13003da202838e4124a9ad2efabb0" }
 
 # Other
@@ -53,16 +53,18 @@ walkdir = "2.3.3"
 zip = "0.6.6"
 
 # Serde
-serde = { version = "1.0.147", features = ["derive"] }
+serde = { version = "1.0.188", features = ["derive"] }
 serde_json = "1.0"
 
 [patch."https://github.com/starkware-libs/blockifier"]
 blockifier = { git = "https://github.com/dojoengine/blockifier", rev = "c794d1b" }
 
+[patch."https://github.com/ethereum/c-kzg-4844"]
+c-kzg = { git = "https://github.com/rjected/c-kzg-4844", branch = "dan/add-serde-feature" }
+
 [patch.crates-io]
 cairo-felt = { git = "https://github.com/dojoengine/cairo-rs.git", rev = "262b7eb4b11ab165a2a936a5f914e78aa732d4a2" }
 cairo-vm = { git = "https://github.com/dojoengine/cairo-rs.git", rev = "262b7eb4b11ab165a2a936a5f914e78aa732d4a2" }
 
 revm = { git = "https://github.com/bluealloy/revm/", branch = "release/v25" }
 revm-primitives = { git = "https://github.com/bluealloy/revm/", branch = "release/v25" }
-ethers-core = { git = "https://github.com/gakonst/ethers-rs.git", rev = "7b7c623" }
diff --git a/crates/ef-testing/Cargo.toml b/crates/ef-testing/Cargo.toml
@@ -19,7 +19,7 @@ reth-rlp = { workspace = true }
 
 # Kakarot deps
 kakarot-rpc-core = { workspace = true }
-hive-utils = { workspace = true }
+kakarot-test-utils = { workspace = true }
 
 # Starknet deps
 dojo-test-utils = { workspace = true }

diff --git a/crates/ef-testing/src/models/case.rs b/crates/ef-testing/src/models/case.rs
@@ -15,14 +15,12 @@ use crate::{
 use async_trait::async_trait;
 use ef_tests::models::BlockchainTest;
 use ef_tests::models::{ForkSpec, RootOrState, State};
-use hive_utils::kakarot::compute_starknet_address;
-use kakarot_rpc_core::{
-    client::api::{KakarotEthApi, KakarotStarknetApi},
-    models::felt::Felt252Wrapper,
-    test_utils::deploy_helpers::{DeployedKakarot, KakarotTestEnvironmentContext},
-};
+use kakarot_rpc_core::{client::api::KakarotEthApi, models::felt::Felt252Wrapper};
+use kakarot_test_utils::deploy_helpers::{DeployedKakarot, KakarotTestEnvironmentContext};
+use kakarot_test_utils::hive_utils::kakarot::compute_starknet_address;
+
 use regex::Regex;
-use starknet::{core::types::FieldElement, providers::Provider};
+use starknet::core::types::FieldElement;
 use starknet_api::{core::ContractAddress as StarknetContractAddress, hash::StarkFelt};
 use std::{
     collections::BTreeMap,
@@ -45,8 +43,11 @@ async fn handle_pre_state(
     let kakarot_address = kakarot.kakarot_address;
 
     let mut starknet = env.sequencer().sequencer.backend.state.write().await;
+    let starknet_db = starknet
+        .maybe_as_cached_db()
+        .ok_or_else(|| RunnerError::SequencerError("failed to get Katana database".to_string()))?;
 
-    let eoa_class_hash = get_eoa_class_hash(env, &starknet).expect("failed to get eoa class hash");
+    let eoa_class_hash = get_eoa_class_hash(env, &starknet_db)?;
     let class_hashes = ClassHashes::new(
         kakarot.proxy_class_hash,
         eoa_class_hash,
@@ -136,14 +137,9 @@ impl BlockchainTestCase {
         )?;
 
         let client = env.client();
-        let hash = client.send_transaction(tx_encoded).await?;
-
-        // we make sure that the transaction has a receipt and fail fast if it doesn't
-        let starknet_provider = env.client().starknet_provider();
-        let transaction_hash: FieldElement = FieldElement::from_bytes_be(&hash)?;
-        starknet_provider
-            .get_transaction_receipt::<FieldElement>(transaction_hash)
-            .await?;
+        // Send the transaction without checking for errors, accounting
+        // for the fact that some transactions might fail.
+        let _ = client.send_transaction(tx_encoded).await;
 
         Ok(())
     }
@@ -168,7 +164,10 @@ impl BlockchainTestCase {
         let kakarot_address = kakarot.kakarot_address;
 
         // Get lock on the Starknet sequencer
-        let starknet = env.sequencer().sequencer.backend.state.read().await;
+        let mut starknet = env.sequencer().sequencer.backend.state.write().await;
+        let starknet_db = starknet.maybe_as_cached_db().ok_or_else(|| {
+            RunnerError::SequencerError("failed to get Katana database".to_string())
+        })?;
 
         for (evm_address, expected_state) in post_state.iter() {
             let addr: FieldElement = Felt252Wrapper::from(*evm_address).into();
@@ -177,11 +176,11 @@ impl BlockchainTestCase {
             let starknet_contract_address =
                 StarknetContractAddress(Into::<StarkFelt>::into(starknet_address).try_into()?);
 
-            let actual_state = starknet.storage.get(&starknet_contract_address);
+            let actual_state = starknet_db.storage.get(&starknet_contract_address);
             match actual_state {
                 None => {
                     // if no state, check post state is empty
-                    let actual_balance = read_balance(evm_address, starknet_address, &starknet)
+                    let actual_balance = read_balance(evm_address, starknet_address, &mut starknet)
                         .map_err(|err| {
                             RunnerError::Assertion(format!("{} {}", test_case_name, err))
                         })?;
@@ -278,16 +277,7 @@ impl Case for BlockchainTestCase {
 
                 // necessary to have our updated state actually applied to transaction
                 // think of it as 'burping' the sequencer
-                env.sequencer()
-                    .sequencer
-                    .backend
-                    .generate_latest_block()
-                    .await;
-                env.sequencer()
-                    .sequencer
-                    .backend
-                    .generate_pending_block()
-                    .await;
+                env.sequencer().sequencer.backend.mine_empty_block().await;
 
                 // handle transaction
                 self.handle_transaction(&env, test_name).await?;
@@ -334,7 +324,7 @@ mod tests {
             .expect("setting tracing default failed");
     }
 
-    #[tokio::test]
+    #[tokio::test(flavor = "multi_thread")]
     async fn test_load_case() {
         // Given
         let path = Path::new(
@@ -349,7 +339,7 @@ mod tests {
         assert!(case.transaction.transaction.secret_key != B256::zero());
     }
 
-    #[tokio::test]
+    #[tokio::test(flavor = "multi_thread")]
     async fn test_run_add() {
         // Given
         let path = Path::new(
@@ -366,7 +356,7 @@ mod tests {
         case.run().await.unwrap();
     }
 
-    #[tokio::test]
+    #[tokio::test(flavor = "multi_thread")]
     async fn test_run_mul() {
         // Given
         let path = Path::new(

diff --git a/crates/ef-testing/src/models/error.rs b/crates/ef-testing/src/models/error.rs
@@ -24,6 +24,9 @@ pub enum RunnerError {
         /// The specific error
         error: String,
     },
+    /// Sequencer error
+    #[error("An error occurred while running the sequencer: {0}")]
+    SequencerError(String),
     /// Skipped test
     #[error("test skipped")]
     Skipped,

diff --git a/crates/ef-testing/src/storage/contract.rs b/crates/ef-testing/src/storage/contract.rs
@@ -1,5 +1,5 @@
 use ef_tests::models::Account;
-use hive_utils::madara::utils::{
+use kakarot_test_utils::hive_utils::madara::utils::{
     genesis_set_bytecode, genesis_set_storage_kakarot_contract_account,
 };
 use reth_primitives::Bytes;

diff --git a/crates/ef-testing/src/storage/eoa.rs b/crates/ef-testing/src/storage/eoa.rs
@@ -1,11 +1,10 @@
-use kakarot_rpc_core::test_utils::deploy_helpers::KakarotTestEnvironmentContext;
-use katana_core::backend::state::MemDb;
+use kakarot_test_utils::deploy_helpers::KakarotTestEnvironmentContext;
+use katana_core::db::cached::AsCachedDb;
 use starknet::core::types::FieldElement;
 use starknet_api::{
     core::ContractAddress as StarknetContractAddress, hash::StarkFelt,
     state::StorageKey as StarknetStorageKey,
 };
-use tokio::sync::RwLockWriteGuard;
 
 use crate::{models::error::RunnerError, utils::starknet::get_starknet_storage_key};
 
@@ -16,15 +15,15 @@ use super::{
 /// Returns the class hash used for the EOA contract.
 pub fn get_eoa_class_hash(
     ctx: &KakarotTestEnvironmentContext,
-    starknet: &RwLockWriteGuard<'_, MemDb>,
+    starknet_db: &AsCachedDb,
 ) -> Result<FieldElement, RunnerError> {
     let eoa = &ctx.kakarot().eoa_addresses;
 
     let eoa_address =
         StarknetContractAddress(Into::<StarkFelt>::into(eoa.starknet_address).try_into()?);
 
     // deriving the eao class hash this way so things are always based off the katana dump file
-    let eoa_class_hash: FieldElement = (*starknet
+    let eoa_class_hash: FieldElement = (*starknet_db
         .storage
         .get(&eoa_address)
         .ok_or_else(|| {

diff --git a/crates/ef-testing/src/storage/fee_token.rs b/crates/ef-testing/src/storage/fee_token.rs
@@ -1,5 +1,7 @@
-use hive_utils::madara::utils::{genesis_approve_kakarot, genesis_fund_starknet_address};
-use katana_core::{backend::state::MemDb, constants::FEE_TOKEN_ADDRESS};
+use kakarot_test_utils::hive_utils::madara::utils::{
+    genesis_approve_kakarot, genesis_fund_starknet_address,
+};
+use katana_core::{constants::FEE_TOKEN_ADDRESS, db::Database};
 use reth_primitives::Address;
 use revm_primitives::U256;
 use starknet::core::types::FieldElement;
@@ -8,7 +10,7 @@ use starknet_api::{
     hash::StarkFelt,
     state::StorageKey,
 };
-use tokio::sync::RwLockReadGuard;
+use tokio::sync::RwLockWriteGuard;
 
 use crate::{models::error::RunnerError, utils::starknet::get_starknet_storage_key};
 
@@ -55,19 +57,13 @@ pub(crate) fn generate_allowance_storage(
 pub(crate) fn read_balance(
     evm_address: &Address,
     starknet_address: FieldElement,
-    starknet: &RwLockReadGuard<'_, MemDb>,
+    starknet: &mut RwLockWriteGuard<'_, dyn Database>,
 ) -> Result<FieldElement, RunnerError> {
     let fee_token_address = ContractAddress(TryInto::<PatriciaKey>::try_into(*FEE_TOKEN_ADDRESS)?);
 
     let storage_key = get_starknet_storage_key("ERC20_balances", &[starknet_address], 0)?;
-    let balance = *starknet
-        .storage
-        .get(&fee_token_address)
-        .ok_or_else(|| {
-            RunnerError::Other(format!("missing fee token address {:?}", fee_token_address))
-        })?
-        .storage
-        .get(&storage_key)
-        .ok_or_else(|| RunnerError::Other(format!("missing balance for {:#20x}", evm_address)))?;
+    let balance = starknet
+        .get_storage_at(fee_token_address, storage_key)
+        .map_err(|_| RunnerError::Other(format!("missing balance for {:#20x}", evm_address)))?;
     Ok(balance.into())
 }
diff --git a/crates/ef-testing/src/storage/mod.rs b/crates/ef-testing/src/storage/mod.rs
@@ -6,15 +6,13 @@ pub mod models;
 
 use ef_tests::models::Account;
 use ef_tests::models::State;
-use hive_utils::{
+use kakarot_rpc_core::models::felt::Felt252Wrapper;
+use kakarot_test_utils::hive_utils::{
     kakarot::compute_starknet_address,
     types::{ContractAddress, StorageKey, StorageValue},
 };
-use kakarot_rpc_core::models::felt::Felt252Wrapper;
-use katana_core::{
-    backend::state::{MemDb, StorageRecord},
-    constants::FEE_TOKEN_ADDRESS,
-};
+use katana_core::constants::FEE_TOKEN_ADDRESS;
+use katana_core::db::Database;
 use starknet::core::types::FieldElement;
 use starknet_api::{
     core::{ClassHash, ContractAddress as StarknetContractAddress, Nonce},
@@ -38,7 +36,7 @@ pub fn write_test_state(
     state: &State,
     kakarot_address: FieldElement,
     class_hashes: ClassHashes,
-    starknet: &mut RwLockWriteGuard<'_, MemDb>,
+    starknet: &mut RwLockWriteGuard<'_, dyn Database>,
 ) -> Result<(), RunnerError> {
     // iterate through pre-state addresses
     let mut kakarot_storage = Vec::new();
@@ -78,12 +76,17 @@ pub fn write_test_state(
         let address =
             StarknetContractAddress(Into::<StarkFelt>::into(starknet_address).try_into()?);
         let account_nonce: FieldElement = Felt252Wrapper::try_from(account_info.nonce.0)?.into();
-        let storage_record = StorageRecord {
-            nonce: Nonce(StarkFelt::from(account_nonce)),
-            class_hash: ClassHash(class_hashes.proxy_class_hash.into()),
-            storage: starknet_contract_storage.into_iter().collect(),
-        };
-        starknet.storage.insert(address, storage_record);
+        starknet
+            .set_class_hash_at(address, ClassHash(class_hashes.proxy_class_hash.into()))
+            .map_err(|err| {
+                RunnerError::SequencerError(format!(
+                    "error setting class hash at {address:#?}: {err}"
+                ))
+            })?;
+        starknet.set_nonce(address, Nonce(StarkFelt::from(account_nonce)));
+        for (k, v) in starknet_contract_storage.into_iter() {
+            starknet.set_storage_at(address, k, v);
+        }
 
         // Update the sequencer state with the fee token balance and allowance
         let fee_token_storage = initialize_fee_token_storage(
@@ -118,15 +121,12 @@ pub fn madara_to_katana_storage(
 pub(crate) fn extend_starknet_state_with_storage(
     address: FieldElement,
     storage: Vec<(StarknetStorageKey, StarkFelt)>,
-    starknet: &mut RwLockWriteGuard<'_, MemDb>,
+    starknet: &mut RwLockWriteGuard<'_, dyn Database>,
 ) -> Result<(), RunnerError> {
     let address = StarknetContractAddress(Into::<StarkFelt>::into(address).try_into()?);
-    starknet
-        .storage
-        .get_mut(&address)
-        .ok_or_else(|| RunnerError::Other(format!("missing address {:?} in storage", address)))?
-        .storage
-        .extend(storage);
+    for (k, v) in storage.into_iter() {
+        starknet.set_storage_at(address, k, v);
+    }
     Ok(())
 }
 

diff --git a/crates/ef-testing/src/utils/assert.rs b/crates/ef-testing/src/utils/assert.rs
@@ -2,7 +2,7 @@ use ef_tests::models::Account;
 use kakarot_rpc_core::{
     client::helpers::split_u256_into_field_elements, models::felt::Felt252Wrapper,
 };
-use katana_core::backend::state::StorageRecord;
+use katana_core::db::cached::StorageRecord;
 use reth_primitives::Address;
 use reth_primitives::JsonU256;
 use revm_primitives::U256;

diff --git a/crates/ef-testing/tests/tests.rs b/crates/ef-testing/tests/tests.rs
@@ -46,7 +46,7 @@ pub fn verify_kakarot_sha() -> Result<String, eyre::Error> {
 
 macro_rules! blockchain_tests {
     ($test_name:ident, $dir:ident) => {
-        #[tokio::test]
+        #[tokio::test(flavor = "multi_thread")]
         async fn $test_name() {
             setup();
             BlockchainTestSuite::new(format!("GeneralStateTests/{}", stringify!($dir)))