Merge branch 'main' into aov-evm-777-add-evm-emulator-option-to-zk-to…

…olbox

.github/release-please/manifest.json

@@ -1,5 +1,5 @@
 {
-  "core": "24.29.0",
+  "core": "25.0.0",
   "prover": "16.5.0",
   "zkstack_cli": "0.1.2"
 }

.github/workflows/build-prover-fri-gpu-gar-and-circuit-prover-gpu-gar.yml

@@ -28,7 +28,6 @@ jobs:
       - name: Download Setup data
         run: |
           gsutil -m rsync -r gs://matterlabs-setup-data-us/${{ inputs.setup_keys_id }} docker/prover-gpu-fri-gar
-          cp -v docker/prover-gpu-fri-gar/*.bin docker/circuit-prover-gpu-gar/
 
       - name: Login to us-central1 GAR
         run: |
@@ -70,6 +69,14 @@ jobs:
             --tag europe-docker.pkg.dev/matterlabs-infra/matterlabs-docker/prover-fri-gpu-gar:2.0-${{ inputs.protocol_version }}-${{ inputs.image_tag_suffix }} \
             us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/prover-fri-gpu-gar:2.0-${{ inputs.protocol_version }}-${{ inputs.image_tag_suffix }}
 
+      - name: Remove prover-gpu-fri-gar image to free space
+        run: |
+          docker image rm us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/prover-fri-gpu-gar:2.0-${{ inputs.protocol_version }}-${{ inputs.image_tag_suffix }}
+
+      - name: Move Setup data from prover-gpu-fri-gar to circuit-prover-gpu-gar
+        run: |
+          mv -v docker/prover-gpu-fri-gar/*.bin docker/circuit-prover-gpu-gar/
+
       - name: Build and push circuit-prover-gpu-gar
         uses: docker/build-push-action@5cd11c3a4ced054e52742c5fd54dca954e0edd85 # v6.7.0
         with:

.github/workflows/ci-core-lint-reusable.yml

@@ -49,6 +49,7 @@ jobs:
           ci_run zkstack dev lint -t js --check
           ci_run zkstack dev lint -t ts --check
           ci_run zkstack dev lint -t rs --check
+          ci_run zkstack dev lint -t autocompletion --check
 
       - name: Check Database
         run: |

core/CHANGELOG.md

@@ -1,5 +1,40 @@
 # Changelog
 
+## [25.0.0](https://github.com/matter-labs/zksync-era/compare/core-v24.29.0...core-v25.0.0) (2024-10-23)
+
+
+### ⚠ BREAKING CHANGES
+
+* **contracts:** integrate protocol defense changes ([#2737](https://github.com/matter-labs/zksync-era/issues/2737))
+
+### Features
+
+* Add CoinMarketCap external API ([#2971](https://github.com/matter-labs/zksync-era/issues/2971)) ([c1cb30e](https://github.com/matter-labs/zksync-era/commit/c1cb30e59ca1d0b5fea5fe0980082aea0eb04aa2))
+* **api:** Implement eth_maxPriorityFeePerGas ([#3135](https://github.com/matter-labs/zksync-era/issues/3135)) ([35e84cc](https://github.com/matter-labs/zksync-era/commit/35e84cc03a7fdd315932fb3020fe41c95a6e4bca))
+* **api:** Make acceptable values cache lag configurable ([#3028](https://github.com/matter-labs/zksync-era/issues/3028)) ([6747529](https://github.com/matter-labs/zksync-era/commit/67475292ff770d2edd6884be27f976a4144778ae))
+* **contracts:** integrate protocol defense changes ([#2737](https://github.com/matter-labs/zksync-era/issues/2737)) ([c60a348](https://github.com/matter-labs/zksync-era/commit/c60a3482ee09b3e371163e62f49e83bc6d6f4548))
+* **external-node:** save protocol version before opening a batch ([#3136](https://github.com/matter-labs/zksync-era/issues/3136)) ([d6de4f4](https://github.com/matter-labs/zksync-era/commit/d6de4f40ddce339c760c95e2bf4b8aceb571af7f))
+* Prover e2e test ([#2975](https://github.com/matter-labs/zksync-era/issues/2975)) ([0edd796](https://github.com/matter-labs/zksync-era/commit/0edd7962429b3530ae751bd7cc947c97193dd0ca))
+* **prover:** Add min_provers and dry_run features. Improve metrics and test. ([#3129](https://github.com/matter-labs/zksync-era/issues/3129)) ([7c28964](https://github.com/matter-labs/zksync-era/commit/7c289649b7b3c418c7193a35b51c264cf4970f3c))
+* **tee_verifier:** speedup SQL query for new jobs ([#3133](https://github.com/matter-labs/zksync-era/issues/3133)) ([30ceee8](https://github.com/matter-labs/zksync-era/commit/30ceee8a48046e349ff0234ebb24d468a0e0876c))
+* vm2 tracers can access storage ([#3114](https://github.com/matter-labs/zksync-era/issues/3114)) ([e466b52](https://github.com/matter-labs/zksync-era/commit/e466b52948e3c4ed1cb5af4fd999a52028e4d216))
+* **vm:** Return compressed bytecodes from `push_transaction()` ([#3126](https://github.com/matter-labs/zksync-era/issues/3126)) ([37f209f](https://github.com/matter-labs/zksync-era/commit/37f209fec8e7cb65c0e60003d46b9ea69c43caf1))
+
+
+### Bug Fixes
+
+* **call_tracer:** Flat call tracer fixes for blocks ([#3095](https://github.com/matter-labs/zksync-era/issues/3095)) ([30ddb29](https://github.com/matter-labs/zksync-era/commit/30ddb292977340beab37a81f75c35480cbdd59d3))
+* **consensus:** preventing config update reverts ([#3148](https://github.com/matter-labs/zksync-era/issues/3148)) ([caee55f](https://github.com/matter-labs/zksync-era/commit/caee55fef4eed0ec58cceaeba277bbdedf5c6f51))
+* **en:** Return `SyncState` health check ([#3142](https://github.com/matter-labs/zksync-era/issues/3142)) ([abeee81](https://github.com/matter-labs/zksync-era/commit/abeee8190d3c3a5e577d71024bdfb30ff516ad03))
+* **external-node:** delete empty unsealed batch on EN initialization ([#3125](https://github.com/matter-labs/zksync-era/issues/3125)) ([5d5214b](https://github.com/matter-labs/zksync-era/commit/5d5214ba983823b306495d34fdd1d46abacce07a))
+* Fix counter metric type to be Counter. ([#3153](https://github.com/matter-labs/zksync-era/issues/3153)) ([08a3fe7](https://github.com/matter-labs/zksync-era/commit/08a3fe7ffd0410c51334193068649905337d5e84))
+* **mempool:** minor mempool improvements ([#3113](https://github.com/matter-labs/zksync-era/issues/3113)) ([cd16083](https://github.com/matter-labs/zksync-era/commit/cd160830a0b7ebe5af4ecbd944da1cd51af3528a))
+* **prover:** Run for zero queue to allow scaling down to 0 ([#3115](https://github.com/matter-labs/zksync-era/issues/3115)) ([bbe1919](https://github.com/matter-labs/zksync-era/commit/bbe191937fa5c5711a7164fd4f0c2ae65cda0833))
+* restore instruction count functionality ([#3081](https://github.com/matter-labs/zksync-era/issues/3081)) ([6159f75](https://github.com/matter-labs/zksync-era/commit/6159f7531a0340a69c4926c4e0325811ed7cabb8))
+* **state-keeper:** save call trace for upgrade txs ([#3132](https://github.com/matter-labs/zksync-era/issues/3132)) ([e1c363f](https://github.com/matter-labs/zksync-era/commit/e1c363f8f5e03c8d62bba1523f17b87d6a0e25ad))
+* **tee_prover:** add zstd compression ([#3144](https://github.com/matter-labs/zksync-era/issues/3144)) ([7241ae1](https://github.com/matter-labs/zksync-era/commit/7241ae139b2b6bf9a9966eaa2f22203583a3786f))
+* **tee_verifier:** correctly initialize storage for re-execution ([#3017](https://github.com/matter-labs/zksync-era/issues/3017)) ([9d88373](https://github.com/matter-labs/zksync-era/commit/9d88373f1b745c489e98e5ef542644a70e815498))
+
 ## [24.29.0](https://github.com/matter-labs/zksync-era/compare/core-v24.28.0...core-v24.29.0) (2024-10-14)
 
 

core/bin/external_node/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "zksync_external_node"
 description = "Non-validator ZKsync node"
-version = "24.29.0" # x-release-please-version
+version = "25.0.0" # x-release-please-version
 edition.workspace = true
 authors.workspace = true
 homepage.workspace = true

core/bin/external_node/src/tests/mod.rs

@@ -22,10 +22,18 @@ const POLL_INTERVAL: Duration = Duration::from_millis(100);
 #[tracing::instrument] // Add args to the test logs
 async fn external_node_basics(components_str: &'static str) {
     let _guard = zksync_vlog::ObservabilityBuilder::new().try_build().ok(); // Enable logging to simplify debugging
-
     let (env, env_handles) = utils::TestEnvironment::with_genesis_block(components_str).await;
 
-    let expected_health_components = utils::expected_health_components(&env.components);
+    let mut expected_health_components = utils::expected_health_components(&env.components);
+    let expected_shutdown_components = expected_health_components.clone();
+    let has_core_or_api = env.components.0.iter().any(|component| {
+        [Component::Core, Component::HttpApi, Component::WsApi].contains(component)
+    });
+    if has_core_or_api {
+        // The `sync_state` component doesn't signal its shutdown, but should be present in the list of components
+        expected_health_components.push("sync_state");
+    }
+
     let l2_client = utils::mock_l2_client(&env);
     let eth_client = utils::mock_eth_client(env.config.diamond_proxy_address());
 
@@ -84,7 +92,7 @@ async fn external_node_basics(components_str: &'static str) {
     let health_data = app_health.check_health().await;
     tracing::info!(?health_data, "final health data");
     assert_matches!(health_data.inner().status(), HealthStatus::ShutDown);
-    for name in expected_health_components {
+    for name in expected_shutdown_components {
         let component_health = &health_data.components()[name];
         assert_matches!(component_health.status(), HealthStatus::ShutDown);
     }

core/bin/zksync_tee_prover/Cargo.toml

@@ -15,7 +15,7 @@ publish = false
 anyhow.workspace = true
 async-trait.workspace = true
 envy.workspace = true
-reqwest.workspace = true
+reqwest = { workspace = true, features = ["zstd"] }
 secp256k1 = { workspace = true, features = ["serde"] }
 serde = { workspace = true, features = ["derive"] }
 thiserror.workspace = true

core/bin/zksync_tee_prover/src/metrics.rs

@@ -2,7 +2,7 @@
 
 use std::time::Duration;
 
-use vise::{Buckets, Gauge, Histogram, Metrics, Unit};
+use vise::{Buckets, Counter, Gauge, Histogram, Metrics, Unit};
 
 #[derive(Debug, Metrics)]
 #[metrics(prefix = "tee_prover")]
@@ -13,7 +13,7 @@ pub(crate) struct TeeProverMetrics {
     pub proof_generation_time: Histogram<Duration>,
     #[metrics(buckets = Buckets::LATENCIES, unit = Unit::Seconds)]
     pub proof_submitting_time: Histogram<Duration>,
-    pub network_errors_counter: Gauge<u64>,
+    pub network_errors_counter: Counter<u64>,
     pub last_batch_number_processed: Gauge<u64>,
 }
 

core/bin/zksync_tee_prover/src/tee_prover.rs

@@ -155,7 +155,7 @@ impl Task for TeeProver {
                     }
                 }
                 Err(err) => {
-                    METRICS.network_errors_counter.inc_by(1);
+                    METRICS.network_errors_counter.inc();
                     if !err.is_retriable() || retries > config.max_retries {
                         return Err(err.into());
                     }

core/lib/config/src/configs/prover_autoscaler.rs

@@ -30,6 +30,9 @@ pub struct ProverAutoscalerAgentConfig {
     pub namespaces: Vec<String>,
     /// Watched cluster name. Also can be set via flag.
     pub cluster_name: Option<String>,
+    /// If dry-run enabled don't do any k8s updates, just report success.
+    #[serde(default = "ProverAutoscalerAgentConfig::default_dry_run")]
+    pub dry_run: bool,
 }
 
 #[derive(Debug, Clone, PartialEq, Deserialize, Default)]
@@ -53,6 +56,8 @@ pub struct ProverAutoscalerScalerConfig {
     pub prover_speed: HashMap<Gpu, u32>,
     /// Maximum number of provers which can be run per cluster/GPU.
     pub max_provers: HashMap<String, HashMap<Gpu, u32>>,
+    /// Minimum number of provers per namespace.
+    pub min_provers: HashMap<String, u32>,
     /// Duration after which pending pod considered long pending.
     #[serde(default = "ProverAutoscalerScalerConfig::default_long_pending_duration")]
     pub long_pending_duration: Duration,
@@ -99,6 +104,10 @@ impl ProverAutoscalerAgentConfig {
     pub fn default_namespaces() -> Vec<String> {
         vec!["prover-blue".to_string(), "prover-red".to_string()]
     }
+
+    pub fn default_dry_run() -> bool {
+        true
+    }
 }
 
 impl ProverAutoscalerScalerConfig {

core/lib/dal/src/consensus_dal/mod.rs

@@ -16,10 +16,48 @@ use crate::{Core, CoreDal};
 #[cfg(test)]
 mod tests;
 
+/// Hash of the batch.
 pub fn batch_hash(info: &StoredBatchInfo) -> attester::BatchHash {
     attester::BatchHash(Keccak256::from_bytes(info.hash().0))
 }
 
+/// Verifies that the transition from `old` to `new` is admissible.
+pub fn verify_config_transition(old: &GlobalConfig, new: &GlobalConfig) -> anyhow::Result<()> {
+    anyhow::ensure!(
+        old.genesis.chain_id == new.genesis.chain_id,
+        "changing chain_id is not allowed: old = {:?}, new = {:?}",
+        old.genesis.chain_id,
+        new.genesis.chain_id,
+    );
+    // Note that it may happen that the fork number didn't change,
+    // in case the binary was updated to support more fields in genesis struct.
+    // In such a case, the old binary was not able to connect to the consensus network,
+    // because of the genesis hash mismatch.
+    // TODO: Perhaps it would be better to deny unknown fields in the genesis instead.
+    // It would require embedding the genesis either as a json string or protobuf bytes within
+    // the global config, so that the global config can be parsed with
+    // `deny_unknown_fields:false` while genesis would be parsed with
+    // `deny_unknown_fields:true`.
+    anyhow::ensure!(
+        old.genesis.fork_number <= new.genesis.fork_number,
+        "transition to a past fork is not allowed: old = {:?}, new = {:?}",
+        old.genesis.fork_number,
+        new.genesis.fork_number,
+    );
+    new.genesis.verify().context("genesis.verify()")?;
+    // This is a temporary hack until the `consensus_genesis()` RPC is disabled.
+    if new
+        == (&GlobalConfig {
+            genesis: old.genesis.clone(),
+            registry_address: None,
+            seed_peers: [].into(),
+        })
+    {
+        anyhow::bail!("new config is equal to truncated old config, which means that it was sourced from the wrong endpoint");
+    }
+    Ok(())
+}
+
 /// Storage access methods for `zksync_core::consensus` module.
 #[derive(Debug)]
 pub struct ConsensusDal<'a, 'c> {
@@ -94,6 +132,8 @@ impl ConsensusDal<'_, '_> {
             if got == want {
                 return Ok(());
             }
+            verify_config_transition(got, want)?;
+
             // If genesis didn't change, just update the config.
             if got.genesis == want.genesis {
                 let s = zksync_protobuf::serde::Serialize;
@@ -112,30 +152,6 @@ impl ConsensusDal<'_, '_> {
                 txn.commit().await?;
                 return Ok(());
             }
-
-            // Verify the genesis change.
-            anyhow::ensure!(
-                got.genesis.chain_id == want.genesis.chain_id,
-                "changing chain_id is not allowed: old = {:?}, new = {:?}",
-                got.genesis.chain_id,
-                want.genesis.chain_id,
-            );
-            // Note that it may happen that the fork number didn't change,
-            // in case the binary was updated to support more fields in genesis struct.
-            // In such a case, the old binary was not able to connect to the consensus network,
-            // because of the genesis hash mismatch.
-            // TODO: Perhaps it would be better to deny unknown fields in the genesis instead.
-            // It would require embedding the genesis either as a json string or protobuf bytes within
-            // the global config, so that the global config can be parsed with
-            // `deny_unknown_fields:false` while genesis would be parsed with
-            // `deny_unknown_fields:true`.
-            anyhow::ensure!(
-                got.genesis.fork_number <= want.genesis.fork_number,
-                "transition to a past fork is not allowed: old = {:?}, new = {:?}",
-                got.genesis.fork_number,
-                want.genesis.fork_number,
-            );
-            want.genesis.verify().context("genesis.verify()")?;
         }
 
         // Reset the consensus state.