Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: Implement gas relay mode and inclusion data for data attestation #3070

Merged
merged 30 commits into from
Oct 24, 2024
Merged
Show file tree
Hide file tree
Changes from 24 commits
Commits
Show all changes
30 commits
Select commit Hold shift + click to select a range
788f5a3
feat: implement gas relay mode
vibhurajeev Oct 9, 2024
feec572
fix(env_config): merge conflicts with da client resolved
vibhurajeev Oct 9, 2024
fc9f7d4
fix(protobuf_config/da_client): minor cleanup
vibhurajeev Oct 9, 2024
b1cc9b1
fix(config/da_client): moved gas relay api key to secrets
vibhurajeev Oct 9, 2024
2c16460
fix(protobuf_config): remove redundant gas_relay_mode parse
QEDK Oct 10, 2024
f235786
feat(avail): implement tokenize for merkleproofinput
QEDK Oct 10, 2024
aab3e78
chore(deps): remove alloy dep
QEDK Oct 10, 2024
c6434c9
fix(.gitignore): readd removed folder
QEDK Oct 10, 2024
dec3b95
fix(protobuf-secrets,avail-client): make da secrets one-of, remove lo…
QEDK Oct 13, 2024
d57cd92
feat(zksync-basic-types): make new api key type and change gas relay …
QEDK Oct 14, 2024
d69cec1
feat(avail-client,protobuf-da-client,env-config): enum-ify modes and …
QEDK Oct 14, 2024
f3a8687
feat(protobuf-da-client): remove timeout and retries
QEDK Oct 14, 2024
0606be1
lint: clippy
QEDK Oct 14, 2024
ca1b381
feat(avail-client,protobuf-da-client): remove fields and add timeout
QEDK Oct 15, 2024
5304bc1
feat(avail-client): introduce timeouts for inclusion checks
QEDK Oct 15, 2024
de45684
fix: proto and env configs
dimazhornyk Oct 16, 2024
e04cbe8
update contracts
dimazhornyk Oct 16, 2024
6dd2671
fix: camel_case
dimazhornyk Oct 16, 2024
f3b524a
fix: linter
dimazhornyk Oct 16, 2024
6c6c0e0
Merge branch 'main' of github.com:matter-labs/zksync-era into dz-sync…
dimazhornyk Oct 17, 2024
ef6531e
decrease polling interval
dimazhornyk Oct 17, 2024
fcf68eb
fix(protobuf-da-client): remove redundant fields from match
QEDK Oct 18, 2024
9963ae2
fix(protobuf-secrets): make matching more logical
QEDK Oct 18, 2024
b9b2ea5
fix(da_clients-avail-sdk): remove superfluous error mappings
QEDK Oct 18, 2024
cd50d1a
fix(avail-client,protobuf-da-client): add max_retries to gas relay, c…
QEDK Oct 21, 2024
2efd527
chore(contracts): set to current commit
QEDK Oct 21, 2024
3ea1ed9
add backon
dimazhornyk Oct 23, 2024
1cef015
resolved conflicts, do some fixes
dimazhornyk Oct 23, 2024
f3d127a
Merge branch 'main' into dz-sync-layer-da-clients-2
dimazhornyk Oct 23, 2024
6c58d88
Merge branch 'main' into dz-sync-layer-da-clients-2
dimazhornyk Oct 23, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,7 @@ prover/data/keys/setup_*
# ZK Stack CLI
chains/era/configs/*
chains/gateway/*
chains/avail/*
configs/*
era-observability/
core/tests/ts-integration/deployments-zk
Expand Down
2 changes: 2 additions & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 2 additions & 1 deletion Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,7 @@ backon = "0.4.4"
bigdecimal = "0.4.5"
bincode = "1"
blake2 = "0.10"
bytes = "1"
chrono = "0.4"
clap = "4.2.2"
codegen = "0.2.0"
Expand Down Expand Up @@ -156,7 +157,7 @@ rayon = "1.3.1"
regex = "1"
reqwest = "0.12"
rlp = "0.5"
rocksdb = "0.21.0"
rocksdb = "0.21"
rustc_version = "0.4.0"
rustls = "0.23"
secp256k1 = { version = "0.27.0", features = ["recovery", "global-context"] }
Expand Down
2 changes: 1 addition & 1 deletion contracts
QEDK marked this conversation as resolved.
Show resolved Hide resolved
Submodule contracts updated 492 files
20 changes: 20 additions & 0 deletions core/lib/basic_types/src/api_key.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
use std::str::FromStr;

use secrecy::{ExposeSecret, Secret};

#[derive(Debug, Clone)]
pub struct APIKey(pub Secret<String>);

impl PartialEq for APIKey {
fn eq(&self, other: &Self) -> bool {
self.0.expose_secret().eq(other.0.expose_secret())
}
}

impl FromStr for APIKey {
type Err = anyhow::Error;

fn from_str(s: &str) -> Result<Self, Self::Err> {
Ok(APIKey(s.parse()?))
}
}
1 change: 1 addition & 0 deletions core/lib/basic_types/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ use serde::{de, Deserialize, Deserializer, Serialize};

#[macro_use]
mod macros;
pub mod api_key;
pub mod basic_fri_types;
pub mod commitment;
pub mod network;
Expand Down
29 changes: 25 additions & 4 deletions core/lib/config/src/configs/da_client/avail.rs
Original file line number Diff line number Diff line change
@@ -1,16 +1,37 @@
use serde::Deserialize;
use zksync_basic_types::seed_phrase::SeedPhrase;
use zksync_basic_types::{api_key::APIKey, seed_phrase::SeedPhrase};

pub const AVAIL_GAS_RELAY_CLIENT_NAME: &str = "GasRelay";
pub const AVAIL_FULL_CLIENT_NAME: &str = "FullClient";

#[derive(Clone, Debug, PartialEq, Deserialize)]
#[serde(tag = "avail_client")]
pub enum AvailClientConfig {
FullClient(AvailDefaultConfig),
GasRelay(AvailGasRelayConfig),
}

#[derive(Clone, Debug, PartialEq, Deserialize)]
pub struct AvailConfig {
pub api_node_url: String,
pub bridge_api_url: String,
pub app_id: u32,
pub timeout: usize,
pub max_retries: usize,
#[serde(flatten)]
pub config: AvailClientConfig,
}

#[derive(Clone, Debug, PartialEq, Deserialize)]
pub struct AvailDefaultConfig {
pub api_node_url: String,
pub app_id: u32,
}

#[derive(Clone, Debug, PartialEq, Deserialize)]
pub struct AvailGasRelayConfig {
pub gas_relay_api_url: String,
}

#[derive(Clone, Debug, PartialEq)]
pub struct AvailSecrets {
pub seed_phrase: Option<SeedPhrase>,
pub gas_relay_api_key: Option<APIKey>,
}
16 changes: 12 additions & 4 deletions core/lib/config/src/testonly.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ use std::num::NonZeroUsize;
use rand::{distributions::Distribution, Rng};
use secrecy::Secret;
use zksync_basic_types::{
api_key::APIKey,
basic_fri_types::CircuitIdRoundTuple,
commitment::L1BatchCommitmentMode,
network::Network,
Expand All @@ -16,7 +17,12 @@ use zksync_crypto_primitives::K256PrivateKey;

use crate::{
configs::{
self, da_client::DAClientConfig::Avail, eth_sender::PubdataSendingMode,
self,
da_client::{
avail::{AvailClientConfig, AvailDefaultConfig},
DAClientConfig::Avail,
},
eth_sender::PubdataSendingMode,
external_price_api_client::ForcedPriceClientConfig,
},
AvailConfig,
Expand Down Expand Up @@ -946,11 +952,12 @@ impl Distribution<configs::en_config::ENConfig> for EncodeDist {
impl Distribution<configs::da_client::DAClientConfig> for EncodeDist {
fn sample<R: Rng + ?Sized>(&self, rng: &mut R) -> configs::da_client::DAClientConfig {
Avail(AvailConfig {
api_node_url: self.sample(rng),
bridge_api_url: self.sample(rng),
app_id: self.sample(rng),
timeout: self.sample(rng),
max_retries: self.sample(rng),
config: AvailClientConfig::FullClient(AvailDefaultConfig {
api_node_url: self.sample(rng),
app_id: self.sample(rng),
}),
})
}
}
Expand All @@ -959,6 +966,7 @@ impl Distribution<configs::secrets::DataAvailabilitySecrets> for EncodeDist {
fn sample<R: Rng + ?Sized>(&self, rng: &mut R) -> configs::secrets::DataAvailabilitySecrets {
configs::secrets::DataAvailabilitySecrets::Avail(configs::da_client::avail::AvailSecrets {
seed_phrase: Some(SeedPhrase(Secret::new(self.sample(rng)))),
gas_relay_api_key: Some(APIKey(Secret::new(self.sample(rng)))),
})
}
}
Expand Down
81 changes: 57 additions & 24 deletions core/lib/env_config/src/da_client.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,19 +2,35 @@ use std::env;

use zksync_config::configs::{
da_client::{
avail::AvailSecrets, DAClientConfig, AVAIL_CLIENT_CONFIG_NAME,
OBJECT_STORE_CLIENT_CONFIG_NAME,
avail::{
AvailClientConfig, AvailSecrets, AVAIL_FULL_CLIENT_NAME, AVAIL_GAS_RELAY_CLIENT_NAME,
},
DAClientConfig, AVAIL_CLIENT_CONFIG_NAME, OBJECT_STORE_CLIENT_CONFIG_NAME,
},
secrets::DataAvailabilitySecrets,
AvailConfig,
};

use crate::{envy_load, FromEnv};

impl FromEnv for DAClientConfig {
fn from_env() -> anyhow::Result<Self> {
let client_tag = std::env::var("DA_CLIENT")?;
let client_tag = env::var("DA_CLIENT")?;
let config = match client_tag.as_str() {
AVAIL_CLIENT_CONFIG_NAME => Self::Avail(envy_load("da_avail_config", "DA_")?),
AVAIL_CLIENT_CONFIG_NAME => Self::Avail(AvailConfig {
bridge_api_url: env::var("DA_BRIDGE_API_URL").ok().unwrap(),
timeout: env::var("DA_TIMEOUT")?.parse()?,

config: match env::var("DA_AVAIL_CLIENT_TYPE")?.as_str() {
AVAIL_FULL_CLIENT_NAME => {
AvailClientConfig::FullClient(envy_load("da_avail_full_client", "DA_")?)
}
AVAIL_GAS_RELAY_CLIENT_NAME => {
AvailClientConfig::GasRelay(envy_load("da_avail_gas_relay", "DA_")?)
}
_ => anyhow::bail!("Unknown Avail DA client type"),
},
}),
OBJECT_STORE_CLIENT_CONFIG_NAME => {
Self::ObjectStore(envy_load("da_object_store", "DA_")?)
}
Expand All @@ -30,11 +46,21 @@ impl FromEnv for DataAvailabilitySecrets {
let client_tag = std::env::var("DA_CLIENT")?;
let secrets = match client_tag.as_str() {
AVAIL_CLIENT_CONFIG_NAME => {
let seed_phrase = env::var("DA_SECRETS_SEED_PHRASE")
.ok()
.map(|s| s.parse())
.transpose()?;
Self::Avail(AvailSecrets { seed_phrase })
let seed_phrase: Option<zksync_basic_types::seed_phrase::SeedPhrase> =
env::var("DA_SECRETS_SEED_PHRASE")
.ok()
.map(|s| s.parse().unwrap());
let gas_relay_api_key: Option<zksync_basic_types::api_key::APIKey> =
env::var("DA_SECRETS_GAS_RELAY_API_KEY")
.ok()
.map(|s| s.parse().unwrap());
if seed_phrase.is_none() && gas_relay_api_key.is_none() {
anyhow::bail!("No secrets provided for Avail DA client");
}
Self::Avail(AvailSecrets {
seed_phrase,
gas_relay_api_key,
})
}
_ => anyhow::bail!("Unknown DA client name: {}", client_tag),
};
Expand All @@ -47,7 +73,10 @@ impl FromEnv for DataAvailabilitySecrets {
mod tests {
use zksync_config::{
configs::{
da_client::{DAClientConfig, DAClientConfig::ObjectStore},
da_client::{
avail::{AvailClientConfig, AvailDefaultConfig},
DAClientConfig::{self, ObjectStore},
},
object_store::ObjectStoreMode::GCS,
},
AvailConfig, ObjectStoreConfig,
Expand Down Expand Up @@ -91,14 +120,14 @@ mod tests {
bridge_api_url: &str,
app_id: u32,
timeout: usize,
max_retries: usize,
) -> DAClientConfig {
DAClientConfig::Avail(AvailConfig {
api_node_url: api_node_url.to_string(),
bridge_api_url: bridge_api_url.to_string(),
app_id,
timeout,
max_retries,
config: AvailClientConfig::FullClient(AvailDefaultConfig {
api_node_url: api_node_url.to_string(),
app_id,
}),
})
}

Expand All @@ -107,11 +136,13 @@ mod tests {
let mut lock = MUTEX.lock();
let config = r#"
DA_CLIENT="Avail"
DA_API_NODE_URL="localhost:12345"
DA_AVAIL_CLIENT_TYPE="FullClient"

DA_BRIDGE_API_URL="localhost:54321"
DA_APP_ID="1"
DA_TIMEOUT="2"
DA_MAX_RETRIES="3"

DA_API_NODE_URL="localhost:12345"
DA_APP_ID="1"
"#;

lock.set_env(config);
Expand All @@ -124,7 +155,6 @@ mod tests {
"localhost:54321",
"1".parse::<u32>().unwrap(),
"2".parse::<usize>().unwrap(),
"3".parse::<usize>().unwrap(),
)
);
}
Expand All @@ -139,15 +169,18 @@ mod tests {

lock.set_env(config);

let actual = match DataAvailabilitySecrets::from_env().unwrap() {
DataAvailabilitySecrets::Avail(avail) => avail.seed_phrase,
let (actual_seed, actual_key) = match DataAvailabilitySecrets::from_env().unwrap() {
DataAvailabilitySecrets::Avail(avail) => (avail.seed_phrase, avail.gas_relay_api_key),
};

assert_eq!(
actual.unwrap(),
"bottom drive obey lake curtain smoke basket hold race lonely fit walk"
.parse()
.unwrap()
(actual_seed.unwrap(), actual_key),
(
"bottom drive obey lake curtain smoke basket hold race lonely fit walk"
.parse()
.unwrap(),
None
)
);
}
}
49 changes: 36 additions & 13 deletions core/lib/protobuf_config/src/da_client.rs
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
use anyhow::Context;
use zksync_config::{
configs::{
da_client::DAClientConfig::{Avail, ObjectStore},
{self},
use zksync_config::configs::{
self,
da_client::{
avail::{AvailClientConfig, AvailConfig, AvailDefaultConfig, AvailGasRelayConfig},
DAClientConfig::{Avail, ObjectStore},
},
AvailConfig,
};
use zksync_protobuf::{required, ProtoRepr};

Expand All @@ -18,15 +18,28 @@ impl ProtoRepr for proto::DataAvailabilityClient {

let client = match config {
proto::data_availability_client::Config::Avail(conf) => Avail(AvailConfig {
api_node_url: required(&conf.api_node_url)
.context("api_node_url")?
.clone(),
bridge_api_url: required(&conf.bridge_api_url)
.context("bridge_api_url")?
.clone(),
app_id: *required(&conf.app_id).context("app_id")?,
timeout: *required(&conf.timeout).context("timeout")? as usize,
max_retries: *required(&conf.max_retries).context("max_retries")? as usize,
config: match conf.config.as_ref() {
Some(proto::avail_config::Config::FullClient(full_client_conf)) => {
AvailClientConfig::FullClient(AvailDefaultConfig {
api_node_url: required(&full_client_conf.api_node_url)
.context("api_node_url")?
.clone(),
app_id: *required(&full_client_conf.app_id).context("app_id")?,
})
}
Some(proto::avail_config::Config::GasRelay(gas_relay_conf)) => {
AvailClientConfig::GasRelay(AvailGasRelayConfig {
gas_relay_api_url: required(&gas_relay_conf.gas_relay_api_url)
.context("gas_relay_api_url")?
.clone(),
})
}
None => return Err(anyhow::anyhow!("Invalid Avail DA configuration")),
},
}),
proto::data_availability_client::Config::ObjectStore(conf) => {
ObjectStore(object_store_proto::ObjectStore::read(conf)?)
Expand All @@ -41,11 +54,21 @@ impl ProtoRepr for proto::DataAvailabilityClient {
Avail(config) => Self {
config: Some(proto::data_availability_client::Config::Avail(
proto::AvailConfig {
api_node_url: Some(config.api_node_url.clone()),
bridge_api_url: Some(config.bridge_api_url.clone()),
app_id: Some(config.app_id),
timeout: Some(config.timeout as u64),
max_retries: Some(config.max_retries as u64),
config: match &config.config {
AvailClientConfig::FullClient(conf) => Some(
proto::avail_config::Config::FullClient(proto::AvailClientConfig {
api_node_url: Some(conf.api_node_url.clone()),
app_id: Some(conf.app_id),
}),
),
AvailClientConfig::GasRelay(conf) => Some(
proto::avail_config::Config::GasRelay(proto::AvailGasRelayConfig {
gas_relay_api_url: Some(conf.gas_relay_api_url.clone()),
}),
),
},
},
)),
},
Expand Down
Loading