diff --git a/CHANGELOG.md b/CHANGELOG.md index 82018148012..570a3130672 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/). - [2321](https://github.com/FuelLabs/fuel-core/pull/2321): New metrics for the txpool: "The size of transactions in the txpool" (`txpool_tx_size`), "The time spent by a transaction in the txpool in seconds" (`txpool_tx_time_in_txpool_seconds`), The number of transactions in the txpool (`txpool_number_of_transactions`), "The number of transactions pending verification before entering the txpool" (`txpool_number_of_transactions_pending_verification`), "The number of executable transactions in the txpool" (`txpool_number_of_executable_transactions`), "The time it took to select transactions for inclusion in a block in nanoseconds" (`txpool_select_transaction_time_nanoseconds`), The time it took to insert a transaction in the txpool in milliseconds (`txpool_insert_transaction_time_milliseconds`). - [2347](https://github.com/FuelLabs/fuel-core/pull/2364): Add activity concept in order to protect against infinitely increasing DA gas price scenarios - [2362](https://github.com/FuelLabs/fuel-core/pull/2362): Added a new request_response protocol version `/fuel/req_res/0.0.2`. In comparison with `/fuel/req/0.0.1`, which returns an empty response when a request cannot be fulfilled, this version returns more meaningful error codes. Nodes still support the version `0.0.1` of the protocol to guarantee backward compatibility with fuel-core nodes. Empty responses received from nodes using the old protocol `/fuel/req/0.0.1` are automatically converted into an error `ProtocolV1EmptyResponse` with error code 0, which is also the only error code implemented. More specific error codes will be added in the future. +- [2386](https://github.com/FuelLabs/fuel-core/pull/2386): Add a flag to define the maximum number of file descriptors that RocksDB can use. By default it's half of the OS limit. ## [Version 0.40.0] diff --git a/Cargo.lock b/Cargo.lock index d4d274e7f71..8e305a0c04a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3295,6 +3295,7 @@ dependencies = [ "pyroscope", "pyroscope_pprofrs", "rand", + "rlimit", "serde", "serde_json", "strum 0.25.0", @@ -7657,6 +7658,15 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3582f63211428f83597b51b2ddb88e2a91a9d52d12831f9d08f5e624e8977422" +[[package]] +name = "rlimit" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7043b63bd0cd1aaa628e476b80e6d4023a3b50eb32789f2728908107bd0c793a" +dependencies = [ + "libc", +] + [[package]] name = "rlp" version = "0.5.2" diff --git a/benches/benches/vm_set/blockchain.rs b/benches/benches/vm_set/blockchain.rs index 2d85a4f2028..cd51bf6dbc9 100644 --- a/benches/benches/vm_set/blockchain.rs +++ b/benches/benches/vm_set/blockchain.rs @@ -75,6 +75,7 @@ impl BenchDb { tmp_dir.path(), None, Default::default(), + -1, ) .unwrap(); let db = Arc::new(db); diff --git a/benches/src/db_lookup_times_utils/mod.rs b/benches/src/db_lookup_times_utils/mod.rs index 9bb12fb7169..0d001a3bc3e 100644 --- a/benches/src/db_lookup_times_utils/mod.rs +++ b/benches/src/db_lookup_times_utils/mod.rs @@ -27,7 +27,7 @@ mod tests { fn setup_test_db() -> RocksDb { let temp_dir = ShallowTempDir::new(); - RocksDb::default_open(temp_dir.path(), None).unwrap() + RocksDb::default_open(temp_dir.path(), None, -1).unwrap() } #[test] diff --git a/benches/src/db_lookup_times_utils/utils.rs b/benches/src/db_lookup_times_utils/utils.rs index 5845079e2b5..4db547a5094 100644 --- a/benches/src/db_lookup_times_utils/utils.rs +++ b/benches/src/db_lookup_times_utils/utils.rs @@ -39,7 +39,7 @@ pub fn get_random_block_height( pub fn open_rocks_db( path: &Path, ) -> Result> { - let db = RocksDb::default_open(path, None)?; + let db = RocksDb::default_open(path, None, -1)?; Ok(db) } diff --git a/bin/fuel-core/Cargo.toml b/bin/fuel-core/Cargo.toml index 76bbb6d0367..41d05c2b69a 100644 --- a/bin/fuel-core/Cargo.toml +++ b/bin/fuel-core/Cargo.toml @@ -35,6 +35,7 @@ hex = { workspace = true } humantime = "2.1" pyroscope = "0.5" pyroscope_pprofrs = "0.2" +rlimit = "0.10.2" serde_json = { workspace = true } tikv-jemallocator = { workspace = true } tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } diff --git a/bin/fuel-core/src/cli/rollback.rs b/bin/fuel-core/src/cli/rollback.rs index 2b85edd0e4f..12e18a0547b 100644 --- a/bin/fuel-core/src/cli/rollback.rs +++ b/bin/fuel-core/src/cli/rollback.rs @@ -5,6 +5,10 @@ use fuel_core::{ combined_database::CombinedDatabase, state::historical_rocksdb::StateRewindPolicy, }; +use rlimit::{ + getrlimit, + Resource, +}; use std::path::PathBuf; /// Rollbacks the state of the blockchain to a specific block height. @@ -19,11 +23,28 @@ pub struct Command { )] pub database_path: PathBuf, + /// Defines a specific number of file descriptors that RocksDB can use. + /// + /// If defined as -1 no limit will be applied and will use the OS limits. + /// If not defined the system default divided by two is used. + #[clap( + long = "rocksdb-max-fds", + env, + default_value = get_default_max_fds().to_string() + )] + pub rocksdb_max_fds: i32, + /// The path to the database. #[clap(long = "target-block-height")] pub target_block_height: u32, } +fn get_default_max_fds() -> i32 { + getrlimit(Resource::NOFILE) + .map(|(_, hard)| i32::try_from(hard.saturating_div(2)).unwrap_or(i32::MAX)) + .expect("Our supported platforms should return max FD.") +} + pub async fn exec(command: Command) -> anyhow::Result<()> { use crate::cli::ShutdownListener; @@ -32,6 +53,7 @@ pub async fn exec(command: Command) -> anyhow::Result<()> { path, 64 * 1024 * 1024, StateRewindPolicy::RewindFullRange, + command.rocksdb_max_fds, ) .map_err(Into::::into) .context(format!("failed to open combined database at path {path:?}"))?; diff --git a/bin/fuel-core/src/cli/run.rs b/bin/fuel-core/src/cli/run.rs index 256ecc1fe3b..06d00219b50 100644 --- a/bin/fuel-core/src/cli/run.rs +++ b/bin/fuel-core/src/cli/run.rs @@ -65,6 +65,10 @@ use pyroscope_pprofrs::{ pprof_backend, PprofConfig, }; +use rlimit::{ + getrlimit, + Resource, +}; use std::{ env, net, @@ -126,6 +130,18 @@ pub struct Command { )] pub database_type: DbType, + #[cfg(feature = "rocksdb")] + /// Defines a specific number of file descriptors that RocksDB can use. + /// + /// If defined as -1 no limit will be applied and will use the OS limits. + /// If not defined the system default divided by two is used. + #[clap( + long = "rocksdb-max-fds", + env, + default_value = get_default_max_fds().to_string() + )] + pub rocksdb_max_fds: i32, + #[cfg(feature = "rocksdb")] /// Defines the state rewind policy for the database when RocksDB is enabled. /// @@ -273,6 +289,8 @@ impl Command { database_path, database_type, #[cfg(feature = "rocksdb")] + rocksdb_max_fds, + #[cfg(feature = "rocksdb")] state_rewind_duration, db_prune, snapshot, @@ -441,6 +459,8 @@ impl Command { max_database_cache_size, #[cfg(feature = "rocksdb")] state_rewind_policy, + #[cfg(feature = "rocksdb")] + max_fds: rocksdb_max_fds, }; let block_importer = fuel_core::service::config::fuel_core_importer::Config::new( @@ -591,6 +611,13 @@ impl Command { } } +#[cfg(feature = "rocksdb")] +fn get_default_max_fds() -> i32 { + getrlimit(Resource::NOFILE) + .map(|(_, hard)| i32::try_from(hard.saturating_div(2)).unwrap_or(i32::MAX)) + .expect("Our supported platforms should return max FD.") +} + pub async fn get_service_with_shutdown_listeners( command: Command, ) -> anyhow::Result<(FuelService, ShutdownListener)> { diff --git a/bin/fuel-core/src/cli/snapshot.rs b/bin/fuel-core/src/cli/snapshot.rs index 0c03f73edfd..3f562e38733 100644 --- a/bin/fuel-core/src/cli/snapshot.rs +++ b/bin/fuel-core/src/cli/snapshot.rs @@ -10,6 +10,10 @@ use fuel_core::{ types::fuel_types::ContractId, }; use fuel_core_chain_config::ChainConfig; +use rlimit::{ + getrlimit, + Resource, +}; use std::path::{ Path, PathBuf, @@ -29,6 +33,17 @@ pub struct Command { )] pub database_path: PathBuf, + /// Defines a specific number of file descriptors that RocksDB can use. + /// + /// If defined as -1 no limit will be applied and will use the OS limits. + /// If not defined the system default divided by two is used. + #[clap( + long = "rocksdb-max-fds", + env, + default_value = get_default_max_fds().to_string() + )] + pub rocksdb_max_fds: i32, + /// Where to save the snapshot #[arg(name = "OUTPUT_DIR", long = "output-directory")] pub output_dir: PathBuf, @@ -112,6 +127,12 @@ pub enum SubCommands { }, } +fn get_default_max_fds() -> i32 { + getrlimit(Resource::NOFILE) + .map(|(_, hard)| i32::try_from(hard.saturating_div(2)).unwrap_or(i32::MAX)) + .expect("Our supported platforms should return max FD.") +} + #[cfg(feature = "rocksdb")] pub async fn exec(command: Command) -> anyhow::Result<()> { use fuel_core::service::genesis::Exporter; @@ -125,6 +146,7 @@ pub async fn exec(command: Command) -> anyhow::Result<()> { let db = open_db( &command.database_path, Some(command.max_database_cache_size), + command.rocksdb_max_fds, )?; let output_dir = command.output_dir; let shutdown_listener = ShutdownListener::spawn(); @@ -180,11 +202,16 @@ fn load_chain_config_or_use_testnet(path: Option<&Path>) -> anyhow::Result) -> anyhow::Result { +fn open_db( + path: &Path, + capacity: Option, + max_fds: i32, +) -> anyhow::Result { CombinedDatabase::open( path, capacity.unwrap_or(1024 * 1024 * 1024), StateRewindPolicy::NoRewind, + max_fds, ) .map_err(Into::::into) .context(format!("failed to open combined database at path {path:?}",)) @@ -668,7 +695,8 @@ mod tests { let db_path = temp_dir.path().join("db"); std::fs::create_dir(&db_path)?; - let mut db = DbPopulator::new(open_db(&db_path, None)?, StdRng::seed_from_u64(2)); + let mut db = + DbPopulator::new(open_db(&db_path, None, 512)?, StdRng::seed_from_u64(2)); let state = db.given_persisted_data(); db.flush(); @@ -681,6 +709,7 @@ mod tests { chain_config: None, encoding_command: Some(EncodingCommand::Encoding { encoding }), }, + rocksdb_max_fds: 512, }); // Because the test_case macro doesn't work with async tests @@ -720,7 +749,8 @@ mod tests { let snapshot_dir = temp_dir.path().join("snapshot"); let db_path = temp_dir.path().join("db"); - let mut db = DbPopulator::new(open_db(&db_path, None)?, StdRng::seed_from_u64(2)); + let mut db = + DbPopulator::new(open_db(&db_path, None, 512)?, StdRng::seed_from_u64(2)); let state = db.given_persisted_data(); db.flush(); @@ -739,6 +769,7 @@ mod tests { }, }), }, + rocksdb_max_fds: 512, }); tokio::runtime::Runtime::new() @@ -763,7 +794,8 @@ mod tests { let snapshot_dir = temp_dir.path().join("snapshot"); let db_path = temp_dir.path().join("db"); - let mut db = DbPopulator::new(open_db(&db_path, None)?, StdRng::seed_from_u64(2)); + let mut db = + DbPopulator::new(open_db(&db_path, None, 512)?, StdRng::seed_from_u64(2)); let original_state = db.given_persisted_data().sorted().into_state_config(); @@ -789,6 +821,7 @@ mod tests { output_dir: snapshot_dir.clone(), max_database_cache_size: DEFAULT_DATABASE_CACHE_SIZE, subcommand: SubCommands::Contract { contract_id }, + rocksdb_max_fds: 512, }) .await?; diff --git a/crates/fuel-core/src/combined_database.rs b/crates/fuel-core/src/combined_database.rs index f69649d2605..8696a8b3969 100644 --- a/crates/fuel-core/src/combined_database.rs +++ b/crates/fuel-core/src/combined_database.rs @@ -39,6 +39,8 @@ pub struct CombinedDatabaseConfig { pub max_database_cache_size: usize, #[cfg(feature = "rocksdb")] pub state_rewind_policy: StateRewindPolicy, + #[cfg(feature = "rocksdb")] + pub max_fds: i32, } /// A database that combines the on-chain, off-chain and relayer databases into one entity. @@ -79,13 +81,22 @@ impl CombinedDatabase { path: &std::path::Path, capacity: usize, state_rewind_policy: StateRewindPolicy, + max_fds: i32, ) -> crate::database::Result { + // Split the fds in equitable manner between the databases + let max_fds = match max_fds { + -1 => -1, + _ => max_fds.saturating_div(4), + }; // TODO: Use different cache sizes for different databases - let on_chain = Database::open_rocksdb(path, capacity, state_rewind_policy)?; - let off_chain = Database::open_rocksdb(path, capacity, state_rewind_policy)?; + let on_chain = + Database::open_rocksdb(path, capacity, state_rewind_policy, max_fds)?; + let off_chain = + Database::open_rocksdb(path, capacity, state_rewind_policy, max_fds)?; let relayer = - Database::open_rocksdb(path, capacity, StateRewindPolicy::NoRewind)?; - let gas_price = Database::open_rocksdb(path, capacity, state_rewind_policy)?; + Database::open_rocksdb(path, capacity, StateRewindPolicy::NoRewind, max_fds)?; + let gas_price = + Database::open_rocksdb(path, capacity, state_rewind_policy, max_fds)?; Ok(Self { on_chain, off_chain, @@ -115,6 +126,7 @@ impl CombinedDatabase { &config.database_path, config.max_database_cache_size, config.state_rewind_policy, + config.max_fds, )? } } diff --git a/crates/fuel-core/src/database.rs b/crates/fuel-core/src/database.rs index d1d85dfe17e..5061692ed92 100644 --- a/crates/fuel-core/src/database.rs +++ b/crates/fuel-core/src/database.rs @@ -199,12 +199,14 @@ where path: &Path, capacity: impl Into>, state_rewind_policy: StateRewindPolicy, + max_fds: i32, ) -> Result { use anyhow::Context; let db = HistoricalRocksDB::::default_open( path, capacity.into(), state_rewind_policy, + max_fds, ) .map_err(Into::::into) .with_context(|| { @@ -1076,6 +1078,7 @@ mod tests { temp_dir.path(), 1024 * 1024 * 1024, Default::default(), + 512, ) .unwrap(); // rocks db fails diff --git a/crates/fuel-core/src/service/config.rs b/crates/fuel-core/src/service/config.rs index 2dd6602a979..0e093c2b8b6 100644 --- a/crates/fuel-core/src/service/config.rs +++ b/crates/fuel-core/src/service/config.rs @@ -126,6 +126,8 @@ impl Config { #[cfg(feature = "rocksdb")] state_rewind_policy: crate::state::historical_rocksdb::StateRewindPolicy::RewindFullRange, + #[cfg(feature = "rocksdb")] + max_fds: 512, }; let starting_gas_price = 0; let gas_price_change_percent = 0; diff --git a/crates/fuel-core/src/state/historical_rocksdb.rs b/crates/fuel-core/src/state/historical_rocksdb.rs index a95368ec12f..52887fdaf2e 100644 --- a/crates/fuel-core/src/state/historical_rocksdb.rs +++ b/crates/fuel-core/src/state/historical_rocksdb.rs @@ -101,8 +101,10 @@ where path: P, capacity: Option, state_rewind_policy: StateRewindPolicy, + max_fds: i32, ) -> DatabaseResult { - let db = RocksDb::>::default_open(path, capacity)?; + let db = + RocksDb::>::default_open(path, capacity, max_fds)?; Ok(Self { state_rewind_policy, db, diff --git a/crates/fuel-core/src/state/rocks_db.rs b/crates/fuel-core/src/state/rocks_db.rs index c54ee3765c1..c751ebe7c55 100644 --- a/crates/fuel-core/src/state/rocks_db.rs +++ b/crates/fuel-core/src/state/rocks_db.rs @@ -127,6 +127,7 @@ where path, enum_iterator::all::().collect::>(), capacity, + 512, ); let mut db = result?; @@ -146,11 +147,13 @@ where pub fn default_open>( path: P, capacity: Option, + max_fds: i32, ) -> DatabaseResult { Self::open( path, enum_iterator::all::().collect::>(), capacity, + max_fds, ) } @@ -165,8 +168,9 @@ where path: P, columns: Vec, capacity: Option, + max_fds: i32, ) -> DatabaseResult { - Self::open_with(DB::open_cf_descriptors, path, columns, capacity) + Self::open_with(DB::open_cf_descriptors, path, columns, capacity, max_fds) } pub fn open_read_only>( @@ -174,6 +178,7 @@ where columns: Vec, capacity: Option, error_if_log_file_exist: bool, + max_fds: i32, ) -> DatabaseResult { Self::open_with( |options, primary_path, cfs| { @@ -187,6 +192,7 @@ where path, columns, capacity, + max_fds, ) } @@ -195,6 +201,7 @@ where secondary_path: SecondaryPath, columns: Vec, capacity: Option, + max_fds: i32, ) -> DatabaseResult where PrimaryPath: AsRef, @@ -212,6 +219,7 @@ where path, columns, capacity, + max_fds, ) } @@ -220,6 +228,7 @@ where path: P, columns: Vec, capacity: Option, + max_fds: i32, ) -> DatabaseResult where F: Fn( @@ -279,9 +288,7 @@ where } opts.set_max_background_jobs(6); opts.set_bytes_per_sync(1048576); - - #[cfg(feature = "test-helpers")] - opts.set_max_open_files(512); + opts.set_max_open_files(max_fds); let existing_column_families = DB::list_cf(&opts, &path).unwrap_or_default(); @@ -880,7 +887,7 @@ mod tests { fn create_db() -> (RocksDb, TempDir) { let tmp_dir = TempDir::new().unwrap(); ( - RocksDb::default_open(tmp_dir.path(), None).unwrap(), + RocksDb::default_open(tmp_dir.path(), None, 512).unwrap(), tmp_dir, ) } @@ -893,7 +900,7 @@ mod tests { let old_columns = vec![Column::Coins, Column::Messages, Column::UploadedBytecodes]; let database_with_old_columns = - RocksDb::::open(tmp_dir.path(), old_columns.clone(), None) + RocksDb::::open(tmp_dir.path(), old_columns.clone(), None, 512) .expect("Failed to open database with old columns"); drop(database_with_old_columns); @@ -902,7 +909,7 @@ mod tests { new_columns.push(Column::ContractsAssets); new_columns.push(Column::Metadata); let database_with_new_columns = - RocksDb::::open(tmp_dir.path(), new_columns, None).map(|_| ()); + RocksDb::::open(tmp_dir.path(), new_columns, None, 512).map(|_| ()); // Then assert_eq!(Ok(()), database_with_new_columns); @@ -1071,7 +1078,7 @@ mod tests { // When let columns = enum_iterator::all::<::Column>() .collect::>(); - let result = RocksDb::::open(tmp_dir.path(), columns, None); + let result = RocksDb::::open(tmp_dir.path(), columns, None, 512); // Then assert!(result.is_err()); @@ -1090,6 +1097,7 @@ mod tests { old_columns.clone(), None, false, + 512, ) .map(|_| ()); @@ -1111,6 +1119,7 @@ mod tests { secondary_temp.path(), old_columns.clone(), None, + 512, ) .map(|_| ()); @@ -1203,7 +1212,7 @@ mod tests { .skip(1) .collect::>(); let open_with_part_of_columns = - RocksDb::::open(tmp_dir.path(), part_of_columns, None); + RocksDb::::open(tmp_dir.path(), part_of_columns, None, 512); // Then let _ = open_with_part_of_columns diff --git a/tests/tests/health.rs b/tests/tests/health.rs index 5cdcf5b8a39..d0b241fcb5a 100644 --- a/tests/tests/health.rs +++ b/tests/tests/health.rs @@ -29,7 +29,8 @@ async fn can_restart_node() { // start node once { let database = - Database::open_rocksdb(tmp_dir.path(), None, Default::default()).unwrap(); + Database::open_rocksdb(tmp_dir.path(), None, Default::default(), 512) + .unwrap(); let first_startup = FuelService::from_database(database, Config::local_node()) .await .unwrap(); @@ -41,7 +42,8 @@ async fn can_restart_node() { { let database = - Database::open_rocksdb(tmp_dir.path(), None, Default::default()).unwrap(); + Database::open_rocksdb(tmp_dir.path(), None, Default::default(), 512) + .unwrap(); let _second_startup = FuelService::from_database(database, Config::local_node()) .await .unwrap(); @@ -56,7 +58,8 @@ async fn can_restart_node_with_transactions() { { // Given let database = - CombinedDatabase::open(tmp_dir.path(), capacity, Default::default()).unwrap(); + CombinedDatabase::open(tmp_dir.path(), capacity, Default::default(), 512) + .unwrap(); let service = FuelService::from_combined_database(database, Config::local_node()) .await .unwrap(); @@ -74,7 +77,8 @@ async fn can_restart_node_with_transactions() { { // When let database = - CombinedDatabase::open(tmp_dir.path(), capacity, Default::default()).unwrap(); + CombinedDatabase::open(tmp_dir.path(), capacity, Default::default(), 512) + .unwrap(); let service = FuelService::from_combined_database(database, Config::local_node()) .await .unwrap(); diff --git a/tests/tests/poa.rs b/tests/tests/poa.rs index 792064fbade..0f4f5a4fb97 100644 --- a/tests/tests/poa.rs +++ b/tests/tests/poa.rs @@ -137,8 +137,8 @@ async fn can_get_sealed_block_from_poa_produced_block_when_signing_with_kms() { // stop the node and just grab the database let db_path = driver.kill().await; - let db = - CombinedDatabase::open(db_path.path(), 1024 * 1024, Default::default()).unwrap(); + let db = CombinedDatabase::open(db_path.path(), 1024 * 1024, Default::default(), 512) + .unwrap(); let view = db.on_chain().latest_view().unwrap(); diff --git a/tests/tests/relayer.rs b/tests/tests/relayer.rs index 28afbf03d89..9ea80b1e242 100644 --- a/tests/tests/relayer.rs +++ b/tests/tests/relayer.rs @@ -331,7 +331,8 @@ async fn can_restart_node_with_relayer_data() { { // Given let database = - CombinedDatabase::open(tmp_dir.path(), capacity, Default::default()).unwrap(); + CombinedDatabase::open(tmp_dir.path(), capacity, Default::default(), 512) + .unwrap(); let service = FuelService::from_combined_database(database, config.clone()) .await @@ -350,7 +351,8 @@ async fn can_restart_node_with_relayer_data() { { // When let database = - CombinedDatabase::open(tmp_dir.path(), capacity, Default::default()).unwrap(); + CombinedDatabase::open(tmp_dir.path(), capacity, Default::default(), 512) + .unwrap(); let service = FuelService::from_combined_database(database, config) .await .unwrap();