Add new flag for maximum file descriptors in rocksdb.

CHANGELOG.md

@@ -17,6 +17,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 ### Added
 - [2321](https://github.com/FuelLabs/fuel-core/pull/2321): New metrics for the txpool: "The size of transactions in the txpool" (`txpool_tx_size`), "The time spent by a transaction in the txpool in seconds" (`txpool_tx_time_in_txpool_seconds`), The number of transactions in the txpool (`txpool_number_of_transactions`), "The number of transactions pending verification before entering the txpool" (`txpool_number_of_transactions_pending_verification`), "The number of executable transactions in the txpool" (`txpool_number_of_executable_transactions`), "The time it took to select transactions for inclusion in a block in nanoseconds" (`txpool_select_transaction_time_nanoseconds`), The time it took to insert a transaction in the txpool in milliseconds (`txpool_insert_transaction_time_milliseconds`).
 - [2362](https://github.com/FuelLabs/fuel-core/pull/2362): Added a new request_response protocol version `/fuel/req_res/0.0.2`. In comparison with `/fuel/req/0.0.1`, which returns an empty response when a request cannot be fulfilled, this version returns more meaningful error codes. Nodes still support the version `0.0.1` of the protocol to guarantee backward compatibility with fuel-core nodes. Empty responses received from nodes using the old protocol `/fuel/req/0.0.1` are automatically converted into an error `ProtocolV1EmptyResponse` with error code 0, which is also the only error code implemented. More specific error codes will be added in the future.
+- [2386](https://github.com/FuelLabs/fuel-core/pull/2386): Add a flag to define the maximum number of file descriptors that RocksDB can use. By default it's half of the OS limit.
 
 ## [Version 0.40.0]
 

benches/benches/vm_set/blockchain.rs

@@ -75,6 +75,7 @@ impl BenchDb {
             tmp_dir.path(),
             None,
             Default::default(),
+            -1,
         )
         .unwrap();
         let db = Arc::new(db);

benches/src/db_lookup_times_utils/mod.rs

@@ -27,7 +27,7 @@ mod tests {
 
     fn setup_test_db() -> RocksDb<BenchDatabase> {
         let temp_dir = ShallowTempDir::new();
-        RocksDb::default_open(temp_dir.path(), None).unwrap()
+        RocksDb::default_open(temp_dir.path(), None, -1).unwrap()
     }
 
     #[test]

benches/src/db_lookup_times_utils/utils.rs

@@ -39,7 +39,7 @@ pub fn get_random_block_height(
 pub fn open_rocks_db<Description: DatabaseDescription>(
     path: &Path,
 ) -> Result<RocksDb<Description>> {
-    let db = RocksDb::default_open(path, None)?;
+    let db = RocksDb::default_open(path, None, -1)?;
     Ok(db)
 }
 

bin/fuel-core/Cargo.toml

@@ -35,6 +35,7 @@ hex = { workspace = true }
 humantime = "2.1"
 pyroscope = "0.5"
 pyroscope_pprofrs = "0.2"
+rlimit = "0.10.2"
 serde_json = { workspace = true }
 tikv-jemallocator = { workspace = true }
 tokio = { workspace = true, features = ["macros", "rt-multi-thread"] }

bin/fuel-core/src/cli/rollback.rs

@@ -5,6 +5,10 @@ use fuel_core::{
     combined_database::CombinedDatabase,
     state::historical_rocksdb::StateRewindPolicy,
 };
+use rlimit::{
+    getrlimit,
+    Resource,
+};
 use std::path::PathBuf;
 
 /// Rollbacks the state of the blockchain to a specific block height.
@@ -19,6 +23,17 @@ pub struct Command {
     )]
     pub database_path: PathBuf,
 
+    /// Defines a specific number of file descriptors that RocksDB can use.
+    ///
+    /// If defined as -1 no limit will be applied and will use the OS limits.
+    /// If not defined the system default divided by two is used.
+    #[clap(
+        long = "rocksdb-max-fds",
+        env,
+        default_value = getrlimit(Resource::NOFILE).map(|(_, hard)| i32::try_from(hard.saturating_div(2)).unwrap_or(i32::MAX)).unwrap().to_string()
+    )]
+    pub rocksdb_max_fds: i32,
+
     /// The path to the database.
     #[clap(long = "target-block-height")]
     pub target_block_height: u32,
@@ -32,6 +47,7 @@ pub async fn exec(command: Command) -> anyhow::Result<()> {
         path,
         64 * 1024 * 1024,
         StateRewindPolicy::RewindFullRange,
+        command.rocksdb_max_fds,
     )
     .map_err(Into::<anyhow::Error>::into)
     .context(format!("failed to open combined database at path {path:?}"))?;

bin/fuel-core/src/cli/run.rs

@@ -65,6 +65,10 @@ use pyroscope_pprofrs::{
     pprof_backend,
     PprofConfig,
 };
+use rlimit::{
+    getrlimit,
+    Resource,
+};
 use std::{
     env,
     net,
@@ -126,6 +130,19 @@ pub struct Command {
     )]
     pub database_type: DbType,
 
+    #[cfg(feature = "rocksdb")]
+
+    /// Defines a specific number of file descriptors that RocksDB can use.
+    ///
+    /// If defined as -1 no limit will be applied and will use the OS limits.
+    /// If not defined the system default divided by two is used.
+    #[clap(
+        long = "rocksdb-max-fds",
+        env,
+        default_value = getrlimit(Resource::NOFILE).map(|(_, hard)| i32::try_from(hard.saturating_div(2)).unwrap_or(i32::MAX)).unwrap().to_string()
+    )]
+    pub rocksdb_max_fds: i32,
+
     #[cfg(feature = "rocksdb")]
     /// Defines the state rewind policy for the database when RocksDB is enabled.
     ///
@@ -273,6 +290,8 @@ impl Command {
             database_path,
             database_type,
             #[cfg(feature = "rocksdb")]
+            rocksdb_max_fds,
+            #[cfg(feature = "rocksdb")]
             state_rewind_duration,
             db_prune,
             snapshot,
@@ -441,6 +460,8 @@ impl Command {
             max_database_cache_size,
             #[cfg(feature = "rocksdb")]
             state_rewind_policy,
+            #[cfg(feature = "rocksdb")]
+            max_fds: rocksdb_max_fds,
         };
 
         let block_importer = fuel_core::service::config::fuel_core_importer::Config::new(

bin/fuel-core/src/cli/snapshot.rs

@@ -10,6 +10,10 @@ use fuel_core::{
     types::fuel_types::ContractId,
 };
 use fuel_core_chain_config::ChainConfig;
+use rlimit::{
+    getrlimit,
+    Resource,
+};
 use std::path::{
     Path,
     PathBuf,
@@ -29,6 +33,17 @@ pub struct Command {
     )]
     pub database_path: PathBuf,
 
+    /// Defines a specific number of file descriptors that RocksDB can use.
+    ///
+    /// If defined as -1 no limit will be applied and will use the OS limits.
+    /// If not defined the system default divided by two is used.
+    #[clap(
+        long = "rocksdb-max-fds",
+        env,
+        default_value = getrlimit(Resource::NOFILE).map(|(_, hard)| i32::try_from(hard.saturating_div(2)).unwrap_or(i32::MAX)).unwrap().to_string()
+    )]
+    pub rocksdb_max_fds: i32,
+
     /// Where to save the snapshot
     #[arg(name = "OUTPUT_DIR", long = "output-directory")]
     pub output_dir: PathBuf,
@@ -125,6 +140,7 @@ pub async fn exec(command: Command) -> anyhow::Result<()> {
     let db = open_db(
         &command.database_path,
         Some(command.max_database_cache_size),
+        command.rocksdb_max_fds,
     )?;
     let output_dir = command.output_dir;
     let shutdown_listener = ShutdownListener::spawn();
@@ -180,11 +196,16 @@ fn load_chain_config_or_use_testnet(path: Option<&Path>) -> anyhow::Result<Chain
     }
 }
 
-fn open_db(path: &Path, capacity: Option<usize>) -> anyhow::Result<CombinedDatabase> {
+fn open_db(
+    path: &Path,
+    capacity: Option<usize>,
+    max_fds: i32,
+) -> anyhow::Result<CombinedDatabase> {
     CombinedDatabase::open(
         path,
         capacity.unwrap_or(1024 * 1024 * 1024),
         StateRewindPolicy::NoRewind,
+        max_fds,
     )
     .map_err(Into::<anyhow::Error>::into)
     .context(format!("failed to open combined database at path {path:?}",))
@@ -668,7 +689,8 @@ mod tests {
         let db_path = temp_dir.path().join("db");
         std::fs::create_dir(&db_path)?;
 
-        let mut db = DbPopulator::new(open_db(&db_path, None)?, StdRng::seed_from_u64(2));
+        let mut db =
+            DbPopulator::new(open_db(&db_path, None, -1)?, StdRng::seed_from_u64(2));
         let state = db.given_persisted_data();
         db.flush();
 
@@ -681,6 +703,7 @@ mod tests {
                 chain_config: None,
                 encoding_command: Some(EncodingCommand::Encoding { encoding }),
             },
+            rocksdb_max_fds: -1,
         });
 
         // Because the test_case macro doesn't work with async tests
@@ -720,7 +743,8 @@ mod tests {
 
         let snapshot_dir = temp_dir.path().join("snapshot");
         let db_path = temp_dir.path().join("db");
-        let mut db = DbPopulator::new(open_db(&db_path, None)?, StdRng::seed_from_u64(2));
+        let mut db =
+            DbPopulator::new(open_db(&db_path, None, -1)?, StdRng::seed_from_u64(2));
 
         let state = db.given_persisted_data();
         db.flush();
@@ -739,6 +763,7 @@ mod tests {
                     },
                 }),
             },
+            rocksdb_max_fds: -1,
         });
 
         tokio::runtime::Runtime::new()
@@ -763,7 +788,8 @@ mod tests {
         let snapshot_dir = temp_dir.path().join("snapshot");
 
         let db_path = temp_dir.path().join("db");
-        let mut db = DbPopulator::new(open_db(&db_path, None)?, StdRng::seed_from_u64(2));
+        let mut db =
+            DbPopulator::new(open_db(&db_path, None, -1)?, StdRng::seed_from_u64(2));
 
         let original_state = db.given_persisted_data().sorted().into_state_config();
 
@@ -789,6 +815,7 @@ mod tests {
             output_dir: snapshot_dir.clone(),
             max_database_cache_size: DEFAULT_DATABASE_CACHE_SIZE,
             subcommand: SubCommands::Contract { contract_id },
+            rocksdb_max_fds: -1,
         })
         .await?;
 

crates/fuel-core/src/combined_database.rs

@@ -39,6 +39,8 @@ pub struct CombinedDatabaseConfig {
     pub max_database_cache_size: usize,
     #[cfg(feature = "rocksdb")]
     pub state_rewind_policy: StateRewindPolicy,
+    #[cfg(feature = "rocksdb")]
+    pub max_fds: i32,
 }
 
 /// A database that combines the on-chain, off-chain and relayer databases into one entity.
@@ -79,13 +81,22 @@ impl CombinedDatabase {
         path: &std::path::Path,
         capacity: usize,
         state_rewind_policy: StateRewindPolicy,
+        max_fds: i32,
     ) -> crate::database::Result<Self> {
+        // Split the fds in equitable manner between the databases
+        let max_fds = match max_fds {
+            -1 => -1,
+            _ => max_fds.saturating_div(4),
+        };
         // TODO: Use different cache sizes for different databases
-        let on_chain = Database::open_rocksdb(path, capacity, state_rewind_policy)?;
-        let off_chain = Database::open_rocksdb(path, capacity, state_rewind_policy)?;
+        let on_chain =
+            Database::open_rocksdb(path, capacity, state_rewind_policy, max_fds)?;
+        let off_chain =
+            Database::open_rocksdb(path, capacity, state_rewind_policy, max_fds)?;
         let relayer =
-            Database::open_rocksdb(path, capacity, StateRewindPolicy::NoRewind)?;
-        let gas_price = Database::open_rocksdb(path, capacity, state_rewind_policy)?;
+            Database::open_rocksdb(path, capacity, StateRewindPolicy::NoRewind, max_fds)?;
+        let gas_price =
+            Database::open_rocksdb(path, capacity, state_rewind_policy, max_fds)?;
         Ok(Self {
             on_chain,
             off_chain,
@@ -115,6 +126,7 @@ impl CombinedDatabase {
                         &config.database_path,
                         config.max_database_cache_size,
                         config.state_rewind_policy,
+                        config.max_fds,
                     )?
                 }
             }

crates/fuel-core/src/database.rs

@@ -199,12 +199,14 @@ where
         path: &Path,
         capacity: impl Into<Option<usize>>,
         state_rewind_policy: StateRewindPolicy,
+        max_fds: i32,
     ) -> Result<Self> {
         use anyhow::Context;
         let db = HistoricalRocksDB::<Description>::default_open(
             path,
             capacity.into(),
             state_rewind_policy,
+            max_fds,
         )
         .map_err(Into::<anyhow::Error>::into)
         .with_context(|| {
@@ -1076,6 +1078,7 @@ mod tests {
             temp_dir.path(),
             1024 * 1024 * 1024,
             Default::default(),
+            -1,
         )
         .unwrap();
         // rocks db fails

crates/fuel-core/src/service/config.rs

@@ -126,6 +126,8 @@ impl Config {
             #[cfg(feature = "rocksdb")]
             state_rewind_policy:
                 crate::state::historical_rocksdb::StateRewindPolicy::RewindFullRange,
+            #[cfg(feature = "rocksdb")]
+            max_fds: -1,
         };
         let starting_gas_price = 0;
         let gas_price_change_percent = 0;

crates/fuel-core/src/state/historical_rocksdb.rs

@@ -101,8 +101,10 @@ where
         path: P,
         capacity: Option<usize>,
         state_rewind_policy: StateRewindPolicy,
+        max_fds: i32,
     ) -> DatabaseResult<Self> {
-        let db = RocksDb::<Historical<Description>>::default_open(path, capacity)?;
+        let db =
+            RocksDb::<Historical<Description>>::default_open(path, capacity, max_fds)?;
         Ok(Self {
             state_rewind_policy,
             db,