From 2ec0ac62a43bd6c26d553dafa559d129917b51c5 Mon Sep 17 00:00:00 2001 From: teor Date: Mon, 21 Feb 2022 10:20:29 +1000 Subject: [PATCH] fix(state): use the new `increase_nofile_limit` function from rlimit 0.7.0 (#3539) Also: - upgrades to rlimit 0.7.0 - updates types to match the breaking changes in rlimit - deletes a manual implementation that was similar to `increase_nofile_limit`, but not as good on macOS and some BSDs Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- Cargo.lock | 8 +- zebra-state/Cargo.toml | 2 +- zebra-state/src/config.rs | 196 +++++++++++++++----------------------- 3 files changed, 80 insertions(+), 126 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c3a7f855ea1..78f8fe92858 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2058,9 +2058,9 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" [[package]] name = "libc" -version = "0.2.113" +version = "0.2.117" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eef78b64d87775463c549fbd80e19249ef436ea3bf1de2a1eb7e717ec7fab1e9" +checksum = "e74d72e0f9b65b5b4ca49a346af3976df0f9c61d550727f349ecd559f251a26c" [[package]] name = "libgit2-sys" @@ -3399,9 +3399,9 @@ dependencies = [ [[package]] name = "rlimit" -version = "0.5.4" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81a9ed03edbed449d6897c2092c71ab5f7b5fb80f6f0b1a3ed6d40a6f9fc0720" +checksum = "347703a5ae47adf1e693144157be231dde38c72bd485925cae7407ad3e52480b" dependencies = [ "libc", ] diff --git a/zebra-state/Cargo.toml b/zebra-state/Cargo.toml index 1ffd70d9912..5ed401b8e23 100644 --- a/zebra-state/Cargo.toml +++ b/zebra-state/Cargo.toml @@ -26,7 +26,7 @@ multiset = { git = "https://github.com/jmitchell/multiset", rev = "91ef8550b518f proptest = { version = "0.10.1", optional = true } proptest-derive = { version = "0.3.0", optional = true } regex = "1" -rlimit = "0.5.4" +rlimit = "0.7.0" rocksdb = "0.17.0" serde = { version = "1", features = ["serde_derive"] } tempfile = "3.3.0" diff --git a/zebra-state/src/config.rs b/zebra-state/src/config.rs index 9955d579b77..ad9aaea9fbb 100644 --- a/zebra-state/src/config.rs +++ b/zebra-state/src/config.rs @@ -1,5 +1,9 @@ -use serde::{Deserialize, Serialize}; use std::{convert::TryInto, path::PathBuf}; + +use rlimit::increase_nofile_limit; +use serde::{Deserialize, Serialize}; +use tracing::{info, warn}; + use zebra_chain::parameters::Network; /// Configuration for the state service. @@ -54,7 +58,7 @@ fn gen_temp_path(prefix: &str) -> PathBuf { impl Config { /// The ideal open file limit for Zebra - const IDEAL_OPEN_FILE_LIMIT: usize = 1024; + const IDEAL_OPEN_FILE_LIMIT: u64 = 1024; /// The minimum number of open files for Zebra to operate normally. Also used /// as the default open file limit, when the OS doesn't tell us how many @@ -65,13 +69,13 @@ impl Config { /// On Windows, the default limit is 512 high-level I/O files, and 8192 /// low-level I/O files: /// https://docs.microsoft.com/en-us/cpp/c-runtime-library/reference/setmaxstdio?view=msvc-160#remarks - const MIN_OPEN_FILE_LIMIT: usize = 512; + const MIN_OPEN_FILE_LIMIT: u64 = 512; /// The number of files used internally by Zebra. /// /// Zebra uses file descriptors for OS libraries (10+), polling APIs (10+), /// stdio (3), and other OS facilities (2+). - const RESERVED_FILE_COUNT: usize = 48; + const RESERVED_FILE_COUNT: u64 = 48; /// Returns the path and database options for the finalized state database pub(crate) fn db_config(&self, network: Network) -> (PathBuf, rocksdb::Options) { @@ -100,12 +104,13 @@ impl Config { let open_file_limit = Config::increase_open_file_limit(); let db_file_limit = Config::get_db_open_file_limit(open_file_limit); + // If the current limit is very large, set the DB limit using the ideal limit - let db_file_limit = db_file_limit.try_into().unwrap_or_else(|_| { - Config::get_db_open_file_limit(Config::IDEAL_OPEN_FILE_LIMIT) - .try_into() - .expect("ideal open file limit fits in a config int") - }); + let ideal_limit = Config::get_db_open_file_limit(Config::IDEAL_OPEN_FILE_LIMIT) + .try_into() + .expect("ideal open file limit fits in a c_int"); + let db_file_limit = db_file_limit.try_into().unwrap_or(ideal_limit); + opts.set_max_open_files(db_file_limit); (path, opts) @@ -120,7 +125,7 @@ impl Config { } /// Calculate the database's share of `open_file_limit` - fn get_db_open_file_limit(open_file_limit: usize) -> usize { + fn get_db_open_file_limit(open_file_limit: u64) -> u64 { // Give the DB half the files, and reserve half the files for peers (open_file_limit - Config::RESERVED_FILE_COUNT) / 2 } @@ -136,124 +141,73 @@ impl Config { /// # Panics /// /// If the open file limit can not be increased to `MIN_OPEN_FILE_LIMIT`. - #[cfg(unix)] - fn increase_open_file_limit() -> usize { - use rlimit::{getrlimit, Resource}; - - let (old_limit, hard_rlimit) = match getrlimit(Resource::NOFILE) { - Ok((soft_limit, hard_rlimit)) => (soft_limit.try_into().ok(), Some(hard_rlimit)), - Err(_) => (None, None), - }; - - // There's no API for reliably setting the soft limit to the lower of the - // hard limit and a desired limit, because: - // * the returned hard limit can be invalid or unrepresentable, and - // * some OS versions (macOS) return larger values than the actual hard - // limit. - // So we try setting the ideal limit, then the minimum limit. - if let Ok(actual_limit) = - Config::set_open_file_limit(Config::IDEAL_OPEN_FILE_LIMIT, hard_rlimit, old_limit) - { - return actual_limit; - } - - // Try the hard limit or the minimum, whichever is greater - let min_limit = - if let Some(hard_limit) = hard_rlimit.map(TryInto::try_into).and_then(Result::ok) { - std::cmp::max(Config::MIN_OPEN_FILE_LIMIT, hard_limit) - } else { - Config::MIN_OPEN_FILE_LIMIT - }; - if let Ok(actual_limit) = Config::set_open_file_limit(min_limit, hard_rlimit, old_limit) { - tracing::warn!(?actual_limit, - ?hard_rlimit, - ?old_limit, - min_limit = ?Config::MIN_OPEN_FILE_LIMIT, - ideal_limit = ?Config::IDEAL_OPEN_FILE_LIMIT, - "the maximum number of open files is below Zebra's ideal limit. Hint: Increase the open file limit to {} before launching Zebra", - Config::IDEAL_OPEN_FILE_LIMIT); - return actual_limit; - } - - panic!("open file limit too low: unable to set the number of open files to {}, the minimum number of files required by Zebra. Current soft limit is {:?} and hard limit is {:?}. Hint: Increase the open file limit to {} before launching Zebra", - Config::MIN_OPEN_FILE_LIMIT, - old_limit, - hard_rlimit, - Config::IDEAL_OPEN_FILE_LIMIT); - } - - /// Increase the soft open file limit for this process to `new_limit`, - /// and the hard open file limit to `hard_rlimit`. - /// - /// If `hard_rlimit` is `None`, also sets the hard limit to `new_limit`. - /// - /// If `old_limit` is already greater than or equal to `new_limit`, - /// returns `Ok(old_limit)`. - /// - /// Otherwise, tries to set the limit. Returns `Ok(new_limit)` if the - /// limit is set successfully. - #[cfg(unix)] - fn set_open_file_limit( - new_limit: usize, - hard_rlimit: Option, - old_limit: Option, - ) -> Result { - use rlimit::{setrlimit, Resource}; + fn increase_open_file_limit() -> u64 { + // `increase_nofile_limit` doesn't do anything on Windows in rlimit 0.7.0. + // + // On Windows, the default limit is: + // - 512 high-level stream I/O files (via the C standard functions), and + // - 8192 low-level I/O files (via the Unix C functions). + // https://docs.microsoft.com/en-us/cpp/c-runtime-library/reference/setmaxstdio?view=msvc-160#remarks + // + // If we need more high-level I/O files on Windows, + // use `setmaxstdio` and `getmaxstdio` from the `rlimit` crate: + // https://docs.rs/rlimit/latest/rlimit/#windows + // + // Then panic if `setmaxstdio` fails to set the minimum value, + // and `getmaxstdio` is below the minimum value. - if let Some(old_limit) = old_limit { - if old_limit >= new_limit { - tracing::info!(?new_limit, - current_limit = ?old_limit, - ?hard_rlimit, - "the open file limit is at or above the specified limit"); - return Ok(old_limit); + // We try setting the ideal limit, then the minimum limit. + let current_limit = match increase_nofile_limit(Config::IDEAL_OPEN_FILE_LIMIT) { + Ok(current_limit) => current_limit, + Err(limit_error) => { + info!( + ?limit_error, + min_limit = ?Config::MIN_OPEN_FILE_LIMIT, + ideal_limit = ?Config::IDEAL_OPEN_FILE_LIMIT, + "unable to increase the open file limit, \ + assuming Zebra can open a minimum number of files" + ); + + return Config::MIN_OPEN_FILE_LIMIT; } - } + }; - let new_rlimit = new_limit - .try_into() - .expect("new_limit is a valid rlimit value"); - if setrlimit( - Resource::NOFILE, - new_rlimit, - hard_rlimit.unwrap_or(new_rlimit), - ) - .is_ok() - { - tracing::info!( - ?new_limit, - ?old_limit, - ?hard_rlimit, - "set the open file limit for Zebra" + if current_limit < Config::MIN_OPEN_FILE_LIMIT { + panic!( + "open file limit too low: \ + unable to set the number of open files to {}, \ + the minimum number of files required by Zebra. \ + Current limit is {:?}. \ + Hint: Increase the open file limit to {} before launching Zebra", + Config::MIN_OPEN_FILE_LIMIT, + current_limit, + Config::IDEAL_OPEN_FILE_LIMIT + ); + } else if current_limit < Config::IDEAL_OPEN_FILE_LIMIT { + warn!( + ?current_limit, + min_limit = ?Config::MIN_OPEN_FILE_LIMIT, + ideal_limit = ?Config::IDEAL_OPEN_FILE_LIMIT, + "the maximum number of open files is below Zebra's ideal limit. \ + Hint: Increase the open file limit to {} before launching Zebra", + Config::IDEAL_OPEN_FILE_LIMIT + ); + } else if cfg!(windows) { + info!( + min_limit = ?Config::MIN_OPEN_FILE_LIMIT, + ideal_limit = ?Config::IDEAL_OPEN_FILE_LIMIT, + "assuming the open file limit is high enough for Zebra", ); - Ok(new_limit) } else { - Err(()) + info!( + ?current_limit, + min_limit = ?Config::MIN_OPEN_FILE_LIMIT, + ideal_limit = ?Config::IDEAL_OPEN_FILE_LIMIT, + "the open file limit is high enough for Zebra", + ); } - } - - /// Assumes that Zebra can open at least the minimum number of files, and - /// returns `MIN_OPEN_FILE_LIMIT`. - /// - /// Increasing the open file limit is not yet implemented on Windows. (And - /// other non-unix platforms). - #[cfg(not(unix))] - fn increase_open_file_limit() -> usize { - // On Windows, the default limit is 512 high-level I/O files, and 8192 - // low-level I/O files: - // https://docs.microsoft.com/en-us/cpp/c-runtime-library/reference/setmaxstdio?view=msvc-160#remarks - // - // If we need more high-level I/O files on Windows, we could implement - // support for `setmaxstdio` and `getmaxstdio` in the `rlimit` crate: - // https://github.com/Nugine/rlimit/issues/16#issuecomment-723393017 - // - // We should panic if `setmaxstdio` fails to set the minimum value, - // and `getmaxstdio` is below the minimum value. - tracing::info!(min_limit = ?Config::MIN_OPEN_FILE_LIMIT, - ideal_limit = ?Config::IDEAL_OPEN_FILE_LIMIT, - "assuming Zebra can open a minimum number of files"); - Config::MIN_OPEN_FILE_LIMIT + current_limit } }