Skip to content

Commit

Permalink
fix(state): use the new increase_nofile_limit function from rlimit …
Browse files Browse the repository at this point in the history 
…0.7.0 (#3539)

Also:
- upgrades to rlimit 0.7.0
- updates types to match the breaking changes in rlimit
- deletes a manual implementation that was similar to `increase_nofile_limit`,
  but not as good on macOS and some BSDs

Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
  • Loading branch information
@teor2345 @mergify
teor2345 and mergify[bot] authored Feb 21, 2022
1 parent ab03a37 commit 2ec0ac6
Show file tree
Hide file tree
Showing 3 changed files with 80 additions and 126 deletions.
8 changes: 4 additions & 4 deletions Cargo.lock
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion zebra-state/Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ multiset = { git = "https://github.com/jmitchell/multiset", rev = "91ef8550b518f
proptest = { version = "0.10.1", optional = true }
proptest-derive = { version = "0.3.0", optional = true }
regex = "1"
rlimit = "0.5.4"
rlimit = "0.7.0"
rocksdb = "0.17.0"
serde = { version = "1", features = ["serde_derive"] }
tempfile = "3.3.0"
Expand Down
196 changes: 75 additions & 121 deletions zebra-state/src/config.rs
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,9 @@
use serde::{Deserialize, Serialize};
use std::{convert::TryInto, path::PathBuf};

use rlimit::increase_nofile_limit;
use serde::{Deserialize, Serialize};
use tracing::{info, warn};

use zebra_chain::parameters::Network;

/// Configuration for the state service.
Expand Down Expand Up @@ -54,7 +58,7 @@ fn gen_temp_path(prefix: &str) -> PathBuf {

impl Config {
/// The ideal open file limit for Zebra
const IDEAL_OPEN_FILE_LIMIT: usize = 1024;
const IDEAL_OPEN_FILE_LIMIT: u64 = 1024;

/// The minimum number of open files for Zebra to operate normally. Also used
/// as the default open file limit, when the OS doesn't tell us how many
Expand All @@ -65,13 +69,13 @@ impl Config {
/// On Windows, the default limit is 512 high-level I/O files, and 8192
/// low-level I/O files:
/// https://docs.microsoft.com/en-us/cpp/c-runtime-library/reference/setmaxstdio?view=msvc-160#remarks
const MIN_OPEN_FILE_LIMIT: usize = 512;
const MIN_OPEN_FILE_LIMIT: u64 = 512;

/// The number of files used internally by Zebra.
///
/// Zebra uses file descriptors for OS libraries (10+), polling APIs (10+),
/// stdio (3), and other OS facilities (2+).
const RESERVED_FILE_COUNT: usize = 48;
const RESERVED_FILE_COUNT: u64 = 48;

/// Returns the path and database options for the finalized state database
pub(crate) fn db_config(&self, network: Network) -> (PathBuf, rocksdb::Options) {
Expand Down Expand Up @@ -100,12 +104,13 @@ impl Config {

let open_file_limit = Config::increase_open_file_limit();
let db_file_limit = Config::get_db_open_file_limit(open_file_limit);

// If the current limit is very large, set the DB limit using the ideal limit
let db_file_limit = db_file_limit.try_into().unwrap_or_else(|_| {
Config::get_db_open_file_limit(Config::IDEAL_OPEN_FILE_LIMIT)
.try_into()
.expect("ideal open file limit fits in a config int")
});
let ideal_limit = Config::get_db_open_file_limit(Config::IDEAL_OPEN_FILE_LIMIT)
.try_into()
.expect("ideal open file limit fits in a c_int");
let db_file_limit = db_file_limit.try_into().unwrap_or(ideal_limit);

opts.set_max_open_files(db_file_limit);

(path, opts)
Expand All @@ -120,7 +125,7 @@ impl Config {
}

/// Calculate the database's share of `open_file_limit`
fn get_db_open_file_limit(open_file_limit: usize) -> usize {
fn get_db_open_file_limit(open_file_limit: u64) -> u64 {
// Give the DB half the files, and reserve half the files for peers
(open_file_limit - Config::RESERVED_FILE_COUNT) / 2
}
Expand All @@ -136,124 +141,73 @@ impl Config {
/// # Panics
///
/// If the open file limit can not be increased to `MIN_OPEN_FILE_LIMIT`.
#[cfg(unix)]
fn increase_open_file_limit() -> usize {
use rlimit::{getrlimit, Resource};

let (old_limit, hard_rlimit) = match getrlimit(Resource::NOFILE) {
Ok((soft_limit, hard_rlimit)) => (soft_limit.try_into().ok(), Some(hard_rlimit)),
Err(_) => (None, None),
};

// There's no API for reliably setting the soft limit to the lower of the
// hard limit and a desired limit, because:
// * the returned hard limit can be invalid or unrepresentable, and
// * some OS versions (macOS) return larger values than the actual hard
// limit.
// So we try setting the ideal limit, then the minimum limit.
if let Ok(actual_limit) =
Config::set_open_file_limit(Config::IDEAL_OPEN_FILE_LIMIT, hard_rlimit, old_limit)
{
return actual_limit;
}

// Try the hard limit or the minimum, whichever is greater
let min_limit =
if let Some(hard_limit) = hard_rlimit.map(TryInto::try_into).and_then(Result::ok) {
std::cmp::max(Config::MIN_OPEN_FILE_LIMIT, hard_limit)
} else {
Config::MIN_OPEN_FILE_LIMIT
};
if let Ok(actual_limit) = Config::set_open_file_limit(min_limit, hard_rlimit, old_limit) {
tracing::warn!(?actual_limit,
?hard_rlimit,
?old_limit,
min_limit = ?Config::MIN_OPEN_FILE_LIMIT,
ideal_limit = ?Config::IDEAL_OPEN_FILE_LIMIT,
"the maximum number of open files is below Zebra's ideal limit. Hint: Increase the open file limit to {} before launching Zebra",
Config::IDEAL_OPEN_FILE_LIMIT);
return actual_limit;
}

panic!("open file limit too low: unable to set the number of open files to {}, the minimum number of files required by Zebra. Current soft limit is {:?} and hard limit is {:?}. Hint: Increase the open file limit to {} before launching Zebra",
Config::MIN_OPEN_FILE_LIMIT,
old_limit,
hard_rlimit,
Config::IDEAL_OPEN_FILE_LIMIT);
}

/// Increase the soft open file limit for this process to `new_limit`,
/// and the hard open file limit to `hard_rlimit`.
///
/// If `hard_rlimit` is `None`, also sets the hard limit to `new_limit`.
///
/// If `old_limit` is already greater than or equal to `new_limit`,
/// returns `Ok(old_limit)`.
///
/// Otherwise, tries to set the limit. Returns `Ok(new_limit)` if the
/// limit is set successfully.
#[cfg(unix)]
fn set_open_file_limit(
new_limit: usize,
hard_rlimit: Option<rlimit::Rlim>,
old_limit: Option<usize>,
) -> Result<usize, ()> {
use rlimit::{setrlimit, Resource};
fn increase_open_file_limit() -> u64 {
// `increase_nofile_limit` doesn't do anything on Windows in rlimit 0.7.0.
//
// On Windows, the default limit is:
// - 512 high-level stream I/O files (via the C standard functions), and
// - 8192 low-level I/O files (via the Unix C functions).
// https://docs.microsoft.com/en-us/cpp/c-runtime-library/reference/setmaxstdio?view=msvc-160#remarks
//
// If we need more high-level I/O files on Windows,
// use `setmaxstdio` and `getmaxstdio` from the `rlimit` crate:
// https://docs.rs/rlimit/latest/rlimit/#windows
//
// Then panic if `setmaxstdio` fails to set the minimum value,
// and `getmaxstdio` is below the minimum value.

if let Some(old_limit) = old_limit {
if old_limit >= new_limit {
tracing::info!(?new_limit,
current_limit = ?old_limit,
?hard_rlimit,
"the open file limit is at or above the specified limit");
return Ok(old_limit);
// We try setting the ideal limit, then the minimum limit.
let current_limit = match increase_nofile_limit(Config::IDEAL_OPEN_FILE_LIMIT) {
Ok(current_limit) => current_limit,
Err(limit_error) => {
info!(
?limit_error,
min_limit = ?Config::MIN_OPEN_FILE_LIMIT,
ideal_limit = ?Config::IDEAL_OPEN_FILE_LIMIT,
"unable to increase the open file limit, \
assuming Zebra can open a minimum number of files"
);

return Config::MIN_OPEN_FILE_LIMIT;
}
}
};

let new_rlimit = new_limit
.try_into()
.expect("new_limit is a valid rlimit value");
if setrlimit(
Resource::NOFILE,
new_rlimit,
hard_rlimit.unwrap_or(new_rlimit),
)
.is_ok()
{
tracing::info!(
?new_limit,
?old_limit,
?hard_rlimit,
"set the open file limit for Zebra"
if current_limit < Config::MIN_OPEN_FILE_LIMIT {
panic!(
"open file limit too low: \
unable to set the number of open files to {}, \
the minimum number of files required by Zebra. \
Current limit is {:?}. \
Hint: Increase the open file limit to {} before launching Zebra",
Config::MIN_OPEN_FILE_LIMIT,
current_limit,
Config::IDEAL_OPEN_FILE_LIMIT
);
} else if current_limit < Config::IDEAL_OPEN_FILE_LIMIT {
warn!(
?current_limit,
min_limit = ?Config::MIN_OPEN_FILE_LIMIT,
ideal_limit = ?Config::IDEAL_OPEN_FILE_LIMIT,
"the maximum number of open files is below Zebra's ideal limit. \
Hint: Increase the open file limit to {} before launching Zebra",
Config::IDEAL_OPEN_FILE_LIMIT
);
} else if cfg!(windows) {
info!(
min_limit = ?Config::MIN_OPEN_FILE_LIMIT,
ideal_limit = ?Config::IDEAL_OPEN_FILE_LIMIT,
"assuming the open file limit is high enough for Zebra",
);
Ok(new_limit)
} else {
Err(())
info!(
?current_limit,
min_limit = ?Config::MIN_OPEN_FILE_LIMIT,
ideal_limit = ?Config::IDEAL_OPEN_FILE_LIMIT,
"the open file limit is high enough for Zebra",
);
}
}

/// Assumes that Zebra can open at least the minimum number of files, and
/// returns `MIN_OPEN_FILE_LIMIT`.
///
/// Increasing the open file limit is not yet implemented on Windows. (And
/// other non-unix platforms).
#[cfg(not(unix))]
fn increase_open_file_limit() -> usize {
// On Windows, the default limit is 512 high-level I/O files, and 8192
// low-level I/O files:
// https://docs.microsoft.com/en-us/cpp/c-runtime-library/reference/setmaxstdio?view=msvc-160#remarks
//
// If we need more high-level I/O files on Windows, we could implement
// support for `setmaxstdio` and `getmaxstdio` in the `rlimit` crate:
// https://github.com/Nugine/rlimit/issues/16#issuecomment-723393017
//
// We should panic if `setmaxstdio` fails to set the minimum value,
// and `getmaxstdio` is below the minimum value.

tracing::info!(min_limit = ?Config::MIN_OPEN_FILE_LIMIT,
ideal_limit = ?Config::IDEAL_OPEN_FILE_LIMIT,
"assuming Zebra can open a minimum number of files");
Config::MIN_OPEN_FILE_LIMIT
current_limit
}
}

Expand Down

0 comments on commit 2ec0ac6

Please sign in to comment.