Merge remote-tracking branch 'upstream/master' into double-write

.github/workflows/rust.yml

@@ -19,7 +19,7 @@ jobs:
         uses: actions-rs/toolchain@v1
         with:
           profile: minimal
-          toolchain: nightly-2023-01-01
+          toolchain: nightly-2023-07-01
           override: true
           components: rustfmt, clippy, rust-src
       - uses: Swatinem/rust-cache@v1
@@ -87,7 +87,7 @@ jobs:
         uses: actions-rs/toolchain@v1
         with:
           profile: minimal
-          toolchain: nightly-2023-01-01
+          toolchain: nightly-2023-07-01
           override: true
           components: llvm-tools-preview
       - uses: Swatinem/rust-cache@v1
@@ -97,8 +97,7 @@ jobs:
         run: if [[ ! -e ~/.cargo/bin/grcov ]]; then cargo install --locked grcov; fi
       - name: Run tests
         run: |
-          make test
-          env WITH_STABLE_TOOLCHAIN=auto make test
+          make test_matrix
         env:
           RUSTFLAGS: '-Zinstrument-coverage'
           LLVM_PROFILE_FILE: '%p-%m.profraw'

CHANGELOG.md

@@ -2,6 +2,15 @@
 
 ## [Unreleased]
 
+## [0.4.1] - 2023-09-14
+
+### Behavior Changes
+
+* When log recycling is enabled, Raft Engine will now retain 50% more log files to reduce the chance of running out.
+* Reduce the scope of keys reserved for internal use.
+
+## [0.4.0] - 2023-09-01
+
 ### Behavior Changes
 
 * `LogBatch::put` returns a `Result<()>` instead of `()`. It errs when the key is reserved for internal use.
@@ -18,6 +27,7 @@
 * Support preparing prefilled logs to enable log recycling when start-up. The amount of logs to prepare is controlled by `Config::prefill_limit`.
 * Add a new configuration `spill-dir` to allow automatic placement of logs into an auxiliary directory when `dir` is full.
 * Add a new method `Engine::fork` to duplicate an `Engine` to a new place, with a few disk file copies.
+* Support configuring lz4 acceleration factor with `compression-level`.
 
 ## [0.3.0] - 2022-09-14
 

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "raft-engine"
-version = "0.3.0"
+version = "0.4.1"
 authors = ["The TiKV Project Developers"]
 edition = "2018"
 rust-version = "1.66.0"
@@ -49,7 +49,7 @@ log = { version = "0.4", features = ["max_level_trace", "release_max_level_debug
 lz4-sys = "1.9"
 memmap2 = { version = "0.7", optional = true }
 nix = "0.26"
-num-derive = "0.3"
+num-derive = "0.4"
 num-traits = "0.2"
 parking_lot = "0.12"
 prometheus = { version = "0.13" }
@@ -64,33 +64,43 @@ strum = { version = "0.25.0", features = ["derive"] }
 thiserror = "1.0"
 
 [dev-dependencies]
-criterion = "0.5"
+criterion = "0.4"
 ctor = "0.2"
 env_logger = "0.10"
 kvproto = { git = "https://github.com/pingcap/kvproto.git", default-features = false, features = ["protobuf-codec"] }
 raft = { git = "https://github.com/tikv/raft-rs", branch = "master", default-features = false, features = ["protobuf-codec"] }
 rand = "0.8"
 rand_distr = "0.4"
-tempfile = "3.1"
-toml = "0.7"
+tempfile = "3.6"
+toml = "0.8"
 md-5 = "0.10.5"
 
 [features]
+default = ["internals", "scripting"]
 internals = []
-nightly = ["prometheus/nightly"]
-failpoints = ["fail/failpoints"]
-scripting = ["rhai"]
-swap = ["nightly", "memmap2"]
+nightly = [
+  "prometheus/nightly",
+]
+failpoints = [
+  "fail/failpoints",
+]
+scripting = [
+  "rhai",
+]
+swap = [
+  "nightly",
+  "memmap2",
+]
+std_fs = []
 
-# Shortcuts
-all_except_failpoints = ["internals", "scripting", "nightly", "swap"]
-all_stable = ["internals", "scripting", "failpoints"]
-all_stable_except_failpoints = ["internals", "scripting"]
+nightly_group = ["nightly", "swap"]
 
 [patch.crates-io]
 raft-proto = { git = "https://github.com/tikv/raft-rs", branch = "master" }
 protobuf = { git = "https://github.com/pingcap/rust-protobuf", branch = "v2.8" }
 protobuf-codegen = { git = "https://github.com/pingcap/rust-protobuf", branch = "v2.8" }
+# TODO: Use official grpc-rs once https://github.com/tikv/grpc-rs/pull/622 is merged.
+grpcio = { git = "https://github.com/tabokie/grpc-rs", branch = "v0.10.x-win" }
 
 [workspace]
 members = ["stress", "ctl"]

Makefile

@@ -4,7 +4,7 @@
 EXTRA_CARGO_ARGS ?=
 ## How to test stable toolchain.
 ## - auto: use current default toolchain, disable nightly features.
-## - force: always use stable toolchain, disable nightly features.
+## - force: explicitly use stable toolchain, disable nightly features.
 WITH_STABLE_TOOLCHAIN ?=
 
 WITH_NIGHTLY_FEATURES =
@@ -41,22 +41,35 @@ clean:
 format:
 	cargo ${TOOLCHAIN_ARGS} fmt --all
 
+CLIPPY_WHITELIST += -A clippy::bool_assert_comparison
 ## Run clippy.
 clippy:
 ifdef WITH_NIGHTLY_FEATURES
-	cargo ${TOOLCHAIN_ARGS} clippy --all --all-features --all-targets -- -D clippy::all
+	cargo ${TOOLCHAIN_ARGS} clippy --all --features nightly_group,failpoints --all-targets -- -D clippy::all ${CLIPPY_WHITELIST}
 else
-	cargo ${TOOLCHAIN_ARGS} clippy --all --features all_stable --all-targets -- -D clippy::all
+	cargo ${TOOLCHAIN_ARGS} clippy --all --features failpoints --all-targets -- -D clippy::all ${CLIPPY_WHITELIST}
 endif
 
 ## Run tests.
 test:
 ifdef WITH_NIGHTLY_FEATURES
-	cargo ${TOOLCHAIN_ARGS} test --all --features all_except_failpoints ${EXTRA_CARGO_ARGS} -- --nocapture
-	cargo ${TOOLCHAIN_ARGS} test --test failpoints --all-features ${EXTRA_CARGO_ARGS} -- --test-threads 1 --nocapture
+	cargo ${TOOLCHAIN_ARGS} test --all --features nightly_group ${EXTRA_CARGO_ARGS} -- --nocapture
+	cargo ${TOOLCHAIN_ARGS} test --test failpoints --features nightly_group,failpoints ${EXTRA_CARGO_ARGS} -- --test-threads 1 --nocapture
 else
-	cargo ${TOOLCHAIN_ARGS} test --all --features all_stable_except_failpoints ${EXTRA_CARGO_ARGS} -- --nocapture
-	cargo ${TOOLCHAIN_ARGS} test --test failpoints --features all_stable ${EXTRA_CARGO_ARGS} -- --test-threads 1 --nocapture
+	cargo ${TOOLCHAIN_ARGS} test --all ${EXTRA_CARGO_ARGS} -- --nocapture
+	cargo ${TOOLCHAIN_ARGS} test --test failpoints --features failpoints ${EXTRA_CARGO_ARGS} -- --test-threads 1 --nocapture
+endif
+
+## Run tests with various features for maximum code coverage.
+ifndef WITH_NIGHTLY_FEATURES
+test_matrix:
+	$(error Must run test matrix with nightly features. Please reset WITH_STABLE_TOOLCHAIN.)
+else
+test_matrix: test
+	cargo ${TOOLCHAIN_ARGS} test --all ${EXTRA_CARGO_ARGS} -- --nocapture
+	cargo ${TOOLCHAIN_ARGS} test --test failpoints --features failpoints ${EXTRA_CARGO_ARGS} -- --test-threads 1 --nocapture
+	cargo ${TOOLCHAIN_ARGS} test --all --features nightly_group,std_fs ${EXTRA_CARGO_ARGS} -- --nocapture
+	cargo ${TOOLCHAIN_ARGS} test --test failpoints --features nightly_group,std_fs,failpoints ${EXTRA_CARGO_ARGS} -- --test-threads 1 --nocapture
 endif
 
 ## Build raft-engine-ctl.

README.md

@@ -54,7 +54,7 @@ Put this in your Cargo.toml:
 
 ```rust
 [dependencies]
-raft-engine = "0.3.0"
+raft-engine = "0.4"
 ```
 
 Available Cargo features:

ctl/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "raft-engine-ctl"
-version = "0.3.0"
+version = "0.4.1"
 authors = ["The TiKV Project Developers"]
 edition = "2018"
 rust-version = "1.61.0"
@@ -11,4 +11,4 @@ license = "Apache-2.0"
 [dependencies]
 clap = { version = "3.1", features = ["derive", "cargo"] }
 env_logger = "0.10"
-raft-engine = { path = "..", version = "0.3.0", features = ["scripting", "internals"] }
+raft-engine = { path = "..", version = "0.4.1", features = ["scripting", "internals"] }

src/codec.rs

@@ -670,6 +670,7 @@ mod tests {
             decode_var_u64(&mut buf.as_slice()),
             ErrorKind::UnexpectedEof
         );
+        check_error!(decode_var_u64(&mut [].as_slice()), ErrorKind::UnexpectedEof);
 
         buf.push(0);
         assert_eq!(0, decode_var_u64(&mut buf.as_slice()).unwrap());

src/config.rs

@@ -60,6 +60,12 @@ pub struct Config {
     ///
     /// Default: "8KB"
     pub batch_compression_threshold: ReadableSize,
+    /// Acceleration factor for LZ4 compression. It can be fine tuned, with each
+    /// successive value providing roughly +~3% to speed. The value will be
+    /// capped within [1, 65537] by LZ4.
+    ///
+    /// Default: 1.
+    pub compression_level: Option<usize>,
     /// Deprecated.
     /// Incrementally sync log files after specified bytes have been written.
     /// Setting it to zero disables incremental sync.
@@ -112,7 +118,7 @@ pub struct Config {
     pub prefill_for_recycle: bool,
 
     /// Maximum capacity for preparing log files for recycling when start.
-    /// If not `None`, its size is equal to `purge-threshold`.
+    /// If `None`, its size is equal to `purge-threshold`*1.5.
     /// Only available for `prefill-for-recycle` is true.
     ///
     /// Default: None
@@ -130,6 +136,7 @@ impl Default for Config {
             recovery_read_block_size: ReadableSize::kb(16),
             recovery_threads: 4,
             batch_compression_threshold: ReadableSize::kb(8),
+            compression_level: None,
             bytes_per_sync: None,
             format_version: Version::V2,
             target_file_size: ReadableSize::mb(128),
@@ -215,10 +222,10 @@ impl Config {
         }
         if self.enable_log_recycle && self.purge_threshold.0 >= self.target_file_size.0 {
             // (1) At most u32::MAX so that the file number can be capped into an u32
-            // without colliding. (2) Add some more file as an additional buffer to
-            // avoid jitters.
+            // without colliding. (2) Increase the threshold by 50% to add some more file
+            // as an additional buffer to avoid jitters.
             std::cmp::min(
-                (self.purge_threshold.0 / self.target_file_size.0) as usize + 2,
+                (self.purge_threshold.0 / self.target_file_size.0) as usize * 3 / 2,
                 u32::MAX as usize,
             )
         } else {
@@ -237,7 +244,7 @@ impl Config {
         if self.prefill_for_recycle && prefill_limit >= self.target_file_size.0 {
             // Keep same with the maximum setting of `recycle_capacity`.
             std::cmp::min(
-                (prefill_limit / self.target_file_size.0) as usize + 2,
+                (prefill_limit / self.target_file_size.0) as usize * 3 / 2,
                 u32::MAX as usize,
             )
         } else {
@@ -280,6 +287,8 @@ mod tests {
         assert_eq!(load.target_file_size, ReadableSize::mb(1));
         assert_eq!(load.purge_threshold, ReadableSize::mb(3));
         assert_eq!(load.format_version, Version::V1);
+        assert_eq!(load.enable_log_recycle, false);
+        assert_eq!(load.prefill_for_recycle, false);
         load.sanitize().unwrap();
     }
 
@@ -293,14 +302,16 @@ mod tests {
         assert!(hard_load.sanitize().is_err());
 
         let soft_error = r#"
-            recovery-read-block-size = "1KB"
+            recovery-read-block-size = 1
             recovery-threads = 0
             target-file-size = "5000MB"
             format-version = 2
             enable-log-recycle = true
             prefill-for-recycle = true
         "#;
         let soft_load: Config = toml::from_str(soft_error).unwrap();
+        assert!(soft_load.recovery_read_block_size.0 < MIN_RECOVERY_READ_BLOCK_SIZE as u64);
+        assert!(soft_load.recovery_threads < MIN_RECOVERY_THREADS);
         let mut soft_sanitized = soft_load;
         soft_sanitized.sanitize().unwrap();
         assert!(soft_sanitized.recovery_read_block_size.0 >= MIN_RECOVERY_READ_BLOCK_SIZE as u64);
@@ -309,8 +320,6 @@ mod tests {
             soft_sanitized.purge_rewrite_threshold.unwrap(),
             soft_sanitized.target_file_size
         );
-        assert_eq!(soft_sanitized.format_version, Version::V2);
-        assert!(soft_sanitized.enable_log_recycle);
 
         let recycle_error = r#"
             enable-log-recycle = true