Merge branch 'master' into zaidoon/0.22.6

Author: zaidoon1

.github/workflows/audit.yml

@@ -0,0 +1,29 @@
+name: Security Audit
+
+on:
+  push:
+    branches:
+      - master
+    paths:
+      - "**/Cargo.toml"
+  schedule:
+    - cron: "0 2 * * *" # run at 2 AM UTC
+
+permissions:
+  contents: read
+
+jobs:
+  security-audit:
+    permissions:
+      checks: write # for rustsec/audit-check to create check
+      contents: read # for actions/checkout to fetch code
+      issues: write # for rustsec/audit-check to create issues
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Audit Check
+        # https://github.com/rustsec/audit-check/issues/2
+        uses: rustsec/audit-check@master
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/rust.yml

@@ -1,79 +1,60 @@
 name: RocksDB CI
 
 on: [push, pull_request]
-env:
-  RUST_VERSION: 1.75.0
 
 jobs:
   style:
     name: Rustfmt
     runs-on: ubuntu-latest
     steps:
       - name: Checkout sources
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
       - name: Install rust
-        uses: actions-rs/toolchain@v1
+        uses: dtolnay/rust-toolchain@stable
         with:
-          toolchain: ${{ env.RUST_VERSION }}
           components: rustfmt
-          profile: minimal
-          override: true
 
       - name: Run rustfmt
-        uses: actions-rs/cargo@v1
-        with:
-          command: fmt
-          args: --all -- --check
+        run: cargo fmt --all -- --check
 
   doc-check:
     name: Rustdoc-check
     runs-on: ubuntu-latest
     steps:
       - name: Checkout sources
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
       - name: Install rust
-        uses: actions-rs/toolchain@v1
+        uses: dtolnay/rust-toolchain@stable
         with:
-          toolchain: ${{ env.RUST_VERSION }}
           components: rust-docs
-          profile: minimal
-          override: true
 
       - name: Run cargo rustdoc
-        uses: actions-rs/cargo@v1
-        with:
-          command: rustdoc
-          args: -- -D warnings
+        run: cargo rustdoc -- -D warnings
 
   clippy:
     name: Clippy
     runs-on: ubuntu-latest
     steps:
       - name: Checkout sources
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
       - name: Install rust
-        uses: actions-rs/toolchain@v1
+        uses: dtolnay/rust-toolchain@stable
         with:
-          toolchain: ${{ env.RUST_VERSION }}
           components: clippy
-          profile: minimal
-          override: true
 
       - name: Run clippy
-        uses: actions-rs/clippy-check@v1
-        with:
-          token: ${{ secrets.GITHUB_TOKEN }}
-          args: --all-targets -- -D warnings
+        run: cargo clippy --all --all-targets -- -Dwarnings
 
   audit:
     name: Security audit
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions-rs/audit-check@v1
+      - uses: actions/checkout@v4
+      # https://github.com/rustsec/audit-check/issues/2
+      - uses: rustsec/audit-check@master
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
 
@@ -92,15 +73,10 @@ jobs:
             os: windows-latest
     steps:
       - name: Checkout sources
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
       - name: Install rust
-        uses: actions-rs/toolchain@v1
-        with:
-          toolchain: ${{ env.RUST_VERSION }}
-          target: ${{ matrix.target }}
-          profile: minimal
-          override: true
+        uses: dtolnay/rust-toolchain@stable
 
       - name: Remove msys64 # Workaround to resolve link error with C:\msys64\mingw64\bin\libclang.dll
         if: runner.os == 'Windows'
@@ -109,10 +85,6 @@ jobs:
       - name: Install dependencies
         if: runner.os == 'Windows'
         run: choco install llvm -y
-      
-      - name: Install dependencies (Linux)
-        if: runner.os == 'Linux'
-        run: sudo apt-get install libjemalloc-dev
 
       - name: Run rocksdb tests
         run: |

CHANGELOG.md

@@ -1,6 +1,8 @@
 # Changelog
 
 ## [Unreleased]
+* Document that `default` column family doesn't inherit open options of db (
+  0xdeafbeef)
 
 ## 0.22.4 (2024-02-20)
 

Cargo.toml

@@ -29,13 +29,14 @@ zstd = ["rust-librocksdb-sys/zstd"]
 zlib = ["rust-librocksdb-sys/zlib"]
 bzip2 = ["rust-librocksdb-sys/bzip2"]
 rtti = ["rust-librocksdb-sys/rtti"]
+mt_static = ["rust-librocksdb-sys/mt_static"]
 multi-threaded-cf = []
 serde1 = ["serde"]
 malloc-usable-size = ["rust-librocksdb-sys/malloc-usable-size"]
 
 [dependencies]
 libc = "0.2"
-rust-librocksdb-sys = { path = "librocksdb-sys", version = "0.18.0" }
+rust-librocksdb-sys = { path = "librocksdb-sys", version = "0.22.0" }
 serde = { version = "1", features = ["derive"], optional = true }
 
 [dev-dependencies]

README.md

@@ -37,7 +37,9 @@ compression submodules:
 git submodule update --init --recursive
 ```
 
-## Compression Support
+## Features
+
+### Compression Support
 
 By default, support for [Snappy](https://github.com/google/snappy),
 [LZ4](https://github.com/lz4/lz4), [Zstd](https://github.com/facebook/zstd),
@@ -53,11 +55,45 @@ default-features = false
 features = ["lz4"]
 ```
 
-## Multithreaded ColumnFamily alternation
+### Multithreaded ColumnFamily alternation
 
 RocksDB allows column families to be created and dropped
 from multiple threads concurrently, but this crate doesn't allow it by default
 for compatibility. If you need to modify column families concurrently, enable
 the crate feature `multi-threaded-cf`, which makes this binding's
 data structures use `RwLock` by default. Alternatively, you can directly create
 `DBWithThreadMode<MultiThreaded>` without enabling the crate feature.
+
+### Switch between /MT or /MD run time library (Only for Windows)
+
+The feature `mt_static` will request the library to be built with [/MT](https://learn.microsoft.com/en-us/cpp/build/reference/md-mt-ld-use-run-time-library?view=msvc-170)
+flag, which results in library using the static version of the run-time library.
+*This can be useful in case there's a conflict in the dependecy tree between different
+run-time versions.*
+
+### Jemalloc
+
+The feature `jemalloc` will enable the
+`unprefixed_malloc_on_supported_platforms` feature of `tikv-jemalloc-sys`,
+hooking the actual malloc and free, so jemalloc is used to allocate memory. On
+Supported platforms such as Linux, Rocksdb will also be properly informed that
+Jemalloc is enabled so that it can apply internal optimizations gated behind
+Jemalloc being enabled. On [unsupported
+platforms](https://github.com/zaidoon1/rust-rocksdb/blob/master/librocksdb-sys/build.rs#L4-L7),
+Rocksdb won't be properly
+informed that Jemalloc is being used so some internal optimizations are skipped
+BUT you will still get the benefits of Jemalloc memory allocation. Note that by
+default, Rust uses libc malloc on Linux which is known to have more memory
+fragmentation than Jemalloc especially with Rocksdb. See [github
+issue](https://github.com/facebook/rocksdb/issues/12364) for more information.
+In general, I highly suggest enabling Jemalloc unless there is a specific reason
+not to (your system doesn't support it, etc.)
+
+### Malloc Usable Size
+
+The feature `malloc-usable-size` will inform Rocksdb that malloc_usable_size is
+supported by the platform and is necessary if you want to use the
+`optimize_filters_for_memory` rocksdb feature as this feature is gated behind
+malloc_usable_size being available. See
+[rocksdb](https://github.com/facebook/rocksdb/blob/v9.0.0/include/rocksdb/table.h#L401-L434)
+for more information on the feature.

librocksdb-sys/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "rust-librocksdb-sys"
-version = "0.18.0+8.11.3"
+version = "0.22.0+9.2.1"
 edition = "2021"
 rust-version = "1.75.0"
 authors = [
@@ -29,8 +29,9 @@ exclude = [
 
 [features]
 default = ["static"]
-jemalloc = ["tikv-jemalloc-sys", "pkg-config"]
+jemalloc = ["tikv-jemalloc-sys"]
 static = ["libz-sys?/static", "bzip2-sys?/static"]
+mt_static = []
 io-uring = ["pkg-config"]
 snappy = []
 lz4 = ["lz4-sys"]
@@ -51,7 +52,7 @@ libz-sys = { version = "1.1", default-features = false, optional = true }
 bzip2-sys = { version = "0.1", default-features = false, optional = true }
 
 [dev-dependencies]
-const-cstr = "0.3"
+const-str = "0.5"
 uuid = { version = "1", features = ["v4"] }
 
 [build-dependencies]

librocksdb-sys/build.rs

@@ -1,6 +1,11 @@
 use std::path::Path;
 use std::{env, fs, path::PathBuf, process::Command};
 
+// On these platforms jemalloc-sys will use a prefixed jemalloc which cannot be linked together
+// with RocksDB.
+// See https://github.com/tikv/jemallocator/blob/tikv-jemalloc-sys-0.5.3/jemalloc-sys/src/env.rs#L25
+const NO_JEMALLOC_TARGETS: &[&str] = &["android", "dragonfly", "musl", "darwin"];
+
 fn link(name: &str, bundled: bool) {
     use std::env::var;
     let target = var("TARGET").unwrap();
@@ -218,18 +223,11 @@ fn build_rocksdb() {
 
     config.define("ROCKSDB_SUPPORT_THREAD_LOCAL", None);
 
-    #[cfg(feature = "jemalloc")]
-    {
-        if target.contains("linux") {
-            pkg_config::probe_library("jemalloc")
-                .expect("The jemalloc feature was requested but jemalloc is not installed");
-            config.define("ROCKSDB_JEMALLOC", Some("1"));
-            config.define("JEMALLOC_NO_DEMANGLE", Some("1"));
-        } else {
-            // this doesn't actually enable jemalloc, but mac/windows needs more
-            // work to make it work (statically link tikv-jemalloc). For now,
-            // I'll keep this as is.
-            config.define("WITH_JEMALLOC", "ON");
+    if cfg!(feature = "jemalloc") && NO_JEMALLOC_TARGETS.iter().all(|i| !target.contains(i)) {
+        config.define("ROCKSDB_JEMALLOC", Some("1"));
+        config.define("JEMALLOC_NO_DEMANGLE", Some("1"));
+        if let Some(jemalloc_root) = env::var_os("DEP_JEMALLOC_ROOT") {
+            config.include(Path::new(&jemalloc_root).join("include"));
         }
     }
 
@@ -246,6 +244,9 @@ fn build_rocksdb() {
     }
 
     if target.contains("msvc") {
+        if cfg!(feature = "mt_static") {
+            config.static_crt(true);
+        }
         config.flag("-EHsc");
         config.flag("-std:c++17");
     } else {
@@ -261,7 +262,10 @@ fn build_rocksdb() {
         config.flag("-Wno-strict-aliasing");
         config.flag("-Wno-invalid-offsetof");
     }
-
+    if target.contains("riscv64gc") {
+        // link libatomic required to build for riscv64gc
+        println!("cargo:rustc-link-lib=atomic");
+    }
     for file in lib_sources {
         config.file(format!("rocksdb/{file}"));
     }
@@ -285,6 +289,9 @@ fn build_snappy() {
 
     if target.contains("msvc") {
         config.flag("-EHsc");
+        if cfg!(feature = "mt_static") {
+            config.static_crt(true);
+        }
     } else {
         // Snappy requires C++11.
         // See: https://github.com/google/snappy/blob/master/CMakeLists.txt#L32-L38

librocksdb-sys/build_version.cc

@@ -8,17 +8,17 @@
 
 // The build script may replace these values with real values based
 // on whether or not GIT is available and the platform settings
-static const std::string rocksdb_build_git_sha  = "c2467b141e840fdba5b3a1810763043e56449fb9";
-static const std::string rocksdb_build_git_tag = "rocksdb_build_git_tag:v8.11.3";
+static const std::string rocksdb_build_git_sha  = "08f93221f50700f19f11555fb46abfe708a716d1";
+static const std::string rocksdb_build_git_tag = "rocksdb_build_git_tag:v9.2.1";
 #define HAS_GIT_CHANGES 0
 #if HAS_GIT_CHANGES == 0
 // If HAS_GIT_CHANGES is 0, the GIT date is used.
 // Use the time the branch/tag was last modified
-static const std::string rocksdb_build_date = "rocksdb_build_date:2024-02-27 16:24:11";
+static const std::string rocksdb_build_date = "rocksdb_build_date:2024-05-08 16:03:15";
 #else
 // If HAS_GIT_CHANGES is > 0, the branch/tag has modifications.
 // Use the time the build was created.
-static const std::string rocksdb_build_date = "rocksdb_build_date:2024-02-27 16:24:11";
+static const std::string rocksdb_build_date = "rocksdb_build_date:2024-05-08 16:03:15";
 #endif
 
 std::unordered_map<std::string, ROCKSDB_NAMESPACE::RegistrarFunc> ROCKSDB_NAMESPACE::ObjectRegistry::builtins_ = {};

librocksdb-sys/rocksdb

@@ -12,6 +12,7 @@ cache/secondary_cache_adapter.cc
 cache/sharded_cache.cc
 cache/tiered_secondary_cache.cc
 db/arena_wrapped_db_iter.cc
+db/attribute_group_iterator_impl.cc
 db/blob/blob_contents.cc
 db/blob/blob_fetcher.cc
 db/blob/blob_file_addition.cc
@@ -28,6 +29,7 @@ db/blob/blob_source.cc
 db/blob/prefetch_buffer_collection.cc
 db/builder.cc
 db/c.cc
+db/coalescing_iterator.cc
 db/column_family.cc
 db/compaction/compaction.cc
 db/compaction/compaction_iterator.cc
@@ -49,6 +51,7 @@ db/db_impl/db_impl_compaction_flush.cc
 db/db_impl/db_impl_debug.cc
 db/db_impl/db_impl_experimental.cc
 db/db_impl/db_impl_files.cc
+db/db_impl/db_impl_follower.cc
 db/db_impl/db_impl_open.cc
 db/db_impl/db_impl_readonly.cc
 db/db_impl/db_impl_secondary.cc
@@ -105,6 +108,7 @@ env/env_chroot.cc
 env/env_encryption.cc
 env/env_posix.cc
 env/file_system.cc
+env/fs_on_demand.cc
 env/fs_posix.cc
 env/fs_remap.cc
 env/file_system_tracer.cc
@@ -311,6 +315,7 @@ utilities/transactions/write_prepared_txn_db.cc
 utilities/transactions/write_unprepared_txn.cc
 utilities/transactions/write_unprepared_txn_db.cc
 utilities/ttl/db_ttl_impl.cc
+utilities/types_util.cc
 utilities/wal_filter.cc
 utilities/write_batch_with_index/write_batch_with_index.cc
 utilities/write_batch_with_index/write_batch_with_index_internal.cc