From e229058294f908b8371c1737f8587e1d20c72a33 Mon Sep 17 00:00:00 2001 From: Arron Date: Sun, 5 Mar 2023 19:57:11 +0800 Subject: [PATCH 1/7] Fix rocksdb depend --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index d320ff7..241cb02 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,7 +15,7 @@ persy = { version = "1.3.1", features = ["background_ops"] } r2d2 = "0.8.10" rand = "0.8.5" rayon = "1.5.3" -rocksdb = { path = "./lib/rust-rocksdb", features = [ +rocksdb = { features = [ "zstd" ], default-features = false } rusty-leveldb = "1.0.2" From c1ad0a316dd6f5d0c3eff54704a6a6c13abf9220 Mon Sep 17 00:00:00 2001 From: Arron Date: Sun, 5 Mar 2023 19:59:45 +0800 Subject: [PATCH 2/7] Add lmdb test --- Cargo.lock | 55 +++++++++++++++++++++++++++++++---------------------- Cargo.toml | 1 + src/main.rs | 36 ++++++++++++++++++++++++++++++++++- 3 files changed, 68 insertions(+), 24 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1854a29..e8e22d6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -70,6 +70,7 @@ dependencies = [ "btree-slab", "dashmap", "duckdb", + "lmdb-rkv", "num_cpus", "parking_lot 0.12.1", "persy", @@ -82,25 +83,6 @@ dependencies = [ "yakv", ] -[[package]] -name = "bindgen" -version = "0.59.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2bd2a9a458e8f4304c52c43ebb0cfbd520289f8379a52e329a38afda99bf8eb8" -dependencies = [ - "bitflags", - "cexpr", - "clang-sys", - "lazy_static", - "lazycell", - "peeking_take_while", - "proc-macro2", - "quote", - "regex", - "rustc-hash", - "shlex", -] - [[package]] name = "bindgen" version = "0.60.1" @@ -613,7 +595,7 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d39db247d9ab409ce8e3fb87bd08b49edea055d391625c2270e2ac59d302e01d" dependencies = [ - "bindgen 0.60.1", + "bindgen", "cc", "pkg-config", "vcpkg", @@ -631,9 +613,11 @@ dependencies = [ [[package]] name = "librocksdb-sys" -version = "0.7.1+7.3.1" +version = "0.8.0+7.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "611804e4666a25136fcc5f8cf425ab4d26c7f74ea245ffe92ea23b85b6420b5d" dependencies = [ - "bindgen 0.59.2", + "bindgen", "bzip2-sys", "cc", "glob", @@ -659,6 +643,29 @@ version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7fb9b38af92608140b86b693604b9ffcc5824240a484d1ecd4795bacb2fe88f3" +[[package]] +name = "lmdb-rkv" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "447a296f7aca299cfbb50f4e4f3d49451549af655fb7215d7f8c0c3d64bad42b" +dependencies = [ + "bitflags", + "byteorder", + "libc", + "lmdb-rkv-sys", +] + +[[package]] +name = "lmdb-rkv-sys" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61b9ce6b3be08acefa3003c57b7565377432a89ec24476bbe72e11d101f852fe" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + [[package]] name = "lock_api" version = "0.4.7" @@ -1055,7 +1062,9 @@ checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b" [[package]] name = "rocksdb" -version = "0.18.0" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e9562ea1d70c0cc63a34a22d977753b50cca91cc6b6527750463bd5dd8697bc" dependencies = [ "libc", "librocksdb-sys", diff --git a/Cargo.toml b/Cargo.toml index 241cb02..b8b7dd9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,6 +9,7 @@ anyhow = "1.0.58" btree-slab = "0.4.0" dashmap = "5.3.4" duckdb = { version = "0.4.0", features = ["bundled", "r2d2"] } +lmdb-rkv = "0.14.0" num_cpus = "1.13.1" parking_lot = "0.12.1" persy = { version = "1.3.1", features = ["background_ops"] } diff --git a/src/main.rs b/src/main.rs index 86816de..c907b8d 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,6 +1,7 @@ #![feature(try_trait_v2)] use anyhow::Result; +use lmdb::Transaction; use parking_lot::Mutex; use rand::Rng; use rayon::prelude::*; @@ -9,7 +10,7 @@ use std::sync::Arc; use std::{ collections::{BTreeMap, HashMap}, env, - fs::{remove_dir_all, remove_file}, + fs::{create_dir, remove_dir_all, remove_file}, mem::MaybeUninit, time::Instant, }; @@ -338,6 +339,39 @@ pub fn run() -> Result<()> { Ok(()) }); } + + { + let filename = "lmdb"; + println!("\n# {filename}"); + let dbpath = dir.join(filename); + let _ = remove_dir_all(&dbpath); + let _ = create_dir(dbpath.clone()); + let env = lmdb::Environment::new().open(&dbpath)?; + let db = env.open_db(None)?; + + elapsed!(insert, |kv| -> Result<()> { + let [k, v] = kv; + let mut txn = env.begin_rw_txn()?; + txn.put( + db, + &k.to_be_bytes(), + &v.to_le_bytes(), + lmdb::WriteFlags::empty(), + )?; + txn.commit()?; + Ok(()) + }); + elapsed!(get, |kv| -> Result<()> { + let [k, _] = kv; + let mut txn = env.begin_rw_txn()?; + { + let i = txn.get(db, &k.to_be_bytes())?; + n_add!(i); + } + txn.commit()?; + Ok(()) + }); + } Ok(()) } From bb0a325f6404530f94a64c7702b65ff04110286e Mon Sep 17 00:00:00 2001 From: Arron Date: Sun, 5 Mar 2023 20:05:37 +0800 Subject: [PATCH 3/7] Fix mut warn --- src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main.rs b/src/main.rs index c907b8d..7b1a31d 100644 --- a/src/main.rs +++ b/src/main.rs @@ -363,7 +363,7 @@ pub fn run() -> Result<()> { }); elapsed!(get, |kv| -> Result<()> { let [k, _] = kv; - let mut txn = env.begin_rw_txn()?; + let txn = env.begin_rw_txn()?; { let i = txn.get(db, &k.to_be_bytes())?; n_add!(i); From 01f8d9c08a1adc8cbbb0d497f3133950311b4836 Mon Sep 17 00:00:00 2001 From: Arron Date: Sat, 18 Mar 2023 20:59:40 +0800 Subject: [PATCH 4/7] Add rocksdb transaction bench --- src/main.rs | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/src/main.rs b/src/main.rs index 7b1a31d..c85e349 100644 --- a/src/main.rs +++ b/src/main.rs @@ -238,6 +238,38 @@ pub fn run() -> Result<()> { Ok(()) }); } + + { + let filename = "rocksdb-transaction"; + println!("\n# {filename}"); + let dbpath = dir.join(filename); + let _ = remove_dir_all(&dbpath); + + let db: Arc = + Arc::new(rocksdb::TransactionDB::open_default(dbpath)?); + // let db = Arc::new(rocksdb::DB::open(&opt, dbpath)?); + + let mut write_opts = rocksdb::WriteOptions::new(); + write_opts.set_sync(true); + let txn_opts = rocksdb::TransactionOptions::new(); + + elapsed!(insert, |kv| -> Result<()> { + let txn = db.transaction_opt(&write_opts, &txn_opts); + let [k, v] = kv; + txn.put(&k.to_be_bytes(), &v.to_le_bytes())?; + txn.commit()?; + Ok(()) + }); + + elapsed!(get, |kv| -> Result<()> { + let [k, _] = kv; + if let Some(i) = db.get_pinned(&k.to_be_bytes())? { + n_add!(i) + } + Ok(()) + }); + } + { use persy::{Config, Persy, TransactionConfig, ValueMode}; let filename = "persy"; From d29eae5b10d2fbe6407b1cbd34f2f5c1bfe674b5 Mon Sep 17 00:00:00 2001 From: Arron Date: Sun, 19 Mar 2023 15:35:14 +0800 Subject: [PATCH 5/7] Fix lmdb read tnx --- src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main.rs b/src/main.rs index c85e349..7566df8 100644 --- a/src/main.rs +++ b/src/main.rs @@ -395,7 +395,7 @@ pub fn run() -> Result<()> { }); elapsed!(get, |kv| -> Result<()> { let [k, _] = kv; - let txn = env.begin_rw_txn()?; + let txn = env.begin_ro_txn()?; { let i = txn.get(db, &k.to_be_bytes())?; n_add!(i); From 27e594dcc750d7751066c1d73f62ff470b61d497 Mon Sep 17 00:00:00 2001 From: Arron Date: Sun, 19 Mar 2023 16:10:11 +0800 Subject: [PATCH 6/7] Add lmdb bulk insert --- src/main.rs | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/main.rs b/src/main.rs index 7566df8..e052714 100644 --- a/src/main.rs +++ b/src/main.rs @@ -393,6 +393,24 @@ pub fn run() -> Result<()> { txn.commit()?; Ok(()) }); + + let mut txn = env.begin_rw_txn()?; + elapsed!( + insert_bulk, + |kv| -> Result<()> { + let [k, v] = kv; + txn.put( + db, + &k.to_be_bytes(), + &v.to_le_bytes(), + lmdb::WriteFlags::empty(), + )?; + Ok(()) + }, + iter + ); + txn.commit()?; + elapsed!(get, |kv| -> Result<()> { let [k, _] = kv; let txn = env.begin_ro_txn()?; From 82de1f368238bb35e51a29a251bb8a2d88fc0dfb Mon Sep 17 00:00:00 2001 From: Arron Date: Sun, 19 Mar 2023 16:13:22 +0800 Subject: [PATCH 7/7] Add rocksdb bulk insert --- src/main.rs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/main.rs b/src/main.rs index e052714..cfbcc65 100644 --- a/src/main.rs +++ b/src/main.rs @@ -261,6 +261,18 @@ pub fn run() -> Result<()> { Ok(()) }); + let txn = db.transaction_opt(&write_opts, &txn_opts); + elapsed!( + insert_bulk, + |kv| -> Result<()> { + let [k, v] = kv; + txn.put(&k.to_be_bytes(), &v.to_le_bytes())?; + Ok(()) + }, + iter + ); + txn.commit()?; + elapsed!(get, |kv| -> Result<()> { let [k, _] = kv; if let Some(i) = db.get_pinned(&k.to_be_bytes())? {