From 2be54b51a3410557e6651cbe6e0735b701cbe230 Mon Sep 17 00:00:00 2001 From: lihz Date: Fri, 16 Jun 2017 22:23:36 +0800 Subject: [PATCH 1/4] add cache_index_and_filter_blocks --- rocksdb/_rocksdb.pyx | 1 + rocksdb/table_factory.pxd | 1 + 2 files changed, 2 insertions(+) diff --git a/rocksdb/_rocksdb.pyx b/rocksdb/_rocksdb.pyx index da216d3a..b1195036 100644 --- a/rocksdb/_rocksdb.pyx +++ b/rocksdb/_rocksdb.pyx @@ -579,6 +579,7 @@ cdef class BlockBasedTableFactory(PyTableFactory): block_size=None, block_size_deviation=None, block_restart_interval=None, + cache_index_and_filter_blocks=True, whole_key_filtering=None): cdef table_factory.BlockBasedTableOptions table_options diff --git a/rocksdb/table_factory.pxd b/rocksdb/table_factory.pxd index 2359292a..95acc574 100644 --- a/rocksdb/table_factory.pxd +++ b/rocksdb/table_factory.pxd @@ -27,6 +27,7 @@ cdef extern from "rocksdb/table.h" namespace "rocksdb": int block_size_deviation int block_restart_interval cpp_bool whole_key_filtering + cpp_bool cache_index_and_filter_blocks shared_ptr[Cache] block_cache shared_ptr[Cache] block_cache_compressed shared_ptr[FilterPolicy] filter_policy From caca2729e693361fadbd8a91ee158e2f62d7de1a Mon Sep 17 00:00:00 2001 From: lihz Date: Mon, 19 Jun 2017 13:04:17 +0800 Subject: [PATCH 2/4] test cache_index_and_filter_blocks --- rocksdb/tests/test_options.py | 1 + 1 file changed, 1 insertion(+) diff --git a/rocksdb/tests/test_options.py b/rocksdb/tests/test_options.py index c585190f..9c20f86b 100644 --- a/rocksdb/tests/test_options.py +++ b/rocksdb/tests/test_options.py @@ -90,6 +90,7 @@ def test_block_options(self): rocksdb.BlockBasedTableFactory( block_size=4096, filter_policy=TestFilterPolicy(), + cache_index_and_filter_blocks=True, block_cache=rocksdb.LRUCache(100)) def test_unicode_path(self): From c5ee70be90179e19e41a1c834870e3a9a28ecc7f Mon Sep 17 00:00:00 2001 From: lihz Date: Fri, 5 Jan 2018 14:50:50 +0800 Subject: [PATCH 3/4] rocksdb for column family --- rocksdb/__init__.py | 2 +- rocksdb/_rocksdb.pyx | 154 +++++++++++++++++++++++++++++--------- rocksdb/db.pxd | 69 ++++++++++++++--- rocksdb/options.pxd | 107 ++++++++++++++++++++++++++ rocksdb/table_factory.pxd | 15 ++++ setup.py | 16 ++-- 6 files changed, 313 insertions(+), 50 deletions(-) diff --git a/rocksdb/__init__.py b/rocksdb/__init__.py index 0417ff0e..b949a4f5 100644 --- a/rocksdb/__init__.py +++ b/rocksdb/__init__.py @@ -1 +1 @@ -from ._rocksdb import * +from ._rocksdb import * \ No newline at end of file diff --git a/rocksdb/_rocksdb.pyx b/rocksdb/_rocksdb.pyx index da216d3a..13fafd02 100644 --- a/rocksdb/_rocksdb.pyx +++ b/rocksdb/_rocksdb.pyx @@ -2,6 +2,7 @@ import cython from libcpp.string cimport string from libcpp.deque cimport deque from libcpp.vector cimport vector +from libcpp.map cimport map as cpp_map from cpython cimport bool as py_bool from libcpp cimport bool as cpp_bool from libc.stdint cimport uint32_t @@ -38,7 +39,8 @@ from options cimport kCompactionStyleLevel from options cimport kCompactionStyleUniversal from options cimport kCompactionStyleFIFO from options cimport kCompactionStyleNone - +from options cimport ColumnFamilyOptions +from db cimport ColumnFamilyHandle, ColumnFamilyDescriptor from slice_ cimport Slice from status cimport Status @@ -56,6 +58,9 @@ ctypedef const filter_policy.FilterPolicy ConstFilterPolicy cdef extern from "cpp/utils.hpp" namespace "py_rocks": cdef const Slice* vector_data(vector[Slice]&) +cdef extern from "rocksdb/db.h" namespace "rocksdb": + cdef string kDefaultColumnFamilyName + # Prepare python for threaded usage. # Python callbacks (merge, comparator) # could be executed in a rocksdb background thread (eg. compaction). @@ -105,6 +110,13 @@ cdef Slice bytes_to_slice(ob) except *: cdef slice_to_bytes(Slice sl): return PyBytes_FromStringAndSize(sl.data(), sl.size()) +cdef ColumnFamilyDescriptor column_name_to_cfd(string column): + cdef ColumnFamilyDescriptor cdf + cdf.name = column + cdef ColumnFamilyOptions opt + cdf.options = opt + return cdf + ## only for filsystem paths cdef string path_to_string(object path) except *: if isinstance(path, bytes): @@ -579,6 +591,7 @@ cdef class BlockBasedTableFactory(PyTableFactory): block_size=None, block_size_deviation=None, block_restart_interval=None, + cache_index_and_filter_blocks=True, whole_key_filtering=None): cdef table_factory.BlockBasedTableOptions table_options @@ -1394,38 +1407,47 @@ cdef class WriteBatchIterator(object): self.pos += 1 return ret + @cython.no_gc_clear cdef class DB(object): cdef Options opts cdef db.DB* db + cdef vector[ColumnFamilyHandle*]* cfh + cdef cpp_map[string, ColumnFamilyHandle*]* cfh_map - def __cinit__(self, db_name, Options opts, read_only=False): + def __cinit__(self, db_name, Options opts, cfs, read_only=False): cdef Status st cdef string db_path self.db = NULL self.opts = None - + self.cfh = new vector[ColumnFamilyHandle*]() if opts.in_use: raise Exception("Options object is already used by another DB") - + cdef vector[ColumnFamilyDescriptor] cf + for c in cfs: + cf.push_back(column_name_to_cfd(c)) db_path = path_to_string(db_name) if read_only: with nogil: - st = db.DB_OpenForReadOnly( + st = db.DB_OpenForReadOnly_CF( deref(opts.opts), db_path, + cf, + self.cfh, cython.address(self.db), False) else: with nogil: - st = db.DB_Open( + st = db.DB_Open_CF( deref(opts.opts), db_path, + cf, + self.cfh, cython.address(self.db)) check_status(st) # Inject the loggers into the python callbacks - cdef shared_ptr[logger.Logger] info_log = self.db.GetOptions().info_log + cdef shared_ptr[logger.Logger] info_log = self.db.GetOptions(self.cfh[0][0]).info_log if opts.py_comparator is not None: opts.py_comparator.set_info_log(info_log) @@ -1437,16 +1459,50 @@ cdef class DB(object): self.opts = opts self.opts.in_use = True + self.cfh_map = new cpp_map[string, ColumnFamilyHandle*]() + for cfh1 in self.cfh[0]: + self.cfh_map[0][cfh1.GetName()] = cfh1 + + def close(self): + if not self.db == NULL: + with nogil: + for cf in self.cfh[0]: + self.db.DestroyColumnFamilyHandle(cf) + del self.cfh + del self.db + self.db = NULL + if self.opts is not None: + self.opts.in_use = False def __dealloc__(self): if not self.db == NULL: with nogil: + for cf in self.cfh[0]: + self.db.DestroyColumnFamilyHandle(cf) + del self.cfh del self.db if self.opts is not None: self.opts.in_use = False - def put(self, key, value, sync=False, disable_wal=False): + @staticmethod + cdef ColumnFamilyHandle* get_cfh(cpp_map[string, ColumnFamilyHandle*]* cfh_map, string column): + if cfh_map[0].count(column) <= 0: + return NULL + return cfh_map[0][column] + + def create_column_family(self, columns): + cdef ColumnFamilyOptions opt + cdef vector[ColumnFamilyHandle*] cfhs + cdef vector[string] cf + for c in columns: + cf.push_back(c) + with nogil: + self.db.CreateColumnFamilies(opt, cf, &cfhs) + for cfh in cfhs: + self.db.DestroyColumnFamilyHandle(cfh) + + def put(self, column, key, value, sync=False, disable_wal=False): cdef Status st cdef options.WriteOptions opts opts.sync = sync @@ -1454,23 +1510,28 @@ cdef class DB(object): cdef Slice c_key = bytes_to_slice(key) cdef Slice c_value = bytes_to_slice(value) - + cdef ColumnFamilyHandle* cfh = DB.get_cfh(self.cfh_map, column) + if cfh == NULL: + raise ValueError("column family {} doesn't exist".format(column)) with nogil: - st = self.db.Put(opts, c_key, c_value) + st = self.db.Put(opts, cfh, c_key, c_value) check_status(st) - def delete(self, key, sync=False, disable_wal=False): + def delete(self, column, key, sync=False, disable_wal=False): cdef Status st cdef options.WriteOptions opts opts.sync = sync opts.disableWAL = disable_wal cdef Slice c_key = bytes_to_slice(key) + cdef ColumnFamilyHandle* cfh = DB.get_cfh(self.cfh_map, column) + if cfh == NULL: + raise ValueError("column family {} doesn't exist".format(column)) with nogil: - st = self.db.Delete(opts, c_key) + st = self.db.Delete(opts, cfh, c_key) check_status(st) - def merge(self, key, value, sync=False, disable_wal=False): + def merge(self, column, key, value, sync=False, disable_wal=False): cdef Status st cdef options.WriteOptions opts opts.sync = sync @@ -1478,8 +1539,11 @@ cdef class DB(object): cdef Slice c_key = bytes_to_slice(key) cdef Slice c_value = bytes_to_slice(value) + cdef ColumnFamilyHandle* cfh = DB.get_cfh(self.cfh_map, column) + if cfh == NULL: + raise ValueError("column family {} doesn't exist".format(column)) with nogil: - st = self.db.Merge(opts, c_key, c_value) + st = self.db.Merge(opts, cfh, c_key, c_value) check_status(st) def write(self, WriteBatch batch, sync=False, disable_wal=False): @@ -1492,16 +1556,18 @@ cdef class DB(object): st = self.db.Write(opts, batch.batch) check_status(st) - def get(self, key, *args, **kwargs): + def get(self, column, key, *args, **kwargs): cdef string res cdef Status st cdef options.ReadOptions opts - + cdef ColumnFamilyHandle* cfh = DB.get_cfh(self.cfh_map, column) + if cfh == NULL: + raise ValueError("column family {} doesn't exist".format(column)) opts = self.build_read_opts(self.__parse_read_opts(*args, **kwargs)) cdef Slice c_key = bytes_to_slice(key) with nogil: - st = self.db.Get(opts, c_key, cython.address(res)) + st = self.db.Get(opts, cfh, c_key, cython.address(res)) if st.ok(): return string_to_bytes(res) @@ -1510,13 +1576,16 @@ cdef class DB(object): else: check_status(st) - def multi_get(self, keys, *args, **kwargs): + def multi_get(self, columns, keys, *args, **kwargs): cdef vector[string] values values.resize(len(keys)) cdef vector[Slice] c_keys for key in keys: c_keys.push_back(bytes_to_slice(key)) + cdef vector[ColumnFamilyHandle*] cfhs + for column in columns: + cfhs.push_back(DB.get_cfh(self.cfh_map, column)) cdef options.ReadOptions opts opts = self.build_read_opts(self.__parse_read_opts(*args, **kwargs)) @@ -1525,6 +1594,7 @@ cdef class DB(object): with nogil: res = self.db.MultiGet( opts, + cfhs, c_keys, cython.address(values)) @@ -1539,14 +1609,16 @@ cdef class DB(object): return ret_dict - def key_may_exist(self, key, fetch=False, *args, **kwargs): + def key_may_exist(self, column, key, fetch=False, *args, **kwargs): cdef string value cdef cpp_bool value_found cdef cpp_bool exists cdef options.ReadOptions opts cdef Slice c_key opts = self.build_read_opts(self.__parse_read_opts(*args, **kwargs)) - + cdef ColumnFamilyHandle* cfh = DB.get_cfh(self.cfh_map, column) + if cfh == NULL: + raise ValueError("column family {} doesn't exist".format(column)) c_key = bytes_to_slice(key) exists = False @@ -1555,6 +1627,7 @@ cdef class DB(object): with nogil: exists = self.db.KeyMayExist( opts, + cfh, c_key, cython.address(value), cython.address(value_found)) @@ -1570,56 +1643,65 @@ cdef class DB(object): with nogil: exists = self.db.KeyMayExist( opts, + cfh, c_key, cython.address(value)) return (exists, None) - def iterkeys(self, *args, **kwargs): + def iterkeys(self, column, *args, **kwargs): cdef options.ReadOptions opts cdef KeysIterator it - + cdef ColumnFamilyHandle* cfh = DB.get_cfh(self.cfh_map, column) + if cfh == NULL: + raise ValueError("column family {} doesn't exist".format(column)) opts = self.build_read_opts(self.__parse_read_opts(*args, **kwargs)) it = KeysIterator(self) with nogil: - it.ptr = self.db.NewIterator(opts) + it.ptr = self.db.NewIterator(opts, cfh) return it - def itervalues(self, *args, **kwargs): + def itervalues(self, column, *args, **kwargs): cdef options.ReadOptions opts cdef ValuesIterator it opts = self.build_read_opts(self.__parse_read_opts(*args, **kwargs)) - + cdef ColumnFamilyHandle* cfh = DB.get_cfh(self.cfh_map, column) + if cfh == NULL: + raise ValueError("column family {} doesn't exist".format(column)) it = ValuesIterator(self) with nogil: - it.ptr = self.db.NewIterator(opts) + it.ptr = self.db.NewIterator(opts, cfh) return it - def iteritems(self, *args, **kwargs): + def iteritems(self, column, *args, **kwargs): cdef options.ReadOptions opts cdef ItemsIterator it - + cdef ColumnFamilyHandle* cfh = DB.get_cfh(self.cfh_map, column) + if cfh == NULL: + raise ValueError("column family {} doesn't exist".format(column)) opts = self.build_read_opts(self.__parse_read_opts(*args, **kwargs)) it = ItemsIterator(self) with nogil: - it.ptr = self.db.NewIterator(opts) + it.ptr = self.db.NewIterator(opts, cfh) return it def snapshot(self): return Snapshot(self) - def get_property(self, prop): + def get_property(self, column, prop): cdef string value cdef Slice c_prop = bytes_to_slice(prop) cdef cpp_bool ret = False - + cdef ColumnFamilyHandle* cfh = DB.get_cfh(self.cfh_map, column) + if cfh == NULL: + raise ValueError("column family {} doesn't exist".format(column)) with nogil: - ret = self.db.GetProperty(c_prop, cython.address(value)) + ret = self.db.GetProperty(cfh, c_prop, cython.address(value)) if ret: return string_to_bytes(value) @@ -1647,9 +1729,11 @@ cdef class DB(object): return ret - def compact_range(self, begin=None, end=None, **py_options): + def compact_range(self, column, begin=None, end=None, **py_options): cdef options.CompactRangeOptions c_options - + cdef ColumnFamilyHandle* cfh = DB.get_cfh(self.cfh_map, column) + if cfh == NULL: + raise ValueError("column family {} doesn't exist".format(column)) c_options.change_level = py_options.get('change_level', False) c_options.target_level = py_options.get('target_level', -1) @@ -1681,7 +1765,7 @@ cdef class DB(object): end_val = bytes_to_slice(end) end_ptr = cython.address(end_val) - st = self.db.CompactRange(c_options, begin_ptr, end_ptr) + st = self.db.CompactRange(c_options, cfh, begin_ptr, end_ptr) check_status(st) @staticmethod diff --git a/rocksdb/db.pxd b/rocksdb/db.pxd index a83786ba..fa9d7b32 100644 --- a/rocksdb/db.pxd +++ b/rocksdb/db.pxd @@ -1,5 +1,6 @@ cimport options from libc.stdint cimport uint64_t +from libc.stdint cimport uint32_t from status cimport Status from libcpp cimport bool as cpp_bool from libcpp.string cimport string @@ -7,6 +8,8 @@ from libcpp.vector cimport vector from slice_ cimport Slice from snapshot cimport Snapshot from iterator cimport Iterator +from logger cimport Logger +from options cimport ColumnFamilyOptions cdef extern from "rocksdb/write_batch.h" namespace "rocksdb": cdef cppclass WriteBatch: @@ -36,7 +39,7 @@ cdef extern from "cpp/write_batch_iter_helper.hpp" namespace "py_rocks": cdef extern from "rocksdb/db.h" namespace "rocksdb": ctypedef uint64_t SequenceNumber - + cdef string kDefaultColumnFamilyName cdef struct LiveFileMetaData: string name int level @@ -49,18 +52,43 @@ cdef extern from "rocksdb/db.h" namespace "rocksdb": cdef cppclass Range: Range(const Slice&, const Slice&) + cdef struct ColumnFamilyDescriptor: + string name + ColumnFamilyOptions options + cdef cppclass ColumnFamilyHandle: + string GetName() nogil except+ + uint32_t GetID() nogil except+ + Status GetDescriptor( + ColumnFamilyDescriptor* desc) nogil except+ + + cdef cppclass DB: + Status CreateColumnFamily( + const ColumnFamilyOptions&, + const string&, + ColumnFamilyHandle**) nogil except+ + + Status CreateColumnFamilies( + const ColumnFamilyOptions&, + const vector[string]&, + vector[ColumnFamilyHandle*]*) nogil except+ + Status DestroyColumnFamilyHandle( + ColumnFamilyHandle*) nogil except+ + Status Put( const options.WriteOptions&, + ColumnFamilyHandle*, const Slice&, const Slice&) nogil except+ Status Delete( const options.WriteOptions&, + ColumnFamilyHandle*, const Slice&) nogil except+ Status Merge( const options.WriteOptions&, + ColumnFamilyHandle*, const Slice&, const Slice&) nogil except+ @@ -70,54 +98,62 @@ cdef extern from "rocksdb/db.h" namespace "rocksdb": Status Get( const options.ReadOptions&, + ColumnFamilyHandle*, const Slice&, string*) nogil except+ vector[Status] MultiGet( const options.ReadOptions&, + const vector[ColumnFamilyHandle*]&, const vector[Slice]&, vector[string]*) nogil except+ cpp_bool KeyMayExist( const options.ReadOptions&, + ColumnFamilyHandle*, Slice&, string*, cpp_bool*) nogil except+ cpp_bool KeyMayExist( const options.ReadOptions&, + ColumnFamilyHandle*, Slice&, string*) nogil except+ Iterator* NewIterator( - const options.ReadOptions&) nogil except+ + const options.ReadOptions&, + ColumnFamilyHandle*) nogil except+ const Snapshot* GetSnapshot() nogil except+ void ReleaseSnapshot(const Snapshot*) nogil except+ cpp_bool GetProperty( + ColumnFamilyHandle*, const Slice&, string*) nogil except+ void GetApproximateSizes( - const Range* + ColumnFamilyHandle*, + const Range*, int, uint64_t*) nogil except+ Status CompactRange( const options.CompactRangeOptions&, + ColumnFamilyHandle*, const Slice*, const Slice*) nogil except+ - int NumberLevels() nogil except+ - int MaxMemCompactionLevel() nogil except+ - int Level0StopWriteTrigger() nogil except+ + int NumberLevels(ColumnFamilyHandle*) nogil except+ + int MaxMemCompactionLevel(ColumnFamilyHandle*) nogil except+ + int Level0StopWriteTrigger(ColumnFamilyHandle*) nogil except+ const string& GetName() nogil except+ - const options.Options& GetOptions() nogil except+ - Status Flush(const options.FlushOptions&) nogil except+ + const options.Options& GetOptions(ColumnFamilyHandle*) nogil except+ + Status Flush(const options.FlushOptions&, ColumnFamilyHandle*) nogil except+ Status DisableFileDeletions() nogil except+ - Status EnableFileDeletions() nogil except+ + Status EnableFileDeletions(cpp_bool) nogil except+ # TODO: Status GetSortedWalFiles(VectorLogPtr& files) # TODO: SequenceNumber GetLatestSequenceNumber() @@ -140,4 +176,19 @@ cdef extern from "rocksdb/db.h" namespace "rocksdb": DB**, cpp_bool) nogil except+ + cdef Status DB_Open_CF "rocksdb::DB::Open"( + const options.Options&, + const string&, + const vector[ColumnFamilyDescriptor]&, + vector[ColumnFamilyHandle*]*, + DB**) nogil except+ + + cdef Status DB_OpenForReadOnly_CF "rocksdb::DB::OpenForReadOnly"( + const options.Options&, + const string&, + const vector[ColumnFamilyDescriptor]&, + vector[ColumnFamilyHandle*]*, + DB**, + cpp_bool) nogil except+ + cdef Status RepairDB(const string& dbname, const options.Options&) diff --git a/rocksdb/options.pxd b/rocksdb/options.pxd index e54bf698..7a4e5c64 100644 --- a/rocksdb/options.pxd +++ b/rocksdb/options.pxd @@ -4,6 +4,7 @@ from libcpp.vector cimport vector from libc.stdint cimport uint64_t from libc.stdint cimport uint32_t from std_memory cimport shared_ptr +from libcpp.memory cimport unique_ptr from comparator cimport Comparator from merge_operator cimport MergeOperator from logger cimport Logger @@ -11,10 +12,90 @@ from slice_ cimport Slice from snapshot cimport Snapshot from slice_transform cimport SliceTransform from table_factory cimport TableFactory +from table_factory cimport TablePropertiesCollectorFactory from memtablerep cimport MemTableRepFactory from universal_compaction cimport CompactionOptionsUniversal from cache cimport Cache + +cdef extern from "rocksdb/compaction_filter.h" namespace "rocksdb::CompactionFilter": + ctypedef enum ValueType: + kValue + kMergeOperand + ctypedef enum Decision: + kKeep + kRemove + kChangeValue + kRemoveAndSkipUntil + cdef struct Context: + cpp_bool is_full_compaction + cpp_bool is_manual_compaction + uint32_t column_family_id + + +cdef extern from "rocksdb/compaction_filter.h" namespace "rocksdb": + cdef cppclass CompactionFilter: + cpp_bool Filter( + int, + const Slice& key, + const Slice& existing_value, + string* new_value, + cpp_bool* value_changed) const + + cdef cppclass CompactionFilterFactory: + unique_ptr[CompactionFilter] CreateCompactionFilter( + const Context& context) nogil except+ + const char* Name() const + + +cdef extern from "rocksdb/advanced_options.h" namespace "rocksdb": + ctypedef vector[shared_ptr[TablePropertiesCollectorFactory]] TablePropertiesCollectorFactories + + ctypedef enum CompactionPri: + kByCompensatedSize + kOldestLargestSeqFirst + kOldestSmallestSeqFirst + kMinOverlappingRatio + + cdef struct CompactionOptionsFIFO: + uint64_t max_table_files_size + + cdef cppclass AdvancedColumnFamilyOptions: + int max_write_buffer_number + int min_write_buffer_number_to_merge + int max_write_buffer_number_to_maintain + cpp_bool inplace_update_support + size_t inplace_update_num_locks + double memtable_prefix_bloom_size_ratio + size_t memtable_huge_page_size + shared_ptr[const SliceTransform] memtable_insert_with_hint_prefix_extractor + uint32_t bloom_locality + size_t arena_block_size + vector[CompressionType] compression_per_level + int num_levels + int level0_slowdown_writes_trigger + int level0_stop_writes_trigger + uint64_t target_file_size_base + int target_file_size_multiplier + cpp_bool level_compaction_dynamic_level_bytes + double max_bytes_for_level_multiplier + vector[int] max_bytes_for_level_multiplier_additional + uint64_t max_compaction_bytes + uint64_t soft_pending_compaction_bytes_limit + uint64_t hard_pending_compaction_bytes_limit + CompactionPri compaction_pri + CompactionOptionsUniversal compaction_options_universal + CompactionOptionsFIFO compaction_options_fifo + uint64_t max_sequential_skip_in_iterations + shared_ptr[MemTableRepFactory] memtable_factory + TablePropertiesCollectorFactories table_properties_collector_factories + size_t max_successive_merges + cpp_bool optimize_filters_for_hits + cpp_bool paranoid_file_checks + cpp_bool force_consistency_checks + cpp_bool report_bg_io_stats + + cdef extern from "rocksdb/options.h" namespace "rocksdb": cdef cppclass CompressionOptions: int window_bits; @@ -154,3 +235,29 @@ cdef extern from "rocksdb/options.h" namespace "rocksdb": int target_level uint32_t target_path_id BottommostLevelCompaction bottommost_level_compaction + + cdef cppclass ColumnFamilyOptions(AdvancedColumnFamilyOptions): + ColumnFamilyOptions* OldDefaults( + int rocksdb_major_version, + int rocksdb_minor_version) nogil except+ + + ColumnFamilyOptions* OptimizeLevelStyleCompaction( + uint64_t memtable_memory_budget) nogil except+ + + ColumnFamilyOptions* OptimizeUniversalStyleCompaction( + uint64_t memtable_memory_budget) nogil except+ + + const Comparator* comparator + shared_ptr[MergeOperator] merge_operator + const CompactionFilter* compaction_filter + shared_ptr[CompactionFilterFactory] compaction_filter_factory + size_t write_buffer_size + CompressionType compression + CompressionType bottommost_compression + CompressionOptions compression_opts + int level0_file_num_compaction_trigger + shared_ptr[const SliceTransform] prefix_extractor + uint64_t max_bytes_for_level_base + cpp_bool disable_auto_compactions + shared_ptr[TableFactory] table_factory + void Dump(Logger* log) diff --git a/rocksdb/table_factory.pxd b/rocksdb/table_factory.pxd index 2359292a..f41159a2 100644 --- a/rocksdb/table_factory.pxd +++ b/rocksdb/table_factory.pxd @@ -5,6 +5,20 @@ from std_memory cimport shared_ptr from cache cimport Cache from filter_policy cimport FilterPolicy + +cdef extern from "rocksdb/table_properties.h" namespace "rocksdb::TablePropertiesCollectorFactory": + cdef struct Context: + uint32_t column_family_id + +cdef extern from "rocksdb/table_properties.h" namespace "rocksdb": + cdef cppclass TablePropertiesCollector: + char* Name() const + + cdef cppclass TablePropertiesCollectorFactory: + TablePropertiesCollector* CreateTablePropertiesCollector( + Context context) nogil except+ + char* Name() const + cdef extern from "rocksdb/table.h" namespace "rocksdb": cdef cppclass TableFactory: TableFactory() @@ -27,6 +41,7 @@ cdef extern from "rocksdb/table.h" namespace "rocksdb": int block_size_deviation int block_restart_interval cpp_bool whole_key_filtering + cpp_bool cache_index_and_filter_blocks shared_ptr[Cache] block_cache shared_ptr[Cache] block_cache_compressed shared_ptr[FilterPolicy] filter_policy diff --git a/setup.py b/setup.py index 0b3f59c3..0b815380 100644 --- a/setup.py +++ b/setup.py @@ -13,20 +13,26 @@ def cythonize(extensions): return extensions mod1 = Extension( 'rocksdb._rocksdb', sources, + include_dirs = ['D:/lhz/proj/xd/dev/cpp/rocksdb-5.4.6/include', 'D:/lhz/proj/xd/dev/cpp/rocksdb-5.4.6/options'], + library_dirs = ['D:/lhz/proj/xd/dev/cpp/rocksdb-5.4.6/msvc14/Release', 'D:/lhz/proj/xd/dev/cpp/rocksdb-5.4.6/third-party/Snappy.Library/bin/retail/amd64', + 'D:/lhz/proj/xd/dev/cpp/rocksdb-5.4.6/third-party/LZ4.Library/bin/retail/amd64', 'D:/lhz/proj/xd/dev/cpp/rocksdb-5.4.6/third-party/ZLIB.Library/bin/retail/amd64', + 'D:/lhz/proj/xd/dev/cpp/rocksdb-5.4.6/third-party/Gflags.Library/bin/retail/amd64'], extra_compile_args=[ '-std=c++11', '-O3', '-Wall', - '-Wextra', - '-Wconversion', + # '-Wextra', + # '-Wconversion', '-fno-strict-aliasing' ], language='c++', libraries=[ - 'rocksdb', + 'rocksdblib', 'snappy', - 'bz2', - 'z' + 'lz4', + 'zlib', + 'gflags', + 'Rpcrt4' ] ) From 591313a6d9844192f97c6a330b0bd5eaa6860fab Mon Sep 17 00:00:00 2001 From: lhz Date: Sat, 6 Mar 2021 08:37:40 +0800 Subject: [PATCH 4/4] linux --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 0a73f29a..d001f319 100644 --- a/setup.py +++ b/setup.py @@ -36,7 +36,7 @@ ['rocksdb/_rocksdb.pyx'], extra_compile_args=extra_compile_args, language='c++', - libraries=['rocksdb', 'snappy', 'bz2', 'z', 'lz4'], + libraries=['rocksdb', 'snappy', 'bz2', 'zstd', 'lz4'], )], extras_require={ "doc": ['sphinx_rtd_theme', 'sphinx'],