diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 3b91778..78049ba 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -5,34 +5,32 @@ on: - cron: '10 20 * * *' workflow_dispatch: pull_request: - paths: - - '**.zig' - - '**.yml' + paths-ignore: + - 'README.org' push: branches: - main - paths: - - '**.zig' - - '**.yml' + paths-ignore: + - 'README.org' + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true jobs: test: - timeout-minutes: 10 + timeout-minutes: 15 runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: - os: [ubuntu-latest] - # os: [ubuntu-latest, macos-latest] - zig-version: [0.12.0, master] + os: [ubuntu-latest, macos-latest] + zig-version: [0.13.0, master] steps: - uses: actions/checkout@v4 - - uses: goto-bus-stop/setup-zig@v2 + - uses: mlugg/setup-zig@v1 with: version: ${{ matrix.zig-version }} - - name: Install deps - run: | - make install-deps - name: Run tests run: | make test @@ -42,4 +40,5 @@ jobs: - name: Memory leak detect if: matrix.os == 'ubuntu-latest' run: | + sudo apt install -y librocksdb-dev valgrind make valgrind diff --git a/.gitignore b/.gitignore index 7dc0ebe..83ecaec 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ # Zig programming language zig-cache/ +.zig-cache/ zig-out/ build/ build-*/ diff --git a/Makefile b/Makefile index e6e158a..b1223af 100644 --- a/Makefile +++ b/Makefile @@ -10,6 +10,9 @@ run: test: zig build test -freference-trace +clean: + rm -rf .zig-cache zig-out + valgrind: zig build ./scripts/valgrind.sh diff --git a/README.org b/README.org index e7c1a8e..be22a87 100644 --- a/README.org +++ b/README.org @@ -1,20 +1,35 @@ #+TITLE: Zig-rocksdb #+DATE: 2024-04-27T11:27:04+0800 -#+LASTMOD: 2024-04-30T09:05:31+0800 +#+LASTMOD: 2024-10-27T14:34:21+0800 #+AUTHOR: Jiacai Liu +[[https://img.shields.io/badge/zig%20version-0.13.0-blue.svg]] [[https://github.com/jiacai2050/zig-rocksdb/actions/workflows/CI.yml][https://github.com/jiacai2050/zig-rocksdb/actions/workflows/CI.yml/badge.svg]] [[https://github.com/facebook/rocksdb/][RocksDB]] binding for Zig. * Usage +See [[file:examples/basic.zig]], [[file:examples/cf.zig]] for details. +* Installation #+begin_src bash zig fetch --save=rocksdb https://github.com/jiacai2050/zig-rocksdb/archive/${COMMIT}.tar.gz #+end_src -See [[file:examples/basic.zig]], [[file:examples/cf.zig]] for details. -* Other bindings -https://github.com/facebook/rocksdb/blob/main/LANGUAGE-BINDINGS.md +Replace ~${COMMIT}~ with a real one, then in your =build.zig=, import the module like this: +#+begin_src zig +const dep_rocksdb = b.dependency("rocksdb", .{}); +exe.root_module.addImport("rocksdb", dep_rocksdb.module("rocksdb")); +exe.linkLibC(); +#+end_src + +This library will link to a vendored [[https://github.com/facebook/rocksdb/releases/tag/v9.0.0][librocksdb(v9.0.0)]] by default, you can disable it and link to system-wide with this +#+begin_src zig +const dep_rocksdb = b.dependency("rocksdb", .{ .link_vendor = false }); +exe.linkSystemLibrary("rocksdb"); +exe.linkLibC(); +#+end_src +* Acknowledge +Thanks to [[https://github.com/rust-rocksdb/rust-rocksdb][rust-rocksdb]], I benefit a lot from it when build this building library. * License [[file:LICENSE][MIT]] diff --git a/build.zig b/build.zig index 03d8dbe..ebbd7ad 100644 --- a/build.zig +++ b/build.zig @@ -1,10 +1,14 @@ const std = @import("std"); +const Build = std.Build; +const Step = Build.Step; -pub fn build(b: *std.Build) void { +pub fn build(b: *std.Build) !void { const target = b.standardTargetOptions(.{}); const optimize = b.standardOptimizeOption(.{}); + const link_vendor = b.option(bool, "link_vendor", "Whether link to vendored rocksdb(default: true)") orelse true; + const strip_lib = b.option(bool, "strip_lib", "Whether strip librocksdb(default: false)") orelse false; - const module = b.createModule(.{ + const module = b.addModule("rocksdb", .{ .root_source_file = b.path("src/root.zig"), .target = target, .optimize = optimize, @@ -12,7 +16,17 @@ pub fn build(b: *std.Build) void { .link_libcpp = true, }); - module.linkSystemLibrary("rocksdb", .{}); + var librocksdb: ?*Step.Compile = null; + if (link_vendor) { + if (try buildStaticRocksdb(b, target, optimize, strip_lib)) |v| { + librocksdb = v; + module.linkLibrary(v); + } else { + return; + } + } else { + module.linkSystemLibrary("rocksdb", .{}); + } const lib_unit_tests = b.addTest(.{ .root_source_file = b.path("src/root.zig"), @@ -27,6 +41,78 @@ pub fn build(b: *std.Build) void { buildExample(b, "cf", run_step, target, optimize, module); } +fn buildStaticRocksdb( + b: *std.Build, + target: std.Build.ResolvedTarget, + optimize: std.builtin.OptimizeMode, + strip_lib: bool, +) !?*Step.Compile { + const is_darwin = target.result.isDarwin(); + const is_linux = target.result.os.tag == .linux; + + const rocksdb_dep = b.lazyDependency("rocksdb", .{ + .target = target, + .optimize = optimize, + }) orelse return null; + const lib = b.addStaticLibrary(.{ + .name = "rocksdb", + .target = target, + .optimize = optimize, + .link_libc = true, + .strip = if (strip_lib) true else false, + }); + lib.root_module.sanitize_c = false; + if (optimize != .Debug) { + lib.defineCMacro("NDEBUG", "1"); + } + + lib.defineCMacro("ROCKSDB_PLATFORM_POSIX", null); + lib.defineCMacro("ROCKSDB_LIB_IO_POSIX", null); + lib.defineCMacro("ROCKSDB_SUPPORT_THREAD_LOCAL", null); + if (is_darwin) { + lib.defineCMacro("OS_MACOSX", null); + } else if (is_linux) { + lib.defineCMacro("OS_LINUX", null); + } + + lib.linkLibCpp(); + lib.addIncludePath(rocksdb_dep.path("include")); + lib.addIncludePath(rocksdb_dep.path(".")); + const cflags = &.{ + "-std=c++17", + "-Wsign-compare", + "-Wshadow", + "-Wno-unused-parameter", + "-Wno-unused-variable", + "-Woverloaded-virtual", + "-Wnon-virtual-dtor", + "-Wno-missing-field-initializers", + "-Wno-strict-aliasing", + "-Wno-invalid-offsetof", + }; + const src_file = b.path("sys/rocksdb_lib_sources.txt").getPath2(b, null); + var f = try std.fs.openFileAbsolute(src_file, .{}); + const body = try f.readToEndAlloc(b.allocator, 1024_1000); + var it = std.mem.splitScalar(u8, body, '\n'); + while (it.next()) |src| { + // We have a pre-generated a version of build_version.cc in the local directory + if (std.mem.eql(u8, "util/build_version.cc", src) or src.len == 0) { + continue; + } + lib.addCSourceFile(.{ + .file = rocksdb_dep.path(src), + .flags = cflags, + }); + } + lib.addCSourceFile(.{ + .file = b.path("sys/build_version.cc"), + .flags = cflags, + }); + b.installArtifact(lib); + lib.installHeadersDirectory(rocksdb_dep.path("include/rocksdb"), "rocksdb", .{}); + return lib; +} + fn buildExample( b: *std.Build, comptime name: []const u8, diff --git a/build.zig.zon b/build.zig.zon index 50e0cb6..7a40bce 100644 --- a/build.zig.zon +++ b/build.zig.zon @@ -1,12 +1,18 @@ .{ .name = "zig-rocksdb", - .version = "0.0.0", + .version = "0.1.0", .minimum_zig_version = "0.12.0", - .dependencies = .{}, + .dependencies = .{ + .rocksdb = .{ + .url = "https://github.com/facebook/rocksdb/archive/refs/tags/v9.0.0.zip", + .hash = "1220bd8f94fd5fcab321fa2b433f0f69b33d6af3fc5cc548ddda793633d629ae3647", + }, + }, .paths = .{ "build.zig", "build.zig.zon", "src", + "sys", // librocksdb "LICENSE", "README.org", }, diff --git a/src/root.zig b/src/root.zig index 0b27a51..8f385bb 100644 --- a/src/root.zig +++ b/src/root.zig @@ -5,6 +5,7 @@ const WriteOptions = @import("options.zig").WriteOptions; const ColumnFamily = @import("ColumnFamily.zig"); const mem = std.mem; const Allocator = mem.Allocator; +const is_latest_zig = @import("builtin").zig_version.minor > 13; const testing = std.testing; pub const c = @cImport({ @@ -292,7 +293,7 @@ pub fn Database(comptime tm: ThreadMode) type { fn FFIReturnType(Func: type) type { const info = @typeInfo(Func); const fn_info = switch (info) { - .Fn => |fn_info| fn_info, + if (is_latest_zig) .@"fn" else .Fn => |fn_info| fn_info, else => @compileError("expecting a function"), }; diff --git a/sys/build_version.cc b/sys/build_version.cc new file mode 100644 index 0000000..a7e0256 --- /dev/null +++ b/sys/build_version.cc @@ -0,0 +1,77 @@ +// Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + +#include + +#include "rocksdb/version.h" +#include "rocksdb/utilities/object_registry.h" +#include "util/string_util.h" + +// The build script may replace these values with real values based +// on whether or not GIT is available and the platform settings +static const std::string rocksdb_build_git_sha = "f4441966592636253fd5ab0bb9ed44fc2697fc53"; +static const std::string rocksdb_build_git_tag = "rocksdb_build_git_tag:v9.0.0"; +#define HAS_GIT_CHANGES 0 +#if HAS_GIT_CHANGES == 0 +// If HAS_GIT_CHANGES is 0, the GIT date is used. +// Use the time the branch/tag was last modified +static const std::string rocksdb_build_date = "rocksdb_build_date:2024-03-11 11:26:24"; +#else +// If HAS_GIT_CHANGES is > 0, the branch/tag has modifications. +// Use the time the build was created. +static const std::string rocksdb_build_date = "rocksdb_build_date:2024-03-11 11:26:24"; +#endif + +std::unordered_map ROCKSDB_NAMESPACE::ObjectRegistry::builtins_ = {}; + +extern "C" bool RocksDbIOUringEnable() { + return true; +} + +namespace ROCKSDB_NAMESPACE { +static void AddProperty(std::unordered_map *props, const std::string& name) { + size_t colon = name.find(":"); + if (colon != std::string::npos && colon > 0 && colon < name.length() - 1) { + // If we found a "@:", then this property was a build-time substitution that failed. Skip it + size_t at = name.find("@", colon); + if (at != colon + 1) { + // Everything before the colon is the name, after is the value + (*props)[name.substr(0, colon)] = name.substr(colon + 1); + } + } +} + +static std::unordered_map* LoadPropertiesSet() { + auto * properties = new std::unordered_map(); + AddProperty(properties, rocksdb_build_git_sha); + AddProperty(properties, rocksdb_build_git_tag); + AddProperty(properties, rocksdb_build_date); + return properties; +} + +const std::unordered_map& GetRocksBuildProperties() { + static std::unique_ptr> props(LoadPropertiesSet()); + return *props; +} + +std::string GetRocksVersionAsString(bool with_patch) { + std::string version = std::to_string(ROCKSDB_MAJOR) + "." + std::to_string(ROCKSDB_MINOR); + if (with_patch) { + return version + "." + std::to_string(ROCKSDB_PATCH); + } else { + return version; + } +} + +std::string GetRocksBuildInfoAsString(const std::string& program, bool verbose) { + std::string info = program + " (RocksDB) " + GetRocksVersionAsString(true); + if (verbose) { + for (const auto& it : GetRocksBuildProperties()) { + info.append("\n "); + info.append(it.first); + info.append(": "); + info.append(it.second); + } + } + return info; +} +} // namespace ROCKSDB_NAMESPACE diff --git a/sys/rocksdb_lib_sources.txt b/sys/rocksdb_lib_sources.txt new file mode 100644 index 0000000..f7159dd --- /dev/null +++ b/sys/rocksdb_lib_sources.txt @@ -0,0 +1,316 @@ +cache/cache.cc +cache/cache_entry_roles.cc +cache/cache_key.cc +cache/cache_helpers.cc +cache/cache_reservation_manager.cc +cache/charged_cache.cc +cache/clock_cache.cc +cache/lru_cache.cc +cache/compressed_secondary_cache.cc +cache/secondary_cache.cc +cache/secondary_cache_adapter.cc +cache/sharded_cache.cc +cache/tiered_secondary_cache.cc +db/arena_wrapped_db_iter.cc +db/blob/blob_contents.cc +db/blob/blob_fetcher.cc +db/blob/blob_file_addition.cc +db/blob/blob_file_builder.cc +db/blob/blob_file_cache.cc +db/blob/blob_file_garbage.cc +db/blob/blob_file_meta.cc +db/blob/blob_file_reader.cc +db/blob/blob_garbage_meter.cc +db/blob/blob_log_format.cc +db/blob/blob_log_sequential_reader.cc +db/blob/blob_log_writer.cc +db/blob/blob_source.cc +db/blob/prefetch_buffer_collection.cc +db/builder.cc +db/c.cc +db/column_family.cc +db/compaction/compaction.cc +db/compaction/compaction_iterator.cc +db/compaction/compaction_job.cc +db/compaction/compaction_picker.cc +db/compaction/compaction_picker_fifo.cc +db/compaction/compaction_picker_level.cc +db/compaction/compaction_picker_universal.cc +db/compaction/compaction_service_job.cc +db/compaction/compaction_state.cc +db/compaction/compaction_outputs.cc +db/compaction/sst_partitioner.cc +db/compaction/subcompaction_state.cc +db/convenience.cc +db/db_filesnapshot.cc +db/db_impl/compacted_db_impl.cc +db/db_impl/db_impl.cc +db/db_impl/db_impl_compaction_flush.cc +db/db_impl/db_impl_debug.cc +db/db_impl/db_impl_experimental.cc +db/db_impl/db_impl_files.cc +db/db_impl/db_impl_open.cc +db/db_impl/db_impl_readonly.cc +db/db_impl/db_impl_secondary.cc +db/db_impl/db_impl_write.cc +db/db_info_dumper.cc +db/db_iter.cc +db/dbformat.cc +db/error_handler.cc +db/event_helpers.cc +db/experimental.cc +db/external_sst_file_ingestion_job.cc +db/file_indexer.cc +db/flush_job.cc +db/flush_scheduler.cc +db/forward_iterator.cc +db/import_column_family_job.cc +db/internal_stats.cc +db/logs_with_prep_tracker.cc +db/log_reader.cc +db/log_writer.cc +db/malloc_stats.cc +db/memtable.cc +db/memtable_list.cc +db/merge_helper.cc +db/merge_operator.cc +db/output_validator.cc +db/periodic_task_scheduler.cc +db/range_del_aggregator.cc +db/range_tombstone_fragmenter.cc +db/repair.cc +db/seqno_to_time_mapping.cc +db/snapshot_impl.cc +db/table_cache.cc +db/table_properties_collector.cc +db/transaction_log_impl.cc +db/trim_history_scheduler.cc +db/version_builder.cc +db/version_edit.cc +db/version_edit_handler.cc +db/version_set.cc +db/wal_edit.cc +db/wal_manager.cc +db/wide/wide_column_serialization.cc +db/wide/wide_columns.cc +db/wide/wide_columns_helper.cc +db/write_batch.cc +db/write_batch_base.cc +db/write_controller.cc +db/write_stall_stats.cc +db/write_thread.cc +env/composite_env.cc +env/env.cc +env/env_chroot.cc +env/env_encryption.cc +env/env_posix.cc +env/file_system.cc +env/fs_posix.cc +env/fs_remap.cc +env/file_system_tracer.cc +env/io_posix.cc +env/mock_env.cc +env/unique_id_gen.cc +file/delete_scheduler.cc +file/file_prefetch_buffer.cc +file/file_util.cc +file/filename.cc +file/line_file_reader.cc +file/random_access_file_reader.cc +file/read_write_util.cc +file/readahead_raf.cc +file/sequence_file_reader.cc +file/sst_file_manager_impl.cc +file/writable_file_writer.cc +logging/auto_roll_logger.cc +logging/event_logger.cc +logging/log_buffer.cc +memory/arena.cc +memory/concurrent_arena.cc +memory/jemalloc_nodump_allocator.cc +memory/memkind_kmem_allocator.cc +memory/memory_allocator.cc +memtable/alloc_tracker.cc +memtable/hash_linklist_rep.cc +memtable/hash_skiplist_rep.cc +memtable/skiplistrep.cc +memtable/vectorrep.cc +memtable/write_buffer_manager.cc +monitoring/histogram.cc +monitoring/histogram_windowing.cc +monitoring/in_memory_stats_history.cc +monitoring/instrumented_mutex.cc +monitoring/iostats_context.cc +monitoring/perf_context.cc +monitoring/perf_level.cc +monitoring/persistent_stats_history.cc +monitoring/statistics.cc +monitoring/thread_status_impl.cc +monitoring/thread_status_updater.cc +monitoring/thread_status_updater_debug.cc +monitoring/thread_status_util.cc +monitoring/thread_status_util_debug.cc +options/cf_options.cc +options/configurable.cc +options/customizable.cc +options/db_options.cc +options/offpeak_time_info.cc +options/options.cc +options/options_helper.cc +options/options_parser.cc +port/mmap.cc +port/port_posix.cc +port/stack_trace.cc +table/adaptive/adaptive_table_factory.cc +table/block_based/binary_search_index_reader.cc +table/block_based/block.cc +table/block_based/block_based_table_builder.cc +table/block_based/block_based_table_factory.cc +table/block_based/block_based_table_iterator.cc +table/block_based/block_based_table_reader.cc +table/block_based/block_builder.cc +table/block_based/block_cache.cc +table/block_based/block_prefetcher.cc +table/block_based/block_prefix_index.cc +table/block_based/data_block_hash_index.cc +table/block_based/data_block_footer.cc +table/block_based/filter_block_reader_common.cc +table/block_based/filter_policy.cc +table/block_based/flush_block_policy.cc +table/block_based/full_filter_block.cc +table/block_based/hash_index_reader.cc +table/block_based/index_builder.cc +table/block_based/index_reader_common.cc +table/block_based/parsed_full_filter_block.cc +table/block_based/partitioned_filter_block.cc +table/block_based/partitioned_index_iterator.cc +table/block_based/partitioned_index_reader.cc +table/block_based/reader_common.cc +table/block_based/uncompression_dict_reader.cc +table/block_fetcher.cc +table/cuckoo/cuckoo_table_builder.cc +table/cuckoo/cuckoo_table_factory.cc +table/cuckoo/cuckoo_table_reader.cc +table/format.cc +table/get_context.cc +table/iterator.cc +table/merging_iterator.cc +table/compaction_merging_iterator.cc +table/meta_blocks.cc +table/persistent_cache_helper.cc +table/plain/plain_table_bloom.cc +table/plain/plain_table_builder.cc +table/plain/plain_table_factory.cc +table/plain/plain_table_index.cc +table/plain/plain_table_key_coding.cc +table/plain/plain_table_reader.cc +table/sst_file_dumper.cc +table/sst_file_reader.cc +table/sst_file_writer.cc +table/table_factory.cc +table/table_properties.cc +table/two_level_iterator.cc +table/unique_id.cc +test_util/sync_point.cc +test_util/sync_point_impl.cc +test_util/transaction_test_util.cc +tools/dump/db_dump_tool.cc +trace_replay/trace_record_handler.cc +trace_replay/trace_record_result.cc +trace_replay/trace_record.cc +trace_replay/trace_replay.cc +trace_replay/block_cache_tracer.cc +trace_replay/io_tracer.cc +util/async_file_reader.cc +util/build_version.cc +util/cleanable.cc +util/coding.cc +util/compaction_job_stats_impl.cc +util/comparator.cc +util/compression.cc +util/compression_context_cache.cc +util/concurrent_task_limiter_impl.cc +util/crc32c.cc +util/crc32c_arm64.cc +util/data_structure.cc +util/dynamic_bloom.cc +util/hash.cc +util/murmurhash.cc +util/random.cc +util/rate_limiter.cc +util/ribbon_config.cc +util/slice.cc +util/file_checksum_helper.cc +util/status.cc +util/stderr_logger.cc +util/string_util.cc +util/thread_local.cc +util/threadpool_imp.cc +util/udt_util.cc +util/write_batch_util.cc +util/xxhash.cc +utilities/agg_merge/agg_merge.cc +utilities/backup/backup_engine.cc +utilities/blob_db/blob_compaction_filter.cc +utilities/blob_db/blob_db.cc +utilities/blob_db/blob_db_impl.cc +utilities/blob_db/blob_db_impl_filesnapshot.cc +utilities/blob_db/blob_file.cc +utilities/cache_dump_load.cc +utilities/cache_dump_load_impl.cc +utilities/cassandra/cassandra_compaction_filter.cc +utilities/cassandra/format.cc +utilities/cassandra/merge_operator.cc +utilities/checkpoint/checkpoint_impl.cc +utilities/compaction_filters.cc +utilities/compaction_filters/remove_emptyvalue_compactionfilter.cc +utilities/convenience/info_log_finder.cc +utilities/counted_fs.cc +utilities/debug.cc +utilities/env_mirror.cc +utilities/env_timed.cc +utilities/fault_injection_env.cc +utilities/fault_injection_fs.cc +utilities/fault_injection_secondary_cache.cc +utilities/leveldb_options/leveldb_options.cc +utilities/memory/memory_util.cc +utilities/merge_operators.cc +utilities/merge_operators/max.cc +utilities/merge_operators/put.cc +utilities/merge_operators/sortlist.cc +utilities/merge_operators/string_append/stringappend.cc +utilities/merge_operators/string_append/stringappend2.cc +utilities/merge_operators/uint64add.cc +utilities/merge_operators/bytesxor.cc +utilities/object_registry.cc +utilities/option_change_migration/option_change_migration.cc +utilities/options/options_util.cc +utilities/persistent_cache/block_cache_tier.cc +utilities/persistent_cache/block_cache_tier_file.cc +utilities/persistent_cache/block_cache_tier_metadata.cc +utilities/persistent_cache/persistent_cache_tier.cc +utilities/persistent_cache/volatile_tier_impl.cc +utilities/simulator_cache/cache_simulator.cc +utilities/simulator_cache/sim_cache.cc +utilities/table_properties_collectors/compact_on_deletion_collector.cc +utilities/trace/file_trace_reader_writer.cc +utilities/trace/replayer_impl.cc +utilities/transactions/lock/lock_manager.cc +utilities/transactions/lock/point/point_lock_tracker.cc +utilities/transactions/lock/point/point_lock_manager.cc +utilities/transactions/optimistic_transaction.cc +utilities/transactions/optimistic_transaction_db_impl.cc +utilities/transactions/pessimistic_transaction.cc +utilities/transactions/pessimistic_transaction_db.cc +utilities/transactions/snapshot_checker.cc +utilities/transactions/transaction_base.cc +utilities/transactions/transaction_db_mutex_impl.cc +utilities/transactions/transaction_util.cc +utilities/transactions/write_prepared_txn.cc +utilities/transactions/write_prepared_txn_db.cc +utilities/transactions/write_unprepared_txn.cc +utilities/transactions/write_unprepared_txn_db.cc +utilities/ttl/db_ttl_impl.cc +utilities/wal_filter.cc +utilities/write_batch_with_index/write_batch_with_index.cc +utilities/write_batch_with_index/write_batch_with_index_internal.cc