feat: add vendered rocksdb (#1)

* feat: add vendered rocksdb

* fix ci

* fix zig master

.github/workflows/CI.yml

@@ -5,34 +5,32 @@ on:
     - cron: '10 20 * * *'
   workflow_dispatch:
   pull_request:
-    paths:
-      - '**.zig'
-      - '**.yml'
+    paths-ignore:
+      - 'README.org'
   push:
     branches:
       - main
-    paths:
-      - '**.zig'
-      - '**.yml'
+    paths-ignore:
+      - 'README.org'
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
 
 jobs:
   test:
-    timeout-minutes: 10
+    timeout-minutes: 15
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest]
-        # os: [ubuntu-latest, macos-latest]
-        zig-version: [0.12.0, master]
+        os: [ubuntu-latest, macos-latest]
+        zig-version: [0.13.0, master]
     steps:
       - uses: actions/checkout@v4
-      - uses: goto-bus-stop/setup-zig@v2
+      - uses: mlugg/setup-zig@v1
         with:
           version: ${{ matrix.zig-version }}
-      - name: Install deps
-        run: |
-          make install-deps
       - name: Run tests
         run: |
           make test
@@ -42,4 +40,5 @@ jobs:
       - name: Memory leak detect
         if: matrix.os == 'ubuntu-latest'
         run: |
+          sudo apt install -y librocksdb-dev valgrind
           make valgrind

.gitignore

@@ -5,6 +5,7 @@
 # Zig programming language
 
 zig-cache/
+.zig-cache/
 zig-out/
 build/
 build-*/

Makefile

@@ -10,6 +10,9 @@ run:
 test:
 	zig build test -freference-trace
 
+clean:
+	rm -rf .zig-cache zig-out
+
 valgrind:
 	zig build
 	./scripts/valgrind.sh

README.org

@@ -1,20 +1,35 @@
 #+TITLE: Zig-rocksdb
 #+DATE: 2024-04-27T11:27:04+0800
-#+LASTMOD: 2024-04-30T09:05:31+0800
+#+LASTMOD: 2024-10-27T14:34:21+0800
 #+AUTHOR: Jiacai Liu
 
+[[https://img.shields.io/badge/zig%20version-0.13.0-blue.svg]]
 [[https://github.com/jiacai2050/zig-rocksdb/actions/workflows/CI.yml][https://github.com/jiacai2050/zig-rocksdb/actions/workflows/CI.yml/badge.svg]]
 
 [[https://github.com/facebook/rocksdb/][RocksDB]] binding for Zig.
 
 * Usage
+See [[file:examples/basic.zig]], [[file:examples/cf.zig]] for details.
+* Installation
 #+begin_src bash
 zig fetch --save=rocksdb https://github.com/jiacai2050/zig-rocksdb/archive/${COMMIT}.tar.gz
 #+end_src
 
-See [[file:examples/basic.zig]], [[file:examples/cf.zig]] for details.
-* Other bindings
-https://github.com/facebook/rocksdb/blob/main/LANGUAGE-BINDINGS.md
+Replace ~${COMMIT}~ with a real one, then in your =build.zig=, import the module like this:
+#+begin_src zig
+const dep_rocksdb = b.dependency("rocksdb", .{});
+exe.root_module.addImport("rocksdb", dep_rocksdb.module("rocksdb"));
+exe.linkLibC();
+#+end_src
+
+This library will link to a vendored [[https://github.com/facebook/rocksdb/releases/tag/v9.0.0][librocksdb(v9.0.0)]] by default, you can disable it and link to system-wide with this
+#+begin_src zig
+const dep_rocksdb = b.dependency("rocksdb", .{ .link_vendor = false });
+exe.linkSystemLibrary("rocksdb");
+exe.linkLibC();
+#+end_src
+* Acknowledge
+Thanks to [[https://github.com/rust-rocksdb/rust-rocksdb][rust-rocksdb]], I benefit a lot from it when build this building library.
 
 * License
 [[file:LICENSE][MIT]]

build.zig

@@ -1,18 +1,32 @@
 const std = @import("std");
+const Build = std.Build;
+const Step = Build.Step;
 
-pub fn build(b: *std.Build) void {
+pub fn build(b: *std.Build) !void {
     const target = b.standardTargetOptions(.{});
     const optimize = b.standardOptimizeOption(.{});
+    const link_vendor = b.option(bool, "link_vendor", "Whether link to vendored rocksdb(default: true)") orelse true;
+    const strip_lib = b.option(bool, "strip_lib", "Whether strip librocksdb(default: false)") orelse false;
 
-    const module = b.createModule(.{
+    const module = b.addModule("rocksdb", .{
         .root_source_file = b.path("src/root.zig"),
         .target = target,
         .optimize = optimize,
         .link_libc = true,
         .link_libcpp = true,
     });
 
-    module.linkSystemLibrary("rocksdb", .{});
+    var librocksdb: ?*Step.Compile = null;
+    if (link_vendor) {
+        if (try buildStaticRocksdb(b, target, optimize, strip_lib)) |v| {
+            librocksdb = v;
+            module.linkLibrary(v);
+        } else {
+            return;
+        }
+    } else {
+        module.linkSystemLibrary("rocksdb", .{});
+    }
 
     const lib_unit_tests = b.addTest(.{
         .root_source_file = b.path("src/root.zig"),
@@ -27,6 +41,78 @@ pub fn build(b: *std.Build) void {
     buildExample(b, "cf", run_step, target, optimize, module);
 }
 
+fn buildStaticRocksdb(
+    b: *std.Build,
+    target: std.Build.ResolvedTarget,
+    optimize: std.builtin.OptimizeMode,
+    strip_lib: bool,
+) !?*Step.Compile {
+    const is_darwin = target.result.isDarwin();
+    const is_linux = target.result.os.tag == .linux;
+
+    const rocksdb_dep = b.lazyDependency("rocksdb", .{
+        .target = target,
+        .optimize = optimize,
+    }) orelse return null;
+    const lib = b.addStaticLibrary(.{
+        .name = "rocksdb",
+        .target = target,
+        .optimize = optimize,
+        .link_libc = true,
+        .strip = if (strip_lib) true else false,
+    });
+    lib.root_module.sanitize_c = false;
+    if (optimize != .Debug) {
+        lib.defineCMacro("NDEBUG", "1");
+    }
+
+    lib.defineCMacro("ROCKSDB_PLATFORM_POSIX", null);
+    lib.defineCMacro("ROCKSDB_LIB_IO_POSIX", null);
+    lib.defineCMacro("ROCKSDB_SUPPORT_THREAD_LOCAL", null);
+    if (is_darwin) {
+        lib.defineCMacro("OS_MACOSX", null);
+    } else if (is_linux) {
+        lib.defineCMacro("OS_LINUX", null);
+    }
+
+    lib.linkLibCpp();
+    lib.addIncludePath(rocksdb_dep.path("include"));
+    lib.addIncludePath(rocksdb_dep.path("."));
+    const cflags = &.{
+        "-std=c++17",
+        "-Wsign-compare",
+        "-Wshadow",
+        "-Wno-unused-parameter",
+        "-Wno-unused-variable",
+        "-Woverloaded-virtual",
+        "-Wnon-virtual-dtor",
+        "-Wno-missing-field-initializers",
+        "-Wno-strict-aliasing",
+        "-Wno-invalid-offsetof",
+    };
+    const src_file = b.path("sys/rocksdb_lib_sources.txt").getPath2(b, null);
+    var f = try std.fs.openFileAbsolute(src_file, .{});
+    const body = try f.readToEndAlloc(b.allocator, 1024_1000);
+    var it = std.mem.splitScalar(u8, body, '\n');
+    while (it.next()) |src| {
+        // We have a pre-generated a version of build_version.cc in the local directory
+        if (std.mem.eql(u8, "util/build_version.cc", src) or src.len == 0) {
+            continue;
+        }
+        lib.addCSourceFile(.{
+            .file = rocksdb_dep.path(src),
+            .flags = cflags,
+        });
+    }
+    lib.addCSourceFile(.{
+        .file = b.path("sys/build_version.cc"),
+        .flags = cflags,
+    });
+    b.installArtifact(lib);
+    lib.installHeadersDirectory(rocksdb_dep.path("include/rocksdb"), "rocksdb", .{});
+    return lib;
+}
+
 fn buildExample(
     b: *std.Build,
     comptime name: []const u8,

build.zig.zon

@@ -1,12 +1,18 @@
 .{
     .name = "zig-rocksdb",
-    .version = "0.0.0",
+    .version = "0.1.0",
     .minimum_zig_version = "0.12.0",
-    .dependencies = .{},
+    .dependencies = .{
+        .rocksdb = .{
+            .url = "https://github.com/facebook/rocksdb/archive/refs/tags/v9.0.0.zip",
+            .hash = "1220bd8f94fd5fcab321fa2b433f0f69b33d6af3fc5cc548ddda793633d629ae3647",
+        },
+    },
     .paths = .{
         "build.zig",
         "build.zig.zon",
         "src",
+        "sys", // librocksdb
         "LICENSE",
         "README.org",
     },

src/root.zig

@@ -5,6 +5,7 @@ const WriteOptions = @import("options.zig").WriteOptions;
 const ColumnFamily = @import("ColumnFamily.zig");
 const mem = std.mem;
 const Allocator = mem.Allocator;
+const is_latest_zig = @import("builtin").zig_version.minor > 13;
 
 const testing = std.testing;
 pub const c = @cImport({
@@ -292,7 +293,7 @@ pub fn Database(comptime tm: ThreadMode) type {
 fn FFIReturnType(Func: type) type {
     const info = @typeInfo(Func);
     const fn_info = switch (info) {
-        .Fn => |fn_info| fn_info,
+        if (is_latest_zig) .@"fn" else .Fn => |fn_info| fn_info,
         else => @compileError("expecting a function"),
     };
 

sys/build_version.cc

@@ -0,0 +1,77 @@
+// Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
+
+#include <memory>
+
+#include "rocksdb/version.h"
+#include "rocksdb/utilities/object_registry.h"
+#include "util/string_util.h"
+
+// The build script may replace these values with real values based
+// on whether or not GIT is available and the platform settings
+static const std::string rocksdb_build_git_sha  = "f4441966592636253fd5ab0bb9ed44fc2697fc53";
+static const std::string rocksdb_build_git_tag = "rocksdb_build_git_tag:v9.0.0";
+#define HAS_GIT_CHANGES 0
+#if HAS_GIT_CHANGES == 0
+// If HAS_GIT_CHANGES is 0, the GIT date is used.
+// Use the time the branch/tag was last modified
+static const std::string rocksdb_build_date = "rocksdb_build_date:2024-03-11 11:26:24";
+#else
+// If HAS_GIT_CHANGES is > 0, the branch/tag has modifications.
+// Use the time the build was created.
+static const std::string rocksdb_build_date = "rocksdb_build_date:2024-03-11 11:26:24";
+#endif
+
+std::unordered_map<std::string, ROCKSDB_NAMESPACE::RegistrarFunc> ROCKSDB_NAMESPACE::ObjectRegistry::builtins_ = {};
+
+extern "C" bool RocksDbIOUringEnable() {
+  return true;
+}
+
+namespace ROCKSDB_NAMESPACE {
+static void AddProperty(std::unordered_map<std::string, std::string> *props, const std::string& name) {
+  size_t colon = name.find(":");
+  if (colon != std::string::npos && colon > 0 && colon < name.length() - 1) {
+    // If we found a "@:", then this property was a build-time substitution that failed.  Skip it
+    size_t at = name.find("@", colon);
+    if (at != colon + 1) {
+      // Everything before the colon is the name, after is the value
+      (*props)[name.substr(0, colon)] = name.substr(colon + 1);
+    }
+  }
+}
+
+static std::unordered_map<std::string, std::string>* LoadPropertiesSet() {
+  auto * properties = new std::unordered_map<std::string, std::string>();
+  AddProperty(properties, rocksdb_build_git_sha);
+  AddProperty(properties, rocksdb_build_git_tag);
+  AddProperty(properties, rocksdb_build_date);
+  return properties;
+}
+
+const std::unordered_map<std::string, std::string>& GetRocksBuildProperties() {
+  static std::unique_ptr<std::unordered_map<std::string, std::string>> props(LoadPropertiesSet());
+  return *props;
+}
+
+std::string GetRocksVersionAsString(bool with_patch) {
+  std::string version = std::to_string(ROCKSDB_MAJOR) + "." + std::to_string(ROCKSDB_MINOR);
+  if (with_patch) {
+    return version + "." + std::to_string(ROCKSDB_PATCH);
+  } else {
+    return version;
+ }
+}
+
+std::string GetRocksBuildInfoAsString(const std::string& program, bool verbose) {
+  std::string info = program + " (RocksDB) " + GetRocksVersionAsString(true);
+  if (verbose) {
+    for (const auto& it : GetRocksBuildProperties()) {
+      info.append("\n    ");
+      info.append(it.first);
+      info.append(": ");
+      info.append(it.second);
+    }
+  }
+  return info;
+}
+} // namespace ROCKSDB_NAMESPACE