From 3d3a9a37f4db8f690ba3d30f975764cb04157f35 Mon Sep 17 00:00:00 2001 From: dterazhao Date: Mon, 11 Nov 2024 16:55:42 +0800 Subject: [PATCH] feat: compiler static library for phe_kit --- .bazelrc | 5 +- .bazelversion | 2 +- MODULE.bazel | 21 ++- MODULE.bazel.lock | 12 +- bazel/algo.bzl | 31 ++- bazel/build.sh | 11 +- bazel/cc_static_library.bzl | 112 +++++++++++ bazel/repositories.bzl | 13 ++ bazel/third_party/gmp.BUILD | 4 +- native/pom.xml | 86 ++++++++- .../algorithm/javacpp/heu/Ciphertext.java | 23 +++ .../cstn/algorithm/javacpp/heu/Constants.java | 59 ++++++ .../cn/cstn/algorithm/javacpp/heu/PheKit.java | 39 ++++ .../algorithm/javacpp/heu/SchemaType.java | 20 ++ .../cn/cstn/algorithm/javacpp/preset/heu.java | 177 ++++++++++++++++++ .../cn/cstn/algorithm/javacpp/api/BUILD.bazel | 6 +- .../cn/cstn/algorithm/javacpp/api/heu_api.cc | 8 +- .../cn/cstn/algorithm/javacpp/api/heu_api.h | 6 +- .../cstn/algorithm/javacpp/bench/BUILD.bazel | 2 +- .../cn/cstn/algorithm/javacpp/heu/BUILD.bazel | 46 +++++ .../cn/cstn/algorithm/javacpp/heu/phe_kit.cc | 65 +++++++ .../cn/cstn/algorithm/javacpp/heu/phe_kit.h | 37 ++++ .../algorithm/javacpp/heu/phe_kit_test.cc | 33 ++++ .../test/java/cn/cstn/algorithm/AppTest.java | 33 ---- .../java/cn/cstn/algorithm/PheKitTest.java | 43 +++++ 25 files changed, 829 insertions(+), 65 deletions(-) create mode 100644 bazel/cc_static_library.bzl create mode 100644 native/src/main/java/cn/cstn/algorithm/javacpp/heu/Ciphertext.java create mode 100644 native/src/main/java/cn/cstn/algorithm/javacpp/heu/Constants.java create mode 100644 native/src/main/java/cn/cstn/algorithm/javacpp/heu/PheKit.java create mode 100644 native/src/main/java/cn/cstn/algorithm/javacpp/heu/SchemaType.java create mode 100644 native/src/main/java/cn/cstn/algorithm/javacpp/preset/heu.java create mode 100644 native/src/main/native/cn/cstn/algorithm/javacpp/heu/BUILD.bazel create mode 100644 native/src/main/native/cn/cstn/algorithm/javacpp/heu/phe_kit.cc create mode 100644 native/src/main/native/cn/cstn/algorithm/javacpp/heu/phe_kit.h create mode 100644 native/src/main/native/cn/cstn/algorithm/javacpp/heu/phe_kit_test.cc delete mode 100644 native/src/test/java/cn/cstn/algorithm/AppTest.java create mode 100644 native/src/test/java/cn/cstn/algorithm/PheKitTest.java diff --git a/.bazelrc b/.bazelrc index 89e3c4f..3cfc853 100644 --- a/.bazelrc +++ b/.bazelrc @@ -34,7 +34,6 @@ build --linkopt -fvisibility=hidden build --linkopt -fvisibility-inlines-hidden build --linkopt -lm build --linkopt -ldl -build --action_env=PATH=/opt/homebrew/bin:/opt/homebrew/sbin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin # Binary safety flags @@ -42,6 +41,7 @@ build --host_copt=-fPIE build --host_copt=-fstack-protector-strong build:linux --host_copt=-Wl,-z,noexecstack build:macos --host_copt=-Wa,--noexecstack +build:macos --action_env=PATH=/opt/homebrew/bin:/opt/homebrew/sbin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin # platform specific config # Bazel will automatic pick platform config since we have enable_platform_specific_config set @@ -70,6 +70,9 @@ build:ubsan --copt -g build:ubsan --copt -fno-omit-frame-pointer build:ubsan --linkopt -fsanitize=undefined +#build:monolithic --define framework_shared_object=false +build:exp --experimental_cc_static_library + test --keep_going test --test_output=errors test --test_timeout=600 diff --git a/.bazelversion b/.bazelversion index 4be2c72..f22d756 100644 --- a/.bazelversion +++ b/.bazelversion @@ -1 +1 @@ -6.5.0 \ No newline at end of file +6.5.0 diff --git a/MODULE.bazel b/MODULE.bazel index 00bb183..41d301a 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -1,6 +1,15 @@ -############################################################################### -# Bazel now uses Bzlmod by default to manage external dependencies. -# Please consider migrating your external dependencies from WORKSPACE to MODULE.bazel. -# -# For more details, please check https://github.com/bazelbuild/bazel/issues/18958 -############################################################################### +"""Bazel build and test dependencies.""" + +# NOTE: When editing this file, also update the lockfile. +# bazel mod deps --lockfile_mode=update + +module( + name = "algo", + repo_name = "algorithm", +) + +# ========================================= +# Bazel module dependencies +# ========================================= + +#bazel_dep(name = "rules_cc", version = "0.0.13") diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index 21b358e..d62a47c 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -64,19 +64,19 @@ "@@apple_support~//crosstool:setup.bzl%apple_cc_configure_extension": { "general": { "bzlTransitiveDigest": "PjIds3feoYE8SGbbIq2SFTZy3zmxeO2tQevJZNDo7iY=", - "usagesDigest": "aLmqbvowmHkkBPve05yyDNGN7oh7QE9kBADr3QIZTZs=", + "usagesDigest": "+hz7IHWN6A1oVJJWNDB6yZRG+RYhF76wAYItpAeIUIg=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, "envVariables": {}, "generatedRepoSpecs": { - "local_config_apple_cc": { + "local_config_apple_cc_toolchains": { "bzlFile": "@@apple_support~//crosstool:setup.bzl", - "ruleClassName": "_apple_cc_autoconf", + "ruleClassName": "_apple_cc_autoconf_toolchains", "attributes": {} }, - "local_config_apple_cc_toolchains": { + "local_config_apple_cc": { "bzlFile": "@@apple_support~//crosstool:setup.bzl", - "ruleClassName": "_apple_cc_autoconf_toolchains", + "ruleClassName": "_apple_cc_autoconf", "attributes": {} } }, @@ -92,7 +92,7 @@ "@@platforms//host:extension.bzl%host_platform": { "general": { "bzlTransitiveDigest": "xelQcPZH8+tmuOHVjL9vDxMnnQNMlwj0SlvgoqBkm4U=", - "usagesDigest": "meSzxn3DUCcYEhq4HQwExWkWtU4EjriRBQLsZN+Q0SU=", + "usagesDigest": "pCYpDQmqMbmiiPI1p2Kd3VLm5T48rRAht5WdW0X2GlA=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, "envVariables": {}, diff --git a/bazel/algo.bzl b/bazel/algo.bzl index 6e561ab..8a099ce 100644 --- a/bazel/algo.bzl +++ b/bazel/algo.bzl @@ -15,15 +15,20 @@ """ wrapper bazel cc_xx to modify flags. """ +#load("@_builtins//:common/cc/experimental_cc_static_library.bzl", "cc_static_library") +load("//bazel:cc_static_library.bzl", "cc_static_library") load("@yacl//bazel:yacl.bzl", "yacl_cc_library", "yacl_cc_binary", "yacl_cc_test", "yacl_cmake_external", "yacl_configure_make") +load("@rules_cc//cc/private/toolchain:cc_configure.bzl", _cc_configure = "cc_configure") + INCLUDE_COPTS = ["-Inative/src/main/native/cn/cstn/algorithm/javacpp"] + ["-Ibazel/third_party/include"] WARNING_FLAGS = [ "-Wall", "-Wextra", "-Werror", + "-Wno-unknown-pragmas", ] # set `SPDLOG_ACTIVE_LEVEL=1(SPDLOG_LEVEL_DEBUG)` to enable debug level log DEBUG_FLAGS = ["-DSPDLOG_ACTIVE_LEVEL=1", "-O0", "-g"] @@ -38,12 +43,25 @@ def _algo_copts(): "//conditions:default": FAST_FLAGS, }) + WARNING_FLAGS +def add_prefix_for_local_deps(deps=[]): + prefix_deps=[] + if type(deps) == "list": + for dep in deps: + if not dep.startswith(("@", "//", ":")): + dep = "//native/src/main/native/cn/cstn/algorithm/javacpp/" + dep + prefix_deps.append(dep) + return prefix_deps + else: + return deps + def algo_cc_binary( copts = [], + deps = [], linkopts = [], **kargs): yacl_cc_binary( copts = _algo_copts() + copts, + deps = add_prefix_for_local_deps(deps), linkopts = linkopts, **kargs ) @@ -54,7 +72,16 @@ def algo_cc_library( **kargs): yacl_cc_library( copts = _algo_copts() + copts, - deps = deps, + deps = add_prefix_for_local_deps(deps), + **kargs + ) + + +def algo_cc_static_library( + deps = [], + **kargs): + cc_static_library( + deps = add_prefix_for_local_deps(deps), **kargs ) @@ -65,7 +92,7 @@ def algo_cc_test( **kwargs): yacl_cc_test( copts = _algo_copts() + copts, - deps = deps, + deps = add_prefix_for_local_deps(deps), linkopts = linkopts, **kwargs ) diff --git a/bazel/build.sh b/bazel/build.sh index 25770d5..25fd29d 100755 --- a/bazel/build.sh +++ b/bazel/build.sh @@ -8,4 +8,13 @@ echo "Current Directory: $CD" #"$CD"/third_party/build.sh cd .. -bazel build -c opt //... +OPTS="-c opt" +if [[ "$OSTYPE" == "darwin"* ]]; then + echo 6.5.0 > .bazelversion +else + echo 7.4.0 > .bazelversion + OPTS="$OPTS --config=exp" +fi +# --config=monolithic --spawn_strategy=standalone --genrule_strategy=standalone --output_filter=DONT_MATCH_ANYTHING +# shellcheck disable=SC2086 +bazel build $OPTS //native/src/main/native/cn/cstn/algorithm/javacpp/... diff --git a/bazel/cc_static_library.bzl b/bazel/cc_static_library.bzl new file mode 100644 index 0000000..fe78b24 --- /dev/null +++ b/bazel/cc_static_library.bzl @@ -0,0 +1,112 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +""" +`cc_static_library` creates a self-contained static library from the rule's +dependencies. +""" + +load("@rules_cc//cc:action_names.bzl", "CPP_LINK_STATIC_LIBRARY_ACTION_NAME") +load("@rules_cc//cc:toolchain_utils.bzl", "find_cpp_toolchain") + +def _cc_static_library_impl(ctx): + output_file_name = "lib%s.a" % ctx.label.name + output_file = ctx.actions.declare_file(output_file_name) + + input_objects = [] + for dep_target in ctx.attr.deps: + for dep in dep_target[CcInfo].linking_context.linker_inputs.to_list(): + for library in dep.libraries: + input_objects += library.objects + + cc_toolchain = find_cpp_toolchain(ctx) + + feature_configuration = cc_common.configure_features( + ctx = ctx, + cc_toolchain = cc_toolchain, + requested_features = ctx.features, + unsupported_features = ctx.disabled_features, + ) + + library = cc_common.create_library_to_link( + actions = ctx.actions, + feature_configuration = feature_configuration, + cc_toolchain = cc_toolchain, + static_library = output_file, + ) + linker_input = cc_common.create_linker_input( + owner = ctx.label, + libraries = depset(direct = [library]), + ) + + compilation_context = cc_common.create_compilation_context() + linking_context = cc_common.create_linking_context( + linker_inputs = depset(direct = [linker_input]), + ) + + archiver_path = cc_common.get_tool_for_action( + feature_configuration = feature_configuration, + action_name = CPP_LINK_STATIC_LIBRARY_ACTION_NAME, + ) + archiver_variables = cc_common.create_link_variables( + cc_toolchain = cc_toolchain, + feature_configuration = feature_configuration, + output_file = output_file.path, + is_using_linker = False, + ) + command_line = cc_common.get_memory_inefficient_command_line( + feature_configuration = feature_configuration, + action_name = CPP_LINK_STATIC_LIBRARY_ACTION_NAME, + variables = archiver_variables, + ) + + arguments = ctx.actions.args() + arguments.add_all(command_line) + arguments.add_all([obj.path for obj in input_objects]) + + env = cc_common.get_environment_variables( + feature_configuration = feature_configuration, + action_name = CPP_LINK_STATIC_LIBRARY_ACTION_NAME, + variables = archiver_variables, + ) + + ctx.actions.run( + outputs = [output_file], + inputs = depset( + direct = input_objects, + transitive = [cc_toolchain.all_files], + ), + executable = archiver_path, + arguments = [arguments], + mnemonic = "StaticArchive", + progress_message = "Creating static archive %s" % output_file_name, + env = env, + ) + + default_info = DefaultInfo( + files = depset([output_file]), + runfiles = ctx.runfiles(files = [output_file]), + ) + this_cc_info = CcInfo( + compilation_context = compilation_context, + linking_context = linking_context, + ) + deps_cc_infos = [dep[CcInfo] for dep in ctx.attr.deps] + cc_info = cc_common.merge_cc_infos( + cc_infos = [this_cc_info] + deps_cc_infos, + ) + return [default_info, cc_info] + +cc_static_library = rule( + implementation = _cc_static_library_impl, + attrs = { + "deps": attr.label_list(providers = [CcInfo]), + "_cc_toolchain": attr.label( + default = Label("@bazel_tools//tools/cpp:current_cc_toolchain"), + ), + }, + fragments = ["cpp"], + toolchains = ["@bazel_tools//tools/cpp:toolchain_type"], + incompatible_use_toolchain_transition = True, + doc = "Create a self-contained static library from the dependencies.", +) diff --git a/bazel/repositories.bzl b/bazel/repositories.bzl index 3e732dd..72275af 100644 --- a/bazel/repositories.bzl +++ b/bazel/repositories.bzl @@ -17,11 +17,21 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe") def algo_deps(): + #_bazel() _rules_cc() _heu() _spu() _gmp() +def _bazel(): + maybe( + http_archive, + name = "bazel", + sha256 = "2b75cc9aa0a23cf594b96c4ad1cbf5eae9360dba98949b1b42e5d37360333149", + strip_prefix = "bazel-7.4.0", + url = "https://github.com/bazelbuild/bazel/archive/refs/tags/7.4.0.tar.gz", + ) + def _rules_cc(): maybe( http_archive, @@ -29,6 +39,9 @@ def _rules_cc(): sha256 = "2037875b9a4456dce4a79d112a8ae885bbc4aad968e6587dca6e64f3a0900cdf", strip_prefix = "rules_cc-0.0.9", url = "https://github.com/bazelbuild/rules_cc/releases/download/0.0.9/rules_cc-0.0.9.tar.gz", + #sha256 = "906e89286acc67c20819c3c88b3283de0d5868afda33635d70acae0de9777bb7", + #strip_prefix = "rules_cc-0.0.14", + #url = "https://github.com/bazelbuild/rules_cc/releases/download/0.0.14/rules_cc-0.0.14.tar.gz", ) def _heu(): diff --git a/bazel/third_party/gmp.BUILD b/bazel/third_party/gmp.BUILD index 1c4f0f7..846bbb9 100644 --- a/bazel/third_party/gmp.BUILD +++ b/bazel/third_party/gmp.BUILD @@ -27,9 +27,9 @@ filegroup( configure_make( name = "gmp", args = ["-j 4"], - configure_command = "Configure", + configure_command = "configure", configure_in_place = True, - configure_options = [ "--enable-fat", "--enable-cxx", "CFLAGS=-Dredacted" ], + configure_options = [ "--enable-fat" ], env = select({ "@platforms//os:macos": { "AR": "", diff --git a/native/pom.xml b/native/pom.xml index c18a5db..e3db23b 100644 --- a/native/pom.xml +++ b/native/pom.xml @@ -34,7 +34,7 @@ windows-x86_64${javacpp.platform.extension} - ${project.build.directory}/dist/ + ${project.build.directory}/dist true ${project.basedir}/../bazel-${project.parent.artifactId} @@ -55,6 +55,10 @@ org.scijava native-lib-loader + + org.projectlombok + lombok + junit @@ -146,17 +150,95 @@ ${project.build.outputDirectory} + + -fPIE + + true ${dist.download.folder}/include/ ${native.include.src.dir} ${native.include.external.dir}/com_dtera_heu + ${native.include.external.dir}/yacl + ${native.include.external.dir}/com_google_absl + ${native.lib.external.dir}/com_github_fmtlib_fmt/fmtlib/include + ${native.lib.external.dir}/com_github_gabime_spdlog/spdlog/include + ${native.lib.external.dir}/com_github_msgpack_msgpack/msgpack/include + ${native.lib.external.dir}/com_github_libtom_libtommath/libtommath/include + ${dist.download.folder}/ ${dist.download.folder}/lib/ ${native.lib.src.dir} - ${native.lib.external.dir}/com_dtera_heu + ${native.lib.src.dir}/heu + + ${project.basedir}/../ diff --git a/native/src/main/java/cn/cstn/algorithm/javacpp/heu/Ciphertext.java b/native/src/main/java/cn/cstn/algorithm/javacpp/heu/Ciphertext.java new file mode 100644 index 0000000..fbec30f --- /dev/null +++ b/native/src/main/java/cn/cstn/algorithm/javacpp/heu/Ciphertext.java @@ -0,0 +1,23 @@ +package cn.cstn.algorithm.javacpp.heu; + +import cn.cstn.algorithm.javacpp.preset.heu; +import org.bytedeco.javacpp.Loader; +import org.bytedeco.javacpp.Pointer; +import org.bytedeco.javacpp.annotation.Namespace; +import org.bytedeco.javacpp.annotation.Properties; + +@SuppressWarnings("unused") +@Properties(inherit = heu.class) +@Namespace("heu::lib::phe") +public class Ciphertext extends Pointer { + static { + Loader.load(); + } + + public Ciphertext() { + allocate(); + } + + private native void allocate(); + +} \ No newline at end of file diff --git a/native/src/main/java/cn/cstn/algorithm/javacpp/heu/Constants.java b/native/src/main/java/cn/cstn/algorithm/javacpp/heu/Constants.java new file mode 100644 index 0000000..7517932 --- /dev/null +++ b/native/src/main/java/cn/cstn/algorithm/javacpp/heu/Constants.java @@ -0,0 +1,59 @@ +package cn.cstn.algorithm.javacpp.heu; + +@SuppressWarnings("ALL") +public class Constants { + public static final String[] links = {"phe_kit", "decryptor", "encryptor", "evaluator", "key_generator", "public_key", "secret_key", "ciphertext", + "lookup_table", "paillier_float", "pb_utils", "phe", "schema", "serializable_types", "legacy_coding", "blake3_c", + "spdlog", "gflags", "benchmark", "mcl", "libsodium", "tommath", "FourQ", "crypto", "ssl", "base", "log_severity", + "malloc_internal", "raw_logging_internal", "spinlock_wait", "throw_delegate", "debugging_internal", + "decode_rust_punycode", "demangle_internal", "demangle_rust", "stacktrace", "symbolize", "utf8_for_code_point", + "int128", "internal", "string_view", "strings", "bad_optional_access", "gtest", "gtest_main", "protobuf", + "protobuf_lite", "gmp", "secparam", "buffer", "exception", "int128", "curve_meta", "ec_point", "spi", + "FourQ_group", "ed25519_group", "sodium_group", "x25519_group", "util", "common", "montgomery", "weierstrass", + "blake3", "hash_utils", "ssl_hash", "montgomery_math", "mpint", "tommath_ext_features", "tommath_ext_types", + "parallel", "thread_pool", "item", "arg_impl", "argument", "util", "zlib"}; + + public static final String[] linkPaths = {"gmp/lib/libgmp.a", "libgflags.a", "spdlog/lib/libspdlog.a", "liblibsodium.a", + "libzlib.a", "libbenchmark.a", "interconnection/runtime/libphe.a", "openssl/lib/libcrypto.a", "openssl/lib/libssl.a", + "mcl-cmake/lib/libmcl.a", "libprotobuf.a", "libprotobuf_lite.a", "yacl/crypto/hash/libblake3.a", + "yacl/crypto/hash/libssl_hash.a", "yacl/crypto/hash/libhash_utils.a", "yacl/crypto/ecc/toy/libmontgomery.a", + "yacl/crypto/ecc/toy/libcommon.a", "yacl/crypto/ecc/toy/libweierstrass.a", "yacl/crypto/ecc/libcurve_meta.a", + "yacl/crypto/ecc/FourQlib/libFourQ_group.a", "yacl/crypto/ecc/mcl/libutil.a", "yacl/crypto/ecc/libspi.a", + "yacl/crypto/ecc/libec_point.a", "yacl/crypto/ecc/libsodium/libed25519_group.a", + "yacl/crypto/ecc/libsodium/libx25519_group.a", "yacl/crypto/ecc/libsodium/libsodium_group.a", "yacl/libsecparam.a", + "yacl/utils/libparallel.a", "yacl/utils/libthread_pool.a", "yacl/utils/spi/libitem.a", + "yacl/utils/spi/argument/libargument.a", "yacl/utils/spi/argument/libutil.a", "yacl/utils/spi/argument/libarg_impl.a", + "yacl/math/mpint/libmpint.a", "yacl/math/mpint/libmontgomery_math.a", "yacl/math/mpint/libtommath_ext_types.a", + "yacl/math/mpint/libtommath_ext_features.a", "yacl/base/libexception.a", "yacl/base/libint128.a", + "yacl/base/libbuffer.a", "libblake3_c.a", "libgtest_main.a", "libgtest.a", "FourQlib/lib/libFourQ.a", + "libtommath/lib/libtommath.a", "heu/library/algorithms/dgk/libkey_generator.a", + "heu/library/algorithms/dgk/libsecret_key.a", "heu/library/algorithms/dgk/libdecryptor.a", + "heu/library/algorithms/dgk/libpublic_key.a", "heu/library/algorithms/dgk/libencryptor.a", + "heu/library/algorithms/dgk/libevaluator.a", "heu/library/algorithms/elgamal/libciphertext.a", + "heu/library/algorithms/elgamal/libkey_generator.a", "heu/library/algorithms/elgamal/libsecret_key.a", + "heu/library/algorithms/elgamal/utils/liblookup_table.a", "heu/library/algorithms/elgamal/libdecryptor.a", + "heu/library/algorithms/elgamal/libpublic_key.a", "heu/library/algorithms/elgamal/libencryptor.a", + "heu/library/algorithms/elgamal/libevaluator.a", "heu/library/algorithms/mock/libkey_generator.a", + "heu/library/algorithms/mock/libdecryptor.a", "heu/library/algorithms/mock/libencryptor.a", + "heu/library/algorithms/mock/libevaluator.a", "heu/library/algorithms/paillier_float/libpaillier_float.a", + "heu/library/algorithms/dj/libkey_generator.a", "heu/library/algorithms/dj/libsecret_key.a", + "heu/library/algorithms/dj/libdecryptor.a", "heu/library/algorithms/dj/libpublic_key.a", + "heu/library/algorithms/dj/libencryptor.a", "heu/library/algorithms/dj/libevaluator.a", + "heu/library/algorithms/paillier_zahlen/libkey_generator.a", "heu/library/algorithms/paillier_zahlen/libsecret_key.a", + "heu/library/algorithms/paillier_zahlen/libdecryptor.a", "heu/library/algorithms/paillier_zahlen/libpublic_key.a", + "heu/library/algorithms/paillier_zahlen/libencryptor.a", "heu/library/algorithms/paillier_zahlen/libevaluator.a", + "heu/library/algorithms/paillier_ic/libciphertext.a", "heu/library/algorithms/paillier_ic/libkey_generator.a", + "heu/library/algorithms/paillier_ic/libpb_utils.a", "heu/library/algorithms/paillier_ic/libdecryptor.a", + "heu/library/algorithms/paillier_ic/libpublic_key.a", "heu/library/algorithms/paillier_ic/libencryptor.a", + "heu/library/algorithms/paillier_ic/libevaluator.a", "heu/library/algorithms/ou/libkey_generator.a", + "heu/library/algorithms/ou/libdecryptor.a", "heu/library/algorithms/ou/libpublic_key.a", + "heu/library/algorithms/ou/libencryptor.a", "heu/library/algorithms/ou/libevaluator.a", "heu/library/phe/libphe.a", + "heu/library/phe/encoding/liblegacy_coding.a", "heu/library/phe/libdecryptor.a", "heu/library/phe/libencryptor.a", + "heu/library/phe/libevaluator.a", "heu/library/phe/base/libschema.a", "heu/library/phe/base/libserializable_types.a", + "absl/strings/libstring_view.a", "absl/strings/libinternal.a", "absl/strings/libstrings.a", + "absl/types/libbad_optional_access.a", "absl/debugging/libstacktrace.a", "absl/debugging/libdemangle_rust.a", + "absl/debugging/libutf8_for_code_point.a", "absl/debugging/libdecode_rust_punycode.a", + "absl/debugging/libdebugging_internal.a", "absl/debugging/libdemangle_internal.a", "absl/debugging/libsymbolize.a", + "absl/numeric/libint128.a", "absl/base/libthrow_delegate.a", "absl/base/liblog_severity.a", "absl/base/libbase.a", + "absl/base/libmalloc_internal.a", "absl/base/libraw_logging_internal.a", "absl/base/libspinlock_wait.a"}; +} diff --git a/native/src/main/java/cn/cstn/algorithm/javacpp/heu/PheKit.java b/native/src/main/java/cn/cstn/algorithm/javacpp/heu/PheKit.java new file mode 100644 index 0000000..4bf0df1 --- /dev/null +++ b/native/src/main/java/cn/cstn/algorithm/javacpp/heu/PheKit.java @@ -0,0 +1,39 @@ +package cn.cstn.algorithm.javacpp.heu; + +import cn.cstn.algorithm.javacpp.preset.heu; +import org.bytedeco.javacpp.Loader; +import org.bytedeco.javacpp.Pointer; +import org.bytedeco.javacpp.annotation.ByRef; +import org.bytedeco.javacpp.annotation.Const; +import org.bytedeco.javacpp.annotation.Properties; + +@SuppressWarnings("unused") +@Properties(inherit = heu.class) +public class PheKit extends Pointer { + static { + Loader.load(); + } + + public PheKit(SchemaType schemaType, int keySize, int scale) { + allocate(schemaType.value, keySize, scale); + } + + public PheKit(SchemaType schemaType, int keySize) { + this(schemaType, keySize, (int) 1e6); + } + + public PheKit(SchemaType schemaType) { + this(schemaType, 2048); + } + + private native void allocate(int schemaType, int keySize, int scale); + + public native Ciphertext encrypt(double data); + + public native double decrypt(@Const @ByRef Ciphertext ct); + + public native Ciphertext add(@Const @ByRef Ciphertext ct1, @Const @ByRef Ciphertext ct2); + + public native void addInplace(@ByRef Ciphertext ct1, @Const @ByRef Ciphertext ct2); + +} \ No newline at end of file diff --git a/native/src/main/java/cn/cstn/algorithm/javacpp/heu/SchemaType.java b/native/src/main/java/cn/cstn/algorithm/javacpp/heu/SchemaType.java new file mode 100644 index 0000000..5aeaf31 --- /dev/null +++ b/native/src/main/java/cn/cstn/algorithm/javacpp/heu/SchemaType.java @@ -0,0 +1,20 @@ +package cn.cstn.algorithm.javacpp.heu; + +import lombok.RequiredArgsConstructor; + +@RequiredArgsConstructor +public enum SchemaType { + Mock(0), + OU(1), + IPCL(2), + GPaillier(3), + ZPaillier(4), + FPaillier(5), + IcPaillier(6), + ClustarFPGA(7), + ElGamal(8), + DGK(10), + DJ(11); + + public final int value; +} diff --git a/native/src/main/java/cn/cstn/algorithm/javacpp/preset/heu.java b/native/src/main/java/cn/cstn/algorithm/javacpp/preset/heu.java new file mode 100644 index 0000000..f6f6839 --- /dev/null +++ b/native/src/main/java/cn/cstn/algorithm/javacpp/preset/heu.java @@ -0,0 +1,177 @@ +package cn.cstn.algorithm.javacpp.preset; + +import org.bytedeco.javacpp.ClassProperties; +import org.bytedeco.javacpp.LoadEnabled; +import org.bytedeco.javacpp.Loader; +import org.bytedeco.javacpp.annotation.NoException; +import org.bytedeco.javacpp.annotation.Platform; +import org.bytedeco.javacpp.annotation.Properties; +import org.bytedeco.javacpp.tools.InfoMap; +import org.bytedeco.javacpp.tools.InfoMapper; + +import java.util.List; + +@Properties( + value = { + @Platform( + value = {"linux", "macosx", "windows"}, + compiler = "cpp17", + include = { + "heu/phe_kit.h", + }, + link = {"phe_kit_all"}, + define = {"MSGPACK_NO_BOOST", "SPDLOG_FMT_EXTERNAL", "SPDLOG_NO_TLS"} + ), + @Platform( + value = {"linux", "macosx", "windows"}, + extension = {"-gpu"}) + }) +@NoException +public class heu implements LoadEnabled, InfoMapper { + @Override + public void map(InfoMap infoMap) { + + } + + @Override + public void init(ClassProperties properties) { + String platform = properties.getProperty("platform"); + String extension = properties.getProperty("platform.extension"); + List preloads = properties.get("platform.preload"); + List resources = properties.get("platform.preloadresource"); + List preloadpaths = properties.get("platform.preloadpath"); + + String vcredistdir = System.getenv("VCToolsRedistDir"); + if (vcredistdir != null && !vcredistdir.isEmpty()) { + switch (platform) { + case "windows-x86": + preloadpaths.add(0, vcredistdir + "\\x86\\Microsoft.VC142.CRT"); + preloadpaths.add(1, vcredistdir + "\\x86\\Microsoft.VC142.OpenMP"); + preloadpaths.add(2, vcredistdir + "\\x86\\Microsoft.VC141.CRT"); + preloadpaths.add(3, vcredistdir + "\\x86\\Microsoft.VC141.OpenMP"); + break; + case "windows-x86_64": + preloadpaths.add(0, vcredistdir + "\\x64\\Microsoft.VC142.CRT"); + preloadpaths.add(1, vcredistdir + "\\x64\\Microsoft.VC142.OpenMP"); + preloadpaths.add(2, vcredistdir + "\\x64\\Microsoft.VC141.CRT"); + preloadpaths.add(3, vcredistdir + "\\x64\\Microsoft.VC141.OpenMP"); + break; + default: + // not Windows + } + } + + // Only apply this at load time + if (!Loader.isLoadLibraries()) { + return; + } + + // Let users enable loading of the full version of MKL + String load = + System.getProperty( + "org.bytedeco.openblas.load", System.getProperty("org.bytedeco.mklml.load", "")) + .toLowerCase(); + + int i = 0; + if (load.equals("mkl") || load.equals("mkl_rt")) { + String[] libs = { + "iomp5", + "libiomp5md", + "mkl_core", + "mkl_avx", + "mkl_avx2", + "mkl_avx512", + "mkl_avx512_mic", + "mkl_def", + "mkl_mc", + "mkl_mc3", + "mkl_intel_lp64", + "mkl_intel_thread", + "mkl_gnu_thread", + "mkl_rt" + }; + for (i = 0; i < libs.length; i++) { + preloads.add(i, libs[i] + "#" + libs[i]); + } + load = "mkl_rt"; + resources.add("/org/bytedeco/mkl/"); + } + + if (!load.isEmpty()) { + if (platform.startsWith("linux")) { + preloads.add(i, load + "#mklml_intel"); + } else if (platform.startsWith("macosx")) { + preloads.add(i, load + "#mklml"); + } else if (platform.startsWith("windows")) { + preloads.add(i, load + "#mklml"); + } + } + + // Only apply this at load time since we don't want to copy the CUDA libraries here + if (!Loader.isLoadLibraries() || extension == null || !extension.endsWith("-gpu")) { + return; + } + String[] libs = { + "cudart", + "cublasLt", + "cublas", + "cufft", + "curand", + "cusolver", + "cusparse", + "cudnn", + "nccl", + "nvrtc", + "myelin", + "nvinfer", + "cudnn_ops_infer", + "cudnn_ops_train", + "cudnn_adv_infer", + "cudnn_adv_train", + "cudnn_cnn_infer", + "cudnn_cnn_train" + }; + for (String lib : libs) { + if (platform.startsWith("linux")) { + lib += + lib.startsWith("cudnn") + ? "@.8" + : lib.equals("nccl") + ? "@.2" + : lib.equals("myelin") + ? "@.1" + : lib.equals("nvinfer") + ? "@.7" + : lib.equals("cufft") || lib.equals("curand") || lib.equals("cusolver") + ? "@.10" + : lib.equals("cudart") + ? "@.11.0" + : lib.equals("nvrtc") ? "@.11.0" : "@.11"; + } else if (platform.startsWith("windows")) { + lib += + lib.startsWith("cudnn") + ? "64_8" + : lib.equals("nccl") + ? "64_2" + : lib.equals("myelin") + ? "64_1" + : lib.equals("nvinfer") + ? "64_7" + : lib.equals("cufft") || lib.equals("curand") || lib.equals("cusolver") + ? "64_10" + : lib.equals("cudart") + ? "64_110" + : lib.equals("nvrtc") ? "64_110_0" : "64_11"; + } else { + continue; // no CUDA + } + if (!preloads.contains(lib)) { + preloads.add(i++, lib); + } + } + if (i > 0) { + resources.add("/org/bytedeco/cuda/"); + resources.add("/org/bytedeco/tensorrt/"); + } + } +} diff --git a/native/src/main/native/cn/cstn/algorithm/javacpp/api/BUILD.bazel b/native/src/main/native/cn/cstn/algorithm/javacpp/api/BUILD.bazel index a17d382..4a1f4a4 100644 --- a/native/src/main/native/cn/cstn/algorithm/javacpp/api/BUILD.bazel +++ b/native/src/main/native/cn/cstn/algorithm/javacpp/api/BUILD.bazel @@ -19,11 +19,11 @@ test_suite( ) algo_cc_library( - name = "api", + name = "heu_api", srcs = ["heu_api.cc"], hdrs = ["heu_api.h"], deps = [ - "//native/src/main/native/cn/cstn/algorithm/javacpp/util:utils", + "util:utils", "@com_dtera_heu//heu/library/phe", ], ) @@ -32,6 +32,6 @@ algo_cc_test( name = "heu_test", srcs = ["heu_test.cc"], deps = [ - ":api", + ":heu_api", ], ) diff --git a/native/src/main/native/cn/cstn/algorithm/javacpp/api/heu_api.cc b/native/src/main/native/cn/cstn/algorithm/javacpp/api/heu_api.cc index de64e50..25e8077 100644 --- a/native/src/main/native/cn/cstn/algorithm/javacpp/api/heu_api.cc +++ b/native/src/main/native/cn/cstn/algorithm/javacpp/api/heu_api.cc @@ -379,7 +379,7 @@ HEU_DLL BlockedSpace2d blocked2D( len, [&](size_t i) { return row_size[i]; }, grain_size); ParallelFor2d(blocked2D, n_threads, [&](size_t i, const Range1d &r) { - for (int j = r.begin(); j < r.end(); ++j) { + for (unsigned long j = r.begin(); j < r.end(); ++j) { evaluator->AddInplace(&cs1[i], cs2[i][j]); } }); @@ -402,7 +402,7 @@ HEU_DLL [[maybe_unused]] int AddCiphersInplaceAxis0__( API_DHEKIT_HANDLE({ auto evaluator = dhekit->GetEvaluator(); ParallelFor(cs1.size(), n_threads, [&](int i) { - for (int j = 0; j < cs2[i].size(); ++j) { + for (unsigned long j = 0; j < cs2[i].size(); ++j) { evaluator->AddInplace(cs1[i], *cs2[i][j]); } }); @@ -426,7 +426,7 @@ HEU_DLL [[maybe_unused]] int AddCiphersInplaceAxis0___( API_DHEKIT_HANDLE({ auto evaluator = dhekit->GetEvaluator(); ParallelFor(len, n_threads, [&](int i) { - for (int j = 0; j < cs2[i].size(); ++j) { + for (unsigned long j = 0; j < cs2[i].size(); ++j) { evaluator->AddInplace(&cs1[i], *cs2[i][j]); } }); @@ -576,7 +576,7 @@ int SumCiphers_(DestinationHeKitHandle handle, heu::lib::phe::Ciphertext *out, int32_t min_work_size, const int32_t n_threads) { CheckCall(DestinationHeKitEncrypt(handle, 0, out, 1), "DHeKitEncrypt"); - auto len = ciphers.size(); + int32_t len = ciphers.size(); API_DHEKIT_HANDLE({ auto evaluator = dhekit->GetEvaluator(); auto add_op = [&](int size, std::vector &cs) { diff --git a/native/src/main/native/cn/cstn/algorithm/javacpp/api/heu_api.h b/native/src/main/native/cn/cstn/algorithm/javacpp/api/heu_api.h index 3f488bb..4d31146 100644 --- a/native/src/main/native/cn/cstn/algorithm/javacpp/api/heu_api.h +++ b/native/src/main/native/cn/cstn/algorithm/javacpp/api/heu_api.h @@ -486,7 +486,7 @@ template Encrypt(encoder.Encode(data[i][j])); } }); @@ -777,7 +777,7 @@ template Encrypt(encoder.Encode(data[i][j])); } }); @@ -821,7 +821,7 @@ void DHeKitCiphersInit_(DestinationHeKitHandle handle, std::vector &ciphers) { CheckCall(DestinationHeKitEncrypt(handle, 0, ciphers[0], 1), "DHeKitEncrypt"); auto buf = ciphers[0]->Serialize(); - for (int i = 1; i < ciphers.size(); ++i) { + for (unsigned long i = 1; i < ciphers.size(); ++i) { ciphers[i]->Deserialize(buf); } } diff --git a/native/src/main/native/cn/cstn/algorithm/javacpp/bench/BUILD.bazel b/native/src/main/native/cn/cstn/algorithm/javacpp/bench/BUILD.bazel index c0b774c..970b26c 100644 --- a/native/src/main/native/cn/cstn/algorithm/javacpp/bench/BUILD.bazel +++ b/native/src/main/native/cn/cstn/algorithm/javacpp/bench/BUILD.bazel @@ -15,7 +15,7 @@ load("//bazel:algo.bzl", "algo_cc_binary") algo_cc_binary( - name = "phe", + name = "phe_bench", srcs = ["phe_bench.cc"], deps = [ "@com_dtera_heu//heu/library/phe", diff --git a/native/src/main/native/cn/cstn/algorithm/javacpp/heu/BUILD.bazel b/native/src/main/native/cn/cstn/algorithm/javacpp/heu/BUILD.bazel new file mode 100644 index 0000000..ecdf508 --- /dev/null +++ b/native/src/main/native/cn/cstn/algorithm/javacpp/heu/BUILD.bazel @@ -0,0 +1,46 @@ +# Copyright 2024 dterazhao, Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load("//bazel:algo.bzl", "algo_cc_library", "algo_cc_static_library", "algo_cc_test") + +test_suite( + name = "phe_tests", +) + +algo_cc_library( + name = "phe_kit", + srcs = ["phe_kit.cc"], + hdrs = ["phe_kit.h"], + deps = [ + "util:utils", + "@com_dtera_heu//heu/library/phe", + ], +) + +algo_cc_static_library( + name = "phe_kit_all", + deps = [ + ":phe_kit", + ], + visibility = ["//visibility:public"], +) + +algo_cc_test( + name = "phe_test", + srcs = ["phe_kit_test.cc"], + deps = [ + ":phe_kit", + "util:stopwatch", + ], +) diff --git a/native/src/main/native/cn/cstn/algorithm/javacpp/heu/phe_kit.cc b/native/src/main/native/cn/cstn/algorithm/javacpp/heu/phe_kit.cc new file mode 100644 index 0000000..d68fce3 --- /dev/null +++ b/native/src/main/native/cn/cstn/algorithm/javacpp/heu/phe_kit.cc @@ -0,0 +1,65 @@ +// +// Created by 2024 dterazhao. +// + +#include "heu/phe_kit.h" + +PheKit::PheKit(SchemaType schema_type, size_t key_size, int64_t scale) { + he_kit_ = std::make_shared(schema_type, key_size); + encryptor_ = he_kit_->GetEncryptor(); + decryptor_ = he_kit_->GetDecryptor(); + evaluator_ = he_kit_->GetEvaluator(); + encoder_ = std::make_shared( + he_kit_->GetEncoder(scale)); + batch_encoder_ = std::make_shared( + he_kit_->GetEncoder(scale)); +} + +PheKit::PheKit(uint8_t schema_type, size_t key_size, int64_t scale) + : PheKit((SchemaType)schema_type, key_size, scale) {} + +PheKit::PheKit(yacl::ByteContainerView pk_buffer) { + dhe_kit_ = std::make_shared(pk_buffer); + encryptor_ = dhe_kit_->GetEncryptor(); + evaluator_ = dhe_kit_->GetEvaluator(); +} + +bool PheKit::hasSecretKey() const { return has_secret_key; } + +const std::shared_ptr &PheKit::getPublicKey() const { + if (has_secret_key) { + return he_kit_->GetPublicKey(); + } + return dhe_kit_->GetPublicKey(); +} + +const std::shared_ptr &PheKit::getSecretKey() const { + if (has_secret_key) { + return he_kit_->GetSecretKey(); + } + throw std::invalid_argument("have no secret key"); +} + +Ciphertext *PheKit::encrypt(double data) { + auto pt = encoder_->Encode(data); + auto res = new Ciphertext(); + *res = encryptor_->Encrypt(pt); + return res; +} + +double PheKit::decrypt(const Ciphertext &ct) { + heu::lib::phe::Plaintext out; + decryptor_->Decrypt(ct, &out); + return encoder_->Decode(out); +} + +Ciphertext *PheKit::add(const Ciphertext &ct1, const Ciphertext &ct2) { + auto res = new Ciphertext(); + *res = ct1; + evaluator_->AddInplace(res, ct2); + return res; +} + +void PheKit::addInplace(Ciphertext &ct1, const Ciphertext &ct2) { + evaluator_->AddInplace(&ct1, ct2); +} diff --git a/native/src/main/native/cn/cstn/algorithm/javacpp/heu/phe_kit.h b/native/src/main/native/cn/cstn/algorithm/javacpp/heu/phe_kit.h new file mode 100644 index 0000000..23eb133 --- /dev/null +++ b/native/src/main/native/cn/cstn/algorithm/javacpp/heu/phe_kit.h @@ -0,0 +1,37 @@ +// +// Created by 2024 dterazhao. +// + +#pragma once + +#include "heu/library/phe/encoding/encoding.h" +#include "heu/library/phe/phe.h" + +using SchemaType = heu::lib::phe::SchemaType; +using Ciphertext = heu::lib::phe::Ciphertext; + +class PheKit { + private: + std::shared_ptr he_kit_; + std::shared_ptr dhe_kit_; + std::shared_ptr encryptor_; + std::shared_ptr decryptor_; + std::shared_ptr evaluator_; + std::shared_ptr encoder_; + std::shared_ptr batch_encoder_; + bool has_secret_key; + + public: + PheKit(SchemaType schema_type, size_t key_size = 2048, int64_t scale = 1e6); + PheKit(uint8_t schema_type, size_t key_size = 2048, int64_t scale = 1e6); + PheKit(yacl::ByteContainerView pk_buffer); + + bool hasSecretKey() const; + const std::shared_ptr &getPublicKey() const; + const std::shared_ptr &getSecretKey() const; + + Ciphertext *encrypt(double data); + double decrypt(const Ciphertext &ct); + Ciphertext *add(const Ciphertext &ct1, const Ciphertext &ct2); + void addInplace(Ciphertext &ct1, const Ciphertext &ct2); +}; diff --git a/native/src/main/native/cn/cstn/algorithm/javacpp/heu/phe_kit_test.cc b/native/src/main/native/cn/cstn/algorithm/javacpp/heu/phe_kit_test.cc new file mode 100644 index 0000000..dd9eb35 --- /dev/null +++ b/native/src/main/native/cn/cstn/algorithm/javacpp/heu/phe_kit_test.cc @@ -0,0 +1,33 @@ +// +// Created by 2024 dterazhao. +// + +#include + +#include + +#include "util/stopwatch.hpp" + +#include "heu/phe_kit.h" + +TEST(phe_kit, t1) { + StopWatch sw; + + sw.Mark("init"); + PheKit pheKit(1); + sw.PrintWithMills("init"); + std::cout << pheKit.getPublicKey()->ToString() << std::endl; + + double a = 2.36, b = 5.12; + sw.Mark("encrypt"); + auto ct1 = pheKit.encrypt(a); + auto ct2 = pheKit.encrypt(b); + sw.PrintWithMills("encrypt"); + sw.Mark("add"); + auto ct = pheKit.add(*ct1, *ct2); + sw.PrintWithMills("add"); + sw.Mark("decrypt"); + auto res = pheKit.decrypt(*ct); + sw.PrintWithMills("decrypt"); + std::cout << "real: " << a + b << ", res: " << res << std::endl; +} diff --git a/native/src/test/java/cn/cstn/algorithm/AppTest.java b/native/src/test/java/cn/cstn/algorithm/AppTest.java deleted file mode 100644 index 835ce04..0000000 --- a/native/src/test/java/cn/cstn/algorithm/AppTest.java +++ /dev/null @@ -1,33 +0,0 @@ -package cn.cstn.algorithm; - -import junit.framework.Test; -import junit.framework.TestCase; -import junit.framework.TestSuite; - -/** - * Unit test for simple App. - */ -public class AppTest extends TestCase { - /** - * Create the test case - * - * @param testName name of the test case - */ - public AppTest(String testName) { - super(testName); - } - - /** - * @return the suite of tests being tested - */ - public static Test suite() { - return new TestSuite(AppTest.class); - } - - /** - * Rigourous Test :-) - */ - public void testApp() { - assertTrue(true); - } -} diff --git a/native/src/test/java/cn/cstn/algorithm/PheKitTest.java b/native/src/test/java/cn/cstn/algorithm/PheKitTest.java new file mode 100644 index 0000000..31e96ce --- /dev/null +++ b/native/src/test/java/cn/cstn/algorithm/PheKitTest.java @@ -0,0 +1,43 @@ +package cn.cstn.algorithm; + +import cn.cstn.algorithm.javacpp.heu.Ciphertext; +import cn.cstn.algorithm.javacpp.heu.PheKit; +import cn.cstn.algorithm.javacpp.heu.SchemaType; +import junit.framework.Test; +import junit.framework.TestCase; +import junit.framework.TestSuite; + +/** + * Unit test for simple App. + */ +public class PheKitTest extends TestCase { + /** + * Create the test case + * + * @param testName name of the test case + */ + public PheKitTest(String testName) { + super(testName); + } + + /** + * @return the suite of tests being tested + */ + @SuppressWarnings({"TextBlockMigration"}) + public static Test suite() { + return new TestSuite(PheKitTest.class); + } + + /** + * Rigourous Test :-) + */ + public void test1() { + try (PheKit pheKit = new PheKit(SchemaType.OU)) { + Ciphertext ct1 = pheKit.encrypt(2); + Ciphertext ct2 = pheKit.encrypt(3); + Ciphertext addRes = pheKit.add(ct1, ct2); + System.out.printf("addRes: %f", pheKit.decrypt(addRes)); + + } + } +}