From 5ea2a8d4df6e76c43efdb599a7bfff9dcafd90c4 Mon Sep 17 00:00:00 2001 From: Yingchun Lai Date: Tue, 12 Dec 2023 14:51:40 +0800 Subject: [PATCH] feat(snappy/lz4/zstd): Build snappy/lz4/zstd as a thirdparty libraries (#1709) Build snappy/lz4/zstd as thirdparty libraries, and do not dependent on the library in system. It would be more flexible to upgrade the libraries version no matter what the library versions in system are. Another issue this patch resolve is https://github.com/apache/incubator-pegasus/issues/1632. --- .github/workflows/lint_and_test_cpp.yaml | 3 - .licenserc.yaml | 1 + cmake_modules/BaseFunctions.cmake | 8 +- docker/pegasus-build-env/centos7/Dockerfile | 4 - .../pegasus-build-env/ubuntu1804/Dockerfile | 4 - .../pegasus-build-env/ubuntu2004/Dockerfile | 4 - .../pegasus-build-env/ubuntu2204/Dockerfile | 4 - src/aio/CMakeLists.txt | 7 +- src/aio/test/CMakeLists.txt | 12 ++- src/block_service/hdfs/CMakeLists.txt | 7 +- src/block_service/local/CMakeLists.txt | 6 +- src/block_service/test/CMakeLists.txt | 25 ++++--- src/client/test/CMakeLists.txt | 15 ++-- src/common/test/CMakeLists.txt | 5 +- src/failure_detector/test/CMakeLists.txt | 19 +++-- src/geo/bench/CMakeLists.txt | 8 +- src/http/test/CMakeLists.txt | 14 ++-- src/meta/CMakeLists.txt | 31 ++++---- src/nfs/test/CMakeLists.txt | 12 ++- src/perf_counter/test/CMakeLists.txt | 10 ++- src/replica/CMakeLists.txt | 29 +++---- src/replica/backup/test/CMakeLists.txt | 5 +- src/replica/bulk_load/test/CMakeLists.txt | 7 +- src/replica/duplication/test/CMakeLists.txt | 5 +- src/replica/storage/simple_kv/CMakeLists.txt | 11 ++- .../storage/simple_kv/test/CMakeLists.txt | 28 ++++--- src/replica/test/CMakeLists.txt | 34 +++++---- src/runtime/ranger/CMakeLists.txt | 14 ++-- src/runtime/test/CMakeLists.txt | 15 ++-- src/server/CMakeLists.txt | 32 ++++---- src/server/test/CMakeLists.txt | 7 +- src/shell/CMakeLists.txt | 40 +++++----- src/test/bench_test/CMakeLists.txt | 8 +- .../function_test/bulk_load/CMakeLists.txt | 25 ++++--- src/test_util/CMakeLists.txt | 7 +- src/utils/CMakeLists.txt | 7 +- src/utils/long_adder_bench/CMakeLists.txt | 8 +- src/utils/test/CMakeLists.txt | 16 ++-- .../test/nth_element_bench/CMakeLists.txt | 8 +- src/zookeeper/test/CMakeLists.txt | 17 +++-- thirdparty/CMakeLists.txt | 75 ++++++++++++++++++- .../fix_snappy-Wsign-compare-warning.patch | 26 +++++++ 42 files changed, 415 insertions(+), 208 deletions(-) create mode 100644 thirdparty/fix_snappy-Wsign-compare-warning.patch diff --git a/.github/workflows/lint_and_test_cpp.yaml b/.github/workflows/lint_and_test_cpp.yaml index a9b7724ebf..85b8eeb09d 100644 --- a/.github/workflows/lint_and_test_cpp.yaml +++ b/.github/workflows/lint_and_test_cpp.yaml @@ -673,9 +673,6 @@ jobs: run: | # Preinstalled softwares: https://github.com/actions/virtual-environments/blob/main/images/macos/macos-12-Readme.md brew install ccache - brew install snappy - brew install lz4 - brew install zstd brew install openssl@1.1 - uses: actions/checkout@v3 - name: Setup cache diff --git a/.licenserc.yaml b/.licenserc.yaml index 3ddf87be61..1c58047590 100644 --- a/.licenserc.yaml +++ b/.licenserc.yaml @@ -71,6 +71,7 @@ header: - 'thirdparty/fix_libevent_for_macos.patch' - 'thirdparty/fix_prometheus-cpp_limits.patch' - 'thirdparty/fix_rocksdb-cmake-PORTABLE-option.patch' + - 'thirdparty/fix_snappy-Wsign-compare-warning.patch' - 'thirdparty/fix_s2_build_with_absl_and_gtest.patch' - 'thirdparty/fix_thrift_for_cpp11.patch' # TODO(yingchun): shell/* files are import from thirdparties, we can move them to thirdparty later. diff --git a/cmake_modules/BaseFunctions.cmake b/cmake_modules/BaseFunctions.cmake index 1882353c83..dc0e3b28f7 100644 --- a/cmake_modules/BaseFunctions.cmake +++ b/cmake_modules/BaseFunctions.cmake @@ -326,15 +326,15 @@ function(dsn_setup_thirdparty_libs) find_package(fmt REQUIRED) set(DEFAULT_THIRDPARTY_LIBS ${THRIFT_LIB} fmt::fmt CACHE STRING "default thirdparty libs" FORCE) - # rocksdb + # rocksdb and dependencies file(GLOB ROCKSDB_DEPENDS_MODULE_PATH ${THIRDPARTY_ROOT}/build/Source/rocksdb/cmake/modules) if(NOT ROCKSDB_DEPENDS_MODULE_PATH) message(WARNING "Cannot find RocksDB depends cmake modules path, might not find snappy, zstd, lz4") endif() list(APPEND CMAKE_MODULE_PATH "${ROCKSDB_DEPENDS_MODULE_PATH}") - find_package(snappy) - find_package(zstd) - find_package(lz4) + find_package(Snappy REQUIRED) + find_package(zstd REQUIRED) + find_package(lz4 REQUIRED) if(USE_JEMALLOC) find_package(Jemalloc REQUIRED) endif() diff --git a/docker/pegasus-build-env/centos7/Dockerfile b/docker/pegasus-build-env/centos7/Dockerfile index 0e89f316b1..c8f8da6640 100644 --- a/docker/pegasus-build-env/centos7/Dockerfile +++ b/docker/pegasus-build-env/centos7/Dockerfile @@ -42,12 +42,8 @@ RUN yum -y install centos-release-scl \ which \ openssl-devel \ libaio-devel \ - snappy-devel \ - bzip2-devel \ zlib \ zlib-devel \ - libzstd-devel \ - lz4-devel \ bison \ flex \ krb5-devel \ diff --git a/docker/pegasus-build-env/ubuntu1804/Dockerfile b/docker/pegasus-build-env/ubuntu1804/Dockerfile index fb4c9a47ac..55189b1997 100644 --- a/docker/pegasus-build-env/ubuntu1804/Dockerfile +++ b/docker/pegasus-build-env/ubuntu1804/Dockerfile @@ -30,10 +30,6 @@ RUN apt-get update -y; \ openjdk-8-jdk \ python3-pip \ libaio-dev \ - libsnappy-dev \ - libbz2-dev \ - libzstd-dev \ - liblz4-dev \ zlib1g \ zlib1g.dev \ patch \ diff --git a/docker/pegasus-build-env/ubuntu2004/Dockerfile b/docker/pegasus-build-env/ubuntu2004/Dockerfile index 604b5a0b1e..73d58e059a 100644 --- a/docker/pegasus-build-env/ubuntu2004/Dockerfile +++ b/docker/pegasus-build-env/ubuntu2004/Dockerfile @@ -30,10 +30,6 @@ RUN apt-get update -y; \ openjdk-8-jdk \ python3-pip \ libaio-dev \ - libsnappy-dev \ - libbz2-dev \ - libzstd-dev \ - liblz4-dev \ zlib1g \ zlib1g.dev \ patch \ diff --git a/docker/pegasus-build-env/ubuntu2204/Dockerfile b/docker/pegasus-build-env/ubuntu2204/Dockerfile index 0eed16c6cf..d7efdb2867 100644 --- a/docker/pegasus-build-env/ubuntu2204/Dockerfile +++ b/docker/pegasus-build-env/ubuntu2204/Dockerfile @@ -31,10 +31,6 @@ RUN apt-get update -y; \ pkg-config \ python3-pip \ libaio-dev \ - libsnappy-dev \ - libbz2-dev \ - libzstd-dev \ - liblz4-dev \ zlib1g \ zlib1g.dev \ patch \ diff --git a/src/aio/CMakeLists.txt b/src/aio/CMakeLists.txt index 3754361d06..3a27fbe01a 100644 --- a/src/aio/CMakeLists.txt +++ b/src/aio/CMakeLists.txt @@ -33,7 +33,12 @@ set(MY_PROJ_SRC "") #"GLOB" for non - recursive search set(MY_SRC_SEARCH_MODE "GLOB") -set(MY_PROJ_LIBS dsn_runtime rocksdb) +set(MY_PROJ_LIBS + dsn_runtime + rocksdb + lz4 + zstd + snappy) #Extra files that will be installed set(MY_BINPLACES "") diff --git a/src/aio/test/CMakeLists.txt b/src/aio/test/CMakeLists.txt index b8d4ad675b..2eb4371467 100644 --- a/src/aio/test/CMakeLists.txt +++ b/src/aio/test/CMakeLists.txt @@ -33,7 +33,17 @@ set(MY_PROJ_SRC "") # "GLOB" for non-recursive search set(MY_SRC_SEARCH_MODE "GLOB") -set(MY_PROJ_LIBS gtest dsn_runtime dsn_aio test_utils rocksdb) +set(MY_PROJ_LIBS + dsn_meta_server + dsn_replication_common + gtest + dsn_runtime + dsn_aio + test_utils + rocksdb + lz4 + zstd + snappy) set(MY_BOOST_LIBS Boost::system Boost::filesystem) diff --git a/src/block_service/hdfs/CMakeLists.txt b/src/block_service/hdfs/CMakeLists.txt index 2bba8b96bb..ae821f2bf2 100644 --- a/src/block_service/hdfs/CMakeLists.txt +++ b/src/block_service/hdfs/CMakeLists.txt @@ -26,7 +26,12 @@ set(MY_PROJ_SRC "") #"GLOB" for non - recursive search set(MY_SRC_SEARCH_MODE "GLOB") -set(MY_PROJ_LIBS hdfs rocksdb) +set(MY_PROJ_LIBS + hdfs + rocksdb + lz4 + zstd + snappy) #Extra files that will be installed set(MY_BINPLACES "") diff --git a/src/block_service/local/CMakeLists.txt b/src/block_service/local/CMakeLists.txt index 9d830f7825..8fdd0d53ac 100644 --- a/src/block_service/local/CMakeLists.txt +++ b/src/block_service/local/CMakeLists.txt @@ -26,7 +26,11 @@ set(MY_PROJ_SRC "") #"GLOB" for non - recursive search set(MY_SRC_SEARCH_MODE "GLOB") -set(MY_PROJ_LIBS rocksdb) +set(MY_PROJ_LIBS + rocksdb + lz4 + zstd + snappy) #Extra files that will be installed set(MY_BINPLACES "") diff --git a/src/block_service/test/CMakeLists.txt b/src/block_service/test/CMakeLists.txt index 6fbfa9ca11..b8aaad6520 100644 --- a/src/block_service/test/CMakeLists.txt +++ b/src/block_service/test/CMakeLists.txt @@ -22,17 +22,20 @@ set(MY_PROJ_NAME dsn_block_service_test) set(MY_SRC_SEARCH_MODE "GLOB") set(MY_PROJ_LIBS - dsn_replication_common - dsn.block_service - dsn.block_service.local - dsn.block_service.hdfs - dsn_runtime - dsn_utils - gtest - gtest_main - hdfs - test_utils - rocksdb) + dsn_replication_common + dsn.block_service + dsn.block_service.local + dsn.block_service.hdfs + dsn_runtime + dsn_utils + gtest + gtest_main + hdfs + test_utils + rocksdb + lz4 + zstd + snappy) set(MY_BOOST_LIBS Boost::system Boost::filesystem) diff --git a/src/client/test/CMakeLists.txt b/src/client/test/CMakeLists.txt index cfe591ce0f..60f3762760 100644 --- a/src/client/test/CMakeLists.txt +++ b/src/client/test/CMakeLists.txt @@ -22,12 +22,15 @@ set(MY_PROJ_SRC "") set(MY_SRC_SEARCH_MODE "GLOB") set(MY_PROJ_LIBS - dsn_client - dsn_replication_common - dsn_runtime - dsn_utils - gtest - rocksdb) + dsn_client + dsn_replication_common + dsn_runtime + dsn_utils + gtest + rocksdb + lz4 + zstd + snappy) set(MY_BOOST_LIBS Boost::system Boost::filesystem) diff --git a/src/common/test/CMakeLists.txt b/src/common/test/CMakeLists.txt index 12f53aa52c..fab0926249 100644 --- a/src/common/test/CMakeLists.txt +++ b/src/common/test/CMakeLists.txt @@ -31,7 +31,10 @@ set(MY_PROJ_LIBS dsn_replication_common dsn_runtime gtest - rocksdb) + rocksdb + lz4 + zstd + snappy) set(MY_BOOST_LIBS Boost::system Boost::filesystem) diff --git a/src/failure_detector/test/CMakeLists.txt b/src/failure_detector/test/CMakeLists.txt index 6bd1e05553..5b5a6832a9 100644 --- a/src/failure_detector/test/CMakeLists.txt +++ b/src/failure_detector/test/CMakeLists.txt @@ -34,14 +34,17 @@ set(MY_PROJ_SRC "") set(MY_SRC_SEARCH_MODE "GLOB") set(MY_PROJ_LIBS - dsn_runtime - dsn_meta_server - dsn_replica_server - dsn_replication_common - dsn.failure_detector - gtest - hashtable - rocksdb) + dsn_runtime + dsn_meta_server + dsn_replica_server + dsn_replication_common + dsn.failure_detector + gtest + hashtable + rocksdb + lz4 + zstd + snappy) set(MY_BOOST_LIBS Boost::system Boost::filesystem) diff --git a/src/geo/bench/CMakeLists.txt b/src/geo/bench/CMakeLists.txt index 6df54a6bcb..559747b59d 100644 --- a/src/geo/bench/CMakeLists.txt +++ b/src/geo/bench/CMakeLists.txt @@ -34,9 +34,11 @@ set(MY_PROJ_LIBS s2testing s2 pegasus_client_static - RocksDB::rocksdb - dsn_utils - ) + rocksdb + lz4 + zstd + snappy + dsn_utils) set(MY_BOOST_LIBS Boost::system Boost::filesystem) diff --git a/src/http/test/CMakeLists.txt b/src/http/test/CMakeLists.txt index 5ebc142a55..4400c0183f 100644 --- a/src/http/test/CMakeLists.txt +++ b/src/http/test/CMakeLists.txt @@ -22,12 +22,14 @@ set(MY_PROJ_SRC "") set(MY_SRC_SEARCH_MODE "GLOB") set(MY_PROJ_LIBS - dsn_http - dsn_runtime - curl - gtest - rocksdb - ) + dsn_http + dsn_runtime + curl + gtest + rocksdb + lz4 + zstd + snappy) set(MY_BOOST_LIBS Boost::system Boost::filesystem) diff --git a/src/meta/CMakeLists.txt b/src/meta/CMakeLists.txt index 477671acb3..fcc3a12329 100644 --- a/src/meta/CMakeLists.txt +++ b/src/meta/CMakeLists.txt @@ -34,20 +34,23 @@ set(DUPLICATION_SRC set(MY_PROJ_SRC "${DUPLICATION_SRC}") set(MY_PROJ_LIBS - dsn_replication_common - dsn.block_service - dsn.block_service.local - dsn.block_service.hdfs - dsn.failure_detector - dsn.replication.zookeeper_provider - dsn_dist_cmd - dsn_http - dsn_runtime - dsn_aio - zookeeper - hashtable - hdfs - rocksdb) + dsn_replication_common + dsn.block_service + dsn.block_service.local + dsn.block_service.hdfs + dsn.failure_detector + dsn.replication.zookeeper_provider + dsn_dist_cmd + dsn_http + dsn_runtime + dsn_aio + zookeeper + hashtable + hdfs + rocksdb + lz4 + zstd + snappy) set(MY_BOOST_LIBS Boost::system Boost::filesystem) diff --git a/src/nfs/test/CMakeLists.txt b/src/nfs/test/CMakeLists.txt index ff591b7e21..4a22ce4867 100644 --- a/src/nfs/test/CMakeLists.txt +++ b/src/nfs/test/CMakeLists.txt @@ -33,7 +33,17 @@ set(MY_PROJ_SRC "") # "GLOB" for non-recursive search set(MY_SRC_SEARCH_MODE "GLOB") -set(MY_PROJ_LIBS dsn_nfs dsn_runtime gtest dsn_aio dsn_http rocksdb test_utils) +set(MY_PROJ_LIBS + dsn_nfs + dsn_runtime + gtest + dsn_aio + dsn_http + rocksdb + lz4 + zstd + snappy + test_utils) set(MY_BOOST_LIBS Boost::system Boost::filesystem) diff --git a/src/perf_counter/test/CMakeLists.txt b/src/perf_counter/test/CMakeLists.txt index 2e02cf868f..f924437dec 100644 --- a/src/perf_counter/test/CMakeLists.txt +++ b/src/perf_counter/test/CMakeLists.txt @@ -33,7 +33,15 @@ set(MY_PROJ_SRC "") # "GLOB" for non-recursive search set(MY_SRC_SEARCH_MODE "GLOB") -set(MY_PROJ_LIBS gtest dsn_runtime rocksdb) +set(MY_PROJ_LIBS + dsn_meta_server + dsn_replication_common + gtest + dsn_runtime + rocksdb + lz4 + zstd + snappy) set(MY_BOOST_LIBS Boost::system Boost::filesystem) diff --git a/src/replica/CMakeLists.txt b/src/replica/CMakeLists.txt index fcf7891017..5bcd3f57bc 100644 --- a/src/replica/CMakeLists.txt +++ b/src/replica/CMakeLists.txt @@ -57,19 +57,22 @@ set(MY_PROJ_SRC # "GLOB" for non-recursive search set(MY_SRC_SEARCH_MODE "GLOB") -set(MY_PROJ_LIBS - dsn_replication_common - dsn.failure_detector - dsn.block_service - dsn.block_service.local - dsn.block_service.hdfs - dsn_nfs - dsn_dist_cmd - dsn_http - dsn_runtime - dsn_aio - dsn_meta_server - rocksdb) +set(MY_PROJ_LIBS + dsn_replication_common + dsn.failure_detector + dsn.block_service + dsn.block_service.local + dsn.block_service.hdfs + dsn_nfs + dsn_dist_cmd + dsn_http + dsn_runtime + dsn_aio + dsn_meta_server + rocksdb + lz4 + zstd + snappy) set(MY_BOOST_LIBS Boost::filesystem) diff --git a/src/replica/backup/test/CMakeLists.txt b/src/replica/backup/test/CMakeLists.txt index 47158ca93c..fed60e3120 100644 --- a/src/replica/backup/test/CMakeLists.txt +++ b/src/replica/backup/test/CMakeLists.txt @@ -29,7 +29,10 @@ set(MY_PROJ_LIBS dsn_meta_server dsn_utils hashtable gtest - rocksdb) + rocksdb + lz4 + zstd + snappy) set(MY_BOOST_LIBS Boost::system Boost::filesystem) diff --git a/src/replica/bulk_load/test/CMakeLists.txt b/src/replica/bulk_load/test/CMakeLists.txt index 0b0d083851..bda1a89819 100644 --- a/src/replica/bulk_load/test/CMakeLists.txt +++ b/src/replica/bulk_load/test/CMakeLists.txt @@ -28,9 +28,12 @@ set(MY_PROJ_LIBS dsn_meta_server hashtable gtest test_utils - rocksdb) + rocksdb + lz4 + zstd + snappy) -set(MY_BOOST_LIBS Boost::system Boost::filesystem rocksdb test_utils) +set(MY_BOOST_LIBS Boost::system Boost::filesystem) set(MY_BINPLACES config-test.ini diff --git a/src/replica/duplication/test/CMakeLists.txt b/src/replica/duplication/test/CMakeLists.txt index 5cc139f916..26e461bd75 100644 --- a/src/replica/duplication/test/CMakeLists.txt +++ b/src/replica/duplication/test/CMakeLists.txt @@ -31,7 +31,10 @@ set(MY_PROJ_LIBS dsn_meta_server hashtable gtest test_utils - rocksdb) + rocksdb + lz4 + zstd + snappy) set(MY_BOOST_LIBS Boost::system Boost::filesystem) diff --git a/src/replica/storage/simple_kv/CMakeLists.txt b/src/replica/storage/simple_kv/CMakeLists.txt index 8584848f11..a2f8506770 100644 --- a/src/replica/storage/simple_kv/CMakeLists.txt +++ b/src/replica/storage/simple_kv/CMakeLists.txt @@ -37,7 +37,16 @@ set(MY_PROJ_SRC ${SIMPLE_KV_THRIFT_SRCS}) # "GLOB" for non-recursive search set(MY_SRC_SEARCH_MODE "GLOB") -set(MY_PROJ_LIBS dsn_replica_server dsn_meta_server dsn_client dsn_runtime hashtable rocksdb) +set(MY_PROJ_LIBS + dsn_replica_server + dsn_meta_server + dsn_client + dsn_runtime + hashtable + rocksdb + lz4 + zstd + snappy) set(MY_BOOST_LIBS Boost::system Boost::filesystem) diff --git a/src/replica/storage/simple_kv/test/CMakeLists.txt b/src/replica/storage/simple_kv/test/CMakeLists.txt index a1e0a69b21..0bce41b712 100644 --- a/src/replica/storage/simple_kv/test/CMakeLists.txt +++ b/src/replica/storage/simple_kv/test/CMakeLists.txt @@ -29,18 +29,22 @@ set(MY_PROJ_NAME dsn.rep_tests.simple_kv) # "GLOB" for non-recursive search set(MY_SRC_SEARCH_MODE "GLOB") -set(MY_PROJ_LIBS dsn_replica_server - dsn_meta_server - dsn_replication_common - dsn_client - dsn.failure_detector - dsn.replication.zookeeper_provider - dsn_runtime - zookeeper - hashtable - gtest - dsn_utils - rocksdb) +set(MY_PROJ_LIBS + dsn_replica_server + dsn_meta_server + dsn_replication_common + dsn_client + dsn.failure_detector + dsn.replication.zookeeper_provider + dsn_runtime + zookeeper + hashtable + gtest + dsn_utils + rocksdb + lz4 + zstd + snappy) set(MY_BOOST_LIBS Boost::system Boost::filesystem) diff --git a/src/replica/test/CMakeLists.txt b/src/replica/test/CMakeLists.txt index 83af542d10..94b5c89348 100644 --- a/src/replica/test/CMakeLists.txt +++ b/src/replica/test/CMakeLists.txt @@ -33,21 +33,25 @@ set(MY_PROJ_SRC "") #"GLOB" for non - recursive search set(MY_SRC_SEARCH_MODE "GLOB") -set(MY_PROJ_LIBS dsn_meta_server - dsn_replica_server - dsn.replication.zookeeper_provider - dsn_replication_common - dsn.block_service - dsn.block_service.local - dsn.block_service.hdfs - dsn.failure_detector - dsn_http - dsn_runtime - zookeeper - hashtable - gtest - test_utils - rocksdb) +set(MY_PROJ_LIBS + dsn_meta_server + dsn_replica_server + dsn.replication.zookeeper_provider + dsn_replication_common + dsn.block_service + dsn.block_service.local + dsn.block_service.hdfs + dsn.failure_detector + dsn_http + dsn_runtime + zookeeper + hashtable + gtest + test_utils + rocksdb + lz4 + zstd + snappy) set(MY_BOOST_LIBS Boost::system Boost::filesystem) diff --git a/src/runtime/ranger/CMakeLists.txt b/src/runtime/ranger/CMakeLists.txt index 3c3e441a4f..fbbceb570a 100644 --- a/src/runtime/ranger/CMakeLists.txt +++ b/src/runtime/ranger/CMakeLists.txt @@ -17,14 +17,10 @@ set(MY_PROJ_NAME dsn_ranger) -# Search mode for source files under CURRENT project directory? -# "GLOB_RECURSE" for recursive search -# "GLOB" for non-recursive search +set(MY_PROJ_SRC "") set(MY_SRC_SEARCH_MODE "GLOB") +set(MY_PROJ_LIBS + dsn_meta_server + dsn_replication_common) -set(MY_PROJ_LIBS "") - -# Extra files that will be installed -set(MY_BINPLACES "") - -dsn_add_object() +dsn_add_static_library() diff --git a/src/runtime/test/CMakeLists.txt b/src/runtime/test/CMakeLists.txt index cd5e0daac8..38e52a4f5b 100644 --- a/src/runtime/test/CMakeLists.txt +++ b/src/runtime/test/CMakeLists.txt @@ -29,12 +29,15 @@ set(MY_PROJ_NAME dsn_runtime_tests) # "GLOB" for non-recursive search set(MY_SRC_SEARCH_MODE "GLOB") -set(MY_PROJ_LIBS gtest - dsn_runtime - dsn_aio - dsn_meta_server - rocksdb - ) +set(MY_PROJ_LIBS + gtest + dsn_runtime + dsn_aio + dsn_meta_server + rocksdb + lz4 + zstd + snappy) set(MY_BOOST_LIBS Boost::system Boost::filesystem) diff --git a/src/server/CMakeLists.txt b/src/server/CMakeLists.txt index d19408e276..8c8daa4d12 100644 --- a/src/server/CMakeLists.txt +++ b/src/server/CMakeLists.txt @@ -27,21 +27,23 @@ set(MY_PROJ_SRC "") set(MY_SRC_SEARCH_MODE "GLOB") set(MY_PROJ_LIBS - dsn_replica_server - dsn_meta_server - dsn_replication_common - dsn_client - dsn.block_service.local - dsn.block_service - dsn.failure_detector - dsn.replication.zookeeper_provider - dsn_utils - RocksDB::rocksdb - pegasus_base - pegasus_client_static - event - hashtable - ) + dsn_replica_server + dsn_meta_server + dsn_replication_common + dsn_client + dsn.block_service.local + dsn.block_service + dsn.failure_detector + dsn.replication.zookeeper_provider + dsn_utils + rocksdb + lz4 + zstd + snappy + pegasus_base + pegasus_client_static + event + hashtable) set(MY_BOOST_LIBS Boost::system Boost::filesystem) diff --git a/src/server/test/CMakeLists.txt b/src/server/test/CMakeLists.txt index 8a2fbd6f1c..2b4c96ac1f 100644 --- a/src/server/test/CMakeLists.txt +++ b/src/server/test/CMakeLists.txt @@ -45,14 +45,17 @@ set(MY_PROJ_LIBS dsn.failure_detector dsn.replication.zookeeper_provider dsn_utils - RocksDB::rocksdb + rocksdb + lz4 + zstd + snappy pegasus_client_static event pegasus_base gtest gmock hashtable - ) +) add_definitions(-DPEGASUS_UNIT_TEST) add_definitions(-DENABLE_FAIL) diff --git a/src/shell/CMakeLists.txt b/src/shell/CMakeLists.txt index 909fda9ab6..81271006e1 100644 --- a/src/shell/CMakeLists.txt +++ b/src/shell/CMakeLists.txt @@ -28,25 +28,27 @@ set(MY_PROJ_SRC "linenoise/linenoise.c" "sds/sds.c") set(MY_SRC_SEARCH_MODE "GLOB_RECURSE") set(MY_PROJ_LIBS - pegasus_base - dsn.replication.tool - dsn_replica_server - dsn_meta_server - dsn_replication_common - dsn_client - dsn_utils - dsn.block_service.local - dsn.block_service.hdfs - dsn.block_service - dsn.failure_detector - pegasus_client_static - pegasus_geo_lib - RocksDB::rocksdb - absl::flat_hash_set - absl::strings - s2 - hdfs - ) + pegasus_base + dsn.replication.tool + dsn_replica_server + dsn_meta_server + dsn_replication_common + dsn_client + dsn_utils + dsn.block_service.local + dsn.block_service.hdfs + dsn.block_service + dsn.failure_detector + pegasus_client_static + pegasus_geo_lib + rocksdb + lz4 + zstd + snappy + absl::flat_hash_set + absl::strings + s2 + hdfs) set(MY_BINPLACES "${CMAKE_CURRENT_SOURCE_DIR}/config.ini") diff --git a/src/test/bench_test/CMakeLists.txt b/src/test/bench_test/CMakeLists.txt index 307d543d09..5a52808d8d 100644 --- a/src/test/bench_test/CMakeLists.txt +++ b/src/test/bench_test/CMakeLists.txt @@ -30,11 +30,13 @@ set(MY_SRC_SEARCH_MODE "GLOB") set(MY_PROJ_LIBS pegasus_client_static dsn_utils - RocksDB::rocksdb + rocksdb + lz4 + zstd + snappy sasl2 gssapi_krb5 - krb5 - ) + krb5) set(MY_BOOST_LIBS Boost::system Boost::filesystem) diff --git a/src/test/function_test/bulk_load/CMakeLists.txt b/src/test/function_test/bulk_load/CMakeLists.txt index f9f1976491..ce17323ea7 100644 --- a/src/test/function_test/bulk_load/CMakeLists.txt +++ b/src/test/function_test/bulk_load/CMakeLists.txt @@ -28,17 +28,20 @@ set(MY_PROJ_SRC "") set(MY_SRC_SEARCH_MODE "GLOB") set(MY_PROJ_LIBS - dsn_client - dsn_replication_common - dsn_utils - pegasus_client_static - gtest - sasl2 - gssapi_krb5 - krb5 - function_test_utils - test_utils - rocksdb) + dsn_client + dsn_replication_common + dsn_utils + pegasus_client_static + gtest + sasl2 + gssapi_krb5 + krb5 + function_test_utils + test_utils + rocksdb + lz4 + zstd + snappy) set(MY_BOOST_LIBS Boost::system Boost::filesystem) diff --git a/src/test_util/CMakeLists.txt b/src/test_util/CMakeLists.txt index b1e7ac2dff..fd4cf1dc16 100644 --- a/src/test_util/CMakeLists.txt +++ b/src/test_util/CMakeLists.txt @@ -22,6 +22,11 @@ set(MY_PROJ_NAME test_utils) # "GLOB" for non-recursive search set(MY_SRC_SEARCH_MODE "GLOB") -set(MY_PROJ_LIBS gtest rocksdb) +set(MY_PROJ_LIBS + gtest + rocksdb + lz4 + zstd + snappy) dsn_add_static_library() diff --git a/src/utils/CMakeLists.txt b/src/utils/CMakeLists.txt index 8c04e3adc4..38390f0ed6 100644 --- a/src/utils/CMakeLists.txt +++ b/src/utils/CMakeLists.txt @@ -31,7 +31,12 @@ set(MY_SRC_SEARCH_MODE "GLOB") set(MY_BOOST_LIBS Boost::system Boost::filesystem) -set(MY_PROJ_LIBS dsn_http rocksdb) +set(MY_PROJ_LIBS + dsn_http + rocksdb + lz4 + zstd + snappy) # Extra files that will be installed set(MY_BINPLACES "") diff --git a/src/utils/long_adder_bench/CMakeLists.txt b/src/utils/long_adder_bench/CMakeLists.txt index d5ace78cac..71568c61ae 100644 --- a/src/utils/long_adder_bench/CMakeLists.txt +++ b/src/utils/long_adder_bench/CMakeLists.txt @@ -27,7 +27,13 @@ set(MY_PROJ_SRC "") # "GLOB" for non-recursive search set(MY_SRC_SEARCH_MODE "GLOB") -set(MY_PROJ_LIBS dsn_runtime dsn_utils rocksdb) +set(MY_PROJ_LIBS + dsn_runtime + dsn_utils + rocksdb + lz4 + zstd + snappy) set(MY_BOOST_LIBS Boost::system Boost::filesystem) diff --git a/src/utils/test/CMakeLists.txt b/src/utils/test/CMakeLists.txt index 684a5faaf0..8502cabb76 100644 --- a/src/utils/test/CMakeLists.txt +++ b/src/utils/test/CMakeLists.txt @@ -29,12 +29,16 @@ set(MY_PROJ_NAME dsn_utils_tests) # "GLOB" for non-recursive search set(MY_SRC_SEARCH_MODE "GLOB") -set(MY_PROJ_LIBS dsn_http - dsn_runtime - dsn_utils - gtest - test_utils - rocksdb) +set(MY_PROJ_LIBS + dsn_http + dsn_runtime + dsn_utils + gtest + test_utils + rocksdb + lz4 + zstd + snappy) set(MY_BOOST_LIBS Boost::system Boost::filesystem) diff --git a/src/utils/test/nth_element_bench/CMakeLists.txt b/src/utils/test/nth_element_bench/CMakeLists.txt index 7bf0af804b..3aae31e4a6 100644 --- a/src/utils/test/nth_element_bench/CMakeLists.txt +++ b/src/utils/test/nth_element_bench/CMakeLists.txt @@ -27,7 +27,13 @@ set(MY_PROJ_SRC "") # "GLOB" for non-recursive search set(MY_SRC_SEARCH_MODE "GLOB") -set(MY_PROJ_LIBS dsn_runtime dsn_utils rocksdb) +set(MY_PROJ_LIBS + dsn_runtime + dsn_utils + rocksdb + lz4 + zstd + snappy) set(MY_BOOST_LIBS Boost::system Boost::filesystem) diff --git a/src/zookeeper/test/CMakeLists.txt b/src/zookeeper/test/CMakeLists.txt index cdb6d4cec2..7dabd1faf5 100644 --- a/src/zookeeper/test/CMakeLists.txt +++ b/src/zookeeper/test/CMakeLists.txt @@ -33,13 +33,16 @@ set(MY_PROJ_SRC "") # "GLOB" for non-recursive search set(MY_SRC_SEARCH_MODE "GLOB") -set(MY_PROJ_LIBS - dsn.replication.zookeeper_provider - dsn_runtime - zookeeper - hashtable - gtest - rocksdb) +set(MY_PROJ_LIBS + dsn.replication.zookeeper_provider + dsn_runtime + zookeeper + hashtable + gtest + rocksdb + lz4 + zstd + snappy) set(MY_BOOST_LIBS Boost::system Boost::filesystem) diff --git a/thirdparty/CMakeLists.txt b/thirdparty/CMakeLists.txt index e95e24f83e..c3a9498f76 100644 --- a/thirdparty/CMakeLists.txt +++ b/thirdparty/CMakeLists.txt @@ -361,6 +361,75 @@ ExternalProject_Add(jemalloc DOWNLOAD_NO_PROGRESS true ) +set(SNAPPY_OPTIONS + -DCMAKE_INSTALL_PREFIX=${TP_OUTPUT} + -DCMAKE_POSITION_INDEPENDENT_CODE=ON + -DCMAKE_BUILD_TYPE=Release + -DBUILD_SHARED_LIBS=OFF + -DSNAPPY_BUILD_TESTS=OFF + -DSNAPPY_BUILD_BENCHMARKS=OFF + -DSNAPPY_FUZZING_BUILD=OFF + -DSNAPPY_INSTALL=ON) +execute_process(COMMAND arch OUTPUT_VARIABLE ARCH_NAME OUTPUT_STRIP_TRAILING_WHITESPACE) +message(STATUS "ARCH_NAME = ${ARCH_NAME}") +if (ARCH_NAME EQUAL "x86_64") + set(SNAPPY_OPTIONS + ${SNAPPY_OPTIONS} + -DSNAPPY_REQUIRE_AVX=ON + -DSNAPPY_REQUIRE_AVX2=ON) +endif () +ExternalProject_Add(snappy + URL ${OSS_URL_PREFIX}/snappy-1.1.10.tar.gz + https://github.com/google/snappy/archive/refs/tags/1.1.10.tar.gz + URL_MD5 70153395ebe6d72febe2cf2e40026a44 + PATCH_COMMAND patch -p1 < ${TP_DIR}/fix_snappy-Wsign-compare-warning.patch + CMAKE_ARGS ${SNAPPY_OPTIONS} + BUILD_COMMAND make -j${PARALLEL} + INSTALL_COMMAND make install + DOWNLOAD_EXTRACT_TIMESTAMP true + DOWNLOAD_NO_PROGRESS true +) + +ExternalProject_Add(zstd + URL ${OSS_URL_PREFIX}/zstd-1.5.5.tar.gz + https://github.com/facebook/zstd/releases/download/v1.5.5/zstd-1.5.5.tar.gz + URL_MD5 63251602329a106220e0a5ad26ba656f + PATCH_COMMAND "" + CONFIGURE_COMMAND cmake build/cmake + -DCMAKE_INSTALL_PREFIX=${TP_OUTPUT} + -DCMAKE_POSITION_INDEPENDENT_CODE=ON + -DCMAKE_BUILD_TYPE=Release + -DZSTD_BUILD_PROGRAMS=OFF + -DZSTD_BUILD_TESTS=OFF + -DZSTD_BUILD_CONTRIB=OFF + -DZSTD_BUILD_SHARED=OFF + -DZSTD_BUILD_STATIC=ON + BUILD_COMMAND make -j${PARALLEL} + INSTALL_COMMAND make install + BUILD_IN_SOURCE 1 + DOWNLOAD_EXTRACT_TIMESTAMP true + DOWNLOAD_NO_PROGRESS true +) + +ExternalProject_Add(lz4 + URL ${OSS_URL_PREFIX}/lz4-1.9.4.tar.gz + https://github.com/lz4/lz4/releases/download/v1.9.4/lz4-1.9.4.tar.gz + URL_MD5 e9286adb64040071c5e23498bf753261 + PATCH_COMMAND "" + CONFIGURE_COMMAND cmake build/cmake + -DCMAKE_INSTALL_PREFIX=${TP_OUTPUT} + -DLZ4_POSITION_INDEPENDENT_LIB=ON + -DCMAKE_BUILD_TYPE=Release + -DLZ4_BUILD_CLI=OFF + -DBUILD_SHARED_LIBS=OFF + -DBUILD_STATIC_LIBS=ON + BUILD_COMMAND make -j${PARALLEL} + INSTALL_COMMAND make install + BUILD_IN_SOURCE 1 + DOWNLOAD_EXTRACT_TIMESTAMP true + DOWNLOAD_NO_PROGRESS true +) + option(ROCKSDB_PORTABLE "Minimum CPU arch to support, or 0 = current CPU, 1 = baseline CPU" 0) set(ROCKSDB_OPTIONS -DFAIL_ON_WARNINGS=OFF @@ -378,6 +447,7 @@ set(ROCKSDB_OPTIONS -DJEMALLOC_ROOT_DIR=${TP_OUTPUT} -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} + -DCMAKE_INSTALL_PREFIX=${TP_OUTPUT} -DPORTABLE=${ROCKSDB_PORTABLE}) # Link error on MacOS, disable building encfs plugin. # See https://github.com/pegasus-kv/encfs/issues/4 @@ -393,9 +463,8 @@ ExternalProject_Add(rocksdb PATCH_COMMAND patch -p1 < ${TP_DIR}/fix_rocksdb-cmake-PORTABLE-option.patch COMMAND rm -rf ${TP_DIR}/build/Source/rocksdb/plugin/encfs COMMAND git clone -b main --depth=1 https://github.com/pegasus-kv/encfs.git ${TP_DIR}/build/Source/rocksdb/plugin/encfs - DEPENDS googletest jemalloc - CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${TP_OUTPUT} - ${ROCKSDB_OPTIONS} + DEPENDS googletest jemalloc lz4 snappy zstd + CMAKE_ARGS ${ROCKSDB_OPTIONS} DOWNLOAD_EXTRACT_TIMESTAMP true DOWNLOAD_NO_PROGRESS true ) diff --git a/thirdparty/fix_snappy-Wsign-compare-warning.patch b/thirdparty/fix_snappy-Wsign-compare-warning.patch new file mode 100644 index 0000000000..0540d0177c --- /dev/null +++ b/thirdparty/fix_snappy-Wsign-compare-warning.patch @@ -0,0 +1,26 @@ +From 27f34a580be4a3becf5f8c0cba13433f53c21337 Mon Sep 17 00:00:00 2001 +From: Richard O'Grady +Date: Wed, 12 Jul 2023 10:12:01 -0700 +Subject: [PATCH] Fix -Wsign-compare warning + +PiperOrigin-RevId: 547529709 +--- + snappy.cc | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/snappy.cc b/snappy.cc +index 688065b..6473123 100644 +--- a/snappy.cc ++++ b/snappy.cc +@@ -1289,7 +1289,7 @@ std::pair DecompressBranchless( + DeferMemCopy(&deferred_src, &deferred_length, from, len); + } + } while (ip < ip_limit_min_slop && +- (op + deferred_length) < op_limit_min_slop); ++ static_cast(op + deferred_length) < op_limit_min_slop); + exit: + ip--; + assert(ip <= ip_limit); +-- +2.42.1 +