feat(snappy/lz4/zstd): Build snappy/lz4/zstd as a thirdparty libraries (

#1709) Build snappy/lz4/zstd as thirdparty libraries, and do not dependent on the library in system. It would be more flexible to upgrade the libraries version no matter what the library versions in system are. Another issue this patch resolve is #1632.
apache · Dec 12, 2023 · 5ea2a8d · 5ea2a8d
1 parent d85837c
commit 5ea2a8d
Show file tree

Hide file tree

Showing 42 changed files with 415 additions and 208 deletions.
diff --git a/.github/workflows/lint_and_test_cpp.yaml b/.github/workflows/lint_and_test_cpp.yaml
@@ -673,9 +673,6 @@ jobs:
         run: |
           # Preinstalled softwares: https://github.com/actions/virtual-environments/blob/main/images/macos/macos-12-Readme.md
           brew install ccache
-          brew install snappy
-          brew install lz4
-          brew install zstd
           brew install [email protected]
       - uses: actions/checkout@v3
       - name: Setup cache

diff --git a/.licenserc.yaml b/.licenserc.yaml
@@ -71,6 +71,7 @@ header:
     - 'thirdparty/fix_libevent_for_macos.patch'
     - 'thirdparty/fix_prometheus-cpp_limits.patch'
     - 'thirdparty/fix_rocksdb-cmake-PORTABLE-option.patch'
+    - 'thirdparty/fix_snappy-Wsign-compare-warning.patch'
     - 'thirdparty/fix_s2_build_with_absl_and_gtest.patch'
     - 'thirdparty/fix_thrift_for_cpp11.patch'
     # TODO(yingchun): shell/* files are import from thirdparties, we can move them to thirdparty later.

diff --git a/cmake_modules/BaseFunctions.cmake b/cmake_modules/BaseFunctions.cmake
@@ -326,15 +326,15 @@ function(dsn_setup_thirdparty_libs)
   find_package(fmt REQUIRED)
   set(DEFAULT_THIRDPARTY_LIBS ${THRIFT_LIB} fmt::fmt CACHE STRING "default thirdparty libs" FORCE)
 
-  # rocksdb
+  # rocksdb and dependencies
   file(GLOB ROCKSDB_DEPENDS_MODULE_PATH ${THIRDPARTY_ROOT}/build/Source/rocksdb/cmake/modules)
   if(NOT ROCKSDB_DEPENDS_MODULE_PATH)
     message(WARNING "Cannot find RocksDB depends cmake modules path, might not find snappy, zstd, lz4")
   endif()
   list(APPEND CMAKE_MODULE_PATH "${ROCKSDB_DEPENDS_MODULE_PATH}")
-  find_package(snappy)
-  find_package(zstd)
-  find_package(lz4)
+  find_package(Snappy REQUIRED)
+  find_package(zstd REQUIRED)
+  find_package(lz4 REQUIRED)
   if(USE_JEMALLOC)
     find_package(Jemalloc REQUIRED)
   endif()

diff --git a/docker/pegasus-build-env/centos7/Dockerfile b/docker/pegasus-build-env/centos7/Dockerfile
@@ -42,12 +42,8 @@ RUN yum -y install centos-release-scl \
                    which \
                    openssl-devel \
                    libaio-devel \
-                   snappy-devel \
-                   bzip2-devel \
                    zlib \
                    zlib-devel \
-                   libzstd-devel \
-                   lz4-devel \
                    bison \
                    flex \
                    krb5-devel \

diff --git a/docker/pegasus-build-env/ubuntu1804/Dockerfile b/docker/pegasus-build-env/ubuntu1804/Dockerfile
@@ -30,10 +30,6 @@ RUN apt-get update -y; \
                        openjdk-8-jdk \
                        python3-pip \
                        libaio-dev \
-                       libsnappy-dev \
-                       libbz2-dev \
-                       libzstd-dev \
-                       liblz4-dev \
                        zlib1g \
                        zlib1g.dev \
                        patch \

diff --git a/docker/pegasus-build-env/ubuntu2004/Dockerfile b/docker/pegasus-build-env/ubuntu2004/Dockerfile
@@ -30,10 +30,6 @@ RUN apt-get update -y; \
                        openjdk-8-jdk \
                        python3-pip \
                        libaio-dev \
-                       libsnappy-dev \
-                       libbz2-dev \
-                       libzstd-dev \
-                       liblz4-dev \
                        zlib1g \
                        zlib1g.dev \
                        patch \

diff --git a/docker/pegasus-build-env/ubuntu2204/Dockerfile b/docker/pegasus-build-env/ubuntu2204/Dockerfile
@@ -31,10 +31,6 @@ RUN apt-get update -y; \
                        pkg-config \
                        python3-pip \
                        libaio-dev \
-                       libsnappy-dev \
-                       libbz2-dev \
-                       libzstd-dev \
-                       liblz4-dev \
                        zlib1g \
                        zlib1g.dev \
                        patch \

diff --git a/src/aio/CMakeLists.txt b/src/aio/CMakeLists.txt
@@ -33,7 +33,12 @@ set(MY_PROJ_SRC "")
 #"GLOB" for non - recursive search
 set(MY_SRC_SEARCH_MODE "GLOB")
 
-set(MY_PROJ_LIBS dsn_runtime rocksdb)
+set(MY_PROJ_LIBS
+        dsn_runtime
+        rocksdb
+        lz4
+        zstd
+        snappy)
 
 #Extra files that will be installed
 set(MY_BINPLACES "")

diff --git a/src/aio/test/CMakeLists.txt b/src/aio/test/CMakeLists.txt
@@ -33,7 +33,17 @@ set(MY_PROJ_SRC "")
 # "GLOB" for non-recursive search
 set(MY_SRC_SEARCH_MODE "GLOB")
 
-set(MY_PROJ_LIBS gtest dsn_runtime dsn_aio test_utils rocksdb)
+set(MY_PROJ_LIBS
+        dsn_meta_server
+        dsn_replication_common
+        gtest
+        dsn_runtime
+        dsn_aio
+        test_utils
+        rocksdb
+        lz4
+        zstd
+        snappy)
 
 set(MY_BOOST_LIBS Boost::system Boost::filesystem)
 

diff --git a/src/block_service/hdfs/CMakeLists.txt b/src/block_service/hdfs/CMakeLists.txt
@@ -26,7 +26,12 @@ set(MY_PROJ_SRC "")
 #"GLOB" for non - recursive search
 set(MY_SRC_SEARCH_MODE "GLOB")
 
-set(MY_PROJ_LIBS hdfs rocksdb)
+set(MY_PROJ_LIBS
+        hdfs
+        rocksdb
+        lz4
+        zstd
+        snappy)
 
 #Extra files that will be installed
 set(MY_BINPLACES "")

diff --git a/src/block_service/local/CMakeLists.txt b/src/block_service/local/CMakeLists.txt
@@ -26,7 +26,11 @@ set(MY_PROJ_SRC "")
 #"GLOB" for non - recursive search
 set(MY_SRC_SEARCH_MODE "GLOB")
 
-set(MY_PROJ_LIBS rocksdb)
+set(MY_PROJ_LIBS
+        rocksdb
+        lz4
+        zstd
+        snappy)
 
 #Extra files that will be installed
 set(MY_BINPLACES "")

diff --git a/src/block_service/test/CMakeLists.txt b/src/block_service/test/CMakeLists.txt
@@ -22,17 +22,20 @@ set(MY_PROJ_NAME dsn_block_service_test)
 set(MY_SRC_SEARCH_MODE "GLOB")
 
 set(MY_PROJ_LIBS
-    dsn_replication_common
-    dsn.block_service
-    dsn.block_service.local
-    dsn.block_service.hdfs
-    dsn_runtime
-    dsn_utils
-    gtest
-    gtest_main
-    hdfs
-    test_utils
-    rocksdb)
+        dsn_replication_common
+        dsn.block_service
+        dsn.block_service.local
+        dsn.block_service.hdfs
+        dsn_runtime
+        dsn_utils
+        gtest
+        gtest_main
+        hdfs
+        test_utils
+        rocksdb
+        lz4
+        zstd
+        snappy)
 
 set(MY_BOOST_LIBS Boost::system Boost::filesystem)
 

diff --git a/src/client/test/CMakeLists.txt b/src/client/test/CMakeLists.txt
@@ -22,12 +22,15 @@ set(MY_PROJ_SRC "")
 set(MY_SRC_SEARCH_MODE "GLOB")
 
 set(MY_PROJ_LIBS
-    dsn_client
-    dsn_replication_common
-    dsn_runtime
-    dsn_utils
-    gtest
-    rocksdb)
+        dsn_client
+        dsn_replication_common
+        dsn_runtime
+        dsn_utils
+        gtest
+        rocksdb
+        lz4
+        zstd
+        snappy)
 
 set(MY_BOOST_LIBS Boost::system Boost::filesystem)
 

diff --git a/src/common/test/CMakeLists.txt b/src/common/test/CMakeLists.txt
@@ -31,7 +31,10 @@ set(MY_PROJ_LIBS
         dsn_replication_common
         dsn_runtime
         gtest
-        rocksdb)
+        rocksdb
+        lz4
+        zstd
+        snappy)
 
 set(MY_BOOST_LIBS Boost::system Boost::filesystem)
 

diff --git a/src/failure_detector/test/CMakeLists.txt b/src/failure_detector/test/CMakeLists.txt
@@ -34,14 +34,17 @@ set(MY_PROJ_SRC "")
 set(MY_SRC_SEARCH_MODE "GLOB")
 
 set(MY_PROJ_LIBS
-    dsn_runtime
-    dsn_meta_server
-    dsn_replica_server
-    dsn_replication_common
-    dsn.failure_detector
-    gtest
-    hashtable
-    rocksdb)
+        dsn_runtime
+        dsn_meta_server
+        dsn_replica_server
+        dsn_replication_common
+        dsn.failure_detector
+        gtest
+        hashtable
+        rocksdb
+        lz4
+        zstd
+        snappy)
 
 set(MY_BOOST_LIBS Boost::system Boost::filesystem)
 

diff --git a/src/geo/bench/CMakeLists.txt b/src/geo/bench/CMakeLists.txt
@@ -34,9 +34,11 @@ set(MY_PROJ_LIBS
         s2testing
         s2
         pegasus_client_static
-        RocksDB::rocksdb
-        dsn_utils
-        )
+        rocksdb
+        lz4
+        zstd
+        snappy
+        dsn_utils)
 
 set(MY_BOOST_LIBS Boost::system Boost::filesystem)
 

diff --git a/src/http/test/CMakeLists.txt b/src/http/test/CMakeLists.txt
@@ -22,12 +22,14 @@ set(MY_PROJ_SRC "")
 set(MY_SRC_SEARCH_MODE "GLOB")
 
 set(MY_PROJ_LIBS
-    dsn_http
-    dsn_runtime
-    curl
-    gtest
-    rocksdb
-    )
+        dsn_http
+        dsn_runtime
+        curl
+        gtest
+        rocksdb
+        lz4
+        zstd
+        snappy)
 
 set(MY_BOOST_LIBS Boost::system Boost::filesystem)
 

diff --git a/src/meta/CMakeLists.txt b/src/meta/CMakeLists.txt
@@ -34,20 +34,23 @@ set(DUPLICATION_SRC
 set(MY_PROJ_SRC "${DUPLICATION_SRC}")
 
 set(MY_PROJ_LIBS
-    dsn_replication_common
-    dsn.block_service
-    dsn.block_service.local
-    dsn.block_service.hdfs
-    dsn.failure_detector
-    dsn.replication.zookeeper_provider
-    dsn_dist_cmd
-    dsn_http
-    dsn_runtime
-    dsn_aio
-    zookeeper
-    hashtable
-    hdfs
-    rocksdb)
+        dsn_replication_common
+        dsn.block_service
+        dsn.block_service.local
+        dsn.block_service.hdfs
+        dsn.failure_detector
+        dsn.replication.zookeeper_provider
+        dsn_dist_cmd
+        dsn_http
+        dsn_runtime
+        dsn_aio
+        zookeeper
+        hashtable
+        hdfs
+        rocksdb
+        lz4
+        zstd
+        snappy)
 
 set(MY_BOOST_LIBS Boost::system Boost::filesystem)
 

diff --git a/src/nfs/test/CMakeLists.txt b/src/nfs/test/CMakeLists.txt
@@ -33,7 +33,17 @@ set(MY_PROJ_SRC "")
 # "GLOB" for non-recursive search
 set(MY_SRC_SEARCH_MODE "GLOB")
 
-set(MY_PROJ_LIBS dsn_nfs dsn_runtime gtest dsn_aio dsn_http rocksdb test_utils)
+set(MY_PROJ_LIBS
+        dsn_nfs
+        dsn_runtime
+        gtest
+        dsn_aio
+        dsn_http
+        rocksdb
+        lz4
+        zstd
+        snappy
+        test_utils)
 
 set(MY_BOOST_LIBS Boost::system Boost::filesystem)
 

diff --git a/src/perf_counter/test/CMakeLists.txt b/src/perf_counter/test/CMakeLists.txt
@@ -33,7 +33,15 @@ set(MY_PROJ_SRC "")
 # "GLOB" for non-recursive search
 set(MY_SRC_SEARCH_MODE "GLOB")
 
-set(MY_PROJ_LIBS gtest dsn_runtime rocksdb)
+set(MY_PROJ_LIBS
+        dsn_meta_server
+        dsn_replication_common
+        gtest
+        dsn_runtime
+        rocksdb
+        lz4
+        zstd
+        snappy)
 
 set(MY_BOOST_LIBS Boost::system Boost::filesystem)
 

diff --git a/src/replica/CMakeLists.txt b/src/replica/CMakeLists.txt
@@ -57,19 +57,22 @@ set(MY_PROJ_SRC
 # "GLOB" for non-recursive search
 set(MY_SRC_SEARCH_MODE "GLOB")
 
-set(MY_PROJ_LIBS 
-    dsn_replication_common
-    dsn.failure_detector
-    dsn.block_service
-    dsn.block_service.local
-    dsn.block_service.hdfs
-    dsn_nfs
-    dsn_dist_cmd
-    dsn_http
-    dsn_runtime
-    dsn_aio
-    dsn_meta_server
-    rocksdb)
+set(MY_PROJ_LIBS
+        dsn_replication_common
+        dsn.failure_detector
+        dsn.block_service
+        dsn.block_service.local
+        dsn.block_service.hdfs
+        dsn_nfs
+        dsn_dist_cmd
+        dsn_http
+        dsn_runtime
+        dsn_aio
+        dsn_meta_server
+        rocksdb
+        lz4
+        zstd
+        snappy)
 
 set(MY_BOOST_LIBS Boost::filesystem)
 

diff --git a/src/replica/backup/test/CMakeLists.txt b/src/replica/backup/test/CMakeLists.txt
@@ -29,7 +29,10 @@ set(MY_PROJ_LIBS dsn_meta_server
         dsn_utils
         hashtable
         gtest
-        rocksdb)
+        rocksdb
+        lz4
+        zstd
+        snappy)
 
 set(MY_BOOST_LIBS Boost::system Boost::filesystem)