Removed brotli from contrib (#107)

ydb-platform · Mar 27, 2024 · 5411ee5 · 5411ee5
1 parent cffa8aa
commit 5411ee5
Show file tree

Hide file tree

Showing 128 changed files with 261 additions and 35,023 deletions.
diff --git a/.github/actions/prepare_vm/action.yaml b/.github/actions/prepare_vm/action.yaml
@@ -20,19 +20,24 @@ runs:
         tar -xvzf libiconv-1.15.tar.gz
         cd libiconv-1.15
         ./configure --prefix=/usr/local
-        sudo make
+        make
         sudo make install
-        cd ..
+        cd ../
+        wget https://github.com/aklomp/base64/archive/refs/tags/v0.5.2.tar.gz
+        tar -xvzf v0.5.2.tar.gz
+        cd base64-0.5.2
+        mkdir build && cd build
+        cmake -DCMAKE_BUILD_TYPE=Release ..
+        sudo cmake --build . --config Release --target install
+        cd ../../
+        wget https://github.com/google/brotli/archive/refs/tags/v1.1.0.tar.gz
+        tar -xvzf v1.1.0.tar.gz
+        cd brotli-1.1.0
+        mkdir build && cd build
+        cmake -DCMAKE_BUILD_TYPE=Release ..
+        sudo cmake --build . --config Release --target install
         (V=4.8.1; curl -L https://github.com/ccache/ccache/releases/download/v${V}/ccache-${V}-linux-x86_64.tar.xz | \
         sudo tar -xJ -C /usr/local/bin/ --strip-components=1 --no-same-owner ccache-${V}-linux-x86_64/ccache)
-        sudo rm -rf llvm.sh libiconv-1.15
-        cd ..
-        sudo wget https://github.com/aklomp/base64/archive/refs/tags/v0.5.2.tar.gz
-        sudo tar -xvzf v0.5.2.tar.gz
-        cd base64-0.5.2
-        sudo mkdir build
-        cd build
-        sudo cmake ..
-        sudo cmake --build . --target install
         cd ../../
-        sudo rm -r ./base64-0.5.2
+        sudo rm -r llvm.sh libiconv-1.15.tar.gz v0.5.2.tar.gz v1.1.0.tar.gz \
+          libiconv-1.15 base64-0.5.2 brotli-1.1.0
diff --git a/README.md b/README.md
@@ -8,7 +8,7 @@ If you ok with this warning, then...
 ## Prerequisites
 
 - cmake 3.22+
-- llvm 16+
+- clang 16+
 - git 2.20+
 - ninja 1.10+
 - ragel
@@ -29,6 +29,7 @@ If you ok with this warning, then...
 - lz4
 - snappy 1.1.8+
 - base64
+- brotli 1.1.10+
 
 ## Runtime requirements
 
@@ -48,21 +49,23 @@ chmod u+x llvm.sh
 sudo ./llvm.sh 16
 
 wget https://ftp.gnu.org/pub/gnu/libiconv/libiconv-1.15.tar.gz
-tar -xvzf libiconv-1.15.tar.gz
-cd libiconv-1.15
+tar -xvzf libiconv-1.15.tar.gz && cd libiconv-1.15
 ./configure --prefix=/usr/local
-sudo make
+make
 sudo make install
 
-sudo wget https://github.com/aklomp/base64/archive/refs/tags/v0.5.2.tar.gz
-sudo tar -xvzf v0.5.2.tar.gz
-cd base64-0.5.2
-sudo mkdir build
-cd build
-sudo cmake ..
-sudo cmake --build . --target install
-cd ../../
-
+wget https://github.com/aklomp/base64/archive/refs/tags/v0.5.2.tar.gz
+tar -xvzf v0.5.2.tar.gz && cd base64-0.5.2
+mkdir build && cd build
+cmake -DCMAKE_BUILD_TYPE=Release ..
+sudo cmake --build . --config Release --target install
+
+wget https://github.com/google/brotli/archive/refs/tags/v1.1.0.tar.gz
+tar -xvzf v1.1.0.tar.gz
+cd brotli-1.1.0
+mkdir build && cd build
+cmake -DCMAKE_BUILD_TYPE=Release ..
+sudo cmake --build . --config Release --target install
 ```
 
 ## Create the work directory

diff --git a/cmake/FindBrotli.cmake b/cmake/FindBrotli.cmake
@@ -0,0 +1,67 @@
+# - Find IDN
+#
+# IDN_INCLUDE - Where to find LibIDN public headers
+# IDN_LIBS - List of libraries when using LibIDN.
+# IDN_FOUND - True if LibIDN found.
+
+find_path(Brotli_INCLUDE_DIR
+  "brotli/decode.h"
+  HINTS $ENV{Brotli_ROOT}/include
+)
+
+find_library(Brotli_common_LIBRARY
+  brotlicommon
+  HINTS $ENV{Brotli_ROOT}/lib
+)
+
+find_library(Brotli_enc_LIBRARY
+  brotlienc
+  HINTS $ENV{Brotli_ROOT}/lib
+)
+
+find_library(Brotli_dec_LIBRARY
+  brotlidec
+  HINTS $ENV{Brotli_ROOT}/lib
+)
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(Brotli
+  DEFAULT_MSG
+  Brotli_common_LIBRARY
+  Brotli_enc_LIBRARY
+  Brotli_dec_LIBRARY
+  Brotli_INCLUDE_DIR
+)
+
+mark_as_advanced(Brotli_INCLUDE_DIR Brotli_common_LIBRARY Brotli_enc_LIBRARY Brotli_dec_LIBRARY)
+
+if (Brotli_FOUND)
+  if (NOT TARGET Brotli::common)
+    add_library(Brotli::common UNKNOWN IMPORTED)
+    set_target_properties(Brotli::common PROPERTIES
+      IMPORTED_LOCATION ${Brotli_common_LIBRARY}
+      INTERFACE_INCLUDE_DIRECTORIES ${Brotli_INCLUDE_DIR}
+    )
+  endif()
+  if (NOT TARGET Brotli::dec)
+    add_library(Brotli::dec UNKNOWN IMPORTED)
+    set_target_properties(Brotli::dec PROPERTIES
+      IMPORTED_LOCATION ${Brotli_dec_LIBRARY}
+      INTERFACE_INCLUDE_DIRECTORIES ${Brotli_INCLUDE_DIR}
+    )
+    set_property(TARGET Brotli::dec APPEND PROPERTY INTERFACE_LINK_LIBRARIES 
+      Brotli::common
+    )
+  endif()
+  if (NOT TARGET Brotli::enc)
+    add_library(Brotli::enc UNKNOWN IMPORTED)
+    set_target_properties(Brotli::enc PROPERTIES
+      IMPORTED_LOCATION ${Brotli_enc_LIBRARY}
+      INTERFACE_INCLUDE_DIRECTORIES ${Brotli_INCLUDE_DIR}
+    )
+    set_property(TARGET Brotli::enc APPEND PROPERTY INTERFACE_LINK_LIBRARIES 
+      Brotli::common
+      Brotli::dec
+    )
+  endif()
+endif()
diff --git a/cmake/FindIDN.cmake b/cmake/FindIDN.cmake
@@ -6,11 +6,13 @@
 
 find_path(IDN_INCLUDE_DIR
   idna.h
-  HINTS $ENV{IDN_ROOT}/include)
+  HINTS $ENV{IDN_ROOT}/include
+)
 
 find_library(IDN_LIBRARIES
   idn
-  HINTS $ENV{IDN_ROOT}/lib)
+  HINTS $ENV{IDN_ROOT}/lib
+)
 
 include(FindPackageHandleStandardArgs)
 find_package_handle_standard_args(IDN DEFAULT_MSG IDN_LIBRARIES IDN_INCLUDE_DIR)

diff --git a/cmake/FindZSTD.cmake b/cmake/FindZSTD.cmake
@@ -6,11 +6,13 @@
 
 find_path(ZSTD_INCLUDE_DIR
   zstd.h
-  HINTS $ENV{ZSTD_ROOT}/include)
+  HINTS $ENV{ZSTD_ROOT}/include
+)
 
 find_library(ZSTD_LIBRARIES
   zstd
-  HINTS $ENV{ZSTD_ROOT}/lib)
+  HINTS $ENV{ZSTD_ROOT}/lib
+)
 
 include(FindPackageHandleStandardArgs)
 find_package_handle_standard_args(ZSTD DEFAULT_MSG ZSTD_LIBRARIES ZSTD_INCLUDE_DIR)

diff --git a/cmake/FindgRPC.cmake b/cmake/FindgRPC.cmake
@@ -1,24 +1,24 @@
 if (NOT Threads_FOUND)
-    find_package(Threads REQUIRED)
+  find_package(Threads REQUIRED)
 endif()
 
 if (NOT Protobuf_FOUND)
-    find_package(Protobuf REQUIRED)
+  find_package(Protobuf REQUIRED)
 endif()
 
 find_library(gRPC_grpc_LIBRARY
-    grpc
-    HINTS $ENV{gRPC_ROOT}/lib
+  grpc
+  HINTS $ENV{gRPC_ROOT}/lib
 )
 
 find_library(gRPC_grpc++_LIBRARY
-    grpc++
-    HINTS $ENV{gRPC_ROOT}/lib
+  grpc++
+  HINTS $ENV{gRPC_ROOT}/lib
 )
 
 find_path(gRPC_INCLUDE_DIR
-    grpcpp/grpcpp.h
-    HINTS $ENV{gRPC_ROOT}/include
+  grpcpp/grpcpp.h
+  HINTS $ENV{gRPC_ROOT}/include
 )
 
 include(FindPackageHandleStandardArgs)
@@ -27,40 +27,40 @@ find_package_handle_standard_args(gRPC DEFAULT_MSG gRPC_grpc++_LIBRARY gRPC_grpc
 mark_as_advanced(gRPC_grcp++_LIBRARIES gRPC_grpc_LIBRARIES gRPC_INCLUDE_DIR)
 
 if (gRPC_FOUND)
-    if (NOT TARGET gRPC::grpc)
-        add_library(gRPC::grpc UNKNOWN IMPORTED)
-        set_target_properties(gRPC::grpc PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${gRPC_INCLUDE_DIR}")
-        set_target_properties(gRPC::grpc PROPERTIES IMPORTED_LOCATION "${gRPC_grpc_LIBRARY}")
-        set_property(TARGET gRPC::grpc APPEND PROPERTY INTERFACE_LINK_LIBRARIES 
-            protobuf::libprotobuf
-            Threads::Threads
-        )
-    endif()
+  if (NOT TARGET gRPC::grpc)
+    add_library(gRPC::grpc UNKNOWN IMPORTED)
+    set_target_properties(gRPC::grpc PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${gRPC_INCLUDE_DIR}")
+    set_target_properties(gRPC::grpc PROPERTIES IMPORTED_LOCATION "${gRPC_grpc_LIBRARY}")
+    set_property(TARGET gRPC::grpc APPEND PROPERTY INTERFACE_LINK_LIBRARIES 
+      protobuf::libprotobuf
+      Threads::Threads
+    )
+  endif()
 
-    if (NOT TARGET gRPC::grpc++)
-        add_library(gRPC::grpc++ UNKNOWN IMPORTED)
-        set_target_properties(gRPC::grpc++ PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${gRPC_INCLUDE_DIR}")
-        set_target_properties(gRPC::grpc++ PROPERTIES IMPORTED_LOCATION "${gRPC_grpc++_LIBRARY}")
-        set_property(TARGET gRPC::grpc++ APPEND PROPERTY INTERFACE_LINK_LIBRARIES 
-            gRPC::grpc
-            protobuf::libprotobuf
-            Threads::Threads
-        )
-    endif()
+  if (NOT TARGET gRPC::grpc++)
+    add_library(gRPC::grpc++ UNKNOWN IMPORTED)
+    set_target_properties(gRPC::grpc++ PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${gRPC_INCLUDE_DIR}")
+    set_target_properties(gRPC::grpc++ PROPERTIES IMPORTED_LOCATION "${gRPC_grpc++_LIBRARY}")
+    set_property(TARGET gRPC::grpc++ APPEND PROPERTY INTERFACE_LINK_LIBRARIES 
+      gRPC::grpc
+      protobuf::libprotobuf
+      Threads::Threads
+    )
+  endif()
 
-    if (NOT TARGET gRPC::grpc_cpp_plugin)
-        add_executable(gRPC::grpc_cpp_plugin IMPORTED)
-    endif()
+  if (NOT TARGET gRPC::grpc_cpp_plugin)
+    add_executable(gRPC::grpc_cpp_plugin IMPORTED)
+  endif()
 
-    get_target_property(_gRPC_CPP_PLUGIN_EXECUTABLE gRPC::grpc_cpp_plugin IMPORTED_LOCATION)
+  get_target_property(_gRPC_CPP_PLUGIN_EXECUTABLE gRPC::grpc_cpp_plugin IMPORTED_LOCATION)
 
-    if (NOT _gRPC_CPP_PLUGIN_EXECUTABLE)
-        find_program(_gRPC_CPP_PLUGIN_EXECUTABLE grpc_cpp_plugin DOC "The gRPC C++ plugin for protoc")
-        mark_as_advanced(_gRPC_CPP_PLUGIN_EXECUTABLE)
-        if (_gRPC_CPP_PLUGIN_EXECUTABLE)
-            set_property(TARGET gRPC::grpc_cpp_plugin PROPERTY IMPORTED_LOCATION ${_gRPC_CPP_PLUGIN_EXECUTABLE})
-        else()
-            set(gRPC_FOUND "grpc_cpp_plugin-NOTFOUND")
-        endif()
+  if (NOT _gRPC_CPP_PLUGIN_EXECUTABLE)
+    find_program(_gRPC_CPP_PLUGIN_EXECUTABLE grpc_cpp_plugin DOC "The gRPC C++ plugin for protoc")
+    mark_as_advanced(_gRPC_CPP_PLUGIN_EXECUTABLE)
+    if (_gRPC_CPP_PLUGIN_EXECUTABLE)
+      set_property(TARGET gRPC::grpc_cpp_plugin PROPERTY IMPORTED_LOCATION ${_gRPC_CPP_PLUGIN_EXECUTABLE})
+    else()
+      set(gRPC_FOUND "grpc_cpp_plugin-NOTFOUND")
     endif()
-endif()
+  endif()
+endif()
diff --git a/cmake/FindxxHash.cmake b/cmake/FindxxHash.cmake
@@ -6,11 +6,13 @@
 
 find_path(xxHash_INCLUDE_DIR
   xxhash.h
-  HINTS $ENV{xxHash_ROOT}/include)
+  HINTS $ENV{xxHash_ROOT}/include
+)
 
 find_library(xxHash_LIBRARIES
   xxhash
-  HINTS $ENV{xxHash_ROOT}/lib)
+  HINTS $ENV{xxHash_ROOT}/lib
+)
 
 include(FindPackageHandleStandardArgs)
 find_package_handle_standard_args(xxHash DEFAULT_MSG xxHash_LIBRARIES xxHash_INCLUDE_DIR)

diff --git a/cmake/common.cmake b/cmake/common.cmake
@@ -5,21 +5,10 @@ find_package(Python3 REQUIRED)
 add_compile_definitions(CATBOOST_OPENSOURCE=yes)
 
 # assumes ToolName is always both the binary and the target name
-function(get_built_tool_path OutBinPath OutDependency SrcPath ToolName)
-  if (CMAKE_GENERATOR MATCHES "Visual.Studio.*")
-    set(BinPath "${TOOLS_ROOT}/${SrcPath}/\$(Configuration)/${ToolName}${CMAKE_EXECUTABLE_SUFFIX}")
-  else()
-    set(BinPath "${TOOLS_ROOT}/${SrcPath}/${ToolName}${CMAKE_EXECUTABLE_SUFFIX}")
-  endif()
-  set(${OutBinPath} ${BinPath} PARENT_SCOPE)
-  if (CMAKE_CROSSCOMPILING)
-    set(${OutDependency} ${BinPath} PARENT_SCOPE)
-  else()
-    set(${OutDependency} ${ToolName} PARENT_SCOPE)
-  endif()
+function(get_built_tool_path OutBinPath SrcPath ToolName)
+  set(${OutBinPath} "${CMAKE_BINARY_DIR}/${SrcPath}/${ToolName}${CMAKE_EXECUTABLE_SUFFIX}" PARENT_SCOPE)
 endfunction()
 
-
 function(target_ragel_lexers TgtName Key Src)
   SET(RAGEL_BIN ragel${CMAKE_EXECUTABLE_SUFFIX})
   get_filename_component(OutPath ${Src} NAME_WLE)
@@ -112,7 +101,7 @@ function(generate_enum_serilization Tgt Input)
     ${ARGN}
   )
 
-  get_built_tool_path(enum_parser_bin enum_parser_dependency tools/enum_parser/enum_parser enum_parser)
+  get_built_tool_path(enum_parser_bin tools/enum_parser/enum_parser enum_parser)
 
   get_filename_component(BaseName ${Input} NAME)
   add_custom_command(
@@ -122,7 +111,7 @@ function(generate_enum_serilization Tgt Input)
       ${Input}
       --include-path ${ENUM_SERIALIZATION_ARGS_INCLUDE_HEADERS}
       --output ${CMAKE_CURRENT_BINARY_DIR}/${BaseName}_serialized.cpp
-    DEPENDS ${Input} ${enum_parser_dependency}
+    DEPENDS ${Input} ${enum_parser_bin}
   )
   target_sources(${Tgt} PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/${BaseName}_serialized.cpp)
 endfunction()
@@ -189,12 +178,12 @@ function(resources Tgt Output)
     list(APPEND ResourcesList ${Key})
   endforeach()
 
-  get_built_tool_path(rescompiler_bin rescompiler_dependency tools/rescompiler/bin rescompiler)
+  get_built_tool_path(rescompiler_bin tools/rescompiler/bin rescompiler)
 
   add_custom_command(
     OUTPUT ${Output}
     COMMAND ${rescompiler_bin} ${Output} ${ResourcesList}
-    DEPENDS ${RESOURCE_ARGS_INPUTS} ${rescompiler_dependency}
+    DEPENDS ${RESOURCE_ARGS_INPUTS} ${rescompiler_bin}
   )
 endfunction()