HPCC-29432 rebase and fix issues dependency/package issues for 9.4.x

Signed-off-by: Michael Gardner <[email protected]>
hpcc-systems · Sep 22, 2023 · 3c10138 · 3c10138
1 parent 85f7c98
commit 3c10138
Show file tree

Hide file tree

Showing 26 changed files with 253 additions and 99 deletions.
diff --git a/.github/workflows/build-assets.yml b/.github/workflows/build-assets.yml
@@ -185,7 +185,7 @@ jobs:
         run: |
           mkdir -p ${{ needs.preamble.outputs.folder_build }}
           echo "${{ secrets.SIGNING_SECRET }}" > ${{ needs.preamble.outputs.folder_build }}/private.key
-          plugins=("CASSANDRAEMBED" "COUCHBASEEMBED" "ECLBLAS" "H3" "JAVAEMBED" "KAFKA" "MEMCACHED" "MONGODBEMBED" "MYSQLEMBED" "NLP" "REDIS" "REMBED" "SQLITE3EMBED" "SQS" "PLATFORM" "CLIENTTOOLS_ONLY")
+          plugins=("COUCHBASEEMBED" "H3" "JAVAEMBED" "KAFKA" "MEMCACHED" "MONGODBEMBED" "MYSQLEMBED" "NLP" "REDIS" "REMBED" "SQLITE3EMBED" "SQS" "PLATFORM" "CLIENTTOOLS_ONLY")
           for plugin in "${plugins[@]}"; do
             sudo rm -f ${{ needs.preamble.outputs.folder_build }}/CMakeCache.txt
             sudo rm -rf ${{ needs.preamble.outputs.folder_build }}/CMakeFiles

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -150,22 +150,22 @@ endif()
 include(${HPCC_SOURCE_DIR}/cmake_modules/commonSetup.cmake)
 
 if ( PLUGIN )
-    HPCC_ADD_SUBDIRECTORY (common/deftype)
-    HPCC_ADD_SUBDIRECTORY (common/dllserver)
-    HPCC_ADD_SUBDIRECTORY (common/environment)
-    HPCC_ADD_SUBDIRECTORY (common/remote)
-    HPCC_ADD_SUBDIRECTORY (common/workunit)
-    HPCC_ADD_SUBDIRECTORY (ecl/hql)
-    HPCC_ADD_SUBDIRECTORY (fs/dafsclient)
+    HPCC_ADD_SUBDIRECTORY (common/deftype "NLP")
+    HPCC_ADD_SUBDIRECTORY (common/dllserver "NLP")
+    HPCC_ADD_SUBDIRECTORY (common/environment "NLP")
+    HPCC_ADD_SUBDIRECTORY (common/remote "NLP")
+    HPCC_ADD_SUBDIRECTORY (common/workunit "NLP")
+    HPCC_ADD_SUBDIRECTORY (ecl/hql "NLP")
+    HPCC_ADD_SUBDIRECTORY (fs/dafsclient "NLP")
     HPCC_ADD_SUBDIRECTORY (roxie/roxiemem)
     HPCC_ADD_SUBDIRECTORY (rtl/nbcd)
     HPCC_ADD_SUBDIRECTORY (rtl/eclrtl)
     HPCC_ADD_SUBDIRECTORY (system/jlib)
-    HPCC_ADD_SUBDIRECTORY (system/jhtree)
-    HPCC_ADD_SUBDIRECTORY (system/hrpc)
-    HPCC_ADD_SUBDIRECTORY (system/mp)
-    HPCC_ADD_SUBDIRECTORY (system/security/securesocket)
-    HPCC_ADD_SUBDIRECTORY (system/security/zcrypt)
+    HPCC_ADD_SUBDIRECTORY (system/jhtree "NLP")
+    HPCC_ADD_SUBDIRECTORY (system/hrpc "NLP")
+    HPCC_ADD_SUBDIRECTORY (system/mp "NLP")
+    HPCC_ADD_SUBDIRECTORY (system/security/securesocket "NLP")
+    HPCC_ADD_SUBDIRECTORY (system/security/zcrypt "NLP")
     HPCC_ADD_SUBDIRECTORY (dali/base)
     HPCC_ADD_SUBDIRECTORY (plugins/Rembed "REMBED")
     HPCC_ADD_SUBDIRECTORY (plugins/v8embed "V8EMBED")
@@ -218,7 +218,7 @@ if(APPLE)
     HPCC_ADD_SUBDIRECTORY(package)
 endif(APPLE)
 
-if (NOT CLIENTTOOLS_ONLY)
+if (PLATFORM)
     install(DIRECTORY ${VCPKG_INSTALLED_DIR}/${VCPKG_TARGET_TRIPLET}/include DESTINATION ${COMPONENTFILES_DIR}/cl FILES_MATCHING PATTERN "*")
 endif()
 

diff --git a/cmake_modules/options.cmake b/cmake_modules/options.cmake
@@ -9,6 +9,7 @@ option(DEVEL "Enable the building/inclusion of a Development component." OFF)
 option(CLIENTTOOLS_ONLY "Enable the building of Client Tools only." OFF)
 option(INCLUDE_PLUGINS "Enable the building of platform and all plugins for testing purposes" OFF)
 option(USE_CASSANDRA "Include the Cassandra plugin in the base package" ON)
+option(USE_ECLBLAS "Include the ECLBLAS plugin in the base package" ON)
 option(PLUGIN "Enable building of a plugin" OFF)
 option(USE_SHLIBDEPS "Enable the use of dpkg-shlibdeps on ubuntu packaging" OFF)
 

diff --git a/cmake_modules/plugins.cmake b/cmake_modules/plugins.cmake
@@ -26,9 +26,7 @@ set(VCPKG_INCLUDE "(windows | osx | linux)")
 set(VCPKG_SUPPRESS "(!windows & !osx & !linux)")
 
 set(PLUGINS_LIST
-    CASSANDRAEMBED
     COUCHBASEEMBED
-    ECLBLAS
     H3
     JAVAEMBED
     KAFKA
@@ -70,6 +68,10 @@ if (USE_CASSANDRA)
     set(VCPKG_CASSANDRAEMBED "${VCPKG_INCLUDE}")
 endif()
 
+if (USE_ECLBLAS)
+    set(VCPKG_ECLBLAS "${VCPKG_INCLUDE}")
+endif()
+
 if (USE_LIBMEMCACHED)
     set(VCPKG_MEMCACHED "${VCPKG_INCLUDE}")
 endif()

diff --git a/cmake_modules/vcpkg.cmake b/cmake_modules/vcpkg.cmake
@@ -22,7 +22,7 @@ foreach(VCPKG_PACKAGE ${VCPKG_PACKAGES})
     list(APPEND VCPKG_PACKAGE_LIST "-----------------\n")
 endforeach()
 file(WRITE ${CMAKE_BINARY_DIR}/vcpkg-catalog.txt ${VCPKG_PACKAGE_LIST})
-if (INSTALL_VCPKG_CATALOG)
+if (INSTALL_VCPKG_CATALOG AND PLATFORM)
     install(FILES ${CMAKE_BINARY_DIR}/vcpkg-catalog.txt DESTINATION "." COMPONENT Runtime)
 endif()
 

diff --git a/common/deftype/CMakeLists.txt b/common/deftype/CMakeLists.txt
@@ -52,7 +52,14 @@ endif()
 ADD_DEFINITIONS( -D_USRDLL -DDEFTYPE_EXPORTS )
 
 HPCC_ADD_LIBRARY( deftype SHARED ${SRCS} )
-install ( TARGETS deftype RUNTIME DESTINATION ${EXEC_DIR} LIBRARY DESTINATION ${LIB_DIR} )
+
+if(NOT PLUGIN)
+     install(
+          TARGETS deftype
+          RUNTIME DESTINATION ${EXEC_DIR}
+          LIBRARY DESTINATION ${LIB_DIR})
+endif()
+
 target_link_libraries ( deftype 
          jlib 
          nbcd

diff --git a/common/dllserver/CMakeLists.txt b/common/dllserver/CMakeLists.txt
@@ -55,7 +55,14 @@ ENDIF()
 ADD_DEFINITIONS( -D_USRDLL -DDLLSERVER_EXPORTS )
 
 HPCC_ADD_LIBRARY( dllserver SHARED ${SRCS} )
-install ( TARGETS dllserver RUNTIME DESTINATION ${EXEC_DIR} LIBRARY DESTINATION ${LIB_DIR} )
+
+if(NOT PLUGIN)
+  install(
+    TARGETS dllserver
+    RUNTIME DESTINATION ${EXEC_DIR}
+    LIBRARY DESTINATION ${LIB_DIR})
+endif()
+
 target_link_libraries ( dllserver
          ${ZLIB_LIBRARIES}
          jlib 

diff --git a/common/environment/CMakeLists.txt b/common/environment/CMakeLists.txt
@@ -46,7 +46,14 @@ include_directories (
 
 ADD_DEFINITIONS( -D_USRDLL -DENVIRONMENT_EXPORTS )
 HPCC_ADD_LIBRARY( environment SHARED ${SRCS} ${INCLUDES} )
-install ( TARGETS environment RUNTIME DESTINATION ${EXEC_DIR} LIBRARY DESTINATION ${LIB_DIR} )
+
+if(NOT PLUGIN)
+     install(
+          TARGETS environment
+          RUNTIME DESTINATION ${EXEC_DIR}
+          LIBRARY DESTINATION ${LIB_DIR})
+endif()
+
 target_link_libraries ( environment 
          jlib 
          mp 

diff --git a/common/remote/CMakeLists.txt b/common/remote/CMakeLists.txt
@@ -62,7 +62,13 @@ include_directories (
 ADD_DEFINITIONS( -D_USRDLL -DREMOTE_EXPORTS )
 
 HPCC_ADD_LIBRARY( remote SHARED ${SRCS}  )
-install ( TARGETS remote RUNTIME DESTINATION ${EXEC_DIR} LIBRARY DESTINATION ${LIB_DIR} )
+
+if(NOT PLUGIN)
+    install(
+        TARGETS remote
+        RUNTIME DESTINATION ${EXEC_DIR}
+        LIBRARY DESTINATION ${LIB_DIR})
+endif()
 
 target_link_libraries ( remote 
     eclrtl

diff --git a/common/remote/hooks/azure/CMakeLists.txt b/common/remote/hooks/azure/CMakeLists.txt
@@ -45,7 +45,13 @@ include_directories (
 ADD_DEFINITIONS( -D_USRDLL -DAZURE_FILE_EXPORTS )
 
 HPCC_ADD_LIBRARY( azurefile SHARED ${SRCS}  )
-install ( TARGETS azurefile DESTINATION filehooks CALC_DEPS)
+
+if(NOT PLUGIN)
+    install(
+        TARGETS azurefile
+        DESTINATION filehooks
+        CALC_DEPS)
+endif()
 
 target_link_libraries ( azurefile
         jlib

diff --git a/common/remote/hooks/git/CMakeLists.txt b/common/remote/hooks/git/CMakeLists.txt
@@ -40,7 +40,13 @@ include_directories (
 ADD_DEFINITIONS( -D_USRDLL -DGITFILE_EXPORTS )
 
 HPCC_ADD_LIBRARY( gitfile SHARED ${SRCS}  )
-install ( TARGETS gitfile DESTINATION filehooks CALC_DEPS)
+
+if(NOT PLUGIN)
+    install(
+        TARGETS gitfile
+        DESTINATION filehooks
+        CALC_DEPS)
+endif()
 
 target_link_libraries ( gitfile
     jlib

diff --git a/common/remote/hooks/libarchive/CMakeLists.txt b/common/remote/hooks/libarchive/CMakeLists.txt
@@ -42,7 +42,14 @@ if ( WIN32 )
 endif ( WIN32 )
 
 HPCC_ADD_LIBRARY( archivefile SHARED ${SRCS}  )
-install ( TARGETS archivefile RUNTIME DESTINATION filehooks LIBRARY DESTINATION filehooks CALC_DEPS )
+
+if(NOT PLUGIN)
+    install(
+        TARGETS archivefile
+        RUNTIME DESTINATION filehooks
+        LIBRARY DESTINATION filehooks
+        CALC_DEPS)
+endif()
 
 target_link_libraries ( archivefile
     jlib

diff --git a/common/remote/hooks/s3/CMakeLists.txt b/common/remote/hooks/s3/CMakeLists.txt
@@ -43,7 +43,13 @@ include_directories (
 ADD_DEFINITIONS( -D_USRDLL -DS3FILE_EXPORTS )
 
 HPCC_ADD_LIBRARY( s3file SHARED ${SRCS}  )
-install ( TARGETS s3file DESTINATION filehooks CALC_DEPS)
+
+if(NOT PLUGIN)
+    install(
+        TARGETS s3file
+        DESTINATION filehooks
+        CALC_DEPS)
+endif()
 
 target_link_libraries ( s3file
     jlib

diff --git a/ecl/hql/CMakeLists.txt b/ecl/hql/CMakeLists.txt
@@ -176,7 +176,14 @@ endif()
 ADD_DEFINITIONS( -D_USRDLL -DHQL_EXPORTS -DHQLFOLD_EXPORTS -DHQLTRANS_EXPORTS )
 
 HPCC_ADD_LIBRARY( hql SHARED ${SRCS} ${CMAKE_CURRENT_BINARY_DIR}/hqlgram.cpp ${CMAKE_CURRENT_BINARY_DIR}/hqllex.cpp  )
-install ( TARGETS hql RUNTIME DESTINATION ${EXEC_DIR} LIBRARY DESTINATION ${LIB_DIR} )
+
+if(NOT PLUGIN)
+  install(
+    TARGETS hql
+    RUNTIME DESTINATION ${EXEC_DIR}
+    LIBRARY DESTINATION ${LIB_DIR})
+endif()
+
 target_link_libraries ( hql
          jlib
          nbcd

diff --git a/fs/dafsclient/CMakeLists.txt b/fs/dafsclient/CMakeLists.txt
@@ -48,7 +48,13 @@ include_directories (
 ADD_DEFINITIONS( -D_USRDLL -DDAFSCLIENT_EXPORTS )
 
 HPCC_ADD_LIBRARY( dafsclient SHARED ${SRCS}  )
-install ( TARGETS dafsclient RUNTIME DESTINATION ${EXEC_DIR} LIBRARY DESTINATION ${LIB_DIR} )
+
+if(NOT PLUGIN)
+    install(
+        TARGETS dafsclient
+        RUNTIME DESTINATION ${EXEC_DIR}
+        LIBRARY DESTINATION ${LIB_DIR})
+endif()
 
 target_link_libraries ( dafsclient 
     jlib

diff --git a/plugins/cassandra/CMakeLists.txt b/plugins/cassandra/CMakeLists.txt
@@ -24,9 +24,8 @@
 
 project(cassandraembed)
 
-if(CASSANDRAEMBED OR USE_CASSANDRA)
+if(USE_CASSANDRA)
   ADD_PLUGIN(cassandraembed)
-  if(MAKE_CASSANDRAEMBED OR USE_CASSANDRA)
     find_path(CPP_DRIVER_INCLUDE_DIRS "cassandra.h" REQUIRED)
     find_library(CPP_DRIVER_LIBRARY NAMES "cassandra" REQUIRED)
     find_package(libuv CONFIG REQUIRED)
@@ -75,7 +74,6 @@ if(CASSANDRAEMBED OR USE_CASSANDRA)
       ${CPP_DRIVER_LIBRARY}
       $<IF:$<TARGET_EXISTS:libuv::uv_a>,libuv::uv_a,libuv::uv>)
 
-  endif()
 endif()
 
 if(PLATFORM OR CLIENTTOOLS_ONLY)

diff --git a/plugins/couchbase/CMakeLists.txt b/plugins/couchbase/CMakeLists.txt
@@ -50,10 +50,25 @@ IF ( COUCHBASEEMBED )
         ADD_DEFINITIONS(-D_USRDLL -DCOUCHBASEEMBED_EXPORTS)
         HPCC_ADD_LIBRARY(couchbaseembed SHARED ${SRCS})
         install(CODE "set(ENV{LD_LIBRARY_PATH} \"\$ENV{LD_LIBRARY_PATH}:${PROJECT_BINARY_DIR}:${PROJECT_BINARY_DIR}/libcouchbase-build/lib\")")
-        install(
-            TARGETS couchbaseembed
-            DESTINATION plugins
-            CALC_DEPS)
+
+        if(WIN32 OR APPLE)
+            install(
+                TARGETS couchbaseembed
+                DESTINATION plugins
+                CALC_DEPS)
+        else()
+            install(
+                RUNTIME_DEPENDENCY_SET couchbaseembed_deps
+                DIRECTORIES "${VCPKG_FILES_DIR}/vcpkg_installed"
+                PRE_EXCLUDE_REGEXES "libcrypto|libssl"
+                POST_INCLUDE_REGEXES "^${VCPKG_FILES_DIR}\/vcpkg_installed\/.*"
+                POST_EXCLUDE_REGEXES ".*")
+            install(
+                TARGETS couchbaseembed
+                RUNTIME_DEPENDENCY_SET couchbaseembed_deps
+                DESTINATION plugins)
+        endif()
+
 
         target_link_libraries(
             couchbaseembed

diff --git a/plugins/eclblas/CMakeLists.txt b/plugins/eclblas/CMakeLists.txt
@@ -25,46 +25,44 @@
 
 
 project(eclblas)
+# Added as default library, no longer 'plugin'
 
-if(ECLBLAS)
+
+if(USE_ECLBLAS)
     ADD_PLUGIN(eclblas)
-    if(MAKE_ECLBLAS)
-        find_package(OpenBLAS REQUIRED)
+    find_package(OpenBLAS REQUIRED)
 
-        add_definitions(-D_USE_CBLAS)
+    add_definitions(-D_USE_CBLAS)
 
-        set(SRCS
-            dasum.cpp
-            daxpy.cpp
-            dgemm.cpp
-            dgetf2.cpp
-            dpotf2.cpp
-            dscal.cpp
-            dsyrk.cpp
-            dtrsm.cpp
-            eclblas.cpp
-            extract_tri.cpp
-            make_diag.cpp)
+    set(SRCS
+        dasum.cpp
+        daxpy.cpp
+        dgemm.cpp
+        dgetf2.cpp
+        dpotf2.cpp
+        dscal.cpp
+        dsyrk.cpp
+        dtrsm.cpp
+        eclblas.cpp
+        extract_tri.cpp
+        make_diag.cpp)
 
-        include_directories(
-            ./../../system/include
-            ./../../rtl/eclrtl
-            ./../../rtl/include
-            ./../../common/deftype
-            ./../../system/jlib
-            ${OpenBLAS_DIR})
+    include_directories(
+        ./../../system/include
+        ./../../rtl/eclrtl
+        ./../../rtl/include
+        ./../../common/deftype
+        ./../../system/jlib
+        ${OpenBLAS_DIR})
 
-        ADD_DEFINITIONS(-D_USRDLL -DECLBLAS_EXPORTS)
+    ADD_DEFINITIONS(-D_USRDLL -DECLBLAS_EXPORTS)
 
-        HPCC_ADD_LIBRARY(eclblas SHARED ${SRCS})
-        install ( TARGETS eclblas RUNTIME DESTINATION ${EXEC_DIR} LIBRARY DESTINATION ${LIB_DIR} CALC_DEPS)  # No need to put in plugins dir
-        target_link_libraries(eclblas
-            eclrtl
-            OpenBLAS::OpenBLAS)
-    else()
-        message(AUTHOR_WARNING "Not building eclblas library for standard library due to lacking libcblas")
-    endif()
-endif()
+    HPCC_ADD_LIBRARY(eclblas SHARED ${SRCS})
+    install ( TARGETS eclblas RUNTIME DESTINATION ${EXEC_DIR} LIBRARY DESTINATION ${LIB_DIR} CALC_DEPS)  # No need to put in plugins dir
+    target_link_libraries(eclblas
+        eclrtl
+        OpenBLAS::OpenBLAS)
+endif(USE_ECLBLAS)
 
 if(PLATFORM OR CLIENTTOOLS_ONLY)
     install(