trying to add mlpack to clickhouse

CMakeLists.txt

@@ -415,6 +415,7 @@ elseif (OS_FREEBSD)
 endif ()
 link_libraries(global-group)
 
+set (WERROR OFF)
 if (NOT (OS_LINUX OR OS_DARWIN))
     # Using system libs can cause a lot of warnings in includes (on macro expansion).
     option(WERROR "Enable -Werror compiler option" OFF)

cmake/find/FindMLPACK.cmake

@@ -0,0 +1,51 @@
+#.rst:
+# FindMLPACK
+# -------------
+#
+# Find MLPACK
+#
+# Find the MLPACK C++ library
+#
+# Using MLPACK::
+#
+#   find_package(MLPACK REQUIRED)
+#   include_directories(${MLPACK_INCLUDE_DIRS})
+#   add_executable(foo foo.cc)
+#   target_link_libraries(foo ${MLPACK_LIBRARIES})
+#
+# This module sets the following variables::
+#
+#   MLPACK_FOUND - set to true if the library is found
+#   MLPACK_INCLUDE_DIRS - list of required include directories
+#   MLPACK_LIBRARIES - list of libraries to be linked
+#   MLPACK_VERSION_MAJOR - major version number
+#   MLPACK_VERSION_MINOR - minor version number
+#   MLPACK_VERSION_PATCH - patch version number
+#   MLPACK_VERSION_STRING - version number as a string (ex: "1.0.4")
+
+include(FindPackageHandleStandardArgs)
+
+# UNIX paths are standard, no need to specify them.
+find_library(MLPACK_LIBRARY
+	NAMES mlpack
+	PATHS "$ENV{ProgramFiles}/mlpack/lib"  "$ENV{ProgramFiles}/mlpack/lib64" "$ENV{ProgramFiles}/mlpack"
+)
+find_path(MLPACK_INCLUDE_DIR
+	NAMES mlpack/core.hpp mlpack/prereqs.hpp
+	PATHS "$ENV{ProgramFiles}/mlpack"
+)
+
+find_package_handle_standard_args(MLPACK
+	REQUIRED_VARS MLPACK_LIBRARY MLPACK_INCLUDE_DIR
+)
+
+if(MLPACK_FOUND)
+	set(MLPACK_INCLUDE_DIRS ${MLPACK_INCLUDE_DIR})
+	set(MLPACK_LIBRARIES ${MLPACK_LIBRARY})
+endif()
+
+# Hide internal variables
+mark_as_advanced(
+	MLPACK_INCLUDE_DIR
+	MLPACK_LIBRARY
+)

contrib/boost-cmake/CMakeLists.txt

@@ -183,3 +183,58 @@ target_include_directories(_boost_circular_buffer SYSTEM BEFORE INTERFACE ${LIBR
 add_library(_boost_heap INTERFACE)
 add_library(boost::heap ALIAS _boost_heap)
 target_include_directories(_boost_heap SYSTEM BEFORE INTERFACE ${LIBRARY_DIR})
+
+set (SRCS_SERIALIZATION
+    "${LIBRARY_DIR}/libs/serialization/src/archive_exception.cpp"
+    "${LIBRARY_DIR}/libs/serialization/src/basic_archive.cpp"
+    "${LIBRARY_DIR}/libs/serialization/src/basic_iarchive.cpp"
+    "${LIBRARY_DIR}/libs/serialization/src/basic_iserializer.cpp"
+    "${LIBRARY_DIR}/libs/serialization/src/basic_oarchive.cpp"
+    "${LIBRARY_DIR}/libs/serialization/src/basic_oserializer.cpp"
+    "${LIBRARY_DIR}/libs/serialization/src/basic_pointer_iserializer.cpp"
+    "${LIBRARY_DIR}/libs/serialization/src/basic_pointer_oserializer.cpp"
+    "${LIBRARY_DIR}/libs/serialization/src/basic_serializer_map.cpp"
+    "${LIBRARY_DIR}/libs/serialization/src/basic_text_iprimitive.cpp"
+    "${LIBRARY_DIR}/libs/serialization/src/basic_text_oprimitive.cpp"
+    "${LIBRARY_DIR}/libs/serialization/src/basic_text_wiprimitive.cpp"
+    "${LIBRARY_DIR}/libs/serialization/src/basic_text_woprimitive.cpp"
+    "${LIBRARY_DIR}/libs/serialization/src/basic_xml_archive.cpp"
+    "${LIBRARY_DIR}/libs/serialization/src/basic_xml_grammar.ipp"
+    "${LIBRARY_DIR}/libs/serialization/src/binary_iarchive.cpp"
+    "${LIBRARY_DIR}/libs/serialization/src/binary_oarchive.cpp"
+    "${LIBRARY_DIR}/libs/serialization/src/binary_wiarchive.cpp"
+    "${LIBRARY_DIR}/libs/serialization/src/binary_woarchive.cpp"
+    "${LIBRARY_DIR}/libs/serialization/src/codecvt_null.cpp"
+    "${LIBRARY_DIR}/libs/serialization/src/extended_type_info_no_rtti.cpp"
+    "${LIBRARY_DIR}/libs/serialization/src/extended_type_info_typeid.cpp"
+    "${LIBRARY_DIR}/libs/serialization/src/extended_type_info.cpp"
+    "${LIBRARY_DIR}/libs/serialization/src/polymorphic_binary_iarchive.cpp"
+    "${LIBRARY_DIR}/libs/serialization/src/polymorphic_iarchive.cpp"
+    "${LIBRARY_DIR}/libs/serialization/src/polymorphic_oarchive.cpp"
+    "${LIBRARY_DIR}/libs/serialization/src/polymorphic_text_iarchive.cpp"
+    "${LIBRARY_DIR}/libs/serialization/src/polymorphic_text_oarchive.cpp"
+    "${LIBRARY_DIR}/libs/serialization/src/polymorphic_text_wiarchive.cpp"
+    "${LIBRARY_DIR}/libs/serialization/src/polymorphic_text_woarchive.cpp"
+    "${LIBRARY_DIR}/libs/serialization/src/polymorphic_xml_iarchive.cpp"
+    "${LIBRARY_DIR}/libs/serialization/src/polymorphic_xml_oarchive.cpp"
+    "${LIBRARY_DIR}/libs/serialization/src/polymorphic_xml_wiarchive.cpp"
+    "${LIBRARY_DIR}/libs/serialization/src/polymorphic_xml_woarchive.cpp"
+    "${LIBRARY_DIR}/libs/serialization/src/stl_port.cpp"
+    "${LIBRARY_DIR}/libs/serialization/src/text_iarchive.cpp"
+    "${LIBRARY_DIR}/libs/serialization/src/text_oarchive.cpp"
+    "${LIBRARY_DIR}/libs/serialization/src/text_wiarchive.cpp"
+    "${LIBRARY_DIR}/libs/serialization/src/text_woarchive.cpp"
+    "${LIBRARY_DIR}/libs/serialization/src/utf8_codecvt_facet.cpp"
+    "${LIBRARY_DIR}/libs/serialization/src/void_cast.cpp"
+    "${LIBRARY_DIR}/libs/serialization/src/xml_archive_exception.cpp"
+    "${LIBRARY_DIR}/libs/serialization/src/xml_grammar.cpp"
+    "${LIBRARY_DIR}/libs/serialization/src/xml_iarchive.cpp"
+    "${LIBRARY_DIR}/libs/serialization/src/xml_oarchive.cpp"
+    "${LIBRARY_DIR}/libs/serialization/src/xml_wgrammar.cpp"
+    "${LIBRARY_DIR}/libs/serialization/src/xml_wiarchive.cpp"
+    "${LIBRARY_DIR}/libs/serialization/src/xml_woarchive.cpp"
+)
+
+add_library (_boost_serialization ${SRCS_SERIALIZATION})
+add_library (boost::serialization ALIAS _boost_serialization)
+target_include_directories (_boost_serialization PRIVATE ${LIBRARY_DIR})

src/Access/Common/AccessType.h

@@ -185,6 +185,7 @@ enum class AccessType
     M(ODBC, "", GLOBAL, SOURCES) \
     M(JDBC, "", GLOBAL, SOURCES) \
     M(HDFS, "", GLOBAL, SOURCES) \
+    M(MLPACK, "", GLOBAL, SOURCES) \
     M(S3, "", GLOBAL, SOURCES) \
     M(HIVE, "", GLOBAL, SOURCES) \
     M(SOURCES, "", GROUP, ALL) \

src/CMakeLists.txt

@@ -87,6 +87,7 @@ add_headers_and_sources(clickhouse_common_io IO/S3)
 list (REMOVE_ITEM clickhouse_common_io_sources Common/malloc.cpp Common/new_delete.cpp)
 
 add_headers_and_sources(dbms Disks/IO)
+add_headers_and_sources(dbms Storages/MLpack)
 add_headers_and_sources(dbms Disks/ObjectStorages)
 if (TARGET ch_contrib::sqlite)
     add_headers_and_sources(dbms Databases/SQLite)
@@ -308,6 +309,25 @@ macro (dbms_target_link_libraries)
     endforeach ()
 endmacro ()
 
+include (../cmake/find/FindMLPACK.cmake)
+find_package(MLPACK REQUIRED)
+# include_directories(${MLPACK_INCLUDE_DIRS})
+# target_link_libraries(clickhouse_interpreters PUBLIC ${MLPACK_LIBRARIES})
+
+find_package(Armadillo REQUIRED)
+
+dbms_target_link_libraries(PUBLIC ${MLPACK_LIBRARIES}  ${ARMADILLO_LIBRARIES})
+target_link_libraries (clickhouse_common_io PUBLIC ${MLPACK_LIBRARIES} ${ARMADILLO_LIBRARIES})
+dbms_target_include_directories(PUBLIC ${MLPACK_INCLUDE_DIRS} ${ARMADILLO_INCLUDE_DIRS})
+
+target_link_libraries(${MLPACK_LIBRARIES} INTERFACE ${ARMADILLO_LIBRARIES})
+
+target_link_libraries(clickhouse_common_io PUBLIC _boost_serialization)
+# dbms_target_include_directories(PUBLIC _boost_serialization)
+# if (TARGET ${MLPACK_LIBRARIES})
+#     add_headers_and_sources(dbms Storages/MLpack)
+# endif()
+
 dbms_target_include_directories (PUBLIC "${ClickHouse_SOURCE_DIR}/src" "${ClickHouse_BINARY_DIR}/src")
 target_include_directories (clickhouse_common_io PUBLIC "${ClickHouse_SOURCE_DIR}/src" "${ClickHouse_BINARY_DIR}/src")
 

src/Common/ErrorCodes.cpp

@@ -630,6 +630,11 @@
     M(659, UNKNOWN_STATUS_OF_TRANSACTION) \
     M(660, HDFS_ERROR) \
     M(661, CANNOT_SEND_SIGNAL) \
+    M(662, FAILED_TO_RESTORE_METADATA_ON_OTHER_NODE) \
+    M(663, CANNOT_SAVE_MLPACK_MODEL) \
+    M(664, INSUFFICIENT_NUMBER_OF_COLUMNS) \
+    M(665, CANNOT_CONVERT_TO_FLOAT64) \
+    M(666, FEATURE_BUFFERS_EMPTY)\
     \
     M(999, KEEPER_EXCEPTION) \
     M(1000, POCO_EXCEPTION) \

src/Functions/modelEvaluate.cpp

@@ -14,7 +14,11 @@
 #include <Common/assert_cast.h>
 #include <Functions/IFunction.h>
 #include <Interpreters/Context_fwd.h>
-
+#include <Interpreters/DatabaseCatalog.h>
+#include <Storages/IStorage_fwd.h>
+#include <Storages/IStorage.h>
+#include <Databases/IDatabase.h>
+#include <Storages/MLpack/StorageMLmodel.h>
 
 namespace DB
 {
@@ -78,7 +82,11 @@ class FunctionModelEvaluate final : public IFunction
         for (size_t i = 1; i < arguments.size(); ++i)
             has_nullable = has_nullable || arguments[i].type->isNullable();
 
-        auto model = models_loader.getModel(name_col->getValue<String>());
+        const std::string model_name = name_col->getValue<String>();
+        if (model_name.length() > 7 && model_name.rfind("mlpack", 0) == 0) {
+            return std::make_shared<DataTypeFloat64>();
+        }
+        auto model = models_loader.getModel(model_name);
         auto type = model->getReturnType();
 
         if (has_nullable)
@@ -106,8 +114,6 @@ class FunctionModelEvaluate final : public IFunction
             throw Exception("First argument of function " + getName() + " must be a constant string",
                             ErrorCodes::ILLEGAL_COLUMN);
 
-        auto model = models_loader.getModel(name_col->getValue<String>());
-
         ColumnRawPtrs column_ptrs;
         Columns materialized_columns;
         ColumnPtr null_map;
@@ -144,7 +150,22 @@ class FunctionModelEvaluate final : public IFunction
             }
         }
 
-        auto res = model->evaluate(column_ptrs);
+
+        ColumnPtr res;
+        const std::string model_name = name_col->getValue<String>();
+
+        if (model_name.length() > 7 && model_name.rfind("mlpack", 0) == 0) {
+            const std::string table_name = model_name.substr(7);
+            StoragePtr tableptr = DatabaseCatalog::instance().getTable(StorageID{"default", table_name}, CurrentThread::get().getQueryContext());
+            std::shared_ptr<StorageMLmodel> casted = std::dynamic_pointer_cast<StorageMLmodel>(tableptr);
+            res = casted->getModel()->evaluateModel(column_ptrs);
+        }
+
+        else
+        {
+            auto model = models_loader.getModel(name_col->getValue<String>());
+            res = model->evaluate(column_ptrs);
+        }
 
         if (null_map)
         {

src/Interpreters/ExternalModelsLoader.cpp

@@ -22,6 +22,7 @@ std::shared_ptr<const IExternalLoadable> ExternalModelsLoader::create(
     const std::string & config_prefix, const std::string & /* repository_name */) const
 {
     String type = config.getString(config_prefix + ".type");
+    LOG_FATAL(&Poco::Logger::root(), "type  {}", type);
     ExternalLoadableLifetime lifetime(config, config_prefix + ".lifetime");
 
     /// TODO: add models factory.
@@ -33,6 +34,15 @@ std::shared_ptr<const IExternalLoadable> ExternalModelsLoader::create(
                 lifetime
         );
     }
+    else if (type == "mlpack")
+    {
+        return std::make_unique<MlpackModel>(
+                name,
+                config.getString(config_prefix + ".path"),
+                config.getString(config_prefix + ".method"),
+                lifetime
+        );
+    }
     else
     {
         throw Exception("Unknown model type: " + type, ErrorCodes::INVALID_CONFIG_PARAMETER);

src/Interpreters/ExternalModelsLoader.h

@@ -1,6 +1,7 @@
 #pragma once
 
 #include <Interpreters/CatBoostModel.h>
+#include <Interpreters/MlpackModel.h>
 #include <Interpreters/Context_fwd.h>
 #include <Interpreters/ExternalLoader.h>
 #include <Common/logger_useful.h>