Implement Treelite v4 specification (#524)

* Bring in treelite-ng

* Apply linting to CMake files

* Fix CMakeLists.txt

* Fix C++ app example

* Fix build on MacOS

* Fix build on MacOS

* Fix scripts

* Use MacOS 10.15+

* Fix test on MacOS: size_t != uint64_t on MacOS

* Install latest XGBoost in CI

* Don't install if TEST_COVERAGE

* Don't install fmt

* Don't use O0 for test coverage

* Use latest MacOS and Windows workers

* Require MSVC 2022

* Use latest Windows

* Ensure that we use positive int for seeding rng

* Fix build with USE_OPENMP=OFF

* Fix flaky test

* Add back LightGBM

* Install XGBoost from the source

* Fix tests

.github/workflows/coverage-tests.yml

@@ -41,7 +41,7 @@ jobs:
         bash ops/cpp-python-coverage.sh
   win-python-coverage:
     name: Run Python and C++ tests with test coverage (Windows)
-    runs-on: windows-2019
+    runs-on: windows-latest
     steps:
     - uses: actions/checkout@v2
       with:
@@ -63,7 +63,7 @@ jobs:
         call ops/win-python-coverage.bat
   macos-python-coverage:
     name: Run Python and C++ tests with test coverage (MacOS)
-    runs-on: macos-11
+    runs-on: macos-latest
     steps:
     - uses: actions/checkout@v2
       with:

.github/workflows/macos-wheel-builder.yml

@@ -25,7 +25,7 @@ env:
 jobs:
   macos-wheel-builder:
     name: Build and test Python wheels (MacOS)
-    runs-on: macos-11
+    runs-on: macos-latest
     strategy:
       fail-fast: false
       matrix:

.github/workflows/misc-tests.yml

@@ -48,7 +48,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, macos-11]
+        os: [ubuntu-latest, macos-latest]
     steps:
     - uses: actions/checkout@v2
       with:
@@ -72,7 +72,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, macos-11]
+        os: [ubuntu-latest, macos-latest]
     steps:
     - uses: actions/checkout@v2
       with:

.github/workflows/windows-wheel-builder.yml

@@ -25,7 +25,7 @@ env:
 jobs:
   windows-wheel-builder:
     name: Build and test Python wheels (Windows)
-    runs-on: windows-2019
+    runs-on: windows-latest
     steps:
     - uses: actions/checkout@v2
       with:

.pre-commit-config.yaml

@@ -18,6 +18,11 @@ repos:
   hooks:
   - id: isort
     args: ["--profile", "black", "--filter-files"]
+- repo: https://github.com/MarcoGorelli/cython-lint
+  rev: v0.15.0
+  hooks:
+  - id: cython-lint
+  - id: double-quote-cython-strings
 - repo: https://github.com/PyCQA/flake8
   rev: 5.0.4
   hooks:
@@ -58,5 +63,4 @@ repos:
   rev: v1.2.0
   hooks:
     - id: mypy
-      exclude: setup.py
-      additional_dependencies: [types-setuptools]
+      additional_dependencies: [types-setuptools, numpy]

CMakeLists.txt

@@ -1,14 +1,15 @@
 cmake_policy(SET CMP0025 NEW)
 cmake_policy(SET CMP0091 NEW)
 set(CMAKE_FIND_NO_INSTALL_PREFIX TRUE FORCE)
-cmake_minimum_required (VERSION 3.16)
+cmake_minimum_required(VERSION 3.16 FATAL_ERROR)
+
 project(treelite LANGUAGES CXX C VERSION 4.0.0)
 
 # Check compiler versions
 # Use latest compilers to ensure that std::filesystem is available
 if(MSVC)
-  if(MSVC_VERSION LESS 1920)
-    message(FATAL_ERROR "Need Visual Studio 2019 or newer to build Treelite")
+  if(MSVC_VERSION LESS 1930)
+    message(FATAL_ERROR "Need Visual Studio 2022 or newer to build Treelite")
   endif()
 elseif(CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
   if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS "8.1")
@@ -37,11 +38,10 @@ option(BUILD_DOXYGEN "Build documentation for C/C++ functions using Doxygen." OF
 option(BUILD_CPP_TEST "Build C++ tests" OFF)
 option(DETECT_CONDA_ENV "Enable detection of conda environment for dependencies" ON)
 option(HIDE_CXX_SYMBOLS "Hide all C++ symbols. Useful when building Pip package" OFF)
-option(BUILD_JVM_RUNTIME "Build Treelite runtime for JVM" OFF)
 option(ENABLE_ALL_WARNINGS "Enable all compiler warnings. Only effective for GCC/Clang" OFF)
 option(USE_SANITIZER "Use sanitizer flags" OFF)
 SET(ENABLED_SANITIZERS "address" "leak" "undefined" CACHE STRING
-        "Semicolon separated list of sanitizer names. E.g 'address;leak'.")
+    "Semicolon separated list of sanitizer names. E.g 'address;leak'.")
 
 if(MSVC)
   if(Treelite_USE_DYNAMIC_MSVC_RUNTIME)
@@ -72,11 +72,13 @@ endif()
 # When installing dependencies, use Conda environment if available
 if(DETECT_CONDA_ENV)
   if(DEFINED ENV{CONDA_PREFIX})
-    set(CMAKE_PREFIX_PATH "$ENV{CONDA_PREFIX};${CMAKE_PREFIX_PATH}")
+    string(REPLACE "\\" "/" CONDA_PREFIX "$ENV{CONDA_PREFIX}")
+    set(CMAKE_PREFIX_PATH "${CONDA_PREFIX};${CMAKE_PREFIX_PATH}")
     message(STATUS "Detected Conda environment, CMAKE_PREFIX_PATH set to: ${CMAKE_PREFIX_PATH}")
-    if(CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT)
-      message(STATUS "No CMAKE_INSTALL_PREFIX argument detected, setting to: $ENV{CONDA_PREFIX}")
-      set(CMAKE_INSTALL_PREFIX $ENV{CONDA_PREFIX})
+    if(CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT OR USING_CONDA_INSTALL_PREFIX)
+      message(STATUS "No CMAKE_INSTALL_PREFIX argument detected, setting to: ${CONDA_PREFIX}")
+      set(CMAKE_INSTALL_PREFIX "${CONDA_PREFIX}")
+      set(USING_CONDA_INSTALL_PREFIX ON CACHE BOOL "Installing into Conda prefix")
     endif()
   else()
     message(STATUS "No Conda environment detected")
@@ -88,9 +90,7 @@ include(cmake/Utils.cmake)
 include(cmake/Version.cmake)
 
 add_subdirectory(src)
-if(BUILD_JVM_RUNTIME)
-  add_subdirectory(runtime/java)
-endif()
+
 if(BUILD_CPP_TEST)
   add_subdirectory(tests/cpp)
 endif()
@@ -117,35 +117,31 @@ if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR)
 endif()
 include(GNUInstallDirs)
 include(CMakePackageConfigHelpers)
-set(INSTALL_TARGETS ${TREELITE_TARGETS} objtreelite rapidjson)
-if(NOT FMTLIB_FROM_SYSTEM_ROOT)
-  list(APPEND INSTALL_TARGETS fmt-header-only)
-endif()
+set(INSTALL_TARGETS ${TREELITE_TARGETS} objtreelite rapidjson mdspan)
 install(TARGETS ${INSTALL_TARGETS}
     EXPORT TreeliteTargets
-    ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
-    LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
-    RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
+    ARCHIVE DESTINATION "${CMAKE_INSTALL_LIBDIR}"
+    LIBRARY DESTINATION "${CMAKE_INSTALL_LIBDIR}"
+    RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}"
     INCLUDES DESTINATION include)
-install(DIRECTORY include/treelite ${PROJECT_BINARY_DIR}/include/treelite
-    DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
+install(DIRECTORY include/treelite "${PROJECT_BINARY_DIR}/include/treelite"
+    DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}")
 install(EXPORT TreeliteTargets
     FILE TreeliteTargets.cmake
     NAMESPACE treelite::
-    DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/treelite)
+    DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/treelite")
 configure_package_config_file(
     cmake/TreeliteConfig.cmake.in
-    ${PROJECT_BINARY_DIR}/cmake/TreeliteConfig.cmake
-    INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/treelite)
+    "${PROJECT_BINARY_DIR}/cmake/TreeliteConfig.cmake"
+    INSTALL_DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/treelite")
 write_basic_package_version_file(
-    ${PROJECT_BINARY_DIR}/cmake/TreeliteConfigVersion.cmake
+    "${PROJECT_BINARY_DIR}/cmake/TreeliteConfigVersion.cmake"
     VERSION ${PROJECT_VERSION}
     COMPATIBILITY SameMajorVersion)
 install(FILES
-    ${PROJECT_BINARY_DIR}/cmake/TreeliteConfig.cmake
-    ${PROJECT_BINARY_DIR}/cmake/TreeliteConfigVersion.cmake
-    DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/treelite)
+    "${PROJECT_BINARY_DIR}/cmake/TreeliteConfig.cmake"
+    "${PROJECT_BINARY_DIR}/cmake/TreeliteConfigVersion.cmake"
+    DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/treelite")
 
 write_version()
-
 set_default_configuration_release()

cmake/ExternalLibs.cmake

@@ -1,26 +1,8 @@
 include(FetchContent)
-include(cmake/FetchContentMakeAvailable.cmake)
-
-# fmtlib
-find_package(fmt)
-if(fmt_FOUND)
-  get_target_property(fmt_loc fmt::fmt INTERFACE_INCLUDE_DIRECTORIES)
-  message(STATUS "Found fmtlib at ${fmt_loc}")
-  set(FMTLIB_FROM_SYSTEM_ROOT TRUE)
-else()
-  message(STATUS "Did not find fmtlib in the system root. Fetching fmtlib now...")
-  FetchContent_Declare(
-    fmtlib
-    GIT_REPOSITORY  https://github.com/fmtlib/fmt.git
-    GIT_TAG         7.0.3
-  )
-  FetchContent_MakeAvailable(fmtlib)
-  set_target_properties(fmt PROPERTIES EXCLUDE_FROM_ALL TRUE)
-  set(FMTLIB_FROM_SYSTEM_ROOT FALSE)
-endif()
 
 # RapidJSON (header-only library)
 add_library(rapidjson INTERFACE)
+target_compile_definitions(rapidjson INTERFACE -DRAPIDJSON_HAS_STDSTRING=1)
 find_package(RapidJSON)
 if(RapidJSON_FOUND)
   target_include_directories(rapidjson INTERFACE ${RAPIDJSON_INCLUDE_DIRS})
@@ -37,9 +19,27 @@ else()
 endif()
 add_library(RapidJSON::rapidjson ALIAS rapidjson)
 
+# mdspan (header-only library)
+message(STATUS "Fetching mdspan...")
+set(MDSPAN_CXX_STANDARD 17 CACHE STRING "")
+FetchContent_Declare(
+  mdspan
+  GIT_REPOSITORY https://github.com/kokkos/mdspan.git
+  GIT_TAG        mdspan-0.6.0
+)
+FetchContent_GetProperties(mdspan)
+if(NOT mdspan_POPULATED)
+  FetchContent_Populate(mdspan)
+  add_subdirectory(${mdspan_SOURCE_DIR} ${mdspan_BINARY_DIR} EXCLUDE_FROM_ALL)
+  message(STATUS "mdspan was downloaded at ${mdspan_SOURCE_DIR}.")
+endif()
+if(MSVC)  # workaround for MSVC 19.x: https://github.com/kokkos/mdspan/issues/276
+  target_compile_options(mdspan INTERFACE "/permissive-")
+endif()
+
 # Google C++ tests
 if(BUILD_CPP_TEST)
-  find_package(GTest 1.11.0 CONFIG)
+  find_package(GTest 1.11.0)
   if(NOT GTEST_FOUND)
     message(STATUS "Did not find Google Test in the system root. Fetching Google Test now...")
     FetchContent_Declare(
@@ -56,3 +56,24 @@ if(BUILD_CPP_TEST)
     endif()
   endif()
 endif()
+
+# fmtlib
+if(BUILD_CPP_TEST)
+  find_package(fmt 10.1)
+  if(fmt_FOUND)
+    get_target_property(fmt_loc fmt::fmt INTERFACE_INCLUDE_DIRECTORIES)
+    message(STATUS "Found fmtlib at ${fmt_loc}")
+    set(FMTLIB_FROM_SYSTEM_ROOT TRUE)
+  else()
+    message(STATUS "Did not find fmtlib in the system root. Fetching fmtlib now...")
+    set(FMT_INSTALL OFF CACHE BOOL "" FORCE)
+    FetchContent_Declare(
+        fmtlib
+        GIT_REPOSITORY  https://github.com/fmtlib/fmt.git
+        GIT_TAG         10.1.1
+    )
+    FetchContent_MakeAvailable(fmtlib)
+    set_target_properties(fmt PROPERTIES EXCLUDE_FROM_ALL TRUE)
+    set(FMTLIB_FROM_SYSTEM_ROOT FALSE)
+  endif()
+endif()

cmake/Utils.cmake

@@ -1,18 +1,3 @@
-# Automatically set source group based on folder
-function(auto_source_group SOURCES)
-
-  foreach(FILE ${SOURCES})
-      get_filename_component(PARENT_DIR "${FILE}" PATH)
-
-      # skip src or include and changes /'s to \\'s
-      string(REPLACE "${CMAKE_CURRENT_LIST_DIR}" "" GROUP "${PARENT_DIR}")
-      string(REPLACE "/" "\\\\" GROUP "${GROUP}")
-      string(REGEX REPLACE "^\\\\" "" GROUP "${GROUP}")
-
-      source_group("${GROUP}" FILES "${FILE}")
-  endforeach()
-endfunction(auto_source_group)
-
 # Set output directory of target, ignoring debug or release
 function(set_output_directory target dir)
   set_target_properties(${target} PROPERTIES

cmake/version.h.in

@@ -1,11 +1,12 @@
 /*!
- * Copyright (c) 2021 by Contributors
+ * Copyright (c) 2023 by Contributors
  */
-#ifndef TREELITE_VERSION_CONFIG_H_
-#define TREELITE_VERSION_CONFIG_H_
+#ifndef TREELITE_VERSION_H_
+#define TREELITE_VERSION_H_
 
 #define TREELITE_VER_MAJOR @treelite_VERSION_MAJOR@
 #define TREELITE_VER_MINOR @treelite_VERSION_MINOR@
 #define TREELITE_VER_PATCH @treelite_VERSION_PATCH@
+#define TREELITE_VERSION_STR "@treelite_VERSION_MAJOR@.@treelite_VERSION_MINOR@.@treelite_VERSION_PATCH@"
 
-#endif  // TREELITE_VERSION_CONFIG_H_
+#endif  // TREELITE_VERSION_H_

dev/change_version.py

@@ -35,6 +35,7 @@ def update_pypkg(
     rc_ver: Optional[int] = None,
 ) -> None:
     """Change version in the Python package"""
+    # pylint: disable=too-many-arguments
     version = f"{major}.{minor}.{patch}"
     if is_rc:
         assert rc_ver

docs/install.rst

@@ -29,7 +29,7 @@ For other operating systems, see the next section.
 .. note:: Windows users need to install Visual C++ Redistributable
 
   Treelite requires DLLs from `Visual C++ Redistributable
-  <https://www.microsoft.com/en-us/download/details.aspx?id=48145>`_
+  <https://learn.microsoft.com/en-us/cpp/windows/latest-supported-vc-redist?view=msvc-170#visual-studio-2015-2017-2019-and-2022>`_
   in order to function, so make sure to install it. Exception: If
   you have Visual Studio installed, you already have access to
   necessary libraries and thus don't need to install Visual C++