From 46da701deeebff356d9b8e73cb43607453c4258e Mon Sep 17 00:00:00 2001 From: "Brian C. Van Essen" Date: Mon, 11 Mar 2024 07:56:06 -0700 Subject: [PATCH 1/3] Adding additional HDF5 plugins and ZFP to the superbuild. Updating the cuda-distconv.sh superbuild example. --- scripts/superbuild/CMakeLists.txt | 14 +- scripts/superbuild/examples/cuda-distconv.sh | 26 +- scripts/superbuild/h5z-zfp/CMakeLists.txt | 42 ++++ scripts/superbuild/hdf5/CMakeLists.txt | 36 +-- .../superbuild/hdf5_plugins/CMakeLists.txt | 227 ++++++++++++++++++ scripts/superbuild/zfp/CMakeLists.txt | 45 ++++ 6 files changed, 368 insertions(+), 22 deletions(-) mode change 100644 => 100755 scripts/superbuild/examples/cuda-distconv.sh create mode 100644 scripts/superbuild/h5z-zfp/CMakeLists.txt create mode 100644 scripts/superbuild/hdf5_plugins/CMakeLists.txt create mode 100644 scripts/superbuild/zfp/CMakeLists.txt diff --git a/scripts/superbuild/CMakeLists.txt b/scripts/superbuild/CMakeLists.txt index 164915f64b9..f9ce300875a 100644 --- a/scripts/superbuild/CMakeLists.txt +++ b/scripts/superbuild/CMakeLists.txt @@ -77,6 +77,9 @@ lbann_sb_add_packages( Aluminum # Hydrogen, DiHydrogen Catch2 # Hydrogen, DiHydrogen HDF5 # Conduit + ZFP # LBANN + H5Z-ZFP # LBANN + HDF5_PLUGINS # LBANN JPEG-TURBO # OpenCV OpenBLAS # Hydrogen spdlog # DiHydrogen, LBANN @@ -112,7 +115,11 @@ foreach (pkg ${LBANN_SB_BUILD_PKGS}) endforeach () # Print a helpful(?) message -set(LBANN_SB_SUGG_CMAKE_PREFIX_PATH_TMP "\$\{CMAKE_PREFIX_PATH\}") +set(LBANN_SB_SUGG_CMAKE_PREFIX_PATH_TMP "${CMAKE_PREFIX_PATH}:\$\{CMAKE_PREFIX_PATH\}") +message("BVE I think that there is a prefix path ${CMAKE_PREFIX_PATH}") +message("BVE I think that there is a foo ${FOO}") +message("BVE I think that there is a suggesed prefix path ${LBANN_SB_SUGG_CMAKE_PREFIX_PATH_TMP}") +#set(LBANN_SB_SUGG_CMAKE_PREFIX_PATH_TMP "\$\{CMAKE_PREFIX_PATH\}") message("\n-----------------------------------------------------------------\n") message("LBANN SuperBuild will build the following packages:\n") foreach (pkg ${LBANN_SB_BUILD_PKGS}) @@ -140,6 +147,11 @@ message("\n-----------------------------------------------------------------\n") file(WRITE "${CMAKE_BINARY_DIR}/lbann_sb_suggested_cmake_prefix_path.sh" "export CMAKE_PREFIX_PATH=${LBANN_SB_SUGG_CMAKE_PREFIX_PATH}\n") +if (LBANN_SB_BUILD_H5Z-ZFP) +file(APPEND "${CMAKE_BINARY_DIR}/lbann_sb_suggested_cmake_prefix_path.sh" + "\nexport HDF5_PLUGIN_PATH=${H5Z-ZFP_DIR}/plugin\n") +endif (LBANN_SB_BUILD_H5Z-ZFP) + # Add a custom target for bundling all things up if (UNIX) find_program(__FIND_EXE find) diff --git a/scripts/superbuild/examples/cuda-distconv.sh b/scripts/superbuild/examples/cuda-distconv.sh old mode 100644 new mode 100755 index 9780f16bf02..217bbe817e6 --- a/scripts/superbuild/examples/cuda-distconv.sh +++ b/scripts/superbuild/examples/cuda-distconv.sh @@ -24,6 +24,11 @@ ## permissions and limitations under the license. ################################################################################ +#source $LMOD_PKG/init/zsh +#ml cuda/11.8 cmake/3.23 ninja gcc/11 +ml load cuda/12.2.2 cmake/3.23.1 gcc/11.2.1 spectrum-mpi/rolling-release python/3.8.2 +export LIBRARY_PATH=$LD_LIBRARY_PATH + # Set to ON (or any CMake truthy value) to build all of the # dependencies of the LBANN stack BUILD_EXTERNAL_TPLS=ON @@ -53,18 +58,21 @@ LBANN_SRC_DIR=$(git rev-parse --show-toplevel) SUPERBUILD_SRC_DIR=${LBANN_SRC_DIR}/scripts/superbuild # Set to the preferred install directory -INSTALL_PREFIX=${PWD}/install-cuda-distconv +#INSTALL_PREFIX=${PWD}/install-cuda-distconv +INSTALL_PREFIX=/p/vast1/lbann/stable_dependencies/lassen/gcc-11.2.1/cuda-12.2.2/spectrum-mpi-rolling-release # Set to the preferred build directory BUILD_DIR=${TMPDIR}/lbann-superbuild +# -G Ninja \ cmake \ - -G Ninja \ + -G "Unix Makefiles" \ -S ${SUPERBUILD_SRC_DIR} \ -B ${BUILD_DIR} \ \ -D CMAKE_BUILD_TYPE=Release \ -D CMAKE_INSTALL_PREFIX=${INSTALL_PREFIX} \ + -D BUILD_SHARED_LIBS=ON \ \ -D CMAKE_C_COMPILER=$(command -v gcc) \ -D CMAKE_CXX_COMPILER=$(command -v g++) \ @@ -74,7 +82,7 @@ cmake \ \ -D CMAKE_CXX_STANDARD=17 \ -D CMAKE_CUDA_STANDARD=17 \ - -D CMAKE_CUDA_ARCHITECTURES=${GPU_ARCH} \ + -D CMAKE_CUDA_ARCHITECTURES=${CUDA_GPU_ARCH} \ \ -D CMAKE_POSITION_INDEPENDENT_CODE=ON \ \ @@ -107,12 +115,13 @@ cmake \ -D LBANN_SB_BUILD_Aluminum=${BUILD_LBANN_STACK} \ -D LBANN_SB_Aluminum_CXX_FLAGS="${EXTRA_CXX_FLAGS}" \ -D LBANN_SB_Aluminum_CUDA_FLAGS="${EXTRA_CUDA_FLAGS}" \ - -D LBANN_SB_FWD_Aluminum_ALUMINUM_ENABLE_CALIPER=ON \ + -D LBANN_SB_FWD_Aluminum_ALUMINUM_ENABLE_CALIPER=OFF \ -D LBANN_SB_FWD_Aluminum_ALUMINUM_ENABLE_NCCL=ON \ -D LBANN_SB_FWD_Aluminum_ALUMINUM_ENABLE_HOST_TRANSFER=ON \ -D LBANN_SB_FWD_Aluminum_ALUMINUM_ENABLE_TESTS=OFF \ -D LBANN_SB_FWD_Aluminum_ALUMINUM_ENABLE_BENCHMARKS=OFF \ -D LBANN_SB_FWD_Aluminum_ALUMINUM_ENABLE_THREAD_MULTIPLE=OFF \ + -D LBANN_SB_FWD_Aluminum_NCCL_DIR=${INSTALL_PREFIX}/nccl_2.20.3-1+cuda12.2_ppc64le \ \ -D LBANN_SB_BUILD_Hydrogen=${BUILD_LBANN_STACK} \ -D LBANN_SB_Hydrogen_CXX_FLAGS="${EXTRA_CXX_FLAGS}" \ @@ -122,12 +131,15 @@ cmake \ -D LBANN_SB_FWD_Hydrogen_Hydrogen_ENABLE_HALF=OFF \ -D LBANN_SB_FWD_Hydrogen_Hydrogen_ENABLE_TESTING=ON \ -D LBANN_SB_FWD_Hydrogen_Hydrogen_ENABLE_UNIT_TESTS=OFF \ + -D LBANN_SB_FWD_Hydrogen_NCCL_DIR=${INSTALL_PREFIX}/nccl_2.20.3-1+cuda12.2_ppc64le \ \ -D LBANN_SB_BUILD_DiHydrogen=${BUILD_LBANN_STACK} \ -D LBANN_SB_DiHydrogen_CXX_FLAGS="${EXTRA_CXX_FLAGS}" \ -D LBANN_SB_DiHydrogen_CUDA_FLAGS="${EXTRA_CUDA_FLAGS}" \ -D LBANN_SB_FWD_DiHydrogen_BLA_VENDOR="Generic" \ -D LBANN_SB_FWD_DiHydrogen_H2_ENABLE_DISTCONV_LEGACY=${BUILD_WITH_DISTCONV} \ + -D LBANN_SB_FWD_DiHydrogen_NCCL_DIR=${INSTALL_PREFIX}/nccl_2.20.3-1+cuda12.2_ppc64le \ + -D LBANN_SB_FWD_DiHydrogen_CUDNN_DIR=${INSTALL_PREFIX}/cudnn-linux-ppc64le-8.9.7.29_cuda12-archive \ \ -D LBANN_SB_BUILD_LBANN=${BUILD_LBANN_STACK} \ -D LBANN_SB_LBANN_SOURCE_DIR=${LBANN_SRC_DIR} \ @@ -137,7 +149,9 @@ cmake \ -D LBANN_SB_FWD_LBANN_BLA_VENDOR="Generic" \ -D LBANN_SB_FWD_LBANN_CMAKE_EXPORT_COMPILE_COMMANDS=ON \ -D LBANN_SB_FWD_LBANN_LBANN_DATATYPE=float \ - -D LBANN_SB_FWD_LBANN_LBANN_WITH_CALIPER=ON \ + -D LBANN_SB_FWD_LBANN_LBANN_WITH_CALIPER=OFF \ -D LBANN_SB_FWD_LBANN_LBANN_WITH_DISTCONV=${BUILD_WITH_DISTCONV} \ -D LBANN_SB_FWD_LBANN_LBANN_WITH_TBINF=OFF \ - -D LBANN_SB_FWD_LBANN_LBANN_WITH_UNIT_TESTING=ON + -D LBANN_SB_FWD_LBANN_LBANN_WITH_UNIT_TESTING=ON \ + -D LBANN_SB_FWD_LBANN_NCCL_DIR=${INSTALL_PREFIX}/nccl_2.20.3-1+cuda12.2_ppc64le \ + -D LBANN_SB_FWD_LBANN_CUDNN_DIR=${INSTALL_PREFIX}/cudnn-linux-ppc64le-8.9.7.29_cuda12-archive diff --git a/scripts/superbuild/h5z-zfp/CMakeLists.txt b/scripts/superbuild/h5z-zfp/CMakeLists.txt new file mode 100644 index 00000000000..e9ac1a68eff --- /dev/null +++ b/scripts/superbuild/h5z-zfp/CMakeLists.txt @@ -0,0 +1,42 @@ +################################################################################ +## Copyright (c) 2014-2024, Lawrence Livermore National Security, LLC. +## Produced at the Lawrence Livermore National Laboratory. +## Written by the LBANN Research Team (B. Van Essen, et al.) listed in +## the CONTRIBUTORS file. +## +## LLNL-CODE-697807. +## All rights reserved. +## +## This file is part of LBANN: Livermore Big Artificial Neural Network +## Toolkit. For details, see http://software.llnl.gov/LBANN or +## https://github.com/LLNL/LBANN. +## +## Licensed under the Apache License, Version 2.0 (the "Licensee"); you +## may not use this file except in compliance with the License. You may +## obtain a copy of the License at: +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +## implied. See the License for the specific language governing +## permissions and limitations under the license. +################################################################################ + +# Conduit is "cute" about finding HDF5. It's not a CMake option() -- +# you opt in by setting HDF5_DIR explicitly. So let's do that. +# if (TARGET HDF5 AND NOT LBANN_SB_FWD_ZFP_HDF5_DIR) +# set(LBANN_SB_FWD_ZFP_HDF5_DIR ${HDF5_DIR}) +# endif () + +lbann_sb_add_cmake_extern_pkg( + NAME H5Z-ZFP + LANGUAGES C CXX ${lbann_sb_ftn_lang} + GITHUB_URL llnl/h5z-zfp.git + GIT_TAG "master" + DEPENDS_ON HDF5 ZFP +) + +set(H5Z-ZFP_DIR ${LBANN_SB_H5Z-ZFP_PREFIX} + CACHE INTERNAL "The install prefix of H5Z-ZFP.") diff --git a/scripts/superbuild/hdf5/CMakeLists.txt b/scripts/superbuild/hdf5/CMakeLists.txt index c9e4bb56739..c69971e071b 100644 --- a/scripts/superbuild/hdf5/CMakeLists.txt +++ b/scripts/superbuild/hdf5/CMakeLists.txt @@ -24,8 +24,8 @@ ## permissions and limitations under the license. ################################################################################ # These defaults are set so as to be most amenable to LBANN(/Conduit). -set(LBANN_SB_FWD_HDF5_DEFAULT_API_VERSION "v18" - CACHE BOOL "Default API version for HDF5") +set(LBANN_SB_FWD_HDF5_DEFAULT_API_VERSION "v112" + CACHE STRING "Default API version for HDF5") option(LBANN_SB_FWD_HDF5_BUILD_TESTING "Build HDF5 tests." @@ -37,24 +37,30 @@ option(LBANN_SB_FWD_HDF5_HDF5_BUILD_EXAMPLES option(LBANN_SB_FWD_HDF5_HDF5_ENABLE_PARALLEL "Build HDF5 with parallel (MPI) support." ON) -option(LBANN_SB_FWD_HDF5_HDF5_USE_16_API_DEFAULT - "Use 1.6 API by default" - OFF) -option(LBANN_SB_FWD_HDF5_HDF5_USE_18_API_DEFAULT - "Use 1.8 API by default" - ON) -option(LBANN_SB_FWD_HDF5_HDF5_USE_110_API_DEFAULT - "Use 1.10 API by default" - OFF) -option(LBANN_SB_FWD_HDF5_HDF5_USE_112_API_DEFAULT - "Use 1.12 API by default" - OFF) +# option(LBANN_SB_FWD_HDF5_HDF5_USE_16_API_DEFAULT +# "Use 1.6 API by default" +# OFF) +# option(LBANN_SB_FWD_HDF5_HDF5_USE_18_API_DEFAULT +# "Use 1.8 API by default" +# OFF) +# option(LBANN_SB_FWD_HDF5_HDF5_USE_110_API_DEFAULT +# "Use 1.10 API by default" +# OFF) +# option(LBANN_SB_FWD_HDF5_HDF5_USE_112_API_DEFAULT +# "Use 1.12 API by default" +# ON) option(LBANN_SB_FWD_HDF5_HDF5_BUILD_FORTRAN "Build HDF5 with fortran support" OFF) option(LBANN_SB_FWD_HDF5_HDF5_GENERATE_HEADERS "See HDF5 docs." ON) +option(LBANN_SB_FWD_HDF5_ONLY_SHARED_LIBS + "Only Build Shared Libraries." + ON) +# option(LBANN_SB_FWD_HDF5_DEFAULT_API_VERSION +# "Only Build Shared Libraries." +# v112) # At present, this is required for LBANN/JAG use. option(LBANN_SB_FWD_HDF5_HDF5_ENABLE_Z_LIB_SUPPORT @@ -75,7 +81,7 @@ lbann_sb_add_cmake_extern_pkg( NAME HDF5 LANGUAGES C CXX ${MAYBE_HDF5_FORTRAN} GITHUB_URL HDFGroup/hdf5 - GIT_TAG "hdf5-1_10_9") + GIT_TAG "hdf5-1_12_3") set(HDF5_DIR ${LBANN_SB_HDF5_PREFIX} CACHE INTERNAL "The install prefix of HDF5.") diff --git a/scripts/superbuild/hdf5_plugins/CMakeLists.txt b/scripts/superbuild/hdf5_plugins/CMakeLists.txt new file mode 100644 index 00000000000..65a0bece1e7 --- /dev/null +++ b/scripts/superbuild/hdf5_plugins/CMakeLists.txt @@ -0,0 +1,227 @@ +################################################################################ +## Copyright (c) 2014-2024, Lawrence Livermore National Security, LLC. +## Produced at the Lawrence Livermore National Laboratory. +## Written by the LBANN Research Team (B. Van Essen, et al.) listed in +## the CONTRIBUTORS file. +## +## LLNL-CODE-697807. +## All rights reserved. +## +## This file is part of LBANN: Livermore Big Artificial Neural Network +## Toolkit. For details, see http://software.llnl.gov/LBANN or +## https://github.com/LLNL/LBANN. +## +## Licensed under the Apache License, Version 2.0 (the "Licensee"); you +## may not use this file except in compliance with the License. You may +## obtain a copy of the License at: +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +## implied. See the License for the specific language governing +## permissions and limitations under the license. +################################################################################ + +# https://forum.hdfgroup.org/t/thread-parallel-compression-filters-feature-request/10656/2 + +# Conduit is "cute" about finding HDF5. It's not a CMake option() -- +# you opt in by setting HDF5_DIR explicitly. So let's do that. +# if (TARGET HDF5 AND NOT LBANN_SB_FWD_ZFP_HDF5_DIR) +# set(LBANN_SB_FWD_ZFP_HDF5_DIR ${HDF5_DIR}) +# endif () + +set(LBANN_SB_FWD_HDF5_PLUGINS_H5PL_ALLOW_EXTERNAL_SUPPORT "TGZ" + CACHE STRING "The external compression library sources are provided in the PLUGIN_SOURCE folder") + +set(LBANN_SB_FWD_HDF5_PLUGINS_H5PL_TGZPATH:PATH "${CMAKE_CURRENT_BINARY_DIR}/src/libs" + CACHE STRING "") + + # set(BUILD_OPTIONS " ") + # set(BUILD_OPTIONS "${BUILD_OPTIONS} -DH5PL_TGZPATH:PATH=${PLUGIN_SOURCE}/libs ) + # cmake -C ${PLUGIN_SOURCE}/config/cmake/cacheinit.cmake -DCMAKE_BUILD_TYPE:STRING=Release ${BUILD_OPTIONS} "-G${CMAKE_GENERATOR}" ${PLUGIN_SOURCE} + + + +# Copyright by The HDF Group. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the COPYING file, which can be found at the root of the source code +# distribution tree, or in https://www.hdfgroup.org/licenses. +# If you do not have access to either file, you may request a copy from +# help@hdfgroup.org. +# +# This is the CMakeCache file. + +######################## +# EXTERNAL cache entries +######################## + +# set (LBANN_SB_FWD_HDF5_PLUGINS_USE_SHARED_LIBS ON CACHE BOOL "Use Shared Libraries" FORCE) + +set (LBANN_SB_FWD_HDF5_PLUGINS_HDF_ENABLE_BLOSC_SUPPORT OFF CACHE BOOL "Enable BLOSC compression" FORCE) +set (LBANN_SB_FWD_HDF5_PLUGINS_HDF_HDF_ENABLE_BLOSC_ZLIB_SUPPORT OFF CACHE BOOL "Enable BLOSC compression" FORCE) + +set (LBANN_SB_FWD_HDF5_PLUGINS_BUILD_TESTING OFF CACHE BOOL "Build h5blosc Unit Testing" FORCE) +# set (LBANN_SB_FWD_HDF5_PLUGINS_H5PL_BUILD_TESTING ON CACHE BOOL "Enable h5pl examples" FORCE) + +# set (LBANN_SB_FWD_HDF5_PLUGINS_BUILD_EXAMPLES ON CACHE BOOL "Build h5pl Examples" FORCE) + +# set (LBANN_SB_FWD_HDF5_PLUGINS_HDF_ENABLE_PARALLEL ON CACHE BOOL "Enable parallel build (requires MPI)" FORCE) +# #set (HDF_ENABLE_PARALLEL OFF CACHE BOOL "Enable parallel build (requires MPI)" FORCE) + +# set (LBANN_SB_FWD_HDF5_PLUGINS_HDF5_PACKAGE_NAME "hdf5" CACHE STRING "Name of HDF5 package" FORCE) + +# set (LBANN_SB_FWD_HDF5_PLUGINS_HDF5_NAMESPACE "hdf5::" CACHE STRING "Name space of HDF5 library" FORCE) + +# # set (H5PL_ALLOW_EXTERNAL_SUPPORT "NO" CACHE STRING "Allow External Library Building (NO GIT TGZ)" FORCE) +# # set_property (CACHE H5PL_ALLOW_EXTERNAL_SUPPORT PROPERTY STRINGS NO GIT TGZ) + +# set (LBANN_SB_FWD_HDF5_PLUGINS_H5PL_GIT_URL "https://github.com/HDFGroup/h5plugin.git" CACHE STRING "Use plugins from HDF repository" FORCE) +# set (LBANN_SB_FWD_HDF5_PLUGINS_H5PL_GIT_BRANCH "master" CACHE STRING "" FORCE) + +# set (LBANN_SB_FWD_HDF5_PLUGINS_H5PL_TGZ_ORIGPATH "https://github.com/HDFGroup/hdf5_plugins/releases/download/snapshots" CACHE STRING "Use PLUGINS from original location" FORCE) +# set (LBANN_SB_FWD_HDF5_PLUGINS_H5PL_TGZ_NAME "hdf5_plugins-master.tar.gz" CACHE STRING "Use plugins from compressed file" FORCE) + +# set (LBANN_SB_FWD_HDF5_PLUGINS_PL_PACKAGE_NAME "pl" CACHE STRING "Name of plugins package" FORCE) +# set (LBANN_SB_FWD_HDF5_PLUGINS_H5PL_CPACK_ENABLE ON CACHE BOOL "Enable the CPACK include and components" FORCE) + +############################# bitshuffle ################################################ + +set (LBANN_SB_FWD_HDF5_PLUGINS_BUILD_BSHUF_LZ4_LIBRARY_SOURCE OFF CACHE BOOL "build the lz4 library within the BSHUF plugin" FORCE) + +# set (BSHUF_GIT_URL "https://github.com/kiyo-masui/bitshuffle.git" CACHE STRING "Use BSHUF from HDF repository" FORCE) +# set (BSHUF_GIT_BRANCH "master" CACHE STRING "" FORCE) + +# set (BSHUF_TGZ_ORIGPATH "https://github.com/kiyo-masui/bitshuffle/archive/refs/tags" CACHE STRING "Use PLUGINS from original location" FORCE) +# set (BSHUF_TGZ_NAME "bitshuffle-0.5.1.tar.gz" CACHE STRING "Use BSHUF from compressed file" FORCE) + +# set (BSHUF_PACKAGE_NAME "bshuf" CACHE STRING "Name of BSHUF package" FORCE) + +# ############################# blosc ################################################ + +# set (BLOSC_GIT_URL "https://github.com/Blosc/c-blosc.git" CACHE STRING "Use BLOSC from Github repository" FORCE) +# set (BLOSC_GIT_BRANCH "main" CACHE STRING "" FORCE) + +# set (BLOSC_TGZ_ORIGPATH "https://github.com/Blosc/c-blosc/archive/refs/tags" CACHE STRING "Use PLUGINS from original location" FORCE) +# set (BLOSC_TGZ_NAME "c-blosc-1.21.5.tar.gz" CACHE STRING "Use BLOSC from compressed file" FORCE) + +# set (BLOSC_PACKAGE_NAME "blosc" CACHE STRING "Name of BLOSC package" FORCE) + +# set (BLOSC_ZLIB_GIT_URL "https://github.com/madler/zlib.git" CACHE STRING "Use ZLIB from GitHub repository" FORCE) +# set (BLOSC_ZLIB_GIT_BRANCH "develop" CACHE STRING "" FORCE) + +# set (BLOSC_ZLIB_TGZ_ORIGPATH "https://github.com/madler/zlib/releases/download/v1.3" CACHE STRING "Use PLUGINS from original location" FORCE) +# set (BLOSC_ZLIB_TGZ_NAME "zlib-1.3.tar.gz" CACHE STRING "Use ZLib from compressed file" FORCE) + +# set (BLOSC_ZLIB_PACKAGE_NAME "zlib" CACHE STRING "Name of BLOSC_ZLIB package" FORCE) + + +# ############################# bzip2 ################################################ + +set (LBANN_SB_FWD_HDF5_PLUGINS_HDF_ENABLE_BZ2_USE_EXTERNAL OFF CACHE BOOL "Use External Library Building for BZ2" FORCE) + +# set (BZ2_GIT_URL "https://github.com/libarchive/bzip2.git" CACHE STRING "Use BZ2 from GitHub repository" FORCE) +# set (BZ2_GIT_BRANCH "bzip2-1.0.8" CACHE STRING "" FORCE) + +# set (BZ2_TGZ_ORIGPATH "https://github.com/libarchive/bzip2/archive/refs/tags" CACHE STRING "Use PLUGINS from original location" FORCE) +# set (BZ2_TGZ_NAME "bzip2-bzip2-1.0.8.tar.gz" CACHE STRING "Use BZ2 from compressed file" FORCE) + +# set (BZ2_PACKAGE_NAME "bz2" CACHE STRING "Name of BZ2 package" FORCE) + +# ############################# fpzip ################################################ + +# set (FPZIP_GIT_URL "https://github.com/LLNL/fpzip.git" CACHE STRING "Use FPZIP from GitHub repository" FORCE) +# set (FPZIP_GIT_BRANCH "develop" CACHE STRING "" FORCE) + +# set (FPZIP_TGZ_ORIGPATH "https://github.com/LLNL/fpzip/releases/download/1.3.0" CACHE STRING "Use PLUGINS from original location" FORCE) +# set (FPZIP_TGZ_NAME "fpzip-1.3.0.tar.gz" CACHE STRING "Use FPZIP from compressed file" FORCE) + +# set (FPZIP_PACKAGE_NAME "fpzip" CACHE STRING "Name of FPZIP package" FORCE) + +# ############################# jpeg ################################################ + +# set (JPEG_GIT_URL "https://github.com/libjpeg-turbo/libjpeg-turbo.git" CACHE STRING "Use JPEG from TurboJPEG" FORCE) +# set (JPEG_GIT_BRANCH "jpeg-9e" CACHE STRING "" FORCE) + +# set (JPEG_TGZ_ORIGPATH "https://www.ijg.org/files" CACHE STRING "Use PLUGINS from original location" FORCE) +# set (JPEG_TGZ_NAME "jpegsrc.v9e.tar.gz" CACHE STRING "Use JPEG from compressed file" FORCE) + +# set (JPEG_PACKAGE_NAME "jpeg" CACHE STRING "Name of JPEG package" FORCE) + +# ############################# lz4 ################################################ + +set (LBANN_SB_FWD_HDF5_PLUGINS_BUILD_LZ4_LIBRARY_SOURCE OFF CACHE BOOL "build the lz4 library within the plugin" FORCE) + +# set (LZ4_GIT_URL "https://github.com/lz4/lz4.git" CACHE STRING "Use LZ4 from GitHub repository" FORCE) +# set (LZ4_GIT_BRANCH "dev" CACHE STRING "" FORCE) + +# set (LZ4_TGZ_ORIGPATH "https://github.com/lz4/lz4/releases/download/v1.9.4" CACHE STRING "Use PLUGINS from original location" FORCE) +# set (LZ4_TGZ_NAME "lz4-1.9.4.tar.gz" CACHE STRING "Use LZ4 from compressed file" FORCE) + +# set (LZ4_PACKAGE_NAME "lz4" CACHE STRING "Name of LZ4 package" FORCE) + +# ############################# lzf ################################################ + +# set (LZF_URL "http://software.schmorp.de/pkg/liblzf.html" CACHE STRING "LZF home" FORCE) + +# set (LZF_TGZ_ORIGPATH "http://dist.schmorp.de/liblzf" CACHE STRING "Use LZF from original location" FORCE) +# set (LZF_TGZ_NAME "liblzf-3.6.tar.gz" CACHE STRING "Use LZF from compressed file" FORCE) + +# set (LZF_PACKAGE_NAME "lzf" CACHE STRING "Name of LZF package" FORCE) + +# ############################# mafisc ################################################ + +# #set (BUILD_MAFISC_LIBRARY_SOURCE OFF CACHE BOOL "build the mafisc library within the plugin" FORCE) + +# #set (MAFISC_TGZ_ORIGPATH "" CACHE STRING "Use PLUGINS from original location" FORCE) +# #set (MAFISC_TGZ_NAME ".tar.gz" CACHE STRING "Use MAFISC from compressed file" FORCE) +# #set (MAFISC_PACKAGE_NAME "mafisc" CACHE STRING "Name of MAFISC package" FORCE) + +# ############################# sz ################################################ + +# set (SZ_GIT_URL "https://github.com/szcompressor/SZ.git" CACHE STRING "Use SZ filter from GitHub repository" FORCE) +# set (SZ_GIT_BRANCH "master" CACHE STRING "" FORCE) + +# set (SZ_TGZ_ORIGPATH "https://github.com/szcompressor/SZ/releases/download/v2.1.12.5" CACHE STRING "Use PLUGINS from original location" FORCE) +# set (SZ_TGZ_NAME "SZ-2.1.12.5.tar.gz" CACHE STRING "Use SZ filter from compressed file" FORCE) + +# set (SZ_PACKAGE_NAME "SZ" CACHE STRING "Name of SZ filter package" FORCE) + +# ############################# zfp ################################################ + +# set (ZFP_GIT_URL "https://github.com/LLNL/zfp.git" CACHE STRING "Use ZFP from GitHub repository" FORCE) +# set (ZFP_GIT_BRANCH "develop" CACHE STRING "" FORCE) + +# set (ZFP_TGZ_ORIGPATH "https://github.com/LLNL/zfp/releases/download/1.0.0" CACHE STRING "Use PLUGINS from original location" FORCE) +# set (ZFP_TGZ_NAME "zfp-1.0.0.tar.gz" CACHE STRING "Use ZFP from compressed file" FORCE) + +# set (ZFP_PACKAGE_NAME "zfp" CACHE STRING "Name of ZFP package" FORCE) + +############################# zstd ################################################ + +# set (LBANN_SB_FWD_HDF5_PLUGINS_ZSTD_GIT_URL "https://github.com/facebook/zstd.git" CACHE STRING "Use ZSTD from GitHub repository" FORCE) +# set (LBANN_SB_FWD_HDF5_PLUGINS_ZSTD_GIT_BRANCH "dev" CACHE STRING "" FORCE) + +# set (LBANN_SB_FWD_HDF5_PLUGINS_ZSTD_TGZ_ORIGPATH "https://github.com/facebook/zstd/releases/download/v1.5.5" CACHE STRING "Use PLUGINS from original location" FORCE) +# set (LBANN_SB_FWD_HDF5_PLUGINS_ZSTD_TGZ_NAME "zstd-1.5.5.tar.gz" CACHE STRING "Use ZSTD from compressed file" FORCE) + +# set (LBANN_SB_FWD_HDF5_PLUGINS_ZSTD_PACKAGE_NAME "zstd" CACHE STRING "Name of ZSTD package" FORCE) + +lbann_sb_add_cmake_extern_pkg( + NAME HDF5_PLUGINS + LANGUAGES C CXX ${lbann_sb_ftn_lang} + GITHUB_URL HDFGroup/hdf5_plugins.git + GIT_TAG "master" + DEPENDS_ON HDF5 + EXTRA_CMAKE_ARGS + -C${CMAKE_CURRENT_BINARY_DIR}/src/config/cmake/cacheinit.cmake + -DUSE_SHARED_LIBS:BOOL=ON + -DBUILD_SHARED_LIBS:BOOL=ON +) + +set(HDF5_PLUGINS_DIR ${LBANN_SB_HDF5_PLUGINS_PREFIX} + CACHE INTERNAL "The install prefix of HDF5_PLUGINS.") diff --git a/scripts/superbuild/zfp/CMakeLists.txt b/scripts/superbuild/zfp/CMakeLists.txt new file mode 100644 index 00000000000..70a7749c69c --- /dev/null +++ b/scripts/superbuild/zfp/CMakeLists.txt @@ -0,0 +1,45 @@ +################################################################################ +## Copyright (c) 2014-2024, Lawrence Livermore National Security, LLC. +## Produced at the Lawrence Livermore National Laboratory. +## Written by the LBANN Research Team (B. Van Essen, et al.) listed in +## the CONTRIBUTORS file. +## +## LLNL-CODE-697807. +## All rights reserved. +## +## This file is part of LBANN: Livermore Big Artificial Neural Network +## Toolkit. For details, see http://software.llnl.gov/LBANN or +## https://github.com/LLNL/LBANN. +## +## Licensed under the Apache License, Version 2.0 (the "Licensee"); you +## may not use this file except in compliance with the License. You may +## obtain a copy of the License at: +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +## implied. See the License for the specific language governing +## permissions and limitations under the license. +################################################################################ + +# if (TARGET HDF5 AND NOT LBANN_SB_FWD_ZFP_HDF5_DIR) +# set(LBANN_SB_FWD_ZFP_HDF5_DIR ${HDF5_DIR}) +# endif () + +# Conduit is "cute" about finding HDF5. It's not a CMake option() -- +# you opt in by setting HDF5_DIR explicitly. So let's do that. +# if (TARGET HDF5 AND NOT LBANN_SB_FWD_ZFP_HDF5_DIR) +# set(LBANN_SB_FWD_ZFP_HDF5_DIR ${HDF5_DIR}) +# endif () + +lbann_sb_add_cmake_extern_pkg( + NAME ZFP + LANGUAGES C CXX ${lbann_sb_ftn_lang} + GITHUB_URL llnl/zfp.git + GIT_TAG "develop" +) + +set(ZFP_DIR ${LBANN_SB_ZFP_PREFIX} + CACHE INTERNAL "The install prefix of ZFP.") From 180e3510e3b4fee54468390dbd496ac07d714e71 Mon Sep 17 00:00:00 2001 From: "Brian C. Van Essen" Date: Tue, 12 Mar 2024 05:13:55 -0700 Subject: [PATCH 2/3] Added a field for an initial config in an external package. --- .../LBANNSuperBuildAddCMakeExternPkg.cmake | 8 +++++++- scripts/superbuild/hdf5_plugins/CMakeLists.txt | 15 ++++++++------- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/scripts/superbuild/cmake/modules/LBANNSuperBuildAddCMakeExternPkg.cmake b/scripts/superbuild/cmake/modules/LBANNSuperBuildAddCMakeExternPkg.cmake index 5794aacada0..97c8346e314 100644 --- a/scripts/superbuild/cmake/modules/LBANNSuperBuildAddCMakeExternPkg.cmake +++ b/scripts/superbuild/cmake/modules/LBANNSuperBuildAddCMakeExternPkg.cmake @@ -186,7 +186,8 @@ macro(lbann_sb_add_cmake_extern_pkg) NAME GITHUB_URL GIT_TAG - SOURCE_SUBDIR) + SOURCE_SUBDIR + INIT_CONFIG) set(_macro_multi_val LANGUAGES EXTRA_CMAKE_ARGS @@ -274,6 +275,11 @@ macro(lbann_sb_add_cmake_extern_pkg) message("\nLBANN!\n") endif () + if (lbann_sb_add_INIT_CONFIG) + list(PREPEND LBANN_SB_${PKG_NAME}_CMAKE_LANG_ARGS + "-C${lbann_sb_add_INIT_CONFIG}") + endif () + # Finally add the project. ExternalProject_Add(${PKG_NAME} # Standard stuff; users shouldn't care. diff --git a/scripts/superbuild/hdf5_plugins/CMakeLists.txt b/scripts/superbuild/hdf5_plugins/CMakeLists.txt index 65a0bece1e7..185a822046a 100644 --- a/scripts/superbuild/hdf5_plugins/CMakeLists.txt +++ b/scripts/superbuild/hdf5_plugins/CMakeLists.txt @@ -62,10 +62,10 @@ set(LBANN_SB_FWD_HDF5_PLUGINS_H5PL_TGZPATH:PATH "${CMAKE_CURRENT_BINARY_DIR}/src # set (LBANN_SB_FWD_HDF5_PLUGINS_USE_SHARED_LIBS ON CACHE BOOL "Use Shared Libraries" FORCE) -set (LBANN_SB_FWD_HDF5_PLUGINS_HDF_ENABLE_BLOSC_SUPPORT OFF CACHE BOOL "Enable BLOSC compression" FORCE) -set (LBANN_SB_FWD_HDF5_PLUGINS_HDF_HDF_ENABLE_BLOSC_ZLIB_SUPPORT OFF CACHE BOOL "Enable BLOSC compression" FORCE) +# set (LBANN_SB_FWD_HDF5_PLUGINS_HDF_ENABLE_BLOSC_SUPPORT OFF CACHE BOOL "Enable BLOSC compression" FORCE) +# set (LBANN_SB_FWD_HDF5_PLUGINS_HDF_HDF_ENABLE_BLOSC_ZLIB_SUPPORT OFF CACHE BOOL "Enable BLOSC compression" FORCE) -set (LBANN_SB_FWD_HDF5_PLUGINS_BUILD_TESTING OFF CACHE BOOL "Build h5blosc Unit Testing" FORCE) +# set (LBANN_SB_FWD_HDF5_PLUGINS_BUILD_TESTING OFF CACHE BOOL "Build h5blosc Unit Testing" FORCE) # set (LBANN_SB_FWD_HDF5_PLUGINS_H5PL_BUILD_TESTING ON CACHE BOOL "Enable h5pl examples" FORCE) # set (LBANN_SB_FWD_HDF5_PLUGINS_BUILD_EXAMPLES ON CACHE BOOL "Build h5pl Examples" FORCE) @@ -91,7 +91,7 @@ set (LBANN_SB_FWD_HDF5_PLUGINS_BUILD_TESTING OFF CACHE BOOL "Build h5blosc Unit ############################# bitshuffle ################################################ -set (LBANN_SB_FWD_HDF5_PLUGINS_BUILD_BSHUF_LZ4_LIBRARY_SOURCE OFF CACHE BOOL "build the lz4 library within the BSHUF plugin" FORCE) +# set (LBANN_SB_FWD_HDF5_PLUGINS_BUILD_BSHUF_LZ4_LIBRARY_SOURCE OFF CACHE BOOL "build the lz4 library within the BSHUF plugin" FORCE) # set (BSHUF_GIT_URL "https://github.com/kiyo-masui/bitshuffle.git" CACHE STRING "Use BSHUF from HDF repository" FORCE) # set (BSHUF_GIT_BRANCH "master" CACHE STRING "" FORCE) @@ -122,7 +122,8 @@ set (LBANN_SB_FWD_HDF5_PLUGINS_BUILD_BSHUF_LZ4_LIBRARY_SOURCE OFF CACHE BOOL "bu # ############################# bzip2 ################################################ -set (LBANN_SB_FWD_HDF5_PLUGINS_HDF_ENABLE_BZ2_USE_EXTERNAL OFF CACHE BOOL "Use External Library Building for BZ2" FORCE) +#BVE +# set (LBANN_SB_FWD_HDF5_PLUGINS_BZ2_USE_EXTERNAL ON CACHE BOOL "Use External Library Building for BZ2" FORCE) # set (BZ2_GIT_URL "https://github.com/libarchive/bzip2.git" CACHE STRING "Use BZ2 from GitHub repository" FORCE) # set (BZ2_GIT_BRANCH "bzip2-1.0.8" CACHE STRING "" FORCE) @@ -154,7 +155,7 @@ set (LBANN_SB_FWD_HDF5_PLUGINS_HDF_ENABLE_BZ2_USE_EXTERNAL OFF CACHE BOOL "Use E # ############################# lz4 ################################################ -set (LBANN_SB_FWD_HDF5_PLUGINS_BUILD_LZ4_LIBRARY_SOURCE OFF CACHE BOOL "build the lz4 library within the plugin" FORCE) +# set (LBANN_SB_FWD_HDF5_PLUGINS_BUILD_LZ4_LIBRARY_SOURCE OFF CACHE BOOL "build the lz4 library within the plugin" FORCE) # set (LZ4_GIT_URL "https://github.com/lz4/lz4.git" CACHE STRING "Use LZ4 from GitHub repository" FORCE) # set (LZ4_GIT_BRANCH "dev" CACHE STRING "" FORCE) @@ -216,9 +217,9 @@ lbann_sb_add_cmake_extern_pkg( LANGUAGES C CXX ${lbann_sb_ftn_lang} GITHUB_URL HDFGroup/hdf5_plugins.git GIT_TAG "master" + INIT_CONFIG ${CMAKE_CURRENT_BINARY_DIR}/src/config/cmake/cacheinit.cmake DEPENDS_ON HDF5 EXTRA_CMAKE_ARGS - -C${CMAKE_CURRENT_BINARY_DIR}/src/config/cmake/cacheinit.cmake -DUSE_SHARED_LIBS:BOOL=ON -DBUILD_SHARED_LIBS:BOOL=ON ) From 2c690f3ef4ca9c9d8c06af3d2f27432783098e7f Mon Sep 17 00:00:00 2001 From: "Brian C. Van Essen" Date: Tue, 12 Mar 2024 08:02:57 -0700 Subject: [PATCH 3/3] Added the HDF5_PLUGIN_PATH field for the hdf5_plugins. --- scripts/superbuild/CMakeLists.txt | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/scripts/superbuild/CMakeLists.txt b/scripts/superbuild/CMakeLists.txt index f9ce300875a..64ce0755206 100644 --- a/scripts/superbuild/CMakeLists.txt +++ b/scripts/superbuild/CMakeLists.txt @@ -149,9 +149,14 @@ file(WRITE "${CMAKE_BINARY_DIR}/lbann_sb_suggested_cmake_prefix_path.sh" if (LBANN_SB_BUILD_H5Z-ZFP) file(APPEND "${CMAKE_BINARY_DIR}/lbann_sb_suggested_cmake_prefix_path.sh" - "\nexport HDF5_PLUGIN_PATH=${H5Z-ZFP_DIR}/plugin\n") + "\nexport HDF5_PLUGIN_PATH=${H5Z-ZFP_DIR}/plugin:\$\{HDF5_PLUGIN_PATH\}\n") endif (LBANN_SB_BUILD_H5Z-ZFP) +if (LBANN_SB_BUILD_HDF5_PLUGINS) +file(APPEND "${CMAKE_BINARY_DIR}/lbann_sb_suggested_cmake_prefix_path.sh" + "\nexport HDF5_PLUGIN_PATH=${HDF5_PLUGINS_DIR}/lib/plugin:\$\{HDF5_PLUGIN_PATH\}\n") +endif (LBANN_SB_BUILD_HDF5_PLUGINS) + # Add a custom target for bundling all things up if (UNIX) find_program(__FIND_EXE find)