diff --git a/scripts/superbuild/CMakeLists.txt b/scripts/superbuild/CMakeLists.txt new file mode 100644 index 00000000000..164915f64b9 --- /dev/null +++ b/scripts/superbuild/CMakeLists.txt @@ -0,0 +1,180 @@ +################################################################################ +## Copyright (c) 2014-2024, Lawrence Livermore National Security, LLC. +## Produced at the Lawrence Livermore National Laboratory. +## Written by the LBANN Research Team (B. Van Essen, et al.) listed in +## the CONTRIBUTORS file. +## +## LLNL-CODE-697807. +## All rights reserved. +## +## This file is part of LBANN: Livermore Big Artificial Neural Network +## Toolkit. For details, see http://software.llnl.gov/LBANN or +## https://github.com/LLNL/LBANN. +## +## Licensed under the Apache License, Version 2.0 (the "Licensee"); you +## may not use this file except in compliance with the License. You may +## obtain a copy of the License at: +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +## implied. See the License for the specific language governing +## permissions and limitations under the license. +################################################################################ +cmake_minimum_required(VERSION 3.22) +project(LBANN_SuperBuild NONE) + +cmake_policy(SET CMP0097 NEW) + +message("\nWelcome to the LBANN SuperBuild system.\n\nGood luck!\n") + +if (PROJECT_SOURCE_DIR STREQUAL PROJECT_BINARY_DIR) + message(FATAL_ERROR "In-source builds are not permitted.") +endif () + +list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/modules") +include(LBANNSuperBuildAddCMakeExternPkg) +include(LBANNSuperBuildAddPackages) +include(LBANNSuperBuildCreateCMakeArguments) +include(LBANNSuperBuildInitExternPkg) + +# SuperBuild options +option(LBANN_SB_DEFAULT_CUDA_OPTS + "Set Tom's preferred default options for CUDA builds." + OFF) + +option(LBANN_SB_DEFAULT_ROCM_OPTS + "Set Tom's preferred default options for ROCm builds." + OFF) + +option(LBANN_SB_CLONE_VIA_SSH + "Use SSH protocol instead of HTTPS for github." OFF) + +set(LBANN_SB_DEFAULT_INSTALL_PATH_STRATEGY "COMMON" + CACHE STRING "Package prefix selection strategy. [COMMON, PKG, PKG_LC].") + +# List packages that we can build. The names are chosen first to align +# with variables specific to the package (e.g., Aluminum uses +# "ALUMINUM_" as its variable prefix, so we simplify our lives by +# using "ALUMINUM" as the package name). When the variable names do +# not include the package, we fall back on however the package refers +# to itself. +# +# NOTE: This list is ORDERED. "Second-order" dependencies come before +# "first-order" dependencies. +# +# FIXME: Address the above node so that the packages are known before +# descending into subdirectories. +lbann_sb_add_packages( + # Ack, a "third-order" dependency + RCCL + + # These are "second-order" dependencies + adiak # Caliper + Caliper # Aluminum, LBANN + Aluminum # Hydrogen, DiHydrogen + Catch2 # Hydrogen, DiHydrogen + HDF5 # Conduit + JPEG-TURBO # OpenCV + OpenBLAS # Hydrogen + spdlog # DiHydrogen, LBANN + + # These are the "first-order" dependencies + cereal + Clara + CNPY + Conduit + Hydrogen # DiHydrogen depends on H. + DiHydrogen + OpenCV + protobuf + zstr + + # And finally add the option to build LBANN + LBANN) + +# Note: This changes things around slightly from the original +# version. I have flattened this list to IGNORE the dependency +# graph. This makes it slightly more general-purpose (that is, I don't +# care WHY you want to build {HDF5, JPEG-turbo, OpenBLAS} but not +# {Conduit, OpenCV, Hydrogen}, so I'm just going to let you do that). + +# Add the TPL subdirectories +set(_GIT_REPOSITORY_TAG "GIT_REPOSITORY") +set(_GIT_TAG_TAG "GIT_TAG") +foreach (pkg ${LBANN_SB_BUILD_PKGS}) + if (LBANN_SB_BUILD_${pkg}) + string(TOLOWER "${pkg}" pkg_lower) + add_subdirectory(${pkg_lower}) + endif () +endforeach () + +# Print a helpful(?) message +set(LBANN_SB_SUGG_CMAKE_PREFIX_PATH_TMP "\$\{CMAKE_PREFIX_PATH\}") +message("\n-----------------------------------------------------------------\n") +message("LBANN SuperBuild will build the following packages:\n") +foreach (pkg ${LBANN_SB_BUILD_PKGS}) + if (${pkg}_CMAKE_INSTALL_PREFIX) + message(" -- ${pkg} (${${pkg}_CMAKE_INSTALL_PREFIX})") + list(PREPEND LBANN_SB_SUGG_CMAKE_PREFIX_PATH_TMP + "${${pkg}_CMAKE_INSTALL_PREFIX}") + elseif (LBANN_SB_${pkg}_PREFIX) + message(" -- ${pkg} (${LBANN_SB_${pkg}_PREFIX})") + list(PREPEND LBANN_SB_SUGG_CMAKE_PREFIX_PATH_TMP + "${LBANN_SB_${pkg}_PREFIX}") + else () + message(" -- ${pkg} (??????)") + endif () +endforeach () +list(REMOVE_DUPLICATES LBANN_SB_SUGG_CMAKE_PREFIX_PATH_TMP) +string(REPLACE ";" ":" LBANN_SB_SUGG_CMAKE_PREFIX_PATH + "${LBANN_SB_SUGG_CMAKE_PREFIX_PATH_TMP}") +message("\nIt may be useful to do the following:\n") +message("export CMAKE_PREFIX_PATH=${LBANN_SB_SUGG_CMAKE_PREFIX_PATH}\n") +message("or\n") +message("source lbann_sb_suggested_cmake_prefix_path.sh\n") +message("Note that these assume a Bourne-compatible shell.") +message("\n-----------------------------------------------------------------\n") +file(WRITE "${CMAKE_BINARY_DIR}/lbann_sb_suggested_cmake_prefix_path.sh" + "export CMAKE_PREFIX_PATH=${LBANN_SB_SUGG_CMAKE_PREFIX_PATH}\n") + +# Add a custom target for bundling all things up +if (UNIX) + find_program(__FIND_EXE find) + mark_as_advanced(__FIND_EXE) + set(__WORKING_DIR "${CMAKE_BINARY_DIR}") + if (__FIND_EXE) + set(__cmd "${__FIND_EXE};.;\(;-ipath;*/stamp/*.log;-o;-ipath;*/CMakeFiles/CMake*.log;-o;-name;CMakeCache.txt;\);-exec;${CMAKE_COMMAND};-E;tar;czf;all_output_logs.tar.gz;--;{};+") + add_custom_target(gather-logs + COMMAND "${__cmd}" + BYPRODUCTS "${__WORKING_DIR}/all_output_logs.tar.gz" + WORKING_DIRECTORY "${__WORKING_DIR}" + COMMENT "Gathering all output logs." + VERBATIM + COMMAND_EXPAND_LISTS + USES_TERMINAL) + + add_custom_target(gather-all) + add_dependencies(gather-all gather-logs) + if (CMAKE_GENERATOR STREQUAL "Ninja") + set(__cmd "${__FIND_EXE};.;-name;*.ninja;-exec;${CMAKE_COMMAND};-E;tar;czf;all_build_files.tar.gz;{};+") + elseif (CMAKE_GENERATOR STREQUAL "Unix Makefiles") + set(__cmd "${__FIND_EXE};.;\(;-name;link.txt;-o;-name;build.make;-o;-name;flags.make;\);-exec;${CMAKE_COMMAND};-E;tar;czf;all_build_files.tar.gz;{};+") + else () + set(__cmd) + endif () + if (__cmd) + add_custom_target(gather-build + COMMAND "${__cmd}" + BYPRODUCTS "${__WORKING_DIR}/all_build_files.tar.gz" + WORKING_DIRECTORY "${__WORKING_DIR}" + COMMENT "Gathering all build files." + VERBATIM + COMMAND_EXPAND_LISTS + USES_TERMINAL) + add_dependencies(gather-all gather-build) + endif () + endif (__FIND_EXE) +endif (UNIX) diff --git a/scripts/superbuild/README.md b/scripts/superbuild/README.md new file mode 100644 index 00000000000..ad5379ccc7b --- /dev/null +++ b/scripts/superbuild/README.md @@ -0,0 +1,326 @@ +# LBANN SuperBuild + +Welcome to the LBANN SuperBuild project. Let's begin by establishing +some boundaries and expectations. + +## What this is + +This SuperBuild infrastructure was originally developed as part of the +[LBANN Project](https://github.com/LLNL/LBANN) as a way to build the +first-order third-party dependencies for LBANN. It existed for some +time hidden behind the infamous and storied `build_lbann_lc.sh` shell +script. However, it was deprecated when Spack was determined to be +mature and capable enough to build LBANN. It was officially removed +from LBANN when the development team at the time had embraced the +Spack build. + +Under normal circumstances, the official position of the LBANN team +(myself included) is that users and developers should use the LBANN +build script (which uses Spack under the hood). If you're here, it's +likely because the build script just isn't working for you, and +hopefully this will get you started. + +The SuperBuild can build all of the first-order dependencies of LBANN +(except Python, due to its ubiquity), as well as a select few of its +second-order dependencies that are either deemed to have significant +performance impact (JPEG-turbo, OpenBLAS) or subject to version +constraints that are unlikely to be fulfilled by system defaults +(HDF5). It can also build LBANN itself, either from scratch or from a +source directory that a user provides. As long as you have a +mostly-sane operating system, this should get you a working build of +LBANN. + +Above all, bear in mind: this is a *build tool*. It builds stuff, and +that's it. + +The primary build targets are developer workstations (Linux and macOS) +and Department of Energy clusters/supercomputers. It is regularly +tested on a variety of LLNL machines as well as Perlmutter at NERSC. + +## What this is NOT + +This is not a package manager. Use `pacman`, `dnf`, `apt`, etc. for +that. Or use Spack if you're really desperate. + +In particular, I don't track versions all that closely, and I don't do +anything to "update" packages beyond what CMake gives me for free. + +I don't do anything special with Python. If you use this to build +LBANN and you can't run something due to a missing Python dependency, +install it however you would normally install a Python package. + +The SuperBuild will do very little to babysit your build. It will +happily let you metaphorically shoot yourself in the foot. It does +exactly what you tell it to do. + +# Supported packages + +The following packages are known by the SuperBuild framework: + +- [Aluminum](https://github.com/LLNL/Aluminum) - High-performance + communication library that provides a stream-aware interface and + semantics. +- [Catch2](https://github.com/catchorg/catch2) - A unit-testing + framework for C++ packages. (Mostly for developers; also used by + H2 and Hydrogen, if enabled.) +- [cereal](https://github.com/uscilab/cereal) - C++ serialization + library. +- [Clara](https://github.com/catchorg/clara) - Argument parser (yes, + it *is* deprecated, and no, I don't care. It works. It could be + replaced by cxxopts or something if that changes.) +- [CNPY](https://github.com/rogersce/cnpy) - Read and write NumPy data + in C++. +- [Conduit](https://github.com/LLNL/Conduit) - Yet another + serialization format. Apparently there aren't enough of these. +- [DiHydrogen](https://github.com/LLNL/DiHydrogen) - Distributed + tensors and associated operations upon them. This is also the + current location of the legacy "DistConv" library used by LBANN. +- [HDF5](https://github.com/HDFGroup/hdf5) - Hierarchical data format, + but for SCIENCE! +- [Hydrogen](https://github.com/LLNL/Elemental) - GPU-aware matrix + algebra library. +- [JPEG-TURBO](https://github.com/libjpeg-turbo/libjpeg-turbo) - JPEG + but in turbo mode. Zoom zoom zoom. +- [OpenBLAS](https://github.com/xianyi/OpenBLAS.git) - BLAS library + for when your vendor doesn't do a good job. +- [OpenCV](https://github.com/opencv/opencv) - Computer vision + library. +- [protobuf](https://github.com/protocolbuffers/protobuf.git) - And + yet *another* serialization format that LBANN (and others) (ab)use + for model topology description and configuration. +- [spdlog](https://github.com/gabime/spdlog) - Fast C++ logging + library. +- [zstr](https://github.com/mateidavid/zstr) - C++ ZLib wrapper. + +The framework is "opt-in". No package will be built unless the user +explicitly enables it when they configure the SuperBuild project. + +# CMake configuration + +The SuperBuild project is configured and run through +[CMake](https://cmake.org). I personally advocate using the +[Ninja](https://ninja-build.org/) generator, but they should +work. Under the hood, the system works by defining a series of [CMake +External +Projects](https://cmake.org/cmake/help/git-stage/module/ExternalProject.html) +that is controlled by special CMake arguments, documented below. + +An important bit to note is that `ExternalProject` generates the +*infratructure* to configure, build and install other projects. +*However*, these projects are *NOT* configured when the SuperBuild is +generated. This implies that, for things like library- or package- +finding in these projects, the environment that matters is *not* the +environment that exists when `cmake` is run to configure the +SuperBuild but rather the environment that is exported to the +subsequent invocation of the generated build system (e.g., `ninja` or +`make` or whatever). This should be borne in mind when thinking about +how to ensure that packages are resolving dependencies appropriately. + +On this note, the SuperBuild only offers dependency-resolution +assistance with packages built by the SuperBuild. For example, if a +user builds HDF5 *and* Conduit with *the same* SuperBuild, the +SuperBuild itself will attempt to forward the correct HDF5 directory +to Conduit so that Conduit's configuration finds the HDF5 that was +built by this SuperBuild build system. However, if some other HDF5 is +to be used, the onus is on the user to forward the necessary +information about its existence and location to the Conduit package in +the SuperBuild (e.g., by setting +`LBANN_SB_FWD_Conduit_HDF5_DIR=/path/to/hdf5/prefix` in the SuperBuild +configuration line, [as described +below](#passing-options-to-superbuild-packages). + +The main SuperBuild project is designed to be a passive task +manager. In CMake-speak, the language is `NONE` and it never calls a +`find_{path,package,executable,library}` function. It simply +configures a system of `ExternalProject` calls that do ALL of the work +of building these packages. This has the benefit that it's nearly +impossible for the configuration of the SuperBuild project itself to +fail. The downside is that it pushes all configuration issues to the +individual packages being built. + +## SuperBuild CMake Options + +There are several options and variables that can be used to control +the SuperBuild project itself. All arguments used by the SuperBuild +package are either standard CMake options (prefixed with `CMAKE_`) or +use the `LBANN_SB_` prefix. + +- `LBANN_SB_BUILD_` enables the build for ``, which must + match a package (case-sensitive) in the list of packages above. + +- `LBANN_SB_CLONE_VIA_SSH` switches HTTPS GitHub URLs for SSH-based + ones. The onus is on the user to ensure their SSH authentication + method is setup and working. + +- `LBANN_SB_DEFAULT_INSTALL_PATH_STRATEGY` defines how to modify the + top-level `CMAKE_INSTALL_PREFIX` at the package level. Acceptible + values are: + - `COMMON` (default): Any package that doesn't specify its own + prefix will install to the SuperBuild's `CMAKE_INSTALL_PREFIX` (if + set). + - `PKG`: Any package that doesn't specify its own prefix will + install to `CMAKE_INSTALL_PREFIX/`, with the casing of `PKG` + matching that in the list of packages above. + - `PKG_LC`: Any package that doesn't specify its own prefix will + install to `CMAKE_INSTALL_PREFIX/`, where `` is the + lower-cased version of the package name in the list of packages + above. + +The following variables will be forwarded to all GPU-enabled packages +if it can be detected that they are being built with GPU support. CUDA +packages will inherit + +- `CMAKE_CUDA_ARCHITECTURES` + +ROCm packages will inherit all of + +- `CMAKE_HIP_ARCHITECTURES` +- `AMDGPU_TARGETS` +- `GPU_TARGETS` + +If any of those are set, all will inherit that value and be forwarded +appropriately. (I have no explanation for why these are ALL needed, +but I get sporadic and nonsensical failures if I omit any. It is +zero-cost-to-me to forward them all, and therefore zero-value-to-me to +diagnose this (non-)issue.) + +## Passing options to SuperBuild packages + +The SuperBuild project attempts to expose mechanisms to allow even +fine-grained control over the build of its component packages. The +level of control is much higher if the package uses a CMake build, but +an effort is made for other types of projects. Packages have some +common options that are used to configure the `ExternalProject` +command in CMake. These are prefixed with `LBANN_SB__`, where +`` matches a package (case-sensitive) in the list of packages +above. The mechanism for package-specific options are described in +detail [below](#options-specific-to-a-package). + +### Common options + +The following options are available to any package in the SuperBuild +system and may be used to customize the properties of the build of +those packages. It is the user's responsibility to ensure no +compatibility issues arise (e.g., using two different and incompatible +compilers to build two different packages). The options that are tied +to CMake options work by forwarding values to the +`ExternalProject`. This is (usually) easy for CMake projects, and a +"best effort" is made to map them to non-CMake-based build systems. + +- `LBANN_SB__PREFIX` - The installation prefix for the `` + package. If not provided, this defaults to `CMAKE_INSTALL_PREFIX` + modified according to `LBANN_SB_DEFAULT_INSTALL_PATH_STRATEGY`. +- `LBANN_SB___COMPILER` - The compiler for the `` + language for the `` package. The default is the top-level + `CMAKE__COMPILER`, if provided. Otherwise no value is + forwarded and it is left to the package configure stage to deduce. +- `LBANN_SB___FLAGS` - The flags to pass to the compiler + for the `` language for the `` package. The default is + the top-level `CMAKE__FLAGS`, if provided. Otherwise no value + is forwarded and it is left to the package configure stage to + deduce. +- `LBANN_SB__SOURCE_DIR` - Specify a local directory to use to + build the `` package. If this is set, it must be the top-level + directory for the build system of `` (for a CMake package, the + directory containing the top-level `CMakeLists.txt`). Using this + option will disable ALL automatic Git steps and the source code will + be built as-is. +- `LBANN_SB__URL` - The URL from which to clone the `` + package. This must be a Git repository. This defaults to some flavor + of the "official" project repository (with a preference toward + Github if multiple candidates exist). This is ignored if + `LBANN_SB__SOURCE_DIR` is set. +- `LBANN_SB__TAG` - The Git tag to check out for the `` + package. The SuperBuild specifies a default tag for each package + that is a version acceptible to use with the `develop` branch of + LBANN. +- `LBANN_SB__CMAKE_GENERATOR` - Setup the CMake generator to use + for the `` package. This has no effect if the package is not a + CMake package. It will fail at *SuperBuild build time* (during the + configure step for ``) if this is not a valid CMake generator + string. The default value is the generator used to configure the + SuperBuild. +- `LBANN_SB__BUILD_SHARED_LIBS` - Build shared libraries for the + `` package. This may not have an effect if the package is not a + CMake package. The default value is the value of the top-level + `BUILD_SHARED_LIBS`. +- `LBANN_SB__IPO` - Attempt to enable interprocedural + optimization for the `` package. The default value is the value + of the top-level `CMAKE_INTERPROCEDURAL_OPTIMIZATION`. This only has + an effect in CMake packages. Non-CMake packages should use compiler + flags to manually specify IPO options. +- `LBANN_SB__PIC` - Attempt to enable + position-independent code for the `` package. This may override + any inherited behavior from `LBANN_SB__BUILD_SHARED_LIBS`. This + only has an effect in CMake packages. Non-CMake packages should use + compiler flags to manually specify PIC/PIE flags. + +### Options specific to a package + +Packages in the SuperBuild that use CMake may have other options +forwarded to them by using a special variable syntax: +`LBANN_SB_FWD__=value`. For a project named `Foo`, +we could set a variable, `WITH_SOME_FEATURE=ON`, in the `Foo` build +system by passing `LBANN_SB_FWD_Foo_WITH_SOME_FEATURE=ON` to the +SuperBuild configuration. When the CMake `ExternalProject` is +generated for `Foo`, the configuration step will include `cmake +... -DWITH_SOME_FEATURE=ON ... /path/to/Foo/src` automatically. + +Packages that do NOT use CMake builds (notably OpenBLAS) may have +additional options associated with them if they are coded into that +package's `CMakeLists.txt` format. At this time, there's no "hard and +fast rule" for the format of these options, and they may not follow +the `LBANN_SB_FWD_` syntax. Indeed, the option may not be forwarding +anything but rather manipulating the `ExternalProject_Add` +specification, so `_FWD_` may be semantically inappropriate. I haven't +decided yet if I want to push for it anyway for "consistency". In the +meantime, users are encouraged to consult the corresponding +`CMakeLists.txt` file or their CMake cache (`ccmake`, e.g.) for a list +of exposed options for these packages. + +Some packages in the SuperBuild will have default settings that are +configured to match the "usual usage" for LBANN. These are exposed by +`CACHE` variables or explicit `option()` calls in the SuperBuild CMake +configuration, and users will see these appear in their +`CMakeCache.txt` even if they haven't explicitly set these +variables. They generally follow the same variable name syntax as +described above for consistency, though users are encouraged to verify +this in the appropriate `CMakeLists.txt` files. + +# Debugging SuperBuild issues + +SuperBuild packages are all built with a common structure. Unless +`LBANN_SB__SOURCE_DIR` was specified, the source clone will be +located in `//src` The scripts that the SuperBuild +build stage will execute, as well as the log files generated by these +commands, are located in `//stamp`. Auxiliary scripts +may be located in `//tmp`. All CMake-based packages +have their build directories located in `//build`. +Packages that build in the source tree (e.g., OpenBLAS) will not have +such a directory. + +If configuration was successful, a fully-functional build system should +be present in the `//build` directory, and a user +should be able to invoke `cmake --build //build` to +perform a standalone (re)build. This can be useful for incremental +rebuilds while debugging and/or editing the package. + +(TODO: Finish this section.) + +# Getting help + +If something fails at SuperBuild configuration time (i.e., when you +run `cmake` for this project), please [open an +issue](https://github.com/llnl/lbann/issues/new) and begin the title +with "[SuperBuild]" so I know to look. Please attach *THE WHOLE +OUTPUT* of CMake. I may also ask you for the log files it notes at the +end, usually `/CMakeFiles/CMake{Output,Error.log}`, so it's +not a bad idea to send those along, too. + +If something fails in one of the package builds, and you're using the +Ninja or Makefiles generators, there's a target called `gather-all` +that will attempt to collect the relevant build system files and log +files for every package that was enabled. I will probably ask for the +tarballs generated by this target. When filing an issue, please +indicate which package is failing and if you have uncovered any +insight into why that's happening. diff --git a/scripts/superbuild/adiak/CMakeLists.txt b/scripts/superbuild/adiak/CMakeLists.txt new file mode 100644 index 00000000000..dc829adfc9d --- /dev/null +++ b/scripts/superbuild/adiak/CMakeLists.txt @@ -0,0 +1,72 @@ +################################################################################ +## Copyright (c) 2014-2024, Lawrence Livermore National Security, LLC. +## Produced at the Lawrence Livermore National Laboratory. +## Written by the LBANN Research Team (B. Van Essen, et al.) listed in +## the CONTRIBUTORS file. +## +## LLNL-CODE-697807. +## All rights reserved. +## +## This file is part of LBANN: Livermore Big Artificial Neural Network +## Toolkit. For details, see http://software.llnl.gov/LBANN or +## https://github.com/LLNL/LBANN. +## +## Licensed under the Apache License, Version 2.0 (the "Licensee"); you +## may not use this file except in compliance with the License. You may +## obtain a copy of the License at: +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +## implied. See the License for the specific language governing +## permissions and limitations under the license. +################################################################################ +# adiak-specific configuration options explicitly exposed +lbann_sb_default_pkg_option(adiak + ENABLE_MPI + "Enable MPI support in adiak." + ON) + +lbann_sb_default_pkg_option(adiak + ENABLE_OPENMP + "Enable OpenMP support in adiak." + ON) + +lbann_sb_default_cuda_option( + adiak + ENABLE_CUDA + "Enable CUDA support in adiak." + ON) + +lbann_sb_default_rocm_option( + adiak + ENABLE_HIP + "Enable HIP support in adiak." + ON) + +lbann_sb_default_pkg_option(adiak + ENABLE_TESTS + "Build adiak unit tests." + OFF) + +lbann_sb_default_pkg_option(adiak + ENABLE_FORTRAN + "Build adiak with fortran support" + OFF) + +set(lbann_sb_ftn_lang) +if (LBANN_SB_FWD_adiak_ENABLE_FORTRAN) + set(lbann_sb_ftn_lang Fortran) +endif () + +lbann_sb_add_cmake_extern_pkg( + NAME adiak + LANGUAGES C CXX ${lbann_sb_ftn_lang} + GITHUB_URL llnl/adiak.git + GIT_TAG "master" +) + +set(adiak_DIR ${LBANN_SB_adiak_PREFIX} + CACHE INTERNAL "The install prefix of adiak.") diff --git a/scripts/superbuild/aluminum/CMakeLists.txt b/scripts/superbuild/aluminum/CMakeLists.txt new file mode 100644 index 00000000000..c04e78c0100 --- /dev/null +++ b/scripts/superbuild/aluminum/CMakeLists.txt @@ -0,0 +1,67 @@ +################################################################################ +## Copyright (c) 2014-2024, Lawrence Livermore National Security, LLC. +## Produced at the Lawrence Livermore National Laboratory. +## Written by the LBANN Research Team (B. Van Essen, et al.) listed in +## the CONTRIBUTORS file. +## +## LLNL-CODE-697807. +## All rights reserved. +## +## This file is part of LBANN: Livermore Big Artificial Neural Network +## Toolkit. For details, see http://software.llnl.gov/LBANN or +## https://github.com/LLNL/LBANN. +## +## Licensed under the Apache License, Version 2.0 (the "Licensee"); you +## may not use this file except in compliance with the License. You may +## obtain a copy of the License at: +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +## implied. See the License for the specific language governing +## permissions and limitations under the license. +################################################################################ +lbann_sb_default_pkg_option( + Aluminum + ALUMINUM_ENABLE_CALIPER + "Enable Caliper support in Aluminum" + ${LBANN_SB_BUILD_Caliper}) + +lbann_sb_default_cuda_option( + Aluminum + ALUMINUM_ENABLE_CUDA + "Enable the base CUDA support in Aluminum" + ON) + +lbann_sb_default_rocm_option( + Aluminum + ALUMINUM_ENABLE_ROCM + "Enable the base ROCM support in Aluminum" + ON) + +lbann_sb_default_gpu_option( + Aluminum + ALUMINUM_ENABLE_HOST_TRANSFER + "Enable the host-transfer backend in Aluminum" + ON) + +if (LBANN_SB_BUILD_RCCL) + lbann_sb_default_rocm_option( + Aluminum + ALUMINUM_ENABLE_NCCL + "Enable RCCL support in Aluminum" + ON) +endif () + +lbann_sb_add_cmake_extern_pkg( + NAME Aluminum + LANGUAGES CXX + OPTIONAL_LANGUAGES CUDA HIP + GITHUB_URL llnl/Aluminum.git + GIT_TAG "master" + DEPENDS_ON Caliper RCCL) + +set(Aluminum_DIR ${LBANN_SB_Aluminum_PREFIX} + CACHE INTERNAL "The install prefix of Aluminum.") diff --git a/scripts/superbuild/caliper/CMakeLists.txt b/scripts/superbuild/caliper/CMakeLists.txt new file mode 100644 index 00000000000..93783f31ac8 --- /dev/null +++ b/scripts/superbuild/caliper/CMakeLists.txt @@ -0,0 +1,95 @@ +################################################################################ +## Copyright (c) 2014-2024, Lawrence Livermore National Security, LLC. +## Produced at the Lawrence Livermore National Laboratory. +## Written by the LBANN Research Team (B. Van Essen, et al.) listed in +## the CONTRIBUTORS file. +## +## LLNL-CODE-697807. +## All rights reserved. +## +## This file is part of LBANN: Livermore Big Artificial Neural Network +## Toolkit. For details, see http://software.llnl.gov/LBANN or +## https://github.com/LLNL/LBANN. +## +## Licensed under the Apache License, Version 2.0 (the "Licensee"); you +## may not use this file except in compliance with the License. You may +## obtain a copy of the License at: +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +## implied. See the License for the specific language governing +## permissions and limitations under the license. +################################################################################ +# Caliper-specific configuration options explicitly exposed +lbann_sb_default_pkg_option( + Caliper + WITH_ADIAK + "Enable adiak support in Caliper." + ${LBANN_SB_BUILD_adiak}) + +lbann_sb_default_pkg_option( + Caliper + WITH_FORTRAN + "Enable fortran support in Caliper." + OFF) + +lbann_sb_default_pkg_option( + Caliper + WITH_OMPT + "Enable OMPT support in Caliper." + OFF) + +lbann_sb_default_pkg_option( + Caliper + WITH_MPI + "Enable MPI support in Caliper." + ON) + +lbann_sb_default_cuda_option( + Caliper + WITH_CUPTI + "Enable cupty support in Caliper." + ON) + +lbann_sb_default_cuda_option( + Caliper + WITH_NVTX + "Enable nvtx support in Caliper." + ON) + +lbann_sb_default_rocm_option( + Caliper + WITH_ROCM + "Enable ROCm support in Caliper." + ON) + +lbann_sb_default_rocm_option( + Caliper + WITH_ROCTRACER + "Enable roctracer support in Caliper." + ON) + +lbann_sb_default_rocm_option( + Caliper + WITH_ROCTX + "Enable roctx support in Caliper." + ON) + +set(lbann_sb_ftn_lang) +if (LBANN_SB_FWD_Caliper_WITH_FORTRAN) + set(lbann_sb_ftn_lang Fortran) +endif () + +lbann_sb_add_cmake_extern_pkg( + NAME Caliper + LANGUAGES C CXX ${lbann_sb_ftn_lang} + GITHUB_URL llnl/caliper.git + GIT_TAG "master" + DEPENDS_ON adiak +) + +set(Caliper_DIR ${LBANN_SB_Caliper_PREFIX} + CACHE INTERNAL "The install prefix of Caliper.") diff --git a/scripts/superbuild/catch2/CMakeLists.txt b/scripts/superbuild/catch2/CMakeLists.txt new file mode 100644 index 00000000000..70fe220d9ce --- /dev/null +++ b/scripts/superbuild/catch2/CMakeLists.txt @@ -0,0 +1,40 @@ +################################################################################ +## Copyright (c) 2014-2024, Lawrence Livermore National Security, LLC. +## Produced at the Lawrence Livermore National Laboratory. +## Written by the LBANN Research Team (B. Van Essen, et al.) listed in +## the CONTRIBUTORS file. +## +## LLNL-CODE-697807. +## All rights reserved. +## +## This file is part of LBANN: Livermore Big Artificial Neural Network +## Toolkit. For details, see http://software.llnl.gov/LBANN or +## https://github.com/LLNL/LBANN. +## +## Licensed under the Apache License, Version 2.0 (the "Licensee"); you +## may not use this file except in compliance with the License. You may +## obtain a copy of the License at: +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +## implied. See the License for the specific language governing +## permissions and limitations under the license. +################################################################################ + +# This prefers Catch2 v3 +lbann_sb_add_cmake_extern_pkg( + NAME Catch2 + LANGUAGES CXX + GITHUB_URL catchorg/catch2.git + GIT_TAG "devel" + EXTRA_CMAKE_ARGS + -DCATCH_BUILD_TESTING=OFF + -DCATCH_BUILD_EXAMPLES=OFF + -DCATCH_ENABLE_WERROR=OFF + ) + +set(CATCH2_DIR ${LBANN_SB_Catch2_PREFIX} + CACHE INTERNAL "The install prefix of Catch2.") diff --git a/scripts/superbuild/cereal/CMakeLists.txt b/scripts/superbuild/cereal/CMakeLists.txt new file mode 100644 index 00000000000..2a843004f82 --- /dev/null +++ b/scripts/superbuild/cereal/CMakeLists.txt @@ -0,0 +1,39 @@ +################################################################################ +## Copyright (c) 2014-2024, Lawrence Livermore National Security, LLC. +## Produced at the Lawrence Livermore National Laboratory. +## Written by the LBANN Research Team (B. Van Essen, et al.) listed in +## the CONTRIBUTORS file. +## +## LLNL-CODE-697807. +## All rights reserved. +## +## This file is part of LBANN: Livermore Big Artificial Neural Network +## Toolkit. For details, see http://software.llnl.gov/LBANN or +## https://github.com/LLNL/LBANN. +## +## Licensed under the Apache License, Version 2.0 (the "Licensee"); you +## may not use this file except in compliance with the License. You may +## obtain a copy of the License at: +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +## implied. See the License for the specific language governing +## permissions and limitations under the license. +################################################################################ +option(LBANN_SB_FWD_cereal_JUST_INSTALL_CEREAL + "cereal value-forward for JUST_INSTALL_CEREAL" ON) +option(LBANN_SB_FWD_cereal_WITH_WERROR + "CEREAL value-forward for WITH_WERROR" OFF) + +lbann_sb_add_cmake_extern_pkg( + NAME cereal + LANGUAGES C CXX + GITHUB_URL uscilab/cereal.git + GIT_TAG v1.3.0 +) + +set(cereal_DIR ${LBANN_SB_cereal_PREFIX} + CACHE INTERNAL "The install prefix of cereal.") diff --git a/scripts/superbuild/clara/CMakeLists.txt b/scripts/superbuild/clara/CMakeLists.txt new file mode 100644 index 00000000000..4c9fe65cfa5 --- /dev/null +++ b/scripts/superbuild/clara/CMakeLists.txt @@ -0,0 +1,61 @@ +################################################################################ +## Copyright (c) 2014-2024, Lawrence Livermore National Security, LLC. +## Produced at the Lawrence Livermore National Laboratory. +## Written by the LBANN Research Team (B. Van Essen, et al.) listed in +## the CONTRIBUTORS file. +## +## LLNL-CODE-697807. +## All rights reserved. +## +## This file is part of LBANN: Livermore Big Artificial Neural Network +## Toolkit. For details, see http://software.llnl.gov/LBANN or +## https://github.com/LLNL/LBANN. +## +## Licensed under the Apache License, Version 2.0 (the "Licensee"); you +## may not use this file except in compliance with the License. You may +## obtain a copy of the License at: +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +## implied. See the License for the specific language governing +## permissions and limitations under the license. +################################################################################ +lbann_sb_init_extern_pkg( + NAME Clara + LANGUAGES CXX + GITHUB_URL catchorg/clara.git + GIT_TAG v1.1.5) + +# Manually spec the external project +include(ExternalProject) +ExternalProject_Add(Clara + PREFIX ${CMAKE_CURRENT_BINARY_DIR} + TMP_DIR ${CMAKE_CURRENT_BINARY_DIR}/tmp + STAMP_DIR ${CMAKE_CURRENT_BINARY_DIR}/stamp + + SOURCE_DIR ${LBANN_SB_Clara_SOURCE_DIR} + INSTALL_DIR ${LBANN_SB_Clara_PREFIX} + BUILD_IN_SOURCE 1 + + ${_GIT_REPOSITORY_TAG} ${LBANN_SB_Clara_URL} + ${_GIT_TAG_TAG} ${LBANN_SB_Clara_TAG} + + USES_TERMINAL_INSTALL 1 + LOG_DOWNLOAD 1 + LOG_UPDATE 1 + LOG_INSTALL 1 + + CONFIGURE_COMMAND "" + BUILD_COMMAND "" + + INSTALL_COMMAND + ${CMAKE_COMMAND} -E copy_directory + ${LBANN_SB_Clara_SOURCE_DIR}/single_include + ${LBANN_SB_Clara_PREFIX}/include +) + +set(Clara_DIR ${LBANN_SB_Clara_PREFIX} + CACHE INTERNAL "The install prefix of Clara.") diff --git a/scripts/superbuild/cmake/modules/LBANNSuperBuildAddCMakeExternPkg.cmake b/scripts/superbuild/cmake/modules/LBANNSuperBuildAddCMakeExternPkg.cmake new file mode 100644 index 00000000000..5794aacada0 --- /dev/null +++ b/scripts/superbuild/cmake/modules/LBANNSuperBuildAddCMakeExternPkg.cmake @@ -0,0 +1,339 @@ +################################################################################ +## Copyright (c) 2014-2024, Lawrence Livermore National Security, LLC. +## Produced at the Lawrence Livermore National Laboratory. +## Written by the LBANN Research Team (B. Van Essen, et al.) listed in +## the CONTRIBUTORS file. +## +## LLNL-CODE-697807. +## All rights reserved. +## +## This file is part of LBANN: Livermore Big Artificial Neural Network +## Toolkit. For details, see http://software.llnl.gov/LBANN or +## https://github.com/LLNL/LBANN. +## +## Licensed under the Apache License, Version 2.0 (the "Licensee"); you +## may not use this file except in compliance with the License. You may +## obtain a copy of the License at: +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +## implied. See the License for the specific language governing +## permissions and limitations under the license. +################################################################################ +function(lbann_sb_create_extern_pkg_cmake_args OUTPUT_VAR) + set(_fn_opts) + set(_fn_one_val PKG_NAME) + set(_fn_multi_val VARIABLES) + cmake_parse_arguments(lbann_sb_args + "${_fn_opts}" + "${_fn_one_val}" + "${_fn_multi_val}" ${ARGN}) + + set(lbann_sb_args_strip_regex "^LBANN_SB_FWD_${lbann_sb_args_PKG_NAME}_") + + set(lbann_sb_args_arg_list) + foreach (var ${lbann_sb_args_VARIABLES}) + string(REGEX REPLACE "${lbann_sb_args_strip_regex}" "" + lbann_sb_args_arg_name "${var}") + list(APPEND lbann_sb_args_arg_list + "-D${lbann_sb_args_arg_name}=${${var}}") + endforeach () + + # Return + set(${OUTPUT_VAR} "${lbann_sb_args_arg_list}" PARENT_SCOPE) +endfunction () + +function(lbann_sb_create_pkg_lang_cmake_args OUTPUT_VAR) + set(_fn_opts) + set(_fn_one_val PKG_NAME) + set(_fn_multi_val LANGUAGES) + cmake_parse_arguments(lbann_sb_args + "${_fn_opts}" + "${_fn_one_val}" + "${_fn_multi_val}" ${ARGN}) + + set(NAME "${lbann_sb_args_PKG_NAME}") + set(lbann_sb_args_arg_list) + foreach (lang ${lbann_sb_args_LANGUAGES}) + if ("${lang}" STREQUAL "NONE") + continue() + endif () + if (LBANN_SB_${NAME}_${lang}_COMPILER) + list(APPEND lbann_sb_args_arg_list + "-DCMAKE_${lang}_COMPILER:STRING=${LBANN_SB_${NAME}_${lang}_COMPILER}") + endif () + if (LBANN_SB_${NAME}_${lang}_STANDARD) + list(APPEND lbann_sb_args_arg_list + "-DCMAKE_${lang}_STANDARD:STRING=${LBANN_SB_${NAME}_${lang}_STANDARD}") + endif () + if (LBANN_SB_${NAME}_${lang}_FLAGS) + list(APPEND lbann_sb_args_arg_list + "-DCMAKE_${lang}_FLAGS:STRING=${LBANN_SB_${NAME}_${lang}_FLAGS}") + endif () + if (LBANN_SB_${NAME}_${lang}_HOST_COMPILER) + list(APPEND lbann_sb_args_arg_list + "-DCMAKE_${lang}_HOST_COMPILER:STRING=${LBANN_SB_${NAME}_${lang}_HOST_COMPILER}") + endif () + if ("${lang}" STREQUAL "CUDA") + list(APPEND lbann_sb_args_arg_list + "-DCMAKE_${lang}_ARCHITECTURES:STRING=${LBANN_SB_${NAME}_${lang}_ARCHITECTURES}") + endif () + if ("${lang}" STREQUAL "HIP") + list(APPEND lbann_sb_args_arg_list + "-DCMAKE_${lang}_ARCHITECTURES:STRING=${LBANN_SB_${NAME}_${lang}_ARCHITECTURES}" + "-DAMDGPU_TARGETS:STRING=${LBANN_SB_${NAME}_${lang}_ARCHITECTURES}" + "-DGPU_TARGETS:STRING=${LBANN_SB_${NAME}_${lang}_ARCHITECTURES}") + endif () + + if (LBANN_SB_${PKG_NAME}_BUILD_TYPE) + string(TOUPPER "${LBANN_SB_${PKG_NAME}_BUILD_TYPE}" build_type) + if (LBANN_SB_${NAME}_${lang}_FLAGS_${build_type}) + string(CONCAT _argstring + "-DCMAKE_${lang}_FLAGS_${build_type}:STRING=" + "${LBANN_SB_${NAME}_${lang}_FLAGS_${build_type}}") + list(APPEND lbann_sb_args_arg_list "${_argstring}") + endif () + endif () + endforeach () + + # Not language-specific, but this seems like the best place for now. + if (LBANN_SB_${NAME}_SHARED_LINKER_FLAGS) + string(CONCAT _argstring + "-DCMAKE_SHARED_LINKER_FLAGS:STRING=" + "${LBANN_SB_${NAME}_SHARED_LINKER_FLAGS}") + list(APPEND lbann_sb_args_arg_list "${_argstring}") + endif () + if (LBANN_SB_${NAME}_STATIC_LINKER_FLAGS) + string(CONCAT _argstring + "-DCMAKE_STATIC_LINKER_FLAGS:STRING=" + "${LBANN_SB_${NAME}_STATIC_LINKER_FLAGS}") + list(APPEND lbann_sb_args_arg_list "${_argstring}") + endif () + if (LBANN_SB_${NAME}_EXE_LINKER_FLAGS) + string(CONCAT _argstring + "-DCMAKE_EXE_LINKER_FLAGS:STRING=" + "${LBANN_SB_${NAME}_EXE_LINKER_FLAGS}") + list(APPEND lbann_sb_args_arg_list "${_argstring}") + endif () + if (LBANN_SB_${PKG_NAME}_BUILD_TYPE) + string(TOUPPER "${LBANN_SB_${PKG_NAME}_BUILD_TYPE}" build_type) + if (LBANN_SB_${NAME}_SHARED_LINKER_FLAGS_${build_type}) + string(CONCAT _argstring + "-DCMAKE_SHARED_LINKER_FLAGS_${build_type}:STRING=" + "${LBANN_SB_${NAME}_SHARED_LINKER_FLAGS_${build_type}}") + list(APPEND lbann_sb_args_arg_list "${_argstring}") + endif () + if (LBANN_SB_${NAME}_STATIC_LINKER_FLAGS_${build_type}) + string(CONCAT _argstring + "-DCMAKE_STATIC_LINKER_FLAGS_${build_type}:STRING=" + "${LBANN_SB_${NAME}_STATIC_LINKER_FLAGS_${build_type}}") + list(APPEND lbann_sb_args_arg_list "${_argstring}") + endif () + if (LBANN_SB_${NAME}_EXE_LINKER_FLAGS_${build_type}) + string(CONCAT _argstring + "-DCMAKE_EXE_LINKER_FLAGS_${build_type}:STRING=" + "${LBANN_SB_${NAME}_EXE_LINKER_FLAGS_${build_type}}") + list(APPEND lbann_sb_args_arg_list "${_argstring}") + endif () + endif () + + # RPATHs + if (LBANN_SB_${NAME}_BUILD_RPATH) + list(APPEND lbann_sb_args_arg_list + "-DCMAKE_BUILD_RPATH:STRING=${LBANN_SB_${NAME}_BUILD_RPATH}") + endif () + if (LBANN_SB_${NAME}_INSTALL_RPATH) + list(APPEND lbann_sb_args_arg_list + "-DCMAKE_INSTALL_RPATH:STRING=${LBANN_SB_${NAME}_INSTALL_RPATH}") + endif () + + # Return + set(${OUTPUT_VAR} "${lbann_sb_args_arg_list}" PARENT_SCOPE) +endfunction () + +# Keywords: +# NAME -- the name of the package +# +# LANGUAGES -- the languages for the package +# +# DEPENDS_ON -- Any dependencies of the package. These are packages +# known to the superbuild and MUST be initialized +# prior to invoking this macro. NOTE: this is strictly +# an ordering mechanism required for correct build +# system generation! If detection is not automatic, +# the user is responsible for ensuring this package +# knows to find the dependencies at its build time +# (e.g., by setting CMAKE_PREFIX_PATH or similar). +# +# EXTRA_CMAKE_ARGS -- strings to be forwarded as arguments to +# CMake for this package. +# +# SOURCE_SUBDIR -- Path in the source directory to the toplevel +# CMakeLists.txt file. +# +# This macro also accepts the following keywords, which are forwarded +# directly to lbann_sb_init_extern_pkg: +# +# GITHUB_URL -- the github URL for the package +# GIT_TAG -- the git tag to checkout for this package +include(ExternalProject) +macro(lbann_sb_add_cmake_extern_pkg) + set(_macro_opts) + set(_macro_one_val + NAME + GITHUB_URL + GIT_TAG + SOURCE_SUBDIR) + set(_macro_multi_val + LANGUAGES + EXTRA_CMAKE_ARGS + DEPENDS_ON + OPTIONAL_LANGUAGES) + cmake_parse_arguments(lbann_sb_add + "${_macro_opts}" + "${_macro_one_val}" + "${_macro_multi_val}" ${ARGN}) + + # Perform the common initialization tasks. A non-CMake package (or + # something with weird stuff going on such that they need to + # customize the ExternalProject_Add call) would just call this + # directly. The subset of arguments that it cares about are noted in + # that macro's documentation. + lbann_sb_init_extern_pkg( + NAME ${lbann_sb_add_NAME} + LANGUAGES ${lbann_sb_add_LANGUAGES} + OPTIONAL_LANGUAGES ${lbann_sb_add_OPTIONAL_LANGUAGES} + GITHUB_URL ${lbann_sb_add_GITHUB_URL} + GIT_TAG ${lbann_sb_add_GIT_TAG}) + + # Handle the DEPENDS_ON tag. This assumes that any package name + # passed here has already been initialized. + set(LBANN_SB_DEPENDS_TAG) + set(LBANN_SB_${PKG_NAME}_DEPENDS) + set(LBANN_SB_${PKG_NAME}_DEPENDS_PATHS) + foreach (pkg ${lbann_sb_add_DEPENDS_ON}) + message(STATUS "${PKG_NAME}: Checking for ${pkg}") + if (TARGET ${pkg}) + list(APPEND LBANN_SB_${PKG_NAME}_DEPENDS ${pkg}) + list(APPEND + LBANN_SB_${PKG_NAME}_DEPENDS_PATHS + "${LBANN_SB_${pkg}_PREFIX}") + set(LBANN_SB_FWD_${PKG_NAME}_${pkg}_ROOT "${LBANN_SB_${pkg}_PREFIX}") + endif () + endforeach () + if (LBANN_SB_${PKG_NAME}_DEPENDS) + set(LBANN_SB_DEPENDS_TAG "DEPENDS") + string(REPLACE ";" "|" + LBANN_SB_FWD_${PKG_NAME}_CMAKE_PREFIX_PATH + "${LBANN_SB_${PKG_NAME}_DEPENDS_PATHS}") + message(STATUS "${PKG_NAME} depends on: ${LBANN_SB_${PKG_NAME}_DEPENDS}") + endif () + + # Get the variables to forward. + get_property(PKG_VARIABLES DIRECTORY PROPERTY VARIABLES) + set(lbann_sb_add_var_incl_regex "^LBANN_SB_FWD_${PKG_NAME}_.*") + list(FILTER PKG_VARIABLES INCLUDE REGEX "${lbann_sb_add_var_incl_regex}") + # Unlike previous versions of the code, we don't need the EXCLUDE + # REGEX clause because we now prefix any and all SuperBuild + # variables with "LBANN_SB". Of those, we only care about those + # followed by "_FWD_${PKG_NAME}", which we collect via the INCLUDE + # REGEX. + + # This function will take the list of variables and copy them into a + # string of the form "-D${REAL_VAR_NAME}=${${VAR_NAME}}", where + # "${REAL_VAR_NAME}" is the name of the variable with + # "LBANN_SB_FWD_${PKG_NAME}_" stripped off. + lbann_sb_create_extern_pkg_cmake_args( + LBANN_SB_${PKG_NAME}_CMAKE_ARGS + PKG_NAME ${PKG_NAME} + VARIABLES ${PKG_VARIABLES}) + + # This uses the output variables of "lbann_sb_init_extern_pkg" to + # forward basic language compiler and flag information. + lbann_sb_create_pkg_lang_cmake_args( + LBANN_SB_${PKG_NAME}_CMAKE_LANG_ARGS + PKG_NAME ${PKG_NAME} + LANGUAGES ${lbann_sb_add_LANGUAGES} ${lbann_sb_add_OPTIONAL_LANGUAGES}) + + # Handle the SOURCE_SUBDIR tag. + set(LBANN_SB_SOURCE_SUBDIR_TAG) + set(LBANN_SB_${PKG_NAME}_SOURCE_SUBDIR) + if (lbann_sb_add_SOURCE_SUBDIR) + set(LBANN_SB_SOURCE_SUBDIR_TAG "SOURCE_SUBDIR") + set(LBANN_SB_${PKG_NAME}_SOURCE_SUBDIR "${lbann_sb_add_SOURCE_SUBDIR}") + endif () + + set(LBANN_SB_GIT_SUBMODULES_TAG) + set(LBANN_SB_GIT_SUBMODULES_VAL) + if ("${PKG_NAME}" STREQUAL "LBANN") + set(LBANN_SB_GIT_SUBMODULES_TAG "GIT_SUBMODULES") + set(LBANN_SB_GIT_SUBMODULES_VAL "") + message("\nLBANN!\n") + endif () + + # Finally add the project. + ExternalProject_Add(${PKG_NAME} + # Standard stuff; users shouldn't care. + PREFIX ${CMAKE_CURRENT_BINARY_DIR} + TMP_DIR ${CMAKE_CURRENT_BINARY_DIR}/tmp + STAMP_DIR ${CMAKE_CURRENT_BINARY_DIR}/stamp + BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR}/build + + # User-configured paths. + SOURCE_DIR ${LBANN_SB_${PKG_NAME}_SOURCE_DIR} + INSTALL_DIR ${LBANN_SB_${PKG_NAME}_PREFIX} + + ${LBANN_SB_SOURCE_SUBDIR_TAG} ${LBANN_SB_${PKG_NAME}_SOURCE_SUBDIR} + + # Hack to handle building from an existing source directory. + ${LBANN_SB_GIT_REPOSITORY_TAG} ${LBANN_SB_GIT_REPOSITORY} + ${LBANN_SB_GIT_TAG_TAG} ${LBANN_SB_GIT_TAG} + + # Setup any dependencies + ${LBANN_SB_DEPENDS_TAG} ${LBANN_SB_${PKG_NAME}_DEPENDS} + + # FIXME (trb 03/07/2023): This needs to be conditionally set! + #GIT_SUBMODULES "" + #${LBANN_SB_GIT_SUBMODULES_TAG} ${LBANN_SB_GIT_SUBMODULES_VAL} + GIT_SHALLOW 1 + + # Log everything. + LOG_DOWNLOAD 1 + LOG_UPDATE 1 + LOG_CONFIGURE 1 + LOG_BUILD 1 + LOG_INSTALL 1 + LOG_TEST 1 + + USES_TERMINAL_BUILD 1 + LIST_SEPARATOR | + + CMAKE_GENERATOR ${LBANN_SB_${PKG_NAME}_CMAKE_GENERATOR} + CMAKE_ARGS + # Compilers and flags + ${LBANN_SB_${PKG_NAME}_CMAKE_LANG_ARGS} + + # Standard CMakery + -D BUILD_SHARED_LIBS=${LBANN_SB_${PKG_NAME}_BUILD_SHARED_LIBS} + -D CMAKE_INSTALL_PREFIX=${LBANN_SB_${PKG_NAME}_PREFIX} + -D CMAKE_BUILD_TYPE=${LBANN_SB_${PKG_NAME}_BUILD_TYPE} + -D CMAKE_POSITION_INDEPENDENT_CODE=${LBANN_SB_${PKG_NAME}_PIC} + -D CMAKE_INTERPROCEDURAL_OPTIMIZATION=${LBANN_SB_${PKG_NAME}_IPO} + + -D CMAKE_INSTALL_RPATH_USE_LINK_PATH=${LBANN_SB_${PKG_NAME}_INSTALL_RPATH_USE_LINK_PATH} + -D CMAKE_SKIP_RPATH=${LBANN_SB_${PKG_NAME}_SKIP_RPATH} + -D CMAKE_BUILD_RPATH_USE_ORIGIN=${LBANN_SB_${PKG_NAME}_BUILD_RPATH_USE_ORIGIN} + -D CMAKE_BUILD_WITH_INSTALL_RPATH=${LBANN_SB_${PKG_NAME}_BUILD_WITH_INSTALL_RPATH} + -D CMAKE_INSTALL_REMOVE_ENVIRONMENT_RPATH=${LBANN_SB_${PKG_NAME}_INSTALL_REMOVE_ENVIRONMENT_RPATH} + -D CMAKE_SKIP_BUILD_RPATH=${LBANN_SB_${PKG_NAME}_SKIP_BUILD_RPATH} + -D CMAKE_SKIP_INSTALL_RPATH=${LBANN_SB_${PKG_NAME}_SKIP_INSTALL_RPATH} + + # Extra arguments for CMake + ${LBANN_SB_${PKG_NAME}_CMAKE_ARGS} + ${lbann_sb_add_EXTRA_CMAKE_ARGS} + ) + +endmacro () diff --git a/scripts/superbuild/cmake/modules/LBANNSuperBuildAddPackages.cmake b/scripts/superbuild/cmake/modules/LBANNSuperBuildAddPackages.cmake new file mode 100644 index 00000000000..e57c3ee9690 --- /dev/null +++ b/scripts/superbuild/cmake/modules/LBANNSuperBuildAddPackages.cmake @@ -0,0 +1,74 @@ +################################################################################ +## Copyright (c) 2014-2024, Lawrence Livermore National Security, LLC. +## Produced at the Lawrence Livermore National Laboratory. +## Written by the LBANN Research Team (B. Van Essen, et al.) listed in +## the CONTRIBUTORS file. +## +## LLNL-CODE-697807. +## All rights reserved. +## +## This file is part of LBANN: Livermore Big Artificial Neural Network +## Toolkit. For details, see http://software.llnl.gov/LBANN or +## https://github.com/LLNL/LBANN. +## +## Licensed under the Apache License, Version 2.0 (the "Licensee"); you +## may not use this file except in compliance with the License. You may +## obtain a copy of the License at: +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +## implied. See the License for the specific language governing +## permissions and limitations under the license. +################################################################################ +include(CMakeDependentOption) + +macro(lbann_sb_default_pkg_option PKG_NAME OPTION_NAME DOC_STR VALUE) + option(LBANN_SB_FWD_${PKG_NAME}_${OPTION_NAME} + "${DOC_STR}" + ${VALUE}) +endmacro () + +macro(lbann_sb_default_cuda_option PKG_NAME OPTION_NAME DOC_STR VALUE) + cmake_dependent_option( + LBANN_SB_FWD_${PKG_NAME}_${OPTION_NAME} + "${DOC_STR}" + ${VALUE} + "LBANN_SB_DEFAULT_CUDA_OPTS" + OFF) +endmacro () + +macro(lbann_sb_default_rocm_option PKG_NAME OPTION_NAME DOC_STR VALUE) + cmake_dependent_option( + LBANN_SB_FWD_${PKG_NAME}_${OPTION_NAME} + "${DOC_STR}" + ${VALUE} + "LBANN_SB_DEFAULT_ROCM_OPTS" + OFF) +endmacro () + +macro(lbann_sb_default_gpu_option PKG_NAME OPTION_NAME DOC_STR VALUE) + cmake_dependent_option( + LBANN_SB_FWD_${PKG_NAME}_${OPTION_NAME} + "${DOC_STR}" + ${VALUE} + "LBANN_SB_DEFAULT_CUDA_OPTS OR LBANN_SB_DEFAULT_ROCM_OPTS" + OFF) +endmacro () + +macro(lbann_sb_add_package PKG_NAME) + option(LBANN_SB_BUILD_${PKG_NAME} + "Optionally download and build ${PKG_NAME}?" + OFF) + if (LBANN_SB_BUILD_${PKG_NAME}) + list(APPEND LBANN_SB_BUILD_PKGS ${PKG_NAME}) + endif () +endmacro () + +macro(lbann_sb_add_packages) + foreach (pkg ${ARGN}) + lbann_sb_add_package(${pkg}) + endforeach () +endmacro () diff --git a/scripts/superbuild/cmake/modules/LBANNSuperBuildCreateCMakeArguments.cmake b/scripts/superbuild/cmake/modules/LBANNSuperBuildCreateCMakeArguments.cmake new file mode 100644 index 00000000000..2c98261d581 --- /dev/null +++ b/scripts/superbuild/cmake/modules/LBANNSuperBuildCreateCMakeArguments.cmake @@ -0,0 +1,124 @@ +################################################################################ +## Copyright (c) 2014-2024, Lawrence Livermore National Security, LLC. +## Produced at the Lawrence Livermore National Laboratory. +## Written by the LBANN Research Team (B. Van Essen, et al.) listed in +## the CONTRIBUTORS file. +## +## LLNL-CODE-697807. +## All rights reserved. +## +## This file is part of LBANN: Livermore Big Artificial Neural Network +## Toolkit. For details, see http://software.llnl.gov/LBANN or +## https://github.com/LLNL/LBANN. +## +## Licensed under the Apache License, Version 2.0 (the "Licensee"); you +## may not use this file except in compliance with the License. You may +## obtain a copy of the License at: +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +## implied. See the License for the specific language governing +## permissions and limitations under the license. +################################################################################ +# This function takes a list of variables and writes a CMake argument string of the format: +# -DARGNAME:ARGTYPE=ARGVALUE ... +# +# Arguments: +# -- REMOVE_PKG_NAME: _