From c1a29c8d5d8299c4f472386b92a7de84fefff80a Mon Sep 17 00:00:00 2001 From: "Brian C. Van Essen" Date: Mon, 30 Jan 2023 14:57:03 -0800 Subject: [PATCH 01/10] Added a Include What You Use (IWYU) mapping file to help control inclusion of header files. Added a new header file to contain standard C++ STL libraries. This should enable a smooth transition to C++20's import stl feature as well as allow for precompiled headers. --- include/lbann/lbann_stl.hpp | 47 +++++++++++++++++++++++++++++++++++++ lbann_iwyu_mapping.imp | 24 +++++++++++++++++++ 2 files changed, 71 insertions(+) create mode 100644 include/lbann/lbann_stl.hpp create mode 100644 lbann_iwyu_mapping.imp diff --git a/include/lbann/lbann_stl.hpp b/include/lbann/lbann_stl.hpp new file mode 100644 index 00000000000..830e91261ed --- /dev/null +++ b/include/lbann/lbann_stl.hpp @@ -0,0 +1,47 @@ +//////////////////////////////////////////////////////////////////////////////// +// Copyright (c) 2014-2023, Lawrence Livermore National Security, LLC. +// Produced at the Lawrence Livermore National Laboratory. +// Written by the LBANN Research Team (B. Van Essen, et al.) listed in +// the CONTRIBUTORS file. +// +// LLNL-CODE-697807. +// All rights reserved. +// +// This file is part of LBANN: Livermore Big Artificial Neural Network +// Toolkit. For details, see http://software.llnl.gov/LBANN or +// https://github.com/LLNL/LBANN. +// +// Licensed under the Apache License, Version 2.0 (the "Licensee"); you +// may not use this file except in compliance with the License. You may +// obtain a copy of the License at: +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the license. +//////////////////////////////////////////////////////////////////////////////// + +#ifndef LBANN_LBANN_STL_HPP_INCLUDED +#define LBANN_LBANN_STL_HPP_INCLUDED + +#ifdef __cpp_modules && __cpp_modules +import std; +#else +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#endif + +#endif // LBANN_LBANN_STL_HPP_INCLUDED diff --git a/lbann_iwyu_mapping.imp b/lbann_iwyu_mapping.imp new file mode 100644 index 00000000000..8f2c89e63e5 --- /dev/null +++ b/lbann_iwyu_mapping.imp @@ -0,0 +1,24 @@ +[ + { include: ["@", "private", "", "public"] }, + { include: ["@

", "private", "\"lbann/utils/h2_tmp.hpp\"", "public"] }, + { include: ["@", "private", "\"lbann/utils/serialization/cereal_utils.hpp\"", "public"] }, + { include: ["@", "private", "", "public"] }, + { include: ["@", "private", "", "public"] }, + { include: ["@", "private", "\"lbann/base.hpp\"", "public"] }, + { symbol: ["std::vector", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, + { symbol: ["std::string", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, + { symbol: ["std::basic_string", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, + { symbol: ["std::unique_ptr", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, + { symbol: ["std::make_unique", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, + { symbol: ["size_t", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, + { symbol: ["std::allocator", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, + { symbol: ["std::__1::operator==", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, + { symbol: ["std::char_traits", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, + { symbol: ["std::operator+", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, + { symbol: ["std::max", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, + { symbol: ["std::min", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, + { symbol: ["std::move", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, + { symbol: ["std::shared_ptr", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, + { symbol: ["std::unordered_map", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, + +] From 291601c875723d914ab985b1308868b779157d68 Mon Sep 17 00:00:00 2001 From: "Brian C. Van Essen" Date: Mon, 30 Jan 2023 17:17:29 -0800 Subject: [PATCH 02/10] Working through the data_reader classes --- include/lbann/lbann_stl.hpp | 1 + lbann_iwyu_mapping.imp | 101 ++++++++++++++++++++++++++++++++++-- 2 files changed, 98 insertions(+), 4 deletions(-) diff --git a/include/lbann/lbann_stl.hpp b/include/lbann/lbann_stl.hpp index 830e91261ed..39f0149f89b 100644 --- a/include/lbann/lbann_stl.hpp +++ b/include/lbann/lbann_stl.hpp @@ -37,6 +37,7 @@ import std; #include #include #include +#include #include #include #include diff --git a/lbann_iwyu_mapping.imp b/lbann_iwyu_mapping.imp index 8f2c89e63e5..cd236e75233 100644 --- a/lbann_iwyu_mapping.imp +++ b/lbann_iwyu_mapping.imp @@ -3,22 +3,115 @@ { include: ["@

", "private", "\"lbann/utils/h2_tmp.hpp\"", "public"] }, { include: ["@", "private", "\"lbann/utils/serialization/cereal_utils.hpp\"", "public"] }, { include: ["@", "private", "", "public"] }, +# { include: ["@", "private", "", "public"] }, + { include: ["@", "private", "", "public"] }, { include: ["@", "private", "", "public"] }, { include: ["@", "private", "\"lbann/base.hpp\"", "public"] }, + { include: ["\"lbann_config.hpp\"", "private", "\"lbann/base.hpp\"", "public"] }, +# Vector { symbol: ["std::vector", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, + { symbol: ["std::swap", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, + { symbol: ["size_t", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, +# String { symbol: ["std::string", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, { symbol: ["std::basic_string", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, - { symbol: ["std::unique_ptr", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, - { symbol: ["std::make_unique", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, - { symbol: ["size_t", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, { symbol: ["std::allocator", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, { symbol: ["std::__1::operator==", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, { symbol: ["std::char_traits", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, { symbol: ["std::operator+", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, + { symbol: ["std::to_string", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, + { symbol: ["std::operator<", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, +# Memory + { symbol: ["std::unique_ptr", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, + { symbol: ["std::make_unique", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, + { symbol: ["std::shared_ptr", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, +# Misc { symbol: ["std::max", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, { symbol: ["std::min", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, { symbol: ["std::move", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, - { symbol: ["std::shared_ptr", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, + { symbol: ["int64_t", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, +# iostream + { symbol: ["std::__1::operator<<", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, + { symbol: ["std::basic_ostream", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, + { symbol: ["std::cout", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, + { symbol: ["std::ostream", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, + { symbol: ["std::endl", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, + { symbol: ["std::__1::operator>>", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, + { symbol: ["std::cerr", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, +# unordered_map { symbol: ["std::unordered_map", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, + { symbol: ["std::copy", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, + { symbol: ["std::pair", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, + { symbol: ["std::cbegin", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, + { symbol: ["std::cend", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, + { symbol: ["std::next", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, + { symbol: ["__hash_map_iterator", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, + { symbol: ["__hash_map_const_iterator", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, +# Set + { symbol: ["std::__tree_const_iterator", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, + { symbol: ["std::__1::operator!=", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, + { symbol: ["std::set", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, + { symbol: ["std::unordered_set", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, +# Tuple + { symbol: ["std::tuple", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, + { symbol: ["std::get", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, +# Map + { symbol: ["std::map", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, + { symbol: ["std::__map_iterator", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, +# Fstream #include + { symbol: ["std::streampos", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, + { symbol: ["std::ifstream", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, + { symbol: ["std::getline", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, + { symbol: ["std::basic_istream", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, + { symbol: ["std::basic_ifstream", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, + { symbol: ["std::ios", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, + { symbol: ["std::stringstream", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, + { symbol: ["std::basic_ios", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, + { symbol: ["std::streamsize", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, + + { symbol: ["std::function", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, + +#include + { symbol: ["std::ofstream", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, + { symbol: ["std::stringstream", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, + { symbol: ["std::basic_ios", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, +# Misc + { symbol: ["std::__bind", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, + { symbol: ["std::reference_wrapper", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, + { symbol: ["std::bind", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, + { symbol: ["std::ref", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, + { symbol: ["std::sort", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, + { symbol: ["std::shuffle", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, + +# { symbol: ["", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, ] + + +#/Users/vanessen1/Research/DeepLearning/lbann.git/src/data_readers/data_reader_csv.cpp:59:3: warning: std::streampos is defined in , which isn't directly #included. + +#arning/lbann.git/src/data_readers/data_reader_csv.cpp:41:3: warning: std::map is defined in , which isn't directly #included. + +#/Users/vanessen1/Research/DeepLearning/lbann.git/src/callbacks/check_dataset.cpp:69:10: warning: std::__tree_const_iterator is defined in <__tree>, which isn't directly #included. +#/Users/vanessen1/Research/DeepLearning/lbann.git/src/callbacks/check_dataset.cpp:69:13: warning: std::__1::operator!= is defined in <__tree>, which isn't directly #included. +#/Users/vanessen1/Research/DeepLearning/lbann.git/src/callbacks/check_dataset.cpp:69:20: warning: std::set is defined in , which isn't directly #included. + +#/Users/vanessen1/Research/DeepLearning/lbann.git/src/callbacks/alternate_updates.cpp:85:5: warning: std::__1::operator>> is defined in , which isn't directly #included. + +# std::__1::operator!= is defined in <__tree>, which isn't directly #included. +# std::__tree_const_iterator is defined in <__tree>, which isn't directly #included. +# std::set is defined in , which isn't directly #included. +# std::set is defined in , which isn't directly #included. +#/Users/vanessen1/Research/DeepLearning/lbann.git/src/callbacks/timeline.cpp:104:40: warning: std::pair is defined in <__utility/pair.h>, which isn't directly #included. + +#/Users/vanessen1/Research/DeepLearning/lbann.git/src/callbacks/alternate_updates.cpp:73:21: warning: std::cbegin is defined in <__iterator/access.h>, which isn't directly #included. +#/Users/vanessen1/Research/DeepLearning/lbann.git/src/callbacks/alternate_updates.cpp:73:21: warning: std::cend is defined in <__iterator/access.h>, which isn't directly #included. +#/Users/vanessen1/Research/DeepLearning/lbann.git/src/callbacks/alternate_updates.cpp:73:21: warning: std::next is defined in <__iterator/next.h>, which isn't directly #included. + + +#include <__algorithm/shuffle.h> // for shuffle +#include <__algorithm/sort.h> // for sort +#include <__functional/bind.h> // for __bind, bind +#include <__functional/reference_wrapper.h> // for reference_wrapper, ref +#include // for __map_iterator +#include // for char_traits, allocator, operator< From ecb477127ef34e09a99bc5bbe87f9d5e9b489d0e Mon Sep 17 00:00:00 2001 From: "Brian C. Van Essen" Date: Tue, 31 Jan 2023 09:24:25 -0800 Subject: [PATCH 03/10] Used the IWYU export feature to capture all of the standard library symbols and a clever trick to grab the implementation details as well. --- include/lbann/lbann_stl.hpp | 16 ++++- lbann_iwyu_mapping.imp | 114 +++--------------------------------- 2 files changed, 24 insertions(+), 106 deletions(-) diff --git a/include/lbann/lbann_stl.hpp b/include/lbann/lbann_stl.hpp index 39f0149f89b..03b4558b970 100644 --- a/include/lbann/lbann_stl.hpp +++ b/include/lbann/lbann_stl.hpp @@ -27,22 +27,36 @@ #ifndef LBANN_LBANN_STL_HPP_INCLUDED #define LBANN_LBANN_STL_HPP_INCLUDED -#ifdef __cpp_modules && __cpp_modules +#if defined(__cpp_modules) && __cpp_modules import std; #else +// IWYU pragma: begin_exports +#include #include #include #include #include +#include +#include +#include +#include #include +#include #include +#include +#include #include +#include +#include +#include #include #include #include #include #include +#include #include +// IWYU pragma: end_exports #endif #endif // LBANN_LBANN_STL_HPP_INCLUDED diff --git a/lbann_iwyu_mapping.imp b/lbann_iwyu_mapping.imp index cd236e75233..ff7e5ae0463 100644 --- a/lbann_iwyu_mapping.imp +++ b/lbann_iwyu_mapping.imp @@ -1,117 +1,21 @@ [ { include: ["@", "private", "", "public"] }, { include: ["@

", "private", "\"lbann/utils/h2_tmp.hpp\"", "public"] }, + { include: ["@", "private", "\"lbann/utils/serialization/cereal_utils.hpp\"", "public"] }, { include: ["@", "private", "", "public"] }, -# { include: ["@", "private", "", "public"] }, - { include: ["@", "private", "", "public"] }, + + # Standard packages in the Conduit library, outside of the relay sublibrary should be in a standard header + { include: ["@", "private", "", "public"] }, + { include: ["@", "private", "", "public"] }, + { include: ["@", "private", "\"lbann/base.hpp\"", "public"] }, { include: ["\"lbann_config.hpp\"", "private", "\"lbann/base.hpp\"", "public"] }, -# Vector - { symbol: ["std::vector", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, - { symbol: ["std::swap", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, - { symbol: ["size_t", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, -# String - { symbol: ["std::string", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, - { symbol: ["std::basic_string", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, - { symbol: ["std::allocator", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, - { symbol: ["std::__1::operator==", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, - { symbol: ["std::char_traits", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, - { symbol: ["std::operator+", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, - { symbol: ["std::to_string", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, - { symbol: ["std::operator<", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, -# Memory - { symbol: ["std::unique_ptr", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, - { symbol: ["std::make_unique", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, - { symbol: ["std::shared_ptr", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, -# Misc - { symbol: ["std::max", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, - { symbol: ["std::min", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, - { symbol: ["std::move", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, - { symbol: ["int64_t", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, -# iostream - { symbol: ["std::__1::operator<<", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, - { symbol: ["std::basic_ostream", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, - { symbol: ["std::cout", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, - { symbol: ["std::ostream", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, - { symbol: ["std::endl", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, - { symbol: ["std::__1::operator>>", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, - { symbol: ["std::cerr", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, -# unordered_map - { symbol: ["std::unordered_map", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, - { symbol: ["std::copy", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, - { symbol: ["std::pair", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, - { symbol: ["std::cbegin", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, - { symbol: ["std::cend", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, - { symbol: ["std::next", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, - { symbol: ["__hash_map_iterator", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, - { symbol: ["__hash_map_const_iterator", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, -# Set - { symbol: ["std::__tree_const_iterator", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, - { symbol: ["std::__1::operator!=", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, - { symbol: ["std::set", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, - { symbol: ["std::unordered_set", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, -# Tuple - { symbol: ["std::tuple", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, - { symbol: ["std::get", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, -# Map - { symbol: ["std::map", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, - { symbol: ["std::__map_iterator", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, -# Fstream #include - { symbol: ["std::streampos", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, - { symbol: ["std::ifstream", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, - { symbol: ["std::getline", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, - { symbol: ["std::basic_istream", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, - { symbol: ["std::basic_ifstream", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, - { symbol: ["std::ios", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, - { symbol: ["std::stringstream", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, - { symbol: ["std::basic_ios", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, - { symbol: ["std::streamsize", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, - { symbol: ["std::function", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, + # Grab all of the C++ implementation details and tie them into a common header + { include: ["@<__.*>", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, -#include - { symbol: ["std::ofstream", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, - { symbol: ["std::stringstream", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, - { symbol: ["std::basic_ios", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, -# Misc - { symbol: ["std::__bind", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, - { symbol: ["std::reference_wrapper", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, - { symbol: ["std::bind", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, - { symbol: ["std::ref", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, - { symbol: ["std::sort", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, - { symbol: ["std::shuffle", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, - -# { symbol: ["", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, + { symbol: ["hid_t", "private", "\"hdf5.h\"", "public"] }, ] - - -#/Users/vanessen1/Research/DeepLearning/lbann.git/src/data_readers/data_reader_csv.cpp:59:3: warning: std::streampos is defined in , which isn't directly #included. - -#arning/lbann.git/src/data_readers/data_reader_csv.cpp:41:3: warning: std::map is defined in , which isn't directly #included. - -#/Users/vanessen1/Research/DeepLearning/lbann.git/src/callbacks/check_dataset.cpp:69:10: warning: std::__tree_const_iterator is defined in <__tree>, which isn't directly #included. -#/Users/vanessen1/Research/DeepLearning/lbann.git/src/callbacks/check_dataset.cpp:69:13: warning: std::__1::operator!= is defined in <__tree>, which isn't directly #included. -#/Users/vanessen1/Research/DeepLearning/lbann.git/src/callbacks/check_dataset.cpp:69:20: warning: std::set is defined in , which isn't directly #included. - -#/Users/vanessen1/Research/DeepLearning/lbann.git/src/callbacks/alternate_updates.cpp:85:5: warning: std::__1::operator>> is defined in , which isn't directly #included. - -# std::__1::operator!= is defined in <__tree>, which isn't directly #included. -# std::__tree_const_iterator is defined in <__tree>, which isn't directly #included. -# std::set is defined in , which isn't directly #included. -# std::set is defined in , which isn't directly #included. -#/Users/vanessen1/Research/DeepLearning/lbann.git/src/callbacks/timeline.cpp:104:40: warning: std::pair is defined in <__utility/pair.h>, which isn't directly #included. - -#/Users/vanessen1/Research/DeepLearning/lbann.git/src/callbacks/alternate_updates.cpp:73:21: warning: std::cbegin is defined in <__iterator/access.h>, which isn't directly #included. -#/Users/vanessen1/Research/DeepLearning/lbann.git/src/callbacks/alternate_updates.cpp:73:21: warning: std::cend is defined in <__iterator/access.h>, which isn't directly #included. -#/Users/vanessen1/Research/DeepLearning/lbann.git/src/callbacks/alternate_updates.cpp:73:21: warning: std::next is defined in <__iterator/next.h>, which isn't directly #included. - - -#include <__algorithm/shuffle.h> // for shuffle -#include <__algorithm/sort.h> // for sort -#include <__functional/bind.h> // for __bind, bind -#include <__functional/reference_wrapper.h> // for reference_wrapper, ref -#include // for __map_iterator -#include // for char_traits, allocator, operator< From d37c7ab8e19e7ad5d45e157e7849d4fbfdea78e5 Mon Sep 17 00:00:00 2001 From: "Brian C. Van Essen" Date: Tue, 31 Jan 2023 12:25:15 -0800 Subject: [PATCH 04/10] Updating the IWYU implementation to handle Python.h objects --- include/lbann/base.hpp | 2 ++ include/lbann/lbann_stl.hpp | 3 +++ lbann_iwyu_mapping.imp | 18 ++++++++++++++++++ 3 files changed, 23 insertions(+) diff --git a/include/lbann/base.hpp b/include/lbann/base.hpp index 5f86f6b3fdd..88e9f6f8869 100644 --- a/include/lbann/base.hpp +++ b/include/lbann/base.hpp @@ -27,6 +27,8 @@ #ifndef LBANN_BASE_HPP_INCLUDED #define LBANN_BASE_HPP_INCLUDED +#include "lbann/lbann_stl.hpp" + #include // Defines, among other things, DataType. diff --git a/include/lbann/lbann_stl.hpp b/include/lbann/lbann_stl.hpp index 03b4558b970..13f8604483c 100644 --- a/include/lbann/lbann_stl.hpp +++ b/include/lbann/lbann_stl.hpp @@ -39,16 +39,19 @@ import std; #include #include #include +#include #include #include #include #include #include +#include #include #include #include #include #include +#include #include #include #include diff --git a/lbann_iwyu_mapping.imp b/lbann_iwyu_mapping.imp index ff7e5ae0463..96d6c2d083c 100644 --- a/lbann_iwyu_mapping.imp +++ b/lbann_iwyu_mapping.imp @@ -15,7 +15,25 @@ # Grab all of the C++ implementation details and tie them into a common header { include: ["@<__.*>", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, + { include: ["@<_.*>", "private", "\"lbann/lbann_stl.hpp\"", "public"] }, { symbol: ["hid_t", "private", "\"hdf5.h\"", "public"] }, + # Handle obnoxious Python.h symbols + { symbol: ["PyObject_CallMethod", "private", "", "public"] }, + { symbol: ["PyIter_Next", "private", "", "public"] }, + { symbol: ["PyObject_CallObject", "private", "", "public"] }, + { symbol: ["PyObject_GetIter", "private", "", "public"] }, + { symbol: ["PyObject_Length", "private", "", "public"] }, + { symbol: ["PyImport_ImportModule", "private", "", "public"] }, + { symbol: ["PyList_Append", "private", "", "public"] }, + { symbol: ["PyList_New", "private", "", "public"] }, + { symbol: ["PyLong_AsLong", "private", "", "public"] }, + { symbol: ["Py_BuildValue", "private", "", "public"] }, + { symbol: ["PyObject_GetAttrString", "private", "", "public"] }, + { symbol: ["PyObject_SetAttrString", "private", "", "public"] }, + { symbol: ["Py_DECREF", "private", "", "public"] }, + { symbol: ["PySys_GetObject", "private", "", "public"] }, + { symbol: ["PyRun_SimpleString", "private", "", "public"] }, + ] From 2d49a7dd6ca82d67db784926aa0728ea70bb7950 Mon Sep 17 00:00:00 2001 From: "Brian C. Van Essen" Date: Tue, 31 Jan 2023 13:08:03 -0800 Subject: [PATCH 05/10] Applied IWYU on the data reader classes. --- .../data_readers/data_reader_cifar10.hpp | 2 + .../data_reader_merge_features.hpp | 4 ++ .../data_reader_merge_samples.hpp | 3 ++ .../lbann/data_readers/data_reader_mesh.hpp | 3 ++ .../lbann/data_readers/data_reader_mnist.hpp | 2 + .../lbann/data_readers/data_reader_nci.hpp | 1 + .../lbann/data_readers/data_reader_python.hpp | 9 ++++ .../data_readers/data_reader_synthetic.hpp | 4 ++ src/data_readers/data_reader.cpp | 23 +++++++---- src/data_readers/data_reader_HDF5.cpp | 16 +++++++- src/data_readers/data_reader_cifar10.cpp | 8 +++- src/data_readers/data_reader_csv.cpp | 11 +++-- src/data_readers/data_reader_image.cpp | 17 ++++++-- src/data_readers/data_reader_jag_conduit.cpp | 41 ++++++++++--------- .../data_reader_merge_features.cpp | 8 +++- .../data_reader_merge_samples.cpp | 5 ++- src/data_readers/data_reader_mesh.cpp | 7 ++++ src/data_readers/data_reader_mnist.cpp | 11 ++++- src/data_readers/data_reader_nci.cpp | 5 +-- src/data_readers/data_reader_node2vec.cpp | 8 ++-- src/data_readers/data_reader_python.cpp | 13 +++--- src/data_readers/data_reader_smiles.cpp | 15 ++++--- src/data_readers/data_reader_synthetic.cpp | 7 +++- 23 files changed, 163 insertions(+), 60 deletions(-) diff --git a/include/lbann/data_readers/data_reader_cifar10.hpp b/include/lbann/data_readers/data_reader_cifar10.hpp index f3e1389dd91..8b66c6ad3bb 100644 --- a/include/lbann/data_readers/data_reader_cifar10.hpp +++ b/include/lbann/data_readers/data_reader_cifar10.hpp @@ -30,6 +30,8 @@ #define LBANN_DATA_READER_CIFAR10_HPP #include "data_reader_image.hpp" +#include "lbann/base.hpp" +#include "lbann/lbann_stl.hpp" namespace lbann { diff --git a/include/lbann/data_readers/data_reader_merge_features.hpp b/include/lbann/data_readers/data_reader_merge_features.hpp index aa8375f38c9..c2a8f874c99 100644 --- a/include/lbann/data_readers/data_reader_merge_features.hpp +++ b/include/lbann/data_readers/data_reader_merge_features.hpp @@ -30,8 +30,12 @@ #define LBANN_DATA_READER_MERGE_FEATURES_HPP #include "compound_data_reader.hpp" +#include "lbann/base.hpp" +#include "lbann/data_readers/data_reader.hpp" +#include "lbann/lbann_stl.hpp" namespace lbann { +class thread_pool; /** * Data reader for merging multiple data readers. diff --git a/include/lbann/data_readers/data_reader_merge_samples.hpp b/include/lbann/data_readers/data_reader_merge_samples.hpp index 1c4b66fa005..5af7eef55b3 100644 --- a/include/lbann/data_readers/data_reader_merge_samples.hpp +++ b/include/lbann/data_readers/data_reader_merge_samples.hpp @@ -30,6 +30,9 @@ #define LBANN_DATA_READER_MERGE_SAMPLES_HPP #include "compound_data_reader.hpp" +#include "lbann/base.hpp" +#include "lbann/data_readers/data_reader.hpp" +#include "lbann/lbann_stl.hpp" namespace lbann { diff --git a/include/lbann/data_readers/data_reader_mesh.hpp b/include/lbann/data_readers/data_reader_mesh.hpp index e9c1e8949cf..9dac8e74b03 100644 --- a/include/lbann/data_readers/data_reader_mesh.hpp +++ b/include/lbann/data_readers/data_reader_mesh.hpp @@ -30,8 +30,11 @@ #define LBANN_DATA_READER_MESH_HPP #include "data_reader.hpp" +#include "lbann/base.hpp" +#include "lbann/lbann_stl.hpp" namespace lbann { +class thread_pool; /** * Data reader for reading dumped mesh images. diff --git a/include/lbann/data_readers/data_reader_mnist.hpp b/include/lbann/data_readers/data_reader_mnist.hpp index cbe0741feb8..ca6f6d22526 100644 --- a/include/lbann/data_readers/data_reader_mnist.hpp +++ b/include/lbann/data_readers/data_reader_mnist.hpp @@ -30,6 +30,8 @@ #define LBANN_DATA_READER_MNIST_HPP #include "data_reader_image.hpp" +#include "lbann/base.hpp" +#include "lbann/lbann_stl.hpp" namespace lbann { diff --git a/include/lbann/data_readers/data_reader_nci.hpp b/include/lbann/data_readers/data_reader_nci.hpp index c007612756b..5fbdc35d78f 100644 --- a/include/lbann/data_readers/data_reader_nci.hpp +++ b/include/lbann/data_readers/data_reader_nci.hpp @@ -30,6 +30,7 @@ #define LBANN_DATA_READER_NCI_HPP #include "data_reader_csv.hpp" +#include "lbann/lbann_stl.hpp" #define NCI_HAS_HEADER diff --git a/include/lbann/data_readers/data_reader_python.hpp b/include/lbann/data_readers/data_reader_python.hpp index f6124e85564..03ed03566b0 100644 --- a/include/lbann/data_readers/data_reader_python.hpp +++ b/include/lbann/data_readers/data_reader_python.hpp @@ -27,7 +27,16 @@ #ifndef LBANN_DATA_READERS_PYTHON_HPP_INCLUDED #define LBANN_DATA_READERS_PYTHON_HPP_INCLUDED +#include + #include "data_reader.hpp" +#include "lbann/base.hpp" +#include "lbann/data_readers/utils/input_data_type.hpp" +#include "lbann/lbann_stl.hpp" + +namespace lbann { +class thread_pool; +} // namespace lbann #ifdef LBANN_HAS_EMBEDDED_PYTHON #include "lbann/utils/python.hpp" diff --git a/include/lbann/data_readers/data_reader_synthetic.hpp b/include/lbann/data_readers/data_reader_synthetic.hpp index 151c7d47c2b..ddf29aef33d 100644 --- a/include/lbann/data_readers/data_reader_synthetic.hpp +++ b/include/lbann/data_readers/data_reader_synthetic.hpp @@ -30,6 +30,10 @@ #include "data_reader.hpp" #include "lbann/utils/dim_helpers.hpp" +#include "lbann/base.hpp" +#include "lbann/data_readers/utils/input_data_type.hpp" +#include "lbann/lbann_stl.hpp" +#include "lbann/utils/exception.hpp" // Forward declaration class DataReaderSyntheticWhiteboxTester; diff --git a/src/data_readers/data_reader.cpp b/src/data_readers/data_reader.cpp index d6ecfd864e4..f3a514db3ae 100644 --- a/src/data_readers/data_reader.cpp +++ b/src/data_readers/data_reader.cpp @@ -26,23 +26,30 @@ // lbann_data_reader .hpp .cpp - Input data base class for training, testing //////////////////////////////////////////////////////////////////////////////// -#include "lbann/comm_impl.hpp" +#include +#include +#include + #include "lbann/data_readers/data_reader.hpp" #include "lbann/data_store/data_store_conduit.hpp" #include "lbann/execution_algorithms/sgd_execution_context.hpp" -#include "lbann/io/persist.hpp" #include "lbann/io/persist_impl.hpp" #include "lbann/trainers/trainer.hpp" -#include "lbann/utils/serialize.hpp" #include "lbann/utils/threads/thread_pool.hpp" #include "lbann/utils/timer.hpp" - -#include "conduit/conduit_node.hpp" - -#include -#include +#include "lbann/base.hpp" +#include "lbann/comm.hpp" +#include "lbann/data_coordinator/data_coordinator.hpp" +#include "lbann/data_readers/utils/input_data_type.hpp" +#include "lbann/lbann_stl.hpp" +#include "lbann/utils/argument_parser.hpp" +#include "lbann/utils/enum_iterator.hpp" +#include "lbann/utils/exception.hpp" +#include "lbann/utils/options.hpp" +#include "lbann/utils/random_number_generators.hpp" namespace lbann { +class persist; #undef DEBUG //#define DEBUG diff --git a/src/data_readers/data_reader_HDF5.cpp b/src/data_readers/data_reader_HDF5.cpp index 6ccb9fb81fb..67ea29dc820 100644 --- a/src/data_readers/data_reader_HDF5.cpp +++ b/src/data_readers/data_reader_HDF5.cpp @@ -24,13 +24,27 @@ // permissions and limitations under the license. // ///////////////////////////////////////////////////////////////////////////////// -#include "conduit/conduit_relay_mpi.hpp" +#include +#include +#include +#include +#include "conduit/conduit_relay_mpi.hpp" #include "lbann/data_readers/data_reader_HDF5.hpp" #include "lbann/data_readers/data_reader_sample_list_impl.hpp" #include "lbann/data_readers/sample_list_impl.hpp" #include "lbann/data_readers/sample_list_open_files_impl.hpp" #include "lbann/utils/timer.hpp" +#include "hdf5.h" +#include "lbann/comm.hpp" +#include "lbann/data_readers/data_reader_sample_list.hpp" +#include "lbann/data_readers/sample_list_hdf5.hpp" +#include "lbann/data_readers/utils/input_data_type.hpp" +#include "lbann/data_store/data_store_conduit.hpp" +#include "lbann/lbann_stl.hpp" +#include "lbann/utils/argument_parser.hpp" +#include "lbann/utils/exception.hpp" +#include "lbann/utils/options.hpp" namespace lbann { namespace { diff --git a/src/data_readers/data_reader_cifar10.cpp b/src/data_readers/data_reader_cifar10.cpp index 396e976d3c8..19e5e31c569 100644 --- a/src/data_readers/data_reader_cifar10.cpp +++ b/src/data_readers/data_reader_cifar10.cpp @@ -27,7 +27,13 @@ //////////////////////////////////////////////////////////////////////////////// #include "lbann/data_readers/data_reader_cifar10.hpp" -#include "lbann/data_readers/sample_list_impl.hpp" + +#include + +#include "lbann/data_readers/data_reader_image.hpp" +#include "lbann/data_readers/utils/input_data_type.hpp" +#include "lbann/transforms/transform_pipeline.hpp" +#include "lbann/utils/exception.hpp" namespace lbann { diff --git a/src/data_readers/data_reader_csv.cpp b/src/data_readers/data_reader_csv.cpp index 60d44d49577..fc519f194c6 100644 --- a/src/data_readers/data_reader_csv.cpp +++ b/src/data_readers/data_reader_csv.cpp @@ -26,12 +26,17 @@ // lbann_data_reader_csv .hpp .cpp - generic_data_reader class for CSV files //////////////////////////////////////////////////////////////////////////////// +#include + #include "lbann/comm_impl.hpp" #include "lbann/data_readers/data_reader_csv.hpp" -#include "lbann/utils/options.hpp" #include "lbann/utils/threads/thread_pool.hpp" - -#include +#include "lbann/base.hpp" +#include "lbann/comm.hpp" +#include "lbann/data_readers/data_reader.hpp" +#include "lbann/data_readers/utils/input_data_type.hpp" +#include "lbann/lbann_stl.hpp" +#include "lbann/utils/exception.hpp" namespace lbann { diff --git a/src/data_readers/data_reader_image.cpp b/src/data_readers/data_reader_image.cpp index ca1988ca597..c75727d8422 100644 --- a/src/data_readers/data_reader_image.cpp +++ b/src/data_readers/data_reader_image.cpp @@ -26,17 +26,28 @@ // data_reader_image .hpp .cpp - generic data reader class for image dataset //////////////////////////////////////////////////////////////////////////////// +#include +#include +#include + #include "lbann/comm_impl.hpp" #include "lbann/data_readers/data_reader_image.hpp" #include "lbann/data_readers/sample_list_impl.hpp" #include "lbann/data_store/data_store_conduit.hpp" #include "lbann/utils/file_utils.hpp" #include "lbann/utils/lbann_library.hpp" -#include "lbann/utils/threads/thread_utils.hpp" #include "lbann/utils/timer.hpp" #include "lbann/utils/vectorwrapbuf.hpp" - -#include +#include "lbann/base.hpp" +#include "lbann/comm.hpp" +#include "lbann/data_readers/data_reader.hpp" +#include "lbann/data_readers/sample_list.hpp" +#include "lbann/lbann_stl.hpp" +#include "lbann/transforms/transform_pipeline.hpp" +#include "lbann/utils/argument_parser.hpp" +#include "lbann/utils/exception.hpp" +#include "lbann/utils/options.hpp" +#include "lbann/utils/threads/thread_pool.hpp" namespace lbann { diff --git a/src/data_readers/data_reader_jag_conduit.cpp b/src/data_readers/data_reader_jag_conduit.cpp index 741fcfbdd34..150adbf0d30 100644 --- a/src/data_readers/data_reader_jag_conduit.cpp +++ b/src/data_readers/data_reader_jag_conduit.cpp @@ -25,38 +25,41 @@ // //////////////////////////////////////////////////////////////////////////////// +#include +#include +#include +#include +#include +#include + #include "lbann/comm_impl.hpp" #include "lbann/data_readers/data_reader_jag_conduit.hpp" #include "lbann/data_store/data_store_conduit.hpp" #include "lbann/trainers/trainer.hpp" #include "lbann/execution_algorithms/sgd_execution_context.hpp" #include "lbann/utils/dim_helpers.hpp" -#include "lbann/utils/lbann_library.hpp" -#include "lbann/utils/serialize.hpp" #include "lbann/transforms/repack_HWC_to_CHW_layout.hpp" #include "lbann/transforms/scale_and_translate.hpp" - #include "lbann/utils/file_utils.hpp" // for add_delimiter() in load() #include "lbann/data_readers/sample_list_impl.hpp" #include "lbann/data_readers/sample_list_open_files_impl.hpp" #include "lbann/utils/vectorwrapbuf.hpp" - -#include // numeric_limits -#include // max_element -#include // accumulate -#include // multiplies -#include // is_same -#include -#include -#include #include "lbann/utils/timer.hpp" -#include "lbann/utils/glob.hpp" -#include "lbann/utils/peek_map.hpp" -#include "conduit/conduit_relay.hpp" -#include "conduit/conduit_relay_io_hdf5.hpp" - -#include -#include +#include "hdf5.h" +#include "lbann/base.hpp" +#include "lbann/comm.hpp" +#include "lbann/data_coordinator/data_coordinator.hpp" +#include "lbann/data_coordinator/data_coordinator_metadata.hpp" +#include "lbann/data_readers/data_reader.hpp" +#include "lbann/data_readers/sample_list.hpp" +#include "lbann/data_readers/utils/input_data_type.hpp" +#include "lbann/lbann_stl.hpp" +#include "lbann/utils/argument_parser.hpp" +#include "lbann/utils/exception.hpp" +#include "lbann/utils/options.hpp" +#include "lbann/utils/random_number_generators.hpp" +#include "lbann/utils/threads/thread_pool.hpp" +#include "lbann/utils/type_erased_matrix.hpp" // This comes after all the headers, and is only visible within the current implementation file. // To make sure, we put '#undef _CN_' at the end of this file diff --git a/src/data_readers/data_reader_merge_features.cpp b/src/data_readers/data_reader_merge_features.cpp index c302a925e21..3dcb6cec1ec 100644 --- a/src/data_readers/data_reader_merge_features.cpp +++ b/src/data_readers/data_reader_merge_features.cpp @@ -27,10 +27,16 @@ //////////////////////////////////////////////////////////////////////////////// #include "lbann/data_readers/data_reader_merge_features.hpp" -#include "lbann/utils/options.hpp" + +#include + #include "lbann/utils/timer.hpp" +#include "lbann/comm.hpp" +#include "lbann/data_readers/compound_data_reader.hpp" +#include "lbann/utils/exception.hpp" namespace lbann { +class thread_pool; data_reader_merge_features::data_reader_merge_features( std::vector data_readers, diff --git a/src/data_readers/data_reader_merge_samples.cpp b/src/data_readers/data_reader_merge_samples.cpp index d7430514e91..7733af5cc8e 100644 --- a/src/data_readers/data_reader_merge_samples.cpp +++ b/src/data_readers/data_reader_merge_samples.cpp @@ -27,7 +27,10 @@ //////////////////////////////////////////////////////////////////////////////// #include "lbann/data_readers/data_reader_merge_samples.hpp" -#include "lbann/utils/options.hpp" + +#include "lbann/data_readers/compound_data_reader.hpp" +#include "lbann/data_readers/utils/input_data_type.hpp" +#include "lbann/utils/exception.hpp" namespace lbann { diff --git a/src/data_readers/data_reader_mesh.cpp b/src/data_readers/data_reader_mesh.cpp index 5879dfe29de..69b0e222ffc 100644 --- a/src/data_readers/data_reader_mesh.cpp +++ b/src/data_readers/data_reader_mesh.cpp @@ -27,8 +27,15 @@ //////////////////////////////////////////////////////////////////////////////// #include "lbann/data_readers/data_reader_mesh.hpp" + +#include + #include "lbann/utils/glob.hpp" #include "lbann/utils/threads/thread_pool.hpp" +#include "lbann/data_readers/data_reader.hpp" +#include "lbann/data_readers/utils/input_data_type.hpp" +#include "lbann/utils/exception.hpp" +#include "lbann/utils/random_number_generators.hpp" namespace lbann { diff --git a/src/data_readers/data_reader_mnist.cpp b/src/data_readers/data_reader_mnist.cpp index 43d8b501d15..1ab9b3ddc20 100644 --- a/src/data_readers/data_reader_mnist.cpp +++ b/src/data_readers/data_reader_mnist.cpp @@ -27,10 +27,17 @@ //////////////////////////////////////////////////////////////////////////////// #include "lbann/data_readers/data_reader_mnist.hpp" -#include "lbann/data_readers/sample_list_impl.hpp" -#include "lbann/utils/file_utils.hpp" + +#include #include +#include "lbann/utils/file_utils.hpp" +#include "lbann/comm.hpp" +#include "lbann/data_readers/data_reader_image.hpp" +#include "lbann/data_readers/utils/input_data_type.hpp" +#include "lbann/transforms/transform_pipeline.hpp" +#include "lbann/utils/exception.hpp" + namespace lbann { mnist_reader::mnist_reader(bool shuffle) diff --git a/src/data_readers/data_reader_nci.cpp b/src/data_readers/data_reader_nci.cpp index cc58fd69450..92af2d9ea17 100644 --- a/src/data_readers/data_reader_nci.cpp +++ b/src/data_readers/data_reader_nci.cpp @@ -27,9 +27,8 @@ //////////////////////////////////////////////////////////////////////////////// #include "lbann/data_readers/data_reader_nci.hpp" -#include -#include -#include + +#include "lbann/data_readers/data_reader_csv.hpp" namespace lbann { diff --git a/src/data_readers/data_reader_node2vec.cpp b/src/data_readers/data_reader_node2vec.cpp index 8b5009f96e3..d1f97f65c2f 100644 --- a/src/data_readers/data_reader_node2vec.cpp +++ b/src/data_readers/data_reader_node2vec.cpp @@ -24,18 +24,16 @@ // permissions and limitations under the license. //////////////////////////////////////////////////////////////////////////////// -#include "lbann/comm_impl.hpp" -#include "lbann/data_readers/data_reader_node2vec.hpp" #ifdef LBANN_HAS_LARGESCALE_NODE2VEC -#include "lbann/utils/memory.hpp" -#include "lbann/utils/random.hpp" #include #include #include - #include #include +#include "lbann/utils/memory.hpp" +#include "lbann/utils/random.hpp" + namespace lbann { namespace node2vec_reader_impl { diff --git a/src/data_readers/data_reader_python.cpp b/src/data_readers/data_reader_python.cpp index 73f07f4e232..1d5f96ef7b1 100644 --- a/src/data_readers/data_reader_python.cpp +++ b/src/data_readers/data_reader_python.cpp @@ -25,15 +25,18 @@ //////////////////////////////////////////////////////////////////////////////// #include "lbann/data_readers/data_reader_python.hpp" + #include "lbann/trainers/trainer.hpp" -#ifdef LBANN_HAS_EMBEDDED_PYTHON -#include -#include -#include +#include "lbann/data_readers/data_reader.hpp" +#include "lbann/utils/dim_helpers.hpp" +#include "lbann/utils/exception.hpp" +namespace lbann { +class thread_pool; +} // namespace lbann +#ifdef LBANN_HAS_EMBEDDED_PYTHON #include -#include "lbann/trainers/trainer.hpp" #include "lbann/utils/python.hpp" namespace lbann { diff --git a/src/data_readers/data_reader_smiles.cpp b/src/data_readers/data_reader_smiles.cpp index bcea535bdd1..3ce4ade910d 100644 --- a/src/data_readers/data_reader_smiles.cpp +++ b/src/data_readers/data_reader_smiles.cpp @@ -25,6 +25,8 @@ // //////////////////////////////////////////////////////////////////////////////// +#include + #include "lbann/comm_impl.hpp" #include "lbann/data_readers/data_reader_sample_list_impl.hpp" #include "lbann/data_readers/data_reader_smiles.hpp" @@ -35,12 +37,13 @@ #include "lbann/utils/file_utils.hpp" #include "lbann/utils/timer.hpp" #include "lbann/utils/commify.hpp" -#include "lbann/utils/lbann_library.hpp" -#include "lbann/utils/vectorwrapbuf.hpp" -#include -#include -#include -#include +#include "lbann/base.hpp" +#include "lbann/comm.hpp" +#include "lbann/data_readers/data_reader_sample_list.hpp" +#include "lbann/data_readers/sample_list_ifstream.hpp" +#include "lbann/lbann_stl.hpp" +#include "lbann/utils/exception.hpp" +#include "lbann/utils/options.hpp" namespace lbann { diff --git a/src/data_readers/data_reader_synthetic.cpp b/src/data_readers/data_reader_synthetic.cpp index cea0ce11c86..d4029e26d1b 100644 --- a/src/data_readers/data_reader_synthetic.cpp +++ b/src/data_readers/data_reader_synthetic.cpp @@ -28,9 +28,12 @@ //////////////////////////////////////////////////////////////////////////////// #include "lbann/data_readers/data_reader_synthetic.hpp" + +#include + #include "lbann/utils/random_number_generators.hpp" -#include -#include +#include "lbann/data_readers/data_reader.hpp" +#include "lbann/utils/random.hpp" namespace lbann { From d1b61f78bfec7396d0f964d38b4aa2fc6771c8db Mon Sep 17 00:00:00 2001 From: "Brian C. Van Essen" Date: Wed, 1 Feb 2023 06:52:43 -0800 Subject: [PATCH 06/10] Updated the mapping rules to ensure that IWYU doesn't break up the serialization header files. --- lbann_iwyu_mapping.imp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lbann_iwyu_mapping.imp b/lbann_iwyu_mapping.imp index 96d6c2d083c..22003d02fc4 100644 --- a/lbann_iwyu_mapping.imp +++ b/lbann_iwyu_mapping.imp @@ -10,7 +10,8 @@ { include: ["@", "private", "", "public"] }, - { include: ["@", "private", "\"lbann/base.hpp\"", "public"] }, + { include: ["@\"lbann/utils/serialization/.*\"", "public", "\"lbann/utils/serialize.hpp\"", "public"] }, +# { include: ["@", "private", "\"lbann/base.hpp\"", "public"] }, { include: ["\"lbann_config.hpp\"", "private", "\"lbann/base.hpp\"", "public"] }, # Grab all of the C++ implementation details and tie them into a common header From 9ccc0cff43653931eb77382bce8f568672c451bf Mon Sep 17 00:00:00 2001 From: "Brian C. Van Essen" Date: Wed, 1 Feb 2023 06:54:47 -0800 Subject: [PATCH 07/10] IWYU for activations cereal registration. --- src/layers/activations/cereal_registration/elu.cpp | 7 ++++++- src/layers/activations/cereal_registration/identity.cpp | 6 +++++- src/layers/activations/cereal_registration/leaky_relu.cpp | 7 ++++++- src/layers/activations/cereal_registration/log_softmax.cpp | 6 +++++- src/layers/activations/cereal_registration/relu.cpp | 6 +++++- src/layers/activations/cereal_registration/softmax.cpp | 7 ++++++- 6 files changed, 33 insertions(+), 6 deletions(-) diff --git a/src/layers/activations/cereal_registration/elu.cpp b/src/layers/activations/cereal_registration/elu.cpp index cff58f79609..17e3b5a7c32 100644 --- a/src/layers/activations/cereal_registration/elu.cpp +++ b/src/layers/activations/cereal_registration/elu.cpp @@ -23,10 +23,15 @@ // implied. See the License for the specific language governing // permissions and limitations under the license. //////////////////////////////////////////////////////////////////////////////// -#include "lbann/utils/serialize.hpp" #include +#include +#include + +#include "lbann/utils/serialize.hpp" +#include "lbann/layers/data_type_layer.hpp" namespace lbann { +enum class data_layout; template template diff --git a/src/layers/activations/cereal_registration/identity.cpp b/src/layers/activations/cereal_registration/identity.cpp index 9f5ca9893a2..a049a4fad63 100644 --- a/src/layers/activations/cereal_registration/identity.cpp +++ b/src/layers/activations/cereal_registration/identity.cpp @@ -23,10 +23,14 @@ // implied. See the License for the specific language governing // permissions and limitations under the license. //////////////////////////////////////////////////////////////////////////////// -#include "lbann/utils/serialize.hpp" #include +#include + +#include "lbann/utils/serialize.hpp" +#include "lbann/layers/data_type_layer.hpp" namespace lbann { +enum class data_layout; template template diff --git a/src/layers/activations/cereal_registration/leaky_relu.cpp b/src/layers/activations/cereal_registration/leaky_relu.cpp index 6f584503f11..c11e97c696d 100644 --- a/src/layers/activations/cereal_registration/leaky_relu.cpp +++ b/src/layers/activations/cereal_registration/leaky_relu.cpp @@ -23,10 +23,15 @@ // implied. See the License for the specific language governing // permissions and limitations under the license. //////////////////////////////////////////////////////////////////////////////// -#include "lbann/utils/serialize.hpp" #include +#include +#include + +#include "lbann/utils/serialize.hpp" +#include "lbann/layers/data_type_layer.hpp" namespace lbann { +enum class data_layout; template template diff --git a/src/layers/activations/cereal_registration/log_softmax.cpp b/src/layers/activations/cereal_registration/log_softmax.cpp index b3596d37930..3cfdaf44af9 100644 --- a/src/layers/activations/cereal_registration/log_softmax.cpp +++ b/src/layers/activations/cereal_registration/log_softmax.cpp @@ -23,10 +23,14 @@ // implied. See the License for the specific language governing // permissions and limitations under the license. //////////////////////////////////////////////////////////////////////////////// -#include "lbann/utils/serialize.hpp" #include +#include + +#include "lbann/utils/serialize.hpp" +#include "lbann/layers/data_type_layer.hpp" namespace lbann { +enum class data_layout; template template diff --git a/src/layers/activations/cereal_registration/relu.cpp b/src/layers/activations/cereal_registration/relu.cpp index d1570789964..da25ae6c173 100644 --- a/src/layers/activations/cereal_registration/relu.cpp +++ b/src/layers/activations/cereal_registration/relu.cpp @@ -23,10 +23,14 @@ // implied. See the License for the specific language governing // permissions and limitations under the license. //////////////////////////////////////////////////////////////////////////////// -#include "lbann/utils/serialize.hpp" #include +#include + +#include "lbann/utils/serialize.hpp" +#include "lbann/layers/data_type_layer.hpp" namespace lbann { +enum class data_layout; template template diff --git a/src/layers/activations/cereal_registration/softmax.cpp b/src/layers/activations/cereal_registration/softmax.cpp index 753ff85dda6..ff8a1ba4a53 100644 --- a/src/layers/activations/cereal_registration/softmax.cpp +++ b/src/layers/activations/cereal_registration/softmax.cpp @@ -23,10 +23,15 @@ // implied. See the License for the specific language governing // permissions and limitations under the license. //////////////////////////////////////////////////////////////////////////////// -#include "lbann/utils/serialize.hpp" #include +#include +#include + +#include "lbann/utils/serialize.hpp" +#include "lbann/layers/data_type_layer.hpp" namespace lbann { +enum class data_layout; template template From deb37c2f99f184fb455552698ab29ff16262911e Mon Sep 17 00:00:00 2001 From: "Brian C. Van Essen" Date: Wed, 1 Feb 2023 09:27:52 -0800 Subject: [PATCH 08/10] Fixed a typo in the include. --- src/layers/math/matmul.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/layers/math/matmul.cpp b/src/layers/math/matmul.cpp index e7f118b14f5..fca1e3af73e 100644 --- a/src/layers/math/matmul.cpp +++ b/src/layers/math/matmul.cpp @@ -31,8 +31,8 @@ #include "lbann/utils/gpu/helpers.hpp" #endif // LBANN_HAS_GPU #include "lbann/proto/layers.pb.h" -#include -namespace lbann +#include +namespace lbann { #ifdef LBANN_HAS_DISTCONV @@ -111,13 +111,13 @@ matmul_distconv_adapter ::get_activations_local_shape(int index) const{ const auto &layer = dynamic_cast< const matmul_layer&>(this->layer()); - const auto output_shape = + const auto output_shape = ::distconv::get_matmul_local_tensor_shape( this->get_prev_activations(0), this->get_prev_activations(1), layer.m_transpose_a, layer.m_transpose_b); - return output_shape; + return output_shape; } // ============================================================= // DistConv-enabled MatMul member functions @@ -142,7 +142,7 @@ template const matmul_distconv_adapter& matmul_layer ::get_distconv_adapter() const{ - return dynamic_cast&>(data_type_layer::get_distconv_adapter()); } @@ -512,7 +512,7 @@ void matmul_layer::fp_compute() { } template void matmul_layer::bp_compute() { - + #ifdef LBANN_HAS_DISTCONV // We are guaranteed to have if(this->distconv_enabled()){ From b81f72abbb26ab8dcd8540ee9c3ae55771f96312 Mon Sep 17 00:00:00 2001 From: "Brian C. Van Essen" Date: Mon, 6 Feb 2023 10:50:48 +0900 Subject: [PATCH 09/10] Mark El.hpp as exporting for IWYU --- include/lbann/base.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/lbann/base.hpp b/include/lbann/base.hpp index 88e9f6f8869..1f3cc864386 100644 --- a/include/lbann/base.hpp +++ b/include/lbann/base.hpp @@ -29,7 +29,7 @@ #include "lbann/lbann_stl.hpp" -#include +#include // IWYU pragma: export // Defines, among other things, DataType. #include "lbann_config.hpp" From 15ac3acf92ee084cd480732e936e544d59955557 Mon Sep 17 00:00:00 2001 From: "Brian C. Van Essen" Date: Mon, 6 Feb 2023 10:51:16 +0900 Subject: [PATCH 10/10] Include --- include/lbann/lbann_stl.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/include/lbann/lbann_stl.hpp b/include/lbann/lbann_stl.hpp index 13f8604483c..89aa07e6351 100644 --- a/include/lbann/lbann_stl.hpp +++ b/include/lbann/lbann_stl.hpp @@ -43,6 +43,7 @@ import std; #include #include #include +#include #include #include #include