From 7a9a123d7cc4269d61816f2bf762e150a43ebd40 Mon Sep 17 00:00:00 2001 From: bxq2011hust Date: Thu, 18 Jan 2024 13:45:03 +0800 Subject: [PATCH] support log archive and compress --- .clang-tidy | 2 +- bcos-utilities/CMakeLists.txt | 3 - bcos-utilities/bcos-utilities/BoostLog.cpp | 506 ++++++++++++++++++ bcos-utilities/bcos-utilities/BoostLog.h | 9 + .../bcos-utilities/BoostLogInitializer.cpp | 4 +- .../unittests/libutilities/TestBoostLog.cpp | 39 ++ 6 files changed, 557 insertions(+), 6 deletions(-) create mode 100644 bcos-utilities/test/unittests/libutilities/TestBoostLog.cpp diff --git a/.clang-tidy b/.clang-tidy index 56bafee34b..a34358922a 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -235,6 +235,6 @@ CheckOptions: - key: llvm-else-after-return.WarnOnUnfixable value: 'false' - key: readability-identifier-length.IgnoredVariableNames - value: 'it|ar|i|j|k|e|id|ss|os|tx|db' + value: 'it|ar|i|j|k|e|id|ss|os|tx|db|to' ... diff --git a/bcos-utilities/CMakeLists.txt b/bcos-utilities/CMakeLists.txt index 81e5aca030..defbe623cc 100644 --- a/bcos-utilities/CMakeLists.txt +++ b/bcos-utilities/CMakeLists.txt @@ -32,9 +32,6 @@ if(TESTS) add_subdirectory(test) endif() -if (NOT WIN32) - target_compile_options(bcos-utilities PRIVATE -Wno-error=unused-function) #FIXME: remove this line -endif() include(GNUInstallDirs) install(TARGETS bcos-utilities EXPORT fiscobcosTargets ARCHIVE DESTINATION "${CMAKE_INSTALL_LIBDIR}") install(DIRECTORY "bcos-utilities" DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/" FILES_MATCHING PATTERN "*.h") \ No newline at end of file diff --git a/bcos-utilities/bcos-utilities/BoostLog.cpp b/bcos-utilities/bcos-utilities/BoostLog.cpp index dae8a95d0c..8a7c711918 100644 --- a/bcos-utilities/bcos-utilities/BoostLog.cpp +++ b/bcos-utilities/bcos-utilities/BoostLog.cpp @@ -454,5 +454,511 @@ class file_collector_repository; using file_collector_hook = boost::intrusive::list_base_hook>; +//! Log file collector implementation +class file_collector : public boost::log::sinks::file::collector, + public file_collector_hook, + public boost::enable_shared_from_this +{ +private: + //! Information about a single stored file + struct file_info + { + //! Ordering predicate by timestamp + struct order_by_timestamp + { + using result_type = bool; + + result_type operator()( + file_info const& left, file_info const& right) const BOOST_NOEXCEPT + { + return left.m_TimeStamp < right.m_TimeStamp; + } + }; + + //! Predicate for testing if a file_info refers to a file equivalent to another path + class equivalent_file + { + public: + using result_type = bool; + explicit equivalent_file(boost::filesystem::path const& path) BOOST_NOEXCEPT + : m_Path(path) + {} + + result_type operator()(file_info const& info) const + { + return boost::filesystem::equivalent(info.m_Path, m_Path); + } + + private: + boost::filesystem::path const& m_Path; + }; + + uintmax_t m_Size; + std::time_t m_TimeStamp; + boost::filesystem::path m_Path; + }; + //! A list of the stored files + using file_list = std::list; + //! The string type compatible with the universal path type + using path_string_type = boost::filesystem::path::string_type; + + //! A reference to the repository this collector belongs to + boost::shared_ptr m_pRepository; + +#if !defined(BOOST_LOG_NO_THREADS) + //! Synchronization mutex + std::mutex m_Mutex; +#endif // !defined(BOOST_LOG_NO_THREADS) + + //! Total file size upper limit + uintmax_t m_MaxSize; + //! Free space lower limit + uintmax_t m_MinFreeSpace; + //! File count upper limit + uintmax_t m_MaxFiles; + + //! The current path at the point when the collector is created + /* + * The special member is required to calculate absolute paths with no + * dependency on the current path for the application, which may change + */ + const boost::filesystem::path m_BasePath; + //! Target directory to store files to + boost::filesystem::path m_StorageDir; + + //! The list of stored files + file_list m_Files; + //! Total size of the stored files + uintmax_t m_TotalSize; + bool m_ConvertTarGZ = false; + +public: + //! Constructor + file_collector(boost::shared_ptr const& repo, + boost::filesystem::path const& target_dir, uintmax_t max_size, uintmax_t min_free_space, + uintmax_t max_files); + + //! Destructor + ~file_collector() BOOST_OVERRIDE; + + //! The function stores the specified file in the storage + void store_file(boost::filesystem::path const& src_path) BOOST_OVERRIDE; + + //! The function checks if the specified path refers to an existing file in the storage + bool is_in_storage(boost::filesystem::path const& src_path) const BOOST_OVERRIDE; + + //! Scans the target directory for the files that have already been stored + boost::log::sinks::file::scan_result scan_for_files(boost::log::sinks::file::scan_method method, + boost::filesystem::path const& pattern) BOOST_OVERRIDE; + + //! The function updates storage restrictions + void update(uintmax_t max_size, uintmax_t min_free_space, uintmax_t max_files); + + //! The function checks if the directory is governed by this collector + bool is_governed(boost::filesystem::path const& dir) const + { + return boost::filesystem::equivalent(m_StorageDir, dir); + } + + void convert_tar_gz(bool ConvertTarGZ) + { + m_ConvertTarGZ = ConvertTarGZ; + } + +private: + //! Makes relative path absolute with respect to the base path + boost::filesystem::path make_absolute(boost::filesystem::path const& path) const + { + return boost::filesystem::absolute(path, m_BasePath); + } + //! Acquires file name string from the path + static path_string_type filename_string(boost::filesystem::path const& path) + { + return path.filename().string(); + } +}; + + +//! The singleton of the list of file collectors +class file_collector_repository : public boost::log::aux::lazy_singleton> +{ +private: + //! Base type + using base_type = boost::log::aux::lazy_singleton>; + +#if !defined(BOOST_LOG_BROKEN_FRIEND_TEMPLATE_SPECIALIZATIONS) + friend class boost::log::aux::lazy_singleton>; +#else + friend class base_type; +#endif + + //! The type of the list of collectors + using file_collectors = + boost::intrusive::list>; + +#if !defined(BOOST_LOG_NO_THREADS) + //! Synchronization mutex + std::mutex m_Mutex; +#endif // !defined(BOOST_LOG_NO_THREADS) + //! The list of file collectors + file_collectors m_Collectors; + +public: + //! Finds or creates a file collector + boost::shared_ptr get_collector( + boost::filesystem::path const& target_dir, uintmax_t max_size, uintmax_t min_free_space, + uintmax_t max_files, bool convert_tar_gz); + + //! Removes the file collector from the list + void remove_collector(file_collector* fileCollector); + +private: + //! Initializes the singleton instance + static void init_instance() + { + base_type::get_instance() = boost::make_shared(); + } +}; + +//! Constructor +file_collector::file_collector(boost::shared_ptr const& repo, + boost::filesystem::path const& target_dir, uintmax_t max_size, uintmax_t min_free_space, + uintmax_t max_files) + : m_pRepository(repo), + m_MaxSize(max_size), + m_MinFreeSpace(min_free_space), + m_MaxFiles(max_files), + m_BasePath(boost::filesystem::current_path()), + m_TotalSize(0) +{ + m_StorageDir = make_absolute(target_dir); + boost::filesystem::create_directories(m_StorageDir); +} + +//! Destructor +file_collector::~file_collector() +{ + m_pRepository->remove_collector(this); +} + +//! The function stores the specified file in the storage +void file_collector::store_file(boost::filesystem::path const& src_path) +{ + // NOTE FOR THE FOLLOWING CODE: + // Avoid using Boost.Filesystem functions that would call path::codecvt(). store_file() can be + // called at process termination, and the global codecvt facet can already be destroyed at this + // point. https://svn.boost.org/trac/boost/ticket/8642 + + // Let's construct the new file name + file_info info; + info.m_TimeStamp = boost::filesystem::last_write_time(src_path); + info.m_Size = boost::filesystem::file_size(src_path); + + const boost::filesystem::path file_name_path = src_path.filename(); + path_string_type const& file_name = file_name_path.native(); + info.m_Path = m_StorageDir / file_name_path; + + // Check if the file is already in the target directory + boost::filesystem::path src_dir = + src_path.has_parent_path() ? boost::filesystem::system_complete(src_path.parent_path()) : + m_BasePath; + const bool is_in_target_dir = boost::filesystem::equivalent(src_dir, m_StorageDir); + if (!is_in_target_dir) + { + if (boost::filesystem::exists(info.m_Path)) + { + // If the file already exists, try to mangle the file name + // to ensure there's no conflict. I'll need to make this customizable some day. + file_counter_formatter formatter(file_name.size(), 5); + unsigned int n = 0; + while (true) + { + path_string_type alt_file_name = formatter(file_name, n); + info.m_Path = m_StorageDir / boost::filesystem::path(alt_file_name); + if (!boost::filesystem::exists(info.m_Path)) + { + break; + } + + if (BOOST_UNLIKELY(n == (std::numeric_limits::max)())) + { + BOOST_THROW_EXCEPTION(boost::filesystem::filesystem_error( + "Target file exists and an unused fallback file name could not be found", + info.m_Path, + boost::system::error_code( + boost::system::errc::io_error, boost::system::generic_category()))); + } + + ++n; + } + } + + // The directory should have been created in constructor, but just in case it got deleted + // since then... + boost::filesystem::create_directories(m_StorageDir); + } + + std::lock_guard lock(m_Mutex); + + auto it = m_Files.begin(); + const auto end = m_Files.end(); + if (is_in_target_dir) + { + // If the sink writes log file into the target dir (is_in_target_dir == true), it is + // possible that after scanning an old file entry refers to the file that is picked up by + // the sink for writing. Later on, the sink attempts to store the file in the storage. At + // best, this would result in duplicate file entries. At worst, if the storage limits + // trigger a deletion and this file get deleted, we may have an entry that refers to no + // actual file. In any case, the total size of files in the storage will be incorrect. Here + // we work around this problem and simply remove the old file entry without removing the + // file. The entry will be re-added to the list later. + while (it != end) + { + boost::system::error_code ec; + if (boost::filesystem::equivalent(it->m_Path, info.m_Path, ec)) + { + m_TotalSize -= it->m_Size; + m_Files.erase(it); + break; + } + ++it; + } + + it = m_Files.begin(); + } + + // Check if an old file should be erased + uintmax_t free_space = m_MinFreeSpace != 0U ? boost::filesystem::space(m_StorageDir).available : + static_cast(0); + while (it != end && (m_TotalSize + info.m_Size > m_MaxSize || + ((m_MinFreeSpace != 0U) && m_MinFreeSpace > free_space) || + m_MaxFiles <= m_Files.size())) + { + file_info& old_info = *it; + boost::system::error_code ec; + boost::filesystem::file_status status = boost::filesystem::status(old_info.m_Path, ec); + + if (status.type() == boost::filesystem::regular_file) + { + try + { + boost::filesystem::remove(old_info.m_Path); + // Free space has to be queried as it may not increase equally + // to the erased file size on compressed filesystems + if (m_MinFreeSpace != 0U) + { + free_space = boost::filesystem::space(m_StorageDir).available; + } + m_TotalSize -= old_info.m_Size; + it = m_Files.erase(it); + } + catch (boost::system::system_error&) + { + // Can't erase the file. Maybe it's locked? Never mind... + ++it; + } + } + else + { + // If it's not a file or is absent, just remove it from the list + m_TotalSize -= old_info.m_Size; + it = m_Files.erase(it); + } + } + + if (!is_in_target_dir) + { + // Move/rename the file to the target storage + move_file(src_path, info.m_Path); + } + if (m_ConvertTarGZ) + { + auto from = info.m_Path; + info.m_Path = info.m_Path.string() + std::string(".gz"); + convert_to_tar_gz(from, info.m_Path); + info.m_Size = boost::filesystem::file_size(info.m_Path); + } + + m_Files.push_back(info); + m_TotalSize += info.m_Size; +} + +//! The function checks if the specified path refers to an existing file in the storage +bool file_collector::is_in_storage(boost::filesystem::path const& src_path) const +{ + const boost::filesystem::path file_name_path = src_path.filename(); + const boost::filesystem::path trg_path = m_StorageDir / file_name_path; + + // Check if the file is already in the target directory + boost::system::error_code ec; + boost::filesystem::path src_dir = + src_path.has_parent_path() ? + boost::filesystem::system_complete(src_path.parent_path(), ec) : + m_BasePath; + if (ec) + { + return false; + } + + boost::filesystem::file_status status = boost::filesystem::status(trg_path, ec); + if (ec || status.type() != boost::filesystem::regular_file) + { + return false; + } + bool equiv = boost::filesystem::equivalent(src_dir / file_name_path, trg_path, ec); + if (ec) + { + return false; + } + + return equiv; +} + +//! Scans the target directory for the files that have already been stored +boost::log::sinks::file::scan_result file_collector::scan_for_files( + boost::log::sinks::file::scan_method method, boost::filesystem::path const& pattern) +{ + boost::log::sinks::file::scan_result result; + if (method != boost::log::sinks::file::no_scan) + { + boost::filesystem::path dir = m_StorageDir; + path_string_type mask; + if (method == boost::log::sinks::file::scan_matching) + { + mask = filename_string(pattern); + if (m_ConvertTarGZ) + { + mask = mask + std::string(".gz"); + } + if (pattern.has_parent_path()) + { + dir = make_absolute(pattern.parent_path()); + } + } + + boost::system::error_code ec; + boost::filesystem::file_status status = boost::filesystem::status(dir, ec); + if (status.type() == boost::filesystem::directory_file) + { + std::lock_guard lock(m_Mutex); + + file_list files; + boost::filesystem::directory_iterator it(dir); + boost::filesystem::directory_iterator end; + uintmax_t total_size = 0U; + for (; it != end; ++it) + { + boost::filesystem::directory_entry const& dir_entry = *it; + file_info info; + info.m_Path = dir_entry.path(); + status = dir_entry.status(ec); + if (status.type() == boost::filesystem::regular_file) + { + // Check that there are no duplicates in the resulting list + if (std::find_if(m_Files.begin(), m_Files.end(), + file_info::equivalent_file(info.m_Path)) == m_Files.end()) + { + // Check that the file name matches the pattern + unsigned int file_number = 0U; + bool file_number_parsed = false; + if (method != boost::log::sinks::file::scan_matching || + match_pattern(filename_string(info.m_Path), mask, file_number, + file_number_parsed)) + { + info.m_Size = boost::filesystem::file_size(info.m_Path); + total_size += info.m_Size; + info.m_TimeStamp = boost::filesystem::last_write_time(info.m_Path); + files.push_back(info); + ++result.found_count; + + // Test that the file_number >= result.last_file_counter accounting for + // the integer overflow + if (file_number_parsed && + (!result.last_file_counter || + (file_number - *result.last_file_counter) < + ((~0U) ^ ((~0U) >> 1U)))) + { + result.last_file_counter = file_number; + } + } + } + } + } + + // Sort files chronologically + m_Files.splice(m_Files.end(), files); + m_TotalSize += total_size; + m_Files.sort(file_info::order_by_timestamp()); + } + } + + return result; +} + +//! The function updates storage restrictions +void file_collector::update(uintmax_t max_size, uintmax_t min_free_space, uintmax_t max_files) +{ + std::lock_guard lock(m_Mutex); + + m_MaxSize = (std::min)(m_MaxSize, max_size); + m_MinFreeSpace = (std::max)(m_MinFreeSpace, min_free_space); + m_MaxFiles = (std::min)(m_MaxFiles, max_files); +} + + +//! Finds or creates a file collector +boost::shared_ptr file_collector_repository::get_collector( + boost::filesystem::path const& target_dir, uintmax_t max_size, uintmax_t min_free_space, + uintmax_t max_files, bool convert_tar_gz) +{ + std::lock_guard lock(m_Mutex); + + file_collectors::iterator it = std::find_if( + m_Collectors.begin(), m_Collectors.end(), [&target_dir](file_collector const& collector) { + return collector.is_governed(target_dir); + }); + boost::shared_ptr collector; + if (it != m_Collectors.end()) + { + try + { + // This may throw if the collector is being currently destroyed + collector = it->shared_from_this(); + collector->update(max_size, min_free_space, max_files); + } + catch (std::bad_weak_ptr&) + {} + } + + if (!collector) + { + collector = boost::make_shared( + file_collector_repository::get(), target_dir, max_size, min_free_space, max_files); + m_Collectors.push_back(*collector); + } + collector->convert_tar_gz(convert_tar_gz); + return collector; +} + +//! Removes the file collector from the list +void file_collector_repository::remove_collector(file_collector* fileCollector) +{ + std::lock_guard lock(m_Mutex); + m_Collectors.erase(m_Collectors.iterator_to(*fileCollector)); +} } // namespace +namespace log +{ + +boost::shared_ptr make_collector( + boost::filesystem::path const& target_dir, uintmax_t max_size, uintmax_t min_free_space, + uintmax_t max_files = (std::numeric_limits::max)(), bool convert_tar_gz = false) +{ + return file_collector_repository::get()->get_collector( + target_dir, max_size, min_free_space, max_files, convert_tar_gz); +} +} // namespace log } // namespace bcos diff --git a/bcos-utilities/bcos-utilities/BoostLog.h b/bcos-utilities/bcos-utilities/BoostLog.h index a7d526d6c5..6696e1d543 100644 --- a/bcos-utilities/bcos-utilities/BoostLog.h +++ b/bcos-utilities/bcos-utilities/BoostLog.h @@ -23,6 +23,7 @@ #include #include +#include #include #include @@ -83,4 +84,12 @@ void setStatLogLevel(LogLevel const& _level); bcos::FileLoggerHandler, (boost::log::trivial::severity_level)(bcos::LogLevel::level)) // for block number log #define BLOCK_NUMBER(NUMBER) "[blk-" << (NUMBER) << "]" + +namespace log +{ +boost::shared_ptr make_collector( + boost::filesystem::path const& target_dir, uintmax_t max_size, uintmax_t min_free_space, + uintmax_t max_files, bool convert_tar_gz); +} + } // namespace bcos \ No newline at end of file diff --git a/bcos-utilities/bcos-utilities/BoostLogInitializer.cpp b/bcos-utilities/bcos-utilities/BoostLogInitializer.cpp index 6bd05b98ae..e30a2ed384 100644 --- a/bcos-utilities/bcos-utilities/BoostLogInitializer.cpp +++ b/bcos-utilities/bcos-utilities/BoostLogInitializer.cpp @@ -281,13 +281,14 @@ boost::shared_ptr BoostLogInitializer::initLo sink->set_filter(boost::log::expressions::attr("Channel") == channel); if (!m_archivePath.empty()) { +#if 0 sink->locked_backend()->set_file_collector(boost::log::sinks::file::make_collector( boost::log::keywords::target = m_archivePath, // to store rotated files boost::log::keywords::max_size = m_maxArchiveSize, // maximum size(bytes) boost::log::keywords::min_free_space = m_minFreeSpace, // minimum free space(bytes) boost::log::keywords::max_files = m_maxArchiveFiles // maximum number of stored files )); -#if 0 +#endif boost::filesystem::path targetDir(m_archivePath); sink->locked_backend()->set_file_collector( bcos::log::make_collector(targetDir, // where to store rotated files @@ -295,7 +296,6 @@ boost::shared_ptr BoostLogInitializer::initLo m_minFreeSpace, // minimum free space, in bytes m_maxArchiveFiles, // maximum number of stored files m_compressArchive)); -#endif } sink->locked_backend()->scan_for_files(); boost::log::core::get()->add_sink(sink); diff --git a/bcos-utilities/test/unittests/libutilities/TestBoostLog.cpp b/bcos-utilities/test/unittests/libutilities/TestBoostLog.cpp new file mode 100644 index 0000000000..fb13d196a5 --- /dev/null +++ b/bcos-utilities/test/unittests/libutilities/TestBoostLog.cpp @@ -0,0 +1,39 @@ +/** + * Copyright (C) 2021 FISCO BCOS. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file TestBoostLog + * @author: xingqiangbai + * @date: 2021-01-18 + */ + +#include "bcos-utilities/BoostLog.h" +#include +#include +#include +using namespace bcos; +using namespace std; + +namespace bcos +{ +namespace test +{ +BOOST_AUTO_TEST_CASE(testCreatFileColletctor) +{ + boost::filesystem::path targetDir("./testCreatFileColletctor"); + auto collector = bcos::log::make_collector(targetDir, 0, 0, 0, false); + BOOST_CHECK(collector != nullptr); +} +} // namespace test +} // namespace bcos