From 83c70ac04f6782a33d861610b52233781484ad6c Mon Sep 17 00:00:00 2001 From: "Raphael S. Carvalho" Date: Mon, 19 Jun 2023 15:21:06 -0300 Subject: [PATCH] utils: Extract pretty printers into a header Can be easily reused elsewhere. Signed-off-by: Raphael S. Carvalho --- compaction/compaction.cc | 25 +++----------------- compaction/compaction.hh | 16 +------------ compaction/task_manager_module.cc | 3 ++- configure.py | 1 + replica/distributed_loader.cc | 4 ++-- test/boost/sstable_compaction_test.cc | 10 ++++---- utils/pretty_printers.cc | 32 ++++++++++++++++++++++++++ utils/pretty_printers.hh | 33 +++++++++++++++++++++++++++ 8 files changed, 79 insertions(+), 45 deletions(-) create mode 100644 utils/pretty_printers.cc create mode 100644 utils/pretty_printers.hh diff --git a/compaction/compaction.cc b/compaction/compaction.cc index 09eefc69cd7e..ab8fd9df8451 100644 --- a/compaction/compaction.cc +++ b/compaction/compaction.cc @@ -148,25 +148,6 @@ std::ostream& operator<<(std::ostream& os, compaction_type_options::scrub::quara return os << to_string(quarantine_mode); } -std::ostream& operator<<(std::ostream& os, pretty_printed_data_size data) { - static constexpr const char* suffixes[] = { " bytes", "kB", "MB", "GB", "TB", "PB" }; - - unsigned exp = 0; - while ((data._size >= 1000) && (exp < sizeof(suffixes))) { - exp++; - data._size /= 1000; - } - - os << data._size << suffixes[exp]; - return os; -} - -std::ostream& operator<<(std::ostream& os, pretty_printed_throughput tp) { - uint64_t throughput = tp._duration.count() > 0 ? tp._size / tp._duration.count() : 0; - os << pretty_printed_data_size(throughput) << "/s"; - return os; -} - static api::timestamp_type get_max_purgeable_timestamp(const table_state& table_s, sstable_set::incremental_selector& selector, const std::unordered_set& compacting_set, const dht::decorated_key& dk, uint64_t& bloom_filter_checks) { if (!table_s.tombstone_gc_enabled()) [[unlikely]] { @@ -806,8 +787,8 @@ class compaction { // By the time being, using estimated key count. log_info("{} {} sstables to {}. {} to {} (~{}% of original) in {}ms = {}. ~{} total partitions merged to {}.", report_finish_desc(), - _input_sstable_generations.size(), new_sstables_msg, pretty_printed_data_size(_start_size), pretty_printed_data_size(_end_size), int(ratio * 100), - std::chrono::duration_cast(duration).count(), pretty_printed_throughput(_end_size, duration), + _input_sstable_generations.size(), new_sstables_msg, utils::pretty_printed_data_size(_start_size), utils::pretty_printed_data_size(_end_size), int(ratio * 100), + std::chrono::duration_cast(duration).count(), utils::pretty_printed_throughput(_end_size, duration), _cdata.total_partitions, _cdata.total_keys_written); return ret; @@ -944,7 +925,7 @@ void compacted_fragments_writer::split_large_partition() { _c.log_debug("Closing active tombstone {} with {} for partition {}", _current_partition.current_emitted_tombstone, rtc, *_current_partition.dk); _compaction_writer->writer.consume(std::move(rtc)); } - _c.log_debug("Splitting large partition {} in order to respect SSTable size limit of {}", *_current_partition.dk, pretty_printed_data_size(_c._max_sstable_size)); + _c.log_debug("Splitting large partition {} in order to respect SSTable size limit of {}", *_current_partition.dk, utils::pretty_printed_data_size(_c._max_sstable_size)); // Close partition in current writer, and open it again in a new writer. do_consume_end_of_partition(); stop_current_writer(); diff --git a/compaction/compaction.hh b/compaction/compaction.hh index 4db544221609..652c70fb5657 100644 --- a/compaction/compaction.hh +++ b/compaction/compaction.hh @@ -14,6 +14,7 @@ #include "gc_clock.hh" #include "compaction_weight_registration.hh" #include "utils/UUID.hh" +#include "utils/pretty_printers.hh" #include "table_state.hh" #include #include @@ -24,21 +25,6 @@ namespace sstables { bool is_eligible_for_compaction(const sstables::shared_sstable& sst) noexcept; -class pretty_printed_data_size { - uint64_t _size; -public: - pretty_printed_data_size(uint64_t size) : _size(size) {} - friend std::ostream& operator<<(std::ostream&, pretty_printed_data_size); -}; - -class pretty_printed_throughput { - uint64_t _size; - std::chrono::duration _duration; -public: - pretty_printed_throughput(uint64_t size, std::chrono::duration dur) : _size(size), _duration(std::move(dur)) {} - friend std::ostream& operator<<(std::ostream&, pretty_printed_throughput); -}; - // Return the name of the compaction type // as used over the REST api, e.g. "COMPACTION" or "CLEANUP". sstring compaction_name(compaction_type type); diff --git a/compaction/task_manager_module.cc b/compaction/task_manager_module.cc index 84ca9e463531..d8988454ecf9 100644 --- a/compaction/task_manager_module.cc +++ b/compaction/task_manager_module.cc @@ -11,6 +11,7 @@ #include "replica/database.hh" #include "sstables/sstables.hh" #include "sstables/sstable_directory.hh" +#include "utils/pretty_printers.hh" namespace compaction { @@ -295,7 +296,7 @@ future<> table_reshaping_compaction_task_impl::run() { if (total_size > 0) { auto duration = std::chrono::duration_cast>(std::chrono::steady_clock::now() - start); - dblog.info("Reshaped {} in {:.2f} seconds, {}", sstables::pretty_printed_data_size(total_size), duration.count(), sstables::pretty_printed_throughput(total_size, duration)); + dblog.info("Reshaped {} in {:.2f} seconds, {}", utils::pretty_printed_data_size(total_size), duration.count(), utils::pretty_printed_throughput(total_size, duration)); } } diff --git a/configure.py b/configure.py index 891cfe30d04a..9b38cb45e355 100755 --- a/configure.py +++ b/configure.py @@ -726,6 +726,7 @@ def find_headers(repodir, excluded_dirs): 'utils/rjson.cc', 'utils/human_readable.cc', 'utils/histogram_metrics_helper.cc', + 'utils/pretty_printers.cc', 'converting_mutation_partition_applier.cc', 'readers/combined.cc', 'readers/multishard.cc', diff --git a/replica/distributed_loader.cc b/replica/distributed_loader.cc index af0eb012c187..14944e2f77da 100644 --- a/replica/distributed_loader.cc +++ b/replica/distributed_loader.cc @@ -278,7 +278,7 @@ future<> run_resharding_jobs(sharded& dir, std::vec } auto start = std::chrono::steady_clock::now(); - dblog.info("Resharding {} for {}.{}", sstables::pretty_printed_data_size(total_size), ks_name, table_name); + dblog.info("Resharding {} for {}.{}", utils::pretty_printed_data_size(total_size), ks_name, table_name); co_await dir.invoke_on_all(coroutine::lambda([&] (sstables::sstable_directory& d) -> future<> { auto& table = db.local().find_column_family(ks_name, table_name); @@ -293,7 +293,7 @@ future<> run_resharding_jobs(sharded& dir, std::vec })); auto duration = std::chrono::duration_cast>(std::chrono::steady_clock::now() - start); - dblog.info("Resharded {} for {}.{} in {:.2f} seconds, {}", sstables::pretty_printed_data_size(total_size), ks_name, table_name, duration.count(), sstables::pretty_printed_throughput(total_size, duration)); + dblog.info("Resharded {} for {}.{} in {:.2f} seconds, {}", utils::pretty_printed_data_size(total_size), ks_name, table_name, duration.count(), utils::pretty_printed_throughput(total_size, duration)); } // Global resharding function. Done in two parts: diff --git a/test/boost/sstable_compaction_test.cc b/test/boost/sstable_compaction_test.cc index 5f06243bac20..cfae56cf31ec 100644 --- a/test/boost/sstable_compaction_test.cc +++ b/test/boost/sstable_compaction_test.cc @@ -4518,7 +4518,7 @@ SEASTAR_TEST_CASE(simple_backlog_controller_test) { auto backlog_before = t.as_table_state().get_backlog_tracker().backlog(); t->add_sstable_and_update_cache(sst).get(); testlog.debug("\tNew sstable of size={} level={}; Backlog diff={};", - sstables::pretty_printed_data_size(data_size), level, + utils::pretty_printed_data_size(data_size), level, t.as_table_state().get_backlog_tracker().backlog() - backlog_before); }; @@ -4556,7 +4556,7 @@ SEASTAR_TEST_CASE(simple_backlog_controller_test) { for (auto target_table_count : target_table_count_s) { const uint64_t per_table_max_disk_usage = std::ceil(all_tables_disk_usage / target_table_count); - testlog.info("Creating tables, with max size={}", sstables::pretty_printed_data_size(per_table_max_disk_usage)); + testlog.info("Creating tables, with max size={}", utils::pretty_printed_data_size(per_table_max_disk_usage)); std::vector tables; uint64_t tables_total_size = 0; @@ -4577,18 +4577,18 @@ SEASTAR_TEST_CASE(simple_backlog_controller_test) { } auto table_size = t->get_stats().live_disk_space_used; - testlog.debug("T{}: {} tiers, with total size={}", t_idx, tiers, sstables::pretty_printed_data_size(table_size)); + testlog.debug("T{}: {} tiers, with total size={}", t_idx, tiers, utils::pretty_printed_data_size(table_size)); tables.push_back(t); tables_total_size += table_size; } - testlog.debug("Created {} tables, with total size={}", tables.size(), sstables::pretty_printed_data_size(tables_total_size)); + testlog.debug("Created {} tables, with total size={}", tables.size(), utils::pretty_printed_data_size(tables_total_size)); results.push_back(result{ tables.size(), per_table_max_disk_usage, normalize_backlog(manager.backlog()) }); for (auto& t : tables) { t.stop().get(); } } for (auto& r : results) { - testlog.info("Tables={} with max size={} -> NormalizedBacklog={}", r.table_count, sstables::pretty_printed_data_size(r.per_table_max_disk_usage), r.normalized_backlog); + testlog.info("Tables={} with max size={} -> NormalizedBacklog={}", r.table_count, utils::pretty_printed_data_size(r.per_table_max_disk_usage), r.normalized_backlog); // Expect 0 backlog as tiers are all perfectly compacted // With LCS, the size of levels *set up by the test* can slightly exceed their target size, // so let's account for the microscopical amount of backlog returned. diff --git a/utils/pretty_printers.cc b/utils/pretty_printers.cc new file mode 100644 index 000000000000..c4d51ea31734 --- /dev/null +++ b/utils/pretty_printers.cc @@ -0,0 +1,32 @@ +/* + * Copyright (C) 2023-present ScyllaDB + */ + +/* + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +#include "pretty_printers.hh" + +namespace utils { + +std::ostream& operator<<(std::ostream& os, pretty_printed_data_size data) { + static constexpr const char *suffixes[] = {" bytes", "kB", "MB", "GB", "TB", "PB"}; + + unsigned exp = 0; + while ((data._size >= 1000) && (exp < sizeof(suffixes))) { + exp++; + data._size /= 1000; + } + + os << data._size << suffixes[exp]; + return os; +} + +std::ostream& operator<<(std::ostream& os, pretty_printed_throughput tp) { + uint64_t throughput = tp._duration.count() > 0 ? tp._size / tp._duration.count() : 0; + os << pretty_printed_data_size(throughput) << "/s"; + return os; +} + +} diff --git a/utils/pretty_printers.hh b/utils/pretty_printers.hh new file mode 100644 index 000000000000..e90e1c51646c --- /dev/null +++ b/utils/pretty_printers.hh @@ -0,0 +1,33 @@ +/* + * Copyright (C) 2023-present ScyllaDB + */ + +/* + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +#pragma once + +#include +#include + +namespace utils { + +class pretty_printed_data_size { + uint64_t _size; +public: + pretty_printed_data_size(uint64_t size) : _size(size) {} + + friend std::ostream& operator<<(std::ostream&, pretty_printed_data_size); +}; + +class pretty_printed_throughput { + uint64_t _size; + std::chrono::duration _duration; +public: + pretty_printed_throughput(uint64_t size, std::chrono::duration dur) : _size(size), _duration(std::move(dur)) {} + + friend std::ostream& operator<<(std::ostream&, pretty_printed_throughput); +}; + +}