diff --git a/src/v/storage/mvlog/BUILD b/src/v/storage/mvlog/BUILD new file mode 100644 index 0000000000000..7c803b25a7f22 --- /dev/null +++ b/src/v/storage/mvlog/BUILD @@ -0,0 +1,61 @@ +load("//bazel:build.bzl", "redpanda_cc_library") + +package( + default_visibility = [ + "//src/v/storage/mvlog/tests:__pkg__", + ], +) + +redpanda_cc_library( + name = "mvlog", + srcs = [ + "batch_collecting_stream_utils.cc", + "batch_collector.cc", + "entry.cc", + "entry_stream.cc", + "entry_stream_utils.cc", + "file.cc", + "logger.cc", + "readable_segment.cc", + "segment_appender.cc", + "segment_reader.cc", + "skipping_data_source.cc", + "versioned_log.cc", + ], + hdrs = [ + "batch_collecting_stream_utils.h", + "batch_collector.h", + "entry.h", + "entry_stream.h", + "entry_stream_utils.h", + "errc.h", + "file.h", + "file_gap.h", + "logger.h", + "readable_segment.h", + "reader_outcome.h", + "segment_appender.h", + "segment_identifier.h", + "segment_reader.h", + "skipping_data_source.h", + "versioned_log.h", + ], + include_prefix = "storage/mvlog", + deps = [ + "//src/v/base", + "//src/v/bytes:iobuf", + "//src/v/bytes:iostream", + "//src/v/container:fragmented_vector", + "//src/v/container:interval_set", + "//src/v/hashing:crc32c", + "//src/v/io", + "//src/v/model", + "//src/v/serde", + "//src/v/storage", + "//src/v/storage:record_batch_utils", + "//src/v/utils:mutex", + "//src/v/utils:named_type", + "@fmt", + "@seastar", + ], +) diff --git a/src/v/storage/mvlog/tests/BUILD b/src/v/storage/mvlog/tests/BUILD new file mode 100644 index 0000000000000..35bb0e8964fb2 --- /dev/null +++ b/src/v/storage/mvlog/tests/BUILD @@ -0,0 +1,161 @@ +load("//bazel:test.bzl", "redpanda_cc_gtest") + +redpanda_cc_gtest( + name = "active_segment_test", + timeout = "short", + srcs = [ + "active_segment_test.cc", + ], + deps = [ + "//src/v/base", + "//src/v/model/tests:random", + "//src/v/random:generators", + "//src/v/storage", + "//src/v/storage/mvlog", + "//src/v/test_utils:gtest", + "@googletest//:gtest", + "@seastar", + "@seastar//:testing", + ], +) + +redpanda_cc_gtest( + name = "batch_collecting_stream_utils_test", + timeout = "short", + srcs = [ + "batch_collecting_stream_utils_test.cc", + ], + deps = [ + "//src/v/bytes:iostream", + "//src/v/hashing:crc32", + "//src/v/model", + "//src/v/model/tests:random", + "//src/v/storage", + "//src/v/storage:record_batch_utils", + "//src/v/storage/mvlog", + "//src/v/test_utils:gtest", + "@googletest//:gtest", + "@seastar", + "@seastar//:testing", + ], +) + +redpanda_cc_gtest( + name = "batch_collector_test", + timeout = "short", + srcs = [ + "batch_collector_test.cc", + ], + deps = [ + "//src/v/model", + "//src/v/storage", + "//src/v/storage/mvlog", + "//src/v/test_utils:gtest", + "@googletest//:gtest", + "@seastar", + "@seastar//:testing", + ], +) + +redpanda_cc_gtest( + name = "entry_stream_utils_test", + timeout = "short", + srcs = [ + "entry_stream_utils_test.cc", + ], + deps = [ + "//src/v/bytes:iostream", + "//src/v/bytes:random", + "//src/v/random:generators", + "//src/v/storage/mvlog", + "//src/v/test_utils:gtest", + "@googletest//:gtest", + "@seastar//:testing", + ], +) + +redpanda_cc_gtest( + name = "file_test", + timeout = "short", + srcs = [ + "file_test.cc", + ], + deps = [ + "//src/v/bytes:random", + "//src/v/container:fragmented_vector", + "//src/v/storage/mvlog", + "//src/v/test_utils:gtest", + "@googletest//:gtest", + "@seastar", + "@seastar//:testing", + ], +) + +redpanda_cc_gtest( + name = "segment_appender_test", + timeout = "short", + srcs = [ + "segment_appender_test.cc", + ], + deps = [ + "//src/v/model", + "//src/v/model/tests:random", + "//src/v/storage:record_batch_utils", + "//src/v/storage/mvlog", + "//src/v/test_utils:gtest", + "@googletest//:gtest", + "@seastar", + "@seastar//:testing", + ], +) + +redpanda_cc_gtest( + name = "segment_io_test", + timeout = "short", + srcs = [ + "segment_io_test.cc", + ], + deps = [ + "//src/v/model/tests:random", + "//src/v/storage/mvlog", + "//src/v/test_utils:gtest", + "@googletest//:gtest", + "@seastar", + "@seastar//:testing", + ], +) + +redpanda_cc_gtest( + name = "segment_reader_test", + timeout = "short", + srcs = [ + "segment_reader_test.cc", + ], + deps = [ + "//src/v/random:generators", + "//src/v/storage/mvlog", + "//src/v/test_utils:gtest", + "@googletest//:gtest", + "@seastar", + "@seastar//:testing", + ], +) + +redpanda_cc_gtest( + name = "skipping_data_source_test", + timeout = "short", + srcs = [ + "skipping_data_source_test.cc", + ], + deps = [ + "//src/v/base", + "//src/v/bytes:iostream", + "//src/v/bytes:random", + "//src/v/random:generators", + "//src/v/storage/mvlog", + "//src/v/test_utils:gtest", + "@googletest//:gtest", + "@seastar", + "@seastar//:testing", + ], +) diff --git a/src/v/storage/mvlog/tests/entry_stream_utils_test.cc b/src/v/storage/mvlog/tests/entry_stream_utils_test.cc index 86fd08f6b783a..aa4c5c5814d97 100644 --- a/src/v/storage/mvlog/tests/entry_stream_utils_test.cc +++ b/src/v/storage/mvlog/tests/entry_stream_utils_test.cc @@ -98,7 +98,7 @@ TEST(EntryStream, TestShortHeader) { entry_header hdr{0, 0, entry_type::record_batch}; hdr_buf.append(entry_header_to_iobuf(hdr)); ASSERT_GT(hdr_buf.size_bytes(), 1); - for (int i = 1; i < hdr_buf.size_bytes(); i++) { + for (size_t i = 1; i < hdr_buf.size_bytes(); i++) { auto copy_buf = hdr_buf.copy(); copy_buf.trim_back(i); auto short_stream = make_iobuf_input_stream(std::move(copy_buf)); diff --git a/src/v/storage/mvlog/tests/segment_reader_test.cc b/src/v/storage/mvlog/tests/segment_reader_test.cc index 67aeeda63dd9d..f55eb04078151 100644 --- a/src/v/storage/mvlog/tests/segment_reader_test.cc +++ b/src/v/storage/mvlog/tests/segment_reader_test.cc @@ -80,7 +80,7 @@ TEST_F(SegmentReaderTest, TestBasicReads) { buf.append(data.data(), data.size()); paging_file_->append(std::move(buf)).get(); readable_segment readable_seg(paging_file_.get()); - for (int i = 0; i < data.size(); i++) { + for (size_t i = 0; i < data.size(); i++) { auto reader = readable_seg.make_reader(); auto stream = reader->make_stream(i); auto buf = stream.read_up_to(data.size()).get(); @@ -194,7 +194,7 @@ TEST_F(SegmentReaderTest, TestReadVersionBeforeGaps) { expectations_map_t expected_per_version; auto reader = readable_seg.make_reader(); readable_seg.add_gap(file_gap(1, 3)); - for (int i = 0; i < data.size(); i++) { + for (size_t i = 0; i < data.size(); i++) { // Reading below the gapped version should ignore the gap entirely. ASSERT_NO_FATAL_FAILURE( validate_reader_exactly(*reader, i, data.substr(i))); @@ -285,7 +285,7 @@ TEST_F(SegmentReaderTest, TestReadsWithRandomGaps) { } // Rebuild the expected string, removing all gaps. ss::sstring expected; - for (int d = 0; d < data.size(); d++) { + for (size_t d = 0; d < data.size(); d++) { if (removed_pos.contains(d)) { continue; } @@ -317,12 +317,12 @@ TEST_F(SegmentReaderTest, TestRandomReadsWithGaps) { for (int i = 0; i < 10; i++) { auto gap_start = random_generators::get_int(size); auto gap_len = random_generators::get_int(size_t{1}, size); - for (int l = 0; l < gap_len; l++) { + for (size_t l = 0; l < gap_len; l++) { removed_pos.emplace(gap_start + l); } // Rebuild the expected string, removing all gaps. ss::sstring expected; - for (int d = 0; d < data.size(); d++) { + for (size_t d = 0; d < data.size(); d++) { if (removed_pos.contains(d)) { continue; } @@ -333,7 +333,7 @@ TEST_F(SegmentReaderTest, TestRandomReadsWithGaps) { expected_per_version.emplace_back(readable_seg.make_reader(), expected); for (const auto& [reader, full_expected_str] : expected_per_version) { // Read starting at every position, and ensure nothing funny. - for (int i = 0; i < size; i++) { + for (size_t i = 0; i < size; i++) { auto stream = reader->make_stream(i); auto buf = stream.read_exactly(size).get(); diff --git a/src/v/storage/tests/BUILD b/src/v/storage/tests/BUILD index 5912b9922f87e..5aa1a387cf228 100644 --- a/src/v/storage/tests/BUILD +++ b/src/v/storage/tests/BUILD @@ -1,5 +1,33 @@ load("//bazel:test.bzl", "redpanda_cc_btest", "redpanda_cc_gtest", "redpanda_test_cc_library") +redpanda_test_cc_library( + name = "storage_e2e_fixture", + hdrs = [ + "storage_e2e_fixture.h", + ], + include_prefix = "storage/tests", + visibility = ["//visibility:public"], + deps = [ + "//src/v/kafka/server/tests:kafka_test_utils", + "//src/v/redpanda/tests:fixture_btest", + "//src/v/storage", + "//src/v/test_utils:scoped_config", + "@seastar", + ], +) + +redpanda_test_cc_library( + name = "manual_mixin", + hdrs = [ + "manual_mixin.h", + ], + include_prefix = "storage/tests", + visibility = ["//visibility:public"], + deps = [ + "//src/v/test_utils:scoped_config", + ], +) + redpanda_test_cc_library( name = "kvstore_fixture", hdrs = [ @@ -541,3 +569,287 @@ redpanda_cc_btest( "@seastar", ], ) + +redpanda_cc_btest( + name = "segment_size_jitter_test", + timeout = "short", + srcs = [ + "segment_size_jitter_test.cc", + ], + deps = [ + "//src/v/storage", + "//src/v/test_utils:seastar_boost", + "@boost//:test", + "@seastar", + "@seastar//:testing", + ], +) + +redpanda_cc_btest( + name = "log_segment_reader_test", + timeout = "short", + srcs = [ + "log_segment_reader_test.cc", + ], + deps = [ + "//src/v/model", + "//src/v/model/tests:random", + "//src/v/random:generators", + "//src/v/storage", + "//src/v/storage:logger", + "//src/v/storage:parser_utils", + "//src/v/storage:record_batch_utils", + "//src/v/storage:segment_appender", + "//src/v/storage/tests:disk_log_builder", + "//src/v/test_utils:seastar_boost", + "@boost//:test", + "@seastar", + "@seastar//:testing", + ], +) + +redpanda_cc_btest( + name = "offset_assignment_test", + timeout = "short", + srcs = [ + "offset_assignment_test.cc", + ], + deps = [ + "//src/v/model/tests:random", + "//src/v/storage", + "//src/v/test_utils:seastar_boost", + "@boost//:test", + "@seastar", + "@seastar//:testing", + ], +) + +redpanda_cc_btest( + name = "log_truncate_test", + timeout = "short", + srcs = [ + "log_truncate_test.cc", + ], + deps = [ + "//src/v/model", + "//src/v/storage", + "//src/v/storage/tests:disk_log_builder", + "//src/v/storage/tests:storage_test_fixture", + "//src/v/test_utils:fixture", + "//src/v/test_utils:seastar_boost", + "@boost//:range", + "@boost//:test", + "@seastar", + "@seastar//:testing", + ], +) + +redpanda_cc_btest( + name = "compaction_index_format_tests", + timeout = "short", + srcs = [ + "compaction_index_format_tests.cc", + ], + deps = [ + "//src/v/base", + "//src/v/bytes", + "//src/v/bytes:iobuf_parser", + "//src/v/bytes:random", + "//src/v/model/tests:random", + "//src/v/random:generators", + "//src/v/reflection:adl", + "//src/v/storage", + "//src/v/storage:logger", + "//src/v/test_utils:fixture", + "//src/v/test_utils:random", + "//src/v/test_utils:seastar_boost", + "//src/v/test_utils:tmpbuf_file", + "//src/v/utils:vint", + "@boost//:test", + "@seastar//:testing", + ], +) + +redpanda_cc_btest( + name = "disk_log_builder_test", + timeout = "short", + srcs = [ + "disk_log_builder_test.cc", + ], + deps = [ + "//src/v/base", + "//src/v/model", + "//src/v/model/tests:random", + "//src/v/storage", + "//src/v/storage/tests:disk_log_builder_fixture", + "//src/v/test_utils:fixture", + "//src/v/test_utils:seastar_boost", + "@boost//:test", + "@seastar", + "@seastar//:testing", + ], +) + +redpanda_cc_btest( + name = "log_retention_tests", + timeout = "short", + srcs = [ + "log_retention_tests.cc", + ], + deps = [ + "//src/v/model", + "//src/v/storage/tests:disk_log_builder", + "//src/v/test_utils:fixture", + "//src/v/test_utils:seastar_boost", + "@boost//:test", + "@seastar", + "@seastar//:testing", + ], +) + +redpanda_cc_btest( + name = "half_page_concurrent_dispatch", + timeout = "short", + srcs = [ + "half_page_concurrent_dispatch.cc", + ], + deps = [ + "//src/v/storage", + "//src/v/storage:record_batch_builder", + "//src/v/storage/tests:disk_log_builder", + "//src/v/test_utils:fixture", + "//src/v/test_utils:seastar_boost", + "@boost//:test", + "@seastar", + "@seastar//:testing", + ], +) + +redpanda_cc_btest( + name = "file_sanitizer_test", + timeout = "short", + srcs = [ + "file_sanitizer_test.cc", + ], + deps = [ + "//src/v/bytes:iobuf", + "//src/v/bytes:iostream", + "//src/v/storage", + "//src/v/test_utils:fixture", + "//src/v/test_utils:seastar_boost", + "@boost//:test", + "@seastar", + "@seastar//:testing", + ], +) + +redpanda_cc_btest( + name = "compaction_reducer_test", + timeout = "short", + srcs = [ + "compaction_reducer_test.cc", + ], + deps = [ + "//src/v/bytes:random", + "//src/v/random:generators", + "//src/v/storage", + "//src/v/storage:logger", + "//src/v/test_utils:seastar_boost", + "@boost//:test", + "@seastar", + "@seastar//:testing", + ], +) + +redpanda_cc_btest( + name = "batch_consumer_utils_test", + timeout = "short", + srcs = [ + "batch_consumer_utils_test.cc", + ], + deps = [ + "//src/v/bytes:iostream", + "//src/v/model/tests:random", + "//src/v/storage", + "//src/v/storage/tests:disk_log_builder", + "//src/v/test_utils:fixture", + "//src/v/test_utils:seastar_boost", + "@boost//:test", + "@seastar", + "@seastar//:testing", + ], +) + +redpanda_cc_btest( + name = "log_replayer_test", + timeout = "short", + srcs = [ + "log_replayer_test.cc", + ], + deps = [ + "//src/v/base", + "//src/v/features", + "//src/v/model", + "//src/v/model/tests:random", + "//src/v/random:generators", + "//src/v/storage", + "//src/v/storage:logger", + "//src/v/storage:record_batch_utils", + "//src/v/storage:resources", + "//src/v/test_utils:seastar_boost", + "@boost//:test", + "@seastar", + "@seastar//:testing", + ], +) + +redpanda_cc_btest( + name = "storage_e2e_fixture_test", + timeout = "moderate", + srcs = [ + "storage_e2e_fixture_test.cc", + ], + cpu = 1, + deps = [ + "//src/v/kafka/server/tests:kafka_test_utils", + "//src/v/model", + "//src/v/random:generators", + "//src/v/redpanda/tests:fixture_btest", + "//src/v/storage", + "//src/v/storage/tests:storage_e2e_fixture", + "//src/v/test_utils:fixture", + "//src/v/test_utils:scoped_config", + "//src/v/test_utils:seastar_boost", + "@boost//:test", + "@seastar", + "@seastar//:testing", + ], +) + +redpanda_cc_gtest( + name = "compaction_e2e_test", + timeout = "moderate", + srcs = [ + "compaction_e2e_test.cc", + ], + cpu = 1, + deps = [ + "//src/v/base", + "//src/v/kafka/server/tests:kafka_test_utils", + "//src/v/model", + "//src/v/random:generators", + "//src/v/redpanda/tests:fixture_btest", + "//src/v/redpanda/tests:fixture_gtest", + "//src/v/storage", + "//src/v/storage/tests:manual_mixin", + "//src/v/test_utils:fixture", + "//src/v/test_utils:gtest", + "//src/v/test_utils:scoped_config", + "//src/v/test_utils:seastar_boost", + "//src/v/utils:directory_walker", + "@abseil-cpp//absl/container:btree", + "@googletest//:gtest", + "@seastar", + "@seastar//:testing", + ], +) diff --git a/src/v/storage/tests/compaction_e2e_test.cc b/src/v/storage/tests/compaction_e2e_test.cc index ba7a8fee2a855..12e0f668d889c 100644 --- a/src/v/storage/tests/compaction_e2e_test.cc +++ b/src/v/storage/tests/compaction_e2e_test.cc @@ -127,7 +127,7 @@ class CompactionFixtureTest // Generate some segments. size_t val_count = starting_value; for (size_t i = 0; i < num_segments; i++) { - for (int r = 0; r < batches_per_segment; r++) { + for (size_t r = 0; r < batches_per_segment; r++) { auto kvs = tests::kv_t::sequence( val_count, records_per_batch, @@ -459,7 +459,7 @@ TEST_F(CompactionFixtureTest, TestDedupeMultiPassAddedSegment) { ASSERT_LT(segments_compacted, segments_compacted_2); // segs.size() - 2 to account for active segment. - for (int i = 0; i < segs.size() - 2; ++i) { + for (size_t i = 0; i < segs.size() - 2; ++i) { auto& seg = segs[i]; ASSERT_TRUE(seg->finished_windowed_compaction()); ASSERT_TRUE(seg->finished_self_compaction()); @@ -938,7 +938,7 @@ TEST_P(CompactionFixtureTombstonesParamTest, TestTombstonesCompletelyEmptyLog) { .get(); ASSERT_TRUE(did_compact); - for (int i = 0; i < num_segments; ++i) { + for (size_t i = 0; i < num_segments; ++i) { ASSERT_TRUE(log->segments()[i]->index().has_clean_compact_timestamp()); } diff --git a/src/v/storage/tests/log_segment_reader_test.cc b/src/v/storage/tests/log_segment_reader_test.cc index 7d159e5d02da5..772437e84e489 100644 --- a/src/v/storage/tests/log_segment_reader_test.cc +++ b/src/v/storage/tests/log_segment_reader_test.cc @@ -21,7 +21,7 @@ #include "storage/segment.h" #include "storage/segment_appender.h" #include "storage/segment_reader.h" -#include "utils/disk_log_builder.h" +#include "storage/tests/utils/disk_log_builder.h" #include #include diff --git a/src/v/storage/tests/storage_e2e_fixture.h b/src/v/storage/tests/storage_e2e_fixture.h index 8f7e3352e2f94..e58ef0f934003 100644 --- a/src/v/storage/tests/storage_e2e_fixture.h +++ b/src/v/storage/tests/storage_e2e_fixture.h @@ -7,6 +7,7 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0 +#include "kafka/server/tests/produce_consume_utils.h" #include "redpanda/tests/fixture.h" #include "storage/disk_log_impl.h" #include "storage/segment.h"