From 541d1efb48e7a12086df38f3da7740b52490bedd Mon Sep 17 00:00:00 2001 From: Abhijat Malviya Date: Mon, 18 Mar 2024 13:17:43 +0530 Subject: [PATCH] compression: Allocate memory for compressEnd Before this change the code assumed that the destination temp. buffer always had enough free space for LZ4F_compressEnd operation. This change checks the free size before calling LZ4F_compressEnd and allocates a new buffer if necessary. Note that the free space required is computed using LZ4F_compressBound with a srcSize of 0. By looking at LZ4F_compressEnd it may appear that we can just check if 12 bytes are free in the destination buffer, whereas LZ4F_compressBound always assumes at least 65443 ie 65535 + block header size (4) + frame footer (4). However, lz4frame docstring for LZ4F_compressBound states that: "When srcSize==0, LZ4F_compressBound() provides an upper bound for LZ4F_flush() and LZ4F_compressEnd() instead." Similar directions are mentioned on LZ4 gh issues, so it is safer to be conservative and possibly allocate the extra bytes. (cherry picked from commit 1493c6d8e28fb18801fb360319f3b5c65cab1e25) --- src/v/compression/internal/lz4_frame_compressor.cc | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/v/compression/internal/lz4_frame_compressor.cc b/src/v/compression/internal/lz4_frame_compressor.cc index 1b032f242102b..4028ee2581237 100644 --- a/src/v/compression/internal/lz4_frame_compressor.cc +++ b/src/v/compression/internal/lz4_frame_compressor.cc @@ -145,6 +145,16 @@ iobuf lz4_frame_compressor::compress(const iobuf& b) { output_cursor += code; } + if (const auto sz_for_compress_end = LZ4F_compressBound(0, &prefs); + output_sz - output_cursor < sz_for_compress_end) { + obuf.trim(output_cursor); + ret.append(std::move(obuf)); + obuf = ss::temporary_buffer(sz_for_compress_end); + output = obuf.get_write(); + output_sz = obuf.size(); + output_cursor = 0; + } + code = LZ4F_compressEnd( ctx, output + output_cursor, output_sz - output_cursor, nullptr); check_lz4_error("lz4f_compressend:{}", code);