From 18dc324746e66fa880a04cf7cc5bf761cc6f4e82 Mon Sep 17 00:00:00 2001 From: Brant Burnett Date: Sun, 15 Dec 2024 09:08:29 -0500 Subject: [PATCH] Improve the efficiency of decompressing to IBufferWriter --- Snappier/Internal/IsExternalInit.cs | 15 +++++++ Snappier/Internal/SnappyDecompressor.cs | 56 ++++++++++++++++++++++++- Snappier/Internal/ThrowHelper.cs | 4 +- Snappier/Snappy.cs | 19 ++++++++- 4 files changed, 88 insertions(+), 6 deletions(-) create mode 100644 Snappier/Internal/IsExternalInit.cs diff --git a/Snappier/Internal/IsExternalInit.cs b/Snappier/Internal/IsExternalInit.cs new file mode 100644 index 0000000..0c06a73 --- /dev/null +++ b/Snappier/Internal/IsExternalInit.cs @@ -0,0 +1,15 @@ +#if NETSTANDARD2_0 + +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +#pragma warning disable IDE0079 +#pragma warning disable S3903 + +namespace System.Runtime.CompilerServices; + +internal static class IsExternalInit +{ +} + +#endif diff --git a/Snappier/Internal/SnappyDecompressor.cs b/Snappier/Internal/SnappyDecompressor.cs index 79dcb09..9c40cb4 100644 --- a/Snappier/Internal/SnappyDecompressor.cs +++ b/Snappier/Internal/SnappyDecompressor.cs @@ -1,6 +1,7 @@ using System; using System.Buffers; using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; using System.IO; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; @@ -49,6 +50,11 @@ private struct ScratchBuffer /// public void Decompress(ReadOnlySpan input) { + if (AllDataDecompressed) + { + ThrowHelper.ThrowInvalidOperationException("All data has been decompressed"); + } + if (!ExpectedLength.HasValue) { OperationStatus status = TryReadUncompressedLength(input, out int bytesConsumed); @@ -82,6 +88,15 @@ public void Decompress(ReadOnlySpan input) DecompressAllTags(input); } } + + if (BufferWriter is not null && AllDataDecompressed) + { + // Advance the buffer writer to the end of the data + BufferWriter.Advance(_lookbackPosition); + + // Release the lookback buffer + _lookbackBuffer = default; + } } public void Reset() @@ -92,6 +107,12 @@ public void Reset() _lookbackPosition = 0; _readPosition = 0; ExpectedLength = null; + + if (BufferWriter is not null) + { + // Don't reuse the lookback buffer when it came from a BufferWriter + _lookbackBuffer = default; + } } private OperationStatus TryReadUncompressedLength(ReadOnlySpan input, out int bytesConsumed) @@ -483,6 +504,11 @@ private uint RefillTag(ref byte input, ref byte inputEnd) #region Loopback Writer + /// + /// Buffer writer for the output data. Incompatible with and . + /// + public IBufferWriter? BufferWriter { get; init; } + private byte[]? _lookbackBufferArray; private Memory _lookbackBuffer; private int _lookbackPosition = 0; @@ -503,8 +529,15 @@ private int? ExpectedLength ArrayPool.Shared.Return(_lookbackBufferArray); } - _lookbackBufferArray = ArrayPool.Shared.Rent(value.GetValueOrDefault()); - _lookbackBuffer = _lookbackBufferArray.AsMemory(0, _lookbackBufferArray.Length); + if (BufferWriter is not null) + { + _lookbackBuffer = BufferWriter.GetMemory(value.GetValueOrDefault()); + } + else + { + _lookbackBufferArray = ArrayPool.Shared.Rent(value.GetValueOrDefault()); + _lookbackBuffer = _lookbackBufferArray.AsMemory(0, _lookbackBufferArray.Length); + } } } } @@ -587,6 +620,11 @@ private static void AppendFromSelf(ref byte op, ref byte buffer, ref byte buffer public int Read(Span destination) { + if (BufferWriter is not null) + { + ThrowCannotUseWithBufferWriter(nameof(Read)); + } + var bytesToRead = Math.Min(destination.Length, UnreadBytes); if (bytesToRead <= 0) { @@ -609,6 +647,11 @@ public int Read(Span destination) /// public IMemoryOwner ExtractData() { + if (BufferWriter is not null) + { + ThrowCannotUseWithBufferWriter(nameof(ExtractData)); + } + byte[]? data = _lookbackBufferArray; if (!ExpectedLength.HasValue) { @@ -637,6 +680,15 @@ public IMemoryOwner ExtractData() return returnBuffer; } + [DoesNotReturn] + [MethodImpl(MethodImplOptions.NoInlining)] + private static void ThrowCannotUseWithBufferWriter(string method) + { + // This is intentionally not inlined to keep the size of Read and ExtractData smaller, + // making it more likely they may be inlined. + ThrowHelper.ThrowNotSupportedException($"Cannot use {method} when using a BufferWriter."); + } + #endregion #region Test Helpers diff --git a/Snappier/Internal/ThrowHelper.cs b/Snappier/Internal/ThrowHelper.cs index f7fe14e..b16cc84 100644 --- a/Snappier/Internal/ThrowHelper.cs +++ b/Snappier/Internal/ThrowHelper.cs @@ -41,8 +41,8 @@ public static void ThrowInvalidOperationException(string? message) => throw new InvalidOperationException(message); [DoesNotReturn] - public static void ThrowNotSupportedException() => - throw new NotSupportedException(); + public static void ThrowNotSupportedException(string? message = null) => + throw new NotSupportedException(message); [DoesNotReturn] public static void ThrowObjectDisposedException(string? objectName) => diff --git a/Snappier/Snappy.cs b/Snappier/Snappy.cs index 4faa46d..8d2bbac 100644 --- a/Snappier/Snappy.cs +++ b/Snappier/Snappy.cs @@ -52,6 +52,8 @@ public static int Compress(ReadOnlySpan input, Span output) /// public static void Compress(ReadOnlySequence input, IBufferWriter output) { + ThrowHelper.ThrowIfNull(output); + using var compressor = new SnappyCompressor(); compressor.Compress(input, output); @@ -149,9 +151,22 @@ public static int Decompress(ReadOnlySpan input, Span output) /// Invalid Snappy block. public static void Decompress(ReadOnlySequence input, IBufferWriter output) { - using IMemoryOwner buffer = DecompressToMemory(input); + ThrowHelper.ThrowIfNull(output); + + using var decompressor = new SnappyDecompressor() + { + BufferWriter = output + }; + + foreach (ReadOnlyMemory segment in input) + { + decompressor.Decompress(segment.Span); + } - output.Write(buffer.Memory.Span); + if (!decompressor.AllDataDecompressed) + { + ThrowHelper.ThrowInvalidDataException("Incomplete Snappy block."); + } } ///