Switch LZ4 compressor to smallz4 for an improvement in compression ra…

…tio and speed
maxim-zhao · Sep 8, 2022 · 18e6418 · 18e6418
1 parent 14ac9cd
commit 18e6418
Show file tree

Hide file tree

Showing 6 changed files with 48 additions and 2,069 deletions.
diff --git a/benchmark.png b/benchmark.png
diff --git a/compressors/gfxcomp_lz4.cpp b/compressors/gfxcomp_lz4.cpp
@@ -1,4 +1,6 @@
-#include "lz4/lz4.h"
+#pragma warning(push,0)
+#include "smallz4/smallz4.h"
+#pragma warning(pop)
 #include <string>
 
 extern "C" __declspec(dllexport) const char* getName()
@@ -9,7 +11,7 @@ extern "C" __declspec(dllexport) const char* getName()
     if (name.empty())
     {
         name = "LZ4 (raw, v";
-        name += LZ4_versionString();
+        name += smallz4::getVersion();
         name += ")";
     }
     return name.c_str();
@@ -21,22 +23,45 @@ extern "C" __declspec(dllexport) const char* getExt()
     return "lz4";
 }
 
+const uint8_t* g_pSource;
+uint32_t g_sourceRemaining;
+uint8_t* g_pDestination;
+uint32_t g_destinationRemaining;
+
+size_t getBytes(void* data, size_t numBytes, void* /*userPtr*/)
+{
+    const auto count = std::min(g_sourceRemaining, numBytes);
+    memcpy_s(data, numBytes, g_pSource, g_sourceRemaining);
+    g_pSource += count;
+    g_sourceRemaining -= count;
+    return count;
+}
+
+void sendBytes(const void* data, size_t numBytes, void* /*userPtr*/)
+{
+    const auto count = std::min(g_destinationRemaining, numBytes);
+    memcpy_s(g_pDestination, g_destinationRemaining, data, count);
+    g_pDestination += count;
+    g_destinationRemaining -= count;
+}
+
 extern "C" __declspec(dllexport) int compressTiles(
     const uint8_t* pSource,
     const uint32_t numTiles,
     uint8_t* pDestination,
     const uint32_t destinationLength)
 {
-    const auto sourceLength = numTiles * 32;
-    if (destinationLength < static_cast<uint32_t>(LZ4_compressBound(sourceLength)))
-    {
-        return 0;
-    }
-    return LZ4_compress_default(
-        reinterpret_cast<const char*>(pSource),
-        reinterpret_cast<char*>(pDestination),
-        static_cast<int>(sourceLength),
-        static_cast<int>(destinationLength));
+    g_pSource = pSource;
+    g_sourceRemaining = numTiles * 32;
+    g_pDestination = pDestination;
+    g_destinationRemaining = destinationLength;
+    smallz4::lz4(getBytes, sendBytes);
+    // Remove framing
+    constexpr auto headerSize = 7 + 4; // Header + block size
+    constexpr auto footerSize = 4; // Empty block terminator
+    const auto size = destinationLength - g_destinationRemaining - headerSize - footerSize;
+    memcpy_s(pDestination, destinationLength, pDestination + headerSize, size);
+    return static_cast<int>(size);
 }
 
 extern "C" __declspec(dllexport) int compressTilemap(
@@ -46,14 +71,15 @@ extern "C" __declspec(dllexport) int compressTilemap(
     uint8_t* pDestination,
     const uint32_t destinationLength)
 {
-    const auto sourceLen = width * height * 2;
-    if (destinationLength < static_cast<uint32_t>(LZ4_compressBound(sourceLen)))
-    {
-        return 0;
-    }
-    return LZ4_compress_default(
-        reinterpret_cast<const char*>(pSource),
-        reinterpret_cast<char*>(pDestination),
-        static_cast<int>(sourceLen),
-        static_cast<int>(destinationLength));
+    g_pSource = pSource;
+    g_sourceRemaining = width * height * 2;
+    g_pDestination = pDestination;
+    g_destinationRemaining = destinationLength;
+    smallz4::lz4(getBytes, sendBytes);
+    // Remove framing
+    constexpr auto headerSize = 7 + 4; // Header + block size
+    constexpr auto footerSize = 4; // Empty block terminator
+    const auto size = destinationLength - g_destinationRemaining - headerSize - footerSize;
+    memcpy_s(pDestination, destinationLength, pDestination + headerSize, size);
+    return static_cast<int>(size);
 }
diff --git a/compressors/gfxcomp_lz4.vcxproj b/compressors/gfxcomp_lz4.vcxproj
@@ -73,7 +73,6 @@
   </ItemDefinitionGroup>
   <ItemGroup>
     <ClCompile Include="gfxcomp_lz4.cpp" />
-    <ClCompile Include="lz4\lz4.c" />
   </ItemGroup>
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
   <ImportGroup Label="ExtensionTargets">

diff --git a/compressors/gfxcomp_lz4.vcxproj.filters b/compressors/gfxcomp_lz4.vcxproj.filters
@@ -15,9 +15,6 @@
     </Filter>
   </ItemGroup>
   <ItemGroup>
-    <ClCompile Include="lz4\lz4.c">
-      <Filter>Source Files</Filter>
-    </ClCompile>
     <ClCompile Include="gfxcomp_lz4.cpp">
       <Filter>Source Files</Filter>
     </ClCompile>