HPCC-29917 Refactor compressToBuffer to support different compression…

… methods Also add some tests, and fix some issues that the tests showed up with small input sizes. Signed-off-by: Richard Chapman <[email protected]>
hpcc-systems · Dec 4, 2023 · 1a2e915 · 1a2e915
1 parent b330baf
commit 1a2e915
Show file tree

Hide file tree

Showing 7 changed files with 253 additions and 127 deletions.
diff --git a/common/dllserver/thorplugin.cpp b/common/dllserver/thorplugin.cpp
@@ -679,10 +679,7 @@ extern DLLSERVER_API bool decompressResource(size32_t len, const void *data, Str
 extern DLLSERVER_API void appendResource(MemoryBuffer & mb, size32_t len, const void *data, bool compress)
 {
     mb.append((byte)0x80).append(resourceHeaderVersion);
-    if (compress)
-        compressToBuffer(mb, len, data);
-    else
-        appendToBuffer(mb, len, data);
+    compressToBuffer(mb, len, data, compress ? COMPRESS_METHOD_LZW : COMPRESS_METHOD_NONE);
 }
 
 extern DLLSERVER_API void compressResource(MemoryBuffer & compressed, size32_t len, const void *data)

diff --git a/system/jlib/jfcmp.hpp b/system/jlib/jfcmp.hpp
@@ -72,8 +72,6 @@ class jlib_decl CFcmpCompressor : public CSimpleInterfaceOf<ICompressor>
 
     virtual void open(void *buf,size32_t max) override
     {
-        if (max<1024)
-            throw MakeStringException(-1,"CFcmpCompressor::open - block size (%d) not large enough", max);
         wrmax = max;
         originalMax = max;
         if (buf)
@@ -103,8 +101,6 @@ class jlib_decl CFcmpCompressor : public CSimpleInterfaceOf<ICompressor>
     {
         if (!initialSize)
             initialSize = FCMP_BUFFER_SIZE; // 1MB
-        if (initialSize<1024)
-            throw MakeStringException(-1,"CFcmpCompressor::open - block size (%d) not large enough", initialSize);
         wrmax = initialSize;
         if (bufalloc)
         {

diff --git a/system/jlib/jlz4.cpp b/system/jlib/jlz4.cpp
@@ -34,18 +34,17 @@ class CLZ4Compressor final : public CFcmpCompressor
 protected:
     virtual void setinmax() override
     {
-        inmax = blksz-outlen-sizeof(size32_t);
-        if (inmax<256)
+        if (blksz <= outlen+sizeof(size32_t))
             trailing = true;    // too small to bother compressing
         else
         {
             trailing = false;
+            inmax = blksz-outlen-sizeof(size32_t);
             size32_t slack = LZ4_COMPRESSBOUND(inmax) - inmax;
-            int inmax2 = inmax - (slack + sizeof(size32_t));
-            if (inmax2<256)
+            if (inmax <= (slack + sizeof(size32_t)))
                 trailing = true;
             else
-                inmax = inmax2;
+                inmax = inmax - (slack + sizeof(size32_t));
         }
     }
 
@@ -73,12 +72,6 @@ class CLZ4Compressor final : public CFcmpCompressor
         if (toflush == 0)
             return;
 
-        if (toflush < 256)
-        {
-            trailing = true;
-            return;
-        }
-
         size32_t outSzRequired = outlen+sizeof(size32_t)*2+LZ4_COMPRESSBOUND(toflush);
         if (!dynamicOutSz)
             assertex(outSzRequired<=blksz);