HPCC-29917 Refactor compressToBuffer to support different compression…

… methods Ensure back compatibility for LZW LE compression Signed-off-by: Richard Chapman <[email protected]>
hpcc-systems · Dec 4, 2023 · 2e6ca90 · 2e6ca90
1 parent 1a2e915
commit 2e6ca90
Show file tree

Hide file tree

Showing 2 changed files with 15 additions and 5 deletions.
diff --git a/system/jlib/jlzw.cpp b/system/jlib/jlzw.cpp
@@ -769,11 +769,11 @@ void compressToBuffer(MemoryBuffer & out, size32_t len, const void * src, Compre
             throw makeStringException(0, s.str());
         }
         unsigned originalLength = out.length();
-        // For back-compatibility, we always store COMPRESS_METHOD_LZW as 1 as earlier versions stored a boolean here
+        // For back-compatibility, we always store COMPRESS_METHOD_LZW_LITTLE_ENDIAN as 1 as earlier versions stored a boolean here
         // rather than an enum
         // This means that compressToBuffer/decompressToBuffer cannot bs used for rowdiff compression - this is not likely to be an issue
         // Alternative would be a separate enum for compressToBuffer formats, but that seems more likely to cause confusion
-        out.append((byte) (method == COMPRESS_METHOD_LZW ? COMPRESS_METHOD_LZWLEGACY : method));
+        out.append((byte) (method == COMPRESS_METHOD_LZW_LITTLE_ENDIAN ? COMPRESS_METHOD_LZWLEGACY : method));
         out.append((size32_t)0);
         size32_t newSize = len * 4 / 5; // Copy if compresses less than 80% ...
         Owned<ICompressor> compressor = handler->getCompressor(options);
@@ -814,7 +814,7 @@ void decompressToBuffer(MemoryBuffer & out, MemoryBuffer & in, const char *optio
     else
     {
         if (method==COMPRESS_METHOD_LZWLEGACY)
-            method = COMPRESS_METHOD_LZW;    // Back compatibilty
+            method = COMPRESS_METHOD_LZW_LITTLE_ENDIAN;    // Back compatibilty
         ICompressHandler *handler = queryCompressHandler(method);
         if (!handler)
         {
@@ -3023,6 +3023,15 @@ MODULE_INIT(INIT_PRIORITY_STANDARD)
         virtual ICompressor *getCompressor(const char *options) { return createLZWCompressor(true); }
         virtual IExpander *getExpander(const char *options) { return createLZWExpander(true); }
     };
+    class CLZWLECompressHandler : public CCompressHandlerBase
+    {
+    public:
+        virtual const char *queryType() const { return "LZWLE"; }
+        virtual CompressionMethod queryMethod() const { return COMPRESS_METHOD_LZW_LITTLE_ENDIAN; }
+        virtual ICompressor *getCompressor(const char *options) { return createLZWCompressor(false); }
+        virtual IExpander *getExpander(const char *options) { return createLZWExpander(false); }
+    };
+    addCompressorHandler(new CLZWLECompressHandler());
     addCompressorHandler(new CLZWCompressHandler());
     addCompressorHandler(new CAESCompressHandler());
     addCompressorHandler(new CDiffCompressHandler());

diff --git a/system/jlib/jlzw.hpp b/system/jlib/jlzw.hpp
@@ -35,11 +35,12 @@ enum CompressionMethod
     COMPRESS_METHOD_LZ4,
     COMPRESS_METHOD_LZ4HC,
     COMPRESS_METHOD_RANDROW,
+    COMPRESS_METHOD_LZW_LITTLE_ENDIAN,
     COMPRESS_METHOD_LAST,
 
 
     COMPRESS_METHOD_AES = 0x80,
-    COMPRESS_METHOD_LZWLEGACY = 1,  // Matches value of boolean 'true' used to indicate LZW compression by legacy compressToBuffer
+    COMPRESS_METHOD_LZWLEGACY = 1,  // Matches value of boolean 'true' used to indicate LZW little endian compression by legacy compressToBuffer
 };
 
 
@@ -120,7 +121,7 @@ extern jlib_decl IRandRowExpander *createRandRDiffExpander(); // NB only support
 
 
 // Helper functions to make it easy to compress/decompress to memorybuffers.
-extern jlib_decl void compressToBuffer(MemoryBuffer & out, size32_t len, const void * src, CompressionMethod method=COMPRESS_METHOD_LZW, const char *options=nullptr);
+extern jlib_decl void compressToBuffer(MemoryBuffer & out, size32_t len, const void * src, CompressionMethod method=COMPRESS_METHOD_LZW_LITTLE_ENDIAN, const char *options=nullptr);
 extern jlib_decl void decompressToBuffer(MemoryBuffer & out, MemoryBuffer & in, const char *options=nullptr);