HPCC-33113 Ensure empty compressed part size is correct

When writing a logical file that is wider than Thor with, Thor pads the file at publish time with empty parts. If compressed it creates an empty compressed file, which will typically be non-zero in size (e.g. contains compressed header). The logical file part meta data did not relect this correct physical size. Consequently, as a result of the check added in HPCC-33064, reading these files was provoking this error check. Signed-off-by: Jake Smith <[email protected]>
hpcc-systems · Dec 11, 2024 · 7ba4780 · 7ba4780
1 parent 761dda9
commit 7ba4780
Showing 1 changed file with 9 additions and 2 deletions.
diff --git a/thorlcr/activities/thdiskbase.cpp b/thorlcr/activities/thdiskbase.cpp
@@ -324,6 +324,9 @@ void CWriteMasterBase::publish()
                 compMethod = COMPRESS_METHOD_LZ4HC;
             bool blockCompressed;
             bool compressed = fileDesc->isCompressed(&blockCompressed);
+
+            // NB: it would be far preferable to avoid this and have the file reference a group with the correct number of parts
+            // Possibly could use subgroup syntax: 'data[1..n]'
             for (unsigned clusterIdx=0; clusterIdx<fileDesc->numClusters(); clusterIdx++)
             {
                 StringBuffer clusterName;
@@ -336,6 +339,7 @@ void CWriteMasterBase::publish()
                         p += queryJob().querySlaves();
                     IPartDescriptor *partDesc = fileDesc->queryPart(p);
                     CDateTime createTime, modifiedTime;
+                    offset_t compSize = 0;
                     for (unsigned c=0; c<partDesc->numCopies(); c++)
                     {
                         RemoteFilename rfn;
@@ -347,15 +351,18 @@ void CWriteMasterBase::publish()
                             ensureDirectoryForFile(path.str());
                             OwnedIFile iFile = createIFile(path.str());
                             Owned<IFileIO> iFileIO;
-                            if (compressed)
+                            if (compressed) // NB: this would not be necessary if all builds have the changes in HPCC-32651
                                 iFileIO.setown(createCompressedFileWriter(iFile, recordSize, false, true, NULL, compMethod));
                             else
                                 iFileIO.setown(iFile->open(IFOcreate));
                             dbgassertex(iFileIO.get());
                             iFileIO.clear();
                             // ensure copies have matching datestamps, as they would do normally (backupnode expects it)
                             if (0 == c)
+                            {
                                 iFile->getTime(&createTime, &modifiedTime, NULL);
+                                compSize = iFile->size();
+                            }
                             else
                                 iFile->setTime(&createTime, &modifiedTime, NULL);
                         }
@@ -376,7 +383,7 @@ void CWriteMasterBase::publish()
                     props.setPropInt64("@recordCount", 0);
                     props.setPropInt64("@size", 0);
                     if (compressed)
-                        props.setPropInt64("@compressedSize", 0);
+                        props.setPropInt64("@compressedSize", compSize);
                     p++;
                 }
             }