Merge pull request #18600 from jackdelv/allowFieldTranslation

HPCC-31496 Allow field translation that only removes fields Reviewed-by: Gavin Halliday <[email protected]> Merged-by: Gavin Halliday <[email protected]>
hpcc-systems · May 2, 2024 · 1cbacfd · 1cbacfd
2 parents 54f70aa + cd63ef9
commit 1cbacfd
Show file tree

Hide file tree

Showing 7 changed files with 36 additions and 1 deletion.
diff --git a/common/thorhelper/thorcommon.cpp b/common/thorhelper/thorcommon.cpp
@@ -2148,6 +2148,9 @@ static bool getTranslators(Owned<const IDynamicTransform> &translator, Owned<con
             if (!translator->canTranslate())
                 throw MakeStringException(0, "Untranslatable record layout mismatch detected for file %s", tracing);
 
+            if (mode == RecordTranslationMode::PayloadRemoveOnly && translator->hasNewFields())
+                throw MakeStringException(0, "Translatable file layout mismatch reading file %s but translation disabled when expected fields are missing from source.", tracing);
+
             if (translator->needsTranslate())
             {
                 if (keyedTranslator && (sourceFormat != expectedFormat))

diff --git a/common/thorhelper/thorread.cpp b/common/thorhelper/thorread.cpp
@@ -176,6 +176,9 @@ void DiskReadMapping::ensureTranslators() const
         if (!translator->canTranslate())
             throw MakeStringException(0, "Untranslatable record layout mismatch detected for file %s", filename);
 
+        if (mode == RecordTranslationMode::PayloadRemoveOnly && translator->hasNewFields())
+            throw MakeStringException(0, "Translatable file layout mismatch reading file %s but translation disabled when expected fields are missing from source.", filename);
+
         if (translator->needsTranslate())
         {
             if (sourceMeta != expectedMeta)

diff --git a/ecl/hthor/hthorkey.cpp b/ecl/hthor/hthorkey.cpp
@@ -700,6 +700,8 @@ const IDynamicTransform * CHThorIndexReadActivityBase::getLayoutTranslator(IDist
     Owned<const IDynamicTransform> payloadTranslator =  createRecordTranslator(projectedFormat->queryRecordAccessor(true), actualFormat->queryRecordAccessor(true));
     if (!payloadTranslator->canTranslate())
         throw MakeStringException(0, "Untranslatable key layout mismatch reading index %s", f->queryLogicalName());
+    if (getLayoutTranslationMode() == RecordTranslationMode::PayloadRemoveOnly && payloadTranslator->hasNewFields())
+        throw MakeStringException(0, "Translatable file layout mismatch reading file %s but translation disabled when expected fields are missing from source.", f->queryLogicalName());
     if (payloadTranslator->needsTranslate())
         return payloadTranslator.getClear();
     return nullptr;
@@ -715,6 +717,8 @@ void CHThorIndexReadActivityBase::verifyIndex(IKeyIndex * idx)
         {
             if (!layoutTrans->canTranslate())
                 throw MakeStringException(0, "Untranslatable key layout mismatch reading index %s", df->queryLogicalName());
+            if (getLayoutTranslationMode() == RecordTranslationMode::PayloadRemoveOnly && layoutTrans->hasNewFields())
+                throw MakeStringException(0, "Translatable file layout mismatch reading file %s but translation disabled when expected fields are missing from source.", df->queryLogicalName());
         }
         else
         {
@@ -2462,6 +2466,8 @@ class CHThorFlatFetchActivity : public CHThorFetchActivityBase, public IFlatFetc
                     translator->describe();
                     if (translator->canTranslate())
                     {
+                        if (getLayoutTranslationMode()==RecordTranslationMode::PayloadRemoveOnly && translator->hasNewFields())
+                            throw MakeStringException(0, "Translatable file layout mismatch reading file %s but translation disabled when expected fields are missing from source.", f->queryLogicalName());
                         if (getLayoutTranslationMode()==RecordTranslationMode::None)
                             throw MakeStringException(0, "Translatable file layout mismatch reading file %s but translation disabled", f->queryLogicalName());
                         VStringBuffer msg("Record layout translation required for %s", f->queryLogicalName());
@@ -4102,6 +4108,8 @@ class CHThorKeyedJoinActivity  : public CHThorThreadedActivityBase, implements I
                 throw MakeStringException(0, "Untranslatable key layout mismatch reading index %s", f->queryLogicalName());
             if (payloadTranslator->keyedTranslated())
                 throw MakeStringException(0, "Untranslatable key layout mismatch reading index %s - keyed fields do not match", f->queryLogicalName());
+            if (getLayoutTranslationMode()==RecordTranslationMode::PayloadRemoveOnly && payloadTranslator->hasNewFields())
+                throw MakeStringException(0, "Translatable file layout mismatch reading file %s but translation disabled when expected fields are missing from source.", f->queryLogicalName());
             if (getLayoutTranslationMode()==RecordTranslationMode::None)
                 throw MakeStringException(0, "Translatable file layout mismatch reading file %s but translation disabled", f->queryLogicalName());
             VStringBuffer msg("Record layout translation required for %s", f->queryLogicalName());
@@ -4119,6 +4127,8 @@ class CHThorKeyedJoinActivity  : public CHThorThreadedActivityBase, implements I
             {
                 if (!trans->canTranslate())
                     throw MakeStringException(0, "Untranslatable key layout mismatch reading index %s", f->queryLogicalName());
+                if (getLayoutTranslationMode() == RecordTranslationMode::PayloadRemoveOnly && trans->hasNewFields())
+                    throw MakeStringException(0, "Translatable file layout mismatch reading file %s but translation disabled when expected fields are missing from source.", f->queryLogicalName());
             }
             else
             {
@@ -4149,6 +4159,8 @@ class CHThorKeyedJoinActivity  : public CHThorThreadedActivityBase, implements I
                     translator.setown(createRecordTranslator(helper.queryProjectedDiskRecordSize()->queryRecordAccessor(true), actualDiskMeta->queryRecordAccessor(true)));
                     if (translator->canTranslate())
                     {
+                        if (getLayoutTranslationMode()==RecordTranslationMode::PayloadRemoveOnly && translator->hasNewFields())
+                            throw MakeStringException(0, "Translatable file layout mismatch reading file %s but translation disabled when expected fields are missing from source.", f->queryLogicalName());
                         if (getLayoutTranslationMode()==RecordTranslationMode::None)
                             throw MakeStringException(0, "Translatable file layout mismatch reading file %s but translation disabled", f->queryLogicalName());
                         VStringBuffer msg("Record layout translation required for %s", f->queryLogicalName());

diff --git a/roxie/ccd/ccdfile.cpp b/roxie/ccd/ccdfile.cpp
@@ -3013,6 +3013,8 @@ class CResolvedFile : implements IResolvedFileCreator, implements ISafeSDSSubscr
                             }
                             if (!translator || !translator->canTranslate())
                                 throw MakeStringException(ROXIE_MISMATCH, "Untranslatable record layout mismatch detected for file %s", subname);
+                            else if (mode == RecordTranslationMode::PayloadRemoveOnly && translator->hasNewFields())
+                                throw MakeStringException(0, "Translatable file layout mismatch reading file %s but translation disabled when expected fields are missing from source.", subname);
                             else if (translator->needsTranslate())
                             {
                                 if (fileMode==FileFormatMode::index && translator->keyedTranslated())

diff --git a/rtl/eclrtl/rtldynfield.cpp b/rtl/eclrtl/rtldynfield.cpp
@@ -38,6 +38,8 @@ extern ECLRTL_API RecordTranslationMode getTranslationMode(const char *val, bool
 {
     if (isEmptyString(val) || strToBool(val) || strieq(val, "payload"))
         return RecordTranslationMode::Payload;
+    else if (strieq(val, "payloadRemoveOnly"))
+        return RecordTranslationMode::PayloadRemoveOnly;
     else if (strieq(val, "alwaysDisk") || strieq(val, "disk"))
     {
         if (!isLocal)
@@ -60,6 +62,7 @@ extern ECLRTL_API const char *getTranslationModeText(RecordTranslationMode val)
     case RecordTranslationMode::AlwaysDisk: return "alwaysDisk";
     case RecordTranslationMode::AlwaysECL: return "alwaysECL";
     case RecordTranslationMode::Payload: return "payload";
+    case RecordTranslationMode::PayloadRemoveOnly: return "payloadRemoveOnly";
     case RecordTranslationMode::None: return "off";
     }
     throwUnexpected();
@@ -1085,6 +1088,10 @@ class GeneralRecordTranslator : public CInterfaceOf<IDynamicTransform>
     {
         return (matchFlags & match_keychange) != 0;
     }
+    virtual bool hasNewFields() const override
+    {
+        return (matchFlags & match_none) != 0;
+    }
 private:
     void doDescribe(unsigned indent) const
     {
@@ -1948,6 +1955,10 @@ class CloneVirtualRecordTranslator : public CInterfaceOf<IDynamicTransform>
     {
         return false;
     }
+    virtual bool hasNewFields() const override
+    {
+        return false;
+    }
 private:
     void doDescribe(unsigned indent) const
     {

diff --git a/rtl/eclrtl/rtldynfield.hpp b/rtl/eclrtl/rtldynfield.hpp
@@ -110,7 +110,8 @@ enum class RecordTranslationMode : byte
     Payload = 2,    // Translate all fields in datasets, and only payload fields in indexes
     AlwaysDisk = 3, // Always translate - even if wouldn't normally (e.g. csv/xml source read as binary), or crcs happen to match
     AlwaysECL = 4,  // Ignore the published format - can make sense to force no translation e.g. when field names have changed
-    Unspecified = 5
+    PayloadRemoveOnly = 5, // Allow fields to be removed from the incoming dataset, but not allow fields to be missing
+    Unspecified = 6
 };  // AlwaysDisk and AlwaysECL are for testing purposes only, and can only be set per file (not globally)
 
 extern ECLRTL_API RecordTranslationMode getTranslationMode(const char *modeStr, bool isLocal);
@@ -139,6 +140,7 @@ interface IDynamicTransform : public IInterface
     virtual bool needsTranslate() const = 0;
     virtual bool keyedTranslated() const = 0;
     virtual bool needsNonVirtualTranslate() const = 0;
+    virtual bool hasNewFields() const = 0;
 };
 
 interface IKeyTranslator : public IInterface

diff --git a/thorlcr/slave/slavmain.cpp b/thorlcr/slave/slavmain.cpp
@@ -353,6 +353,8 @@ class CKJService : public CSimpleInterfaceOf<IKJService>, implements IThreaded,
                     translator.setown(createRecordTranslator(projectedFormat->queryRecordAccessor(true), publishedFormat->queryRecordAccessor(true)));
                     if (!translator->canTranslate())
                         throw MakeStringException(0, "Untranslatable record layout mismatch detected for: %s", tracing);
+                    if (RecordTranslationMode::PayloadRemoveOnly == translationMode && translator->hasNewFields())
+                        throw MakeStringException(0, "Translatable file layout mismatch reading file %s but translation disabled when expected fields are missing from source.", tracing);
                 }
                 DBGLOG("Record layout translator created for %s", tracing);
                 translator->describe();