diff --git a/common/thorhelper/thorcommon.cpp b/common/thorhelper/thorcommon.cpp index 6c9668cfcb8..38f29328615 100644 --- a/common/thorhelper/thorcommon.cpp +++ b/common/thorhelper/thorcommon.cpp @@ -2148,6 +2148,9 @@ static bool getTranslators(Owned &translator, OwnedcanTranslate()) throw MakeStringException(0, "Untranslatable record layout mismatch detected for file %s", tracing); + if (mode == RecordTranslationMode::PayloadRemoveOnly && translator->hasNewFields()) + throw MakeStringException(0, "Translatable file layout mismatch reading file %s but translation disabled when expected fields are missing from source.", tracing); + if (translator->needsTranslate()) { if (keyedTranslator && (sourceFormat != expectedFormat)) diff --git a/common/thorhelper/thorread.cpp b/common/thorhelper/thorread.cpp index 591d5d553cc..cc8386ffd21 100644 --- a/common/thorhelper/thorread.cpp +++ b/common/thorhelper/thorread.cpp @@ -176,6 +176,9 @@ void DiskReadMapping::ensureTranslators() const if (!translator->canTranslate()) throw MakeStringException(0, "Untranslatable record layout mismatch detected for file %s", filename); + if (mode == RecordTranslationMode::PayloadRemoveOnly && translator->hasNewFields()) + throw MakeStringException(0, "Translatable file layout mismatch reading file %s but translation disabled when expected fields are missing from source.", filename); + if (translator->needsTranslate()) { if (sourceMeta != expectedMeta) diff --git a/ecl/hthor/hthorkey.cpp b/ecl/hthor/hthorkey.cpp index 7fb6c7952ef..1c9094a3a63 100644 --- a/ecl/hthor/hthorkey.cpp +++ b/ecl/hthor/hthorkey.cpp @@ -700,6 +700,8 @@ const IDynamicTransform * CHThorIndexReadActivityBase::getLayoutTranslator(IDist Owned payloadTranslator = createRecordTranslator(projectedFormat->queryRecordAccessor(true), actualFormat->queryRecordAccessor(true)); if (!payloadTranslator->canTranslate()) throw MakeStringException(0, "Untranslatable key layout mismatch reading index %s", f->queryLogicalName()); + if (getLayoutTranslationMode() == RecordTranslationMode::PayloadRemoveOnly && payloadTranslator->hasNewFields()) + throw MakeStringException(0, "Translatable file layout mismatch reading file %s but translation disabled when expected fields are missing from source.", f->queryLogicalName()); if (payloadTranslator->needsTranslate()) return payloadTranslator.getClear(); return nullptr; @@ -715,6 +717,8 @@ void CHThorIndexReadActivityBase::verifyIndex(IKeyIndex * idx) { if (!layoutTrans->canTranslate()) throw MakeStringException(0, "Untranslatable key layout mismatch reading index %s", df->queryLogicalName()); + if (getLayoutTranslationMode() == RecordTranslationMode::PayloadRemoveOnly && layoutTrans->hasNewFields()) + throw MakeStringException(0, "Translatable file layout mismatch reading file %s but translation disabled when expected fields are missing from source.", df->queryLogicalName()); } else { @@ -2462,6 +2466,8 @@ class CHThorFlatFetchActivity : public CHThorFetchActivityBase, public IFlatFetc translator->describe(); if (translator->canTranslate()) { + if (getLayoutTranslationMode()==RecordTranslationMode::PayloadRemoveOnly && translator->hasNewFields()) + throw MakeStringException(0, "Translatable file layout mismatch reading file %s but translation disabled when expected fields are missing from source.", f->queryLogicalName()); if (getLayoutTranslationMode()==RecordTranslationMode::None) throw MakeStringException(0, "Translatable file layout mismatch reading file %s but translation disabled", f->queryLogicalName()); VStringBuffer msg("Record layout translation required for %s", f->queryLogicalName()); @@ -4102,6 +4108,8 @@ class CHThorKeyedJoinActivity : public CHThorThreadedActivityBase, implements I throw MakeStringException(0, "Untranslatable key layout mismatch reading index %s", f->queryLogicalName()); if (payloadTranslator->keyedTranslated()) throw MakeStringException(0, "Untranslatable key layout mismatch reading index %s - keyed fields do not match", f->queryLogicalName()); + if (getLayoutTranslationMode()==RecordTranslationMode::PayloadRemoveOnly && payloadTranslator->hasNewFields()) + throw MakeStringException(0, "Translatable file layout mismatch reading file %s but translation disabled when expected fields are missing from source.", f->queryLogicalName()); if (getLayoutTranslationMode()==RecordTranslationMode::None) throw MakeStringException(0, "Translatable file layout mismatch reading file %s but translation disabled", f->queryLogicalName()); VStringBuffer msg("Record layout translation required for %s", f->queryLogicalName()); @@ -4119,6 +4127,8 @@ class CHThorKeyedJoinActivity : public CHThorThreadedActivityBase, implements I { if (!trans->canTranslate()) throw MakeStringException(0, "Untranslatable key layout mismatch reading index %s", f->queryLogicalName()); + if (getLayoutTranslationMode() == RecordTranslationMode::PayloadRemoveOnly && trans->hasNewFields()) + throw MakeStringException(0, "Translatable file layout mismatch reading file %s but translation disabled when expected fields are missing from source.", f->queryLogicalName()); } else { @@ -4149,6 +4159,8 @@ class CHThorKeyedJoinActivity : public CHThorThreadedActivityBase, implements I translator.setown(createRecordTranslator(helper.queryProjectedDiskRecordSize()->queryRecordAccessor(true), actualDiskMeta->queryRecordAccessor(true))); if (translator->canTranslate()) { + if (getLayoutTranslationMode()==RecordTranslationMode::PayloadRemoveOnly && translator->hasNewFields()) + throw MakeStringException(0, "Translatable file layout mismatch reading file %s but translation disabled when expected fields are missing from source.", f->queryLogicalName()); if (getLayoutTranslationMode()==RecordTranslationMode::None) throw MakeStringException(0, "Translatable file layout mismatch reading file %s but translation disabled", f->queryLogicalName()); VStringBuffer msg("Record layout translation required for %s", f->queryLogicalName()); diff --git a/roxie/ccd/ccdfile.cpp b/roxie/ccd/ccdfile.cpp index e1e7cb3f1c5..a08ee799ceb 100644 --- a/roxie/ccd/ccdfile.cpp +++ b/roxie/ccd/ccdfile.cpp @@ -3013,6 +3013,8 @@ class CResolvedFile : implements IResolvedFileCreator, implements ISafeSDSSubscr } if (!translator || !translator->canTranslate()) throw MakeStringException(ROXIE_MISMATCH, "Untranslatable record layout mismatch detected for file %s", subname); + else if (mode == RecordTranslationMode::PayloadRemoveOnly && translator->hasNewFields()) + throw MakeStringException(0, "Translatable file layout mismatch reading file %s but translation disabled when expected fields are missing from source.", subname); else if (translator->needsTranslate()) { if (fileMode==FileFormatMode::index && translator->keyedTranslated()) diff --git a/rtl/eclrtl/rtldynfield.cpp b/rtl/eclrtl/rtldynfield.cpp index 14f60d4ca22..31d8ad9f02c 100644 --- a/rtl/eclrtl/rtldynfield.cpp +++ b/rtl/eclrtl/rtldynfield.cpp @@ -38,6 +38,8 @@ extern ECLRTL_API RecordTranslationMode getTranslationMode(const char *val, bool { if (isEmptyString(val) || strToBool(val) || strieq(val, "payload")) return RecordTranslationMode::Payload; + else if (strieq(val, "payloadRemoveOnly")) + return RecordTranslationMode::PayloadRemoveOnly; else if (strieq(val, "alwaysDisk") || strieq(val, "disk")) { if (!isLocal) @@ -60,6 +62,7 @@ extern ECLRTL_API const char *getTranslationModeText(RecordTranslationMode val) case RecordTranslationMode::AlwaysDisk: return "alwaysDisk"; case RecordTranslationMode::AlwaysECL: return "alwaysECL"; case RecordTranslationMode::Payload: return "payload"; + case RecordTranslationMode::PayloadRemoveOnly: return "payloadRemoveOnly"; case RecordTranslationMode::None: return "off"; } throwUnexpected(); @@ -1085,6 +1088,10 @@ class GeneralRecordTranslator : public CInterfaceOf { return (matchFlags & match_keychange) != 0; } + virtual bool hasNewFields() const override + { + return (matchFlags & match_none) != 0; + } private: void doDescribe(unsigned indent) const { @@ -1948,6 +1955,10 @@ class CloneVirtualRecordTranslator : public CInterfaceOf { return false; } + virtual bool hasNewFields() const override + { + return false; + } private: void doDescribe(unsigned indent) const { diff --git a/rtl/eclrtl/rtldynfield.hpp b/rtl/eclrtl/rtldynfield.hpp index 2e7ae6f1df6..d9ddae13068 100644 --- a/rtl/eclrtl/rtldynfield.hpp +++ b/rtl/eclrtl/rtldynfield.hpp @@ -110,7 +110,8 @@ enum class RecordTranslationMode : byte Payload = 2, // Translate all fields in datasets, and only payload fields in indexes AlwaysDisk = 3, // Always translate - even if wouldn't normally (e.g. csv/xml source read as binary), or crcs happen to match AlwaysECL = 4, // Ignore the published format - can make sense to force no translation e.g. when field names have changed - Unspecified = 5 + PayloadRemoveOnly = 5, // Allow fields to be removed from the incoming dataset, but not allow fields to be missing + Unspecified = 6 }; // AlwaysDisk and AlwaysECL are for testing purposes only, and can only be set per file (not globally) extern ECLRTL_API RecordTranslationMode getTranslationMode(const char *modeStr, bool isLocal); @@ -139,6 +140,7 @@ interface IDynamicTransform : public IInterface virtual bool needsTranslate() const = 0; virtual bool keyedTranslated() const = 0; virtual bool needsNonVirtualTranslate() const = 0; + virtual bool hasNewFields() const = 0; }; interface IKeyTranslator : public IInterface diff --git a/thorlcr/slave/slavmain.cpp b/thorlcr/slave/slavmain.cpp index 4e3943e13a1..037d5e8fc11 100644 --- a/thorlcr/slave/slavmain.cpp +++ b/thorlcr/slave/slavmain.cpp @@ -353,6 +353,8 @@ class CKJService : public CSimpleInterfaceOf, implements IThreaded, translator.setown(createRecordTranslator(projectedFormat->queryRecordAccessor(true), publishedFormat->queryRecordAccessor(true))); if (!translator->canTranslate()) throw MakeStringException(0, "Untranslatable record layout mismatch detected for: %s", tracing); + if (RecordTranslationMode::PayloadRemoveOnly == translationMode && translator->hasNewFields()) + throw MakeStringException(0, "Translatable file layout mismatch reading file %s but translation disabled when expected fields are missing from source.", tracing); } DBGLOG("Record layout translator created for %s", tracing); translator->describe();