diff --git a/common/thorhelper/engineerr.hpp b/common/thorhelper/engineerr.hpp new file mode 100644 index 00000000000..20e3cba9b79 --- /dev/null +++ b/common/thorhelper/engineerr.hpp @@ -0,0 +1,22 @@ +/*############################################################################## + HPCC SYSTEMS software Copyright (C) 2023 HPCC Systems®. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +############################################################################## */ + +#pragma once + +#include "errorlist.h" + +#define ENGINEERR_EXTEND_CLUSTER_WRITE ENGINE_ERROR_START +#define ENGINEERR_MIXED_COMPRESSED_WRITE ENGINE_ERROR_START+1 +#define ENGINEERR_FILE_TYPE_MISMATCH ENGINE_ERROR_START+2 +#define ENGINEERR_MISSING_OPTIONAL_FILE ENGINE_ERROR_START+3 +#define ENGINEERR_FILE_UPTODATE ENGINE_ERROR_START+4 diff --git a/ecl/hql/hqlatoms.cpp b/ecl/hql/hqlatoms.cpp index af6cded1200..6e798b7298b 100644 --- a/ecl/hql/hqlatoms.cpp +++ b/ecl/hql/hqlatoms.cpp @@ -302,6 +302,7 @@ IAtom * newSetAtom; IAtom * _nlpParse_Atom; IAtom * noBoundCheckAtom; IAtom * noCaseAtom; +IAtom * noCompressAtom; IAtom * noConstAtom; IAtom * _noDuplicate_Atom; IAtom * nofoldAtom; @@ -782,6 +783,7 @@ MODULE_INIT(INIT_PRIORITY_HQLATOM) MAKESYSATOM(nlpParse); MAKEATOM(noBoundCheck); MAKEATOM(noCase); + MAKEATOM(noCompress); MAKEATOM(noConst); MAKESYSATOM(noDuplicate); MAKEATOM(nofold); diff --git a/ecl/hql/hqlatoms.hpp b/ecl/hql/hqlatoms.hpp index b3937ee6539..70eb07d144b 100644 --- a/ecl/hql/hqlatoms.hpp +++ b/ecl/hql/hqlatoms.hpp @@ -306,6 +306,7 @@ extern HQL_API IAtom * newSetAtom; extern HQL_API IAtom * _nlpParse_Atom; extern HQL_API IAtom * noBoundCheckAtom; extern HQL_API IAtom * noCaseAtom; +extern HQL_API IAtom * noCompressAtom; extern HQL_API IAtom * noConstAtom; extern HQL_API IAtom * _noDuplicate_Atom; extern HQL_API IAtom * nofoldAtom; diff --git a/ecl/hql/hqlgram.y b/ecl/hql/hqlgram.y index 4caa7375c6a..75eda3b6c99 100644 --- a/ecl/hql/hqlgram.y +++ b/ecl/hql/hqlgram.y @@ -472,6 +472,7 @@ static void eclsyntaxerror(HqlGram * parser, const char * s, short yystate, int TOK_TRUE TYPE TYPEOF + UNCOMPRESSED UNICODEORDER UNGROUP UNLIKELY @@ -3490,6 +3491,21 @@ outputFlags { $$.setExpr(createComma($1.getExpr(), $3.getExpr())); $$.setPosition($1); } ; +compressionOptions + : COMPRESSED { + $$.setExpr(createAttribute(compressedAtom)); + $$.setPosition($1); + } + | UNCOMPRESSED + { + $$.setExpr(createAttribute(noCompressAtom)); + $$.setPosition($1); + } + | __COMPRESSED__ { + $$.setExpr(createAttribute(__compressed__Atom)); + $$.setPosition($1); + } + outputFlag : EXTEND { $$.setExpr(createAttribute(extendAtom)); @@ -3505,14 +3521,7 @@ outputFlag $$.setExpr(createExprAttribute(csvAtom, args)); $$.setPosition($1); } - | COMPRESSED { - $$.setExpr(createAttribute(compressedAtom)); - $$.setPosition($1); - } - | __COMPRESSED__ { - $$.setExpr(createAttribute(__compressed__Atom)); - $$.setPosition($1); - } + | compressionOptions | __GROUPED__ { $$.setExpr(createAttribute(groupedAtom)); $$.setPosition($1); diff --git a/ecl/hql/hqlgram2.cpp b/ecl/hql/hqlgram2.cpp index 08df8e840a2..8ef067f8809 100644 --- a/ecl/hql/hqlgram2.cpp +++ b/ecl/hql/hqlgram2.cpp @@ -11763,6 +11763,7 @@ static void getTokenText(StringBuffer & msg, int token) case TOK_TRUE: msg.append("TRUE"); break; case TYPE: msg.append("TYPE"); break; case TYPEOF: msg.append("TYPEOF"); break; + case UNCOMPRESSED: msg.append("UNCOMPRESSED"); break; case UNGROUP: msg.append("UNGROUP"); break; case UNICODEORDER: msg.append("UNICODEORDER"); break; case UNLIKELY: msg.append("UNLIKELY"); break; diff --git a/ecl/hql/hqllex.l b/ecl/hql/hqllex.l index 1467ab228c9..b558e55df4f 100644 --- a/ecl/hql/hqllex.l +++ b/ecl/hql/hqllex.l @@ -992,6 +992,7 @@ TRIM { RETURNSYM(TRIM); } TRUNCATE { RETURNSYM(TRUNCATE); } TYPE { RETURNSYM(TYPE); } TYPEOF { RETURNSYM(TYPEOF); } +UNCOMPRESSED { RETURNSYM(UNCOMPRESSED); } UNICODEORDER { RETURNSYM(UNICODEORDER); } UNGROUP { RETURNSYM(UNGROUP); } UNLIKELY { RETURNSYM(UNLIKELY); } diff --git a/ecl/hql/reservedwords.cpp b/ecl/hql/reservedwords.cpp index 0eaea04c386..afa19fa55c0 100644 --- a/ecl/hql/reservedwords.cpp +++ b/ecl/hql/reservedwords.cpp @@ -476,6 +476,7 @@ static const char * eclReserved14[] = { //Attribute functions (some might actual "timelimit", "timeout", "token", + "uncompressed", "unstable", "update", "use", diff --git a/ecl/hqlcpp/hqlhtcpp.cpp b/ecl/hqlcpp/hqlhtcpp.cpp index cec050e886e..36b33a9194e 100644 --- a/ecl/hqlcpp/hqlhtcpp.cpp +++ b/ecl/hqlcpp/hqlhtcpp.cpp @@ -11068,6 +11068,7 @@ ABoundActivity * HqlCppTranslator::doBuildActivityOutput(BuildCtx & ctx, IHqlExp if (expr->hasAttribute(groupedAtom)) flags.append("|TDXgrouped"); if (expr->hasAttribute(compressedAtom)) flags.append("|TDWnewcompress"); if (expr->hasAttribute(__compressed__Atom)) flags.append("|TDXcompress"); + if (expr->hasAttribute(noCompressAtom)) flags.append("|TDWnocompress"); if (expr->hasAttribute(extendAtom)) flags.append("|TDWextend"); if (expr->hasAttribute(overwriteAtom)) flags.append("|TDWoverwrite"); if (expr->hasAttribute(noOverwriteAtom)) flags.append("|TDWnooverwrite"); diff --git a/ecl/hthor/hthor.cpp b/ecl/hthor/hthor.cpp index bcf85f94b94..18e0016fcbd 100644 --- a/ecl/hthor/hthor.cpp +++ b/ecl/hthor/hthor.cpp @@ -59,6 +59,7 @@ #include "thorread.hpp" #include "ws_dfsclient.hpp" +#include "hthorerr.hpp" #define EMPTY_LOOP_LIMIT 1000 @@ -361,6 +362,7 @@ ClusterWriteHandler *createClusterWriteHandler(IAgentContext &agent, IHThorIndex getDefaultStoragePlane(defaultCluster); Owned clusterHandler; unsigned clusterIdx = 0; + while(true) { OwnedRoxieString helperCluster(iwHelper ? iwHelper->getCluster(clusterIdx++) : dwHelper->getCluster(clusterIdx++)); @@ -372,10 +374,10 @@ ClusterWriteHandler *createClusterWriteHandler(IAgentContext &agent, IHThorIndex } if (!cluster) break; - if(!clusterHandler) + if (!clusterHandler) { - if(extend) - throw MakeStringException(0, "Cannot combine EXTEND and CLUSTER flags on disk write of file %s", lfn); + if (extend) + throw makeStringExceptionV(ENGINEERR_EXTEND_CLUSTER_WRITE, "Cannot combine EXTEND and CLUSTER flags on disk write of file %s", lfn); clusterHandler.setown(new CHThorClusterWriteHandler(lfn, "OUTPUT", agent)); } clusterHandler->addCluster(cluster); @@ -540,6 +542,20 @@ void CHThorDiskWriteActivity::resolve() } clusterHandler.setown(createClusterWriteHandler(agent, NULL, &helper, dfsLogicalName.get(), filename, extend, false)); + StringBuffer planeName; + if (clusterHandler) + { + StringArray clusterNames; + clusterHandler->getClusters(clusterNames); + planeName.set(clusterNames.item(0)); // NB: only bother with 1st, if multiple createClusterWriteHandler validates if same + } + else + getDefaultStoragePlane(planeName); + bool outputCompressionDefault = agent.queryWorkUnit()->getDebugValueBool("compressAllOutputs", isContainerized()); + outputPlaneCompressed = outputCompressionDefault; + Owned plane = getStoragePlane(planeName); + if (plane) + outputPlaneCompressed = plane->getPropBool("@compressLogicalFiles", outputCompressionDefault); } } else @@ -560,7 +576,13 @@ void CHThorDiskWriteActivity::open() Linked groupedMeta = input->queryOutputMeta()->querySerializedDiskMeta(); if (grouped) groupedMeta.setown(createDeltaRecordSize(groupedMeta, +1)); - blockcompressed = checkWriteIsCompressed(helper.getFlags(), serializedOutputMeta.getFixedSize(), grouped);//TDWnewcompress for new compression, else check for row compression + blockcompressed=false; + if (0 == (helper.getFlags() & TDWnocompress)) + { + blockcompressed = checkWriteIsCompressed(helper.getFlags(), serializedOutputMeta.getFixedSize(), grouped);//TDWnewcompress for new compression, else check for row compression + if (!blockcompressed) // if ECL doesn't specify, default to plane definition + blockcompressed = outputPlaneCompressed; + } void *ekey; size32_t ekeylen; helper.getEncryptKey(ekeylen,ekey); @@ -8314,7 +8336,6 @@ void CHThorDiskReadBaseActivity::stop() CHThorActivityBase::stop(); } -#define TE_FileTypeMismatch 10138 // NB: duplicated from thorlcr/shared/thexception.hpp, but be moved to common header void CHThorDiskReadBaseActivity::checkFileType(IDistributedFile *file) { if (rt_csv == readType) @@ -8346,7 +8367,7 @@ void CHThorDiskReadBaseActivity::checkFileType(IDistributedFile *file) return; if (!strieq(kind, expectedType)) { - Owned e = makeStringExceptionV(TE_FileTypeMismatch, "File format mismatch reading file: '%s'. Expected type '%s', but file is type '%s'", file->queryLogicalName(), expectedType, kind); + Owned e = makeStringExceptionV(ENGINEERR_FILE_TYPE_MISMATCH, "File format mismatch reading file: '%s'. Expected type '%s', but file is type '%s'", file->queryLogicalName(), expectedType, kind); if (!warningOnly) throw e.getClear(); StringBuffer tmp; diff --git a/ecl/hthor/hthor.ipp b/ecl/hthor/hthor.ipp index b76e2c1a214..688f080d689 100644 --- a/ecl/hthor/hthor.ipp +++ b/ecl/hthor/hthor.ipp @@ -292,6 +292,7 @@ protected: bool grouped; bool blockcompressed; bool encrypted; + bool outputPlaneCompressed = false; CachedOutputMetaData serializedOutputMeta; offset_t uncompressedBytesWritten; Owned outSeq; diff --git a/ecl/hthor/hthorerr.hpp b/ecl/hthor/hthorerr.hpp new file mode 100644 index 00000000000..5ddadf7ec10 --- /dev/null +++ b/ecl/hthor/hthorerr.hpp @@ -0,0 +1,16 @@ +/*############################################################################## + HPCC SYSTEMS software Copyright (C) 2023 HPCC Systems®. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +############################################################################## */ + +#pragma once + +#include "engineerr.hpp" \ No newline at end of file diff --git a/helm/hpcc/values.schema.json b/helm/hpcc/values.schema.json index 5525f1916c2..20cf262b67a 100644 --- a/helm/hpcc/values.schema.json +++ b/helm/hpcc/values.schema.json @@ -560,6 +560,11 @@ "type": "integer", "default": 0 }, + "compressLogicalFiles" : { + "description": "Compress all logical file outputs on this plane.", + "type": "boolean", + "default": true + }, "eclwatchVisible": { "type": "boolean" }, diff --git a/roxie/ccd/ccdserver.cpp b/roxie/ccd/ccdserver.cpp index a6c4bf59e37..56eaf6e69e3 100644 --- a/roxie/ccd/ccdserver.cpp +++ b/roxie/ccd/ccdserver.cpp @@ -75,6 +75,7 @@ #include "keybuild.hpp" #include "thorstrand.hpp" #include "rtldynfield.hpp" +#include "engineerr.hpp" #define MAX_HTTP_HEADERSIZE 8000 @@ -11902,6 +11903,7 @@ class CRoxieServerDiskWriteActivity : public CRoxieServerInternalSinkActivity, i bool overwrite; bool encrypted; bool grouped; + bool outputPlaneCompressed = false; IHThorDiskWriteArg &helper; Owned diskmeta; Owned writer; @@ -11959,6 +11961,7 @@ class CRoxieServerDiskWriteActivity : public CRoxieServerInternalSinkActivity, i assertex((helper.getFlags() & TDXtemporary) == 0); StringArray clusters; unsigned clusterIdx = 0; + bool outputCompressionDefault = isContainerized(); while(true) { OwnedRoxieString cluster(helper.getCluster(clusterIdx)); @@ -11966,6 +11969,14 @@ class CRoxieServerDiskWriteActivity : public CRoxieServerInternalSinkActivity, i break; clusters.append(cluster); clusterIdx++; + + if (1 == clusterIdx) + { + // establish default compression from 1st plane, but ECL compression attributes take precedence + Owned plane = getStoragePlane(cluster); + if (plane) + outputPlaneCompressed = plane->getPropBool("@compressLogicalFiles", outputCompressionDefault); + } } if (clusters.length()) { @@ -11976,7 +11987,12 @@ class CRoxieServerDiskWriteActivity : public CRoxieServerInternalSinkActivity, i { StringBuffer defaultCluster; if (getDefaultStoragePlane(defaultCluster)) + { clusters.append(defaultCluster); + Owned plane = getStoragePlane(defaultCluster); + if (plane) + outputPlaneCompressed = plane->getPropBool("@compressLogicalFiles", outputCompressionDefault); + } else if (roxieName.length()) clusters.append(roxieName.str()); else @@ -11997,8 +12013,12 @@ class CRoxieServerDiskWriteActivity : public CRoxieServerInternalSinkActivity, i diskmeta.set(helper.queryDiskRecordSize()->querySerializedDiskMeta()); if (grouped) diskmeta.setown(createDeltaRecordSize(diskmeta, +1)); - size32_t fixedSize = diskmeta->getFixedSize(); - blockcompressed = (((helper.getFlags() & TDWnewcompress) != 0) || (((helper.getFlags() & TDXcompress) != 0) && ((0 == fixedSize) || (fixedSize >= MIN_ROWCOMPRESS_RECSIZE)))); //always use new compression + blockcompressed = false; + if (0 == (helper.getFlags() & TDWnocompress)) + { + size32_t fixedSize = diskmeta->getFixedSize(); + blockcompressed = (((helper.getFlags() & TDWnewcompress) != 0) || (((helper.getFlags() & TDXcompress) != 0) && ((0 == fixedSize) || (fixedSize >= MIN_ROWCOMPRESS_RECSIZE)))); //always use new compression + } encrypted = false; // set later tallycrc = true; uncompressedBytesWritten = 0; @@ -12031,6 +12051,11 @@ class CRoxieServerDiskWriteActivity : public CRoxieServerInternalSinkActivity, i encrypted = true; blockcompressed = true; } + else + { + if ((0 == (helper.getFlags() & TDWnocompress)) && !blockcompressed) + blockcompressed = outputPlaneCompressed; + } if (blockcompressed) io.setown(createCompressedFileWriter(writer->queryFile(), (diskmeta->isFixedSize() ? diskmeta->getFixedSize() : 0), extend, true, ecomp, COMPRESS_METHOD_LZ4)); else diff --git a/rtl/include/eclhelper.hpp b/rtl/include/eclhelper.hpp index 0eaf18f8599..a7c2019a42a 100644 --- a/rtl/include/eclhelper.hpp +++ b/rtl/include/eclhelper.hpp @@ -1155,6 +1155,7 @@ enum TDWupdatecrc = 0x80000, // has format crc TDWexpires = 0x100000, TDWrestricted = 0x200000, + TDWnocompress = 0x400000, }; //flags for thor index read diff --git a/system/include/errorlist.h b/system/include/errorlist.h index ae76d8efcf2..badad7e2588 100644 --- a/system/include/errorlist.h +++ b/system/include/errorlist.h @@ -88,6 +88,14 @@ #define FILEVIEW_ERROR_START 6700 #define FILEVIEW_ERROR_END 6749 +//HThor +#define HTHOR_ERROR_START 6800 +#define HTHOR_ERROR_END 7099 + +//Common to engines +#define ENGINE_ERROR_START 7100 +#define ENGINE_ERROR_END 7299 + #define REMOTE_ERROR_START 8000 // dafilesrv etc - see common/remote/remoteerr.hpp #define REMOTE_ERROR_END 8099 diff --git a/testing/regress/ecl/dfs.ecl b/testing/regress/ecl/dfs.ecl index 2a882f77cbe..f7640bdffc3 100644 --- a/testing/regress/ecl/dfs.ecl +++ b/testing/regress/ecl/dfs.ecl @@ -17,7 +17,7 @@ //class=file -#onwarning(10140, ignore); +#onwarning(7103, ignore); import $.setup; Files := setup.Files(false, false, false); diff --git a/testing/regress/ecl/dfsi.ecl b/testing/regress/ecl/dfsi.ecl index b3193c3cacf..46cc47c173c 100644 --- a/testing/regress/ecl/dfsi.ecl +++ b/testing/regress/ecl/dfsi.ecl @@ -17,7 +17,7 @@ //class=file -#onwarning(10140, ignore); +#onwarning(7103, ignore); import $.setup; Files := setup.Files(false, false, false); diff --git a/testing/regress/ecl/dfsirecordof.ecl b/testing/regress/ecl/dfsirecordof.ecl index 048d4716304..809beaab537 100644 --- a/testing/regress/ecl/dfsirecordof.ecl +++ b/testing/regress/ecl/dfsirecordof.ecl @@ -17,7 +17,7 @@ //class=file -#onwarning(10140, ignore); +#onwarning(7103, ignore); import $.setup; Files := setup.Files(false, false, false); diff --git a/testing/regress/ecl/dfsj.ecl b/testing/regress/ecl/dfsj.ecl index 00aa971551a..549b5defd75 100644 --- a/testing/regress/ecl/dfsj.ecl +++ b/testing/regress/ecl/dfsj.ecl @@ -17,7 +17,7 @@ //class=file -#onwarning(10140, ignore); +#onwarning(7103, ignore); import $.setup; Files := setup.Files(false, false, false); diff --git a/testing/regress/ecl/dfsrecordof.ecl b/testing/regress/ecl/dfsrecordof.ecl index 5518df35c50..03af2c63a58 100644 --- a/testing/regress/ecl/dfsrecordof.ecl +++ b/testing/regress/ecl/dfsrecordof.ecl @@ -17,7 +17,7 @@ //class=file -#onwarning(10140, ignore); +#onwarning(7103, ignore); import $.setup; Files := setup.Files(false, false, false); diff --git a/testing/regress/ecl/dynamicoptflag.ecl b/testing/regress/ecl/dynamicoptflag.ecl index 6fab56feded..a993781c7d8 100644 --- a/testing/regress/ecl/dynamicoptflag.ecl +++ b/testing/regress/ecl/dynamicoptflag.ecl @@ -18,9 +18,9 @@ import $.setup; prefix := setup.Files(false, false).QueryFilePrefix; -#onwarning (4522, ignore); -#onwarning (4523, ignore); -#onwarning(10140, ignore); +#onwarning(4522, ignore); +#onwarning(4523, ignore); +#onwarning(7103, ignore); d := dataset(DYNAMIC(prefix + 'no_such_file'), {string10 f}, FLAT, OPT); output(d); diff --git a/testing/regress/ecl/keyed_join4.ecl b/testing/regress/ecl/keyed_join4.ecl index 440df3dbc48..4cf1506b9a9 100644 --- a/testing/regress/ecl/keyed_join4.ecl +++ b/testing/regress/ecl/keyed_join4.ecl @@ -17,7 +17,7 @@ #option('warnOnImplicitJoinLimit', false); -#onwarning(10140, ignore); +#onwarning(7103, ignore); import $.setup; prefix := setup.Files(false, false).QueryFilePrefix; diff --git a/testing/regress/ecl/nestedtranslate.ecl b/testing/regress/ecl/nestedtranslate.ecl index 8c0421bc277..cc852a238f9 100644 --- a/testing/regress/ecl/nestedtranslate.ecl +++ b/testing/regress/ecl/nestedtranslate.ecl @@ -18,7 +18,7 @@ //class=file //version multiPart=true -#onwarning(10138, ignore); +#onwarning(7102, ignore); import ^ as root; multiPart := #IFDEFINED(root.multiPart, true); diff --git a/testing/regress/ecl/optflag.ecl b/testing/regress/ecl/optflag.ecl index 8023d0de084..66825ee93c8 100644 --- a/testing/regress/ecl/optflag.ecl +++ b/testing/regress/ecl/optflag.ecl @@ -19,7 +19,7 @@ import $.setup; prefix := setup.Files(false, false).QueryFilePrefix; #onwarning(4523, ignore); #onwarning(4522, ignore); -#onwarning(10140, ignore); +#onwarning(7103, ignore); d := dataset(prefix + 'no_such_file', {string10 f}, FLAT, OPT); output(d); diff --git a/testing/regress/ecl/optflag2.ecl b/testing/regress/ecl/optflag2.ecl index c4faf381158..94f260340c1 100644 --- a/testing/regress/ecl/optflag2.ecl +++ b/testing/regress/ecl/optflag2.ecl @@ -18,7 +18,7 @@ import $.setup; import Std.File; -#onwarning(10140, ignore); +#onwarning(7103, ignore); //version optRemoteRead=false //version optRemoteRead=true diff --git a/testing/regress/ecl/splitter-partialstart.ecl b/testing/regress/ecl/splitter-partialstart.ecl index 3c2bfff6783..4f209b1defe 100644 --- a/testing/regress/ecl/splitter-partialstart.ecl +++ b/testing/regress/ecl/splitter-partialstart.ecl @@ -23,7 +23,7 @@ prefix := setup.Files(false, false).QueryFilePrefix; import Std; -#onwarning(10125, ignore); // ignore UPDATE 'up to date' messages, so that output is consistent across engines +#onwarning(7104, ignore); // ignore UPDATE 'up to date' messages, so that output is consistent across engines rec := RECORD unsigned4 id; diff --git a/testing/regress/ecl/spray_test_json.ecl b/testing/regress/ecl/spray_test_json.ecl index 7612b59dbc3..0a5e0645e36 100644 --- a/testing/regress/ecl/spray_test_json.ecl +++ b/testing/regress/ecl/spray_test_json.ecl @@ -19,7 +19,7 @@ //nohthor //nothor -#onwarning(10138, ignore); +#onwarning(7102, ignore); //version isSmallFile=true,isUnBallanced=false //version isSmallFile=true,isUnBallanced=true diff --git a/testing/regress/ecl/update.ecl b/testing/regress/ecl/update.ecl index ff728f0ec6c..4ae159e4a7b 100644 --- a/testing/regress/ecl/update.ecl +++ b/testing/regress/ecl/update.ecl @@ -18,7 +18,7 @@ //noRoxie //nolocal -#onwarning(10125, ignore); // ignore UPDATE 'up to date' messages, so that output is consistent across engines +#onwarning(7104, ignore); // ignore UPDATE 'up to date' messages, so that output is consistent across engines import Std.File AS FileServices; import $.setup; diff --git a/testing/regress/ecl/xmlread.ecl b/testing/regress/ecl/xmlread.ecl index 57332367d4d..b4eb70eac92 100644 --- a/testing/regress/ecl/xmlread.ecl +++ b/testing/regress/ecl/xmlread.ecl @@ -18,7 +18,7 @@ //class=file //version multiPart=true -#onwarning(10138, ignore); +#onwarning(7102, ignore); import ^ as root; multiPart := #IFDEFINED(root.multiPart, true); diff --git a/thorlcr/activities/fetch/thfetch.cpp b/thorlcr/activities/fetch/thfetch.cpp index 97f59eac25f..78d11706d49 100644 --- a/thorlcr/activities/fetch/thfetch.cpp +++ b/thorlcr/activities/fetch/thfetch.cpp @@ -58,7 +58,7 @@ class CFetchActivityMaster : public CMasterActivity if (fetchFile) { if (isFileKey(fetchFile)) - throw MakeActivityException(this, TE_FileTypeMismatch, "Attempting to read index as a flat file: %s", fname.get()); + throw MakeActivityException(this, ENGINEERR_FILE_TYPE_MISMATCH, "Attempting to read index as a flat file: %s", fname.get()); Owned fileDesc = getConfiguredFileDescriptor(*fetchFile); void *ekey; size32_t ekeylen; diff --git a/thorlcr/activities/hashdistrib/thhashdistrib.cpp b/thorlcr/activities/hashdistrib/thhashdistrib.cpp index abe2c00f037..8a347c9b3fe 100644 --- a/thorlcr/activities/hashdistrib/thhashdistrib.cpp +++ b/thorlcr/activities/hashdistrib/thhashdistrib.cpp @@ -104,7 +104,7 @@ class IndexDistributeActivityMaster : public HashDistributeMasterBase if (0 == file->numParts()) throw MakeActivityException(this, 0, "KeyedDistribute: Can't distribute based on an empty key: %s", scoped.str()); if (!isFileKey(file)) - throw MakeActivityException(this, TE_FileTypeMismatch, "Attempting to read flat file as an index: %s", indexFileName.get()); + throw MakeActivityException(this, ENGINEERR_FILE_TYPE_MISMATCH, "Attempting to read flat file as an index: %s", indexFileName.get()); checkFormatCrc(this, file, helper->getFormatCrc(), nullptr, helper->getFormatCrc(), nullptr, true); Owned fileDesc = file->getFileDescriptor(); diff --git a/thorlcr/activities/keydiff/thkeydiff.cpp b/thorlcr/activities/keydiff/thkeydiff.cpp index c45b9388b03..8f688ee8abd 100644 --- a/thorlcr/activities/keydiff/thkeydiff.cpp +++ b/thorlcr/activities/keydiff/thkeydiff.cpp @@ -59,9 +59,9 @@ class CKeyDiffMaster : public CMasterActivity originalIndexFile.setown(lookupReadFile(originalHelperName, AccessMode::readRandom, false, false, false)); newIndexFile.setown(lookupReadFile(updatedHelperName, AccessMode::readRandom, false, false, false)); if (!isFileKey(originalIndexFile)) - throw MakeActivityException(this, TE_FileTypeMismatch, "Attempting to read flat file as an index: %s", originalHelperName.get()); + throw MakeActivityException(this, ENGINEERR_FILE_TYPE_MISMATCH, "Attempting to read flat file as an index: %s", originalHelperName.get()); if (!isFileKey(newIndexFile)) - throw MakeActivityException(this, TE_FileTypeMismatch, "Attempting to read flat file as an index: %s", updatedHelperName.get()); + throw MakeActivityException(this, ENGINEERR_FILE_TYPE_MISMATCH, "Attempting to read flat file as an index: %s", updatedHelperName.get()); if (originalIndexFile->numParts() != newIndexFile->numParts()) throw MakeActivityException(this, TE_KeyDiffIndexSizeMismatch, "Index %s and %s differ in width", originalName.get(), updatedName.get()); if (originalIndexFile->querySuperFile() || newIndexFile->querySuperFile()) diff --git a/thorlcr/activities/keyedjoin/thkeyedjoin-legacy.cpp b/thorlcr/activities/keyedjoin/thkeyedjoin-legacy.cpp index 7b1292e0916..877c109adae 100644 --- a/thorlcr/activities/keyedjoin/thkeyedjoin-legacy.cpp +++ b/thorlcr/activities/keyedjoin/thkeyedjoin-legacy.cpp @@ -80,12 +80,12 @@ class CKeyedJoinMaster : public CMasterActivity if (indexFile) { if (!isFileKey(indexFile)) - throw MakeActivityException(this, TE_FileTypeMismatch, "Attempting to read flat file as an index: %s", indexFileName.get()); + throw MakeActivityException(this, ENGINEERR_FILE_TYPE_MISMATCH, "Attempting to read flat file as an index: %s", indexFileName.get()); unsigned numParts = 0; localKey = indexFile->queryAttributes().getPropBool("@local"); if (container.queryLocalData() && !localKey) - throw MakeActivityException(this, TE_FileTypeMismatch, "Keyed Join cannot be LOCAL unless supplied index is local"); + throw MakeActivityException(this, ENGINEERR_FILE_TYPE_MISMATCH, "Keyed Join cannot be LOCAL unless supplied index is local"); checkFormatCrc(this, indexFile, helper->getIndexFormatCrc(), helper->queryIndexRecordSize(), helper->getProjectedIndexFormatCrc(), helper->queryProjectedIndexRecordSize(), true); Owned indexFileDesc = indexFile->getFileDescriptor(); diff --git a/thorlcr/activities/keyedjoin/thkeyedjoin.cpp b/thorlcr/activities/keyedjoin/thkeyedjoin.cpp index 5e709c770a1..8a687180a98 100644 --- a/thorlcr/activities/keyedjoin/thkeyedjoin.cpp +++ b/thorlcr/activities/keyedjoin/thkeyedjoin.cpp @@ -308,7 +308,7 @@ class CKeyedJoinMaster : public CMasterActivity if (indexFile) { if (!isFileKey(indexFile)) - throw MakeActivityException(this, TE_FileTypeMismatch, "Attempting to read flat file as an index: %s", indexFileName.get()); + throw MakeActivityException(this, ENGINEERR_FILE_TYPE_MISMATCH, "Attempting to read flat file as an index: %s", indexFileName.get()); IDistributedSuperFile *superIndex = indexFile->querySuperFile(); unsigned numSuperIndexSubs = superIndex?superIndex->numSubFiles(true):1; @@ -321,7 +321,7 @@ class CKeyedJoinMaster : public CMasterActivity if (dataFile) { if (isFileKey(dataFile)) - throw MakeActivityException(this, TE_FileTypeMismatch, "Full-Keyed-Join: Attempting to read index as a flat file (fetch file): %s", fetchFilename.get()); + throw MakeActivityException(this, ENGINEERR_FILE_TYPE_MISMATCH, "Full-Keyed-Join: Attempting to read index as a flat file (fetch file): %s", fetchFilename.get()); if (superIndex) throw MakeActivityException(this, 0, "Full-Keyed-Join: Superkeys with full keyed joins are not supported"); diff --git a/thorlcr/activities/keypatch/thkeypatch.cpp b/thorlcr/activities/keypatch/thkeypatch.cpp index 66c0bfed593..3f279cbe82c 100644 --- a/thorlcr/activities/keypatch/thkeypatch.cpp +++ b/thorlcr/activities/keypatch/thkeypatch.cpp @@ -57,10 +57,10 @@ class CKeyPatchMaster : public CMasterActivity originalIndexFile.setown(lookupReadFile(originalHelperName, AccessMode::readRandom, false, false, false)); if (!isFileKey(originalIndexFile)) - throw MakeActivityException(this, TE_FileTypeMismatch, "Attempting to read flat file as an index: %s", originalHelperName.get()); + throw MakeActivityException(this, ENGINEERR_FILE_TYPE_MISMATCH, "Attempting to read flat file as an index: %s", originalHelperName.get()); patchFile.setown(lookupReadFile(patchHelperName, AccessMode::readRandom, false, false, false)); if (isFileKey(patchFile)) - throw MakeActivityException(this, TE_FileTypeMismatch, "Attempting to read index as a patch file: %s", patchHelperName.get()); + throw MakeActivityException(this, ENGINEERR_FILE_TYPE_MISMATCH, "Attempting to read index as a patch file: %s", patchHelperName.get()); if (originalIndexFile->numParts() != patchFile->numParts()) throw MakeActivityException(this, TE_KeyPatchIndexSizeMismatch, "Index %s and patch %s differ in width", originalName.get(), patchName.get()); diff --git a/thorlcr/activities/msort/thmsort.cpp b/thorlcr/activities/msort/thmsort.cpp index 5222a544ebc..981a4f87820 100644 --- a/thorlcr/activities/msort/thmsort.cpp +++ b/thorlcr/activities/msort/thmsort.cpp @@ -93,7 +93,7 @@ class CMSortActivityMaster : public CSortBaseActivityMaster { Owned coSortFile = lookupReadFile(cosortlogname, AccessMode::readSequential, false, false, false); if (isFileKey(coSortFile)) - throw MakeActivityException(this, TE_FileTypeMismatch, "Attempting to read index as a flat file: %s", cosortlogname.get()); + throw MakeActivityException(this, ENGINEERR_FILE_TYPE_MISMATCH, "Attempting to read index as a flat file: %s", cosortlogname.get()); Owned fileDesc = coSortFile->getFileDescriptor(); unsigned o; for (o=0; onumParts(); o++) diff --git a/thorlcr/activities/thdiskbase.cpp b/thorlcr/activities/thdiskbase.cpp index dae7d5d870f..269d55be407 100644 --- a/thorlcr/activities/thdiskbase.cpp +++ b/thorlcr/activities/thdiskbase.cpp @@ -163,6 +163,8 @@ void CWriteMasterBase::init() } if (dlfn.isExternal()) mpTag = container.queryJob().allocateMPTag(); // used + bool outputCompressionDefault = getOptBool(THOROPT_COMPRESS_ALLFILES, isContainerized()); + bool outputPlaneCompressed = outputCompressionDefault; if (NULL == fileDesc.get()) { bool overwriteok = 0!=(TDWoverwrite & diskHelperBase->getFlags()); @@ -175,6 +177,14 @@ void CWriteMasterBase::init() break; clusters.append(cluster); idx++; + + if (1 == idx) + { + // establish default compression from 1st plane, but ECL compression attributes take precedence + Owned plane = getStoragePlane(cluster); + if (plane) + outputPlaneCompressed = plane->getPropBool("@compressLogicalFiles", outputCompressionDefault); + } } IArrayOf groups; @@ -196,7 +206,12 @@ void CWriteMasterBase::init() { StringBuffer defaultCluster; if (getDefaultStoragePlane(defaultCluster)) + { clusters.append(defaultCluster); + Owned plane = getStoragePlane(defaultCluster); + if (plane) + outputPlaneCompressed = plane->getPropBool("@compressLogicalFiles", outputCompressionDefault); + } } } if (0 == groups.ordinality()) // may be filled if temp (see above) @@ -225,8 +240,15 @@ void CWriteMasterBase::init() props.setPropBool("@encrypted", true); blockCompressed = true; } - else if (0 != (diskHelperBase->getFlags() & TDWnewcompress) || 0 != (diskHelperBase->getFlags() & TDXcompress)) - blockCompressed = true; + else + { + if (0 == (diskHelperBase->getFlags() & TDWnocompress)) + { + blockCompressed = (0 != (diskHelperBase->getFlags() & TDWnewcompress) || 0 != (diskHelperBase->getFlags() & TDXcompress)); + if (!blockCompressed) // if ECL doesn't specify, default to plane definition + blockCompressed = outputPlaneCompressed; + } + } if (blockCompressed) props.setPropBool("@blockCompressed", true); props.setProp("@kind", "flat"); diff --git a/thorlcr/graph/thgraphmaster.cpp b/thorlcr/graph/thgraphmaster.cpp index 696472f604f..c4cbdeb240d 100644 --- a/thorlcr/graph/thgraphmaster.cpp +++ b/thorlcr/graph/thgraphmaster.cpp @@ -770,7 +770,7 @@ bool CMasterGraphElement::checkUpdate() if ((eclCRC == (unsigned)props.getPropInt("@eclCRC")) && (totalCRC == (unsigned __int64)props.getPropInt64("@totalCRC"))) { // so this needs pruning - Owned e = MakeActivityWarning(this, TE_UpToDate, "output file = '%s' - is up to date - it will not be rebuilt", file->queryLogicalName()); + Owned e = MakeActivityWarning(this, ENGINEERR_FILE_UPTODATE, "output file = '%s' - is up to date - it will not be rebuilt", file->queryLogicalName()); queryOwner().fireException(e); return true; } diff --git a/thorlcr/mfilemanager/thmfilemanager.cpp b/thorlcr/mfilemanager/thmfilemanager.cpp index 14c3968691c..b26528dce45 100644 --- a/thorlcr/mfilemanager/thmfilemanager.cpp +++ b/thorlcr/mfilemanager/thmfilemanager.cpp @@ -367,7 +367,7 @@ class CFileManager : public CSimpleInterface, implements IThorFileManager throw MakeStringException(TE_MachineOrderNotFound, "Missing logical file %s\n", scopedName.str()); if (reportOptional) { - Owned e = MakeThorException(TE_MissingOptionalFile, "Input file '%s' was missing but declared optional", scopedName.str()); + Owned e = MakeThorException(ENGINEERR_MISSING_OPTIONAL_FILE, "Input file '%s' was missing but declared optional", scopedName.str()); e->setAction(tea_warning); e->setSeverity(SeverityWarning); reportExceptionToWorkunitCheckIgnore(job.queryWorkUnit(), e); diff --git a/thorlcr/shared/thexception.hpp b/thorlcr/shared/thexception.hpp index c816dee6e47..4d928550a67 100644 --- a/thorlcr/shared/thexception.hpp +++ b/thorlcr/shared/thexception.hpp @@ -20,151 +20,146 @@ #ifndef _thexception_hpp #define _thexception_hpp +#include "engineerr.hpp" - -#define TE_Base 10000 - -#define TE_MachineOrderNotFound TE_Base + 1 -#define TE_NoResultFromNode TE_Base + 2 -#define TE_FailedToCreateProcess TE_Base + 3 -#define TE_FileNotFound TE_Base + 4 -#define TE_BadFileLength TE_Base + 5 -#define TE_NoFileName TE_Base + 6 -#define TE_FileCreationFailed TE_Base + 7 -#define TE_NonOrdered TE_Base + 8 -#define TE_HelperUnavailable TE_Base + 10 -#define TE_SocketReadFailed TE_Base + 12 -#define TE_JoinTypeNotSupported TE_Base + 13 -#define TE_DuplicateInRight TE_Base + 14 -#define TE_NoMachineOrder TE_Base + 15 -#define TE_VariableSizeRecordsNotSupported TE_Base + 16 -#define TE_ThorTimeout TE_Base + 17 -#define TE_SlaveCreationError TE_Base + 18 -#define TE_ActivityNotFound TE_Base + 19 -#define TE_HelperLoadError TE_Base + 20 -#define TE_ActivityConnectionError TE_Base + 21 -#define TE_UnexpectedParameters TE_Base + 22 -#define TE_FailedToRetrieveWorkunitValue TE_Base + 23 -#define TE_NoPhysicalForLogicalActivity TE_Base + 24 -#define TE_PollingForNonExistentGraph TE_Base + 25 -#define TE_CannotLocateFilePart TE_Base + 26 -#define TE_NoSlavesForLogicalFile TE_Base + 27 -#define TE_AddLogPhysNameListError TE_Base + 28 -#define TE_ResolveAllError TE_Base + 29 -#define TE_UnknownException TE_Base + 30 -#define TE_LogicalFileNotFound TE_Base + 31 -#define TE_NoSuchPartForLogicalFile TE_Base + 32 -#define TE_OutOfMemory TE_Base + 33 -#define TE_CannotConnectToSlave TE_Base + 34 -#define TE_SpillPositionsOutOfOrder TE_Base + 35 -#define TE_NoSuchActivityKindRegistered TE_Base + 36 -#define TE_ActivityKindAlreadyPresent TE_Base + 37 -#define TE_CouldNotResolveWildcard TE_Base + 38 -#define TE_CloneInputNotSupported TE_Base + 39 -#define TE_FatalResourceError TE_Base + 40 -#define TE_CouldNotExtractResourceChains TE_Base + 41 -#define TE_NoMachinesFoundForFile TE_Base + 42 -#define TE_NoMemoryResourceFound TE_Base + 43 -#define TE_InsufficientMachines TE_Base + 44 -#define TE_FailedToLoadSharedProcedure TE_Base + 45 -#define TE_FileAlreadyUsedAsTempFile TE_Base + 46 -#define TE_FailedToLoadDll TE_Base + 47 -#define TE_FailedToConnectSlaves TE_Base + 48 -#define TE_BadGraphID TE_Base + 49 -#define TE_StartingNonProcessActivity TE_Base + 50 -#define TE_FailedToWriteDll TE_Base + 51 -#define TE_CouldNotFindActivity TE_Base + 52 -#define TE_CouldNotFindMethod TE_Base + 53 -#define TE_UnexpectedEOF TE_Base + 54 -#define TE_NoResponseFromSlave TE_Base + 55 -#define TE_AbortException TE_Base + 56 -#define TE_ErrorInTCONTROLShutdown TE_Base + 57 -#define TE_DeleteNameException TE_Base + 58 -#define TE_JackClientException TE_Base + 59 -#define TE_NameScanFault TE_Base + 60 -#define TE_UnableToStartGraph TE_Base + 61 -#define TE_ErrorReadyingActivity TE_Base + 62 -#define TE_ErrorStartingActivity TE_Base + 63 -#define TE_ErrorCreatingActivity TE_Base + 64 -#define TE_ErrorEndingActivity TE_Base + 65 -#define TE_ErrorReleasingActivity TE_Base + 66 -#define TE_ErrorInBatchCall TE_Base + 67 -#define TE_ErrorCommunicatingWithSlaveManagers TE_Base + 68 -#define TE_FailedQueryActivityManager TE_Base + 69 -#define TE_ExceptionPassingNSFileEntry TE_Base + 70 -#define TE_SplitPostionsOutOfOrder TE_Base + 71 -#define TE_BarrierNoMaster TE_Base + 72 -#define TE_QueryTimeoutError TE_Base + 74 -#define TE_ErrorCleaningGraph TE_Base + 75 -#define TE_ErrorClosingGraph TE_Base + 76 -#define TE_ErrorInProcessDone TE_Base + 77 -#define TE_JackResovleError TE_Base + 78 -#define TE_FailedToAbortSlaves TE_Base + 79 -#define TE_WorkUnitAborting TE_Base + 80 -#define TE_SEH TE_Base + 81 -#define TE_MasterProcessError TE_Base + 82 -#define TE_JoinFailedSkewExceeded TE_Base + 83 -#define TE_SortFailedSkewExceeded TE_Base + 84 -#define TE_TooMuchData TE_Base + 85 -#define TE_DiskReadMachineGroupMismatch TE_Base + 86 -#define TE_WriteToPipeFailed TE_Base + 87 -#define TE_ReadPartialFromPipe TE_Base + 88 -#define TE_MissingRecordSizeHelper TE_Base + 89 -#define TE_FailedToStartJoinStreams TE_Base + 90 -#define TE_FailedToStartSplitterInput TE_Base + 91 -#define TE_RecursiveDependency TE_Base + 92 -#define TE_OverwriteNotSpecified TE_Base + 93 -#define TE_RecordSizeMismatch TE_Base + 94 -#define TE_CompressMismatch TE_Base + 95 -#define TE_PipeReturnedFailure TE_Base + 96 -#define TE_IdleRestart TE_Base + 97 -#define TE_NotEnoughFreeSpace TE_Base + 98 -#define TE_WorkUnitWriteLimitExceeded TE_Base + 99 -#define TE_CsvLineLenghtExceeded TE_Base + 100 -#define TE_ActivityBufferingLimitExceeded TE_Base + 101 -#define TE_CouldNotCreateLookAhead TE_Base + 102 -#define TE_FailedToRegisterSlave TE_Base + 103 -#define TE_LargeBufferWarning TE_Base + 104 -#define TE_KeySizeMismatch TE_Base + 105 -#define TE_BarrierAborted TE_Base + 106 -#define TE_UnsupportedActivityKind TE_Base + 107 -#define TE_BufferTooSmall TE_Base + 108 -#define TE_GroupMismatch TE_Base + 109 -#define TE_CannotFetchOnCompressedFile TE_Base + 110 -#define TE_UnexpectedMultipleSlaveResults TE_Base + 111 -#define TE_MoxieIndarOverflow TE_Base + 112 -#define TE_KeyDiffIndexSizeMismatch TE_Base + 113 -#define TE_KeyPatchIndexSizeMismatch TE_Base + 114 -#define TE_FileCrc TE_Base + 115 -#define TE_RowCrc TE_Base + 116 -#define TE_SpillAdded TE_Base + 117 -#define TE_MemberOfSuperFile TE_Base + 118 -#define TE_SelfJoinMatchWarning TE_Base + 119 -#define TE_BuildIndexFewExcess TE_Base + 120 -#define TE_FetchMisaligned TE_Base + 121 -#define TE_FetchOutOfRange TE_Base + 122 -#define TE_IndexMissing TE_Base + 123 -#define TE_FormatCrcMismatch TE_Base + 124 -#define TE_UpToDate TE_Base + 125 -#define TE_UnimplementedFeature TE_Base + 126 -#define TE_CompressionMismatch TE_Base + 127 -#define TE_EncryptionMismatch TE_Base + 128 -#define TE_DistributeFailedSkewExceeded TE_Base + 129 -#define TE_SeriailzationError TE_Base + 130 -#define TE_NotSorted TE_Base + 131 -#define TE_LargeAggregateTable TE_Base + 132 -#define TE_SkewWarning TE_Base + 133 -#define TE_SkewError TE_Base + 134 -#define TE_KERN TE_Base + 135 -#define TE_WorkUnitAbortingDumpInfo TE_Base + 136 -#define TE_RowLeaksDetected TE_Base + 137 -#define TE_FileTypeMismatch TE_Base + 138 -#define TE_RemoteReadFailure TE_Base + 139 -#define TE_MissingOptionalFile TE_Base + 140 -#define TE_UnsupportedSortOrder TE_Base + 141 -#define TE_CostExceeded TE_Base + 142 -#define TE_InvalidSortConnect TE_Base + 143 -#define TE_Final TE_Base + 144 // keep this last -#define ISTHOREXCEPTION(n) (n > TE_Base && n < TE_Final) +#define TE_MachineOrderNotFound THOR_ERROR_START + 1 +#define TE_NoResultFromNode THOR_ERROR_START + 2 +#define TE_FailedToCreateProcess THOR_ERROR_START + 3 +#define TE_FileNotFound THOR_ERROR_START + 4 +#define TE_BadFileLength THOR_ERROR_START + 5 +#define TE_NoFileName THOR_ERROR_START + 6 +#define TE_FileCreationFailed THOR_ERROR_START + 7 +#define TE_NonOrdered THOR_ERROR_START + 8 +#define TE_HelperUnavailable THOR_ERROR_START + 10 +#define TE_SocketReadFailed THOR_ERROR_START + 12 +#define TE_JoinTypeNotSupported THOR_ERROR_START + 13 +#define TE_DuplicateInRight THOR_ERROR_START + 14 +#define TE_NoMachineOrder THOR_ERROR_START + 15 +#define TE_VariableSizeRecordsNotSupported THOR_ERROR_START + 16 +#define TE_ThorTimeout THOR_ERROR_START + 17 +#define TE_SlaveCreationError THOR_ERROR_START + 18 +#define TE_ActivityNotFound THOR_ERROR_START + 19 +#define TE_HelperLoadError THOR_ERROR_START + 20 +#define TE_ActivityConnectionError THOR_ERROR_START + 21 +#define TE_UnexpectedParameters THOR_ERROR_START + 22 +#define TE_FailedToRetrieveWorkunitValue THOR_ERROR_START + 23 +#define TE_NoPhysicalForLogicalActivity THOR_ERROR_START + 24 +#define TE_PollingForNonExistentGraph THOR_ERROR_START + 25 +#define TE_CannotLocateFilePart THOR_ERROR_START + 26 +#define TE_NoSlavesForLogicalFile THOR_ERROR_START + 27 +#define TE_AddLogPhysNameListError THOR_ERROR_START + 28 +#define TE_ResolveAllError THOR_ERROR_START + 29 +#define TE_UnknownException THOR_ERROR_START + 30 +#define TE_LogicalFileNotFound THOR_ERROR_START + 31 +#define TE_NoSuchPartForLogicalFile THOR_ERROR_START + 32 +#define TE_OutOfMemory THOR_ERROR_START + 33 +#define TE_CannotConnectToSlave THOR_ERROR_START + 34 +#define TE_SpillPositionsOutOfOrder THOR_ERROR_START + 35 +#define TE_NoSuchActivityKindRegistered THOR_ERROR_START + 36 +#define TE_ActivityKindAlreadyPresent THOR_ERROR_START + 37 +#define TE_CouldNotResolveWildcard THOR_ERROR_START + 38 +#define TE_CloneInputNotSupported THOR_ERROR_START + 39 +#define TE_FatalResourceError THOR_ERROR_START + 40 +#define TE_CouldNotExtractResourceChains THOR_ERROR_START + 41 +#define TE_NoMachinesFoundForFile THOR_ERROR_START + 42 +#define TE_NoMemoryResourceFound THOR_ERROR_START + 43 +#define TE_InsufficientMachines THOR_ERROR_START + 44 +#define TE_FailedToLoadSharedProcedure THOR_ERROR_START + 45 +#define TE_FileAlreadyUsedAsTempFile THOR_ERROR_START + 46 +#define TE_FailedToLoadDll THOR_ERROR_START + 47 +#define TE_FailedToConnectSlaves THOR_ERROR_START + 48 +#define TE_BadGraphID THOR_ERROR_START + 49 +#define TE_StartingNonProcessActivity THOR_ERROR_START + 50 +#define TE_FailedToWriteDll THOR_ERROR_START + 51 +#define TE_CouldNotFindActivity THOR_ERROR_START + 52 +#define TE_CouldNotFindMethod THOR_ERROR_START + 53 +#define TE_UnexpectedEOF THOR_ERROR_START + 54 +#define TE_NoResponseFromSlave THOR_ERROR_START + 55 +#define TE_AbortException THOR_ERROR_START + 56 +#define TE_ErrorInTCONTROLShutdown THOR_ERROR_START + 57 +#define TE_DeleteNameException THOR_ERROR_START + 58 +#define TE_JackClientException THOR_ERROR_START + 59 +#define TE_NameScanFault THOR_ERROR_START + 60 +#define TE_UnableToStartGraph THOR_ERROR_START + 61 +#define TE_ErrorReadyingActivity THOR_ERROR_START + 62 +#define TE_ErrorStartingActivity THOR_ERROR_START + 63 +#define TE_ErrorCreatingActivity THOR_ERROR_START + 64 +#define TE_ErrorEndingActivity THOR_ERROR_START + 65 +#define TE_ErrorReleasingActivity THOR_ERROR_START + 66 +#define TE_ErrorInBatchCall THOR_ERROR_START + 67 +#define TE_ErrorCommunicatingWithSlaveManagers THOR_ERROR_START + 68 +#define TE_FailedQueryActivityManager THOR_ERROR_START + 69 +#define TE_ExceptionPassingNSFileEntry THOR_ERROR_START + 70 +#define TE_SplitPostionsOutOfOrder THOR_ERROR_START + 71 +#define TE_BarrierNoMaster THOR_ERROR_START + 72 +#define TE_QueryTimeoutError THOR_ERROR_START + 74 +#define TE_ErrorCleaningGraph THOR_ERROR_START + 75 +#define TE_ErrorClosingGraph THOR_ERROR_START + 76 +#define TE_ErrorInProcessDone THOR_ERROR_START + 77 +#define TE_JackResovleError THOR_ERROR_START + 78 +#define TE_FailedToAbortSlaves THOR_ERROR_START + 79 +#define TE_WorkUnitAborting THOR_ERROR_START + 80 +#define TE_SEH THOR_ERROR_START + 81 +#define TE_MasterProcessError THOR_ERROR_START + 82 +#define TE_JoinFailedSkewExceeded THOR_ERROR_START + 83 +#define TE_SortFailedSkewExceeded THOR_ERROR_START + 84 +#define TE_TooMuchData THOR_ERROR_START + 85 +#define TE_DiskReadMachineGroupMismatch THOR_ERROR_START + 86 +#define TE_WriteToPipeFailed THOR_ERROR_START + 87 +#define TE_ReadPartialFromPipe THOR_ERROR_START + 88 +#define TE_MissingRecordSizeHelper THOR_ERROR_START + 89 +#define TE_FailedToStartJoinStreams THOR_ERROR_START + 90 +#define TE_FailedToStartSplitterInput THOR_ERROR_START + 91 +#define TE_RecursiveDependency THOR_ERROR_START + 92 +#define TE_OverwriteNotSpecified THOR_ERROR_START + 93 +#define TE_RecordSizeMismatch THOR_ERROR_START + 94 +#define TE_CompressMismatch THOR_ERROR_START + 95 +#define TE_PipeReturnedFailure THOR_ERROR_START + 96 +#define TE_IdleRestart THOR_ERROR_START + 97 +#define TE_NotEnoughFreeSpace THOR_ERROR_START + 98 +#define TE_WorkUnitWriteLimitExceeded THOR_ERROR_START + 99 +#define TE_CsvLineLenghtExceeded THOR_ERROR_START + 100 +#define TE_ActivityBufferingLimitExceeded THOR_ERROR_START + 101 +#define TE_CouldNotCreateLookAhead THOR_ERROR_START + 102 +#define TE_FailedToRegisterSlave THOR_ERROR_START + 103 +#define TE_LargeBufferWarning THOR_ERROR_START + 104 +#define TE_KeySizeMismatch THOR_ERROR_START + 105 +#define TE_BarrierAborted THOR_ERROR_START + 106 +#define TE_UnsupportedActivityKind THOR_ERROR_START + 107 +#define TE_BufferTooSmall THOR_ERROR_START + 108 +#define TE_GroupMismatch THOR_ERROR_START + 109 +#define TE_CannotFetchOnCompressedFile THOR_ERROR_START + 110 +#define TE_UnexpectedMultipleSlaveResults THOR_ERROR_START + 111 +#define TE_MoxieIndarOverflow THOR_ERROR_START + 112 +#define TE_KeyDiffIndexSizeMismatch THOR_ERROR_START + 113 +#define TE_KeyPatchIndexSizeMismatch THOR_ERROR_START + 114 +#define TE_FileCrc THOR_ERROR_START + 115 +#define TE_RowCrc THOR_ERROR_START + 116 +#define TE_SpillAdded THOR_ERROR_START + 117 +#define TE_MemberOfSuperFile THOR_ERROR_START + 118 +#define TE_SelfJoinMatchWarning THOR_ERROR_START + 119 +#define TE_BuildIndexFewExcess THOR_ERROR_START + 120 +#define TE_FetchMisaligned THOR_ERROR_START + 121 +#define TE_FetchOutOfRange THOR_ERROR_START + 122 +#define TE_IndexMissing THOR_ERROR_START + 123 +#define TE_FormatCrcMismatch THOR_ERROR_START + 124 +#define TE_UnimplementedFeature THOR_ERROR_START + 125 +#define TE_CompressionMismatch THOR_ERROR_START + 126 +#define TE_EncryptionMismatch THOR_ERROR_START + 127 +#define TE_DistributeFailedSkewExceeded THOR_ERROR_START + 128 +#define TE_SeriailzationError THOR_ERROR_START + 129 +#define TE_NotSorted THOR_ERROR_START + 130 +#define TE_LargeAggregateTable THOR_ERROR_START + 131 +#define TE_SkewWarning THOR_ERROR_START + 132 +#define TE_SkewError THOR_ERROR_START + 133 +#define TE_KERN THOR_ERROR_START + 134 +#define TE_WorkUnitAbortingDumpInfo THOR_ERROR_START + 135 +#define TE_RowLeaksDetected THOR_ERROR_START + 136 +#define TE_RemoteReadFailure THOR_ERROR_START + 137 +#define TE_UnsupportedSortOrder THOR_ERROR_START + 138 +#define TE_CostExceeded THOR_ERROR_START + 139 +#define TE_InvalidSortConnect THOR_ERROR_START + 140 +#define TE_Final THOR_ERROR_START + 141 // keep this last +#define ISTHOREXCEPTION(n) (n > THOR_ERROR_START && n < TE_Final) #endif diff --git a/thorlcr/thorutil/thormisc.cpp b/thorlcr/thorutil/thormisc.cpp index f2c6c2a2343..3e4509ed273 100644 --- a/thorlcr/thorutil/thormisc.cpp +++ b/thorlcr/thorutil/thormisc.cpp @@ -1646,7 +1646,7 @@ void checkFileType(CActivityBase *activity, IDistributedFile *file, const char * return; if (!strieq(kind, expectedType)) { - Owned e = MakeActivityException(activity, TE_FileTypeMismatch, "File format mismatch reading file: '%s'. Expected type '%s', but file is type '%s'", file->queryLogicalName(), expectedType, kind); + Owned e = MakeActivityException(activity, ENGINEERR_FILE_TYPE_MISMATCH, "File format mismatch reading file: '%s'. Expected type '%s', but file is type '%s'", file->queryLogicalName(), expectedType, kind); if (throwException) throw e.getClear(); e->setAction(tea_warning); diff --git a/thorlcr/thorutil/thormisc.hpp b/thorlcr/thorutil/thormisc.hpp index 895c325164d..02c74d6dace 100644 --- a/thorlcr/thorutil/thormisc.hpp +++ b/thorlcr/thorutil/thormisc.hpp @@ -114,6 +114,7 @@ #define THOROPT_FAIL_ON_LEAKS "failOnLeaks" // If any leaks are detected at the end of graph, fail the query (default=false) #define THOROPT_SOAP_TRACE_LEVEL "soapTraceLevel" // The trace SOAP level (default=1) #define THOROPT_SORT_ALGORITHM "sortAlgorithm" // The algorithm used to sort records (quicksort/mergesort) +#define THOROPT_COMPRESS_ALLFILES "compressAllOutputs" // Compress all output files (default: bare-metal=off, cloud=on) #define INITIAL_SELFJOIN_MATCH_WARNING_LEVEL 20000 // max of row matches before selfjoin emits warning