diff --git a/.github/workflows/build-assets.yml b/.github/workflows/build-assets.yml index 1138242ffc1..4babda36283 100644 --- a/.github/workflows/build-assets.yml +++ b/.github/workflows/build-assets.yml @@ -245,7 +245,6 @@ jobs: with: name: html-help-documents path: | - ${{ needs.preamble.outputs.folder_build }}/docs/EN_US/EclipseHelp/*.zip ${{ needs.preamble.outputs.folder_build }}/docs/EN_US/HTMLHelp/*.zip ${{ needs.preamble.outputs.folder_build }}/docs/PT_BR/HTMLHelp/*.zip compression-level: 0 @@ -263,7 +262,7 @@ jobs: ${{ needs.preamble.outputs.folder_build }}/Release/docs/*.zip, ${{ needs.preamble.outputs.folder_build }}/Release/docs/EN_US/*.zip, ${{ needs.preamble.outputs.folder_build }}/Release/docs/PT_BR/*.zip, - ${{ needs.preamble.outputs.folder_build }}/docs/EN_US/EclipseHelp/*.zip, + ${{ needs.preamble.outputs.folder_build }}/docs/EN_US/HTMLHelp/*.zip, ${{ needs.preamble.outputs.folder_build }}/docs/PT_BR/HTMLHelp/*.zip, ${{ needs.preamble.outputs.folder_build }}/*.md5sum diff --git a/docs/BuildTools/cmake_config/EclipseHelp.txt b/docs/BuildTools/cmake_config/EclipseHelp.txt deleted file mode 100644 index a868917c57f..00000000000 --- a/docs/BuildTools/cmake_config/EclipseHelp.txt +++ /dev/null @@ -1,29 +0,0 @@ -################################################################################ -# HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -################################################################################ -IF(MAKE_DOCS) - set (ECLIPSE_HTML_DIR ${CMAKE_CURRENT_BINARY_DIR}/eclipse_html) - - GET_PROPERTY(Current_Targets GLOBAL PROPERTY DOC_TARGETS) - SET(HELP_DEPENDENCIES) - FOREACH(T ${Current_Targets}) - IF("${T}" MATCHES ".*ECLReference_${DOC_LANG}.*") - LIST(APPEND HELP_DEPENDENCIES ${T}) - ENDIF() - ENDFOREACH() - - DOCBOOK_TO_HTML(${ECLIPSE_HTML_XSL} ${ECL_REFERENCE_XML} ${ECLIPSE_HTML_DIR} "eclipse_html_${DOC_LANG}" "${HPCC_SOURCE_DIR}/docs/common/eclipsehelp.css" "doc_generate_html_${DOC_LANG}_zip") - -ENDIF(MAKE_DOCS) diff --git a/docs/EN_US/CMakeLists.txt b/docs/EN_US/CMakeLists.txt index 48817c4b3de..e74ca33065e 100644 --- a/docs/EN_US/CMakeLists.txt +++ b/docs/EN_US/CMakeLists.txt @@ -68,7 +68,6 @@ add_subdirectory(ECLProgrammersGuide) add_subdirectory(ECLStandardLibraryReference) add_subdirectory(ECLReference) -add_subdirectory(EclipseHelp) add_subdirectory(HTMLHelp) add_subdirectory(ECLPlayground) add_subdirectory(HPCCClientTools) diff --git a/docs/EN_US/EclipseHelp/CMakeLists.txt b/docs/EN_US/EclipseHelp/CMakeLists.txt deleted file mode 100644 index 630f4204600..00000000000 --- a/docs/EN_US/EclipseHelp/CMakeLists.txt +++ /dev/null @@ -1,17 +0,0 @@ -################################################################################ -# HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -################################################################################ -get_filename_component(DOC_DIR_NAME ${CMAKE_CURRENT_SOURCE_DIR} NAME) -include(${CMAKE_CURRENT_SOURCE_DIR}/../../BuildTools/cmake_config/${DOC_DIR_NAME}.txt) diff --git a/docs/EN_US/HPCCDataTutorial/DataTutorial.xml b/docs/EN_US/HPCCDataTutorial/DataTutorial.xml index fb2b847925b..8748a57c5bd 100644 --- a/docs/EN_US/HPCCDataTutorial/DataTutorial.xml +++ b/docs/EN_US/HPCCDataTutorial/DataTutorial.xml @@ -955,13 +955,7 @@ DATASET('~tutorial::YN::TutorialPerson', - Check the syntax, if no errors press the Submit button. - - - - When it completes, it displays a green checkmark - . + Check the syntax, if no errors you can continue. @@ -985,7 +979,7 @@ INDEX(TutorialYourName.File_TutorialPerson,{zip,fpos},'~tutorial::YN::PeopleByZi - Check the syntax. + Check the syntax, if no errors you can continue. Next, we will build the index file. diff --git a/ecl/eclagent/eclagent.ipp b/ecl/eclagent/eclagent.ipp index 5362b60c929..17109a2c6f4 100644 --- a/ecl/eclagent/eclagent.ipp +++ b/ecl/eclagent/eclagent.ipp @@ -795,6 +795,7 @@ public: void ready(); void execute() { if (!alreadyUpdated) activity->execute(); } + void onStart(const byte * parentExtract, CHThorDebugContext * debugContext); void stop() { if (!alreadyUpdated) activity->stop(); } IHThorException * makeWrappedException(IException * e); @@ -829,6 +830,7 @@ public: CICopyArrayOf dependentOnActivity; IntArray dependentControlId; IProbeManager * probeManager; + const byte * savedParentExtract = nullptr; Owned loopGraph; }; diff --git a/ecl/eclagent/eclgraph.cpp b/ecl/eclagent/eclgraph.cpp index a14266c4c07..a285b7d9cdc 100644 --- a/ecl/eclagent/eclgraph.cpp +++ b/ecl/eclagent/eclgraph.cpp @@ -747,6 +747,24 @@ void EclGraphElement::ready() activity->ready(); } +void EclGraphElement::onStart(const byte * parentExtract, CHThorDebugContext * debugContext) +{ + savedParentExtract = parentExtract; + if (arg) + { + try + { + arg->onStart(parentExtract, NULL); + } + catch(IException * e) + { + if (debugContext) + debugContext->checkBreakpoint(DebugStateException, NULL, e); + throw makeWrappedException(e); + } + } +} + IHThorException * EclGraphElement::makeWrappedException(IException * e) { throw makeHThorException(kind, id, subgraph->id, e); @@ -962,19 +980,7 @@ void EclSubGraph::doExecute(const byte * parentExtract, bool checkDependencies) ForEachItemIn(idx, elements) { EclGraphElement & cur = elements.item(idx); - if (cur.arg) - { - try - { - cur.arg->onStart(parentExtract, NULL); - } - catch(IException * e) - { - if (debugContext) - debugContext->checkBreakpoint(DebugStateException, NULL, e); - throw cur.makeWrappedException(e); - } - } + cur.onStart(parentExtract, debugContext); } ForEachItemIn(ir, sinks) diff --git a/ecl/hthor/hthor.cpp b/ecl/hthor/hthor.cpp index 4ccb5c9860c..ee9dc1bc5ab 100644 --- a/ecl/hthor/hthor.cpp +++ b/ecl/hthor/hthor.cpp @@ -6719,13 +6719,13 @@ CHThorWhenActionActivity::CHThorWhenActionActivity(IAgentContext &_agent, unsign void CHThorWhenActionActivity::ready() { CHThorSimpleActivityBase::ready(); - graphElement->executeDependentActions(agent, NULL, WhenBeforeId); - graphElement->executeDependentActions(agent, NULL, WhenParallelId); + graphElement->executeDependentActions(agent, graphElement->savedParentExtract, WhenBeforeId); + graphElement->executeDependentActions(agent, graphElement->savedParentExtract, WhenParallelId); } void CHThorWhenActionActivity::execute() { - graphElement->executeDependentActions(agent, NULL, 1); + graphElement->executeDependentActions(agent, graphElement->savedParentExtract, 1); } const void * CHThorWhenActionActivity::nextRow() @@ -6735,7 +6735,7 @@ const void * CHThorWhenActionActivity::nextRow() void CHThorWhenActionActivity::stop() { - graphElement->executeDependentActions(agent, NULL, WhenSuccessId); + graphElement->executeDependentActions(agent, graphElement->savedParentExtract, WhenSuccessId); CHThorSimpleActivityBase::stop(); } diff --git a/esp/scm/ws_logaccess.ecm b/esp/scm/ws_logaccess.ecm index bd214a668fa..5232f983275 100644 --- a/esp/scm/ws_logaccess.ecm +++ b/esp/scm/ws_logaccess.ecm @@ -79,7 +79,8 @@ ESPenum LogColumnValueType : string str("string"), numeric("numeric"), datetime("datetime"), - enum("enum") + enum("enum"), + epoch("epoch") }; ESPStruct [nil_remove] LogColumn diff --git a/esp/services/ws_logaccess/WsLogAccessService.cpp b/esp/services/ws_logaccess/WsLogAccessService.cpp index 0e8b12c329d..98e4a98ad90 100644 --- a/esp/services/ws_logaccess/WsLogAccessService.cpp +++ b/esp/services/ws_logaccess/WsLogAccessService.cpp @@ -71,6 +71,10 @@ bool Cws_logaccessEx::onGetLogAccessInfo(IEspContext &context, IEspGetLogAccessI WARNLOG("Invalid col type found in logaccess logmap config"); } } + else + { + espLogColumn->setColumnType("string"); + } logColumns.append(*espLogColumn.getClear()); } else diff --git a/esp/src/package-lock.json b/esp/src/package-lock.json index 28aef7baf94..e15b7782995 100644 --- a/esp/src/package-lock.json +++ b/esp/src/package-lock.json @@ -15,22 +15,23 @@ "@fluentui/react-hooks": "8.8.10", "@fluentui/react-icons-mdl2": "1.3.72", "@fluentui/react-migration-v8-v9": "9.6.22", - "@hpcc-js/chart": "2.83.4", - "@hpcc-js/codemirror": "2.62.1", - "@hpcc-js/common": "2.71.18", - "@hpcc-js/comms": "2.94.1", + "@hpcc-js/chart": "2.84.1", + "@hpcc-js/codemirror": "2.63.0", + "@hpcc-js/common": "2.72.0", + "@hpcc-js/comms": "2.95.0", "@hpcc-js/dataflow": "8.1.7", - "@hpcc-js/eclwatch": "2.74.8", - "@hpcc-js/graph": "2.85.16", - "@hpcc-js/html": "2.42.21", - "@hpcc-js/layout": "2.49.23", - "@hpcc-js/map": "2.77.22", - "@hpcc-js/other": "2.15.23", - "@hpcc-js/phosphor": "2.18.9", - "@hpcc-js/react": "2.53.17", - "@hpcc-js/tree": "2.40.18", - "@hpcc-js/util": "2.51.1", - "@hpcc-js/wasm": "2.18.0", + "@hpcc-js/eclwatch": "2.75.3", + "@hpcc-js/graph": "2.86.0", + "@hpcc-js/html": "2.43.0", + "@hpcc-js/layout": "2.50.1", + "@hpcc-js/map": "2.78.1", + "@hpcc-js/other": "2.16.1", + "@hpcc-js/phosphor": "2.19.1", + "@hpcc-js/react": "2.54.0", + "@hpcc-js/timeline": "2.53.0", + "@hpcc-js/tree": "2.41.0", + "@hpcc-js/util": "2.52.0", + "@hpcc-js/wasm": "2.18.1", "@kubernetes/client-node": "0.20.0", "clipboard": "2.0.11", "d3-dsv": "3.0.1", @@ -2026,41 +2027,41 @@ } }, "node_modules/@hpcc-js/api": { - "version": "2.12.18", - "resolved": "https://registry.npmjs.org/@hpcc-js/api/-/api-2.12.18.tgz", - "integrity": "sha512-Iq00B6q9DGGoLe+PTm4mJjaopfHRQbd6A3rNW968vZDmtgBh3Vcn2ie/Ym/Dbcf3UjSuWjaldaLKksqvgIbmtw==", + "version": "2.13.0", + "resolved": "https://registry.npmjs.org/@hpcc-js/api/-/api-2.13.0.tgz", + "integrity": "sha512-bqMLQ/jlCSqBe3CfC2f2vaWZt12FBPW/5LfzW0I0Z30L+FRmjvBpEPMgr3Jd9JIv/igpq49F3xEAKiCxeMYaeA==", "license": "Apache-2.0", "dependencies": { - "@hpcc-js/common": "^2.71.18" + "@hpcc-js/common": "^2.72.0" } }, "node_modules/@hpcc-js/chart": { - "version": "2.83.4", - "resolved": "https://registry.npmjs.org/@hpcc-js/chart/-/chart-2.83.4.tgz", - "integrity": "sha512-rIE6OSEbh9Z75l0fkP0rDjZuBi60lXDMsBlaG42YBgnZpWKIQHydkGDOnq7q7jC3PedvSgOnr3ZRJL8ZBA1iXg==", + "version": "2.84.1", + "resolved": "https://registry.npmjs.org/@hpcc-js/chart/-/chart-2.84.1.tgz", + "integrity": "sha512-ddqZh6rnp93FIRLtcfE3bKpV+o194aOnSyYwW2pCkKYoAiGtFPfZAiOY2AaYsqqhQ96Pqk+6bOEfi4Asr6cj5A==", "license": "Apache-2.0", "dependencies": { - "@hpcc-js/api": "^2.12.18", - "@hpcc-js/common": "^2.71.18", - "@hpcc-js/util": "^2.51.1" + "@hpcc-js/api": "^2.13.0", + "@hpcc-js/common": "^2.72.0", + "@hpcc-js/util": "^2.52.0" } }, "node_modules/@hpcc-js/codemirror": { - "version": "2.62.1", - "resolved": "https://registry.npmjs.org/@hpcc-js/codemirror/-/codemirror-2.62.1.tgz", - "integrity": "sha512-p/sG+En3UG5grGx00maZ8wjUuuUGb/6flzlq+bSIDNldwqaheWZscGPaJdxLa+aSvcFbXXrDgcV4dhhd70xEyA==", + "version": "2.63.0", + "resolved": "https://registry.npmjs.org/@hpcc-js/codemirror/-/codemirror-2.63.0.tgz", + "integrity": "sha512-Vx/EJWOhSzAqdg5RqA9w/wOjtIuBBNYAWjvvz+mqwHtzWN9bK9ciwRFeL+LyrunT66W49qV0sSt8sgQ4fa7n+Q==", "license": "Apache-2.0", "dependencies": { - "@hpcc-js/common": "^2.71.18" + "@hpcc-js/common": "^2.72.0" } }, "node_modules/@hpcc-js/common": { - "version": "2.71.18", - "resolved": "https://registry.npmjs.org/@hpcc-js/common/-/common-2.71.18.tgz", - "integrity": "sha512-9hsDYXsjB2ltmomAAPQUfNUv/FPrHDkE4e7t62nNH7HCN3ZrkgOgbNjdrfPPHwtcz5+j4kH6hjV0hbhVgocs5Q==", + "version": "2.72.0", + "resolved": "https://registry.npmjs.org/@hpcc-js/common/-/common-2.72.0.tgz", + "integrity": "sha512-5GxU2RmNrB/Bq1gG9Ng+NjojxBLDQhPnDxo5l3PhiMLdBe9HCOsAWSkGg55AQvlOQu4o9RLZ78Gtwkjw5BYiHw==", "license": "Apache-2.0", "dependencies": { - "@hpcc-js/util": "^2.51.1", + "@hpcc-js/util": "^2.52.0", "@types/d3-array": "1.2.12", "@types/d3-brush": "1.1.8", "@types/d3-collection": "1.0.13", @@ -2079,9 +2080,10 @@ } }, "node_modules/@hpcc-js/comms": { - "version": "2.94.1", - "resolved": "https://registry.npmjs.org/@hpcc-js/comms/-/comms-2.94.1.tgz", - "integrity": "sha512-ROCHmHogsZ5/G9LsRPHRIx25rpVoR9d5TakbSkXeSugRkhPV+16+3C/Lfd4d8ZzNLR99Jmnvjd0NXZAMgpAkGQ==", + "version": "2.95.0", + "resolved": "https://registry.npmjs.org/@hpcc-js/comms/-/comms-2.95.0.tgz", + "integrity": "sha512-kzEyDxf1Msus5rhU0yO826JxgIl2kh/bI7yNTxHAlCfLKp4SU//zrj/MK5SuEDs2lUHvmTkfx1jbfGUAK3RGFg==", + "license": "Apache-2.0", "dependencies": { "@hpcc-js/ddl-shim": "^2.21.0", "@hpcc-js/util": "^2.52.0", @@ -2093,14 +2095,6 @@ "undici": "5.28.4" } }, - "node_modules/@hpcc-js/comms/node_modules/@hpcc-js/util": { - "version": "2.52.0", - "resolved": "https://registry.npmjs.org/@hpcc-js/util/-/util-2.52.0.tgz", - "integrity": "sha512-WHm/0ApEdWktpPCUG+AFuMnnrDHOTqKXK2oVgyRUAAQJhSWMFxTJxbqIQG7SM9myK58tXzNJrKsP8huzt8X2dg==", - "dependencies": { - "tslib": "2.6.3" - } - }, "node_modules/@hpcc-js/comms/node_modules/safe-buffer": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", @@ -2120,11 +2114,6 @@ } ] }, - "node_modules/@hpcc-js/comms/node_modules/tslib": { - "version": "2.6.3", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.3.tgz", - "integrity": "sha512-xNvxJEOUiWPGhUuUdQgAJPKOOJfGnIyKySOc09XkKsgdUV/3E2zvwZYdejjmRgPCgcym1juLH3226yA7sEFJKQ==" - }, "node_modules/@hpcc-js/dataflow": { "version": "8.1.7", "resolved": "https://registry.npmjs.org/@hpcc-js/dataflow/-/dataflow-8.1.7.tgz", @@ -2144,92 +2133,92 @@ } }, "node_modules/@hpcc-js/dgrid": { - "version": "2.32.23", - "resolved": "https://registry.npmjs.org/@hpcc-js/dgrid/-/dgrid-2.32.23.tgz", - "integrity": "sha512-wWMJ+YJNeuW+aXdWjDIEfLfcnfSf2TydsZ6weTbJxFAydvn7dkito2qWwPfXYSlMQ8iZztXng8M/ddJ0P2M7zg==", + "version": "2.33.0", + "resolved": "https://registry.npmjs.org/@hpcc-js/dgrid/-/dgrid-2.33.0.tgz", + "integrity": "sha512-MyTybKnc1a83AsYLb/hpwARa9REfT2UbDEzowhqvcSwASPUwdv7di1Gb3iZkws1yvByinfWOPSprWHqAKEDwmw==", "license": "Apache-2.0", "dependencies": { - "@hpcc-js/common": "^2.71.18", - "@hpcc-js/ddl-shim": "^2.20.7", - "@hpcc-js/dgrid-shim": "^2.24.11", - "@hpcc-js/util": "^2.51.1" + "@hpcc-js/common": "^2.72.0", + "@hpcc-js/ddl-shim": "^2.21.0", + "@hpcc-js/dgrid-shim": "^2.25.0", + "@hpcc-js/util": "^2.52.0" } }, "node_modules/@hpcc-js/dgrid-shim": { - "version": "2.24.11", - "resolved": "https://registry.npmjs.org/@hpcc-js/dgrid-shim/-/dgrid-shim-2.24.11.tgz", - "integrity": "sha512-rWKAKrlO0GA5FwktWmVBEw0TxmRQp+x2fw+MHL0ksYjMGMcRgaV4sD4DKH5GlLlMQwrmIsmbWdOgz7mV5dPOtA==", + "version": "2.25.0", + "resolved": "https://registry.npmjs.org/@hpcc-js/dgrid-shim/-/dgrid-shim-2.25.0.tgz", + "integrity": "sha512-QYYivb9NKq33OZ8RBCwcr2AYvNLVWgURPAdxvZznRt1qdIdUCaDRxUi0D1RLi+8eBWQx9EHkTWw5lqERNU1/1w==", "license": "Apache-2.0" }, "node_modules/@hpcc-js/dgrid2": { - "version": "2.3.20", - "resolved": "https://registry.npmjs.org/@hpcc-js/dgrid2/-/dgrid2-2.3.20.tgz", - "integrity": "sha512-+9e0LBFKELADY2UhZ6Pj15EGfoW1OVzMiUli2xprebpTtzrJaFESQ6c+8OfqiSuFGvACxlVABq4GvlB8ohHkKw==", + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/@hpcc-js/dgrid2/-/dgrid2-2.4.0.tgz", + "integrity": "sha512-92uYB7Cqh7ZGAWeCtZOClKZJJjOVnFRKDgWuHVCexSZ/o54RUXc5WYWqpSp3gas/6K1zAZVFaJA+Yqh6E4aM6g==", "license": "Apache-2.0", "dependencies": { - "@hpcc-js/common": "^2.71.18", - "@hpcc-js/preact-shim": "^2.16.11", - "@hpcc-js/util": "^2.51.1" + "@hpcc-js/common": "^2.72.0", + "@hpcc-js/preact-shim": "^2.17.0", + "@hpcc-js/util": "^2.52.0" } }, "node_modules/@hpcc-js/eclwatch": { - "version": "2.74.8", - "resolved": "https://registry.npmjs.org/@hpcc-js/eclwatch/-/eclwatch-2.74.8.tgz", - "integrity": "sha512-kcszxO1eAKYirVTpgzvUKjn9W6hnOdOOSrC9L+bu8fGf8j62C2D0aFOrNsXg3ZV3BnQRxjZt+1z2K3TwttB+hg==", + "version": "2.75.3", + "resolved": "https://registry.npmjs.org/@hpcc-js/eclwatch/-/eclwatch-2.75.3.tgz", + "integrity": "sha512-PFvk0LFeo+kf679J5oMEr/nwL0B/tSPMqFm8g/ZKQRhxNz6KhfTQMZnUXRqHf+M+ppYetFTkfcJJq0ujvqA+Fw==", "license": "Apache-2.0", "dependencies": { - "@hpcc-js/codemirror": "^2.62.1", - "@hpcc-js/common": "^2.71.18", - "@hpcc-js/comms": "^2.93.0", - "@hpcc-js/dgrid": "^2.32.23", - "@hpcc-js/graph": "^2.85.16", - "@hpcc-js/layout": "^2.49.23", - "@hpcc-js/phosphor": "^2.18.9", - "@hpcc-js/timeline": "^2.51.26", - "@hpcc-js/tree": "^2.40.18", - "@hpcc-js/util": "^2.51.1" + "@hpcc-js/codemirror": "^2.63.0", + "@hpcc-js/common": "^2.72.0", + "@hpcc-js/comms": "^2.95.0", + "@hpcc-js/dgrid": "^2.33.0", + "@hpcc-js/graph": "^2.86.0", + "@hpcc-js/layout": "^2.50.1", + "@hpcc-js/phosphor": "^2.19.1", + "@hpcc-js/timeline": "^2.53.0", + "@hpcc-js/tree": "^2.41.0", + "@hpcc-js/util": "^2.52.0" } }, "node_modules/@hpcc-js/graph": { - "version": "2.85.16", - "resolved": "https://registry.npmjs.org/@hpcc-js/graph/-/graph-2.85.16.tgz", - "integrity": "sha512-egJdVtAv8PPGVMj1ZColHYROZvXO/KZq/Y/+1aEuXmS8WIXoJo8fCV2SRHo9LmPUrh57ffT/DYhg3xpvcWw+VQ==", + "version": "2.86.0", + "resolved": "https://registry.npmjs.org/@hpcc-js/graph/-/graph-2.86.0.tgz", + "integrity": "sha512-b8yAULHxhMGTSDbhJg4hzx64WdcF5wEAtDD6AerxS4NhgOWS1GpWnZE8ToQWCckdZFFWyKSV7N10d3xB57cVHg==", "license": "Apache-2.0", "dependencies": { - "@hpcc-js/api": "^2.12.18", - "@hpcc-js/common": "^2.71.18", - "@hpcc-js/html": "^2.42.21", - "@hpcc-js/react": "^2.53.17", - "@hpcc-js/util": "^2.51.1" + "@hpcc-js/api": "^2.13.0", + "@hpcc-js/common": "^2.72.0", + "@hpcc-js/html": "^2.43.0", + "@hpcc-js/react": "^2.54.0", + "@hpcc-js/util": "^2.52.0" } }, "node_modules/@hpcc-js/html": { - "version": "2.42.21", - "resolved": "https://registry.npmjs.org/@hpcc-js/html/-/html-2.42.21.tgz", - "integrity": "sha512-acNRDb2jnHzEue1irdGut2zSxaxvOVd3uFPdTCJjJpOpqNqQ2jphGYU2EdOiyO/uApVGmmvONeGLfnWtqeGPig==", + "version": "2.43.0", + "resolved": "https://registry.npmjs.org/@hpcc-js/html/-/html-2.43.0.tgz", + "integrity": "sha512-464r3luThDbp72CPhfcirZyqWC+EDhTBXdS76+bxFK5CEsiTzE6sFpx4hCwLE9JFvjDsu0pS0dbFGIHQkJ1Fmw==", "license": "Apache-2.0", "dependencies": { - "@hpcc-js/common": "^2.71.18", - "@hpcc-js/preact-shim": "^2.16.11", - "@hpcc-js/util": "^2.51.1" + "@hpcc-js/common": "^2.72.0", + "@hpcc-js/preact-shim": "^2.17.0", + "@hpcc-js/util": "^2.52.0" } }, "node_modules/@hpcc-js/layout": { - "version": "2.49.23", - "resolved": "https://registry.npmjs.org/@hpcc-js/layout/-/layout-2.49.23.tgz", - "integrity": "sha512-JhKf9UqzRt0H9PlRnghRfAiKPucvbsHwuLzjR3w965VKSMgX2M+bqZDbG0ch45kLq81sNcpmn6UupEq/8E4Wpw==", + "version": "2.50.1", + "resolved": "https://registry.npmjs.org/@hpcc-js/layout/-/layout-2.50.1.tgz", + "integrity": "sha512-0OoLv+kOwuzCqGPUviUDtty4yeaRBZONMIUyV+qZNIZGVPvmrMTqlYV6rj8LSTnX8cylz4aqGUfs6llI0UDq4g==", "license": "Apache-2.0", "dependencies": { - "@hpcc-js/api": "^2.12.18", - "@hpcc-js/chart": "^2.83.4", - "@hpcc-js/common": "^2.71.18", - "@hpcc-js/dgrid2": "^2.3.20" + "@hpcc-js/api": "^2.13.0", + "@hpcc-js/chart": "^2.84.1", + "@hpcc-js/common": "^2.72.0", + "@hpcc-js/dgrid2": "^2.4.0" } }, "node_modules/@hpcc-js/leaflet-shim": { - "version": "2.3.6", - "resolved": "https://registry.npmjs.org/@hpcc-js/leaflet-shim/-/leaflet-shim-2.3.6.tgz", - "integrity": "sha512-7eXs0/8gika93i3hb2oaxXiQJxQuV5tO8tQm7PpzzwyzjGI+t622/msrFR1NnAGnywso8YGGBqnQkqxsS1sE3Q==", + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/@hpcc-js/leaflet-shim/-/leaflet-shim-2.4.0.tgz", + "integrity": "sha512-ue3UmHYA9hKOxDc7XFP63+k5OKU6/da8QHr5ElBX5XHDrYDw+TY7/ZknAyFUTS9fv8Sp1YiecPiTI0+DEdMcbA==", "license": "Apache-2.0", "dependencies": { "@types/leaflet": "1.9.8", @@ -2237,47 +2226,47 @@ } }, "node_modules/@hpcc-js/map": { - "version": "2.77.22", - "resolved": "https://registry.npmjs.org/@hpcc-js/map/-/map-2.77.22.tgz", - "integrity": "sha512-znQ6UioWkkJJkuOPoPndk/OHDmcPSOu07EqQVr+GtEacJqmI6B/z9Ltdl920A/2Mr2SDYLaZAgvvR6wjvlInPw==", + "version": "2.78.1", + "resolved": "https://registry.npmjs.org/@hpcc-js/map/-/map-2.78.1.tgz", + "integrity": "sha512-R/FGKO4p3hfplhja4CeEjrCsZAqucdDvnakS9ec2V9UK9SlODM6o0FbOjA9l54vJ0wjQf6luPNV1/9LGQBUUtQ==", "license": "Apache-2.0", "dependencies": { - "@hpcc-js/api": "^2.12.18", - "@hpcc-js/common": "^2.71.18", - "@hpcc-js/graph": "^2.85.16", - "@hpcc-js/layout": "^2.49.23", - "@hpcc-js/leaflet-shim": "^2.3.6", - "@hpcc-js/other": "^2.15.23", - "@hpcc-js/util": "^2.51.1" + "@hpcc-js/api": "^2.13.0", + "@hpcc-js/common": "^2.72.0", + "@hpcc-js/graph": "^2.86.0", + "@hpcc-js/layout": "^2.50.1", + "@hpcc-js/leaflet-shim": "^2.4.0", + "@hpcc-js/other": "^2.16.1", + "@hpcc-js/util": "^2.52.0" } }, "node_modules/@hpcc-js/other": { - "version": "2.15.23", - "resolved": "https://registry.npmjs.org/@hpcc-js/other/-/other-2.15.23.tgz", - "integrity": "sha512-79oV6+lEyKUwxqtMRje2Dh32N6g0Qdcu2+YRUo351A6rU0KWwPTnMl3+u24IfxjVUC6b39PNKoU9c70JjohYOQ==", + "version": "2.16.1", + "resolved": "https://registry.npmjs.org/@hpcc-js/other/-/other-2.16.1.tgz", + "integrity": "sha512-S78frHoqjcIJw+A3GK3g+OoCsQ3N3m9o1d3fZmKoldkbw74hzPbcFgBa9+Y5h9jYfEAFOvwMx1nPQsq7EEFg4g==", "license": "Apache-2.0", "dependencies": { - "@hpcc-js/api": "^2.12.18", - "@hpcc-js/common": "^2.71.18", - "@hpcc-js/layout": "^2.49.23" + "@hpcc-js/api": "^2.13.0", + "@hpcc-js/common": "^2.72.0", + "@hpcc-js/layout": "^2.50.1" } }, "node_modules/@hpcc-js/phosphor": { - "version": "2.18.9", - "resolved": "https://registry.npmjs.org/@hpcc-js/phosphor/-/phosphor-2.18.9.tgz", - "integrity": "sha512-xhJMgMvrWVkzmzOEFKPLltf9jV8eEN8cZ3nkcm6GXEQte84RsHLbKHeUfUH2iZZjJeSA/xNTivdp7aDZaC+Xhw==", + "version": "2.19.1", + "resolved": "https://registry.npmjs.org/@hpcc-js/phosphor/-/phosphor-2.19.1.tgz", + "integrity": "sha512-hR6ngzFY6GydqjiCMLgp+stqZZhKaemgzrr1axFi1U+JEzZSz2rY9Gt0rJ7IeLwcj8+hc07jZzsN9/GtZbu6BQ==", "license": "Apache-2.0", "dependencies": { - "@hpcc-js/common": "^2.71.18", - "@hpcc-js/other": "^2.15.23", - "@hpcc-js/phosphor-shim": "^2.14.7", - "@hpcc-js/util": "^2.51.1" + "@hpcc-js/common": "^2.72.0", + "@hpcc-js/other": "^2.16.1", + "@hpcc-js/phosphor-shim": "^2.15.0", + "@hpcc-js/util": "^2.52.0" } }, "node_modules/@hpcc-js/phosphor-shim": { - "version": "2.14.7", - "resolved": "https://registry.npmjs.org/@hpcc-js/phosphor-shim/-/phosphor-shim-2.14.7.tgz", - "integrity": "sha512-Xu/kSr1kqs//4yGAkiX08Kq7VaGAPRU2Cbvj47NDMdwEtmzT8qm9npTomKvaK0yYYjcLZNwC5liRb74z1HKdIg==", + "version": "2.15.0", + "resolved": "https://registry.npmjs.org/@hpcc-js/phosphor-shim/-/phosphor-shim-2.15.0.tgz", + "integrity": "sha512-WyRhZou8IgP3kwvsZ41VtNUlnuVwtddBp9qd3mY7W59sYerTT1GIIpyRVSs/Jbyy8/piKOogthJPrDptn6k1Zg==", "license": "Apache-2.0", "dependencies": { "@lumino/algorithm": "1.9.2", @@ -2287,52 +2276,52 @@ } }, "node_modules/@hpcc-js/preact-shim": { - "version": "2.16.11", - "resolved": "https://registry.npmjs.org/@hpcc-js/preact-shim/-/preact-shim-2.16.11.tgz", - "integrity": "sha512-pYaRMTpNXEam3qUoCZPoKJvvePVMMhZ9rBbQ6q0z7Ri4Lsq6chXCGTKVrTImafPbNWoMOZVPQE6L8+0Iu1PYzw==", + "version": "2.17.0", + "resolved": "https://registry.npmjs.org/@hpcc-js/preact-shim/-/preact-shim-2.17.0.tgz", + "integrity": "sha512-Thbyu1zycHef+6M/MYT4VE72Sjem3ZZ9jS81T67Ixnd84rHKPMU1na0kBnLfk7jzyiKBnjXOH0Vl2RIXrtesKQ==", "license": "Apache-2.0", "dependencies": { "preact": "10.22.0" } }, "node_modules/@hpcc-js/react": { - "version": "2.53.17", - "resolved": "https://registry.npmjs.org/@hpcc-js/react/-/react-2.53.17.tgz", - "integrity": "sha512-r2K+LqjPQ0VgCUxF+fOOhmCuBHQw2nr+SL4DL9MNrkqUDjFMNWG6YkiLEqb6dx6b/mVvWM1P+IWMrD4zIYLsfA==", + "version": "2.54.0", + "resolved": "https://registry.npmjs.org/@hpcc-js/react/-/react-2.54.0.tgz", + "integrity": "sha512-ciWvp77WyiXZeg+uA5L+a7S8X8nglrdHLj7D+lSiz7n3ApRhGxNvuFpU3pE9AWmkDp5O5r+0Oxd0IO1cipvoew==", "license": "Apache-2.0", "dependencies": { - "@hpcc-js/common": "^2.71.18", - "@hpcc-js/preact-shim": "^2.16.11" + "@hpcc-js/common": "^2.72.0", + "@hpcc-js/preact-shim": "^2.17.0" } }, "node_modules/@hpcc-js/timeline": { - "version": "2.51.26", - "resolved": "https://registry.npmjs.org/@hpcc-js/timeline/-/timeline-2.51.26.tgz", - "integrity": "sha512-CS73qPo7dVru/0M00IzJnL2YDxlhYMq7VNWDzBpNjLxRhUvvmKAUS4O7/dkuskBpT5SrpPcQc6gkOKvY4qjJbQ==", + "version": "2.53.0", + "resolved": "https://registry.npmjs.org/@hpcc-js/timeline/-/timeline-2.53.0.tgz", + "integrity": "sha512-HgZiPLLaDcQexEH9XPpvJzF1103+CeomwJ9sRulyO3hjctw3yDVHKblIkdQbxBcXvjCNIWCiHcs53tJ3yBcrXw==", "license": "Apache-2.0", "dependencies": { - "@hpcc-js/api": "^2.12.18", - "@hpcc-js/chart": "^2.83.4", - "@hpcc-js/common": "^2.71.18", - "@hpcc-js/html": "^2.42.21", - "@hpcc-js/layout": "^2.49.23", - "@hpcc-js/react": "^2.53.17" + "@hpcc-js/api": "^2.13.0", + "@hpcc-js/chart": "^2.84.1", + "@hpcc-js/common": "^2.72.0", + "@hpcc-js/html": "^2.43.0", + "@hpcc-js/layout": "^2.50.1", + "@hpcc-js/react": "^2.54.0" } }, "node_modules/@hpcc-js/tree": { - "version": "2.40.18", - "resolved": "https://registry.npmjs.org/@hpcc-js/tree/-/tree-2.40.18.tgz", - "integrity": "sha512-SWXgKs1JCSL1S1Xx0zBrbUd35fcDxhum573w4JiZzV5S8L0M7zhNTWvPn7joW5Q8Ox3jn1Hk/IQzcyBtVDyp0Q==", + "version": "2.41.0", + "resolved": "https://registry.npmjs.org/@hpcc-js/tree/-/tree-2.41.0.tgz", + "integrity": "sha512-IairckImHOibB1LlnzvpCpfUk6VWTk82aLaG2sDK9M53qWAJPjxb2ftxND1nzeDvWZz4VxpR7F14tgJlRzaEqw==", "license": "Apache-2.0", "dependencies": { - "@hpcc-js/api": "^2.12.18", - "@hpcc-js/common": "^2.71.18" + "@hpcc-js/api": "^2.13.0", + "@hpcc-js/common": "^2.72.0" } }, "node_modules/@hpcc-js/util": { - "version": "2.51.1", - "resolved": "https://registry.npmjs.org/@hpcc-js/util/-/util-2.51.1.tgz", - "integrity": "sha512-BJuqg6FGqcV4RR8/BU5e7fASDtkl0Na7dWY+Th7r5ciWKI5AXsO0GtlgwDBt2uLUOlcGOMpYozmdbGSCoSHAvQ==", + "version": "2.52.0", + "resolved": "https://registry.npmjs.org/@hpcc-js/util/-/util-2.52.0.tgz", + "integrity": "sha512-WHm/0ApEdWktpPCUG+AFuMnnrDHOTqKXK2oVgyRUAAQJhSWMFxTJxbqIQG7SM9myK58tXzNJrKsP8huzt8X2dg==", "license": "Apache-2.0", "dependencies": { "tslib": "2.6.3" @@ -2345,9 +2334,9 @@ "license": "0BSD" }, "node_modules/@hpcc-js/wasm": { - "version": "2.18.0", - "resolved": "https://registry.npmjs.org/@hpcc-js/wasm/-/wasm-2.18.0.tgz", - "integrity": "sha512-M9XVIvAXGH4Xcyb5UoiohWcn6fil89pcos/gClNdBZG2v+W48xSf2bjcA8BW131X/AFHUerVY28n1P1Jw81/9A==", + "version": "2.18.1", + "resolved": "https://registry.npmjs.org/@hpcc-js/wasm/-/wasm-2.18.1.tgz", + "integrity": "sha512-fT8NCOTaF0NDnT+ZwWpV2VQ6ywFEqw+fG87GSPNQemEmg7FFqUaKRQOW9MBICrkZcXaJBb7VHo1t5UF6bi/JgQ==", "license": "Apache-2.0", "dependencies": { "yargs": "17.7.2" @@ -12333,4 +12322,4 @@ } } } -} +} \ No newline at end of file diff --git a/esp/src/package.json b/esp/src/package.json index 7a4508f1cf3..c1ec1651488 100644 --- a/esp/src/package.json +++ b/esp/src/package.json @@ -41,22 +41,23 @@ "@fluentui/react-hooks": "8.8.10", "@fluentui/react-icons-mdl2": "1.3.72", "@fluentui/react-migration-v8-v9": "9.6.22", - "@hpcc-js/chart": "2.83.4", - "@hpcc-js/codemirror": "2.62.1", - "@hpcc-js/common": "2.71.18", - "@hpcc-js/comms": "2.94.1", + "@hpcc-js/chart": "2.84.1", + "@hpcc-js/codemirror": "2.63.0", + "@hpcc-js/common": "2.72.0", + "@hpcc-js/comms": "2.95.0", "@hpcc-js/dataflow": "8.1.7", - "@hpcc-js/eclwatch": "2.74.8", - "@hpcc-js/graph": "2.85.16", - "@hpcc-js/html": "2.42.21", - "@hpcc-js/layout": "2.49.23", - "@hpcc-js/map": "2.77.22", - "@hpcc-js/other": "2.15.23", - "@hpcc-js/phosphor": "2.18.9", - "@hpcc-js/react": "2.53.17", - "@hpcc-js/tree": "2.40.18", - "@hpcc-js/util": "2.51.1", - "@hpcc-js/wasm": "2.18.0", + "@hpcc-js/eclwatch": "2.75.3", + "@hpcc-js/graph": "2.86.0", + "@hpcc-js/html": "2.43.0", + "@hpcc-js/layout": "2.50.1", + "@hpcc-js/map": "2.78.1", + "@hpcc-js/other": "2.16.1", + "@hpcc-js/phosphor": "2.19.1", + "@hpcc-js/react": "2.54.0", + "@hpcc-js/timeline": "2.53.0", + "@hpcc-js/tree": "2.41.0", + "@hpcc-js/util": "2.52.0", + "@hpcc-js/wasm": "2.18.1", "@kubernetes/client-node": "0.20.0", "clipboard": "2.0.11", "d3-dsv": "3.0.1", diff --git a/esp/src/src-react/components/Frame.tsx b/esp/src/src-react/components/Frame.tsx index ad77d0b45c1..9821ffdfd43 100644 --- a/esp/src/src-react/components/Frame.tsx +++ b/esp/src/src-react/components/Frame.tsx @@ -20,6 +20,13 @@ import { useUserSession } from "../hooks/user"; const logger = scopedLogger("../components/Frame.tsx"); const envLogger = scopedLogger("environment"); +const USER_COOKIE_CONSENT = "user_cookie_consent"; + +export function resetCookieConsent() { + const store = userKeyValStore(); + return store.delete(USER_COOKIE_CONSENT); +} + interface FrameProps { } @@ -111,7 +118,7 @@ export const Frame: React.FunctionComponent = () => { />} /> { - userKeyValStore().set("user_cookie_consent", n ? "1" : "0"); + userKeyValStore().set(USER_COOKIE_CONSENT, n ? "1" : "0"); }} /> ; diff --git a/esp/src/src-react/components/Metrics.tsx b/esp/src/src-react/components/Metrics.tsx index 798d7843a79..f0df3a94f6a 100644 --- a/esp/src/src-react/components/Metrics.tsx +++ b/esp/src/src-react/components/Metrics.tsx @@ -7,7 +7,7 @@ import { bundleIcon, Folder20Filled, Folder20Regular, FolderOpen20Filled, Folder import { Database } from "@hpcc-js/common"; import { WorkunitsServiceEx, IScope, splitMetric } from "@hpcc-js/comms"; import { CellFormatter, ColumnFormat, ColumnType, DBStore, RowType, Table } from "@hpcc-js/dgrid"; -import { compare, scopedLogger } from "@hpcc-js/util"; +import { scopedLogger } from "@hpcc-js/util"; import nlsHPCC from "src/nlsHPCC"; import { WUTimelineNoFetch } from "src/Timings"; import * as Utility from "src/Utility"; @@ -89,7 +89,7 @@ class TableEx extends Table { } _rawDataMap: { [id: number]: string } = {}; - metrics(metrics: any[], options: MetricsOptionsT, timelineFilter: string, scopeFilter: string, matchCase: boolean): this { + metrics(metrics: any[], options: MetricsOptionsT, scopeFilter: string, matchCase: boolean): this { this .columns(["##"]) // Reset hash to force recalculation of default widths .columns(["##", nlsHPCC.Type, "StdDevs", nlsHPCC.Scope, ...options.properties, "__StdDevs"]) @@ -106,8 +106,7 @@ class TableEx extends Table { .data(metrics .filter(m => this.scopeFilterFunc(m, scopeFilter, matchCase)) .filter(row => { - return (timelineFilter === "" || row.name?.indexOf(timelineFilter) === 0) && - (options.scopeTypes.indexOf(row.type) >= 0); + return options.scopeTypes.indexOf(row.type) >= 0; }).map((row, idx) => { if (idx === 0) { this._rawDataMap = { @@ -175,7 +174,6 @@ export const Metrics: React.FunctionComponent = ({ selection }) => { const [_uiState, _setUIState] = React.useState({ ...defaultUIState }); - const [timelineFilter, setTimelineFilter] = React.useState(""); const [selectedMetricsSource, setSelectedMetricsSource] = React.useState(""); const [selectedMetrics, setSelectedMetrics] = React.useState([]); const [selectedMetricsPtr, setSelectedMetricsPtr] = React.useState(-1); @@ -227,13 +225,15 @@ export const Metrics: React.FunctionComponent = ({ .maxZoom(Number.MAX_SAFE_INTEGER) ); + const [scopeFilter, setScopeFilter] = React.useState(""); React.useEffect(() => { timeline .on("click", (row, col, sel) => { - setTimelineFilter(sel ? row[7].__hpcc_id : ""); if (sel) { + timeline.selection([]); setSelectedMetricsSource("scopesTable"); - pushUrl(`${parentUrl}/${row[7].Id}`); + setScopeFilter(`name:${row[7].__hpcc_id}`); + pushUrl(`${parentUrl}/${row[7].id}`); } }, true) ; @@ -247,7 +247,6 @@ export const Metrics: React.FunctionComponent = ({ }, [metrics, timeline]); // Scopes Table --- - const [scopeFilter, setScopeFilter] = React.useState(""); const onChangeScopeFilter = React.useCallback((event: React.FormEvent, newValue?: string) => { setScopeFilter(newValue || ""); }, []); @@ -259,7 +258,7 @@ export const Metrics: React.FunctionComponent = ({ const scopesTable = useConst(() => new TableEx() .multiSelect(true) - .metrics([], options, timelineFilter, scopeFilter, matchCase) + .metrics([], options, scopeFilter, matchCase) .sortable(true) ); @@ -274,24 +273,22 @@ export const Metrics: React.FunctionComponent = ({ }, [scopesSelectionChanged, scopesTable]); React.useEffect(() => { - if (!scopeFilter || scopeFilter.indexOf("name:") === 0) { - setScopeFilter(timelineFilter ? `name:${timelineFilter}` : ""); - } scopesTable - .metrics(metrics, options, timelineFilter, scopeFilter, matchCase) + .metrics(metrics, options, scopeFilter, matchCase) .lazyRender() ; - }, [matchCase, metrics, options, scopeFilter, scopesTable, timelineFilter]); + }, [matchCase, metrics, options, scopeFilter, scopesTable]); const updateScopesTable = React.useCallback((selection: IScope[]) => { if (scopesTable?.renderCount() > 0) { - const prevSelection = scopesTable.selection().map(row => row.__lparam.id); - const newSelection = selection.map(row => row.id); - const diffs = compare(prevSelection, newSelection); - if (diffs.enter.length || diffs.exit.length) { - scopesTable.selection(scopesTable.data().filter(row => { + scopesTable.selection([]); + if (selection.length) { + const selRows = scopesTable.data().filter(row => { return selection.indexOf(row[row.length - 1]) >= 0; - })); + }); + scopesTable.render(() => { + scopesTable.selection(selRows); + }); } } }, [scopesTable]); @@ -508,7 +505,7 @@ export const Metrics: React.FunctionComponent = ({ } } setIsLayoutComplete(true); - }); + }).catch(err => logger.error(err)); } return () => { cancelled = true; @@ -622,13 +619,13 @@ export const Metrics: React.FunctionComponent = ({ const setShowMetricOptionsHook = React.useCallback((show: boolean) => { setShowMetricOptions(show); scopesTable - .metrics(metrics, options, timelineFilter, scopeFilter, matchCase) + .metrics(metrics, options, scopeFilter, matchCase) .render(() => { updateScopesTable(selectedMetrics); }) ; - }, [matchCase, metrics, options, scopeFilter, scopesTable, selectedMetrics, timelineFilter, updateScopesTable]); + }, [matchCase, metrics, options, scopeFilter, scopesTable, selectedMetrics, updateScopesTable]); return diff --git a/esp/src/src-react/components/Reset.tsx b/esp/src/src-react/components/Reset.tsx new file mode 100644 index 00000000000..5954d6fe990 --- /dev/null +++ b/esp/src/src-react/components/Reset.tsx @@ -0,0 +1,87 @@ +import * as React from "react"; +import { PrimaryButton, DefaultButton, mergeStyleSets, Checkbox, Stack } from "@fluentui/react"; +import { resetCookies, resetModernMode } from "src/Session"; +import nlsHPCC from "src/nlsHPCC"; +import { pushUrl, replaceUrl } from "../util/history"; +import { resetMetricsViews } from "../hooks/metrics"; +import { resetHistory } from "../util/history"; +import { MessageBox } from "../layouts/MessageBox"; +import { resetTheme, useUserTheme } from "../hooks/theme"; +import { resetFavorites } from "../hooks/favorite"; +import { resetCookieConsent } from "./Frame"; +import { resetWorkunitOptions } from "./Workunits"; + +export interface ResetDialogProps { +} + +export const ResetDialog: React.FunctionComponent = ({ +}) => { + + const [show, setShow] = React.useState(true); + const [checkMetricOptions, setCheckMetricOptions] = React.useState(true); + const [checkWorkunitOptions, setCheckWorkunitOptions] = React.useState(true); + const [checkHistory, setCheckHistoryCheckbox] = React.useState(true); + const [checkFavorites, setCheckFavorites] = React.useState(true); + const [checkEclWatchVersion, setCheckEclWatchVersion] = React.useState(true); + const [checkTheme, setCheckTheme] = React.useState(true); + const [checkCookies, setCheckCookies] = React.useState(false); + const { theme } = useUserTheme(); + + const styles = React.useMemo(() => mergeStyleSets({ + root: { + height: "100%", + backgroundColor: theme.semanticColors.bodyBackground + }, + }), [theme.semanticColors.bodyBackground]); + + return
+ + { + if (checkMetricOptions) { + await resetMetricsViews(); + } + if (checkWorkunitOptions) { + await resetWorkunitOptions(); + } + if (checkHistory) { + await resetHistory(); + } + if (checkFavorites) { + await resetFavorites(); + } + if (checkHistory) { + await resetHistory(); + } + if (checkEclWatchVersion) { + await resetModernMode(); + } + if (checkTheme) { + await resetTheme(); + } + if (checkCookies) { + await resetCookies(); + await resetCookieConsent(); + } + setShow(false); + replaceUrl("/"); + window.location.reload(); + }} /> + { + setShow(false); + pushUrl("/"); + }} /> + + }> + + setCheckMetricOptions(checked)} /> + setCheckWorkunitOptions(checked)} /> + setCheckHistoryCheckbox(checked)} /> + setCheckFavorites(checked)} /> + setCheckEclWatchVersion(checked)} /> + setCheckTheme(checked)} /> + setCheckCookies(checked)} /> + + +
; +}; diff --git a/esp/src/src-react/components/Title.tsx b/esp/src/src-react/components/Title.tsx index 5257ba34747..e4910f7ec42 100644 --- a/esp/src/src-react/components/Title.tsx +++ b/esp/src/src-react/components/Title.tsx @@ -182,6 +182,12 @@ export const DevTitle: React.FunctionComponent = ({ isChecked: true, onClick: onTechPreviewClick }, + { key: "divider_4", itemType: ContextualMenuItemType.Divider }, + { + key: "reset", + href: "/esp/files/index.html#/reset", + text: nlsHPCC.ResetUserSettings + }, { key: "about", text: nlsHPCC.About, onClick: () => setShowAbout(true) } ], directionalHintFixed: true diff --git a/esp/src/src-react/components/Workunits.tsx b/esp/src/src-react/components/Workunits.tsx index b4ff6a3a291..0fce3d69f16 100644 --- a/esp/src/src-react/components/Workunits.tsx +++ b/esp/src/src-react/components/Workunits.tsx @@ -5,6 +5,7 @@ import { SizeMe } from "react-sizeme"; import { CreateWUQueryStore, defaultSort, emptyFilter, Get, WUQueryStore, formatQuery } from "src/ESPWorkunit"; import * as WsWorkunits from "src/WsWorkunits"; import { formatCost } from "src/Session"; +import { userKeyValStore } from "src/KeyValStore"; import nlsHPCC from "src/nlsHPCC"; import { useConfirm } from "../hooks/confirm"; import { useMyAccount } from "../hooks/user"; @@ -45,6 +46,13 @@ const defaultUIState = { hasNotCompleted: false }; +const WORKUNITS_SHOWTIMELINE = "workunits_showTimeline"; + +export function resetWorkunitOptions() { + const store = userKeyValStore(); + return store?.delete(WORKUNITS_SHOWTIMELINE); +} + interface WorkunitsProps { filter?: { [id: string]: any }; sort?: QuerySortItem; @@ -64,7 +72,7 @@ export const Workunits: React.FunctionComponent = ({ const [showFilter, setShowFilter] = React.useState(false); const { currentUser } = useMyAccount(); const [uiState, setUIState] = React.useState({ ...defaultUIState }); - const [showTimeline, setShowTimeline] = useUserStore("workunits_showTimeline", true); + const [showTimeline, setShowTimeline] = useUserStore(WORKUNITS_SHOWTIMELINE, true); const { selection, setSelection, pageNum, setPageNum, diff --git a/esp/src/src-react/components/controls/Grid.tsx b/esp/src/src-react/components/controls/Grid.tsx index f3cdd7a67f1..7a445acac93 100644 --- a/esp/src/src-react/components/controls/Grid.tsx +++ b/esp/src/src-react/components/controls/Grid.tsx @@ -240,8 +240,9 @@ const FluentStoreGrid: React.FunctionComponent = ({ setTotal(total); }); storeQuery.then(items => { + const selectedIndices = selectionHandler.getSelectedIndices(); setItems(items); - setSelection(selectionHandler.getSelection()); + selectedIndices.forEach(index => selectionHandler.setIndexSelected(index, true, false)); }); }, [count, selectionHandler, start, store], [query, sorted]); diff --git a/esp/src/src-react/hooks/favorite.ts b/esp/src/src-react/hooks/favorite.ts index 183d547c3ca..114282fee9d 100644 --- a/esp/src/src-react/hooks/favorite.ts +++ b/esp/src/src-react/hooks/favorite.ts @@ -7,6 +7,11 @@ import { hashHistory } from "../util/history"; const STORE_FAVORITES_ID = "favorites"; const STORE_CACHE_TIMEOUT = 10000; +export function resetFavorites() { + const store = userKeyValStore(); + return store?.delete(STORE_FAVORITES_ID); +} + interface Payload { // TODO: Will be used for labels and extra info... } diff --git a/esp/src/src-react/hooks/metrics.ts b/esp/src/src-react/hooks/metrics.ts index 24d4a5341e4..72f78b279f1 100644 --- a/esp/src/src-react/hooks/metrics.ts +++ b/esp/src/src-react/hooks/metrics.ts @@ -9,7 +9,13 @@ import { useCounter } from "./util"; const logger = scopedLogger("src-react/hooks/metrics.ts"); -const MetricOptionsVersion = 2; +const METRIC_OPTIONS_VERSION = 2; +const METRIC_OPTIONS_KEY = `MetricOptions-${METRIC_OPTIONS_VERSION}`; + +export function resetMetricsViews() { + const store = userKeyValStore(); + return store?.delete(METRIC_OPTIONS_KEY); +} export interface MetricsOptions { scopeTypes: string[]; @@ -58,7 +64,7 @@ export function useMetricsOptions(): [MetricsOptions, (opts: MetricsOptions) => const save = React.useCallback(() => { if (checkLayout(options)) { - store?.set(`MetricOptions-${MetricOptionsVersion}`, JSON.stringify(options), true); + store?.set(METRIC_OPTIONS_KEY, JSON.stringify(options), true); } }, [store]); @@ -66,7 +72,7 @@ export function useMetricsOptions(): [MetricsOptions, (opts: MetricsOptions) => if (toDefaults) { setOptions({ ...defaults }); } else { - store?.get(`MetricOptions-${MetricOptionsVersion}`).then(opts => { + store?.get(METRIC_OPTIONS_KEY).then(opts => { const options = JSON.parse(opts); checkLayout(options); setOptions({ ...defaults, ...options }); diff --git a/esp/src/src-react/hooks/store.ts b/esp/src/src-react/hooks/store.ts index 95f48517129..58af4736634 100644 --- a/esp/src/src-react/hooks/store.ts +++ b/esp/src/src-react/hooks/store.ts @@ -38,7 +38,7 @@ function fromString(value: string, defaultValue: T): T { } } -function useStore(store: IKeyValStore, key: string, defaultValue: T, monitor: boolean = false): [value: T, setValue: (value: T) => void, reset: () => void] { +function useStore(store: IKeyValStore, key: string, defaultValue: T, monitor: boolean = false): [value: T, setValue: (value: T) => Promise, reset: () => Promise] { const [value, setValue] = React.useState(); @@ -66,13 +66,13 @@ function useStore(store: IKeyValStore, key: string, defaultValue: T, monitor: }, [defaultValue, key, monitor, store]); const extSetValue = React.useCallback((value: T) => { - store.set(key, toString(value, defaultValue), monitor).then(() => { + return store.set(key, toString(value, defaultValue), monitor).then(() => { setValue(value); }); }, [defaultValue, key, monitor, store]); const reset = React.useCallback(() => { - store.delete(key, monitor).then(() => { + return store.delete(key, monitor).then(() => { setValue(defaultValue); }); }, [defaultValue, key, monitor, store]); diff --git a/esp/src/src-react/hooks/theme.ts b/esp/src/src-react/hooks/theme.ts index e1f909a456a..85b50d163da 100644 --- a/esp/src/src-react/hooks/theme.ts +++ b/esp/src/src-react/hooks/theme.ts @@ -1,11 +1,19 @@ import { Theme } from "@fluentui/react"; import { Theme as ThemeV9 } from "@fluentui/react-components"; +import { userKeyValStore } from "src/KeyValStore"; import { darkTheme, lightTheme, darkThemeV9, lightThemeV9 } from "../themes"; import { useUserStore } from "./store"; +const THEME = "theme"; + +export function resetTheme() { + const store = userKeyValStore(); + return store?.delete(THEME); +} + export function useUserTheme(): { theme: Theme, themeV9: ThemeV9, setTheme: (value: "light" | "dark") => void, isDark: boolean } { - const [theme, setTheme] = useUserStore("theme", "light", true); + const [theme, setTheme] = useUserStore(THEME, "light", true); return { theme: theme === "dark" ? darkTheme : lightTheme, diff --git a/esp/src/src-react/layouts/MessageBox.tsx b/esp/src/src-react/layouts/MessageBox.tsx index a7286cc7c3b..caa17875078 100644 --- a/esp/src/src-react/layouts/MessageBox.tsx +++ b/esp/src/src-react/layouts/MessageBox.tsx @@ -48,6 +48,7 @@ interface MessageBoxProps { show: boolean; modeless?: boolean; blocking?: boolean; + onDismiss?: () => void; setShow: (_: boolean) => void; footer?: React.ReactNode; children?: React.ReactNode; @@ -59,6 +60,7 @@ export const MessageBox: React.FunctionComponent = ({ show, modeless = true, blocking = true, + onDismiss, setShow, footer, children @@ -71,7 +73,12 @@ export const MessageBox: React.FunctionComponent = ({ body: { padding: "12px 24px 12px 24px", overflowY: "hidden" }, }), [theme.palette.themePrimary, minWidth]); - const close = React.useCallback(() => setShow(false), [setShow]); + const close = React.useCallback(() => { + if (onDismiss) { + onDismiss(); + } + setShow(false); + }, [onDismiss, setShow]); return diff --git a/esp/src/src-react/routes.tsx b/esp/src/src-react/routes.tsx index 9a3a59a66b3..fc0326d001f 100644 --- a/esp/src/src-react/routes.tsx +++ b/esp/src/src-react/routes.tsx @@ -45,12 +45,17 @@ export const routes: RoutesEx = [ path: "/login", action: (ctx) => import("./components/forms/Login").then(_ => <_.Login />) }, + { + mainNav: [], + name: "reset", + path: "/reset", + action: (ctx) => import("./components/Reset").then(_ => <_.ResetDialog />) + }, // Main --- { mainNav: ["activities"], path: "", action: (context) => pushUrl("/activities") - }, { mainNav: ["activities"], diff --git a/esp/src/src-react/util/history.ts b/esp/src/src-react/util/history.ts index 5fe0bc2c1aa..d4420fc4d7e 100644 --- a/esp/src/src-react/util/history.ts +++ b/esp/src/src-react/util/history.ts @@ -92,6 +92,11 @@ const globalHistory = globalThis.history; const STORE_HISTORY_ID = "history"; +export function resetHistory() { + const store = userKeyValStore(); + return store?.delete(STORE_HISTORY_ID); +} + class History { location: HistoryLocation = { diff --git a/esp/src/src/KeyValStore.ts b/esp/src/src/KeyValStore.ts index 6dbb01ef0cf..ede424f2561 100644 --- a/esp/src/src/KeyValStore.ts +++ b/esp/src/src/KeyValStore.ts @@ -250,6 +250,13 @@ class CookieStorage implements IKeyValStore { }); } + async deleteAll(broadcast?: boolean): Promise { + const cookies = Utility.parseCookies(); + for (const cookie in cookies) { + await this.delete(cookie, broadcast); + } + } + monitor(callback: (messages: ValueChangedMessage[]) => void): IObserverHandle { return this._dispatch.attach(callback); } diff --git a/esp/src/src/Session.ts b/esp/src/src/Session.ts index f7a43044e3d..10c6fe5e186 100644 --- a/esp/src/src/Session.ts +++ b/esp/src/src/Session.ts @@ -35,6 +35,15 @@ export async function fetchModernMode(): Promise { }); } +export async function resetModernMode() { + await sessionStore.delete(ModernMode); + await userStore.delete(ModernMode); +} + +export async function resetCookies() { + await cookieStore.deleteAll(true); +} + const isV5DirectURL = () => !!parseSearch(window.location.search)?.["Widget"]; const isV9DirectURL = () => window.location.hash && window.location.hash.indexOf("#/stub/") !== 0; diff --git a/esp/src/src/nls/hpcc.ts b/esp/src/src/nls/hpcc.ts index 6f0389e9cdd..03f5324c2d0 100644 --- a/esp/src/src/nls/hpcc.ts +++ b/esp/src/src/nls/hpcc.ts @@ -151,6 +151,7 @@ export = { Content: "Content", Contents: "Contents", ContentType: "Content Type", + Cookies: "Cookies", CookiesNoticeLinkText: "Our Cookie Notice", CookiesAcceptButtonText: "Allow Cookies", Copy: "Copy", @@ -199,6 +200,7 @@ export = { Deactivate: "Deactivate", Debug: "Debug", DEF: "DEF", + Default: "Default", Defaults: "Defaults", Definition: "Definition", DefinitionID: "Definition ID", @@ -271,6 +273,7 @@ export = { ECL: "ECL", ECLWatchRequiresCookies: "ECL Watch requires cookies enabled to continue.", ECLWatchSessionManagement: "ECL Watch session management", + ECLWatchVersion: "ECL Watch Version", ECLWorkunit: "ECL Workunit", EdgeLabel: "Edge Label", Edges: "Edges", @@ -317,6 +320,7 @@ export = { ExcludeIndexes: "Exclude Indexes", ExpireDays: "Expire in (days)", ExpirationDate: "Expiration Date", + FactoryReset: "Factory Reset", FailIfNoSourceFile: "Fail If No Source File", Fatal: "Fatal", Favorites: "Favorites", @@ -557,6 +561,7 @@ export = { Message: "Message", Methods: "Methods", Metrics: "Metrics", + MetricOptions: "Metric Options", MetricsGraph: "Metrics/Graph", MetricsSQL: "Metrics (SQL)", Min: "Min", @@ -806,6 +811,7 @@ export = { RequiredForXML: "Required for spraying XML", Reschedule: "Reschedule", Reset: "Reset", + ResetUserSettings: "Reset User Settings", ResetThisQuery: "Reset This Query?", ResetViewToSelection: "Reset View to Selection", Resource: "Resource", @@ -839,7 +845,10 @@ export = { SampleResponse: "Sample Response", Sasha: "Sasha", Save: "Save", + SaveAs: "Save As", Scope: "Scope", + ScopeColumns: "Scope Columns", + ScopeTypes: "Scope Types", SearchResults: "Search Results", Seconds: "Seconds", SecondsRemaining: "Seconds Remaining", @@ -957,6 +966,7 @@ export = { TechPreview: "Tech Preview", Terminators: "Terminators", TestPages: "Test Pages", + Theme: "Theme", TheReturnedResults: "The returned results", ThorNetworkAddress: "Thor Network Address", ThorMasterAddress: "Thor Master Address", @@ -1137,6 +1147,7 @@ export = { Workflows: "Workflows", Workunit: "Workunit", WorkunitNotFound: "Workunit not found", + WorkunitOptions: "Workunit Options", Workunits: "Workunits", WorkUnitScopeDefaultPermissions: "Workunit Scope Default Permissions", Wrap: "Wrap", diff --git a/helm/examples/azure/log-analytics/loganalytics-hpcc-logaccessV2.yaml b/helm/examples/azure/log-analytics/loganalytics-hpcc-logaccessV2.yaml index ca249e5f262..ca4aa230a22 100644 --- a/helm/examples/azure/log-analytics/loganalytics-hpcc-logaccessV2.yaml +++ b/helm/examples/azure/log-analytics/loganalytics-hpcc-logaccessV2.yaml @@ -82,6 +82,14 @@ global: searchColumn: "PodName" columnMode: "DEFAULT" columnType: "string" + - type: "spanid" + searchColumn: "hpcc_log_spanid" + columnMode: "DEFAULT" + columnType: "string" + - type: "traceid" + searchColumn: "hpcc_log_traceid" + columnMode: "DEFAULT" + columnType: "string" secrets: esp: azure-logaccess: "azure-logaccess" diff --git a/helm/hpcc/values.schema.json b/helm/hpcc/values.schema.json index b2e1e742d85..312019680bc 100644 --- a/helm/hpcc/values.schema.json +++ b/helm/hpcc/values.schema.json @@ -3205,7 +3205,7 @@ "type": { "type": "string", "description" : "The searchable HPCC log column to be mapped - 'global' applies to all known fields", - "enum": [ "global", "workunits", "components", "audience", "class", "instance", "host", "node", "message", "logid", "processid", "threadid", "timestamp", "pod"] + "enum": [ "global", "workunits", "components", "audience", "class", "instance", "host", "node", "message", "logid", "processid", "threadid", "timestamp", "pod", "traceid", "spanid"] }, "timeStampColumn": { "description" : "Name of timestamp column related to mapped field (only requried for 'global' mapping)", diff --git a/helm/managed/logging/elastic/elastic4hpcclogs-hpcc-logaccess.yaml b/helm/managed/logging/elastic/elastic4hpcclogs-hpcc-logaccess.yaml index 45d5bac31d3..3edd1a7d668 100644 --- a/helm/managed/logging/elastic/elastic4hpcclogs-hpcc-logaccess.yaml +++ b/helm/managed/logging/elastic/elastic4hpcclogs-hpcc-logaccess.yaml @@ -77,3 +77,11 @@ global: searchColumn: "kubernetes.pod.name" columnMode: "DEFAULT" columnType: "string" + - type: "traceid" + searchColumn: "hpcc.log.traceid" + columnMode: "DEFAULT" + columnType: "string" + - type: "spanid" + searchColumn: "hpcc.log.spanid" + columnMode: "DEFAULT" + columnType: "string" \ No newline at end of file diff --git a/helm/managed/logging/loki-stack/grafana-hpcc-logaccess.yaml b/helm/managed/logging/loki-stack/grafana-hpcc-logaccess.yaml index 70d09058960..1b2375f09c8 100644 --- a/helm/managed/logging/loki-stack/grafana-hpcc-logaccess.yaml +++ b/helm/managed/logging/loki-stack/grafana-hpcc-logaccess.yaml @@ -21,17 +21,27 @@ global: - type: "components" storeName: "stream" searchColumn: "component" - columnMode: "MIN" + columnMode: "ALL" columnType: "string" - type: "timestamp" storeName: "values" - searchColumn: "time" - columnMode: "ALL" + searchColumn: "tsNs" + columnMode: "MIN" columnType: "datetime" - type: "pod" storeName: "stream" searchColumn: "pod" + columnMode: "DEFAULT" + columnType: "string" + - type: "message" + storeName: "values" + searchColumn: "log" + columnMode: "MIN" + columnType: "string" + - type: "node" + storeName: "stream" columnMode: "ALL" + searchColumn: "node_name" columnType: "string" secrets: esp: diff --git a/plugins/parquet/parquetembed.cpp b/plugins/parquet/parquetembed.cpp index 88cb29bd3f9..2808e699dc8 100644 --- a/plugins/parquet/parquetembed.cpp +++ b/plugins/parquet/parquetembed.cpp @@ -386,6 +386,7 @@ arrow::Status ParquetReader::processReadFile() else { __int64 totalTables = 0; + fileTableCounts.reserve(parquetFileReaders.size()); for (int i = 0; i < parquetFileReaders.size(); i++) { @@ -770,10 +771,17 @@ arrow::Status ParquetWriter::fieldToNode(const RtlFieldInfo *field, std::vector< case type_unicode: case type_varunicode: case type_decimal: - arrowFields.push_back(std::make_shared(name.str(), arrow::utf8())); //TODO add decimal encoding + arrowFields.push_back(std::make_shared(name.str(), arrow::utf8())); // TODO: add decimal encoding break; case type_data: - arrowFields.push_back(std::make_shared(name.str(), arrow::large_binary())); + if (field->type->length > 0) + { + arrowFields.push_back(std::make_shared(name.str(), arrow::fixed_size_binary(field->type->length))); + } + else + { + arrowFields.push_back(std::make_shared(name.str(), arrow::large_binary())); + } break; case type_record: arrowFields.push_back(std::make_shared(name.str(), makeChildRecord(field))); @@ -992,20 +1000,26 @@ void ParquetWriter::addFieldToBuilder(const RtlFieldInfo *field, unsigned len, c arrow::ArrayBuilder *fieldBuilder = getFieldBuilder(field); switch(fieldBuilder->type()->id()) { - case arrow::Type::type::STRING: + case arrow::Type::STRING: { arrow::StringBuilder *stringBuilder = static_cast(fieldBuilder); reportIfFailure(stringBuilder->Append(data, len)); break; } - case arrow::Type::type::LARGE_BINARY: + case arrow::Type::LARGE_BINARY: { arrow::LargeBinaryBuilder *largeBinaryBuilder = static_cast(fieldBuilder); reportIfFailure(largeBinaryBuilder->Append(data, len)); break; } + case arrow::Type::FIXED_SIZE_BINARY: + { + arrow::FixedSizeBinaryBuilder *fixedSizeBinaryBuilder = static_cast(fieldBuilder); + reportIfFailure(fixedSizeBinaryBuilder->Append(data)); + break; + } default: - failx("Incorrect type for String/Large_Binary addFieldToBuilder: %s", fieldBuilder->type()->ToString().c_str()); + failx("Incorrect type for String/Large_Binary/Fixed_Size_Binary addFieldToBuilder: %s", fieldBuilder->type()->ToString().c_str()); } } @@ -1178,6 +1192,8 @@ std::string_view ParquetRowBuilder::getCurrView(const RtlFieldInfo *field) return arrayVisitor->largeStringArr->GetView(currArrayIndex()); case DecimalType: return arrayVisitor->size == 128 ? arrayVisitor->decArr->GetView(currArrayIndex()) : arrayVisitor->largeDecArr->GetView(currArrayIndex()); + case FixedSizeBinaryType: + return arrayVisitor->fixedSizeBinaryArr->GetView(currArrayIndex()); default: failx("Unimplemented Parquet type for field with name %s.", field->name); } @@ -1459,9 +1475,8 @@ void ParquetRowBuilder::processBeginSet(const RtlFieldInfo *field, bool &isAll) if (arrayVisitor->type == ListType) { - ParquetColumnTracker newPathNode(field, arrayVisitor->listArr, CPNTSet); - newPathNode.childCount = arrayVisitor->listArr->value_slice(currentRow)->length(); - pathStack.push_back(newPathNode); + pathStack.emplace_back(field, arrayVisitor->listArr, CPNTSet); + pathStack.back().childCount = arrayVisitor->listArr->value_slice(currentRow)->length(); } else if (arrayVisitor->type == LargeListType) { @@ -1509,7 +1524,7 @@ void ParquetRowBuilder::processBeginRow(const RtlFieldInfo *field) nextField(field); if (arrayVisitor->type == StructType) { - pathStack.push_back(ParquetColumnTracker(field, arrayVisitor->structArr, CPNTScalar)); + pathStack.emplace_back(field, arrayVisitor->structArr, CPNTScalar); } else { diff --git a/plugins/parquet/parquetembed.hpp b/plugins/parquet/parquetembed.hpp index f7545026cae..e95e19967d1 100644 --- a/plugins/parquet/parquetembed.hpp +++ b/plugins/parquet/parquetembed.hpp @@ -133,7 +133,8 @@ enum ParquetArrayType ListType, LargeListType, StructType, - RealType + RealType, + FixedSizeBinaryType }; /** @@ -271,6 +272,13 @@ class ParquetArrayVisitor : public arrow::ArrayVisitor size = 8; return arrow::Status::OK(); } + arrow::Status Visit(const arrow::FixedSizeBinaryArray &array) + { + fixedSizeBinaryArr = &array; + type = FixedSizeBinaryType; + size = array.byte_width(); + return arrow::Status::OK(); + } arrow::Status Visit(const arrow::StringArray &array) { stringArr = &array; @@ -348,6 +356,7 @@ class ParquetArrayVisitor : public arrow::ArrayVisitor const arrow::HalfFloatArray *halfFloatArr = nullptr; const arrow::FloatArray *floatArr = nullptr; const arrow::DoubleArray *doubleArr = nullptr; + const arrow::FixedSizeBinaryArray *fixedSizeBinaryArr = nullptr; const arrow::StringArray *stringArr = nullptr; const arrow::LargeStringArray *largeStringArr = nullptr; const arrow::BinaryArray *binArr = nullptr; diff --git a/system/jlib/jdebug.cpp b/system/jlib/jdebug.cpp index 17154d4a99c..d939b8a417d 100644 --- a/system/jlib/jdebug.cpp +++ b/system/jlib/jdebug.cpp @@ -354,9 +354,16 @@ void calibrate_timing() useRDTSC = false; } #endif +#if defined (__APPLE__) + verifyex(mach_timebase_info(&timebase_info) == KERN_SUCCESS); + cycleToNanoScale = static_cast(timebase_info.numer)/timebase_info.denom; + cycleToMicroScale = cycleToNanoScale/1000.0; + cycleToMilliScale = cycleToNanoScale/1000000.0; +#else cycleToNanoScale = 1.0; - cycleToMicroScale = 1.0; - cycleToMilliScale = 1.0; + cycleToMicroScale = cycleToNanoScale/1000.0; + cycleToMilliScale = cycleToNanoScale/1000000.0; +#endif } @@ -405,7 +412,11 @@ __int64 jlib_decl cycle_to_microsec(cycle_t cycles) if (useRDTSC) return (__int64)((double)cycles * cycleToMicroScale); #endif +#ifdef __APPLE__ + return cycles * (uint64_t) timebase_info.numer / ((uint64_t)timebase_info.denom*1000); +#else return cycles / 1000; +#endif } __int64 jlib_decl cycle_to_millisec(cycle_t cycles) @@ -414,7 +425,11 @@ __int64 jlib_decl cycle_to_millisec(cycle_t cycles) if (useRDTSC) return (__int64)((double)cycles * cycleToMilliScale); #endif +#ifdef __APPLE__ + return cycles * (uint64_t) timebase_info.numer / ((uint64_t)timebase_info.denom*1000000); +#else return cycles / 1000000; +#endif } cycle_t nanosec_to_cycle(__int64 ns) @@ -423,7 +438,11 @@ cycle_t nanosec_to_cycle(__int64 ns) if (useRDTSC) return (__int64)((double)ns / cycleToNanoScale); #endif +#ifdef __APPLE__ + return ns * (uint64_t) timebase_info.denom / ((uint64_t)timebase_info.numer); +#else return ns; +#endif } double getCycleToNanoScale() diff --git a/system/jlib/jexcept.cpp b/system/jlib/jexcept.cpp index 1cbfb8bb150..313fbf9bef9 100644 --- a/system/jlib/jexcept.cpp +++ b/system/jlib/jexcept.cpp @@ -36,6 +36,8 @@ #include #include #include +#include +#include #include #ifdef __linux__ #include // comment out if not present @@ -1663,6 +1665,30 @@ void jlib_decl disableSEHtoExceptionMapping() #endif } +void jlib_decl raiseSignalInFuture(int signo, unsigned timeoutSec) +{ +#if defined(__linux__) + int ret; + timer_t timerId; + struct sigevent sigev; + struct itimerspec itSpec; + + sigev.sigev_notify = SIGEV_SIGNAL; + sigev.sigev_signo = signo; + sigev.sigev_value.sival_ptr = &timerId; + sigev.sigev_notify_function = nullptr; + sigev.sigev_notify_attributes = NULL; + + itSpec.it_value.tv_sec = timeoutSec; + itSpec.it_value.tv_nsec = 0; + itSpec.it_interval.tv_sec = 0; + itSpec.it_interval.tv_nsec = 0; + + ret = timer_create(CLOCK_MONOTONIC, &sigev, &timerId); + if (!ret) + timer_settime(timerId, 0, &itSpec, 0); +#endif +} StringBuffer & formatSystemError(StringBuffer & out, unsigned errcode) { diff --git a/system/jlib/jexcept.hpp b/system/jlib/jexcept.hpp index 3a80701c673..733513d4e49 100644 --- a/system/jlib/jexcept.hpp +++ b/system/jlib/jexcept.hpp @@ -128,6 +128,7 @@ void jlib_decl disableSEHtoExceptionMapping(); void jlib_decl *setSEHtoExceptionHandler(IExceptionHandler *handler); // sets handler and return old value +void jlib_decl raiseSignalInFuture(int signo, unsigned timeoutSec); void jlib_decl setTerminateOnSEHInSystemDLLs(bool set=true); void jlib_decl setTerminateOnSEH(bool set=true); diff --git a/system/jlib/jfile.cpp b/system/jlib/jfile.cpp index 9a62b29801e..a63073d87fd 100644 --- a/system/jlib/jfile.cpp +++ b/system/jlib/jfile.cpp @@ -7589,7 +7589,7 @@ class CFileEventWatcher : public CInterfaceOf, implements ITh { inotifyFd = inotify_init1(IN_NONBLOCK|IN_CLOEXEC); if (-1 == inotifyFd) - throw makeStringException(-1, "CFileEventWatcher - inotify_init1"); + throw makeErrnoException(-1, "CFileEventWatcher - inotify_init1"); if (pipe2(pipefd, O_NONBLOCK|O_CLOEXEC)) throw makeErrnoException(-1, "CFileEventWatcher - create pipe failed"); diff --git a/system/jlib/jstats.cpp b/system/jlib/jstats.cpp index e6b939e98c6..dd0ad0b2034 100644 --- a/system/jlib/jstats.cpp +++ b/system/jlib/jstats.cpp @@ -1329,6 +1329,8 @@ void StatisticsMapping::createMappings() ForEachItemIn(i2, indexToKind) { unsigned kind = indexToKind.item(i2); + unsigned existing = kindToIndex.item(kind); + assertex(existing == numStatistics()); // Throw an error if there are duplicate statistics in the mapping kindToIndex.replace(i2, kind); } diff --git a/system/logaccess/Azure/LogAnalytics/CurlClient/AzureLogAnalyticsCurlClient.cpp b/system/logaccess/Azure/LogAnalytics/CurlClient/AzureLogAnalyticsCurlClient.cpp index 529919dc36a..49a8213e5ff 100644 --- a/system/logaccess/Azure/LogAnalytics/CurlClient/AzureLogAnalyticsCurlClient.cpp +++ b/system/logaccess/Azure/LogAnalytics/CurlClient/AzureLogAnalyticsCurlClient.cpp @@ -344,6 +344,8 @@ AzureLogAnalyticsCurlClient::AzureLogAnalyticsCurlClient(IPropertyTree & logAcce m_workunitSearchColName.set(defaultHPCCLogJobIDCol); m_componentsSearchColName.set(defaultHPCCLogComponentCol); m_audienceSearchColName.set(defaultHPCCLogAudCol); + m_traceSearchColName.set(defaultHPCCLogTraceIDCol); + m_spanSearchColName.set(defaultHPCCLogSpanIDCol); Owned logMapIter = m_pluginCfg->getElements("logMaps"); ForEach(*logMapIter) @@ -443,19 +445,21 @@ AzureLogAnalyticsCurlClient::AzureLogAnalyticsCurlClient(IPropertyTree & logAcce if (logMap.hasProp(logMapSearchColAtt)) m_podSearchColName = logMap.queryProp(logMapSearchColAtt); } - else if (streq(logMapType, "trace")) + else if (streq(logMapType, "traceid")) { if (logMap.hasProp(logMapIndexPatternAtt)) m_traceIndexSearchPattern = logMap.queryProp(logMapIndexPatternAtt); - m_traceSearchColName = logMap.hasProp(logMapSearchColAtt) ? logMap.queryProp(logMapSearchColAtt) : defaultHPCCLogTraceIDCol; + if (logMap.hasProp(logMapSearchColAtt)) + m_traceSearchColName.set(logMap.queryProp(logMapSearchColAtt)); } - else if (streq(logMapType, "span")) + else if (streq(logMapType, "spanid")) { if (logMap.hasProp(logMapIndexPatternAtt)) m_spanIndexSearchPattern = logMap.queryProp(logMapIndexPatternAtt); - m_spanSearchColName = logMap.hasProp(logMapSearchColAtt) ? logMap.queryProp(logMapSearchColAtt) : defaultHPCCLogSpanIDCol; + if (logMap.hasProp(logMapSearchColAtt)) + m_spanSearchColName.set(logMap.queryProp(logMapSearchColAtt)); } else { diff --git a/system/logaccess/ElasticStack/ElasticStackLogAccess.cpp b/system/logaccess/ElasticStack/ElasticStackLogAccess.cpp index d87ff800779..878946703a1 100644 --- a/system/logaccess/ElasticStack/ElasticStackLogAccess.cpp +++ b/system/logaccess/ElasticStack/ElasticStackLogAccess.cpp @@ -94,6 +94,8 @@ ElasticStackLogAccess::ElasticStackLogAccess(const std::vector &hos m_componentsSearchColName.set(DEFAULT_HPCC_LOG_COMPONENT_COL); m_audienceSearchColName.set(DEFAULT_HPCC_LOG_AUD_COL); m_podSearchColName.set(DEFAULT_HPCC_LOG_POD_COL); + m_traceSearchColName.set(DEFAULT_HPCC_LOG_TRACE_COL); + m_spanSearchColName.set(DEFAULT_HPCC_LOG_SPAN_COL); Owned logMapIter = m_pluginCfg->getElements("logMaps"); ForEach(*logMapIter) @@ -171,19 +173,21 @@ ElasticStackLogAccess::ElasticStackLogAccess(const std::vector &hos else OERRLOG("%s: Possible LogMap collision detected, 'host' and 'node' refer to same log column!", COMPONENT_NAME); } - else if (streq(logMapType, "trace")) + else if (streq(logMapType, "traceid")) { if (logMap.hasProp(LOGMAP_INDEXPATTERN_ATT)) m_traceIndexSearchPattern = logMap.queryProp(LOGMAP_INDEXPATTERN_ATT); - m_traceSearchColName = logMap.hasProp(LOGMAP_SEARCHCOL_ATT) ? logMap.queryProp(LOGMAP_SEARCHCOL_ATT) : DEFAULT_HPCC_LOG_TRACE_COL; + if (logMap.hasProp(LOGMAP_SEARCHCOL_ATT)) + m_traceSearchColName = logMap.queryProp(LOGMAP_SEARCHCOL_ATT); } - else if (streq(logMapType, "span")) + else if (streq(logMapType, "spanid")) { if (logMap.hasProp(LOGMAP_INDEXPATTERN_ATT)) m_spanIndexSearchPattern = logMap.queryProp(LOGMAP_INDEXPATTERN_ATT); - m_spanSearchColName = logMap.hasProp(LOGMAP_SEARCHCOL_ATT) ? logMap.queryProp(LOGMAP_SEARCHCOL_ATT) : DEFAULT_HPCC_LOG_SPAN_COL; + if (logMap.hasProp(LOGMAP_SEARCHCOL_ATT)) + m_spanSearchColName = logMap.queryProp(LOGMAP_SEARCHCOL_ATT); } else { diff --git a/system/logaccess/Grafana/CurlClient/GrafanaCurlClient.cpp b/system/logaccess/Grafana/CurlClient/GrafanaCurlClient.cpp index 8e58aa8c596..505e2a82778 100644 --- a/system/logaccess/Grafana/CurlClient/GrafanaCurlClient.cpp +++ b/system/logaccess/Grafana/CurlClient/GrafanaCurlClient.cpp @@ -190,18 +190,25 @@ void GrafanaLogAccessCurlClient::processDatasourceJsonResp(const std::string & r } /* - * This method consumes a logLine string from a successful Grafana Loki query - * The LogLine is wrapped in the desired output format + * This method consumes a logLine represented as a set of field names and values + * The LogLine is wrapped in the requested output format */ -void formatResultLine(StringBuffer & returnbuf, const char * resultLine, const char * resultLineName, LogAccessLogFormat format, bool & isFirstLine) +void formatResultLine(StringBuffer & returnbuf, const IProperties * resultLine, LogAccessLogFormat format, bool & isFirstLine) { switch (format) { case LOGACCESS_LOGFORMAT_xml: { - returnbuf.appendf("<%s>", resultLineName); - encodeXML(resultLine, returnbuf); - returnbuf.appendf("", resultLineName); + returnbuf.append(""); + Owned fieldsIter = resultLine->getIterator(); + ForEach(*fieldsIter) + { + const char * prop = fieldsIter->queryPropValue(); + returnbuf.appendf("<%s>", fieldsIter->getPropKey()); + encodeXML(prop, returnbuf); + returnbuf.appendf("", fieldsIter->getPropKey()); + } + returnbuf.appendf(""); isFirstLine = false; break; } @@ -210,15 +217,41 @@ void formatResultLine(StringBuffer & returnbuf, const char * resultLine, const c if (!isFirstLine) returnbuf.append(", "); - returnbuf.append("\""); - encodeJSON(returnbuf,resultLine); - returnbuf.append("\""); + returnbuf.appendf("{\"fields\": [ "); + bool firstField = true; + Owned fieldsIter = resultLine->getIterator(); + ForEach(*fieldsIter) + { + if (!firstField) + returnbuf.append(", "); + else + firstField = false; + + const char * prop = fieldsIter->queryPropValue(); + returnbuf.appendf("{\"%s\":\"", fieldsIter->getPropKey()); + encodeJSON(returnbuf,prop); + returnbuf.append("\"}"); + } + returnbuf.append(" ]}"); + isFirstLine = false; break; } case LOGACCESS_LOGFORMAT_csv: { - encodeCSVColumn(returnbuf, resultLine); //Currently treating entire log line as a single CSV column + bool firstField = true; + Owned fieldsIter = resultLine->getIterator(); + ForEach(*fieldsIter) + { + if (!firstField) + returnbuf.append(", "); + else + firstField = false; + + const char * fieldValue = resultLine->queryProp(fieldsIter->getPropKey()); + encodeCSVColumn(returnbuf, fieldValue); + } + returnbuf.newline(); isFirstLine = false; break; @@ -230,18 +263,54 @@ void formatResultLine(StringBuffer & returnbuf, const char * resultLine, const c /* * This method consumes an Iterator of values elements from a successful Grafana Loki query - * It ignores the 1st child (ingest timestamp in ns), and formats the 2nd child (log line) into the desired format + * It extracts the appropriate "stream" values based on return column mode, and the values' 1st and 2nd children + * which represent timestamp in ns, and the log line, and formats into the requested format */ -void processValues(StringBuffer & returnbuf, IPropertyTreeIterator * valuesIter, LogAccessLogFormat format, bool & isFirstLine) +void GrafanaLogAccessCurlClient::processValues(StringBuffer & returnbuf, IPropertyTreeIterator * valuesIter, IPropertyTree * stream, LogAccessLogFormat format, const LogAccessReturnColsMode retcolmode, bool & isFirstLine) { + if (!valuesIter) + return; + + Owned fieldValues = createProperties(true); + + //extract the requested fields from the stream if it's available + if (stream) + { + switch (retcolmode) + { + case RETURNCOLS_MODE_all: + { + fieldValues->setProp(m_nodeColumn.name, stream->queryProp(m_nodeColumn.name)); + fieldValues->setProp(m_containerColumn.name, stream->queryProp(m_containerColumn.name)); + fieldValues->setProp(m_instanceColumn.name, stream->queryProp(m_instanceColumn.name)); + [[fallthrough]]; + } + case RETURNCOLS_MODE_default: + { + fieldValues->setProp(m_podColumn.name, stream->queryProp(m_podColumn.name)); + [[fallthrough]]; + } + case RETURNCOLS_MODE_min: + { + fieldValues->setProp(m_logDateTimstampColumn.name, stream->queryProp(m_logDateTimstampColumn.name)); + fieldValues->setProp(m_messageColumn.name, stream->queryProp(m_messageColumn.name)); + break; + } + case RETURNCOLS_MODE_custom: //not supported yet + default: + break; + } + } + ForEach(*valuesIter) { IPropertyTree & values = valuesIter->query(); int numofvalues = values.getCount("values"); if (values.getCount("values") == 2) { - //const char * insertTimeStamp = values.queryProp("values[1]"); - formatResultLine(returnbuf, values.queryProp("values[2]"), "line", format, isFirstLine); + fieldValues->setProp(m_logDateTimstampColumn.name, values.queryProp("values[1]")); + fieldValues->setProp(m_messageColumn.name, values.queryProp("values[2]")); + formatResultLine(returnbuf, fieldValues, format, isFirstLine); } else { @@ -268,14 +337,6 @@ inline void resultsWrapStart(StringBuffer & returnbuf, LogAccessLogFormat format break; } case LOGACCESS_LOGFORMAT_csv: - { - if (reportHeader) - { - returnbuf.append("line"); // this is the entire header for CSV if we're only reporting the line - returnbuf.newline(); - } - break; - } default: break; } @@ -305,31 +366,12 @@ inline void resultsWrapEnd(StringBuffer & returnbuf, LogAccessLogFormat format) } } -/* - * This method consumes JSON formatted elements from a successful Grafana Loki query - * It extracts all values elements processes them into the desired format - */ -void wrapResult(StringBuffer & returnbuf, IPropertyTree * result, LogAccessLogFormat format, bool & isFirstLine) -{ - Owned logLineIter; - - if (result->hasProp("values")) - { - logLineIter.setown(result->getElements("values")); - } - - processValues(returnbuf, logLineIter, format, isFirstLine); -} - /* * This method consumes the JSON response from a Grafana Loki query * It attempts to unwrap the response and extract the log payload, and reports it in the desired format */ -void GrafanaLogAccessCurlClient::processQueryJsonResp(LogQueryResultDetails & resultDetails, const std::string & retrievedDocument, StringBuffer & returnbuf, LogAccessLogFormat format, bool reportHeader) +void GrafanaLogAccessCurlClient::processQueryJsonResp(LogQueryResultDetails & resultDetails, const std::string & retrievedDocument, StringBuffer & returnbuf, const LogAccessLogFormat format, const LogAccessReturnColsMode retcolmode, bool reportHeader) { - resultDetails.totalReceived = 0; - resultDetails.totalAvailable = 0; - Owned tree = createPTreeFromJSONString(retrievedDocument.c_str()); if (!tree) throw makeStringExceptionV(-1, "%s: Could not parse log query response", COMPONENT_NAME); @@ -370,11 +412,17 @@ void GrafanaLogAccessCurlClient::processQueryJsonResp(LogQueryResultDetails & re bool isFirstLine = true; Owned resultIter = data->getElements("result"); - //many result elements can be returned, each with a unique set of labels + //many result elements can be returned, each with a unique set of labels and a common set of stream values ForEach(*resultIter) { IPropertyTree & result = resultIter->query(); - wrapResult(returnbuf, &result, format, isFirstLine); + Owned logLineIter; + + if (result.hasProp("values")) + { + logLineIter.setown(result.getElements("values")); // if no values elements found, will get NullPTreeIterator + processValues(returnbuf, logLineIter, result.queryPropTree("stream"), format, retcolmode, isFirstLine); + } } //Adds the format postfix to the return buffer @@ -465,9 +513,6 @@ void GrafanaLogAccessCurlClient::populateQueryFilterAndStreamSelector(StringBuff } case LOGACCESS_FILTER_wildcard: { - if (queryValue.isEmpty()) - throw makeStringExceptionV(-1, "%s: Wildcard filter cannot be empty!", COMPONENT_NAME); - DBGLOG("%s: Searching log entries by wildcard filter: '%s %s %s'...", COMPONENT_NAME, queryField.str(), queryOperator, queryValue.str()); break; } @@ -605,14 +650,14 @@ bool GrafanaLogAccessCurlClient::fetchLog(LogQueryResultDetails & resultDetails, //from https://grafana.com/docs/loki/latest/query/log_queries/ //Adding | json to your pipeline will extract all json properties as labels if the log line is a valid json document. Nested properties are flattened into label keys using the _ separator. logLineParser.set(" | json log"); //this parses the log entry and extracts the log field into a label - logLineParser.append(" | line_format \"{{.log}}\""); //Formats output line to only contain log label + logLineParser.append(" | line_format \"{{.log | trim}}\""); //Formats output line to only contain log label //This drops the stream, and various insert timestamps //we're always going to get a stream container, and a the log line... //the stream container contains unnecessary, and redundant lines //there's documentation of a 'drop' command whch doesn't work in practice //online recomendation is to clear those stream entries... - logLineParser.append(" | label_format log=\"\", filename=\"\", namespace=\"\", node_name=\"\", job=\"\"");// app=\"\", component=\"\", container=\"\", instance=\"\"); + logLineParser.append(" | label_format log=\"\", filename=\"\"");//, namespace=\"\", job=\"\""// app=\"\", component=\"\", container=\"\", instance=\"\"); /* we're not going to attempt to parse the log line for now, return the entire log line in raw format @@ -664,8 +709,7 @@ bool GrafanaLogAccessCurlClient::fetchLog(LogQueryResultDetails & resultDetails, std::string readBuffer; submitQuery(readBuffer, fullQuery.str()); - processQueryJsonResp(resultDetails, readBuffer, returnbuf, format, true); - //DBGLOG("Query fetchLog result: %s", readBuffer.c_str()); + processQueryJsonResp(resultDetails, readBuffer, returnbuf, format, options.getReturnColsMode(), true); } catch(IException * e) { @@ -676,6 +720,19 @@ bool GrafanaLogAccessCurlClient::fetchLog(LogQueryResultDetails & resultDetails, return false; } +void processLogMapConfig(const IPropertyTree * logMapConfig, LogField * targetField) +{ + if (!logMapConfig || !targetField) + return; + + if (logMapConfig->hasProp(logMapIndexPatternAtt)) + if (strcmp(logMapConfig->queryProp(logMapIndexPatternAtt), "stream")==0) + targetField->isStream = true; + + if (logMapConfig->hasProp(logMapSearchColAtt)) + targetField->name = logMapConfig->queryProp(logMapSearchColAtt); +} + GrafanaLogAccessCurlClient::GrafanaLogAccessCurlClient(IPropertyTree & logAccessPluginConfig) { m_pluginCfg.set(&logAccessPluginConfig); @@ -769,73 +826,29 @@ GrafanaLogAccessCurlClient::GrafanaLogAccessCurlClient(IPropertyTree & logAccess IPropertyTree & logMap = logMapIter->query(); const char * logMapType = logMap.queryProp("@type"); if (streq(logMapType, "global")) - { - if (logMap.hasProp(logMapIndexPatternAtt)) - if (strcmp(logMap.queryProp(logMapIndexPatternAtt), "stream")==0) - m_globalSearchCol.isStream = true; - - if (logMap.hasProp(logMapSearchColAtt)) - m_globalSearchCol.name = logMap.queryProp(logMapSearchColAtt); - } + processLogMapConfig(&logMap, &m_globalSearchCol); else if (streq(logMapType, "workunits")) - { - if (logMap.hasProp(logMapSearchColAtt)) - m_workunitsColumn = logMap.queryProp(logMapSearchColAtt); - } + processLogMapConfig(&logMap, &m_workunitsColumn); else if (streq(logMapType, "components")) - { - if (logMap.hasProp(logMapIndexPatternAtt)) - if (strcmp(logMap.queryProp(logMapIndexPatternAtt), "stream")==0) - m_componentsColumn.isStream = true; - - if (logMap.hasProp(logMapSearchColAtt)) - m_componentsColumn.name = logMap.queryProp(logMapSearchColAtt); - } + processLogMapConfig(&logMap, &m_componentsColumn); else if (streq(logMapType, "class")) - { - if (logMap.hasProp(logMapSearchColAtt)) - m_classColumn = logMap.queryProp(logMapSearchColAtt); - } + processLogMapConfig(&logMap, &m_classColumn); else if (streq(logMapType, "audience")) - { - if (logMap.hasProp(logMapSearchColAtt)) - m_audienceColumn = logMap.queryProp(logMapSearchColAtt); - } + processLogMapConfig(&logMap, &m_audienceColumn); else if (streq(logMapType, "instance")) - { - if (logMap.hasProp(logMapIndexPatternAtt)) - if (strcmp(logMap.queryProp(logMapIndexPatternAtt), "stream")==0) - m_instanceColumn.isStream = true; - - if (logMap.hasProp(logMapSearchColAtt)) - m_instanceColumn.name = logMap.queryProp(logMapSearchColAtt); - } + processLogMapConfig(&logMap, &m_instanceColumn); else if (streq(logMapType, "node")) - { - if (logMap.hasProp(logMapIndexPatternAtt)) - if (strcmp(logMap.queryProp(logMapIndexPatternAtt), "stream")==0) - m_nodeColumn.isStream = true; - - if (logMap.hasProp(logMapSearchColAtt)) - m_nodeColumn.name = logMap.queryProp(logMapSearchColAtt); - } + processLogMapConfig(&logMap, &m_nodeColumn); else if (streq(logMapType, "host")) - { OWARNLOG("%s: 'host' LogMap entry is NOT supported!", COMPONENT_NAME); - } else if (streq(logMapType, "pod")) - { - if (logMap.hasProp(logMapIndexPatternAtt)) - if (strcmp(logMap.queryProp(logMapIndexPatternAtt), "stream")==0) - m_podColumn.isStream = true; - - if (logMap.hasProp(logMapSearchColAtt)) - m_podColumn.name = logMap.queryProp(logMapSearchColAtt); - } + processLogMapConfig(&logMap, &m_podColumn); + else if (streq(logMapType, "message")) + processLogMapConfig(&logMap, &m_messageColumn); + else if (streq(logMapType, "timestamp")) + processLogMapConfig(&logMap, &m_logDateTimstampColumn); else - { ERRLOG("Encountered invalid LogAccess field map type: '%s'", logMapType); - } } DBGLOG("%s: targeting: '%s' - datasource: '%s'", COMPONENT_NAME, m_grafanaConnectionStr.str(), m_dataSourcesAPIURI.str()); diff --git a/system/logaccess/Grafana/CurlClient/GrafanaCurlClient.hpp b/system/logaccess/Grafana/CurlClient/GrafanaCurlClient.hpp index fb6f71cff98..5953161f5a9 100644 --- a/system/logaccess/Grafana/CurlClient/GrafanaCurlClient.hpp +++ b/system/logaccess/Grafana/CurlClient/GrafanaCurlClient.hpp @@ -73,13 +73,14 @@ class GrafanaLogAccessCurlClient : public CInterfaceOf LogField m_instanceColumn = LogField("instance", true); LogField m_podColumn = LogField("pod", true); LogField m_containerColumn = LogField("container", true); - LogField m_messageColumn = LogField("MSG"); + LogField m_messageColumn = LogField("log"); LogField m_nodeColumn = LogField("node_name", true); - LogField m_logTimestampColumn = LogField("TIME"); - LogField m_logDatestampColumn = LogField("DATE"); - LogField m_logSequesnceColumn = LogField("MID"); - LogField m_logProcIDColumn = LogField("PID"); - LogField m_logThreadIDColumn = LogField("TID"); + LogField m_logDateTimstampColumn = LogField("tsNs"); + //LogField m_logTimestampColumn = LogField("TIME"); + //LogField m_logDatestampColumn = LogField("DATE"); + //LogField m_logSequesnceColumn = LogField("MID"); + //LogField m_logProcIDColumn = LogField("PID"); + //LogField m_logThreadIDColumn = LogField("TID"); //LogField m_logTraceIDColumn = LogField("TRC"); //LogField m_logSpanIDColumn = LogField("SPN"); @@ -87,8 +88,9 @@ class GrafanaLogAccessCurlClient : public CInterfaceOf public: GrafanaLogAccessCurlClient(IPropertyTree & logAccessPluginConfig); - void processQueryJsonResp(LogQueryResultDetails & resultDetails, const std::string & retrievedDocument, StringBuffer & returnbuf, LogAccessLogFormat format, bool reportHeader); + void processQueryJsonResp(LogQueryResultDetails & resultDetails, const std::string & retrievedDocument, StringBuffer & returnbuf, const LogAccessLogFormat format, const LogAccessReturnColsMode retcolmode, bool reportHeader); void processDatasourceJsonResp(const std::string & retrievedDocument); + void processValues(StringBuffer & returnbuf, IPropertyTreeIterator * valuesIter, IPropertyTree * stream, LogAccessLogFormat format, const LogAccessReturnColsMode retcolmode, bool & isFirstLine); void fetchDatasourceByName(const char * targetDataSourceName); void fetchDatasources(std::string & readBuffer); void fetchLabels(std::string & readBuffer); diff --git a/system/mp/mpcomm.cpp b/system/mp/mpcomm.cpp index e68a7a13562..6a3d17658ff 100644 --- a/system/mp/mpcomm.cpp +++ b/system/mp/mpcomm.cpp @@ -836,6 +836,7 @@ class CMPServer: private CMPChannelHT, implements IMPServer bool tryReopenChannel = false; bool useTLS = false; unsigned mpTraceLevel = 0; + bool dumpQueue = true; // packet handlers PingPacketHandler *pingpackethandler; // TAG_SYS_PING @@ -2724,10 +2725,13 @@ CMPServer::CMPServer(unsigned __int64 _role, unsigned _port, bool _listen) CMPServer::~CMPServer() { #ifdef _TRACEORPHANS - StringBuffer buf; - getReceiveQueueDetails(buf); - if (buf.length()) - LOG(MCdebugInfo, "MP: Orphan check\n%s",buf.str()); + if (dumpQueue) + { + StringBuffer buf; + getReceiveQueueDetails(buf); + if (buf.length()) + LOG(MCdebugInfo, "MP: Orphan check\n%s",buf.str()); + } #endif _releaseAll(); selecthandler->stop(true); @@ -3625,6 +3629,7 @@ class CGlobalMPServer : public CMPServer unsigned queryNest() { return nestLevel; } bool isPaused() const { return paused; } void setPaused(bool onOff) { paused = onOff; } + void setDumpQueue(bool onOff) { dumpQueue = onOff; } }; CriticalSection CGlobalMPServer::sect; static CGlobalMPServer *globalMPServer; @@ -3669,7 +3674,7 @@ void startMPServer(unsigned port, bool paused, bool listen) startMPServer(0, port, paused, listen); } -void stopMPServer() +void stopMPServer(bool dumpQueue) { CGlobalMPServer *_globalMPServer = NULL; { @@ -3688,6 +3693,7 @@ void stopMPServer() } if (NULL == _globalMPServer) return; + _globalMPServer->setDumpQueue(dumpQueue); _globalMPServer->stop(); _globalMPServer->Release(); #ifdef _TRACE diff --git a/system/mp/mpcomm.hpp b/system/mp/mpcomm.hpp index 4825640e071..e91af5f9f5b 100644 --- a/system/mp/mpcomm.hpp +++ b/system/mp/mpcomm.hpp @@ -107,7 +107,7 @@ interface IMPServer : extends IInterface extern mp_decl void startMPServer(unsigned port, bool paused=false, bool listen=false); extern mp_decl void startMPServer(unsigned __int64 role, unsigned port, bool paused=false, bool listen=false); -extern mp_decl void stopMPServer(); +extern mp_decl void stopMPServer(bool dumpQueue=true); extern mp_decl IMPServer *getMPServer(); extern mp_decl IMPServer *startNewMPServer(unsigned port, bool listen=false); diff --git a/system/security/LdapSecurity/ldapconnection.cpp b/system/security/LdapSecurity/ldapconnection.cpp index 35e74cd1450..ef4eb7ab07a 100644 --- a/system/security/LdapSecurity/ldapconnection.cpp +++ b/system/security/LdapSecurity/ldapconnection.cpp @@ -1671,8 +1671,8 @@ class CLdapClient : implements ILdapClient, public CInterface //Create base LDAP OU tree. Specify PT_ADMINISTRATORS_ONLY to ensure each OU //grants access to Administrators only createLdapBasedn(NULL, m_ldapconfig->getResourceBasedn(RT_DEFAULT), PT_ADMINISTRATORS_ONLY, nullptr); - createLdapBasedn(NULL, m_ldapconfig->getResourceBasedn(RT_FILE_SCOPE), PT_DEFAULT, nullptr); - createLdapBasedn(NULL, m_ldapconfig->getResourceBasedn(RT_WORKUNIT_SCOPE), PT_DEFAULT, nullptr); + createLdapBasedn(NULL, m_ldapconfig->getResourceBasedn(RT_FILE_SCOPE), PT_ADMINISTRATORS_ONLY, nullptr); + createLdapBasedn(NULL, m_ldapconfig->getResourceBasedn(RT_WORKUNIT_SCOPE), PT_ADMINISTRATORS_ONLY, nullptr); createLdapBasedn(NULL, m_ldapconfig->getUserBasedn(), PT_ADMINISTRATORS_ONLY, nullptr); createLdapBasedn(NULL, m_ldapconfig->getGroupBasedn(), PT_ADMINISTRATORS_ONLY, nullptr); diff --git a/testing/regress/ecl/assert.ecl b/testing/regress/ecl/assert.ecl index 4b2ec214ff3..e45df9154b9 100644 --- a/testing/regress/ecl/assert.ecl +++ b/testing/regress/ecl/assert.ecl @@ -63,9 +63,21 @@ rec t(ds l) := transform end; o3 := output(project(ds, t(LEFT))); -sequential(o1, o2, o3); -ASSERT(COUNT(ds) = 9); // should be 2 + +o4 := ASSERT(COUNT(ds) = 9); // should be 2 // Test unicode formatting utf8 euro := u'€' : stored('euro'); -ASSERT(euro=u'€€'); // fail! +o5 := ASSERT(euro=u'€€'); // fail! + + +childds(integer id) := Function + + base := DATASET(3, TRANSFORM({unsigned x}, SELF.x := id + COUNTER)); + check := ASSERT(id != 0, 'Oh no'); + RETURN WHEN(base, check, BEFORE); +END; + +o6 := output(ds1(SUM(childds(val1), x) != 3 * val1 + 6)); + +sequential(o1, o2, o3, o4, o5, o6); diff --git a/testing/regress/ecl/key/assert.xml b/testing/regress/ecl/key/assert.xml index 2f06173f743..6677864dd82 100644 --- a/testing/regress/ecl/key/assert.xml +++ b/testing/regress/ecl/key/assert.xml @@ -16,6 +16,7 @@ 100000assert.ecl47userAbc7 100000assert.ecl67userAssert (2 = 9) failed [COUNT(ds) = 9] 100000assert.ecl71userAssert (€ = €€) failed [euro = U'€€'] +100000assert.ecl36userAssert (2 = 1) failed [val1 = 1] 1 2 @@ -28,3 +29,5 @@ 1One 2Two + + diff --git a/thorlcr/slave/slavmain.cpp b/thorlcr/slave/slavmain.cpp index d3eb593354b..1b6cc826f77 100644 --- a/thorlcr/slave/slavmain.cpp +++ b/thorlcr/slave/slavmain.cpp @@ -53,6 +53,8 @@ #include "rtlcommon.hpp" #include "../activities/keyedjoin/thkeyedjoincommon.hpp" +bool recvShutdown = false; + //--------------------------------------------------------------------------- //--------------------------------------------------------------------------- @@ -1497,7 +1499,8 @@ class CKJService : public CSimpleInterfaceOf, implements IThreaded, } catch (IMP_Exception *e) { - EXCLOG(e, nullptr); + if (!recvShutdown) + EXCLOG(e, nullptr); e->Release(); break; } @@ -2094,6 +2097,7 @@ class CJobListener : public CSimpleInterface case Shutdown: { stopped = true; + recvShutdown = true; PROGLOG("Shutdown received"); if (watchdog) watchdog->stop(); diff --git a/thorlcr/slave/slavmain.hpp b/thorlcr/slave/slavmain.hpp index 4e8af88bde0..2ce336977bf 100644 --- a/thorlcr/slave/slavmain.hpp +++ b/thorlcr/slave/slavmain.hpp @@ -24,6 +24,7 @@ void slaveMain(bool &jobListenerStopped, ILogMsgHandler *logHandler); void enableThorSlaveAsDaliClient(); void disableThorSlaveAsDaliClient(); +extern bool recvShutdown; #endif diff --git a/thorlcr/slave/thslavemain.cpp b/thorlcr/slave/thslavemain.cpp index c1fa682a57c..c7c8d3107fc 100644 --- a/thorlcr/slave/thslavemain.cpp +++ b/thorlcr/slave/thslavemain.cpp @@ -50,8 +50,6 @@ #include "dafdesc.hpp" #include "rmtfile.hpp" -#include "slavmain.hpp" - #ifdef _CONTAINERIZED #include "dafsserver.hpp" #endif @@ -287,10 +285,22 @@ bool UnregisterSelf(IException *e) bool ControlHandler(ahType type) { + static bool recvdSig = false; + if (recvdSig) + { + if (ahInterrupt == type) + _exit(128+SIGINT); + else + _exit(128+SIGTERM); + } + recvdSig = true; + raiseSignalInFuture(SIGTERM, 20); + if (ahInterrupt == type) LOG(MCdebugProgress, "CTRL-C detected"); else if (!jobListenerStopped) LOG(MCdebugProgress, "SIGTERM detected"); + bool unregOK = false; if (!jobListenerStopped) { @@ -298,6 +308,8 @@ bool ControlHandler(ahType type) unregOK = UnregisterSelf(NULL); abortSlave(); } + if (recvShutdown) + return false; return !unregOK; } @@ -599,7 +611,7 @@ int main( int argc, const char *argv[] ) setMultiThorMemoryNotify(0,NULL); roxiemem::releaseRoxieHeap(); - if (unregisterException.get()) + if (!recvShutdown && unregisterException.get()) UnregisterSelf(unregisterException); if (getExpertOptBool("slaveDaliClient")) @@ -608,7 +620,7 @@ int main( int argc, const char *argv[] ) #ifdef USE_MP_LOG stopLogMsgReceivers(); #endif - stopMPServer(); + stopMPServer(!recvShutdown); releaseAtoms(); // don't know why we can't use a module_exit to destruct these... ExitModuleObjects(); // not necessary, atexit will call, but good for leak checking diff --git a/thorlcr/thorutil/thormisc.cpp b/thorlcr/thorutil/thormisc.cpp index fa2a6164204..286c18b6791 100644 --- a/thorlcr/thorutil/thormisc.cpp +++ b/thorlcr/thorutil/thormisc.cpp @@ -85,15 +85,15 @@ const StatisticsMapping keyedJoinActivityStatistics({ StNumIndexAccepted, StNumP const StatisticsMapping commonJoinActivityStatistics({StNumMatchLeftRowsMax, StNumMatchRightRowsMax, StNumMatchCandidates, StNumMatchCandidatesMax}, basicActivityStatistics); const StatisticsMapping hashJoinActivityStatistics({StNumLeftRows, StNumRightRows}, commonJoinActivityStatistics); const StatisticsMapping allJoinActivityStatistics({}, commonJoinActivityStatistics); -const StatisticsMapping lookupJoinActivityStatistics({StNumSmartJoinSlavesDegradedToStd, StNumSmartJoinDegradedToLocal}, spillStatistics, commonJoinActivityStatistics); -const StatisticsMapping joinActivityStatistics({StNumLeftRows, StNumRightRows}, commonJoinActivityStatistics, spillStatistics); +const StatisticsMapping lookupJoinActivityStatistics({StNumSmartJoinSlavesDegradedToStd, StNumSmartJoinDegradedToLocal}, commonJoinActivityStatistics); +const StatisticsMapping joinActivityStatistics({StNumLeftRows, StNumRightRows}, commonJoinActivityStatistics); const StatisticsMapping diskReadActivityStatistics({StNumDiskRowsRead, }, basicActivityStatistics, diskReadRemoteStatistics); const StatisticsMapping diskWriteActivityStatistics({StPerReplicated}, basicActivityStatistics, diskWriteRemoteStatistics); -const StatisticsMapping sortActivityStatistics({}, basicActivityStatistics, spillStatistics); +const StatisticsMapping sortActivityStatistics({}, basicActivityStatistics); const StatisticsMapping diskReadPartStatistics({StNumDiskRowsRead}, diskReadRemoteStatistics); const StatisticsMapping indexDistribActivityStatistics({}, basicActivityStatistics, jhtreeCacheStatistics); const StatisticsMapping soapcallActivityStatistics({}, basicActivityStatistics, soapcallStatistics); -const StatisticsMapping hashDedupActivityStatistics({}, spillStatistics, diskWriteRemoteStatistics, basicActivityStatistics); +const StatisticsMapping hashDedupActivityStatistics({}, diskWriteRemoteStatistics, basicActivityStatistics); const StatisticsMapping hashDistribActivityStatistics({StNumLocalRows, StNumRemoteRows, StSizeRemoteWrite}, basicActivityStatistics); const StatisticsMapping loopActivityStatistics({StNumIterations}, basicActivityStatistics); const StatisticsMapping graphStatistics({StNumExecutions, StSizeSpillFile, StSizeGraphSpill, StSizePeakTempDisk, StSizePeakEphemeralDisk, StTimeUser, StTimeSystem, StNumContextSwitches, StSizeMemory, StSizePeakMemory, StSizeRowMemory, StSizePeakRowMemory}, executeStatistics);