diff --git a/.github/workflows/build-assets.yml b/.github/workflows/build-assets.yml index 7e5c7c8c33a..ad8e387ab29 100644 --- a/.github/workflows/build-assets.yml +++ b/.github/workflows/build-assets.yml @@ -425,20 +425,6 @@ jobs: cmake -S /hpcc-dev/LN -B /hpcc-dev/build -DHPCC_SOURCE_DIR=/hpcc-dev/HPCC-Platform ${{ needs.preamble.outputs.cmake_docker_config }} -DBUILD_LEVEL=ENTERPRISE -DSIGN_MODULES_PASSPHRASE=${{ secrets.SIGN_MODULES_PASSPHRASE }} -DSIGN_MODULES_KEYID=${{ secrets.SIGN_MODULES_KEYID }} -DPLATFORM=ON -DINCLUDE_PLUGINS=ON -DCONTAINERIZED=OFF -DSUPPRESS_REMBED=ON -DSUPPRESS_V8EMBED=ON -DSUPPRESS_SPARK=ON -DCPACK_STRIP_FILES=OFF && \ cmake --build /hpcc-dev/build --parallel $(nproc) --target package" - - name: Upload Assets (enterprise) - if: ${{ matrix.ee }} - uses: ncipollo/release-action@v1.12.0 - with: - allowUpdates: true - generateReleaseNotes: false - prerelease: ${{ contains(github.ref, '-rc') }} - owner: ${{ secrets.LNB_ACTOR }} - repo: LN - token: ${{ secrets.LNB_TOKEN }} - tag: ${{ needs.preamble.outputs.internal_tag }} - artifacts: "${{ needs.preamble.outputs.folder_build }}/hpccsystems-platform-enterprise*.deb,${{ needs.preamble.outputs.folder_build }}/hpccsystems-platform-enterprise*.rpm" - - # Common --- - name: Cleanup Environment if: always() @@ -704,12 +690,22 @@ jobs: C:\"Program Files (x86)"\"Microsoft SDKs"\ClickOnce\SignTool\signtool.exe sign /debug /f ../../sign/hpcc_code_signing.pfx /p ${{ secrets.SIGNING_CERTIFICATE_PASSPHRASE}} /t http://timestamp.digicert.com /fd SHA256 hpccsystems-eclide*.exe - name: Upload Assets - uses: ncipollo/release-action@v1.12.0 + uses: ncipollo/release-action@v1.14.0 with: allowUpdates: true generateReleaseNotes: false prerelease: ${{ contains(github.ref, '-rc') }} artifacts: "./ECLIDE/build/*.exe" + + - name: Upload Assets to Jfrog (windows) + if: ${{ github.repository_owner == 'hpcc-systems' }} + shell: bash + run: | + cd ./ECLIDE/build + packages=($(ls -1 hpccsystems-*.exe )) + for _package in ${packages[@]}; do + curl -u${{ secrets.JFROG_USERNAME }}:${{ secrets.JFROG_PASSWORD }} "https://${{ secrets.JFROG_REGISTRY }}/hpccpl-windows-local/LN/windows/x86_64/${_package}" -T ${_package} + done - name: Upload error logs if: ${{ failure() || cancelled() }} diff --git a/common/thorhelper/thorsoapcall.cpp b/common/thorhelper/thorsoapcall.cpp index 000d4a80e81..e7ea8283878 100644 --- a/common/thorhelper/thorsoapcall.cpp +++ b/common/thorhelper/thorsoapcall.cpp @@ -2559,10 +2559,16 @@ class CWSCAsyncFor : implements IWSCAsyncFor, public CInterface, public CAsyncFo { checkTimeLimitExceeded(&remainingMS); checkRoxieAbortMonitor(master->roxieAbortMonitor); - OwnedSpanScope socketOperationSpan = master->activitySpanScope->createClientSpan("Socket Write"); - setSpanURLAttributes(socketOperationSpan, url); - Owned traceHeaders = ::getClientHeaders(socketOperationSpan); + StringBuffer spanName; + spanName.appendf("%s %s %s:%d", getWsCallTypeName(master->wscType), master->service.str(), url.host.str(), url.port); + OwnedSpanScope requestSpan = master->activitySpanScope->createClientSpan(spanName.str()); + + setSpanURLAttributes(requestSpan, url); + requestSpan->setSpanAttribute("request.type", getWsCallTypeName(master->wscType)); + requestSpan->setSpanAttribute("service.name", master->service.str()); + + Owned traceHeaders = ::getClientHeaders(requestSpan); createHttpRequest(request, url, traceHeaders); socket->write(request.str(), request.length()); @@ -2575,7 +2581,7 @@ class CWSCAsyncFor : implements IWSCAsyncFor, public CInterface, public CAsyncFo bool keepAlive2; StringBuffer contentType; int rval = readHttpResponse(response, socket, keepAlive2, contentType); - socketOperationSpan->setSpanAttribute("http.response.status_code", (int64_t)rval); + requestSpan->setSpanAttribute("http.response.status_code", (int64_t)rval); keepAlive = keepAlive && keepAlive2; if (soapTraceLevel > 4) @@ -2583,22 +2589,22 @@ class CWSCAsyncFor : implements IWSCAsyncFor, public CInterface, public CAsyncFo if (rval != 200) { - socketOperationSpan->setSpanStatusSuccess(false); + requestSpan->setSpanStatusSuccess(false); if (rval == 503) { - socketOperationSpan->recordError(SpanError("Server Too Busy", 1001, true, true)); + requestSpan->recordError(SpanError("Server Too Busy", 1001, true, true)); throw new ReceivedRoxieException(1001, "Server Too Busy"); } StringBuffer text; text.appendf("HTTP error (%d) in processQuery",rval); rtlAddExceptionTag(text, "soapresponse", response.str()); - socketOperationSpan->recordError(SpanError(text.str(), -1, true, true)); + requestSpan->recordError(SpanError(text.str(), -1, true, true)); throw MakeStringExceptionDirect(-1, text.str()); } if (response.length() == 0) { - socketOperationSpan->recordError(SpanError("Zero length response in processQuery", -1, true, true)); + requestSpan->recordError(SpanError("Zero length response in processQuery", -1, true, true)); throw MakeStringException(-1, "Zero length response in processQuery"); } checkTimeLimitExceeded(&remainingMS); @@ -2614,7 +2620,7 @@ class CWSCAsyncFor : implements IWSCAsyncFor, public CInterface, public CAsyncFo persistentHandler->add(socket, &ep, proto); } - socketOperationSpan->setSpanStatusSuccess(true); + requestSpan->setSpanStatusSuccess(true); break; } catch (IReceivedRoxieException *e) diff --git a/esp/services/ws_store/espstorelib/daliKVStore.cpp b/esp/services/ws_store/espstorelib/daliKVStore.cpp index b3f1a40419d..cc45f00ce40 100644 --- a/esp/services/ws_store/espstorelib/daliKVStore.cpp +++ b/esp/services/ws_store/espstorelib/daliKVStore.cpp @@ -17,18 +17,29 @@ #include "daliKVStore.hpp" +void getEncodedLowerCaseUserName(StringBuffer & out, ISecUser * username) +{ + StringBuffer userlowercased; + userlowercased.set(username->getName()).toLowerCase(); + + encodePTreeName(out, userlowercased.str()); +} + bool CDALIKVStore::createStore(const char * apptype, const char * storename, const char * description, ISecUser * owner, unsigned int maxvalsize=DALI_KVSTORE_MAXVALSIZE_DEFAULT) { if (!storename || !*storename) throw MakeStringException(-1, "DALI Keystore createStore(): Store name not provided"); + StringBuffer encodedStoreName; + encodePTreeName(encodedStoreName, storename); + ensureAttachedToDali(); //throws if in offline mode Owned conn = querySDS().connect(DALI_KVSTORE_PATH, myProcessSession(), RTM_LOCK_READ, SDS_LOCK_TIMEOUT_KVSTORE); if (!conn) throw MakeStringException(-1, "Unable to connect to DALI KeyValue store root: '%s'", DALI_KVSTORE_PATH); - VStringBuffer xpath("Store[%s='%s'][1]", DALI_KVSTORE_NAME_ATT, storename); + VStringBuffer xpath("Store[%s='%s'][1]", DALI_KVSTORE_NAME_ATT, encodedStoreName.str()); { Owned root = conn->getRoot(); if (root->hasProp(xpath.str())) @@ -53,7 +64,7 @@ bool CDALIKVStore::createStore(const char * apptype, const char * storename, con } Owned apptree = createPTree(); - apptree->setProp(DALI_KVSTORE_NAME_ATT, storename); + apptree->setProp(DALI_KVSTORE_NAME_ATT, encodedStoreName.str()); CDateTime dt; dt.setNow(); StringBuffer str; @@ -89,15 +100,21 @@ bool CDALIKVStore::set(const char * storename, const char * thenamespace, const if (isEmptyString(storename)) throw MakeStringException(-1, "DALI Keystore set(): Store name not provided"); + StringBuffer encodedStoreName; + encodePTreeName(encodedStoreName, storename); + if (!global && (!owner || isEmptyString(owner->getName()))) throw MakeStringException(-1, "DALI Keystore set(): Attempting to set non-global entry but owner name not provided"); if (isEmptyString(thenamespace)) throw MakeStringException(-1, "DALI Keystore set(): namespace not provided"); + StringBuffer encodedNameSpace; + encodePTreeName(encodedNameSpace, thenamespace); + ensureAttachedToDali(); //throws if in offline mode - VStringBuffer xpath("%s/Store[%s='%s'][1]", DALI_KVSTORE_PATH, DALI_KVSTORE_NAME_ATT, storename); + VStringBuffer xpath("%s/Store[%s='%s'][1]", DALI_KVSTORE_PATH, DALI_KVSTORE_NAME_ATT, encodedStoreName.str()); Owned conn = querySDS().connect(xpath.str(), myProcessSession(), RTM_LOCK_WRITE, SDS_LOCK_TIMEOUT_KVSTORE); if (!conn) throw MakeStringException(-1, "DALI Keystore set(): Unable to connect to DALI KeyValue store path '%s'", xpath.str()); //rodrigo, not sure if this is too much info @@ -111,7 +128,7 @@ bool CDALIKVStore::set(const char * storename, const char * thenamespace, const if (global) xpath.set(DALI_KVSTORE_GLOBAL); else - xpath.set(owner->getName()).toLowerCase(); + getEncodedLowerCaseUserName(xpath.clear(), owner); Owned ownertree = storetree->getPropTree(xpath.str()); if (!ownertree) @@ -121,18 +138,20 @@ bool CDALIKVStore::set(const char * storename, const char * thenamespace, const dt.setNow(); StringBuffer str; - Owned nstree = ownertree->getPropTree(thenamespace); + Owned nstree = ownertree->getPropTree(encodedNameSpace.str()); if (!nstree) { - nstree.setown(createPTree(thenamespace)); + nstree.setown(createPTree(encodedNameSpace.str())); nstree->setProp(DALI_KVSTORE_CREATEDTIME_ATT,dt.getString(str).str()); } - Owned valuetree = nstree->getPropTree(key); + StringBuffer encodedKey; + encodePTreeName(encodedKey, key); + Owned valuetree = nstree->getPropTree(encodedKey.str()); if (!valuetree) { - nstree->setProp(key, value); - valuetree.setown(nstree->getPropTree(key)); + nstree->setProp(encodedKey.str(), value); + valuetree.setown(nstree->getPropTree(encodedKey.str())); valuetree->setProp(DALI_KVSTORE_CREATEDTIME_ATT,dt.getString(str).str()); valuetree->setProp(DALI_KVSTORE_CREATEDBY_ATT, owner ? owner->getName(): ""); } @@ -143,7 +162,7 @@ bool CDALIKVStore::set(const char * storename, const char * thenamespace, const valuetree->setProp(".", value); } - ownertree->setPropTree(thenamespace, LINK(nstree)); + ownertree->setPropTree(encodedNameSpace.str(), LINK(nstree)); storetree->setPropTree(xpath.str(), LINK(ownertree)); conn->commit(); @@ -156,26 +175,31 @@ IPropertyTree * CDALIKVStore::getAllKeyProperties(const char * storename, const if (isEmptyString(storename)) throw MakeStringException(-1, "DALI Keystore fetchKeyProperties(): Store name not provided"); + StringBuffer encodedStoreName; + encodePTreeName(encodedStoreName, storename); + if (!global && (!username || isEmptyString(username->getName()))) throw MakeStringException(-1, "DALI Keystore fetchKeyProperties(): Attempting to set non-global entry but owner name not provided"); if (isEmptyString(ns)) throw MakeStringException(-1, "DALI Keystore fetchKeyProperties(): namespace not provided"); + StringBuffer encodedNS; + encodePTreeName(encodedNS, ns); + ensureAttachedToDali(); //throws if in offline mode - VStringBuffer xpath("%s/Store[%s='%s'][1]/", DALI_KVSTORE_PATH, DALI_KVSTORE_NAME_ATT, storename); + VStringBuffer xpath("%s/Store[%s='%s'][1]/", DALI_KVSTORE_PATH, DALI_KVSTORE_NAME_ATT, encodedStoreName.str()); if (global) xpath.append(DALI_KVSTORE_GLOBAL); else - { - StringBuffer userlowercased; - userlowercased.set(username->getName()).toLowerCase(); - xpath.append(userlowercased); - } + getEncodedLowerCaseUserName(xpath, username); - xpath.appendf("/%s/%s", ns, key); + StringBuffer encodedKey; + encodePTreeName(encodedKey, key); + + xpath.appendf("/%s/%s", encodedNS.str(), encodedKey.str()); Owned conn = querySDS().connect(xpath.str(), myProcessSession(), RTM_LOCK_READ, SDS_LOCK_TIMEOUT_KVSTORE); if (!conn) @@ -189,26 +213,31 @@ bool CDALIKVStore::fetchKeyProperty(StringBuffer & propval , const char * storen if (isEmptyString(storename)) throw MakeStringException(-1, "DALI Keystore fetchKeyProperty(): Store name not provided"); + StringBuffer encodedStoreName; + encodePTreeName(encodedStoreName, storename); + if (!global && (!username || isEmptyString(username->getName()))) throw MakeStringException(-1, "DALI Keystore fetchKeyProperty(): Attempting to set non-global entry but owner name not provided"); if (isEmptyString(ns)) throw MakeStringException(-1, "DALI Keystore fetchKeyProperty(): namespace not provided"); + StringBuffer encodedNamespace; + encodePTreeName(encodedNamespace, ns); + ensureAttachedToDali(); //throws if in offline mode - VStringBuffer xpath("%s/Store[%s='%s'][1]/", DALI_KVSTORE_PATH, DALI_KVSTORE_NAME_ATT, storename); + VStringBuffer xpath("%s/Store[%s='%s'][1]/", DALI_KVSTORE_PATH, DALI_KVSTORE_NAME_ATT, encodedStoreName.str()); if (global) xpath.append(DALI_KVSTORE_GLOBAL); else - { - StringBuffer userlowercased; - userlowercased.set(username->getName()).toLowerCase(); - xpath.append(userlowercased); - } + getEncodedLowerCaseUserName(xpath, username); + + StringBuffer encodedKey; + encodePTreeName(encodedKey, key); - xpath.appendf("/%s/%s", ns, key); + xpath.appendf("/%s/%s", encodedNamespace.str(), encodedKey.str()); Owned conn = querySDS().connect(xpath.str(), myProcessSession(), RTM_LOCK_READ, SDS_LOCK_TIMEOUT_KVSTORE); if (!conn) @@ -216,7 +245,10 @@ bool CDALIKVStore::fetchKeyProperty(StringBuffer & propval , const char * storen Owned keytree = conn->getRoot(); - keytree->getProp(property,propval.clear()); + StringBuffer encodedPropName; + encodePTreeName(encodedPropName, property); + + keytree->getProp(encodedPropName.str(),propval.clear()); return true; } @@ -225,18 +257,27 @@ bool CDALIKVStore::deletekey(const char * storename, const char * thenamespace, if (!storename || !*storename) throw MakeStringException(-1, "DALI Keystore deletekey(): Store name not provided"); + StringBuffer encodedStoreName; + encodePTreeName(encodedStoreName, storename); + if (!thenamespace || !*thenamespace) throw MakeStringException(-1, "DALI KV Store deletekey(): target namespace not provided!"); + StringBuffer encodedNS; + encodePTreeName(encodedNS, thenamespace); + if (!key || !*key) throw MakeStringException(-1, "DALI KV Store deletekey(): target key not provided!"); + StringBuffer encodedKey; + encodePTreeName(encodedKey, key); + if (!global && (!user || isEmptyString(user->getName()))) throw MakeStringException(-1, "DALI Keystore set(): Attempting to set non-global entry but user not provided"); ensureAttachedToDali(); //throws if in offline mode - VStringBuffer xpath("%s/Store[%s='%s'][1]", DALI_KVSTORE_PATH, DALI_KVSTORE_NAME_ATT, storename); + VStringBuffer xpath("%s/Store[%s='%s'][1]", DALI_KVSTORE_PATH, DALI_KVSTORE_NAME_ATT, encodedStoreName.str()); Owned conn = querySDS().connect(xpath.str(), myProcessSession(), RTM_LOCK_WRITE, SDS_LOCK_TIMEOUT_KVSTORE); if (!conn) throw MakeStringException(-1, "DALI Keystore deletekey(): Unable to connect to DALI KeyValue store root path '%s'", DALI_KVSTORE_PATH); @@ -246,9 +287,9 @@ bool CDALIKVStore::deletekey(const char * storename, const char * thenamespace, if (global) xpath.set(DALI_KVSTORE_GLOBAL); else - xpath.set(user->getName()).toLowerCase(); + getEncodedLowerCaseUserName(xpath.clear(), user); - xpath.appendf("/%s/%s", thenamespace,key); + xpath.appendf("/%s/%s", encodedNS.str(), encodedKey.str()); if(!storetree->hasProp(xpath.str())) throw MakeStringException(-1, "DALI KV Store deletekey(): Could not find '%s/%s/%s' for user '%s'", storename, thenamespace, key, global ? "GLOBAL USER" : user->getName()); @@ -264,15 +305,21 @@ bool CDALIKVStore::deleteNamespace(const char * storename, const char * thenames if (!storename || !*storename) throw MakeStringException(-1, "DALI Keystore deletekey(): Store name not provided"); + StringBuffer encodedStoreName; + encodePTreeName(encodedStoreName, storename); + if (!global && (!user || isEmptyString(user->getName()))) throw MakeStringException(-1, "DALI Keystore deleteNamespace(): Attempting to fetch non-global keys but user not provided"); if (isEmptyString(thenamespace)) throw MakeStringException(-1, "DALI KV Store deleteNamespace(): target namespace not provided!"); + StringBuffer encodedNS; + encodePTreeName(encodedNS, thenamespace); + ensureAttachedToDali(); //throws if in offline mode - VStringBuffer xpath("%s/Store[%s='%s']", DALI_KVSTORE_PATH, DALI_KVSTORE_NAME_ATT, storename); + VStringBuffer xpath("%s/Store[%s='%s']", DALI_KVSTORE_PATH, DALI_KVSTORE_NAME_ATT, encodedStoreName.str()); Owned conn = querySDS().connect(xpath.str(), myProcessSession(), RTM_LOCK_WRITE, SDS_LOCK_TIMEOUT_KVSTORE); if (!conn) throw MakeStringException(-1, "DALI Keystore deleteNamespace(): Unable to connect to DALI KeyValue store path '%s'", xpath.str()); @@ -282,9 +329,9 @@ bool CDALIKVStore::deleteNamespace(const char * storename, const char * thenames if (global) xpath.set(DALI_KVSTORE_GLOBAL); else - xpath.set(user->getName()).toLowerCase(); + getEncodedLowerCaseUserName(xpath.clear(), user); - xpath.appendf("/%s", thenamespace); //we're interested in the children of the namespace + xpath.appendf("/%s", encodedNS.str()); //we're interested in the children of the namespace if(!storetree->hasProp(xpath.str())) throw MakeStringException(-1, "DALI KV Store deleteNamespace(): invalid namespace detected '%s/%s' for user '%s'", storename, thenamespace, global ? "GLOBAL USER" : user->getName()); @@ -300,12 +347,15 @@ bool CDALIKVStore::fetchAllNamespaces(StringArray & namespaces, const char * sto if (!storename || !*storename) throw MakeStringException(-1, "DALI Keystore fetchAllNamespaces(): Store name not provided"); + StringBuffer encodedStoreName; + encodePTreeName(encodedStoreName, storename); + if (!global && (!user || isEmptyString(user->getName()))) throw MakeStringException(-1, "DALI Keystore fetchAllNamespaces(): Attempting to fetch non-global keys but requester name not provided"); ensureAttachedToDali(); //throws if in offline mode - VStringBuffer xpath("%s/Store[%s='%s']", DALI_KVSTORE_PATH, DALI_KVSTORE_NAME_ATT, storename); + VStringBuffer xpath("%s/Store[%s='%s']", DALI_KVSTORE_PATH, DALI_KVSTORE_NAME_ATT, encodedStoreName.str()); Owned conn = querySDS().connect(xpath.str(), myProcessSession(), RTM_LOCK_READ, SDS_LOCK_TIMEOUT_KVSTORE); if (!conn) throw MakeStringException(-1, "DALI Keystore fetchAllNamespaces: Unable to connect to DALI KeyValue store path '%s'", xpath.str()); @@ -313,16 +363,21 @@ bool CDALIKVStore::fetchAllNamespaces(StringArray & namespaces, const char * sto Owned storetree = conn->getRoot(); if (global) - xpath.setf("%s/*", DALI_KVSTORE_GLOBAL); //we're interested in the children of the namespace + xpath.set(DALI_KVSTORE_GLOBAL); else - xpath.setf("%s/*", user->getName()).toLowerCase(); //we're interested in the children of the namespace + getEncodedLowerCaseUserName(xpath.clear(), user); + + xpath.append("/*"); //we're interested in the children of the namespace StringBuffer name; Owned iter = storetree->getElements(xpath.str()); ForEach(*iter) { iter->query().getName(name.clear()); - namespaces.append(name.str()); + StringBuffer decodedName; + decodePtreeName(decodedName, name.str()); + + namespaces.append(decodedName.str()); } return true; @@ -333,15 +388,21 @@ bool CDALIKVStore::fetchKeySet(StringArray & keyset, const char * storename, con if (!storename || !*storename) throw MakeStringException(-1, "DALI Keystore fetchKeySet(): Store name not provided"); + StringBuffer encodedStoreName; + encodePTreeName(encodedStoreName, storename); + if (!global && (!user || isEmptyString(user->getName()))) throw MakeStringException(-1, "DALI Keystore fetchKeySet(): Attempting to fetch non-global keys but requester name not provided"); if (isEmptyString(ns)) throw MakeStringException(-1, "DALI Keystore fetchKeySet: Namespace not provided!"); + StringBuffer encodedNS; + encodePTreeName(encodedNS, ns); + ensureAttachedToDali(); //throws if in offline mode - VStringBuffer xpath("%s/Store[%s='%s']", DALI_KVSTORE_PATH, DALI_KVSTORE_NAME_ATT, storename); + VStringBuffer xpath("%s/Store[%s='%s']", DALI_KVSTORE_PATH, DALI_KVSTORE_NAME_ATT, encodedStoreName.str()); Owned conn = querySDS().connect(xpath.str(), myProcessSession(), RTM_LOCK_READ, SDS_LOCK_TIMEOUT_KVSTORE); if (!conn) throw MakeStringException(-1, "DALI Keystore fetchKeySet: Unable to connect to DALI KeyValue store path '%s'", DALI_KVSTORE_PATH); @@ -351,9 +412,10 @@ bool CDALIKVStore::fetchKeySet(StringArray & keyset, const char * storename, con if (global) xpath.set(DALI_KVSTORE_GLOBAL); else - xpath.set(user->getName()).toLowerCase(); + getEncodedLowerCaseUserName(xpath.clear(), user); + + xpath.appendf("/%s/*", encodedNS.str()); //we're interested in the children of the namespace - xpath.appendf("/%s/*", ns); //we're interested in the children of the namespace if(!storetree->hasProp(xpath.str())) throw MakeStringException(-1, "DALI Keystore fetchKeySet: invalid namespace '%s' detected!", ns); @@ -362,7 +424,9 @@ bool CDALIKVStore::fetchKeySet(StringArray & keyset, const char * storename, con ForEach(*iter) { iter->query().getName(name.clear()); - keyset.append(name.str()); + StringBuffer decodedName; + decodePtreeName(decodedName, name.str()); + keyset.append(decodedName.str()); } return true; @@ -373,15 +437,21 @@ bool CDALIKVStore::fetch(const char * storename, const char * ns, const char * k if (!storename || !*storename) throw MakeStringException(-1, "DALI Keystore fetch(): Store name not provided"); - if (!global && (!user || isEmptyString(user->getName()))) + StringBuffer encodedStoreName; + encodePTreeName(encodedStoreName, storename); + + if (!global && (!user || isEmptyString(user->getName()))) throw MakeStringException(-1, "DALI Keystore fetch(): Attempting to fetch non-global entry but requester name not provided"); - if (isEmptyString(ns)) - throw MakeStringException(-1, "DALI Keystore fetch: key not provided!"); + if (isEmptyString(ns)) + throw MakeStringException(-1, "DALI Keystore fetch: key not provided!"); + + StringBuffer encodedNS; + encodePTreeName(encodedNS, ns); ensureAttachedToDali(); //throws if in offline mode - VStringBuffer xpath("%s/Store[%s='%s']", DALI_KVSTORE_PATH, DALI_KVSTORE_NAME_ATT, storename); + VStringBuffer xpath("%s/Store[%s='%s']", DALI_KVSTORE_PATH, DALI_KVSTORE_NAME_ATT, encodedStoreName.str()); Owned conn = querySDS().connect(xpath.str(), myProcessSession(), RTM_LOCK_READ, SDS_LOCK_TIMEOUT_KVSTORE); if (!conn) throw MakeStringException(-1, "DALI Keystore fetch: Unable to connect to DALI KeyValue store path '%s'", xpath.str()); @@ -391,25 +461,23 @@ bool CDALIKVStore::fetch(const char * storename, const char * ns, const char * k if (global) xpath.set(DALI_KVSTORE_GLOBAL); else - xpath.set(user->getName()).toLowerCase(); + getEncodedLowerCaseUserName(xpath.clear(), user); - xpath.appendf("/%s", ns); + xpath.appendf("/%s", encodedNS.str()); if(!storetree->hasProp(xpath.str())) throw MakeStringException(-1, "DALI Keystore fetch: invalid namespace '%s' detected!", ns); if (key && *key) { - xpath.appendf("/%s", key); + StringBuffer encodedKey; + encodePTreeName(encodedKey, key); + xpath.appendf("/%s", encodedKey.str()); if(!storetree->hasProp(xpath.str())) - { - throw makeStringExceptionV(ECLWATCH_INVALID_QUERY_KEY, "DALI Keystore fetch: invalid key '%s' detected!", key); - } + return false; else - { value.set(storetree->queryProp(xpath.str())); - } - return value.str(); + return true; } else throw makeStringException(-1, "DALI Keystore fetch: Key not provided!"); @@ -422,15 +490,21 @@ IPropertyTree * CDALIKVStore::getAllPairs(const char * storename, const char * n if (!storename || !*storename) throw MakeStringException(-1, "DALI Keystore fetchAll(): Store name not provided"); + StringBuffer encodedStoreName; + encodePTreeName(encodedStoreName, storename); + if (!global && (!user || isEmptyString(user->getName()))) throw MakeStringException(-1, "DALI Keystore fetchAll(): Attempting to fetch non-global entries but requester name not provided"); if (isEmptyString(ns)) throw MakeStringException(-1, "DALI Keystore fetchAll: Namespace not provided!"); + StringBuffer encodedNS; + encodePTreeName(encodedNS, ns); + ensureAttachedToDali(); //throws if in offline mode - VStringBuffer xpath("%s/Store[%s='%s']", DALI_KVSTORE_PATH, DALI_KVSTORE_NAME_ATT, storename); + VStringBuffer xpath("%s/Store[%s='%s']", DALI_KVSTORE_PATH, DALI_KVSTORE_NAME_ATT, encodedStoreName.str()); Owned conn = querySDS().connect(xpath.str(), myProcessSession(), RTM_LOCK_READ, SDS_LOCK_TIMEOUT_KVSTORE); if (!conn) throw MakeStringException(-1, "DALI Keystore fetchAll: Unable to connect to DALI KeyValue store path '%s'", xpath.str()); @@ -440,9 +514,9 @@ IPropertyTree * CDALIKVStore::getAllPairs(const char * storename, const char * n if (global) xpath.set(DALI_KVSTORE_GLOBAL); else - xpath.set(user->getName()).toLowerCase(); + getEncodedLowerCaseUserName(xpath.clear(), user); - xpath.appendf("/%s", ns); + xpath.appendf("/%s", encodedNS.str()); if(!storetree->hasProp(xpath.str())) throw MakeStringException(-1, "DALI Keystore fetchAll: invalid namespace '%s' detected!", ns); diff --git a/esp/services/ws_store/ws_storeService.cpp b/esp/services/ws_store/ws_storeService.cpp index 75ddde7d6e5..24b9dfea8fb 100644 --- a/esp/services/ws_store/ws_storeService.cpp +++ b/esp/services/ws_store/ws_storeService.cpp @@ -271,6 +271,7 @@ bool CwsstoreEx::onListKeys(IEspContext &context, IEspListKeysRequest &req, IEsp StringArray keys; m_storeProvider->fetchKeySet(keys, storename, ns, secuser.get(), !req.getUserSpecific()); + resp.setKeySet(keys); resp.setNamespace(ns); resp.setStoreName(storename); @@ -312,25 +313,11 @@ bool CwsstoreEx::onFetch(IEspContext &context, IEspFetchRequest &req, IEspFetchR storename = m_defaultStore.get(); } - try - { - m_storeProvider->fetch(storename, req.getNamespace(), req.getKey(), value, secuser.get(), !req.getUserSpecific()); + bool success = m_storeProvider->fetch(storename, req.getNamespace(), req.getKey(), value, secuser.get(), !req.getUserSpecific()); + if (success) resp.setValue(value.str()); - } - catch(IException * e) - { - if (e->errorCode() == ECLWATCH_INVALID_QUERY_KEY) - { - StringBuffer msg; - LOG(MCuserInfo, "WsStore: %s", e->errorMessage(msg).str()); - e->Release(); - return false; - } - else - throw e; - } - return true; + return success; } bool CwsstoreEx::onFetchKeyMetadata(IEspContext &context, IEspFetchKeyMDRequest &req, IEspFetchKeyMDResponse &resp) @@ -385,22 +372,28 @@ bool CwsstoreEx::onFetchAll(IEspContext &context, IEspFetchAllRequest &req, IEsp storename = m_defaultStore.get(); } - Owned nstree = m_storeProvider->getAllPairs(storename, ns, secuser.get(), !req.getUserSpecific()); - IArrayOf pairs; - - Owned iter = nstree->getElements("*"); - ForEach(*iter) { - StringBuffer name; - StringBuffer value; - iter->query().getName(name); - nstree->getProp(name.str(), value); - - Owned kvpair = createKVPair("",""); - kvpair->setKey(name.str()); - kvpair->setValue(value.str()); - pairs.append(*kvpair.getClear()); + Owned nstree = m_storeProvider->getAllPairs(storename, ns, secuser.get(), !req.getUserSpecific()); + + Owned iter = nstree->getElements("*"); + ForEach(*iter) + { + StringBuffer name; + StringBuffer value; + iter->query().getName(name); + nstree->getProp(name.str(), value); + + Owned kvpair = createKVPair("",""); + + //it's possible this has been encoded, so decode it + StringBuffer decoded; + decodePtreeName(decoded, name.str()); + + kvpair->setKey(decoded.str()); + kvpair->setValue(value.str()); + pairs.append(*kvpair.getClear()); + } } resp.setPairs(pairs); diff --git a/esp/src/src-react/components/Files.tsx b/esp/src/src-react/components/Files.tsx index 14939bd6164..08f5f953c34 100644 --- a/esp/src/src-react/components/Files.tsx +++ b/esp/src/src-react/components/Files.tsx @@ -171,7 +171,7 @@ export const Files: React.FunctionComponent = ({ }, Name: { label: nlsHPCC.LogicalName, - width: 360, + width: 180, formatter: (name, row) => { const file = Get(row.NodeGroup, name, row); if (row.__hpcc_isDir) { diff --git a/helm/hpcc/templates/dali.yaml b/helm/hpcc/templates/dali.yaml index d07f8677a5c..1c60bcd3840 100644 --- a/helm/hpcc/templates/dali.yaml +++ b/helm/hpcc/templates/dali.yaml @@ -77,6 +77,7 @@ spec: run: {{ $dali.name | quote }} server: {{ $dali.name | quote }} app: dali + serviceName: {{ $dali.name | quote }} updateStrategy: type: RollingUpdate rollingUpdate: diff --git a/system/jlib/jfile.cpp b/system/jlib/jfile.cpp index e25bbac59e7..0d33fa28a1d 100644 --- a/system/jlib/jfile.cpp +++ b/system/jlib/jfile.cpp @@ -7961,9 +7961,12 @@ unsigned __int64 getPlaneAttributeValue(const char *planeName, PlaneAttributeTyp CriticalBlock b(planeAttriubuteMapCrit); auto it = planeAttributesMap.find(planeName); if (it != planeAttributesMap.end()) - return it->second[planeAttrType]; - else - return defaultValue; + { + unsigned v = it->second[planeAttrType]; + if (v) // a plane attribute value of 0 is considered as not set + return v; + } + return defaultValue; } size32_t getBlockedFileIOSize(const char *planeName, size32_t defaultSize)