Merge remote-tracking branch 'origin/candidate-9.2.x' into candidate-…

…9.4.x

.github/workflows/build-assets.yml

@@ -248,7 +248,7 @@ jobs:
           sudo rm -f ${{ needs.preamble.outputs.folder_build }}/CMakeCache.txt
           sudo rm -rf ${{ needs.preamble.outputs.folder_build }}/CMakeFiles
           docker run --rm --mount ${{ needs.preamble.outputs.mount_platform }} --mount ${{ needs.preamble.outputs.mount_build }} ${{ steps.vars.outputs.docker_tag_candidate_base }} "\
-            cmake -S /hpcc-dev/HPCC-Platform -B /hpcc-dev/build ${{ needs.preamble.outputs.cmake_docker_config }} -DSIGN_MODULES=OFF -DMAKE_DOCS_ONLY=ON -DUSE_NATIVE_LIBRARIES=ON -DDOCS_AUTO=ON -DDOC_LANGS=ALL && \
+            cmake -S /hpcc-dev/HPCC-Platform -B /hpcc-dev/build -DVCPKG_FILES_DIR=/hpcc-dev -DMAKE_DOCS_ONLY=ON -DUSE_NATIVE_LIBRARIES=ON -DDOCS_AUTO=ON -DDOC_LANGS=ALL && \
             cmake --build /hpcc-dev/build --parallel $(nproc) --target all"
           docker run --rm --mount ${{ needs.preamble.outputs.mount_platform }} --mount ${{ needs.preamble.outputs.mount_build }} ${{ steps.vars.outputs.docker_tag_candidate_base }} "cd /hpcc-dev/build/Release/docs/EN_US && zip ALL_HPCC_DOCS_EN_US-${{ needs.preamble.outputs.community_tag }}.zip *.pdf"
           docker run --rm --mount ${{ needs.preamble.outputs.mount_platform }} --mount ${{ needs.preamble.outputs.mount_build }} ${{ steps.vars.outputs.docker_tag_candidate_base }} "cd /hpcc-dev/build/Release/docs/PT_BR && zip ALL_HPCC_DOCS_PT_BR-${{ needs.preamble.outputs.community_tag }}.zip *.pdf"

common/workunit/workunit.cpp

@@ -14022,13 +14022,16 @@ extern WORKUNIT_API double getThorManagerRate()
 
 extern WORKUNIT_API double getThorWorkerRate()
 {
+    // Note: (bare-metal) the use of getAffinityCpus to get the number of CPUs used by workers
+    // doesn't really make sense since the caller is likely to be running on thor manager (so it will
+    // return cpu affinity for the manager, rather than for the worker).  This needs rethinking.
     double numCpus = isContainerized() ? getResourcedCpus("workerResources") : getAffinityCpus();
     return getCostCpuHour() * numCpus;
 }
 
-extern WORKUNIT_API double calculateThorCost(unsigned __int64 ms, unsigned numberOfMachines)
+extern WORKUNIT_API double calculateThorCost(unsigned __int64 ms, unsigned numberOfWorkers)
 {
-    return calcCost(getThorManagerRate(), ms) + calcCost(getThorWorkerRate(), ms) * numberOfMachines;
+    return calcCost(getThorManagerRate(), ms) + calcCost(getThorWorkerRate(), ms) * numberOfWorkers;
 }
 
 void aggregateStatistic(StatsAggregation & result, IConstWorkUnit * wu, const WuScopeFilter & filter, StatisticKind search)

common/workunit/workunit.hpp

@@ -1746,7 +1746,7 @@ extern WORKUNIT_API void addTimeStamp(IWorkUnit * wu, unsigned wfid, const char
 extern WORKUNIT_API double getMachineCostRate();
 extern WORKUNIT_API double getThorManagerRate();
 extern WORKUNIT_API double getThorWorkerRate();
-extern WORKUNIT_API double calculateThorCost(unsigned __int64 ms, unsigned numberMachines);
+extern WORKUNIT_API double calculateThorCost(unsigned __int64 ms, unsigned numberOfWorkers);
 
 extern WORKUNIT_API IPropertyTree * getWUGraphProgress(const char * wuid, bool readonly);
 

esp/src/Login.html

@@ -143,7 +143,7 @@
                         var d = new Date();
                         d.setTime(d.getTime() + (exMins * 60 * 1000));
                         var expires = "expires=" + d.toUTCString();
-                        document.cookie = cname + "=" + cvalue + ";" + expires + ";path=/";
+                        document.cookie = cname + "=" + cvalue + ";" + expires + ";";
                     }
 
                     if (document.cookie.indexOf("ESPAuthenticationMSG") > -1) {

esp/src/eclwatch/HPCCPlatformWidget.js

@@ -253,7 +253,7 @@ define([
                     context.checkIfAdmin(context.userName);
                     context.refreshUserName();
                     if (!cookie("PasswordExpiredCheck")) {
-                        cookie("PasswordExpiredCheck", "true", { expires: 1 });
+                        cookie("PasswordExpiredCheck", "true", { expires: 1, path: "/" });
                         if (lang.exists("MyAccountResponse.passwordDaysRemaining", response)) {
                             switch (response.MyAccountResponse.passwordDaysRemaining) {
                                 case null:

esp/src/eclwatch/LockDialogWidget.js

@@ -58,7 +58,6 @@ define([
         show: function (event) {
             var context = this;
             if (!dojoConfig.username) {
-                cookie("Status", "Unlocked");
                 context.storage.setItem("Status", "Unlocked");
                 topic.publish("hpcc/session_management_status", {
                     status: "Unlocked"
@@ -96,7 +95,6 @@ define([
             var context = this;
 
             if (this.unlockForm.validate()) {
-                cookie("Status", "login_attempt");
                 WsAccount.Unlock({
                     request: {
                         username: this.unlockUserName.get("value"),
@@ -113,7 +111,6 @@ define([
                         topic.publish("hpcc/session_management_status", {
                             status: "Unlocked"
                         });
-                        cookie("Status", "Unlocked");
                         context.storage.setItem("Status", "Unlocked");
                         if (context.idleFired) {
                             dojo.publish("hpcc/brToaster", {
@@ -127,7 +124,6 @@ define([
                         }
                     } else {
                         context.unlockStatus.innerHTML = response.UnlockResponse.Message;
-                        cookie("Status", "Locked");
                     }
                 });
             }
@@ -150,9 +146,8 @@ define([
                 topic.publish("hpcc/session_management_status", {
                     status: "Locked"
                 });
-                cookie("Status", "Locked");
                 context.storage.setItem("Status", "Locked");
-            } else if (cookie("Status") === "Unlocked") {
+            } else {
                 xhr("/esp/lock", {
                     method: "post"
                 }).then(function (response) {
@@ -163,7 +158,6 @@ define([
                         topic.publish("hpcc/session_management_status", {
                             status: "Locked"
                         });
-                        cookie("Status", "Locked");
                         context.storage.setItem("Status", "Locked");
                     }
                 });

esp/src/index.html

@@ -38,7 +38,7 @@
             }
 
             function checkCookie() {
-                document.cookie.indexOf("ESPAuthenticated=true") > -1 ? window.location.href : window.location.href = location.origin + "#/login";
+                document.cookie.indexOf("ESPAuthenticated=true") > -1 ? window.location.href : window.location.href = location.origin + "/esp/files/Login.html";
             }
 
             function sendAuthRequest() {

esp/src/lws.config.js

@@ -38,7 +38,7 @@ let rewrite = [
     { from: "/esp/files/dist/(.*)", to: "/build/dist/$1" },
     { from: "/esp/files/img/(.*)", to: "build/esp/files/img/$1" },
     { from: "/esp/files/(.*/*.css)", to: "/build/esp/files/$1" },
-    { from: "/esp/files/(.*)", to: "/$1" },
+    { from: "/esp/files/(.*)", to: "/build/$1" },
     { from: "/main", to: protocol + "://" + ip + ":" + port + "/main" },
     { from: "/FileSpray/(.*)", to: protocol + "://" + ip + ":" + port + "/FileSpray/$1" },
     { from: "/WsCloud/(.*)", to: protocol + "://" + ip + ":" + port + "/WsCloud/$1" },

esp/src/src-react/components/Frame.tsx

@@ -65,7 +65,7 @@ export const Frame: React.FunctionComponent<FrameProps> = () => {
                 fireIdle();
             } else if (publishedMessage.status === "Idle") {
                 window.localStorage.setItem("pageOnLock", window.location.hash.substring(1));
-                setUserSession({ ...userSession, Status: "Locked" });
+                setUserSession({ ...userSession });
                 window.location.reload();
             }
         });

esp/src/src-react/components/Title.tsx

@@ -142,7 +142,7 @@ export const DevTitle: React.FunctionComponent<DevTitleProps> = ({
                         fetch("/esp/lock", {
                             method: "post"
                         }).then(() => {
-                            setUserSession({ ...userSession, Status: "Locked" });
+                            setUserSession({ ...userSession });
                             replaceUrl("/login", true);
                         });
                     }
@@ -235,7 +235,7 @@ export const DevTitle: React.FunctionComponent<DevTitleProps> = ({
         if (!currentUser.username) return;
         if (!cookie("PasswordExpiredCheck")) {
             // cookie expires option expects whole number of days, use a decimal < 1 for hours
-            cookie("PasswordExpiredCheck", "true", { expires: 0.5 });
+            cookie("PasswordExpiredCheck", "true", { expires: 0.5, path: "/" });
             if (currentUser.passwordIsExpired) {
                 alert(nlsHPCC.PasswordExpired);
                 setShowMyAccount(true);

esp/src/src-react/components/Workunits.tsx

@@ -143,6 +143,7 @@ export const Workunits: React.FunctionComponent<WorkunitsProps> = ({
             },
             Wuid: {
                 label: nlsHPCC.WUID, width: 120,
+                sortable: true,
                 formatter: (Wuid, row) => {
                     const wu = Get(Wuid);
                     return <>

esp/src/src-react/components/controls/Grid.tsx

@@ -165,7 +165,7 @@ export function useFluentStoreState({ page }: FluentStoreStateProps): FluentStor
     const [selection, setSelection] = React.useState([]);
     const [pageNum, setPageNum] = React.useState(page);
     const [pageSize, setPageSize] = React.useState<number>();
-    const [total, setTotal] = React.useState(0);
+    const [total, setTotal] = React.useState(-1);
     const refreshTable = useRefreshTable();
 
     return { selection, setSelection, pageNum, setPageNum, pageSize, setPageSize, total, setTotal, refreshTable };
@@ -462,7 +462,7 @@ export const FluentPagedFooter: React.FunctionComponent<FluentPagedFooterProps>
     return <Stack horizontal className={paginationStyles.root}>
         <Stack.Item className={paginationStyles.pageControls}>
             <Pagination
-                selectedPageIndex={page} itemsPerPage={pageSize} totalItemCount={total}
+                selectedPageIndex={page} itemsPerPage={pageSize} totalItemCount={total >= 0 ? total : -1}
                 pageCount={Math.ceil(total / pageSize)} format="buttons" onPageChange={index => {
                     setPage(Math.round(index));
                     updatePage(Math.round(index + 1).toString());
@@ -471,7 +471,7 @@ export const FluentPagedFooter: React.FunctionComponent<FluentPagedFooterProps>
                     const start = props.totalItemCount === 0 ? 0 : props.selectedPageIndex === 0 ? 1 : (props.selectedPageIndex * props.itemsPerPage) + 1;
                     const end = (props.itemsPerPage * (props.selectedPageIndex + 1)) > props.totalItemCount ? props.totalItemCount : props.itemsPerPage * (props.selectedPageIndex + 1);
                     return <div className={paginationStyles.paginationLabel}>
-                        {start} {props.strings.divider} {end} {nlsHPCC.Of.toLowerCase()} {props.totalItemCount} {nlsHPCC.Rows} {selectionCount ? `(${selectionCount} ${nlsHPCC.Selected})` : ""}
+                        {start} {props.strings.divider} {end >= 0 ? end : 1} {nlsHPCC.Of.toLowerCase()} {props.totalItemCount >= 0 ? props.totalItemCount : "???"} {nlsHPCC.Rows} {selectionCount ? `(${selectionCount} ${nlsHPCC.Selected})` : ""}
                     </div>;
                 }}
             />

esp/src/src-react/components/forms/Login.tsx

@@ -116,7 +116,6 @@ export const Login: React.FunctionComponent<LoginProps> = ({
                     setErrorMessage(cookies.ESPAuthenticationMSG);
                     setShowError(true);
                 } else {
-                    cookies["Status"] = "Unlocked";
                     cookies["ESPAuthenticated"] = "true";
                     createUserSession(cookies).then(() => {
                         setErrorMessage("");

esp/src/src-react/hooks/user.ts

@@ -7,7 +7,6 @@ const defaults = {
     ESPSessionTimeout: 7200,
     ESPAuthenticated: false,
     ECLWatchUser: false,
-    Status: "Unlocked",
     ESPSessionState: false
 };
 
@@ -17,7 +16,6 @@ export interface UserSession {
     ESPSessionTimeout: number;
     ESPAuthenticated: boolean;
     ECLWatchUser: boolean;
-    Status: string;
     ESPAuthenticationMSG?: string;
     ESPSessionState: boolean;
 }
@@ -56,7 +54,6 @@ export function useUserSession(): {
                 default:
                     store.set("ESPSessionState", "false");
             }
-            store.set("Status", "Unlocked");
             store.set("ECLWatchUser", "true");
         }).catch(err => {
             store.set("ESPSessionState", "false");

esp/src/src-react/index.tsx

@@ -43,7 +43,7 @@ async function loadUI() {
     const authType = await authTypeResp?.text() ?? "None";
     const userStore = cookieKeyValStore();
     const userSession = await userStore.getAll();
-    if (authType.indexOf("None") < 0 && (userSession["ESPSessionState"] === "false" || userSession["ECLWatchUser"] === "false" || (!userSession["Status"] || userSession["Status"] === "Locked"))) {
+    if (authType.indexOf("None") < 0 && (userSession["ESPSessionState"] === "false" || userSession["ECLWatchUser"] === "false")) {
         if (window.location.hash.indexOf("login") < 0) {
             replaceUrl("/login");
         }

esp/src/src/ESPRequest.ts

@@ -122,7 +122,6 @@ class RequestHelper {
                             status: "DoIdle"
                         });
                     }
-                    cookie("Status", "Locked");
                     ESPUtil.LocalStorage.removeItem("Status");
                 }
             }

esp/src/src/Session.ts

@@ -3,12 +3,12 @@ import * as xhr from "dojo/request/xhr";
 import * as topic from "dojo/topic";
 import { format as d3Format } from "@hpcc-js/common";
 import { SMCService } from "@hpcc-js/comms";
+import { scopedLogger } from "@hpcc-js/util";
 import { cookieKeyValStore, sessionKeyValStore, userKeyValStore } from "src/KeyValStore";
 import { singletonDebounce } from "../src-react/util/throttle";
 import { parseSearch } from "../src-react/util/history";
 import { ModernMode } from "./BuildInfo";
 import * as ESPUtil from "./ESPUtil";
-import { scopedLogger } from "@hpcc-js/util";
 
 const logger = scopedLogger("src/Session.ts");
 
@@ -43,6 +43,10 @@ export async function needsRedirectV5(): Promise<boolean> {
         window.location.replace(`/esp/files/index.html${window.location.hash}`);
         return true;
     }
+    if (window.location.pathname.indexOf("/esp/files/stub.htm") < 0) {
+        window.location.replace(`/esp/files/stub.htm${window.location.search}${window.location.hash}`);
+        return true;
+    }
     if (isV5DirectURL()) {
         return false;
     }
@@ -143,21 +147,16 @@ export function initSession() {
         });
 
         idleWatcher.start();
-        if (!cookie("Status")) {
-            document.cookie = "Status=Unlocked;Path=/";
-        }
     } else if (cookie("ECLWatchUser")) {
         window.location.replace(dojoConfig.urlInfo.basePath + "/Login.html");
     }
 }
 
 export function lock() {
-    cookie("Status", "Locked");
     idleWatcher.stop();
 }
 
 export function unlock() {
-    cookie("Status", "Unlocked");
     idleWatcher.start();
 }
 

esp/src/src/Utility.ts

@@ -1089,7 +1089,7 @@ export function parseCookies(): Record<string, any> {
 export function deleteCookie(name: string) {
     const expireDate = new Date();
     expireDate.setSeconds(expireDate.getSeconds() + 1);
-    document.cookie = `${name}=; domain=${window.location.hostname}; path=/; expires=${expireDate.toUTCString()}`;
+    document.cookie = `${name}=; domain=${window.location.hostname}; expires=${expireDate.toUTCString()}`;
 }
 
 const d3FormatDecimal = d3Format(",.2f");

thorlcr/master/thdemonserver.cpp

@@ -40,7 +40,6 @@ class DeMonServer : public CSimpleInterface, implements IDeMonServer
     UnsignedArray graphStarts;
     double thorManagerRate = 0;
     double thorWorkerRate = 0;
-    unsigned numberOfMachines = 0;
     cost_type costLimit = 0;
     cost_type workunitCost = 0;
 
@@ -90,7 +89,7 @@ class DeMonServer : public CSimpleInterface, implements IDeMonServer
         updateWorkunitStat(wu, SSTsubgraph, graphScope, StTimeElapsed, timer, milliToNano(duration));
         if (costLimit || finished)
         {
-            const cost_type sgCost = money2cost_type(calcCost(thorManagerRate, duration) + calcCost(thorWorkerRate, duration) * numberOfMachines);
+            const cost_type sgCost = money2cost_type(calcCost(thorManagerRate, duration) + calcCost(thorWorkerRate, duration) * queryNodeClusterWidth());
             cost_type costDiskAccess = graph.getDiskAccessCost();
             if (finished)
             {
@@ -268,7 +267,6 @@ class DeMonServer : public CSimpleInterface, implements IDeMonServer
             costLimit = money2cost_type(hardLimit);
         else
             costLimit = money2cost_type(tmpcostLimit);
-        numberOfMachines = queryNodeClusterWidth() / globals->getPropInt("@numWorkersPerPod", 1); // Number of Pods or physical machines
         activeGraphs.append(*LINK(graph));
         unsigned startTime = msTick();
         graphStarts.append(startTime);

thorlcr/master/thgraphmanager.cpp

@@ -1144,8 +1144,7 @@ bool CJobManager::executeGraph(IConstWorkUnit &workunit, const char *graphName,
         updateWorkunitStat(wu, SSTgraph, graphName, StTimeElapsed, graphTimeStr, graphTimeNs, wfid);
 
         addTimeStamp(wu, SSTgraph, graphName, StWhenFinished, wfid);
-        unsigned numberOfMachines = queryNodeClusterWidth() / globals->getPropInt("@numWorkersPerPod", 1); // Number of Pods or physical machines
-        cost_type cost = money2cost_type(calculateThorCost(nanoToMilli(graphTimeNs), numberOfMachines));
+        cost_type cost = money2cost_type(calculateThorCost(nanoToMilli(graphTimeNs), queryNodeClusterWidth()));
         if (cost)
             wu->setStatistic(queryStatisticsComponentType(), queryStatisticsComponentName(), SSTgraph, graphScope, StCostExecute, NULL, cost, 1, 0, StatsMergeReplace);
         updateSpillSize(wu, graphScope, SSTgraph);