Merge pull request #288 from skadefro/master

close 1.5.5

OpenFlow/src/Config.ts

@@ -68,7 +68,8 @@ export class dbConfig extends Base {
             if(["license_key", "otel_trace_url", "cache_store_type", "cache_store_redis_host", "cache_store_max", "grafana_url", "workitem_queue_monitoring_interval",
             "NODE_ENV", "validate_emails", "amqp_url", "port", "saml_issuer", "saml_federation_metadata", "api_ws_url",
             "domain", "enable_openapi", "enable_openapiauth", "ping_clients_interval", "tls_crt", "tls_key", "tls_ca",
-            "otel_metric_url", "otel_trace_url", "multi_tenant", "auto_hourly_housekeeping" ].indexOf(key) > -1 ) {
+            "otel_metric_url", "otel_trace_url", "multi_tenant", "auto_hourly_housekeeping", "housekeeping_skip_calculate_size", "housekeeping_skip_update_user_size"].indexOf(key) > -1 ) {
+
                 if(os.hostname().toLowerCase() == "nixos") {
                     continue;
                 }
@@ -118,7 +119,7 @@ export class dbConfig extends Base {
                 if(["license_key", "otel_trace_url", "cache_store_type", "cache_store_redis_host", "cache_store_max", "grafana_url", "workitem_queue_monitoring_interval",
                 "NODE_ENV", "validate_emails", "amqp_url", "port", "saml_issuer", "saml_federation_metadata", "api_ws_url",
                 "domain", "enable_openapi", "enable_openapiauth", "ping_clients_interval", "tls_crt", "tls_key", "tls_ca",
-                "otel_metric_url", "otel_trace_url", "multi_tenant", "auto_hourly_housekeeping" ].indexOf(key) > -1 ) {
+                "otel_metric_url", "otel_trace_url", "multi_tenant", "auto_hourly_housekeeping", "housekeeping_skip_calculate_size", "housekeeping_skip_update_user_size" ].indexOf(key) > -1 ) {
                     if(os.hostname().toLowerCase() == "nixos") {
                         continue;
                     }
@@ -256,6 +257,8 @@ export class Config {
         enable_nodered_tours: true,
         grafana_url: "",
         auto_hourly_housekeeping: true,
+        housekeeping_skip_calculate_size: false,
+        housekeeping_skip_update_user_size: false,
         housekeeping_skip_collections: "",
         housekeeping_remomve_unvalidated_user_days: 0, // if above 0, remove unvalidated users after x days
         housekeeping_cleanup_openrpa_instances: false,
@@ -267,6 +270,7 @@ export class Config {
         agent_HTTP_PROXY: "",
         agent_HTTPS_PROXY: "",
         agent_NO_PROXY: "",
+        agent_NPM_REGISTRY: "",
 
         stripe_api_key: "",
         stripe_api_secret: "",
@@ -524,6 +528,9 @@ export class Config {
     public static enable_nodered_tours: boolean = Config.parseBoolean(Config.getEnv("enable_nodered_tours"));
     public static grafana_url:string = Config.getEnv("grafana_url");
     public static auto_hourly_housekeeping: boolean = Config.parseBoolean(Config.getEnv("auto_hourly_housekeeping"));
+    public static housekeeping_skip_calculate_size: boolean = Config.parseBoolean(Config.getEnv("housekeeping_skip_calculate_size"));
+    public static housekeeping_skip_update_user_size: boolean = Config.parseBoolean(Config.getEnv("housekeeping_skip_update_user_size"));
+
     public static housekeeping_skip_collections: string = Config.getEnv("housekeeping_skip_collections");
     public static housekeeping_remomve_unvalidated_user_days: number = parseInt(Config.getEnv("housekeeping_remomve_unvalidated_user_days"));
     public static housekeeping_cleanup_openrpa_instances: boolean = Config.parseBoolean(Config.getEnv("housekeeping_cleanup_openrpa_instances"));
@@ -541,6 +548,9 @@ export class Config {
     public static agent_HTTP_PROXY: string = Config.getEnv("agent_HTTP_PROXY");
     public static agent_HTTPS_PROXY: string = Config.getEnv("agent_HTTPS_PROXY");
     public static agent_NO_PROXY: string = Config.getEnv("agent_NO_PROXY");
+    public static agent_NPM_REGISTRY: string = Config.getEnv("agent_NPM_REGISTRY");
+    public static agent_NPM_TOKEN: string = Config.getEnv("agent_NPM_TOKEN");
+
 
     public static stripe_api_key: string = Config.getEnv("stripe_api_key");
     public static stripe_api_secret: string = Config.getEnv("stripe_api_secret");

OpenFlow/src/Messages/Message.ts

@@ -2108,11 +2108,15 @@ export class Message {
         file = null;
         if (metadata == null) { metadata = new Base(); }
         metadata = Base.assign(metadata);
+        const user: TokenUser = await Message.verityToken(jwt);
         if (NoderedUtil.IsNullUndefinded(metadata._acl)) {
             metadata._acl = [];
-            Base.addRight(metadata, WellknownIds.filestore_users, "filestore users", [Rights.read]);
+            Base.addRight(metadata, WellknownIds.filestore_admins, "filestore admins", [Rights.full_control]);
+            if(!Config.multi_tenant) {
+                Base.addRight(metadata, WellknownIds.filestore_users, "filestore users", [Rights.read]);
+            }
+            Base.addRight(metadata, user._id, user.name, [Rights.full_control]);            
         }
-        const user: TokenUser = await Message.verityToken(jwt);
         metadata._createdby = user.name;
         metadata._createdbyid = user._id;
         metadata._created = new Date(new Date().toISOString());
@@ -3632,7 +3636,10 @@ export class Message {
                 Logger.instanse.debug("Ensure Indexes", span);
                 await Config.db.ensureindexes(span);
             } catch (error) {
-
+            }
+            if(Config.auto_hourly_housekeeping == false) {
+                Logger.instanse.debug("HouseKeeping disabled, quit.", span);
+                return;
             }
             try {
                 if(Logger.agentdriver != null) {

OpenFlow/src/WebServer.ts

@@ -344,7 +344,11 @@ export class WebServer {
             metadata._created = new Date(new Date().toISOString());
             metadata._modified = metadata._created;
 
-            Base.addRight(metadata, WellknownIds.filestore_users, "filestore users", [Rights.read]);
+            Base.addRight(metadata, WellknownIds.filestore_admins, "filestore admins", [Rights.full_control]);
+            if(!Config.multi_tenant) {
+                Base.addRight(metadata, WellknownIds.filestore_users, "filestore users", [Rights.read]);
+            }
+
 
             const rs = new stream.Readable;
             rs._read = () => { };

OpenFlow/src/dockerdriver.ts

@@ -153,6 +153,20 @@ export class dockerdriver implements i_agent_driver {
             if((oidc_config == null || oidc_config == "") && Config.agent_oidc_issuer == "") {
                 if(Config.domain != "localhost.openiap.io") oidc_config = Config.protocol + "://" + Config.domain + "/oidc/.well-known/openid-configuration"
             }
+            var HTTP_PROXY = Config.HTTP_PROXY;
+            var HTTPS_PROXY = Config.HTTPS_PROXY;
+            var NO_PROXY = Config.NO_PROXY;
+            var NPM_REGISTRY = Config.agent_NPM_REGISTRY;
+            var NPM_TOKEN = Config.agent_NPM_TOKEN;
+
+            if(HTTP_PROXY == null) HTTP_PROXY = "";
+            if(Config.agent_HTTP_PROXY != null && Config.agent_HTTP_PROXY != "") HTTP_PROXY = Config.agent_HTTP_PROXY;
+            if(HTTPS_PROXY == null) HTTPS_PROXY = "";
+            if(Config.agent_HTTPS_PROXY != null && Config.agent_HTTPS_PROXY != "") HTTPS_PROXY = Config.agent_HTTPS_PROXY;
+            if(NO_PROXY == null) NO_PROXY = "";
+            if(Config.agent_NO_PROXY != null && Config.agent_NO_PROXY != "") NO_PROXY = Config.agent_NO_PROXY;
+            if(NPM_REGISTRY == null) NPM_REGISTRY = "";
+            if(NPM_TOKEN == null) NPM_TOKEN = "";
             const Env = [
                 "jwt=" + agentjwt,
                 "agentid=" + agent._id,
@@ -163,9 +177,11 @@ export class dockerdriver implements i_agent_driver {
                 "protocol=" + Config.protocol,
                 "port=" + agentport.toString(),
                 "NODE_ENV=" + Config.NODE_ENV,
-                "HTTP_PROXY=" + Config.HTTP_PROXY,
-                "HTTPS_PROXY=" + Config.HTTPS_PROXY,
-                "NO_PROXY=" + Config.NO_PROXY,
+                "HTTP_PROXY=" + HTTP_PROXY,
+                "HTTPS_PROXY=" + HTTPS_PROXY,
+                "NO_PROXY=" + NO_PROXY,
+                "NPM_REGISTRY=" + NPM_REGISTRY,
+                "NPM_TOKEN=" + NPM_TOKEN,
                 "enable_analytics=" + Config.enable_analytics.toString(),
                 "enable_detailed_analytic=" + Config.enable_detailed_analytic.toString(),
                 "otel_trace_url=" + Config.otel_trace_url,

OpenFlow/src/index.ts

@@ -62,9 +62,12 @@ function doHouseKeeping(span: Span) {
     var dt = new Date(Message.lastHouseKeeping.toISOString());
     var msg2 = new Message(); msg2.jwt = Crypt.rootToken();
     var h = dt.getHours();
-    var skipUpdateUsage: boolean = !(dt.getHours() == 1 || dt.getHours() == 13);
+    var housekeeping_skip_calculate_size: boolean = !(dt.getHours() == 1 || dt.getHours() == 13);
+    var housekeeping_skip_update_user_size: boolean = !(dt.getHours() == 1 || dt.getHours() == 13);
+    if(Config.housekeeping_skip_calculate_size) housekeeping_skip_calculate_size = true;
+    if(Config.housekeeping_skip_update_user_size) housekeeping_skip_update_user_size = true;
     if (Config.NODE_ENV == "production") {
-        msg2._Housekeeping(false, skipUpdateUsage, skipUpdateUsage, null).catch((error) => Logger.instanse.error(error, null));
+        msg2._Housekeeping(false, housekeeping_skip_calculate_size, housekeeping_skip_update_user_size, null).catch((error) => Logger.instanse.error(error, null));
     } else {
         // While debugging, always do all calculations
         msg2._Housekeeping(false, false, false, null).catch((error) => Logger.instanse.error(error, null));
@@ -293,7 +296,7 @@ async function initDatabase(parent: Span): Promise<boolean> {
         }
         await Logger.DBHelper.Save(workitem_queue_users, jwt, span);
 
-        if (Config.auto_hourly_housekeeping) {
+        // if (Config.auto_hourly_housekeeping) {
             const crypto = require('crypto');
             const randomNum = crypto.randomInt(1, 100);
             // Every 15 minutes, give and take a few minutes, send out a message to do house keeping, if ready
@@ -333,7 +336,7 @@ async function initDatabase(parent: Span): Promise<boolean> {
                     doHouseKeeping(span);
                 }
             }, randomNum2 * 1000);
-        }
+        // }
         await Config.db.ParseTimeseries(span);
         return true;
     } catch (error) {

OpenFlow/src/public/Config.html

@@ -60,20 +60,25 @@
 <div ng-repeat="setting in ctrl.settings" class="form-group" ng-show="ctrl.Show(setting)">
     <label ng-class="{'dimmed': ctrl.model[setting.name] == null}">{{ setting.name }} ({{ setting.type }}) 
         <a href="" ng-click="ctrl.deletekey(setting.name); $event.preventDefault(); $event.stopPropagation();">clear</a>
-        ( default: {{ setting.default }})
+        ( default: {{ setting.default }}) <em class="fas fa-cog fa-spin" ng-show="ctrl.loading==true" title="Hang on, loading data"></em>
     </label>
     <!-- no-model does not work for boolens if type has not been set before parsing ng-type -->
-
     <input ng-model="ctrl.model[setting.name]" ng-if="setting.type != 'boolean' && !ctrl.isobject(ctrl.model[setting.name])"
-        ng-model="ctrl.model[setting.name]"
+        ng-model="ctrl.model[setting.name]" ng-enabled="ctrl.loading == false"
         ng-type step="{{ctrl.getstep(setting.name, ctrl.model[setting.name])}}"
         ng-on-blur="ctrl.submit()"
         ng-class="{'form-control': setting.type != 'boolean', 'secret': ctrl.model._encrypt.indexOf(setting.name) > -1, 'dimmed': ctrl.model[setting.name] == null }" />
-    <input ng-model="ctrl.model[setting.name]" ng-if="setting.type == 'boolean' && !ctrl.isobject(ctrl.model[setting.name])"
-    type="checkbox" ng-model="ctrl.model[setting.name]"
-    ng-click="ctrl.delayedUpdate()"
+    <!-- <three-stage-checkmark ng-if="setting.type == 'boolean' && !ctrl.isobject(ctrl.model[setting.name])"
+    ng-model="ctrl.model[setting.name]"
+    ng-click="ctrl.delayedUpdate()"></three-stage-checkmark> -->
+
+    <input ng-checked="ctrl.model[setting.name]" ng-if="setting.type == 'boolean' && !ctrl.isobject(ctrl.model[setting.name])"
+    type="checkbox" ng-enabled="ctrl.loading == false"
+    ng-click="ctrl.Toggle(this, setting.name)"
     ng-class="{'form-control': setting.type != 'boolean', 'secret': ctrl.model._encrypt.indexOf(setting.name) > -1, 'dimmed': ctrl.model[setting.name] == null}" />
 
-    <textarea  ng-class="{'dimmed': ctrl.model[setting.name] == null}" json-text class='form-control' rows='1' data-min-rows='1' ng-model="ctrl.model[setting.name]" ng-if="ctrl.isobject(ctrl.model[setting.name])"></textarea>
+    <textarea  ng-class="{'dimmed': ctrl.model[setting.name] == null}" json-text class='form-control' rows='1' data-min-rows='1' ng-model="ctrl.model[setting.name]" 
+    ng-enabled="ctrl.loading == false" ng-if="ctrl.isobject(ctrl.model[setting.name])"></textarea>
+
 
 </div>

OpenFlow/src/public/Controllers.ts

@@ -7368,6 +7368,8 @@ export class ConfigCtrl extends entityCtrl<RPAWorkflow> {
             {"name": "enable_nodered_tours", "type": "boolean", "default": "true"},
             {"name": "grafana_url", "type": "string", "default": ""},
             {"name": "auto_hourly_housekeeping", "type": "boolean", "default": "true"},
+            {"name": "housekeeping_skip_calculate_size", "type": "boolean", "default": "false"},
+            {"name": "housekeeping_skip_update_user_size", "type": "boolean", "default": "false"},    
             {"name": "housekeeping_skip_collections", "type": "string", "default": ""},
             {"name": "housekeeping_remomve_unvalidated_user_days", "type": "number", "default": "0"},
             {"name": "housekeeping_cleanup_openrpa_instances", "type": "boolean", "default": "false"},
@@ -7381,6 +7383,8 @@ export class ConfigCtrl extends entityCtrl<RPAWorkflow> {
             {"name": "agent_HTTP_PROXY", "type": "string", "default": ""},
             {"name": "agent_HTTPS_PROXY", "type": "string", "default": ""},
             {"name": "agent_NO_PROXY", "type": "string", "default": ""},
+            {"name": "agent_NPM_REGISTRY", "type": "string", "default": ""},
+            {"name": "agent_NPM_TOKEN", "type": "string", "default": ""},            
             {"name": "stripe_api_key", "type": "string", "default": ""},
             {"name": "stripe_api_secret", "type": "string", "default": ""},
             {"name": "stripe_force_vat", "type": "boolean", "default": "false"},
@@ -7564,12 +7568,33 @@ export class ConfigCtrl extends entityCtrl<RPAWorkflow> {
         if (!this.$scope.$$phase) { this.$scope.$apply(); }
     }
     Show(setting) {
+        if(this.model == null) return false;
         if(this.show == "all") return true;
         var isset = this.model[setting.name] != null;
         if(this.show == "set" && isset) return true;
         if(this.show == "unset" && !isset) return true;
         return false;
     }
+    Toggle(el, name) {
+        if(this.model == null) return;
+        console.log(name, this.model[name])
+        if(this.model[name] == null) {
+            var de = this.settings.filter(x => x.name == name)[0].default;
+            if(de == "true") {
+                de = true;
+            } else {
+                de = false;
+            }
+            console.log("default", de)
+            this.model[name] = !de;
+            el.checked = this.model[name];
+        } else {
+            this.model[name] = !this.model[name];
+            el.checked = this.model[name];
+        }
+        console.log(name, this.model[name])
+        this.delayedUpdate();
+    }
     async RegisterWatch() {
         try {
             if (!NoderedUtil.IsNullEmpty(this.watchid)) {
@@ -7588,19 +7613,10 @@ export class ConfigCtrl extends entityCtrl<RPAWorkflow> {
     }
     deletekey(key: string) {
         delete this.model[key]; 
+        this.delayedUpdate();
         this.submit();
     }
     processdata() {
-        const ids: string[] = [];
-        if (this.collection == "files") {
-            for (let i: number = 0; i < (this.model as any).metadata._acl.length; i++) {
-                ids.push((this.model as any).metadata._acl[i]._id);
-            }
-        } else {
-            for (let i: number = 0; i < this.model._acl.length; i++) {
-                ids.push(this.model._acl[i]._id);
-            }
-        }
         if (this.model._encrypt == null) { this.model._encrypt = []; }
         if (!this.$scope.$$phase) { this.$scope.$apply(); }
         this.fixtextarea();
@@ -7617,6 +7633,7 @@ export class ConfigCtrl extends entityCtrl<RPAWorkflow> {
         } catch (error) {
             this.errormessage = error.message ? error.message : error;
         }
+        this.loading = false;
         if (!this.$scope.$$phase) { this.$scope.$apply(); }
     }
     hasprop(name) {
@@ -7659,12 +7676,15 @@ export class ConfigCtrl extends entityCtrl<RPAWorkflow> {
     }
     delayhandler = null;
     delayedUpdate() {
+        console.log("delayedUpdate")
         if(this.loading == true) return;
         if(this.delayhandler ! = null) return;
+        this.loading = true;
         this.delayhandler = setTimeout(() => {
             this.delayhandler = null;
             this.submit();
         }, 500);
+        if (!this.$scope.$$phase) { this.$scope.$apply(); }
     }
 }
 

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openiap/openflow",
-  "version": "1.5.4",
+  "version": "1.5.5",
   "description": "Simple wrapper around NodeRed, RabbitMQ and MongoDB to support a more scaleable NodeRed implementation.\r Also the \"backend\" for [OpenRPA](https://github.com/skadefro/OpenRPA)",
   "main": "index.js",
   "scripts": {