diff --git a/.vscode/launch.json b/.vscode/launch.json index f55bc261..55de22d0 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -34,7 +34,8 @@ "runtimeArgs": [ "--nolazy", "--trace-warnings", - "--inspect" + "--inspect", + "--preserve-symlinks" ], "showAsyncStacks": true, "trace": true, diff --git a/OpenFlow/src/Config.ts b/OpenFlow/src/Config.ts index f85203a8..32f8f22d 100644 --- a/OpenFlow/src/Config.ts +++ b/OpenFlow/src/Config.ts @@ -1,10 +1,10 @@ import { fetch, toPassportConfig } from "passport-saml-metadata"; import * as fs from "fs"; import * as path from "path"; -import * as retry from "async-retry"; import { DatabaseConnection } from "./DatabaseConnection"; // import { Logger } from "./Logger"; import { NoderedUtil } from "@openiap/openflow-api"; +import { promiseRetry } from "./Logger"; export class Config { public static getversion(): string { let versionfile: string = path.join(__dirname, "VERSION"); @@ -428,22 +428,16 @@ export class Config { } public static async parse_federation_metadata(url: string): Promise { // if anything throws, we retry - return retry(async bail => { + return promiseRetry(async () => { const reader: any = await fetch({ url }); - if (NoderedUtil.IsNullUndefinded(reader)) { bail(new Error("Failed getting result")); return; } + if (NoderedUtil.IsNullUndefinded(reader)) { throw new Error("Failed getting result"); return; } const config: any = toPassportConfig(reader); // we need this, for Office 365 :-/ if (reader.signingCerts && reader.signingCerts.length > 1) { config.cert = reader.signingCerts; } return config; - }, { - retries: 50, - onRetry: function (error: Error, count: number): void { - console.log("retry " + count + " error " + error.message + " getting " + url); - // Logger.instanse.warn("retry " + count + " error " + error.message + " getting " + url); - } - }); + }, 10, 1000); } public static parseArray(s: string): string[] { let arr = s.split(","); diff --git a/OpenFlow/src/Crypt.ts b/OpenFlow/src/Crypt.ts index 5a7fe030..1d437745 100644 --- a/OpenFlow/src/Crypt.ts +++ b/OpenFlow/src/Crypt.ts @@ -126,6 +126,12 @@ export class Crypt { throw new Error('jwt must be provided'); } const o: any = jsonwebtoken.verify(token, Crypt.encryption_key); + let impostor: string = null; + if (!NoderedUtil.IsNullUndefinded(o) && !NoderedUtil.IsNullUndefinded(o.data) && !NoderedUtil.IsNullEmpty(o.data._id)) { + if (!NoderedUtil.IsNullEmpty(o.data.impostor)) { + impostor = o.data.impostor; + } + } if (!NoderedUtil.IsNullUndefinded(o) && !NoderedUtil.IsNullUndefinded(o.data) && !NoderedUtil.IsNullEmpty(o.data._id) && o.data._id != WellknownIds.root) { var id = o.data._id; o.data = await DBHelper.FindById(o.data._id, token, null); @@ -136,6 +142,7 @@ export class Crypt { var b = true; } } + if (!NoderedUtil.IsNullEmpty(impostor)) o.data.impostor = impostor; return TokenUser.assign(o.data); } diff --git a/OpenFlow/src/DBHelper.ts b/OpenFlow/src/DBHelper.ts index 8bf6e694..f579f0b4 100644 --- a/OpenFlow/src/DBHelper.ts +++ b/OpenFlow/src/DBHelper.ts @@ -10,13 +10,30 @@ import { LoginProvider } from "./LoginProvider"; import * as cacheManager from "cache-manager"; // var cacheManager = require('cache-manager'); var redisStore = require('cache-manager-ioredis'); +var mongoStore = require('@skadefro/cache-manager-mongodb'); export class DBHelper { public static memoryCache: any; + public static mongoCache: any; public static async init() { if (!NoderedUtil.IsNullUndefinded(this.memoryCache)) return; - if (Config.cache_store_type == "redis") { + + this.mongoCache = cacheManager.caching({ + store: mongoStore, + uri: Config.mongodb_url, + options: { + collection: "_cache", + compression: false, + poolSize: 5 + } + }); + + if (Config.cache_store_type == "mongodb") { + this.memoryCache = this.mongoCache; + DBHelper.ensureotel(); + return; + } else if (Config.cache_store_type == "redis") { this.memoryCache = cacheManager.caching({ store: redisStore, host: Config.cache_store_redis_host, @@ -32,6 +49,7 @@ export class DBHelper { redisClient.on('error', (error) => { console.log(error); }); + DBHelper.ensureotel(); return; } @@ -45,8 +63,6 @@ export class DBHelper { } public static async clearCache(reason: string) { this.init(); - // Auth.ensureotel(); - // this.memoryCache.reset(); var keys: string[]; if (Config.cache_store_type == "redis") { keys = await this.memoryCache.keys('*'); @@ -71,16 +87,6 @@ export class DBHelper { DBHelper.item_cache = Logger.otel.meter.createValueObserver("openflow_item_cache_count", { description: 'Total number of cached items' }, async (res) => { - // let keys: string[] = Object.keys(this.authorizationCache); - // let types = {}; - // for (let i = keys.length - 1; i >= 0; i--) { - // if (!types[this.authorizationCache[keys[i]].type]) types[this.authorizationCache[keys[i]].type] = 0; - // types[this.authorizationCache[keys[i]].type]++; - // } - // keys = Object.keys(types); - // for (let i = keys.length - 1; i >= 0; i--) { - // res.observe(types[keys[i]], { ...Logger.otel.defaultlabels, type: keys[i] }) - // } var keys: any = null; try { if (Config.cache_store_type == "redis") { @@ -105,7 +111,6 @@ export class DBHelper { try { if (NoderedUtil.IsNullEmpty(_id)) return null; let item = await this.memoryCache.wrap("users" + _id, () => { - // if (jwt === null || jwt == undefined || jwt == "") { jwt = Crypt.rootToken(); } if (Config.log_cache) Logger.instanse.debug("Add user to cache : " + _id); return Config.db.getbyid(_id, "users", Crypt.rootToken(), true, span); }); @@ -120,12 +125,16 @@ export class DBHelper { Logger.otel.endSpan(span); } } - public static async FindRequestTokenID(key: string, parent: Span): Promise { + public static async FindRequestTokenID(key: string, parent: Span): Promise { this.init(); const span: Span = Logger.otel.startSubSpan("dbhelper.FindRequestTokenID", parent); try { if (NoderedUtil.IsNullEmpty(key)) return null; - return await this.memoryCache.get("requesttoken" + key); + if (Config.cache_store_type == "redis") { + return await this.memoryCache.get("requesttoken" + key); + } else { + return await this.mongoCache.get("requesttoken" + key); + } } catch (error) { span?.recordException(error); throw error; @@ -133,11 +142,15 @@ export class DBHelper { Logger.otel.endSpan(span); } } - public static async AdddRequestTokenID(key: string, data: any, parent: Span): Promise { + public static async AdddRequestTokenID(key: string, data: any, parent: Span): Promise { this.init(); const span: Span = Logger.otel.startSubSpan("dbhelper.FindRequestTokenID", parent); try { - return await this.memoryCache.set("requesttoken" + key, data); + if (Config.cache_store_type == "redis") { + return await this.memoryCache.set("requesttoken" + key, data); + } else { + return await this.mongoCache.set("requesttoken" + key, data); + } } catch (error) { span?.recordException(error); throw error; @@ -145,11 +158,15 @@ export class DBHelper { Logger.otel.endSpan(span); } } - public static async RemoveRequestTokenID(key: string, parent: Span): Promise { + public static async RemoveRequestTokenID(key: string, parent: Span): Promise { this.init(); const span: Span = Logger.otel.startSubSpan("dbhelper.FindRequestTokenID", parent); try { - return await this.memoryCache.del("requesttoken" + key); + if (Config.cache_store_type == "redis") { + return await this.memoryCache.del("requesttoken" + key); + } else { + return await this.mongoCache.del("requesttoken" + key); + } } catch (error) { span?.recordException(error); throw error; @@ -598,4 +615,13 @@ export class DBHelper { return { $set: updatedoc }; } } +} +export class TokenRequest extends Base { + constructor(code: string) { + super(); + this._type = "tokenrequest"; + if (NoderedUtil.IsNullEmpty(code)) this.code = ""; + } + public code: string; + public jwt: string; } \ No newline at end of file diff --git a/OpenFlow/src/DatabaseConnection.ts b/OpenFlow/src/DatabaseConnection.ts index 3346f978..c0509a51 100644 --- a/OpenFlow/src/DatabaseConnection.ts +++ b/OpenFlow/src/DatabaseConnection.ts @@ -28,20 +28,6 @@ const jsondiffpatch = require('jsondiffpatch').create({ }); -Object.defineProperty(Promise, 'retry', { - configurable: true, - writable: true, - value: function retry(retries, executor) { - // console.warn(`${retries} retries left!`) - if (typeof retries !== 'number') { - throw new TypeError('retries is not a number') - } - return new Promise(executor).catch(error => retries > 0 - ? (Promise as any).retry(retries - 1, executor) - : Promise.reject(error) - ) - } -}) export type GetDocumentVersionOptions = { collectionname: string, id: string, @@ -159,18 +145,22 @@ export class DatabaseConnection extends events.EventEmitter { } const span: Span = Logger.otel.startSubSpan("db.connect", parent); this.streams = []; - this.cli = await (Promise as any).retry(100, (resolve, reject) => { - const options: MongoClientOptions = { minPoolSize: Config.mongodb_minpoolsize, autoReconnect: false, useNewUrlParser: true, useUnifiedTopology: true }; - MongoClient.connect(this.mongodburl, options).then((cli) => { - this.replicat = (cli as any).s.options.replicaSet; - resolve(cli); - span?.addEvent("Connected to mongodb"); - }).catch((reason) => { - span?.recordException(reason); - console.error(reason); - reject(reason); - }); - }); + // this.cli = await (Promise as any).retry(100, (resolve, reject) => { + // const options: MongoClientOptions = { minPoolSize: Config.mongodb_minpoolsize, autoReconnect: false, useNewUrlParser: true, useUnifiedTopology: true }; + // MongoClient.connect(this.mongodburl, options).then((cli) => { + // this.replicat = (cli as any).s.options.replicaSet; + // resolve(cli); + // span?.addEvent("Connected to mongodb"); + // }).catch((reason) => { + // span?.recordException(reason); + // console.error(reason); + // reject(reason); + // }); + // }); + span?.addEvent("connecting to mongodb"); + const options: MongoClientOptions = { minPoolSize: Config.mongodb_minpoolsize, autoReconnect: false, useNewUrlParser: true, useUnifiedTopology: true }; + this.cli = await MongoClient.connect(this.mongodburl, options); + span?.addEvent("Connected to mongodb"); Logger.instanse.silly(`Really connected to mongodb`); const errEvent = (error) => { diff --git a/OpenFlow/src/Logger.ts b/OpenFlow/src/Logger.ts index 7aa291c4..898d4d5b 100644 --- a/OpenFlow/src/Logger.ts +++ b/OpenFlow/src/Logger.ts @@ -4,6 +4,22 @@ import { i_license_file, i_nodered_driver, i_otel } from "./commoninterfaces"; import { Config } from "./Config"; import { dockerdriver } from "./dockerdriver"; const path = require('path'); + +const MAX_RETRIES_DEFAULT = 5 +export async function promiseRetry( + fn: () => Promise, + retries = MAX_RETRIES_DEFAULT, + retryIntervalMillis: number, + previousError?: Error +): Promise { + return !retries + ? Promise.reject(previousError) + : fn().catch(async (error) => { + await new Promise((resolve) => setTimeout(resolve, retryIntervalMillis)) + return promiseRetry(fn, retries - 1, retryIntervalMillis, error) + }) +} + export class Logger { public static otel: i_otel; public static License: i_license_file; diff --git a/OpenFlow/src/LoginProvider.ts b/OpenFlow/src/LoginProvider.ts index 447a6bb9..3fb0d929 100644 --- a/OpenFlow/src/LoginProvider.ts +++ b/OpenFlow/src/LoginProvider.ts @@ -13,8 +13,8 @@ const multer = require('multer'); // const GridFsStorage = require('multer-gridfs-storage'); import { GridFsStorage } from "multer-gridfs-storage"; import { GridFSBucket, ObjectID, Binary } from "mongodb"; -import { Base, User, NoderedUtil, TokenUser, WellknownIds, Rights, Role } from "@openiap/openflow-api"; -import { DBHelper } from "./DBHelper"; +import { Base, User, NoderedUtil, TokenUser, WellknownIds, Rights, Role, InsertOrUpdateOneMessage } from "@openiap/openflow-api"; +import { DBHelper, TokenRequest } from "./DBHelper"; import { Span } from "@opentelemetry/api"; import { Logger } from "./Logger"; import { DatabaseConnection } from "./DatabaseConnection"; @@ -409,7 +409,7 @@ export class LoginProvider { const span: Span = Logger.otel.startSpan("LoginProvider.login"); try { const key = req.body.key; - var exists = await DBHelper.FindRequestTokenID(key, span); + let exists: TokenRequest = await DBHelper.FindRequestTokenID(key, span); if (!NoderedUtil.IsNullUndefinded(exists)) return res.status(500).send({ message: "Illegal key" }); await DBHelper.AdddRequestTokenID(key, {}, span); res.status(200).send({ message: "ok" }); @@ -424,11 +424,11 @@ export class LoginProvider { const span: Span = Logger.otel.startSpan("LoginProvider.login"); try { const key = req.query.key; - var exists = await DBHelper.FindRequestTokenID(key, span); + let exists: TokenRequest = null; + exists = await DBHelper.FindRequestTokenID(key, span); if (NoderedUtil.IsNullUndefinded(exists)) { res.status(200).send({ message: "Illegal key" }); return; - // return res.status(500).send({ message: "Illegal key" }); } if (!NoderedUtil.IsNullEmpty(exists.jwt)) { @@ -478,7 +478,7 @@ export class LoginProvider { if (!NoderedUtil.IsNullEmpty(key)) { if (req.user) { const user: User = await DBHelper.FindById(req.user._id, undefined, span); - var exists = await DBHelper.FindRequestTokenID(key, span); + var exists: TokenRequest = await DBHelper.FindRequestTokenID(key, span); if (!NoderedUtil.IsNullUndefinded(exists)) { await DBHelper.AdddRequestTokenID(key, { jwt: Crypt.createToken(user, Config.longtoken_expires_in) }, span); res.cookie("requesttoken", "", { expires: new Date(0) }); @@ -523,7 +523,10 @@ export class LoginProvider { } catch (error) { span?.recordException(error); console.error(error.message ? error.message : error); - return res.status(500).send({ message: error.message ? error.message : error }); + try { + return res.status(500).send({ message: error.message ? error.message : error }); + } catch (error) { + } } Logger.otel.endSpan(span); }); diff --git a/OpenFlow/src/Messages/Message.ts b/OpenFlow/src/Messages/Message.ts index 8a29d54e..0d768b21 100644 --- a/OpenFlow/src/Messages/Message.ts +++ b/OpenFlow/src/Messages/Message.ts @@ -1917,6 +1917,8 @@ export class Message { } // await DBHelper.Save(user, Crypt.rootToken()); await Config.db._UpdateOne({ "_id": user._id }, UpdateDoc, "users", 1, false, Crypt.rootToken(), span) + DBHelper.memoryCache.del("users" + user._id); + if (NoderedUtil.IsNullEmpty(tuser.impostor)) DBHelper.memoryCache.del("users" + tuser.impostor); } } catch (error) { if (NoderedUtil.IsNullUndefinded(msg)) { (msg as any) = {}; } @@ -2008,7 +2010,7 @@ export class Message { msg = DeleteNoderedInstanceMessage.assign(this.data); const _tuser = await Crypt.verityToken(this.jwt); const instancename = await this.GetInstanceName(msg._id, _tuser._id, _tuser.username, this.jwt, span); - await Logger.nodereddriver.DeleteNoderedInstance(this.jwt, _tuser, msg._id, instancename, false, span); + await Logger.nodereddriver.DeleteNoderedInstance(this.jwt, _tuser, msg._id, instancename, span); } catch (error) { span?.recordException(error); this.data = ""; @@ -2032,7 +2034,7 @@ export class Message { msg = DeleteNoderedPodMessage.assign(this.data); const _tuser = await Crypt.verityToken(this.jwt); const instancename = await this.GetInstanceName(msg._id, _tuser._id, _tuser.username, this.jwt, span); - await Logger.nodereddriver.DeleteNoderedPod(this.jwt, _tuser, msg._id, instancename, msg.instancename, false, span); + await Logger.nodereddriver.DeleteNoderedPod(this.jwt, _tuser, msg._id, instancename, msg.instancename, span); } catch (error) { span?.recordException(error); this.data = ""; @@ -2056,7 +2058,7 @@ export class Message { msg = RestartNoderedInstanceMessage.assign(this.data); const _tuser = await Crypt.verityToken(this.jwt); const instancename = await this.GetInstanceName(msg._id, _tuser._id, _tuser.username, this.jwt, span); - await Logger.nodereddriver.RestartNoderedInstance(this.jwt, _tuser, msg._id, instancename, false, span); + await Logger.nodereddriver.RestartNoderedInstance(this.jwt, _tuser, msg._id, instancename, span); } catch (error) { span?.recordException(error); this.data = ""; @@ -2103,7 +2105,7 @@ export class Message { msg = GetNoderedInstanceMessage.assign(this.data); const _tuser = await Crypt.verityToken(this.jwt); const instancename = await this.GetInstanceName(msg._id, _tuser._id, _tuser.username, this.jwt, span); - msg.results = await Logger.nodereddriver.GetNoderedInstance(this.jwt, _tuser, msg._id, instancename, false, span); + msg.results = await Logger.nodereddriver.GetNoderedInstance(this.jwt, _tuser, msg._id, instancename, span); } catch (error) { span?.recordException(error); this.data = ""; @@ -2127,7 +2129,7 @@ export class Message { msg = GetNoderedInstanceLogMessage.assign(this.data); const _tuser = await Crypt.verityToken(this.jwt); const instancename = await this.GetInstanceName(msg._id, _tuser._id, _tuser.username, this.jwt, span); - msg.result = await Logger.nodereddriver.GetNoderedInstanceLog(this.jwt, _tuser, msg._id, instancename, msg.instancename, false, span); + msg.result = await Logger.nodereddriver.GetNoderedInstanceLog(this.jwt, _tuser, msg._id, instancename, msg.instancename, span); } catch (error) { span?.recordException(error); this.data = ""; @@ -4375,6 +4377,8 @@ export class Message { } if (wiq == null) throw new Error("Work item queue not found " + msg.wiq + " (" + msg.wiqid + ") not found."); + var additems = []; + // isRelevant = (msg.items.length > 0); for (let i = 0; i < msg.items.length; i++) { let item = msg.items[i]; @@ -4463,8 +4467,10 @@ export class Message { } } delete item.files; - wi = await Config.db.InsertOne(wi, "workitems", 1, true, jwt, parent); + // wi = await Config.db.InsertOne(wi, "workitems", 1, true, jwt, parent); + additems.push(wi); } + await Config.db.InsertMany(additems, "workitems", 1, true, jwt, parent); delete msg.items; msg.items = []; @@ -4549,6 +4555,7 @@ export class Message { } else if (["failed", "successful", "retry", "processing"].indexOf(msg.state) == -1) { throw new Error("Illegal state " + msg.state + " on Workitem, must be failed, successful, processing or retry"); } + if (msg.errortype == "business") msg.state == "failed"; if (msg.state == "retry") { if (NoderedUtil.IsNullEmpty(wi.retries)) wi.retries = 0; if (wi.retries < wiq.maxretries || msg.ignoremaxretries) { diff --git a/OpenFlow/src/OAuthProvider.ts b/OpenFlow/src/OAuthProvider.ts index 0185ce5d..6fc3f976 100644 --- a/OpenFlow/src/OAuthProvider.ts +++ b/OpenFlow/src/OAuthProvider.ts @@ -1,6 +1,6 @@ import * as OAuthServer from "oauth2-server"; import * as express from "express"; -import { TokenUser, Base, NoderedUtil, User } from "@openiap/openflow-api"; +import { TokenUser, Base, NoderedUtil, User, InsertOrUpdateOneMessage } from "@openiap/openflow-api"; import { Config } from "./Config"; import { Crypt } from "./Crypt"; import { Provider, KoaContextWithOIDC } from "oidc-provider"; @@ -715,12 +715,6 @@ export class Account { DBHelper.DeleteKey("user" + accountId); if (user == null) throw new Error("Cannot create Account from null user for id ${this.accountId}"); user = Object.assign(user, { accountId: accountId, sub: accountId }); - // var roles = []; - // user.roles.forEach(role => { - // roles.push(role.name); - // }); - // user.roles = roles; - // node-bb username hack if (NoderedUtil.IsNullEmpty(user.email)) user.email = user.username; if (user.name == user.email && user.email.indexOf("@") > -1) { @@ -741,11 +735,6 @@ export class Account { if (acc == null) { acc = await DBHelper.FindById(id, undefined, undefined); } - // let acc = Auth.getUser(id, "oidc"); - // if (!acc) { - // const user = await DBHelper.FindById(id, undefined, undefined); - // await Auth.AddUser(user, id, "oidc") - // } var res = new Account(id, TokenUser.From(acc)) return res; } @@ -765,78 +754,25 @@ export class Account { // DBHelper.DeleteKey("user" + tuser._id); var res = new Account(tuser._id, tuser); return res; - // let acc = Auth.getUser(tuser._id, "oidc"); - // if (!acc) { - // let role = client.defaultrole; - // const keys: string[] = Object.keys(client.rolemappings); - // for (let i = 0; i < keys.length; i++) { - // if (tuser.HasRoleName(keys[i])) role = client.rolemappings[keys[i]]; - // } - // (tuser as any).role = role; - // Auth.AddUser(tuser, tuser._id, "oidc") - // } - // return new Account(tuser._id, TokenUser.From(tuser)); } catch (error) { console.error(error); } return undefined; } + static async GetTokenRequest(code: string, parent: Span) { + let tokens = await Config.db.query({ query: { _type: "tokenrequest", "code": code }, top: 1, collectionname: "oauthtokens", jwt: Crypt.rootToken() }, parent); + if (tokens.length == 0) return null; + return tokens[0]; + } + static async AddTokenRequest(code: string, item: Base, parent: Span) { + var q: InsertOrUpdateOneMessage = new InsertOrUpdateOneMessage(); + q.item = item; q.uniqeness = "_type,code"; q.collectionname = "oauthtokens", q.jwt = Crypt.rootToken(); + q.w = 1; q.j = true; + let token = await Config.db.InsertOrUpdateOne(q, parent); + return token.item; + } + static async RemoveTokenRequest(code: string, parent: Span) { + let tokens = await Config.db.DeleteMany({ _type: "tokenrequest", "code": code }, null, "oauthtokens", Crypt.rootToken(), parent); + return tokens[0]; + } } - -// const accountStorage = new Map(); -// export class Account { -// constructor(public accountId: string, public user: TokenUser) { -// accountStorage.set(`Account:${this.accountId}`, this); -// if (user == null) throw new Error("Cannot create Account from null user for id ${this.accountId}"); -// user = Object.assign(user, { accountId: accountId, sub: accountId }); -// // var roles = []; -// // user.roles.forEach(role => { -// // roles.push(role.name); -// // }); -// // user.roles = roles; - -// // node-bb username hack -// if (NoderedUtil.IsNullEmpty(user.email)) user.email = user.username; -// if (user.name == user.email && user.email.indexOf("@") > -1) { -// user.name = user.email.substr(0, user.email.indexOf("@") - 1); -// } -// if (user.name == user.email && user.email.indexOf("@") == -1) { -// user.email = user.email + "@unknown.local" -// } -// if (user.name == user.email) { -// user.name = "user " + user.email; -// } -// } -// static get storage() { -// return accountStorage; -// } -// claims() { -// return this.user; -// } -// static async findAccount(ctx: KoaContextWithOIDC, id) { -// let acc = accountStorage.get(`Account:${id}`); -// if (!acc) { -// const user = await DBHelper.FindById(id, undefined, undefined); -// acc = new Account(id, TokenUser.From(user)); -// } -// return acc; -// } -// static AddAccount(tuser: TokenUser, client: any) { -// try { -// let acc = accountStorage.get(`Account:${tuser._id}`); -// if (!acc) { -// let role = client.defaultrole; -// const keys: string[] = Object.keys(client.rolemappings); -// for (let i = 0; i < keys.length; i++) { -// if (tuser.HasRoleName(keys[i])) role = client.rolemappings[keys[i]]; -// } -// (tuser as any).role = role; -// acc = new Account(tuser._id, tuser); -// } -// return acc; -// } catch (error) { -// console.error(error); -// } -// return undefined; -// } -// } \ No newline at end of file diff --git a/OpenFlow/src/QueueClient.ts b/OpenFlow/src/QueueClient.ts index 1186eef1..ec121239 100644 --- a/OpenFlow/src/QueueClient.ts +++ b/OpenFlow/src/QueueClient.ts @@ -108,8 +108,9 @@ export class QueueClient { } catch (error) { if (NoderedUtil.IsNullUndefinded(this.queue)) { Logger.instanse.warn("SendForProcessing queue is null, shutdown amqp connection"); - amqpwrapper.Instance().shutdown(); - amqpwrapper.Instance().connect(null); + process.exit(406); + // amqpwrapper.Instance().shutdown(); + // amqpwrapper.Instance().connect(null); } else { Logger.instanse.error(error); } diff --git a/OpenFlow/src/amqpwrapper.ts b/OpenFlow/src/amqpwrapper.ts index 2e177c08..62afa7cc 100644 --- a/OpenFlow/src/amqpwrapper.ts +++ b/OpenFlow/src/amqpwrapper.ts @@ -138,6 +138,11 @@ export class amqpwrapper extends events.EventEmitter { try { span?.addEvent("AddReplyQueue"); await this.AddReplyQueue(span); + this.channel.on('error', (error) => { + if (error.code != 404) { + Logger.instanse.error(error); + } + }); } catch (error) { Logger.instanse.error(error); if (Config.NODE_ENV == "production") { @@ -145,11 +150,6 @@ export class amqpwrapper extends events.EventEmitter { process.exit(405); } } - this.channel.on('error', (error) => { - if (error.code != 404) { - Logger.instanse.error(error); - } - }); try { await this.Adddlx(span); await this.AddOFExchange(span); @@ -256,22 +256,24 @@ export class amqpwrapper extends events.EventEmitter { } }) this.channel.on('close', async () => { - this.connected = false; - try { - if (this.conn != null) await this.conn.close(); - } catch (error) { - } - this.channel = null; - if (this.timeout != null) { - clearTimeout(this.timeout); - this.timeout = null; - } - this.timeout = setTimeout(this.connect.bind(this), 1000); + Logger.instanse.error("Exit, when we cannot create dead letter exchange and/or Openflow exchange"); + process.exit(406); + // this.connected = false; + // try { + // if (this.conn != null) await this.conn.close(); + // } catch (error) { + // } + // this.channel = null; + // if (this.timeout != null) { + // clearTimeout(this.timeout); + // this.timeout = null; + // } + // this.timeout = setTimeout(this.connect.bind(this), 1000); }); } catch (error) { span?.recordException(error); Logger.instanse.error(error); - return; + throw error; } finally { Logger.otel.endSpan(span); } diff --git a/OpenFlow/src/cli.ts b/OpenFlow/src/cli.ts index 5bd33516..5b66d4ce 100644 --- a/OpenFlow/src/cli.ts +++ b/OpenFlow/src/cli.ts @@ -2,6 +2,8 @@ // npm link --force // npm i npm install --global --production windows-build-tools import * as fs from "fs"; +import { Logger } from './Logger'; +Logger.configure(true, true); import { Config } from "./Config"; import { logger, loadenv, envfilename, envfilepathname, servicename, isOpenFlow } from "./nodeclient/cliutil"; import { WebSocketClient, NoderedUtil } from "@openiap/openflow-api"; diff --git a/OpenFlow/src/commoninterfaces.ts b/OpenFlow/src/commoninterfaces.ts index e16712ee..2082d018 100644 --- a/OpenFlow/src/commoninterfaces.ts +++ b/OpenFlow/src/commoninterfaces.ts @@ -76,10 +76,10 @@ export interface i_otel { export interface i_nodered_driver { detect(): Promise; EnsureNoderedInstance(jwt: string, tokenUser: TokenUser, _id: string, name: string, skipcreate: boolean, parent: Span): Promise; - GetNoderedInstance(jwt: string, tokenUser: TokenUser, _id: string, name: string, skipcreate: boolean, parent: Span): Promise; - RestartNoderedInstance(jwt: string, tokenUser: TokenUser, _id: string, name: string, skipcreate: boolean, parent: Span): Promise; - DeleteNoderedInstance(jwt: string, tokenUser: TokenUser, _id: string, name: string, skipcreate: boolean, parent: Span): Promise; - DeleteNoderedPod(jwt: string, user: TokenUser, _id: string, name: string, podname: string, skipcreate: boolean, parent: Span): Promise; - GetNoderedInstanceLog(jwt: string, user: TokenUser, _id: string, name: string, podname: string, skipcreate: boolean, parent: Span): Promise; + GetNoderedInstance(jwt: string, tokenUser: TokenUser, _id: string, name: string, parent: Span): Promise; + RestartNoderedInstance(jwt: string, tokenUser: TokenUser, _id: string, name: string, parent: Span): Promise; + DeleteNoderedInstance(jwt: string, tokenUser: TokenUser, _id: string, name: string, parent: Span): Promise; + DeleteNoderedPod(jwt: string, user: TokenUser, _id: string, name: string, podname: string, parent: Span): Promise; + GetNoderedInstanceLog(jwt: string, user: TokenUser, _id: string, name: string, podname: string, parent: Span): Promise; NodeLabels(parent: Span): Promise; } \ No newline at end of file diff --git a/OpenFlow/src/dockerdriver.ts b/OpenFlow/src/dockerdriver.ts index a190797a..a8a07900 100644 --- a/OpenFlow/src/dockerdriver.ts +++ b/OpenFlow/src/dockerdriver.ts @@ -186,7 +186,7 @@ export class dockerdriver implements i_nodered_driver { } } - public async GetNoderedInstance(jwt: string, tokenUser: TokenUser, _id: string, name: string, skipcreate: boolean, parent: Span): Promise { + public async GetNoderedInstance(jwt: string, tokenUser: TokenUser, _id: string, name: string, parent: Span): Promise { const span: Span = Logger.otel.startSubSpan("message.EnsureNoderedInstance", parent); try { span?.addEvent("init Docker()"); @@ -199,6 +199,33 @@ export class dockerdriver implements i_nodered_driver { var Created = new Date(item.Created * 1000); item.metadata = { creationTimestamp: Created, name: (item.Names[0] as string).substr(1) }; item.status = { phase: item.State } + + + // const itemname = item.metadata.name; + // const billed = item.metadata.labels.billed; + // const image = item.spec.containers[0].image + // const userid = item.metadata.labels.userid; + // const image = item.Image; + // const date = new Date(); + // const a: number = (date as any) - (Created as any); + // // const diffminutes = a / (1000 * 60); + // const diffhours = a / (1000 * 60 * 60); + // if ((image.indexOf("openflownodered") > -1 || image.indexOf("openiap/nodered") > -1) && !NoderedUtil.IsNullEmpty(userid)) { + // try { + // if (billed != "true" && diffhours > 24) { + // Logger.instanse.debug("[" + tokenUser.username + "] Remove un billed nodered instance " + itemname + " that has been running for " + diffhours + " hours"); + // await this.DeleteNoderedInstance(jwt, tokenUser, _id, name, true, span); + // } + // } catch (error) { + // } + // } else if (image.indexOf("openflownodered") > -1 || image.indexOf("openiap/nodered") > -1) { + // if (billed != "true" && diffhours > 24) { + // console.debug("unbilled " + name + " with no userid, should be removed, it has been running for " + diffhours + " hours"); + // } else { + // console.debug("unbilled " + name + " with no userid, has been running for " + diffhours + " hours"); + // } + // } + if (item.Names[0] == "/" + name) { span?.addEvent("getContainer(" + item.Id + ")"); const container = docker.getContainer(item.Id); @@ -227,7 +254,7 @@ export class dockerdriver implements i_nodered_driver { Logger.otel.endSpan(span); } } - public async RestartNoderedInstance(jwt: string, tuser: TokenUser, _id: string, name: string, skipcreate: boolean, parent: Span): Promise { + public async RestartNoderedInstance(jwt: string, tuser: TokenUser, _id: string, name: string, parent: Span): Promise { const span: Span = Logger.otel.startSubSpan("message.DockerRestartNoderedInstance", parent); try { span?.addEvent("init Docker()"); @@ -275,7 +302,7 @@ export class dockerdriver implements i_nodered_driver { }); }) } - public async GetNoderedInstanceLog(jwt: string, user: TokenUser, _id: string, name: string, podname: string, skipcreate: boolean, parent: Span): Promise { + public async GetNoderedInstanceLog(jwt: string, user: TokenUser, _id: string, name: string, podname: string, parent: Span): Promise { const span: Span = Logger.otel.startSubSpan("message.GetNoderedInstanceLog", parent); try { var result: string = null; @@ -314,10 +341,10 @@ export class dockerdriver implements i_nodered_driver { } } - public async DeleteNoderedInstance(jwt: string, tokenUser: TokenUser, _id: string, name: string, skipcreate: boolean, parent: Span): Promise { - this.DeleteNoderedPod(jwt, tokenUser, _id, name, null, skipcreate, parent); + public async DeleteNoderedInstance(jwt: string, tokenUser: TokenUser, _id: string, name: string, parent: Span): Promise { + this.DeleteNoderedPod(jwt, tokenUser, _id, name, null, parent); } - public async DeleteNoderedPod(jwt: string, user: TokenUser, _id: string, name: string, podname: string, skipcreate: boolean, parent: Span): Promise { + public async DeleteNoderedPod(jwt: string, user: TokenUser, _id: string, name: string, podname: string, parent: Span): Promise { const span: Span = Logger.otel.startSubSpan("message.dockerDeleteNoderedPod", parent); try { Logger.instanse.debug("[" + user.username + "] dockerDeleteNoderedPod"); diff --git a/OpenFlow/src/rabbitmq.ts b/OpenFlow/src/rabbitmq.ts index f94e2c89..1a8ac94f 100644 --- a/OpenFlow/src/rabbitmq.ts +++ b/OpenFlow/src/rabbitmq.ts @@ -1,9 +1,8 @@ import { Config } from "./Config"; -import * as retry from "async-retry"; import * as url from "url"; import { AssertQueue } from "./amqpwrapper"; import { NoderedUtil } from "@openiap/openflow-api"; -import { Logger } from "./Logger"; +import { Logger, promiseRetry } from "./Logger"; const got = require("got"); export class rabbitmq { @@ -46,7 +45,7 @@ export class rabbitmq { async checkQueueConsumerCount(queuename: string): Promise { let result: boolean = false; try { - result = await retry(async bail => { + result = await promiseRetry(async () => { const queue = await rabbitmq.getqueue(Config.amqp_url, '/', queuename); // const queue = await amqpwrapper.getqueue(queuename); let hasConsumers: boolean = false; @@ -66,15 +65,7 @@ export class rabbitmq { // return bail(); } return hasConsumers; - }, { - retries: 10, - minTimeout: 500, - maxTimeout: 500, - onRetry: function (error: Error, count: number): void { - result = false; - console.warn("retry " + count + " error " + error.message + " getting " + url); - } - }); + }, 10, 1000); } catch (error) { Logger.instanse.debug(error.message ? error.message : error); } diff --git a/OpenFlowNodeRED/package-lock.json b/OpenFlowNodeRED/package-lock.json index e855345c..5f27dc03 100644 --- a/OpenFlowNodeRED/package-lock.json +++ b/OpenFlowNodeRED/package-lock.json @@ -1,12 +1,12 @@ { "name": "@openiap/nodered", - "version": "1.4.3", + "version": "1.4.6", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@openiap/nodered", - "version": "1.4.3", + "version": "1.4.6", "license": "MPL-2.0", "dependencies": { "@nodemailer/mailparser2": "^1.0.3", @@ -16,7 +16,6 @@ "@opentelemetry/exporter-collector-grpc": "0.19.0", "@opentelemetry/metrics": "0.19.0", "@opentelemetry/tracing": "0.19.0", - "async-retry": "^1.3.3", "body-parser": "^1.19.0", "command-line-args": "^5.2.0", "compression": "^1.7.4", @@ -1383,14 +1382,6 @@ "tslib": "^2.3.1" } }, - "node_modules/async-retry": { - "version": "1.3.3", - "resolved": "https://registry.npmjs.org/async-retry/-/async-retry-1.3.3.tgz", - "integrity": "sha512-wfr/jstw9xNi/0teMHrRW7dsz3Lt5ARhYNZ2ewpadnhaIp5mbALhOAP+EAdsC7t4Z6wqsDVv9+W6gm1Dk9mEyw==", - "dependencies": { - "retry": "0.13.1" - } - }, "node_modules/asynckit": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", @@ -5700,14 +5691,6 @@ "lowercase-keys": "^2.0.0" } }, - "node_modules/retry": { - "version": "0.13.1", - "resolved": "https://registry.npmjs.org/retry/-/retry-0.13.1.tgz", - "integrity": "sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==", - "engines": { - "node": ">= 4" - } - }, "node_modules/rfdc": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/rfdc/-/rfdc-1.3.0.tgz", @@ -7653,14 +7636,6 @@ "tslib": "^2.3.1" } }, - "async-retry": { - "version": "1.3.3", - "resolved": "https://registry.npmjs.org/async-retry/-/async-retry-1.3.3.tgz", - "integrity": "sha512-wfr/jstw9xNi/0teMHrRW7dsz3Lt5ARhYNZ2ewpadnhaIp5mbALhOAP+EAdsC7t4Z6wqsDVv9+W6gm1Dk9mEyw==", - "requires": { - "retry": "0.13.1" - } - }, "asynckit": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", @@ -11013,11 +10988,6 @@ "lowercase-keys": "^2.0.0" } }, - "retry": { - "version": "0.13.1", - "resolved": "https://registry.npmjs.org/retry/-/retry-0.13.1.tgz", - "integrity": "sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==" - }, "rfdc": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/rfdc/-/rfdc-1.3.0.tgz", diff --git a/OpenFlowNodeRED/package.json b/OpenFlowNodeRED/package.json index 3c954983..9931f3ad 100644 --- a/OpenFlowNodeRED/package.json +++ b/OpenFlowNodeRED/package.json @@ -1,6 +1,6 @@ { "name": "@openiap/nodered", - "version": "1.4.5", + "version": "1.4.7", "description": "Simple wrapper around NodeRed, RabbitMQ and MongoDB to support a more scaleable NodeRed implementation.\r Also the \"backend\" for [OpenRPA](https://github.com/skadefro/OpenRPA)", "main": "index.js", "scripts": { @@ -31,7 +31,6 @@ "@opentelemetry/exporter-collector-grpc": "0.19.0", "@opentelemetry/metrics": "0.19.0", "@opentelemetry/tracing": "0.19.0", - "async-retry": "^1.3.3", "body-parser": "^1.19.0", "command-line-args": "^5.2.0", "compression": "^1.7.4", diff --git a/OpenFlowNodeRED/src/Config.ts b/OpenFlowNodeRED/src/Config.ts index 5df5d60f..db3c1aa3 100644 --- a/OpenFlowNodeRED/src/Config.ts +++ b/OpenFlowNodeRED/src/Config.ts @@ -1,9 +1,9 @@ import * as https from "https"; -import * as retry from "async-retry"; import * as fs from "fs"; import * as path from "path"; import { fetch, toPassportConfig } from "passport-saml-metadata"; import { NoderedUtil } from "@openiap/openflow-api"; +import { promiseRetry } from "./Logger"; const { networkInterfaces } = require('os'); export class Config { public static getversion(): string { @@ -250,23 +250,18 @@ export class Config { } // if anything throws, we retry - const metadata: any = await retry(async bail => { + const metadata: any = await promiseRetry(async () => { if (Config.saml_ignore_cert) process.env.NODE_TLS_REJECT_UNAUTHORIZED = "0"; const reader: any = await fetch({ url }); if (Config.saml_ignore_cert) process.env.NODE_TLS_REJECT_UNAUTHORIZED = "1"; - if (reader === null || reader === undefined) { bail(new Error("Failed getting result")); return; } + if (reader === null || reader === undefined) { throw new Error("Failed getting result"); return; } const config: any = toPassportConfig(reader); // we need this, for Office 365 :-/ if (reader.signingCerts && reader.signingCerts.length > 1) { config.cert = reader.signingCerts; } return config; - }, { - retries: 50, - onRetry: function (error: Error, count: number): void { - console.log("retry " + count + " error " + error.message + " getting " + url); - } - }); + }, 10, 1000); return metadata; } diff --git a/OpenFlowNodeRED/src/Logger.ts b/OpenFlowNodeRED/src/Logger.ts index 29b3a27b..89e93cfa 100644 --- a/OpenFlowNodeRED/src/Logger.ts +++ b/OpenFlowNodeRED/src/Logger.ts @@ -3,6 +3,36 @@ import { Config } from "./Config"; const path = require('path'); import { format } from 'winston'; import { i_otel } from "./commoninterfaces"; + +const MAX_RETRIES_DEFAULT = 5 +export async function promiseRetry( + fn: () => Promise, + retries = MAX_RETRIES_DEFAULT, + retryIntervalMillis: number, + previousError?: Error +): Promise { + return !retries + ? Promise.reject(previousError) + : fn().catch(async (error) => { + await new Promise((resolve) => setTimeout(resolve, retryIntervalMillis)) + return promiseRetry(fn, retries - 1, retryIntervalMillis, error) + }) +} + +// Object.defineProperty(Promise, 'retry', { +// configurable: true, +// writable: true, +// value: function retry(retries, executor) { +// // console.warn(`${retries} retries left!`) +// if (typeof retries !== 'number') { +// throw new TypeError('retries is not a number') +// } +// return new Promise(executor).catch(error => { +// retries > 0 ? (Promise as any).retry(retries - 1, executor) : Promise.reject(error); +// } +// ) +// } +// }) export class Logger { public static otel: i_otel; static configure(): winston.Logger { diff --git a/OpenFlowNodeRED/src/cli.ts b/OpenFlowNodeRED/src/cli.ts index 95847166..d8fc2c7d 100644 --- a/OpenFlowNodeRED/src/cli.ts +++ b/OpenFlowNodeRED/src/cli.ts @@ -2,10 +2,12 @@ // npm link --force // npm i npm install --global --production windows-build-tools import * as fs from "fs"; +import { Logger } from './Logger'; +Logger.configure(); import { Config } from "./Config"; import { logger, loadenv, envfilename, envfilepathname, servicename, isOpenFlow } from "./nodeclient/cliutil"; -import { WebSocketClient, SigninMessage, Message, NoderedUtil } from "@openiap/openflow-api"; -import { pm2stop, pm2delete, pm2start, pm2restart, pm2list, pm2disconnect, pm2dump, pm2startup, pm2exists } from "./nodeclient/pm2util"; +import { WebSocketClient, NoderedUtil } from "@openiap/openflow-api"; +import { pm2stop, pm2delete, pm2start, pm2restart, pm2disconnect, pm2dump, pm2startup, pm2exists } from "./nodeclient/pm2util"; const optionDefinitions = [ { name: 'verbose', alias: 'v', type: Boolean }, diff --git a/OpenFlowNodeRED/src/node-red-contrib-auth-saml.ts b/OpenFlowNodeRED/src/node-red-contrib-auth-saml.ts index a7b2f1c4..e7bb6a95 100644 --- a/OpenFlowNodeRED/src/node-red-contrib-auth-saml.ts +++ b/OpenFlowNodeRED/src/node-red-contrib-auth-saml.ts @@ -3,8 +3,7 @@ import * as path from "path"; import * as SAMLStrategy from "passport-saml"; import { fetch, toPassportConfig } from "passport-saml-metadata"; import * as https from "https"; -import * as retry from "async-retry"; -import { Logger } from "./Logger"; +import { Logger, promiseRetry } from "./Logger"; import { Config } from "./Config"; export const logger = Logger.configure(); import { FileSystemCache } from "@openiap/openflow-api"; @@ -73,24 +72,19 @@ export class noderedcontribauthsaml { console.error(error); } // if anything throws, we retry - const metadata: any = await retry(async bail => { + const metadata: any = await promiseRetry(async () => { if (Config.saml_ignore_cert) process.env.NODE_TLS_REJECT_UNAUTHORIZED = "0"; const backupStore = new FileSystemCache(path.join(Config.logpath, '.cache-' + Config.nodered_id)); const reader: any = await fetch({ url, backupStore }); if (Config.saml_ignore_cert) process.env.NODE_TLS_REJECT_UNAUTHORIZED = "1"; - if (reader === null || reader === undefined) { bail(new Error("Failed getting result")); return; } + if (reader === null || reader === undefined) { throw new Error("Failed getting result"); return; } const config: any = toPassportConfig(reader); // we need this, for Office 365 :-/ if (reader.signingCerts && reader.signingCerts.length > 1) { config.cert = reader.signingCerts; } return config; - }, { - retries: 50, - onRetry: function (error: Error, count: number): void { - console.error("retry " + count + " error " + error.message + " getting " + url); - } - }); + }, 50, 1000); return metadata; } constructor(baseURL: string) { diff --git a/VERSION b/VERSION index 03e5161d..b000a6a0 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.4.5 \ No newline at end of file +1.4.7 \ No newline at end of file diff --git a/gulpfile.js b/gulpfile.js index ad884dbc..cda7156c 100644 --- a/gulpfile.js +++ b/gulpfile.js @@ -246,12 +246,12 @@ gulp.task("compose", shell.task([ ])); gulp.task("latest", shell.task([ - 'docker tag openiap/nodered:edge openiap/nodered:latest', - 'echo "Push openiap/nodered"', - 'docker push openiap/nodered:latest', 'docker tag openiap/openflow:edge openiap/openflow:latest', 'echo "Push openiap/openflow"', 'docker push openiap/openflow:latest', + 'docker tag openiap/nodered:edge openiap/nodered:latest', + 'echo "Push openiap/nodered"', + 'docker push openiap/nodered:latest', 'docker tag openiap/nodered-puppeteer:edge openiap/nodered-puppeteer:latest', 'echo "Push openiap/nodered-puppeteer"', 'docker push openiap/nodered-puppeteer:latest', diff --git a/package-lock.json b/package-lock.json index aba9f170..5f42b51b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@openiap/openflow", - "version": "1.4.3", + "version": "1.4.6", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@openiap/openflow", - "version": "1.4.3", + "version": "1.4.6", "license": "MPL-2.0", "dependencies": { "@kubernetes/client-node": "0.14.3", @@ -16,10 +16,10 @@ "@opentelemetry/exporter-collector-grpc": "0.19.0", "@opentelemetry/metrics": "0.19.0", "@opentelemetry/tracing": "0.19.0", + "@skadefro/cache-manager-mongodb": "^0.3.2", "@types/amqplib": "^0.8.2", "@types/node": "^15.12.4", "amqplib": "^0.8.0", - "async-retry": "^1.3.1", "bcryptjs": "^2.4.3", "body-parser": "^1.19.0", "cache-manager": "^3.6.1", @@ -43,7 +43,7 @@ "jsonpath-plus": "^5.0.7", "jsonwebtoken": "^8.5.1", "mimetype": "0.0.8", - "mongodb": "^3.6.9", + "mongodb": "^3.7.3", "morgan": "^1.10.0", "multer": "^1.4.2", "multer-gridfs-storage": "^5.0.0", @@ -137,6 +137,25 @@ "url": "https://github.com/sponsors/skadefro" } }, + "../PR/node-cache-manager-mongodb": { + "name": "@skadefro/cache-manager-mongodb", + "version": "0.3.2", + "extraneous": true, + "license": "MIT", + "dependencies": { + "cache-manager": "^2.9.0", + "lodash": "^4.17.15", + "mongodb": "^3.7.3" + }, + "devDependencies": { + "cache-manager": "^3.4.0", + "grunt": "^1.0.4", + "grunt-contrib-jshint": "^2.0.0", + "jshint-stylish": "^2.2.1", + "load-grunt-tasks": "^4.0.0", + "mocha": "^8.2.1" + } + }, "node_modules/@auth0/thumbprint": { "version": "0.0.6", "resolved": "https://registry.npmjs.org/@auth0/thumbprint/-/thumbprint-0.0.6.tgz", @@ -1476,6 +1495,45 @@ "url": "https://github.com/sindresorhus/is?sponsor=1" } }, + "node_modules/@skadefro/cache-manager-mongodb": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/@skadefro/cache-manager-mongodb/-/cache-manager-mongodb-0.3.2.tgz", + "integrity": "sha512-z5Ojv3d4KgvI/NZggLbg1jdcZcIhe8gQzjTfcz1hRRTU18L09nU63+EQkfe4CYXUgITXqIrZ/iySxJL5jjhanQ==", + "dependencies": { + "cache-manager": "^2.9.0", + "lodash": "^4.17.15", + "mongodb": "^3.7.3" + } + }, + "node_modules/@skadefro/cache-manager-mongodb/node_modules/async": { + "version": "1.5.2", + "resolved": "https://registry.npmjs.org/async/-/async-1.5.2.tgz", + "integrity": "sha1-7GphrlZIDAw8skHJVhjiCJL5Zyo=" + }, + "node_modules/@skadefro/cache-manager-mongodb/node_modules/cache-manager": { + "version": "2.11.1", + "resolved": "https://registry.npmjs.org/cache-manager/-/cache-manager-2.11.1.tgz", + "integrity": "sha512-XhUuc9eYwkzpK89iNewFwtvcDYMUsvtwzHeyEOPJna/WsVsXcrzsA1ft2M0QqPNunEzLhNCYPo05tEfG+YuNow==", + "dependencies": { + "async": "1.5.2", + "lodash.clonedeep": "4.5.0", + "lru-cache": "4.0.0" + } + }, + "node_modules/@skadefro/cache-manager-mongodb/node_modules/lru-cache": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-4.0.0.tgz", + "integrity": "sha1-tcvwFVbBaWb+vlTO7A+03JDfbCg=", + "dependencies": { + "pseudomap": "^1.0.1", + "yallist": "^2.0.0" + } + }, + "node_modules/@skadefro/cache-manager-mongodb/node_modules/yallist": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-2.1.2.tgz", + "integrity": "sha1-HBH5IY8HYImkfdUS+TxmmaaoHVI=" + }, "node_modules/@szmarczak/http-timer": { "version": "4.0.5", "resolved": "https://registry.npmjs.org/@szmarczak/http-timer/-/http-timer-4.0.5.tgz", @@ -2593,14 +2651,6 @@ "node": "<=0.11.8 || >0.11.10" } }, - "node_modules/async-retry": { - "version": "1.3.3", - "resolved": "https://registry.npmjs.org/async-retry/-/async-retry-1.3.3.tgz", - "integrity": "sha512-wfr/jstw9xNi/0teMHrRW7dsz3Lt5ARhYNZ2ewpadnhaIp5mbALhOAP+EAdsC7t4Z6wqsDVv9+W6gm1Dk9mEyw==", - "dependencies": { - "retry": "0.13.1" - } - }, "node_modules/async-settle": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/async-settle/-/async-settle-1.0.0.tgz", @@ -9229,6 +9279,11 @@ "resolved": "https://registry.npmjs.org/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz", "integrity": "sha1-soqmKIorn8ZRA1x3EfZathkDMaY=" }, + "node_modules/lodash.clonedeep": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/lodash.clonedeep/-/lodash.clonedeep-4.5.0.tgz", + "integrity": "sha1-4j8/nE+Pvd6HJSnBBxhXoIblzO8=" + }, "node_modules/lodash.defaults": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz", @@ -10610,14 +10665,14 @@ } }, "node_modules/mongodb": { - "version": "3.7.0", - "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-3.7.0.tgz", - "integrity": "sha512-JOAYjT9WYeRFkIP6XtDidAr3qvpfLRJhT2iokRWWH0tgqCQr9kmSfOJBZ3Ry0E5A3EqKxVPVhN3MV8Gn03o7pA==", + "version": "3.7.3", + "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-3.7.3.tgz", + "integrity": "sha512-Psm+g3/wHXhjBEktkxXsFMZvd3nemI0r3IPsE0bU+4//PnvNWKkzhZcEsbPcYiWqe8XqXJJEg4Tgtr7Raw67Yw==", "dependencies": { "bl": "^2.2.1", "bson": "^1.1.4", "denque": "^1.4.1", - "optional-require": "^1.0.3", + "optional-require": "^1.1.8", "safe-buffer": "^5.1.2" }, "engines": { @@ -11701,9 +11756,9 @@ } }, "node_modules/optional-require": { - "version": "1.1.7", - "resolved": "https://registry.npmjs.org/optional-require/-/optional-require-1.1.7.tgz", - "integrity": "sha512-cIeRZocXsZnZYn+SevbtSqNlLbeoS4mLzuNn4fvXRMDRNhTGg0sxuKXl0FnZCtnew85LorNxIbZp5OeliILhMw==", + "version": "1.1.8", + "resolved": "https://registry.npmjs.org/optional-require/-/optional-require-1.1.8.tgz", + "integrity": "sha512-jq83qaUb0wNg9Krv1c5OQ+58EK+vHde6aBPzLvPPqJm89UQWsvSuFy9X/OSNJnFeSOKo7btE0n8Nl2+nE+z5nA==", "dependencies": { "require-at": "^1.0.6" }, @@ -12992,6 +13047,11 @@ "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==" }, + "node_modules/pseudomap": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/pseudomap/-/pseudomap-1.0.2.tgz", + "integrity": "sha1-8FKijacOYYkX7wqKw0wa5aaChrM=" + }, "node_modules/psl": { "version": "1.8.0", "resolved": "https://registry.npmjs.org/psl/-/psl-1.8.0.tgz", @@ -13785,14 +13845,6 @@ "node": ">=0.12" } }, - "node_modules/retry": { - "version": "0.13.1", - "resolved": "https://registry.npmjs.org/retry/-/retry-0.13.1.tgz", - "integrity": "sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==", - "engines": { - "node": ">= 4" - } - }, "node_modules/rfc4648": { "version": "1.5.0", "resolved": "https://registry.npmjs.org/rfc4648/-/rfc4648-1.5.0.tgz", @@ -18171,6 +18223,47 @@ "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-4.0.0.tgz", "integrity": "sha512-FyD2meJpDPjyNQejSjvnhpgI/azsQkA4lGbuu5BQZfjvJ9cbRZXzeWL2HceCekW4lixO9JPesIIQkSoLjeJHNQ==" }, + "@skadefro/cache-manager-mongodb": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/@skadefro/cache-manager-mongodb/-/cache-manager-mongodb-0.3.2.tgz", + "integrity": "sha512-z5Ojv3d4KgvI/NZggLbg1jdcZcIhe8gQzjTfcz1hRRTU18L09nU63+EQkfe4CYXUgITXqIrZ/iySxJL5jjhanQ==", + "requires": { + "cache-manager": "^2.9.0", + "lodash": "^4.17.15", + "mongodb": "^3.7.3" + }, + "dependencies": { + "async": { + "version": "1.5.2", + "resolved": "https://registry.npmjs.org/async/-/async-1.5.2.tgz", + "integrity": "sha1-7GphrlZIDAw8skHJVhjiCJL5Zyo=" + }, + "cache-manager": { + "version": "2.11.1", + "resolved": "https://registry.npmjs.org/cache-manager/-/cache-manager-2.11.1.tgz", + "integrity": "sha512-XhUuc9eYwkzpK89iNewFwtvcDYMUsvtwzHeyEOPJna/WsVsXcrzsA1ft2M0QqPNunEzLhNCYPo05tEfG+YuNow==", + "requires": { + "async": "1.5.2", + "lodash.clonedeep": "4.5.0", + "lru-cache": "4.0.0" + } + }, + "lru-cache": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-4.0.0.tgz", + "integrity": "sha1-tcvwFVbBaWb+vlTO7A+03JDfbCg=", + "requires": { + "pseudomap": "^1.0.1", + "yallist": "^2.0.0" + } + }, + "yallist": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-2.1.2.tgz", + "integrity": "sha1-HBH5IY8HYImkfdUS+TxmmaaoHVI=" + } + } + }, "@szmarczak/http-timer": { "version": "4.0.5", "resolved": "https://registry.npmjs.org/@szmarczak/http-timer/-/http-timer-4.0.5.tgz", @@ -19157,14 +19250,6 @@ "shimmer": "^1.1.0" } }, - "async-retry": { - "version": "1.3.3", - "resolved": "https://registry.npmjs.org/async-retry/-/async-retry-1.3.3.tgz", - "integrity": "sha512-wfr/jstw9xNi/0teMHrRW7dsz3Lt5ARhYNZ2ewpadnhaIp5mbALhOAP+EAdsC7t4Z6wqsDVv9+W6gm1Dk9mEyw==", - "requires": { - "retry": "0.13.1" - } - }, "async-settle": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/async-settle/-/async-settle-1.0.0.tgz", @@ -24597,6 +24682,11 @@ "resolved": "https://registry.npmjs.org/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz", "integrity": "sha1-soqmKIorn8ZRA1x3EfZathkDMaY=" }, + "lodash.clonedeep": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/lodash.clonedeep/-/lodash.clonedeep-4.5.0.tgz", + "integrity": "sha1-4j8/nE+Pvd6HJSnBBxhXoIblzO8=" + }, "lodash.defaults": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz", @@ -25690,14 +25780,14 @@ } }, "mongodb": { - "version": "3.7.0", - "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-3.7.0.tgz", - "integrity": "sha512-JOAYjT9WYeRFkIP6XtDidAr3qvpfLRJhT2iokRWWH0tgqCQr9kmSfOJBZ3Ry0E5A3EqKxVPVhN3MV8Gn03o7pA==", + "version": "3.7.3", + "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-3.7.3.tgz", + "integrity": "sha512-Psm+g3/wHXhjBEktkxXsFMZvd3nemI0r3IPsE0bU+4//PnvNWKkzhZcEsbPcYiWqe8XqXJJEg4Tgtr7Raw67Yw==", "requires": { "bl": "^2.2.1", "bson": "^1.1.4", "denque": "^1.4.1", - "optional-require": "^1.0.3", + "optional-require": "^1.1.8", "safe-buffer": "^5.1.2", "saslprep": "^1.0.0" }, @@ -26535,9 +26625,9 @@ } }, "optional-require": { - "version": "1.1.7", - "resolved": "https://registry.npmjs.org/optional-require/-/optional-require-1.1.7.tgz", - "integrity": "sha512-cIeRZocXsZnZYn+SevbtSqNlLbeoS4mLzuNn4fvXRMDRNhTGg0sxuKXl0FnZCtnew85LorNxIbZp5OeliILhMw==", + "version": "1.1.8", + "resolved": "https://registry.npmjs.org/optional-require/-/optional-require-1.1.8.tgz", + "integrity": "sha512-jq83qaUb0wNg9Krv1c5OQ+58EK+vHde6aBPzLvPPqJm89UQWsvSuFy9X/OSNJnFeSOKo7btE0n8Nl2+nE+z5nA==", "requires": { "require-at": "^1.0.6" } @@ -27531,6 +27621,11 @@ "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==" }, + "pseudomap": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/pseudomap/-/pseudomap-1.0.2.tgz", + "integrity": "sha1-8FKijacOYYkX7wqKw0wa5aaChrM=" + }, "psl": { "version": "1.8.0", "resolved": "https://registry.npmjs.org/psl/-/psl-1.8.0.tgz", @@ -28207,11 +28302,6 @@ "integrity": "sha512-TTlYpa+OL+vMMNG24xSlQGEJ3B/RzEfUlLct7b5G/ytav+wPrplCpVMFuwzXbkecJrb6IYo1iFb0S9v37754mg==", "dev": true }, - "retry": { - "version": "0.13.1", - "resolved": "https://registry.npmjs.org/retry/-/retry-0.13.1.tgz", - "integrity": "sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==" - }, "rfc4648": { "version": "1.5.0", "resolved": "https://registry.npmjs.org/rfc4648/-/rfc4648-1.5.0.tgz", diff --git a/package.json b/package.json index 57d91432..1a334d93 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@openiap/openflow", - "version": "1.4.5", + "version": "1.4.7", "description": "Simple wrapper around NodeRed, RabbitMQ and MongoDB to support a more scaleable NodeRed implementation.\r Also the \"backend\" for [OpenRPA](https://github.com/skadefro/OpenRPA)", "main": "index.js", "scripts": { @@ -38,10 +38,10 @@ "@opentelemetry/exporter-collector-grpc": "0.19.0", "@opentelemetry/metrics": "0.19.0", "@opentelemetry/tracing": "0.19.0", + "@skadefro/cache-manager-mongodb": "^0.3.2", "@types/amqplib": "^0.8.2", "@types/node": "^15.12.4", "amqplib": "^0.8.0", - "async-retry": "^1.3.1", "bcryptjs": "^2.4.3", "body-parser": "^1.19.0", "cache-manager": "^3.6.1", @@ -65,7 +65,7 @@ "jsonpath-plus": "^5.0.7", "jsonwebtoken": "^8.5.1", "mimetype": "0.0.8", - "mongodb": "^3.6.9", + "mongodb": "^3.7.3", "morgan": "^1.10.0", "multer": "^1.4.2", "multer-gridfs-storage": "^5.0.0",