From 5e8bea956f9210b7d1a647b0e9924ba919c3b74f Mon Sep 17 00:00:00 2001 From: Fritz Ray Date: Fri, 8 Dec 2023 13:35:05 -0800 Subject: [PATCH] Introduced SKYREPO_DEBUG, disable HTTP2 in library use by default. --- docker/standalone/Dockerfile | 1 + docker/standalone/node/Dockerfile | 1 + src/main/server.js | 4 +- src/main/server/shims/auditLogger.js | 2 +- src/main/server/shims/levr.js | 174 +++++++++++++-------------- src/main/server/skyRepo.js | 56 ++++----- src/main/server/util.js | 164 ++++++++++++------------- 7 files changed, 202 insertions(+), 200 deletions(-) diff --git a/docker/standalone/Dockerfile b/docker/standalone/Dockerfile index 0cf5432fc..3240a80b1 100644 --- a/docker/standalone/Dockerfile +++ b/docker/standalone/Dockerfile @@ -53,5 +53,6 @@ USER root RUN mkdir /logs RUN chown -R elasticsearch:root /logs +ENV HTTP2=false USER elasticsearch:root ENTRYPOINT (cd CASS && npm run run:standalone && npm run logs) & /bin/tini -- /usr/local/bin/docker-entrypoint.sh \ No newline at end of file diff --git a/docker/standalone/node/Dockerfile b/docker/standalone/node/Dockerfile index ba0acb061..c406aee12 100644 --- a/docker/standalone/node/Dockerfile +++ b/docker/standalone/node/Dockerfile @@ -28,4 +28,5 @@ COPY cass.key cass.key COPY copyright.txt copyright.txt COPY LICENSE LICENSE EXPOSE 80 +ENV HTTP2=false CMD node --force-fips ./src/main/server.js \ No newline at end of file diff --git a/src/main/server.js b/src/main/server.js index 9ebed44cb..78f24634e 100644 --- a/src/main/server.js +++ b/src/main/server.js @@ -22,7 +22,7 @@ const express = require('express'); const https = require('https'); const spdy = require('spdy'); const baseUrl = global.baseUrl = process.env.CASS_BASE || ''; -const envHttp2 = process.env.HTTP2 != null ? process.env.HTTP2.trim() == 'true' : true; +const envHttp2 = process.env.HTTP2_SERVER != null ? process.env.HTTP2_SERVER.trim() == 'true' : true; let app = global.app = express(); global.auditLogger = require('./server/shims/auditLogger.js'); require('cassproject'); @@ -57,7 +57,7 @@ repo.selectedServerProxy = process.env.CASS_LOOPBACK_PROXY || null; global.elasticEndpoint = process.env.ELASTICSEARCH_ENDPOINT || 'http://localhost:9200'; -global.skyrepoDebug = false; +global.skyrepoDebug = process.env.SKYREPO_DEBUG || false; global.thisEndpoint = function() { return repo.selectedServer; }; diff --git a/src/main/server/shims/auditLogger.js b/src/main/server/shims/auditLogger.js index 1a111921a..5d34221d2 100644 --- a/src/main/server/shims/auditLogger.js +++ b/src/main/server/shims/auditLogger.js @@ -112,7 +112,7 @@ let report = function(system, severity, message, ...data) { if (data.length == 1) { data = data[0]; } - if (severity <= 6) { + if (severity <= 6 || global.skyrepoDebug) { try { if (EcArray.isArray(data)) data = JSON.stringify(data); console.log(new Date(), system, InverseSeverity[severity], '', message.substr(0, 13), '\t:', data); diff --git a/src/main/server/shims/levr.js b/src/main/server/shims/levr.js index 981a2aea2..219cd4b79 100644 --- a/src/main/server/shims/levr.js +++ b/src/main/server/shims/levr.js @@ -6,19 +6,19 @@ const dns = require('dns').promises; // LEVR shims if (global.fileLoad === undefined) { - global.fileLoad = function(filepath) { + global.fileLoad = function (filepath) { return fs.readFileSync(filepath); }; }; if (global.fileExists === undefined) { - global.fileExists = function(filepath) { + global.fileExists = function (filepath) { return fs.existsSync(filepath); }; }; if (global.fileToString === undefined) { - global.fileToString = function(file) { + global.fileToString = function (file) { if (file === undefined || file == null) return null; return file + ''; }; @@ -34,19 +34,19 @@ try { } if (global.fileSave === undefined) { - global.fileSave = function(text, filepath) { + global.fileSave = function (text, filepath) { fs.writeFileSync(filepath, text); }; }; if (global.bindWebService === undefined) { - global.bindWebService = function(endpoint, callback) { - let get = async function(req, res) { + global.bindWebService = function (endpoint, callback) { + let get = async function (req, res) { let ctx = { - get: function(field) { + get: function (field) { return ctx[field]; }, - put: function(field, value) { + put: function (field, value) { ctx[field] = value; }, }; @@ -61,12 +61,12 @@ if (global.bindWebService === undefined) { ctx: ctx, params: req.query, }); - if (typeof(result) == 'string') { + if (typeof (result) == 'string') { try { JSON.parse(result); res.setHeader('Content-Type', 'application/json; charset=utf-8'); } catch (e) { - // not JSON + // not JSON } res.end(result); } else { @@ -79,16 +79,16 @@ if (global.bindWebService === undefined) { } else { res.status(500); } - res.end(ex && ex.data ? ex.data : ex +''); + res.end(ex && ex.data ? ex.data : ex + ''); } global.auditLogger.report(global.auditLogger.LogCategory.NETWORK, global.auditLogger.Severity.INFO, 'CassHttpGetSuccess', `${endpoint} ${req.isSpdy ? 'spdy' : req.httpVersion} Response: ${res.statusCode} (${(new Date().getTime() - ms)} ms) ${JSON.stringify(req.query)}`, process.env.LOG_HEADERS ? req.headers : undefined); }; - let put = async function(req, res) { + let put = async function (req, res) { let ctx = { - get: function(field) { + get: function (field) { return ctx[field]; }, - put: function(field, value) { + put: function (field, value) { ctx[field] = value; }, }; @@ -101,10 +101,10 @@ if (global.bindWebService === undefined) { global.auditLogger.report(global.auditLogger.LogCategory.NETWORK, global.auditLogger.Severity.INFO, 'CassHttpPutStart', `${endpoint} ${req.isSpdy ? 'spdy' : req.httpVersion} Request: ${JSON.stringify(req.query)}`, process.env.LOG_HEADERS ? req.headers : undefined); req.setEncoding('utf8'); req.body = ''; - req.on('data', function(chunk) { + req.on('data', function (chunk) { req.body += chunk; }); - req.on('end', async function() { + req.on('end', async function () { let dataStreams = {}; if (req.body != '') { dataStreams.body = JSON.parse(req.body); @@ -114,12 +114,12 @@ if (global.bindWebService === undefined) { params: req.query, dataStreams: dataStreams, }); - if (typeof(result) == 'string') { + if (typeof (result) == 'string') { try { JSON.parse(result); res.setHeader('Content-Type', 'application/json; charset=utf-8'); } catch (e) { - // not JSON + // not JSON } res.end(result); } else { @@ -133,16 +133,16 @@ if (global.bindWebService === undefined) { } else { res.status(500); } - res.end(ex.data ? ex.data : ex +''); + res.end(ex.data ? ex.data : ex + ''); } global.auditLogger.report(global.auditLogger.LogCategory.NETWORK, global.auditLogger.Severity.INFO, 'CassHttpPutSuccess', `${endpoint} ${req.isSpdy ? 'spdy' : req.httpVersion} Response: ${res.statusCode} (${(new Date().getTime() - ms)} ms) ${JSON.stringify(req.query)}`, process.env.LOG_HEADERS ? req.headers : undefined); }; - let deleet = async function(req, res) { + let deleet = async function (req, res) { let ctx = { - get: function(field) { + get: function (field) { return ctx[field]; }, - put: function(field, value) { + put: function (field, value) { ctx[field] = value; }, }; @@ -157,12 +157,12 @@ if (global.bindWebService === undefined) { ctx: ctx, params: req.query, }); - if (typeof(result) == 'string') { + if (typeof (result) == 'string') { try { JSON.parse(result); res.setHeader('Content-Type', 'application/json; charset=utf-8'); } catch (e) { - // not JSON + // not JSON } res.end(result); } else { @@ -175,16 +175,16 @@ if (global.bindWebService === undefined) { } else { res.status(500); } - res.end(ex.data ? ex.data : ex +''); + res.end(ex.data ? ex.data : ex + ''); } global.auditLogger.report(global.auditLogger.LogCategory.NETWORK, global.auditLogger.Severity.INFO, 'CassHttpDeleteSuccess', `${endpoint} ${req.isSpdy ? 'spdy' : req.httpVersion} Response: ${res.statusCode} (${(new Date().getTime() - ms)} ms) ${JSON.stringify(req.query)}`, process.env.LOG_HEADERS ? req.headers : undefined); }; - let post = async function(req, res) { + let post = async function (req, res) { let ctx = { - get: function(field) { + get: function (field) { return ctx[field]; }, - put: function(field, value) { + put: function (field, value) { ctx[field] = value; }, req: req, @@ -200,20 +200,20 @@ if (global.bindWebService === undefined) { return await put(req, res); } } - const bb = busboy({headers: req.headers, limits: {parts: 100, fieldSize: global.postMaxSize, fileSize: global.postMaxSize}}); + const bb = busboy({ headers: req.headers, limits: { parts: 100, fieldSize: global.postMaxSize, fileSize: global.postMaxSize } }); req.query.methodType = 'POST'; req.query.urlRemainder = req.params[0]; let fields = {}; bb.on('file', (name, file, info) => { - const {filename, encoding, mimeType} = info; + const { filename, encoding, mimeType } = info; fields[name || filename] = getStream(file); }); bb.on('field', (name, val, info) => { fields[name] = val; }); - bb.on('close', async ()=>{ + bb.on('close', async () => { try { - global.auditLogger.report(global.auditLogger.LogCategory.NETWORK, global.auditLogger.Severity.INFO, 'CassHttpPostStart', endpoint+ ' ' + (req.isSpdy ? 'spdy' : req.httpVersion) + ' Request: ' + JSON.stringify(req.query) + ' - Parts: ' + JSON.stringify(EcObject.keys(fields)), process.env.LOG_HEADERS ? req.headers : undefined); + global.auditLogger.report(global.auditLogger.LogCategory.NETWORK, global.auditLogger.Severity.INFO, 'CassHttpPostStart', endpoint + ' ' + (req.isSpdy ? 'spdy' : req.httpVersion) + ' Request: ' + JSON.stringify(req.query) + ' - Parts: ' + JSON.stringify(EcObject.keys(fields)), process.env.LOG_HEADERS ? req.headers : undefined); for (let key in fields) { fields[key] = await fields[key]; } @@ -222,18 +222,18 @@ if (global.bindWebService === undefined) { params: req.query, dataStreams: fields, }); - if (typeof(result) == 'string') { + if (typeof (result) == 'string') { try { JSON.parse(result); res.setHeader('Content-Type', 'application/json; charset=utf-8'); } catch (e) { - // not JSON + // not JSON } res.end(result); } else { res.end(); } - global.auditLogger.report(global.auditLogger.LogCategory.NETWORK, global.auditLogger.Severity.INFO, 'CassHttpPostSuccess', endpoint+` Response: ${res.statusCode} (` + (new Date().getTime() - ms) + ' ms) ' + JSON.stringify(req.query), process.env.LOG_HEADERS ? req.headers : undefined); + global.auditLogger.report(global.auditLogger.LogCategory.NETWORK, global.auditLogger.Severity.INFO, 'CassHttpPostSuccess', endpoint + ` Response: ${res.statusCode} (` + (new Date().getTime() - ms) + ' ms) ' + JSON.stringify(req.query), process.env.LOG_HEADERS ? req.headers : undefined); } catch (ex) { global.auditLogger.report(global.auditLogger.LogCategory.NETWORK, global.auditLogger.Severity.ERROR, 'CassHttpPostError', `${endpoint} ${(req.isSpdy ? 'spdy' : req.httpVersion)} Request: ${JSON.stringify(req.query)} - Parts: ${JSON.stringify(EcObject.keys(fields))}`, process.env.LOG_HEADERS ? req.headers : undefined); if (ex.status !== undefined && ex.status != null) { @@ -241,7 +241,7 @@ if (global.bindWebService === undefined) { } else { res.status(500); } - res.end(ex.data ? ex.data : ex +''); + res.end(ex.data ? ex.data : ex + ''); } }); req.pipe(bb); @@ -252,7 +252,7 @@ if (global.bindWebService === undefined) { } else { res.status(500); } - res.end(ex.data ? ex.data : ex +''); + res.end(ex.data ? ex.data : ex + ''); } }; global.auditLogger.report(global.auditLogger.LogCategory.SYSTEM, global.auditLogger.Severity.INFO, 'CassBindEndpoint', `Binding endpoint: /api${endpoint}`); @@ -264,9 +264,9 @@ if (global.bindWebService === undefined) { }; if (global.fileFromDatastream === undefined) { - global.fileFromDatastream = function(dataStream) { + global.fileFromDatastream = function (dataStream) { if (this.dataStreams === undefined || this.dataStreams == null) return null; - if ((dataStream === undefined||dataStream == null) && EcObject.keys(this.dataStreams).length == 1) { + if ((dataStream === undefined || dataStream == null) && EcObject.keys(this.dataStreams).length == 1) { return this.dataStreams[EcObject.keys(this.dataStreams)[0]]; } if (this.dataStreams[dataStream] === undefined || this.dataStreams[dataStream] == null) { @@ -277,13 +277,13 @@ if (global.fileFromDatastream === undefined) { } if (global.headers === undefined) { - global.headers = function() { + global.headers = function () { return this.ctx.req.headers; }; }; if (global.httpGet === undefined) { - global.httpGet = async function(url, flag, headers) { + global.httpGet = async function (url, flag, headers) { let failCounter = 0; while (failCounter++ < 1000) { let ip = ''; @@ -293,17 +293,17 @@ if (global.httpGet === undefined) { global.auditLogger.report(global.auditLogger.LogCategory.NETWORK, global.auditLogger.Severity.ERROR, "DNSLookup", url, error); } try { - const response = await fetch(url, {headers: headers}); + const response = await fetch(url, { headers: headers }); let result = null; const contentType = response.headers.get("content-type"); if (contentType && contentType.indexOf("application/json") !== -1) { result = await response.json(); } else { result = await response.text(); - try{ + try { result = JSON.parse(result); } - catch(ex) { + catch (ex) { // Text is not json } } @@ -313,7 +313,7 @@ if (global.httpGet === undefined) { response: { status: response.status, statusText: response.statusText } }; } - if (skyrepoDebug) { + if (global.skyrepoDebug) { global.auditLogger.report(global.auditLogger.LogCategory.NETWORK, global.auditLogger.Severity.NETWORK, 'CassHttpGetSuccess', 'get success: ' + JSON.stringify(result)); } global.auditLogger.report(global.auditLogger.LogCategory.NETWORK, global.auditLogger.Severity.NETWORK, 'CassHttpGetSuccess', ip, url); @@ -326,10 +326,10 @@ if (global.httpGet === undefined) { resp = error.data; } } - if (skyrepoDebug) { + if (global.skyrepoDebug) { global.auditLogger.report(global.auditLogger.LogCategory.NETWORK, global.auditLogger.Severity.NETWORK, 'CassHttpGetError', 'get error: ' + error, url); } - if (skyrepoDebug) { + if (global.skyrepoDebug) { if (resp != null) { global.auditLogger.report(global.auditLogger.LogCategory.NETWORK, global.auditLogger.Severity.NETWORK, 'CassHttpGetError', resp); } else @@ -355,7 +355,7 @@ if (global.debug === undefined) { } if (global.httpDelete === undefined) { - global.httpDelete = async function(url, headers) { + global.httpDelete = async function (url, headers) { let ip = ''; try { ip = (await dns.lookup(new URL(url).hostname)).address; @@ -363,17 +363,17 @@ if (global.httpDelete === undefined) { global.auditLogger.report(global.auditLogger.LogCategory.NETWORK, global.auditLogger.Severity.ERROR, "DNSLookup", url, error); } try { - const response = await fetch(url, {headers: headers, method: 'DELETE'}); + const response = await fetch(url, { headers: headers, method: 'DELETE' }); let result = null; const contentType = response.headers.get("content-type"); if (contentType && contentType.indexOf("application/json") !== -1) { result = await response.json(); } else { result = await response.text(); - try{ + try { result = JSON.parse(result); } - catch(ex) { + catch (ex) { // Text is not json } } @@ -383,7 +383,7 @@ if (global.httpDelete === undefined) { response: { status: response.status, statusText: response.statusText } }; } - if (skyrepoDebug) { + if (global.skyrepoDebug) { global.auditLogger.report(global.auditLogger.LogCategory.NETWORK, global.auditLogger.Severity.NETWORK, 'CassHttpDeleteSuccess', 'delete success: ' + JSON.stringify(response.data)); } global.auditLogger.report(global.auditLogger.LogCategory.NETWORK, global.auditLogger.Severity.NETWORK, 'CassHttpDeleteSuccess', ip, url); @@ -396,10 +396,10 @@ if (global.httpDelete === undefined) { resp = error.data; } } - if (skyrepoDebug) { + if (global.skyrepoDebug) { global.auditLogger.report(global.auditLogger.LogCategory.NETWORK, global.auditLogger.Severity.NETWORK, 'CassHttpDeleteError', 'delete error', url); } - if (skyrepoDebug) { + if (global.skyrepoDebug) { if (resp != null) { global.auditLogger.report(global.auditLogger.LogCategory.NETWORK, global.auditLogger.Severity.NETWORK, 'CassHttpDeleteError', resp); } @@ -410,7 +410,7 @@ if (global.httpDelete === undefined) { }; if (global.httpPut === undefined) { - global.httpPut = async function(data, url, contentType, headers) { + global.httpPut = async function (data, url, contentType, headers) { let ip = ''; try { ip = (await dns.lookup(new URL(url).hostname)).address; @@ -432,10 +432,10 @@ if (global.httpPut === undefined) { result = await response.json(); } else { result = await response.text(); - try{ + try { result = JSON.parse(result); } - catch(ex) { + catch (ex) { // Text is not json } } @@ -445,7 +445,7 @@ if (global.httpPut === undefined) { response: { status: response.status, statusText: response.statusText } }; } - if (skyrepoDebug) { + if (global.skyrepoDebug) { global.auditLogger.report(global.auditLogger.LogCategory.NETWORK, global.auditLogger.Severity.NETWORK, 'CassHttpPutSuccess', 'put success: ' + JSON.stringify(response.data)); } global.auditLogger.report(global.auditLogger.LogCategory.NETWORK, global.auditLogger.Severity.NETWORK, 'CassHttpPutSuccess', ip, url); @@ -458,10 +458,10 @@ if (global.httpPut === undefined) { resp = error.data; } } - if (skyrepoDebug) { + if (global.skyrepoDebug) { global.auditLogger.report(global.auditLogger.LogCategory.NETWORK, global.auditLogger.Severity.NETWORK, 'CassHttpPutError', 'put error', url); } - if (skyrepoDebug) { + if (global.skyrepoDebug) { if (resp != null) { global.auditLogger.report(global.auditLogger.LogCategory.NETWORK, global.auditLogger.Severity.NETWORK, 'CassHttpPutError', resp); } @@ -472,7 +472,7 @@ if (global.httpPut === undefined) { }; if (global.httpPost === undefined) { - global.httpPost = async function(data, url, contentType, multipart, something, something2, simple, headers) { + global.httpPost = async function (data, url, contentType, multipart, something, something2, simple, headers) { let failCounter = 0; while (failCounter++ < 1000) { let ip = ''; @@ -490,17 +490,17 @@ if (global.httpPost === undefined) { }, body: contentType === 'application/json' ? JSON.stringify(data) : data }); - + let result = null; const _contentType = response.headers.get("content-type"); if (_contentType && _contentType.indexOf("application/json") !== -1) { result = await response.json(); } else { result = await response.text(); - try{ + try { result = JSON.parse(result); } - catch(ex) { + catch (ex) { // Text is not json } } @@ -510,7 +510,7 @@ if (global.httpPost === undefined) { response: { status: response.status, statusText: response.statusText } }; } - if (skyrepoDebug) { + if (global.skyrepoDebug) { global.auditLogger.report(global.auditLogger.LogCategory.NETWORK, global.auditLogger.Severity.NETWORK, 'CassHttpPostSuccess', 'post success: ' + JSON.stringify(response.data)); } global.auditLogger.report(global.auditLogger.LogCategory.NETWORK, global.auditLogger.Severity.NETWORK, 'CassHttpPostSuccess', ip, url); @@ -523,10 +523,10 @@ if (global.httpPost === undefined) { resp = error.data; } } - if (skyrepoDebug) { - global.auditLogger.report(global.auditLogger.LogCategory.NETWORK, global.auditLogger.Severity.NETWORK, 'CassHttpPostError', 'post error: ' +error.response.status + ': '+ error.response.statusText, url); + if (global.skyrepoDebug) { + global.auditLogger.report(global.auditLogger.LogCategory.NETWORK, global.auditLogger.Severity.NETWORK, 'CassHttpPostError', 'post error: ' + error.response.status + ': ' + error.response.statusText, url); } - if (skyrepoDebug) { + if (global.skyrepoDebug) { if (resp != null) { global.auditLogger.report(global.auditLogger.LogCategory.NETWORK, global.auditLogger.Severity.NETWORK, 'CassHttpPostError', resp); } else { @@ -547,7 +547,7 @@ if (global.httpPost === undefined) { }; if (global.error === undefined) { - global.error = function(errormessage, status) { + global.error = function (errormessage, status) { let e = {}; e.data = errormessage; e.status = status; @@ -558,7 +558,7 @@ if (global.error === undefined) { }; if (global.warn === undefined) { - global.warn = function(errormessage, status) { + global.warn = function (errormessage, status) { let e = {}; e.data = errormessage; e.status = status; @@ -568,20 +568,20 @@ if (global.warn === undefined) { }; if (global.randomString === undefined) { - global.randomString = function(len) { + global.randomString = function (len) { return forge.random.getBytesSync(len); }; }; if (global.rsaGenerate === undefined) { - global.rsaGenerate = function(len) { + global.rsaGenerate = function (len) { return EcPpk.generateKey().toPem(); }; }; if (global.jsonLdExpand === undefined) { - global.jsonLdExpand = function(json) { - return new Promise(async function(resolve, reject) { + global.jsonLdExpand = function (json) { + return new Promise(async function (resolve, reject) { try { let actual = await jsonld.expand(JSON.parse(json)); resolve(actual); @@ -594,11 +594,11 @@ if (global.jsonLdExpand === undefined) { }; if (global.jsonLdCompact === undefined) { - global.jsonLdCompact = function(actual, finalTargetContext) { - return new Promise(async function(resolve, reject) { + global.jsonLdCompact = function (actual, finalTargetContext) { + return new Promise(async function (resolve, reject) { try { finalTargetContext = JSON.parse(finalTargetContext); - } catch (ex) {} + } catch (ex) { } try { let o = await jsonld.compact(JSON.parse(actual), finalTargetContext); (o)['@context'] = finalTargetContext; @@ -630,9 +630,9 @@ if (global.jsonLdCompact === undefined) { const $rdf = require('rdflib'); if (global.jsonLdToRdfXml === undefined) { - global.jsonLdToRdfXml = function(o) { - return new Promise(async function(resolve, reject) { - let rdf = await jsonld.toRDF(o, {format: 'application/n-quads'}); + global.jsonLdToRdfXml = function (o) { + return new Promise(async function (resolve, reject) { + let rdf = await jsonld.toRDF(o, { format: 'application/n-quads' }); let store = $rdf.graph(); $rdf.parse(rdf, store, 'whatever', 'application/n-quads', (err, str) => { resolve($rdf.serialize(null, str, '*', 'application/rdf+xml')); @@ -642,14 +642,14 @@ if (global.jsonLdToRdfXml === undefined) { }; if (global.jsonLdToTurtle === undefined) { - global.jsonLdToTurtle = async function(o) { + global.jsonLdToTurtle = async function (o) { return (await toTurtleInternal(o)); }; }; -toTurtleInternal = function(o) { - return new Promise(async function(resolve, reject) { - let rdf = await jsonld.toRDF(o, {format: 'application/n-quads'}); +toTurtleInternal = function (o) { + return new Promise(async function (resolve, reject) { + let rdf = await jsonld.toRDF(o, { format: 'application/n-quads' }); let store = $rdf.graph(); $rdf.parse(rdf, store, 'whatever', 'application/n-quads', (err, str) => { let result = ($rdf.serialize(null, str, '*', 'text/turtle')); @@ -659,12 +659,12 @@ toTurtleInternal = function(o) { }; if (global.jsonLdToNQuads === undefined) { - global.jsonLdToNQuads = async function(o) { - return (await jsonld.toRDF(o, {format: 'application/n-quads'})); + global.jsonLdToNQuads = async function (o) { + return (await jsonld.toRDF(o, { format: 'application/n-quads' })); }; }; // Shim to allow require of modules that are intended to be used to import. -require('module').Module._extensions['.js'] = function(module, filename) { +require('module').Module._extensions['.js'] = function (module, filename) { module._compile(require('fs').readFileSync(filename, 'utf8'), filename); }; diff --git a/src/main/server/skyRepo.js b/src/main/server/skyRepo.js index d902d9afc..e28a0531e 100644 --- a/src/main/server/skyRepo.js +++ b/src/main/server/skyRepo.js @@ -315,7 +315,7 @@ const putUrl = function (o, id, version, type) { url += '/' + typeFromObj; } url += '/' + encodeURIComponent(id) + versionPart; - if (skyrepoDebug) { + if (global.skyrepoDebug) { global.auditLogger.report(global.auditLogger.LogCategory.SYSTEM, global.auditLogger.Severity.DEBUG, 'SkyrepoPutUrl', 'Put:' + url); } return url; @@ -339,7 +339,7 @@ const putPermanentUrl = function (o, id, version, type) { url += '/permanent'; } url += '/' + encodeURIComponent(id) + '.' + (version === undefined || version == null ? '' : version) + versionPart; - if (skyrepoDebug) { + if (global.skyrepoDebug) { global.auditLogger.report(global.auditLogger.LogCategory.SYSTEM, global.auditLogger.Severity.DEBUG, 'SkyrepoPutPermanentUrl', 'PutPermanent:' + url); } return url; @@ -363,7 +363,7 @@ const putPermanentBaseUrl = function (o, id, version, type) { url += '/permanent'; } url += '/' + encodeURIComponent(id) + '.' + versionPart; - if (skyrepoDebug) { + if (global.skyrepoDebug) { global.auditLogger.report(global.auditLogger.LogCategory.SYSTEM, global.auditLogger.Severity.DEBUG, 'SkyrepoPutPermanentBaseUrl', 'PutPermanentBase:' + url); } return url; @@ -384,7 +384,7 @@ const getUrl = function (index, id, version, type) { } else { url += '/' + encodeURIComponent(id); } - if (skyrepoDebug) { + if (global.skyrepoDebug) { global.auditLogger.report(global.auditLogger.LogCategory.SYSTEM, global.auditLogger.Severity.DEBUG, 'SkyrepoGetUrl', 'Get:' + url); } return url; @@ -404,7 +404,7 @@ const deleteUrl = function (id, version, type) { } url += '/' + encodeURIComponent(id); url += '?' + refreshPart; - if (skyrepoDebug) { + if (global.skyrepoDebug) { global.auditLogger.report(global.auditLogger.LogCategory.SYSTEM, global.auditLogger.Severity.DEBUG, 'SkyrepoDeleteUrl', 'Delete:' + url); } return url; @@ -418,7 +418,7 @@ const deletePermanentBaseUrl = function (id, version, type) { url += '/permanent'; } url += '/' + encodeURIComponent(id) + '.'; - if (skyrepoDebug) { + if (global.skyrepoDebug) { global.auditLogger.report(global.auditLogger.LogCategory.SYSTEM, global.auditLogger.Severity.DEBUG, 'SkyrepoDeletePermBase', 'DeletePermanentBase:' + url); } return url; @@ -8477,7 +8477,7 @@ const skyrepoPutInternalIndex = async function (o, id, version, type) { delete (o)['payload']; delete (o)['secret']; } - if (skyrepoDebug) { + if (global.skyrepoDebug) { global.auditLogger.report(global.auditLogger.LogCategory.NETWORK, global.auditLogger.Severity.DEBUG, 'SkyrepoPutInternalIndex', JSON.stringify(o)); } const response = await httpPost(o, url, 'application/json', false, null, null, true, elasticHeaders()); @@ -8498,7 +8498,7 @@ const skyrepoPutInternalPermanent = async function (o, id, version, type) { } (doc)['enabled'] = false; const result = await httpPut(mappings, elasticEndpoint + '/permanent', 'application/json', elasticHeaders()); - if (skyrepoDebug) { + if (global.skyrepoDebug) { global.auditLogger.report(global.auditLogger.LogCategory.NETWORK, global.auditLogger.Severity.DEBUG, 'SkyrepoPutInternalPerm', JSON.stringify(result)); } permanentCreated = true; @@ -8515,7 +8515,7 @@ const skyrepoPutInternalPermanent = async function (o, id, version, type) { url = putPermanentUrl.call(this, o, id, version, type); results = await httpPost(data, url, 'application/json', false, null, null, true, elasticHeaders()); } - if (skyrepoDebug) { + if (global.skyrepoDebug) { global.auditLogger.report(global.auditLogger.LogCategory.NETWORK, global.auditLogger.Severity.DEBUG, 'SkyrepoPutInternalPerm', JSON.stringify(results)); } return JSON.stringify(results); @@ -8788,7 +8788,7 @@ let skyrepoPutInternal = global.skyrepoPutInternal = async function (o, id, vers } } } - if (skyrepoDebug) { + if (global.skyrepoDebug) { global.auditLogger.report(global.auditLogger.LogCategory.NETWORK, global.auditLogger.Severity.DEBUG, 'SkyrepoPutInternal', JSON.stringify(obj)); } const permanentIds = [id]; @@ -8827,7 +8827,7 @@ let skyrepoPutInternal = global.skyrepoPutInternal = async function (o, id, vers } }; const skyrepoGetIndexInternal = async function (index, id, version, type) { - if (skyrepoDebug) { + if (global.skyrepoDebug) { global.auditLogger.report(global.auditLogger.LogCategory.NETWORK, global.auditLogger.Severity.DEBUG, 'SkyrepGetIndexInternal', 'Fetching from ' + index + ' : ' + type + ' / ' + id + ' / ' + version); } const response = await httpGet(getUrl.call(this, index, id, version, type), true, elasticHeaders()); @@ -8835,7 +8835,7 @@ const skyrepoGetIndexInternal = async function (index, id, version, type) { }; const skyrepoManyGetIndexInternal = async function (index, manyParseParams) { - if (skyrepoDebug) { + if (global.skyrepoDebug) { global.auditLogger.report(global.auditLogger.LogCategory.NETWORK, global.auditLogger.Severity.DEBUG, 'SkyrepManyGetIndexInternal', 'Fetching from ' + index + ' : ' + manyParseParams); } @@ -8867,7 +8867,7 @@ const skyrepoManyGetIndexInternal = async function (index, manyParseParams) { const skyrepoGetIndexSearch = async function (id, version, type) { const microSearchUrl = elasticEndpoint + '/_search?version&q=_id:' + id + ''; const microSearch = await httpGet(microSearchUrl, true, elasticHeaders()); - if (skyrepoDebug) { + if (global.skyrepoDebug) { global.auditLogger.report(global.auditLogger.LogCategory.NETWORK, global.auditLogger.Severity.DEBUG, 'SkyrepGetIndexSearch', microSearchUrl); } if (microSearch == null) { @@ -8899,7 +8899,7 @@ const skyrepoManyGetIndexSearch = async function (ary) { const microSearch = await httpGet(microSearchUrl, true, elasticHeaders()); - if (skyrepoDebug) { + if (global.skyrepoDebug) { global.auditLogger.report(global.auditLogger.LogCategory.NETWORK, global.auditLogger.Severity.DEBUG, 'SkyrepManyGetIndexSearch', microSearchUrl); } if (microSearch == null) { @@ -8920,7 +8920,7 @@ let skyrepoGetIndexRecords = async function (id) { const hashId = EcCrypto.md5(id); const microSearchUrl = elasticEndpoint + '/_search?version&q=@id:"' + id + '" OR @id:"' + hashId + '"'; const microSearch = await httpGet(microSearchUrl, true, elasticHeaders()); - if (skyrepoDebug) { + if (global.skyrepoDebug) { global.auditLogger.report(global.auditLogger.LogCategory.NETWORK, global.auditLogger.Severity.DEBUG, 'SkyrepGetIndexRecords', microSearchUrl); } if (microSearch == null) { @@ -8954,12 +8954,12 @@ let skyrepoManyGetIndexRecords = async function (ary) { } const searchParameters = await (searchObj).call(this, microSearchUrl, null, 10000); - if (skyrepoDebug) { + if (global.skyrepoDebug) { global.auditLogger.report(global.auditLogger.LogCategory.NETWORK, global.auditLogger.Severity.DEBUG, 'SkyrepSearch', JSON.stringify(searchParameters)); } const microSearch = await httpPost(searchParameters, searchUrl(), 'application/json', false, null, null, true, elasticHeaders()); - if (skyrepoDebug) { + if (global.skyrepoDebug) { global.auditLogger.report(global.auditLogger.LogCategory.NETWORK, global.auditLogger.Severity.DEBUG, 'SkyrepManyGetIndexRecords', microSearchUrl); } if (microSearch == null) { @@ -9022,7 +9022,7 @@ global.skyrepoGetInternal = async function (id, version, type) { if ((result)['found'] == true) { return JSON.parse(((result)['_source'])['data']); } - if (skyrepoDebug) { + if (global.skyrepoDebug) { global.auditLogger.report(global.auditLogger.LogCategory.NETWORK, global.auditLogger.Severity.DEBUG, 'SkyrepGetInternal', 'Failed to find ' + type + '/' + id + '/' + version + ' -- trying degraded form from search index.'); } result = await (skyrepoGetIndex).call(this, id, version, type); @@ -9078,7 +9078,7 @@ global.skyrepoHistoryInternal = async function (id, version, type) { } return hits.map((h) => JSON.parse(h._source.data)); } - if (skyrepoDebug) { + if (global.skyrepoDebug) { global.auditLogger.report(global.auditLogger.LogCategory.NETWORK, global.auditLogger.Severity.DEBUG, 'SkyrepoHistoryInternal', 'Failed to find ' + type + '/' + id + '/' + version + '.'); } return null; @@ -9104,7 +9104,7 @@ global.skyrepoManyGetInternal = async function (manyParseParams) { } } - if (skyrepoDebug && notFoundInPermanent.length > 0) { + if (global.skyrepoDebug && notFoundInPermanent.length > 0) { global.auditLogger.report(global.auditLogger.LogCategory.NETWORK, global.auditLogger.Severity.DEBUG, 'SkyrepManyGetInternal', 'Failed to find ' + manyParseParams + ' -- trying degraded form from search index.'); } @@ -9132,10 +9132,10 @@ global.skyrepoGet = async function (parseParams) { (parseParams)['version'] = this.params.version; (parseParams)['history'] = this.params.history; } - if (skyrepoDebug) { + if (global.skyrepoDebug) { global.auditLogger.report(global.auditLogger.LogCategory.NETWORK, global.auditLogger.Severity.DEBUG, 'SkyrepGet', JSON.stringify(parseParams)); } - if (skyrepoDebug) { + if (global.skyrepoDebug) { global.auditLogger.report(global.auditLogger.LogCategory.NETWORK, global.auditLogger.Severity.DEBUG, 'SkyrepGet', JSON.stringify(this.params.obj)); } const id = (parseParams)['id']; @@ -9198,10 +9198,10 @@ global.skyrepoPut = async function (parseParams) { (parseParams)['version'] = this.params.version; (parseParams)['obj'] = this.params.obj; } - if (skyrepoDebug) { + if (global.skyrepoDebug) { global.auditLogger.report(global.auditLogger.LogCategory.NETWORK, global.auditLogger.Severity.DEBUG, 'SkyrepPut', 'put pp:' + JSON.stringify(parseParams)); } - if (skyrepoDebug) { + if (global.skyrepoDebug) { global.auditLogger.report(global.auditLogger.LogCategory.NETWORK, global.auditLogger.Severity.DEBUG, 'SkyrepPut', 'put obj:' + JSON.stringify(this.params.obj)); } if (parseParams == null && EcObject.isObject(this.params.obj)) { @@ -9409,20 +9409,20 @@ const searchUrl = function (urlRemainder, index_hint) { url += '/'; } url += '_search'; - if (skyrepoDebug) { + if (global.skyrepoDebug) { global.auditLogger.report(global.auditLogger.LogCategory.SYSTEM, global.auditLogger.Severity.DEBUG, 'SkyrepSearchUrl', url); } return url; }; const skyrepoSearch = async function (q, urlRemainder, start, size, sort, track_scores, index_hint) { const searchParameters = await (searchObj).call(this, q, start, size, sort, track_scores); - if (skyrepoDebug) { + if (global.skyrepoDebug) { global.auditLogger.report(global.auditLogger.LogCategory.NETWORK, global.auditLogger.Severity.DEBUG, 'SkyrepSearch', JSON.stringify(searchParameters)); } const results = await httpPost(searchParameters, searchUrl(urlRemainder, index_hint), 'application/json', false, null, null, true, elasticHeaders()); // console.log(results); - if (skyrepoDebug) { + if (global.skyrepoDebug) { global.auditLogger.report(global.auditLogger.LogCategory.NETWORK, global.auditLogger.Severity.DEBUG, 'SkyrepSearch', JSON.stringify(results)); } if (results == null) { @@ -9850,7 +9850,7 @@ const endpointMultiPut = async function () { } (doc)['enabled'] = false; const result = await httpPut(mappings, elasticEndpoint + '/permanent', 'application/json', elasticHeaders()); - if (skyrepoDebug) { + if (global.skyrepoDebug) { global.auditLogger.report(global.auditLogger.LogCategory.NETWORK, global.auditLogger.Severity.DEBUG, 'SkyrepEndpointMultiput', JSON.stringify(result)); } permanentCreated = true; diff --git a/src/main/server/util.js b/src/main/server/util.js index 48a11ad97..207c3dc3d 100644 --- a/src/main/server/util.js +++ b/src/main/server/util.js @@ -22,15 +22,15 @@ require('./shims/levr.js'); require('./shims/stjs.js'); -global.skyrepoMigrate = async function(after) { +global.skyrepoMigrate = async function (after) { let elasticState = await httpGet(elasticEndpoint + '/', true, global.elasticHeaders()); - const result = await httpPut({'index.mapping.total_fields.limit': 10000}, elasticEndpoint + '/schema.cassproject.org.0.4.configuration/_settings', 'application/json', global.elasticHeaders()); - if (skyrepoDebug) { + const result = await httpPut({ 'index.mapping.total_fields.limit': 10000 }, elasticEndpoint + '/schema.cassproject.org.0.4.configuration/_settings', 'application/json', global.elasticHeaders()); + if (global.skyrepoDebug) { global.auditLogger.report(global.auditLogger.LogCategory.NETWORK, global.auditLogger.Severity.DEBUG, 'SkyrepoSetTotalFieldLimits', JSON.stringify(result)); } if (elasticState == null) { - global.auditLogger.report(global.auditLogger.LogCategory.SYSTEM, global.auditLogger.Severity.INFO, 'SkyrepMigrate', 'Waiting for Elasticsearch to appear at '+elasticEndpoint+'...'); - setTimeout(function() { + global.auditLogger.report(global.auditLogger.LogCategory.SYSTEM, global.auditLogger.Severity.INFO, 'SkyrepMigrate', 'Waiting for Elasticsearch to appear at ' + elasticEndpoint + '...'); + setTimeout(function () { skyrepoMigrate(after); }, 1000); return; @@ -41,7 +41,7 @@ global.skyrepoMigrate = async function(after) { health = (await httpGet(elasticEndpoint + '/_cluster/health', true, global.elasticHeaders())).status; if (health != 'yellow' && health != 'green') { global.auditLogger.report(global.auditLogger.LogCategory.SYSTEM, global.auditLogger.Severity.INFO, 'SkyrepMigrate', 'Waiting for cluster health...', health); - setTimeout(function() { + setTimeout(function () { skyrepoMigrate(after); }, 1000); return; @@ -72,13 +72,13 @@ global.skyrepoMigrate = async function(after) { // console.log(r2 = httpDelete(elasticEndpoint+"/"+index,true)); // continue; // } - global.auditLogger.report(global.auditLogger.LogCategory.SYSTEM, global.auditLogger.Severity.INFO, 'SkyrepMigrate', 'Reindexing ' + index + ' -> .temp.'+index.replace('https:..', '').replace(':', '.')); + global.auditLogger.report(global.auditLogger.LogCategory.SYSTEM, global.auditLogger.Severity.INFO, 'SkyrepMigrate', 'Reindexing ' + index + ' -> .temp.' + index.replace('https:..', '').replace(':', '.')); if (index == 'permanent') { let mappings = {}; let doc = {}; (mappings)['mappings'] = doc; (doc)['enabled'] = false; - let result = await httpPut(mappings, elasticEndpoint + '/.temp.'+index.replace('https:..', '').replace(':', '.'), 'application/json', global.elasticHeaders()); + let result = await httpPut(mappings, elasticEndpoint + '/.temp.' + index.replace('https:..', '').replace(':', '.'), 'application/json', global.elasticHeaders()); global.auditLogger.report(global.auditLogger.LogCategory.NETWORK, global.auditLogger.Severity.INFO, 'SkyrepMigrate', JSON.stringify(result)); } else if (index.endsWith('assertion')) { await httpGet(elasticEndpoint + '/' + index + '/_mapping', true, global.elasticHeaders()); @@ -86,11 +86,11 @@ global.skyrepoMigrate = async function(after) { let doc = {}; (mappings)['mappings'] = doc; (doc).properties = { - '@version': {type: 'long'}, - 'confidence': {type: 'float'}, - 'assertionDateDecrypted': {type: 'long'}, + '@version': { type: 'long' }, + 'confidence': { type: 'float' }, + 'assertionDateDecrypted': { type: 'long' }, }; - let result = await httpPut(mappings, elasticEndpoint + '/.temp.'+index.replace('https:..', '').replace(':', '.'), 'application/json', global.elasticHeaders()); + let result = await httpPut(mappings, elasticEndpoint + '/.temp.' + index.replace('https:..', '').replace(':', '.'), 'application/json', global.elasticHeaders()); global.auditLogger.report(global.auditLogger.LogCategory.NETWORK, global.auditLogger.Severity.INFO, 'SkyrepMigrate', JSON.stringify(result)); } else if (index.endsWith('competency')) { await httpGet(elasticEndpoint + '/' + index + '/_mapping', true, global.elasticHeaders()); @@ -98,7 +98,7 @@ global.skyrepoMigrate = async function(after) { let doc = {}; (mappings)['mappings'] = doc; (doc).properties = { - '@version': {type: 'long'}, + '@version': { type: 'long' }, 'ceasn:codedNotation': { 'type': 'text', 'fields': { @@ -109,7 +109,7 @@ global.skyrepoMigrate = async function(after) { }, }, }; - let result = await httpPut(mappings, elasticEndpoint + '/.temp.'+index.replace('https:..', '').replace(':', '.'), 'application/json', global.elasticHeaders()); + let result = await httpPut(mappings, elasticEndpoint + '/.temp.' + index.replace('https:..', '').replace(':', '.'), 'application/json', global.elasticHeaders()); global.auditLogger.report(global.auditLogger.LogCategory.NETWORK, global.auditLogger.Severity.INFO, 'SkyrepMigrate', JSON.stringify(result)); } else if (index.endsWith('conceptscheme')) { await httpGet(elasticEndpoint + '/' + index + '/_mapping', true, global.elasticHeaders()); @@ -117,7 +117,7 @@ global.skyrepoMigrate = async function(after) { let doc = {}; (mappings)['mappings'] = doc; (doc).properties = { - '@version': {type: 'long'}, + '@version': { type: 'long' }, 'skos:hasTopConcept': { 'type': 'text', 'fields': { @@ -128,7 +128,7 @@ global.skyrepoMigrate = async function(after) { }, }, }; - let result = await httpPut(mappings, elasticEndpoint + '/.temp.'+index.replace('https:..', '').replace(':', '.'), 'application/json', global.elasticHeaders()); + let result = await httpPut(mappings, elasticEndpoint + '/.temp.' + index.replace('https:..', '').replace(':', '.'), 'application/json', global.elasticHeaders()); global.auditLogger.report(global.auditLogger.LogCategory.NETWORK, global.auditLogger.Severity.INFO, 'SkyrepMigrate', JSON.stringify(result)); } else { await httpGet(elasticEndpoint + '/' + index + '/_mapping', true, global.elasticHeaders()); @@ -140,20 +140,20 @@ global.skyrepoMigrate = async function(after) { let mappings = {}; let doc = {}; (mappings)['mappings'] = doc; - (doc).properties = {'@version': {type: 'long'}}; - mappings['settings'] = {'index.mapping.total_fields.limit': fields}; - let result = await httpPut(mappings, elasticEndpoint + '/.temp.'+index.replace('https:..', '').replace(':', '.'), 'application/json', global.elasticHeaders()); + (doc).properties = { '@version': { type: 'long' } }; + mappings['settings'] = { 'index.mapping.total_fields.limit': fields }; + let result = await httpPut(mappings, elasticEndpoint + '/.temp.' + index.replace('https:..', '').replace(':', '.'), 'application/json', global.elasticHeaders()); global.auditLogger.report(global.auditLogger.LogCategory.NETWORK, global.auditLogger.Severity.INFO, 'SkyrepMigrate', JSON.stringify(result)); } let r = null; global.auditLogger.report(global.auditLogger.LogCategory.NETWORK, global.auditLogger.Severity.INFO, 'SkyrepMigrate', r = await httpPost(JSON.stringify({ - source: {index: index}, - dest: {index: '.temp.'+index.replace('https:..', '').replace(':', '.'), version_type: 'external'}, - }), elasticEndpoint+'/_reindex?refresh=true', 'application/json', 'false', global.elasticHeaders())); + source: { index: index }, + dest: { index: '.temp.' + index.replace('https:..', '').replace(':', '.'), version_type: 'external' }, + }), elasticEndpoint + '/_reindex?refresh=true', 'application/json', 'false', global.elasticHeaders())); if (r.error != null) continue; global.auditLogger.report(global.auditLogger.LogCategory.NETWORK, global.auditLogger.Severity.INFO, 'SkyrepMigrate', 'Deleting ' + index); let r2 = null; - global.auditLogger.report(global.auditLogger.LogCategory.NETWORK, global.auditLogger.Severity.INFO, 'SkyrepMigrate', r2 = await httpDelete(elasticEndpoint+'/'+index, true, global.elasticHeaders())); + global.auditLogger.report(global.auditLogger.LogCategory.NETWORK, global.auditLogger.Severity.INFO, 'SkyrepMigrate', r2 = await httpDelete(elasticEndpoint + '/' + index, true, global.elasticHeaders())); if (r2.error != null) continue; // if (r.total == 0) continue; if (index == 'permanent') { @@ -169,7 +169,7 @@ global.skyrepoMigrate = async function(after) { let doc = {}; (mappings)['mappings'] = doc; (doc).properties = { - '@version': {type: 'long'}, + '@version': { type: 'long' }, 'ceasn:codedNotation': { 'type': 'text', 'fields': { @@ -180,7 +180,7 @@ global.skyrepoMigrate = async function(after) { }, }, }; - let result = await httpPut(mappings, elasticEndpoint + '/'+index.replace('https:..', '').replace(':', '.'), 'application/json', global.elasticHeaders()); + let result = await httpPut(mappings, elasticEndpoint + '/' + index.replace('https:..', '').replace(':', '.'), 'application/json', global.elasticHeaders()); global.auditLogger.report(global.auditLogger.LogCategory.NETWORK, global.auditLogger.Severity.INFO, 'SkyrepMigrate', JSON.stringify(result)); } else if (index.endsWith('conceptscheme')) { await httpGet(elasticEndpoint + '/.temp.' + index.replace('https:..', '').replace(':', '.') + '/_mapping', true, global.elasticHeaders()); @@ -188,7 +188,7 @@ global.skyrepoMigrate = async function(after) { let doc = {}; (mappings)['mappings'] = doc; (doc).properties = { - '@version': {type: 'long'}, + '@version': { type: 'long' }, 'skos:hasTopConcept': { 'type': 'text', 'fields': { @@ -199,7 +199,7 @@ global.skyrepoMigrate = async function(after) { }, }, }; - let result = await httpPut(mappings, elasticEndpoint + '/'+index.replace('https:..', '').replace(':', '.'), 'application/json', global.elasticHeaders()); + let result = await httpPut(mappings, elasticEndpoint + '/' + index.replace('https:..', '').replace(':', '.'), 'application/json', global.elasticHeaders()); global.auditLogger.report(global.auditLogger.LogCategory.NETWORK, global.auditLogger.Severity.INFO, 'SkyrepMigrate', JSON.stringify(result)); } else if (index.endsWith('assertion')) { await httpGet(elasticEndpoint + '/.temp.' + index.replace('https:..', '').replace(':', '.') + '/_mapping', true, global.elasticHeaders()); @@ -207,11 +207,11 @@ global.skyrepoMigrate = async function(after) { let doc = {}; (mappings)['mappings'] = doc; (doc).properties = { - '@version': {type: 'long'}, - 'confidence': {type: 'float'}, - 'assertionDateDecrypted': {type: 'long'}, + '@version': { type: 'long' }, + 'confidence': { type: 'float' }, + 'assertionDateDecrypted': { type: 'long' }, }; - let result = await httpPut(mappings, elasticEndpoint + '/'+index.replace('https:..', '').replace(':', '.'), 'application/json', global.elasticHeaders()); + let result = await httpPut(mappings, elasticEndpoint + '/' + index.replace('https:..', '').replace(':', '.'), 'application/json', global.elasticHeaders()); global.auditLogger.report(global.auditLogger.LogCategory.NETWORK, global.auditLogger.Severity.INFO, 'SkyrepMigrate', JSON.stringify(result)); } else { await httpGet(elasticEndpoint + '/.temp.' + index.replace('https:..', '').replace(':', '.') + '/_mapping', true, global.elasticHeaders()); @@ -223,19 +223,19 @@ global.skyrepoMigrate = async function(after) { let mappings = {}; let doc = {}; (mappings)['mappings'] = doc; - (doc).properties = {'@version': {type: 'long'}}; - mappings['settings'] = {'index.mapping.total_fields.limit': fields}; - let result = await httpPut(mappings, elasticEndpoint + '/'+index.replace('https:..', '').replace(':', '.'), 'application/json', global.elasticHeaders()); + (doc).properties = { '@version': { type: 'long' } }; + mappings['settings'] = { 'index.mapping.total_fields.limit': fields }; + let result = await httpPut(mappings, elasticEndpoint + '/' + index.replace('https:..', '').replace(':', '.'), 'application/json', global.elasticHeaders()); global.auditLogger.report(global.auditLogger.LogCategory.NETWORK, global.auditLogger.Severity.INFO, 'SkyrepMigrate', JSON.stringify(result)); } - global.auditLogger.report(global.auditLogger.LogCategory.NETWORK, global.auditLogger.Severity.INFO, 'SkyrepMigrate', 'Reindexing .temp.' + index.replace('https:..', '').replace(':', '.') + ' -> '+index.replace('https:..', '').replace(':', '.')); + global.auditLogger.report(global.auditLogger.LogCategory.NETWORK, global.auditLogger.Severity.INFO, 'SkyrepMigrate', 'Reindexing .temp.' + index.replace('https:..', '').replace(':', '.') + ' -> ' + index.replace('https:..', '').replace(':', '.')); global.auditLogger.report(global.auditLogger.LogCategory.NETWORK, global.auditLogger.Severity.INFO, 'SkyrepMigrate', r = await httpPost(JSON.stringify({ - source: {index: '.temp.'+index.replace('https:..', '').replace(':', '.')}, - dest: {index: index.replace('https:..', '').replace(':', '.'), version_type: 'external'}, - }), elasticEndpoint+'/_reindex?refresh=true', 'application/json', 'false', global.elasticHeaders())); + source: { index: '.temp.' + index.replace('https:..', '').replace(':', '.') }, + dest: { index: index.replace('https:..', '').replace(':', '.'), version_type: 'external' }, + }), elasticEndpoint + '/_reindex?refresh=true', 'application/json', 'false', global.elasticHeaders())); if (r.error != null) continue; global.auditLogger.report(global.auditLogger.LogCategory.NETWORK, global.auditLogger.Severity.INFO, 'SkyrepMigrate', 'Deleting .temp.' + index.replace('https:..', '').replace(':', '.')); - global.auditLogger.report(global.auditLogger.LogCategory.NETWORK, global.auditLogger.Severity.INFO, 'SkyrepMigrate', await httpDelete(elasticEndpoint+'/.temp.'+index.replace('https:..', '').replace(':', '.'), true)); + global.auditLogger.report(global.auditLogger.LogCategory.NETWORK, global.auditLogger.Severity.INFO, 'SkyrepMigrate', await httpDelete(elasticEndpoint + '/.temp.' + index.replace('https:..', '').replace(':', '.'), true)); } } if (elasticState.version.number.startsWith('6.') && elasticState.version.minimum_index_compatibility_version == '5.0.0') { @@ -250,7 +250,7 @@ global.skyrepoMigrate = async function(after) { if (settings[index].settings.index.version.created != '5061299' && settings[index].settings.index.version.created != '5061699' && settings[index].settings.index.version.created != '5040099') { continue; } - global.auditLogger.report(global.auditLogger.LogCategory.NETWORK, global.auditLogger.Severity.INFO, 'SkyrepMigrate', 'Reindexing ' + index + ' -> .temp.'+index); + global.auditLogger.report(global.auditLogger.LogCategory.NETWORK, global.auditLogger.Severity.INFO, 'SkyrepMigrate', 'Reindexing ' + index + ' -> .temp.' + index); if (index == 'permanent') { let mappings = {}; let permNoIndex = {}; @@ -258,7 +258,7 @@ global.skyrepoMigrate = async function(after) { (mappings)['mappings'] = permNoIndex; (permNoIndex)['permanent'] = doc; (doc)['enabled'] = false; - let result = await httpPut(mappings, elasticEndpoint + '/.temp.'+index, 'application/json', global.elasticHeaders()); + let result = await httpPut(mappings, elasticEndpoint + '/.temp.' + index, 'application/json', global.elasticHeaders()); global.auditLogger.report(global.auditLogger.LogCategory.NETWORK, global.auditLogger.Severity.INFO, 'SkyrepMigrate', JSON.stringify(result)); } else if (index.endsWith('assertion')) { let mapping = await httpGet(elasticEndpoint + '/' + index + '/_mapping', true, global.elasticHeaders()); @@ -270,11 +270,11 @@ global.skyrepoMigrate = async function(after) { (permNoIndex)[EcObject.keys(mapping[index].mappings)[0]] = doc; } (doc).properties = { - '@version': {type: 'long'}, - 'confidence': {type: 'float'}, - 'assertionDateDecrypted': {type: 'long'}, + '@version': { type: 'long' }, + 'confidence': { type: 'float' }, + 'assertionDateDecrypted': { type: 'long' }, }; - let result = await httpPut(mappings, elasticEndpoint + '/.temp.'+index, 'application/json', global.elasticHeaders()); + let result = await httpPut(mappings, elasticEndpoint + '/.temp.' + index, 'application/json', global.elasticHeaders()); global.auditLogger.report(global.auditLogger.LogCategory.NETWORK, global.auditLogger.Severity.INFO, 'SkyrepMigrate', JSON.stringify(result)); } else if (index.endsWith('competency')) { let mapping = await httpGet(elasticEndpoint + '/' + index + '/_mapping', true, global.elasticHeaders()); @@ -289,7 +289,7 @@ global.skyrepoMigrate = async function(after) { (permNoIndex)[substring + 'EncryptedValue'] = doc; } (doc).properties = { - '@version': {type: 'long'}, + '@version': { type: 'long' }, 'ceasn:codedNotation': { 'type': 'text', 'fields': { @@ -300,7 +300,7 @@ global.skyrepoMigrate = async function(after) { }, }, }; - let result = await httpPut(mappings, elasticEndpoint + '/.temp.'+index, 'application/json', global.elasticHeaders()); + let result = await httpPut(mappings, elasticEndpoint + '/.temp.' + index, 'application/json', global.elasticHeaders()); global.auditLogger.report(global.auditLogger.LogCategory.NETWORK, global.auditLogger.Severity.INFO, 'SkyrepMigrate', JSON.stringify(result)); } else { let mapping = await httpGet(elasticEndpoint + '/' + index + '/_mapping', true, global.elasticHeaders()); @@ -316,19 +316,19 @@ global.skyrepoMigrate = async function(after) { if (EcObject.keys(mapping[index].mappings)[0]) { (permNoIndex)[EcObject.keys(mapping[index].mappings)[0]] = doc; } - (doc).properties = {'@version': {type: 'long'}}; - mappings['settings'] = {'index.mapping.total_fields.limit': fields}; - let result = await httpPut(mappings, elasticEndpoint + '/.temp.'+index, 'application/json', global.elasticHeaders()); + (doc).properties = { '@version': { type: 'long' } }; + mappings['settings'] = { 'index.mapping.total_fields.limit': fields }; + let result = await httpPut(mappings, elasticEndpoint + '/.temp.' + index, 'application/json', global.elasticHeaders()); global.auditLogger.report(global.auditLogger.LogCategory.NETWORK, global.auditLogger.Severity.INFO, 'SkyrepMigrate', JSON.stringify(result)); } let r = null; global.auditLogger.report(global.auditLogger.LogCategory.NETWORK, global.auditLogger.Severity.INFO, 'SkyrepMigrate', r = await httpPost(JSON.stringify({ - source: {index: index}, - dest: {index: '.temp.'+index, version_type: 'external'}, - }), elasticEndpoint+'/_reindex?refresh=true', 'application/json', 'false', global.elasticHeaders())); + source: { index: index }, + dest: { index: '.temp.' + index, version_type: 'external' }, + }), elasticEndpoint + '/_reindex?refresh=true', 'application/json', 'false', global.elasticHeaders())); if (r.error != null) continue; let r2 = null; - global.auditLogger.report(global.auditLogger.LogCategory.NETWORK, global.auditLogger.Severity.INFO, 'SkyrepMigrate', r2 = await httpDelete(elasticEndpoint+'/'+index, true, global.elasticHeaders())); + global.auditLogger.report(global.auditLogger.LogCategory.NETWORK, global.auditLogger.Severity.INFO, 'SkyrepMigrate', r2 = await httpDelete(elasticEndpoint + '/' + index, true, global.elasticHeaders())); if (r2.error != null) continue; // if (r.total == 0) continue; if (index == 'permanent') { @@ -346,11 +346,11 @@ global.skyrepoMigrate = async function(after) { let permNoIndex = {}; let doc = {}; (mappings)['mappings'] = permNoIndex; - if (EcObject.keys(mapping['.temp.'+index].mappings)[0]) { - (permNoIndex)[EcObject.keys(mapping['.temp.'+index].mappings)[0]] = doc; + if (EcObject.keys(mapping['.temp.' + index].mappings)[0]) { + (permNoIndex)[EcObject.keys(mapping['.temp.' + index].mappings)[0]] = doc; } (doc).properties = { - '@version': {type: 'long'}, + '@version': { type: 'long' }, 'ceasn:codedNotation': { 'type': 'text', 'fields': { @@ -361,7 +361,7 @@ global.skyrepoMigrate = async function(after) { }, }, }; - let result = await httpPut(mappings, elasticEndpoint + '/'+index, 'application/json', global.elasticHeaders()); + let result = await httpPut(mappings, elasticEndpoint + '/' + index, 'application/json', global.elasticHeaders()); global.auditLogger.report(global.auditLogger.LogCategory.NETWORK, global.auditLogger.Severity.INFO, 'SkyrepMigrate', JSON.stringify(result)); } else if (index.endsWith('assertion')) { let mapping = await httpGet(elasticEndpoint + '/.temp.' + index + '/_mapping', true, global.elasticHeaders()); @@ -369,15 +369,15 @@ global.skyrepoMigrate = async function(after) { let permNoIndex = {}; let doc = {}; (mappings)['mappings'] = permNoIndex; - if (EcObject.keys(mapping['.temp.'+index].mappings)[0]) { - (permNoIndex)[EcObject.keys(mapping['.temp.'+index].mappings)[0]] = doc; + if (EcObject.keys(mapping['.temp.' + index].mappings)[0]) { + (permNoIndex)[EcObject.keys(mapping['.temp.' + index].mappings)[0]] = doc; } (doc).properties = { - '@version': {type: 'long'}, - 'confidence': {type: 'float'}, - 'assertionDateDecrypted': {type: 'long'}, + '@version': { type: 'long' }, + 'confidence': { type: 'float' }, + 'assertionDateDecrypted': { type: 'long' }, }; - let result = await httpPut(mappings, elasticEndpoint + '/'+index, 'application/json', global.elasticHeaders()); + let result = await httpPut(mappings, elasticEndpoint + '/' + index, 'application/json', global.elasticHeaders()); global.auditLogger.report(global.auditLogger.LogCategory.NETWORK, global.auditLogger.Severity.INFO, 'SkyrepMigrate', JSON.stringify(result)); } else { let mapping = await httpGet(elasticEndpoint + '/.temp.' + index + '/_mapping', true, global.elasticHeaders()); @@ -390,30 +390,30 @@ global.skyrepoMigrate = async function(after) { let permNoIndex = {}; let doc = {}; (mappings)['mappings'] = permNoIndex; - if (EcObject.keys(mapping['.temp.'+index].mappings)[0]) { - (permNoIndex)[EcObject.keys(mapping['.temp.'+index].mappings)[0]] = doc; + if (EcObject.keys(mapping['.temp.' + index].mappings)[0]) { + (permNoIndex)[EcObject.keys(mapping['.temp.' + index].mappings)[0]] = doc; } - (doc).properties = {'@version': {type: 'long'}}; - mappings['settings'] = {'index.mapping.total_fields.limit': fields}; - let result = await httpPut(mappings, elasticEndpoint + '/'+index, 'application/json', global.elasticHeaders()); + (doc).properties = { '@version': { type: 'long' } }; + mappings['settings'] = { 'index.mapping.total_fields.limit': fields }; + let result = await httpPut(mappings, elasticEndpoint + '/' + index, 'application/json', global.elasticHeaders()); global.auditLogger.report(global.auditLogger.LogCategory.NETWORK, global.auditLogger.Severity.INFO, 'SkyrepMigrate', JSON.stringify(result)); } - global.auditLogger.report(global.auditLogger.LogCategory.NETWORK, global.auditLogger.Severity.INFO, 'SkyrepMigrate', 'Reindexing .temp.' + index + ' -> '+index); + global.auditLogger.report(global.auditLogger.LogCategory.NETWORK, global.auditLogger.Severity.INFO, 'SkyrepMigrate', 'Reindexing .temp.' + index + ' -> ' + index); global.auditLogger.report(global.auditLogger.LogCategory.NETWORK, global.auditLogger.Severity.INFO, 'SkyrepMigrate', r = await httpPost(JSON.stringify({ - source: {index: '.temp.'+index}, - dest: {index: index, version_type: 'external'}, - }), elasticEndpoint+'/_reindex?refresh=true', 'application/json', 'false')); + source: { index: '.temp.' + index }, + dest: { index: index, version_type: 'external' }, + }), elasticEndpoint + '/_reindex?refresh=true', 'application/json', 'false')); if (r.error != null) continue; global.auditLogger.report(global.auditLogger.LogCategory.NETWORK, global.auditLogger.Severity.INFO, 'SkyrepMigrate', 'Deleting .temp.' + index); - global.auditLogger.report(global.auditLogger.LogCategory.NETWORK, global.auditLogger.Severity.INFO, 'SkyrepMigrate', await httpDelete(elasticEndpoint+'/.temp.'+index, true, global.elasticHeaders())); + global.auditLogger.report(global.auditLogger.LogCategory.NETWORK, global.auditLogger.Severity.INFO, 'SkyrepMigrate', await httpDelete(elasticEndpoint + '/.temp.' + index, true, global.elasticHeaders())); } } after(); }; -skyrepoReindex = async function() { +skyrepoReindex = async function () { if (this.params.debug != null) { - skyrepoDebug = true; + global.skyrepoDebug = true; } if (this.params.secret.trim() !== skyIdSecret().trim()) { error('You must provide secret=`cat skyId.secret` to invoke reindex.', 401); @@ -421,7 +421,7 @@ skyrepoReindex = async function() { let firstQueryPost = { query: { - query_string: {query: '*:*'}, + query_string: { query: '*:*' }, }, explain: 'false', size: '50', @@ -440,7 +440,7 @@ skyrepoReindex = async function() { for (let i = 0; i < hits.length; i++) { let hit = hits[i]; let id = hit._id; - if (id.indexOf('.') == id.length-1) { + if (id.indexOf('.') == id.length - 1) { if (++counter % 1000 == 0) { global.auditLogger.report(global.auditLogger.LogCategory.NETWORK, global.auditLogger.Severity.INFO, 'SkyrepReindex', 'Reindexed ' + counter + ' records.'); } @@ -451,7 +451,7 @@ skyrepoReindex = async function() { } global.auditLogger.report(global.auditLogger.LogCategory.NETWORK, global.auditLogger.Severity.INFO, 'SkyrepReindex', 'Reindexed ' + counter + ' records.'); if (this.params.debug != null) { - skyrepoDebug = false; + global.skyrepoDebug = false; } }; bindWebService('/util/reindex', skyrepoReindex); @@ -517,12 +517,12 @@ bindWebService('/util/reindex', skyrepoReindex); // return JSON.stringify(log, null, 2); // } // bindWebService("/util/restore", skyrepoRestore); -let skyIdSecret = ()=>{ - return loadConfigurationFile('skyId.secret', function() { +let skyIdSecret = () => { + return loadConfigurationFile('skyId.secret', function () { return randomString(2048); }); }; -skyrepoPurge = async function() { +skyrepoPurge = async function () { if (this.params.secret != skyIdSecret()) { error('You must provide secret=`cat skyId.secret` to invoke purge.', 401); }