From 3f18319c5118a99c2079f149aec835d29125b2f0 Mon Sep 17 00:00:00 2001 From: Georgii Petrov Date: Tue, 20 Feb 2024 09:45:00 +0300 Subject: [PATCH 01/48] [feature] WOPI Save As implementation --- DocService/sources/canvasservice.js | 12 ++++++++++++ DocService/sources/wopiClient.js | 9 +++++++++ 2 files changed, 21 insertions(+) diff --git a/DocService/sources/canvasservice.js b/DocService/sources/canvasservice.js index 7873634f7..92ab9f349 100644 --- a/DocService/sources/canvasservice.js +++ b/DocService/sources/canvasservice.js @@ -229,6 +229,18 @@ var getOutputData = co.wrap(function* (ctx, cmd, outputData, key, optConn, optAd optAdditionalOutput.needUrlMethod = 2; optAdditionalOutput.needUrlType = commonDefines.c_oAscUrlTypes.Temporary; } + if (command === 'save') { + const info = yield docsCoServer.getCallback(ctx, cmd.getDocId(), cmd.getUserIndex()); + // info.wopiParams is null if it is not wopi + // TODO: pass 'save as' flag from web-apps and check here + if (info.wopiParams && wopiClient.isPutRelativeFileImplemented()) { + const suggestedTargetType = `.${formatChecker.getStringFromFormat(cmd.getOutputFormat())}`; + const storageFilePath = `${cmd.getSaveKey()}/${cmd.getOutputPath()}`; + const stream = yield storage.createReadStream(ctx, storageFilePath); + const { wopiSrc, access_token } = info.wopiParams.userAuth; + yield wopiClient.putRelativeFile(ctx, wopiSrc, access_token, null, stream.readStream, stream.ContentLength, suggestedTargetType, false); + } + } } else { let encryptedUserPassword = cmd.getPassword(); let userPassword; diff --git a/DocService/sources/wopiClient.js b/DocService/sources/wopiClient.js index c6c3f5186..bce535568 100644 --- a/DocService/sources/wopiClient.js +++ b/DocService/sources/wopiClient.js @@ -89,6 +89,8 @@ const cfgWopiDummySampleFilePath = config.get('wopi.dummy.sampleFilePath'); let templatesFolderLocalesCache = null; const templateFilesSizeCache = {}; +let isPutRelativeFileFlag = false; + let mimeTypesByExt = (function() { let mimeTypesByExt = {}; for (let mimeType in mimeDB) { @@ -411,6 +413,8 @@ function getEditorHtml(req, res) { return; } + isPutRelativeFileFlag = fileInfo.SupportsUpdate && !fileInfo.UserCanNotWriteRelative; + if (!fileInfo.UserCanWrite) { mode = 'view'; } @@ -934,6 +938,10 @@ function dummyOk(req, res) { res.sendStatus(200); } +function isPutRelativeFileImplemented() { + return isPutRelativeFileFlag; +} + exports.discovery = discovery; exports.collaboraCapabilities = collaboraCapabilities; exports.parseWopiCallback = parseWopiCallback; @@ -953,3 +961,4 @@ exports.getFileTypeByInfo = getFileTypeByInfo; exports.dummyCheckFileInfo = dummyCheckFileInfo; exports.dummyGetFile = dummyGetFile; exports.dummyOk = dummyOk; +exports.isPutRelativeFileImplemented = isPutRelativeFileImplemented; From 92c16d1bb6403b46e6fef221171189ecaa4f4a6d Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Wed, 21 Feb 2024 20:20:06 +0300 Subject: [PATCH 02/48] [feature] Fix wopi save as --- Common/sources/commondefines.js | 8 ++++++ DocService/sources/canvasservice.js | 42 ++++++++++++++++------------- DocService/sources/wopiClient.js | 15 +++-------- 3 files changed, 35 insertions(+), 30 deletions(-) diff --git a/Common/sources/commondefines.js b/Common/sources/commondefines.js index 83fe464cc..3a1162f74 100644 --- a/Common/sources/commondefines.js +++ b/Common/sources/commondefines.js @@ -108,6 +108,7 @@ function InputCommand(data, copyExplicit) { this['status_info_in'] = data['status_info_in']; this['attempt'] = data['attempt']; this['convertToOrigin'] = data['convertToOrigin']; + this['isSaveAs'] = data['isSaveAs']; if (copyExplicit) { this['withAuthorization'] = data['withAuthorization']; this['externalChangeInfo'] = data['externalChangeInfo']; @@ -171,6 +172,7 @@ function InputCommand(data, copyExplicit) { this['attempt'] = undefined; this['convertToOrigin'] = undefined; this['originformat'] = undefined; + this['isSaveAs'] = undefined; } } InputCommand.prototype = { @@ -503,6 +505,12 @@ InputCommand.prototype = { }, setConvertToOrigin: function(data) { this['convertToOrigin'] = data; + }, + getIsSaveAs: function() { + return this['isSaveAs']; + }, + setIsSaveAs: function(data) { + this['isSaveAs'] = data; } }; diff --git a/DocService/sources/canvasservice.js b/DocService/sources/canvasservice.js index 92ab9f349..2789b3421 100644 --- a/DocService/sources/canvasservice.js +++ b/DocService/sources/canvasservice.js @@ -229,18 +229,6 @@ var getOutputData = co.wrap(function* (ctx, cmd, outputData, key, optConn, optAd optAdditionalOutput.needUrlMethod = 2; optAdditionalOutput.needUrlType = commonDefines.c_oAscUrlTypes.Temporary; } - if (command === 'save') { - const info = yield docsCoServer.getCallback(ctx, cmd.getDocId(), cmd.getUserIndex()); - // info.wopiParams is null if it is not wopi - // TODO: pass 'save as' flag from web-apps and check here - if (info.wopiParams && wopiClient.isPutRelativeFileImplemented()) { - const suggestedTargetType = `.${formatChecker.getStringFromFormat(cmd.getOutputFormat())}`; - const storageFilePath = `${cmd.getSaveKey()}/${cmd.getOutputPath()}`; - const stream = yield storage.createReadStream(ctx, storageFilePath); - const { wopiSrc, access_token } = info.wopiParams.userAuth; - yield wopiClient.putRelativeFile(ctx, wopiSrc, access_token, null, stream.readStream, stream.ContentLength, suggestedTargetType, false); - } - } } else { let encryptedUserPassword = cmd.getPassword(); let userPassword; @@ -1737,6 +1725,18 @@ exports.saveFromChanges = function(ctx, docId, statusInfo, optFormat, opt_userId } }); }; + +async function processWopiSaveAs(ctx, cmd) { + const info = await docsCoServer.getCallback(ctx, cmd.getDocId(), cmd.getUserIndex()); + // info.wopiParams is null if it is not wopi + if (info.wopiParams) { + const suggestedTargetType = `.${formatChecker.getStringFromFormat(cmd.getOutputFormat())}`; + const storageFilePath = `${cmd.getSaveKey()}/${cmd.getOutputPath()}`; + const stream = await storage.createReadStream(ctx, storageFilePath); + const { wopiSrc, access_token } = info.wopiParams.userAuth; + await wopiClient.putRelativeFile(ctx, wopiSrc, access_token, null, stream.readStream, stream.ContentLength, suggestedTargetType, false); + } +} exports.receiveTask = function(data, ack) { return co(function* () { let ctx = new operationContext.Context(); @@ -1753,17 +1753,21 @@ exports.receiveTask = function(data, ack) { var outputData = new OutputData(cmd.getCommand()); var command = cmd.getCommand(); var additionalOutput = {needUrlKey: null, needUrlMethod: null, needUrlType: null, needUrlIsCorrectPassword: undefined, creationDate: undefined, openedAt: undefined}; - if ('open' == command || 'reopen' == command) { + if ('open' === command || 'reopen' === command) { yield getOutputData(ctx, cmd, outputData, cmd.getDocId(), null, additionalOutput); - } else if ('save' == command || 'savefromorigin' == command) { - yield getOutputData(ctx, cmd, outputData, cmd.getSaveKey(), null, additionalOutput); - } else if ('sfcm' == command) { + } else if ('save' === command || 'savefromorigin' === command) { + let status = yield getOutputData(ctx, cmd, outputData, cmd.getSaveKey(), null, additionalOutput); + if (commonDefines.FileStatus.Ok === status && cmd.getIsSaveAs()) { + yield processWopiSaveAs(ctx, cmd); + //todo in case of wopi no need to send url. send it to avoid stubs in sdk + } + } else if ('sfcm' === command) { yield commandSfcCallback(ctx, cmd, true); - } else if ('sfc' == command) { + } else if ('sfc' === command) { yield commandSfcCallback(ctx, cmd, false); - } else if ('sendmm' == command) { + } else if ('sendmm' === command) { yield* commandSendMMCallback(ctx, cmd); - } else if ('conv' == command) { + } else if ('conv' === command) { //nothing } if (outputData.getStatus()) { diff --git a/DocService/sources/wopiClient.js b/DocService/sources/wopiClient.js index bce535568..6af117665 100644 --- a/DocService/sources/wopiClient.js +++ b/DocService/sources/wopiClient.js @@ -89,8 +89,6 @@ const cfgWopiDummySampleFilePath = config.get('wopi.dummy.sampleFilePath'); let templatesFolderLocalesCache = null; const templateFilesSizeCache = {}; -let isPutRelativeFileFlag = false; - let mimeTypesByExt = (function() { let mimeTypesByExt = {}; for (let mimeType in mimeDB) { @@ -413,8 +411,6 @@ function getEditorHtml(req, res) { return; } - isPutRelativeFileFlag = fileInfo.SupportsUpdate && !fileInfo.UserCanNotWriteRelative; - if (!fileInfo.UserCanWrite) { mode = 'view'; } @@ -637,8 +633,10 @@ function putRelativeFile(ctx, wopiSrc, access_token, data, dataStream, dataSize, return postRes; } - let headers = {'X-WOPI-Override': 'PUT_RELATIVE', 'X-WOPI-SuggestedTarget': utf7.encode(suggestedTarget), - 'X-WOPI-FileConversion': isFileConversion}; + let headers = {'X-WOPI-Override': 'PUT_RELATIVE', 'X-WOPI-SuggestedTarget': utf7.encode(suggestedTarget)}; + if (isFileConversion) { + headers['X-WOPI-FileConversion'] = isFileConversion; + } fillStandardHeaders(ctx, headers, uri, access_token); ctx.logger.debug('wopi putRelativeFile request uri=%s headers=%j', uri, headers); @@ -938,10 +936,6 @@ function dummyOk(req, res) { res.sendStatus(200); } -function isPutRelativeFileImplemented() { - return isPutRelativeFileFlag; -} - exports.discovery = discovery; exports.collaboraCapabilities = collaboraCapabilities; exports.parseWopiCallback = parseWopiCallback; @@ -961,4 +955,3 @@ exports.getFileTypeByInfo = getFileTypeByInfo; exports.dummyCheckFileInfo = dummyCheckFileInfo; exports.dummyGetFile = dummyGetFile; exports.dummyOk = dummyOk; -exports.isPutRelativeFileImplemented = isPutRelativeFileImplemented; From fb7b1c42572c8ff040e1d29cd341f89138a3ac86 Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Mon, 26 Feb 2024 14:09:00 +0300 Subject: [PATCH 03/48] [wopi] Fix downloadfile handler for wopi; fix bug 66612 # Conflicts: # DocService/sources/canvasservice.js # DocService/sources/wopiClient.js --- DocService/sources/DocsCoServer.js | 6 +++--- DocService/sources/canvasservice.js | 10 +++++++--- DocService/sources/wopiClient.js | 22 ++++++++++++++++++++++ FileConverter/sources/converter.js | 15 +-------------- 4 files changed, 33 insertions(+), 20 deletions(-) diff --git a/DocService/sources/DocsCoServer.js b/DocService/sources/DocsCoServer.js index 5541656ad..d444a8fbb 100644 --- a/DocService/sources/DocsCoServer.js +++ b/DocService/sources/DocsCoServer.js @@ -2242,7 +2242,7 @@ exports.install = function(server, callbackFunction) { let queryParams = decoded.queryParams; data.lang = queryParams.lang || queryParams.ui || "en-US"; } - if (decoded.fileInfo) { + if (wopiClient.isWopiJwtToken(decoded)) { let fileInfo = decoded.fileInfo; if (openCmd) { openCmd.format = wopiClient.getFileTypeByInfo(fileInfo); @@ -2397,7 +2397,7 @@ exports.install = function(server, callbackFunction) { } //todo make required fields - if (decoded.url || decoded.payload|| (decoded.key && !decoded.fileInfo)) { + if (decoded.url || decoded.payload|| (decoded.key && !wopiClient.isWopiJwtToken(decoded))) { ctx.logger.warn('fillDataFromJwt token has invalid format'); res = false; } @@ -2438,7 +2438,7 @@ exports.install = function(server, callbackFunction) { isDecoded = true; let decoded = checkJwtRes.decoded; let fillDataFromJwtRes = false; - if (decoded.fileInfo) { + if (wopiClient.isWopiJwtToken(decoded)) { //wopi fillDataFromJwtRes = fillDataFromWopiJwt(decoded, data); } else if (decoded.editorConfig && undefined !== decoded.editorConfig.ds_view) { diff --git a/DocService/sources/canvasservice.js b/DocService/sources/canvasservice.js index 2789b3421..9eda8a459 100644 --- a/DocService/sources/canvasservice.js +++ b/DocService/sources/canvasservice.js @@ -1598,6 +1598,7 @@ exports.downloadFile = function(req, res) { let authorization; let isInJwtToken = false; let errorDescription; + let headers; let authRes = yield docsCoServer.getRequestParams(ctx, req); if (authRes.code === constants.NO_ERROR) { let decoded = authRes.params; @@ -1610,6 +1611,8 @@ exports.downloadFile = function(req, res) { } else if (decoded.url && -1 !== tenDownloadFileAllowExt.indexOf(decoded.fileType)) { url = decoded.url; isInJwtToken = true; + } else if (wopiClient.isWopiJwtToken(decoded)) { + ({url, headers} = wopiClient.getWopiFileUrl(ctx, decoded.fileInfo, decoded.userAuth)); } else if (!tenTokenEnableBrowser) { //todo token required if (decoded.url) { @@ -1638,11 +1641,12 @@ exports.downloadFile = function(req, res) { res.sendStatus(filterStatus); return; } - let headers; + if (req.get('Range')) { - headers = { - 'Range': req.get('Range') + if (!headers) { + headers = {}; } + headers['Range'] = req.get('Range'); } yield utils.downloadUrlPromise(ctx, url, tenDownloadTimeout, tenDownloadMaxBytes, authorization, isInJwtToken, headers, res); diff --git a/DocService/sources/wopiClient.js b/DocService/sources/wopiClient.js index 6af117665..8ccc6dc4f 100644 --- a/DocService/sources/wopiClient.js +++ b/DocService/sources/wopiClient.js @@ -63,6 +63,7 @@ const cfgTokenEnableBrowser = config.get('services.CoAuthoring.token.enable.brow const cfgCallbackRequestTimeout = config.get('services.CoAuthoring.server.callbackRequestTimeout'); const cfgNewFileTemplate = config.get('services.CoAuthoring.server.newFileTemplate'); const cfgDownloadTimeout = config.get('FileConverter.converter.downloadTimeout'); +const cfgMaxDownloadBytes = config.get('FileConverter.converter.maxDownloadBytes'); const cfgWopiFileInfoBlockList = config.get('wopi.fileInfoBlockList'); const cfgWopiWopiZone = config.get('wopi.wopiZone'); const cfgWopiPdfView = config.get('wopi.pdfView'); @@ -289,6 +290,25 @@ function getFileTypeByInfo(fileInfo) { fileType = fileInfo.FileExtension ? fileInfo.FileExtension.substr(1) : fileType; return fileType.toLowerCase(); } +function getWopiFileUrl(ctx, fileInfo, userAuth) { + const tenMaxDownloadBytes = ctx.getCfg('FileConverter.converter.maxDownloadBytes', cfgMaxDownloadBytes); + let url; + let headers = {'X-WOPI-MaxExpectedSize': tenMaxDownloadBytes}; + if (fileInfo?.FileUrl) { + //Requests to the FileUrl can not be signed using proof keys. The FileUrl is used exactly as provided by the host, so it does not necessarily include the access token, which is required to construct the expected proof. + url = fileInfo.FileUrl; + } else if (fileInfo?.TemplateSource) { + url = fileInfo.TemplateSource; + } else if (userAuth) { + url = `${userAuth.wopiSrc}/contents?access_token=${userAuth.access_token}`; + fillStandardHeaders(ctx, headers, url, userAuth.access_token); + } + ctx.logger.debug('getWopiFileUrl url=%s; headers=%j', url, headers); + return {url, headers}; +} +function isWopiJwtToken(decoded) { + return !!decoded.fileInfo; +} function getLastModifiedTimeFromCallbacks(callbacks) { for (let i = callbacks.length; i >= 0; --i) { let callback = callbacks[i]; @@ -952,6 +972,8 @@ exports.fillStandardHeaders = fillStandardHeaders; exports.getWopiUnlockMarker = getWopiUnlockMarker; exports.getWopiModifiedMarker = getWopiModifiedMarker; exports.getFileTypeByInfo = getFileTypeByInfo; +exports.getWopiFileUrl = getWopiFileUrl; +exports.isWopiJwtToken = isWopiJwtToken; exports.dummyCheckFileInfo = dummyCheckFileInfo; exports.dummyGetFile = dummyGetFile; exports.dummyOk = dummyOk; diff --git a/FileConverter/sources/converter.js b/FileConverter/sources/converter.js index cb2b0df63..a66800166 100644 --- a/FileConverter/sources/converter.js +++ b/FileConverter/sources/converter.js @@ -429,7 +429,6 @@ function* downloadFile(ctx, uri, fileFrom, withAuthorization, isInJwtToken, opt_ return res; } function* downloadFileFromStorage(ctx, strPath, dir, opt_specialDir) { - const tenMaxDownloadBytes = ctx.getCfg('FileConverter.converter.maxDownloadBytes', cfgMaxDownloadBytes); var list = yield storage.listObjects(ctx, strPath, opt_specialDir); ctx.logger.debug('downloadFileFromStorage list %s', list.toString()); //create dirs @@ -1002,7 +1001,6 @@ function* spawnProcess(ctx, builderParams, tempDirs, dataConvert, authorProps, g } function* ExecuteTask(ctx, task) { - const tenMaxDownloadBytes = ctx.getCfg('FileConverter.converter.maxDownloadBytes', cfgMaxDownloadBytes); const tenForgottenFiles = ctx.getCfg('services.CoAuthoring.server.forgottenfiles', cfgForgottenFiles); const tenForgottenFilesName = ctx.getCfg('services.CoAuthoring.server.forgottenfilesname', cfgForgottenFilesName); var startDate = null; @@ -1039,19 +1037,8 @@ function* ExecuteTask(ctx, task) { withAuthorization = false; isInJwtToken = true; let fileInfo = wopiParams.commonInfo?.fileInfo; - let userAuth = wopiParams.userAuth; fileSize = fileInfo?.Size; - if (fileInfo?.FileUrl) { - //Requests to the FileUrl can not be signed using proof keys. The FileUrl is used exactly as provided by the host, so it does not necessarily include the access token, which is required to construct the expected proof. - url = fileInfo.FileUrl; - } else if (fileInfo?.TemplateSource) { - url = fileInfo.TemplateSource; - } else if (userAuth) { - url = `${userAuth.wopiSrc}/contents?access_token=${userAuth.access_token}`; - headers = {'X-WOPI-MaxExpectedSize': tenMaxDownloadBytes}; - wopiClient.fillStandardHeaders(ctx, headers, url, userAuth.access_token); - } - ctx.logger.debug('wopi url=%s; headers=%j', url, headers); + ({url, headers} = wopiClient.getWopiFileUrl(ctx, fileInfo, wopiParams.userAuth)); } if (undefined === fileSize || fileSize > 0) { error = yield* downloadFile(ctx, url, dataConvert.fileFrom, withAuthorization, isInJwtToken, headers); From 23c7f8ceb55360c8935ca5244f09f1dd2826764b Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Mon, 4 Mar 2024 13:01:33 +0300 Subject: [PATCH 04/48] [wopi] Add "default" action flag for non-editable formats;Fix bug 66714 --- DocService/sources/wopiClient.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DocService/sources/wopiClient.js b/DocService/sources/wopiClient.js index 8ccc6dc4f..73ddf2a74 100644 --- a/DocService/sources/wopiClient.js +++ b/DocService/sources/wopiClient.js @@ -167,7 +167,7 @@ function discovery(req, res) { let urlTemplateMobileEdit = `${templateStart}/${documentTypes[i]}/edit?mobile=1&${templateEnd}`; let xmlApp = xmlZone.ele('app', {name: name, favIconUrl: favIconUrl}); for (let j = 0; j < ext.view.length; ++j) { - xmlApp.ele('action', {name: 'view', ext: ext.view[j], urlsrc: urlTemplateView}).up(); + xmlApp.ele('action', {name: 'view', ext: ext.view[j], default: 'true', urlsrc: urlTemplateView}).up(); xmlApp.ele('action', {name: 'embedview', ext: ext.view[j], urlsrc: urlTemplateEmbedView}).up(); xmlApp.ele('action', {name: 'mobileView', ext: ext.view[j], urlsrc: urlTemplateMobileView}).up(); if (-1 === tenWopiPdfView.indexOf(ext.view[j])) { From 7ba3d158fc63c49a5a7eb238d14f11b48c6f3a25 Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Mon, 4 Mar 2024 18:41:17 +0300 Subject: [PATCH 05/48] [wopi] Add "formsubmit" discovery action; For bug 66720 --- Common/config/default.json | 5 +++-- DocService/sources/wopiClient.js | 15 ++++++++++++++- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/Common/config/default.json b/Common/config/default.json index 081762413..c962d7a81 100644 --- a/Common/config/default.json +++ b/Common/config/default.json @@ -86,8 +86,9 @@ "favIconUrlSlide" : "/web-apps/apps/presentationeditor/main/resources/img/favicon.ico", "fileInfoBlockList" : ["FileUrl"], "pdfView": ["pdf", "djvu", "xps", "oxps"], - "wordView": ["doc", "dotx", "dotm", "dot", "fodt", "ott", "rtf", "mht", "mhtml", "html", "htm", "xml", "epub", "fb2", "sxw", "stw", "wps", "wpt"], - "wordEdit": ["docx", "docm", "docxf", "oform", "odt", "txt"], + "wordView": ["doc", "dotx", "dotm", "dot", "fodt", "ott", "rtf", "mht", "mhtml", "html", "htm", "xml", "epub", "fb2", "sxw", "stw", "wps", "wpt", "oform"], + "wordEdit": ["docx", "docm", "docxf", "odt", "txt"], + "wordForm": ["pdf"], "cellView": ["xls", "xlsb", "xltx", "xltm", "xlt", "fods", "ots", "sxc", "xml", "et", "ett"], "cellEdit": ["xlsx", "xlsm", "ods", "csv"], "slideView": ["ppt", "ppsx", "ppsm", "pps", "potx", "potm", "pot", "fodp", "otp", "sxi", "dps", "dpt"], diff --git a/DocService/sources/wopiClient.js b/DocService/sources/wopiClient.js index 73ddf2a74..c5c4afa10 100644 --- a/DocService/sources/wopiClient.js +++ b/DocService/sources/wopiClient.js @@ -73,6 +73,7 @@ const cfgWopiCellView = config.get('wopi.cellView'); const cfgWopiCellEdit = config.get('wopi.cellEdit'); const cfgWopiSlideView = config.get('wopi.slideView'); const cfgWopiSlideEdit = config.get('wopi.slideEdit'); +const cfgWopiWordForm = config.get('wopi.wordForm'); const cfgWopiFavIconUrlWord = config.get('wopi.favIconUrlWord'); const cfgWopiFavIconUrlCell = config.get('wopi.favIconUrlCell'); const cfgWopiFavIconUrlSlide = config.get('wopi.favIconUrlSlide'); @@ -125,6 +126,7 @@ function discovery(req, res) { const tenWopiCellEdit = ctx.getCfg('wopi.cellEdit', cfgWopiCellEdit); const tenWopiSlideView = ctx.getCfg('wopi.slideView', cfgWopiSlideView); const tenWopiSlideEdit = ctx.getCfg('wopi.slideEdit', cfgWopiSlideEdit); + const tenWopiWordForm = ctx.getCfg('wopi.wordForm', cfgWopiWordForm); const tenWopiFavIconUrlWord = ctx.getCfg('wopi.favIconUrlWord', cfgWopiFavIconUrlWord); const tenWopiFavIconUrlCell = ctx.getCfg('wopi.favIconUrlCell', cfgWopiFavIconUrlCell); const tenWopiFavIconUrlSlide = ctx.getCfg('wopi.favIconUrlSlide', cfgWopiFavIconUrlSlide); @@ -140,7 +142,7 @@ function discovery(req, res) { let names = ['Word','Excel','PowerPoint']; let favIconUrls = [tenWopiFavIconUrlWord, tenWopiFavIconUrlCell, tenWopiFavIconUrlSlide]; let exts = [ - {targetext: 'docx', view: tenWopiPdfView.concat(tenWopiWordView), edit: tenWopiWordEdit}, + {targetext: 'docx', view: tenWopiPdfView.concat(tenWopiWordView), edit: tenWopiWordEdit, form: tenWopiWordForm}, {targetext: 'xlsx', view: tenWopiCellView, edit: tenWopiCellEdit}, {targetext: 'pptx', view: tenWopiSlideView, edit: tenWopiSlideEdit} ]; @@ -165,6 +167,7 @@ function discovery(req, res) { let urlTemplateMobileView = `${templateStart}/${documentTypes[i]}/view?mobile=1&${templateEnd}`; let urlTemplateEdit = `${templateStart}/${documentTypes[i]}/edit?${templateEnd}`; let urlTemplateMobileEdit = `${templateStart}/${documentTypes[i]}/edit?mobile=1&${templateEnd}`; + let urlTemplateFormSubmit = `${templateStart}/${documentTypes[i]}/edit?formsubmit=1&${templateEnd}`; let xmlApp = xmlZone.ele('app', {name: name, favIconUrl: favIconUrl}); for (let j = 0; j < ext.view.length; ++j) { xmlApp.ele('action', {name: 'view', ext: ext.view[j], default: 'true', urlsrc: urlTemplateView}).up(); @@ -182,6 +185,9 @@ function discovery(req, res) { xmlApp.ele('action', {name: 'edit', ext: ext.edit[j], default: 'true', requires: 'locks,update', urlsrc: urlTemplateEdit}).up(); xmlApp.ele('action', {name: 'mobileEdit', ext: ext.edit[j], requires: 'locks,update', urlsrc: urlTemplateMobileEdit}).up(); } + for (let extention of (ext.form || [])) { + xmlApp.ele('action', {name: 'formsubmit', ext: extention, requires: 'locks,update', urlsrc: urlTemplateFormSubmit}).up(); + } constants.SUPPORTED_TEMPLATES_EXTENSIONS[name].forEach( extension => xmlApp.ele('action', {name: 'editnew', ext: extension, requires: 'locks,update', urlsrc: urlTemplateEdit}).up() ); @@ -196,6 +202,7 @@ function discovery(req, res) { let urlTemplateMobileView = `${templateStart}/${documentTypes[i]}/view?mobile=1&${templateEnd}`; let urlTemplateEdit = `${templateStart}/${documentTypes[i]}/edit?${templateEnd}`; let urlTemplateMobileEdit = `${templateStart}/${documentTypes[i]}/edit?mobile=1&${templateEnd}`; + let urlTemplateFormSubmit = `${templateStart}/${documentTypes[i]}/edit?formsubmit=1&${templateEnd}`; for (let j = 0; j < ext.view.length; ++j) { let mimeTypes = mimeTypesByExt[ext.view[j]]; if (mimeTypes) { @@ -208,6 +215,9 @@ function discovery(req, res) { let urlConvert = `${templateStart}/convert-and-edit/${ext.view[j]}/${ext.targetext}?${templateEnd}`; xmlApp.ele('action', {name: 'convert', ext: '', targetext: ext.targetext, requires: 'update', urlsrc: urlConvert}).up(); } + if (ext.form && -1 !== ext.form.indexOf(ext.view[j])) { + xmlApp.ele('action', {name: 'formsubmit', ext: '', requires: 'locks,update', urlsrc: urlTemplateFormSubmit}).up(); + } xmlApp.up(); }); } @@ -219,6 +229,9 @@ function discovery(req, res) { let xmlApp = xmlZone.ele('app', {name: value}); xmlApp.ele('action', {name: 'edit', ext: '', default: 'true', requires: 'locks,update', urlsrc: urlTemplateEdit}).up(); xmlApp.ele('action', {name: 'mobileEdit', ext: '', requires: 'locks,update', urlsrc: urlTemplateMobileEdit}).up(); + if (ext.form && -1 !== ext.form.indexOf(ext.edit[j])) { + xmlApp.ele('action', {name: 'formsubmit', ext: '', requires: 'locks,update', urlsrc: urlTemplateFormSubmit}).up(); + } xmlApp.up(); }); } From 7326335713ea3e248d083f0a539760bb4ea51dea Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Mon, 11 Mar 2024 18:06:31 +0300 Subject: [PATCH 06/48] [bug] Fix downloadFile for wopi; For bug https://bugzilla.onlyoffice.com/show_bug.cgi?id=66818 --- DocService/sources/canvasservice.js | 8 ++++++++ DocService/sources/wopiClient.js | 4 +++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/DocService/sources/canvasservice.js b/DocService/sources/canvasservice.js index 9eda8a459..cd700d424 100644 --- a/DocService/sources/canvasservice.js +++ b/DocService/sources/canvasservice.js @@ -1613,6 +1613,14 @@ exports.downloadFile = function(req, res) { isInJwtToken = true; } else if (wopiClient.isWopiJwtToken(decoded)) { ({url, headers} = wopiClient.getWopiFileUrl(ctx, decoded.fileInfo, decoded.userAuth)); + let filterStatus = yield wopiClient.checkIpFilter(ctx, url); + if (0 === filterStatus) { + //todo false? (true because it passed checkIpFilter for wopi) + //todo use directIfIn + isInJwtToken = true; + } else { + errorDescription = 'access deny'; + } } else if (!tenTokenEnableBrowser) { //todo token required if (decoded.url) { diff --git a/DocService/sources/wopiClient.js b/DocService/sources/wopiClient.js index c5c4afa10..a867c9379 100644 --- a/DocService/sources/wopiClient.js +++ b/DocService/sources/wopiClient.js @@ -748,7 +748,8 @@ function checkFileInfo(ctx, wopiSrc, access_token, opt_sc) { } fillStandardHeaders(ctx, headers, uri, access_token); ctx.logger.debug('wopi checkFileInfo request uri=%s headers=%j', uri, headers); - //todo false? + //todo false? (true because it passed checkIpFilter for wopi) + //todo use directIfIn let isInJwtToken = true; let getRes = yield utils.downloadUrlPromise(ctx, uri, tenDownloadTimeout, undefined, undefined, isInJwtToken, headers); ctx.logger.debug(`wopi checkFileInfo headers=%j body=%s`, getRes.response.headers, getRes.body); @@ -969,6 +970,7 @@ function dummyOk(req, res) { res.sendStatus(200); } +exports.checkIpFilter = checkIpFilter; exports.discovery = discovery; exports.collaboraCapabilities = collaboraCapabilities; exports.parseWopiCallback = parseWopiCallback; From 21f172d1f7e07fc0480b94e8574ffca4eab3ecf0 Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Tue, 12 Mar 2024 10:53:56 +0300 Subject: [PATCH 07/48] [bug] Init context cache on 'connection' event; Fix bug 66841 --- DocService/sources/DocsCoServer.js | 297 +++++++++++++++-------------- 1 file changed, 151 insertions(+), 146 deletions(-) diff --git a/DocService/sources/DocsCoServer.js b/DocService/sources/DocsCoServer.js index d444a8fbb..56b916333 100644 --- a/DocService/sources/DocsCoServer.js +++ b/DocService/sources/DocsCoServer.js @@ -1566,167 +1566,172 @@ exports.install = function(server, callbackFunction) { }); }); - io.on('connection', function(conn) { - if (!conn) { - operationContext.global.logger.error("null == conn"); - return; - } + io.on('connection', async function(conn) { let ctx = new operationContext.Context(); - ctx.initFromConnection(conn); - //todo - //yield ctx.initTenantCache(); - if (getIsShutdown()) { - sendFileError(ctx, conn, 'Server shutdow'); - return; - } - conn.baseUrl = utils.getBaseUrlByConnection(ctx, conn); - conn.sessionIsSendWarning = false; - conn.sessionTimeConnect = conn.sessionTimeLastAction = new Date().getTime(); + try { + if (!conn) { + operationContext.global.logger.error("null == conn"); + return; + } + ctx.initFromConnection(conn); + await ctx.initTenantCache(); + if (getIsShutdown()) { + sendFileError(ctx, conn, 'Server shutdow'); + return; + } + conn.baseUrl = utils.getBaseUrlByConnection(ctx, conn); + conn.sessionIsSendWarning = false; + conn.sessionTimeConnect = conn.sessionTimeLastAction = new Date().getTime(); - conn.on('message', function(data) { - return co(function* () { - var docId = 'null'; - let ctx = new operationContext.Context(); - try { - ctx.initFromConnection(conn); - yield ctx.initTenantCache(); - const tenErrorFiles = ctx.getCfg('FileConverter.converter.errorfiles', cfgErrorFiles); + conn.on('message', function(data) { + return co(function* () { + var docId = 'null'; + let ctx = new operationContext.Context(); + try { + ctx.initFromConnection(conn); + yield ctx.initTenantCache(); + const tenErrorFiles = ctx.getCfg('FileConverter.converter.errorfiles', cfgErrorFiles); - var startDate = null; - if(clientStatsD) { - startDate = new Date(); - } + var startDate = null; + if(clientStatsD) { + startDate = new Date(); + } - docId = conn.docId; - ctx.logger.info('data.type = %s', data.type); - if(getIsShutdown()) - { - ctx.logger.debug('Server shutdown receive data'); - return; - } - if (conn.isCiriticalError && ('message' == data.type || 'getLock' == data.type || 'saveChanges' == data.type || - 'isSaveLock' == data.type)) { - ctx.logger.warn("conn.isCiriticalError send command: type = %s", data.type); - sendDataDisconnectReason(ctx, conn, constants.ACCESS_DENIED_CODE, constants.ACCESS_DENIED_REASON); - conn.disconnect(true); - return; - } - if ((conn.isCloseCoAuthoring || (conn.user && conn.user.view)) && - ('getLock' == data.type || 'saveChanges' == data.type || 'isSaveLock' == data.type)) { - ctx.logger.warn("conn.user.view||isCloseCoAuthoring access deny: type = %s", data.type); - sendDataDisconnectReason(ctx, conn, constants.ACCESS_DENIED_CODE, constants.ACCESS_DENIED_REASON); - conn.disconnect(true); - return; - } - yield encryptPasswordParams(ctx, data); - switch (data.type) { - case 'auth' : - try { - yield* auth(ctx, conn, data); - } catch(err){ - ctx.logger.error('auth error: %s', err.stack); + docId = conn.docId; + ctx.logger.info('data.type = %s', data.type); + if(getIsShutdown()) + { + ctx.logger.debug('Server shutdown receive data'); + return; + } + if (conn.isCiriticalError && ('message' == data.type || 'getLock' == data.type || 'saveChanges' == data.type || + 'isSaveLock' == data.type)) { + ctx.logger.warn("conn.isCiriticalError send command: type = %s", data.type); sendDataDisconnectReason(ctx, conn, constants.ACCESS_DENIED_CODE, constants.ACCESS_DENIED_REASON); conn.disconnect(true); return; } - break; - case 'message' : - yield* onMessage(ctx, conn, data); - break; - case 'cursor' : - yield* onCursor(ctx, conn, data); - break; - case 'getLock' : - yield* getLock(ctx, conn, data, false); - break; - case 'saveChanges' : - yield* saveChanges(ctx, conn, data); - break; - case 'isSaveLock' : - yield* isSaveLock(ctx, conn, data); - break; - case 'unSaveLock' : - yield* unSaveLock(ctx, conn, -1, -1, -1); - break; // The index is sent -1, because this is an emergency withdrawal without saving - case 'getMessages' : - yield* getMessages(ctx, conn, data); - break; - case 'unLockDocument' : - yield* checkEndAuthLock(ctx, data.unlock, data.isSave, docId, conn.user.id, data.releaseLocks, data.deleteIndex, conn); - break; - case 'close': - yield* closeDocument(ctx, conn); - break; - case 'versionHistory' : { - let cmd = new commonDefines.InputCommand(data.cmd); - yield* versionHistory(ctx, conn, cmd); - break; - } - case 'openDocument' : { - var cmd = new commonDefines.InputCommand(data.message); - cmd.fillFromConnection(conn); - yield canvasService.openDocument(ctx, conn, cmd); - break; - } - case 'clientLog': - let level = data.level?.toLowerCase(); - if("trace" === level || "debug" === level || "info" === level || "warn" === level || "error" === level || "fatal" === level) { - ctx.logger[level]("clientLog: %s", data.msg); + if ((conn.isCloseCoAuthoring || (conn.user && conn.user.view)) && + ('getLock' == data.type || 'saveChanges' == data.type || 'isSaveLock' == data.type)) { + ctx.logger.warn("conn.user.view||isCloseCoAuthoring access deny: type = %s", data.type); + sendDataDisconnectReason(ctx, conn, constants.ACCESS_DENIED_CODE, constants.ACCESS_DENIED_REASON); + conn.disconnect(true); + return; } - if ("error" === level && tenErrorFiles && docId) { - let destDir = 'browser/' + docId; - yield storage.copyPath(ctx, docId, destDir, undefined, tenErrorFiles); - yield* saveErrorChanges(ctx, docId, destDir); + yield encryptPasswordParams(ctx, data); + switch (data.type) { + case 'auth' : + try { + yield* auth(ctx, conn, data); + } catch(err){ + ctx.logger.error('auth error: %s', err.stack); + sendDataDisconnectReason(ctx, conn, constants.ACCESS_DENIED_CODE, constants.ACCESS_DENIED_REASON); + conn.disconnect(true); + return; + } + break; + case 'message' : + yield* onMessage(ctx, conn, data); + break; + case 'cursor' : + yield* onCursor(ctx, conn, data); + break; + case 'getLock' : + yield* getLock(ctx, conn, data, false); + break; + case 'saveChanges' : + yield* saveChanges(ctx, conn, data); + break; + case 'isSaveLock' : + yield* isSaveLock(ctx, conn, data); + break; + case 'unSaveLock' : + yield* unSaveLock(ctx, conn, -1, -1, -1); + break; // The index is sent -1, because this is an emergency withdrawal without saving + case 'getMessages' : + yield* getMessages(ctx, conn, data); + break; + case 'unLockDocument' : + yield* checkEndAuthLock(ctx, data.unlock, data.isSave, docId, conn.user.id, data.releaseLocks, data.deleteIndex, conn); + break; + case 'close': + yield* closeDocument(ctx, conn); + break; + case 'versionHistory' : { + let cmd = new commonDefines.InputCommand(data.cmd); + yield* versionHistory(ctx, conn, cmd); + break; + } + case 'openDocument' : { + var cmd = new commonDefines.InputCommand(data.message); + cmd.fillFromConnection(conn); + yield canvasService.openDocument(ctx, conn, cmd); + break; + } + case 'clientLog': + let level = data.level?.toLowerCase(); + if("trace" === level || "debug" === level || "info" === level || "warn" === level || "error" === level || "fatal" === level) { + ctx.logger[level]("clientLog: %s", data.msg); + } + if ("error" === level && tenErrorFiles && docId) { + let destDir = 'browser/' + docId; + yield storage.copyPath(ctx, docId, destDir, undefined, tenErrorFiles); + yield* saveErrorChanges(ctx, docId, destDir); + } + break; + case 'extendSession' : + ctx.logger.debug("extendSession idletime: %d", data.idletime); + conn.sessionIsSendWarning = false; + conn.sessionTimeLastAction = new Date().getTime() - data.idletime; + break; + case 'forceSaveStart' : + var forceSaveRes; + if (conn.user) { + forceSaveRes = yield startForceSave(ctx, docId, commonDefines.c_oAscForceSaveTypes.Button, undefined, undefined, conn.user.idOriginal, conn.user.id, undefined, conn.user.indexUser); + } else { + forceSaveRes = {code: commonDefines.c_oAscServerCommandErrors.UnknownError, time: null}; + } + sendData(ctx, conn, {type: "forceSaveStart", messages: forceSaveRes}); + break; + case 'rpc' : + yield* startRPC(ctx, conn, data.responseKey, data.data); + break; + case 'authChangesAck' : + delete conn.authChangesAck; + break; + default: + ctx.logger.debug("unknown command %j", data); + break; } - break; - case 'extendSession' : - ctx.logger.debug("extendSession idletime: %d", data.idletime); - conn.sessionIsSendWarning = false; - conn.sessionTimeLastAction = new Date().getTime() - data.idletime; - break; - case 'forceSaveStart' : - var forceSaveRes; - if (conn.user) { - forceSaveRes = yield startForceSave(ctx, docId, commonDefines.c_oAscForceSaveTypes.Button, undefined, undefined, conn.user.idOriginal, conn.user.id, undefined, conn.user.indexUser); - } else { - forceSaveRes = {code: commonDefines.c_oAscServerCommandErrors.UnknownError, time: null}; + if(clientStatsD) { + if('openDocument' != data.type) { + clientStatsD.timing('coauth.data.' + data.type, new Date() - startDate); + } } - sendData(ctx, conn, {type: "forceSaveStart", messages: forceSaveRes}); - break; - case 'rpc' : - yield* startRPC(ctx, conn, data.responseKey, data.data); - break; - case 'authChangesAck' : - delete conn.authChangesAck; - break; - default: - ctx.logger.debug("unknown command %j", data); - break; - } - if(clientStatsD) { - if('openDocument' != data.type) { - clientStatsD.timing('coauth.data.' + data.type, new Date() - startDate); + } catch (e) { + ctx.logger.error("error receiving response: type = %s %s", (data && data.type) ? data.type : 'null', e.stack); } - } - } catch (e) { - ctx.logger.error("error receiving response: type = %s %s", (data && data.type) ? data.type : 'null', e.stack); - } + }); }); - }); - conn.on("disconnect", function(reason) { - return co(function* () { - let ctx = new operationContext.Context(); - try { - ctx.initFromConnection(conn); - yield ctx.initTenantCache(); - yield* closeDocument(ctx, conn, reason); - } catch (err) { - ctx.logger.error('Error conn close: %s', err.stack); - } + conn.on("disconnect", function(reason) { + return co(function* () { + let ctx = new operationContext.Context(); + try { + ctx.initFromConnection(conn); + yield ctx.initTenantCache(); + yield* closeDocument(ctx, conn, reason); + } catch (err) { + ctx.logger.error('Error conn close: %s', err.stack); + } + }); }); - }); - _checkLicense(ctx, conn); + _checkLicense(ctx, conn); + } catch(err){ + ctx.logger.error('connection error: %s', err.stack); + sendDataDisconnectReason(ctx, conn, constants.DROP_CODE, constants.DROP_REASON); + conn.disconnect(true); + } }); io.engine.on("connection_error", (err) => { operationContext.global.logger.warn('io.connection_error code=%s, message=%s', err.code, err.message); From cf9db259445b59bd9e911f0b84c17d52fe8ad2c4 Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Mon, 18 Mar 2024 11:28:20 +0300 Subject: [PATCH 08/48] [bug] Fix exception with undefined permissions in config --- DocService/sources/DocsCoServer.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/DocService/sources/DocsCoServer.js b/DocService/sources/DocsCoServer.js index 56b916333..9c870fc02 100644 --- a/DocService/sources/DocsCoServer.js +++ b/DocService/sources/DocsCoServer.js @@ -2542,7 +2542,7 @@ exports.install = function(server, callbackFunction) { const curUserId = curUserIdOriginal + curIndexUser; conn.tenant = tenantManager.getTenantByConnection(ctx, conn); conn.docId = data.docid; - conn.permissions = data.permissions; + conn.permissions = data.permissions || {}; conn.user = { id: curUserId, idOriginal: curUserIdOriginal, @@ -2988,7 +2988,7 @@ exports.install = function(server, callbackFunction) { } function* onMessage(ctx, conn, data) { - if (false === conn.permissions.chat) { + if (false === conn.permissions?.chat) { ctx.logger.warn("insert message permissions.chat==false"); return; } From 1050bb79de32778c838793b91d49eb570ae741b1 Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Thu, 21 Mar 2024 01:53:24 +0300 Subject: [PATCH 09/48] [tenant] Add services.CoAuthoring.server.editorStatStorage config param for multitenant server;For bug 66591 --- Common/config/default.json | 9 +- Common/sources/utils.js | 1 - DocService/sources/DocsCoServer.js | 114 ++++++---- DocService/sources/canvasservice.js | 4 +- DocService/sources/changes2forgotten.js | 12 +- DocService/sources/editorDataMemory.js | 287 +++++++++++------------- DocService/sources/shutdown.js | 8 +- 7 files changed, 220 insertions(+), 215 deletions(-) diff --git a/Common/config/default.json b/Common/config/default.json index c962d7a81..61ef36ee1 100644 --- a/Common/config/default.json +++ b/Common/config/default.json @@ -162,6 +162,7 @@ "openProtectedFile": true, "isAnonymousSupport": true, "editorDataStorage": "editorDataMemory", + "editorStatStorage": "", "assemblyFormatAsOrigin": true, "newFileTemplate" : "../../document-templates/new", "downloadFileAllowExt": ["pdf", "xlsx"], @@ -235,10 +236,14 @@ "port": 6379, "options": {}, "optionsCluster": {}, - "iooptions": {}, + "iooptions": { + "lazyConnect": true + }, "iooptionsClusterNodes": [ ], - "iooptionsClusterOptions": {} + "iooptionsClusterOptions": { + "lazyConnect": true + } }, "pubsub": { "maxChanges": 1000 diff --git a/Common/sources/utils.js b/Common/sources/utils.js index f5f2b0f51..ae3eab59c 100644 --- a/Common/sources/utils.js +++ b/Common/sources/utils.js @@ -51,7 +51,6 @@ const NodeCache = require( "node-cache" ); const ms = require('ms'); const constants = require('./constants'); const commonDefines = require('./commondefines'); -const logger = require('./logger'); const forwarded = require('forwarded'); const { RequestFilteringHttpAgent, RequestFilteringHttpsAgent } = require("request-filtering-agent"); const openpgp = require('openpgp'); diff --git a/DocService/sources/DocsCoServer.js b/DocService/sources/DocsCoServer.js index 9c870fc02..f834ac254 100644 --- a/DocService/sources/DocsCoServer.js +++ b/DocService/sources/DocsCoServer.js @@ -102,7 +102,10 @@ const queueService = require('./../../Common/sources/taskqueueRabbitMQ'); const operationContext = require('./../../Common/sources/operationContext'); const tenantManager = require('./../../Common/sources/tenantManager'); -const editorDataStorage = require('./' + config.get('services.CoAuthoring.server.editorDataStorage')); +const cfgEditorDataStorage = config.get('services.CoAuthoring.server.editorDataStorage'); +const cfgEditorStatStorage = config.get('services.CoAuthoring.server.editorStatStorage'); +const editorDataStorage = require('./' + cfgEditorDataStorage); +const editorStatStorage = require('./' + (cfgEditorStatStorage || cfgEditorDataStorage)); const cfgEditSingleton = config.get('services.CoAuthoring.server.edit_singleton'); const cfgEditor = config.get('services.CoAuthoring.editor'); @@ -152,7 +155,9 @@ const EditorTypes = { }; const defaultHttpPort = 80, defaultHttpsPort = 443; // Default ports (for http and https) -const editorData = new editorDataStorage(); +//todo remove editorDataStorage constructor usage after 8.1 +const editorData = editorDataStorage.EditorData ? new editorDataStorage.EditorData() : new editorDataStorage(); +const editorStat = editorStatStorage.EditorStat ? new editorStatStorage.EditorStat() : new editorDataStorage(); const clientStatsD = statsDClient.getClient(); let connections = []; // Active connections let lockDocumentsTimerId = {};//to drop connection that can't unlockDocument @@ -435,30 +440,30 @@ function updatePresenceCounters(ctx, conn, val) { //yield ctx.initTenantCache(); //no need.only global config } if (utils.isLiveViewer(conn)) { - yield editorData.incrLiveViewerConnectionsCountByShard(ctx, SHARD_ID, val); + yield editorStat.incrLiveViewerConnectionsCountByShard(ctx, SHARD_ID, val); if (aggregationCtx) { - yield editorData.incrLiveViewerConnectionsCountByShard(aggregationCtx, SHARD_ID, val); + yield editorStat.incrLiveViewerConnectionsCountByShard(aggregationCtx, SHARD_ID, val); } if (clientStatsD) { - let countLiveView = yield editorData.getLiveViewerConnectionsCount(ctx, connections); + let countLiveView = yield editorStat.getLiveViewerConnectionsCount(ctx, connections); clientStatsD.gauge('expireDoc.connections.liveview', countLiveView); } } else if (conn.isCloseCoAuthoring || (conn.user && conn.user.view)) { - yield editorData.incrViewerConnectionsCountByShard(ctx, SHARD_ID, val); + yield editorStat.incrViewerConnectionsCountByShard(ctx, SHARD_ID, val); if (aggregationCtx) { - yield editorData.incrViewerConnectionsCountByShard(aggregationCtx, SHARD_ID, val); + yield editorStat.incrViewerConnectionsCountByShard(aggregationCtx, SHARD_ID, val); } if (clientStatsD) { - let countView = yield editorData.getViewerConnectionsCount(ctx, connections); + let countView = yield editorStat.getViewerConnectionsCount(ctx, connections); clientStatsD.gauge('expireDoc.connections.view', countView); } } else { - yield editorData.incrEditorConnectionsCountByShard(ctx, SHARD_ID, val); + yield editorStat.incrEditorConnectionsCountByShard(ctx, SHARD_ID, val); if (aggregationCtx) { - yield editorData.incrEditorConnectionsCountByShard(aggregationCtx, SHARD_ID, val); + yield editorStat.incrEditorConnectionsCountByShard(aggregationCtx, SHARD_ID, val); } if (clientStatsD) { - let countEditors = yield editorData.getEditorConnectionsCount(ctx, connections); + let countEditors = yield editorStat.getEditorConnectionsCount(ctx, connections); clientStatsD.gauge('expireDoc.connections.edit', countEditors); } } @@ -625,11 +630,11 @@ function* updateEditUsers(ctx, licenseInfo, userId, anonym, isLiveViewer) { licenseInfo.usersExpire - 1; let period = utils.getLicensePeriod(licenseInfo.startDate, now); if (isLiveViewer) { - yield editorData.addPresenceUniqueViewUser(ctx, userId, expireAt, {anonym: anonym}); - yield editorData.addPresenceUniqueViewUsersOfMonth(ctx, userId, period, {anonym: anonym, firstOpenDate: now.toISOString()}); + yield editorStat.addPresenceUniqueViewUser(ctx, userId, expireAt, {anonym: anonym}); + yield editorStat.addPresenceUniqueViewUsersOfMonth(ctx, userId, period, {anonym: anonym, firstOpenDate: now.toISOString()}); } else { - yield editorData.addPresenceUniqueUser(ctx, userId, expireAt, {anonym: anonym}); - yield editorData.addPresenceUniqueUsersOfMonth(ctx, userId, period, {anonym: anonym, firstOpenDate: now.toISOString()}); + yield editorStat.addPresenceUniqueUser(ctx, userId, expireAt, {anonym: anonym}); + yield editorStat.addPresenceUniqueUsersOfMonth(ctx, userId, period, {anonym: anonym, firstOpenDate: now.toISOString()}); } } function* getEditorsCount(ctx, docId, opt_hvals) { @@ -1474,6 +1479,7 @@ function getOpenFormatByEditor(editorType) { exports.c_oAscServerStatus = c_oAscServerStatus; exports.editorData = editorData; +exports.editorStat = editorStat; exports.sendData = sendData; exports.modifyConnectionForPassword = modifyConnectionForPassword; exports.parseUrl = parseUrl; @@ -3481,13 +3487,13 @@ exports.install = function(server, callbackFunction) { if (licenseInfo.usersCount) { const nowUTC = getLicenseNowUtc(); if(isLiveViewer) { - const arrUsers = yield editorData.getPresenceUniqueViewUser(ctx, nowUTC); + const arrUsers = yield editorStat.getPresenceUniqueViewUser(ctx, nowUTC); if (arrUsers.length >= licenseInfo.usersViewCount && (-1 === arrUsers.findIndex((element) => {return element.userid === userId}))) { licenseType = c_LR.UsersViewCount; } licenseWarningLimitUsersView = licenseInfo.usersViewCount * tenWarningLimitPercents <= arrUsers.length; } else { - const arrUsers = yield editorData.getPresenceUniqueUser(ctx, nowUTC); + const arrUsers = yield editorStat.getPresenceUniqueUser(ctx, nowUTC); if (arrUsers.length >= licenseInfo.usersCount && (-1 === arrUsers.findIndex((element) => {return element.userid === userId}))) { licenseType = c_LR.UsersCount; } @@ -3495,14 +3501,14 @@ exports.install = function(server, callbackFunction) { } } else if(isLiveViewer) { const connectionsLiveCount = licenseInfo.connectionsView; - const liveViewerConnectionsCount = yield editorData.getLiveViewerConnectionsCount(ctx, connections); + const liveViewerConnectionsCount = yield editorStat.getLiveViewerConnectionsCount(ctx, connections); if (liveViewerConnectionsCount >= connectionsLiveCount) { licenseType = c_LR.ConnectionsLive; } licenseWarningLimitConnectionsLive = connectionsLiveCount * tenWarningLimitPercents <= liveViewerConnectionsCount; } else { const connectionsCount = licenseInfo.connections; - const editConnectionsCount = yield editorData.getEditorConnectionsCount(ctx, connections); + const editConnectionsCount = yield editorStat.getEditorConnectionsCount(ctx, connections); if (editConnectionsCount >= connectionsCount) { licenseType = c_LR.Connections; } @@ -3698,7 +3704,7 @@ exports.install = function(server, callbackFunction) { case commonDefines.c_oPublishType.shutdown: //flag prevent new socket connections and receive data from exist connections shutdownFlag = data.status; - ctx.logger.warn('start shutdown:%b', shutdownFlag); + ctx.logger.warn('start shutdown:%s', shutdownFlag); if (shutdownFlag) { ctx.logger.warn('active connections: %d', connections.length); //do not stop the server, because sockets and all requests will be unavailable @@ -3766,7 +3772,7 @@ exports.install = function(server, callbackFunction) { function* collectStats(ctx, countEdit, countLiveView, countView) { let now = Date.now(); - yield editorData.setEditorConnections(ctx, countEdit, countLiveView, countView, now, PRECISION); + yield editorStat.setEditorConnections(ctx, countEdit, countLiveView, countView, now, PRECISION); } function expireDoc() { return co(function* () { @@ -3843,16 +3849,16 @@ exports.install = function(server, callbackFunction) { ctx.setTenant(tenantId); let tenant = tenants[tenantId]; yield* collectStats(ctx, tenant.countEditByShard, tenant.countLiveViewByShard, tenant.countViewByShard); - yield editorData.setEditorConnectionsCountByShard(ctx, SHARD_ID, tenant.countEditByShard); - yield editorData.setLiveViewerConnectionsCountByShard(ctx, SHARD_ID, tenant.countLiveViewByShard); - yield editorData.setViewerConnectionsCountByShard(ctx, SHARD_ID, tenant.countViewByShard); + yield editorStat.setEditorConnectionsCountByShard(ctx, SHARD_ID, tenant.countEditByShard); + yield editorStat.setLiveViewerConnectionsCountByShard(ctx, SHARD_ID, tenant.countLiveViewByShard); + yield editorStat.setViewerConnectionsCountByShard(ctx, SHARD_ID, tenant.countViewByShard); if (clientStatsD) { //todo with multitenant - let countEdit = yield editorData.getEditorConnectionsCount(ctx, connections); + let countEdit = yield editorStat.getEditorConnectionsCount(ctx, connections); clientStatsD.gauge('expireDoc.connections.edit', countEdit); - let countLiveView = yield editorData.getLiveViewerConnectionsCount(ctx, connections); + let countLiveView = yield editorStat.getLiveViewerConnectionsCount(ctx, connections); clientStatsD.gauge('expireDoc.connections.liveview', countLiveView); - let countView = yield editorData.getViewerConnectionsCount(ctx, connections); + let countView = yield editorStat.getViewerConnectionsCount(ctx, connections); clientStatsD.gauge('expireDoc.connections.view', countView); } } @@ -3863,9 +3869,9 @@ exports.install = function(server, callbackFunction) { aggregationCtx.init(tenantManager.getDefautTenant(), ctx.docId, ctx.userId); //yield ctx.initTenantCache();//no need yield* collectStats(aggregationCtx, countEditByShard, countLiveViewByShard, countViewByShard); - yield editorData.setEditorConnectionsCountByShard(aggregationCtx, SHARD_ID, countEditByShard); - yield editorData.setLiveViewerConnectionsCountByShard(aggregationCtx, SHARD_ID, countLiveViewByShard); - yield editorData.setViewerConnectionsCountByShard(aggregationCtx, SHARD_ID, countViewByShard); + yield editorStat.setEditorConnectionsCountByShard(aggregationCtx, SHARD_ID, countEditByShard); + yield editorStat.setLiveViewerConnectionsCountByShard(aggregationCtx, SHARD_ID, countLiveViewByShard); + yield editorStat.setViewerConnectionsCountByShard(aggregationCtx, SHARD_ID, countViewByShard); } ctx.initDefault(); } catch (err) { @@ -3945,11 +3951,16 @@ exports.install = function(server, callbackFunction) { } }); if (-1 !== index || 0 === res.length) { - return editorData.connect().then(function() { - callbackFunction(); - }).catch(err => { - operationContext.global.logger.error('editorData error: %s', err.stack); - }); + return editorData.connect() + .then(function () { + return editorStat.connect(); + }) + .then(function () { + callbackFunction(); + }) + .catch(err => { + operationContext.global.logger.error('editorData error: %s', err.stack); + }); } else { operationContext.global.logger.error('DB table "%s" does not contain %s column, columns info: %j', tableName, tableRequiredColumn, res); } @@ -3974,13 +3985,18 @@ exports.healthCheck = function(req, res) { yield sqlBase.healthCheck(ctx); ctx.logger.debug('healthCheck database'); //check redis connection - if (editorData.isConnected()) { - yield editorData.ping(); + const healthData = yield editorData.healthCheck(); + if (healthData) { ctx.logger.debug('healthCheck editorData'); } else { - throw new Error('redis disconnected'); + throw new Error('editorData'); + } + const healthStat = yield editorStat.healthCheck(); + if (healthStat) { + ctx.logger.debug('healthCheck editorStat'); + } else { + throw new Error('editorStat'); } - const healthPubsub = yield pubsub.healthCheck(); if (healthPubsub) { ctx.logger.debug('healthCheck pubsub'); @@ -4068,7 +4084,7 @@ exports.licenseInfo = function(req, res) { view: {min: 0, avr: 0, max: 0} }; } - var redisRes = yield editorData.getEditorConnections(ctx); + var redisRes = yield editorStat.getEditorConnections(ctx); const now = Date.now(); if (redisRes.length > 0) { let expDocumentsStep95 = expDocumentsStep * 0.95; @@ -4129,8 +4145,8 @@ exports.licenseInfo = function(req, res) { } const nowUTC = getLicenseNowUtc(); let execRes; - execRes = yield editorData.getPresenceUniqueUser(ctx, nowUTC); - output.quota.edit.connectionsCount = yield editorData.getEditorConnectionsCount(ctx, connections); + execRes = yield editorStat.getPresenceUniqueUser(ctx, nowUTC); + output.quota.edit.connectionsCount = yield editorStat.getEditorConnectionsCount(ctx, connections); output.quota.edit.usersCount.unique = execRes.length; execRes.forEach(function(elem) { if (elem.anonym) { @@ -4138,8 +4154,8 @@ exports.licenseInfo = function(req, res) { } }); - execRes = yield editorData.getPresenceUniqueViewUser(ctx, nowUTC); - output.quota.view.connectionsCount = yield editorData.getLiveViewerConnectionsCount(ctx, connections); + execRes = yield editorStat.getPresenceUniqueViewUser(ctx, nowUTC); + output.quota.view.connectionsCount = yield editorStat.getLiveViewerConnectionsCount(ctx, connections); output.quota.view.usersCount.unique = execRes.length; execRes.forEach(function(elem) { if (elem.anonym) { @@ -4147,8 +4163,8 @@ exports.licenseInfo = function(req, res) { } }); - let byMonth = yield editorData.getPresenceUniqueUsersOfMonth(ctx); - let byMonthView = yield editorData.getPresenceUniqueViewUsersOfMonth(ctx); + let byMonth = yield editorStat.getPresenceUniqueUsersOfMonth(ctx); + let byMonthView = yield editorStat.getPresenceUniqueViewUsersOfMonth(ctx); let byMonthMerged = []; for (let i in byMonth) { if (byMonth.hasOwnProperty(i)) { @@ -4230,8 +4246,8 @@ function* findForgottenFile(ctx, docId) { function* commandLicense(ctx) { const nowUTC = getLicenseNowUtc(); - const users = yield editorData.getPresenceUniqueUser(ctx, nowUTC); - const users_view = yield editorData.getPresenceUniqueViewUser(ctx, nowUTC); + const users = yield editorStat.getPresenceUniqueUser(ctx, nowUTC); + const users_view = yield editorStat.getPresenceUniqueViewUser(ctx, nowUTC); const licenseInfo = yield tenantManager.getTenantLicense(ctx); return { @@ -4389,7 +4405,7 @@ exports.shutdown = function(req, res) { ctx.initFromRequest(req); yield ctx.initTenantCache(); ctx.logger.info('shutdown start'); - output = yield shutdown.shutdown(ctx, editorData, req.method === 'PUT'); + output = yield shutdown.shutdown(ctx, editorStat, req.method === 'PUT'); } catch (err) { ctx.logger.error('shutdown error %s', err.stack); } finally { diff --git a/DocService/sources/canvasservice.js b/DocService/sources/canvasservice.js index cd700d424..755a7aae6 100644 --- a/DocService/sources/canvasservice.js +++ b/DocService/sources/canvasservice.js @@ -1194,7 +1194,7 @@ const commandSfcCallback = co.wrap(function*(ctx, cmd, isSfcm, isEncrypted) { if ((docsCoServer.getIsShutdown() && !isSfcm) || cmd.getRedisKey()) { let keyRedis = cmd.getRedisKey() ? cmd.getRedisKey() : redisKeyShutdown; - yield docsCoServer.editorData.removeShutdown(keyRedis, docId); + yield docsCoServer.editorStat.removeShutdown(keyRedis, docId); } ctx.logger.debug('End commandSfcCallback'); return replyStr; @@ -1720,7 +1720,7 @@ exports.saveFromChanges = function(ctx, docId, statusInfo, optFormat, opt_userId queueData.setFromChanges(true); yield* docsCoServer.addTask(queueData, constants.QUEUE_PRIORITY_NORMAL, opt_queue); if (docsCoServer.getIsShutdown()) { - yield docsCoServer.editorData.addShutdown(redisKeyShutdown, docId); + yield docsCoServer.editorStat.addShutdown(redisKeyShutdown, docId); } ctx.logger.debug('AddTask saveFromChanges'); } else { diff --git a/DocService/sources/changes2forgotten.js b/DocService/sources/changes2forgotten.js index 99fb01aa3..ff9e08b2e 100644 --- a/DocService/sources/changes2forgotten.js +++ b/DocService/sources/changes2forgotten.js @@ -44,7 +44,9 @@ const operationContext = require('./../../Common/sources/operationContext'); const sqlBase = require('./databaseConnectors/baseConnector'); const docsCoServer = require('./DocsCoServer'); const taskResult = require('./taskresult'); -const editorDataStorage = require('./' + config.get('services.CoAuthoring.server.editorDataStorage')); +const cfgEditorDataStorage = config.get('services.CoAuthoring.server.editorDataStorage'); +const cfgEditorStatStorage = config.get('services.CoAuthoring.server.editorStatStorage'); +const editorStatStorage = require('./' + (cfgEditorStatStorage || cfgEditorDataStorage)); const cfgForgottenFiles = config.get('services.CoAuthoring.server.forgottenfiles'); const cfgTableResult = config.get('services.CoAuthoring.sql.tableResult'); @@ -81,7 +83,7 @@ function shutdown() { var res = true; let ctx = new operationContext.Context(); try { - let editorData = new editorDataStorage(); + let editorStat = editorStatStorage.EditorStat ? new editorStatStorage.EditorStat() : new editorStatStorage(); ctx.logger.debug('shutdown start:' + EXEC_TIMEOUT); //redisKeyShutdown is not a simple counter, so it doesn't get decremented by a build that started before Shutdown started @@ -130,7 +132,7 @@ function shutdown() { yield ctx.initTenantCache(); yield updateDoc(ctx, docId, commonDefines.FileStatus.Ok, ""); - yield editorData.addShutdown(redisKeyShutdown, docId); + yield editorStat.addShutdown(redisKeyShutdown, docId); ctx.logger.debug('shutdown createSaveTimerPromise %s', docId); yield docsCoServer.createSaveTimer(ctx, docId, null, null, queue, true); } @@ -140,7 +142,7 @@ function shutdown() { let startTime = new Date().getTime(); while (true) { - let remainingFiles = yield editorData.getShutdownCount(redisKeyShutdown); + let remainingFiles = yield editorStat.getShutdownCount(redisKeyShutdown); ctx.logger.debug('shutdown remaining files:%d', remainingFiles); let curTime = new Date().getTime() - startTime; if (curTime >= EXEC_TIMEOUT || remainingFiles <= 0) { @@ -169,7 +171,7 @@ function shutdown() { //todo needs to check queues, because there may be long conversions running before Shutdown //clean up - yield editorData.cleanupShutdown(redisKeyShutdown); + yield editorStat.cleanupShutdown(redisKeyShutdown); yield pubsub.close(); yield queue.close(); diff --git a/DocService/sources/editorDataMemory.js b/DocService/sources/editorDataMemory.js index 0951c40b4..b6f3ac56f 100644 --- a/DocService/sources/editorDataMemory.js +++ b/DocService/sources/editorDataMemory.js @@ -39,19 +39,29 @@ const tenantManager = require('./../../Common/sources/tenantManager'); const cfgExpMonthUniqueUsers = ms(config.get('services.CoAuthoring.expire.monthUniqueUsers')); +function EditorCommon() { +} +EditorCommon.prototype.connect = async function () {}; +EditorCommon.prototype.isConnected = function() { + return true; +}; +EditorCommon.prototype.ping = async function() {return "PONG"}; +EditorCommon.prototype.close = async function() {}; +EditorCommon.prototype.healthCheck = async function() { + if (this.isConnected()) { + await this.ping(); + return true; + } + return false; +}; + function EditorData() { + EditorCommon.call(this); this.data = {}; this.forceSaveTimer = {}; - this.uniqueUser = {}; - this.uniqueUsersOfMonth = {}; - this.uniqueViewUser = {}; - this.uniqueViewUsersOfMonth = {}; - this.shutdown = {}; - this.stat = {}; } -EditorData.prototype.connect = function() { - return Promise.resolve(); -}; +EditorData.prototype = Object.create(EditorCommon.prototype); +EditorData.prototype.constructor = EditorData; EditorData.prototype._getDocumentData = function(ctx, docId) { let tenantData = this.data[ctx.tenant]; if (!tenantData) { @@ -73,7 +83,7 @@ EditorData.prototype._checkAndLock = function(ctx, name, docId, fencingToken, tt const expireAt = now + ttl * 1000; data[name] = {fencingToken: fencingToken, expireAt: expireAt}; } - return Promise.resolve(res); + return res; }; EditorData.prototype._checkAndUnlock = function(ctx, name, docId, fencingToken) { let data = this._getDocumentData(ctx, docId); @@ -90,106 +100,94 @@ EditorData.prototype._checkAndUnlock = function(ctx, name, docId, fencingToken) res = commonDefines.c_oAscUnlockRes.Empty; delete data[name]; } - return Promise.resolve(res); + return res; }; -EditorData.prototype.addPresence = function(ctx, docId, userId, userInfo) { - return Promise.resolve(); -}; -EditorData.prototype.updatePresence = function(ctx, docId, userId) { - return Promise.resolve(); -}; -EditorData.prototype.removePresence = function(ctx, docId, userId) { - return Promise.resolve(); -}; -EditorData.prototype.getPresence = function(ctx, docId, connections) { +EditorData.prototype.addPresence = async function(ctx, docId, userId, userInfo) {}; +EditorData.prototype.updatePresence = async function(ctx, docId, userId) {}; +EditorData.prototype.removePresence = async function(ctx, docId, userId) {}; +EditorData.prototype.getPresence = async function(ctx, docId, connections) { let hvals = []; - for (let i = 0; i < connections.length; ++i) { - let conn = connections[i]; - if (conn.docId === docId && ctx.tenant === tenantManager.getTenantByConnection(ctx, conn)) { - hvals.push(utils.getConnectionInfoStr(conn)); + if (connections) { + for (let i = 0; i < connections.length; ++i) { + let conn = connections[i]; + if (conn.docId === docId && ctx.tenant === tenantManager.getTenantByConnection(ctx, conn)) { + hvals.push(utils.getConnectionInfoStr(conn)); + } } } - return Promise.resolve(hvals); + return hvals; }; -EditorData.prototype.lockSave = function(ctx, docId, userId, ttl) { +EditorData.prototype.lockSave = async function(ctx, docId, userId, ttl) { return this._checkAndLock(ctx, 'lockSave', docId, userId, ttl); }; -EditorData.prototype.unlockSave = function(ctx, docId, userId) { +EditorData.prototype.unlockSave = async function(ctx, docId, userId) { return this._checkAndUnlock(ctx, 'lockSave', docId, userId); }; -EditorData.prototype.lockAuth = function(ctx, docId, userId, ttl) { +EditorData.prototype.lockAuth = async function(ctx, docId, userId, ttl) { return this._checkAndLock(ctx, 'lockAuth', docId, userId, ttl); }; -EditorData.prototype.unlockAuth = function(ctx, docId, userId) { +EditorData.prototype.unlockAuth = async function(ctx, docId, userId) { return this._checkAndUnlock(ctx, 'lockAuth', docId, userId); }; -EditorData.prototype.getDocumentPresenceExpired = function(now) { - return Promise.resolve([]); -}; -EditorData.prototype.removePresenceDocument = function(ctx, docId) { - return Promise.resolve(); +EditorData.prototype.getDocumentPresenceExpired = async function(now) { + return []; }; +EditorData.prototype.removePresenceDocument = async function(ctx, docId) {}; -EditorData.prototype.addLocks = function(ctx, docId, locks) { +EditorData.prototype.addLocks = async function(ctx, docId, locks) { let data = this._getDocumentData(ctx, docId); if (!data.locks) { data.locks = []; } data.locks = data.locks.concat(locks); - return Promise.resolve(); }; -EditorData.prototype.removeLocks = function(ctx, docId) { +EditorData.prototype.removeLocks = async function(ctx, docId) { let data = this._getDocumentData(ctx, docId); data.locks = undefined; - return Promise.resolve(); }; -EditorData.prototype.getLocks = function(ctx, docId) { +EditorData.prototype.getLocks = async function(ctx, docId) { let data = this._getDocumentData(ctx, docId); - return Promise.resolve(data.locks || []); + return data.locks || []; }; -EditorData.prototype.addMessage = function(ctx, docId, msg) { +EditorData.prototype.addMessage = async function(ctx, docId, msg) { let data = this._getDocumentData(ctx, docId); if (!data.messages) { data.messages = []; } data.messages.push(msg); - return Promise.resolve(); }; -EditorData.prototype.removeMessages = function(ctx, docId) { +EditorData.prototype.removeMessages = async function(ctx, docId) { let data = this._getDocumentData(ctx, docId); data.messages = undefined; - return Promise.resolve(); }; -EditorData.prototype.getMessages = function(ctx, docId) { +EditorData.prototype.getMessages = async function(ctx, docId) { let data = this._getDocumentData(ctx, docId); - return Promise.resolve(data.messages || []); + return data.messages || []; }; -EditorData.prototype.setSaved = function(ctx, docId, status) { +EditorData.prototype.setSaved = async function(ctx, docId, status) { let data = this._getDocumentData(ctx, docId); data.saved = status; - return Promise.resolve(); }; -EditorData.prototype.getdelSaved = function(ctx, docId) { +EditorData.prototype.getdelSaved = async function(ctx, docId) { let data = this._getDocumentData(ctx, docId); let res = data.saved; - data.saved = undefined; - return Promise.resolve(res); + data.saved = null; + return res; }; -EditorData.prototype.setForceSave = function(ctx, docId, time, index, baseUrl, changeInfo, convertInfo) { +EditorData.prototype.setForceSave = async function(ctx, docId, time, index, baseUrl, changeInfo, convertInfo) { let data = this._getDocumentData(ctx, docId); data.forceSave = {time, index, baseUrl, changeInfo, started: false, ended: false, convertInfo}; - return Promise.resolve(); }; -EditorData.prototype.getForceSave = function(ctx, docId) { +EditorData.prototype.getForceSave = async function(ctx, docId) { let data = this._getDocumentData(ctx, docId); - return Promise.resolve(data.forceSave || null); + return data.forceSave || null; }; -EditorData.prototype.checkAndStartForceSave = function(ctx, docId) { +EditorData.prototype.checkAndStartForceSave = async function(ctx, docId) { let data = this._getDocumentData(ctx, docId); let res; if (data.forceSave && !data.forceSave.started) { @@ -197,9 +195,9 @@ EditorData.prototype.checkAndStartForceSave = function(ctx, docId) { data.forceSave.ended = false; res = data.forceSave; } - return Promise.resolve(res); + return res; }; -EditorData.prototype.checkAndSetForceSave = function(ctx, docId, time, index, started, ended, convertInfo) { +EditorData.prototype.checkAndSetForceSave = async function(ctx, docId, time, index, started, ended, convertInfo) { let data = this._getDocumentData(ctx, docId); let res; if (data.forceSave && time === data.forceSave.time && index === data.forceSave.index) { @@ -208,15 +206,14 @@ EditorData.prototype.checkAndSetForceSave = function(ctx, docId, time, index, st data.forceSave.convertInfo = convertInfo; res = data.forceSave; } - return Promise.resolve(res); + return res; }; -EditorData.prototype.removeForceSave = function(ctx, docId) { +EditorData.prototype.removeForceSave = async function(ctx, docId) { let data = this._getDocumentData(ctx, docId); data.forceSave = undefined; - return Promise.resolve(); }; -EditorData.prototype.cleanDocumentOnExit = function(ctx, docId) { +EditorData.prototype.cleanDocumentOnExit = async function(ctx, docId) { let tenantData = this.data[ctx.tenant]; if (tenantData) { delete tenantData[docId]; @@ -225,10 +222,9 @@ EditorData.prototype.cleanDocumentOnExit = function(ctx, docId) { if (tenantTimer) { delete tenantTimer[docId]; } - return Promise.resolve(); }; -EditorData.prototype.addForceSaveTimerNX = function(ctx, docId, expireAt) { +EditorData.prototype.addForceSaveTimerNX = async function(ctx, docId, expireAt) { let tenantTimer = this.forceSaveTimer[ctx.tenant]; if (!tenantTimer) { this.forceSaveTimer[ctx.tenant] = tenantTimer = {}; @@ -236,9 +232,8 @@ EditorData.prototype.addForceSaveTimerNX = function(ctx, docId, expireAt) { if (!tenantTimer[docId]) { tenantTimer[docId] = expireAt; } - return Promise.resolve(); }; -EditorData.prototype.getForceSaveTimer = function(now) { +EditorData.prototype.getForceSaveTimer = async function(now) { let res = []; for (let tenant in this.forceSaveTimer) { if (this.forceSaveTimer.hasOwnProperty(tenant)) { @@ -253,18 +248,29 @@ EditorData.prototype.getForceSaveTimer = function(now) { } } } - return Promise.resolve(res); + return res; }; -EditorData.prototype.addPresenceUniqueUser = function(ctx, userId, expireAt, userInfo) { +function EditorStat() { + EditorCommon.call(this); + this.uniqueUser = {}; + this.uniqueUsersOfMonth = {}; + this.uniqueViewUser = {}; + this.uniqueViewUsersOfMonth = {}; + this.stat = {}; + this.shutdown = {}; + this.license = {}; +} +EditorStat.prototype = Object.create(EditorCommon.prototype); +EditorStat.prototype.constructor = EditorStat; +EditorStat.prototype.addPresenceUniqueUser = async function(ctx, userId, expireAt, userInfo) { let tenantUser = this.uniqueUser[ctx.tenant]; if (!tenantUser) { this.uniqueUser[ctx.tenant] = tenantUser = {}; } tenantUser[userId] = {expireAt: expireAt, userInfo: userInfo}; - return Promise.resolve(); }; -EditorData.prototype.getPresenceUniqueUser = function(ctx, nowUTC) { +EditorStat.prototype.getPresenceUniqueUser = async function(ctx, nowUTC) { let res = []; let tenantUser = this.uniqueUser[ctx.tenant]; if (!tenantUser) { @@ -282,9 +288,9 @@ EditorData.prototype.getPresenceUniqueUser = function(ctx, nowUTC) { } } } - return Promise.resolve(res); + return res; }; -EditorData.prototype.addPresenceUniqueUsersOfMonth = function(ctx, userId, period, userInfo) { +EditorStat.prototype.addPresenceUniqueUsersOfMonth = async function(ctx, userId, period, userInfo) { let tenantUser = this.uniqueUsersOfMonth[ctx.tenant]; if (!tenantUser) { this.uniqueUsersOfMonth[ctx.tenant] = tenantUser = {}; @@ -294,9 +300,8 @@ EditorData.prototype.addPresenceUniqueUsersOfMonth = function(ctx, userId, perio tenantUser[period] = {expireAt: expireAt, data: {}}; } tenantUser[period].data[userId] = userInfo; - return Promise.resolve(); }; -EditorData.prototype.getPresenceUniqueUsersOfMonth = function(ctx) { +EditorStat.prototype.getPresenceUniqueUsersOfMonth = async function(ctx) { let res = {}; let nowUTC = Date.now(); let tenantUser = this.uniqueUsersOfMonth[ctx.tenant]; @@ -313,18 +318,17 @@ EditorData.prototype.getPresenceUniqueUsersOfMonth = function(ctx) { } } } - return Promise.resolve(res); + return res; }; -EditorData.prototype.addPresenceUniqueViewUser = function(ctx, userId, expireAt, userInfo) { +EditorStat.prototype.addPresenceUniqueViewUser = async function(ctx, userId, expireAt, userInfo) { let tenantUser = this.uniqueViewUser[ctx.tenant]; if (!tenantUser) { this.uniqueViewUser[ctx.tenant] = tenantUser = {}; } tenantUser[userId] = {expireAt: expireAt, userInfo: userInfo}; - return Promise.resolve(); }; -EditorData.prototype.getPresenceUniqueViewUser = function(ctx, nowUTC) { +EditorStat.prototype.getPresenceUniqueViewUser = async function(ctx, nowUTC) { let res = []; let tenantUser = this.uniqueViewUser[ctx.tenant]; if (!tenantUser) { @@ -342,9 +346,9 @@ EditorData.prototype.getPresenceUniqueViewUser = function(ctx, nowUTC) { } } } - return Promise.resolve(res); + return res; }; -EditorData.prototype.addPresenceUniqueViewUsersOfMonth = function(ctx, userId, period, userInfo) { +EditorStat.prototype.addPresenceUniqueViewUsersOfMonth = async function(ctx, userId, period, userInfo) { let tenantUser = this.uniqueViewUsersOfMonth[ctx.tenant]; if (!tenantUser) { this.uniqueViewUsersOfMonth[ctx.tenant] = tenantUser = {}; @@ -354,9 +358,8 @@ EditorData.prototype.addPresenceUniqueViewUsersOfMonth = function(ctx, userId, p tenantUser[period] = {expireAt: expireAt, data: {}}; } tenantUser[period].data[userId] = userInfo; - return Promise.resolve(); }; -EditorData.prototype.getPresenceUniqueViewUsersOfMonth = function(ctx) { +EditorStat.prototype.getPresenceUniqueViewUsersOfMonth = async function(ctx) { let res = {}; let nowUTC = Date.now(); let tenantUser = this.uniqueViewUsersOfMonth[ctx.tenant]; @@ -373,10 +376,9 @@ EditorData.prototype.getPresenceUniqueViewUsersOfMonth = function(ctx) { } } } - return Promise.resolve(res); + return res; }; - -EditorData.prototype.setEditorConnections = function(ctx, countEdit, countLiveView, countView, now, precision) { +EditorStat.prototype.setEditorConnections = async function(ctx, countEdit, countLiveView, countView, now, precision) { let tenantStat = this.stat[ctx.tenant]; if (!tenantStat) { this.stat[ctx.tenant] = tenantStat = []; @@ -387,79 +389,69 @@ EditorData.prototype.setEditorConnections = function(ctx, countEdit, countLiveVi i++; } tenantStat.splice(0, i); - return Promise.resolve(); }; -EditorData.prototype.getEditorConnections = function(ctx) { +EditorStat.prototype.getEditorConnections = async function(ctx) { let tenantStat = this.stat[ctx.tenant]; if (!tenantStat) { this.stat[ctx.tenant] = tenantStat = []; } - return Promise.resolve(tenantStat); + return tenantStat; }; -EditorData.prototype.setEditorConnectionsCountByShard = function(ctx, shardId, count) { - return Promise.resolve(); -}; -EditorData.prototype.incrEditorConnectionsCountByShard = function(ctx, shardId, count) { - return Promise.resolve(); -}; -EditorData.prototype.getEditorConnectionsCount = function(ctx, connections) { +EditorStat.prototype.setEditorConnectionsCountByShard = async function(ctx, shardId, count) {}; +EditorStat.prototype.incrEditorConnectionsCountByShard = async function(ctx, shardId, count) {}; +EditorStat.prototype.getEditorConnectionsCount = async function(ctx, connections) { let count = 0; - for (let i = 0; i < connections.length; ++i) { - let conn = connections[i]; - if (!(conn.isCloseCoAuthoring || (conn.user && conn.user.view)) && ctx.tenant === tenantManager.getTenantByConnection(ctx, conn)) { - count++; + if (connections) { + for (let i = 0; i < connections.length; ++i) { + let conn = connections[i]; + if (!(conn.isCloseCoAuthoring || (conn.user && conn.user.view)) && ctx.tenant === tenantManager.getTenantByConnection(ctx, conn)) { + count++; + } } } - return Promise.resolve(count); -}; -EditorData.prototype.setViewerConnectionsCountByShard = function(ctx, shardId, count) { - return Promise.resolve(); -}; -EditorData.prototype.incrViewerConnectionsCountByShard = function(ctx, shardId, count) { - return Promise.resolve(); + return count; }; -EditorData.prototype.getViewerConnectionsCount = function(ctx, connections) { +EditorStat.prototype.setViewerConnectionsCountByShard = async function(ctx, shardId, count) {}; +EditorStat.prototype.incrViewerConnectionsCountByShard = async function(ctx, shardId, count) {}; +EditorStat.prototype.getViewerConnectionsCount = async function(ctx, connections) { let count = 0; - for (let i = 0; i < connections.length; ++i) { - let conn = connections[i]; - if (conn.isCloseCoAuthoring || (conn.user && conn.user.view) && ctx.tenant === tenantManager.getTenantByConnection(ctx, conn)) { - count++; + if (connections) { + for (let i = 0; i < connections.length; ++i) { + let conn = connections[i]; + if (conn.isCloseCoAuthoring || (conn.user && conn.user.view) && ctx.tenant === tenantManager.getTenantByConnection(ctx, conn)) { + count++; + } } } - return Promise.resolve(count); -}; -EditorData.prototype.setLiveViewerConnectionsCountByShard = function(ctx, shardId, count) { - return Promise.resolve(); + return count; }; -EditorData.prototype.incrLiveViewerConnectionsCountByShard = function(ctx, shardId, count) { - return Promise.resolve(); -}; -EditorData.prototype.getLiveViewerConnectionsCount = function(ctx, connections) { +EditorStat.prototype.setLiveViewerConnectionsCountByShard = async function(ctx, shardId, count) {}; +EditorStat.prototype.incrLiveViewerConnectionsCountByShard = async function(ctx, shardId, count) {}; +EditorStat.prototype.getLiveViewerConnectionsCount = async function(ctx, connections) { let count = 0; - for (let i = 0; i < connections.length; ++i) { - let conn = connections[i]; - if (utils.isLiveViewer(conn) && ctx.tenant === tenantManager.getTenantByConnection(ctx, conn)) { - count++; + if (connections) { + for (let i = 0; i < connections.length; ++i) { + let conn = connections[i]; + if (utils.isLiveViewer(conn) && ctx.tenant === tenantManager.getTenantByConnection(ctx, conn)) { + count++; + } } } - return Promise.resolve(count); + return count; }; - -EditorData.prototype.addShutdown = function(key, docId) { +EditorStat.prototype.addShutdown = async function(key, docId) { if (!this.shutdown[key]) { this.shutdown[key] = {}; } this.shutdown[key][docId] = 1; - return Promise.resolve(); }; -EditorData.prototype.removeShutdown = function(key, docId) { +EditorStat.prototype.removeShutdown = async function(key, docId) { if (!this.shutdown[key]) { this.shutdown[key] = {}; } delete this.shutdown[key][docId]; - return Promise.resolve(); }; -EditorData.prototype.getShutdownCount = function(key) { +EditorStat.prototype.getShutdownCount = async function(key) { let count = 0; if (this.shutdown[key]) { for (let docId in this.shutdown[key]) { @@ -468,31 +460,22 @@ EditorData.prototype.getShutdownCount = function(key) { } } } - return Promise.resolve(count); + return count; }; -EditorData.prototype.cleanupShutdown = function(key) { +EditorStat.prototype.cleanupShutdown = async function(key) { delete this.shutdown[key]; - return Promise.resolve(); }; - -EditorData.prototype.setLicense = function(key, val) { - return Promise.resolve(); +EditorStat.prototype.setLicense = async function(key, val) { + this.license[key] = val; }; -EditorData.prototype.getLicense = function(key) { - return Promise.resolve(null); +EditorStat.prototype.getLicense = async function(key) { + return this.license[key] || null; }; -EditorData.prototype.removeLicense = function(key) { - return Promise.resolve(); +EditorStat.prototype.removeLicense = async function(key) { + delete this.license[key]; }; -EditorData.prototype.isConnected = function() { - return true; -}; -EditorData.prototype.ping = function() { - return Promise.resolve(); -}; -EditorData.prototype.close = function() { - return Promise.resolve(); -}; - -module.exports = EditorData; +module.exports = { + EditorData, + EditorStat +} diff --git a/DocService/sources/shutdown.js b/DocService/sources/shutdown.js index edd92320a..d85e9c1db 100644 --- a/DocService/sources/shutdown.js +++ b/DocService/sources/shutdown.js @@ -47,7 +47,7 @@ var WAIT_TIMEOUT = 30000; var LOOP_TIMEOUT = 1000; var EXEC_TIMEOUT = WAIT_TIMEOUT + utils.getConvertionTimeout(undefined); -exports.shutdown = function(ctx, editorData, status) { +exports.shutdown = function(ctx, editorStat, status) { return co(function*() { var res = true; try { @@ -55,7 +55,7 @@ exports.shutdown = function(ctx, editorData, status) { //redisKeyShutdown is not a simple counter, so it doesn't get decremented by a build that started before Shutdown started //reset redisKeyShutdown just in case the previous run didn't finish - yield editorData.cleanupShutdown(redisKeyShutdown); + yield editorStat.cleanupShutdown(redisKeyShutdown); var pubsub = new pubsubService(); yield pubsub.initPromise(); @@ -76,7 +76,7 @@ exports.shutdown = function(ctx, editorData, status) { ctx.logger.debug('shutdown timeout'); break; } - var remainingFiles = yield editorData.getShutdownCount(redisKeyShutdown); + var remainingFiles = yield editorStat.getShutdownCount(redisKeyShutdown); ctx.logger.debug('shutdown remaining files:%d', remainingFiles); if (!isStartWait && remainingFiles <= 0) { break; @@ -85,7 +85,7 @@ exports.shutdown = function(ctx, editorData, status) { } //todo need to check the queues, because there may be long conversions running before Shutdown //clean up - yield editorData.cleanupShutdown(redisKeyShutdown); + yield editorStat.cleanupShutdown(redisKeyShutdown); yield pubsub.close(); ctx.logger.debug('shutdown end'); From f02884bea84d3df11b671571e101ef3e20d6dfac Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Tue, 26 Mar 2024 01:46:32 +0300 Subject: [PATCH 10/48] [bug] Fix bug with opening error after forcesave on forgotten file --- DocService/sources/DocsCoServer.js | 13 ++++++------- DocService/sources/canvasservice.js | 5 ++++- DocService/sources/converterservice.js | 5 ++++- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/DocService/sources/DocsCoServer.js b/DocService/sources/DocsCoServer.js index f834ac254..586d6108f 100644 --- a/DocService/sources/DocsCoServer.js +++ b/DocService/sources/DocsCoServer.js @@ -962,13 +962,6 @@ async function startForceSave(ctx, docId, type, opt_userdata, opt_formdata, opt_ forceSave.setAuthorUserId(opt_userId); forceSave.setAuthorUserIndex(opt_userIndex); - if (commonDefines.c_oAscForceSaveTypes.Timeout === type) { - await co(publish(ctx, { - type: commonDefines.c_oPublishType.forceSave, ctx: ctx, docId: docId, - data: {type: type, time: forceSave.getTime(), start: true} - }, undefined, undefined, opt_pubsub)); - } - let priority; let expiration; if (commonDefines.c_oAscForceSaveTypes.Timeout === type) { @@ -983,6 +976,12 @@ async function startForceSave(ctx, docId, type, opt_userdata, opt_formdata, opt_ opt_queue, undefined, opt_initShardKey); if (constants.NO_ERROR === status.err) { res.time = forceSave.getTime(); + if (commonDefines.c_oAscForceSaveTypes.Timeout === type) { + await co(publish(ctx, { + type: commonDefines.c_oPublishType.forceSave, ctx: ctx, docId: docId, + data: {type: type, time: forceSave.getTime(), start: true} + }, undefined, undefined, opt_pubsub)); + } } else { res.code = commonDefines.c_oAscServerCommandErrors.UnknownError; } diff --git a/DocService/sources/canvasservice.js b/DocService/sources/canvasservice.js index 755a7aae6..8683f0384 100644 --- a/DocService/sources/canvasservice.js +++ b/DocService/sources/canvasservice.js @@ -616,7 +616,7 @@ let commandSfctByCmd = co.wrap(function*(ctx, cmd, opt_priority, opt_expiration, var selectRes = yield taskResult.select(ctx, cmd.getDocId()); var row = selectRes.length > 0 ? selectRes[0] : null; if (!row) { - return; + return false; } if (opt_initShardKey) { ctx.setShardKey(sqlBase.DocumentAdditional.prototype.getShardKey(row.additional)); @@ -632,6 +632,7 @@ let commandSfctByCmd = co.wrap(function*(ctx, cmd, opt_priority, opt_expiration, queueData.setFromChanges(true); let priority = null != opt_priority ? opt_priority : constants.QUEUE_PRIORITY_LOW; yield* docsCoServer.addTask(queueData, priority, opt_queue, opt_expiration); + return true; }); function isDisplayedImage(strName) { var res = 0; @@ -1167,6 +1168,8 @@ const commandSfcCallback = co.wrap(function*(ctx, cmd, isSfcm, isEncrypted) { } if (!isSfcm) { //todo simultaneous opening + //clean redis (redisKeyPresenceSet and redisKeyPresenceHash removed with last element) + yield docsCoServer.editorData.cleanDocumentOnExit(ctx, docId); //to unlock wopi file yield docsCoServer.unlockWopiDoc(ctx, docId, callbackUserIndex); //cleanupRes can be false in case of simultaneous opening. it is OK diff --git a/DocService/sources/converterservice.js b/DocService/sources/converterservice.js index 2dcbe0f62..e1cde9663 100644 --- a/DocService/sources/converterservice.js +++ b/DocService/sources/converterservice.js @@ -213,7 +213,10 @@ async function convertFromChanges(ctx, docId, baseUrl, forceSave, externalChange cmd.setRedisKey(opt_redisKey); } - await canvasService.commandSfctByCmd(ctx, cmd, opt_priority, opt_expiration, opt_queue, opt_initShardKey); + let commandSfctByCmdRes = await canvasService.commandSfctByCmd(ctx, cmd, opt_priority, opt_expiration, opt_queue, opt_initShardKey); + if (!commandSfctByCmdRes) { + return new commonDefines.ConvertStatus(constants.UNKNOWN); + } var fileTo = constants.OUTPUT_NAME; let outputExt = formatChecker.getStringFromFormat(cmd.getOutputFormat()); if (outputExt) { From e6c59a4a25e34e7a543339621fc1899c24f57202 Mon Sep 17 00:00:00 2001 From: Georgii Petrov Date: Thu, 28 Mar 2024 14:29:08 +0300 Subject: [PATCH 11/48] [fix] Git actions now ignoring tags --- .github/workflows/damengDatabaseTests.yml | 4 +++- .github/workflows/mssqlDatabaseTests.yml | 4 +++- .github/workflows/mysqlDatabaseTests.yml | 4 +++- .github/workflows/oracleDatabaseTests.yml | 4 +++- .github/workflows/postgreDatabaseTests.yml | 4 +++- .github/workflows/unitTests.yml | 4 +++- 6 files changed, 18 insertions(+), 6 deletions(-) diff --git a/.github/workflows/damengDatabaseTests.yml b/.github/workflows/damengDatabaseTests.yml index e9ac6aa27..ed465672c 100644 --- a/.github/workflows/damengDatabaseTests.yml +++ b/.github/workflows/damengDatabaseTests.yml @@ -1,6 +1,8 @@ name: Dameng database tests on: push: + branches: + - '**' paths: - 'DocService/sources/databaseConnectors/baseConnector.js' - 'DocService/sources/databaseConnectors/damengConnector.js' @@ -45,4 +47,4 @@ jobs: docker exec dameng bash -c "cat /createdb.sql | /opt/dmdbms/bin/disql SYSDBA/SYSDBA001:5236" - name: Run Jest - run: npm run "integration database tests" \ No newline at end of file + run: npm run "integration database tests" diff --git a/.github/workflows/mssqlDatabaseTests.yml b/.github/workflows/mssqlDatabaseTests.yml index 95a311f85..0529401ff 100644 --- a/.github/workflows/mssqlDatabaseTests.yml +++ b/.github/workflows/mssqlDatabaseTests.yml @@ -1,6 +1,8 @@ name: MSSQL database tests on: push: + branches: + - '**' paths: - 'DocService/sources/databaseConnectors/baseConnector.js' - 'DocService/sources/databaseConnectors/mssqlConnector.js' @@ -45,4 +47,4 @@ jobs: docker exec mssql /opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P onlYoff1ce -i /createdb.sql - name: Run Jest - run: npm run "integration database tests" \ No newline at end of file + run: npm run "integration database tests" diff --git a/.github/workflows/mysqlDatabaseTests.yml b/.github/workflows/mysqlDatabaseTests.yml index e84418e65..3d4e8f188 100644 --- a/.github/workflows/mysqlDatabaseTests.yml +++ b/.github/workflows/mysqlDatabaseTests.yml @@ -1,6 +1,8 @@ name: MYSQL database tests on: push: + branches: + - '**' paths: - 'DocService/sources/databaseConnectors/baseConnector.js' - 'DocService/sources/databaseConnectors/mysqlConnector.js' @@ -42,4 +44,4 @@ jobs: docker exec mysql mysql -h 127.0.0.1 -u root --password=onlyoffice -D onlyoffice -e 'source /createdb.sql' - name: Run Jest - run: npm run "integration database tests" \ No newline at end of file + run: npm run "integration database tests" diff --git a/.github/workflows/oracleDatabaseTests.yml b/.github/workflows/oracleDatabaseTests.yml index 27b65d075..be6a37a47 100644 --- a/.github/workflows/oracleDatabaseTests.yml +++ b/.github/workflows/oracleDatabaseTests.yml @@ -1,6 +1,8 @@ name: Oracle database tests on: push: + branches: + - '**' paths: - 'DocService/sources/databaseConnectors/baseConnector.js' - 'DocService/sources/databaseConnectors/oracleConnector.js' @@ -45,4 +47,4 @@ jobs: docker exec oracle sqlplus -s onlyoffice/onlyoffice@//localhost/xepdb1 @/createdb.sql - name: Run Jest - run: npm run "integration database tests" \ No newline at end of file + run: npm run "integration database tests" diff --git a/.github/workflows/postgreDatabaseTests.yml b/.github/workflows/postgreDatabaseTests.yml index 5ebf66b6a..5df3d7c5e 100644 --- a/.github/workflows/postgreDatabaseTests.yml +++ b/.github/workflows/postgreDatabaseTests.yml @@ -1,6 +1,8 @@ name: Postgre database tests on: push: + branches: + - '**' paths: - 'DocService/sources/databaseConnectors/baseConnector.js' - 'DocService/sources/databaseConnectors/postgreConnector.js' @@ -42,4 +44,4 @@ jobs: docker exec postgres psql -d onlyoffice -U onlyoffice -a -f /createdb.sql - name: Run Jest - run: npm run "integration database tests" \ No newline at end of file + run: npm run "integration database tests" diff --git a/.github/workflows/unitTests.yml b/.github/workflows/unitTests.yml index bdef3f3c5..e1d0d2b67 100644 --- a/.github/workflows/unitTests.yml +++ b/.github/workflows/unitTests.yml @@ -1,6 +1,8 @@ name: Service unit tests on: push: + branches: + - '**' paths: - '**.js' - '!tests/integration/**' @@ -31,4 +33,4 @@ jobs: npm --prefix DocService ci - name: Run Jest - run: npm run "unit tests" \ No newline at end of file + run: npm run "unit tests" From a23df363e8ec89a6676419eec6647105c7cd56ce Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Mon, 1 Apr 2024 01:15:17 +0300 Subject: [PATCH 12/48] [config] Add persistentStorage as alternative storage for forgotten files --- Common/config/default.json | 2 + Common/sources/storage-base.js | 240 ++++++++++++------ Common/sources/storage-fs.js | 71 +++--- Common/sources/storage-s3.js | 180 ++++++------- Common/sources/utils.js | 5 +- DocService/sources/DocsCoServer.js | 16 +- DocService/sources/routes/static.js | 96 +++++++ DocService/sources/server.js | 43 +--- .../withServerInstance/storage.tests.js | 95 ++++++- 9 files changed, 483 insertions(+), 265 deletions(-) create mode 100644 DocService/sources/routes/static.js diff --git a/Common/config/default.json b/Common/config/default.json index 61ef36ee1..06d14044c 100644 --- a/Common/config/default.json +++ b/Common/config/default.json @@ -35,6 +35,8 @@ "s3ForcePathStyle": true, "externalHost": "" }, + "persistentStorage": { + }, "rabbitmq": { "url": "amqp://guest:guest@localhost:5672", "socketOptions": {}, diff --git a/Common/sources/storage-base.js b/Common/sources/storage-base.js index 48ff2ad34..79738fc11 100644 --- a/Common/sources/storage-base.js +++ b/Common/sources/storage-base.js @@ -31,102 +31,180 @@ */ 'use strict'; +const os = require('os'); +const cluster = require('cluster'); var config = require('config'); var utils = require('./utils'); -var storage = require('./' + config.get('storage.name')); -var tenantManager = require('./tenantManager'); +const cfgCacheStorage = config.get('storage'); +const cfgPersistentStorage = utils.deepMergeObjects({}, cfgCacheStorage, config.get('persistentStorage')); -const cfgCacheFolderName = config.get('storage.cacheFolderName'); +const cacheStorage = require('./' + cfgCacheStorage.name); +const persistentStorage = require('./' + cfgPersistentStorage.name); +const tenantManager = require('./tenantManager'); + +const HEALTH_CHECK_KEY_MAX = 10000; function getStoragePath(ctx, strPath, opt_specialDir) { - opt_specialDir = opt_specialDir || cfgCacheFolderName; - return opt_specialDir + '/' + tenantManager.getTenantPathPrefix(ctx) + strPath.replace(/\\/g, '/') + opt_specialDir = opt_specialDir || cfgCacheStorage.cacheFolderName; + return opt_specialDir + '/' + tenantManager.getTenantPathPrefix(ctx) + strPath.replace(/\\/g, '/'); +} +function getStorage(ctx, opt_specialDir) { + return opt_specialDir ? persistentStorage : cacheStorage; +} +function getStorageCfg(ctx, opt_specialDir) { + return opt_specialDir ? cfgPersistentStorage : cfgCacheStorage; +} +function canCopyBetweenStorage(storageCfgSrc, storageCfgDst) { + return storageCfgSrc.name === storageCfgDst.name && storageCfgSrc.endpoint === storageCfgDst.endpoint; +} +function isDiffrentPersistentStorage() { + return !canCopyBetweenStorage(cacheStorage, cfgPersistentStorage); } -exports.headObject = function(ctx, strPath, opt_specialDir) { - return storage.headObject(getStoragePath(ctx, strPath, opt_specialDir)); -}; -exports.getObject = function(ctx, strPath, opt_specialDir) { - return storage.getObject(getStoragePath(ctx, strPath, opt_specialDir)); -}; -exports.createReadStream = function(ctx, strPath, opt_specialDir) { - return storage.createReadStream(getStoragePath(ctx, strPath, opt_specialDir)); -}; -exports.putObject = function(ctx, strPath, buffer, contentLength, opt_specialDir) { - return storage.putObject(getStoragePath(ctx, strPath, opt_specialDir), buffer, contentLength); -}; -exports.uploadObject = function(ctx, strPath, filePath, opt_specialDir) { - return storage.uploadObject(getStoragePath(ctx, strPath, opt_specialDir), filePath); -}; -exports.copyObject = function(ctx, sourceKey, destinationKey, opt_specialDirSrc, opt_specialDirDst) { - let storageSrc = getStoragePath(ctx, sourceKey, opt_specialDirSrc); - let storageDst = getStoragePath(ctx, destinationKey, opt_specialDirDst); - return storage.copyObject(storageSrc, storageDst); -}; -exports.copyPath = function(ctx, sourcePath, destinationPath, opt_specialDirSrc, opt_specialDirDst) { - let storageSrc = getStoragePath(ctx, sourcePath, opt_specialDirSrc); - let storageDst = getStoragePath(ctx, destinationPath, opt_specialDirDst); - return storage.listObjects(storageSrc).then(function(list) { - return Promise.all(list.map(function(curValue) { - return storage.copyObject(curValue, storageDst + '/' + exports.getRelativePath(storageSrc, curValue)); - })); - }); -}; -exports.listObjects = function(ctx, strPath, opt_specialDir) { +async function headObject(ctx, strPath, opt_specialDir) { + let storage = getStorage(ctx, opt_specialDir); + let storageCfg = getStorageCfg(ctx, opt_specialDir); + return await storage.headObject(storageCfg, getStoragePath(storageCfg, strPath, opt_specialDir)); +} +async function getObject(ctx, strPath, opt_specialDir) { + let storage = getStorage(ctx, opt_specialDir); + let storageCfg = getStorageCfg(ctx, opt_specialDir); + return await storage.getObject(storageCfg, getStoragePath(storageCfg, strPath, opt_specialDir)); +} +async function createReadStream(ctx, strPath, opt_specialDir) { + let storage = getStorage(ctx, opt_specialDir); + let storageCfg = getStorageCfg(ctx, opt_specialDir); + return await storage.createReadStream(storageCfg, getStoragePath(storageCfg, strPath, opt_specialDir)); +} +async function putObject(ctx, strPath, buffer, contentLength, opt_specialDir) { + let storage = getStorage(ctx, opt_specialDir); + let storageCfg = getStorageCfg(ctx, opt_specialDir); + return await storage.putObject(storageCfg, getStoragePath(ctx, strPath, opt_specialDir), buffer, contentLength); +} +async function uploadObject(ctx, strPath, filePath, opt_specialDir) { + let storage = getStorage(ctx, opt_specialDir); + let storageCfg = getStorageCfg(ctx, opt_specialDir); + return await storage.uploadObject(storageCfg, getStoragePath(ctx, strPath, opt_specialDir), filePath); +} +async function copyObject(ctx, sourceKey, destinationKey, opt_specialDirSrc, opt_specialDirDst) { + let storageSrc = getStorage(ctx, opt_specialDirSrc); + let storagePathSrc = getStoragePath(ctx, sourceKey, opt_specialDirSrc); + let storagePathDst = getStoragePath(ctx, destinationKey, opt_specialDirDst); + let storageCfgSrc = getStorageCfg(ctx, opt_specialDirSrc); + let storageCfgDst = getStorageCfg(ctx, opt_specialDirDst); + if (canCopyBetweenStorage(storageCfgSrc, storageCfgDst)){ + return await storageSrc.copyObject(storageCfgSrc, storageCfgDst, storagePathSrc, storagePathDst); + } else { + let storageDst = getStorage(ctx, opt_specialDirDst); + //todo stream + let buffer = await storageSrc.getObject(storageCfgSrc, storagePathSrc); + return await storageDst.putObject(storageCfgDst, storagePathDst, buffer, buffer.length); + } +} +async function copyPath(ctx, sourcePath, destinationPath, opt_specialDirSrc, opt_specialDirDst) { + let list = await listObjects(ctx, sourcePath, opt_specialDirSrc); + await Promise.all(list.map(function(curValue) { + return copyObject(ctx, curValue, destinationPath + '/' + getRelativePath(sourcePath, curValue), opt_specialDirSrc, opt_specialDirDst); + })); +} +async function listObjects(ctx, strPath, opt_specialDir) { + let storage = getStorage(ctx, opt_specialDir); + let storageCfg = getStorageCfg(ctx, opt_specialDir); let prefix = getStoragePath(ctx, "", opt_specialDir); - return storage.listObjects(getStoragePath(ctx, strPath, opt_specialDir)).then(function(list) { + try { + let list = await storage.listObjects(storageCfg, getStoragePath(ctx, strPath, opt_specialDir)); return list.map((currentValue) => { return currentValue.substring(prefix.length); }); - }).catch(function(e) { + } catch (e) { ctx.logger.error('storage.listObjects: %s', e.stack); return []; - }); -}; -exports.deleteObject = function(ctx, strPath, opt_specialDir) { - return storage.deleteObject(getStoragePath(ctx, strPath, opt_specialDir)); -}; -exports.deletePath = function(ctx, strPath, opt_specialDir) { - return storage.deletePath(getStoragePath(ctx, strPath, opt_specialDir)); -}; -exports.getSignedUrl = function(ctx, baseUrl, strPath, urlType, optFilename, opt_creationDate, opt_specialDir) { - return storage.getSignedUrl(ctx, baseUrl, getStoragePath(ctx, strPath, opt_specialDir), urlType, optFilename, opt_creationDate); -}; -exports.getSignedUrls = function(ctx, baseUrl, strPath, urlType, opt_creationDate, opt_specialDir) { - let storageSrc = getStoragePath(ctx, strPath, opt_specialDir); - return storage.listObjects(storageSrc).then(function(list) { - return Promise.all(list.map(function(curValue) { - return storage.getSignedUrl(ctx, baseUrl, curValue, urlType, undefined, opt_creationDate); - })).then(function(urls) { - var outputMap = {}; - for (var i = 0; i < list.length && i < urls.length; ++i) { - outputMap[exports.getRelativePath(storageSrc, list[i])] = urls[i]; - } - return outputMap; - }); - }); -}; -exports.getSignedUrlsArrayByArray = function(ctx, baseUrl, list, urlType, opt_specialDir) { - return Promise.all(list.map(function(curValue) { - let storageSrc = getStoragePath(ctx, curValue, opt_specialDir); - return storage.getSignedUrl(ctx, baseUrl, storageSrc, urlType, undefined); + } +} +async function deleteObject(ctx, strPath, opt_specialDir) { + let storage = getStorage(ctx, opt_specialDir); + let storageCfg = getStorageCfg(ctx, opt_specialDir); + return await storage.deleteObject(storageCfg, getStoragePath(ctx, strPath, opt_specialDir)); +} +async function deletePath(ctx, strPath, opt_specialDir) { + let storage = getStorage(ctx, opt_specialDir); + let storageCfg = getStorageCfg(ctx, opt_specialDir); + return await storage.deletePath(storageCfg, getStoragePath(ctx, strPath, opt_specialDir)); +} +async function getSignedUrl(ctx, baseUrl, strPath, urlType, optFilename, opt_creationDate, opt_specialDir) { + let storage = getStorage(ctx, opt_specialDir); + let storageCfg = getStorageCfg(ctx, opt_specialDir); + return await storage.getSignedUrl(ctx, storageCfg, baseUrl, getStoragePath(ctx, strPath, opt_specialDir), urlType, optFilename, opt_creationDate); +} +async function getSignedUrls(ctx, baseUrl, strPath, urlType, opt_creationDate, opt_specialDir) { + let storagePathSrc = getStoragePath(ctx, strPath, opt_specialDir); + let storage = getStorage(ctx, opt_specialDir); + let storageCfg = getStorageCfg(ctx, opt_specialDir); + let list = await storage.listObjects(storageCfg, storagePathSrc, storageCfg); + let urls = await Promise.all(list.map(function(curValue) { + return storage.getSignedUrl(ctx, storageCfg, baseUrl, curValue, urlType, undefined, opt_creationDate); })); -}; -exports.getSignedUrlsByArray = function(ctx, baseUrl, list, optPath, urlType, opt_specialDir) { - return exports.getSignedUrlsArrayByArray(ctx, baseUrl, list, urlType, opt_specialDir).then(function(urls) { - var outputMap = {}; - for (var i = 0; i < list.length && i < urls.length; ++i) { - if (optPath) { - let storageSrc = getStoragePath(ctx, optPath, opt_specialDir); - outputMap[exports.getRelativePath(storageSrc, list[i])] = urls[i]; - } else { - outputMap[list[i]] = urls[i]; - } + let outputMap = {}; + for (let i = 0; i < list.length && i < urls.length; ++i) { + outputMap[getRelativePath(storagePathSrc, list[i])] = urls[i]; + } + return outputMap; +} +async function getSignedUrlsArrayByArray(ctx, baseUrl, list, urlType, opt_specialDir) { + return await Promise.all(list.map(function (curValue) { + let storage = getStorage(ctx, opt_specialDir); + let storageCfg = getStorageCfg(ctx, opt_specialDir); + let storagePathSrc = getStoragePath(ctx, curValue, opt_specialDir); + return storage.getSignedUrl(ctx, storageCfg, baseUrl, storagePathSrc, urlType, undefined); + })); +} +async function getSignedUrlsByArray(ctx, baseUrl, list, optPath, urlType, opt_specialDir) { + let urls = await getSignedUrlsArrayByArray(ctx, baseUrl, list, urlType, opt_specialDir); + var outputMap = {}; + for (var i = 0; i < list.length && i < urls.length; ++i) { + if (optPath) { + let storagePathSrc = getStoragePath(ctx, optPath, opt_specialDir); + outputMap[getRelativePath(storagePathSrc, list[i])] = urls[i]; + } else { + outputMap[list[i]] = urls[i]; } - return outputMap; - }); -}; -exports.getRelativePath = function(strBase, strPath) { + } + return outputMap; +} +function getRelativePath(strBase, strPath) { return strPath.substring(strBase.length + 1); +} +async function healthCheck(ctx, opt_specialDir) { + const clusterId = cluster.isWorker ? cluster.worker.id : ''; + const tempName = 'hc_' + os.hostname() + '_' + clusterId + '_' + Math.round(Math.random() * HEALTH_CHECK_KEY_MAX); + const tempBuffer = Buffer.from([1, 2, 3, 4, 5]); + try { + //It's proper to putObject one tempName + await putObject(ctx, tempName, tempBuffer, tempBuffer.length, opt_specialDir); + //try to prevent case, when another process can remove same tempName + await deleteObject(ctx, tempName, opt_specialDir); + } catch (err) { + ctx.logger.warn('healthCheck storage(%s) error %s', opt_specialDir, err.stack); + } +} + +module.exports = { + headObject, + getObject, + createReadStream, + putObject, + uploadObject, + copyObject, + copyPath, + listObjects, + deleteObject, + deletePath, + getSignedUrl, + getSignedUrls, + getSignedUrlsArrayByArray, + getSignedUrlsByArray, + getRelativePath, + isDiffrentPersistentStorage, + healthCheck }; diff --git a/Common/sources/storage-fs.js b/Common/sources/storage-fs.js index 9b2b2ecfa..1b9670e0c 100644 --- a/Common/sources/storage-fs.js +++ b/Common/sources/storage-fs.js @@ -38,39 +38,33 @@ var path = require('path'); var utils = require("./utils"); var crypto = require('crypto'); const ms = require('ms'); +const config = require('config'); const commonDefines = require('./../../Common/sources/commondefines'); const constants = require('./../../Common/sources/constants'); -var config = require('config'); -var configStorage = config.get('storage'); -var cfgBucketName = configStorage.get('bucketName'); -var cfgStorageFolderName = configStorage.get('storageFolderName'); -var configFs = configStorage.get('fs'); -var cfgStorageFolderPath = configFs.get('folderPath'); -var cfgStorageSecretString = configFs.get('secretString'); -var cfgStorageUrlExpires = configFs.get('urlExpires'); const cfgExpSessionAbsolute = ms(config.get('services.CoAuthoring.expire.sessionabsolute')); -function getFilePath(strPath) { - return path.join(cfgStorageFolderPath, strPath); +function getFilePath(storageCfg, strPath) { + const storageFolderPath = storageCfg.fs.folderPath; + return path.join(storageFolderPath, strPath); } function getOutputPath(strPath) { return strPath.replace(/\\/g, '/'); } -async function headObject(strPath) { - let fsPath = getFilePath(strPath); +async function headObject(storageCfg, strPath) { + let fsPath = getFilePath(storageCfg, strPath); let stats = await stat(fsPath); return {ContentLength: stats.size}; } -async function getObject(strPath) { - let fsPath = getFilePath(strPath); +async function getObject(storageCfg, strPath) { + let fsPath = getFilePath(storageCfg, strPath); return await readFile(fsPath); } -async function createReadStream(strPath) { - let fsPath = getFilePath(strPath); +async function createReadStream(storageCfg, strPath) { + let fsPath = getFilePath(storageCfg, strPath); let stats = await stat(fsPath); let contentLength = stats.size; let readStream = await utils.promiseCreateReadStream(fsPath); @@ -80,8 +74,8 @@ async function createReadStream(strPath) { }; } -async function putObject(strPath, buffer, contentLength) { - var fsPath = getFilePath(strPath); +async function putObject(storageCfg, strPath, buffer, contentLength) { + var fsPath = getFilePath(storageCfg, strPath); await mkdir(path.dirname(fsPath), {recursive: true}); if (Buffer.isBuffer(buffer)) { @@ -92,52 +86,57 @@ async function putObject(strPath, buffer, contentLength) { } } -async function uploadObject(strPath, filePath) { - let fsPath = getFilePath(strPath); +async function uploadObject(storageCfg, strPath, filePath) { + let fsPath = getFilePath(storageCfg, strPath); await cp(filePath, fsPath, {force: true, recursive: true}); } -async function copyObject(sourceKey, destinationKey) { - let fsPathSource = getFilePath(sourceKey); - let fsPathDestination = getFilePath(destinationKey); +async function copyObject(storageCfgSrc, storageCfgDst, sourceKey, destinationKey) { + let fsPathSource = getFilePath(storageCfgSrc, sourceKey); + let fsPathDestination = getFilePath(storageCfgDst, destinationKey); await cp(fsPathSource, fsPathDestination, {force: true, recursive: true}); } -async function listObjects(strPath) { - let fsPath = getFilePath(strPath); +async function listObjects(storageCfg, strPath) { + const storageFolderPath = storageCfg.fs.folderPath; + let fsPath = getFilePath(storageCfg, strPath); let values = await utils.listObjects(fsPath); return values.map(function(curvalue) { - return getOutputPath(curvalue.substring(cfgStorageFolderPath.length + 1)); + return getOutputPath(curvalue.substring(storageFolderPath.length + 1)); }); } -async function deleteObject(strPath) { - const fsPath = getFilePath(strPath); +async function deleteObject(storageCfg, strPath) { + const fsPath = getFilePath(storageCfg, strPath); return rm(fsPath, {force: true, recursive: true}); } -async function deletePath(strPath) { - const fsPath = getFilePath(strPath); +async function deletePath(storageCfg, strPath) { + const fsPath = getFilePath(storageCfg, strPath); return rm(fsPath, {force: true, recursive: true}); } -async function getSignedUrl(ctx, baseUrl, strPath, urlType, optFilename, opt_creationDate) { +async function getSignedUrl(ctx, storageCfg, baseUrl, strPath, urlType, optFilename, opt_creationDate) { + const storageSecretString = storageCfg.fs.secretString; + const storageUrlExpires = storageCfg.fs.urlExpires; + const bucketName = storageCfg.bucketName; + const storageFolderName = storageCfg.storageFolderName; //replace '/' with %2f before encodeURIComponent becase nginx determine %2f as '/' and get wrong system path - var userFriendlyName = optFilename ? encodeURIComponent(optFilename.replace(/\//g, "%2f")) : path.basename(strPath); - var uri = '/' + cfgBucketName + '/' + cfgStorageFolderName + '/' + strPath + '/' + userFriendlyName; + const userFriendlyName = optFilename ? encodeURIComponent(optFilename.replace(/\//g, "%2f")) : path.basename(strPath); + var uri = '/' + bucketName + '/' + storageFolderName + '/' + strPath + '/' + userFriendlyName; //RFC 1123 does not allow underscores https://stackoverflow.com/questions/2180465/can-domain-name-subdomains-have-an-underscore-in-it - var url = utils.checkBaseUrl(ctx, baseUrl).replace(/_/g, "%5f"); + var url = utils.checkBaseUrl(ctx, baseUrl, storageCfg).replace(/_/g, "%5f"); url += uri; var date = Date.now(); let creationDate = opt_creationDate || date; - let expiredAfter = (commonDefines.c_oAscUrlTypes.Session === urlType ? (cfgExpSessionAbsolute / 1000) : cfgStorageUrlExpires) || 31536000; + let expiredAfter = (commonDefines.c_oAscUrlTypes.Session === urlType ? (cfgExpSessionAbsolute / 1000) : storageUrlExpires) || 31536000; //todo creationDate can be greater because mysql CURRENT_TIMESTAMP uses local time, not UTC var expires = creationDate + Math.ceil(Math.abs(date - creationDate) / expiredAfter) * expiredAfter; expires = Math.ceil(expires / 1000); expires += expiredAfter; - var md5 = crypto.createHash('md5').update(expires + decodeURIComponent(uri) + cfgStorageSecretString).digest("base64"); + var md5 = crypto.createHash('md5').update(expires + decodeURIComponent(uri) + storageSecretString).digest("base64"); md5 = md5.replace(/\+/g, "-").replace(/\//g, "_").replace(/=/g, ""); url += '?md5=' + encodeURIComponent(md5); diff --git a/Common/sources/storage-s3.js b/Common/sources/storage-s3.js index 32aaede00..8fabade63 100644 --- a/Common/sources/storage-s3.js +++ b/Common/sources/storage-s3.js @@ -39,190 +39,190 @@ const { GetObjectCommand, PutObjectCommand, CopyObjectCommand} = require("@aws-s const { DeleteObjectsCommand, DeleteObjectCommand } = require("@aws-sdk/client-s3"); const { getSignedUrl } = require("@aws-sdk/s3-request-presigner"); const mime = require('mime'); +const config = require('config'); const utils = require('./utils'); const ms = require('ms'); const commonDefines = require('./../../Common/sources/commondefines'); -const config = require('config'); -const configStorage = require('config').get('storage'); -const cfgRegion = configStorage.get('region'); -const cfgEndpoint = configStorage.get('endpoint'); -const cfgBucketName = configStorage.get('bucketName'); -const cfgStorageFolderName = configStorage.get('storageFolderName'); -const cfgAccessKeyId = configStorage.get('accessKeyId'); -const cfgSecretAccessKey = configStorage.get('secretAccessKey'); -const cfgSslEnabled = configStorage.get('sslEnabled'); -const cfgS3ForcePathStyle = configStorage.get('s3ForcePathStyle'); -const configFs = configStorage.get('fs'); -const cfgStorageUrlExpires = configFs.get('urlExpires'); const cfgExpSessionAbsolute = ms(config.get('services.CoAuthoring.expire.sessionabsolute')); -/** - * Don't hard-code your credentials! - * Export the following environment variables instead: - * - * export AWS_ACCESS_KEY_ID='AKID' - * export AWS_SECRET_ACCESS_KEY='SECRET' - */ -let configS3 = { - region: cfgRegion, - endpoint: cfgEndpoint, - credentials : { - accessKeyId: cfgAccessKeyId, - secretAccessKey: cfgSecretAccessKey - } -}; - -if (configS3.endpoint) { - configS3.tls = cfgSslEnabled; - configS3.forcePathStyle = cfgS3ForcePathStyle; -} -const client = new S3Client(configS3); - //This operation enables you to delete multiple objects from a bucket using a single HTTP request. You may specify up to 1000 keys. const MAX_DELETE_OBJECTS = 1000; +let clients = {}; + +function getS3Client(storageCfg) { + /** + * Don't hard-code your credentials! + * Export the following environment variables instead: + * + * export AWS_ACCESS_KEY_ID='AKID' + * export AWS_SECRET_ACCESS_KEY='SECRET' + */ + let configS3 = { + region: storageCfg.region, + endpoint: storageCfg.endpoint, + credentials : { + accessKeyId: storageCfg.accessKeyId, + secretAccessKey: storageCfg.secretAccessKey + } + }; + + if (configS3.endpoint) { + configS3.tls = storageCfg.sslEnabled; + configS3.forcePathStyle = storageCfg.s3ForcePathStyle; + } + let configJson = JSON.stringify(configS3); + let client = clients[configJson]; + if (!client) { + client = new S3Client(configS3); + clients[configJson] = client; + } + return client; +} -function getFilePath(strPath) { - //todo - return cfgStorageFolderName + '/' + strPath; +function getFilePath(storageCfg, strPath) { + const storageFolderName = storageCfg.storageFolderName; + return storageFolderName + '/' + strPath; } -function joinListObjects(inputArray, outputArray) { +function joinListObjects(storageCfg, inputArray, outputArray) { if (!inputArray) { return; } + const storageFolderName = storageCfg.storageFolderName; let length = inputArray.length; for (let i = 0; i < length; i++) { - outputArray.push(inputArray[i].Key.substring((cfgStorageFolderName + '/').length)); + outputArray.push(inputArray[i].Key.substring((storageFolderName + '/').length)); } } -async function listObjectsExec(output, params) { - const data = await client.send(new ListObjectsCommand(params)); - joinListObjects(data.Contents, output); +async function listObjectsExec(storageCfg, output, params) { + const data = await getS3Client(storageCfg).send(new ListObjectsCommand(params)); + joinListObjects(storageCfg, data.Contents, output); if (data.IsTruncated && (data.NextMarker || (data.Contents && data.Contents.length > 0))) { params.Marker = data.NextMarker || data.Contents[data.Contents.length - 1].Key; - return await listObjectsExec(output, params); + return await listObjectsExec(storageCfg, output, params); } else { return output; } } -async function deleteObjectsHelp(aKeys) { +async function deleteObjectsHelp(storageCfg, aKeys) { //By default, the operation uses verbose mode in which the response includes the result of deletion of each key in your request. //In quiet mode the response includes only keys where the delete operation encountered an error. const input = { - Bucket: cfgBucketName, + Bucket: storageCfg.bucketName, Delete: { Objects: aKeys, Quiet: true } }; const command = new DeleteObjectsCommand(input); - await client.send(command); + await getS3Client(storageCfg).send(command); } -async function headObject(strPath) { +async function headObject(storageCfg, strPath) { const input = { - Bucket: cfgBucketName, - Key: getFilePath(strPath) + Bucket: storageCfg.bucketName, + Key: getFilePath(storageCfg, strPath) }; const command = new HeadObjectCommand(input); - let output = await client.send(command); + let output = await getS3Client(storageCfg).send(command); return {ContentLength: output.ContentLength}; } -async function getObject(strPath) { +async function getObject(storageCfg, strPath) { const input = { - Bucket: cfgBucketName, - Key: getFilePath(strPath) + Bucket: storageCfg.bucketName, + Key: getFilePath(storageCfg, strPath) }; const command = new GetObjectCommand(input); - const output = await client.send(command); + const output = await getS3Client(storageCfg).send(command); return await utils.stream2Buffer(output.Body); } -async function createReadStream(strPath) { +async function createReadStream(storageCfg, strPath) { const input = { - Bucket: cfgBucketName, - Key: getFilePath(strPath) + Bucket: storageCfg.bucketName, + Key: getFilePath(storageCfg, strPath) }; const command = new GetObjectCommand(input); - const output = await client.send(command); + const output = await getS3Client(storageCfg).send(command); return { contentLength: output.ContentLength, readStream: output.Body }; } -async function putObject(strPath, buffer, contentLength) { +async function putObject(storageCfg, strPath, buffer, contentLength) { //todo consider Expires const input = { - Bucket: cfgBucketName, - Key: getFilePath(strPath), + Bucket: storageCfg.bucketName, + Key: getFilePath(storageCfg, strPath), Body: buffer, ContentLength: contentLength, ContentType: mime.getType(strPath) }; const command = new PutObjectCommand(input); - await client.send(command); + await getS3Client(storageCfg).send(command); } -async function uploadObject(strPath, filePath) { +async function uploadObject(storageCfg, strPath, filePath) { const file = fs.createReadStream(filePath); //todo раÑÑмотреть Expires const input = { - Bucket: cfgBucketName, - Key: getFilePath(strPath), + Bucket: storageCfg.bucketName, + Key: getFilePath(storageCfg, strPath), Body: file, ContentType: mime.getType(strPath) }; const command = new PutObjectCommand(input); - await client.send(command); + await getS3Client(storageCfg).send(command); } -async function copyObject(sourceKey, destinationKey) { +async function copyObject(storageCfgSrc, storageCfgDst, sourceKey, destinationKey) { //todo source bucket const input = { - Bucket: cfgBucketName, - Key: getFilePath(destinationKey), - CopySource: `/${cfgBucketName}/${getFilePath(sourceKey)}` + Bucket: storageCfgDst.bucketName, + Key: getFilePath(storageCfgDst, destinationKey), + CopySource: `/${storageCfgSrc.bucketName}/${getFilePath(storageCfgSrc, sourceKey)}` }; const command = new CopyObjectCommand(input); - await client.send(command); + await getS3Client(storageCfgDst).send(command); } -async function listObjects(strPath) { +async function listObjects(storageCfg, strPath) { let params = { - Bucket: cfgBucketName, - Prefix: getFilePath(strPath) + Bucket: storageCfg.bucketName, + Prefix: getFilePath(storageCfg, strPath) }; let output = []; - await listObjectsExec(output, params); + await listObjectsExec(storageCfg, output, params); return output; } -async function deleteObject(strPath) { +async function deleteObject(storageCfg, strPath) { const input = { - Bucket: cfgBucketName, - Key: getFilePath(strPath) + Bucket: storageCfg.bucketName, + Key: getFilePath(storageCfg, strPath) }; const command = new DeleteObjectCommand(input); - await client.send(command); + await getS3Client(storageCfg).send(command); }; -async function deleteObjects(strPaths) { +async function deleteObjects(storageCfg, strPaths) { let aKeys = strPaths.map(function (currentValue) { - return {Key: getFilePath(currentValue)}; + return {Key: getFilePath(storageCfg, currentValue)}; }); for (let i = 0; i < aKeys.length; i += MAX_DELETE_OBJECTS) { - await deleteObjectsHelp(aKeys.slice(i, i + MAX_DELETE_OBJECTS)); + await deleteObjectsHelp(storageCfg, aKeys.slice(i, i + MAX_DELETE_OBJECTS)); } } -async function deletePath(strPath) { - let list = await listObjects(strPath); - await deleteObjects(list); +async function deletePath(storageCfg, strPath) { + let list = await listObjects(storageCfg, strPath); + await deleteObjects(storageCfg, list); } -async function getSignedUrlWrapper(ctx, baseUrl, strPath, urlType, optFilename, opt_creationDate) { - let expires = (commonDefines.c_oAscUrlTypes.Session === urlType ? cfgExpSessionAbsolute / 1000 : cfgStorageUrlExpires) || 31536000; +async function getSignedUrlWrapper(ctx, storageCfg, baseUrl, strPath, urlType, optFilename, opt_creationDate) { + const storageUrlExpires = storageCfg.fs.urlExpires; + let expires = (commonDefines.c_oAscUrlTypes.Session === urlType ? cfgExpSessionAbsolute / 1000 : storageUrlExpires) || 31536000; // Signature version 4 presigned URLs must have an expiration date less than one week in the future expires = Math.min(expires, 604800); let userFriendlyName = optFilename ? optFilename.replace(/\//g, "%2f") : path.basename(strPath); let contentDisposition = utils.getContentDisposition(userFriendlyName, null, null); const input = { - Bucket: cfgBucketName, - Key: getFilePath(strPath), + Bucket: storageCfg.bucketName, + Key: getFilePath(storageCfg, strPath), ResponseContentDisposition: contentDisposition }; const command = new GetObjectCommand(input); @@ -230,7 +230,7 @@ async function getSignedUrlWrapper(ctx, baseUrl, strPath, urlType, optFilename, let options = { expiresIn: expires }; - return await getSignedUrl(client, command, options); + return await getSignedUrl(getS3Client(storageCfg), command, options); //extra query params cause SignatureDoesNotMatch //https://stackoverflow.com/questions/55503009/amazon-s3-signature-does-not-match-when-extra-query-params-ga-added-in-url // return utils.changeOnlyOfficeUrl(url, strPath, optFilename); diff --git a/Common/sources/utils.js b/Common/sources/utils.js index ae3eab59c..48008e612 100644 --- a/Common/sources/utils.js +++ b/Common/sources/utils.js @@ -1087,8 +1087,9 @@ exports.convertLicenseInfoToServerParams = function(licenseInfo) { license.buildNumber = commonDefines.buildNumber; return license; }; -exports.checkBaseUrl = function(ctx, baseUrl) { - const tenStorageExternalHost = ctx.getCfg('storage.externalHost', cfgStorageExternalHost); +exports.checkBaseUrl = function(ctx, baseUrl, opt_storageCfg) { + let storageExternalHost = opt_storageCfg ? opt_storageCfg.externalHost : cfgStorageExternalHost + const tenStorageExternalHost = ctx.getCfg('storage.externalHost', storageExternalHost); return tenStorageExternalHost ? tenStorageExternalHost : baseUrl; }; exports.resolvePath = function(object, path, defaultValue) { diff --git a/DocService/sources/DocsCoServer.js b/DocService/sources/DocsCoServer.js index 586d6108f..e672010e3 100644 --- a/DocService/sources/DocsCoServer.js +++ b/DocService/sources/DocsCoServer.js @@ -4010,18 +4010,12 @@ exports.healthCheck = function(req, res) { } //storage - const clusterId = cluster.isWorker ? cluster.worker.id : ''; - const tempName = 'hc_' + os.hostname() + '_' + clusterId + '_' + Math.round(Math.random() * HEALTH_CHECK_KEY_MAX); - const tempBuffer = Buffer.from([1, 2, 3, 4, 5]); - //It's proper to putObject one tempName - yield storage.putObject(ctx, tempName, tempBuffer, tempBuffer.length); - try { - //try to prevent case, when another process can remove same tempName - yield storage.deleteObject(ctx, tempName); - } catch (err) { - ctx.logger.warn('healthCheck error %s', err.stack); - } + yield storage.healthCheck(ctx); ctx.logger.debug('healthCheck storage'); + if (storage.isDiffrentPersistentStorage()) { + yield storage.healthCheck(ctx, cfgForgottenFiles); + ctx.logger.debug('healthCheck storage persistent'); + } output = true; ctx.logger.info('healthCheck end'); diff --git a/DocService/sources/routes/static.js b/DocService/sources/routes/static.js new file mode 100644 index 000000000..a81a84312 --- /dev/null +++ b/DocService/sources/routes/static.js @@ -0,0 +1,96 @@ +/* + * (c) Copyright Ascensio System SIA 2010-2023 + * + * This program is a free software product. You can redistribute it and/or + * modify it under the terms of the GNU Affero General Public License (AGPL) + * version 3 as published by the Free Software Foundation. In accordance with + * Section 7(a) of the GNU AGPL its Section 15 shall be amended to the effect + * that Ascensio System SIA expressly excludes the warranty of non-infringement + * of any third-party rights. + * + * This program is distributed WITHOUT ANY WARRANTY; without even the implied + * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. For + * details, see the GNU AGPL at: http://www.gnu.org/licenses/agpl-3.0.html + * + * You can contact Ascensio System SIA at 20A-6 Ernesta Birznieka-Upish + * street, Riga, Latvia, EU, LV-1050. + * + * The interactive user interfaces in modified source and object code versions + * of the Program must display Appropriate Legal Notices, as required under + * Section 5 of the GNU AGPL version 3. + * + * Pursuant to Section 7(b) of the License you must retain the original Product + * logo when distributing the program. Pursuant to Section 7(e) we decline to + * grant you any rights under trademark law for use of our trademarks. + * + * All the Product's GUI elements, including illustrations and icon sets, as + * well as technical writing content are licensed under the terms of the + * Creative Commons Attribution-ShareAlike 4.0 International. See the License + * terms at http://creativecommons.org/licenses/by-sa/4.0/legalcode + * + */ + +'use strict'; +const express = require('express'); +const config = require("config"); +const operationContext = require('./../../../Common/sources/operationContext'); +const utils = require('./../../../Common/sources/utils'); +const urlModule = require("url"); +const path = require("path"); +const mime = require("mime"); + +const cfgStaticContent = config.has('services.CoAuthoring.server.static_content') ? config.get('services.CoAuthoring.server.static_content') : {}; +const cfgCacheStorage = config.get('storage'); +const cfgPersistentStorage = utils.deepMergeObjects({}, cfgCacheStorage, config.get('persistentStorage')); +const cfgForgottenFiles = config.get('services.CoAuthoring.server.forgottenfiles'); +const cfgErrorFiles = config.get('FileConverter.converter.errorfiles'); + +const router = express.Router(); + +function initCacheRouter(cfgStorage, routs) { + const bucketName = cfgStorage.bucketName; + const storageFolderName = cfgStorage.storageFolderName; + const folderPath = cfgStorage.fs.folderPath; + routs.forEach((rout) => { + let rootPath = path.join(folderPath, rout); + router.use(`/${bucketName}/${storageFolderName}/${rout}`, (req, res, next) => { + const index = req.url.lastIndexOf('/'); + if ('GET' === req.method && index > 0) { + let sendFileOptions = { + root: rootPath, dotfiles: 'deny', headers: { + 'Content-Disposition': 'attachment' + } + }; + const urlParsed = urlModule.parse(req.url); + if (urlParsed && urlParsed.pathname) { + const filename = decodeURIComponent(path.basename(urlParsed.pathname)); + sendFileOptions.headers['Content-Type'] = mime.getType(filename); + } + const realUrl = decodeURI(req.url.substring(0, index)); + res.sendFile(realUrl, sendFileOptions, (err) => { + if (err) { + operationContext.global.logger.error(err); + res.status(400).end(); + } + }); + } else { + res.sendStatus(404); + } + }); + }); +} + +for (let i in cfgStaticContent) { + if (cfgStaticContent.hasOwnProperty(i)) { + router.use(i, express.static(cfgStaticContent[i]['path'], cfgStaticContent[i]['options'])); + } +} +initCacheRouter(cfgCacheStorage, [cfgCacheStorage.cacheFolderName]); + +let persistentRouts = [cfgForgottenFiles, cfgErrorFiles]; +persistentRouts.filter((rout) => {return rout && rout.length > 0;}); +if (persistentRouts.length > 0) { + initCacheRouter(cfgPersistentStorage, [cfgForgottenFiles, cfgErrorFiles]); +} + +module.exports = router; diff --git a/DocService/sources/server.js b/DocService/sources/server.js index 880983149..09c9e3bf3 100644 --- a/DocService/sources/server.js +++ b/DocService/sources/server.js @@ -57,7 +57,7 @@ const utils = require('./../../Common/sources/utils'); const commonDefines = require('./../../Common/sources/commondefines'); const operationContext = require('./../../Common/sources/operationContext'); const tenantManager = require('./../../Common/sources/tenantManager'); -const configStorage = config.get('storage'); +const staticRouter = require('./routes/static'); const cfgWopiEnable = config.get('wopi.enable'); const cfgWopiDummyEnable = config.get('wopi.dummy.enable'); @@ -132,44 +132,6 @@ updateLicense(); fs.watchFile(cfgLicenseFile, updateLicense); setInterval(updateLicense, 86400000); -if (config.has('services.CoAuthoring.server.static_content')) { - const staticContent = config.get('services.CoAuthoring.server.static_content'); - for (let i in staticContent) { - if (staticContent.hasOwnProperty(i)) { - app.use(i, express.static(staticContent[i]['path'], staticContent[i]['options'])); - } - } -} - -if (configStorage.has('fs.folderPath')) { - const cfgBucketName = configStorage.get('bucketName'); - const cfgStorageFolderName = configStorage.get('storageFolderName'); - app.use('/' + cfgBucketName + '/' + cfgStorageFolderName, (req, res, next) => { - const index = req.url.lastIndexOf('/'); - if ('GET' === req.method && index > 0) { - let sendFileOptions = { - root: configStorage.get('fs.folderPath'), dotfiles: 'deny', headers: { - 'Content-Disposition': 'attachment' - } - }; - const urlParsed = urlModule.parse(req.url); - if (urlParsed && urlParsed.pathname) { - const filename = decodeURIComponent(path.basename(urlParsed.pathname)); - sendFileOptions.headers['Content-Type'] = mime.getType(filename); - } - const realUrl = decodeURI(req.url.substring(0, index)); - res.sendFile(realUrl, sendFileOptions, (err) => { - if (err) { - operationContext.global.logger.error(err); - res.status(400).end(); - } - }); - } else { - res.sendStatus(404); - } - }); -} - try { fs.watch(config.get('services.CoAuthoring.plugins.path'), updatePlugins); } catch (e) { @@ -211,6 +173,9 @@ docsCoServer.install(server, () => { } }); }); + + app.use('/', staticRouter); + const rawFileParser = bodyParser.raw( {inflate: true, limit: config.get('services.CoAuthoring.server.limits_tempfile_upload'), type: function() {return true;}}); const urleEcodedParser = bodyParser.urlencoded({ extended: false }); diff --git a/tests/integration/withServerInstance/storage.tests.js b/tests/integration/withServerInstance/storage.tests.js index 6a91cac0e..5477f03c9 100644 --- a/tests/integration/withServerInstance/storage.tests.js +++ b/tests/integration/withServerInstance/storage.tests.js @@ -21,7 +21,8 @@ const utils = require('../../../Common/sources/utils'); const commonDefines = require("../../../Common/sources/commondefines"); const config = require('../../../Common/node_modules/config'); -const cfgStorageName = config.get('storage.name'); +const cfgCacheStorage = config.get('storage'); +const cfgPersistentStorage = utils.deepMergeObjects({}, cfgCacheStorage, config.get('persistentStorage')); const ctx = operationContext.global; const rand = Math.floor(Math.random() * 1000000); @@ -32,9 +33,15 @@ let testFile1 = testDir + "/test1.txt"; let testFile2 = testDir + "/test2.txt"; let testFile3 = testDir + "/test3.txt"; let testFile4 = testDir + "/test4.txt"; +let specialDirCache = ""; +let specialDirForgotten = "forgotten"; console.debug(`testDir: ${testDir}`) +function getStorageCfg(specialDir) { + return specialDir ? cfgPersistentStorage : cfgCacheStorage; +} + function request(url) { return new Promise(resolve => { let module = url.startsWith('https') ? https : http; @@ -65,7 +72,7 @@ function runTestForDir(specialDir) { let list = await storage.listObjects(ctx, testDir, specialDir); expect(list.sort()).toEqual([testFile1, testFile2].sort()); }); - if ("storage-fs" === cfgStorageName) { + if ("storage-fs" === getStorageCfg(specialDir).name) { test("UploadObject", async () => { let res = await storage.uploadObject(ctx, testFile3, "createReadStream.txt", specialDir); expect(res).toEqual(undefined); @@ -81,6 +88,7 @@ function runTestForDir(specialDir) { let list = await storage.listObjects(ctx, testDir, specialDir); expect(spy).toHaveBeenCalled(); expect(list.sort()).toEqual([testFile1, testFile2, testFile3].sort()); + spy.mockRestore(); }); } test("copyObject", async () => { @@ -140,7 +148,7 @@ function runTestForDir(specialDir) { expect(outputText.toString("utf8")).toEqual(testFileData3); }); test("getSignedUrl", async () => { - let url, data; + let url, urls, data; url = await storage.getSignedUrl(ctx, baseUrl, testFile1, urlType, undefined, undefined, specialDir); data = await request(url); expect(data).toEqual(testFileData1); @@ -157,6 +165,33 @@ function runTestForDir(specialDir) { data = await request(url); expect(data).toEqual(testFileData4); }); + test("getSignedUrls", async () => { + let urls, data; + urls = await storage.getSignedUrls(ctx, baseUrl, testDir, urlType, undefined, specialDir); + data = []; + for(let i in urls) { + data.push(await request(urls[i])); + } + expect(data.sort()).toEqual([testFileData1, testFileData2, testFileData3, testFileData4].sort()); + }); + test("getSignedUrlsArrayByArray", async () => { + let urls, data; + urls = await storage.getSignedUrlsArrayByArray(ctx, baseUrl, [testFile1, testFile2], urlType, specialDir); + data = []; + for(let i = 0; i < urls.length; ++i) { + data.push(await request(urls[i])); + } + expect(data.sort()).toEqual([testFileData1, testFileData2].sort()); + }); + test("getSignedUrlsByArray", async () => { + let urls, data; + urls = await storage.getSignedUrlsByArray(ctx, baseUrl, [testFile3, testFile4], undefined, urlType, specialDir); + data = []; + for(let i in urls) { + data.push(await request(urls[i])); + } + expect(data.sort()).toEqual([testFileData3, testFileData4].sort()); + }); test("deleteObject", async () => { let list; list = await storage.listObjects(ctx, testDir, specialDir); @@ -183,9 +218,57 @@ function runTestForDir(specialDir) { // Assumed, that server is already up. describe('storage common dir', function () { - runTestForDir(""); + runTestForDir(specialDirCache); }); describe('storage forgotten dir', function () { - runTestForDir("forgotten"); -}); \ No newline at end of file + runTestForDir(specialDirForgotten); +}); + +describe('storage mix common and forgotten dir', function () { + test("putObject", async () => { + let buffer = Buffer.from(testFileData1); + let res = await storage.putObject(ctx, testFile1, buffer, buffer.length, specialDirCache); + expect(res).toEqual(undefined); + let list = await storage.listObjects(ctx, testDir, specialDirCache); + expect(list.sort()).toEqual([testFile1].sort()); + + buffer = Buffer.from(testFileData2); + res = await storage.putObject(ctx, testFile2, buffer, buffer.length, specialDirForgotten); + expect(res).toEqual(undefined); + list = await storage.listObjects(ctx, testDir, specialDirForgotten); + expect(list.sort()).toEqual([testFile2].sort()); + }); + + test("copyPath", async () => { + let list, res; + res = await storage.copyPath(ctx, testDir, testDir, specialDirCache, specialDirForgotten); + expect(res).toEqual(undefined); + + list = await storage.listObjects(ctx, testDir, specialDirForgotten); + expect(list.sort()).toEqual([testFile1, testFile2].sort()); + }); + test("copyObject", async () => { + let list, res; + res = await storage.copyObject(ctx, testFile2, testFile2, specialDirForgotten, specialDirCache); + expect(res).toEqual(undefined); + + list = await storage.listObjects(ctx, testDir, specialDirCache); + expect(list.sort()).toEqual([testFile1, testFile2].sort()); + }); + + test("deletePath", async () => { + let list, res; + res = await storage.deletePath(ctx, testDir, specialDirCache); + expect(res).toEqual(undefined); + + list = await storage.listObjects(ctx, testDir, specialDirCache); + expect(list.sort()).toEqual([].sort()); + + res = await storage.deletePath(ctx, testDir, specialDirForgotten); + expect(res).toEqual(undefined); + + list = await storage.listObjects(ctx, testDir, specialDirForgotten); + expect(list.sort()).toEqual([].sort()); + }); +}); From a09ff06aa8fea435c696057d0d08069dc03ce80a Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Mon, 1 Apr 2024 01:53:02 +0300 Subject: [PATCH 13/48] [feature] Add error CONVERT_LIMITS(-10) for conversion service --- Common/sources/utils.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Common/sources/utils.js b/Common/sources/utils.js index 48008e612..02348d523 100644 --- a/Common/sources/utils.js +++ b/Common/sources/utils.js @@ -534,6 +534,8 @@ exports.mapAscServerErrorToOldError = function(error) { res = -7; break; case constants.CONVERT_LIMITS : + res = -10; + break; case constants.CONVERT_NEED_PARAMS : case constants.CONVERT_LIBREOFFICE : case constants.CONVERT_CORRUPTED : From 0d2592f648684a1e6abf4e2cf05575ac19e43dd0 Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Mon, 1 Apr 2024 13:22:57 +0300 Subject: [PATCH 14/48] [bug] Save standard pdf on form submission instead of extended pdf --- FileConverter/sources/converter.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/FileConverter/sources/converter.js b/FileConverter/sources/converter.js index a66800166..777146163 100644 --- a/FileConverter/sources/converter.js +++ b/FileConverter/sources/converter.js @@ -341,13 +341,15 @@ function* isUselessConvertion(ctx, task, cmd) { return constants.NO_ERROR; } async function changeFormatToExtendedPdf(ctx, dataConvert, cmd) { + let forceSave = cmd.getForceSave(); + let isSendForm = forceSave && forceSave.getType() === commonDefines.c_oAscForceSaveTypes.Form; let originFormat = cmd.getOriginFormat(); let isOriginFormatWithForms = constants.AVS_OFFICESTUDIO_FILE_CROSSPLATFORM_PDF === originFormat || constants.AVS_OFFICESTUDIO_FILE_DOCUMENT_OFORM === originFormat || constants.AVS_OFFICESTUDIO_FILE_DOCUMENT_DOCXF === originFormat; let isFormatToPdf = constants.AVS_OFFICESTUDIO_FILE_CROSSPLATFORM_PDF === dataConvert.formatTo || constants.AVS_OFFICESTUDIO_FILE_CROSSPLATFORM_PDFA === dataConvert.formatTo; - if (isFormatToPdf && isOriginFormatWithForms) { + if (isFormatToPdf && isOriginFormatWithForms && !isSendForm) { let format = await formatChecker.getDocumentFormatByFile(dataConvert.fileFrom); if (constants.AVS_OFFICESTUDIO_FILE_CANVAS_WORD === format) { ctx.logger.debug('change format to extended pdf'); From 4acdd458cdfe4fa96e5844e8c030cca3e0cfd082 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 29 Mar 2024 10:49:23 +0000 Subject: [PATCH 15/48] Bump express from 4.18.2 to 4.19.2 in /DocService Bumps [express](https://github.com/expressjs/express) from 4.18.2 to 4.19.2. - [Release notes](https://github.com/expressjs/express/releases) - [Changelog](https://github.com/expressjs/express/blob/master/History.md) - [Commits](https://github.com/expressjs/express/compare/4.18.2...4.19.2) --- updated-dependencies: - dependency-name: express dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- DocService/npm-shrinkwrap.json | 61 +++++++++++++++++++++++----------- DocService/package.json | 2 +- 2 files changed, 42 insertions(+), 21 deletions(-) diff --git a/DocService/npm-shrinkwrap.json b/DocService/npm-shrinkwrap.json index d384c95c5..26c482ab0 100644 --- a/DocService/npm-shrinkwrap.json +++ b/DocService/npm-shrinkwrap.json @@ -1148,9 +1148,9 @@ "integrity": "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==" }, "cookie": { - "version": "0.5.0", - "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.5.0.tgz", - "integrity": "sha512-YZ3GUyn/o8gfKJlnlX7g7xq4gyO6OSuhGPKaaGssGB2qgDUS0gPgtTvoyZLTt9Ab6dC4hfc9dV5arkvc/OCmrw==" + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.6.0.tgz", + "integrity": "sha512-U71cyTamuh1CRNCfpGY6to28lxvNwPG4Guz/EVjgf3Jmzv0vlDp1atT9eS5dDjMYHucpHbWns6Lwf3BKz6svdw==" }, "cookie-signature": { "version": "1.0.6", @@ -1519,16 +1519,16 @@ "integrity": "sha512-c2bQfLNbMzLPmzQuOr8fy0csy84WmwnER81W88DzTp9CYNPJ6yzOj2EZAh9pywYpqHnshVLHQJ8WzldAyfY+Iw==" }, "express": { - "version": "4.18.2", - "resolved": "https://registry.npmjs.org/express/-/express-4.18.2.tgz", - "integrity": "sha512-5/PsL6iGPdfQ/lKM1UuielYgv3BUoJfz1aUwU9vHZ+J7gyvwdQXFEBIEIaxeGf0GIcreATNyBExtalisDbuMqQ==", + "version": "4.19.2", + "resolved": "https://registry.npmjs.org/express/-/express-4.19.2.tgz", + "integrity": "sha512-5T6nhjsT+EOMzuck8JjBHARTHfMht0POzlA60WV2pMD3gyXw2LZnZ+ueGdNxG+0calOJcWKbpFcuzLZ91YWq9Q==", "requires": { "accepts": "~1.3.8", "array-flatten": "1.1.1", - "body-parser": "1.20.1", + "body-parser": "1.20.2", "content-disposition": "0.5.4", "content-type": "~1.0.4", - "cookie": "0.5.0", + "cookie": "0.6.0", "cookie-signature": "1.0.6", "debug": "2.6.9", "depd": "2.0.0", @@ -1556,15 +1556,30 @@ "vary": "~1.1.2" }, "dependencies": { - "accepts": { - "version": "1.3.8", - "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", - "integrity": "sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==", + "body-parser": { + "version": "1.20.2", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.2.tgz", + "integrity": "sha512-ml9pReCu3M61kGlqoTm2umSXTlRTuGTx0bfYj+uIUKKYycG5NtSbeetV3faSU6R7ajOPw0g/J1PvK4qNy7s5bA==", "requires": { - "mime-types": "~2.1.34", - "negotiator": "0.6.3" + "bytes": "3.1.2", + "content-type": "~1.0.5", + "debug": "2.6.9", + "depd": "2.0.0", + "destroy": "1.2.0", + "http-errors": "2.0.0", + "iconv-lite": "0.4.24", + "on-finished": "2.4.1", + "qs": "6.11.0", + "raw-body": "2.5.2", + "type-is": "~1.6.18", + "unpipe": "1.0.0" } }, + "bytes": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", + "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==" + }, "depd": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", @@ -1583,11 +1598,6 @@ "mime-db": "1.52.0" } }, - "negotiator": { - "version": "0.6.3", - "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", - "integrity": "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==" - }, "on-finished": { "version": "2.4.1", "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", @@ -1596,6 +1606,17 @@ "ee-first": "1.1.1" } }, + "raw-body": { + "version": "2.5.2", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.2.tgz", + "integrity": "sha512-8zGqypfENjCIqGhgXToC8aB2r7YrBX+AQAfIPs/Mlk+BtPTztOvTS01NRW/3Eh60J+a48lt8qsCzirQ6loCVfA==", + "requires": { + "bytes": "3.1.2", + "http-errors": "2.0.0", + "iconv-lite": "0.4.24", + "unpipe": "1.0.0" + } + }, "safe-buffer": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", @@ -2884,7 +2905,7 @@ }, "readable-stream": { "version": "2.3.6", - "resolved": "http://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", "integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==", "requires": { "core-util-is": "~1.0.0", diff --git a/DocService/package.json b/DocService/package.json index 5dc7d96aa..2ca09221d 100644 --- a/DocService/package.json +++ b/DocService/package.json @@ -22,7 +22,7 @@ "dmdb": "1.0.14280", "ejs": "3.1.8", "exif-parser": "0.1.12", - "express": "4.18.2", + "express": "4.19.2", "fakeredis": "2.0.0", "ioredis": "5.3.1", "jimp": "0.22.10", From 45d9832f1c3014ded18ade1ab17d7adac8c93a31 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 26 Mar 2024 09:59:51 +0000 Subject: [PATCH 16/48] Bump express from 4.16.4 to 4.19.2 in /SpellChecker Bumps [express](https://github.com/expressjs/express) from 4.16.4 to 4.19.2. - [Release notes](https://github.com/expressjs/express/releases) - [Changelog](https://github.com/expressjs/express/blob/master/History.md) - [Commits](https://github.com/expressjs/express/compare/4.16.4...4.19.2) --- updated-dependencies: - dependency-name: express dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- SpellChecker/npm-shrinkwrap.json | 465 ++++++++++++++++++++----------- SpellChecker/package.json | 2 +- 2 files changed, 309 insertions(+), 158 deletions(-) diff --git a/SpellChecker/npm-shrinkwrap.json b/SpellChecker/npm-shrinkwrap.json index ecc51d9d3..7cd0edf11 100644 --- a/SpellChecker/npm-shrinkwrap.json +++ b/SpellChecker/npm-shrinkwrap.json @@ -5,40 +5,54 @@ "requires": true, "dependencies": { "accepts": { - "version": "1.3.5", - "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.5.tgz", - "integrity": "sha1-63d99gEXI6OxTopywIBcjoZ0a9I=", + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", + "integrity": "sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==", "requires": { - "mime-types": "~2.1.18", - "negotiator": "0.6.1" + "mime-types": "~2.1.34", + "negotiator": "0.6.3" } }, "array-flatten": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", - "integrity": "sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=" + "integrity": "sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==" }, "body-parser": { - "version": "1.18.3", - "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.18.3.tgz", - "integrity": "sha1-WykhmP/dVTs6DyDe0FkrlWlVyLQ=", + "version": "1.20.2", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.2.tgz", + "integrity": "sha512-ml9pReCu3M61kGlqoTm2umSXTlRTuGTx0bfYj+uIUKKYycG5NtSbeetV3faSU6R7ajOPw0g/J1PvK4qNy7s5bA==", "requires": { - "bytes": "3.0.0", - "content-type": "~1.0.4", + "bytes": "3.1.2", + "content-type": "~1.0.5", "debug": "2.6.9", - "depd": "~1.1.2", - "http-errors": "~1.6.3", - "iconv-lite": "0.4.23", - "on-finished": "~2.3.0", - "qs": "6.5.2", - "raw-body": "2.3.3", - "type-is": "~1.6.16" + "depd": "2.0.0", + "destroy": "1.2.0", + "http-errors": "2.0.0", + "iconv-lite": "0.4.24", + "on-finished": "2.4.1", + "qs": "6.11.0", + "raw-body": "2.5.2", + "type-is": "~1.6.18", + "unpipe": "1.0.0" } }, "bytes": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.0.0.tgz", - "integrity": "sha1-0ygVQE1olpn4Wk6k+odV3ROpYEg=" + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", + "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==" + }, + "call-bind": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.7.tgz", + "integrity": "sha512-GHTSNSYICQ7scH7sZ+M2rFopRoLh8t2bLSW6BbgrtLsahOIB5iyAVJf9GjWK3cYTDaMj4XdBpM1cA6pIS0Kv2w==", + "requires": { + "es-define-property": "^1.0.0", + "es-errors": "^1.3.0", + "function-bind": "^1.1.2", + "get-intrinsic": "^1.2.4", + "set-function-length": "^1.2.1" + } }, "co": { "version": "4.6.0", @@ -54,24 +68,34 @@ } }, "content-disposition": { - "version": "0.5.2", - "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.2.tgz", - "integrity": "sha1-DPaLud318r55YcOoUXjLhdunjLQ=" + "version": "0.5.4", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz", + "integrity": "sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==", + "requires": { + "safe-buffer": "5.2.1" + }, + "dependencies": { + "safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==" + } + } }, "content-type": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz", - "integrity": "sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA==" + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.5.tgz", + "integrity": "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==" }, "cookie": { - "version": "0.3.1", - "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.3.1.tgz", - "integrity": "sha1-5+Ch+e9DtMi6klxcWpboBtFoc7s=" + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.6.0.tgz", + "integrity": "sha512-U71cyTamuh1CRNCfpGY6to28lxvNwPG4Guz/EVjgf3Jmzv0vlDp1atT9eS5dDjMYHucpHbWns6Lwf3BKz6svdw==" }, "cookie-signature": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz", - "integrity": "sha1-4wOogrNCzD7oylE6eZmXNNqzriw=" + "integrity": "sha512-QADzlaHc8icV8I7vbaJXJwod9HWYp8uCqf1xa4OfNu1T7JVxQIrUgOWtHdNDtPiywmFbiS12VjotIXLrKM3orQ==" }, "debug": { "version": "2.6.9", @@ -81,71 +105,102 @@ "ms": "2.0.0" } }, + "define-data-property": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/define-data-property/-/define-data-property-1.1.4.tgz", + "integrity": "sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A==", + "requires": { + "es-define-property": "^1.0.0", + "es-errors": "^1.3.0", + "gopd": "^1.0.1" + } + }, "depd": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", - "integrity": "sha1-m81S4UwJd2PnSbJ0xDRu0uVgtak=" + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", + "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==" }, "destroy": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.0.4.tgz", - "integrity": "sha1-l4hXRCxEdJ5CBmE+N5RiBYJqvYA=" + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.2.0.tgz", + "integrity": "sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg==" }, "ee-first": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", - "integrity": "sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0=" + "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==" }, "encodeurl": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", - "integrity": "sha1-rT/0yG7C0CkyL1oCw6mmBslbP1k=" + "integrity": "sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w==" + }, + "es-define-property": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.0.tgz", + "integrity": "sha512-jxayLKShrEqqzJ0eumQbVhTYQM27CfT1T35+gCgDFoL82JLsXqTJ76zv6A0YLOgEnLUMvLzsDsGIrl8NFpT2gQ==", + "requires": { + "get-intrinsic": "^1.2.4" + } + }, + "es-errors": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==" }, "escape-html": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", - "integrity": "sha1-Aljq5NPQwJdN4cFpGI7wBR0dGYg=" + "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==" }, "etag": { "version": "1.8.1", "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", - "integrity": "sha1-Qa4u62XvpiJorr/qg6x9eSmbCIc=" + "integrity": "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==" }, "express": { - "version": "4.16.4", - "resolved": "https://registry.npmjs.org/express/-/express-4.16.4.tgz", - "integrity": "sha512-j12Uuyb4FMrd/qQAm6uCHAkPtO8FDTRJZBDd5D2KOL2eLaz1yUNdUB/NOIyq0iU4q4cFarsUCrnFDPBcnksuOg==", + "version": "4.19.2", + "resolved": "https://registry.npmjs.org/express/-/express-4.19.2.tgz", + "integrity": "sha512-5T6nhjsT+EOMzuck8JjBHARTHfMht0POzlA60WV2pMD3gyXw2LZnZ+ueGdNxG+0calOJcWKbpFcuzLZ91YWq9Q==", "requires": { - "accepts": "~1.3.5", + "accepts": "~1.3.8", "array-flatten": "1.1.1", - "body-parser": "1.18.3", - "content-disposition": "0.5.2", + "body-parser": "1.20.2", + "content-disposition": "0.5.4", "content-type": "~1.0.4", - "cookie": "0.3.1", + "cookie": "0.6.0", "cookie-signature": "1.0.6", "debug": "2.6.9", - "depd": "~1.1.2", + "depd": "2.0.0", "encodeurl": "~1.0.2", "escape-html": "~1.0.3", "etag": "~1.8.1", - "finalhandler": "1.1.1", + "finalhandler": "1.2.0", "fresh": "0.5.2", + "http-errors": "2.0.0", "merge-descriptors": "1.0.1", "methods": "~1.1.2", - "on-finished": "~2.3.0", - "parseurl": "~1.3.2", + "on-finished": "2.4.1", + "parseurl": "~1.3.3", "path-to-regexp": "0.1.7", - "proxy-addr": "~2.0.4", - "qs": "6.5.2", - "range-parser": "~1.2.0", - "safe-buffer": "5.1.2", - "send": "0.16.2", - "serve-static": "1.13.2", - "setprototypeof": "1.1.0", - "statuses": "~1.4.0", - "type-is": "~1.6.16", + "proxy-addr": "~2.0.7", + "qs": "6.11.0", + "range-parser": "~1.2.1", + "safe-buffer": "5.2.1", + "send": "0.18.0", + "serve-static": "1.15.0", + "setprototypeof": "1.2.0", + "statuses": "2.0.1", + "type-is": "~1.6.18", "utils-merge": "1.0.1", "vary": "~1.1.2" + }, + "dependencies": { + "safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==" + } } }, "faye-websocket": { @@ -157,38 +212,90 @@ } }, "finalhandler": { - "version": "1.1.1", - "resolved": "http://registry.npmjs.org/finalhandler/-/finalhandler-1.1.1.tgz", - "integrity": "sha512-Y1GUDo39ez4aHAw7MysnUD5JzYX+WaIj8I57kO3aEPT1fFRL4sr7mjei97FgnwhAyyzRYmQZaTHb2+9uZ1dPtg==", + "version": "1.2.0", + "resolved": "http://registry.npmjs.org/finalhandler/-/finalhandler-1.2.0.tgz", + "integrity": "sha512-5uXcUVftlQMFnWC9qu/svkWv3GTd2PfUhK/3PLkYNAe7FbqJMt3515HaxE6eRL74GdsriiwujiawdaB1BpEISg==", "requires": { "debug": "2.6.9", "encodeurl": "~1.0.2", "escape-html": "~1.0.3", - "on-finished": "~2.3.0", - "parseurl": "~1.3.2", - "statuses": "~1.4.0", + "on-finished": "2.4.1", + "parseurl": "~1.3.3", + "statuses": "2.0.1", "unpipe": "~1.0.0" } }, "forwarded": { - "version": "0.1.2", - "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.1.2.tgz", - "integrity": "sha1-mMI9qxF1ZXuMBXPozszZGw/xjIQ=" + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", + "integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==" }, "fresh": { "version": "0.5.2", "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", - "integrity": "sha1-PYyt2Q2XZWn6g1qx+OSyOhBWBac=" + "integrity": "sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==" + }, + "function-bind": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==" + }, + "get-intrinsic": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.4.tgz", + "integrity": "sha512-5uYhsJH8VJBTv7oslg4BznJYhDoRI6waYCxMmCdnTrcCrHA/fCFKoTFz2JKKE0HdDFUF7/oQuhzumXJK7paBRQ==", + "requires": { + "es-errors": "^1.3.0", + "function-bind": "^1.1.2", + "has-proto": "^1.0.1", + "has-symbols": "^1.0.3", + "hasown": "^2.0.0" + } + }, + "gopd": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.0.1.tgz", + "integrity": "sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA==", + "requires": { + "get-intrinsic": "^1.1.3" + } + }, + "has-property-descriptors": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.2.tgz", + "integrity": "sha512-55JNKuIW+vq4Ke1BjOTjM2YctQIvCT7GFzHwmfZPGo5wnrgkid0YQtnAleFSqumZm4az3n2BS+erby5ipJdgrg==", + "requires": { + "es-define-property": "^1.0.0" + } + }, + "has-proto": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/has-proto/-/has-proto-1.0.3.tgz", + "integrity": "sha512-SJ1amZAJUiZS+PhsVLf5tGydlaVB8EdFpaSO4gmiUKUOxk8qzn5AIy4ZeJUmh22znIdk/uMAUT2pl3FxzVUH+Q==" + }, + "has-symbols": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz", + "integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==" + }, + "hasown": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", + "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==", + "requires": { + "function-bind": "^1.1.2" + } }, "http-errors": { - "version": "1.6.3", - "resolved": "http://registry.npmjs.org/http-errors/-/http-errors-1.6.3.tgz", - "integrity": "sha1-i1VoC7S+KDoLW/TqLjhYC+HZMg0=", + "version": "2.0.0", + "resolved": "http://registry.npmjs.org/http-errors/-/http-errors-2.0.0.tgz", + "integrity": "sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ==", "requires": { - "depd": "~1.1.2", - "inherits": "2.0.3", - "setprototypeof": "1.1.0", - "statuses": ">= 1.4.0 < 2" + "depd": "2.0.0", + "inherits": "2.0.4", + "setprototypeof": "1.2.0", + "statuses": "2.0.1", + "toidentifier": "1.0.1" } }, "http-parser-js": { @@ -197,22 +304,22 @@ "integrity": "sha512-t7hjvef/5HEK7RWTdUzVUhl8zkEu+LlaE0IYzdMuvbSDipxBRpOn4Uhw8ZyECEa808iVT8XCjzo6xmYt4CiLZg==" }, "iconv-lite": { - "version": "0.4.23", - "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.23.tgz", - "integrity": "sha512-neyTUVFtahjf0mB3dZT77u+8O0QB89jFdnBkd5P1JgYPbPaia3gXXOVL2fq8VyU2gMMD7SaN7QukTB/pmXYvDA==", + "version": "0.4.24", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", + "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", "requires": { "safer-buffer": ">= 2.1.2 < 3" } }, "inherits": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", - "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=" + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" }, "ipaddr.js": { - "version": "1.8.0", - "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.8.0.tgz", - "integrity": "sha1-6qM9bd16zo9/b+DJygRA5wZzix4=" + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", + "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==" }, "json5": { "version": "1.0.1", @@ -225,34 +332,34 @@ "media-typer": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", - "integrity": "sha1-hxDXrwqmJvj/+hzgAWhUUmMlV0g=" + "integrity": "sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==" }, "merge-descriptors": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz", - "integrity": "sha1-sAqqVW3YtEVoFQ7J0blT8/kMu2E=" + "integrity": "sha512-cCi6g3/Zr1iqQi6ySbseM1Xvooa98N0w31jzUYrXPX2xqObmFGHJ0tQ5u74H3mVh7wLouTseZyYIq39g8cNp1w==" }, "methods": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz", - "integrity": "sha1-VSmk1nZUE07cxSZmVoNbD4Ua/O4=" + "integrity": "sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w==" }, "mime": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/mime/-/mime-1.4.1.tgz", - "integrity": "sha512-KI1+qOZu5DcW6wayYHSzR/tXKCDC5Om4s1z2QJjDULzLcmf3DvzS7oluY4HCTrc+9FiKmWUgeNLg7W3uIQvxtQ==" + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", + "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==" }, "mime-db": { - "version": "1.37.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.37.0.tgz", - "integrity": "sha512-R3C4db6bgQhlIhPU48fUtdVmKnflq+hRdad7IyKhtFj06VPNVdk2RhiYL3UjQIlso8L+YxAtFkobT0VK+S/ybg==" + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==" }, "mime-types": { - "version": "2.1.21", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.21.tgz", - "integrity": "sha512-3iL6DbwpyLzjR3xHSFNFeb9Nz/M8WDkX33t1GFQnFOllWk8pOrh/LSrB5OXlnlW5P9LH73X6loW/eogc+F5lJg==", + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", "requires": { - "mime-db": "~1.37.0" + "mime-db": "1.52.0" } }, "minimist": { @@ -263,12 +370,12 @@ "ms": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" }, "negotiator": { - "version": "0.6.1", - "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.1.tgz", - "integrity": "sha1-KzJxhOiZIQEXeyhWP7XnECrNDKk=" + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", + "integrity": "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==" }, "node-addon-api": { "version": "3.0.2", @@ -282,51 +389,59 @@ "node-addon-api": "*" } }, + "object-inspect": { + "version": "1.13.1", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.1.tgz", + "integrity": "sha512-5qoj1RUiKOMsCCNLV1CBiPYE10sziTsnmNxkAI/rZhiD63CF7IqdFGC/XzjWjpSgLf0LxXX3bDFIh0E18f6UhQ==" + }, "on-finished": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz", - "integrity": "sha1-IPEzZIGwg811M3mSoWlxqi2QaUc=", + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", + "integrity": "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==", "requires": { "ee-first": "1.1.1" } }, "parseurl": { - "version": "1.3.2", - "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.2.tgz", - "integrity": "sha1-/CidTtiZMRlGDBViUyYs3I3mW/M=" + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", + "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==" }, "path-to-regexp": { "version": "0.1.7", "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz", - "integrity": "sha1-32BBeABfUi8V60SQ5yR6G/qmf4w=" + "integrity": "sha512-5DFkuoqlv1uYQKxy8omFBeJPQcdoE07Kv2sferDCrAq1ohOU+MSDswDIbnx3YAM60qIOnYa53wBhXW0EbMonrQ==" }, "proxy-addr": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.4.tgz", - "integrity": "sha512-5erio2h9jp5CHGwcybmxmVqHmnCBZeewlfJ0pex+UW7Qny7OOZXTtH56TGNyBizkgiOwhJtMKrVzDTeKcySZwA==", + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", + "integrity": "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==", "requires": { - "forwarded": "~0.1.2", - "ipaddr.js": "1.8.0" + "forwarded": "0.2.0", + "ipaddr.js": "1.9.1" } }, "qs": { - "version": "6.5.2", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.2.tgz", - "integrity": "sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA==" + "version": "6.11.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.11.0.tgz", + "integrity": "sha512-MvjoMCJwEarSbUYk5O+nmoSzSutSsTwF85zcHPQ9OrlFoZOYIjaqBAJIqIXjptyD5vThxGq52Xu/MaJzRkIk4Q==", + "requires": { + "side-channel": "^1.0.4" + } }, "range-parser": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.0.tgz", - "integrity": "sha1-9JvmtIeJTdxA3MlKMi9hEJLgDV4=" + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", + "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==" }, "raw-body": { - "version": "2.3.3", - "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.3.3.tgz", - "integrity": "sha512-9esiElv1BrZoI3rCDuOuKCBRbuApGGaDPQfjSflGxdy4oyzqghxu6klEkkVIvBje+FF0BX9coEv8KqW6X/7njw==", + "version": "2.5.2", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.2.tgz", + "integrity": "sha512-8zGqypfENjCIqGhgXToC8aB2r7YrBX+AQAfIPs/Mlk+BtPTztOvTS01NRW/3Eh60J+a48lt8qsCzirQ6loCVfA==", "requires": { - "bytes": "3.0.0", - "http-errors": "1.6.3", - "iconv-lite": "0.4.23", + "bytes": "3.1.2", + "http-errors": "2.0.0", + "iconv-lite": "0.4.24", "unpipe": "1.0.0" } }, @@ -341,40 +456,71 @@ "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" }, "send": { - "version": "0.16.2", - "resolved": "https://registry.npmjs.org/send/-/send-0.16.2.tgz", - "integrity": "sha512-E64YFPUssFHEFBvpbbjr44NCLtI1AohxQ8ZSiJjQLskAdKuriYEP6VyGEsRDH8ScozGpkaX1BGvhanqCwkcEZw==", + "version": "0.18.0", + "resolved": "https://registry.npmjs.org/send/-/send-0.18.0.tgz", + "integrity": "sha512-qqWzuOjSFOuqPjFe4NOsMLafToQQwBSOEpS+FwEt3A2V3vKubTquT3vmLTQpFgMXp8AlFWFuP1qKaJZOtPpVXg==", "requires": { "debug": "2.6.9", - "depd": "~1.1.2", - "destroy": "~1.0.4", + "depd": "2.0.0", + "destroy": "1.2.0", "encodeurl": "~1.0.2", "escape-html": "~1.0.3", "etag": "~1.8.1", "fresh": "0.5.2", - "http-errors": "~1.6.2", - "mime": "1.4.1", - "ms": "2.0.0", - "on-finished": "~2.3.0", - "range-parser": "~1.2.0", - "statuses": "~1.4.0" + "http-errors": "2.0.0", + "mime": "1.6.0", + "ms": "2.1.3", + "on-finished": "2.4.1", + "range-parser": "~1.2.1", + "statuses": "2.0.1" + }, + "dependencies": { + "ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" + } } }, "serve-static": { - "version": "1.13.2", - "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.13.2.tgz", - "integrity": "sha512-p/tdJrO4U387R9oMjb1oj7qSMaMfmOyd4j9hOFoxZe2baQszgHcSWjuya/CiT5kgZZKRudHNOA0pYXOl8rQ5nw==", + "version": "1.15.0", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.15.0.tgz", + "integrity": "sha512-XGuRDNjXUijsUL0vl6nSD7cwURuzEgglbOaFuZM9g3kwDXOWVTck0jLzjPzGD+TazWbboZYu52/9/XPdUgne9g==", "requires": { "encodeurl": "~1.0.2", "escape-html": "~1.0.3", - "parseurl": "~1.3.2", - "send": "0.16.2" + "parseurl": "~1.3.3", + "send": "0.18.0" + } + }, + "set-function-length": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/set-function-length/-/set-function-length-1.2.2.tgz", + "integrity": "sha512-pgRc4hJ4/sNjWCSS9AmnS40x3bNMDTknHgL5UaMBTMyJnU90EgWh1Rz+MC9eFu4BuN/UwZjKQuY/1v3rM7HMfg==", + "requires": { + "define-data-property": "^1.1.4", + "es-errors": "^1.3.0", + "function-bind": "^1.1.2", + "get-intrinsic": "^1.2.4", + "gopd": "^1.0.1", + "has-property-descriptors": "^1.0.2" } }, "setprototypeof": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.1.0.tgz", - "integrity": "sha512-BvE/TwpZX4FXExxOxZyRGQQv651MSwmWKZGqvmPcRIjDqWub67kTKuIMx43cZZrS/cBBzwBcNDWoFxt2XEFIpQ==" + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", + "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==" + }, + "side-channel": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.6.tgz", + "integrity": "sha512-fDW/EZ6Q9RiO8eFG8Hj+7u/oW+XrPTIChwCOM2+th2A6OblDtYYIpve9m+KvI9Z4C9qSEXlaGR6bTEYHReuglA==", + "requires": { + "call-bind": "^1.0.7", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.4", + "object-inspect": "^1.13.1" + } }, "sockjs": { "version": "0.3.21", @@ -387,28 +533,33 @@ } }, "statuses": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.4.0.tgz", - "integrity": "sha512-zhSCtt8v2NDrRlPQpCNtw/heZLtfUDqxBM1udqikb/Hbk52LK4nQSwr10u77iopCW5LsyHpuXS0GnEc48mLeew==" + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz", + "integrity": "sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==" + }, + "toidentifier": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", + "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==" }, "type-is": { - "version": "1.6.16", - "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.16.tgz", - "integrity": "sha512-HRkVv/5qY2G6I8iab9cI7v1bOIdhm94dVjQCPFElW9W+3GeDOSHmy2EBYe4VTApuzolPcmgFTN3ftVJRKR2J9Q==", + "version": "1.6.18", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz", + "integrity": "sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==", "requires": { "media-typer": "0.3.0", - "mime-types": "~2.1.18" + "mime-types": "~2.1.24" } }, "unpipe": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", - "integrity": "sha1-sr9O6FFKrmFltIF4KdIbLvSZBOw=" + "integrity": "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==" }, "utils-merge": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", - "integrity": "sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM=" + "integrity": "sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA==" }, "uuid": { "version": "3.4.0", @@ -418,7 +569,7 @@ "vary": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", - "integrity": "sha1-IpnwLG3tMNSllhsLn3RSShj2NPw=" + "integrity": "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==" }, "websocket-driver": { "version": "0.7.4", diff --git a/SpellChecker/package.json b/SpellChecker/package.json index ebb74c971..62d2d250b 100644 --- a/SpellChecker/package.json +++ b/SpellChecker/package.json @@ -7,7 +7,7 @@ "dependencies": { "co": "4.6.0", "config": "2.0.1", - "express": "4.16.4", + "express": "4.19.2", "nodehun": "git+https://git@github.com/ONLYOFFICE/nodehun.git#2411a56828c7d58214c61781b4a5c63d18adba99", "sockjs": "0.3.21" }, From d8d285e7533d4bc28f5d558931c52c49c2e04184 Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Wed, 3 Apr 2024 01:38:13 +0300 Subject: [PATCH 17/48] [bug] Refactor wopi discovery for new pdf editor; Fix downloadFile for editnew action; For bug 67135 --- Common/config/default.json | 12 ++--- Common/sources/constants.js | 6 +-- DocService/sources/DocsCoServer.js | 2 +- DocService/sources/canvasservice.js | 65 ++++++++++++++++----------- DocService/sources/wopiClient.js | 69 ++++++++++++++++++++--------- FileConverter/sources/converter.js | 28 ++++++++---- 6 files changed, 116 insertions(+), 66 deletions(-) diff --git a/Common/config/default.json b/Common/config/default.json index 06d14044c..fcccf356c 100644 --- a/Common/config/default.json +++ b/Common/config/default.json @@ -86,11 +86,13 @@ "favIconUrlWord" : "/web-apps/apps/documenteditor/main/resources/img/favicon.ico", "favIconUrlCell" : "/web-apps/apps/spreadsheeteditor/main/resources/img/favicon.ico", "favIconUrlSlide" : "/web-apps/apps/presentationeditor/main/resources/img/favicon.ico", + "favIconUrlPdf" : "/web-apps/apps/pdfeditor/main/resources/img/favicon.ico", "fileInfoBlockList" : ["FileUrl"], - "pdfView": ["pdf", "djvu", "xps", "oxps"], - "wordView": ["doc", "dotx", "dotm", "dot", "fodt", "ott", "rtf", "mht", "mhtml", "html", "htm", "xml", "epub", "fb2", "sxw", "stw", "wps", "wpt", "oform"], - "wordEdit": ["docx", "docm", "docxf", "odt", "txt"], - "wordForm": ["pdf"], + "pdfView": ["djvu", "xps", "oxps"], + "pdfEdit": ["pdf"], + "forms": ["pdf"], + "wordView": ["doc", "dotx", "dotm", "dot", "fodt", "ott", "rtf", "mht", "mhtml", "html", "htm", "xml", "epub", "fb2", "sxw", "stw", "wps", "wpt", "docxf", "oform"], + "wordEdit": ["docx", "docm", "odt", "txt"], "cellView": ["xls", "xlsb", "xltx", "xltm", "xlt", "fods", "ots", "sxc", "xml", "et", "ett"], "cellEdit": ["xlsx", "xlsm", "ods", "csv"], "slideView": ["ppt", "ppsx", "ppsm", "pps", "potx", "potm", "pot", "fodp", "otp", "sxi", "dps", "dpt"], @@ -105,7 +107,7 @@ "privateKeyOld": "MIIEowIBAAKCAQEAqnro3nUUjvZK1i7UqeOlXmCrVPiDtHlRgIPReAjt2nKL1GG3SBXO6N0aPbiM5rtK0XRPUoLmKu2rYvSJ/Kmkdp14a/3uiEl788VVn0hb/l9OuQtH3HBjmM0/LKRgJQuU3LgHI67uRVZYtSJ/n9fYdZqnLfveLsrgZpgRCoabrp+H5Uem9N+x0OJR3LpToVRZhzSkYQrxnERJmF3bhR5yF8Zn+3BoSiUpVOCAvJRAYl8cAIs3BwQcTEyXJjnt+wW5Q1VyKr+bXp/39+tnugQeTe1jjdPy6rOTftQwzjro81oZpOMazwwR1aeQuQWCrmHQZqyV3Rvo6X3xYlOQnlo1/wIDAQABAoIBAQCKtUSBs8tNYrGTQTlBHXrwpkDg+u7WSZt5sEcfnkxA39BLtlHU8gGO0E9Ihr8GAL+oWjUsEltJ9GTtN8CJ9lFdPVS8sTiCZR/YQOggmFRZTJyVzMrkXgF7Uwwiu3+KxLiTOZx9eRhfDBlTD8W9fXaegX2i2Xp2ohUhBHthEBLdaZTWFi5Sid/Y0dDzBeP6UIJorZ5D+1ybaeIVHjndpwNsIEUGUxPFLrkeiU8Rm4MJ9ahxfywcP7DjQoPGY9Ge5cBhpxfzERWf732wUD6o3+L9tvOBU00CLVjULbGZKTVE2FJMyXK9jr6Zor9Mkhomp6/8Agkr9rp+TPyelFGYEz8hAoGBAOEc09CrL3eYBkhNEcaMQzxBLvOGpg8kaDX5SaArHfl9+U9yzRqss4ARECanp9HuHfjMQo7iejao0ngDjL7BNMSaH74QlSsPOY2iOm8Qvx8/zb7g4h9r1zLjFZb3mpSA4snRZvvdiZ9ugbuVPmhXnDzRRMg45MibJeeOTJNylofRAoGBAMHfF/WutqKDoX25qZo9m74W4bttOj6oIDk1N4/c6M1Z1v/aptYSE06bkWngj9P46kqjaay4hgMtzyGruc5aojPx5MHHf5bo14+Jv4NzYtR2llrUxO+UJX7aCfUYXI7RC93GUmhpeQ414j7SNAXec58d7e+ETw+6cHiAWO4uOSTPAoGATPq5qDLR4Zi4FUNdn8LZPyKfNqHF6YmupT5hIgd8kZO1jKiaYNPL8jBjkIRmjBBcaXcYD5p85nImvumf2J9jNxPpZOpwyC/Fo5xlVROp97qu1eY7DTmodntXJ6/2SXAlnZQhHmHsrPtyG752f+HtyJJbbgiem8cKWDu+DfHybfECgYBbSLo1WiBwgN4nHqZ3E48jgA6le5azLeKOTTpuKKwNFMIhEkj//t7MYn+jhLL0Mf3PSwZU50Vidc1To1IHkbFSGBGIFHFFEzl8QnXEZS4hr/y3o/teezj0c6HAn8nlDRUzRVBEDXWMdV6kCcGpCccTIrqHzpqTY0vV0UkOTQFnDQKBgAxSEhm/gtCYJIMCBe+KBJT9uECV5xDQopTTjsGOkd4306EN2dyPOIlAfwM6K/0qWisa0Ei5i8TbRRuBeTTdLEYLqXCJ7fj5tdD1begBdSVtHQ2WHqzPJSuImTkFi9NXxd1XUyZFM3y6YQvlssSuL7QSxUIEtZHnrJTt3QDd10dj", "refreshLockInterval": "10m", "dummy" : { - "enable": true, + "enable": false, "sampleFilePath": "" } }, diff --git a/Common/sources/constants.js b/Common/sources/constants.js index 4fe783573..3299ca82c 100644 --- a/Common/sources/constants.js +++ b/Common/sources/constants.js @@ -282,15 +282,11 @@ exports.FILE_STATUS_UPDATE_VERSION = 'updateversion'; exports.ACTIVEMQ_QUEUE_PREFIX = 'queue://'; exports.ACTIVEMQ_TOPIC_PREFIX = 'topic://'; +exports.TEMPLATES_DEFAULT_LOCALE = 'en-US'; exports.TEMPLATES_FOLDER_LOCALE_COLLISON_MAP = { 'en': 'en-US', 'pt': 'pt-BR', 'zh': 'zh-CH', 'pt-PT': 'pt-PT', 'zh-TW': 'zh-TW' -}; -exports.SUPPORTED_TEMPLATES_EXTENSIONS = { - 'Word': ['docx', 'docxf'], - 'Excel': ['xlsx'], - 'PowerPoint': ['pptx'] }; \ No newline at end of file diff --git a/DocService/sources/DocsCoServer.js b/DocService/sources/DocsCoServer.js index e672010e3..e774b05ec 100644 --- a/DocService/sources/DocsCoServer.js +++ b/DocService/sources/DocsCoServer.js @@ -2250,7 +2250,7 @@ exports.install = function(server, callbackFunction) { } if (decoded.queryParams) { let queryParams = decoded.queryParams; - data.lang = queryParams.lang || queryParams.ui || "en-US"; + data.lang = queryParams.lang || queryParams.ui || constants.TEMPLATES_DEFAULT_LOCALE; } if (wopiClient.isWopiJwtToken(decoded)) { let fileInfo = decoded.fileInfo; diff --git a/DocService/sources/canvasservice.js b/DocService/sources/canvasservice.js index 8683f0384..9b92c5eb9 100644 --- a/DocService/sources/canvasservice.js +++ b/DocService/sources/canvasservice.js @@ -53,6 +53,7 @@ var statsDClient = require('./../../Common/sources/statsdclient'); var operationContext = require('./../../Common/sources/operationContext'); var tenantManager = require('./../../Common/sources/tenantManager'); var config = require('config'); +const path = require("path"); const cfgTypesUpload = config.get('services.CoAuthoring.utils.limits_image_types_upload'); const cfgImageSize = config.get('services.CoAuthoring.server.limits_image_size'); @@ -70,6 +71,7 @@ const cfgAssemblyFormatAsOrigin = config.get('services.CoAuthoring.server.assemb const cfgDownloadMaxBytes = config.get('FileConverter.converter.maxDownloadBytes'); const cfgDownloadTimeout = config.get('FileConverter.converter.downloadTimeout'); const cfgDownloadFileAllowExt = config.get('services.CoAuthoring.server.downloadFileAllowExt'); +const cfgNewFileTemplate = config.get('services.CoAuthoring.server.newFileTemplate'); var SAVE_TYPE_PART_START = 0; var SAVE_TYPE_PART = 1; @@ -1597,11 +1599,12 @@ exports.downloadFile = function(req, res) { const tenDownloadMaxBytes = ctx.getCfg('FileConverter.converter.maxDownloadBytes', cfgDownloadMaxBytes); const tenDownloadTimeout = ctx.getCfg('FileConverter.converter.downloadTimeout', cfgDownloadTimeout); const tenDownloadFileAllowExt = ctx.getCfg('services.CoAuthoring.server.downloadFileAllowExt', cfgDownloadFileAllowExt); + const tenNewFileTemplate = ctx.getCfg('services.CoAuthoring.server.newFileTemplate', cfgNewFileTemplate); let authorization; let isInJwtToken = false; let errorDescription; - let headers; + let headers, fromTemplate; let authRes = yield docsCoServer.getRequestParams(ctx, req); if (authRes.code === constants.NO_ERROR) { let decoded = authRes.params; @@ -1615,14 +1618,19 @@ exports.downloadFile = function(req, res) { url = decoded.url; isInJwtToken = true; } else if (wopiClient.isWopiJwtToken(decoded)) { - ({url, headers} = wopiClient.getWopiFileUrl(ctx, decoded.fileInfo, decoded.userAuth)); - let filterStatus = yield wopiClient.checkIpFilter(ctx, url); - if (0 === filterStatus) { - //todo false? (true because it passed checkIpFilter for wopi) - //todo use directIfIn - isInJwtToken = true; + if (decoded.fileInfo.Size === 0) { + //editnew case + fromTemplate = pathModule.extname(decoded.fileInfo.BaseFileName).substring(1); } else { - errorDescription = 'access deny'; + ({url, headers} = wopiClient.getWopiFileUrl(ctx, decoded.fileInfo, decoded.userAuth)); + let filterStatus = yield wopiClient.checkIpFilter(ctx, url); + if (0 === filterStatus) { + //todo false? (true because it passed checkIpFilter for wopi) + //todo use directIfIn + isInJwtToken = true; + } else { + errorDescription = 'access deny'; + } } } else if (!tenTokenEnableBrowser) { //todo token required @@ -1641,26 +1649,33 @@ exports.downloadFile = function(req, res) { res.sendStatus(403); return; } - if (utils.canIncludeOutboxAuthorization(ctx, url)) { - let secret = yield tenantManager.getTenantSecret(ctx, commonDefines.c_oAscSecretType.Outbox); - authorization = utils.fillJwtForRequest(ctx, {url: url}, secret, false); - } - let urlParsed = urlModule.parse(url); - let filterStatus = yield* utils.checkHostFilter(ctx, urlParsed.hostname); - if (0 !== filterStatus) { - ctx.logger.warn('Error downloadFile checkIpFilter error: url = %s', url); - res.sendStatus(filterStatus); - return; - } + if (fromTemplate) { + ctx.logger.debug('downloadFile from file template: %s', fromTemplate); + let locale = constants.TEMPLATES_DEFAULT_LOCALE; + let fileTemplatePath = pathModule.join(tenNewFileTemplate, locale, 'new.' + fromTemplate); + res.sendFile(pathModule.resolve(fileTemplatePath)); + } else { + if (utils.canIncludeOutboxAuthorization(ctx, url)) { + let secret = yield tenantManager.getTenantSecret(ctx, commonDefines.c_oAscSecretType.Outbox); + authorization = utils.fillJwtForRequest(ctx, {url: url}, secret, false); + } + let urlParsed = urlModule.parse(url); + let filterStatus = yield* utils.checkHostFilter(ctx, urlParsed.hostname); + if (0 !== filterStatus) { + ctx.logger.warn('Error downloadFile checkIpFilter error: url = %s', url); + res.sendStatus(filterStatus); + return; + } - if (req.get('Range')) { - if (!headers) { - headers = {}; + if (req.get('Range')) { + if (!headers) { + headers = {}; + } + headers['Range'] = req.get('Range'); } - headers['Range'] = req.get('Range'); - } - yield utils.downloadUrlPromise(ctx, url, tenDownloadTimeout, tenDownloadMaxBytes, authorization, isInJwtToken, headers, res); + yield utils.downloadUrlPromise(ctx, url, tenDownloadTimeout, tenDownloadMaxBytes, authorization, isInJwtToken, headers, res); + } if (clientStatsD) { clientStatsD.timing('coauth.downloadFile', new Date() - startDate); diff --git a/DocService/sources/wopiClient.js b/DocService/sources/wopiClient.js index a867c9379..238e2c97c 100644 --- a/DocService/sources/wopiClient.js +++ b/DocService/sources/wopiClient.js @@ -67,16 +67,18 @@ const cfgMaxDownloadBytes = config.get('FileConverter.converter.maxDownloadBytes const cfgWopiFileInfoBlockList = config.get('wopi.fileInfoBlockList'); const cfgWopiWopiZone = config.get('wopi.wopiZone'); const cfgWopiPdfView = config.get('wopi.pdfView'); +const cfgWopiPdfEdit = config.get('wopi.pdfEdit'); const cfgWopiWordView = config.get('wopi.wordView'); const cfgWopiWordEdit = config.get('wopi.wordEdit'); const cfgWopiCellView = config.get('wopi.cellView'); const cfgWopiCellEdit = config.get('wopi.cellEdit'); const cfgWopiSlideView = config.get('wopi.slideView'); const cfgWopiSlideEdit = config.get('wopi.slideEdit'); -const cfgWopiWordForm = config.get('wopi.wordForm'); +const cfgWopiForms = config.get('wopi.forms'); const cfgWopiFavIconUrlWord = config.get('wopi.favIconUrlWord'); const cfgWopiFavIconUrlCell = config.get('wopi.favIconUrlCell'); const cfgWopiFavIconUrlSlide = config.get('wopi.favIconUrlSlide'); +const cfgWopiFavIconUrlPdf = config.get('wopi.favIconUrlPdf'); const cfgWopiPublicKey = config.get('wopi.publicKey'); const cfgWopiModulus = config.get('wopi.modulus'); const cfgWopiExponent = config.get('wopi.exponent'); @@ -89,6 +91,7 @@ const cfgWopiHost = config.get('wopi.host'); const cfgWopiDummySampleFilePath = config.get('wopi.dummy.sampleFilePath'); let templatesFolderLocalesCache = null; +let templatesFolderExtsCache = null; const templateFilesSizeCache = {}; let mimeTypesByExt = (function() { @@ -109,9 +112,24 @@ let mimeTypesByExt = (function() { return mimeTypesByExt; })(); +async function getTemplatesFolderExts(ctx){ + //find available template files + if (templatesFolderExtsCache === null) { + const tenNewFileTemplate = ctx.getCfg('services.CoAuthoring.server.newFileTemplate', cfgNewFileTemplate); + const dirContent = await readdir(`${tenNewFileTemplate}/${constants.TEMPLATES_DEFAULT_LOCALE}/`, { withFileTypes: true }); + templatesFolderExtsCache = dirContent + .filter(dirObject => dirObject.isFile()) + .reduce((result, item, index, array) => { + let ext = path.extname(item.name).substring(1); + result[ext] = ext; + return result; + }, {}); + } + return templatesFolderExtsCache; +} + function discovery(req, res) { return co(function*() { - let output = ''; const xml = xmlbuilder2.create({version: '1.0', encoding: 'utf-8'}); let ctx = new operationContext.Context(); try { @@ -120,16 +138,18 @@ function discovery(req, res) { ctx.logger.info('wopiDiscovery start'); const tenWopiWopiZone = ctx.getCfg('wopi.wopiZone', cfgWopiWopiZone); const tenWopiPdfView = ctx.getCfg('wopi.pdfView', cfgWopiPdfView); + const tenWopiPdfEdit = ctx.getCfg('wopi.pdfEdit', cfgWopiPdfEdit); const tenWopiWordView = ctx.getCfg('wopi.wordView', cfgWopiWordView); const tenWopiWordEdit = ctx.getCfg('wopi.wordEdit', cfgWopiWordEdit); const tenWopiCellView = ctx.getCfg('wopi.cellView', cfgWopiCellView); const tenWopiCellEdit = ctx.getCfg('wopi.cellEdit', cfgWopiCellEdit); const tenWopiSlideView = ctx.getCfg('wopi.slideView', cfgWopiSlideView); const tenWopiSlideEdit = ctx.getCfg('wopi.slideEdit', cfgWopiSlideEdit); - const tenWopiWordForm = ctx.getCfg('wopi.wordForm', cfgWopiWordForm); + const tenWopiForms = ctx.getCfg('wopi.forms', cfgWopiForms); const tenWopiFavIconUrlWord = ctx.getCfg('wopi.favIconUrlWord', cfgWopiFavIconUrlWord); const tenWopiFavIconUrlCell = ctx.getCfg('wopi.favIconUrlCell', cfgWopiFavIconUrlCell); const tenWopiFavIconUrlSlide = ctx.getCfg('wopi.favIconUrlSlide', cfgWopiFavIconUrlSlide); + const tenWopiFavIconUrlPdf = ctx.getCfg('wopi.favIconUrlSlide', cfgWopiFavIconUrlPdf); const tenWopiPublicKey = ctx.getCfg('wopi.publicKey', cfgWopiPublicKey); const tenWopiModulus = ctx.getCfg('wopi.modulus', cfgWopiModulus); const tenWopiExponent = ctx.getCfg('wopi.exponent', cfgWopiExponent); @@ -139,20 +159,27 @@ function discovery(req, res) { const tenWopiHost = ctx.getCfg('wopi.host', cfgWopiHost); let baseUrl = tenWopiHost || utils.getBaseUrlByRequest(ctx, req); - let names = ['Word','Excel','PowerPoint']; - let favIconUrls = [tenWopiFavIconUrlWord, tenWopiFavIconUrlCell, tenWopiFavIconUrlSlide]; + let names = ['Word','Excel','PowerPoint','Pdf']; + let favIconUrls = [tenWopiFavIconUrlWord, tenWopiFavIconUrlCell, tenWopiFavIconUrlSlide, tenWopiFavIconUrlPdf]; let exts = [ - {targetext: 'docx', view: tenWopiPdfView.concat(tenWopiWordView), edit: tenWopiWordEdit, form: tenWopiWordForm}, + {targetext: 'docx', view: tenWopiWordView, edit: tenWopiWordEdit}, {targetext: 'xlsx', view: tenWopiCellView, edit: tenWopiCellEdit}, - {targetext: 'pptx', view: tenWopiSlideView, edit: tenWopiSlideEdit} + {targetext: 'pptx', view: tenWopiSlideView, edit: tenWopiSlideEdit}, + {targetext: null, view: tenWopiPdfView, edit: tenWopiPdfEdit} ]; + let documentTypes = [`word`, `cell`, `slide`, `pdf`]; + + let templatesFolderExtsCache = yield getTemplatesFolderExts(ctx); + let formsExts = tenWopiForms.reduce((result, item, index, array) => { + result[item] = item; + return result; + }, {}); let templateStart = `${baseUrl}/hosting/wopi`; let templateEnd = `<rs=DC_LLCC&><dchat=DISABLE_CHAT&><embed=EMBEDDED&>`; templateEnd += `<fs=FULLSCREEN&><hid=HOST_SESSION_ID&><rec=RECORDING&>`; templateEnd += `<sc=SESSION_CONTEXT&><thm=THEME_ID&><ui=UI_LLCC&>`; templateEnd += `<wopisrc=WOPI_SOURCE&>&`; - let documentTypes = [`word`, `cell`, `slide`]; let xmlZone = xml.ele('wopi-discovery').ele('net-zone', { name: tenWopiWopiZone }); //start section for MS WOPI connectors for(let i = 0; i < names.length; ++i) { @@ -173,7 +200,7 @@ function discovery(req, res) { xmlApp.ele('action', {name: 'view', ext: ext.view[j], default: 'true', urlsrc: urlTemplateView}).up(); xmlApp.ele('action', {name: 'embedview', ext: ext.view[j], urlsrc: urlTemplateEmbedView}).up(); xmlApp.ele('action', {name: 'mobileView', ext: ext.view[j], urlsrc: urlTemplateMobileView}).up(); - if (-1 === tenWopiPdfView.indexOf(ext.view[j])) { + if (ext.targetext) { let urlConvert = `${templateStart}/convert-and-edit/${ext.view[j]}/${ext.targetext}?${templateEnd}`; xmlApp.ele('action', {name: 'convert', ext: ext.view[j], targetext: ext.targetext, requires: 'update', urlsrc: urlConvert}).up(); } @@ -184,13 +211,13 @@ function discovery(req, res) { xmlApp.ele('action', {name: 'mobileView', ext: ext.edit[j], urlsrc: urlTemplateMobileView}).up(); xmlApp.ele('action', {name: 'edit', ext: ext.edit[j], default: 'true', requires: 'locks,update', urlsrc: urlTemplateEdit}).up(); xmlApp.ele('action', {name: 'mobileEdit', ext: ext.edit[j], requires: 'locks,update', urlsrc: urlTemplateMobileEdit}).up(); + if (formsExts[ext.edit[j]]) { + xmlApp.ele('action', {name: 'formsubmit', ext: ext.edit[j], requires: 'locks,update', urlsrc: urlTemplateFormSubmit}).up(); + } + if (templatesFolderExtsCache[ext.edit[j]]) { + xmlApp.ele('action', {name: 'editnew', ext: ext.edit[j], requires: 'locks,update', urlsrc: urlTemplateEdit}).up(); + } } - for (let extention of (ext.form || [])) { - xmlApp.ele('action', {name: 'formsubmit', ext: extention, requires: 'locks,update', urlsrc: urlTemplateFormSubmit}).up(); - } - constants.SUPPORTED_TEMPLATES_EXTENSIONS[name].forEach( - extension => xmlApp.ele('action', {name: 'editnew', ext: extension, requires: 'locks,update', urlsrc: urlTemplateEdit}).up() - ); xmlApp.up(); } //end section for MS WOPI connectors @@ -211,13 +238,10 @@ function discovery(req, res) { xmlApp.ele('action', {name: 'view', ext: '', default: 'true', urlsrc: urlTemplateView}).up(); xmlApp.ele('action', {name: 'embedview', ext: '', urlsrc: urlTemplateEmbedView}).up(); xmlApp.ele('action', {name: 'mobileView', ext: '', urlsrc: urlTemplateMobileView}).up(); - if (-1 === tenWopiPdfView.indexOf(ext.view[j])) { + if (ext.targetext) { let urlConvert = `${templateStart}/convert-and-edit/${ext.view[j]}/${ext.targetext}?${templateEnd}`; xmlApp.ele('action', {name: 'convert', ext: '', targetext: ext.targetext, requires: 'update', urlsrc: urlConvert}).up(); } - if (ext.form && -1 !== ext.form.indexOf(ext.view[j])) { - xmlApp.ele('action', {name: 'formsubmit', ext: '', requires: 'locks,update', urlsrc: urlTemplateFormSubmit}).up(); - } xmlApp.up(); }); } @@ -229,9 +253,12 @@ function discovery(req, res) { let xmlApp = xmlZone.ele('app', {name: value}); xmlApp.ele('action', {name: 'edit', ext: '', default: 'true', requires: 'locks,update', urlsrc: urlTemplateEdit}).up(); xmlApp.ele('action', {name: 'mobileEdit', ext: '', requires: 'locks,update', urlsrc: urlTemplateMobileEdit}).up(); - if (ext.form && -1 !== ext.form.indexOf(ext.edit[j])) { + if (formsExts[ext.edit[j]]) { xmlApp.ele('action', {name: 'formsubmit', ext: '', requires: 'locks,update', urlsrc: urlTemplateFormSubmit}).up(); } + if (templatesFolderExtsCache[ext.edit[j]]) { + xmlApp.ele('action', {name: 'editnew', ext: '', requires: 'locks,update', urlsrc: urlTemplateEdit}).up(); + } xmlApp.up(); }); } @@ -496,7 +523,7 @@ function getEditorHtml(req, res) { const localePrefix = lang || ui || 'en'; let locale = constants.TEMPLATES_FOLDER_LOCALE_COLLISON_MAP[localePrefix] ?? templatesFolderLocalesCache.find(locale => locale.startsWith(localePrefix)); if (locale === undefined) { - locale = 'en-US'; + locale = constants.TEMPLATES_DEFAULT_LOCALE; } const filePath = `${tenNewFileTemplate}/${locale}/new.${fileType}`; diff --git a/FileConverter/sources/converter.js b/FileConverter/sources/converter.js index 777146163..5396c3d8f 100644 --- a/FileConverter/sources/converter.js +++ b/FileConverter/sources/converter.js @@ -360,7 +360,7 @@ async function changeFormatToExtendedPdf(ctx, dataConvert, cmd) { function* replaceEmptyFile(ctx, fileFrom, ext, _lcid) { const tenNewFileTemplate = ctx.getCfg('services.CoAuthoring.server.newFileTemplate', cfgNewFileTemplate); if (!fs.existsSync(fileFrom) || 0 === fs.lstatSync(fileFrom).size) { - let locale = 'en-US'; + let locale = constants.TEMPLATES_DEFAULT_LOCALE; if (_lcid) { let localeNew = lcid.from(_lcid); if (localeNew) { @@ -372,14 +372,24 @@ function* replaceEmptyFile(ctx, fileFrom, ext, _lcid) { } } } - ctx.logger.debug('replaceEmptyFile format=%s locale=%s', ext, locale); - let format = formatChecker.getFormatFromString(ext); - if (formatChecker.isDocumentFormat(format)) { - fs.copyFileSync(path.join(tenNewFileTemplate, locale, 'new.docx'), fileFrom); - } else if (formatChecker.isSpreadsheetFormat(format)) { - fs.copyFileSync(path.join(tenNewFileTemplate, locale, 'new.xlsx'), fileFrom); - } else if (formatChecker.isPresentationFormat(format)) { - fs.copyFileSync(path.join(tenNewFileTemplate, locale, 'new.pptx'), fileFrom); + let fileTemplatePath = path.join(tenNewFileTemplate, locale, 'new.'); + if (fs.existsSync(fileTemplatePath + ext)) { + ctx.logger.debug('replaceEmptyFile format=%s locale=%s', ext, locale); + fs.copyFileSync(fileTemplatePath + ext, fileFrom); + } else { + let format = formatChecker.getFormatFromString(ext); + let editorFormat; + if (formatChecker.isDocumentFormat(format)) { + editorFormat = 'docx'; + } else if (formatChecker.isSpreadsheetFormat(format)) { + editorFormat = 'xlsx'; + } else if (formatChecker.isPresentationFormat(format)) { + editorFormat = 'pptx'; + } + if (fs.existsSync(fileTemplatePath + editorFormat)) { + ctx.logger.debug('replaceEmptyFile format=%s locale=%s', ext, locale); + fs.copyFileSync(fileTemplatePath + editorFormat, fileFrom); + } } } } From 5373d62df4023945bc87fafeb8d2e62ed285ea30 Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Thu, 4 Apr 2024 13:34:23 +0300 Subject: [PATCH 18/48] [bug] Set interface lang as lcid for file assembling from changes; for bug 66926 --- DocService/sources/DocsCoServer.js | 17 ++++++++++------- DocService/sources/canvasservice.js | 7 +++++-- DocService/sources/changes2forgotten.js | 2 +- DocService/sources/converterservice.js | 14 +++++++++----- DocService/sources/gc.js | 2 +- DocService/sources/utilsDocService.js | 13 ++++++++++++- 6 files changed, 38 insertions(+), 17 deletions(-) diff --git a/DocService/sources/DocsCoServer.js b/DocService/sources/DocsCoServer.js index e774b05ec..7ffb182ca 100644 --- a/DocService/sources/DocsCoServer.js +++ b/DocService/sources/DocsCoServer.js @@ -106,6 +106,7 @@ const cfgEditorDataStorage = config.get('services.CoAuthoring.server.editorDataS const cfgEditorStatStorage = config.get('services.CoAuthoring.server.editorStatStorage'); const editorDataStorage = require('./' + cfgEditorDataStorage); const editorStatStorage = require('./' + (cfgEditorStatStorage || cfgEditorDataStorage)); +const utilsDocService = require("./utilsDocService"); const cfgEditSingleton = config.get('services.CoAuthoring.server.edit_singleton'); const cfgEditor = config.get('services.CoAuthoring.editor'); @@ -933,7 +934,7 @@ async function startForceSave(ctx, docId, type, opt_userdata, opt_formdata, opt_ newChangesLastDate.setMilliseconds(0);//remove milliseconds avoid issues with MySQL datetime rounding let newChangesLastTime = newChangesLastDate.getTime(); let baseUrl = utils.getBaseUrlByConnection(ctx, opt_conn); - let changeInfo = getExternalChangeInfo(opt_conn.user, newChangesLastTime); + let changeInfo = getExternalChangeInfo(opt_conn.user, newChangesLastTime, opt_conn.lang); await editorData.setForceSave(ctx, docId, newChangesLastTime, 0, baseUrl, changeInfo, null); forceSave = await editorData.getForceSave(ctx, docId); } @@ -990,8 +991,8 @@ async function startForceSave(ctx, docId, type, opt_userdata, opt_formdata, opt_ ctx.logger.debug('startForceSave end'); return res; } -function getExternalChangeInfo(user, date) { - return {user_id: user.id, user_id_original: user.idOriginal, user_name: user.username, change_date: date}; +function getExternalChangeInfo(user, date, lang) { + return {user_id: user.id, user_id_original: user.idOriginal, user_name: user.username, lang, change_date: date}; } let resetForceSaveAfterChanges = co.wrap(function*(ctx, docId, newChangesLastTime, puckerIndex, baseUrl, changeInfo) { const tenForceSaveEnable = ctx.getCfg('services.CoAuthoring.autoAssembly.enable', cfgForceSaveEnable); @@ -1323,7 +1324,7 @@ function* cleanDocumentOnExitNoChanges(ctx, docId, opt_userId, opt_userIndex, op yield* cleanDocumentOnExit(ctx, docId, false, opt_userIndex); } -function createSaveTimer(ctx, docId, opt_userId, opt_userIndex, opt_queue, opt_noDelay, opt_initShardKey) { +function createSaveTimer(ctx, docId, opt_userId, opt_userIndex, opt_userLcid, opt_queue, opt_noDelay, opt_initShardKey) { return co(function*(){ const tenAscSaveTimeOutDelay = ctx.getCfg('services.CoAuthoring.server.savetimeoutdelay', cfgAscSaveTimeOutDelay); @@ -1341,7 +1342,7 @@ function createSaveTimer(ctx, docId, opt_userId, opt_userIndex, opt_queue, opt_n } while (true) { if (!sqlBase.isLockCriticalSection(docId)) { - canvasService.saveFromChanges(ctx, docId, updateTask.statusInfo, null, opt_userId, opt_userIndex, opt_queue, opt_initShardKey); + canvasService.saveFromChanges(ctx, docId, updateTask.statusInfo, null, opt_userId, opt_userIndex, opt_userLcid, opt_queue, opt_initShardKey); break; } yield utils.sleep(c_oAscLockTimeOutDelay); @@ -1853,7 +1854,8 @@ exports.install = function(server, callbackFunction) { } if (needSaveChanges && !conn.encrypted) { // Send changes to save server - yield createSaveTimer(ctx, docId, tmpUser.idOriginal, userIndex); + let user_lcid = utilsDocService.localeToLCID(conn.lang); + yield createSaveTimer(ctx, docId, tmpUser.idOriginal, userIndex, user_lcid); } else if (needSendStatus) { yield* cleanDocumentOnExitNoChanges(ctx, docId, tmpUser.idOriginal, userIndex); } else { @@ -2570,6 +2572,7 @@ exports.install = function(server, callbackFunction) { } conn.unsyncTime = null; conn.encrypted = data.encrypted; + conn.lang = data.lang; conn.supportAuthChangesAck = data.supportAuthChangesAck; const c_LR = constants.LICENSE_RESULT; @@ -3237,7 +3240,7 @@ exports.install = function(server, callbackFunction) { // Automatically remove the lock ourselves and send the index to save yield* unSaveLock(ctx, conn, changesIndex, newChangesLastTime, puckerIndex); //last save - let changeInfo = getExternalChangeInfo(conn.user, newChangesLastTime); + let changeInfo = getExternalChangeInfo(conn.user, newChangesLastTime, conn.lang); yield resetForceSaveAfterChanges(ctx, docId, newChangesLastTime, puckerIndex, utils.getBaseUrlByConnection(ctx, conn), changeInfo); } else { let changesToSend = arrNewDocumentChanges; diff --git a/DocService/sources/canvasservice.js b/DocService/sources/canvasservice.js index 9b92c5eb9..58963fc22 100644 --- a/DocService/sources/canvasservice.js +++ b/DocService/sources/canvasservice.js @@ -830,6 +830,7 @@ function* commandSaveFromOrigin(ctx, cmd, outputData, password) { if (docPassword.initial) { cmd.setPassword(docPassword.initial); } + //todo setLCID in browser var queueData = getSaveTask(ctx, cmd); queueData.setFromOrigin(true); queueData.setFromChanges(true); @@ -864,7 +865,7 @@ function* commandSetPassword(ctx, conn, cmd, outputData) { task.password = cmd.getPassword() || ""; let changeInfo = null; if (conn.user) { - changeInfo = task.innerPasswordChange = docsCoServer.getExternalChangeInfo(conn.user, newChangesLastDate.getTime()); + changeInfo = task.innerPasswordChange = docsCoServer.getExternalChangeInfo(conn.user, newChangesLastDate.getTime(), conn.lang); } var upsertRes = yield taskResult.updateIf(ctx, task, updateMask); @@ -1703,7 +1704,7 @@ exports.downloadFile = function(req, res) { } }); }; -exports.saveFromChanges = function(ctx, docId, statusInfo, optFormat, opt_userId, opt_userIndex, opt_queue, opt_initShardKey) { +exports.saveFromChanges = function(ctx, docId, statusInfo, optFormat, opt_userId, opt_userIndex, opt_userLcid, opt_queue, opt_initShardKey) { return co(function* () { try { var startDate = null; @@ -1729,6 +1730,8 @@ exports.saveFromChanges = function(ctx, docId, statusInfo, optFormat, opt_userId cmd.setUserActionId(opt_userId); cmd.setUserActionIndex(opt_userIndex); cmd.setJsonParams(getOpenedAtJSONParams(row)); + //todo lang and region are different + cmd.setLCID(opt_userLcid); let userAuthStr = sqlBase.UserCallback.prototype.getCallbackByUserIndex(ctx, row.callback); cmd.setWopiParams(wopiClient.parseWopiCallback(ctx, userAuthStr, row.callback)); addPasswordToCmd(ctx, cmd, row && row.password); diff --git a/DocService/sources/changes2forgotten.js b/DocService/sources/changes2forgotten.js index ff9e08b2e..75d48873f 100644 --- a/DocService/sources/changes2forgotten.js +++ b/DocService/sources/changes2forgotten.js @@ -134,7 +134,7 @@ function shutdown() { yield updateDoc(ctx, docId, commonDefines.FileStatus.Ok, ""); yield editorStat.addShutdown(redisKeyShutdown, docId); ctx.logger.debug('shutdown createSaveTimerPromise %s', docId); - yield docsCoServer.createSaveTimer(ctx, docId, null, null, queue, true); + yield docsCoServer.createSaveTimer(ctx, docId, null, null, null, queue, true); } ctx.initDefault(); //sleep because of bugs in createSaveTimerPromise diff --git a/DocService/sources/converterservice.js b/DocService/sources/converterservice.js index e1cde9663..acd103ba1 100644 --- a/DocService/sources/converterservice.js +++ b/DocService/sources/converterservice.js @@ -35,7 +35,6 @@ const path = require('path'); var config = require('config'); var co = require('co'); -const locale = require('windows-locale'); const mime = require('mime'); var taskResult = require('./taskresult'); var utils = require('./../../Common/sources/utils'); @@ -50,6 +49,7 @@ var statsDClient = require('./../../Common/sources/statsdclient'); var storageBase = require('./../../Common/sources/storage-base'); var operationContext = require('./../../Common/sources/operationContext'); const sqlBase = require('./databaseConnectors/baseConnector'); +const utilsDocService = require("./utilsDocService"); const cfgTokenEnableBrowser = config.get('services.CoAuthoring.token.enable.browser'); @@ -193,6 +193,10 @@ async function convertFromChanges(ctx, docId, baseUrl, forceSave, externalChange cmd.setDelimiter(commonDefines.c_oAscCsvDelimiter.Comma); cmd.setForceSave(forceSave); cmd.setExternalChangeInfo(externalChangeInfo); + if (externalChangeInfo.lang) { + //todo lang and region are different + cmd.setLCID(utilsDocService.localeToLCID(externalChangeInfo.lang)); + } if (opt_userdata) { cmd.setUserData(opt_userdata); } @@ -284,8 +288,8 @@ function convertRequest(req, res, isJson) { cmd.setDelimiter(parseIntParam(params.delimiter) || commonDefines.c_oAscCsvDelimiter.Comma); if(undefined != params.delimiterChar) cmd.setDelimiterChar(params.delimiterChar); - if (params.region && locale[params.region.toLowerCase()]) { - cmd.setLCID(locale[params.region.toLowerCase()].id); + if (params.region) { + cmd.setLCID(utilsDocService.localeToLCID(params.region)); } let jsonParams = {}; if (params.documentLayout) { @@ -518,8 +522,8 @@ function convertTo(req, res) { cmd.setOutputFormat(outputFormat); cmd.setCodepage(commonDefines.c_oAscCodePageUtf8); cmd.setDelimiter(commonDefines.c_oAscCsvDelimiter.Comma); - if (lang && locale[lang.toLowerCase()]) { - cmd.setLCID(locale[lang.toLowerCase()].id); + if (lang) { + cmd.setLCID(utilsDocService.localeToLCID(lang)); } if (fullSheetPreview) { cmd.setJsonParams(JSON.stringify({'spreadsheetLayout': { diff --git a/DocService/sources/gc.js b/DocService/sources/gc.js index 496e6a934..6fbaf357f 100644 --- a/DocService/sources/gc.js +++ b/DocService/sources/gc.js @@ -125,7 +125,7 @@ var checkDocumentExpire = function() { var hasChanges = yield docsCoServer.hasChanges(ctx, docId); if (hasChanges) { //todo opt_initShardKey from getDocumentPresenceExpired data or from db - yield docsCoServer.createSaveTimer(ctx, docId, null, null, queue, true, true); + yield docsCoServer.createSaveTimer(ctx, docId, null, null, null, queue, true, true); startSaveCount++; } else { yield docsCoServer.cleanDocumentOnExitNoChangesPromise(ctx, docId); diff --git a/DocService/sources/utilsDocService.js b/DocService/sources/utilsDocService.js index a6575ac8a..378b357d8 100644 --- a/DocService/sources/utilsDocService.js +++ b/DocService/sources/utilsDocService.js @@ -34,6 +34,7 @@ const exifParser = require("exif-parser"); const Jimp = require("jimp"); +const locale = require('windows-locale'); async function fixImageExifRotation(ctx, buffer) { if (!buffer) { @@ -59,7 +60,17 @@ async function fixImageExifRotation(ctx, buffer) { } return buffer; } +/** + * + * @param {string} lang + * @returns {number | undefined} + */ +function localeToLCID(lang) { + let elem = locale[lang && lang.toLowerCase()]; + return elem && elem.id; +} module.exports = { - fixImageExifRotation + fixImageExifRotation, + localeToLCID }; \ No newline at end of file From 920ff1fcd5f3ed3870928447fdcb6bc842be9803 Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Fri, 5 Apr 2024 16:56:18 +0300 Subject: [PATCH 19/48] [feature] Add "pdf" param as pdf save options --- DocService/sources/converterservice.js | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/DocService/sources/converterservice.js b/DocService/sources/converterservice.js index acd103ba1..e4ce06af9 100644 --- a/DocService/sources/converterservice.js +++ b/DocService/sources/converterservice.js @@ -275,6 +275,17 @@ function convertRequest(req, res, isJson) { utils.fillResponse(req, res, new commonDefines.ConvertStatus(constants.CONVERT_PARAMS), isJson); return; } + if (params.pdf) { + if (true === params.pdf.pdfa && constants.AVS_OFFICESTUDIO_FILE_CROSSPLATFORM_PDF === outputFormat) { + outputFormat = constants.AVS_OFFICESTUDIO_FILE_CROSSPLATFORM_PDFA; + } else if (false === params.pdf.pdfa && constants.AVS_OFFICESTUDIO_FILE_CROSSPLATFORM_PDFA === outputFormat) { + outputFormat = constants.AVS_OFFICESTUDIO_FILE_CROSSPLATFORM_PDF; + } + if (params.pdf.form && (constants.AVS_OFFICESTUDIO_FILE_CROSSPLATFORM_PDF === outputFormat || + constants.AVS_OFFICESTUDIO_FILE_CROSSPLATFORM_PDFA === outputFormat)) { + outputFormat = constants.AVS_OFFICESTUDIO_FILE_DOCUMENT_OFORM_PDF; + } + } var cmd = new commonDefines.InputCommand(); cmd.setCommand('conv'); cmd.setUrl(params.url); From 6463405961b7fe132b0a8dae894e2eaa0e8cec35 Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Sun, 7 Apr 2024 01:15:31 +0300 Subject: [PATCH 20/48] [bug] Refactor ErrToReload cleaning; Fix bug 67297 --- DocService/sources/canvasservice.js | 41 +++++++++++++++----------- DocService/sources/converterservice.js | 9 +++--- 2 files changed, 29 insertions(+), 21 deletions(-) diff --git a/DocService/sources/canvasservice.js b/DocService/sources/canvasservice.js index 58963fc22..a4c87a1fc 100644 --- a/DocService/sources/canvasservice.js +++ b/DocService/sources/canvasservice.js @@ -280,17 +280,13 @@ var getOutputData = co.wrap(function* (ctx, cmd, outputData, key, optConn, optAd outputData.setData(statusInfo); break; case commonDefines.FileStatus.Err: + outputData.setStatus('err'); + outputData.setData(statusInfo); + break; case commonDefines.FileStatus.ErrToReload: outputData.setStatus('err'); outputData.setData(statusInfo); - if (commonDefines.FileStatus.ErrToReload == status) { - let userAuthStr = sqlBase.UserCallback.prototype.getCallbackByUserIndex(ctx, row.callback); - let wopiParams = wopiClient.parseWopiCallback(ctx, userAuthStr); - if (!wopiParams) { - //todo rework ErrToReload to clean up on next open - yield cleanupCache(ctx, key); - } - } + yield cleanupErrToReload(ctx, key); break; case commonDefines.FileStatus.None: //this status has no handler @@ -379,38 +375,40 @@ function getSaveTask(ctx, cmd) { //} return queueData; } -function* getUpdateResponse(ctx, cmd) { +async function getUpdateResponse(ctx, cmd) { const tenOpenProtectedFile = ctx.getCfg('services.CoAuthoring.server.openProtectedFile', cfgOpenProtectedFile); var updateTask = new taskResult.TaskResultData(); updateTask.tenant = ctx.tenant; updateTask.key = cmd.getSaveKey() ? cmd.getSaveKey() : cmd.getDocId(); var statusInfo = cmd.getStatusInfo(); - if (constants.NO_ERROR == statusInfo) { + if (constants.NO_ERROR === statusInfo) { updateTask.status = commonDefines.FileStatus.Ok; let password = cmd.getPassword(); if (password) { if (false === hasPasswordCol) { - let selectRes = yield taskResult.select(ctx, updateTask.key); + let selectRes = await taskResult.select(ctx, updateTask.key); hasPasswordCol = selectRes.length > 0 && undefined !== selectRes[0].password; } if(hasPasswordCol) { updateTask.password = password; } } - } else if (constants.CONVERT_DOWNLOAD == statusInfo) { + } else if (constants.CONVERT_DOWNLOAD === statusInfo) { + updateTask.status = commonDefines.FileStatus.ErrToReload; + } else if (constants.CONVERT_LIMITS === statusInfo) { updateTask.status = commonDefines.FileStatus.ErrToReload; - } else if (constants.CONVERT_NEED_PARAMS == statusInfo) { + } else if (constants.CONVERT_NEED_PARAMS === statusInfo) { updateTask.status = commonDefines.FileStatus.NeedParams; - } else if (constants.CONVERT_DRM == statusInfo || constants.CONVERT_PASSWORD == statusInfo) { + } else if (constants.CONVERT_DRM === statusInfo || constants.CONVERT_PASSWORD === statusInfo) { if (tenOpenProtectedFile) { updateTask.status = commonDefines.FileStatus.NeedPassword; } else { updateTask.status = commonDefines.FileStatus.Err; } - } else if (constants.CONVERT_DRM_UNSUPPORTED == statusInfo) { + } else if (constants.CONVERT_DRM_UNSUPPORTED === statusInfo) { updateTask.status = commonDefines.FileStatus.Err; - } else if (constants.CONVERT_DEAD_LETTER == statusInfo) { + } else if (constants.CONVERT_DEAD_LETTER === statusInfo) { updateTask.status = commonDefines.FileStatus.ErrToReload; } else { updateTask.status = commonDefines.FileStatus.Err; @@ -441,6 +439,14 @@ var cleanupCacheIf = co.wrap(function* (ctx, mask) { ctx.logger.debug("cleanupCacheIf db.affectedRows=%d", removeRes.affectedRows); return res; }); +async function cleanupErrToReload(ctx, key) { + let updateTask = new taskResult.TaskResultData(); + updateTask.tenant = ctx.tenant; + updateTask.key = key; + updateTask.status = commonDefines.FileStatus.None; + updateTask.statusInfo = constants.NO_ERROR; + await taskResult.update(ctx, updateTask); +} function commandOpenStartPromise(ctx, docId, baseUrl, opt_documentCallbackUrl, opt_format) { var task = new taskResult.TaskResultData(); @@ -1780,7 +1786,7 @@ exports.receiveTask = function(data, ack) { ctx.initFromTaskQueueData(task); yield ctx.initTenantCache(); ctx.logger.info('receiveTask start: %s', data); - var updateTask = yield* getUpdateResponse(ctx, cmd); + var updateTask = yield getUpdateResponse(ctx, cmd); var updateRes = yield taskResult.update(ctx, updateTask); if (updateRes.affectedRows > 0) { var outputData = new OutputData(cmd.getCommand()); @@ -1829,6 +1835,7 @@ exports.receiveTask = function(data, ack) { exports.cleanupCache = cleanupCache; exports.cleanupCacheIf = cleanupCacheIf; +exports.cleanupErrToReload = cleanupErrToReload; exports.getOpenedAt = getOpenedAt; exports.commandSfctByCmd = commandSfctByCmd; exports.commandOpenStartPromise = commandOpenStartPromise; diff --git a/DocService/sources/converterservice.js b/DocService/sources/converterservice.js index e4ce06af9..fe7ca1265 100644 --- a/DocService/sources/converterservice.js +++ b/DocService/sources/converterservice.js @@ -83,12 +83,12 @@ function* getConvertStatus(ctx, docId, encryptedUserPassword, selectRes, opt_che } break; case commonDefines.FileStatus.Err: + status.err = row.status_info; + break; case commonDefines.FileStatus.ErrToReload: case commonDefines.FileStatus.NeedPassword: status.err = row.status_info; - if (commonDefines.FileStatus.ErrToReload == row.status || commonDefines.FileStatus.NeedPassword == row.status) { - yield canvasService.cleanupCache(ctx, docId); - } + yield canvasService.cleanupErrToReload(ctx, docId); break; case commonDefines.FileStatus.NeedParams: case commonDefines.FileStatus.SaveVersion: @@ -147,7 +147,8 @@ function* convertByCmd(ctx, cmd, async, opt_fileTo, opt_taskExist, opt_priority, if (!bCreate) { selectRes = yield taskResult.select(ctx, docId); status = yield* getConvertStatus(ctx, cmd.getDocId() ,cmd.getPassword(), selectRes, opt_checkPassword); - } else { + } + if (bCreate || (commonDefines.FileStatus.None === selectRes?.[0]?.status)) { var queueData = new commonDefines.TaskQueueData(); queueData.setCtx(ctx); queueData.setCmd(cmd); From 515a41cb7582504d3eb7ea05a0efe1e1a176beb8 Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Sat, 13 Apr 2024 16:17:37 +0300 Subject: [PATCH 21/48] [bug] Set isPrint flag while form submission --- Common/sources/commondefines.js | 9 +++++++-- DocService/sources/DocsCoServer.js | 10 +++++++--- DocService/sources/canvasservice.js | 6 +++--- DocService/sources/converterservice.js | 18 +++++++++++------- FileConverter/sources/converter.js | 2 +- 5 files changed, 29 insertions(+), 16 deletions(-) diff --git a/Common/sources/commondefines.js b/Common/sources/commondefines.js index 3a1162f74..fc8756b90 100644 --- a/Common/sources/commondefines.js +++ b/Common/sources/commondefines.js @@ -32,6 +32,7 @@ 'use strict'; +const config = require("config"); const constants = require('./constants'); function InputCommand(data, copyExplicit) { @@ -365,8 +366,12 @@ InputCommand.prototype = { getJsonParams: function() { return this['jsonparams']; }, - setJsonParams: function(data) { - this['jsonparams'] = data; + appendJsonParams: function (data) { + if (this['jsonparams']) { + config.util.extendDeep(this['jsonparams'], data); + } else { + this['jsonparams'] = data; + } }, getLCID: function() { return this['lcid']; diff --git a/DocService/sources/DocsCoServer.js b/DocService/sources/DocsCoServer.js index 7ffb182ca..dc1a8798e 100644 --- a/DocService/sources/DocsCoServer.js +++ b/DocService/sources/DocsCoServer.js @@ -916,7 +916,9 @@ async function applyForceSaveCache(ctx, docId, forceSave, type, opt_userConnecti } return res; } -async function startForceSave(ctx, docId, type, opt_userdata, opt_formdata, opt_userId, opt_userConnectionId, opt_userConnectionDocId, opt_userIndex, opt_responseKey, opt_baseUrl, opt_queue, opt_pubsub, opt_conn, opt_initShardKey) { +async function startForceSave(ctx, docId, type, opt_userdata, opt_formdata, opt_userId, opt_userConnectionId, + opt_userConnectionDocId, opt_userIndex, opt_responseKey, opt_baseUrl, + opt_queue, opt_pubsub, opt_conn, opt_initShardKey, opt_jsonParams) { ctx.logger.debug('startForceSave start'); let res = {code: commonDefines.c_oAscServerCommandErrors.NoError, time: null, inProgress: false}; let startedForceSave; @@ -974,7 +976,7 @@ async function startForceSave(ctx, docId, type, opt_userdata, opt_formdata, opt_ //start new convert let status = await converterService.convertFromChanges(ctx, docId, baseUrl, forceSave, startedForceSave.changeInfo, opt_userdata, opt_formdata, opt_userConnectionId, opt_userConnectionDocId, opt_responseKey, priority, expiration, - opt_queue, undefined, opt_initShardKey); + opt_queue, undefined, opt_initShardKey, opt_jsonParams); if (constants.NO_ERROR === status.err) { res.time = forceSave.getTime(); if (commonDefines.c_oAscForceSaveTypes.Timeout === type) { @@ -1036,9 +1038,11 @@ function* startRPC(ctx, conn, responseKey, data) { case 'sendForm': { let forceSaveRes; if (conn.user) { + //isPrint - to remove forms + let jsonParams = {'documentLayout': {'isPrint': true}}; forceSaveRes = yield startForceSave(ctx, docId, commonDefines.c_oAscForceSaveTypes.Form, undefined, data.formdata, conn.user.idOriginal, conn.user.id, undefined, conn.user.indexUser, - responseKey, undefined, undefined, undefined, conn); + responseKey, undefined, undefined, undefined, conn, undefined, jsonParams); } if (!forceSaveRes || commonDefines.c_oAscServerCommandErrors.NoError !== forceSaveRes.code || forceSaveRes.inProgress) { sendDataRpc(ctx, conn, responseKey, forceSaveRes); diff --git a/DocService/sources/canvasservice.js b/DocService/sources/canvasservice.js index a4c87a1fc..930e5ede1 100644 --- a/DocService/sources/canvasservice.js +++ b/DocService/sources/canvasservice.js @@ -162,7 +162,7 @@ function getOpenedAt(row) { function getOpenedAtJSONParams(row) { let openedAt = getOpenedAt(row); if (openedAt) { - return JSON.stringify({'documentLayout': {'openedAt': openedAt}}); + return {'documentLayout': {'openedAt': openedAt}}; } return undefined; } @@ -635,7 +635,7 @@ let commandSfctByCmd = co.wrap(function*(ctx, cmd, opt_priority, opt_expiration, let userAuthStr = sqlBase.UserCallback.prototype.getCallbackByUserIndex(ctx, row.callback); cmd.setWopiParams(wopiClient.parseWopiCallback(ctx, userAuthStr, row.callback)); cmd.setOutputFormat(changeFormatByOrigin(ctx, row, cmd.getOutputFormat())); - cmd.setJsonParams(getOpenedAtJSONParams(row)); + cmd.appendJsonParams(getOpenedAtJSONParams(row)); var queueData = getSaveTask(ctx, cmd); queueData.setFromChanges(true); let priority = null != opt_priority ? opt_priority : constants.QUEUE_PRIORITY_LOW; @@ -1735,7 +1735,7 @@ exports.saveFromChanges = function(ctx, docId, statusInfo, optFormat, opt_userId cmd.setStatusInfoIn(statusInfo); cmd.setUserActionId(opt_userId); cmd.setUserActionIndex(opt_userIndex); - cmd.setJsonParams(getOpenedAtJSONParams(row)); + cmd.appendJsonParams(getOpenedAtJSONParams(row)); //todo lang and region are different cmd.setLCID(opt_userLcid); let userAuthStr = sqlBase.UserCallback.prototype.getCallbackByUserIndex(ctx, row.callback); diff --git a/DocService/sources/converterservice.js b/DocService/sources/converterservice.js index fe7ca1265..310621613 100644 --- a/DocService/sources/converterservice.js +++ b/DocService/sources/converterservice.js @@ -183,8 +183,9 @@ function* convertByCmd(ctx, cmd, async, opt_fileTo, opt_taskExist, opt_priority, return status; } -async function convertFromChanges(ctx, docId, baseUrl, forceSave, externalChangeInfo, opt_userdata, opt_formdata, opt_userConnectionId, - opt_userConnectionDocId, opt_responseKey, opt_priority, opt_expiration, opt_queue, opt_redisKey, opt_initShardKey) { +async function convertFromChanges(ctx, docId, baseUrl, forceSave, externalChangeInfo, opt_userdata, opt_formdata, + opt_userConnectionId, opt_userConnectionDocId, opt_responseKey, opt_priority, + opt_expiration, opt_queue, opt_redisKey, opt_initShardKey, opt_jsonParams) { var cmd = new commonDefines.InputCommand(); cmd.setCommand('sfcm'); cmd.setDocId(docId); @@ -217,6 +218,9 @@ async function convertFromChanges(ctx, docId, baseUrl, forceSave, externalChange if (opt_redisKey) { cmd.setRedisKey(opt_redisKey); } + if (opt_jsonParams) { + cmd.appendJsonParams(opt_jsonParams); + } let commandSfctByCmdRes = await canvasService.commandSfctByCmd(ctx, cmd, opt_priority, opt_expiration, opt_queue, opt_initShardKey); if (!commandSfctByCmdRes) { @@ -314,7 +318,7 @@ function convertRequest(req, res, isJson) { jsonParams['watermark'] = params.watermark; } if (Object.keys(jsonParams).length > 0) { - cmd.setJsonParams(JSON.stringify(jsonParams)); + cmd.appendJsonParams(jsonParams); } if (params.password) { if (params.password.length > constants.PASSWORD_MAX_LENGTH) { @@ -538,18 +542,18 @@ function convertTo(req, res) { cmd.setLCID(utilsDocService.localeToLCID(lang)); } if (fullSheetPreview) { - cmd.setJsonParams(JSON.stringify({'spreadsheetLayout': { + cmd.appendJsonParams({'spreadsheetLayout': { "ignorePrintArea": true, "fitToWidth": 1, "fitToHeight": 1 - }})); + }}); } else { - cmd.setJsonParams(JSON.stringify({'spreadsheetLayout': { + cmd.appendJsonParams({'spreadsheetLayout': { "ignorePrintArea": true, "fitToWidth": 0, "fitToHeight": 0, "scale": 100 - }})); + }}); } fileTo = constants.OUTPUT_NAME; diff --git a/FileConverter/sources/converter.js b/FileConverter/sources/converter.js index 5396c3d8f..fb80e2af7 100644 --- a/FileConverter/sources/converter.js +++ b/FileConverter/sources/converter.js @@ -123,7 +123,7 @@ function TaskQueueDataConvert(ctx, task) { this.mailMergeSend = cmd.mailmergesend; this.thumbnail = cmd.thumbnail; this.textParams = cmd.getTextParams(); - this.jsonParams = cmd.getJsonParams(); + this.jsonParams = JSON.stringify(cmd.getJsonParams()); this.lcid = cmd.getLCID(); this.password = cmd.getPassword(); this.savePassword = cmd.getSavePassword(); From 11fc8887396974e4430b704d44bf67514e5fee4a Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Mon, 15 Apr 2024 02:09:58 +0300 Subject: [PATCH 22/48] [bug] Fix RequestDefaults absence in postRequestPromise; Fix bug 67402 --- Common/sources/utils.js | 31 ++++++++++++++++------------- DocService/sources/DocsCoServer.js | 3 ++- DocService/sources/canvasservice.js | 2 +- DocService/sources/wopiClient.js | 2 ++ 4 files changed, 22 insertions(+), 16 deletions(-) diff --git a/Common/sources/utils.js b/Common/sources/utils.js index 02348d523..5410cd9d4 100644 --- a/Common/sources/utils.js +++ b/Common/sources/utils.js @@ -444,7 +444,7 @@ function downloadUrlPromiseWithoutRedirect(ctx, uri, optTimeout, optLimit, opt_A } }); } -function postRequestPromise(ctx, uri, postData, postDataStream, postDataSize, optTimeout, opt_Authorization, opt_header) { +function postRequestPromise(ctx, uri, postData, postDataStream, postDataSize, optTimeout, opt_Authorization, opt_headers) { return new Promise(function(resolve, reject) { const tenTenantRequestDefaults = ctx.getCfg('services.CoAuthoring.requestDefaults', cfgRequestDefaults); const tenTokenOutboxHeader = ctx.getCfg('services.CoAuthoring.token.outbox.header', cfgTokenOutboxHeader); @@ -452,29 +452,32 @@ function postRequestPromise(ctx, uri, postData, postDataStream, postDataSize, op //IRI to URI uri = URI.serialize(URI.parse(uri)); var urlParsed = url.parse(uri); - var headers = {'Content-Type': 'application/json'}; + let connectionAndInactivity = optTimeout && optTimeout.connectionAndInactivity && ms(optTimeout.connectionAndInactivity); + let options = config.util.extendDeep({}, tenTenantRequestDefaults); + Object.assign(options, {uri: urlParsed, encoding: 'utf8', timeout: connectionAndInactivity}); + //baseRequest creates new agent(win-ca injects in globalAgent) + options.agentOptions = https.globalAgent.options; + if (postData) { + options.body = postData; + } + if (!options.headers) { + options.headers = {}; + } if (opt_Authorization) { //todo ctx.getCfg - headers[tenTokenOutboxHeader] = tenTokenOutboxPrefix + opt_Authorization; + options.headers[tenTokenOutboxHeader] = tenTokenOutboxPrefix + opt_Authorization; + } + if (opt_headers) { + Object.assign(options.headers, opt_headers); } - headers = opt_header || headers; if (undefined !== postDataSize) { //If no Content-Length is set, data will automatically be encoded in HTTP Chunked transfer encoding, //so that server knows when the data ends. The Transfer-Encoding: chunked header is added. //https://nodejs.org/api/http.html#requestwritechunk-encoding-callback //issue with Transfer-Encoding: chunked wopi and sharepoint 2019 //https://community.alteryx.com/t5/Dev-Space/Download-Tool-amp-Microsoft-SharePoint-Chunked-Request-Error/td-p/735824 - headers['Content-Length'] = postDataSize; + options.headers['Content-Length'] = postDataSize; } - let connectionAndInactivity = optTimeout && optTimeout.connectionAndInactivity && ms(optTimeout.connectionAndInactivity); - let options = config.util.extendDeep({}, tenTenantRequestDefaults); - Object.assign(options, {uri: urlParsed, encoding: 'utf8', headers: headers, timeout: connectionAndInactivity}); - //baseRequest creates new agent(win-ca injects in globalAgent) - options.agentOptions = https.globalAgent.options; - if (postData) { - options.body = postData; - } - let executed = false; let ro = request.post(options, function(err, response, body) { if (executed) { diff --git a/DocService/sources/DocsCoServer.js b/DocService/sources/DocsCoServer.js index dc1a8798e..fcd235d39 100644 --- a/DocService/sources/DocsCoServer.js +++ b/DocService/sources/DocsCoServer.js @@ -747,7 +747,8 @@ async function sendServerRequest(ctx, uri, dataObject, opt_checkAndFixAuthorizat } dataObject.setToken(bodyToken); } - let postRes = await utils.postRequestPromise(ctx, uri, JSON.stringify(dataObject), undefined, undefined, tenCallbackRequestTimeout, auth); + let headers = {'Content-Type': 'application/json'}; + let postRes = await utils.postRequestPromise(ctx, uri, JSON.stringify(dataObject), undefined, undefined, tenCallbackRequestTimeout, auth, headers); ctx.logger.debug('postData response: data = %s', postRes.body); return postRes.body; } diff --git a/DocService/sources/canvasservice.js b/DocService/sources/canvasservice.js index 930e5ede1..bd1ce232f 100644 --- a/DocService/sources/canvasservice.js +++ b/DocService/sources/canvasservice.js @@ -1773,7 +1773,7 @@ async function processWopiSaveAs(ctx, cmd) { const storageFilePath = `${cmd.getSaveKey()}/${cmd.getOutputPath()}`; const stream = await storage.createReadStream(ctx, storageFilePath); const { wopiSrc, access_token } = info.wopiParams.userAuth; - await wopiClient.putRelativeFile(ctx, wopiSrc, access_token, null, stream.readStream, stream.ContentLength, suggestedTargetType, false); + await wopiClient.putRelativeFile(ctx, wopiSrc, access_token, null, stream.readStream, stream.contentLength, suggestedTargetType, false); } } exports.receiveTask = function(data, ack) { diff --git a/DocService/sources/wopiClient.js b/DocService/sources/wopiClient.js index 238e2c97c..47342360a 100644 --- a/DocService/sources/wopiClient.js +++ b/DocService/sources/wopiClient.js @@ -664,6 +664,7 @@ function putFile(ctx, wopiParams, data, dataStream, dataSize, userLastChangeId, //collabora nexcloud connector headers['X-LOOL-WOPI-Timestamp'] = wopiParams.LastModifiedTime; } + headers['Content-Type'] = mime.getType(getFileTypeByInfo(fileInfo)); ctx.logger.debug('wopi PutFile request uri=%s headers=%j', uri, headers); postRes = yield utils.postRequestPromise(ctx, uri, data, dataStream, dataSize, tenCallbackRequestTimeout, undefined, headers); @@ -698,6 +699,7 @@ function putRelativeFile(ctx, wopiSrc, access_token, data, dataStream, dataSize, headers['X-WOPI-FileConversion'] = isFileConversion; } fillStandardHeaders(ctx, headers, uri, access_token); + headers['Content-Type'] = mime.getType(suggestedTarget); ctx.logger.debug('wopi putRelativeFile request uri=%s headers=%j', uri, headers); postRes = yield utils.postRequestPromise(ctx, uri, data, dataStream, dataSize, tenCallbackRequestTimeout, undefined, headers); From e07e14f8660314d11a52a6a79cf0dbfcc876b18c Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Wed, 17 Apr 2024 10:03:37 +0300 Subject: [PATCH 23/48] [feature] Allow "WOPISrc" and "shardkey" query params as shard key --- Common/sources/constants.js | 3 ++- Common/sources/operationContext.js | 21 ++++++++++++------- Common/sources/storage-fs.js | 5 ++++- Common/sources/utils.js | 16 ++++++++++---- DocService/sources/DocsCoServer.js | 5 ++++- DocService/sources/canvasservice.js | 7 ++++++- .../databaseConnectors/connectorUtilities.js | 17 +++++++++++++++ DocService/sources/gc.js | 3 ++- 8 files changed, 61 insertions(+), 16 deletions(-) diff --git a/Common/sources/constants.js b/Common/sources/constants.js index 3299ca82c..ffaabaf78 100644 --- a/Common/sources/constants.js +++ b/Common/sources/constants.js @@ -50,7 +50,8 @@ exports.VIEWER_ONLY = /^(?:(pdf|djvu|xps|oxps))$/; exports.DEFAULT_DOC_ID = 'docId'; exports.DEFAULT_USER_ID = 'userId'; exports.ALLOWED_PROTO = /^https?$/i; -exports.SHARED_KEY_NAME = 'WOPISrc'; +exports.SHARD_KEY_WOPI_NAME = 'WOPISrc'; +exports.SHARD_KEY_API_NAME = 'shardkey'; exports.RIGHTS = { None : 0, diff --git a/Common/sources/operationContext.js b/Common/sources/operationContext.js index 0eeb11a5f..b9f9aaca7 100644 --- a/Common/sources/operationContext.js +++ b/Common/sources/operationContext.js @@ -41,11 +41,12 @@ function Context(){ this.logger = logger.getLogger('nodeJS'); this.initDefault(); } -Context.prototype.init = function(tenant, docId, userId, opt_shardKey) { +Context.prototype.init = function(tenant, docId, userId, opt_shardKey, opt_WopiSrc) { this.setTenant(tenant); this.setDocId(docId); this.setUserId(userId); this.setShardKey(opt_shardKey); + this.setWopiSrc(opt_WopiSrc); this.config = null; this.secret = null; @@ -65,21 +66,23 @@ Context.prototype.initFromConnection = function(conn) { } } let userId = conn.user?.id; - let shardKey = utils.getShardByConnection(this, conn); - this.init(tenant, docId || this.docId, userId || this.userId, shardKey); + let shardKey = utils.getShardKeyByConnection(this, conn); + let wopiSrc = utils.getWopiSrcByConnection(this, conn); + this.init(tenant, docId || this.docId, userId || this.userId, shardKey, wopiSrc); }; Context.prototype.initFromRequest = function(req) { let tenant = tenantManager.getTenantByRequest(this, req); let shardKey = utils.getShardKeyByRequest(this, req); - this.init(tenant, this.docId, this.userId, shardKey); + let wopiSrc = utils.getWopiSrcByRequest(this, req); + this.init(tenant, this.docId, this.userId, shardKey, wopiSrc); }; Context.prototype.initFromTaskQueueData = function(task) { let ctx = task.getCtx(); - this.init(ctx.tenant, ctx.docId, ctx.userId, ctx.shardKey); + this.init(ctx.tenant, ctx.docId, ctx.userId, ctx.shardKey, ctx.wopiSrc); }; Context.prototype.initFromPubSub = function(data) { let ctx = data.ctx; - this.init(ctx.tenant, ctx.docId, ctx.userId, ctx.shardKey); + this.init(ctx.tenant, ctx.docId, ctx.userId, ctx.shardKey, ctx.wopiSrc); }; Context.prototype.initTenantCache = async function() { this.config = await tenantManager.getTenantConfig(this); @@ -101,12 +104,16 @@ Context.prototype.setUserId = function(userId) { Context.prototype.setShardKey = function(shardKey) { this.shardKey = shardKey; }; +Context.prototype.setWopiSrc = function(wopiSrc) { + this.wopiSrc = wopiSrc; +}; Context.prototype.toJSON = function() { return { tenant: this.tenant, docId: this.docId, userId: this.userId, - shardKey: this.shardKey + shardKey: this.shardKey, + wopiSrc: this.wopiSrc } }; Context.prototype.getCfg = function(property, defaultValue) { diff --git a/Common/sources/storage-fs.js b/Common/sources/storage-fs.js index 1b9670e0c..b8e404a2d 100644 --- a/Common/sources/storage-fs.js +++ b/Common/sources/storage-fs.js @@ -142,7 +142,10 @@ async function getSignedUrl(ctx, storageCfg, baseUrl, strPath, urlType, optFilen url += '?md5=' + encodeURIComponent(md5); url += '&expires=' + encodeURIComponent(expires); if (ctx.shardKey) { - url += `&${constants.SHARED_KEY_NAME}=${encodeURIComponent(ctx.shardKey)}`; + url += `&${constants.SHARD_KEY_API_NAME}=${encodeURIComponent(ctx.shardKey)}`; + } + if (ctx.wopiSrc) { + url += `&${constants.SHARD_KEY_WOPI_NAME}=${encodeURIComponent(ctx.wopiSrc)}`; } url += '&filename=' + userFriendlyName; return url; diff --git a/Common/sources/utils.js b/Common/sources/utils.js index 5410cd9d4..52ed83cb3 100644 --- a/Common/sources/utils.js +++ b/Common/sources/utils.js @@ -787,14 +787,22 @@ function getDomainByRequest(ctx, req) { } exports.getDomainByConnection = getDomainByConnection; exports.getDomainByRequest = getDomainByRequest; -function getShardByConnection(ctx, conn) { - return conn?.handshake?.query?.[constants.SHARED_KEY_NAME]; +function getShardKeyByConnection(ctx, conn) { + return conn?.handshake?.query?.[constants.SHARD_KEY_API_NAME]; +} +function getWopiSrcByConnection(ctx, conn) { + return conn?.handshake?.query?.[constants.SHARD_KEY_WOPI_NAME]; } function getShardKeyByRequest(ctx, req) { - return req.query[constants.SHARED_KEY_NAME]; + return req.query?.[constants.SHARD_KEY_API_NAME]; +} +function getWopiSrcByRequest(ctx, req) { + return req.query?.[constants.SHARD_KEY_WOPI_NAME]; } -exports.getShardByConnection = getShardByConnection; +exports.getShardKeyByConnection = getShardKeyByConnection; +exports.getWopiSrcByConnection = getWopiSrcByConnection; exports.getShardKeyByRequest = getShardKeyByRequest; +exports.getWopiSrcByRequest = getWopiSrcByRequest; function stream2Buffer(stream) { return new Promise(function(resolve, reject) { if (!stream.readable) { diff --git a/DocService/sources/DocsCoServer.js b/DocService/sources/DocsCoServer.js index fcd235d39..316547ae8 100644 --- a/DocService/sources/DocsCoServer.js +++ b/DocService/sources/DocsCoServer.js @@ -2522,7 +2522,7 @@ exports.install = function(server, callbackFunction) { upsertRes = yield canvasService.commandOpenStartPromise(ctx, docId, utils.getBaseUrlByConnection(ctx, conn), data.documentCallbackUrl, format); curIndexUser = upsertRes.insertId; //todo update additional in commandOpenStartPromise - if ((upsertRes.isInsert || (wopiParams && 2 === curIndexUser)) && (undefined !== data.timezoneOffset || ctx.shardKey)) { + if ((upsertRes.isInsert || (wopiParams && 2 === curIndexUser)) && (undefined !== data.timezoneOffset || ctx.shardKey || ctx.wopiSrc)) { //todo insert in commandOpenStartPromise. insert here for database compatibility if (false === canvasService.hasAdditionalCol) { let selectRes = yield taskResult.select(ctx, docId); @@ -2539,6 +2539,9 @@ exports.install = function(server, callbackFunction) { if (ctx.shardKey) { task.additional += sqlBase.DocumentAdditional.prototype.setShardKey(ctx.shardKey); } + if (ctx.wopiSrc) { + task.additional += sqlBase.DocumentAdditional.prototype.setWopiSrc(ctx.wopiSrc); + } yield taskResult.update(ctx, task); } else { ctx.logger.warn('auth unknown column "additional"'); diff --git a/DocService/sources/canvasservice.js b/DocService/sources/canvasservice.js index bd1ce232f..dff01cd98 100644 --- a/DocService/sources/canvasservice.js +++ b/DocService/sources/canvasservice.js @@ -628,6 +628,7 @@ let commandSfctByCmd = co.wrap(function*(ctx, cmd, opt_priority, opt_expiration, } if (opt_initShardKey) { ctx.setShardKey(sqlBase.DocumentAdditional.prototype.getShardKey(row.additional)); + ctx.setWopiSrc(sqlBase.DocumentAdditional.prototype.getWopiSrc(row.additional)); } yield* addRandomKeyTaskCmd(ctx, cmd); addPasswordToCmd(ctx, cmd, row.password); @@ -1525,7 +1526,10 @@ function getPrintFileUrl(ctx, docId, baseUrl, filename) { let userFriendlyName = encodeURIComponent(filename.replace(/\//g, "%2f")); let res = `${baseUrl}/printfile/${encodeURIComponent(docId)}/${userFriendlyName}?token=${encodeURIComponent(token)}`; if (ctx.shardKey) { - res += `&${constants.SHARED_KEY_NAME}=${encodeURIComponent(ctx.shardKey)}`; + res += `&${constants.SHARD_KEY_API_NAME}=${encodeURIComponent(ctx.shardKey)}`; + } + if (ctx.wopiSrc) { + res += `&${constants.SHARD_KEY_WOPI_NAME}=${encodeURIComponent(ctx.wopiSrc)}`; } res += `&filename=${userFriendlyName}`; return res; @@ -1727,6 +1731,7 @@ exports.saveFromChanges = function(ctx, docId, statusInfo, optFormat, opt_userId } if (opt_initShardKey) { ctx.setShardKey(sqlBase.DocumentAdditional.prototype.getShardKey(row.additional)); + ctx.setWopiSrc(sqlBase.DocumentAdditional.prototype.getWopiSrc(row.additional)); } var cmd = new commonDefines.InputCommand(); cmd.setCommand('sfc'); diff --git a/DocService/sources/databaseConnectors/connectorUtilities.js b/DocService/sources/databaseConnectors/connectorUtilities.js index b7a988c4c..a1b5c1a7c 100644 --- a/DocService/sources/databaseConnectors/connectorUtilities.js +++ b/DocService/sources/databaseConnectors/connectorUtilities.js @@ -168,6 +168,23 @@ DocumentAdditional.prototype.getShardKey = function(str) { return res; }; +DocumentAdditional.prototype.setWopiSrc = function(wopiSrc) { + let additional = new DocumentAdditional(); + additional.data.push({wopiSrc}); + return additional.toSQLInsert(); +}; +DocumentAdditional.prototype.getWopiSrc = function(str) { + let res; + let val = new DocumentAdditional(); + val.fromString(str); + val.data.forEach((elem) => { + if (elem.wopiSrc) { + res = elem.wopiSrc; + } + }); + return res; +}; + module.exports = { UserCallback, DocumentPassword, diff --git a/DocService/sources/gc.js b/DocService/sources/gc.js index 6fbaf357f..6c734a2f1 100644 --- a/DocService/sources/gc.js +++ b/DocService/sources/gc.js @@ -78,7 +78,8 @@ var checkFileExpire = function(expireSeconds) { let tenant = expired[i].tenant; let docId = expired[i].id; let shardKey = sqlBase.DocumentAdditional.prototype.getShardKey(expired[i].additional); - ctx.init(tenant, docId, ctx.userId, shardKey); + let wopiSrc = sqlBase.DocumentAdditional.prototype.getWopiSrc(expired[i].additional); + ctx.init(tenant, docId, ctx.userId, shardKey, wopiSrc); yield ctx.initTenantCache(); //todo tenant //check that no one is in the document From f33cc2f4454581c358a3508b8dff1d17b01485ba Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Mon, 22 Apr 2024 19:48:34 +0300 Subject: [PATCH 24/48] [feature] Remove unused allowPrivateIPAddressForSignedRequests; bug 63590 --- Common/config/default.json | 3 +-- Common/sources/utils.js | 4 +--- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/Common/config/default.json b/Common/config/default.json index fcccf356c..8e9751fd6 100644 --- a/Common/config/default.json +++ b/Common/config/default.json @@ -170,8 +170,7 @@ "assemblyFormatAsOrigin": true, "newFileTemplate" : "../../document-templates/new", "downloadFileAllowExt": ["pdf", "xlsx"], - "tokenRequiredParams": true, - "allowPrivateIPAddressForSignedRequests": true + "tokenRequiredParams": true }, "requestDefaults": { "headers": { diff --git a/Common/sources/utils.js b/Common/sources/utils.js index 52ed83cb3..7314d7fac 100644 --- a/Common/sources/utils.js +++ b/Common/sources/utils.js @@ -83,7 +83,6 @@ const cfgPasswordEncrypt = config.get('openpgpjs.encrypt'); const cfgPasswordDecrypt = config.get('openpgpjs.decrypt'); const cfgPasswordConfig = config.get('openpgpjs.config'); const cfgRequesFilteringAgent = config.get('services.CoAuthoring.request-filtering-agent'); -const cfgAllowPrivateIPAddressForSignedRequests = config.get('services.CoAuthoring.server.allowPrivateIPAddressForSignedRequests'); const cfgStorageExternalHost = config.get('storage.externalHost'); const cfgExternalRequestDirectIfIn = config.get('externalRequest.directIfIn'); const cfgExternalRequestAction = config.get('externalRequest.action'); @@ -272,7 +271,6 @@ function isRedirectResponse(response) { function isAllowDirectRequest(ctx, uri, isInJwtToken) { let res = false; const tenExternalRequestDirectIfIn = ctx.getCfg('externalRequest.directIfIn', cfgExternalRequestDirectIfIn); - const tenAllowPrivateIPAddressForSignedRequests = ctx.getCfg('services.CoAuthoring.server.allowPrivateIPAddressForSignedRequests', cfgAllowPrivateIPAddressForSignedRequests); let allowList = tenExternalRequestDirectIfIn.allowList; if (allowList.length > 0) { let allowIndex = allowList.findIndex((allowPrefix) => { @@ -280,7 +278,7 @@ function isAllowDirectRequest(ctx, uri, isInJwtToken) { }, uri); res = -1 !== allowIndex; ctx.logger.debug("isAllowDirectRequest check allow list res=%s", res); - } else if (tenExternalRequestDirectIfIn.jwtToken && tenAllowPrivateIPAddressForSignedRequests) { + } else if (tenExternalRequestDirectIfIn.jwtToken) { res = isInJwtToken; ctx.logger.debug("isAllowDirectRequest url in jwt token res=%s", res); } From a5a9db4e576699bf2ade79b26f7fc2058f82e6a7 Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Wed, 24 Apr 2024 02:04:01 +0300 Subject: [PATCH 25/48] [config] Rename proxyAuth->proxyUser --- Common/config/default.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Common/config/default.json b/Common/config/default.json index 8e9751fd6..6c867a733 100644 --- a/Common/config/default.json +++ b/Common/config/default.json @@ -133,7 +133,7 @@ "allow": true, "blockPrivateIP": true, "proxyUrl": "", - "proxyAuth": { + "proxyUser": { "username": "", "password": "" }, From 25f99f8ea5415725664e35148fef196219f09a87 Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Wed, 24 Apr 2024 14:33:56 +0300 Subject: [PATCH 26/48] [bug] Set "formsubmit" as default action; Fix bug 66720 --- DocService/sources/pubsubRabbitMQ.js | 2 +- DocService/sources/wopiClient.js | 16 ++++++++++------ 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/DocService/sources/pubsubRabbitMQ.js b/DocService/sources/pubsubRabbitMQ.js index 57707285f..fb820a8fc 100644 --- a/DocService/sources/pubsubRabbitMQ.js +++ b/DocService/sources/pubsubRabbitMQ.js @@ -151,7 +151,7 @@ function repeat(pubsub) { } function publishRabbit(pubsub, data) { return new Promise(function (resolve, reject) { - //Channels act like stream.Writable when you call publish or sendToQueue: they return either true, meaning “keep sending”, or false, meaning “please wait for a ‘drain’ event”. + //Channels act like stream.Writable when you call publish or sendToQueue: they return either true, meaning “keep sendingâ€, or false, meaning “please wait for a ‘drain’ eventâ€. let keepSending = pubsub.channelPublish.publish(pubsub.exchangePublish, '', data); if (!keepSending) { //todo (node:4308) MaxListenersExceededWarning: Possible EventEmitter memory leak detected. 11 drain listeners added to [Sender]. Use emitter.setMaxListeners() to increase limit diff --git a/DocService/sources/wopiClient.js b/DocService/sources/wopiClient.js index 47342360a..06adc492d 100644 --- a/DocService/sources/wopiClient.js +++ b/DocService/sources/wopiClient.js @@ -209,11 +209,13 @@ function discovery(req, res) { xmlApp.ele('action', {name: 'view', ext: ext.edit[j], urlsrc: urlTemplateView}).up(); xmlApp.ele('action', {name: 'embedview', ext: ext.edit[j], urlsrc: urlTemplateEmbedView}).up(); xmlApp.ele('action', {name: 'mobileView', ext: ext.edit[j], urlsrc: urlTemplateMobileView}).up(); - xmlApp.ele('action', {name: 'edit', ext: ext.edit[j], default: 'true', requires: 'locks,update', urlsrc: urlTemplateEdit}).up(); - xmlApp.ele('action', {name: 'mobileEdit', ext: ext.edit[j], requires: 'locks,update', urlsrc: urlTemplateMobileEdit}).up(); if (formsExts[ext.edit[j]]) { - xmlApp.ele('action', {name: 'formsubmit', ext: ext.edit[j], requires: 'locks,update', urlsrc: urlTemplateFormSubmit}).up(); + xmlApp.ele('action', {name: 'formsubmit', ext: ext.edit[j], default: 'true', requires: 'locks,update', urlsrc: urlTemplateFormSubmit}).up(); + xmlApp.ele('action', {name: 'edit', ext: ext.edit[j], requires: 'locks,update', urlsrc: urlTemplateEdit}).up(); + } else { + xmlApp.ele('action', {name: 'edit', ext: ext.edit[j], default: 'true', requires: 'locks,update', urlsrc: urlTemplateEdit}).up(); } + xmlApp.ele('action', {name: 'mobileEdit', ext: ext.edit[j], requires: 'locks,update', urlsrc: urlTemplateMobileEdit}).up(); if (templatesFolderExtsCache[ext.edit[j]]) { xmlApp.ele('action', {name: 'editnew', ext: ext.edit[j], requires: 'locks,update', urlsrc: urlTemplateEdit}).up(); } @@ -251,11 +253,13 @@ function discovery(req, res) { if (mimeTypes) { mimeTypes.forEach((value) => { let xmlApp = xmlZone.ele('app', {name: value}); - xmlApp.ele('action', {name: 'edit', ext: '', default: 'true', requires: 'locks,update', urlsrc: urlTemplateEdit}).up(); - xmlApp.ele('action', {name: 'mobileEdit', ext: '', requires: 'locks,update', urlsrc: urlTemplateMobileEdit}).up(); if (formsExts[ext.edit[j]]) { - xmlApp.ele('action', {name: 'formsubmit', ext: '', requires: 'locks,update', urlsrc: urlTemplateFormSubmit}).up(); + xmlApp.ele('action', {name: 'formsubmit', ext: '', default: 'true', requires: 'locks,update', urlsrc: urlTemplateFormSubmit}).up(); + xmlApp.ele('action', {name: 'edit', ext: '', requires: 'locks,update', urlsrc: urlTemplateEdit}).up(); + } else { + xmlApp.ele('action', {name: 'edit', ext: '', default: 'true', requires: 'locks,update', urlsrc: urlTemplateEdit}).up(); } + xmlApp.ele('action', {name: 'mobileEdit', ext: '', requires: 'locks,update', urlsrc: urlTemplateMobileEdit}).up(); if (templatesFolderExtsCache[ext.edit[j]]) { xmlApp.ele('action', {name: 'editnew', ext: '', requires: 'locks,update', urlsrc: urlTemplateEdit}).up(); } From 3b9bed32aea6b88354aac726cf963555f9983558 Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Sun, 28 Apr 2024 00:28:33 +0300 Subject: [PATCH 27/48] [bug] Fix "WOPISrc" convert param in getConverterHtml --- DocService/sources/converterservice.js | 2 +- DocService/sources/wopiClient.js | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/DocService/sources/converterservice.js b/DocService/sources/converterservice.js index 310621613..5dad20018 100644 --- a/DocService/sources/converterservice.js +++ b/DocService/sources/converterservice.js @@ -651,7 +651,7 @@ function getConverterHtmlHandler(req, res) { ctx.setDocId(docId); if (!(wopiSrc && access_token && access_token && targetext && docId) || constants.AVS_OFFICESTUDIO_FILE_UNKNOWN === formatChecker.getFormatFromString(targetext)) { - ctx.logger.debug('convert-and-edit-handler invalid params: wopiSrc=%s; access_token=%s; targetext=%s; docId=%s', wopiSrc, access_token, targetext, docId); + ctx.logger.debug('convert-and-edit-handler invalid params: WOPISrc=%s; access_token=%s; targetext=%s; docId=%s', wopiSrc, access_token, targetext, docId); utils.fillResponse(req, res, new commonDefines.ConvertStatus(constants.CONVERT_PARAMS), isJson); return; } diff --git a/DocService/sources/wopiClient.js b/DocService/sources/wopiClient.js index 06adc492d..90861e4b1 100644 --- a/DocService/sources/wopiClient.js +++ b/DocService/sources/wopiClient.js @@ -596,7 +596,7 @@ function getConverterHtml(req, res) { let targetext = req.params.targetext; if (!(wopiSrc && access_token && access_token_ttl && ext && targetext)) { - ctx.logger.debug('convert-and-edit invalid params: wopiSrc=%s; access_token=%s; access_token_ttl=%s; ext=%s; targetext=%s', wopiSrc, access_token, access_token_ttl, ext, targetext); + ctx.logger.debug('convert-and-edit invalid params: WOPISrc=%s; access_token=%s; access_token_ttl=%s; ext=%s; targetext=%s', wopiSrc, access_token, access_token_ttl, ext, targetext); return; } @@ -612,7 +612,7 @@ function getConverterHtml(req, res) { if (docId) { let baseUrl = tenWopiHost || utils.getBaseUrlByRequest(ctx, req); params.statusHandler = `${baseUrl}/hosting/wopi/convert-and-edit-handler`; - params.statusHandler += `?wopiSrc=${encodeURI(wopiSrc)}&access_token=${encodeURI(access_token)}`; + params.statusHandler += `?${constants.SHARD_KEY_WOPI_NAME}=${encodeURI(wopiSrc)}&access_token=${encodeURI(access_token)}`; params.statusHandler += `&targetext=${encodeURI(targetext)}&docId=${encodeURI(docId)}`; if (tenTokenEnableBrowser) { let tokenData = {docId: docId}; From 25d52cab4c6babb628368bcc72ef2c1a677472f0 Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Sun, 28 Apr 2024 15:13:57 +0300 Subject: [PATCH 28/48] [bug] Fix error on reconnection in formsubmit action; For bug 66720 --- DocService/sources/DocsCoServer.js | 9 +++++++-- DocService/sources/wopiClient.js | 6 +++--- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/DocService/sources/DocsCoServer.js b/DocService/sources/DocsCoServer.js index 316547ae8..a2c383025 100644 --- a/DocService/sources/DocsCoServer.js +++ b/DocService/sources/DocsCoServer.js @@ -2261,6 +2261,7 @@ exports.install = function(server, callbackFunction) { } if (wopiClient.isWopiJwtToken(decoded)) { let fileInfo = decoded.fileInfo; + let queryParams = decoded.queryParams; if (openCmd) { openCmd.format = wopiClient.getFileTypeByInfo(fileInfo); openCmd.title = fileInfo.BreadcrumbDocName || fileInfo.BaseFileName; @@ -2276,11 +2277,15 @@ exports.install = function(server, callbackFunction) { openCmd.userid = fileInfo.UserId; } } + let permissionsEdit = !fileInfo.ReadOnly && fileInfo.UserCanWrite && queryParams.formsubmit !== "1"; + let permissionsFillForm = permissionsEdit || queryParams.formsubmit === "1"; let permissions = { - edit: !fileInfo.ReadOnly && fileInfo.UserCanWrite, + edit: permissionsEdit, review: (fileInfo.SupportsReviewing === false) ? false : (fileInfo.UserCanReview === false ? false : fileInfo.UserCanReview), copy: fileInfo.CopyPasteRestrictions !== "CurrentDocumentOnly" && fileInfo.CopyPasteRestrictions !== "BlockAll", - print: !fileInfo.DisablePrint && !fileInfo.HidePrintOption + print: !fileInfo.DisablePrint && !fileInfo.HidePrintOption, + chat: queryParams.dchat!=="1", + fillForms: permissionsFillForm }; //todo (review: undefiend) // res = deepEqual(data.permissions, permissions, {strict: true}); diff --git a/DocService/sources/wopiClient.js b/DocService/sources/wopiClient.js index 90861e4b1..2bb940930 100644 --- a/DocService/sources/wopiClient.js +++ b/DocService/sources/wopiClient.js @@ -612,15 +612,15 @@ function getConverterHtml(req, res) { if (docId) { let baseUrl = tenWopiHost || utils.getBaseUrlByRequest(ctx, req); params.statusHandler = `${baseUrl}/hosting/wopi/convert-and-edit-handler`; - params.statusHandler += `?${constants.SHARD_KEY_WOPI_NAME}=${encodeURI(wopiSrc)}&access_token=${encodeURI(access_token)}`; - params.statusHandler += `&targetext=${encodeURI(targetext)}&docId=${encodeURI(docId)}`; + params.statusHandler += `?${constants.SHARD_KEY_WOPI_NAME}=${encodeURIComponent(wopiSrc)}&access_token=${encodeURIComponent(access_token)}`; + params.statusHandler += `&targetext=${encodeURIComponent(targetext)}&docId=${encodeURIComponent(docId)}`; if (tenTokenEnableBrowser) { let tokenData = {docId: docId}; let options = {algorithm: tenTokenOutboxAlgorithm, expiresIn: tenTokenOutboxExpires}; let secret = yield tenantManager.getTenantSecret(ctx, commonDefines.c_oAscSecretType.Browser); let token = jwt.sign(tokenData, secret, options); - params.statusHandler += `&token=${encodeURI(token)}`; + params.statusHandler += `&token=${encodeURIComponent(token)}`; } } } catch (err) { From 9ac4993965de505a4926056b0317029689e44bfc Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Thu, 2 May 2024 15:14:23 +0300 Subject: [PATCH 29/48] [bug] Add clearTimeout in downloadUrlPromiseWithoutRedirect; Fix bug 67804 --- Common/sources/utils.js | 20 ++++++++++++++---- DocService/sources/canvasservice.js | 32 +++++++++++++++++------------ DocService/sources/wopiClient.js | 7 ++++++- 3 files changed, 41 insertions(+), 18 deletions(-) diff --git a/Common/sources/utils.js b/Common/sources/utils.js index 7314d7fac..909d77333 100644 --- a/Common/sources/utils.js +++ b/Common/sources/utils.js @@ -35,6 +35,7 @@ //Fix EPROTO error in node 8.x at some web sites(https://github.com/nodejs/node/issues/21513) require("tls").DEFAULT_ECDH_CURVE = "auto"; +const { pipeline } = require('node:stream/promises'); var config = require('config'); var fs = require('fs'); var path = require('path'); @@ -350,7 +351,7 @@ function downloadUrlPromiseWithoutRedirect(ctx, uri, optTimeout, optLimit, opt_A uri = URI.serialize(URI.parse(uri)); var urlParsed = url.parse(uri); let sizeLimit = optLimit || Number.MAX_VALUE; - let bufferLength = 0; + let bufferLength = 0, timeoutId; let hash = crypto.createHash('sha256'); //if you expect binary data, you should set encoding: null let connectionAndInactivity = optTimeout && optTimeout.connectionAndInactivity && ms(optTimeout.connectionAndInactivity); @@ -375,6 +376,7 @@ function downloadUrlPromiseWithoutRedirect(ctx, uri, optTimeout, optLimit, opt_A Object.assign(options.headers, opt_headers); } let fError = function(err) { + clearTimeout(timeoutId); reject(err); } if (!opt_streamWriter) { @@ -386,6 +388,7 @@ function downloadUrlPromiseWithoutRedirect(ctx, uri, optTimeout, optLimit, opt_A } executed = true; if (err) { + clearTimeout(timeoutId); reject(err); } else { var contentLength = response.caseless.get('content-length'); @@ -393,6 +396,7 @@ function downloadUrlPromiseWithoutRedirect(ctx, uri, optTimeout, optLimit, opt_A ctx.logger.warn('downloadUrlPromise body size mismatch: uri=%s; content-length=%s; body.length=%d', uri, contentLength, body.length); } let sha256 = hash.digest('hex'); + clearTimeout(timeoutId); resolve({response: response, body: body, sha256: sha256}); } }; @@ -414,13 +418,21 @@ function downloadUrlPromiseWithoutRedirect(ctx, uri, optTimeout, optLimit, opt_A error.response = response; if (opt_streamWriter && !isRedirectResponse(response)) { this.off('error', fError); - resolve(pipeStreams(this, opt_streamWriter, true)); + pipeline(this, opt_streamWriter) + .then(resolve, reject) + .finally(() => { + clearTimeout(timeoutId); + }); } else { raiseErrorObj(this, error); } } else if (opt_streamWriter) { this.off('error', fError); - resolve(pipeStreams(this, opt_streamWriter, true)); + pipeline(this, opt_streamWriter) + .then(resolve, reject) + .finally(() => { + clearTimeout(timeoutId); + }); } }; let fData = function(chunk) { @@ -436,7 +448,7 @@ function downloadUrlPromiseWithoutRedirect(ctx, uri, optTimeout, optLimit, opt_A .on('data', fData) .on('error', fError); if (optTimeout && optTimeout.wholeCycle) { - setTimeout(function() { + timeoutId = setTimeout(function() { raiseError(ro, 'ETIMEDOUT', 'Error: whole request cycle timeout'); }, ms(optTimeout.wholeCycle)); } diff --git a/DocService/sources/canvasservice.js b/DocService/sources/canvasservice.js index dff01cd98..c40854ec6 100644 --- a/DocService/sources/canvasservice.js +++ b/DocService/sources/canvasservice.js @@ -1693,20 +1693,26 @@ exports.downloadFile = function(req, res) { } } catch (err) { - ctx.logger.error('Error downloadFile: %s', err.stack); - //catch errors because status may be sent while piping to response - try { - if (err.code === 'ETIMEDOUT' || err.code === 'ESOCKETTIMEDOUT') { - res.sendStatus(408); - } else if (err.code === 'EMSGSIZE') { - res.sendStatus(413); - } else if (err.response) { - res.sendStatus(err.response.statusCode); - } else { - res.sendStatus(400); - } - } catch (err) { + if (err.code === "ERR_STREAM_PREMATURE_CLOSE") { + ctx.logger.debug('Error downloadFile: %s', err.stack); + } else { ctx.logger.error('Error downloadFile: %s', err.stack); + //catch errors because status may be sent while piping to response + if (!res.headersSent) { + try { + if (err.code === 'ETIMEDOUT' || err.code === 'ESOCKETTIMEDOUT') { + res.sendStatus(408); + } else if (err.code === 'EMSGSIZE') { + res.sendStatus(413); + } else if (err.response) { + res.sendStatus(err.response.statusCode); + } else { + res.sendStatus(400); + } + } catch (err) { + ctx.logger.error('Error downloadFile: %s', err.stack); + } + } } } finally { diff --git a/DocService/sources/wopiClient.js b/DocService/sources/wopiClient.js index 2bb940930..3272e410a 100644 --- a/DocService/sources/wopiClient.js +++ b/DocService/sources/wopiClient.js @@ -992,7 +992,12 @@ async function dummyGetFile(req, res) { res, ); } catch (err) { - ctx.logger.error('dummyGetFile error:%s', err.stack); + if (err.code === "ERR_STREAM_PREMATURE_CLOSE") { + //xhr.abort case + ctx.logger.debug('dummyGetFile error: %s', err.stack); + } else { + ctx.logger.error('dummyGetFile error:%s', err.stack); + } } finally { if (!res.headersSent) { res.sendStatus(400); From 5c8b5534b8cf15ac6d671d0984e6859e48dec9d7 Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Sun, 5 May 2024 01:59:37 +0300 Subject: [PATCH 30/48] [bug] Fix sendForm request for wopi; for bug 66720 --- DocService/sources/DocsCoServer.js | 3 ++- DocService/sources/canvasservice.js | 19 ++++++++++++++++--- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/DocService/sources/DocsCoServer.js b/DocService/sources/DocsCoServer.js index a2c383025..13ebeb09a 100644 --- a/DocService/sources/DocsCoServer.js +++ b/DocService/sources/DocsCoServer.js @@ -845,7 +845,8 @@ function* setForceSave(ctx, docId, forceSave, cmd, success, url) { if (commonDefines.c_oAscForceSaveTypes.Command !== forceSaveType) { let data = {type: forceSaveType, time: forceSave.getTime(), success: success}; if(commonDefines.c_oAscForceSaveTypes.Form === forceSaveType || commonDefines.c_oAscForceSaveTypes.Internal === forceSaveType) { - data = {code: commonDefines.c_oAscServerCommandErrors.NoError, time: null, inProgress: false}; + let code = success ? commonDefines.c_oAscServerCommandErrors.NoError : commonDefines.c_oAscServerCommandErrors.UnknownError; + data = {code: code, time: null, inProgress: false}; if (commonDefines.c_oAscForceSaveTypes.Internal === forceSaveType) { data.url = url; } diff --git a/DocService/sources/canvasservice.js b/DocService/sources/canvasservice.js index c40854ec6..22e4b86e5 100644 --- a/DocService/sources/canvasservice.js +++ b/DocService/sources/canvasservice.js @@ -1077,8 +1077,17 @@ const commandSfcCallback = co.wrap(function*(ctx, cmd, isSfcm, isEncrypted) { } else { try { if (wopiParams) { - let isAutoSave = forceSaveType !== commonDefines.c_oAscForceSaveTypes.Button && forceSaveType !== commonDefines.c_oAscForceSaveTypes.Form; - replyStr = yield processWopiPutFile(ctx, docId, wopiParams, savePathDoc, userLastChangeId, true, isAutoSave, false); + if (outputSfc.getUrl()) { + if (forceSaveType === commonDefines.c_oAscForceSaveTypes.Form) { + yield processWopiSaveAs(ctx, cmd); + replyStr = JSON.stringify({error: 0}); + } else { + let isAutoSave = forceSaveType !== commonDefines.c_oAscForceSaveTypes.Button && forceSaveType !== commonDefines.c_oAscForceSaveTypes.Form; + replyStr = yield processWopiPutFile(ctx, docId, wopiParams, savePathDoc, userLastChangeId, true, isAutoSave, false); + } + } else { + replyStr = JSON.stringify({error: 1, descr: "wopi: no file"}); + } } else { replyStr = yield docsCoServer.sendServerRequest(ctx, uri, outputSfc, checkAndFixAuthorizationLength); } @@ -1111,7 +1120,11 @@ const commandSfcCallback = co.wrap(function*(ctx, cmd, isSfcm, isEncrypted) { updateMask.statusInfo = updateIfTask.statusInfo; try { if (wopiParams) { - replyStr = yield processWopiPutFile(ctx, docId, wopiParams, savePathDoc, userLastChangeId, !notModified, false, true); + if (outputSfc.getUrl()) { + replyStr = yield processWopiPutFile(ctx, docId, wopiParams, savePathDoc, userLastChangeId, !notModified, false, true); + } else { + replyStr = JSON.stringify({error: 1, descr: "wopi: no file"}); + } } else { replyStr = yield docsCoServer.sendServerRequest(ctx, uri, outputSfc, checkAndFixAuthorizationLength); } From 4007f5a7a4c396a0e9272801e468c19e981673c7 Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Mon, 6 May 2024 18:03:11 +0300 Subject: [PATCH 31/48] [refactoring] Refactor publish as async function --- DocService/sources/DocsCoServer.js | 54 ++++++++++++++--------------- DocService/sources/canvasservice.js | 2 +- 2 files changed, 28 insertions(+), 28 deletions(-) diff --git a/DocService/sources/DocsCoServer.js b/DocService/sources/DocsCoServer.js index 13ebeb09a..dc343b1ee 100644 --- a/DocService/sources/DocsCoServer.js +++ b/DocService/sources/DocsCoServer.js @@ -495,7 +495,7 @@ function removePresence(ctx, conn) { let changeConnectionInfo = co.wrap(function*(ctx, conn, cmd) { if (!conn.denyChangeName && conn.user) { - yield* publish(ctx, {type: commonDefines.c_oPublishType.changeConnecitonInfo, ctx: ctx, docId: conn.docId, useridoriginal: conn.user.idOriginal, cmd: cmd}); + yield publish(ctx, {type: commonDefines.c_oPublishType.changeConnecitonInfo, ctx: ctx, docId: conn.docId, useridoriginal: conn.user.idOriginal, cmd: cmd}); return true; } return false; @@ -672,12 +672,12 @@ function* isUserReconnect(ctx, docId, userId, connectionId) { } let pubsubOnMessage = null;//todo move function -function* publish(ctx, data, optDocId, optUserId, opt_pubsub) { +async function publish(ctx, data, optDocId, optUserId, opt_pubsub) { var needPublish = true; let hvals; if (optDocId && optUserId) { needPublish = false; - hvals = yield editorData.getPresence(ctx, optDocId, connections); + hvals = await editorData.getPresence(ctx, optDocId, connections); for (var i = 0; i < hvals.length; ++i) { var elem = JSON.parse(hvals[i]); if (optUserId != elem.id) { @@ -695,7 +695,7 @@ function* publish(ctx, data, optDocId, optUserId, opt_pubsub) { //todo send connections from getLocalConnectionCount to pubsubOnMessage pubsubOnMessage(msg); } else if(realPubsub) { - yield realPubsub.publish(msg); + await realPubsub.publish(msg); } } return needPublish; @@ -852,9 +852,9 @@ function* setForceSave(ctx, docId, forceSave, cmd, success, url) { } let userId = cmd.getUserConnectionId(); docId = cmd.getUserConnectionDocId() || docId; - yield* publish(ctx, {type: commonDefines.c_oPublishType.rpc, ctx, docId, userId, data, responseKey: cmd.getResponseKey()}); + yield publish(ctx, {type: commonDefines.c_oPublishType.rpc, ctx, docId, userId, data, responseKey: cmd.getResponseKey()}); } else { - yield* publish(ctx, {type: commonDefines.c_oPublishType.forceSave, ctx: ctx, docId: docId, data: data}, cmd.getUserConnectionId()); + yield publish(ctx, {type: commonDefines.c_oPublishType.forceSave, ctx: ctx, docId: docId, data: data}, cmd.getUserConnectionId()); } } } @@ -982,10 +982,10 @@ async function startForceSave(ctx, docId, type, opt_userdata, opt_formdata, opt_ if (constants.NO_ERROR === status.err) { res.time = forceSave.getTime(); if (commonDefines.c_oAscForceSaveTypes.Timeout === type) { - await co(publish(ctx, { + await publish(ctx, { type: commonDefines.c_oPublishType.forceSave, ctx: ctx, docId: docId, data: {type: type, time: forceSave.getTime(), start: true} - }, undefined, undefined, opt_pubsub)); + }, undefined, undefined, opt_pubsub); } } else { res.code = commonDefines.c_oAscServerCommandErrors.UnknownError; @@ -1210,7 +1210,7 @@ let onReplySendStatusDocument = co.wrap(function*(ctx, docId, replyData) { var oData = parseReplyData(ctx, replyData); if (!(oData && commonDefines.c_oAscServerCommandErrors.NoError == oData.error)) { // Error subscribing to callback, send warning - yield* publish(ctx, {type: commonDefines.c_oPublishType.warning, ctx: ctx, docId: docId, description: 'Error on save server subscription!'}); + yield publish(ctx, {type: commonDefines.c_oPublishType.warning, ctx: ctx, docId: docId, description: 'Error on save server subscription!'}); } }); function* publishCloseUsersConnection(ctx, docId, users, isOriginalId, code, description) { @@ -1219,7 +1219,7 @@ function* publishCloseUsersConnection(ctx, docId, users, isOriginalId, code, des map[val] = 1; return map; }, {}); - yield* publish(ctx, { + yield publish(ctx, { type: commonDefines.c_oPublishType.closeConnection, ctx: ctx, docId: docId, usersMap: usersMap, isOriginalId: isOriginalId, code: code, description: description }); @@ -1240,7 +1240,7 @@ function closeUsersConnection(ctx, docId, usersMap, isOriginalId, code, descript } function* dropUsersFromDocument(ctx, docId, users) { if (Array.isArray(users)) { - yield* publish(ctx, {type: commonDefines.c_oPublishType.drop, ctx: ctx, docId: docId, users: users, description: ''}); + yield publish(ctx, {type: commonDefines.c_oPublishType.drop, ctx: ctx, docId: docId, users: users, description: ''}); } } @@ -1812,7 +1812,7 @@ exports.install = function(server, callbackFunction) { if (!participantsTimestamp) { participantsTimestamp = Date.now(); } - yield* publish(ctx, {type: commonDefines.c_oPublishType.participantsState, ctx: ctx, docId: docId, userId: tmpUser.id, participantsTimestamp: participantsTimestamp, participants: participants}, docId, tmpUser.id); + yield publish(ctx, {type: commonDefines.c_oPublishType.participantsState, ctx: ctx, docId: docId, userId: tmpUser.id, participantsTimestamp: participantsTimestamp, participants: participants}, docId, tmpUser.id); tmpUser.view = tmpView; // For this user, we remove the lock from saving @@ -1838,7 +1838,7 @@ exports.install = function(server, callbackFunction) { if (0 < userLocks.length) { //todo send nothing in case of close document //sendReleaseLock(conn, userLocks); - yield* publish(ctx, {type: commonDefines.c_oPublishType.releaseLock, ctx: ctx, docId: docId, userId: conn.user.id, locks: userLocks}, docId, conn.user.id); + yield publish(ctx, {type: commonDefines.c_oPublishType.releaseLock, ctx: ctx, docId: docId, userId: conn.user.id, locks: userLocks}, docId, conn.user.id); } // For this user, remove the Lock from the document @@ -1973,7 +1973,7 @@ exports.install = function(server, callbackFunction) { var unlockRes = yield editorData.unlockAuth(ctx, docId, userId); if (commonDefines.c_oAscUnlockRes.Unlocked === unlockRes) { const participantsMap = yield getParticipantMap(ctx, docId); - yield* publish(ctx, { + yield publish(ctx, { type: commonDefines.c_oPublishType.auth, ctx: ctx, docId: docId, @@ -1990,7 +1990,7 @@ exports.install = function(server, callbackFunction) { const userLocks = yield* removeUserLocks(ctx, docId, userId); if (0 < userLocks.length) { sendReleaseLock(ctx, conn, userLocks); - yield* publish(ctx, { + yield publish(ctx, { type: commonDefines.c_oPublishType.releaseLock, ctx: ctx, docId: docId, @@ -2864,7 +2864,7 @@ exports.install = function(server, callbackFunction) { //closing could happen during async action return false; } - yield* publish(ctx, {type: commonDefines.c_oPublishType.participantsState, ctx: ctx, docId: docId, userId: tmpUser.id, participantsTimestamp: participantsTimestamp, participants: participantsMap, waitAuthUserId: waitAuthUserId}, docId, tmpUser.id); + yield publish(ctx, {type: commonDefines.c_oPublishType.participantsState, ctx: ctx, docId: docId, userId: tmpUser.id, participantsTimestamp: participantsTimestamp, participants: participantsMap, waitAuthUserId: waitAuthUserId}, docId, tmpUser.id); return res; } @@ -3023,7 +3023,7 @@ exports.install = function(server, callbackFunction) { var messages = [msg]; sendDataMessage(ctx, conn, messages); - yield* publish(ctx, {type: commonDefines.c_oPublishType.message, ctx: ctx, docId: docId, userId: userId, messages: messages}, docId, userId); + yield publish(ctx, {type: commonDefines.c_oPublishType.message, ctx: ctx, docId: docId, userId: userId, messages: messages}, docId, userId); } function* onCursor(ctx, conn, data) { @@ -3034,7 +3034,7 @@ exports.install = function(server, callbackFunction) { ctx.logger.info("send cursor: %s", msg); var messages = [msg]; - yield* publish(ctx, {type: commonDefines.c_oPublishType.cursor, ctx: ctx, docId: docId, userId: userId, messages: messages}, docId, userId); + yield publish(ctx, {type: commonDefines.c_oPublishType.cursor, ctx: ctx, docId: docId, userId: userId, messages: messages}, docId, userId); } function* getLock(ctx, conn, data, bIsRestore) { @@ -3077,7 +3077,7 @@ exports.install = function(server, callbackFunction) { } //to the one who made the request we return as quickly as possible sendData(ctx, conn, {type: "getLock", locks: documentLocks}); - yield* publish(ctx, {type: commonDefines.c_oPublishType.getLock, ctx: ctx, docId: docId, userId: userId, documentLocks: documentLocks}, docId, userId); + yield publish(ctx, {type: commonDefines.c_oPublishType.getLock, ctx: ctx, docId: docId, userId: userId, documentLocks: documentLocks}, docId, userId); return true; } @@ -3102,7 +3102,7 @@ exports.install = function(server, callbackFunction) { } //to the one who made the request we return as quickly as possible sendData(ctx, conn, {type: "getLock", locks: documentLocks}); - yield* publish(ctx, {type: commonDefines.c_oPublishType.getLock, ctx: ctx, docId: docId, userId: userId, documentLocks: documentLocks}, docId, userId); + yield publish(ctx, {type: commonDefines.c_oPublishType.getLock, ctx: ctx, docId: docId, userId: userId, documentLocks: documentLocks}, docId, userId); return true; } @@ -3127,7 +3127,7 @@ exports.install = function(server, callbackFunction) { } //to the one who made the request we return as quickly as possible sendData(ctx, conn, {type: "getLock", locks: documentLocks}); - yield* publish(ctx, {type: commonDefines.c_oPublishType.getLock, ctx: ctx, docId: docId, userId: userId, documentLocks: documentLocks}, docId, userId); + yield publish(ctx, {type: commonDefines.c_oPublishType.getLock, ctx: ctx, docId: docId, userId: userId, documentLocks: documentLocks}, docId, userId); return true; } @@ -3247,7 +3247,7 @@ exports.install = function(server, callbackFunction) { value.time = value.time.getTime(); }) } - yield* publish(ctx, {type: commonDefines.c_oPublishType.changes, ctx: ctx, docId: docId, userId: userId, + yield publish(ctx, {type: commonDefines.c_oPublishType.changes, ctx: ctx, docId: docId, userId: userId, changes: changesToSend, startIndex: startIndex, changesIndex: puckerIndex, syncChangesIndex: puckerIndex, locks: arrLocks, excelAdditionalInfo: data.excelAdditionalInfo, endSaveChanges: data.endSaveChanges}, docId, userId); } @@ -3265,13 +3265,13 @@ exports.install = function(server, callbackFunction) { value.time = value.time.getTime(); }) } - let isPublished = yield* publish(ctx, {type: commonDefines.c_oPublishType.changes, ctx: ctx, docId: docId, userId: userId, + let isPublished = yield publish(ctx, {type: commonDefines.c_oPublishType.changes, ctx: ctx, docId: docId, userId: userId, changes: changesToSend, startIndex: startIndex, changesIndex: puckerIndex, syncChangesIndex: puckerIndex, locks: [], excelAdditionalInfo: undefined, endSaveChanges: data.endSaveChanges}, docId, userId); sendData(ctx, conn, {type: 'savePartChanges', changesIndex: changesIndex, syncChangesIndex: puckerIndex}); if (!isPublished) { //stub for lockDocumentsTimerId - yield* publish(ctx, {type: commonDefines.c_oPublishType.changesNotify, ctx: ctx, docId: docId}); + yield publish(ctx, {type: commonDefines.c_oPublishType.changesNotify, ctx: ctx, docId: docId}); } } } @@ -3768,7 +3768,7 @@ exports.install = function(server, callbackFunction) { if (hasChanges) { let participants = yield getParticipantMap(ctx, data.docId); let participantsTimestamp = Date.now(); - yield* publish(ctx, {type: commonDefines.c_oPublishType.participantsState, ctx: ctx, docId: data.docId, userId: null, participantsTimestamp: participantsTimestamp, participants: participants}); + yield publish(ctx, {type: commonDefines.c_oPublishType.participantsState, ctx: ctx, docId: data.docId, userId: null, participantsTimestamp: participantsTimestamp, participants: participants}); } break; case commonDefines.c_oPublishType.rpc: @@ -4294,7 +4294,7 @@ function* commandHandle(ctx, params, req, output) { } case 'drop': { if (params.userid) { - yield* publish(ctx, {type: commonDefines.c_oPublishType.drop, ctx: ctx, docId: docId, users: [params.userid], description: params.description}); + yield publish(ctx, {type: commonDefines.c_oPublishType.drop, ctx: ctx, docId: docId, users: [params.userid], description: params.description}); } else if (params.users) { const users = (typeof params.users === 'string') ? JSON.parse(params.users) : params.users; yield* dropUsersFromDocument(ctx, docId, users); @@ -4321,7 +4321,7 @@ function* commandHandle(ctx, params, req, output) { } case 'meta': { if (params.meta) { - yield* publish(ctx, {type: commonDefines.c_oPublishType.meta, ctx: ctx, docId: docId, meta: params.meta}); + yield publish(ctx, {type: commonDefines.c_oPublishType.meta, ctx: ctx, docId: docId, meta: params.meta}); } else { output.error = commonDefines.c_oAscServerCommandErrors.UnknownCommand; } diff --git a/DocService/sources/canvasservice.js b/DocService/sources/canvasservice.js index 22e4b86e5..c34b8b17e 100644 --- a/DocService/sources/canvasservice.js +++ b/DocService/sources/canvasservice.js @@ -1836,7 +1836,7 @@ exports.receiveTask = function(data, ack) { if (outputData.getStatus()) { ctx.logger.debug('receiveTask publish: %s', JSON.stringify(outputData)); var output = new OutputDataWrap('documentOpen', outputData); - yield* docsCoServer.publish(ctx, { + yield docsCoServer.publish(ctx, { type: commonDefines.c_oPublishType.receiveTask, ctx: ctx, cmd: cmd, output: output, needUrlKey: additionalOutput.needUrlKey, needUrlMethod: additionalOutput.needUrlMethod, From 05b15721c1a9ec7acb21ded53e41d8544f6ca419 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 2 May 2024 08:41:40 +0000 Subject: [PATCH 32/48] Bump ejs from 3.1.8 to 3.1.10 in /DocService Bumps [ejs](https://github.com/mde/ejs) from 3.1.8 to 3.1.10. - [Release notes](https://github.com/mde/ejs/releases) - [Commits](https://github.com/mde/ejs/compare/v3.1.8...v3.1.10) --- updated-dependencies: - dependency-name: ejs dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- DocService/npm-shrinkwrap.json | 22 +++++++++++----------- DocService/package.json | 2 +- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/DocService/npm-shrinkwrap.json b/DocService/npm-shrinkwrap.json index 26c482ab0..2f04968ac 100644 --- a/DocService/npm-shrinkwrap.json +++ b/DocService/npm-shrinkwrap.json @@ -843,9 +843,9 @@ } }, "async": { - "version": "3.2.4", - "resolved": "https://registry.npmjs.org/async/-/async-3.2.4.tgz", - "integrity": "sha512-iAB+JbDEGXhyIUavoDl9WP/Jj106Kz9DEn1DPgYw5ruDn0e3Wgi3sKFm55sASdGBNOQB8F59d9qQ7deqrHA8wQ==" + "version": "3.2.5", + "resolved": "https://registry.npmjs.org/async/-/async-3.2.5.tgz", + "integrity": "sha512-baNZyqaaLhyLVKm/DlvdW051MSgO6b8eVfIezl9E5PqWxFgzLm/wQntEW4zOytVburDEr0JlALEpdOFwvErLsg==" }, "asynckit": { "version": "0.4.0", @@ -1332,9 +1332,9 @@ "integrity": "sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0=" }, "ejs": { - "version": "3.1.8", - "resolved": "https://registry.npmjs.org/ejs/-/ejs-3.1.8.tgz", - "integrity": "sha512-/sXZeMlhS0ArkfX2Aw780gJzXSMPnKjtspYZv+f3NiKLlubezAHDU5+9xz6gd3/NhG3txQCo6xlglmTS+oTGEQ==", + "version": "3.1.10", + "resolved": "https://registry.npmjs.org/ejs/-/ejs-3.1.10.tgz", + "integrity": "sha512-UeJmFfOrAQS8OJWPZ4qtgHyWExa088/MtK5UEyoJGFH67cDEXkZSviOiKRCZ4Xij0zxI3JECgYs3oKx+AizQBA==", "requires": { "jake": "^10.8.5" } @@ -2212,14 +2212,14 @@ } }, "jake": { - "version": "10.8.5", - "resolved": "https://registry.npmjs.org/jake/-/jake-10.8.5.tgz", - "integrity": "sha512-sVpxYeuAhWt0OTWITwT98oyV0GsXyMlXCF+3L1SuafBVUIr/uILGRB+NqwkzhgXKvoJpDIpQvqkUALgdmQsQxw==", + "version": "10.8.7", + "resolved": "https://registry.npmjs.org/jake/-/jake-10.8.7.tgz", + "integrity": "sha512-ZDi3aP+fG/LchyBzUM804VjddnwfSfsdeYkwt8NcbKRvo4rFkjhs456iLFn3k2ZUWvNe4i48WACDbza8fhq2+w==", "requires": { "async": "^3.2.3", "chalk": "^4.0.2", - "filelist": "^1.0.1", - "minimatch": "^3.0.4" + "filelist": "^1.0.4", + "minimatch": "^3.1.2" } }, "jimp": { diff --git a/DocService/package.json b/DocService/package.json index 2ca09221d..4728eaded 100644 --- a/DocService/package.json +++ b/DocService/package.json @@ -20,7 +20,7 @@ "cron": "1.5.0", "deep-equal": "1.0.1", "dmdb": "1.0.14280", - "ejs": "3.1.8", + "ejs": "3.1.10", "exif-parser": "0.1.12", "express": "4.19.2", "fakeredis": "2.0.0", From cb18e6a31ed46874fb121092ca2693bb378c3ee8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 23 Apr 2024 22:18:55 +0000 Subject: [PATCH 33/48] Bump mysql2 from 2.3.3 to 3.9.7 in /DocService Bumps [mysql2](https://github.com/sidorares/node-mysql2) from 2.3.3 to 3.9.7. - [Release notes](https://github.com/sidorares/node-mysql2/releases) - [Changelog](https://github.com/sidorares/node-mysql2/blob/master/Changelog.md) - [Commits](https://github.com/sidorares/node-mysql2/compare/v2.3.3...v3.9.7) --- updated-dependencies: - dependency-name: mysql2 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- DocService/npm-shrinkwrap.json | 71 +++++++++++++++------------------- DocService/package.json | 2 +- 2 files changed, 32 insertions(+), 41 deletions(-) diff --git a/DocService/npm-shrinkwrap.json b/DocService/npm-shrinkwrap.json index 2f04968ac..c444b307b 100644 --- a/DocService/npm-shrinkwrap.json +++ b/DocService/npm-shrinkwrap.json @@ -1243,9 +1243,9 @@ "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==" }, "denque": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/denque/-/denque-2.0.1.tgz", - "integrity": "sha512-tfiWc6BQLXNLpNiR5iGd0Ocu3P3VpxfzFiqubLgMfhfOw9WyvgJBd46CClNn9k3qfbjvT//0cf7AlYRX/OslMQ==" + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz", + "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==" }, "depd": { "version": "1.1.2", @@ -2138,7 +2138,7 @@ "is-property": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/is-property/-/is-property-1.0.2.tgz", - "integrity": "sha1-V/4cTkhHTt1lsJkR8msc1Ald2oQ=" + "integrity": "sha512-Ks/IoX00TtClbGQr4TWXemAnktAQvYB7HzcCxDGqEZU6oCmb2INHuOoKxbtR+HFkmYWBKv/dOZtGRiAjDhj92g==" }, "is-regex": { "version": "1.1.4", @@ -2368,9 +2368,9 @@ "integrity": "sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==" }, "long": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/long/-/long-4.0.0.tgz", - "integrity": "sha512-XsP+KhQif4bjX1kbuSiySJFNAehNxgLb6hPRGJ9QsUr8ajHkuXGdrHmFUTUUXhDwVX2R5bY4JNZEwbUiMhV+MA==" + "version": "5.2.3", + "resolved": "https://registry.npmjs.org/long/-/long-5.2.3.tgz", + "integrity": "sha512-lcHwpNoggQTObv5apGNCTdJrO69eHOZMi4BNC+rTLER8iHAqGrUVeLh/irVIM7zTw2bOXA8T6uNPeujwOLg/2Q==" }, "lru-cache": { "version": "6.0.0", @@ -2552,16 +2552,16 @@ } }, "mysql2": { - "version": "2.3.3", - "resolved": "https://registry.npmjs.org/mysql2/-/mysql2-2.3.3.tgz", - "integrity": "sha512-wxJUev6LgMSgACDkb/InIFxDprRa6T95+VEoR+xPvtngtccNH2dGjEB/fVZ8yg1gWv1510c9CvXuJHi5zUm0ZA==", + "version": "3.9.7", + "resolved": "https://registry.npmjs.org/mysql2/-/mysql2-3.9.7.tgz", + "integrity": "sha512-KnJT8vYRcNAZv73uf9zpXqNbvBG7DJrs+1nACsjZP1HMJ1TgXEy8wnNilXAn/5i57JizXKtrUtwDB7HxT9DDpw==", "requires": { - "denque": "^2.0.1", + "denque": "^2.1.0", "generate-function": "^2.3.1", "iconv-lite": "^0.6.3", - "long": "^4.0.0", - "lru-cache": "^6.0.0", - "named-placeholders": "^1.1.2", + "long": "^5.2.1", + "lru-cache": "^8.0.0", + "named-placeholders": "^1.1.3", "seq-queue": "^0.0.5", "sqlstring": "^2.3.2" }, @@ -2574,34 +2574,25 @@ "safer-buffer": ">= 2.1.2 < 3.0.0" } }, - "sqlstring": { - "version": "2.3.3", - "resolved": "https://registry.npmjs.org/sqlstring/-/sqlstring-2.3.3.tgz", - "integrity": "sha512-qC9iz2FlN7DQl3+wjwn3802RTyjCx7sDvfQEXchwa6CWOx07/WVfh91gBmQ9fahw8snwGEWU3xGzOt4tFyHLxg==" + "lru-cache": { + "version": "8.0.5", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-8.0.5.tgz", + "integrity": "sha512-MhWWlVnuab1RG5/zMRRcVGXZLCXrZTgfwMikgzCegsPnG62yDQo5JnqKkrK4jO5iKqDAZGItAqN5CtKBCBWRUA==" } } }, "named-placeholders": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/named-placeholders/-/named-placeholders-1.1.2.tgz", - "integrity": "sha512-wiFWqxoLL3PGVReSZpjLVxyJ1bRqe+KKJVbr4hGs1KWfTZTQyezHFBbuKj9hsizHyGV2ne7EMjHdxEGAybD5SA==", + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/named-placeholders/-/named-placeholders-1.1.3.tgz", + "integrity": "sha512-eLoBxg6wE/rZkJPhU/xRX1WTpkFEwDJEN96oxFrTsqBdbT5ec295Q+CoHrL9IT0DipqKhmGcaZmwOt8OON5x1w==", "requires": { - "lru-cache": "^4.1.3" + "lru-cache": "^7.14.1" }, "dependencies": { "lru-cache": { - "version": "4.1.5", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-4.1.5.tgz", - "integrity": "sha512-sWZlbEP2OsHNkXrMl5GYk/jKk70MBng6UU4YI/qGDYbgf6YbP4EvmqISbXCoJiRKs+1bSpFHVgQxvJ17F2li5g==", - "requires": { - "pseudomap": "^1.0.2", - "yallist": "^2.1.2" - } - }, - "yallist": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/yallist/-/yallist-2.1.2.tgz", - "integrity": "sha1-HBH5IY8HYImkfdUS+TxmmaaoHVI=" + "version": "7.18.3", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-7.18.3.tgz", + "integrity": "sha512-jumlc0BIUrS3qJGgIkWZsyfAM7NCWiBcCDhnd+3NNM5KbBmLTgHVfWBcg6W+rLUsIpzpERPsvwUP7CckAQSOoA==" } } }, @@ -2857,11 +2848,6 @@ "ipaddr.js": "1.9.1" } }, - "pseudomap": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/pseudomap/-/pseudomap-1.0.2.tgz", - "integrity": "sha1-8FKijacOYYkX7wqKw0wa5aaChrM=" - }, "punycode": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz", @@ -3107,7 +3093,7 @@ "seq-queue": { "version": "0.0.5", "resolved": "https://registry.npmjs.org/seq-queue/-/seq-queue-0.0.5.tgz", - "integrity": "sha1-1WgS4cAXpuTnw+Ojeh2m143TyT4=" + "integrity": "sha512-hr3Wtp/GZIc/6DAGPDcV4/9WoZhjrkXsi5B/07QgX8tsdc6ilr7BFM6PM6rbdAX1kFSDYeZGLipIZZKyQP0O5Q==" }, "serve-static": { "version": "1.15.0", @@ -3236,6 +3222,11 @@ "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", "integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==" }, + "sqlstring": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/sqlstring/-/sqlstring-2.3.3.tgz", + "integrity": "sha512-qC9iz2FlN7DQl3+wjwn3802RTyjCx7sDvfQEXchwa6CWOx07/WVfh91gBmQ9fahw8snwGEWU3xGzOt4tFyHLxg==" + }, "standard-as-callback": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.1.0.tgz", diff --git a/DocService/package.json b/DocService/package.json index 4728eaded..1b62239a2 100644 --- a/DocService/package.json +++ b/DocService/package.json @@ -35,7 +35,7 @@ "multer": "1.4.3", "multi-integer-range": "4.0.7", "multiparty": "4.2.1", - "mysql2": "2.3.3", + "mysql2": "3.9.7", "oracledb": "6.3.0", "pg": "8.11.3", "redis": "4.6.11", From d8d2caa9e5a318def59c57bb0348b8be3651bb5e Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Fri, 10 May 2024 20:36:44 +0300 Subject: [PATCH 34/48] [bug] Serve static content for "storage-fs" only; Fix bug 67908 --- DocService/sources/routes/static.js | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/DocService/sources/routes/static.js b/DocService/sources/routes/static.js index a81a84312..19c100639 100644 --- a/DocService/sources/routes/static.js +++ b/DocService/sources/routes/static.js @@ -85,12 +85,15 @@ for (let i in cfgStaticContent) { router.use(i, express.static(cfgStaticContent[i]['path'], cfgStaticContent[i]['options'])); } } -initCacheRouter(cfgCacheStorage, [cfgCacheStorage.cacheFolderName]); - -let persistentRouts = [cfgForgottenFiles, cfgErrorFiles]; -persistentRouts.filter((rout) => {return rout && rout.length > 0;}); -if (persistentRouts.length > 0) { - initCacheRouter(cfgPersistentStorage, [cfgForgottenFiles, cfgErrorFiles]); +if (cfgCacheStorage.name === "storage-fs") { + initCacheRouter(cfgCacheStorage, [cfgCacheStorage.cacheFolderName]); +} +if (cfgPersistentStorage.name === "storage-fs") { + let persistentRouts = [cfgForgottenFiles, cfgErrorFiles]; + persistentRouts.filter((rout) => {return rout && rout.length > 0;}); + if (persistentRouts.length > 0) { + initCacheRouter(cfgPersistentStorage, [cfgForgottenFiles, cfgErrorFiles]); + } } module.exports = router; From adc44fa6b161f3e4d92d02a2fe1d1d73d784afb5 Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Fri, 10 May 2024 20:55:51 +0300 Subject: [PATCH 35/48] [bug] For bug 67908 --- Common/sources/storage-base.js | 35 ++++++++++++++++------------- Common/sources/storage-fs.js | 7 +++++- Common/sources/storage-s3.js | 7 +++++- DocService/sources/routes/static.js | 5 +++-- 4 files changed, 35 insertions(+), 19 deletions(-) diff --git a/Common/sources/storage-base.js b/Common/sources/storage-base.js index 79738fc11..c6665a90d 100644 --- a/Common/sources/storage-base.js +++ b/Common/sources/storage-base.js @@ -49,7 +49,7 @@ function getStoragePath(ctx, strPath, opt_specialDir) { opt_specialDir = opt_specialDir || cfgCacheStorage.cacheFolderName; return opt_specialDir + '/' + tenantManager.getTenantPathPrefix(ctx) + strPath.replace(/\\/g, '/'); } -function getStorage(ctx, opt_specialDir) { +function getStorage(opt_specialDir) { return opt_specialDir ? persistentStorage : cacheStorage; } function getStorageCfg(ctx, opt_specialDir) { @@ -63,32 +63,32 @@ function isDiffrentPersistentStorage() { } async function headObject(ctx, strPath, opt_specialDir) { - let storage = getStorage(ctx, opt_specialDir); + let storage = getStorage(opt_specialDir); let storageCfg = getStorageCfg(ctx, opt_specialDir); return await storage.headObject(storageCfg, getStoragePath(storageCfg, strPath, opt_specialDir)); } async function getObject(ctx, strPath, opt_specialDir) { - let storage = getStorage(ctx, opt_specialDir); + let storage = getStorage(opt_specialDir); let storageCfg = getStorageCfg(ctx, opt_specialDir); return await storage.getObject(storageCfg, getStoragePath(storageCfg, strPath, opt_specialDir)); } async function createReadStream(ctx, strPath, opt_specialDir) { - let storage = getStorage(ctx, opt_specialDir); + let storage = getStorage(opt_specialDir); let storageCfg = getStorageCfg(ctx, opt_specialDir); return await storage.createReadStream(storageCfg, getStoragePath(storageCfg, strPath, opt_specialDir)); } async function putObject(ctx, strPath, buffer, contentLength, opt_specialDir) { - let storage = getStorage(ctx, opt_specialDir); + let storage = getStorage(opt_specialDir); let storageCfg = getStorageCfg(ctx, opt_specialDir); return await storage.putObject(storageCfg, getStoragePath(ctx, strPath, opt_specialDir), buffer, contentLength); } async function uploadObject(ctx, strPath, filePath, opt_specialDir) { - let storage = getStorage(ctx, opt_specialDir); + let storage = getStorage(opt_specialDir); let storageCfg = getStorageCfg(ctx, opt_specialDir); return await storage.uploadObject(storageCfg, getStoragePath(ctx, strPath, opt_specialDir), filePath); } async function copyObject(ctx, sourceKey, destinationKey, opt_specialDirSrc, opt_specialDirDst) { - let storageSrc = getStorage(ctx, opt_specialDirSrc); + let storageSrc = getStorage(opt_specialDirSrc); let storagePathSrc = getStoragePath(ctx, sourceKey, opt_specialDirSrc); let storagePathDst = getStoragePath(ctx, destinationKey, opt_specialDirDst); let storageCfgSrc = getStorageCfg(ctx, opt_specialDirSrc); @@ -96,7 +96,7 @@ async function copyObject(ctx, sourceKey, destinationKey, opt_specialDirSrc, opt if (canCopyBetweenStorage(storageCfgSrc, storageCfgDst)){ return await storageSrc.copyObject(storageCfgSrc, storageCfgDst, storagePathSrc, storagePathDst); } else { - let storageDst = getStorage(ctx, opt_specialDirDst); + let storageDst = getStorage(opt_specialDirDst); //todo stream let buffer = await storageSrc.getObject(storageCfgSrc, storagePathSrc); return await storageDst.putObject(storageCfgDst, storagePathDst, buffer, buffer.length); @@ -109,7 +109,7 @@ async function copyPath(ctx, sourcePath, destinationPath, opt_specialDirSrc, opt })); } async function listObjects(ctx, strPath, opt_specialDir) { - let storage = getStorage(ctx, opt_specialDir); + let storage = getStorage(opt_specialDir); let storageCfg = getStorageCfg(ctx, opt_specialDir); let prefix = getStoragePath(ctx, "", opt_specialDir); try { @@ -123,23 +123,23 @@ async function listObjects(ctx, strPath, opt_specialDir) { } } async function deleteObject(ctx, strPath, opt_specialDir) { - let storage = getStorage(ctx, opt_specialDir); + let storage = getStorage(opt_specialDir); let storageCfg = getStorageCfg(ctx, opt_specialDir); return await storage.deleteObject(storageCfg, getStoragePath(ctx, strPath, opt_specialDir)); } async function deletePath(ctx, strPath, opt_specialDir) { - let storage = getStorage(ctx, opt_specialDir); + let storage = getStorage(opt_specialDir); let storageCfg = getStorageCfg(ctx, opt_specialDir); return await storage.deletePath(storageCfg, getStoragePath(ctx, strPath, opt_specialDir)); } async function getSignedUrl(ctx, baseUrl, strPath, urlType, optFilename, opt_creationDate, opt_specialDir) { - let storage = getStorage(ctx, opt_specialDir); + let storage = getStorage(opt_specialDir); let storageCfg = getStorageCfg(ctx, opt_specialDir); return await storage.getSignedUrl(ctx, storageCfg, baseUrl, getStoragePath(ctx, strPath, opt_specialDir), urlType, optFilename, opt_creationDate); } async function getSignedUrls(ctx, baseUrl, strPath, urlType, opt_creationDate, opt_specialDir) { let storagePathSrc = getStoragePath(ctx, strPath, opt_specialDir); - let storage = getStorage(ctx, opt_specialDir); + let storage = getStorage(opt_specialDir); let storageCfg = getStorageCfg(ctx, opt_specialDir); let list = await storage.listObjects(storageCfg, storagePathSrc, storageCfg); let urls = await Promise.all(list.map(function(curValue) { @@ -153,7 +153,7 @@ async function getSignedUrls(ctx, baseUrl, strPath, urlType, opt_creationDate, o } async function getSignedUrlsArrayByArray(ctx, baseUrl, list, urlType, opt_specialDir) { return await Promise.all(list.map(function (curValue) { - let storage = getStorage(ctx, opt_specialDir); + let storage = getStorage(opt_specialDir); let storageCfg = getStorageCfg(ctx, opt_specialDir); let storagePathSrc = getStoragePath(ctx, curValue, opt_specialDir); return storage.getSignedUrl(ctx, storageCfg, baseUrl, storagePathSrc, urlType, undefined); @@ -188,6 +188,10 @@ async function healthCheck(ctx, opt_specialDir) { ctx.logger.warn('healthCheck storage(%s) error %s', opt_specialDir, err.stack); } } +function needServeStatic(opt_specialDir) { + let storage = getStorage(opt_specialDir); + return storage.needServeStatic(); +} module.exports = { headObject, @@ -206,5 +210,6 @@ module.exports = { getSignedUrlsByArray, getRelativePath, isDiffrentPersistentStorage, - healthCheck + healthCheck, + needServeStatic }; diff --git a/Common/sources/storage-fs.js b/Common/sources/storage-fs.js index b8e404a2d..4729b2b85 100644 --- a/Common/sources/storage-fs.js +++ b/Common/sources/storage-fs.js @@ -151,6 +151,10 @@ async function getSignedUrl(ctx, storageCfg, baseUrl, strPath, urlType, optFilen return url; } +function needServeStatic() { + return true; +} + module.exports = { headObject, getObject, @@ -161,5 +165,6 @@ module.exports = { listObjects, deleteObject, deletePath, - getSignedUrl + getSignedUrl, + needServeStatic }; diff --git a/Common/sources/storage-s3.js b/Common/sources/storage-s3.js index 8fabade63..458b3b807 100644 --- a/Common/sources/storage-s3.js +++ b/Common/sources/storage-s3.js @@ -236,6 +236,10 @@ async function getSignedUrlWrapper(ctx, storageCfg, baseUrl, strPath, urlType, o // return utils.changeOnlyOfficeUrl(url, strPath, optFilename); } +function needServeStatic() { + return false; +} + module.exports = { headObject, getObject, @@ -246,5 +250,6 @@ module.exports = { listObjects, deleteObject, deletePath, - getSignedUrl: getSignedUrlWrapper + getSignedUrl: getSignedUrlWrapper, + needServeStatic }; diff --git a/DocService/sources/routes/static.js b/DocService/sources/routes/static.js index 19c100639..d1996349c 100644 --- a/DocService/sources/routes/static.js +++ b/DocService/sources/routes/static.js @@ -35,6 +35,7 @@ const express = require('express'); const config = require("config"); const operationContext = require('./../../../Common/sources/operationContext'); const utils = require('./../../../Common/sources/utils'); +const storage = require('./../../../Common/sources/storage-base'); const urlModule = require("url"); const path = require("path"); const mime = require("mime"); @@ -85,10 +86,10 @@ for (let i in cfgStaticContent) { router.use(i, express.static(cfgStaticContent[i]['path'], cfgStaticContent[i]['options'])); } } -if (cfgCacheStorage.name === "storage-fs") { +if (storage.needServeStatic()) { initCacheRouter(cfgCacheStorage, [cfgCacheStorage.cacheFolderName]); } -if (cfgPersistentStorage.name === "storage-fs") { +if (storage.needServeStatic(cfgForgottenFiles)) { let persistentRouts = [cfgForgottenFiles, cfgErrorFiles]; persistentRouts.filter((rout) => {return rout && rout.length > 0;}); if (persistentRouts.length > 0) { From 8de0139196e22abe34e60d17de5f85d874e47195 Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Mon, 13 May 2024 01:21:02 +0300 Subject: [PATCH 36/48] [bug] Refactor Locks storage; for bug 65773 --- DocService/sources/DocsCoServer.js | 266 ++++++++----------------- DocService/sources/editorDataMemory.js | 31 ++- 2 files changed, 114 insertions(+), 183 deletions(-) diff --git a/DocService/sources/DocsCoServer.js b/DocService/sources/DocsCoServer.js index dc343b1ee..577b385bb 100644 --- a/DocService/sources/DocsCoServer.js +++ b/DocService/sources/DocsCoServer.js @@ -1649,7 +1649,7 @@ exports.install = function(server, callbackFunction) { yield* onCursor(ctx, conn, data); break; case 'getLock' : - yield* getLock(ctx, conn, data, false); + yield getLock(ctx, conn, data, false); break; case 'saveChanges' : yield* saveChanges(ctx, conn, data); @@ -1834,7 +1834,7 @@ exports.install = function(server, callbackFunction) { } } //Release locks - userLocks = yield* removeUserLocks(ctx, docId, conn.user.id); + userLocks = yield removeUserLocks(ctx, docId, conn.user.id); if (0 < userLocks.length) { //todo send nothing in case of close document //sendReleaseLock(conn, userLocks); @@ -1927,31 +1927,19 @@ exports.install = function(server, callbackFunction) { return objChangesDocument; } - function* getAllLocks(ctx, docId) { - var docLockRes = []; - var docLock = yield editorData.getLocks(ctx, docId); - for (var i = 0; i < docLock.length; ++i) { - docLockRes.push(docLock[i]); - } - return docLockRes; - } - function* removeUserLocks(ctx, docId, userId) { - var userLocks = [], i; - var toCache = []; - var docLock = yield* getAllLocks(ctx, docId); - for (i = 0; i < docLock.length; ++i) { - var elem = docLock[i]; - if (elem.user === userId) { - userLocks.push(elem); - } else { - toCache.push(elem); + async function removeUserLocks(ctx, docId, userId) { + let locks = await editorData.getLocks(ctx, docId); + let res = []; + let toRemove = {}; + for (let lockId in locks) { + let lock = locks[lockId]; + if (lock.user === userId) { + toRemove[lockId] = lock; + res.push(lock); } } - //remove all - yield editorData.removeLocks(ctx, docId); - //set all - yield editorData.addLocks(ctx, docId, toCache); - return userLocks; + await editorData.removeLocks(ctx, docId, toRemove); + return res; } function* checkEndAuthLock(ctx, unlock, isSave, docId, userId, releaseLocks, deleteIndex, conn) { @@ -1987,7 +1975,7 @@ exports.install = function(server, callbackFunction) { //Release locks if (releaseLocks && conn) { - const userLocks = yield* removeUserLocks(ctx, docId, userId); + const userLocks = yield removeUserLocks(ctx, docId, userId); if (0 < userLocks.length) { sendReleaseLock(ctx, conn, userLocks); yield publish(ctx, { @@ -2070,20 +2058,20 @@ exports.install = function(server, callbackFunction) { // Recalculation only for foreign Lock when saving on a client that added/deleted rows or columns function _recalcLockArray(userId, _locks, oRecalcIndexColumns, oRecalcIndexRows) { + let res = {}; if (null == _locks) { - return false; + return res; } - var count = _locks.length; var element = null, oRangeOrObjectId = null; - var i; var sheetId = -1; - var isModify = false; - for (i = 0; i < count; ++i) { + for (let lockId in _locks) { + let isModify = false; + let lock = _locks[lockId]; // we do not count for ourselves - if (userId === _locks[i].user) { + if (userId === lock.user) { continue; } - element = _locks[i].block; + element = lock.block; if (c_oAscLockTypeElem.Range !== element["type"] || c_oAscLockTypeElemSubType.InsertColumns === element["subType"] || c_oAscLockTypeElemSubType.InsertRows === element["subType"]) { @@ -2105,8 +2093,11 @@ exports.install = function(server, callbackFunction) { oRangeOrObjectId["r2"] = oRecalcIndexRows[sheetId].getLockMe2(oRangeOrObjectId["r2"]); isModify = true; } + if (isModify) { + res[lockId] = lock; + } } - return isModify; + return res; } function _addRecalcIndex(oRecalcIndex) { @@ -2745,7 +2736,7 @@ exports.install = function(server, callbackFunction) { if (bIsSuccessRestore) { // check locks var arrayBlocks = data['block']; - var getLockRes = yield* getLock(ctx, conn, data, true); + var getLockRes = yield getLock(ctx, conn, data, true); if (arrayBlocks && (0 === arrayBlocks.length || getLockRes)) { yield* authRestore(ctx, conn, data.sessionId); } else { @@ -2967,17 +2958,13 @@ exports.install = function(server, callbackFunction) { const tenTypesUpload = ctx.getCfg('services.CoAuthoring.utils.limits_image_types_upload', cfgTypesUpload); const docId = conn.docId; - let docLock; - if(EditorTypes.document == conn.editorType){ - docLock = {}; - let elem; - const allLocks = yield* getAllLocks(ctx, docId); - for(let i = 0 ; i < allLocks.length; ++i) { - elem = allLocks[i]; - docLock[elem.block] = elem; + let docLock = yield editorData.getLocks(ctx, docId); + if (EditorTypes.document !== conn.editorType){ + let docLockList = []; + for (let lockId in docLock) { + docLockList.push(docLock[lockId]); } - } else { - docLock = yield* getAllLocks(ctx, docId); + docLock = docLockList; } let allMessages = yield editorData.getMessages(ctx, docId); allMessages = allMessages.length > 0 ? allMessages : undefined;//todo client side @@ -3036,98 +3023,53 @@ exports.install = function(server, callbackFunction) { var messages = [msg]; yield publish(ctx, {type: commonDefines.c_oPublishType.cursor, ctx: ctx, docId: docId, userId: userId, messages: messages}, docId, userId); } - - function* getLock(ctx, conn, data, bIsRestore) { - ctx.logger.info("getLock"); - var fLock = null; + // For Word block is now string "guid" + // For Excel block is now object { sheetId, type, rangeOrObjectId, guid } + // For presentations, this is an object { type, val } or { type, slideId, objId } + async function getLock(ctx, conn, data, bIsRestore) { + ctx.logger.debug("getLock"); + var fCheckLock = null; switch (conn.editorType) { case EditorTypes.document: // Word - fLock = getLockWord; + fCheckLock = _checkLockWord; break; case EditorTypes.spreadsheet: // Excel - fLock = getLockExcel; + fCheckLock = _checkLockExcel; break; case EditorTypes.presentation: // PP - fLock = getLockPresentation; + fCheckLock = _checkLockPresentation; break; + default: + return false; } - return fLock ? yield* fLock(ctx, conn, data, bIsRestore) : false; - } - - function* getLockWord(ctx, conn, data, bIsRestore) { - var docId = conn.docId, userId = conn.user.id, arrayBlocks = data.block; - var i; - var checkRes = yield* _checkLock(ctx, docId, arrayBlocks); - var documentLocks = checkRes.documentLocks; - if (checkRes.res) { - //Ok. take lock - var toCache = []; - for (i = 0; i < arrayBlocks.length; ++i) { - var block = arrayBlocks[i]; - var elem = {time: Date.now(), user: userId, block: block}; - documentLocks[block] = elem; - toCache.push(elem); - } - yield editorData.addLocks(ctx, docId, toCache); - } else if (bIsRestore) { - return false; - } - //to the one who made the request we return as quickly as possible - sendData(ctx, conn, {type: "getLock", locks: documentLocks}); - yield publish(ctx, {type: commonDefines.c_oPublishType.getLock, ctx: ctx, docId: docId, userId: userId, documentLocks: documentLocks}, docId, userId); - return true; - } - - // For Excel block is now object { sheetId, type, rangeOrObjectId, guid } - function* getLockExcel(ctx, conn, data, bIsRestore) { - var docId = conn.docId, userId = conn.user.id, arrayBlocks = data.block; - var i; - var checkRes = yield* _checkLockExcel(ctx, docId, arrayBlocks, userId); - var documentLocks = checkRes.documentLocks; - if (checkRes.res) { - //Ok. take lock - var toCache = []; - for (i = 0; i < arrayBlocks.length; ++i) { - var block = arrayBlocks[i]; - var elem = {time: Date.now(), user: userId, block: block}; - documentLocks.push(elem); - toCache.push(elem); - } - yield editorData.addLocks(ctx, docId, toCache); - } else if (bIsRestore) { - return false; - } - //to the one who made the request we return as quickly as possible - sendData(ctx, conn, {type: "getLock", locks: documentLocks}); - yield publish(ctx, {type: commonDefines.c_oPublishType.getLock, ctx: ctx, docId: docId, userId: userId, documentLocks: documentLocks}, docId, userId); - return true; - } - - // For presentations, this is an object { type, val } or { type, slideId, objId } - function* getLockPresentation(ctx, conn, data, bIsRestore) { - var docId = conn.docId, userId = conn.user.id, arrayBlocks = data.block; - var i; - var checkRes = yield* _checkLockPresentation(ctx, docId, arrayBlocks, userId); - var documentLocks = checkRes.documentLocks; - if (checkRes.res) { - //Ok. take lock - var toCache = []; - for (i = 0; i < arrayBlocks.length; ++i) { - var block = arrayBlocks[i]; - var elem = {time: Date.now(), user: userId, block: block}; - documentLocks.push(elem); - toCache.push(elem); - } - yield editorData.addLocks(ctx, docId, toCache); - } else if (bIsRestore) { - return false; + let docId = conn.docId, userId = conn.user.id, arrayBlocks = data.block; + let locks = arrayBlocks.reduce(function(map, block) { + //todo use one id + map[block.guid || block] = {time: Date.now(), user: userId, block: block}; + return map; + }, {}); + let addRes = await editorData.addLocksNX(ctx, docId, locks); + let documentLocks = addRes.allLocks; + let isAllAdded = Object.keys(addRes.lockConflict).length === 0; + if (!isAllAdded || !fCheckLock(ctx, docId, documentLocks, arrayBlocks, userId)) { + //remove new locks + let toRemove = {}; + for (let lockId in locks) { + if (!addRes.lockConflict[lockId]) { + toRemove[lockId] = locks[lockId]; + delete documentLocks[lockId]; + } + } + await editorData.removeLocks(ctx, docId, toRemove); + if (bIsRestore) { + return false; + } } - //to the one who made the request we return as quickly as possible sendData(ctx, conn, {type: "getLock", locks: documentLocks}); - yield publish(ctx, {type: commonDefines.c_oPublishType.getLock, ctx: ctx, docId: docId, userId: userId, documentLocks: documentLocks}, docId, userId); + await publish(ctx, {type: commonDefines.c_oPublishType.getLock, ctx: ctx, docId: docId, userId: userId, documentLocks: documentLocks}, docId, userId); return true; } @@ -3211,14 +3153,10 @@ exports.install = function(server, callbackFunction) { const oRecalcIndexRows = _addRecalcIndex(tmpAdditionalInfo["indexRows"]); // Now we need to recalculate indexes for lock elements if (null !== oRecalcIndexColumns || null !== oRecalcIndexRows) { - const docLock = yield* getAllLocks(ctx, docId); - if (_recalcLockArray(userId, docLock, oRecalcIndexColumns, oRecalcIndexRows)) { - let toCache = []; - for (let i = 0; i < docLock.length; ++i) { - toCache.push(docLock[i]); - } - yield editorData.removeLocks(ctx, docId); - yield editorData.addLocks(ctx, docId, toCache); + let docLock = yield editorData.getLocks(ctx, docId); + let docLockMod = _recalcLockArray(userId, docLock, oRecalcIndexColumns, oRecalcIndexRows); + if (Object.keys(docLockMod).length > 0) { + yield editorData.addLocks(ctx, docId, docLockMod); } } } @@ -3226,7 +3164,7 @@ exports.install = function(server, callbackFunction) { let userLocks = []; if (data.releaseLocks) { //Release locks - userLocks = yield* removeUserLocks(ctx, docId, userId); + userLocks = yield removeUserLocks(ctx, docId, userId); } // For this user, we remove Lock from the document if the unlock flag has arrived const checkEndAuthLockRes = yield* checkEndAuthLock(ctx, data.unlock, false, docId, userId); @@ -3327,47 +3265,19 @@ exports.install = function(server, callbackFunction) { sendDataMessage(ctx, conn, allMessages); } - function* _checkLock(ctx, docId, arrayBlocks) { - // Data is array now - var isLock = false; - var allLocks = yield* getAllLocks(ctx, docId); - var documentLocks = {}; - for(var i = 0 ; i < allLocks.length; ++i) { - var elem = allLocks[i]; - documentLocks[elem.block] =elem; - } - if (arrayBlocks.length > 0) { - for (var i = 0; i < arrayBlocks.length; ++i) { - var block = arrayBlocks[i]; - ctx.logger.info("getLock id: %s", block); - if (documentLocks.hasOwnProperty(block) && documentLocks[block] !== null) { - isLock = true; - break; - } - } - } else { - isLock = true; - } - return {res: !isLock, documentLocks: documentLocks}; + function _checkLockWord(ctx, docId, documentLocks, arrayBlocks, userId) { + return true; } - - function* _checkLockExcel(ctx, docId, arrayBlocks, userId) { + function _checkLockExcel(ctx, docId, documentLocks, arrayBlocks, userId) { // Data is array now var documentLock; var isLock = false; var isExistInArray = false; var i, blockRange; - var documentLocks = yield* getAllLocks(ctx, docId); var lengthArray = (arrayBlocks) ? arrayBlocks.length : 0; for (i = 0; i < lengthArray && false === isLock; ++i) { blockRange = arrayBlocks[i]; - for (var keyLockInArray in documentLocks) { - if (true === isLock) { - break; - } - if (!documentLocks.hasOwnProperty(keyLockInArray)) { - continue; - } + for (let keyLockInArray in documentLocks) { documentLock = documentLocks[keyLockInArray]; // Checking if an object is in an array (the current user sent a lock again) if (documentLock.user === userId && @@ -3402,41 +3312,39 @@ exports.install = function(server, callbackFunction) { continue; } isLock = compareExcelBlock(blockRange, documentLock.block); + if (true === isLock) { + break; + } } } if (0 === lengthArray) { isLock = true; } - return {res: !isLock && !isExistInArray, documentLocks: documentLocks}; + return !isLock && !isExistInArray; } - function* _checkLockPresentation(ctx, docId, arrayBlocks, userId) { + function _checkLockPresentation(ctx, docId, documentLocks, arrayBlocks, userId) { // Data is array now var isLock = false; - var i, documentLock, blockRange; - var documentLocks = yield* getAllLocks(ctx, docId); + var i, blockRange; var lengthArray = (arrayBlocks) ? arrayBlocks.length : 0; for (i = 0; i < lengthArray && false === isLock; ++i) { blockRange = arrayBlocks[i]; - for (var keyLockInArray in documentLocks) { - if (true === isLock) { - break; - } - if (!documentLocks.hasOwnProperty(keyLockInArray)) { - continue; - } - documentLock = documentLocks[keyLockInArray]; - + for (let keyLockInArray in documentLocks) { + let documentLock = documentLocks[keyLockInArray]; if (documentLock.user === userId || !(documentLock.block)) { continue; } isLock = comparePresentationBlock(blockRange, documentLock.block); + if (true === isLock) { + break; + } } } if (0 === lengthArray) { isLock = true; } - return {res: !isLock, documentLocks: documentLocks}; + return !isLock; } function _checkLicense(ctx, conn) { diff --git a/DocService/sources/editorDataMemory.js b/DocService/sources/editorDataMemory.js index b6f3ac56f..1134f0c71 100644 --- a/DocService/sources/editorDataMemory.js +++ b/DocService/sources/editorDataMemory.js @@ -140,17 +140,40 @@ EditorData.prototype.removePresenceDocument = async function(ctx, docId) {}; EditorData.prototype.addLocks = async function(ctx, docId, locks) { let data = this._getDocumentData(ctx, docId); if (!data.locks) { - data.locks = []; + data.locks = {}; } - data.locks = data.locks.concat(locks); + Object.assign(data.locks, locks); }; -EditorData.prototype.removeLocks = async function(ctx, docId) { +EditorData.prototype.addLocksNX = async function(ctx, docId, locks) { + let data = this._getDocumentData(ctx, docId); + if (!data.locks) { + data.locks = {}; + } + let lockConflict = {}; + for (let lockId in locks) { + if (undefined === data.locks[lockId]) { + data.locks[lockId] = locks[lockId]; + } else { + lockConflict[lockId] = locks[lockId]; + } + } + return {lockConflict, allLocks: data.locks}; +}; +EditorData.prototype.removeLocks = async function(ctx, docId, locks) { + let data = this._getDocumentData(ctx, docId); + if (!data.locks) { + for (let lockId in locks) { + delete data.locks[lockId]; + } + } +}; +EditorData.prototype.removeAllLocks = async function(ctx, docId) { let data = this._getDocumentData(ctx, docId); data.locks = undefined; }; EditorData.prototype.getLocks = async function(ctx, docId) { let data = this._getDocumentData(ctx, docId); - return data.locks || []; + return data.locks || {}; }; EditorData.prototype.addMessage = async function(ctx, docId, msg) { From 0d88c74927e2677cbe6ede4c25ac209a592fa238 Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Mon, 13 May 2024 02:31:01 +0300 Subject: [PATCH 37/48] [bug] For bug 65773 --- DocService/sources/editorDataMemory.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DocService/sources/editorDataMemory.js b/DocService/sources/editorDataMemory.js index 1134f0c71..cfed80559 100644 --- a/DocService/sources/editorDataMemory.js +++ b/DocService/sources/editorDataMemory.js @@ -161,7 +161,7 @@ EditorData.prototype.addLocksNX = async function(ctx, docId, locks) { }; EditorData.prototype.removeLocks = async function(ctx, docId, locks) { let data = this._getDocumentData(ctx, docId); - if (!data.locks) { + if (data.locks) { for (let lockId in locks) { delete data.locks[lockId]; } From d57a5125d64ca025a06df7c0a1e6c85d89ebb370 Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Mon, 13 May 2024 16:05:12 +0300 Subject: [PATCH 38/48] [bug] Do not serve content with empty special dir; For bug 67908 --- DocService/sources/routes/static.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/DocService/sources/routes/static.js b/DocService/sources/routes/static.js index d1996349c..6051ae2c0 100644 --- a/DocService/sources/routes/static.js +++ b/DocService/sources/routes/static.js @@ -53,6 +53,10 @@ function initCacheRouter(cfgStorage, routs) { const storageFolderName = cfgStorage.storageFolderName; const folderPath = cfgStorage.fs.folderPath; routs.forEach((rout) => { + //special dirs are empty by default + if (!rout) { + return; + } let rootPath = path.join(folderPath, rout); router.use(`/${bucketName}/${storageFolderName}/${rout}`, (req, res, next) => { const index = req.url.lastIndexOf('/'); From 3db24b1124325dabd68a731ba0dee2335edfd163 Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Wed, 15 May 2024 18:08:54 +0300 Subject: [PATCH 39/48] [bug] Fix bug 67983 --- DocService/sources/DocsCoServer.js | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/DocService/sources/DocsCoServer.js b/DocService/sources/DocsCoServer.js index 577b385bb..3609444da 100644 --- a/DocService/sources/DocsCoServer.js +++ b/DocService/sources/DocsCoServer.js @@ -3054,7 +3054,7 @@ exports.install = function(server, callbackFunction) { let addRes = await editorData.addLocksNX(ctx, docId, locks); let documentLocks = addRes.allLocks; let isAllAdded = Object.keys(addRes.lockConflict).length === 0; - if (!isAllAdded || !fCheckLock(ctx, docId, documentLocks, arrayBlocks, userId)) { + if (!isAllAdded || !fCheckLock(ctx, docId, documentLocks, locks, arrayBlocks, userId)) { //remove new locks let toRemove = {}; for (let lockId in locks) { @@ -3265,10 +3265,10 @@ exports.install = function(server, callbackFunction) { sendDataMessage(ctx, conn, allMessages); } - function _checkLockWord(ctx, docId, documentLocks, arrayBlocks, userId) { + function _checkLockWord(ctx, docId, documentLocks, newLocks, arrayBlocks, userId) { return true; } - function _checkLockExcel(ctx, docId, documentLocks, arrayBlocks, userId) { + function _checkLockExcel(ctx, docId, documentLocks, newLocks, arrayBlocks, userId) { // Data is array now var documentLock; var isLock = false; @@ -3278,6 +3278,10 @@ exports.install = function(server, callbackFunction) { for (i = 0; i < lengthArray && false === isLock; ++i) { blockRange = arrayBlocks[i]; for (let keyLockInArray in documentLocks) { + if (newLocks[keyLockInArray]) { + //skip just added + continue; + } documentLock = documentLocks[keyLockInArray]; // Checking if an object is in an array (the current user sent a lock again) if (documentLock.user === userId && @@ -3323,7 +3327,7 @@ exports.install = function(server, callbackFunction) { return !isLock && !isExistInArray; } - function _checkLockPresentation(ctx, docId, documentLocks, arrayBlocks, userId) { + function _checkLockPresentation(ctx, docId, documentLocks, newLocks, arrayBlocks, userId) { // Data is array now var isLock = false; var i, blockRange; @@ -3331,6 +3335,10 @@ exports.install = function(server, callbackFunction) { for (i = 0; i < lengthArray && false === isLock; ++i) { blockRange = arrayBlocks[i]; for (let keyLockInArray in documentLocks) { + if (newLocks[keyLockInArray]) { + //skip just added + continue; + } let documentLock = documentLocks[keyLockInArray]; if (documentLock.user === userId || !(documentLock.block)) { continue; From 27e5d646d348940b78b16c19cbd9dba7249a2bb9 Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Sat, 25 May 2024 21:32:50 +0300 Subject: [PATCH 40/48] [bug] Change privateKey format; Make generateProofSign async; For bug 66601 --- Common/config/default.json | 12 ++++---- DocService/sources/canvasservice.js | 2 +- DocService/sources/wopiClient.js | 47 ++++++++++++----------------- FileConverter/sources/converter.js | 2 +- 4 files changed, 28 insertions(+), 35 deletions(-) diff --git a/Common/config/default.json b/Common/config/default.json index 6c867a733..12f9f3d05 100644 --- a/Common/config/default.json +++ b/Common/config/default.json @@ -97,14 +97,14 @@ "cellEdit": ["xlsx", "xlsm", "ods", "csv"], "slideView": ["ppt", "ppsx", "ppsm", "pps", "potx", "potm", "pot", "fodp", "otp", "sxi", "dps", "dpt"], "slideEdit": ["pptx", "pptm", "odp"], - "publicKey": "BgIAAACkAABSU0ExAAgAAAEAAQD/NVqekFNi8X3p6Bvdlaxm0GGuggW5kKfVEQzPGuOkGVrz6DrOMNR+k7Pq8tONY+1NHgS6Z+v3959em78qclVDuQX77Tkml0xMHAQHN4sAHF9iQJS8gOBUKSVKaHD7Z8YXch6F212YSUSc8QphpDSHWVShU7rcUeLQsd/0pkflh5+um4YKEZhm4Mou3vstp5p12NeffyK1WFZF7q4jB7jclAslYKQsP82YY3DcRwu5Tl/+W0ifVcXze0mI7v1reJ12pKn8ifRiq+0q5oJST3TRSrvmjLg9Gt3ozhVIt2HUi3La7Qh40YOAUXm0g/hUq2BepeOp1C7WSvaOFHXe6Hqq", - "modulus": "qnro3nUUjvZK1i7UqeOlXmCrVPiDtHlRgIPReAjt2nKL1GG3SBXO6N0aPbiM5rtK0XRPUoLmKu2rYvSJ/Kmkdp14a/3uiEl788VVn0hb/l9OuQtH3HBjmM0/LKRgJQuU3LgHI67uRVZYtSJ/n9fYdZqnLfveLsrgZpgRCoabrp+H5Uem9N+x0OJR3LpToVRZhzSkYQrxnERJmF3bhR5yF8Zn+3BoSiUpVOCAvJRAYl8cAIs3BwQcTEyXJjnt+wW5Q1VyKr+bXp/39+tnugQeTe1jjdPy6rOTftQwzjro81oZpOMazwwR1aeQuQWCrmHQZqyV3Rvo6X3xYlOQnlo1/w==", + "publicKey": "BgIAAACkAABSU0ExAAgAAAEAAQBpTpiJQ2hD8plpGTfEEmcq4IKyr31HikXpuVSBraMfqyodn2PGXBJ3daNSmdPOc0Nz4HO9Auljn8YYXDPBdpiABptSKvEDPF23Q+Qytg0+vCRyondyBcW91w7KLzXce3fnk8ZfJ8QtbZPL9m11wJIWZueQF+l0HKYx4lty+nccbCanytFTADkGQ3SnmExGEF3rBz6I9+OcrDDK9NKPJgEmCiuyei/d4XbPgKls3EIG0h38X5mVF2VytfWm2Yu850B6z3N4MYhj4b4vsYT62zEC4pMRUeb8dIBy4Jsmr3avtmeO00MUH6DVyPC8nirixj2YIOPKk13CdVqGDSXA3cvl", + "modulus": "E5CBDDC0250D865A75C25D93CAE320983DC6E22A9EBCF0C8D5A01F1443D38E67B6AF76AF269BE0728074FCE6511193E20231DBFA84B12FBEE16388317873CF7A40E7BC8BD9A6F5B572651795995FFC1DD20642DC6CA980CF76E1DD2F7AB22B0A2601268FD2F4CA30AC9CE3F7883E07EB5D10464C98A7744306390053D1CAA7266C1C77FA725BE231A61C74E91790E7661692C0756DF6CB936D2DC4275FC693E7777BDC352FCA0ED7BDC5057277A27224BC3E0DB632E443B75D3C03F12A529B06809876C1335C18C69F63E902BD73E0734373CED39952A37577125CC6639F1D2AAB1FA3AD8154B9E9458A477DAFB282E02A6712C437196999F243684389984E69", "exponent": "AQAB", - "privateKey": "MIIEowIBAAKCAQEAqnro3nUUjvZK1i7UqeOlXmCrVPiDtHlRgIPReAjt2nKL1GG3SBXO6N0aPbiM5rtK0XRPUoLmKu2rYvSJ/Kmkdp14a/3uiEl788VVn0hb/l9OuQtH3HBjmM0/LKRgJQuU3LgHI67uRVZYtSJ/n9fYdZqnLfveLsrgZpgRCoabrp+H5Uem9N+x0OJR3LpToVRZhzSkYQrxnERJmF3bhR5yF8Zn+3BoSiUpVOCAvJRAYl8cAIs3BwQcTEyXJjnt+wW5Q1VyKr+bXp/39+tnugQeTe1jjdPy6rOTftQwzjro81oZpOMazwwR1aeQuQWCrmHQZqyV3Rvo6X3xYlOQnlo1/wIDAQABAoIBAQCKtUSBs8tNYrGTQTlBHXrwpkDg+u7WSZt5sEcfnkxA39BLtlHU8gGO0E9Ihr8GAL+oWjUsEltJ9GTtN8CJ9lFdPVS8sTiCZR/YQOggmFRZTJyVzMrkXgF7Uwwiu3+KxLiTOZx9eRhfDBlTD8W9fXaegX2i2Xp2ohUhBHthEBLdaZTWFi5Sid/Y0dDzBeP6UIJorZ5D+1ybaeIVHjndpwNsIEUGUxPFLrkeiU8Rm4MJ9ahxfywcP7DjQoPGY9Ge5cBhpxfzERWf732wUD6o3+L9tvOBU00CLVjULbGZKTVE2FJMyXK9jr6Zor9Mkhomp6/8Agkr9rp+TPyelFGYEz8hAoGBAOEc09CrL3eYBkhNEcaMQzxBLvOGpg8kaDX5SaArHfl9+U9yzRqss4ARECanp9HuHfjMQo7iejao0ngDjL7BNMSaH74QlSsPOY2iOm8Qvx8/zb7g4h9r1zLjFZb3mpSA4snRZvvdiZ9ugbuVPmhXnDzRRMg45MibJeeOTJNylofRAoGBAMHfF/WutqKDoX25qZo9m74W4bttOj6oIDk1N4/c6M1Z1v/aptYSE06bkWngj9P46kqjaay4hgMtzyGruc5aojPx5MHHf5bo14+Jv4NzYtR2llrUxO+UJX7aCfUYXI7RC93GUmhpeQ414j7SNAXec58d7e+ETw+6cHiAWO4uOSTPAoGATPq5qDLR4Zi4FUNdn8LZPyKfNqHF6YmupT5hIgd8kZO1jKiaYNPL8jBjkIRmjBBcaXcYD5p85nImvumf2J9jNxPpZOpwyC/Fo5xlVROp97qu1eY7DTmodntXJ6/2SXAlnZQhHmHsrPtyG752f+HtyJJbbgiem8cKWDu+DfHybfECgYBbSLo1WiBwgN4nHqZ3E48jgA6le5azLeKOTTpuKKwNFMIhEkj//t7MYn+jhLL0Mf3PSwZU50Vidc1To1IHkbFSGBGIFHFFEzl8QnXEZS4hr/y3o/teezj0c6HAn8nlDRUzRVBEDXWMdV6kCcGpCccTIrqHzpqTY0vV0UkOTQFnDQKBgAxSEhm/gtCYJIMCBe+KBJT9uECV5xDQopTTjsGOkd4306EN2dyPOIlAfwM6K/0qWisa0Ei5i8TbRRuBeTTdLEYLqXCJ7fj5tdD1begBdSVtHQ2WHqzPJSuImTkFi9NXxd1XUyZFM3y6YQvlssSuL7QSxUIEtZHnrJTt3QDd10dj", - "publicKeyOld": "BgIAAACkAABSU0ExAAgAAAEAAQD/NVqekFNi8X3p6Bvdlaxm0GGuggW5kKfVEQzPGuOkGVrz6DrOMNR+k7Pq8tONY+1NHgS6Z+v3959em78qclVDuQX77Tkml0xMHAQHN4sAHF9iQJS8gOBUKSVKaHD7Z8YXch6F212YSUSc8QphpDSHWVShU7rcUeLQsd/0pkflh5+um4YKEZhm4Mou3vstp5p12NeffyK1WFZF7q4jB7jclAslYKQsP82YY3DcRwu5Tl/+W0ifVcXze0mI7v1reJ12pKn8ifRiq+0q5oJST3TRSrvmjLg9Gt3ozhVIt2HUi3La7Qh40YOAUXm0g/hUq2BepeOp1C7WSvaOFHXe6Hqq", - "modulusOld": "qnro3nUUjvZK1i7UqeOlXmCrVPiDtHlRgIPReAjt2nKL1GG3SBXO6N0aPbiM5rtK0XRPUoLmKu2rYvSJ/Kmkdp14a/3uiEl788VVn0hb/l9OuQtH3HBjmM0/LKRgJQuU3LgHI67uRVZYtSJ/n9fYdZqnLfveLsrgZpgRCoabrp+H5Uem9N+x0OJR3LpToVRZhzSkYQrxnERJmF3bhR5yF8Zn+3BoSiUpVOCAvJRAYl8cAIs3BwQcTEyXJjnt+wW5Q1VyKr+bXp/39+tnugQeTe1jjdPy6rOTftQwzjro81oZpOMazwwR1aeQuQWCrmHQZqyV3Rvo6X3xYlOQnlo1/w==", + "privateKey": "LS0tLS1CRUdJTiBQUklWQVRFIEtFWS0tLS0tDQpNSUlFdndJQkFEQU5CZ2txaGtpRzl3MEJBUUVGQUFTQ0JLa3dnZ1NsQWdFQUFvSUJBUURseTkzQUpRMkdXblhDDQpYWlBLNHlDWVBjYmlLcDY4OE1qVm9COFVROU9PWjdhdmRxOG1tK0J5Z0hUODVsRVJrK0lDTWR2NmhMRXZ2dUZqDQppREY0Yzg5NlFPZThpOW1tOWJWeVpSZVZtVi84SGRJR1F0eHNxWURQZHVIZEwzcXlLd29tQVNhUDB2VEtNS3ljDQo0L2VJUGdmclhSQkdUSmluZEVNR09RQlQwY3FuSm13Y2QvcHlXK0l4cGh4MDZSZVE1MllXa3NCMWJmYkxrMjB0DQp4Q2RmeHBQbmQzdmNOUy9LRHRlOXhRVnlkNkp5Skx3K0RiWXk1RU8zWFR3RDhTcFNtd2FBbUhiQk0xd1l4cDlqDQo2UUs5YytCelEzUE8wNWxTbzNWM0VsekdZNThkS3FzZm82MkJWTG5wUllwSGZhK3lndUFxWnhMRU54bHBtZkpEDQphRU9KbUU1cEFnTUJBQUVDZ2dFQUxpTCtSS09yMFh1OEJPZ1EwajFEd0EwM0x4VnJoWGU2ZXRtSkkrSnlTVGNkDQpnS0VOald6aVpWclJJaTJEdlVtNXFNTWw3V2hTd3NsS0sxZWV4eFpKWTd4QVNxU3hjRW9Jd2d6MTdUMDcvanhtDQpmSWRVQmlVS0RaMUt2OFBXbUlyM29LVytma1hXaS9tMXpsSWUwcVhScFRtc0dORXNIUUxFcWkwcm1haVhUWE9SDQovMkxkd2k2a1pSM3NXRng5N1lTNE14L3B1ZUdKVFhFYWk2QVZFWnpONUdvZzZ4RDhIWFIxUnZxK2hoZCtNb2NHDQpmblU0SGdpbEtSZm9KbFdkOUZPc2NnU3VmS0cwTDNWaU80ZlNLVTQ2bDVhdWxsRFlVazVFQ01XaXd1S1NxU0U3DQpxRDQ1akkzbWJPcmU3UzR1M1MzVFdkRDNsendpWEw0OUxkd0tsRUM0bVFLQmdRRDBzTHIwR0g0V3IrUVgyeEpFDQp1QS9DYjhRVzQxbDhpU0NCVFJaWlIvc0pPZCtvM3JiY1ZpZGx6Ty9FYlpibFhHNFpQRG1SamdCQ0dLSVA1RVppDQowRHNMK1d2MzJXT280NExweEpHaHFFeGJtMEgxaVoxelo5N2wwUDhmdkloSEU0MmdtYUxUb09JR0RoUFNYR3Z2DQp6bHFPSGJHYnE0anNFUmMxanAxYmVqNXE2d0tCZ1FEd2F1ZUljNHBSY2hIOThRWWlkY3lyOFZ3ZzlLaGJuZllYDQp5M1c0UlBsWnRCZEYzNGlKYWlvK0FTenVnby96eTFSVGNWcnNDc2tZV1h5S0RVUXoxeXUwaUNuZytmRENVblRtDQpYR21Fb0VHTmhrNHZUSk90N2hCYXYxL0phL2RVaXBHZjZtWFV1YW53SjBlKzEvRXQvQjBhaDVYMVVtNUF5TlpJDQpNK1N5UnozdSt3S0JnUUNqdnRVTlhvcWFnaENCQ21CNlRqWjFwcmV4bldrWUZ1Z0N2MlNTVU1JazFXN2dJbEo2DQp0c2pjcmoxUjFRaWk2cXpmQkZkK0dXb0EwVjA2aDBlMi9xUlZDZy8vcDZHeXRyVzMzSXljZ3ZTK1pQTEo3dExJDQpGUjJyNjZXZlJscG9QaVNMOGVSdC9QN2trRzBoWENuN0s3dWIyVEV1L0thL1cxeU53YWQ2UFI4aUN3S0JnUUM4DQpYY1pTcnRRc3hBYzh3OTllbUpWb0VvOXdjc0NHSjlsdEEwaVV1OVh5WnB2bGJ5SjNKK3M0OFlyV3hRMHNvcDdMDQpVZ0UrOTZSZm81MWtQTWkzSlZ0azgxcDhudGY0S01yV3dva2FGTVhIc1BjSk1DSjFJQlZJUkxFMEM1ZVpjWWh2DQpseU41N0k0dFQxbHpPWllKeFlLNENvdC96cm43b0YvajZtVEJHZmg0aVFLQmdRQ2lKTVV4UnowMS9jekgvWFNYDQpnbzNkVmJIUTRGRU91ZlduRTNFYjkzUzhyMC9lcTFSTTExOHJiMFRxenVpYWRXMnhZRFU0bnVjV1FscmxtcTBkDQpGWS9tK0h5OTdwcXlrNmptb1U1SS9EK3NzQklvWUhXTG5IOS94ZnZERWsySkdTSlNIdHp1MEQ0RURDL3JnUTQ5DQpNYllzTzVvVXJGOHRQbGhqNXZ6YmYzR0tMQT09DQotLS0tLUVORCBQUklWQVRFIEtFWS0tLS0tDQo=", + "publicKeyOld": "BgIAAACkAABSU0ExAAgAAAEAAQBpTpiJQ2hD8plpGTfEEmcq4IKyr31HikXpuVSBraMfqyodn2PGXBJ3daNSmdPOc0Nz4HO9Auljn8YYXDPBdpiABptSKvEDPF23Q+Qytg0+vCRyondyBcW91w7KLzXce3fnk8ZfJ8QtbZPL9m11wJIWZueQF+l0HKYx4lty+nccbCanytFTADkGQ3SnmExGEF3rBz6I9+OcrDDK9NKPJgEmCiuyei/d4XbPgKls3EIG0h38X5mVF2VytfWm2Yu850B6z3N4MYhj4b4vsYT62zEC4pMRUeb8dIBy4Jsmr3avtmeO00MUH6DVyPC8nirixj2YIOPKk13CdVqGDSXA3cvl", + "modulusOld": "E5CBDDC0250D865A75C25D93CAE320983DC6E22A9EBCF0C8D5A01F1443D38E67B6AF76AF269BE0728074FCE6511193E20231DBFA84B12FBEE16388317873CF7A40E7BC8BD9A6F5B572651795995FFC1DD20642DC6CA980CF76E1DD2F7AB22B0A2601268FD2F4CA30AC9CE3F7883E07EB5D10464C98A7744306390053D1CAA7266C1C77FA725BE231A61C74E91790E7661692C0756DF6CB936D2DC4275FC693E7777BDC352FCA0ED7BDC5057277A27224BC3E0DB632E443B75D3C03F12A529B06809876C1335C18C69F63E902BD73E0734373CED39952A37577125CC6639F1D2AAB1FA3AD8154B9E9458A477DAFB282E02A6712C437196999F243684389984E69", "exponentOld": "AQAB", - "privateKeyOld": "MIIEowIBAAKCAQEAqnro3nUUjvZK1i7UqeOlXmCrVPiDtHlRgIPReAjt2nKL1GG3SBXO6N0aPbiM5rtK0XRPUoLmKu2rYvSJ/Kmkdp14a/3uiEl788VVn0hb/l9OuQtH3HBjmM0/LKRgJQuU3LgHI67uRVZYtSJ/n9fYdZqnLfveLsrgZpgRCoabrp+H5Uem9N+x0OJR3LpToVRZhzSkYQrxnERJmF3bhR5yF8Zn+3BoSiUpVOCAvJRAYl8cAIs3BwQcTEyXJjnt+wW5Q1VyKr+bXp/39+tnugQeTe1jjdPy6rOTftQwzjro81oZpOMazwwR1aeQuQWCrmHQZqyV3Rvo6X3xYlOQnlo1/wIDAQABAoIBAQCKtUSBs8tNYrGTQTlBHXrwpkDg+u7WSZt5sEcfnkxA39BLtlHU8gGO0E9Ihr8GAL+oWjUsEltJ9GTtN8CJ9lFdPVS8sTiCZR/YQOggmFRZTJyVzMrkXgF7Uwwiu3+KxLiTOZx9eRhfDBlTD8W9fXaegX2i2Xp2ohUhBHthEBLdaZTWFi5Sid/Y0dDzBeP6UIJorZ5D+1ybaeIVHjndpwNsIEUGUxPFLrkeiU8Rm4MJ9ahxfywcP7DjQoPGY9Ge5cBhpxfzERWf732wUD6o3+L9tvOBU00CLVjULbGZKTVE2FJMyXK9jr6Zor9Mkhomp6/8Agkr9rp+TPyelFGYEz8hAoGBAOEc09CrL3eYBkhNEcaMQzxBLvOGpg8kaDX5SaArHfl9+U9yzRqss4ARECanp9HuHfjMQo7iejao0ngDjL7BNMSaH74QlSsPOY2iOm8Qvx8/zb7g4h9r1zLjFZb3mpSA4snRZvvdiZ9ugbuVPmhXnDzRRMg45MibJeeOTJNylofRAoGBAMHfF/WutqKDoX25qZo9m74W4bttOj6oIDk1N4/c6M1Z1v/aptYSE06bkWngj9P46kqjaay4hgMtzyGruc5aojPx5MHHf5bo14+Jv4NzYtR2llrUxO+UJX7aCfUYXI7RC93GUmhpeQ414j7SNAXec58d7e+ETw+6cHiAWO4uOSTPAoGATPq5qDLR4Zi4FUNdn8LZPyKfNqHF6YmupT5hIgd8kZO1jKiaYNPL8jBjkIRmjBBcaXcYD5p85nImvumf2J9jNxPpZOpwyC/Fo5xlVROp97qu1eY7DTmodntXJ6/2SXAlnZQhHmHsrPtyG752f+HtyJJbbgiem8cKWDu+DfHybfECgYBbSLo1WiBwgN4nHqZ3E48jgA6le5azLeKOTTpuKKwNFMIhEkj//t7MYn+jhLL0Mf3PSwZU50Vidc1To1IHkbFSGBGIFHFFEzl8QnXEZS4hr/y3o/teezj0c6HAn8nlDRUzRVBEDXWMdV6kCcGpCccTIrqHzpqTY0vV0UkOTQFnDQKBgAxSEhm/gtCYJIMCBe+KBJT9uECV5xDQopTTjsGOkd4306EN2dyPOIlAfwM6K/0qWisa0Ei5i8TbRRuBeTTdLEYLqXCJ7fj5tdD1begBdSVtHQ2WHqzPJSuImTkFi9NXxd1XUyZFM3y6YQvlssSuL7QSxUIEtZHnrJTt3QDd10dj", + "privateKeyOld": "LS0tLS1CRUdJTiBQUklWQVRFIEtFWS0tLS0tDQpNSUlFdndJQkFEQU5CZ2txaGtpRzl3MEJBUUVGQUFTQ0JLa3dnZ1NsQWdFQUFvSUJBUURseTkzQUpRMkdXblhDDQpYWlBLNHlDWVBjYmlLcDY4OE1qVm9COFVROU9PWjdhdmRxOG1tK0J5Z0hUODVsRVJrK0lDTWR2NmhMRXZ2dUZqDQppREY0Yzg5NlFPZThpOW1tOWJWeVpSZVZtVi84SGRJR1F0eHNxWURQZHVIZEwzcXlLd29tQVNhUDB2VEtNS3ljDQo0L2VJUGdmclhSQkdUSmluZEVNR09RQlQwY3FuSm13Y2QvcHlXK0l4cGh4MDZSZVE1MllXa3NCMWJmYkxrMjB0DQp4Q2RmeHBQbmQzdmNOUy9LRHRlOXhRVnlkNkp5Skx3K0RiWXk1RU8zWFR3RDhTcFNtd2FBbUhiQk0xd1l4cDlqDQo2UUs5YytCelEzUE8wNWxTbzNWM0VsekdZNThkS3FzZm82MkJWTG5wUllwSGZhK3lndUFxWnhMRU54bHBtZkpEDQphRU9KbUU1cEFnTUJBQUVDZ2dFQUxpTCtSS09yMFh1OEJPZ1EwajFEd0EwM0x4VnJoWGU2ZXRtSkkrSnlTVGNkDQpnS0VOald6aVpWclJJaTJEdlVtNXFNTWw3V2hTd3NsS0sxZWV4eFpKWTd4QVNxU3hjRW9Jd2d6MTdUMDcvanhtDQpmSWRVQmlVS0RaMUt2OFBXbUlyM29LVytma1hXaS9tMXpsSWUwcVhScFRtc0dORXNIUUxFcWkwcm1haVhUWE9SDQovMkxkd2k2a1pSM3NXRng5N1lTNE14L3B1ZUdKVFhFYWk2QVZFWnpONUdvZzZ4RDhIWFIxUnZxK2hoZCtNb2NHDQpmblU0SGdpbEtSZm9KbFdkOUZPc2NnU3VmS0cwTDNWaU80ZlNLVTQ2bDVhdWxsRFlVazVFQ01XaXd1S1NxU0U3DQpxRDQ1akkzbWJPcmU3UzR1M1MzVFdkRDNsendpWEw0OUxkd0tsRUM0bVFLQmdRRDBzTHIwR0g0V3IrUVgyeEpFDQp1QS9DYjhRVzQxbDhpU0NCVFJaWlIvc0pPZCtvM3JiY1ZpZGx6Ty9FYlpibFhHNFpQRG1SamdCQ0dLSVA1RVppDQowRHNMK1d2MzJXT280NExweEpHaHFFeGJtMEgxaVoxelo5N2wwUDhmdkloSEU0MmdtYUxUb09JR0RoUFNYR3Z2DQp6bHFPSGJHYnE0anNFUmMxanAxYmVqNXE2d0tCZ1FEd2F1ZUljNHBSY2hIOThRWWlkY3lyOFZ3ZzlLaGJuZllYDQp5M1c0UlBsWnRCZEYzNGlKYWlvK0FTenVnby96eTFSVGNWcnNDc2tZV1h5S0RVUXoxeXUwaUNuZytmRENVblRtDQpYR21Fb0VHTmhrNHZUSk90N2hCYXYxL0phL2RVaXBHZjZtWFV1YW53SjBlKzEvRXQvQjBhaDVYMVVtNUF5TlpJDQpNK1N5UnozdSt3S0JnUUNqdnRVTlhvcWFnaENCQ21CNlRqWjFwcmV4bldrWUZ1Z0N2MlNTVU1JazFXN2dJbEo2DQp0c2pjcmoxUjFRaWk2cXpmQkZkK0dXb0EwVjA2aDBlMi9xUlZDZy8vcDZHeXRyVzMzSXljZ3ZTK1pQTEo3dExJDQpGUjJyNjZXZlJscG9QaVNMOGVSdC9QN2trRzBoWENuN0s3dWIyVEV1L0thL1cxeU53YWQ2UFI4aUN3S0JnUUM4DQpYY1pTcnRRc3hBYzh3OTllbUpWb0VvOXdjc0NHSjlsdEEwaVV1OVh5WnB2bGJ5SjNKK3M0OFlyV3hRMHNvcDdMDQpVZ0UrOTZSZm81MWtQTWkzSlZ0azgxcDhudGY0S01yV3dva2FGTVhIc1BjSk1DSjFJQlZJUkxFMEM1ZVpjWWh2DQpseU41N0k0dFQxbHpPWllKeFlLNENvdC96cm43b0YvajZtVEJHZmg0aVFLQmdRQ2lKTVV4UnowMS9jekgvWFNYDQpnbzNkVmJIUTRGRU91ZlduRTNFYjkzUzhyMC9lcTFSTTExOHJiMFRxenVpYWRXMnhZRFU0bnVjV1FscmxtcTBkDQpGWS9tK0h5OTdwcXlrNmptb1U1SS9EK3NzQklvWUhXTG5IOS94ZnZERWsySkdTSlNIdHp1MEQ0RURDL3JnUTQ5DQpNYllzTzVvVXJGOHRQbGhqNXZ6YmYzR0tMQT09DQotLS0tLUVORCBQUklWQVRFIEtFWS0tLS0tDQo=", "refreshLockInterval": "10m", "dummy" : { "enable": false, diff --git a/DocService/sources/canvasservice.js b/DocService/sources/canvasservice.js index c34b8b17e..6cbfcdb27 100644 --- a/DocService/sources/canvasservice.js +++ b/DocService/sources/canvasservice.js @@ -1646,7 +1646,7 @@ exports.downloadFile = function(req, res) { //editnew case fromTemplate = pathModule.extname(decoded.fileInfo.BaseFileName).substring(1); } else { - ({url, headers} = wopiClient.getWopiFileUrl(ctx, decoded.fileInfo, decoded.userAuth)); + ({url, headers} = yield wopiClient.getWopiFileUrl(ctx, decoded.fileInfo, decoded.userAuth)); let filterStatus = yield wopiClient.checkIpFilter(ctx, url); if (0 === filterStatus) { //todo false? (true because it passed checkIpFilter for wopi) diff --git a/DocService/sources/wopiClient.js b/DocService/sources/wopiClient.js index 3272e410a..818c1f7fe 100644 --- a/DocService/sources/wopiClient.js +++ b/DocService/sources/wopiClient.js @@ -35,6 +35,7 @@ const path = require('path'); const { pipeline } = require('node:stream/promises'); const crypto = require('crypto'); +let util = require('util'); const {URL} = require('url'); const co = require('co'); const jwt = require('jsonwebtoken'); @@ -90,6 +91,8 @@ const cfgWopiPrivateKeyOld = config.get('wopi.privateKeyOld'); const cfgWopiHost = config.get('wopi.host'); const cfgWopiDummySampleFilePath = config.get('wopi.dummy.sampleFilePath'); +let cryptoSign = util.promisify(crypto.sign); + let templatesFolderLocalesCache = null; let templatesFolderExtsCache = null; const templateFilesSizeCache = {}; @@ -334,7 +337,7 @@ function getFileTypeByInfo(fileInfo) { fileType = fileInfo.FileExtension ? fileInfo.FileExtension.substr(1) : fileType; return fileType.toLowerCase(); } -function getWopiFileUrl(ctx, fileInfo, userAuth) { +async function getWopiFileUrl(ctx, fileInfo, userAuth) { const tenMaxDownloadBytes = ctx.getCfg('FileConverter.converter.maxDownloadBytes', cfgMaxDownloadBytes); let url; let headers = {'X-WOPI-MaxExpectedSize': tenMaxDownloadBytes}; @@ -345,7 +348,7 @@ function getWopiFileUrl(ctx, fileInfo, userAuth) { url = fileInfo.TemplateSource; } else if (userAuth) { url = `${userAuth.wopiSrc}/contents?access_token=${userAuth.access_token}`; - fillStandardHeaders(ctx, headers, url, userAuth.access_token); + await fillStandardHeaders(ctx, headers, url, userAuth.access_token); } ctx.logger.debug('getWopiFileUrl url=%s; headers=%j', url, headers); return {url, headers}; @@ -660,7 +663,7 @@ function putFile(ctx, wopiParams, data, dataStream, dataSize, userLastChangeId, let commonInfo = wopiParams.commonInfo; //todo add all the users who contributed changes to the document in this PutFile request to X-WOPI-Editors let headers = {'X-WOPI-Override': 'PUT', 'X-WOPI-Lock': commonInfo.lockId, 'X-WOPI-Editors': userLastChangeId}; - fillStandardHeaders(ctx, headers, uri, userAuth.access_token); + yield fillStandardHeaders(ctx, headers, uri, userAuth.access_token); headers['X-LOOL-WOPI-IsModifiedByUser'] = isModifiedByUser; headers['X-LOOL-WOPI-IsAutosave'] = isAutosave; headers['X-LOOL-WOPI-IsExitSave'] = isExitSave; @@ -702,7 +705,7 @@ function putRelativeFile(ctx, wopiSrc, access_token, data, dataStream, dataSize, if (isFileConversion) { headers['X-WOPI-FileConversion'] = isFileConversion; } - fillStandardHeaders(ctx, headers, uri, access_token); + yield fillStandardHeaders(ctx, headers, uri, access_token); headers['Content-Type'] = mime.getType(suggestedTarget); ctx.logger.debug('wopi putRelativeFile request uri=%s headers=%j', uri, headers); @@ -741,7 +744,7 @@ function renameFile(ctx, wopiParams, name) { let commonInfo = wopiParams.commonInfo; let headers = {'X-WOPI-Override': 'RENAME_FILE', 'X-WOPI-Lock': commonInfo.lockId, 'X-WOPI-RequestedName': utf7.encode(name)}; - fillStandardHeaders(ctx, headers, uri, userAuth.access_token); + yield fillStandardHeaders(ctx, headers, uri, userAuth.access_token); ctx.logger.debug('wopi RenameFile request uri=%s headers=%j', uri, headers); let postRes = yield utils.postRequestPromise(ctx, uri, undefined, undefined, undefined, tenCallbackRequestTimeout, undefined, headers); @@ -779,7 +782,7 @@ function checkFileInfo(ctx, wopiSrc, access_token, opt_sc) { if (opt_sc) { headers['X-WOPI-SessionContext'] = opt_sc; } - fillStandardHeaders(ctx, headers, uri, access_token); + yield fillStandardHeaders(ctx, headers, uri, access_token); ctx.logger.debug('wopi checkFileInfo request uri=%s headers=%j', uri, headers); //todo false? (true because it passed checkIpFilter for wopi) //todo use directIfIn @@ -815,7 +818,7 @@ function lock(ctx, command, lockId, fileInfo, userAuth) { } let headers = {"X-WOPI-Override": command, "X-WOPI-Lock": lockId}; - fillStandardHeaders(ctx, headers, uri, access_token); + yield fillStandardHeaders(ctx, headers, uri, access_token); ctx.logger.debug('wopi %s request uri=%s headers=%j', command, uri, headers); let postRes = yield utils.postRequestPromise(ctx, uri, undefined, undefined, undefined, tenCallbackRequestTimeout, undefined, headers); ctx.logger.debug('wopi %s response headers=%j', command, postRes.response.headers); @@ -852,7 +855,7 @@ function unlock(ctx, wopiParams) { } let headers = {"X-WOPI-Override": "UNLOCK", "X-WOPI-Lock": lockId}; - fillStandardHeaders(ctx, headers, uri, access_token); + yield fillStandardHeaders(ctx, headers, uri, access_token); ctx.logger.debug('wopi Unlock request uri=%s headers=%j', uri, headers); let postRes = yield utils.postRequestPromise(ctx, uri, undefined, undefined, undefined, tenCallbackRequestTimeout, undefined, headers); ctx.logger.debug('wopi Unlock response headers=%j', postRes.response.headers); @@ -885,28 +888,20 @@ function generateProofBuffer(url, accessToken, timeStamp) { buffer.writeBigUInt64BE(timeStamp, offset); return buffer; } -function generateProofSign(url, accessToken, timeStamp, privateKey) { - let signer = crypto.createSign('RSA-SHA256'); - signer.update(generateProofBuffer(url, accessToken, timeStamp)); - return signer.sign({key:privateKey}, "base64"); -} -function generateProof(ctx, url, accessToken, timeStamp) { - const tenWopiPrivateKey = ctx.getCfg('wopi.privateKey', cfgWopiPrivateKey); - let privateKey = `-----BEGIN RSA PRIVATE KEY-----\n${tenWopiPrivateKey}\n-----END RSA PRIVATE KEY-----`; - return generateProofSign(url, accessToken, timeStamp, privateKey); -} -function generateProofOld(ctx, url, accessToken, timeStamp) { - const tenWopiPrivateKeyOld = ctx.getCfg('wopi.privateKeyOld', cfgWopiPrivateKeyOld); - let privateKey = `-----BEGIN RSA PRIVATE KEY-----\n${tenWopiPrivateKeyOld}\n-----END RSA PRIVATE KEY-----`; - return generateProofSign(url, accessToken, timeStamp, privateKey); + +async function generateProofSign(url, accessToken, timeStamp, privateKey) { + let data = generateProofBuffer(url, accessToken, timeStamp); + let sign = await cryptoSign('RSA-SHA256', data, privateKey); + return sign.toString('base64'); } -function fillStandardHeaders(ctx, headers, url, access_token) { + +async function fillStandardHeaders(ctx, headers, url, access_token) { let timeStamp = utils.getDateTimeTicks(new Date()); const tenWopiPrivateKey = ctx.getCfg('wopi.privateKey', cfgWopiPrivateKey); const tenWopiPrivateKeyOld = ctx.getCfg('wopi.privateKeyOld', cfgWopiPrivateKeyOld); if (tenWopiPrivateKey && tenWopiPrivateKeyOld) { - headers['X-WOPI-Proof'] = generateProof(ctx, url, access_token, timeStamp); - headers['X-WOPI-ProofOld'] = generateProofOld(ctx, url, access_token, timeStamp); + headers['X-WOPI-Proof'] = await generateProofSign(url, access_token, timeStamp, Buffer.from(tenWopiPrivateKey, 'base64')); + headers['X-WOPI-ProofOld'] = await generateProofSign(url, access_token, timeStamp, Buffer.from(tenWopiPrivateKeyOld, 'base64')); headers['X-WOPI-TimeStamp'] = timeStamp; headers['X-WOPI-ClientVersion'] = commonDefines.buildVersion + '.' + commonDefines.buildNumber; // todo @@ -1019,8 +1014,6 @@ exports.putRelativeFile = putRelativeFile; exports.renameFile = renameFile; exports.lock = lock; exports.unlock = unlock; -exports.generateProof = generateProof; -exports.generateProofOld = generateProofOld; exports.fillStandardHeaders = fillStandardHeaders; exports.getWopiUnlockMarker = getWopiUnlockMarker; exports.getWopiModifiedMarker = getWopiModifiedMarker; diff --git a/FileConverter/sources/converter.js b/FileConverter/sources/converter.js index fb80e2af7..fc14c271e 100644 --- a/FileConverter/sources/converter.js +++ b/FileConverter/sources/converter.js @@ -1050,7 +1050,7 @@ function* ExecuteTask(ctx, task) { isInJwtToken = true; let fileInfo = wopiParams.commonInfo?.fileInfo; fileSize = fileInfo?.Size; - ({url, headers} = wopiClient.getWopiFileUrl(ctx, fileInfo, wopiParams.userAuth)); + ({url, headers} = yield wopiClient.getWopiFileUrl(ctx, fileInfo, wopiParams.userAuth)); } if (undefined === fileSize || fileSize > 0) { error = yield* downloadFile(ctx, url, dataConvert.fileFrom, withAuthorization, isInJwtToken, headers); From b32dcf5bbd9c426c05064a67b596c53e6fa1a980 Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Mon, 27 May 2024 19:45:35 +0300 Subject: [PATCH 41/48] [bug] Allow to restrict 'mode' rights; For bug 68198 --- DocService/sources/DocsCoServer.js | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/DocService/sources/DocsCoServer.js b/DocService/sources/DocsCoServer.js index 3609444da..8dd69da0f 100644 --- a/DocService/sources/DocsCoServer.js +++ b/DocService/sources/DocsCoServer.js @@ -2232,7 +2232,10 @@ exports.install = function(server, callbackFunction) { return name; } function isEditMode(permissions, mode) { - //as in web-apps/apps/documenteditor/main/app/controller/Main.js + //like this.api.asc_setViewMode(!this.appOptions.isEdit && !this.appOptions.isRestrictedEdit); + //https://github.com/ONLYOFFICE/web-apps/blob/4a7879b4f88f315fe94d9f7d97c0ed8aa9f82221/apps/documenteditor/main/app/controller/Main.js#L1743 + //todo permissions in embed editor + //https://github.com/ONLYOFFICE/web-apps/blob/72b8350c71e7b314b63b8eec675e76156bb4a2e4/apps/documenteditor/forms/app/controller/ApplicationController.js#L627 return (!mode || mode !== 'view') && (!permissions || permissions.edit !== false || permissions.review === true || permissions.comment === true || permissions.fillForms === true); } @@ -2355,7 +2358,8 @@ exports.install = function(server, callbackFunction) { if (null != edit.lang) { data.lang = edit.lang; } - if (null != edit.mode) { + //allow to restrict rights so don't use token mode in case of 'view' + if (null != edit.mode && 'view' !== data.mode) { data.mode = edit.mode; } if (edit.coEditing?.mode) { From 37da6b0a62ae9c3ed8feb3e008ee1099febb0923 Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Wed, 29 May 2024 20:37:06 +0300 Subject: [PATCH 42/48] [bug] Check permissions in commandSetPassword; Fix bug 68258 --- DocService/sources/canvasservice.js | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/DocService/sources/canvasservice.js b/DocService/sources/canvasservice.js index 6cbfcdb27..594a730dc 100644 --- a/DocService/sources/canvasservice.js +++ b/DocService/sources/canvasservice.js @@ -858,8 +858,11 @@ function* commandSetPassword(ctx, conn, cmd, outputData) { hasDocumentPassword = true; } } - ctx.logger.debug('commandSetPassword isEnterCorrectPassword=%s, hasDocumentPassword=%s, hasPasswordCol=%s', conn.isEnterCorrectPassword, hasDocumentPassword, hasPasswordCol); - if (tenOpenProtectedFile && (conn.isEnterCorrectPassword || !hasDocumentPassword) && hasPasswordCol) { + //https://github.com/ONLYOFFICE/web-apps/blob/4a7879b4f88f315fe94d9f7d97c0ed8aa9f82221/apps/documenteditor/main/app/controller/Main.js#L1652 + //this.appOptions.isPasswordSupport = this.appOptions.isEdit && this.api.asc_isProtectionSupport() && (this.permissions.protect!==false); + let isPasswordSupport = tenOpenProtectedFile && !conn.user?.view && false !== conn.permissions?.protect; + ctx.logger.debug('commandSetPassword isEnterCorrectPassword=%s, hasDocumentPassword=%s, hasPasswordCol=%s, isPasswordSupport=%s', conn.isEnterCorrectPassword, hasDocumentPassword, hasPasswordCol, isPasswordSupport); + if (isPasswordSupport && (conn.isEnterCorrectPassword || !hasDocumentPassword) && hasPasswordCol) { let updateMask = new taskResult.TaskResultData(); updateMask.tenant = ctx.tenant; updateMask.key = cmd.getDocId(); From 3ddf28fef1eddfd3c83811f0ac8146c29fcb9f62 Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Thu, 30 May 2024 01:39:28 +0300 Subject: [PATCH 43/48] [bug] Fix bug with queue closing before document saving in checkDocumentExpire --- DocService/sources/DocsCoServer.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DocService/sources/DocsCoServer.js b/DocService/sources/DocsCoServer.js index 8dd69da0f..fcef4c1cd 100644 --- a/DocService/sources/DocsCoServer.js +++ b/DocService/sources/DocsCoServer.js @@ -1348,7 +1348,7 @@ function createSaveTimer(ctx, docId, opt_userId, opt_userIndex, opt_userLcid, op } while (true) { if (!sqlBase.isLockCriticalSection(docId)) { - canvasService.saveFromChanges(ctx, docId, updateTask.statusInfo, null, opt_userId, opt_userIndex, opt_userLcid, opt_queue, opt_initShardKey); + yield canvasService.saveFromChanges(ctx, docId, updateTask.statusInfo, null, opt_userId, opt_userIndex, opt_userLcid, opt_queue, opt_initShardKey); break; } yield utils.sleep(c_oAscLockTimeOutDelay); From a43ee0ece29f34910289d028b9b270dec7c7118d Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Sun, 2 Jun 2024 23:38:00 +0300 Subject: [PATCH 44/48] [bug] Change wopi exponent format; For bug 66601 --- Common/config/default.json | 4 ++-- DocService/sources/wopiClient.js | 18 ++++++++++++++++-- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/Common/config/default.json b/Common/config/default.json index 12f9f3d05..172d16b53 100644 --- a/Common/config/default.json +++ b/Common/config/default.json @@ -99,11 +99,11 @@ "slideEdit": ["pptx", "pptm", "odp"], "publicKey": "BgIAAACkAABSU0ExAAgAAAEAAQBpTpiJQ2hD8plpGTfEEmcq4IKyr31HikXpuVSBraMfqyodn2PGXBJ3daNSmdPOc0Nz4HO9Auljn8YYXDPBdpiABptSKvEDPF23Q+Qytg0+vCRyondyBcW91w7KLzXce3fnk8ZfJ8QtbZPL9m11wJIWZueQF+l0HKYx4lty+nccbCanytFTADkGQ3SnmExGEF3rBz6I9+OcrDDK9NKPJgEmCiuyei/d4XbPgKls3EIG0h38X5mVF2VytfWm2Yu850B6z3N4MYhj4b4vsYT62zEC4pMRUeb8dIBy4Jsmr3avtmeO00MUH6DVyPC8nirixj2YIOPKk13CdVqGDSXA3cvl", "modulus": "E5CBDDC0250D865A75C25D93CAE320983DC6E22A9EBCF0C8D5A01F1443D38E67B6AF76AF269BE0728074FCE6511193E20231DBFA84B12FBEE16388317873CF7A40E7BC8BD9A6F5B572651795995FFC1DD20642DC6CA980CF76E1DD2F7AB22B0A2601268FD2F4CA30AC9CE3F7883E07EB5D10464C98A7744306390053D1CAA7266C1C77FA725BE231A61C74E91790E7661692C0756DF6CB936D2DC4275FC693E7777BDC352FCA0ED7BDC5057277A27224BC3E0DB632E443B75D3C03F12A529B06809876C1335C18C69F63E902BD73E0734373CED39952A37577125CC6639F1D2AAB1FA3AD8154B9E9458A477DAFB282E02A6712C437196999F243684389984E69", - "exponent": "AQAB", + "exponent": "65537", "privateKey": "LS0tLS1CRUdJTiBQUklWQVRFIEtFWS0tLS0tDQpNSUlFdndJQkFEQU5CZ2txaGtpRzl3MEJBUUVGQUFTQ0JLa3dnZ1NsQWdFQUFvSUJBUURseTkzQUpRMkdXblhDDQpYWlBLNHlDWVBjYmlLcDY4OE1qVm9COFVROU9PWjdhdmRxOG1tK0J5Z0hUODVsRVJrK0lDTWR2NmhMRXZ2dUZqDQppREY0Yzg5NlFPZThpOW1tOWJWeVpSZVZtVi84SGRJR1F0eHNxWURQZHVIZEwzcXlLd29tQVNhUDB2VEtNS3ljDQo0L2VJUGdmclhSQkdUSmluZEVNR09RQlQwY3FuSm13Y2QvcHlXK0l4cGh4MDZSZVE1MllXa3NCMWJmYkxrMjB0DQp4Q2RmeHBQbmQzdmNOUy9LRHRlOXhRVnlkNkp5Skx3K0RiWXk1RU8zWFR3RDhTcFNtd2FBbUhiQk0xd1l4cDlqDQo2UUs5YytCelEzUE8wNWxTbzNWM0VsekdZNThkS3FzZm82MkJWTG5wUllwSGZhK3lndUFxWnhMRU54bHBtZkpEDQphRU9KbUU1cEFnTUJBQUVDZ2dFQUxpTCtSS09yMFh1OEJPZ1EwajFEd0EwM0x4VnJoWGU2ZXRtSkkrSnlTVGNkDQpnS0VOald6aVpWclJJaTJEdlVtNXFNTWw3V2hTd3NsS0sxZWV4eFpKWTd4QVNxU3hjRW9Jd2d6MTdUMDcvanhtDQpmSWRVQmlVS0RaMUt2OFBXbUlyM29LVytma1hXaS9tMXpsSWUwcVhScFRtc0dORXNIUUxFcWkwcm1haVhUWE9SDQovMkxkd2k2a1pSM3NXRng5N1lTNE14L3B1ZUdKVFhFYWk2QVZFWnpONUdvZzZ4RDhIWFIxUnZxK2hoZCtNb2NHDQpmblU0SGdpbEtSZm9KbFdkOUZPc2NnU3VmS0cwTDNWaU80ZlNLVTQ2bDVhdWxsRFlVazVFQ01XaXd1S1NxU0U3DQpxRDQ1akkzbWJPcmU3UzR1M1MzVFdkRDNsendpWEw0OUxkd0tsRUM0bVFLQmdRRDBzTHIwR0g0V3IrUVgyeEpFDQp1QS9DYjhRVzQxbDhpU0NCVFJaWlIvc0pPZCtvM3JiY1ZpZGx6Ty9FYlpibFhHNFpQRG1SamdCQ0dLSVA1RVppDQowRHNMK1d2MzJXT280NExweEpHaHFFeGJtMEgxaVoxelo5N2wwUDhmdkloSEU0MmdtYUxUb09JR0RoUFNYR3Z2DQp6bHFPSGJHYnE0anNFUmMxanAxYmVqNXE2d0tCZ1FEd2F1ZUljNHBSY2hIOThRWWlkY3lyOFZ3ZzlLaGJuZllYDQp5M1c0UlBsWnRCZEYzNGlKYWlvK0FTenVnby96eTFSVGNWcnNDc2tZV1h5S0RVUXoxeXUwaUNuZytmRENVblRtDQpYR21Fb0VHTmhrNHZUSk90N2hCYXYxL0phL2RVaXBHZjZtWFV1YW53SjBlKzEvRXQvQjBhaDVYMVVtNUF5TlpJDQpNK1N5UnozdSt3S0JnUUNqdnRVTlhvcWFnaENCQ21CNlRqWjFwcmV4bldrWUZ1Z0N2MlNTVU1JazFXN2dJbEo2DQp0c2pjcmoxUjFRaWk2cXpmQkZkK0dXb0EwVjA2aDBlMi9xUlZDZy8vcDZHeXRyVzMzSXljZ3ZTK1pQTEo3dExJDQpGUjJyNjZXZlJscG9QaVNMOGVSdC9QN2trRzBoWENuN0s3dWIyVEV1L0thL1cxeU53YWQ2UFI4aUN3S0JnUUM4DQpYY1pTcnRRc3hBYzh3OTllbUpWb0VvOXdjc0NHSjlsdEEwaVV1OVh5WnB2bGJ5SjNKK3M0OFlyV3hRMHNvcDdMDQpVZ0UrOTZSZm81MWtQTWkzSlZ0azgxcDhudGY0S01yV3dva2FGTVhIc1BjSk1DSjFJQlZJUkxFMEM1ZVpjWWh2DQpseU41N0k0dFQxbHpPWllKeFlLNENvdC96cm43b0YvajZtVEJHZmg0aVFLQmdRQ2lKTVV4UnowMS9jekgvWFNYDQpnbzNkVmJIUTRGRU91ZlduRTNFYjkzUzhyMC9lcTFSTTExOHJiMFRxenVpYWRXMnhZRFU0bnVjV1FscmxtcTBkDQpGWS9tK0h5OTdwcXlrNmptb1U1SS9EK3NzQklvWUhXTG5IOS94ZnZERWsySkdTSlNIdHp1MEQ0RURDL3JnUTQ5DQpNYllzTzVvVXJGOHRQbGhqNXZ6YmYzR0tMQT09DQotLS0tLUVORCBQUklWQVRFIEtFWS0tLS0tDQo=", "publicKeyOld": "BgIAAACkAABSU0ExAAgAAAEAAQBpTpiJQ2hD8plpGTfEEmcq4IKyr31HikXpuVSBraMfqyodn2PGXBJ3daNSmdPOc0Nz4HO9Auljn8YYXDPBdpiABptSKvEDPF23Q+Qytg0+vCRyondyBcW91w7KLzXce3fnk8ZfJ8QtbZPL9m11wJIWZueQF+l0HKYx4lty+nccbCanytFTADkGQ3SnmExGEF3rBz6I9+OcrDDK9NKPJgEmCiuyei/d4XbPgKls3EIG0h38X5mVF2VytfWm2Yu850B6z3N4MYhj4b4vsYT62zEC4pMRUeb8dIBy4Jsmr3avtmeO00MUH6DVyPC8nirixj2YIOPKk13CdVqGDSXA3cvl", "modulusOld": "E5CBDDC0250D865A75C25D93CAE320983DC6E22A9EBCF0C8D5A01F1443D38E67B6AF76AF269BE0728074FCE6511193E20231DBFA84B12FBEE16388317873CF7A40E7BC8BD9A6F5B572651795995FFC1DD20642DC6CA980CF76E1DD2F7AB22B0A2601268FD2F4CA30AC9CE3F7883E07EB5D10464C98A7744306390053D1CAA7266C1C77FA725BE231A61C74E91790E7661692C0756DF6CB936D2DC4275FC693E7777BDC352FCA0ED7BDC5057277A27224BC3E0DB632E443B75D3C03F12A529B06809876C1335C18C69F63E902BD73E0734373CED39952A37577125CC6639F1D2AAB1FA3AD8154B9E9458A477DAFB282E02A6712C437196999F243684389984E69", - "exponentOld": "AQAB", + "exponentOld": "65537", "privateKeyOld": "LS0tLS1CRUdJTiBQUklWQVRFIEtFWS0tLS0tDQpNSUlFdndJQkFEQU5CZ2txaGtpRzl3MEJBUUVGQUFTQ0JLa3dnZ1NsQWdFQUFvSUJBUURseTkzQUpRMkdXblhDDQpYWlBLNHlDWVBjYmlLcDY4OE1qVm9COFVROU9PWjdhdmRxOG1tK0J5Z0hUODVsRVJrK0lDTWR2NmhMRXZ2dUZqDQppREY0Yzg5NlFPZThpOW1tOWJWeVpSZVZtVi84SGRJR1F0eHNxWURQZHVIZEwzcXlLd29tQVNhUDB2VEtNS3ljDQo0L2VJUGdmclhSQkdUSmluZEVNR09RQlQwY3FuSm13Y2QvcHlXK0l4cGh4MDZSZVE1MllXa3NCMWJmYkxrMjB0DQp4Q2RmeHBQbmQzdmNOUy9LRHRlOXhRVnlkNkp5Skx3K0RiWXk1RU8zWFR3RDhTcFNtd2FBbUhiQk0xd1l4cDlqDQo2UUs5YytCelEzUE8wNWxTbzNWM0VsekdZNThkS3FzZm82MkJWTG5wUllwSGZhK3lndUFxWnhMRU54bHBtZkpEDQphRU9KbUU1cEFnTUJBQUVDZ2dFQUxpTCtSS09yMFh1OEJPZ1EwajFEd0EwM0x4VnJoWGU2ZXRtSkkrSnlTVGNkDQpnS0VOald6aVpWclJJaTJEdlVtNXFNTWw3V2hTd3NsS0sxZWV4eFpKWTd4QVNxU3hjRW9Jd2d6MTdUMDcvanhtDQpmSWRVQmlVS0RaMUt2OFBXbUlyM29LVytma1hXaS9tMXpsSWUwcVhScFRtc0dORXNIUUxFcWkwcm1haVhUWE9SDQovMkxkd2k2a1pSM3NXRng5N1lTNE14L3B1ZUdKVFhFYWk2QVZFWnpONUdvZzZ4RDhIWFIxUnZxK2hoZCtNb2NHDQpmblU0SGdpbEtSZm9KbFdkOUZPc2NnU3VmS0cwTDNWaU80ZlNLVTQ2bDVhdWxsRFlVazVFQ01XaXd1S1NxU0U3DQpxRDQ1akkzbWJPcmU3UzR1M1MzVFdkRDNsendpWEw0OUxkd0tsRUM0bVFLQmdRRDBzTHIwR0g0V3IrUVgyeEpFDQp1QS9DYjhRVzQxbDhpU0NCVFJaWlIvc0pPZCtvM3JiY1ZpZGx6Ty9FYlpibFhHNFpQRG1SamdCQ0dLSVA1RVppDQowRHNMK1d2MzJXT280NExweEpHaHFFeGJtMEgxaVoxelo5N2wwUDhmdkloSEU0MmdtYUxUb09JR0RoUFNYR3Z2DQp6bHFPSGJHYnE0anNFUmMxanAxYmVqNXE2d0tCZ1FEd2F1ZUljNHBSY2hIOThRWWlkY3lyOFZ3ZzlLaGJuZllYDQp5M1c0UlBsWnRCZEYzNGlKYWlvK0FTenVnby96eTFSVGNWcnNDc2tZV1h5S0RVUXoxeXUwaUNuZytmRENVblRtDQpYR21Fb0VHTmhrNHZUSk90N2hCYXYxL0phL2RVaXBHZjZtWFV1YW53SjBlKzEvRXQvQjBhaDVYMVVtNUF5TlpJDQpNK1N5UnozdSt3S0JnUUNqdnRVTlhvcWFnaENCQ21CNlRqWjFwcmV4bldrWUZ1Z0N2MlNTVU1JazFXN2dJbEo2DQp0c2pjcmoxUjFRaWk2cXpmQkZkK0dXb0EwVjA2aDBlMi9xUlZDZy8vcDZHeXRyVzMzSXljZ3ZTK1pQTEo3dExJDQpGUjJyNjZXZlJscG9QaVNMOGVSdC9QN2trRzBoWENuN0s3dWIyVEV1L0thL1cxeU53YWQ2UFI4aUN3S0JnUUM4DQpYY1pTcnRRc3hBYzh3OTllbUpWb0VvOXdjc0NHSjlsdEEwaVV1OVh5WnB2bGJ5SjNKK3M0OFlyV3hRMHNvcDdMDQpVZ0UrOTZSZm81MWtQTWkzSlZ0azgxcDhudGY0S01yV3dva2FGTVhIc1BjSk1DSjFJQlZJUkxFMEM1ZVpjWWh2DQpseU41N0k0dFQxbHpPWllKeFlLNENvdC96cm43b0YvajZtVEJHZmg0aVFLQmdRQ2lKTVV4UnowMS9jekgvWFNYDQpnbzNkVmJIUTRGRU91ZlduRTNFYjkzUzhyMC9lcTFSTTExOHJiMFRxenVpYWRXMnhZRFU0bnVjV1FscmxtcTBkDQpGWS9tK0h5OTdwcXlrNmptb1U1SS9EK3NzQklvWUhXTG5IOS94ZnZERWsySkdTSlNIdHp1MEQ0RURDL3JnUTQ5DQpNYllzTzVvVXJGOHRQbGhqNXZ6YmYzR0tMQT09DQotLS0tLUVORCBQUklWQVRFIEtFWS0tLS0tDQo=", "refreshLockInterval": "10m", "dummy" : { diff --git a/DocService/sources/wopiClient.js b/DocService/sources/wopiClient.js index 818c1f7fe..a0c212699 100644 --- a/DocService/sources/wopiClient.js +++ b/DocService/sources/wopiClient.js @@ -277,9 +277,11 @@ function discovery(req, res) { //end section for collabora nexcloud connectors let xmlDiscovery = xmlZone.up(); if (tenWopiPublicKeyOld && tenWopiPublicKey) { + let exponent = numberToBase64(tenWopiExponent - 0); + let exponentOld = numberToBase64(tenWopiExponentOld - 0); xmlDiscovery.ele('proof-key', { - oldvalue: tenWopiPublicKeyOld, oldmodulus: tenWopiModulusOld, oldexponent: tenWopiExponentOld, - value: tenWopiPublicKey, modulus: tenWopiModulus, exponent: tenWopiExponent + oldvalue: tenWopiPublicKeyOld, oldmodulus: tenWopiModulusOld, oldexponent: exponentOld, + value: tenWopiPublicKey, modulus: tenWopiModulus, exponent: exponent }).up(); } xmlDiscovery.up(); @@ -895,6 +897,18 @@ async function generateProofSign(url, accessToken, timeStamp, privateKey) { return sign.toString('base64'); } +function numberToBase64(val) { + // Convert to hexadecimal + let hexString = val.toString(16); + //Ensure the hexadecimal string has an even length + if (hexString.length % 2 !== 0) { + hexString = '0' + hexString; + } + //Convert the hexadecimal string to a buffer + const buffer = Buffer.from(hexString, 'hex'); + return buffer.toString('base64'); +} + async function fillStandardHeaders(ctx, headers, url, access_token) { let timeStamp = utils.getDateTimeTicks(new Date()); const tenWopiPrivateKey = ctx.getCfg('wopi.privateKey', cfgWopiPrivateKey); From 59140603e91dad8e7cf9264847e71ccd9be19831 Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Mon, 3 Jun 2024 00:44:53 +0300 Subject: [PATCH 45/48] [bug] Use shutdownFlag for WOPI editing --- DocService/sources/DocsCoServer.js | 1 + DocService/sources/wopiClient.js | 73 ++++++++++++++++-------------- 2 files changed, 41 insertions(+), 33 deletions(-) diff --git a/DocService/sources/DocsCoServer.js b/DocService/sources/DocsCoServer.js index fcef4c1cd..d001a3a81 100644 --- a/DocService/sources/DocsCoServer.js +++ b/DocService/sources/DocsCoServer.js @@ -3640,6 +3640,7 @@ exports.install = function(server, callbackFunction) { case commonDefines.c_oPublishType.shutdown: //flag prevent new socket connections and receive data from exist connections shutdownFlag = data.status; + wopiClient.setIsShutdown(shutdownFlag); ctx.logger.warn('start shutdown:%s', shutdownFlag); if (shutdownFlag) { ctx.logger.warn('active connections: %d', connections.length); diff --git a/DocService/sources/wopiClient.js b/DocService/sources/wopiClient.js index a0c212699..628a8f0d0 100644 --- a/DocService/sources/wopiClient.js +++ b/DocService/sources/wopiClient.js @@ -96,6 +96,7 @@ let cryptoSign = util.promisify(crypto.sign); let templatesFolderLocalesCache = null; let templatesFolderExtsCache = null; const templateFilesSizeCache = {}; +let shutdownFlag = false; let mimeTypesByExt = (function() { let mimeTypesByExt = {}; @@ -358,6 +359,9 @@ async function getWopiFileUrl(ctx, fileInfo, userAuth) { function isWopiJwtToken(decoded) { return !!decoded.fileInfo; } +function setIsShutdown(val) { + shutdownFlag = val; +} function getLastModifiedTimeFromCallbacks(callbacks) { for (let i = callbacks.length; i >= 0; --i) { let callback = callbacks[i]; @@ -512,45 +516,47 @@ function getEditorHtml(req, res) { params.fileInfo = {}; return; } - //save common info - const fileType = getFileTypeByInfo(fileInfo); - if (undefined === lockId) { - lockId = crypto.randomBytes(16).toString('base64'); - let commonInfo = JSON.stringify({lockId: lockId, fileInfo: fileInfo}); - yield canvasService.commandOpenStartPromise(ctx, docId, utils.getBaseUrlByRequest(ctx, req), commonInfo, fileType); - } + if (!shutdownFlag) { + //save common info + const fileType = getFileTypeByInfo(fileInfo); + if (undefined === lockId) { + lockId = crypto.randomBytes(16).toString('base64'); + let commonInfo = JSON.stringify({lockId: lockId, fileInfo: fileInfo}); + yield canvasService.commandOpenStartPromise(ctx, docId, utils.getBaseUrlByRequest(ctx, req), commonInfo, fileType); + } - // TODO: throw error if format not supported? - if (fileInfo.Size === 0 && fileType.length !== 0) { - const wopiParams = getWopiParams(undefined, fileInfo, wopiSrc, access_token, access_token_ttl); + // TODO: throw error if format not supported? + if (fileInfo.Size === 0 && fileType.length !== 0) { + const wopiParams = getWopiParams(undefined, fileInfo, wopiSrc, access_token, access_token_ttl); - if (templatesFolderLocalesCache === null) { - const dirContent = yield readdir(`${tenNewFileTemplate}/`, { withFileTypes: true }); - templatesFolderLocalesCache = dirContent.filter(dirObject => dirObject.isDirectory()).map(dirObject => dirObject.name); - } + if (templatesFolderLocalesCache === null) { + const dirContent = yield readdir(`${tenNewFileTemplate}/`, { withFileTypes: true }); + templatesFolderLocalesCache = dirContent.filter(dirObject => dirObject.isDirectory()).map(dirObject => dirObject.name); + } - const localePrefix = lang || ui || 'en'; - let locale = constants.TEMPLATES_FOLDER_LOCALE_COLLISON_MAP[localePrefix] ?? templatesFolderLocalesCache.find(locale => locale.startsWith(localePrefix)); - if (locale === undefined) { - locale = constants.TEMPLATES_DEFAULT_LOCALE; - } + const localePrefix = lang || ui || 'en'; + let locale = constants.TEMPLATES_FOLDER_LOCALE_COLLISON_MAP[localePrefix] ?? templatesFolderLocalesCache.find(locale => locale.startsWith(localePrefix)); + if (locale === undefined) { + locale = constants.TEMPLATES_DEFAULT_LOCALE; + } - const filePath = `${tenNewFileTemplate}/${locale}/new.${fileType}`; - if (!templateFilesSizeCache[filePath]) { - templateFilesSizeCache[filePath] = yield lstat(filePath); - } + const filePath = `${tenNewFileTemplate}/${locale}/new.${fileType}`; + if (!templateFilesSizeCache[filePath]) { + templateFilesSizeCache[filePath] = yield lstat(filePath); + } - const templateFileInfo = templateFilesSizeCache[filePath]; - const templateFileStream = createReadStream(filePath); - yield putFile(ctx, wopiParams, undefined, templateFileStream, templateFileInfo.size, fileInfo.UserId, false, false, false); - } + const templateFileInfo = templateFilesSizeCache[filePath]; + const templateFileStream = createReadStream(filePath); + yield putFile(ctx, wopiParams, undefined, templateFileStream, templateFileInfo.size, fileInfo.UserId, false, false, false); + } - //Lock - if ('view' !== mode) { - let lockRes = yield lock(ctx, 'LOCK', lockId, fileInfo, userAuth); - if (!lockRes) { - params.fileInfo = {}; - return; + //Lock + if ('view' !== mode) { + let lockRes = yield lock(ctx, 'LOCK', lockId, fileInfo, userAuth); + if (!lockRes) { + params.fileInfo = {}; + return; + } } } @@ -1034,6 +1040,7 @@ exports.getWopiModifiedMarker = getWopiModifiedMarker; exports.getFileTypeByInfo = getFileTypeByInfo; exports.getWopiFileUrl = getWopiFileUrl; exports.isWopiJwtToken = isWopiJwtToken; +exports.setIsShutdown = setIsShutdown; exports.dummyCheckFileInfo = dummyCheckFileInfo; exports.dummyGetFile = dummyGetFile; exports.dummyOk = dummyOk; From 8d401629c5a73b90ad03a368b68d50d2d520c0bc Mon Sep 17 00:00:00 2001 From: Georgii Petrov Date: Mon, 3 Jun 2024 11:30:04 +0300 Subject: [PATCH 46/48] [fix] Added support of MySql ssl connection --- Common/config/default.json | 3 +- .../databaseConnectors/mysqlConnector.js | 202 +++++++++--------- 2 files changed, 105 insertions(+), 100 deletions(-) diff --git a/Common/config/default.json b/Common/config/default.json index 12f9f3d05..e1ffea52b 100644 --- a/Common/config/default.json +++ b/Common/config/default.json @@ -230,7 +230,8 @@ "pool": { "idleTimeoutMillis": 30000 } - } + }, + "mysqlExtraOptions": {} }, "redis": { "name": "redis", diff --git a/DocService/sources/databaseConnectors/mysqlConnector.js b/DocService/sources/databaseConnectors/mysqlConnector.js index 96535bd48..6078d9446 100644 --- a/DocService/sources/databaseConnectors/mysqlConnector.js +++ b/DocService/sources/databaseConnectors/mysqlConnector.js @@ -32,14 +32,14 @@ 'use strict'; -const mysql = require('mysql2'); +const mysql = require('mysql2/promise'); const connectorUtilities = require('./connectorUtilities'); const config = require('config'); const configSql = config.get('services.CoAuthoring.sql'); -const cfgTableResult = config.get('services.CoAuthoring.sql.tableResult'); +const cfgTableResult = configSql.get('tableResult'); -const pool = mysql.createPool({ +const connectionConfiguration = { host : configSql.get('dbHost'), port : parseInt(configSql.get('dbPort')), user : configSql.get('dbUser'), @@ -49,120 +49,124 @@ const pool = mysql.createPool({ connectionLimit : configSql.get('connectionlimit'), timezone : 'Z', flags : '-FOUND_ROWS' -}); +}; -function sqlQuery(ctx, sqlCommand, callbackFunction, opt_noModifyRes = false, opt_noLog = false, opt_values = []) { - pool.getConnection(function(connectionError, connection) { - if (connectionError) { - if (!opt_noLog) { - ctx.logger.error('pool.getConnection error: %s', connectionError); - } +const additionalOptions = configSql.get('mysqlExtraOptions'); +const configuration = Object.assign({}, connectionConfiguration, additionalOptions); + +let pool = null; - callbackFunction?.(connectionError, null); +function sqlQuery(ctx, sqlCommand, callbackFunction, opt_noModifyRes = false, opt_noLog = false, opt_values = []) { + return executeQuery(ctx, sqlCommand, opt_values, opt_noModifyRes, opt_noLog).then( + result => callbackFunction?.(null, result), + error => callbackFunction?.(error) + ); +} - return; +async function executeQuery(ctx, sqlCommand, values = [], noModifyRes = false, noLog = false) { + let connection = null; + try { + if (!pool) { + pool = mysql.createPool(configuration); } - let queryCallback = function (error, result) { - connection.release(); - if (error && !opt_noLog) { - ctx.logger.error('_______________________error______________________'); - ctx.logger.error('sqlQuery: %s sqlCommand: %s', error.code, sqlCommand); - ctx.logger.error(error); - ctx.logger.error('_____________________end_error____________________'); - } + connection = await pool.getConnection(); - let output; - if (!opt_noModifyRes) { - output = result?.affectedRows ? { affectedRows: result.affectedRows } : result; - } else { - output = result; - } + const result = await connection.query(sqlCommand, values); - output = output ?? { rows: [], affectedRows: 0 }; + let output; + if (!noModifyRes) { + output = result[0]?.affectedRows ? { affectedRows: result[0].affectedRows } : result[0]; + } else { + output = result[0]; + } - callbackFunction?.(error, output); - }; + return output ?? { rows: [], affectedRows: 0 }; + } catch (error) { + if (!noLog) { + ctx.logger.error(`sqlQuery() error while executing query: ${sqlCommand}\n${error.stack}`); + } - connection.query(sqlCommand, opt_values, queryCallback); - }); + throw error; + } finally { + if (connection) { + try { + // Put the connection back in the pool + connection.release(); + } catch (error) { + if (!noLog) { + ctx.logger.error(`connection.release() error while executing query: ${sqlCommand}\n${error.stack}`); + } + } + } + } } function closePool() { - return new Promise((resolve, reject) => { - pool.end((error) => { - if (error) { - reject(error); - } else { - resolve(); - } - }); - }); + return pool?.end(); } -function addSqlParameter(val, values) { - values.push(val); +function addSqlParameter(parameter, accumulatedArray) { + accumulatedArray.push(parameter); return '?'; } -function concatParams(val1, val2) { - return `CONCAT(COALESCE(${val1}, ''), COALESCE(${val2}, ''))`; +function concatParams(firstParameter, secondParameter) { + return `CONCAT(COALESCE(${firstParameter}, ''), COALESCE(${secondParameter}, ''))`; } -function upsert(ctx, task) { - return new Promise(function(resolve, reject) { - task.completeDefaults(); - let dateNow = new Date(); - let values = []; - let cbInsert = task.callback; - if (task.callback) { - let userCallback = new connectorUtilities.UserCallback(); - userCallback.fromValues(task.userIndex, task.callback); - cbInsert = userCallback.toSQLInsert(); - } - let p0 = addSqlParameter(task.tenant, values); - let p1 = addSqlParameter(task.key, values); - let p2 = addSqlParameter(task.status, values); - let p3 = addSqlParameter(task.statusInfo, values); - let p4 = addSqlParameter(dateNow, values); - let p5 = addSqlParameter(task.userIndex, values); - let p6 = addSqlParameter(task.changeId, values); - let p7 = addSqlParameter(cbInsert, values); - let p8 = addSqlParameter(task.baseurl, values); - let p9 = addSqlParameter(dateNow, values); - var sqlCommand = `INSERT INTO ${cfgTableResult} (tenant, id, status, status_info, last_open_date, user_index, change_id, callback, baseurl)`+ - ` VALUES (${p0}, ${p1}, ${p2}, ${p3}, ${p4}, ${p5}, ${p6}, ${p7}, ${p8}) ON DUPLICATE KEY UPDATE` + - ` last_open_date = ${p9}`; - if (task.callback) { - let p10 = addSqlParameter(JSON.stringify(task.callback), values); - sqlCommand += `, callback = CONCAT(callback , '${connectorUtilities.UserCallback.prototype.delimiter}{"userIndex":' , (user_index + 1) , ',"callback":', ${p10}, '}')`; - } - if (task.baseurl) { - let p11 = addSqlParameter(task.baseurl, values); - sqlCommand += `, baseurl = ${p11}`; - } - - sqlCommand += ', user_index = LAST_INSERT_ID(user_index + 1);'; - - sqlQuery(ctx, sqlCommand, function(error, result) { - if (error) { - reject(error); - } else { - const insertId = result.affectedRows === 1 ? task.userIndex : result.insertId; - //if CLIENT_FOUND_ROWS don't specify 1 row is inserted , 2 row is updated, and 0 row is set to its current values - //http://dev.mysql.com/doc/refman/5.7/en/insert-on-duplicate.html - const isInsert = result.affectedRows === 1; - - resolve({ isInsert, insertId }); - } - }, true, false, values); - }); +async function upsert(ctx, task) { + task.completeDefaults(); + const dateNow = new Date(); + + let cbInsert = task.callback; + if (task.callback) { + const userCallback = new connectorUtilities.UserCallback(); + userCallback.fromValues(task.userIndex, task.callback); + cbInsert = userCallback.toSQLInsert(); + } + + const values = []; + const valuesPlaceholder = [ + addSqlParameter(task.tenant, values), + addSqlParameter(task.key, values), + addSqlParameter(task.status, values), + addSqlParameter(task.statusInfo, values), + addSqlParameter(dateNow, values), + addSqlParameter(task.userIndex, values), + addSqlParameter(task.changeId, values), + addSqlParameter(cbInsert, values), + addSqlParameter(task.baseurl, values) + ]; + + let updateStatement = `last_open_date = ${addSqlParameter(dateNow, values)}`; + if (task.callback) { + let callbackPlaceholder = addSqlParameter(JSON.stringify(task.callback), values); + updateStatement += `, callback = CONCAT(callback , '${connectorUtilities.UserCallback.prototype.delimiter}{"userIndex":' , (user_index + 1) , ',"callback":', ${callbackPlaceholder}, '}')`; + } + + if (task.baseurl) { + let baseUrlPlaceholder = addSqlParameter(task.baseurl, values); + updateStatement += `, baseurl = ${baseUrlPlaceholder}`; + } + + updateStatement += ', user_index = LAST_INSERT_ID(user_index + 1);'; + + const sqlCommand = `INSERT INTO ${cfgTableResult} (tenant, id, status, status_info, last_open_date, user_index, change_id, callback, baseurl) `+ + `VALUES (${valuesPlaceholder.join(', ')}) ` + + `ON DUPLICATE KEY UPDATE ${updateStatement}`; + + const result = await executeQuery(ctx, sqlCommand, values, true); + const insertId = result.affectedRows === 1 ? task.userIndex : result.insertId; + //if CLIENT_FOUND_ROWS don't specify 1 row is inserted , 2 row is updated, and 0 row is set to its current values + //http://dev.mysql.com/doc/refman/5.7/en/insert-on-duplicate.html + const isInsert = result.affectedRows === 1; + + return { isInsert, insertId }; } -module.exports = { - sqlQuery, - closePool, - addSqlParameter, - concatParams, - upsert -} \ No newline at end of file +module.exports.sqlQuery = sqlQuery; +module.exports.closePool = closePool; +module.exports.addSqlParameter = addSqlParameter; +module.exports.concatParams = concatParams; +module.exports.upsert = upsert; From ee75b4e7cab363247f0ff65e2c8f8f9f673ecc86 Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Mon, 3 Jun 2024 13:24:13 +0300 Subject: [PATCH 47/48] Bump mysql2 from 2.3.3 to 3.9.8 in /DocService --- DocService/npm-shrinkwrap.json | 6 +++--- DocService/package.json | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/DocService/npm-shrinkwrap.json b/DocService/npm-shrinkwrap.json index c444b307b..34ee90889 100644 --- a/DocService/npm-shrinkwrap.json +++ b/DocService/npm-shrinkwrap.json @@ -2552,9 +2552,9 @@ } }, "mysql2": { - "version": "3.9.7", - "resolved": "https://registry.npmjs.org/mysql2/-/mysql2-3.9.7.tgz", - "integrity": "sha512-KnJT8vYRcNAZv73uf9zpXqNbvBG7DJrs+1nACsjZP1HMJ1TgXEy8wnNilXAn/5i57JizXKtrUtwDB7HxT9DDpw==", + "version": "3.9.8", + "resolved": "https://registry.npmjs.org/mysql2/-/mysql2-3.9.8.tgz", + "integrity": "sha512-+5JKNjPuks1FNMoy9TYpl77f+5frbTklz7eb3XDwbpsERRLEeXiW2PDEkakYF50UuKU2qwfGnyXpKYvukv8mGA==", "requires": { "denque": "^2.1.0", "generate-function": "^2.3.1", diff --git a/DocService/package.json b/DocService/package.json index 1b62239a2..f42a1f6e5 100644 --- a/DocService/package.json +++ b/DocService/package.json @@ -35,7 +35,7 @@ "multer": "1.4.3", "multi-integer-range": "4.0.7", "multiparty": "4.2.1", - "mysql2": "3.9.7", + "mysql2": "3.9.8", "oracledb": "6.3.0", "pg": "8.11.3", "redis": "4.6.11", From d779c08027ce3b811b191c3419d5669783d536ca Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Mon, 3 Jun 2024 13:07:33 +0300 Subject: [PATCH 48/48] [fix] Make closePool async; Bump nodejs version for mysql test; For bug 68194 --- .github/workflows/mysqlDatabaseTests.yml | 2 +- .../sources/databaseConnectors/mysqlConnector.js | 10 +++------- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/.github/workflows/mysqlDatabaseTests.yml b/.github/workflows/mysqlDatabaseTests.yml index 3d4e8f188..c8e424e63 100644 --- a/.github/workflows/mysqlDatabaseTests.yml +++ b/.github/workflows/mysqlDatabaseTests.yml @@ -21,7 +21,7 @@ jobs: - name: Caching dependencies uses: actions/setup-node@v3 with: - node-version: '14' + node-version: '16' cache: 'npm' cache-dependency-path: | ./npm-shrinkwrap.json diff --git a/DocService/sources/databaseConnectors/mysqlConnector.js b/DocService/sources/databaseConnectors/mysqlConnector.js index 6078d9446..f295588ce 100644 --- a/DocService/sources/databaseConnectors/mysqlConnector.js +++ b/DocService/sources/databaseConnectors/mysqlConnector.js @@ -54,7 +54,7 @@ const connectionConfiguration = { const additionalOptions = configSql.get('mysqlExtraOptions'); const configuration = Object.assign({}, connectionConfiguration, additionalOptions); -let pool = null; +let pool = mysql.createPool(configuration); function sqlQuery(ctx, sqlCommand, callbackFunction, opt_noModifyRes = false, opt_noLog = false, opt_values = []) { return executeQuery(ctx, sqlCommand, opt_values, opt_noModifyRes, opt_noLog).then( @@ -66,10 +66,6 @@ function sqlQuery(ctx, sqlCommand, callbackFunction, opt_noModifyRes = false, op async function executeQuery(ctx, sqlCommand, values = [], noModifyRes = false, noLog = false) { let connection = null; try { - if (!pool) { - pool = mysql.createPool(configuration); - } - connection = await pool.getConnection(); const result = await connection.query(sqlCommand, values); @@ -102,8 +98,8 @@ async function executeQuery(ctx, sqlCommand, values = [], noModifyRes = false, n } } -function closePool() { - return pool?.end(); +async function closePool() { + return await pool.end(); } function addSqlParameter(parameter, accumulatedArray) {