From 6eb500ee97fd9fa51edf4aab0063066b5a9b78fe Mon Sep 17 00:00:00 2001 From: Denis Hovart Date: Tue, 12 Apr 2022 15:48:42 +0200 Subject: [PATCH 1/5] Bump nwl-components, bis --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index fbd21e46..45f5c15d 100644 --- a/package.json +++ b/package.json @@ -57,7 +57,7 @@ "multer": "^1.4.2", "mustache-express": "^1.3.0", "neuroweblab": "github:neuroanatomy/neuroweblab", - "nwl-components": "^0.0.16", + "nwl-components": "^0.0.17", "pako": "^1.0.11", "passport": "^0.4.1", "passport-github": "^1.1.0", From c0fb46b96a888659a397944ae30382a3e95d2c40 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 12 Apr 2022 23:41:05 +0000 Subject: [PATCH 2/5] Bump async from 2.6.3 to 3.2.2 Bumps [async](https://github.com/caolan/async) from 2.6.3 to 3.2.2. - [Release notes](https://github.com/caolan/async/releases) - [Changelog](https://github.com/caolan/async/blob/master/CHANGELOG.md) - [Commits](https://github.com/caolan/async/compare/v2.6.3...v3.2.2) --- updated-dependencies: - dependency-name: async dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- package-lock.json | 52 +++++++++++++++++++++++++++-------------------- package.json | 2 +- 2 files changed, 31 insertions(+), 23 deletions(-) diff --git a/package-lock.json b/package-lock.json index a076412b..6c892e69 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8,7 +8,7 @@ "name": "brainbox", "version": "0.1.0", "dependencies": { - "async": "^2.6.3", + "async": "^3.2.2", "async-lock": "^1.3.0", "body-parser": "~1.18.2", "chai-as-promised": "^7.1.1", @@ -41,7 +41,7 @@ "multer": "^1.4.2", "mustache-express": "^1.3.0", "neuroweblab": "github:neuroanatomy/neuroweblab", - "nwl-components": "^0.0.12", + "nwl-components": "^0.0.17", "pako": "^1.0.11", "passport": "^0.4.1", "passport-github": "^1.1.0", @@ -1720,12 +1720,9 @@ } }, "node_modules/async": { - "version": "2.6.3", - "resolved": "https://registry.npmjs.org/async/-/async-2.6.3.tgz", - "integrity": "sha512-zflvls11DCy+dQWzTW2dzuilv8Z5X/pjfmZOWba6TNIVDm+2UDaJmXSOXlasHKfNBs8oo3M0aT50fDEWfKZjXg==", - "dependencies": { - "lodash": "^4.17.14" - } + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/async/-/async-3.2.2.tgz", + "integrity": "sha512-H0E+qZaDEfx/FY4t7iLRv1W2fFI6+pyCeTw1uN20AQPiwqwM6ojPxHxdLv4z8hi2DtnW9BOckSspLucW7pIE5g==" }, "node_modules/async-each": { "version": "1.0.3", @@ -9445,14 +9442,16 @@ } }, "node_modules/nwl-components": { - "version": "0.0.12", - "resolved": "https://registry.npmjs.org/nwl-components/-/nwl-components-0.0.12.tgz", - "integrity": "sha512-Qbce+RwJB1N0FBx4JYfNaTMcBA0sz00bpv2hJa6/4/FW+HP6RYo05Jmg/M6e52Kc9J1yIP/qHMtM25GxEasFeg==", + "version": "0.0.17", + "resolved": "https://registry.npmjs.org/nwl-components/-/nwl-components-0.0.17.tgz", + "integrity": "sha512-fScHc9BrWXFmE653AYwZIY0sTvcwmMf1qaUtGkd/mnXSxX9FybW5c9krA97/pIZg7dii20nmLTSzbV7Ol6FJ4A==", "dependencies": { + "dompurify": "^2.3.6", "jdenticon": "^3.1.1", "lodash-es": "^4.17.21", "md5": "^2.3.0", - "nanoid": "^3.3.1" + "nanoid": "^3.3.1", + "splitpanes": "^3.1.0" }, "peerDependencies": { "vue": "^3.2.25" @@ -12123,6 +12122,11 @@ "node": ">=0.10.0" } }, + "node_modules/splitpanes": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/splitpanes/-/splitpanes-3.1.1.tgz", + "integrity": "sha512-VUkxDJfIGSvTM/fm/+OSrx8ha9URwE/9B8FPvfzoBuAxVELIHBWpsfnJXIXv77zVwuex//QQL4kTU9SDBPeHjA==" + }, "node_modules/sprintf-js": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", @@ -16297,12 +16301,9 @@ "dev": true }, "async": { - "version": "2.6.3", - "resolved": "https://registry.npmjs.org/async/-/async-2.6.3.tgz", - "integrity": "sha512-zflvls11DCy+dQWzTW2dzuilv8Z5X/pjfmZOWba6TNIVDm+2UDaJmXSOXlasHKfNBs8oo3M0aT50fDEWfKZjXg==", - "requires": { - "lodash": "^4.17.14" - } + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/async/-/async-3.2.2.tgz", + "integrity": "sha512-H0E+qZaDEfx/FY4t7iLRv1W2fFI6+pyCeTw1uN20AQPiwqwM6ojPxHxdLv4z8hi2DtnW9BOckSspLucW7pIE5g==" }, "async-each": { "version": "1.0.3", @@ -22342,14 +22343,16 @@ } }, "nwl-components": { - "version": "0.0.12", - "resolved": "https://registry.npmjs.org/nwl-components/-/nwl-components-0.0.12.tgz", - "integrity": "sha512-Qbce+RwJB1N0FBx4JYfNaTMcBA0sz00bpv2hJa6/4/FW+HP6RYo05Jmg/M6e52Kc9J1yIP/qHMtM25GxEasFeg==", + "version": "0.0.17", + "resolved": "https://registry.npmjs.org/nwl-components/-/nwl-components-0.0.17.tgz", + "integrity": "sha512-fScHc9BrWXFmE653AYwZIY0sTvcwmMf1qaUtGkd/mnXSxX9FybW5c9krA97/pIZg7dii20nmLTSzbV7Ol6FJ4A==", "requires": { + "dompurify": "^2.3.6", "jdenticon": "^3.1.1", "lodash-es": "^4.17.21", "md5": "^2.3.0", - "nanoid": "^3.3.1" + "nanoid": "^3.3.1", + "splitpanes": "^3.1.0" }, "dependencies": { "canvas-renderer": { @@ -24480,6 +24483,11 @@ "extend-shallow": "^3.0.0" } }, + "splitpanes": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/splitpanes/-/splitpanes-3.1.1.tgz", + "integrity": "sha512-VUkxDJfIGSvTM/fm/+OSrx8ha9URwE/9B8FPvfzoBuAxVELIHBWpsfnJXIXv77zVwuex//QQL4kTU9SDBPeHjA==" + }, "sprintf-js": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", diff --git a/package.json b/package.json index 45f5c15d..460e9ca3 100644 --- a/package.json +++ b/package.json @@ -24,7 +24,7 @@ "prepare": "husky install" }, "dependencies": { - "async": "^2.6.3", + "async": "^3.2.2", "async-lock": "^1.3.0", "body-parser": "~1.18.2", "chai-as-promised": "^7.1.1", From 1ca53059b617988a4a00f49baf023435df82c54d Mon Sep 17 00:00:00 2001 From: Nicolas Date: Mon, 18 Apr 2022 20:20:32 +0200 Subject: [PATCH 3/5] use api to insert test projects --- .vscode/launch.json | 3 +- app.js | 61 +++++++++-------- package-lock.json | 32 ++++++--- package.json | 7 +- test/integration/permissions.js | 42 ++++++------ test/runner.js | 4 +- test/utils.js | 117 +++++++++++++++++--------------- 7 files changed, 144 insertions(+), 122 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index d83598ce..7e96265d 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -32,7 +32,8 @@ "--no-timeout" ], "env": { - "LOCALSIGNIN": "true" + "LOCALSIGNIN": "true", + "MONGODB": "127.0.0.1:27017/brainbox_test" } } ] diff --git a/app.js b/app.js index 9aa11f09..49ce845d 100644 --- a/app.js +++ b/app.js @@ -11,7 +11,7 @@ var compression = require('compression'); const path = require('path'); const favicon = require('serve-favicon'); const logger = require('morgan'); -const tracer = require('tracer').console({format: '[{{file}}:{{line}}] {{message}}'}); +const tracer = require('tracer').console({ format: '[{{file}}:{{line}}] {{message}}' }); const cookieParser = require('cookie-parser'); const bodyParser = require('body-parser'); const mustacheExpress = require('mustache-express'); @@ -31,7 +31,7 @@ const DOCKER_DEVELOP = process.env.DEVELOP; if (DOCKER_DB) { MONGO_DB = DOCKER_DB.replace('tcp://', '') + '/brainbox'; } else { - MONGO_DB = 'localhost:27017/brainbox'; //process.env.MONGODB; + MONGO_DB = process.env.MONGODB || 'localhost:27017/brainbox'; } /** @todo Handle the case when MongoDB is not installed */ @@ -62,8 +62,8 @@ if (DOCKER_DEVELOP === '1') { const start = async function () { const app = express(); - app.use(bodyParser.json({limit: '50mb'})); - app.use(bodyParser.urlencoded({limit: '50mb', extended: true})); + app.use(bodyParser.json({ limit: '50mb' })); + app.use(bodyParser.urlencoded({ limit: '50mb', extended: true })); /* Use the NeuroWebLab (NWL) module for authentication. @@ -85,7 +85,7 @@ const start = async function () { //======================================================================================== // Allow CORS //======================================================================================== - app.use(function(req, res, next) { + app.use(function (req, res, next) { res.header('Access-Control-Allow-Origin', '*'); res.header('Access-Control-Allow-Headers', 'Origin, X-Requested-With, Content-Type, Accept'); next(); @@ -136,7 +136,7 @@ const start = async function () { key: await fs.promises.readFile(Config.ssl_key), cert: await fs.promises.readFile(Config.ssl_cert) }; - if(Config.ssl_chain) { + if (Config.ssl_chain) { options.ca = await fs.promises.readFile(Config.ssl_chain); } atlasmakerServer.server = https.createServer(options, app); @@ -163,33 +163,34 @@ const start = async function () { //======================================================================================== // catch 404 and forward to error handler app.use(function (req, res, next) { - var err = new Error('Not Found'); - err.status = 404; - next(err); + console.log('Not found URL requested: ' + req.url); + next(); }); - // development error handler - // will print stacktrace - if (app.get('env') === 'development') { - app.use(function (err, req, res) { - res.status(err.status || 500); - res.render('error', { - message: err.message, - error: err - }); - }); - } - // production error handler - // no stacktraces leaked to user - app.use(function (err, req, res) { - res.status(err.status || 500); - res.render('error', { - message: err.message, - error: {} - }); - }); + // the following middlewares will not be used by express as we need to pass 4 arguments to + // the use method to handle express errors: https://expressjs.com/fr/guide/error-handling.html + // // development error handler + // // will print stacktrace + // if (app.get('env') === 'development') { + // app.use(function (err, req, res) { + // res.status(err.status || 500); + // res.render('error', { + // message: err.message, + // error: err + // }); + // }); + // } + // // production error handler + // // no stacktraces leaked to user + // app.use(function (err, req, res) { + // res.status(err.status || 500); + // res.render('error', { + // message: err.message, + // error: {} + // }); + // }); return { app, server, atlasmakerServer }; }; -module.exports = {start}; +module.exports = { start }; diff --git a/package-lock.json b/package-lock.json index a076412b..8e289a1b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -41,7 +41,7 @@ "multer": "^1.4.2", "mustache-express": "^1.3.0", "neuroweblab": "github:neuroanatomy/neuroweblab", - "nwl-components": "^0.0.12", + "nwl-components": "^0.0.17", "pako": "^1.0.11", "passport": "^0.4.1", "passport-github": "^1.1.0", @@ -9445,14 +9445,16 @@ } }, "node_modules/nwl-components": { - "version": "0.0.12", - "resolved": "https://registry.npmjs.org/nwl-components/-/nwl-components-0.0.12.tgz", - "integrity": "sha512-Qbce+RwJB1N0FBx4JYfNaTMcBA0sz00bpv2hJa6/4/FW+HP6RYo05Jmg/M6e52Kc9J1yIP/qHMtM25GxEasFeg==", + "version": "0.0.17", + "resolved": "https://registry.npmjs.org/nwl-components/-/nwl-components-0.0.17.tgz", + "integrity": "sha512-fScHc9BrWXFmE653AYwZIY0sTvcwmMf1qaUtGkd/mnXSxX9FybW5c9krA97/pIZg7dii20nmLTSzbV7Ol6FJ4A==", "dependencies": { + "dompurify": "^2.3.6", "jdenticon": "^3.1.1", "lodash-es": "^4.17.21", "md5": "^2.3.0", - "nanoid": "^3.3.1" + "nanoid": "^3.3.1", + "splitpanes": "^3.1.0" }, "peerDependencies": { "vue": "^3.2.25" @@ -12123,6 +12125,11 @@ "node": ">=0.10.0" } }, + "node_modules/splitpanes": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/splitpanes/-/splitpanes-3.1.1.tgz", + "integrity": "sha512-VUkxDJfIGSvTM/fm/+OSrx8ha9URwE/9B8FPvfzoBuAxVELIHBWpsfnJXIXv77zVwuex//QQL4kTU9SDBPeHjA==" + }, "node_modules/sprintf-js": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", @@ -22342,14 +22349,16 @@ } }, "nwl-components": { - "version": "0.0.12", - "resolved": "https://registry.npmjs.org/nwl-components/-/nwl-components-0.0.12.tgz", - "integrity": "sha512-Qbce+RwJB1N0FBx4JYfNaTMcBA0sz00bpv2hJa6/4/FW+HP6RYo05Jmg/M6e52Kc9J1yIP/qHMtM25GxEasFeg==", + "version": "0.0.17", + "resolved": "https://registry.npmjs.org/nwl-components/-/nwl-components-0.0.17.tgz", + "integrity": "sha512-fScHc9BrWXFmE653AYwZIY0sTvcwmMf1qaUtGkd/mnXSxX9FybW5c9krA97/pIZg7dii20nmLTSzbV7Ol6FJ4A==", "requires": { + "dompurify": "^2.3.6", "jdenticon": "^3.1.1", "lodash-es": "^4.17.21", "md5": "^2.3.0", - "nanoid": "^3.3.1" + "nanoid": "^3.3.1", + "splitpanes": "^3.1.0" }, "dependencies": { "canvas-renderer": { @@ -24480,6 +24489,11 @@ "extend-shallow": "^3.0.0" } }, + "splitpanes": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/splitpanes/-/splitpanes-3.1.1.tgz", + "integrity": "sha512-VUkxDJfIGSvTM/fm/+OSrx8ha9URwE/9B8FPvfzoBuAxVELIHBWpsfnJXIXv77zVwuex//QQL4kTU9SDBPeHjA==" + }, "sprintf-js": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", diff --git a/package.json b/package.json index 45f5c15d..973965dd 100644 --- a/package.json +++ b/package.json @@ -15,12 +15,11 @@ "dev-brainbox": "webpack --mode production --config webpack.brainbox.config.js && cp view/brainbox/dist/brainbox.js public/lib/brainbox.js", "dev-pages": "webpack --mode development --config webpack.pages.config.js && cp view/brainbox/dist/*-page.js public/js/", "dev": "npm run dev-atlasmaker && npm run dev-atlasmaker-tools && npm run dev-brainbox && npm run dev-pages", - "test-unit": "nyc --report-dir test/unit/coverage mocha --timeout 5000 ./test/runner.js test/unit/*.js", - "test-integration": "cross-env LOCALSIGNIN=true nyc --report-dir test/integration/coverage mocha --timeout 5000 ./test/runner.js test/integration/*.js", - "mocha-test": "cross-env LOCALSIGNIN=true nyc mocha --timeout 5000 ./test/runner.js test/unit/*.js test/integration/*.js", + "test-unit": "cross-env MONGODB='127.0.0.1:27017/brainbox_test' nyc --report-dir test/unit/coverage mocha --timeout 5000 ./test/runner.js test/unit/*.js", + "test-integration": "cross-env MONGODB='127.0.0.1:27017/brainbox_test' LOCALSIGNIN=true nyc --report-dir test/integration/coverage mocha --timeout 5000 ./test/runner.js test/integration/*.js", + "mocha-test": "cross-env MONGODB='127.0.0.1:27017/brainbox_test' LOCALSIGNIN=true nyc mocha --timeout 5000 ./test/runner.js test/unit/*.js test/integration/*.js", "lint": "eslint .", "test": "docker exec brainbox_web_1 /bin/bash -c 'cd /brainbox && npm run mocha-test'", - "coverage": "nyc mocha --exit ./test/runner.js test/unit/*.js test/integration/*.js", "prepare": "husky install" }, "dependencies": { diff --git a/test/integration/permissions.js b/test/integration/permissions.js index 820574ec..3dfd00de 100644 --- a/test/integration/permissions.js +++ b/test/integration/permissions.js @@ -112,50 +112,50 @@ describe('TESTING PERMISSIONS', function () { describe('Test specific edit / add / remove permissions of logged users', function() { const projects = {}; - const setupProjectWithAccess = (access, name) => { + const setupProjectWithAccess = function (access, name) { const project = U.createProjectWithPermission(name, access); projects[project.name] = project; - U.insertProject(project); + + return U.insertProject(project); }; - before(function () { - ['none', 'view'].forEach((access) => { - setupProjectWithAccess( + before(async function () { + await Promise.all(['none', 'view'].map((acs) => (async function (access) { + await setupProjectWithAccess( { collaborators: access, files: 'edit' }, `collaborators${access}filesedit` ); - setupProjectWithAccess( + await setupProjectWithAccess( { collaborators: 'edit', files: access }, `collaboratorseditfiles${access}` ); - }); - ['edit', 'add', 'remove'].forEach((access) => { - setupProjectWithAccess( + }(acs)))); + await Promise.all(['edit', 'add', 'remove'].map((acs) => (async function (access) { + await setupProjectWithAccess( { collaborators: access, files: 'none' }, `collaborators${access}filesnone` ); - setupProjectWithAccess( + await setupProjectWithAccess( { collaborators: access, files: 'edit' }, `collaborators${access}filesedit` ); - setupProjectWithAccess( + await setupProjectWithAccess( { collaborators: 'edit', files: access }, `collaboratorseditfiles${access}` ); - }); - ['none', 'view', 'add', 'edit', 'remove'].forEach((access) => { - setupProjectWithAccess( + }(acs)))); + await Promise.all(['none', 'view', 'add', 'edit', 'remove'].map((acs) => (async function (access) { + await setupProjectWithAccess( { collaborators: 'edit', files: 'edit', annotations: access }, `collaboratorseditfileseditannotations${access}` ); - }); - + }(acs)))); }); - after(function() { - Object.keys(projects).forEach((shortname) => { - U.removeProject(shortname); - }); + after(async function() { + await Promise.all(Object.keys(projects).map((shortname) => + U.removeProject(shortname) + )); }); @@ -477,7 +477,7 @@ describe('TESTING PERMISSIONS', function () { it(`Checks that collaborators cannot remove project files if set to add (${userStatus})`, async function() { let project = U.createProjectWithPermission('permissionTest', { files: 'add' }); projects.permissionTest = project; - U.insertProject(project); + await U.insertProject(project); const initialProjectState = _.cloneDeep(project); project = _.cloneDeep(project); diff --git a/test/runner.js b/test/runner.js index 705d9a60..aae55a2c 100644 --- a/test/runner.js +++ b/test/runner.js @@ -6,17 +6,17 @@ before(async function () { this.timeout(U.longTimeout); await U.initResources(); await U.insertUser(U.userFoo); + await U.insertTestTokenForUser('foo'); await U.insertProject(U.privateProjectTest); await U.insertProject(U.projectTest); - await U.insertTestTokenForUser('foo'); // await browser.init(); }); after(async function () { - await U.removeUser(U.userFoo.nickname); await U.removeProject(U.projectTest.shortname); await U.removeProject(U.privateProjectTest.shortname); await U.removeTestTokenForUser('foo'); + await U.removeUser(U.userFoo.nickname); // await browser.close(); await U.closeResources(); }); diff --git a/test/utils.js b/test/utils.js index 8e54ad90..fa4026c4 100644 --- a/test/utils.js +++ b/test/utils.js @@ -2,13 +2,17 @@ var fs = require('fs'); const path = require('path'); const rimraf = require('rimraf'); -const {PNG} = require('pngjs'); +const { PNG } = require('pngjs'); var jpeg = require('jpeg-js'); const pixelmatch = require('pixelmatch'); const { exec } = require('child_process'); // const { constants } = require('buffer'); const brainboxApp = require('../app'); +const chai = require('chai'); +const chaiHttp = require('chai-http'); +chai.use(chaiHttp); + process.env.NODE_TLS_REJECT_UNAUTHORIZED = '0'; const serverURL = 'http://127.0.0.1:3001'; @@ -77,11 +81,11 @@ const projectTest = { }, files: { list: [ - serverURL + '/test_data/bert_brain.nii.gz', - 'https://zenodo.org/record/44855/files/MRI-n4.nii.gz', - 'http://files.figshare.com/2284784/MRI_n4.nii.gz', - 'https://dl.dropbox.com/s/cny5b3so267bv94/p32-f18-uchar.nii.gz', - 'https://s3.amazonaws.com/fcp-indi/data/Projects/ABIDE_Initiative/Outputs/freesurfer/5.1/Caltech_0051456/mri/T1.mgz' + {source: serverURL + '/test_data/bert_brain.nii.gz', name: 'bert_brain'}, + {source: 'https://zenodo.org/record/44855/files/MRI-n4.nii.gz', name: 'MRI-n4'}, + {source: 'http://files.figshare.com/2284784/MRI_n4.nii.gz', name: 'MRI_n4'}, + {source: 'https://dl.dropbox.com/s/cny5b3so267bv94/p32-f18-uchar.nii.gz', name: 'p32-f18-uchar'}, + {source: 'https://s3.amazonaws.com/fcp-indi/data/Projects/ABIDE_Initiative/Outputs/freesurfer/5.1/Caltech_0051456/mri/T1.mgz', name: 'T1'} ] }, annotations: { @@ -170,7 +174,7 @@ const currentDirectory = function () { }; const queryUser = function (nickname) { - return db.get('user').findOne({nickname}); + return db.get('user').findOne({ nickname }); }; const insertUser = function (user) { @@ -178,22 +182,23 @@ const insertUser = function (user) { }; const removeUser = function (nickname) { - return db.get('user').remove({nickname}); + return db.get('user').remove({ nickname }); }; const insertProject = function (project) { - return db.get('project').insert(project); + return chai.request(serverURL).post(`/project/json/${project.shortname}?token=${testToken}foo`) + .send({data: project}); }; const removeProject = function (shortname) { - return db.get('project').remove({shortname}); + return db.get('project').remove({ shortname }); }; const queryProject = function (shortname) { return db.get('project').findOne({ shortname, backup: { $exists: 0 } }); }; -const insertTestTokenForUser =async function (nickname) { +const insertTestTokenForUser = async function (nickname) { const now = new Date(); const obj = { token: testToken + nickname, @@ -206,19 +211,19 @@ const insertTestTokenForUser =async function (nickname) { return res; }; -const removeTestTokenForUser =async function (nickname) { - await db.get('log').remove({token: testToken + nickname}); +const removeTestTokenForUser = async function (nickname) { + await db.get('log').remove({ token: testToken + nickname }); }; -const delay=async function (delayTimeout) { +const delay = async function (delayTimeout) { await new Promise((resolve) => { setTimeout(resolve, delayTimeout); }); }; -const removeMRI=async function ({dirPath, srcURL}) { +const removeMRI = async function ({ dirPath, srcURL }) { rimraf.sync(dirPath, {}, (err) => console.log(new Error(err))); - const res = await db.get('mri').remove({source: srcURL}); + const res = await db.get('mri').remove({ source: srcURL }); return res; }; @@ -227,14 +232,14 @@ const compareImages = async function (pathImg1, pathImg2) { const data1 = await fs.promises.readFile(pathImg1); const data2 = await fs.promises.readFile(pathImg2); let img1, img2; - if(pathImg1.split('.').pop() === 'png') { + if (pathImg1.split('.').pop() === 'png') { img1 = PNG.sync.read(data1); - } else if(pathImg1.split('.').pop() === 'jpg') { + } else if (pathImg1.split('.').pop() === 'jpg') { img1 = jpeg.decode(data1); } - if(pathImg2.split('.').pop() === 'png') { + if (pathImg2.split('.').pop() === 'png') { img2 = PNG.sync.read(data2); - } else if(pathImg2.split('.').pop() === 'jpg') { + } else if (pathImg2.split('.').pop() === 'jpg') { img2 = jpeg.decode(data2); } const pixdiff = pixelmatch(img1.data, img2.data, null, img1.width, img1.height); @@ -243,7 +248,7 @@ const compareImages = async function (pathImg1, pathImg2) { }; // eslint-disable-next-line max-statements -const waitUntilHTMLRendered=async function (page, timeout = 30000) { +const waitUntilHTMLRendered = async function (page, timeout = 30000) { const checkDurationMsecs = 1000; const maxChecks = timeout / checkDurationMsecs; let lastHTMLSize = 0; @@ -251,7 +256,7 @@ const waitUntilHTMLRendered=async function (page, timeout = 30000) { let countStableSizeIterations = 0; const minStableSizeIterations = 3; - while(checkCounts++ <= maxChecks) { + while (checkCounts++ <= maxChecks) { // eslint-disable-next-line no-await-in-loop const html = await page.content(); const currentHTMLSize = html.length; @@ -260,13 +265,13 @@ const waitUntilHTMLRendered=async function (page, timeout = 30000) { // console.log('last: ', lastHTMLSize, ' <> curr: ', currentHTMLSize, " body html size: ", bodyHTMLSize); - if(lastHTMLSize !== 0 && currentHTMLSize === lastHTMLSize) { + if (lastHTMLSize !== 0 && currentHTMLSize === lastHTMLSize) { countStableSizeIterations++; } else { countStableSizeIterations = 0; //reset the counter } - if(countStableSizeIterations >= minStableSizeIterations) { + if (countStableSizeIterations >= minStableSizeIterations) { // console.log("Page rendered fully.."); break; } @@ -276,20 +281,20 @@ const waitUntilHTMLRendered=async function (page, timeout = 30000) { } }; -const comparePageScreenshots=async function (testPage, url, filename) { +const comparePageScreenshots = async function (testPage, url, filename) { const newPath = './test/screenshots/' + filename; const refPath = './test/data/reference-screenshots/' + filename; - await testPage.goto(url, {waitUntil: 'networkidle2', timeout: 90000}); + await testPage.goto(url, { waitUntil: 'networkidle2', timeout: 90000 }); await waitUntilHTMLRendered(testPage); fs.promises.mkdir(path.dirname(newPath), { recursive: true }); - await testPage.screenshot({path:'./test/screenshots/' + filename}); + await testPage.screenshot({ path: './test/screenshots/' + filename }); const pixdiff = await compareImages(newPath, refPath); return pixdiff; }; -const waitForDOMReady =async function (testPage, url) { - await testPage.goto(url, {waitUntil: 'networkidle2', timeout: 90000}); +const waitForDOMReady = async function (testPage, url) { + await testPage.goto(url, { waitUntil: 'networkidle2', timeout: 90000 }); await waitUntilHTMLRendered(testPage); }; @@ -323,7 +328,7 @@ const testingCredentials = { password: 'baz' }; -const createProjectWithPermission = function(name, accessProp) { +const createProjectWithPermission = function (name, accessProp) { const access = Object.assign({}, { collaborators: 'view', annotations: 'none', @@ -337,36 +342,38 @@ const createProjectWithPermission = function(name, accessProp) { brainboxURL: '/project/' + name, created: (new Date()).toJSON(), owner: 'foo', - collaborators: { list: [ - { - userID: 'anyone', - access: { - collaborators: 'none', - annotations: 'none', - files: 'view' - }, - username: 'anyone', - nickname: 'anyone', - name: 'Any User' - }, - { - userID: 'bar', - access: { - collaborators: 'view', - annotations: 'view', - files: 'view' + collaborators: { + list: [ + { + userID: 'anyone', + access: { + collaborators: 'none', + annotations: 'none', + files: 'view' + }, + username: 'anyone', + nickname: 'anyone', + name: 'Any User' }, - username: 'foo', - name: 'Foo' - } - ] }, + { + userID: 'bar', + access: { + collaborators: 'view', + annotations: 'view', + files: 'view' + }, + username: 'foo', + name: 'Foo' + } + ] + }, files: { - list: [{source: 'https://zenodo.org/record/44855/files/MRI-n4.nii.gz', name: 'MRI-n4.nii.gz'}] + list: [{ source: 'https://zenodo.org/record/44855/files/MRI-n4.nii.gz', name: 'MRI-n4.nii.gz' }] }, annotations: { list: [ - {'type':'volume', 'name':'Test', 'values':'axolotl_labels.json', 'display':'true'}, - {'type':'volume', 'name':'Test2', 'values':'axolotl_labels.json', 'display':'true'} + { 'type': 'volume', 'name': 'Test', 'values': 'axolotl_labels.json', 'display': 'true' }, + { 'type': 'volume', 'name': 'Test2', 'values': 'axolotl_labels.json', 'display': 'true' } ] } }; From 91f4df919bbb1277cfbd6d65b1aa9c01b5500301 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Mon, 18 Apr 2022 20:24:40 +0200 Subject: [PATCH 4/5] fix promise not awaited when inserting mri names --- controller/project/project.controller.js | 153 ++++++++++++----------- 1 file changed, 80 insertions(+), 73 deletions(-) diff --git a/controller/project/project.controller.js b/controller/project/project.controller.js index b1559be5..5fa552dd 100644 --- a/controller/project/project.controller.js +++ b/controller/project/project.controller.js @@ -10,7 +10,7 @@ const _ = require('lodash'); const createDOMPurify = require('dompurify'); const { JSDOM } = require('jsdom'); const { ForbiddenAccessError } = require('../../errors.js'); -const {window} = (new JSDOM('', { +const { window } = (new JSDOM('', { features: { FetchExternalResources: false, // disables resource loading over HTTP / filesystem ProcessExternalResources: false // do not execute JS within script blocks @@ -59,11 +59,17 @@ const isProjectObject = async function (req, res, object) { // files if (object.files) { for (const file of object.files.list) { + if (typeof file.source !== 'string') { + throw (new Error('File source not specified')); + } + if (typeof file.name !== 'string') { + throw (new Error('File name not specified')); + } if (!validatorNPM.isURL(file.source)) { - throw(new Error('Invalid file URL' )); + throw (new Error(`Invalid file URL: ${file.source}`)); } if (!validatorNPM.isWhitelisted(file.name, allowed)) { - throw(new Error(`Invalid file name "${file.name}"`)); + throw (new Error(`Invalid file name "${file.name}"`)); } } } @@ -71,27 +77,27 @@ const isProjectObject = async function (req, res, object) { // description if (object.description && !validatorNPM.isWhitelisted(object.description, allowed)) { - throw(new Error('Invalid project description')); + throw (new Error('Invalid project description')); // delete object.description; } console.log('> description ok'); // name if (object.name && !validatorNPM.isWhitelisted(object.name, allowed)) { - throw(new Error('Invalid name')); + throw (new Error('Invalid name')); //delete object.name; } console.log('> name ok'); // check that owner and shortname are present if (!object.owner || !object.shortname) { - throw(new Error('Invalid owner or project shortname, not present')); + throw (new Error('Invalid owner or project shortname, not present')); } console.log('> owner and project shortname present'); // check that shortname is alphanumeric if (!validatorNPM.isWhitelisted(object.owner, allowedAlphanumericHyphen) || !validatorNPM.isWhitelisted(object.shortname, allowedAlphanumericHyphen)) { - throw(new Error('Invalid owner or project shortname, not alphanumeric')); + throw (new Error('Invalid owner or project shortname, not alphanumeric')); } console.log('> owner and project shortname valid'); @@ -106,7 +112,7 @@ const isProjectObject = async function (req, res, object) { } } if (flag === false) { - throw(new Error('User \'anyone\' is not present')); + throw (new Error('User \'anyone\' is not present')); } // check that collaborator's access values are valid @@ -129,7 +135,7 @@ const isProjectObject = async function (req, res, object) { } } if (flag === false) { - throw(new Error('Access values are invalid')); + throw (new Error('Access values are invalid')); } console.log('> Access values ok'); @@ -142,7 +148,7 @@ const isProjectObject = async function (req, res, object) { } } if (flag === false) { - throw(new Error('Annotations must contain at least 1 volume-type entry')); + throw (new Error('Annotations must contain at least 1 volume-type entry')); } @@ -163,7 +169,7 @@ const isProjectObject = async function (req, res, object) { } } if (notFound === true) { - throw(new Error('Users are invalid, one or more do not exist')); + throw (new Error('Users are invalid, one or more do not exist')); } // All checks are successful, resolve the promisse @@ -316,8 +322,8 @@ const apiProjectFiles = async function (req, res) { try { const list = await dataSlices.getProjectFilesSlice(req, projShortname, start, length, namesFlag); res.send(list); - } catch(err) { - if(err instanceof ForbiddenAccessError) { + } catch (err) { + if (err instanceof ForbiddenAccessError) { res.status(403).send({ error: err.message }); } else { res.status(500).send({ error: err.message }); @@ -454,68 +460,67 @@ const newProject = function (req, res) { } }; +// eslint-disable-next-line max-statements const insertMRInames = function (req, res, list) { // insert MRI names, but only if they don't exist - for (var i = 0; i < list.length; i++) { - var {name} = list[i]; - var {source} = list[i]; - var filename = url.parse(source).pathname.split('/').pop(); + return Promise.all(list.map((el) => (async function (file) { + const { name, source } = file; + const filename = url.parse(source).pathname.split('/').pop(); // it there's no name, continue to the next mri - if (!name) { continue; } + if (!name) { return; } // check if the mri entry already exists - (async function (na, so, fi) { // without a closure, only the last name in the list is used and repeated - let mri = await req.db.get('mri').findOne({ source: so, backup: { $exists: 0 } }); - var hash = crypto.createHash('md5').update(so) - .digest('hex'); - - // if mri exists, and has no name, insert the name - if (!mri) { - mri = { - filename: fi, - source: so, - url: '/data/' + hash + '/', - included: (new Date()).toJSON(), - owner: req.user.username, - mri: { - brain: fi, - atlas: [ - { - owner: req.user.username, - created: (new Date()).toJSON(), - modified: (new Date()).toJSON(), - type: 'volume', - filename: 'Atlas.nii.gz', - labels: 'foreground.json' - } - ] - } - }; - } else { - delete mri._id; - } - mri.modified = (new Date()).toJSON(); - mri.modifiedBy = req.user.username; + // without a closure, only the last name in the list is used and repeated + let mri = await req.db.get('mri').findOne({ source, backup: { $exists: 0 } }); + var hash = crypto.createHash('md5').update(source) + .digest('hex'); + + // if mri exists, and has no name, insert the name + if (!mri) { + mri = { + filename, + source, + url: '/data/' + hash + '/', + included: (new Date()).toJSON(), + owner: req.user.username, + mri: { + brain: filename, + atlas: [ + { + owner: req.user.username, + created: (new Date()).toJSON(), + modified: (new Date()).toJSON(), + type: 'volume', + filename: 'Atlas.nii.gz', + labels: 'foreground.json' + } + ] + } + }; + } else { + delete mri._id; + } + mri.modified = (new Date()).toJSON(); + mri.modifiedBy = req.user.username; - /* Use this if you want imported names to overwrite existing ones */ - mri.name = na; + /* Use this if you want imported names to overwrite existing ones */ + mri.name = name; - /* Use this if you want imported names to be used only if no previous name exists */ - /* - if(!mri.name) { - mri.name=na; - } - */ + /* Use this if you want imported names to be used only if no previous name exists */ + /* + if(!mri.name) { + mri.name=name; + } + */ - // sanitise json - mri = JSON.parse(DOMPurify.sanitize(JSON.stringify(mri))); // sanitize works on strings, not objects + // sanitise json + mri = JSON.parse(DOMPurify.sanitize(JSON.stringify(mri))); // sanitize works on strings, not objects - // update and insert - await req.db.get('mri').update({ source: mri.source }, { $set: { backup: true } }, { multi: true }); - await req.db.get('mri').insert(mri); - }(name, source, filename)); - } + // update and insert + await req.db.get('mri').update({ source: mri.source }, { $set: { backup: true } }, { multi: true }); + await req.db.get('mri').insert(mri); + }(el)))); }; /** @@ -546,20 +551,21 @@ const postProject = async function (req, res) { } catch (err) { console.log('ERROR'); console.log({ clean, obj }); - res.status(500).send({error: err.message}); + res.status(500).send({ error: err.message }); return; } var k; // eslint-disable-next-line max-statements - await lock.acquire(['project', 'mri'], async function() { + await lock.acquire(['project', 'mri'], async function () { + console.log('enter lock block'); let object; try { object = await isProjectObject(req, res, obj); - } catch(err) { + } catch (err) { console.error(err.message); - res.status(500).send({error: err.message}); + res.status(500).send({ error: err.message }); return; } @@ -570,7 +576,7 @@ const postProject = async function (req, res) { }); // update/insert project if (oldProject) { - // project exists, save update + // project exists, save update if (!AccessControlService.hasFilesAccess(AccessLevel.EDIT, oldProject, loggedUser)) { console.log('User does not have edit rights'); res.status(403).json({ error: 'error', message: 'User does not have edit rights' }); @@ -611,13 +617,13 @@ const postProject = async function (req, res) { console.log('success: true'); let successMessage = 'Project settings updated.'; - if(ignoredChanges.length > 0) { + if (ignoredChanges.length > 0) { successMessage += ` Some changes (on ${ignoredChanges.join(', ')}) were ignored due to a lack of permissions.`; } res.json({ success: true, message: successMessage }); } else { - // new project, insert + // new project, insert console.log('inserting...'); console.log('insert mri names'); await insertMRInames(req, res, obj.files.list); @@ -635,6 +641,7 @@ const postProject = async function (req, res) { console.log('success: true'); res.json({ success: true, message: 'New project inserted' }); } + console.log('leave lock block'); }); }; @@ -661,7 +668,7 @@ const deleteProject = async function (req, res) { try { // eslint-disable-next-line max-statements - await lock.acquire(['project', 'mri'], async function() { + await lock.acquire(['project', 'mri'], async function () { shortname = req.params.projectName; const oldProject = await req.db.get('project').findOne({ shortname: shortname, backup: { $exists: 0 } }); From be304d6ce22b4e7543bee06c7066540cd44cb68e Mon Sep 17 00:00:00 2001 From: Nicolas Date: Mon, 18 Apr 2022 20:26:34 +0200 Subject: [PATCH 5/5] fix backup not set to true on old mri entry after download --- controller/mri/mri.controller.js | 34 ++++++++++-------- test/integration/project.controller.test.js | 38 ++++++++++++--------- 2 files changed, 42 insertions(+), 30 deletions(-) diff --git a/controller/mri/mri.controller.js b/controller/mri/mri.controller.js index 76dff6d0..2970f4cc 100644 --- a/controller/mri/mri.controller.js +++ b/controller/mri/mri.controller.js @@ -11,6 +11,8 @@ const dataSlices = require('../dataSlices/dataSlices.js'); const { AccessType, AccessLevel } = require('neuroweblab'); const BrainboxAccessControlService = require('../../services/BrainboxAccessControlService'); const _ = require('lodash'); +const AsyncLock = require('async-lock'); +const lock = new AsyncLock(); const downloadQueue = {}; let atlasmakerServer; @@ -92,20 +94,20 @@ const validatorPost = function (req, res, next) { --------------------- */ // @todo Change this function callback into a promise // eslint-disable-next-line max-statements -const downloadMRI = async function(myurl, req) { +const downloadMRI = async function (myurl, req) { console.log('downloadMRI'); const hash = crypto .createHash('md5') .update(myurl) .digest('hex'); - const mridb = await req.db.get('mri').findOne({source: myurl, backup: {$exists: 0}}); + const mridb = await req.db.get('mri').findOne({ source: myurl, backup: { $exists: 0 } }); console.log('mridb:', mridb); let filename; if (!mridb || !mridb.filename) { filename = sanitize(url.parse(myurl).pathname.split('/').pop()); } else { - ({filename} = mridb); + ({ filename } = mridb); } let dest = req.dirname + '/public/data/' + hash + '/' + filename; console.log(' source:', myurl); @@ -122,7 +124,7 @@ const downloadMRI = async function(myurl, req) { let cur = 0; return new Promise(function (resolve, reject) { - request({uri: myurl, followAllRedirects: true, rejectUnauthorized : false}) + request({ uri: myurl, followAllRedirects: true, rejectUnauthorized: false }) .on('error', (err) => { console.log('ERROR in downloadMRI', err); reject(err); @@ -162,7 +164,7 @@ const downloadMRI = async function(myurl, req) { .then((mri) => { // Create json file for new dataset let ip = ''; - if(typeof req.headers['x-forwarded-for'] !== 'undefined') { + if (typeof req.headers['x-forwarded-for'] !== 'undefined') { ip = req.headers['x-forwarded-for']; } else if (req.connection.remoteAddress !== 'undefined') { ip = req.connection.remoteAddress; @@ -173,8 +175,8 @@ const downloadMRI = async function(myurl, req) { } let username; - if(req.isAuthenticated()) { - ({username} = req.user); + if (req.isAuthenticated()) { + ({ username } = req.user); } else { username = ip; } @@ -278,9 +280,10 @@ const mri = async function (req, res) { // also query projects that set this MRI as a source projects.push(...await req.db.get('project').find({ $or: [ - { 'files.list': {$eq: myurl }}, - { 'files.list.source': {$eq: myurl }} - ]} + { 'files.list': { $eq: myurl } }, + { 'files.list.source': { $eq: myurl } } + ] + } )); // set access to volume annotations @@ -324,7 +327,7 @@ const apiMriPost = async function (req, res) { try { // eslint-disable-next-line no-new new URL(myurl); - } catch(err) { + } catch (err) { res.send('Invalid URL!'); return; @@ -412,7 +415,10 @@ const apiMriPost = async function (req, res) { console.log('Download succeeded. Insert in DB, remove from queue'); obj.success = true; - return req.db.get('mri').insert(obj); + return lock.acquire('mri', async function () { + await req.db.get('mri').update({ source: myurl }, { $set: { backup: true } }, { multi: true }); + await req.db.get('mri').insert(obj); + }); }) .then(() => { // downloadQueue[myurl] = obj; @@ -567,7 +573,7 @@ const apiMriGet = async function (req, res) { }; // eslint-disable-next-line func-style -const reset = async function reset(req, res) { +const reset = async function reset (req, res) { const myurl = req.query.url; const hash = crypto.createHash('md5').update(myurl) .digest('hex'); @@ -582,7 +588,7 @@ const reset = async function reset(req, res) { }); console.log(mridb); let filename; - if (mridb) { ({filename} = mridb); } + if (mridb) { ({ filename } = mridb); } const mrires = await atlasmakerServer.getBrainAtPath('/data/' + hash + '/' + filename) .catch((err) => { console.log('ERROR:', err); diff --git a/test/integration/project.controller.test.js b/test/integration/project.controller.test.js index b3e36e02..01d2248c 100644 --- a/test/integration/project.controller.test.js +++ b/test/integration/project.controller.test.js @@ -1,7 +1,7 @@ 'use strict'; const chai = require('chai'); -var {assert} = chai; +var { assert } = chai; const chaiHttp = require('chai-http'); chai.use(chaiHttp); const U = require('../utils.js'); @@ -10,17 +10,17 @@ describe('TESTING THE /project ROUTE', function () { // eslint-disable-next-line no-invalid-this this.timeout(U.longTimeout); - before( async function () { + before(async function () { // add one MRI let shouldContinue = true; - while(shouldContinue) { + while (shouldContinue) { // eslint-disable-next-line no-await-in-loop const res = await chai.request(U.serverURL).post('/mri/json') .send({ url: U.localBertURL, token: U.testToken + U.userFoo.nickname }); - const {body} = res; + const { body } = res; // console.log(body); shouldContinue = (body.success !== true); @@ -38,13 +38,13 @@ describe('TESTING THE /project ROUTE', function () { it('get("/project/json/") should get an error message requesting page', async function () { const { body } = await chai.request(U.serverURL).get('/project/json/'); - const expected = {error: 'Provide the parameter \'page\''}; + const expected = { error: 'Provide the parameter \'page\'' }; assert.deepEqual(body, expected); }); it('get("/project/json?page=0") should return an array', async function () { const { body } = await chai.request(U.serverURL).get('/project/json?page=0') - .query({page: 0}); + .query({ page: 0 }); assert(Array.isArray(body)); }); @@ -54,7 +54,7 @@ describe('TESTING THE /project ROUTE', function () { }); it('get("/project/json/test") should return an object with appropriate keys', async function () { - const {body} = await chai.request(U.serverURL).get(`/project/json/${U.projectTest.shortname}`); + const { body } = await chai.request(U.serverURL).get(`/project/json/${U.projectTest.shortname}`); const expectedKeys = [ 'name', 'shortname', 'url', 'brainboxURL', 'created', 'owner', 'collaborators', 'files', 'annotations', 'description', @@ -64,33 +64,39 @@ describe('TESTING THE /project ROUTE', function () { }); it('get("/project/json/test/files") should return an array with >=1 file', async function () { - const {body} = await chai.request(U.serverURL) + const { body } = await chai.request(U.serverURL) .get(`/project/json/${U.projectTest.shortname}/files`) - .query({start: 0, length: 10}); + .query({ start: 0, length: 10 }); assert.isArray(body); assert.isAtLeast(body.length, 1); }); it('get("/project/json/test/files") should return objects with appropriate keys', async function () { - const {body} = await chai.request(U.serverURL) + const { body } = await chai.request(U.serverURL) .get(`/project/json/${U.projectTest.shortname}/files`) - .query({start: 0, length: 10}); + .query({ start: 0, length: 10 }); + // only the first mri was fetched, dim info can be missing for the others const expectedKeys1 = [ '_id', 'filename', 'success', 'source', 'url', 'included', 'dim', 'pixdim', 'voxel2world', 'worldOrigin', 'owner', 'mri', 'modified', 'modifiedBy', 'name' ]; - const expectedKeys2 = ['source', 'name']; + const expectedKeys2 = [ + '_id', 'filename', 'source', 'url', 'included', + 'owner', 'mri', 'modified', 'modifiedBy', 'name' + ]; // console.log(body); assert.isArray(body); assert.containsAllKeys(body[0], expectedKeys1); - assert.containsAllKeys(body[4], expectedKeys2); + for (let i = 1; i < 5; i++) { + assert.containsAllKeys(body[i], expectedKeys2); + } }); it('get("/project/json/test/files") should return only sources and names if required', async function () { const { body } = await chai.request(U.serverURL) .get(`/project/json/${U.projectTest.shortname}/files`) - .query({start: 0, length: 10, names: true}); + .query({ start: 0, length: 10, names: true }); assert.isArray(body); assert.hasAllKeys(body[0], ['source', 'name']); }); @@ -101,9 +107,9 @@ describe('TESTING THE /project ROUTE', function () { .query({ url: U.localBertURL }); - const {body} = res; + const { body } = res; const dirPath = './public' + body.url; - await U.removeMRI({dirPath, srcURL: U.localBertURL}); + await U.removeMRI({ dirPath, srcURL: U.localBertURL }); }); }); });