From 56a9293a49009e89eb20061fc28ba72977868bd4 Mon Sep 17 00:00:00 2001 From: DudaGod Date: Tue, 7 Jun 2022 20:12:27 +0300 Subject: [PATCH 1/3] fix: remove unused "teamcity" reporter BREAKING CHANGE: "teamcity" reporter is no longer supported, use "hermione-teamcity-reporter" plugin instead --- lib/reporters/teamcity.js | 66 --------------------------------------- package-lock.json | 29 +++++++---------- package.json | 1 - 3 files changed, 12 insertions(+), 84 deletions(-) delete mode 100644 lib/reporters/teamcity.js diff --git a/lib/reporters/teamcity.js b/lib/reporters/teamcity.js deleted file mode 100644 index cd4799b0d..000000000 --- a/lib/reporters/teamcity.js +++ /dev/null @@ -1,66 +0,0 @@ -'use strict'; - -var format = require('util').format, - inherit = require('inherit'), - logger = require('../utils/logger'), - RunnerEvents = require('../constants/runner-events'), - tsm = require('teamcity-service-messages'); - -module.exports = inherit({ - attachRunner: function(runner) { - runner.on(RunnerEvents.TEST_BEGIN, (test) => this._onTestBegin(test)); - runner.on(RunnerEvents.TEST_PASS, (test) => this._onTestPass(test)); - runner.on(RunnerEvents.TEST_FAIL, (test) => this._onTestFail(test)); - runner.on(RunnerEvents.TEST_PENDING, (test) => this._onTestPending(test)); - - runner.on(RunnerEvents.WARNING, (info) => this._onWarning(info)); - runner.on(RunnerEvents.ERROR, (err) => this._onError(err)); - runner.on(RunnerEvents.INFO, (info) => this._onInfo(info)); - }, - - _onTestBegin: function(test) { - tsm.testStarted({name: this._getTestName(test), flowId: test.sessionId}); - }, - - _onTestPass: function(test) { - tsm.testFinished({name: this._getTestName(test), flowId: test.sessionId}); - }, - - _onTestFail: function(test) { - this._failAndFinish(this._getTestName(test), test); - }, - - _failAndFinish: function(testName, data) { - tsm.testFailed({ - name: testName, - flowId: data.sessionId, - message: data.err, - details: data.err && data.err.stack || data.err - }); - tsm.testFinished({name: testName, flowId: data.sessionId}); - }, - - _onTestPending: function(test) { - tsm.testIgnored({name: this._getTestName(test), flowId: test.sessionId}); - }, - - _onWarning: function(info) { - logger.warn(info); - }, - - _onError: function(error) { - logger.error(error); - }, - - _onInfo: function(info) { - logger.log(info); - }, - - _getTestName: function(test) { - return format('%s [%s: %s]', - test.fullTitle().trim(), - test.browserId, - test.sessionId - ); - } -}); diff --git a/package-lock.json b/package-lock.json index c140047e0..bfd324c59 100644 --- a/package-lock.json +++ b/package-lock.json @@ -857,7 +857,7 @@ "acorn": { "version": "1.2.2", "resolved": "https://registry.npmjs.org/acorn/-/acorn-1.2.2.tgz", - "integrity": "sha512-FsqWmApWGMGLKKNpHt12PMc5AK7BaZee0WRh04fCysmTzHe+rrKOa2MKjORhnzfpe4r0JnfdqHn02iDA9Dqj2A==" + "integrity": "sha1-yM4n3grMdtiW0rH6099YjZ6C8BQ=" }, "acorn-globals": { "version": "6.0.0", @@ -953,7 +953,7 @@ "aliasify": { "version": "1.9.0", "resolved": "https://registry.npmjs.org/aliasify/-/aliasify-1.9.0.tgz", - "integrity": "sha512-xpW+RVocx23HPb34UdDKS3AjEyyBtAR42idz99ncNUeWIIa5bvMBGuSSw66pz3pB1etsGNEssBFZDbVPh5p7uQ==", + "integrity": "sha1-A6oaX+W0ysYE47lnvEx86s+VcDA=", "requires": { "browserify-transform-tools": "~1.5.1" } @@ -961,7 +961,7 @@ "align-text": { "version": "0.1.4", "resolved": "https://registry.npmjs.org/align-text/-/align-text-0.1.4.tgz", - "integrity": "sha512-GrTZLRpmp6wIC2ztrWW9MjjTgSKccffgFagbNDOX95/dcjEcYZibYTeaOntySQLcdw1ztBoFkviiUvTMbb9MYg==", + "integrity": "sha1-DNkKVhCT810KmSVsIrcGlDP60Rc=", "requires": { "kind-of": "^3.0.2", "longest": "^1.0.1", @@ -1324,7 +1324,7 @@ "brorand": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/brorand/-/brorand-1.1.0.tgz", - "integrity": "sha512-cKV8tMCEpQs4hK/ik71d6LrPOnpkpGBR0wzxqr68g2m/LB2GxVYQroAjMJZRVM1Y4BCjCKc3vAamxSzOY2RP+w==" + "integrity": "sha1-EsJe/kCkXjwyPrhnWgoM5XsiNx8=" }, "browser-pack": { "version": "6.1.0", @@ -1368,7 +1368,7 @@ "browserify": { "version": "13.3.0", "resolved": "https://registry.npmjs.org/browserify/-/browserify-13.3.0.tgz", - "integrity": "sha512-RC51w//pULmKo3XmyC5Ax0FgQ3OZQk6he1SHbgsH63hSpa1RR0cGFU4s1AJY4exLesSZjJI00PynhjwWryi2bg==", + "integrity": "sha1-tanJAgJD8McORnW+yCI7xifkFc4=", "requires": { "JSONStream": "^1.0.3", "assert": "^1.4.0", @@ -1520,7 +1520,7 @@ "browserify-transform-tools": { "version": "1.5.3", "resolved": "https://registry.npmjs.org/browserify-transform-tools/-/browserify-transform-tools-1.5.3.tgz", - "integrity": "sha512-xFnNtb5hYYoeDaR3SWLOV6V2SbKiDNp/bCVcq8zONLwBcwA12cyEKSp4GccwaodFh5qq3Dx2m6L/bDCM+HkK3g==", + "integrity": "sha1-UJycZS+2sHvw0h787rsdgm+AdUs=", "requires": { "falafel": "^1.0.1", "through": "^2.3.7" @@ -1529,7 +1529,7 @@ "browserify-zlib": { "version": "0.1.4", "resolved": "https://registry.npmjs.org/browserify-zlib/-/browserify-zlib-0.1.4.tgz", - "integrity": "sha512-19OEpq7vWgsH6WkvkBJQDFvJS1uPcbFOQ4v9CU839dO+ZZXUZO6XpE6hNCqvlIIj+4fZvRiJ6DsAQ382GwiyTQ==", + "integrity": "sha1-uzX4pRn2AOD6a4SFJByXnQFB+y0=", "requires": { "pako": "~0.2.0" } @@ -1556,12 +1556,12 @@ "buffer-xor": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/buffer-xor/-/buffer-xor-1.0.3.tgz", - "integrity": "sha512-571s0T7nZWK6vB67HI5dyUF7wXiNcfaPPPTl6zYCNApANjIvYJTg7hlud/+cJpdAhS7dVzqMLmfhfHR3rAcOjQ==" + "integrity": "sha1-JuYe0UIvtw3ULm42cp7VHYVf6Nk=" }, "builtin-status-codes": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/builtin-status-codes/-/builtin-status-codes-3.0.0.tgz", - "integrity": "sha512-HpGFw18DgFWlncDfjTa2rcQ4W88O1mC8e8yZ2AvQY5KDaktSTwo+KRf6nHK6FRI5FyRyb/5T6+TSxfP7QyGsmQ==" + "integrity": "sha1-hZgoeOIbmOHGZCXgPQF0eI9Wnug=" }, "cacheable-lookup": { "version": "5.0.4", @@ -1613,7 +1613,7 @@ "camelcase": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-1.2.1.tgz", - "integrity": "sha512-wzLkDa4K/mzI1OSITC+DUyjgIl/ETNHE9QvYgy6J6Jvqyyz4C0Xfd+lQhb19sX2jMpZV4IssUn0VDVmglV+s4g==" + "integrity": "sha1-m7UwTS4LVmmLLHWLCKPqqdqlijk=" }, "camelcase-keys": { "version": "2.1.0", @@ -1636,7 +1636,7 @@ "center-align": { "version": "0.1.3", "resolved": "https://registry.npmjs.org/center-align/-/center-align-0.1.3.tgz", - "integrity": "sha512-Baz3aNe2gd2LP2qk5U+sDk/m4oSuwSDcBfayTCTBoWpfIGO5XFxPmjILQII4NGiZjD6DoDI6kf7gKaxkf7s3VQ==", + "integrity": "sha1-qg0yYptu6XIgBBHL1EYckHvCt60=", "requires": { "align-text": "^0.1.3", "lazy-cache": "^1.0.3" @@ -4187,7 +4187,7 @@ "ansi-regex": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", - "integrity": "sha512-TIGnTpdo+E3+pCyAluZvtED5p5wCqLdezCyhPZzKPcxvFplEt4i+W7OONCKgeZFT3+y5NZZfOOS/Bdcanm1MYA==" + "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8=" }, "is-fullwidth-code-point": { "version": "1.0.0", @@ -8650,11 +8650,6 @@ } } }, - "teamcity-service-messages": { - "version": "0.1.9", - "resolved": "https://registry.npmjs.org/teamcity-service-messages/-/teamcity-service-messages-0.1.9.tgz", - "integrity": "sha512-agmBUllpL8n02cG/6s16St5yXYEdynkyyGDWM5qsFq9sKEkc+gBAJgcgJQCVsqlxbZZUToRwTI1hLataRjCGcw==" - }, "temp": { "version": "0.8.4", "resolved": "https://registry.npmjs.org/temp/-/temp-0.8.4.tgz", diff --git a/package.json b/package.json index 7e3137124..46d7a5820 100644 --- a/package.json +++ b/package.json @@ -47,7 +47,6 @@ "inherit": "^2.2.2", "lodash": "^4.17.4", "plugins-loader": "^1.1.0", - "teamcity-service-messages": "^0.1.6", "urijs": "^1.17.0", "webdriverio": "^7.16.11", "worker-farm": "^1.7.0" From 066c59072bcb086535797888018df73502486f93 Mon Sep 17 00:00:00 2001 From: DudaGod Date: Tue, 7 Jun 2022 20:18:09 +0300 Subject: [PATCH 2/3] feat: add ability to redirect output of reporter to the file BREAKING CHANGE: reporters specified as function and used through programmatic API must have a static create method for initialization --- README.md | 12 +- lib/cli/index.js | 2 +- lib/constants/test-statuses.js | 6 + lib/hermione.js | 25 +-- lib/reporters/base.js | 25 ++- lib/reporters/flat.js | 22 ++- lib/reporters/index.js | 100 ++++++++++++ lib/reporters/informers/base.js | 21 +++ lib/reporters/informers/console.js | 22 +++ lib/reporters/informers/file.js | 41 +++++ lib/reporters/informers/index.js | 12 ++ lib/reporters/plain.js | 5 +- lib/reporters/utils/helpers.js | 44 ++--- test/lib/hermione.js | 64 ++------ test/lib/{reporter => reporters}/flat.js | 135 +++++++++------ test/lib/reporters/index.js | 190 ++++++++++++++++++++++ test/lib/reporters/informers/console.js | 52 ++++++ test/lib/reporters/informers/file.js | 79 +++++++++ test/lib/reporters/informers/index.js | 56 +++++++ test/lib/{reporter => reporters}/plain.js | 53 ++++-- test/lib/{reporter => reporters}/utils.js | 0 21 files changed, 787 insertions(+), 179 deletions(-) create mode 100644 lib/constants/test-statuses.js create mode 100644 lib/reporters/index.js create mode 100644 lib/reporters/informers/base.js create mode 100644 lib/reporters/informers/console.js create mode 100644 lib/reporters/informers/file.js create mode 100644 lib/reporters/informers/index.js rename test/lib/{reporter => reporters}/flat.js (65%) create mode 100644 test/lib/reporters/index.js create mode 100644 test/lib/reporters/informers/console.js create mode 100644 test/lib/reporters/informers/file.js create mode 100644 test/lib/reporters/informers/index.js rename test/lib/{reporter => reporters}/plain.js (67%) rename test/lib/{reporter => reporters}/utils.js (100%) diff --git a/README.md b/README.md index c5ca28322..e51f2e6fc 100644 --- a/README.md +++ b/README.md @@ -1370,10 +1370,20 @@ hermione --config ./config.js --reporter flat --browser firefox --grep name **Note.** All CLI options override config values. ### Reporters + You can choose `flat` or `plain` reporter by option `-r, --reporter`. Default is `flat`. +Information about test results is displayed to the command line by default. But there is an ability to redirect the output to a file, for example: +``` +hermione --reporter '{"type": "flat", "path": "./some-path/result.txt"}' +``` -* `flat` – all information about failed and retried tests would be grouped by browsers at the end of the report. +In that example specified file path and all directories will be created automatically. Moreover you can use few reporters: +``` +hermione --reporter '{"type": "flat", "path": "./some-path/result.txt"}' --reporter flat +``` +Information about each report type: +* `flat` – all information about failed and retried tests would be grouped by browsers at the end of the report; * `plain` – information about fails and retries would be placed after each test. ### Require modules diff --git a/lib/cli/index.js b/lib/cli/index.js index fae401ef1..da8f51d02 100644 --- a/lib/cli/index.js +++ b/lib/cli/index.js @@ -30,9 +30,9 @@ exports.run = () => { program .on('--help', () => logger.log(info.configOverriding)) - .option('-r, --reporter ', 'test reporters', collect) .option('-b, --browser ', 'run tests only in specified browser', collect) .option('-s, --set ', 'run tests only in the specified set', collect) + .option('-r, --reporter ', 'test reporters', collect) .option('--require ', 'require module', collect) .option('--grep ', 'run only tests matching the pattern') .option('--update-refs', 'update screenshot references or gather them if they do not exist ("assertView" command)') diff --git a/lib/constants/test-statuses.js b/lib/constants/test-statuses.js new file mode 100644 index 000000000..5e6888f65 --- /dev/null +++ b/lib/constants/test-statuses.js @@ -0,0 +1,6 @@ +module.exports = { + SUCCESS: 'success', + FAIL: 'fail', + RETRY: 'retry', + SKIPPED: 'skipped' +}; diff --git a/lib/hermione.js b/lib/hermione.js index 9e76b8841..4437f2b2c 100644 --- a/lib/hermione.js +++ b/lib/hermione.js @@ -12,6 +12,7 @@ const signalHandler = require('./signal-handler'); const TestReader = require('./test-reader'); const TestCollection = require('./test-collection'); const validateUnknownBrowsers = require('./validators').validateUnknownBrowsers; +const {initReporters} = require('./reporters'); const logger = require('./utils/logger'); module.exports = class Hermione extends BaseHermione { @@ -25,7 +26,7 @@ module.exports = class Hermione extends BaseHermione { this.emit(RunnerEvents.CLI, parser); } - async run(testPaths, {browsers, sets, grep, updateRefs, requireModules, inspectMode, reporters} = {}) { + async run(testPaths, {browsers, sets, grep, updateRefs, requireModules, inspectMode, reporters = []} = {}) { validateUnknownBrowsers(browsers, _.keys(this._config.browsers)); RuntimeConfig.getInstance().extend({updateRefs, requireModules, inspectMode}); @@ -36,7 +37,7 @@ module.exports = class Hermione extends BaseHermione { .on(RunnerEvents.TEST_FAIL, () => this._fail()) .on(RunnerEvents.ERROR, (err) => this.halt(err)); - _.forEach(reporters, (reporter) => applyReporter(this, reporter)); + await initReporters(reporters, this); eventsUtils.passthroughEvent(this._runner, this, _.values(RunnerEvents.getSync())); eventsUtils.passthroughEventAsync(this._runner, this, _.values(RunnerEvents.getAsync())); @@ -113,23 +114,3 @@ module.exports = class Hermione extends BaseHermione { }, timeout).unref(); } }; - -function applyReporter(runner, reporter) { - if (typeof reporter === 'string') { - try { - reporter = require('./reporters/' + reporter); - } catch (e) { - if (e.code === 'MODULE_NOT_FOUND') { - throw new Error('No such reporter: ' + reporter); - } - throw e; - } - } - if (typeof reporter !== 'function') { - throw new TypeError('Reporter must be a string or a function'); - } - - var Reporter = reporter; - - new Reporter().attachRunner(runner); -} diff --git a/lib/reporters/base.js b/lib/reporters/base.js index 6d1fb8a56..38ed4f719 100644 --- a/lib/reporters/base.js +++ b/lib/reporters/base.js @@ -1,12 +1,22 @@ 'use strict'; const chalk = require('chalk'); -const logger = require('../utils/logger'); const RunnerEvents = require('../constants/runner-events'); const icons = require('./utils/icons'); const helpers = require('./utils/helpers'); +const {initInformer} = require('./informers'); module.exports = class BaseReporter { + static async create(opts = {}) { + const informer = await initInformer(opts); + + return new this(informer, opts); + } + + constructor(informer) { + this.informer = informer; + } + attachRunner(runner) { runner.on(RunnerEvents.TEST_PASS, (test) => this._onTestPass(test)); runner.on(RunnerEvents.TEST_FAIL, (test) => this._onTestFail(test)); @@ -29,7 +39,7 @@ module.exports = class BaseReporter { _onRetry(test) { this._logTestInfo(test, icons.RETRY); - logger.log('Will be retried. Retries left: %s', chalk.yellow(test.retriesLeft)); + this.informer.log(`Will be retried. Retries left: ${chalk.yellow(test.retriesLeft)}`); } _onTestPending(test) { @@ -45,22 +55,23 @@ module.exports = class BaseReporter { `Retries: ${chalk.yellow(stats.retries)}` ]; - logger.log(message.join(' ')); + const method = this.__proto__.hasOwnProperty('_onRunnerEnd') ? 'log' : 'end'; + this.informer[method](message.join(' ')); } _onWarning(info) { - logger.warn(info); + this.informer.warn(info); } _onError(error) { - logger.error(chalk.red(error)); + this.informer.error(chalk.red(error)); } _onInfo(info) { - logger.log(info); + this.informer.log(info); } _logTestInfo(test, icon) { - logger.log(`${icon}${helpers.formatTestInfo(test)}`); + this.informer.log(`${icon}${helpers.formatTestInfo(test)}`); } }; diff --git a/lib/reporters/flat.js b/lib/reporters/flat.js index 2f77d42b9..58d17d91d 100644 --- a/lib/reporters/flat.js +++ b/lib/reporters/flat.js @@ -1,11 +1,13 @@ 'use strict'; +const _ = require('lodash'); const BaseReporter = require('./base'); const helpers = require('./utils/helpers'); +const icons = require('./utils/icons'); module.exports = class FlatReporter extends BaseReporter { - constructor() { - super(); + constructor(...args) { + super(...args); this._tests = []; } @@ -25,6 +27,20 @@ module.exports = class FlatReporter extends BaseReporter { _onRunnerEnd(stats) { super._onRunnerEnd(stats); - helpers.logFailedTestsInfo(this._tests); + const failedTests = helpers.formatFailedTests(this._tests); + + failedTests.forEach((test, index) => { + this.informer.log(`\n${index + 1}) ${test.fullTitle}`); + this.informer.log(` in file ${test.file}\n`); + + _.forEach(test.browsers, (testCase) => { + const icon = testCase.isFailed ? icons.FAIL : icons.RETRY; + + this.informer.log(` ${testCase.browserId}`); + this.informer.log(` ${icon} ${testCase.error}`); + }); + }); + + this.informer.end(); } }; diff --git a/lib/reporters/index.js b/lib/reporters/index.js new file mode 100644 index 000000000..4fb26357f --- /dev/null +++ b/lib/reporters/index.js @@ -0,0 +1,100 @@ +exports.initReporters = async (rawReporters, runner) => { + await Promise.all([].concat(rawReporters).map((rawReporter) => applyReporter(rawReporter, runner))); +}; + +const reporterHandlers = [ + { + isMatched: (rawReporter) => typeof rawReporter === 'string' && isJSON(rawReporter), + initReporter: (rawReporter) => initReporter(getReporterDefinition(rawReporter, JSON.parse)) + }, + { + isMatched: (rawReporter) => typeof rawReporter === 'string', + initReporter: (rawReporter) => initReporter({...getReporterDefinition(rawReporter), type: rawReporter}) + }, + { + isMatched: (rawReporter) => typeof rawReporter === 'object', + initReporter: (rawReporter) => initReporter(getReporterDefinition(rawReporter, (v) => v)) + }, + { + isMatched: (rawReporter) => typeof rawReporter === 'function', + initReporter: (rawReporter) => { + validateReporter(rawReporter); + return rawReporter.create(getReporterDefinition(rawReporter)); + } + }, + { + isMatched: () => true, + initReporter: (rawReporter) => { + throw new TypeError(`Specified reporter must be a string, object or function, but got: "${typeof rawReporter}"`); + } + } +]; + +async function applyReporter(rawReporter, runner) { + for (const handler of reporterHandlers) { + if (!handler.isMatched(rawReporter)) { + continue; + } + + const reporter = await handler.initReporter(rawReporter); + + if (typeof reporter.attachRunner !== 'function') { + throw new TypeError( + 'Initialized reporter must have an "attachRunner" function for subscribe on test result events' + ); + } + + return reporter.attachRunner(runner); + } +} + +function initReporter(reporter) { + let Reporter; + + try { + Reporter = require(`./${reporter.type}`); + } catch (e) { + if (e.code === 'MODULE_NOT_FOUND') { + throw new Error(`No such reporter: "${reporter.type}"`); + } + throw e; + } + + validateReporter(Reporter); + + return Reporter.create(reporter); +} + +function getReporterDefinition(rawReporter, parser) { + if (!parser) { + return {type: null, path: null}; + } + + const {type, path} = parser(rawReporter); + + if (!type) { + const strRawReporter = typeof rawReporter !== 'string' ? JSON.stringify(rawReporter) : rawReporter; + throw new Error(`Failed to find required "type" field in reporter definition: "${strRawReporter}"`); + } + + return {type, path}; +} + +function validateReporter(Reporter) { + if (typeof Reporter !== 'function') { + throw new TypeError(`Imported reporter must be a function, but got: "${typeof Reporter}"`); + } + + if (typeof Reporter.create !== 'function') { + throw new TypeError('Imported reporter must have a "create" function for initialization'); + } +} + +function isJSON(str) { + try { + JSON.parse(str); + } catch (e) { + return false; + } + return true; +} diff --git a/lib/reporters/informers/base.js b/lib/reporters/informers/base.js new file mode 100644 index 000000000..6359b7a0d --- /dev/null +++ b/lib/reporters/informers/base.js @@ -0,0 +1,21 @@ +module.exports = class BaseInformer { + static create(...args) { + return new this(...args); + } + + log() { + throw new Error('Method must be implemented in child classes'); + } + + warn() { + throw new Error('Method must be implemented in child classes'); + } + + error() { + throw new Error('Method must be implemented in child classes'); + } + + end() { + throw new Error('Method must be implemented in child classes'); + } +}; diff --git a/lib/reporters/informers/console.js b/lib/reporters/informers/console.js new file mode 100644 index 000000000..09ac01db8 --- /dev/null +++ b/lib/reporters/informers/console.js @@ -0,0 +1,22 @@ +const BaseInformer = require('./base'); +const logger = require('../../utils/logger'); + +module.exports = class ConsoleInformer extends BaseInformer { + log(message) { + logger.log(message); + } + + warn(message) { + logger.warn(message); + } + + error(message) { + logger.error(message); + } + + end(message) { + if (message) { + logger.log(message); + } + } +}; diff --git a/lib/reporters/informers/file.js b/lib/reporters/informers/file.js new file mode 100644 index 000000000..e6e5479a0 --- /dev/null +++ b/lib/reporters/informers/file.js @@ -0,0 +1,41 @@ +const fs = require('fs'); +const chalk = require('chalk'); +const BaseInformer = require('./base'); +const logger = require('../../utils/logger'); + +module.exports = class FileInformer extends BaseInformer { + constructor(opts) { + super(opts); + + this._fileStream = fs.createWriteStream(opts.path); + this._reporterType = opts.type; + + logger.log(`Information with test results for report: "${opts.type}" will be saved to a file: "${opts.path}"`); + } + + log(message) { + this._fileStream.write(`${this._prepareMsg(message)}\n`); + } + + warn(message) { + this.log(message); + } + + error(message) { + this.log(message); + } + + end(message) { + if (message) { + this._fileStream.end(`${this._prepareMsg(message)}\n`); + } else { + this._fileStream.end(); + } + } + + _prepareMsg(msg) { + return typeof msg === 'object' + ? JSON.stringify(msg) + : chalk.stripColor(msg); + } +}; diff --git a/lib/reporters/informers/index.js b/lib/reporters/informers/index.js new file mode 100644 index 000000000..62d0e4462 --- /dev/null +++ b/lib/reporters/informers/index.js @@ -0,0 +1,12 @@ +const path = require('path'); +const fs = require('fs-extra'); + +exports.initInformer = async (opts) => { + if (opts.path) { + await fs.ensureDir(path.dirname(opts.path)); + } + + const informerType = opts.path ? 'file' : 'console'; + + return require(`./${informerType}`).create(opts); +}; diff --git a/lib/reporters/plain.js b/lib/reporters/plain.js index c0cb90749..e5a4ee954 100644 --- a/lib/reporters/plain.js +++ b/lib/reporters/plain.js @@ -2,7 +2,6 @@ const chalk = require('chalk'); -const logger = require('../utils/logger'); const BaseReporter = require('./base'); const icons = require('./utils/icons'); const helpers = require('./utils/helpers'); @@ -14,8 +13,8 @@ module.exports = class PlainReporter extends BaseReporter { if (icon === icons.RETRY || icon === icons.FAIL) { const testInfo = helpers.getTestInfo(test); - logger.log(` in file ${testInfo.file}`); - logger.log(` ${chalk.red(testInfo.error)}`); + this.informer.log(` in file ${testInfo.file}`); + this.informer.log(` ${chalk.red(testInfo.error)}`); } } }; diff --git a/lib/reporters/utils/helpers.js b/lib/reporters/utils/helpers.js index 47f1d74aa..d6267c4e2 100644 --- a/lib/reporters/utils/helpers.js +++ b/lib/reporters/utils/helpers.js @@ -1,13 +1,9 @@ 'use strict'; const path = require('path'); - const chalk = require('chalk'); const _ = require('lodash'); -const logger = require('../../utils/logger'); -const icons = require('./icons'); - const getSkipReason = (test) => test && (getSkipReason(test.parent) || test.skipReason); const getFilePath = (test) => test && test.file || test.parent && getFilePath(test.parent); const getRelativeFilePath = (file) => file ? path.relative(process.cwd(), file) : undefined; @@ -33,13 +29,25 @@ exports.formatTestInfo = (test) => { exports.getTestInfo = (test) => { const file = getFilePath(test); - - return { - title: test.fullTitle(), - browser: test.browserId, + const testInfo = { + fullTitle: test.fullTitle(), + browserId: test.browserId, file: getRelativeFilePath(file), - error: getTestError(test) + sessionId: test.sessionId, + duration: test.duration, + startTime: test.startTime, + meta: test.meta }; + + if (test.err) { + testInfo.error = getTestError(test); + } + + if (test.pending) { + testInfo.reason = getSkipReason(test); + } + + return testInfo; }; exports.extendTestInfo = (test, opts) => { @@ -50,7 +58,7 @@ exports.formatFailedTests = (tests) => { const formattedTests = []; tests.forEach((test) => { - const testItem = _.pick(test, ['title', 'file']); + const testItem = _.pick(test, ['fullTitle', 'file']); if (_.find(formattedTests, testItem)) { return; @@ -62,19 +70,3 @@ exports.formatFailedTests = (tests) => { return formattedTests; }; - -exports.logFailedTestsInfo = (tests) => { - const failedTests = exports.formatFailedTests(tests); - - failedTests.forEach((test, index) => { - logger.log(`\n${index + 1}) ${test.title}`); - logger.log(` in file ${test.file}\n`); - - _.forEach(test.browsers, (testCase) => { - const icon = testCase.isFailed ? icons.FAIL : icons.RETRY; - - logger.log(` ${testCase.browser}`); - logger.log(` ${icon} ${testCase.error}`); - }); - }); -}; diff --git a/test/lib/hermione.js b/test/lib/hermione.js index f28495c26..d6cd6b899 100644 --- a/test/lib/hermione.js +++ b/test/lib/hermione.js @@ -11,7 +11,6 @@ const proxyquire = require('proxyquire').noCallThru(); const Config = require('lib/config'); const RuntimeConfig = require('lib/config/runtime-config'); const Errors = require('lib/errors'); -const Hermione = require('lib/hermione'); const RunnerStats = require('lib/stats'); const TestReader = require('lib/test-reader'); const TestCollection = require('lib/test-collection'); @@ -23,6 +22,7 @@ const {makeConfigStub} = require('../utils'); describe('hermione', () => { const sandbox = sinon.sandbox.create(); + let Hermione, initReporters; const mkHermione_ = (config) => { Config.create.returns(config || makeConfigStub()); @@ -43,14 +43,16 @@ describe('hermione', () => { beforeEach(() => { sandbox.stub(logger, 'warn'); sandbox.stub(Config, 'create').returns(makeConfigStub()); - sandbox.stub(pluginsLoader, 'load').returns([]); - sandbox.stub(RuntimeConfig, 'getInstance').returns({extend: sandbox.stub()}); - sandbox.stub(TestReader.prototype, 'read').resolves(); - sandbox.stub(RunnerStats, 'create'); + + initReporters = sandbox.stub().resolves(); + + Hermione = proxyquire('lib/hermione', { + './reporters': {initReporters} + }); }); afterEach(() => sandbox.restore()); @@ -224,64 +226,30 @@ describe('hermione', () => { }); describe('reporters', () => { - let Hermione; - let attachRunner; - - const createReporter = () => { - return function Reporter() { - this.attachRunner = attachRunner; - }; - }; + let runner; beforeEach(() => { - Hermione = proxyquire('lib/hermione', { - './reporters/reporter': createReporter() - }); - - attachRunner = sandbox.stub(); - - mkRunnerStub_(); + runner = mkRunnerStub_(); }); - it('should accept reporter specified as string', async () => { + it('should initialize passed reporters', async () => { const options = {reporters: ['reporter']}; Config.create.returns(makeConfigStub()); const hermione = Hermione.create(); await hermione.run(null, options); - assert.calledOnceWith(attachRunner, hermione); + assert.calledOnceWith(initReporters, ['reporter'], hermione); }); - it('should accept reporter specified as function', async () => { - const options = {reporters: [createReporter()]}; - const hermione = mkHermione_(); + it('should initialize reporters before run tests', async () => { + const options = {reporters: ['reporter']}; + Config.create.returns(makeConfigStub()); + const hermione = Hermione.create(); await hermione.run(null, options); - assert.calledOnceWith(attachRunner, hermione); - }); - - it('should fail if reporter was not found for given identifier', async () => { - const options = {reporters: ['unknown-reporter']}; - const hermione = mkHermione_(); - - try { - await hermione.run(null, options); - } catch (e) { - assert.equal(e.message, 'No such reporter: unknown-reporter'); - } - }); - - it('should fail if reporter is not string or function', async () => { - const options = {reporters: [1234]}; - const hermione = mkHermione_(); - - try { - await hermione.run(null, options); - } catch (e) { - assert.equal(e.message, 'Reporter must be a string or a function'); - } + assert.callOrder(initReporters, runner.run); }); }); diff --git a/test/lib/reporter/flat.js b/test/lib/reporters/flat.js similarity index 65% rename from test/lib/reporter/flat.js rename to test/lib/reporters/flat.js index 4da90566e..1eb33a9fe 100644 --- a/test/lib/reporter/flat.js +++ b/test/lib/reporters/flat.js @@ -2,19 +2,14 @@ const chalk = require('chalk'); const path = require('path'); -const EventEmitter = require('events').EventEmitter; -const FlatReporter = require('lib/reporters/flat'); +const {EventEmitter} = require('events'); +const proxyquire = require('proxyquire'); const RunnerEvents = require('lib/constants/runner-events'); -const logger = require('lib/utils/logger'); -const mkTestStub_ = require('./utils').mkTestStub_; -const getDeserializedResult = require('./utils').getDeserializedResult; +const {mkTestStub_, getDeserializedResult} = require('./utils'); describe('Flat reporter', () => { const sandbox = sinon.sandbox.create(); - - let test; - let emitter; - let stdout; + let FlatReporter, initInformer, informer, emitter, test, stdout; const emit = (event, data) => { emitter.emit(RunnerEvents.RUNNER_START); @@ -24,45 +19,69 @@ describe('Flat reporter', () => { emitter.emit(RunnerEvents.RUNNER_END, {}); }; + const createFlatReporter = async (opts = {}) => { + const reporter = await FlatReporter.create(opts); + reporter.attachRunner(emitter); + }; + beforeEach(() => { test = mkTestStub_(); - - const reporter = new FlatReporter(); - emitter = new EventEmitter(); - reporter.attachRunner(emitter); - stdout = ''; - sandbox.stub(logger, 'log').callsFake((str) => stdout += `${str}\n`); - sandbox.stub(logger, 'warn'); - sandbox.stub(logger, 'error'); + informer = { + log: sandbox.stub().callsFake((str) => stdout += `${str}\n`), + warn: sandbox.stub(), + error: sandbox.stub(), + end: sandbox.stub() + }; + + initInformer = sandbox.stub().resolves(informer); + + FlatReporter = proxyquire('lib/reporters/flat', { + './base': proxyquire('lib/reporters/base', { + './informers': {initInformer} + }) + }); }); - afterEach(() => { - sandbox.restore(); - emitter.removeAllListeners(); + afterEach(() => sandbox.restore()); + + it('should initialize informer with passed args', async () => { + const opts = {type: 'flat', path: './flat.txt'}; + + await createFlatReporter(opts); + + assert.calledOnceWith(initInformer, opts); }); - it('should print info', () => { + it('should inform about info', async () => { + await createFlatReporter(); + emit(RunnerEvents.INFO, 'foo'); - assert.calledWith(logger.log, 'foo'); + assert.calledWith(informer.log, 'foo'); }); - it('should print warning', () => { + it('should inform about warning', async () => { + await createFlatReporter(); + emit(RunnerEvents.WARNING, 'foo'); - assert.calledWith(logger.warn, 'foo'); + assert.calledWith(informer.warn, 'foo'); }); - it('should print error', () => { + it('should inform about error', async () => { + await createFlatReporter(); + emit(RunnerEvents.ERROR, 'foo'); - assert.calledWith(logger.error, chalk.red('foo')); + assert.calledWith(informer.error, chalk.red('foo')); }); - it('should print statistics of the tests execution', () => { + it('should inform about statistics of the tests execution', async () => { + await createFlatReporter(); + emit(RunnerEvents.RUNNER_END, { total: 5, passed: 2, @@ -71,7 +90,7 @@ describe('Flat reporter', () => { retries: 2 }); - const deserealizedResult = chalk.stripColor(logger.log.firstCall.args[0]); + const deserealizedResult = chalk.stripColor(informer.log.firstCall.args[0]); assert.equal(deserealizedResult, 'Total: 5 Passed: 2 Failed: 2 Skipped: 1 Retries: 2'); }); @@ -82,31 +101,33 @@ describe('Flat reporter', () => { TEST_FAIL: 'failed' }; - it('should correctly do the rendering', () => { + it('should correctly do the rendering', async () => { test = mkTestStub_({sessionId: 'test_session'}); + await createFlatReporter(); emit(RunnerEvents.TEST_PASS, test); - const result = getDeserializedResult(logger.log.firstCall.args[0]); + const result = getDeserializedResult(informer.log.firstCall.args[0]); assert.equal(result, 'suite test [chrome:test_session] - 100500ms'); }); describe('skipped tests report', () => { - it('should add skip comment if test was skipped', () => { + it('should add skip comment if test was skipped', async () => { test = mkTestStub_({ pending: true, skipReason: 'some comment' }); + await createFlatReporter(); emit(RunnerEvents.TEST_PENDING, test); - const result = getDeserializedResult(logger.log.firstCall.args[0]); + const result = getDeserializedResult(informer.log.firstCall.args[0]); assert.match(result, /reason: some comment/); }); - it('should use parent skip comment if all describe was skipped', () => { + it('should use parent skip comment if all describe was skipped', async () => { test = mkTestStub_({ pending: true, skipReason: 'test comment', @@ -115,35 +136,38 @@ describe('Flat reporter', () => { } }); + await createFlatReporter(); emit(RunnerEvents.TEST_PENDING, test); - const result = getDeserializedResult(logger.log.firstCall.args[0]); + const result = getDeserializedResult(informer.log.firstCall.args[0]); assert.match(result, /reason: suite comment/); }); - it('should use test skip comment if describe was skipped without comment', () => { + it('should use test skip comment if describe was skipped without comment', async () => { test = mkTestStub_({ pending: true, skipReason: 'test comment', parent: {some: 'data'} }); + await createFlatReporter(); emit(RunnerEvents.TEST_PENDING, test); - const result = getDeserializedResult(logger.log.firstCall.args[0]); + const result = getDeserializedResult(informer.log.firstCall.args[0]); assert.match(result, /reason: test comment/); }); - it('should use default message if test was skipped without comment', () => { + it('should use default message if test was skipped without comment', async () => { test = mkTestStub_({ pending: true }); + await createFlatReporter(); emit(RunnerEvents.TEST_PENDING, test); - const result = getDeserializedResult(logger.log.firstCall.args[0]); + const result = getDeserializedResult(informer.log.firstCall.args[0]); assert.match(result, /reason: no comment/); }); @@ -151,73 +175,79 @@ describe('Flat reporter', () => { ['RETRY', 'TEST_FAIL'].forEach((event) => { describe(`${testStates[event]} tests report`, () => { - it(`should log correct number of ${testStates[event]} suite`, () => { + it(`should log correct number of ${testStates[event]} suite`, async () => { test = mkTestStub_(); + await createFlatReporter(); emit(RunnerEvents[event], test); assert.match(stdout, /1\) .+/); }); - it(`should log full title of ${testStates[event]} suite`, () => { + it(`should log full title of ${testStates[event]} suite`, async () => { test = mkTestStub_(); + await createFlatReporter(); emit(RunnerEvents[event], test); assert.include(stdout, test.fullTitle()); }); - it(`should log path to file of ${testStates[event]} suite`, () => { + it(`should log path to file of ${testStates[event]} suite`, async () => { test = mkTestStub_(); - sandbox.stub(path, 'relative').returns(`relative/${test.file}`); + await createFlatReporter(); emit(RunnerEvents[event], test); assert.include(stdout, test.file); }); - it(`should log browser of ${testStates[event]} suite`, () => { + it(`should log browser of ${testStates[event]} suite`, async () => { test = mkTestStub_({ browserId: 'bro1' }); + await createFlatReporter(); emit(RunnerEvents[event], test); assert.match(stdout, /bro1/); }); - it(`should log an error stack of ${testStates[event]} test`, () => { + it(`should log an error stack of ${testStates[event]} test`, async () => { test = mkTestStub_({ err: {stack: 'some stack'} }); + await createFlatReporter(); emit(RunnerEvents[event], test); assert.match(stdout, /some stack/); }); - it(`should log an error message of ${testStates[event]} test if an error stack does not exist`, () => { + it(`should log an error message of ${testStates[event]} test if an error stack does not exist`, async () => { test = mkTestStub_({ err: {message: 'some message'} }); + await createFlatReporter(); emit(RunnerEvents[event], test); assert.match(stdout, /some message/); }); - it(`should log an error of ${testStates[event]} test if an error stack and message do not exist`, () => { + it(`should log an error of ${testStates[event]} test if an error stack and message do not exist`, async () => { test = mkTestStub_({ err: 'some error' }); + await createFlatReporter(); emit(RunnerEvents[event], test); assert.match(stdout, /some error/); }); - it('should extend error with original selenium error if it exist', () => { + it('should extend error with original selenium error if it exists', async () => { test = mkTestStub_({ err: { stack: 'some stack', @@ -227,14 +257,16 @@ describe('Flat reporter', () => { } }); + await createFlatReporter(); emit(RunnerEvents[event], test); assert.match(stdout, /some stack \(some original message\)/); }); - it(`should log "undefined" if ${testStates[event]} test does not have "err" property`, () => { + it(`should log "undefined" if ${testStates[event]} test does not have "err" property`, async () => { test = mkTestStub_(); + await createFlatReporter(); emit(RunnerEvents[event], test); assert.match(stdout, /undefined/); @@ -243,7 +275,7 @@ describe('Flat reporter', () => { }); describe('failed tests report', () => { - it('should log path to file of failed hook', () => { + it('should log path to file of failed hook', async () => { test = mkTestStub_({ file: null, parent: { @@ -253,17 +285,20 @@ describe('Flat reporter', () => { sandbox.stub(path, 'relative').returns(`relative/${test.parent.file}`); + await createFlatReporter(); emit(RunnerEvents.TEST_FAIL, test); assert.include(stdout, 'relative/path-to-test'); }); - it('should not throw if path to file is not specified on failed hook', () => { + it('should not throw if path to file is not specified on failed hook', async () => { test = mkTestStub_({ file: null, parent: {} }); + await createFlatReporter(); + assert.doesNotThrow(() => emit(RunnerEvents.TEST_FAIL, test)); }); }); diff --git a/test/lib/reporters/index.js b/test/lib/reporters/index.js new file mode 100644 index 000000000..398449519 --- /dev/null +++ b/test/lib/reporters/index.js @@ -0,0 +1,190 @@ +const {EventEmitter} = require('events'); +const Promise = require('bluebird'); +const proxyquire = require('proxyquire').noCallThru(); + +describe('"initReporters" method', () => { + const sandbox = sinon.sandbox.create(); + let initReporters, Reporter, attachRunner, runner; + + const createReporter = (attachRunner) => { + function Reporter() { + this.attachRunner = attachRunner; + } + + Reporter.create = () => new Reporter(); + + return Reporter; + }; + + beforeEach(() => { + attachRunner = sandbox.stub(); + Reporter = createReporter(attachRunner); + runner = new EventEmitter(); + + ({initReporters} = proxyquire('lib/reporters', { + './reporter': Reporter + })); + }); + + afterEach(() => { + sandbox.restore(); + }); + + describe('reporter specified as string', () => { + it('should throw error if reporter was not found', async () => { + await assert.isRejected( + initReporters(['unknown-reporter'], runner), + 'No such reporter: "unknown-reporter"' + ); + }); + + it('should wait until reporter is initialized', async () => { + const afterCreateReporter = sinon.spy().named('afterCreateReporter'); + Reporter.create = () => Promise.delay(10).then(() => { + afterCreateReporter(); + return new Reporter(); + }); + + await initReporters(['reporter'], runner); + + assert.callOrder(afterCreateReporter, attachRunner); + }); + + it('should create reporter with correct args', async () => { + sinon.spy(Reporter, 'create'); + + await initReporters(['reporter'], runner); + + assert.calledOnceWith(Reporter.create, {type: 'reporter', path: null}); + }); + + it('should attach reporter to runner', async () => { + await initReporters(['reporter'], runner); + + assert.calledOnceWith(attachRunner, runner); + }); + + it('should attach few reporters to runner', async () => { + const attachRunner1 = sandbox.stub(); + const attachRunner2 = sandbox.stub(); + + ({initReporters} = proxyquire('lib/reporters', { + './reporter-1': createReporter(attachRunner1), + './reporter-2': createReporter(attachRunner2) + })); + + await initReporters(['reporter-1', 'reporter-2'], runner); + + assert.calledOnceWith(attachRunner1, runner); + assert.calledOnceWith(attachRunner2, runner); + }); + }); + + describe('reporter specified as JSON string', () => { + it('should throw error if "type" field is not specified', async () => { + const jsonReporter = JSON.stringify({foo: 'bar'}); + + await assert.isRejected( + initReporters([jsonReporter], runner), + `Failed to find required "type" field in reporter definition: "${jsonReporter}"` + ); + }); + + it('should create reporter with correct args', async () => { + sinon.spy(Reporter, 'create'); + const jsonReporter = JSON.stringify({type: 'reporter', path: './foo/bar.txt'}); + + await initReporters([jsonReporter], runner); + + assert.calledOnceWith(Reporter.create, JSON.parse(jsonReporter)); + }); + + it('should attach reporter to runner', async () => { + const jsonReporter = JSON.stringify({type: 'reporter', path: './foo/bar.txt'}); + + await initReporters([jsonReporter], runner); + + assert.calledOnceWith(attachRunner, runner); + }); + }); + + describe('reporter specified as object', () => { + it('should throw error if "type" field is not specified', async () => { + const objReporter = {foo: 'bar'}; + + await assert.isRejected( + initReporters([objReporter], runner), + `Failed to find required "type" field in reporter definition: "${JSON.stringify(objReporter)}"` + ); + }); + + it('should create reporter with correct args', async () => { + sinon.spy(Reporter, 'create'); + const objReporter = {type: 'reporter', path: './foo/bar.txt'}; + + await initReporters([objReporter], runner); + + assert.calledOnceWith(Reporter.create, objReporter); + }); + + it('should attach reporter to runner', async () => { + const objReporter = JSON.stringify({type: 'reporter', path: './foo/bar.txt'}); + + await initReporters([objReporter], runner); + + assert.calledOnceWith(attachRunner, runner); + }); + }); + + describe('reporter specified as function', () => { + it('should create reporter with correct args', async () => { + const FnReporter = createReporter(sandbox.stub()); + sinon.spy(FnReporter, 'create'); + + await initReporters([FnReporter], runner); + + assert.calledOnceWith(FnReporter.create, {type: null, path: null}); + }); + + it('should attach reporter to runner', async () => { + const attachRunner = sandbox.stub(); + const FnReporter = createReporter(attachRunner); + + await initReporters([FnReporter], runner); + + assert.calledOnceWith(attachRunner, runner); + }); + + it('should throw error if specified reporter does not have "create" function', async () => { + const FnReporter = createReporter(sandbox.stub()); + FnReporter.create = null; + + try { + await initReporters([FnReporter], runner); + } catch (e) { + assert.equal(e.message, 'Imported reporter must have a "create" function for initialization'); + } + }); + + it('should throw error if specified reporter does not have "attachRunner" function', async () => { + const FnReporter = createReporter(null); + + try { + await initReporters([FnReporter], runner); + } catch (e) { + assert.equal( + e.message, + 'Initialized reporter must have an "attachRunner" function for subscribe on test result events' + ); + } + }); + }); + + it('should throw error if specified reporter has unsupported type', async () => { + try { + await initReporters([12345], runner); + } catch (e) { + assert.equal(e.message, 'Specified reporter must be a string, object or function, but got: "number"'); + } + }); +}); diff --git a/test/lib/reporters/informers/console.js b/test/lib/reporters/informers/console.js new file mode 100644 index 000000000..17ae75bae --- /dev/null +++ b/test/lib/reporters/informers/console.js @@ -0,0 +1,52 @@ +const logger = require('lib/utils/logger'); +const ConsoleInformer = require('lib/reporters/informers/console'); + +describe('reporter/informers/console', () => { + const sandbox = sinon.sandbox.create(); + + beforeEach(() => { + sandbox.stub(logger, 'log'); + sandbox.stub(logger, 'warn'); + sandbox.stub(logger, 'error'); + }); + + afterEach(() => sandbox.restore()); + + describe('"log" method', () => { + it('should send log message to console', () => { + ConsoleInformer.create().log('message'); + + assert.calledOnceWith(logger.log, 'message'); + }); + }); + + describe('"warn" method', () => { + it('should send warn message to console', () => { + ConsoleInformer.create().warn('message'); + + assert.calledOnceWith(logger.warn, 'message'); + }); + }); + + describe('"error" method', () => { + it('should send error message to console', () => { + ConsoleInformer.create().error('message'); + + assert.calledOnceWith(logger.error, 'message'); + }); + }); + + describe('"end" method', () => { + it('should do nothing if message is not passed', () => { + ConsoleInformer.create().end(); + + assert.notCalled(logger.log); + }); + + it('should send end message to console', () => { + ConsoleInformer.create().end('message'); + + assert.calledOnceWith(logger.log, 'message'); + }); + }); +}); diff --git a/test/lib/reporters/informers/file.js b/test/lib/reporters/informers/file.js new file mode 100644 index 000000000..14a1338f8 --- /dev/null +++ b/test/lib/reporters/informers/file.js @@ -0,0 +1,79 @@ +const fs = require('fs'); +const chalk = require('chalk'); +const logger = require('lib/utils/logger'); +const FileInformer = require('lib/reporters/informers/file'); + +describe('reporter/informers/file', () => { + const sandbox = sinon.sandbox.create(); + let fsStream; + + beforeEach(() => { + fsStream = {write: sandbox.stub(), end: sandbox.stub()}; + sandbox.stub(fs, 'createWriteStream').returns(fsStream); + + sandbox.stub(logger, 'log'); + }); + + afterEach(() => sandbox.restore()); + + it('should create write stream to passed file path', () => { + FileInformer.create({path: './some-path/file.txt'}); + + assert.calledOnceWith(fs.createWriteStream, './some-path/file.txt'); + }); + + it('should inform user that test results will be saved to a file', () => { + const opts = {type: 'flat', path: './some-path/file.txt'}; + + FileInformer.create(opts); + + assert.calledOnceWith( + logger.log, + `Information with test results for report: "${opts.type}" will be saved to a file: "${opts.path}"` + ); + }); + + ['log', 'warn', 'error'].forEach((methodName) => { + describe(`"${methodName}" method`, () => { + it('should stringify object message before write it to a file', () => { + const message = {foo: 'bar'}; + + FileInformer.create({type: 'flat', path: './file.txt'})[methodName](message); + + assert.calledOnceWith(fsStream.write, `${JSON.stringify(message)}\n`); + }); + + it('should remove color from string message before write it to a file', () => { + const message = chalk.red('some message'); + + FileInformer.create({type: 'flat', path: './file.txt'})[methodName](message); + + assert.calledOnceWith(fsStream.write, 'some message\n'); + }); + }); + }); + + describe('"end" method', () => { + it('should stringify object message before write it to a file', () => { + const message = {foo: 'bar'}; + + FileInformer.create({type: 'flat', path: './file.txt'}).end(message); + + assert.calledOnceWith(fsStream.end, `${JSON.stringify(message)}\n`); + }); + + it('should remove color from string message before write it to a file', () => { + const message = chalk.red('some message'); + + FileInformer.create({type: 'flat', path: './file.txt'}).end(message); + + assert.calledOnceWith(fsStream.end, 'some message\n'); + }); + + it('should call without args if message is not passed', () => { + FileInformer.create({type: 'flat', path: './file.txt'}).end(); + + assert.calledOnceWithExactly(fsStream.end); + }); + }); +}); diff --git a/test/lib/reporters/informers/index.js b/test/lib/reporters/informers/index.js new file mode 100644 index 000000000..8a4567c71 --- /dev/null +++ b/test/lib/reporters/informers/index.js @@ -0,0 +1,56 @@ +const fs = require('fs-extra'); +const proxyquire = require('proxyquire'); + +describe('reporters/informers', () => { + const sandbox = sinon.sandbox.create(); + + afterEach(() => sandbox.restore()); + + describe('"initInformer" method', () => { + let initInformer, createFileInformer, createConsoleInformer; + + beforeEach(() => { + createFileInformer = sandbox.stub(); + createConsoleInformer = sandbox.stub(); + + sandbox.stub(fs, 'ensureDir'); + + ({initInformer} = proxyquire('lib/reporters/informers', { + './file': {create: createFileInformer}, + './console': {create: createConsoleInformer} + })); + }); + + describe('if option "path" is passed', () => { + it('should create directory', async () => { + await initInformer({path: './foo/bar/baz.txt'}); + + assert.calledOnceWith(fs.ensureDir, './foo/bar'); + }); + + it('should create file informer', async () => { + const opts = {path: './foo/bar/baz.txt', type: 'jsonl'}; + + await initInformer(opts); + + assert.calledOnceWith(createFileInformer, opts); + }); + }); + + describe('if option "path" is not passed', () => { + it('should not create directory', async () => { + await initInformer({}); + + assert.notCalled(fs.ensureDir); + }); + + it('should create console informer', async () => { + const opts = {type: 'jsonl'}; + + await initInformer(opts); + + assert.calledOnceWith(createConsoleInformer, opts); + }); + }); + }); +}); diff --git a/test/lib/reporter/plain.js b/test/lib/reporters/plain.js similarity index 67% rename from test/lib/reporter/plain.js rename to test/lib/reporters/plain.js index a25857a0d..34143df95 100644 --- a/test/lib/reporter/plain.js +++ b/test/lib/reporters/plain.js @@ -1,19 +1,14 @@ 'use strict'; const EventEmitter = require('events').EventEmitter; -const logger = require('lib/utils/logger'); -const PlainReporter = require('lib/reporters/plain'); const RunnerEvents = require('lib/constants/runner-events'); - +const proxyquire = require('proxyquire'); const mkTestStub_ = require('./utils').mkTestStub_; const getDeserializedResult = require('./utils').getDeserializedResult; describe('Plain reporter', () => { const sandbox = sinon.sandbox.create(); - - let test; - let emitter; - let stdout; + let PlainReporter, initInformer, informer, emitter, test, stdout; const emit = (event, data) => { emitter.emit(RunnerEvents.RUNNER_START); @@ -23,25 +18,44 @@ describe('Plain reporter', () => { emitter.emit(RunnerEvents.RUNNER_END, {}); }; + const createPlainReporter = async (opts = {}) => { + const reporter = await PlainReporter.create(opts); + reporter.attachRunner(emitter); + }; + beforeEach(() => { test = mkTestStub_(); - - const reporter = new PlainReporter(); - emitter = new EventEmitter(); - reporter.attachRunner(emitter); - stdout = ''; - sandbox.stub(logger, 'log').callsFake((str) => stdout += `${str}\n`); - sandbox.stub(logger, 'warn'); - sandbox.stub(logger, 'error'); + informer = { + log: sandbox.stub().callsFake((str) => stdout += `${str}\n`), + warn: sandbox.stub(), + error: sandbox.stub(), + end: sandbox.stub() + }; + + initInformer = sandbox.stub().resolves(informer); + + PlainReporter = proxyquire('lib/reporters/plain', { + './base': proxyquire('lib/reporters/base', { + './informers': {initInformer} + }) + }); }); afterEach(() => sandbox.restore()); + it('should initialize informer with passed args', async () => { + const opts = {type: 'plain', path: './plain.txt'}; + + await createPlainReporter(opts); + + assert.calledOnceWith(initInformer, opts); + }); + describe('success tests report', () => { - it('should log correct info about test', () => { + it('should log correct info about test', async () => { test = mkTestStub_({ fullTitle: () => 'some test title', title: 'test title', @@ -49,6 +63,7 @@ describe('Plain reporter', () => { duration: '100' }); + await createPlainReporter(); emit(RunnerEvents.TEST_PASS, test); assert.match( @@ -65,7 +80,7 @@ describe('Plain reporter', () => { ['RETRY', 'TEST_FAIL'].forEach((event) => { describe(`${testStates[event]} tests report`, () => { - it(`should log correct info about test`, () => { + it(`should log correct info about test`, async () => { test = mkTestStub_({ fullTitle: () => 'some test title', title: 'test title', @@ -75,6 +90,7 @@ describe('Plain reporter', () => { err: {stack: 'some error stack'} }); + await createPlainReporter(); emit(RunnerEvents[event], test); assert.match( @@ -83,7 +99,7 @@ describe('Plain reporter', () => { ); }); - it('should extend error with original selenium error if it exist', () => { + it('should extend error with original selenium error if it exists', async () => { test = mkTestStub_({ err: { stack: 'some stack', @@ -93,6 +109,7 @@ describe('Plain reporter', () => { } }); + await createPlainReporter(); emit(RunnerEvents[event], test); assert.match(stdout, /some stack \(some original message\)/); diff --git a/test/lib/reporter/utils.js b/test/lib/reporters/utils.js similarity index 100% rename from test/lib/reporter/utils.js rename to test/lib/reporters/utils.js From 08140fdbc1d630e2cbfbfba16457fc813e9f1aee Mon Sep 17 00:00:00 2001 From: DudaGod Date: Tue, 7 Jun 2022 20:22:15 +0300 Subject: [PATCH 3/3] feat: add "jsonl" reporter --- README.md | 9 +- lib/reporters/jsonl.js | 40 ++++++ test/lib/reporters/jsonl.js | 245 ++++++++++++++++++++++++++++++++++++ 3 files changed, 290 insertions(+), 4 deletions(-) create mode 100644 lib/reporters/jsonl.js create mode 100644 test/lib/reporters/jsonl.js diff --git a/README.md b/README.md index e51f2e6fc..e56eb8053 100644 --- a/README.md +++ b/README.md @@ -1371,20 +1371,21 @@ hermione --config ./config.js --reporter flat --browser firefox --grep name ### Reporters -You can choose `flat` or `plain` reporter by option `-r, --reporter`. Default is `flat`. +You can choose `flat`, `plain` or `jsonl` reporter by option `-r, --reporter`. Default is `flat`. Information about test results is displayed to the command line by default. But there is an ability to redirect the output to a file, for example: ``` -hermione --reporter '{"type": "flat", "path": "./some-path/result.txt"}' +hermione --reporter '{"type": "jsonl", "path": "./some-path/result.jsonl"}' ``` In that example specified file path and all directories will be created automatically. Moreover you can use few reporters: ``` -hermione --reporter '{"type": "flat", "path": "./some-path/result.txt"}' --reporter flat +hermione --reporter '{"type": "jsonl", "path": "./some-path/result.jsonl"}' --reporter flat ``` Information about each report type: * `flat` – all information about failed and retried tests would be grouped by browsers at the end of the report; -* `plain` – information about fails and retries would be placed after each test. +* `plain` – information about fails and retries would be placed after each test; +* `jsonl` - displays detailed information about each test result in [jsonl](https://jsonlines.org/) format. ### Require modules diff --git a/lib/reporters/jsonl.js b/lib/reporters/jsonl.js new file mode 100644 index 000000000..c552b1811 --- /dev/null +++ b/lib/reporters/jsonl.js @@ -0,0 +1,40 @@ +'use strict'; + +const BaseReporter = require('./base'); +const {extendTestInfo} = require('./utils/helpers'); +const {SUCCESS, FAIL, RETRY, SKIPPED} = require('../constants/test-statuses'); + +module.exports = class JsonlReporter extends BaseReporter { + _onTestPass(test) { + const testInfo = extendTestInfo(test, {status: SUCCESS}); + this.informer.log(testInfo); + } + + _onTestFail(test) { + this.informer.log(extendTestInfo(test, {status: FAIL})); + } + + _onRetry(test) { + this.informer.log(extendTestInfo(test, {status: RETRY})); + } + + _onTestPending(test) { + this.informer.log(extendTestInfo(test, {status: SKIPPED})); + } + + _onRunnerEnd() { + this.informer.end(); + } + + _onWarning() { + // do nothing + } + + _onError() { + // do nothing + } + + _onInfo() { + // do nothing + } +}; diff --git a/test/lib/reporters/jsonl.js b/test/lib/reporters/jsonl.js new file mode 100644 index 000000000..337b858e8 --- /dev/null +++ b/test/lib/reporters/jsonl.js @@ -0,0 +1,245 @@ +const path = require('path'); +const {EventEmitter} = require('events'); +const proxyquire = require('proxyquire'); +const _ = require('lodash'); +const RunnerEvents = require('lib/constants/runner-events'); +const {SUCCESS, FAIL, RETRY, SKIPPED} = require('lib/constants/test-statuses'); + +const testStates = { + RETRY: 'retried', + TEST_FAIL: 'failed' +}; + +describe('Jsonl reporter', () => { + const sandbox = sinon.sandbox.create(); + let JsonlReporter, initInformer, informer, emitter; + + const emit = (event, data) => { + emitter.emit(RunnerEvents.RUNNER_START); + if (event) { + emitter.emit(event, data); + } + emitter.emit(RunnerEvents.RUNNER_END, {}); + }; + + const createJsonlReporter = async (opts = {}) => { + const reporter = await JsonlReporter.create(opts); + reporter.attachRunner(emitter); + }; + + const mkTest_ = (opts = {}) => { + return _.defaults(opts, { + fullTitle: () => 'default_suite default_test', + browserId: 'default_bro', + file: path.resolve(process.cwd(), './default/path'), + sessionId: '100500', + duration: '500100', + startTime: Date.now(), + meta: {browserVersion: '99999'} + }); + }; + + beforeEach(() => { + emitter = new EventEmitter(); + + informer = { + log: sandbox.stub(), + warn: sandbox.stub(), + error: sandbox.stub(), + end: sandbox.stub() + }; + + initInformer = sandbox.stub().resolves(informer); + + JsonlReporter = proxyquire('lib/reporters/jsonl', { + './base': proxyquire('lib/reporters/base', { + './informers': {initInformer} + }) + }); + }); + + afterEach(() => sandbox.restore()); + + it('should initialize informer with passed args', async () => { + const opts = {type: 'jsonl', path: './report.jsonl'}; + + await createJsonlReporter(opts); + + assert.calledOnceWith(initInformer, opts); + }); + + [RunnerEvents.INFO, RunnerEvents.WARNING, RunnerEvents.ERROR].forEach((eventName) => { + it(`should do nothing on ${eventName} event`, async () => { + await createJsonlReporter(); + + emit(eventName, 'foo'); + + assert.notCalled(informer.log); + assert.notCalled(informer.warn); + assert.notCalled(informer.error); + }); + }); + + it('should inform about successful test', async () => { + const test = mkTest_({ + fullTitle: () => 'suite test', + browserId: 'yabro', + file: path.resolve(process.cwd(), './path/to/test'), + sessionId: '12345', + duration: '100500', + startTime: Date.now(), + meta: {browserVersion: '99'} + }); + + await createJsonlReporter(); + emit(RunnerEvents.TEST_PASS, test); + + assert.deepEqual(informer.log.firstCall.args[0], {...test, fullTitle: 'suite test', file: 'path/to/test', status: SUCCESS}); + }); + + [ + {event: 'RETRY', status: RETRY}, + {event: 'TEST_FAIL', status: FAIL} + ].forEach(({event, status}) => { + describe(`"${testStates[event]}" tests report`, () => { + it(`should inform about ${testStates[event]} test`, async () => { + const test = mkTest_({ + fullTitle: () => 'suite test', + browserId: 'yabro', + file: path.resolve(process.cwd(), './path/to/test'), + sessionId: '12345', + duration: '100500', + startTime: Date.now(), + meta: {browserVersion: '99'} + }); + + await createJsonlReporter(); + emit(RunnerEvents[event], test); + + assert.deepEqual(informer.log.firstCall.args[0], {...test, fullTitle: 'suite test', file: 'path/to/test', status}); + }); + + it(`should add "error" field with "stack" from ${testStates[event]} test`, async () => { + const test = mkTest_({ + err: { + stack: 'o.O', + message: 'O.o' + } + }); + + await createJsonlReporter(); + emit(RunnerEvents[event], test); + + assert.equal(informer.log.firstCall.args[0].error, 'o.O'); + }); + + it(`should add "error" field with "message" from ${testStates[event]} test if "stack" does not exist`, async () => { + const test = mkTest_({ + err: { + message: 'O.o' + } + }); + + await createJsonlReporter(); + emit(RunnerEvents[event], test); + + assert.equal(informer.log.firstCall.args[0].error, 'O.o'); + }); + + it(`should add "error" field if it's specified as string in ${testStates[event]} test`, async () => { + const test = mkTest_({ + err: 'o.O' + }); + + await createJsonlReporter(); + emit(RunnerEvents[event], test); + + assert.equal(informer.log.firstCall.args[0].error, 'o.O'); + }); + + it(`should add "error" field with original selenium error if it exists in ${testStates[event]} test`, async () => { + const test = mkTest_({ + err: { + message: 'O.o', + seleniumStack: { + orgStatusMessage: 'some original message' + } + } + }); + + await createJsonlReporter(); + emit(RunnerEvents[event], test); + + assert.equal(informer.log.firstCall.args[0].error, 'O.o (some original message)'); + }); + }); + }); + + describe('skipped tests report', () => { + it('should inform about skipped test', async () => { + const test = mkTest_({ + fullTitle: () => 'suite test', + browserId: 'yabro', + file: path.resolve(process.cwd(), './path/to/test'), + sessionId: '12345', + duration: '100500', + startTime: Date.now(), + meta: {browserVersion: '99'} + }); + + await createJsonlReporter(); + emit(RunnerEvents.TEST_PENDING, test); + + assert.deepEqual(informer.log.firstCall.args[0], {...test, fullTitle: 'suite test', file: 'path/to/test', status: SKIPPED}); + }); + + it('should add skip comment', async () => { + const test = mkTest_({ + pending: true, + skipReason: 'some comment' + }); + + await createJsonlReporter(); + emit(RunnerEvents.TEST_PENDING, test); + + assert.equal(informer.log.firstCall.args[0].reason, 'some comment'); + }); + + it('should use parent skip comment if test suite was skipped', async () => { + const test = mkTest_({ + pending: true, + skipReason: 'test comment', + parent: { + skipReason: 'suite comment' + } + }); + + await createJsonlReporter(); + emit(RunnerEvents.TEST_PENDING, test); + + assert.equal(informer.log.firstCall.args[0].reason, 'suite comment'); + }); + + it('should use test skip comment if suite was skipped without comment', async () => { + const test = mkTest_({ + pending: true, + skipReason: 'test comment', + parent: {some: 'data'} + }); + + await createJsonlReporter(); + emit(RunnerEvents.TEST_PENDING, test); + + assert.equal(informer.log.firstCall.args[0].reason, 'test comment'); + }); + + it('should set undefined if test was skipped without comment', async () => { + const test = mkTest_({pending: true}); + + await createJsonlReporter(); + emit(RunnerEvents.TEST_PENDING, test); + + assert.isUndefined(informer.log.firstCall.args[0].reason); + }); + }); +});