Skip to content

Commit

Permalink
Merge pull request #642 from gemini-testing/FEI-25418.add_jsonl_reporter
Browse files Browse the repository at this point in the history
Add jsonl reporter
  • Loading branch information
DudaGod authored Jun 9, 2022
2 parents e6b2cab + 08140fd commit d303061
Show file tree
Hide file tree
Showing 26 changed files with 1,087 additions and 265 deletions.
17 changes: 14 additions & 3 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -1370,11 +1370,22 @@ hermione --config ./config.js --reporter flat --browser firefox --grep name
**Note.** All CLI options override config values.

### Reporters
You can choose `flat` or `plain` reporter by option `-r, --reporter`. Default is `flat`.

* `flat` – all information about failed and retried tests would be grouped by browsers at the end of the report.
You can choose `flat`, `plain` or `jsonl` reporter by option `-r, --reporter`. Default is `flat`.
Information about test results is displayed to the command line by default. But there is an ability to redirect the output to a file, for example:
```
hermione --reporter '{"type": "jsonl", "path": "./some-path/result.jsonl"}'
```
In that example specified file path and all directories will be created automatically. Moreover you can use few reporters:
```
hermione --reporter '{"type": "jsonl", "path": "./some-path/result.jsonl"}' --reporter flat
```
* `plain` – information about fails and retries would be placed after each test.
Information about each report type:
* `flat` – all information about failed and retried tests would be grouped by browsers at the end of the report;
* `plain` – information about fails and retries would be placed after each test;
* `jsonl` - displays detailed information about each test result in [jsonl](https://jsonlines.org/) format.
### Require modules
Expand Down
2 changes: 1 addition & 1 deletion lib/cli/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -30,9 +30,9 @@ exports.run = () => {

program
.on('--help', () => logger.log(info.configOverriding))
.option('-r, --reporter <reporter>', 'test reporters', collect)
.option('-b, --browser <browser>', 'run tests only in specified browser', collect)
.option('-s, --set <set>', 'run tests only in the specified set', collect)
.option('-r, --reporter <reporter>', 'test reporters', collect)
.option('--require <module>', 'require module', collect)
.option('--grep <grep>', 'run only tests matching the pattern')
.option('--update-refs', 'update screenshot references or gather them if they do not exist ("assertView" command)')
Expand Down
6 changes: 6 additions & 0 deletions lib/constants/test-statuses.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
module.exports = {
SUCCESS: 'success',
FAIL: 'fail',
RETRY: 'retry',
SKIPPED: 'skipped'
};
25 changes: 3 additions & 22 deletions lib/hermione.js
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ const signalHandler = require('./signal-handler');
const TestReader = require('./test-reader');
const TestCollection = require('./test-collection');
const validateUnknownBrowsers = require('./validators').validateUnknownBrowsers;
const {initReporters} = require('./reporters');
const logger = require('./utils/logger');

module.exports = class Hermione extends BaseHermione {
Expand All @@ -25,7 +26,7 @@ module.exports = class Hermione extends BaseHermione {
this.emit(RunnerEvents.CLI, parser);
}

async run(testPaths, {browsers, sets, grep, updateRefs, requireModules, inspectMode, reporters} = {}) {
async run(testPaths, {browsers, sets, grep, updateRefs, requireModules, inspectMode, reporters = []} = {}) {
validateUnknownBrowsers(browsers, _.keys(this._config.browsers));

RuntimeConfig.getInstance().extend({updateRefs, requireModules, inspectMode});
Expand All @@ -36,7 +37,7 @@ module.exports = class Hermione extends BaseHermione {
.on(RunnerEvents.TEST_FAIL, () => this._fail())
.on(RunnerEvents.ERROR, (err) => this.halt(err));

_.forEach(reporters, (reporter) => applyReporter(this, reporter));
await initReporters(reporters, this);

eventsUtils.passthroughEvent(this._runner, this, _.values(RunnerEvents.getSync()));
eventsUtils.passthroughEventAsync(this._runner, this, _.values(RunnerEvents.getAsync()));
Expand Down Expand Up @@ -113,23 +114,3 @@ module.exports = class Hermione extends BaseHermione {
}, timeout).unref();
}
};

function applyReporter(runner, reporter) {
if (typeof reporter === 'string') {
try {
reporter = require('./reporters/' + reporter);
} catch (e) {
if (e.code === 'MODULE_NOT_FOUND') {
throw new Error('No such reporter: ' + reporter);
}
throw e;
}
}
if (typeof reporter !== 'function') {
throw new TypeError('Reporter must be a string or a function');
}

var Reporter = reporter;

new Reporter().attachRunner(runner);
}
25 changes: 18 additions & 7 deletions lib/reporters/base.js
Original file line number Diff line number Diff line change
@@ -1,12 +1,22 @@
'use strict';

const chalk = require('chalk');
const logger = require('../utils/logger');
const RunnerEvents = require('../constants/runner-events');
const icons = require('./utils/icons');
const helpers = require('./utils/helpers');
const {initInformer} = require('./informers');

module.exports = class BaseReporter {
static async create(opts = {}) {
const informer = await initInformer(opts);

return new this(informer, opts);
}

constructor(informer) {
this.informer = informer;
}

attachRunner(runner) {
runner.on(RunnerEvents.TEST_PASS, (test) => this._onTestPass(test));
runner.on(RunnerEvents.TEST_FAIL, (test) => this._onTestFail(test));
Expand All @@ -29,7 +39,7 @@ module.exports = class BaseReporter {

_onRetry(test) {
this._logTestInfo(test, icons.RETRY);
logger.log('Will be retried. Retries left: %s', chalk.yellow(test.retriesLeft));
this.informer.log(`Will be retried. Retries left: ${chalk.yellow(test.retriesLeft)}`);
}

_onTestPending(test) {
Expand All @@ -45,22 +55,23 @@ module.exports = class BaseReporter {
`Retries: ${chalk.yellow(stats.retries)}`
];

logger.log(message.join(' '));
const method = this.__proto__.hasOwnProperty('_onRunnerEnd') ? 'log' : 'end';
this.informer[method](message.join(' '));
}

_onWarning(info) {
logger.warn(info);
this.informer.warn(info);
}

_onError(error) {
logger.error(chalk.red(error));
this.informer.error(chalk.red(error));
}

_onInfo(info) {
logger.log(info);
this.informer.log(info);
}

_logTestInfo(test, icon) {
logger.log(`${icon}${helpers.formatTestInfo(test)}`);
this.informer.log(`${icon}${helpers.formatTestInfo(test)}`);
}
};
22 changes: 19 additions & 3 deletions lib/reporters/flat.js
Original file line number Diff line number Diff line change
@@ -1,11 +1,13 @@
'use strict';

const _ = require('lodash');
const BaseReporter = require('./base');
const helpers = require('./utils/helpers');
const icons = require('./utils/icons');

module.exports = class FlatReporter extends BaseReporter {
constructor() {
super();
constructor(...args) {
super(...args);

this._tests = [];
}
Expand All @@ -25,6 +27,20 @@ module.exports = class FlatReporter extends BaseReporter {
_onRunnerEnd(stats) {
super._onRunnerEnd(stats);

helpers.logFailedTestsInfo(this._tests);
const failedTests = helpers.formatFailedTests(this._tests);

failedTests.forEach((test, index) => {
this.informer.log(`\n${index + 1}) ${test.fullTitle}`);
this.informer.log(` in file ${test.file}\n`);

_.forEach(test.browsers, (testCase) => {
const icon = testCase.isFailed ? icons.FAIL : icons.RETRY;

this.informer.log(` ${testCase.browserId}`);
this.informer.log(` ${icon} ${testCase.error}`);
});
});

this.informer.end();
}
};
100 changes: 100 additions & 0 deletions lib/reporters/index.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,100 @@
exports.initReporters = async (rawReporters, runner) => {
await Promise.all([].concat(rawReporters).map((rawReporter) => applyReporter(rawReporter, runner)));
};

const reporterHandlers = [
{
isMatched: (rawReporter) => typeof rawReporter === 'string' && isJSON(rawReporter),
initReporter: (rawReporter) => initReporter(getReporterDefinition(rawReporter, JSON.parse))
},
{
isMatched: (rawReporter) => typeof rawReporter === 'string',
initReporter: (rawReporter) => initReporter({...getReporterDefinition(rawReporter), type: rawReporter})
},
{
isMatched: (rawReporter) => typeof rawReporter === 'object',
initReporter: (rawReporter) => initReporter(getReporterDefinition(rawReporter, (v) => v))
},
{
isMatched: (rawReporter) => typeof rawReporter === 'function',
initReporter: (rawReporter) => {
validateReporter(rawReporter);
return rawReporter.create(getReporterDefinition(rawReporter));
}
},
{
isMatched: () => true,
initReporter: (rawReporter) => {
throw new TypeError(`Specified reporter must be a string, object or function, but got: "${typeof rawReporter}"`);
}
}
];

async function applyReporter(rawReporter, runner) {
for (const handler of reporterHandlers) {
if (!handler.isMatched(rawReporter)) {
continue;
}

const reporter = await handler.initReporter(rawReporter);

if (typeof reporter.attachRunner !== 'function') {
throw new TypeError(
'Initialized reporter must have an "attachRunner" function for subscribe on test result events'
);
}

return reporter.attachRunner(runner);
}
}

function initReporter(reporter) {
let Reporter;

try {
Reporter = require(`./${reporter.type}`);
} catch (e) {
if (e.code === 'MODULE_NOT_FOUND') {
throw new Error(`No such reporter: "${reporter.type}"`);
}
throw e;
}

validateReporter(Reporter);

return Reporter.create(reporter);
}

function getReporterDefinition(rawReporter, parser) {
if (!parser) {
return {type: null, path: null};
}

const {type, path} = parser(rawReporter);

if (!type) {
const strRawReporter = typeof rawReporter !== 'string' ? JSON.stringify(rawReporter) : rawReporter;
throw new Error(`Failed to find required "type" field in reporter definition: "${strRawReporter}"`);
}

return {type, path};
}

function validateReporter(Reporter) {
if (typeof Reporter !== 'function') {
throw new TypeError(`Imported reporter must be a function, but got: "${typeof Reporter}"`);
}

if (typeof Reporter.create !== 'function') {
throw new TypeError('Imported reporter must have a "create" function for initialization');
}
}

function isJSON(str) {
try {
JSON.parse(str);
} catch (e) {
return false;
}
return true;
}
21 changes: 21 additions & 0 deletions lib/reporters/informers/base.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
module.exports = class BaseInformer {
static create(...args) {
return new this(...args);
}

log() {
throw new Error('Method must be implemented in child classes');
}

warn() {
throw new Error('Method must be implemented in child classes');
}

error() {
throw new Error('Method must be implemented in child classes');
}

end() {
throw new Error('Method must be implemented in child classes');
}
};
22 changes: 22 additions & 0 deletions lib/reporters/informers/console.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
const BaseInformer = require('./base');
const logger = require('../../utils/logger');

module.exports = class ConsoleInformer extends BaseInformer {
log(message) {
logger.log(message);
}

warn(message) {
logger.warn(message);
}

error(message) {
logger.error(message);
}

end(message) {
if (message) {
logger.log(message);
}
}
};
41 changes: 41 additions & 0 deletions lib/reporters/informers/file.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
const fs = require('fs');
const chalk = require('chalk');
const BaseInformer = require('./base');
const logger = require('../../utils/logger');

module.exports = class FileInformer extends BaseInformer {
constructor(opts) {
super(opts);

this._fileStream = fs.createWriteStream(opts.path);
this._reporterType = opts.type;

logger.log(`Information with test results for report: "${opts.type}" will be saved to a file: "${opts.path}"`);
}

log(message) {
this._fileStream.write(`${this._prepareMsg(message)}\n`);
}

warn(message) {
this.log(message);
}

error(message) {
this.log(message);
}

end(message) {
if (message) {
this._fileStream.end(`${this._prepareMsg(message)}\n`);
} else {
this._fileStream.end();
}
}

_prepareMsg(msg) {
return typeof msg === 'object'
? JSON.stringify(msg)
: chalk.stripColor(msg);
}
};
12 changes: 12 additions & 0 deletions lib/reporters/informers/index.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
const path = require('path');
const fs = require('fs-extra');

exports.initInformer = async (opts) => {
if (opts.path) {
await fs.ensureDir(path.dirname(opts.path));
}

const informerType = opts.path ? 'file' : 'console';

return require(`./${informerType}`).create(opts);
};
Loading

0 comments on commit d303061

Please sign in to comment.