From dfd6dd991853dffd350aeeb3baced3184b8865f3 Mon Sep 17 00:00:00 2001 From: DudaGod Date: Wed, 11 Sep 2024 00:18:16 +0300 Subject: [PATCH] docs(cli): add info about new command: "list-tests" --- docs/command-line/index.mdx | 448 +++++++++++++++++++++++++++++------- 1 file changed, 362 insertions(+), 86 deletions(-) diff --git a/docs/command-line/index.mdx b/docs/command-line/index.mdx index 88bc796..eae538b 100644 --- a/docs/command-line/index.mdx +++ b/docs/command-line/index.mdx @@ -2,89 +2,45 @@ import Admonition from "@theme/Admonition"; # CLI {#cli} -Testplane allows setting options via command line interface. +## `testplane` command -## Overview {#overview} - -Available options: - -``` --V, --version output the version number --c, --config path to configuration file --b, --browser run tests only in specified browser --s, --set run tests only in the specified set --r, --require require module ---reporter test reporters ---grep run only tests matching the pattern ---update-refs update screenshot references or gather them if they do not exist ("assertView" command) ---inspect [inspect] nodejs inspector on [=[host:]port] ---inspect-brk [inspect-brk] nodejs inspector with break at the start ---repl [type] run one test, call `browser.switchToRepl` in test code to open repl interface (default: false) ---repl-before-test [type] open repl interface before test run (default: false) ---repl-on-fail [type] open repl interface on test fail only (default: false) ---devtools switches the browser to the devtools mode with using CDP protocol --h, --help output usage information -``` - -## Overriding settings {#overriding-settings} - -All options can be overridden via command-line flags or environment variables with the following priorities, by descending: - -- A command-line option; - -- An environment variable; - -- A config file value; - -- The default value. - -### Overriding with CLI option {#overriding-cli-settings} - -To override a config setting with a CLI option, convert the full option path to `--kebab-case`. - -For example, to override config's `baseUrl` property: +Main command to run tests. ```bash -testplane path/to/mytest.js --base-url http://example.com +> testplane --help + + Usage: testplane [options] [command] [paths...] + + Run tests + + Options: + + -V, --version output the version number + -c, --config path to configuration file + -b, --browser run tests only in specified browser + -s, --set run tests only in the specified set + -r, --require require module + --grep run only tests matching the pattern + --reporter test reporters + --update-refs update screenshot references or gather them if they do not exist ("assertView" command) + --inspect [inspect] nodejs inspector on [=[host:]port] + --inspect-brk [inspect-brk] nodejs inspector with break at the start + --repl [type] run one test, call `browser.switchToRepl` in test code to open repl interface (default: false) + --repl-before-test [type] open repl interface before test run (default: false) + --repl-on-fail [type] open repl interface on test fail only (default: false) + --devtools switches the browser to the devtools mode with using CDP protocol + -h, --help output usage information ``` -It also works for nested properties. Example of overriding `browsers.firefox.sessionsPerBrowser` value (assuming you have defined browser with id "firefox"): +For example, ```bash -testplane path/to/mytest.js --browsers-firefox-sessions-per-browser 7 +npx testplane --config ./config.js --reporter flat --browser firefox --grep name ``` -### Overriding with environment variable {#overriding-env-settings} +### Options {#testplane-options} -To override a setting with an environment variable, convert its full path to `snake_case` and add the `testplane_` prefix. - -Examples of overriding the same `baseUrl` and `browsers.firefox.sessionsPerBrowser` values using environment variables instead of CLI options: - -```bash -testplane_base_url=http://example.com testplane path/to/mytest.js -testplane_browsers_firefox_sessions_per_browser=7 testplane path/to/mytest.js -``` - -Besides overriding config values, there also two extra environment variables: "TESTPLANE_SKIP_BROWSERS" and "TESTPLANE_SETS": - -#### TESTPLANE_SKIP_BROWSERS - -Skip the browsers specified in the config by passing the browser IDs. Multiple browser IDs should be separated by commas -(spaces after commas are allowed). - -```bash -TESTPLANE_SKIP_BROWSERS=ie10,ie11 testplane -``` - -#### TESTPLANE_SETS - -Runs only specified sets (CLI option `--set` alternative). - -```bash -TESTPLANE_SETS=desktop,touch testplane -``` - -## Version {#version} +#### Version {#testplane-version} Print current `testplane` version. @@ -92,7 +48,7 @@ Print current `testplane` version. testplane --version ``` -## Config {#config} +#### Config {#testplane-config} Use custom configuration file. @@ -100,7 +56,7 @@ Use custom configuration file. testplane --config ./local.testplane.config.ts ``` -## Browser {#browser} +#### Browser {#testplane-browser} Run tests only in specified browser. @@ -108,7 +64,7 @@ Run tests only in specified browser. testplane --browser chrome ``` -## Set {#set} +#### Set {#testplane-set} Run tests only in the specified set. @@ -116,7 +72,7 @@ Run tests only in the specified set. testplane --set desktop ``` -## Require {#require} +#### Require {#testplane-require} Load external modules, locally existing in your machine, before running `testplane`. This is useful for loaders, such as ECMAScript modules via [esm](https://www.npmjs.com/package/esm). @@ -124,7 +80,7 @@ Load external modules, locally existing in your machine, before running `testpla testplane --require ./tsconfig-register-paths.js ``` -## Reporter {#reporter} +#### Reporter {#testplane-reporter} Can be used to set one of the following reporters: @@ -148,11 +104,11 @@ testplane --reporter '{"type": "jsonl", "path": "./some-path/result.jsonl"}' --r Aside these terminal reporters, you can use [html-reporter][html-reporter] plugin in order to generate html reports. -## Grep {#grep} +#### Grep {#testplane-grep} Run only tests, which full name matches the pattern. -### Example {#grep-example} +##### Example {#testplane-grep-example} If you have some tests: @@ -177,7 +133,7 @@ testplane --grep "with nested path" testplane --grep "test with nested path" ``` -### Update refs {#update-refs} +#### Update refs {#testplane-update-refs} Run tests, updating all screenshot references, created by [assertView][assert-view] command. @@ -189,7 +145,7 @@ testplane --update-refs Recommended way to update screenshots is using [html-reporter][html-reporter] plugin. -## Inspect {#inspect} +#### Inspect {#testplane-inspect} Runs Testplane tests using [nodejs inspector](https://nodejs.org/en/docs/inspector). @@ -202,7 +158,7 @@ testplane --inspect this mode with option `sessionsPerBrowser=1` in order to debug tests one at a time. -## Inspect break {#inspect-brk} +#### Inspect break {#testplane-inspect-brk} The same as [Inspect](#inspect), but with breakpoint at the start. @@ -210,7 +166,7 @@ The same as [Inspect](#inspect), but with breakpoint at the start. testplane --inspect-brk ``` -## REPL {#repl} +#### REPL {#testplane-repl} Enables a [REPL](https://en.wikipedia.org/wiki/Read–eval–print_loop). It also disables test duration timeout. Can be used by specifying following CLI options: @@ -227,7 +183,7 @@ testplane --repl --grep 'my test name' --browser chrome command documentation. -## Devtools {#devtools} +#### Devtools {#testplane-devtools} Runs Testplane tests using [devtools automation protocol][webdriver-vs-cdp]. @@ -235,7 +191,7 @@ Runs Testplane tests using [devtools automation protocol][webdriver-vs-cdp]. testplane --devtools ``` -## Help {#help} +#### Help {#testplane-help} Prints out information about options and commands. Testplane Plugins can add their own commands and options. @@ -245,6 +201,326 @@ For example, [html-reporter][html-reporter] adds `gui` command. testplane --help ``` +## `list-tests` command + +Command to get list of tests in one of available formats (`list` or `tree`). + +```bash +> testplane list-tests --help + + Usage: list-tests [options] [paths...] + + Lists all tests info in one of available formats + + Options: + + -c, --config path to configuration file + -b, --browser list tests only in specified browser + -s, --set list tests only in the specified set + -r, --require require module + --grep list only tests matching the pattern + --ignore exclude paths from tests read + --silent [type] flag to disable events emitting while reading tests (default: false) + --output-file save results to specified file + --formatter [name] return tests in specified format (default: list) + -h, --help output usage information +``` + +For example, + +``` +npx testplane list-tests --config ./config.js --browser firefox --grep name --formatter tree +``` + +### Options {#list-tests-options} + +#### Config {#list-tests-config} + +Use custom configuration file. + +```bash +testplane list-tests --config ./local.testplane.config.ts +``` + +#### Browser {#list-tests-browser} + +List tests only in specified browser. + +```bash +testplane list-tests --browser chrome +``` + +#### Set {#list-tests-set} + +List tests only in the specified set. + +```bash +testplane list-tests --set desktop +``` + +#### Require {#list-tests-require} + +Load external modules, locally existing in your machine, before listing `testplane` tests. This is useful for loaders, such as ECMAScript modules via [esm](https://www.npmjs.com/package/esm). + +```bash +testplane list-tests --require ./tsconfig-register-paths.js +``` + +#### Grep {#list-tests-grep} + +List only tests, which full name matches the pattern. + +##### Example {#list-tests-grep-example} + +If you have some tests: + +```ts +describe("test", () => { + describe("with", () => { + describe("nested path", () => { + ... + }); + describe("other path", () => { + ... + }) + }); +}); +``` + +You can list tests inside "nested path" suite without listing tests inside "other path" suite with any of these variants: + +```bash +testplane list-tests --grep "nested path" +testplane list-tests --grep "with nested path" +testplane list-tests --grep "test with nested path" +``` + +#### Silent {#list-tests-silent} + +List tests silently (without emitting events). + +```bash +testplane list-tests --silent +``` + +#### Output file {#list-tests-output-file} + +Save listed tests to specified file. + +```bash +testplane list-tests --output-file "./tests.json" +``` + +#### Formatter {#list-tests-formatter} + +List tests in specified format. Available formatters: `list` (default) and `tree`. + +##### List formatter {#list-tests-formatter-list} + +List tests in `list` format. + +```bash +testplane list-tests --formatter "list" +``` + +For following tests: + +```typescript title="example.testplane.ts" +describe("suite1", () => { + it("test1", () => {}); +}); + +it("test2", () => {}); +``` + +Returns the following output: + +```json +[ + { + "id": "d2b3179", + "titlePath": [ + "suite", + "test1" + ], + "browserIds": [ + "yandex", + "chrome" + ], + "file": "tests/example.testplane.ts", + "pending": false, + "skipReason": "" + } + { + "id": "5a105e8", + "titlePath": [ + "test1" + ], + "browserIds": [ + "yandex", + "chrome" + ], + "file": "tests/example.testplane.ts", + "pending": false, + "skipReason": "" + } +] +``` + +Here, we got plain list of unique tests, where: + +- `id` (`String`) - unique identifier of the test; +- `titlePath` (`String[]`) - full name of the test. Each element of the array is the title of a suite or test. To get the full title, you need just join `titlePath` with single whitespace; +- `browserIds` (`String[]`) - list of browsers in which the test will be launched; +- `file` (`String`) - path to the file relative to the working directory; +- `pending` (`Boolean`) - flag that means if the test is disabled or not; +- `skipReason` (`String`) - the reason why the test was disabled. + +##### Tree formatter {#list-tests-formatter-tree} + +List tests in `tree` format. + +```bash +testplane list-tests --formatter "tree" +``` + +For following tests: + +```typescript title="example.testplane.ts" +describe("suite1", () => { + it("test1", () => {}); +}); + +it("test2", () => {}); +``` + +Returns the following output: + +```json +[ + { + "id": "36749990", + "title": "suite1", + "line": 1, + "column": 1, + "file": "example.testplane.ts", + "suites": [], + "tests": [ + { + "id": "d2b3179", + "title": "test1", + "line": 2, + "column": 5, + "browserIds": ["yandex", "chrome"] + } + ], + "pending": false, + "skipReason": "" + }, + { + "id": "5a105e8", + "title": "test2", + "line": 5, + "column": 1, + "browserIds": ["yandex", "chrome"], + "file": "example.testplane.ts", + "pending": false, + "skipReason": "" + } +] +``` + +Here, we got list of unique tests in the form of a tree structure (with parent suites), where `Suite` has following options: + +- `id` (`String`) - unique identifier of the suite; +- `title` (`String`) - unique identifier of the suite; +- `line` (`Number`) - line on which the suite was called; +- `column` (`Number`) - column on which the suite was called; +- `file` (`String`, only in topmost suite) - path to the file relative to the working directory; +- `suites` (`Suite[]`, exists only in suite) - list of child suites; +- `tests` (`Test[]`) - list of tests; +- `pending` (`Boolean`) - flag that means if the test is disabled or not; +- `skipReason` (`String`) - the reason why the test was disabled. + +And `Test` has following options: + +- `id` (`String`) - unique identifier of the test; +- `title` (`String`) - unique identifier of the test; +- `line` (`Number`) - line on which the test was called; +- `column` (`Number`) - column on which the test was called; +- `browserIds` (`String[]`) - list of browsers in which the test will be launched; +- `file` (`String`, only in tests without parent suites) - path to the file relative to the working directory; +- `pending` (`Boolean`) - flag that means if the test is disabled or not; +- `skipReason` (`String`) - the reason why the test was disabled. + +#### Help {#list-tests-help} + +Prints out information about options and commands. Testplane Plugins can add their own commands and options. + +For example, [html-reporter][html-reporter] adds `gui` command. + +```bash +testplane list-tests --help +``` + +## Overriding settings {#overriding-settings} + +All options can be overridden via command-line flags or environment variables with the following priorities, by descending: + +- A command-line option; + +- An environment variable; + +- A config file value; + +- The default value. + +### Overriding with CLI option {#overriding-cli-settings} + +To override a config setting with a CLI option, convert the full option path to `--kebab-case`. + +For example, to override config's `baseUrl` property: + +```bash +testplane path/to/mytest.js --base-url http://example.com +``` + +It also works for nested properties. Example of overriding `browsers.firefox.sessionsPerBrowser` value (assuming you have defined browser with id "firefox"): + +```bash +testplane path/to/mytest.js --browsers-firefox-sessions-per-browser 7 +``` + +### Overriding with environment variable {#overriding-env-settings} + +To override a setting with an environment variable, convert its full path to `snake_case` and add the `testplane_` prefix. + +Examples of overriding the same `baseUrl` and `browsers.firefox.sessionsPerBrowser` values using environment variables instead of CLI options: + +```bash +testplane_base_url=http://example.com testplane path/to/mytest.js +testplane_browsers_firefox_sessions_per_browser=7 testplane path/to/mytest.js +``` + +Besides overriding config values, there also two extra environment variables: "TESTPLANE_SKIP_BROWSERS" and "TESTPLANE_SETS": + +#### TESTPLANE_SKIP_BROWSERS + +Skip the browsers specified in the config by passing the browser IDs. Multiple browser IDs should be separated by commas +(spaces after commas are allowed). + +```bash +TESTPLANE_SKIP_BROWSERS=ie10,ie11 testplane +``` + +#### TESTPLANE_SETS + +Runs only specified sets (CLI option `--set` alternative). + +```bash +TESTPLANE_SETS=desktop,touch testplane +``` + [html-reporter]: ../html-reporter/html-reporter-setup [assert-view]: ../commands/browser/assertView [switch-to-repl]: ../commands/browser/switchToRepl