diff --git a/docs/guides/how-to-add-testing-library.mdx b/docs/guides/how-to-add-testing-library.mdx index 2943e84..ba56f48 100644 --- a/docs/guides/how-to-add-testing-library.mdx +++ b/docs/guides/how-to-add-testing-library.mdx @@ -6,7 +6,7 @@ ## Подключение -Для того, чтобы появилась возможность использования [запросов (queries)][queries] из `testing-library` в testplane-тестах, необходимо выполнить всего несколько шагов. +Для того чтобы появилась возможность использования [запросов (queries)][queries] из `testing-library` в testplane-тестах, необходимо выполнить всего несколько шагов. 1. Установите npm-пакет `'@testing-library/webdriverio'` @@ -56,6 +56,8 @@ it("example", async ({ browser }) => { }); ``` +Полный пример использования можно найти [здесь](https://github.com/gemini-testing/testplane/tree/master/examples/). + ## Полезные ссылки {#useful_links} - [Testing-library][testing-library] diff --git a/docs/guides/how-to-get-rid-of-diffs-in-screenshots.mdx b/docs/guides/how-to-get-rid-of-diffs-in-screenshots.mdx deleted file mode 100644 index 2b0f076..0000000 --- a/docs/guides/how-to-get-rid-of-diffs-in-screenshots.mdx +++ /dev/null @@ -1,6 +0,0 @@ -# Как победить диффы в скриншотах - -## Ключевые слова {#keywords} - -- hermione-reassert-view -- ignore в assertView diff --git a/docs/guides/how-to-mitigate-flaky-tests.mdx b/docs/guides/how-to-mitigate-flaky-tests.mdx deleted file mode 100644 index 02d6632..0000000 --- a/docs/guides/how-to-mitigate-flaky-tests.mdx +++ /dev/null @@ -1,7 +0,0 @@ -# Как победить нестабильные тесты - -## Ключевые слова {#keywords} - -- testplane-muted-tests -- testplane-retry-command -- testplane-retry-progressive diff --git a/docs/guides/how-to-optimize-test-running.mdx b/docs/guides/how-to-optimize-test-running.mdx deleted file mode 100644 index 3544291..0000000 --- a/docs/guides/how-to-optimize-test-running.mdx +++ /dev/null @@ -1,9 +0,0 @@ -# Как ускорить прогон тестов - -## Ключевые слова {#keywords} - -- testsPerSession -- sessionsPerBrowser -- workers -- testplane-chunks -- retry-limiter diff --git a/docs/guides/how-to-profile-tests.mdx b/docs/guides/how-to-profile-tests.mdx deleted file mode 100644 index 0de3423..0000000 --- a/docs/guides/how-to-profile-tests.mdx +++ /dev/null @@ -1,6 +0,0 @@ -# Как проанализировать скорость тестов - -## Ключевые слова {#keywords} - -- testplane-profiler -- testplane-plugins-profiler diff --git a/docs/guides/how-to-remove-useless-screenshots.mdx b/docs/guides/how-to-remove-useless-screenshots.mdx deleted file mode 100644 index 64d6008..0000000 --- a/docs/guides/how-to-remove-useless-screenshots.mdx +++ /dev/null @@ -1,5 +0,0 @@ -# Как удалить лишние скриншоты - -## Ключевые слова {#keywords} - -- html-reporter diff --git a/docs/guides/how-to-write-your-own-plugin.mdx b/docs/guides/how-to-write-your-own-plugin.mdx deleted file mode 100644 index 785d3ae..0000000 --- a/docs/guides/how-to-write-your-own-plugin.mdx +++ /dev/null @@ -1 +0,0 @@ -# Как написать свой собственный плагин для testplane diff --git a/docs/plugins/hermione-reassert-view.mdx b/docs/plugins/hermione-reassert-view.mdx deleted file mode 100644 index 9eddeab..0000000 --- a/docs/plugins/hermione-reassert-view.mdx +++ /dev/null @@ -1 +0,0 @@ -Плагин [hermione-reassert-view](https://github.com/gemini-testing/hermione-reassert-view/blob/master/README.md) diff --git a/docs/plugins/testplane-passive-browsers.mdx b/docs/plugins/testplane-passive-browsers.mdx deleted file mode 100644 index b79de0d..0000000 --- a/docs/plugins/testplane-passive-browsers.mdx +++ /dev/null @@ -1 +0,0 @@ -Плагин [testplane-passive-browsers](https://github.com/gemini-testing/testplane-passive-browsers/blob/master/README.md) diff --git a/docs/plugins/testplane-plugins-profiler.mdx b/docs/plugins/testplane-plugins-profiler.mdx deleted file mode 100644 index b1f7f86..0000000 --- a/docs/plugins/testplane-plugins-profiler.mdx +++ /dev/null @@ -1 +0,0 @@ -Плагин [testplane-plugins-profiler](https://github.com/gemini-testing/testplane-plugins-profiler/blob/master/README.md) diff --git a/docs/plugins/testplane-tabs-closer.mdx b/docs/plugins/testplane-tabs-closer.mdx deleted file mode 100644 index c93a0d9..0000000 --- a/docs/plugins/testplane-tabs-closer.mdx +++ /dev/null @@ -1 +0,0 @@ -Плагин [testplane-tabs-closer](https://github.com/gemini-testing/testplane-tabs-closer/blob/master/README.md) diff --git a/docs/plugins/testplane-test-filter.mdx b/docs/plugins/testplane-test-filter.mdx deleted file mode 100644 index 64af2ae..0000000 --- a/docs/plugins/testplane-test-filter.mdx +++ /dev/null @@ -1 +0,0 @@ -Плагин [testplane-test-filter](https://github.com/gemini-testing/testplane-test-filter/blob/master/README.md) diff --git a/docusaurus.config.ts b/docusaurus.config.ts index a818a11..9bbe141 100644 --- a/docusaurus.config.ts +++ b/docusaurus.config.ts @@ -145,7 +145,7 @@ const config: Config = { }, { label: "Plugins", - href: "/docs/v8/guides/how-to-write-your-own-plugin", + href: "/docs/v8/plugins/hermione-browser-version-changer", }, ], }, diff --git a/i18n/en/docusaurus-plugin-content-blog/component-testing.mdx b/i18n/en/docusaurus-plugin-content-blog/component-testing.mdx new file mode 100644 index 0000000..7bc02cd --- /dev/null +++ b/i18n/en/docusaurus-plugin-content-blog/component-testing.mdx @@ -0,0 +1,185 @@ +--- +title: Component Testing +slug: component-testing-intro +hide_table_of_contents: false +date: 2024-06-10T14:00 +--- + +import Admonition from "@theme/Admonition"; + +In testplane, experimental support for component testing and unit tests running in the browser has been added. + + + +Almost all modern web interfaces are built using frameworks (React, Vue, Svelte, ...) to simplify the creation and reuse of components. It is important to test such components in isolation from each other to ensure that each component correctly performs its function. Just as we write unit tests separately from integration tests. Testplane already supports screenshot testing of components using [Storybook](https://storybook.js.org/), but this tool is not relevant for all projects. Therefore, we developed another option for component testing that does not require the use of Storybook. + +This feature can be useful if your project uses React components. At the same time, there are no tests at all or only heavy integration tests are used (i.e., entire pages containing many components are checked). According to the [testing pyramid](https://martinfowler.com/articles/practical-test-pyramid.html), there should be fewer integration tests because they are more prone to "flaps" and often redundant. Many scenarios can be checked using component tests, thereby reducing test execution time in CI and improving their stability. + +### Component Testing Implementation Options + +Component testing is a type of testing where the logic of a web component is checked in isolation from the web page in which it is used. To perform such a test, you need to be able to correctly render the component. Often, [JSDom](https://github.com/jsdom/jsdom) (used in [Jest](https://jestjs.io/)) is used for this task, which renders web components using virtual rendering in Node.js, i.e., without using a real browser. On the one hand, this works faster (the browser is not launched), but on the other hand, it is less stable because the checks are not performed in a real browser. The second popular solution is to use a very fast dev server [Vite](https://vitejs.dev/), which supports many frameworks (React, Vue, Svelte, ...) and is responsible for rendering components in isolation. + +We chose the option of using Vite because this approach provides testing of the page closer to reality (as if it were opened by a user). At the same time, the tests themselves run a little longer than in jsdom. But for us, the most important thing is the stability and reproducibility of test results, so the choice was obvious. + +
+ Brief information on how this is implemented + + - when specifying the `testRunEnv: 'browser'` option in the Testplane config, a browser runner will be used, which launches Vite on localhost with a random free port (the user can set the required port in the Vite config). It is on this launched server that all user components will be rendered and all necessary commands/checks will be performed (i.e., directly inside the browser); + - then the tests are read in Node.js, i.e., as it is done for integration tests. This is necessary for all plugins to work correctly (we are talking about event triggers when reading tests), as well as to be able to run tests from one file in parallel. If the test were read only in the browser context, it would be necessary to run absolutely all tests inside one file, and a critical failure in one of them would lead to the stopping of all subsequent ones. That is, at this stage, we understand which tests need to be run; + - after that, as usual, the necessary browsers are launched, and the tests are run in them. Each test, before executing user code, navigates to the launched Vite server. When such a request is made, a special index.html is generated, into which all necessary libraries are loaded: + - mocha — for reading tests; + - webdriverio — for using the browser instance inside the browser itself; + - expect — for performing checks; + - and other internal modules necessary for correct operation. + - when opening index.html from Vite, the browser establishes a websocket connection with the Testplane master process to exchange the necessary data. For example, if the construct `await browser.$("body").assertView("plain", "body")` is called in the browser, it is obvious that it cannot be executed in the browser itself, as access to the file system is required inside `assertView`. Therefore, the execution of this command is sent to the Testplane master, which in turn sends it to the worker in which this test is running. And it is the worker that executes the command passed to it. When the result is obtained, it is sent back to the browser in the same way. All communication is implemented using the [socket.io](https://socket.io/) library; + - after that, the specified test begins to run in the browser, which returns the result to the Node.js process upon completion. + +
+ +### How to use? + +We will set up testing of react components written in TypeScript. Therefore, first, we will install the necessary dependencies: + +```bash +npm i testplane vite @vitejs/plugin-react @testing-library/react --save-dev +npm i react --save +``` + +Create a Vite config in which we connect the plugin to support React. Example: + +```javascript +// vite.config.ts +import { defineConfig } from "vite"; +import react from "@vitejs/plugin-react"; + +export default defineConfig({ + plugins: [react()], +}); +``` + +Now we will set up running tests in the browser. To do this, we will specify the [testRunEnv](https://github.com/gemini-testing/testplane/blob/master/docs/config.md#testrunenv) option. Example: + +```javascript +// .testplane.conf.ts +export const { + // ... + system: { + // ... + testRunEnv: ['browser', { viteConfig: './vite.config.ts' }], + }, + sets: { + linux: { + files: [ + 'src/tests/**/*.testplane.tsx' + ], + browsers: [ + 'chrome' + ] + }, + }, +} +``` + +After that, we can write a test in which we simply output the value of `document` to the console without using the [browser.execute](/docs/v8/commands/browser/execute) command: + +```javascript +// src/tests/test.testplane.tsx +it("should log document", async () => { + console.log(document); +}); +``` + +If such a test were run in the Node.js environment, it would fail with the error `ReferenceError: document is not defined`. But in our case, it will be executed directly in the browser, where the global variable `document` is available. Therefore, in the browser log and terminal (we will talk about this feature below), we will see the following: + +``` +{ + location: { + ancestorOrigins: {}, + href: 'http://localhost:56292/run-uuids/23d2af81-4259-425c-8214-c9e770d75ea4', + origin: 'http://localhost:56292', + protocol: 'http:', + host: 'localhost:56292', + hostname: 'localhost', + port: '56292', + pathname: '/run-uuids/23d2af81-4259-425c-8214-c9e770d75ea4', + search: '', + hash: '' + } +} +``` + +Let's write a more complex test with rendering a React component. To do this, first, we will write a small component: + +```javascript +// src/components/Component.tsx +import { useState } from "react"; + +// Simple component with a title and a counter button +function Component() { + const [count, setCount] = useState(0); + + return ( +
+

Testplane Component Testing!

+ +
+ ); +} + +export default Component; +``` + +And write the test itself, which will test our React component: + +```javascript +// src/tests/test.testplane.tsx +import { render } from "@testing-library/react"; +import Component from "../components/Component"; + +it("should render react button", async ({ browser }) => { + render(); // render the component on the generated Vite page + + const button = await browser.$("button"); + + await button.click(); + await button.click(); + + await expect(button).toHaveText("count is 2"); +}); +``` + +You can find fully working examples [here](https://github.com/gemini-testing/testplane/tree/master/examples/component-testing). + + + - only components written in React in `.jsx` and `.tsx` files are supported. Vue support is also + planned; - no access to `currentTest` from hooks and tests; - the @testplane/global-hook plugin + is temporarily not supported. + + +### What additional features are supported? + +#### 1. Hot Module Replacement (HMR) + +Vite supports [HMR](https://vitejs.dev/guide/api-hmr.html). This means that if a loaded file is changed, either the component will be remounted, or the page will be fully reloaded. If the component is described in a separate file (i.e., not in the same file as the test), it will be remounted, but the test will not be restarted. And if the test file is changed, the page will be reloaded, which will cause Testplane to interrupt the current test and start it again. Thanks to this feature in Vite, you can very quickly develop components and write tests for them. It is recommended to use it together with the REPL mode. + +When changing the component's source code, the test is not fully restarted (only the component itself is remounted). However, if the test code is changed, a full restart occurs. + +#### 2. Using the browser instance and expect directly in the browser's DevTools + +In the browser console where the test is running, the `browser` instance and the `expect` instance are available. This is quite convenient to use when debugging the test. + +#### 3. Logs from the browser console in the terminal + +Calling the `log`, `info`, `warn`, `error`, `debug`, and `table` commands on the `console` object in the browser results in the information being displayed not only in the browser's DevTools but also in the terminal from which Testplane was launched. That is, you can call `console.log` in the test/component and then see the result of its execution in the terminal. This is also quite convenient when debugging the test. + +### Conclusion + +This functionality provides our users with new capabilities: + +- isolated testing of React components in a real browser; +- stability and reproducibility of test results compared to JSDom; +- HMR support; +- access to browser/expect instances in the browser's DevTools for convenient debugging; +- log display in the terminal to enhance comfort and speed up development. + +Switch to Testplane and try the new feature yourself. If you encounter any problems, come to the [issue github](https://github.com/gemini-testing/testplane/issues) — we will definitely help you! diff --git a/i18n/en/docusaurus-plugin-content-blog/rebranding.mdx b/i18n/en/docusaurus-plugin-content-blog/rebranding.mdx new file mode 100644 index 0000000..4c94774 --- /dev/null +++ b/i18n/en/docusaurus-plugin-content-blog/rebranding.mdx @@ -0,0 +1,29 @@ +--- +title: Rebranding +slug: testplane-rebranding +hide_table_of_contents: false +date: 2024-05-20T13:00 +--- + +We present to your attention... **Testplane**. Our project Hermione has decided to change its name — meet the time-tested product in a fresh look! + + + +We carefully considered all the pros and cons and concluded that the new name — Testplane — best reflects our vision and future development of the product: + +1. We plan to actively invest in the development of the tool in open source. The new name symbolizes a qualitative leap in the development of the tool. +2. We aim to create a full-fledged brand with a trademark, logo, and corporate visual style. Testplane is a name that both references a "test flight" and can be read as "a plane for testing." + +The essence of the product remains unchanged. Testplane is the same project you have known for many years as Hermione, but in a fresh, updated form. To simplify the transition, we decided to maintain continuous version numbering: Testplane v8.x is equivalent to Hermione v8.x. + +If you are already working with Hermione, updating to Testplane will take just a few minutes. We designed it as a drop-in replacement: + +- Full support for existing Hermione plugins. +- Understanding of all Hermione options and environment variables. +- Full configuration compatibility. +- Two binary files (testplane and hermione) for a smooth transition (you can run both as hermione gui and as testplane gui). +- Type export for TypeScript, including both Testplane's own types and Hermione's types (including hermioneCtx). + +If you haven't tried Testplane yet — check out our documentation and start your exploration with Testplane in just a few minutes! + +Thank you for your interest in the project! ✈️ diff --git a/i18n/en/docusaurus-plugin-content-docs/current/commands/browser/executionContext.mdx b/i18n/en/docusaurus-plugin-content-docs/current/commands/browser/executionContext.mdx index 471af96..e9f8772 100644 --- a/i18n/en/docusaurus-plugin-content-docs/current/commands/browser/executionContext.mdx +++ b/i18n/en/docusaurus-plugin-content-docs/current/commands/browser/executionContext.mdx @@ -4,12 +4,12 @@ import Admonition from "@theme/Admonition"; ## Overview {#overview} -During the test run, testplane adds an `executionContext` property to `browser` to store the execution context. +During the test run, Testplane adds an `executionContext` property to `browser` to store the execution context. The `browser.executionContext` property holds the current test or hook \_Mocha object, supplemented with a browser identifier. - The execution context is added by testplane and is not available in the [API + The execution context is added by Testplane and is not available in the [API WebDriverIO][webdriverio-api]. diff --git a/i18n/en/docusaurus-plugin-content-docs/current/commands/browser/getMeta.mdx b/i18n/en/docusaurus-plugin-content-docs/current/commands/browser/getMeta.mdx index 5cd34cb..1619f97 100644 --- a/i18n/en/docusaurus-plugin-content-docs/current/commands/browser/getMeta.mdx +++ b/i18n/en/docusaurus-plugin-content-docs/current/commands/browser/getMeta.mdx @@ -12,7 +12,7 @@ If the key is not specified, the command will return an object with all the meta To set values in the metadata, use the [setMeta](../setMeta) command. - This command is implemented within testplane and is not available in the [API + This command is implemented within Testplane and is not available in the [API WebDriverIO][webdriverio-api]. diff --git a/i18n/en/docusaurus-plugin-content-docs/current/commands/browser/setMeta.mdx b/i18n/en/docusaurus-plugin-content-docs/current/commands/browser/setMeta.mdx index d6c7f91..3ea4575 100644 --- a/i18n/en/docusaurus-plugin-content-docs/current/commands/browser/setMeta.mdx +++ b/i18n/en/docusaurus-plugin-content-docs/current/commands/browser/setMeta.mdx @@ -9,7 +9,7 @@ Use the `setMeta` command to write a value under a specified key in the test's m To read the metadata, use the [getMeta](../getMeta) command. - This command is implemented within testplane and is not available in the [API + This command is implemented within Testplane and is not available in the [API WebDriverIO][webdriverio-api]. diff --git a/i18n/en/docusaurus-plugin-content-docs/current/commands/expect/browser-matchers.mdx b/i18n/en/docusaurus-plugin-content-docs/current/commands/expect/browser-matchers.mdx new file mode 100644 index 0000000..6cba125 --- /dev/null +++ b/i18n/en/docusaurus-plugin-content-docs/current/commands/expect/browser-matchers.mdx @@ -0,0 +1,51 @@ +# expect for the browser + +## toHaveUrl + +Checks if the browser is on the specified page. + +For example: + +```javascript +await browser.url("https://webdriver.io/"); +await expect(browser).toHaveUrl("https://webdriver.io"); +``` + +## toHaveUrlContaining + +Checks if the specified substring is contained in the URL of the page the browser is on. + +For example: + +```javascript +await browser.url("https://webdriver.io/"); +await expect(browser).toHaveUrlContaining("webdriver"); +``` + +## toHaveTitle + +Checks if the website has the specified title. + +For example: + +```javascript +await browser.url("https://webdriver.io/"); +await expect(browser).toHaveTitle( + "WebdriverIO · Next-gen browser and mobile automation test framework for Node.js", +); +``` + +## toHaveTitleContaining + +Checks if the specified substring is contained in the website's title. + +For example: + +```javascript +await browser.url("https://webdriver.io/"); +await expect(browser).toHaveTitleContaining("WebdriverIO"); +``` + +## References + +We'd like to give credit to the original WebdriverIO docs [article](https://webdriver.io/docs/api/expect-webdriverio), from which we drew some information while writing our version. diff --git a/i18n/en/docusaurus-plugin-content-docs/current/commands/expect/element-matchers.mdx b/i18n/en/docusaurus-plugin-content-docs/current/commands/expect/element-matchers.mdx new file mode 100644 index 0000000..1d0696c --- /dev/null +++ b/i18n/en/docusaurus-plugin-content-docs/current/commands/expect/element-matchers.mdx @@ -0,0 +1,357 @@ +# expect for elements + +## toBeDisplayed + +Calls [isDisplayed](../../element/isDisplayed) on the given element. + +For example: + +```javascript +const elem = await browser.$("#someElem"); +await expect(elem).toBeDisplayed(); +``` + +## toExist + +Calls [isExisting](../../element/isExisting) on the given element. + +For example: + +```javascript +const elem = await browser.$("#someElem"); +await expect(elem).toExist(); +``` + +## toBePresent + +Same as [toExist](#toexist). + +For example: + +```javascript +const elem = await browser.$("#someElem"); +await expect(elem).toBePresent(); +``` + +## toBeExisting + +Same as [toExist](#toexist). + +For example: + +```javascript +const elem = await browser.$("#someElem"); +await expect(elem).toBeExisting(); +``` + +## toBeFocused + +Checks if the element has focus. This assertion works only in the web context. + +For example: + +```javascript +const elem = await browser.$("#someElem"); +await expect(elem).toBeFocused(); +``` + +## toHaveAttribute + +Checks if the element has the given attribute with the specified value. + +For example: + +```javascript +const myInput = await browser.$("input"); +await expect(myInput).toHaveAttribute("class", "form-control"); +``` + +## toHaveAttr + +Same as [toHaveAttribute](#tohaveattribute). + +For example: + +```javascript +const myInput = await browser.$("input"); +await expect(myInput).toHaveAttr("class", "form-control"); +``` + +## toHaveAttributeContaining + +Checks if the given substring is contained in the specified attribute's value of the element. + +For example: + +```javascript +const myInput = await browser.$("input"); +await expect(myInput).toHaveAttributeContaining("class", "form"); +``` + +## toHaveAttrContaining + +Same as [toHaveAttributeContaining](#tohaveattrcontaining). + +For example: + +```javascript +const myInput = await browser.$("input"); +await expect(myInput).toHaveAttrContaining("class", "form"); +``` + +## toHaveElementClass + +Checks if the element has the specified class name. + +For example: + +```javascript +const myInput = await browser.$("input"); +await expect(myInput).toHaveElementClass("form-control", { message: "Not a form control!" }); +``` + +## toHaveElementClassContaining + +Checks if the element's class name contains the specified substring. + +For example: + +```javascript +const myInput = await browser.$("input"); +await expect(myInput).toHaveElementClassContaining("form"); +``` + +## toHaveElementProperty + +Checks if the element has the specified property with the given value. + +For example: + +```javascript +const elem = await browser.$("#elem"); +await expect(elem).toHaveElementProperty("height", 23); +await expect(elem).not.toHaveElementProperty("height", 0); +``` + +## toHaveValue + +Checks if an input element has the given value. + +For example: + +```javascript +const myInput = await browser.$("input"); +await expect(myInput).toHaveValue("user", { ignoreCase: true }); +``` + +## toHaveValueContaining + +Checks if the given substring is contained in the specified input element's value. + +For example: + +```javascript +const myInput = await browser.$("input"); +await expect(myInput).toHaveValueContaining("us"); +``` + +## toBeClickable + +Checks if the element is clickable by calling [isClickable](../../element/isClickable). + +For example: + +```javascript +const elem = await browser.$("#elem"); +await expect(elem).toBeClickable(); +``` + +## toBeDisabled + +Checks if the element is disabled by calling [isEnabled](../../element/isEnabled). + +For example: + +```javascript +const elem = await browser.$("#elem"); +await expect(elem).toBeDisabled(); +// or, equivalently: +await expect(elem).not.toBeEnabled(); +``` + +## toBeEnabled + +Checks if the element is enabled by calling [isEnabled](../../element/isEnabled). + +For example: + +```javascript +const elem = await browser.$("#elem"); +await expect(elem).toBeEnabled(); +// or, equivalently: +await expect(elem).not.toBeDisabled(); +``` + +## toBeSelected + +Checks if the element is selected by calling [isSelected](../../element/isSelected). + +For example: + +```javascript +const elem = await browser.$("#elem"); +await expect(elem).toBeSelected(); +``` + +## toBeChecked + +Same as [toBeSelected](#tobeselected). + +For example: + +```javascript +const elem = await browser.$("#elem"); +await expect(elem).toBeChecked(); +``` + +## toHaveHref + +Checks if the element link has the specified URL. + +For example: + +```javascript +const link = await browser.$("a"); +await expect(link).toHaveHref("https://webdriver.io"); +``` + +## toHaveLink + +Same as [toHaveHref](#tohavehref). + +For example: + +```javascript +const link = await browser.$("a"); +await expect(link).toHaveLink("https://webdriver.io"); +``` + +## toHaveHrefContaining + +Checks if the given substring is contained in the element link's URL. + +For example: + +```javascript +const link = await browser.$("a"); +await expect(link).toHaveHrefContaining("webdriver.io"); +``` + +## toHaveLinkContaining + +Same as [toHaveHrefContaining](#tohavehrefcontaining). + +For example: + +```javascript +const link = await browser.$("a"); +await expect(link).toHaveLinkContaining("webdriver.io"); +``` + +## toHaveId + +Checks if the element has the given ID. + +For example: + +```javascript +const elem = await browser.$("#elem"); +await expect(elem).toHaveId("elem"); +``` + +## toHaveText + +Checks if the element's text matches the specified value. + +It can also be called with an array as a parameter in case the element can have different texts. + +For example: + +```javascript +await browser.url("https://webdriver.io/"); +const elem = await browser.$(".container"); +await expect(elem).toHaveText("Next-gen browser and mobile automation test framework for Node.js"); +await expect(elem).toHaveText([ + "Next-gen browser and mobile automation test framework for Node.js", + "Get Started", +]); +``` + +## toHaveTextContaining + +Checks if the element's text contains the specified substring. + +It can also be called with an array as a parameter in case the element can have different texts. + +For example: + +```javascript +await browser.url("https://webdriver.io/"); +const elem = await browser.$(".container"); +await expect(elem).toHaveTextContaining("browser and mobile automation test framework"); +await expect(elem).toHaveTextContaining([ + "browser and mobile automation test framework", + "Started", +]); +``` + +## toBeDisplayedInViewport + +Checks if the element is within the viewport using the [isDisplayedInViewport](../../element/isDisplayedInViewport) command. + +For example: + +```javascript +const elem = await browser.$("#elem"); +await expect(elem).toBeDisplayedInViewport(); +``` + +## toHaveChildren + +Checks the number of child elements of the given element using the [element.$('./\*')](../../element/_dollar) command. + +For example: + +```javascript +const list = await browser.$("ul"); +await expect(list).toHaveChildren(); // list has at least 1 element +// or, equivalently: +await expect(list).toHaveChildren({ gte: 1 }); +``` + +```javascript +await expect(list).toHaveChildren(3); // list has 3 elements +// or, equivalently: +await expect(list).toHaveChildren({ eq: 3 }); +``` + +## toBeElementsArrayOfSize + +Checks the number of elements obtained using the [$$](../../element/_dollardollar) command. + +For example: + +```javascript +const listItems = await browser.$$("ul>li"); +await expect(listItems).toBeElementsArrayOfSize(5); // 5 items in the list +``` + +```javascript +await expect(listItems).toBeElementsArrayOfSize({ lte: 10 }); +// or, equivalently: +assert.ok(listItems.length <= 10); +``` + +## References + +We'd like to give credit to the original WebdriverIO docs [article](https://webdriver.io/docs/api/expect-webdriverio), from which we drew some information while writing our version. diff --git a/i18n/en/docusaurus-plugin-content-docs/current/commands/expect/mock-matchers.mdx b/i18n/en/docusaurus-plugin-content-docs/current/commands/expect/mock-matchers.mdx new file mode 100644 index 0000000..b565434 --- /dev/null +++ b/i18n/en/docusaurus-plugin-content-docs/current/commands/expect/mock-matchers.mdx @@ -0,0 +1,66 @@ +# expect for network mocks + +## toBeRequested + +Checks if the specified request was made. + +For example: + +```javascript +const mock = browser.mock("**/api/todo*"); +await expect(mock).toBeRequested(); +``` + +## toBeRequestedTimes + +Checks if the specified request was made the expected number of times. + +For example: + +```javascript +const mock = browser.mock("**/api/todo*"); +await expect(mock).toBeRequestedTimes(2); +// or, equivalently: +await expect(mock).toBeRequestedTimes({ eq: 2 }); +``` + +```javascript +const mock = browser.mock("**/api/todo*"); +// Check if the request was made at least 5 times but no more than 10 times +await expect(mock).toBeRequestedTimes({ gte: 5, lte: 10 }); +``` + +## toBeRequestedWith + +Checks if the specified request was made with the given options. + +Most options support partial matchers _expect.* / jasmine.*_ such as [expect.objectContaining](https://jestjs.io/docs/expect#expectobjectcontainingobject). + +For example: + +```javascript +const mock = browser.mock("**/api/todo*", { method: "POST" }); + +await expect(mock).toBeRequestedWith({ + url: "http://localhost:8080/api/todo", // [optional] string | function | custom matcher + method: "POST", // [optional] string | array + statusCode: 200, // [optional] number | array + requestHeaders: { Authorization: "foo" }, // [optional] object | function | custom matcher + responseHeaders: { Authorization: "bar" }, // [optional] object | function | custom matcher + postData: { title: "foo", description: "bar" }, // [optional] object | function | custom matcher + response: { success: true }, // [optional] object | function | custom matcher +}); + +await expect(mock).toBeRequestedWith({ + url: expect.stringMatching(/.*\/api\/.*/i), + method: ["POST", "PUT"], // Either POST or PUT + statusCode: [401, 403], // Either 401 or 403 + requestHeaders: headers => headers.Authorization.startsWith("Bearer "), + postData: expect.objectContaining({ released: true, title: expect.stringContaining("foobar") }), + response: r => Array.isArray(r) && r.data.items.length === 20, +}); +``` + +## References + +We'd like to give credit to the original WebdriverIO docs [article](https://webdriver.io/docs/api/expect-webdriverio), from which we drew some information while writing our version. diff --git a/i18n/en/docusaurus-plugin-content-docs/current/commands/mock/abort.mdx b/i18n/en/docusaurus-plugin-content-docs/current/commands/mock/abort.mdx new file mode 100644 index 0000000..37cd1f1 --- /dev/null +++ b/i18n/en/docusaurus-plugin-content-docs/current/commands/mock/abort.mdx @@ -0,0 +1,52 @@ +# abort + +## Overview {#overview} + +Use the `abort` command to abort a request with a specified error. + +A request can be aborted with one of the following errors: + +- Failed +- Aborted +- TimedOut +- AccessDenied +- ConnectionClosed | ConnectionReset | ConnectionRefused | ConnectionAborted | ConnectionFailed +- NameNotResolved +- InternetDisconnected +- AddressUnreachable +- BlockedByClient | BlockedByResponse + +## Usage {#usage} + +```javascript +mock.abort(errorCode); +``` + +## Command Parameters {#parameters} + + + + + + + + + +
**Name****Type****Description**
errorCodeErrorCodeThe response error code. Possible values are listed [above](#overview).
+ +## Usage Examples {#examples} + +```javascript +it("should block Google Analytics from page", async ({ browser }) => { + const mock = await browser.mock("https://www.google-analytics.com/**"); + mock.abort("Failed"); +}); +``` + +## Related Commands {#related} + +- [abortOnce](../abortOnce) + +## References + +We'd like to give credit to the original WebdriverIO docs [article](https://webdriver.io/docs/api/mock/abort), from which we drew some information while writing our version. diff --git a/i18n/en/docusaurus-plugin-content-docs/current/commands/mock/abortOnce.mdx b/i18n/en/docusaurus-plugin-content-docs/current/commands/mock/abortOnce.mdx new file mode 100644 index 0000000..5361c83 --- /dev/null +++ b/i18n/en/docusaurus-plugin-content-docs/current/commands/mock/abortOnce.mdx @@ -0,0 +1,61 @@ +# abortOnce + +## Overview {#overview} + +Use the `abortOnce` command to abort a request once with a specified error. + +A request can be aborted with one of the following errors: + +- Failed +- Aborted +- TimedOut +- AccessDenied +- ConnectionClosed | ConnectionReset | ConnectionRefused | ConnectionAborted | ConnectionFailed +- NameNotResolved +- InternetDisconnected +- AddressUnreachable +- BlockedByClient | BlockedByResponse + +## Usage {#usage} + +```javascript +mock.abortOnce(errorCode); +``` + +## Command Parameters {#parameters} + + + + + + + + + +
**Name****Type****Description**
errorCodeErrorCodeThe response error code. Possible values are listed [above](#overview).
+ +## Usage Examples {#examples} + +```javascript +it("should block mock only once", async ({ browser }) => { + const mock = await browser.mock("https://webdriver.io"); + mock.abortOnce("Failed"); + + await browser + .url("https://webdriver.io") + // catch exception as the request to the page will fail + .catch(() => console.log('Failed to get the page "https://webdriver.io"')); + + await browser.url("https://webdriver.io"); + console.log(await browser.getTitle()); + // will output: "WebdriverIO · Next-gen browser and mobile automation test framework for Node.js" +}); +``` + +## Related Commands {#related} + +- [abort](../abort) + +## References + +We'd like to give credit to the original WebdriverIO docs [article](https://webdriver.io/docs/api/mock/abortOnce), from which we drew some information while writing our version. diff --git a/i18n/en/docusaurus-plugin-content-docs/current/commands/mock/clear.mdx b/i18n/en/docusaurus-plugin-content-docs/current/commands/mock/clear.mdx new file mode 100644 index 0000000..160f684 --- /dev/null +++ b/i18n/en/docusaurus-plugin-content-docs/current/commands/mock/clear.mdx @@ -0,0 +1,32 @@ +# clear + +## Overview {#overview} + +Use the `clear` command to reset all information stored in the `mock.calls` array. + +## Usage {#usage} + +```javascript +mock.clear(); +``` + +## Usage Examples {#examples} + +```javascript +it("should clear mock", async ({ browser }) => { + const mock = await browser.mock("https://google.com/"); + await browser.url("https://google.com"); + + console.log(mock.calls.length); // outputs: 1 + mock.clear(); + console.log(mock.calls.length); // outputs: 0 +}); +``` + +## Related Commands {#related} + +- [restore](../restore) + +## References + +We'd like to give credit to the original WebdriverIO docs [article](https://webdriver.io/docs/api/mock/clear), from which we drew some information while writing our version. diff --git a/i18n/en/docusaurus-plugin-content-docs/current/commands/mock/respond.mdx b/i18n/en/docusaurus-plugin-content-docs/current/commands/mock/respond.mdx new file mode 100644 index 0000000..f908808 --- /dev/null +++ b/i18n/en/docusaurus-plugin-content-docs/current/commands/mock/respond.mdx @@ -0,0 +1,100 @@ +import Admonition from "@theme/Admonition"; + +# respond + +## Overview {#overview} + +Use the `respond` command to always reply with the same overwrite. + + + Also read the recipe "[How to Track and Intercept Network Requests and + Responses][how-to-intercept-requests-and-responses]". + + +## Usage {#usage} + +```javascript +mock.respond(overwrites, { header, statusCode, fetchResponse }); +``` + +## Command Parameters {#parameters} + + + + + + + + + + + + +
**Name****Type****Description**
overwritesMockOverwrite_Payload_ to overwrite the response.
headerObjectOverwrite specific headers.
statusCodeNumberOverwrite the response status code.
fetchResponseBooleanFetch the actual response before replying with fake data.
+ +## Usage Examples {#examples} + +```javascript +it("should demonstrate response overwrite with static data", async ({ browser }) => { + const mock = await browser.mock("https://todo-backend-express-knex.herokuapp.com/", { + method: "get", + }); + + mock.respond( + [ + { + title: "Injected (non) completed Todo", + order: null, + completed: false, + }, + { + title: "Injected completed Todo", + order: null, + completed: true, + }, + ], + { + statusCode: 200, + fetchResponse: true, // default + }, + ); + + await browser.url( + "https://todobackend.com/client/index.html?https://todo-backend-express-knex.herokuapp.com/", + ); + + await browser.$("#todo-list li").waitForExist(); + + const todoElements = await browser.$$("#todo-list li"); + console.log(await Promise.all(todoElements.map(el => el.getText()))); + // will output: "[ 'Injected (non) completed Todo', 'Injected completed Todo' ]" +}); + +it("should demonstrate response overwrite with dynamic data", async ({ browser }) => { + const mock = await browser.mock("https://todo-backend-express-knex.herokuapp.com/"); + + mock.respond( + request => { + if (request.body.username === "test") { + return { ...request.body, foo: "bar" }; + } + return request.body; + }, + { + statusCode: () => 200, + headers: () => ({ foo: "bar" }), + fetchResponse: false, // do not fetch the actual response + }, + ); +}); +``` + +## Related Commands {#related} + +- [respondOnce](../respondOnce) + +## References + +We'd like to give credit to the original WebdriverIO docs [article](https://webdriver.io/docs/api/mock/respond), from which we drew some information while writing our version. + +[how-to-intercept-requests-and-responses]: ../../../guides/how-to-intercept-requests-and-responses diff --git a/i18n/en/docusaurus-plugin-content-docs/current/commands/mock/respondOnce.mdx b/i18n/en/docusaurus-plugin-content-docs/current/commands/mock/respondOnce.mdx new file mode 100644 index 0000000..d41dc19 --- /dev/null +++ b/i18n/en/docusaurus-plugin-content-docs/current/commands/mock/respondOnce.mdx @@ -0,0 +1,112 @@ +import Admonition from "@theme/Admonition"; + +# respondOnce + +## Overview {#overview} + +Use the `respondOnce` command to respond once with the specified overwrite. + +You can call `respondOnce` multiple times in succession. The responses will be used in the same order the `respondOnce` commands were called. + +If you use only `respondOnce` and access the resource more times than `respondOnce` was called, then after exhausting the fake data, the request will start returning the original response from the resource. + + + Also read the recipe "[How to Track and Intercept Network Requests and + Responses][how-to-intercept-requests-and-responses]". + + +## Usage {#usage} + +```javascript +mock.respondOnce(overwrites, { header, statusCode, fetchResponse }); +``` + +## Command Parameters {#parameters} + + + + + + + + + + + + +
**Name****Type****Description**
overwritesMockOverwrite_Payload_ to overwrite the response.
headerObjectOverwrite specific headers.
statusCodeNumberOverwrite the response status code.
fetchResponseBooleanFetch the actual response before replying with fake data.
+ +## Usage Examples {#examples} + +```javascript +async function getToDos(browser) { + await browser.$("#todo-list li").waitForExist(); + + const todoElements = await browser.$$("#todo-list li"); + + return Promise.all(todoElements.map(el => el.getText())); +} + +it("should demonstrate the respondOnce command", async ({ browser }) => { + const mock = await browser.mock("https://todo-backend-express-knex.herokuapp.com/", { + method: "get", + }); + + mock.respondOnce([ + { + title: "3", + }, + { + title: "2", + }, + { + title: "1", + }, + ]); + + mock.respondOnce([ + { + title: "2", + }, + { + title: "1", + }, + ]); + + mock.respondOnce([ + { + title: "1", + }, + ]); + + await browser.url( + "https://todobackend.com/client/index.html?https://todo-backend-express-knex.herokuapp.com/", + ); + console.log(await getToDos(browser)); // outputs: [ '3', '2', '1' ] + + await browser.url( + "https://todobackend.com/client/index.html?https://todo-backend-express-knex.herokuapp.com/", + ); + console.log(await getToDos(browser)); // outputs: [ '2', '1' ] + + await browser.url( + "https://todobackend.com/client/index.html?https://todo-backend-express-knex.herokuapp.com/", + ); + console.log(await getToDos(browser)); // outputs: [ '1' ] + + await browser.url( + "https://todobackend.com/client/index.html?https://todo-backend-express-knex.herokuapp.com/", + ); + console.log(await getToDos(browser)); // outputs: the actual resource response +}); +``` + +## Related Commands {#related} + +- [respond](../respond) + +## References + +We'd like to give credit to the original WebdriverIO docs [article](https://webdriver.io/docs/api/mock/respondOnce), from which we drew some information while writing our version. + +[how-to-intercept-requests-and-responses]: ../../../guides/how-to-intercept-requests-and-responses diff --git a/i18n/en/docusaurus-plugin-content-docs/current/commands/mock/restore.mdx b/i18n/en/docusaurus-plugin-content-docs/current/commands/mock/restore.mdx new file mode 100644 index 0000000..52b82f5 --- /dev/null +++ b/i18n/en/docusaurus-plugin-content-docs/current/commands/mock/restore.mdx @@ -0,0 +1,32 @@ +# restore + +## Overview {#overview} + +Use the `restore` command to perform all actions that [mock.clear()](../clear) does, as well as remove any fake return values or implementations. + +## Usage {#usage} + +```javascript +mock.restore(); +``` + +## Usage Examples {#examples} + +```javascript +it("should demonstrate the addValue command", async ({ browser }) => { + const mock = await browser.mock("**/googlelogo_color_272x92dp.png"); + mock.respond("https://webdriver.io/img/webdriverio.png"); + + await browser.url("https://google.com"); // will show WebdriverIO logo instead of Google logo + mock.restore(); + await browser.url("https://google.com"); // will show the native Google logo +}); +``` + +## Related Commands {#related} + +- [clear](../clear) + +## References + +We'd like to give credit to the original WebdriverIO docs [article](https://webdriver.io/docs/api/mock/restore), from which we drew some information while writing our version. diff --git a/i18n/en/docusaurus-plugin-content-docs/current/commands/overview.mdx b/i18n/en/docusaurus-plugin-content-docs/current/commands/overview.mdx new file mode 100644 index 0000000..332df8f --- /dev/null +++ b/i18n/en/docusaurus-plugin-content-docs/current/commands/overview.mdx @@ -0,0 +1,50 @@ +--- +sidebar_class_name: hidden +--- + +import Admonition from "@theme/Admonition"; + +# Testplane Commands + +## Overview + + + Only commands for the latest version Testplane v8 and WebDriverIO v8 are described. Commands for + older versions should be referenced in the WebDriverIO documentation (example for [WebDriverIO + v7][webdriverio@7-api]). + + +Since Testplane is based on [WebDriverIO v8][webdriverio-api], all commands provided by WebDriverIO are available in it. + +However, the command descriptions on the [WebDriverIO][webdriverio-api] website for version 8 are not quite suitable _as is_ for Testplane users due to a number of reasons: + +- In WebDriverIO, the `browser` object exists in the global scope, whereas in Testplane you need to either write `this.browser`: + +```javascript +it("should test something", async function () { + const browser = this.browser; + // test code... +}); +``` + +or get the `browser` object from the function argument (note that the `browser` object is passed inside an object!): + +```javascript +it("should test something", async ({ browser }) => { + // test code... +}); +``` + +- Similarly, the `browser.$` and `browser.$$` functions in WebDriverIO are available in tests under the names `$` and `$$`, while in Testplane you need to refer to them by their full path: `browser.$` and `browser.$$`; + +- There are no links established between similar commands: no clustering of commands; + +- All commands are described only in English. + +Nevertheless, our description does not yet include protocol-specific commands. You can find the relevant commands on the WebDriverIO website under the "[Protocols][webdriverio-protocols]" section. + +Also, in the descriptions of some commands, links to individual recipes are still not localized and lead to the WebDriverIO website. + +[webdriverio@7-api]: https://webdriver.io/docs/api +[webdriverio-api]: https://webdriver.io/docs/api +[webdriverio-protocols]: https://webdriver.io/docs/api/webdriver diff --git a/i18n/en/docusaurus-plugin-content-docs/current/config/browsers.mdx b/i18n/en/docusaurus-plugin-content-docs/current/config/browsers.mdx index e114014..65f8080 100644 --- a/i18n/en/docusaurus-plugin-content-docs/current/config/browsers.mdx +++ b/i18n/en/docusaurus-plugin-content-docs/current/config/browsers.mdx @@ -23,13 +23,13 @@ module.exports = { // other browsers... }, - // other testplane settings... + // other Testplane settings... }; ``` Where `` is the browser name that is used to identify it. -In order not to repeat the same settings for different browsers, you can set all the default values you need in the root of testplane config. For example: +In order not to repeat the same settings for different browsers, you can set all the default values you need in the root of Testplane config. For example: ```javascript module.exports = { @@ -45,7 +45,7 @@ module.exports = { }, }, - // other testplane settings... + // other Testplane settings... }; ``` @@ -89,7 +89,7 @@ module.exports = { // other browsers settings... }, - // other testplane settings... + // other Testplane settings... }; ``` @@ -144,7 +144,7 @@ module.exports = { // other browsers settings... }, - // other testplane settings... + // other Testplane settings... }; ``` @@ -215,7 +215,7 @@ Timeout for any requests to the Selenium server, in ms. By default: `30000` ms. ### urlHttpTimeout {#url_http_timeout} -Timeout for the `/url` request to the Selenium server, in ms. Sometimes when opening a link on the server side, a lot of logic is performed in the middlewares, which is why the link takes a long time to open. In order not to raise the timeout for all commands because of this, testplane allows you to set up a separate timeout for the request `/url`. +Timeout for the `/url` request to the Selenium server, in ms. Sometimes when opening a link on the server side, a lot of logic is performed in the middlewares, which is why the link takes a long time to open. In order not to raise the timeout for all commands because of this, Testplane allows you to set up a separate timeout for the request `/url`. ### pageLoadTimeout {#page_load_timeout} @@ -247,6 +247,7 @@ If the value is not set, a common timeout for all browsers will be used, which i [retry](#retry)Number0How many times you need to restart the failing test. [shouldRetry](#should_retry)Function_see description_A function that determines whether a retry is needed. By default, a function is set that returns _true_, if _retry > 0,_ and _false_, if _retry == 0_. [strictTestsOrder](#strict_test_order)BooleanfalseGuarantee a strict order of tests. If _true_, then the API function _testplane.readTests_ will always return the same result. +[passive](#passive)BooleanfalseMakes the browser "passive". In passive browsers, tests do not run by default. _Available from testplane@8.16.0_. @@ -289,6 +290,17 @@ The argument of this function is an object with the following fields: This option enables a guarantee of a strict order of reading tests. By default: `false`. +### passive {#passive} + + + Available from testplane@8.16.0. Doesn't work with deprecated plugin + [hermione-passive-browsers](https://github.com/gemini-testing/testplane-passive-browsers). + + +Makes the browser "passive". In passive browsers, tests do not run by default. You may use the [testplane.also.in][testplane-also-in-helper] helper before a suite or test to run it in corresponding browser. + +По умолчанию: `false`. + ## Information about the test and its failure {#info_when_test_fails} @@ -408,7 +420,7 @@ The orientation of the browser window that you need to set before running each t ### waitOrientationChange {#wait_orientation_change} -Wait for a real change of orientation. By default: `true`. When set to `true`, testplane guarantees that the command `setOrientation` will be completed only after the orientation is actually changed to the specified one. +Wait for a real change of orientation. By default: `true`. When set to `true`, Testplane guarantees that the command `setOrientation` will be completed only after the orientation is actually changed to the specified one. ### resetCursor {#reset_cursor} @@ -511,7 +523,7 @@ The following options are available for `assertView`: - + @@ -604,7 +616,7 @@ Defines whether the SSL certificate should be valid. If `true`, it means that [d ## Cloud settings {#cloud_settings} -The following settings may be useful if you want to run your testplane tests in cloud service browsers. +The following settings may be useful if you want to run your Testplane tests in cloud service browsers. [SauceLabs][sauce-labs] is an example of such a cloud service, which [can provide](https://saucelabs.com/platform/cross-browser-testing) you both desktop and mobile browsers to run tests in them. diff --git a/i18n/en/docusaurus-plugin-content-docs/current/config/plugins.mdx b/i18n/en/docusaurus-plugin-content-docs/current/config/plugins.mdx index 92efe94..201de09 100644 --- a/i18n/en/docusaurus-plugin-content-docs/current/config/plugins.mdx +++ b/i18n/en/docusaurus-plugin-content-docs/current/config/plugins.mdx @@ -13,17 +13,17 @@ For example, plugins [html-reporter][html-reporter] or [testplane-test-repeater] A plugin is a module that exports a single function with the following arguments: -- testplane instance -- plugin options from testplane config +- Testplane instance +- plugin options from Testplane config -All plugins will be loaded before testplane runs the tests. +All plugins will be loaded before Testplane runs the tests. When choosing a name for a plugin, add the prefix _testplane-_ to it. Then it will be easier to search for such a plugin. -If the plugin name starts with the prefix `testplane-`, then you can omit this prefix when adding the plugin to the `plugins` section. If testplane discovers modules with both names on the file system: _testplane-some-module_ and _some-module_, it will give preference to the module with the prefix `testplane-`. +If the plugin name starts with the prefix `testplane-`, then you can omit this prefix when adding the plugin to the `plugins` section. If Testplane discovers modules with both names on the file system: _testplane-some-module_ and _some-module_, it will give preference to the module with the prefix `testplane-`. ## Draft example {#example} @@ -36,10 +36,10 @@ If the plugin name starts with the prefix `testplane-`, then you can omit this p param: 'some-value' }, - // other testplane plugins... + // other Testplane plugins... }, - // other testplane settings... + // other Testplane settings... }; @@ -66,7 +66,7 @@ module.exports = function(testplane, opts) { -## Properties of testplane instance {#testplane_instance} +## Properties of Testplane instance {#testplane_instance}
ignoreElementsArray or StringElements (set as selectors) to ignore when comparing screenshots. Ignore is implemented by painting the listed elements in black. In the case of a single element, the parameter can be set as a string.
toleranceNumberSensitivity to color difference. The value overwrites [browsers.tolerance](#tolerance).
antialiasingToleranceNumberSensitivity in antialiasing. The value overwrites [browsers.antialiasingTolerance](#antialiasing_tolerance).
allowViewportOverflowBooleanBy default, testplane throws an error if the element is outside the boundaries of the viewport. This parameter disables checking for borders, allowing you to take screenshots of elements that do not fit into the viewport. At the same time, only those parts of the element that fit into the viewport will be visible in the screenshot. However, if _compositeImage_ is equal to _true_, then the parts of the element that were below the viewport's _bottom_ border will also be visible in the screenshot. Similarly, if _captureElementFromTop_ is equal to _true_, then the screenshot will also include those parts of the element that were above the viewport's _top_ border.
allowViewportOverflowBooleanBy default, Testplane throws an error if the element is outside the boundaries of the viewport. This parameter disables checking for borders, allowing you to take screenshots of elements that do not fit into the viewport. At the same time, only those parts of the element that fit into the viewport will be visible in the screenshot. However, if _compositeImage_ is equal to _true_, then the parts of the element that were below the viewport's _bottom_ border will also be visible in the screenshot. Similarly, if _captureElementFromTop_ is equal to _true_, then the screenshot will also include those parts of the element that were above the viewport's _top_ border.
captureElementFromTopBooleanTake a screenshot of the element from the very top. If the element is located outside the viewport, then a scroll will be made to it.
compositeImageBooleanAllows you to test elements that do not fit into the viewport in height.
screenshotDelayNumberThe delay in milliseconds before taking a screenshot. It can be useful when there are elements on the page that use animation, or a scrollbar that does not disappear immediately and gets to the resulting screenshot.
diff --git a/i18n/en/docusaurus-plugin-content-docs/current/config/sets.mdx b/i18n/en/docusaurus-plugin-content-docs/current/config/sets.mdx index df74685..9512326 100644 --- a/i18n/en/docusaurus-plugin-content-docs/current/config/sets.mdx +++ b/i18n/en/docusaurus-plugin-content-docs/current/config/sets.mdx @@ -8,7 +8,7 @@ For example, it can be convenient to run tests on platforms: `desktop`, `touch-p Define a set of tests as an array of paths to them or to folders with them on the file system. You can also specify paths to ignore when searching for tests to speed up the process of reading tests by testplane. -If there are no sets in the config, or you do not specify them when starting testplane (see the section "[Usage](#usage)"), then all tests from the `testplane` folder will be run in all browsers that were specified in the [browsers][browsers] section of testplane config. +If there are no sets in the config, or you do not specify them when starting Testplane (see the section "[Usage](#usage)"), then all tests from the `testplane` folder will be run in all browsers that were specified in the [browsers][browsers] section of Testplane config. ## Setup {#setup} @@ -38,7 +38,7 @@ module.exports = { // other sets settings... }, - // other testplane settings... + // other Testplane settings... }; ``` @@ -53,7 +53,7 @@ Where `` is the name of the set to identify it. - +
filesArray or String_N/A_A list of paths to files or folders with tests. In the case of a single path, the parameter can be set as a string. You can also use [masks][fast-glob-patterns].
ignoreFilesArray or String`[ ]`A list of paths or [masks][fast-glob-patterns] to ignore when searching for test files. This parameter allows you to speed up the reading of tests by testplane.
browsersArray_all browsers_The list of browsers to run the tests in. You can only specify browsers that are in the [browsers][browsers] section of testplane config. By default: all browsers from the [browsers][browsers] section.
browsersArray_all browsers_The list of browsers to run the tests in. You can only specify browsers that are in the [browsers][browsers] section of Testplane config. By default: all browsers from the [browsers][browsers] section.
@@ -90,7 +90,7 @@ module.exports = { }, }, - // other testplane settings... + // other Testplane settings... }; ``` @@ -104,7 +104,7 @@ Example of how to run tests for a desktop platform in the case of the configurat testplane --set desktop ``` -If there are no sets in the config, or they exist, but the `--set` option was not specified, and the paths were not passed to testplane via the CLI, then all tests from the `testplane` folder will be run in all [browsers][browsers]. +If there are no sets in the config, or they exist, but the `--set` option was not specified, and the paths were not passed to Testplane via the CLI, then all tests from the `testplane` folder will be run in all [browsers][browsers]. [fast-glob-patterns]: https://github.com/mrmlnc/fast-glob#pattern-syntax [browsers]: ../browsers diff --git a/i18n/en/docusaurus-plugin-content-docs/current/guides/how-to-add-testing-library.mdx b/i18n/en/docusaurus-plugin-content-docs/current/guides/how-to-add-testing-library.mdx new file mode 100644 index 0000000..be079af --- /dev/null +++ b/i18n/en/docusaurus-plugin-content-docs/current/guides/how-to-add-testing-library.mdx @@ -0,0 +1,67 @@ +# How to Connect testing-library in testplane + +## Introduction {#preface} + +[Testing-library][testing-library] is a collection of tools for testing web application user interfaces, focused on creating reliable and maintainable tests by emphasizing user behavior. The main advantage of `testing-library` is its focus on interaction with interface elements. And in testplane, you can use the element search methods provided by the `testing-library` itself. + +## Connection + +To be able to use [queries][queries] from `testing-library` in Testplane tests, you only need to follow a few steps. + +1. Install the npm package `'@testing-library/webdriverio'` + +```bash +npm i -D @testing-library/webdriverio +``` + +2. Include it in the Testplane config in the `prepareBrowser` section: + +```javascript +// .testplane.conf.js +const { setupBrowser } = require("@testing-library/webdriverio"); + +module.exports = { + prepareBrowser(browser) { + setupBrowser(browser); + }, + + // other Testplane settings... +}; +``` + +## Usage + +After configuring, you will be able to use the search by selectors from `testing-library`, as described in the [official documentation](https://testing-library.com/docs/queries/about/). For example, searching for an element by its text: + +```javascript +it("example", async ({ browser }) => { + await browser.url("https://github.com/"); + + const newRepoButton = await browser.getByText("New"); + + await newRepoButton.click(); +}); +``` + +This feature will also be available in the context of found elements: + +```javascript +it("example", async ({ browser }) => { + await browser.url("https://github.com/"); + + const sidebar = await browser.$(".dashboard-sidebar"); + const newRepoButton = await sidebar.getByText("New"); + + await newRepoButton.click(); +}); +``` + +For a complete usage example, visit [this link](https://github.com/gemini-testing/testplane/tree/master/examples/). + +## Useful Links {#useful_links} + +- [Testing-library][testing-library] +- [WebdriverIO Testing Library](https://testing-library.com/docs/webdriverio-testing-library/intro) + +[testing-library]: https://testing-library.com/ +[queries]: https://testing-library.com/docs/queries/about diff --git a/i18n/en/docusaurus-plugin-content-docs/current/guides/how-to-check-accessibility.mdx b/i18n/en/docusaurus-plugin-content-docs/current/guides/how-to-check-accessibility.mdx new file mode 100644 index 0000000..e543b50 --- /dev/null +++ b/i18n/en/docusaurus-plugin-content-docs/current/guides/how-to-check-accessibility.mdx @@ -0,0 +1,83 @@ +import Admonition from "@theme/Admonition"; + +# How to Test Page Accessibility + +## Overview {#overview} + + +This recipe only works when using _Chrome DevTools Protocol (CDP) or Chrome_. + +Read more details in the section “[How to Use CDP in Testplane][how-to-use-cdp]” + + + +[Accessibility tree][accessibility-tree] is an accessibility tree that contains a hierarchical structure of accessible objects. Unlike the DOM tree, which is intended for browsers, the accessibility tree is intended for [screen readers][screen-reader] and other tools that help people with disabilities interact with websites. + +To obtain such a tree, _puppeteer_ has a special [Accessibility class][puppeteer-accessibility]. + +## Example {#example} + +Here's an example of how to use it: + +```javascript +it("should get accessibility tree of yandex.ru", async function () { + // Get puppeteer instance + const puppeteer = await this.browser.getPuppeteer(); + + // Get the first open page (considering it to be currently active) + const [page] = await puppeteer.pages(); + + await this.browser.url("https://yandex.ru"); + + // Get the current state of the accessibility tree + const snapshot = await page.accessibility.snapshot(); + console.log("snapshot:", JSON.stringify(snapshot, null, 4)); +}); +``` + +## Accessibility Tree {#accessibility_tree} + +And here's how the obtained accessibility tree looks: + +```json +{ + "role": "WebArea", + "name": "Yandex", + "children": [ + { + "role": "link", + "name": "Login" + }, + { + "role": "link", + "name": "Mail" + }, + { + "role": "link", + "name": "Disk" + }, + { + "role": "link", + "name": "Try Plus" + }, + + // omitted for brevity... + + { + "role": "link", + "name": "finance" + }, + { + "role": "link", + "name": "politics" + } + ] +} +``` + +Using the obtained tree, we can check that all necessary nodes are contained in the tree and have the correct structure. + +[how-to-use-cdp]: ../how-to-use-cdp +[accessibility-tree]: https://web.dev/the-accessibility-tree/ +[screen-reader]: https://en.wikipedia.org/wiki/Screen_reader +[puppeteer-accessibility]: https://pptr.dev/api/puppeteer.accessibility diff --git a/i18n/en/docusaurus-plugin-content-docs/current/guides/how-to-check-test-stability.mdx b/i18n/en/docusaurus-plugin-content-docs/current/guides/how-to-check-test-stability.mdx new file mode 100644 index 0000000..52cd8ba --- /dev/null +++ b/i18n/en/docusaurus-plugin-content-docs/current/guides/how-to-check-test-stability.mdx @@ -0,0 +1,51 @@ +# How to Check Test Stability + +## Problem {#problem} + +There are many reasons why tests can fail: + +- infrastructure issues – browser unavailability, network malfunctions, expired access, etc. +- problems with external services the test interacts with; +- races within the test itself, where an element does not render in time and the wait period expires; +- unexpected popups during the test execution that obscure the necessary element, cause differences in screenshots, or prevent clicking the needed element; +- and so on. + +Often these failures do not reproduce on the first attempt. Therefore, to ensure a newly written test is stable, it needs to be run multiple times. But there is one problem: if you run your test in Testplane _as is,_ after the first successful run, Testplane will stop running your test. The test passed – everything is OK. But for stability verification, this is not enough. The test might have passed accidentally, and if you rerun it, it might fail. Ideally, you want to run it not just once or twice, but for example, 20 times and see how many times it passes out of 20. Or 30 times. Or... and so on. + +## Solution: testplane-test-repeater {#solution} + +To solve this problem, the [testplane-test-repeater][testplane-test-repeater] plugin was developed. + +The plugin allows you to run the same test (or group of tests) the required number of times. + +The plugin ensures that the tests will be run as many times as you specify, regardless of the results of each run. Moreover, the plugin allows you to start the tests each time in a new browser session. This eliminates browser degradation or other side effects that might occur with repeated runs in the same browser session. + +Read more about how to add this plugin to a project, configure, and use it in the [plugin documentation][testplane-test-repeater]. + +## Usage Examples {#usage} + +Below are examples of test runs, in which they were run 21 times (1 primary + 20 retries) to check their stability. + +### Example of Broken Tests + +As seen in the screenshot, the tests were run 21 times and never completed successfully: + +![Broken Test](/img/docs/guides/how-to-check-test-stability.total-failure.png) + +### Example of Stable Test + +Here, on the contrary, all runs were successful: + +![Stable Test](/img/docs/guides/how-to-check-test-stability.total-success.png) + +### Example of Unstable Tests + +In the next screenshot, the first test is almost non-functional – out of 21 attempts, the test passed only once. The second test is quite stable, although 2 failures out of 21 is still not 100% stability. The developer may try to understand why the test fails occasionally. + +![Unstable Test](/img/docs/guides/how-to-check-test-stability.unstable-test.png) + +## Keywords {#keywords} + +- testplane-test-repeater + +[testplane-test-repeater]: ../../plugins/testplane-test-repeater diff --git a/i18n/en/docusaurus-plugin-content-docs/current/guides/how-to-debug-test.mdx b/i18n/en/docusaurus-plugin-content-docs/current/guides/how-to-debug-test.mdx new file mode 100644 index 0000000..bde9824 --- /dev/null +++ b/i18n/en/docusaurus-plugin-content-docs/current/guides/how-to-debug-test.mdx @@ -0,0 +1,62 @@ +import Admonition from "@theme/Admonition"; + +# How to Debug a Test + +## Problem {#problem} + +In the course of developing tests, any developer will sooner or later encounter errors that are hard to detect through normal code review. At such moments, it's necessary to use software tools to understand where the error occurred, or why the test is behaving differently than expected. + +Let's explore the options available to a Testplane test developer. + +## Solution 1: --inspect or --inspect-brk option {#solution_1} + +To see how a test is executed step by step, Testplane has a debug mode. This mode relies on the [V8 inspector integration with Node.js](https://nodejs.org/dist/latest-v16.x/docs/api/debugger.html#advanced-usage). + +
+ + +About V8 Inspector Integration with Node.js + + + +The integration with the _V8_ inspector allows connecting _Chrome DevTools_ to a _Node.js_ instance for debugging and profiling. This uses the _Chrome DevTools_ protocol. + +The _V8_ inspector can be enabled by passing the `--inspect` option when starting a _Node.js_ application. You can also specify a custom port with this option; for example, `--inspect=9222` will accept `DevTools` connections on port `9222`. + +To stop the code execution at the first line of the application, use the `--inspect-brk` option instead of `--inspect`. + +```bash +$ node --inspect index.js +Debugger listening on ws://127.0.0.1:9229/dc9010dd-f8b8-4ac5-a510-c1a114ec7d29 +For help, see: https://nodejs.org/en/docs/inspector +``` + +_In the example above, the `UUID dc9010dd-f8b8-4ac5-a510-c1a114ec7d29` at the end of the URL is generated "on the fly," and varies in different debugging sessions._ + +If the Chrome browser is older than `66.0.3345.0`, use `inspector.html` instead of `js_app.html` in the above URL. + +_Chrome DevTools_ does not yet support debugging for [worker threads](https://nodejs.org/dist/latest-v16.x/docs/api/worker_threads.html). To debug them, you can use [ndb](https://github.com/GoogleChromeLabs/ndb/). + +
+ +To run a test in this mode, use the `--inspect` option. If you want the debugger to stop at the first line of code, use the `--inspect-brk` option. + +Example: + +```bash +testplane path/to/mytest.js --inspect +``` + + + In debug mode, only one worker process is started, and all tests are run in it. Use this mode + with the parameter _sessionsPerBrowser=1_ to debug tests one at a time. + + +## Keywords {#keywords} + +- --inspect +- --inspect-brk + +## Useful Links {#useful_links} + +- [About Chrome DevTools](https://developer.chrome.com/docs/devtools/) diff --git a/i18n/en/docusaurus-plugin-content-docs/current/guides/how-to-get-report.mdx b/i18n/en/docusaurus-plugin-content-docs/current/guides/how-to-get-report.mdx new file mode 100644 index 0000000..f9f8fd0 --- /dev/null +++ b/i18n/en/docusaurus-plugin-content-docs/current/guides/how-to-get-report.mdx @@ -0,0 +1,115 @@ +# How to Get a Test Run Report + +## Testplane Report {#testplane_report} + +After completing all test runs, Testplane writes the result to the console in the form of a string like this: + +```bash +Total: 1812 Passed: 1792 Failed: 0 Skipped: 20 Retries: 47 +``` + +Where: + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
StatusDescription
TotalTotal number of tests that Testplane read from the file system during launch.
PassedNumber of tests that passed successfully.
FailedNumber of tests that failed.
SkippedNumber of tests that were skipped during the run.
RetriesTotal number of test retries that occurred during the run.
+ +However, this information may not be sufficient, so you can add the [stat-reporter][stat-reporter] plugin to your project. + +## stat-reporter Plugin Report {#stat_reporter_report} + +If you add the [stat-reporter][stat-reporter] plugin to your project, you will get a more detailed report of the run results in the console after completing all the tests. For example: + +```bash +┌──────────────────────┬────────┬───────┬────────┬────────┬─────────┬─────────┬──────────┐ +│ Browser │ Status │ Tests │ Passed │ Failed │ Skipped │ Retries │ Duration │ +├──────────────────────┼────────┼───────┼────────┼────────┼─────────┼─────────┼──────────┤ +│ firefox │ passed │ 25 │ 24 │ 0 │ 1 │ 0 │ 01:02 │ +├──────────────────────┼────────┼───────┼────────┼────────┼─────────┼─────────┼──────────┤ +│ chrome-desktop │ passed │ 466 │ 464 │ 0 │ 2 │ 4 │ 07:40 │ +├──────────────────────┼────────┼───────┼────────┼────────┼─────────┼─────────┼──────────┤ +│ ipad │ passed │ 24 │ 23 │ 0 │ 1 │ 0 │ 01:27 │ +├──────────────────────┼────────┼───────┼────────┼────────┼─────────┼─────────┼──────────┤ +│ iphone │ passed │ 376 │ 372 │ 0 │ 4 │ 7 │ 07:12 │ +├──────────────────────┼────────┼───────┼────────┼────────┼─────────┼─────────┼──────────┤ +│ chrome-phone │ passed │ 427 │ 421 │ 0 │ 6 │ 14 │ 07:32 │ +├──────────────────────┼────────┼───────┼────────┼────────┼─────────┼─────────┼──────────┤ +│ iphone-dark │ passed │ 74 │ 72 │ 0 │ 2 │ 4 │ 02:18 │ +├──────────────────────┼────────┼───────┼────────┼────────┼─────────┼─────────┼──────────┤ +│ searchapp-phone │ passed │ 319 │ 317 │ 0 │ 2 │ 9 │ 10:00 │ +├──────────────────────┼────────┼───────┼────────┼────────┼─────────┼─────────┼──────────┤ +│ safari13 │ passed │ 15 │ 13 │ 0 │ 2 │ 4 │ 02:42 │ +├──────────────────────┼────────┼───────┼────────┼────────┼─────────┼─────────┼──────────┤ +│ chrome-desktop-1920 │ passed │ 3 │ 3 │ 0 │ 0 │ 0 │ 00:57 │ +├──────────────────────┼────────┼───────┼────────┼────────┼─────────┼─────────┼──────────┤ +│ iphoneX │ passed │ 3 │ 3 │ 0 │ 0 │ 0 │ 00:36 │ +├──────────────────────┼────────┼───────┼────────┼────────┼─────────┼─────────┼──────────┤ +│ chrome-desktop-dark │ passed │ 77 │ 77 │ 0 │ 0 │ 5 │ 01:33 │ +├──────────────────────┼────────┼───────┼────────┼────────┼─────────┼─────────┼──────────┤ +│ yandex-browser-phone │ passed │ 1 │ 1 │ 0 │ 0 │ 0 │ 00:28 │ +├──────────────────────┼────────┼───────┼────────┼────────┼─────────┼─────────┼──────────┤ +│ chrome-grid-720 │ passed │ 2 │ 2 │ 0 │ 0 │ 0 │ 00:49 │ +└──────────────────────┴────────┴───────┴────────┴────────┴─────────┴─────────┴──────────┘ +``` + +Unlike the simple Testplane report, the `stat-reporter` plugin report breaks down the results by browser. It also displays the maximum execution time _(Duration)_ in minutes and seconds and the test run result _(Status)_ in each browser. + +Such a report allows for a better understanding of which browsers encountered problems, specifically: where the most tests failed or the execution time sharply increased. + +The `stat-reporter` plugin also allows generating reports in HTML or JSON formats. See details in the [plugin description][stat-reporter]. + +However, in terms of capabilities, the [stat-reporter][stat-reporter] plugin is significantly inferior to the [html-reporter][html-reporter] plugin, which provides much more advanced ways to work with tests and their run results. + +## html-reporter Plugin Report {#html_reporter_report} + +Add the [html-reporter][html-reporter] plugin to your project to get a graphical HTML report with the results of all test runs. Additionally, in the generated report, you will be able to: + +- filter tests by completion status; +- group tests by errors or any parameter from the test metadata; +- view screenshot differences in 6 different ways; +- view all retries or errors in tests separately. + +![html-reporter](/img/docs/guides/how-to-get-report.html-reporter.png) + +Moreover, the [html-reporter][html-reporter] allows running Testplane in a special GUI mode. In this mode, you can run and rerun tests, reshoot screenshots, use special debugging modes, and much more. + +## Keywords {#keywords} + +- stat-reporter +- html-reporter +- gui + +## Useful Links {#useful_links} + +- [stat-reporter Plugin][stat-reporter] +- [html-reporter Plugin][html-reporter] + +[stat-reporter]: ../../plugins/stat-reporter +[html-reporter]: ../../html-reporter/html-reporter-setup diff --git a/i18n/en/docusaurus-plugin-content-docs/current/guides/how-to-hide-scrollbars-by-cdp.mdx b/i18n/en/docusaurus-plugin-content-docs/current/guides/how-to-hide-scrollbars-by-cdp.mdx new file mode 100644 index 0000000..1ab4ae7 --- /dev/null +++ b/i18n/en/docusaurus-plugin-content-docs/current/guides/how-to-hide-scrollbars-by-cdp.mdx @@ -0,0 +1,48 @@ +import Admonition from "@theme/Admonition"; + +# How to Hide Scrollbars Using Chrome DevTools Protocol + +## Overview {#overview} + + +This recipe only works when using _Chrome DevTools Protocol (CDP)_. + +Read details in the section “[How to use CDP in Testplane][how-to-use-cdp]” + + + +One of the reasons for test failures when testing layouts using screenshots is the presence of scrollbars in the browser at the moment the screenshot is taken. You can read more about this problem and some ways to solve it [here](../how-to-hide-scrollbars). This problem arises particularly often in tests with mobile emulation. + +CDP has a special method [Emulation.setScrollbarsHidden][set-scrollbars-hidden] that allows hiding the scrollbar. However, _puppeteer_ lacks a wrapper for this method. Therefore, we will use the [CDPSession.send][cdp-session-send] method to execute the [Emulation.setScrollbarsHidden][set-scrollbars-hidden] command. + +## Example: How to Hide Scrollbars Using CDP {#example} + +Here's how it looks: + +```javascript +it("should hide scrollbar", async function () { + // Get puppeteer instance + const puppeteer = await this.browser.getPuppeteer(); + + // Get the first open page (considering it to be currently active) + const [page] = await puppeteer.pages(); + + // Create a CDP session + const client = await page.target().createCDPSession(); + + // Hide the scrollbar + await client.send("Emulation.setScrollbarsHidden", { hidden: true }); + + await this.browser.url("https://yandex.ru"); +}); +``` + +## Useful Links {#useful_links} + +Also, read our recipe “[How to Hide Scrollbars from Screenshots](../how-to-hide-scrollbars)”. + +There you will also learn about the [hermione-hide-scrollbars](../../plugins/hermione-hide-scrollbars) plugin, which is implemented based on the [Emulation.setScrollbarsHidden][set-scrollbars-hidden] method and which you can use to disable scrollbars in CI for all tests in specific browsers. + +[how-to-use-cdp]: ../how-to-use-cdp +[set-scrollbars-hidden]: https://chromedevtools.github.io/devtools-protocol/tot/Emulation/#method-setScrollbarsHidden +[cdp-session-send]: https://pptr.dev/next/api/puppeteer.cdpsession.send diff --git a/i18n/en/docusaurus-plugin-content-docs/current/guides/how-to-hide-scrollbars.mdx b/i18n/en/docusaurus-plugin-content-docs/current/guides/how-to-hide-scrollbars.mdx new file mode 100644 index 0000000..dea9864 --- /dev/null +++ b/i18n/en/docusaurus-plugin-content-docs/current/guides/how-to-hide-scrollbars.mdx @@ -0,0 +1,63 @@ +import Admonition from "@theme/Admonition"; + +# How to Hide Scrollbars from Screenshots + +## Problem {#problem} + +One of the reasons for test failures when testing layouts using screenshots is the presence of scrollbars in the browser at the moment the screenshot is taken. There can be 3 scenarios where a diff occurs between the reference screenshot and the current one taken during the test run: + +1. There was no scrollbar when the reference screenshot was taken, but it appeared when taking the current screenshot. +2. There was a scrollbar when the reference screenshot was taken, but it did not appear in time when taking the current screenshot. +3. There is a scrollbar on both the reference and current screenshots, but the positions, sizes, or transparency of the scrollbars differ due to the timing of when the screenshots were taken relative to the appearance of the scrollbar. + +The screenshot below shows an example corresponding to the first scenario: + +![diff due to scrollbar](/img/docs/guides/how-to-hide-scrollbars.diff-in-screenshot.png) + +## Solution 1: screenshotDelay {#solution_1} + +In Testplane settings, there is a mandatory option `browsers` that specifies the set of browsers available in the project and their properties. Select the browser where you are experiencing diffs due to scrollbars and add the `screenshotDelay` option for it: + +```javascript +module.exports = { + browsers: { + iphone: { + screenshotDelay: 600, // Delay before taking a screenshot in ms + + // other browser settings... + }, + + // other browsers... + }, + + // other Testplane settings... +}; +``` + +The `screenshotDelay` option sets a pause in milliseconds that Testplane should wait before taking a screenshot (before executing the `assertView` command). + +### How can it help? {#how_can_it_help} + +Often screenshots generate diffs because the test needs to scroll the page to the required element just before taking the screenshot. After the scroll is performed, the scrollbar might still be visible on the screen for some time and thus, might appear in the screenshot if taken immediately. The `screenshotDelay` gives the scrollbar time to disappear. + +However, this method does not always work as it depends on the implementation and behavior of the browsers. + +## Solution 2: Disabling Scrollbars {#solution_2} + +If scrollbars appear in screenshots in the Chrome browser, they can be disabled using the [DevTools protocol][CDP]. + +To do this, add the [hermione-hide-scrollbars][hermione-hide-scrollbars] plugin to your project and specify in its settings the list of browsers for which you want to disable scrollbars in the tests. + + + Update the Chrome browser to version 72.1 or higher for this functionality to work in your + tests. Earlier versions of Chrome do not support the _Emulation.setScrollbarsHidden_ command, + which is used to disable the scrollbars. + + +## Keywords {#keywords} + +- screenshotDelay +- hermione-hide-scrollbars + +[hermione-hide-scrollbars]: ../../plugins/hermione-hide-scrollbars +[CDP]: https://chromedevtools.github.io/devtools-protocol/ diff --git a/i18n/en/docusaurus-plugin-content-docs/current/guides/how-to-intercept-requests-and-responses.mdx b/i18n/en/docusaurus-plugin-content-docs/current/guides/how-to-intercept-requests-and-responses.mdx new file mode 100644 index 0000000..e143155 --- /dev/null +++ b/i18n/en/docusaurus-plugin-content-docs/current/guides/how-to-intercept-requests-and-responses.mdx @@ -0,0 +1,346 @@ +import Admonition from "@theme/Admonition"; + +# How to Track and Intercept Network Requests and Responses + +## Overview {#overview} + +[//]: # "TODO: add screencasts" + + +This recipe only works when using _Chrome DevTools Protocol (CDP)_. + +Read details in the section “[How to Use CDP in Testplane][how-to-use-cdp]” + + + +CDP has [Fetch][fetch] and [Network][network] domains that provide full access to all network requests and responses. If we used the Webdriver protocol, we would have to write a separate proxy server and route all traffic through it. + +In _webdriverio_, there is a [mock][mock] method for working with network requests, which uses the API of the [Fetch][fetch] domain. + +With this method, we can: + +- mock a request to a resource and return our own data; +- cancel a request, returning a necessary error; +- modify a response from a resource; +- redirect from the requested resource to another resource; +- mock a resource and, for example, collect information on how many times this resource was called and what response it returned. + +Let's try writing tests using this API and cover different cases. To clarify, all graphical representations of the test execution process are slowed down by a factor of 2 because locally the tests run very quickly, making it difficult to observe anything. + +## Example 1: Mocking a Request to google.com and Returning Our Own Response {#example_1} + +```javascript +it("should mock google.com", async function () { + // Mocking the request to google.com + const mock = await this.browser.mock("https://google.com"); + + // Returning the string "Hello, world!" instead of the response from the site. + // The "fetchResponse" option dictates whether the request should be made + // to the mocked resource; default is true. + mock.respond("Hello, world!", { fetchResponse: false }); + + await this.browser.url("https://google.com"); +}); +``` + +From the graphical representation, you can see that we returned our text, although the browser's address bar shows we navigated to _google.com._ Also, it's clear that we didn't mock the favicon, which was fetched from an external source. We can write this same example using the puppeteer API. For this, _webdriverio_ has the [getPuppeteer()][get-puppeteer] command: + +```javascript +it("should mock google.com using puppeteer api", async function () { + // Get puppeteer instance + const puppeteer = await this.browser.getPuppeteer(); + + // Get the first open page (considering it to be currently active) + const [page] = await puppeteer.pages(); + + // Enable request interception + await page.setRequestInterception(true); + page.on("request", async request => { + if (request.url() !== "https://google.com/") { + // If the request URL does not match https://google.com/, + // continue the request (i.e., don't intercept it) + return request.continue(); + } + + // Respond with our own data + return request.respond({ body: "Hello, world!" }); + }); + + // Here, we could call "page.goto('https://google.com')", but it's better to call "url", + // because most plugins have wrappers for the "url" command adding additional logic. + // For example, in testplane, the URL is added to the meta. + await this.browser.url("https://google.com"); +}); +``` + +### Hardcore Example Using CDP Directly {#hardcore_example} + +Now, let's imagine that puppeteer doesn't yet have an API for mocking requests, but this is already implemented in the [Fetch][fetch] domain of CDP. In this case, we will use this domain's method by interacting with the CDP session directly. For this, puppeteer has the [CDPSession.send()][cdp-session-send] method: + +```javascript +it("should mock google.com using cdp fetch domain", async function () { + // Get puppeteer instance + const puppeteer = await this.browser.getPuppeteer(); + + // Get the first open page (considering it to be currently active) + const [page] = await puppeteer.pages(); + + // Create a CDP session + const client = await page.target().createCDPSession(); + + // Enable request interception by subscribing to the "requestPaused" event + await client.send("Fetch.enable"); + + client.on("Fetch.requestPaused", event => { + const { + request: { url }, + requestId, + responseHeaders, + } = event; + + if (url !== "https://google.com/") { + // If the request URL does not match https://google.com/, + // continue the request (i.e., don't intercept it) + return client.send("Fetch.continueRequest", { requestId }); + } + + // Replace the response with our own and encode it in base64 + return client.send("Fetch.fulfillRequest", { + requestId, + responseCode: 200, + responseHeaders, + body: Buffer.from("Hello, world!", "utf8").toString("base64"), + }); + }); + + await this.browser.url("https://google.com"); +}); +``` + +Obviously, when using the _webdriverio_ API for mocking requests, the code is much shorter, but the _webdriverio_ API is very limited, and for more complex cases, it is necessary to use puppeteer's API. However, puppeteer itself might not have an API for some new methods or CDP domains. Therefore, in rare cases, direct communication via CDP using [CDPSession.send()][cdp-session-send] might come in handy. + +## Example 2: Canceling the Request for Google's Logo {#example_2} + +```javascript +it("should abort request to logo on google.com", async function () { + // You can use a mask for the URL + const mock = await this.browser.mock("https://www.google.com/images/**/*.png"); + + // Throw an error "ERR_FAILED" when loading a resource that matches the mask + mock.abort("Failed"); + + await this.browser.url("https://google.com"); +}); +``` + +From the graphical representation, it is clear that the logo is not displayed, and there is a `net::ERR_FAILED` error in the log. This solution can be useful for disabling some scripts that hinder the quick execution of the test. For example, analytics collection scripts can be disabled. + +## Example 3: Loading google.com Using a Fixture for the Response {#example_3} + +```javascript +it("should mock google.com and return answer from fixture", async function () { + // Mocking the request to google.com + const mock = await this.browser.mock("https://google.com"); + + // Specify the path from which to take the fixture, and with + // "fetchResponse: false", indicate that the real request should not be made + mock.respond("./fixtures/my-google.com.html", { fetchResponse: false }); + + await this.browser.url("https://google.com"); +}); +``` + +From the graphical representation, it is clear that instead of google.com's content, our fixture's data is displayed. + +## Example 4: Redirecting the Request from google.com to yandex.ru {#example_4} + +```javascript +it("should redirect from google.com to yandex.ru", async function () { + // Mocking the request to google.com + const mock = await this.browser.mock("https://google.com"); + + // For redirection, simply specify the URL + mock.respond("https://yandex.ru"); + + await this.browser.url("https://google.com"); +}); +``` + +## Example 5: Modifying google.com's Response in Real-Time {#example_5} + +Puppeteer still does not have an API for conveniently modifying responses. There is an [issue#1191](https://github.com/puppeteer/puppeteer/issues/1191) on this. But this capability is already supported in CDP. Webdriverio uses CDP directly through [puppeteer][puppeteer], so it works in _webdriverio_. + +Replace all occurrences of the string `Google` with `Yandex` in google.com's response: + +```javascript +it("should modify response from google.com", async function () { + // Here, you need to mock with www because navigating to google.com + // returns a 301 response without a body and redirects to www + const mock = await this.browser.mock("https://www.google.com"); + + mock.respond(req => { + // Replace "Google" with "Yandex" using a regular expression + return req.body.replace(/Google/g, "Yandex"); + }); + + await this.browser.url("https://google.com"); +}); +``` + +Additionally, we can modify responses from unknown sources in advance. For example, let's modify all scripts loaded on _google.com:_ + +```javascript +it("should modify response from google.com", async function () { + // The first argument specifies that we will intercept all requests + const mock = await this.browser.mock("**", { + headers: headers => { + // Filter only the requests where the "content-type" + // header contains values "text/javascript" or "application/javascript" + return ( + headers["content-type"] && + /^(text|application)\/javascript/.test(headers["content-type"]) + ); + }, + }); + + mock.respond(req => { + // Append our own console.log to the end of each script + return (req.body += `\nconsole.log("This script was modified in real time.");`); + }); + + await this.browser.url("https://google.com"); +}); +``` + +## Example 6: Intercepting All Requests to yandex.ru and Collecting a List of All Loaded URLs {#example_6} + +Let's say we need to collect a list of all URLs loaded on the page. Using this information, we could determine if we have requests for external resources or neighboring services that we do not control. This means they could fail at any time and break our tests. Here's what our code might look like: + +```javascript +it("should mock yandex.ru and log all loaded urls", async function () { + // Intercept absolutely all requests + const mock = await this.browser.mock("**"); + + await this.browser.url("https://yandex.ru"); + + // mock.calls contains not only the visited URL information + // but also the response from the source, the request headers, response headers, etc. + const urls = mock.calls.map(({ url }) => url); + + console.log("visited urls:", JSON.stringify(urls, null, "\t")); + console.log("count of visited urls:", urls.length); +}); +``` + +Most likely, your tests are more complex than these examples and involve various clicks on elements that open in new tabs. In such cases, the previous code will not capture the opening of new tabs or that URLs need to be collected there as well. Therefore, in such cases, you need to use puppeteer's API: + +```javascript +it("should mock yandex.ru and log all loaded urls (using puppeteer)", async function () { + // Accumulative list of all URLs + const urls = []; + + // Helper that expects a page instance from puppeteer: + // https://pptr.dev/#?product=Puppeteer&version=v10.1.0&show=api-class-page + function urlsHandler(page) { + // Subscribe to all requests occurring on the given page + page.on("request", req => { + urls.push(req.url()); + }); + } + + // Get puppeteer instance + const puppeteer = await this.browser.getPuppeteer(); + + // Get all open pages at the current moment + const pages = await puppeteer.pages(); + + // Subscribe to all requests occurring on these pages + await Promise.all(pages.map(p => urlsHandler(p))); + + // Subscribe to the creation of new pages + puppeteer.on("targetcreated", async target => { + // Check that the opened target is indeed a new page + const page = await target.page(); + + if (!page) { + return; + } + + // Since the new page opens with some URL, + // it needs to be explicitly recorded (the request subscription will not detect it) + urls.push(target.url()); + + // Subscribe to all requests occurring on the new tab + urlsHandler(page); + }); + + await this.browser.url("https://yandex.ru"); + + // Find the first element in the list of services (at that time it was a football page) + const elem = await this.browser.$(".services-new__list-item > a"); + + // Click the service that opens in a new tab + await elem.click(); + + console.log("visited urls:", JSON.stringify(urls, null, "\t")); + console.log("count of visited urls:", urls.length); +}); +``` + +## Example 7: Mocking the google.com Resource in All Chrome Tests {#example_7} + +To avoid manually mocking the same resources in all tests, you can use the [testplane-global-hook][testplane-global-hook] plugin. Configure it appropriately in the Testplane config: + +```javascript +// .testplane.conf.js +module.exports = { + plugins: { + "testplane-global-hook": { + enabled: true, + + beforeEach: async function () { + // Check that the browser name starts with "chrome" + if (!/^chrome$/i.test(this.browser.capabilities.browserName)) { + return; + } + + // Mocking the request to google.com + const mock = await this.browser.mock("https://google.com"); + mock.respond("hello world", { fetchResponse: false }); + }, + + afterEach: function () { + // Clear all mocks in the current session + this.browser.mockRestoreAll(); + }, + }, + + // other Testplane plugins... + }, + + // other Testplane settings... +}; +``` + +The test code will now only contain the URL transition: + +```javascript +// Explicitly indicate that the test is only executed in browsers whose name starts with chrome +testplane.only.in(/^chrome/); +it("should mock google.com inside global before each", async function () { + await this.browser.url("https://google.com"); +}); +``` + +## Useful Links {#useful_links} + +More usage examples can be found in the "[Mocks and Spies][wdio-mocks-and-spies]" guide on the _webdriverio_ website. + +[how-to-use-cdp]: ../how-to-use-cdp +[wdio-mocks-and-spies]: https://webdriver.io/docs/mocksandspies/ +[fetch]: https://chromedevtools.github.io/devtools-protocol/tot/Fetch/ +[network]: https://chromedevtools.github.io/devtools-protocol/tot/Network/ +[mock]: ../../commands/browser/mock +[get-puppeteer]: ../../commands/browser/getPuppeteer +[cdp-session-send]: https://pptr.dev/next/api/puppeteer.cdpsession.send +[testplane-global-hook]: ../../plugins/testplane-global-hook diff --git a/i18n/en/docusaurus-plugin-content-docs/current/guides/how-to-manage-cpu-performance.mdx b/i18n/en/docusaurus-plugin-content-docs/current/guides/how-to-manage-cpu-performance.mdx new file mode 100644 index 0000000..58de3aa --- /dev/null +++ b/i18n/en/docusaurus-plugin-content-docs/current/guides/how-to-manage-cpu-performance.mdx @@ -0,0 +1,64 @@ +import Admonition from "@theme/Admonition"; + +# How to Manage CPU Performance + +## Overview {#overview} + + +This recipe only works when using _Chrome DevTools Protocol (CDP)_. + +Read details in the section “[How to Use CDP in Testplane][how-to-use-cdp]” + + + +The CPU speed on mobile devices is significantly slower than on computers. Therefore, to emulate CPU speed in _puppeteer_, there is a method called [emulateCPUThrottling][emulate-cpu-throttling]. + +## Example: Slowing Down CPU Speed by 8x {#example} + +Let's use this method to slow down CPU speed by 8 times: + +```javascript +it("should open yandex.ru with emulation 8x slower CPU", async function () { + // Get puppeteer instance + const puppeteer = await this.browser.getPuppeteer(); + + // Get the first open page (considering it to be currently active) + const [page] = await puppeteer.pages(); + + // Slow down the CPU speed by 8 times + await page.emulateCPUThrottling(8); + + await this.browser.url("https://yandex.ru"); +}); +``` + +## A Small Story About a Workaround {#workaround_in_the_past} + +Initially, _webdriverio_ did not support the `page.emulateCPUThrottling` method because _webdriverio_ used _puppeteer-core@9.1.0,_ not _puppeteer-core@10.1.0_, which supports this method. + +However, this limitation could be bypassed using puppeteer's [CDPSession.send()][cdp-session-send] method by sending the browser the [Emulation.setCPUThrottlingRate][emulation-set-cpu-throttling-rate] command via CDP: + +```javascript +it("should open yandex.ru with emulation 8x slower CPU", async function () { + // Get puppeteer instance + const puppeteer = await this.browser.getPuppeteer(); + + // Get the first open page (considering it to be currently active) + const [page] = await puppeteer.pages(); + + // Create a CDP session + const client = await page.target().createCDPSession(); + + // Slow down the CPU speed by 8 times + await client.send("Emulation.setCPUThrottlingRate", { rate: 8 }); + + await this.browser.url("https://yandex.ru"); +}); +``` + +Later, we submitted a requisite [pull request](https://github.com/webdriverio/webdriverio/pull/7135) to _webdriverio_ to update the version of _puppeteer-core_. Now, the [emulateCPUThrottling][emulate-cpu-throttling] method is available in Testplane right out of the box. + +[how-to-use-cdp]: ../how-to-use-cdp +[emulate-cpu-throttling]: https://pptr.dev/api/puppeteer.page.emulatecputhrottling +[cdp-session-send]: https://pptr.dev/api/puppeteer.cdpsession.send +[emulation-set-cpu-throttling-rate]: https://chromedevtools.github.io/devtools-protocol/tot/Emulation/#method-setCPUThrottlingRate diff --git a/i18n/en/docusaurus-plugin-content-docs/current/guides/how-to-manage-network-bandwidth.mdx b/i18n/en/docusaurus-plugin-content-docs/current/guides/how-to-manage-network-bandwidth.mdx new file mode 100644 index 0000000..5ca620d --- /dev/null +++ b/i18n/en/docusaurus-plugin-content-docs/current/guides/how-to-manage-network-bandwidth.mdx @@ -0,0 +1,58 @@ +import Admonition from "@theme/Admonition"; + +# How to Manage Network Bandwidth + +## Overview {#overview} + + +This recipe only works when using _Chrome DevTools Protocol (CDP)_. + +Read details in the section “[How to use CDP in Testplane][how-to-use-cdp]”. + + + +A large number of users access services from mobile devices where internet speed can be quite slow or may even drop out intermittently. In _webdriverio_, we can limit network bandwidth using the [throttle][throttle] method, thereby testing the website's behavior under various network conditions. + +Besides custom settings, the [throttle][throttle] method supports the following ready-made presets: + +- offline | online +- GPRS +- Regular2G | Good2G +- Regular3G | Good3G +- Regular4G +- DSL +- WiFi + +## Example 1: Emulating a 2G Connection {#example_1} + +Let's emulate a 2G connection and open yandex.ru in Chrome with phone emulation: + +```javascript +it("should open yandex.ru with emulation of 2G-connection", async function () { + // Emulate a 2G connection + await this.browser.throttle("Good2G"); + + await this.browser.url("https://yandex.ru"); +}); +``` + +## Example 2: Emulating a Network with Given Characteristics {#example_2} + +We can also emulate a connection with specific characteristics: + +```javascript +it("should open yandex.ru with emulation of custom connection", async function () { + // Emulate a network connection with specified characteristics + await this.browser.throttle({ + offline: false, // emulate offline state + downloadThroughput: (10 * 1024) / 8, // max download bandwidth (byte/sec) + uploadThroughput: (10 * 1024) / 8, // max upload bandwidth (byte/sec) + latency: 10, // min latency from sending the request to receiving the response headers + }); + + await this.browser.url("https://yandex.ru"); +}); +``` + +[how-to-use-cdp]: ../how-to-use-cdp +[throttle]: ../../commands/browser/throttle diff --git a/i18n/en/docusaurus-plugin-content-docs/current/guides/how-to-minify-screenshots.mdx b/i18n/en/docusaurus-plugin-content-docs/current/guides/how-to-minify-screenshots.mdx new file mode 100644 index 0000000..3429c10 --- /dev/null +++ b/i18n/en/docusaurus-plugin-content-docs/current/guides/how-to-minify-screenshots.mdx @@ -0,0 +1,36 @@ +import Admonition from "@theme/Admonition"; + +# How to Reduce Screenshot Size + +## Introduction {#preface} + +One of the key features of Testplane is the ability to test layouts using screenshots. For this, you can take a screenshot during different states of a web page in your tests and compare it with a reference. + +An example of such a comparison: + +![Screenshot Comparison](/img/docs/guides/how-to-minify-screenshots.diff-in-screenshot.png) + + +**Where do reference screenshots come from?** + +You wrote a test using the Testplane _assertView_ command. The _assertView_ command compares the actual screenshot with the reference screenshot. An actual screenshot is one that the test takes during its run. It reflects the current state of your web page. On the first run of your test, you don't yet have a reference screenshot. Therefore, the _assertView_ command will fail with a message that the reference screenshot is not found. At the same time, you will have an actual screenshot. If the actual screenshot reflects the state of your web page you expect (consider as reference), then you need to _accept_ the actual screenshot as the reference one. To do this, you need to run Testplane with an additional _--update-refs_ option. Alternatively, you can use the GUI mode provided by the _html-reporter_ plugin. With it, you can view the run report in GUI mode, ensure the actual screenshots are as you expect, and accept them using the _Accept_ button. + + + +## Problem {#problem} + +If your project has many tests that use layout validation via screenshots, then your project will have many screenshots. Sometimes these screenshots are quite large and can take up a lot of space on the file system. + +If the project is actively developed, each pull request will generate its own test run report, containing hundreds (or thousands) of reference screenshots. The larger the screenshots, the larger your reports will be. This means they will take longer to download, occupy more disk space, and so on. + +## Solution: testplane-image-minifier {#solution} + +To reduce the space screenshots occupy on disk, add the [testplane-image-minifier][testplane-image-minifier] plugin to your project. This plugin supports 8 levels of lossless image compression. + +How to add and configure the plugin is detailed in the [plugin documentation][testplane-image-minifier]. + +## Keywords {#keywords} + +- testplane-image-minifier + +[testplane-image-minifier]: ../../plugins/testplane-image-minifier diff --git a/i18n/en/docusaurus-plugin-content-docs/current/guides/how-to-optimize-test-code.mdx b/i18n/en/docusaurus-plugin-content-docs/current/guides/how-to-optimize-test-code.mdx new file mode 100644 index 0000000..7f7514f --- /dev/null +++ b/i18n/en/docusaurus-plugin-content-docs/current/guides/how-to-optimize-test-code.mdx @@ -0,0 +1,23 @@ +# How to Eliminate Duplication in Tests + +## Problem {#problem} + +Often, before running the next Testplane test, certain preparatory work needs to be done, such as: + +- clearing all cookies; +- cleaning localStorage; +- initializing some test variable. + +Similarly, after completing the main checks in a Testplane test, you may always want to check for errors in the client code, the triggering of required metrics, etc. + +## Solution: testplane-global-hook {#solution} + +To avoid repeating these actions each time in your tests, add the [testplane-global-hook][testplane-global-hook] plugin and describe them in the plugin settings as functions for the `beforeEach` and `afterEach` hooks. + +Learn more about how to add and configure the plugin in the [plugin documentation][testplane-global-hook]. + +## Keywords {#keywords} + +- testplane-global-hook + +[testplane-global-hook]: ../../plugins/testplane-global-hook diff --git a/i18n/en/docusaurus-plugin-content-docs/current/guides/how-to-run-specified-test.mdx b/i18n/en/docusaurus-plugin-content-docs/current/guides/how-to-run-specified-test.mdx new file mode 100644 index 0000000..950e1f8 --- /dev/null +++ b/i18n/en/docusaurus-plugin-content-docs/current/guides/how-to-run-specified-test.mdx @@ -0,0 +1,51 @@ +# How to Run Specific Tests + +## Problem {#problem} + +Sometimes you may need to run specific tests rather than the entire set of tests in your project. + +For example, you developed a feature and covered it with functional tests. You might want to first check the correctness of the new tests. + +Or you are fixing a flaky test, found bugs, fixed them, and want to verify that the test now passes correctly. + +## Solution 1: Running a Specific File {#solution_1} + +If you want to run a whole group of tests located in a specific file, specify the path to this file as an input parameter for testplane: + +```bash +testplane src/features/Reviews/Reviews.test/MyReview/MyReview.a11y@touch-phone.testplane.js +``` + +## Solution 2: --grep Option {#solution_2} + +If you want to run a specific test, use the `--grep` option by providing the full name of the test as its value: + +```bash +testplane --grep "Accessibility Leaving a review" +``` + +## Solution 3: .only Directive {#solution_3} + +You can also use the `.only` directive for a suite of tests (`describe`) or a specific test (`it`), similar to what is implemented in `mocha` (see the [exclusive tests](https://mochajs.org/#exclusive-tests) section): + +For example: + +```javascript +describe.only("Accessibility", function () { + // Test suite... +}); +``` + +or + +```javascript +it.only("Leaving a review", async function () { + // Test code... +}); +``` + +## Keywords {#keywords} + +- grep +- describe.only +- it.only diff --git a/i18n/en/docusaurus-plugin-content-docs/current/guides/how-to-skip-test-in-browsers.mdx b/i18n/en/docusaurus-plugin-content-docs/current/guides/how-to-skip-test-in-browsers.mdx new file mode 100644 index 0000000..8a755cc --- /dev/null +++ b/i18n/en/docusaurus-plugin-content-docs/current/guides/how-to-skip-test-in-browsers.mdx @@ -0,0 +1,81 @@ +# How to Skip a Test in a Specific Browser + +## Problem {#problem} + +Sometimes you need to skip running a test in a specific browser, rather than in all browsers. That is, you don't want to disable or delete the entire test, but only want to limit the number of browsers in which it will run. + +For example, this could be due to the limited functionality of the respective browser: the absence of necessary features that are used on the web page and checked by the test. + +Another reason could be the unstable behavior of the test in a particular browser due to certain implementation nuances in the browser. + +In testplane, you can do this using special helpers (directives) `skip` and `only`. + +## Solution 1: .skip.in Directive {#solution_1} + +For example, if you don't want to run the test in `IE8`: + +```javascript +describe("feature", function () { + testplane.skip.in("ie8", "it cannot work in this browser"); + it("nowaday functionality", function () { + // ... + }); +}); +``` + +When using the `testplane.skip.in` directive, you will see a message in the report indicating that the run was skipped in the respective browser. + +To skip the test runs without notifications in the report, you can pass a special flag `silent` to the helper as the third argument: + +```javascript +testplane.skip.in("ie8", "skipReason", { silent: true }); +``` + +## Solution 2: .skip.notIn Directive {#solution_2} + +You might also want to run the test only in a specific browser, for example, in `Chrome`: + +```javascript +describe("feature", function () { + testplane.skip.notIn("chrome", "it should work only in Chrome"); + it("specific functionality", function () { + // ... + }); +}); +``` + +Similarly, to avoid notifications in the report, you can pass a special flag `silent` to the helper as the third argument: + +```javascript +testplane.skip.notIn("chrome", "skipReason", { silent: true }); +``` + +## Solution 3: .only.in and .only.notIn Directives {#solution_3} + +You can also use the helpers `only.in` and `only.notIn`, whose logic is the opposite of the helpers `skip.in` and `.skip.notIn`. Additionally, these helpers do not, by default, produce any notifications in the report: + +```javascript +testplane.only.in("chrome"); // run the test only in Chrome +``` + +```javascript +testplane.only.notIn("ie8"); // run the test in all browsers except IE8 +``` + +## Solution 4: .also.in Directive and Passive Browser Option {#solution_4} + +If you are introducing a new browser and need to run it only in a few tests while having thousands of them, using the `.skip.in` helper may be inconvenient. To solve this problem, you can use the [passive][passive-option] browser option and the helper `.also.in`: + +```javascript +testplane.also.in("ie8"); // run the test in the passive browser IE8 +``` + +## Keywords {#keywords} + +- testplane.skip.in +- testplane.skip.notIn +- testplane.only.in +- testplane.only.notIn +- testplane.also.in + +[passive-option]: ../../config/browsers#passive diff --git a/i18n/en/docusaurus-plugin-content-docs/current/guides/how-to-update-browsers.mdx b/i18n/en/docusaurus-plugin-content-docs/current/guides/how-to-update-browsers.mdx new file mode 100644 index 0000000..0fc15d8 --- /dev/null +++ b/i18n/en/docusaurus-plugin-content-docs/current/guides/how-to-update-browsers.mdx @@ -0,0 +1,92 @@ +import Tabs from "@theme/Tabs"; +import TabItem from "@theme/TabItem"; + +import Admonition from "@theme/Admonition"; + +# How to Update Browsers in Large Projects + +## Problem + +In a large project, handled by multiple teams, there are usually a lot of tests. Updating browsers in such a project can become a real headache. Changing the browser version often requires retaking screenshots because the browser's rendering, animation, and antialiasing mechanisms might change. Even the slightest changes in the internal rendering implementation of a browser can lead to diffs between reference and actual screenshots in tests. + +After retaking thousands of screenshots, a developer creates a pull request that needs to be merged into the main branch of the project. Merging such a pull request becomes extremely challenging due to the constantly occurring merge conflicts. Development doesn’t stop, and teams might change tests, retake reference screenshots to account for new functionality, or delete outdated tests and their screenshots. This means a pull request with a large number of changed files quickly becomes outdated and requires constant rebasing relative to the main branch. Moreover, this rebase involves a series of actions: not only updating the codebase but also rerunning the tests to retake the screenshots. Simply resolving the merge conflict between screenshots will not work. Both conflicting screenshots will be incorrect: one outdated and the other not corresponding to the new browser version. + +Thus, if a team wants to update the version of the browser in which their tests are run, they are forced to update the tests for the entire project and face the difficulties described above. + +There are three ways to solve this problem. + +## Solution 1: testplane.browser().version() + +Testplane allows you to override the browser version for a specific test or set of tests using the `testplane.browser().version()` helper. + +For example: + +```javascript +// Override the browser version for chrome-desktop to 70.3 for the entire test suite +testplane.browser("chrome-desktop").version("70.3"); +describe("suite", function () { + it("test 1", function () { + // ... + }); + + // Override the browser version for chrome-desktop to 70.1 for a specific test + testplane.browser("chrome-desktop").version("70.1"); + it("test 2", function () { + // ... + }); +}); +``` + +The drawback of this approach is that you need to manually change the test files themselves. With many files, this can take quite a long time. + +## Solution 2: Browser Version Changer + +You can use the [hermione-browser-version-changer][hermione-browser-version-changer] plugin, which allows you to define the browser version for a specific test based on a special dictionary _(store)_ and predicates for all available browser versions in the project. + +Example usage of the plugin: + +```javascript +module.exports = { + plugins: { + "hermione-browser-version-changer": { + enabled: true, + initStore: async () => { + // Prepare a dictionary with arbitrary tags for labeling + return { + "title-test1": "tag1", + "title-test2": "tag1", + "title-test3": "tag2", + }; + }, + browsers: { + "chrome-desktop": { + // Key "70.1" – browser version, value – predicate + // If the function returns a true value, this version will be set for the browser + 70.1: (test, version, store) => { + // test – current test + // version – proposed version "70.1" + // store – dictionary prepared in the initStore method + + // Set browser version 70.1 if the test belongs to "tag1" + return store[test.title] === "tag1"; + }, + 80: (test, version, store) => { + return store[test.title] === "tag2"; + }, + }, + }, + }, + + // other Testplane plugins... + }, + + // other Testplane settings... +}; +``` + +The provided example is very conditional, and it is possible that in your project you won’t even need to use a _store_, and checking the test against a given regex pattern will be sufficient. + +[hermione-browser-version-changer]: ../../plugins/hermione-browser-version-changer +[testplane]: https://github.com/gemini-testing/testplane +[html-reporter]: ../../html-reporter/html-reporter-setup +[gh-issues]: https://github.com/gemini-testing/testplane/issues/ diff --git a/i18n/en/docusaurus-plugin-content-docs/current/guides/how-to-use-cdp.mdx b/i18n/en/docusaurus-plugin-content-docs/current/guides/how-to-use-cdp.mdx new file mode 100644 index 0000000..07463cd --- /dev/null +++ b/i18n/en/docusaurus-plugin-content-docs/current/guides/how-to-use-cdp.mdx @@ -0,0 +1,118 @@ +import Admonition from "@theme/Admonition"; + +# How to Use Chrome DevTools Protocol in Testplane + +## Introduction {#preface} + + + Install Hermione (Testplane) version 4 or higher in your project to use _Chrome DevTools + Protocol (CDP)_ in Testplane tests. + + +The `WebDriver` protocol has been used in Testplane for a long time, but the possibility of using [CDP][CDP] appeared only after migrating to _[WebdriverIO@7](https://webdriver.io/versions)_ in Hermione version 4. + +CDP support in _WebdriverIO@7_ is implemented using [puppeteer][puppeteer], which is a wrapper with a convenient API over CDP. + +For a comparison of the _WebDriver_ and _CDP_ protocols, see the "WebDriver vs CDP" section in the Reference Manual. + +## Local Usage {#local_usage} + +To work with the browser via CDP locally, add the `automationProtocol: 'devtools'` option to the browser settings in the Testplane config: + +```javascript +// .testplane.conf.js + +module.exports = { + browsers: { + chrome: { + automationProtocol: "devtools", + desiredCapabilities: { + // ... + }, + }, + + // other browser settings... + }, + + // other Testplane settings... +}; +``` + +After this, all subsequent runs will be performed in your locally installed Chrome. + +But if you need to run a browser with CDP support just once, it's more convenient to override this option using an environment variable: + +```bash +testplane_browsers_chrome_automation_protocol=devtools npx testplane ... +``` + +Or set it as a CLI option: + +```bash +npx testplane ... --browsers-chrome-automation-protocol=devtools +``` + +## Remote Usage {#remote_usage} + +When using CDP on a remote machine (e.g., in a grid), Testplane will first start the browser using the WebDriver protocol and then, upon user request (i.e., when calling a CDP command), switch to CDP connection. Thus, in a single test scenario with a remote browser, we will interact using both protocols. + +It looks something like this: + +![Remote CDP Usage Diagram](/img/docs/guides/how-to-use-cdp.remote-scheme.png) + +To connect to a remote browser via CDP, you need to: + +- use `automationProtocol: webdriver` (default value); +- add the vendor-specific field `selenoid:options` in the browser’s `desiredCapabilities`: this option is necessary for _webdriverio_ to understand that it needs to connect to a remote machine instead of a local browser. + +```javascript +// .testplane.conf.js + +module.exports = { + browsers: { + chrome: { + desiredCapabilities: { + "selenoid:options": {}, + // ... + }, + + // other browser settings... + }, + + // other browser settings... + }, + + // other Testplane settings... +}; +``` + + + Full CDP usage is only supported from **Chrome@77** and higher. This is due to the internal + implementation in _webdriverio._ + + +## What Capabilities Does CDP Provide {#what_does_cdp_give} + +With CDP, you can: + +- [track and intercept network requests and responses][how-to-intercept-requests-and-responses] +- [test page accessibility][how-to-check-accessibility] +- [manage network bandwidth][how-to-manage-network-bandwidth] +- [control CPU performance][how-to-manage-cpu-performance] +- [hide scrollbars][how-to-hide-scrollbars-by-cdp] + +## Useful Links {#useful_links} + +- [WebDriver vs CDP][webdriver-vs-cdp] +- [Web Performance Recipes With Puppeteer](https://addyosmani.com/blog/puppeteer-recipes/) +- [About Chrome DevTools Protocol][CDP] +- [puppeteer][puppeteer] + +[CDP]: https://chromedevtools.github.io/devtools-protocol/ +[puppeteer]: https://pptr.dev/ +[how-to-intercept-requests-and-responses]: ../how-to-intercept-requests-and-responses +[how-to-check-accessibility]: ../how-to-check-accessibility +[how-to-manage-network-bandwidth]: ../how-to-manage-network-bandwidth +[how-to-manage-cpu-performance]: ../how-to-manage-cpu-performance +[how-to-hide-scrollbars-by-cdp]: ../how-to-hide-scrollbars-by-cdp +[webdriver-vs-cdp]: ../../reference/webdriver-vs-cdp diff --git a/i18n/en/docusaurus-plugin-content-docs/current/html-reporter/html-reporter-setup.mdx b/i18n/en/docusaurus-plugin-content-docs/current/html-reporter/html-reporter-setup.mdx index 24c9185..0a63ac4 100644 --- a/i18n/en/docusaurus-plugin-content-docs/current/html-reporter/html-reporter-setup.mdx +++ b/i18n/en/docusaurus-plugin-content-docs/current/html-reporter/html-reporter-setup.mdx @@ -158,7 +158,7 @@ Save or not save error details in json files (to the `error-details` folder). By default, "do not save": `false`. -Any testplane plugin can add any details to the error object when it occurs. These details can help the user in debugging problems that have occurred in the test. Html-reporter saves these details in the `error-details` folder in a file named: `-__.json`. +Any Testplane plugin can add any details to the error object when it occurs. These details can help the user in debugging problems that have occurred in the test. Html-reporter saves these details in the `error-details` folder in a file named: `-__.json`. Under the stack trace, the html-reporter adds an `Error details` section with a `` link pointing to the json file. The user can open this file either in the browser or in any IDE. @@ -375,10 +375,10 @@ module.exports = { // other plugin settings... }, - // other testplane plugins... + // other Testplane plugins... }, - // other testplane settings... + // other Testplane settings... }; ``` diff --git a/i18n/en/docusaurus-plugin-content-docs/current/index.mdx b/i18n/en/docusaurus-plugin-content-docs/current/index.mdx index 5314fc7..2d51eae 100644 --- a/i18n/en/docusaurus-plugin-content-docs/current/index.mdx +++ b/i18n/en/docusaurus-plugin-content-docs/current/index.mdx @@ -88,7 +88,7 @@ And that’s not all. - dozens of [ready-made plugins][plugins]; - write your own plugin to implement any functionality; - [over 20 events you can subscribe to][testplane-events]; -- available [extension points][html-reporter-extension-points] in the testplane report; +- available [extension points][html-reporter-extension-points] in the Testplane report; - [extend the Testplane CLI from your plugin][testplane-cli], adding new commands and options. [create-testplane]: https://github.com/gemini-testing/create-testplane diff --git a/i18n/en/docusaurus-plugin-content-docs/current/migrations/how-to-upgrade-hermione-to-4.mdx b/i18n/en/docusaurus-plugin-content-docs/current/migrations/how-to-upgrade-hermione-to-4.mdx new file mode 100644 index 0000000..7cd9236 --- /dev/null +++ b/i18n/en/docusaurus-plugin-content-docs/current/migrations/how-to-upgrade-hermione-to-4.mdx @@ -0,0 +1,476 @@ +import Tabs from "@theme/Tabs"; +import TabItem from "@theme/TabItem"; + +import Admonition from "@theme/Admonition"; + +# How to Upgrade hermione to Version 4.x + +<Admonition type="warning"> + This guide is relevant only for projects using hermione versions older than 4.x. +</Admonition> + +## Why Should You Upgrade? {#why_to_upgrade} + +A long time ago, hermione switched to a "temporary" fork of the [webdriverio@4][webdriverio] package (abbr. wdio) that it used "under the hood" because issues in the _external_ _wdio_ were slowing down its development: constant bugs in _wdio_, disagreements over changes, etc. Initially, the fork was regularly updated by the hermione team to provide users with up-to-date functionality, but over time, the fork significantly lagged behind the current version of _wdio_ globally. + +By the time many new features appeared in _wdio_ that interested developers, such as [Chrome DevTools Protocol (CDP)][how-to-use-cdp], [mocking external requests][how-to-intercept-requests-and-responses], extended mobile device capabilities, etc., the hermione team had no choice but to abandon the _webdriverio@4_ fork and switch to the latest version of _webdriverio@7_. + +<Admonition type="info"> + Version 8 of _webdriverio_ is already available, and _hermione@7_ uses it. +</Admonition> + +Additionally, users found it increasingly inconvenient to use outdated commands: typings had to be included from a separate package (in the new _wdio_, they are included out of the box), and documentation for commands had to be accessed on the [old site][webdriverio-v4-api], while some users occasionally visited the [current page][webdriverio-api] and couldn't understand why the commands from the documentation didn't work in hermione. + +Thus, there were plenty of reasons for a radical upgrade — jumping up by three major versions. + +## What Has Changed? {#what_is_new} + +There are many changes, so only the most important/interesting ones will be listed below. + +### Command API {#api_changes} + +#### async/await Instead of Chaining {#feature_async_await} + +In the new version, you can no longer write tests using _chaining_. Only _async/await_ syntax is available: + +<Tabs> +<TabItem value="Old" label="Old"> +```javascript + it('some test', function() { + return this.browser + .foo() + .bar() + .baz(); + }); + ``` +</TabItem> + +<TabItem value="New" label="New"> +```javascript + it('some test', async function() { + await this.browser.foo(); + await this.browser.bar(); + await this.browser.baz(); + }); + ``` +</TabItem> + +</Tabs> + +Starting from version [hermione@4.9.0](https://github.com/gemini-testing/hermione/blob/master/CHANGELOG.md#490-2022-05-24), you can write tests even shorter, as hermione now passes an object with a `browser` field to the function: + +```javascript +it("some test", async function ({ browser }) { + await browser.foo(); + await browser.bar(); + await browser.baz(); +}); +``` + +<Admonition type="warning"> + In the "New" examples going forward, it is assumed that the hermione version is at least + _4.9.0_. If you plan to use hermione _4+_ but at a lower version than _4.9.0_, you should still + access the browser in tests through _this_, for example: _await + this.browser.getText('.selector')_. +</Admonition> + +#### Direct Result Instead of Object with value Key {#feature_direct_result} + +Now, instead of returning an object with a `value` key, the actual result is directly returned from command results (old behavior often led to errors in tests): + +<Tabs> +<TabItem value="Old" label="Old"> +```javascript + it('some test', async function() { + const { value } = await this.browser.getText('.selector'); + console.log(value); // some text + }); + ``` +</TabItem> + +<TabItem value="New" label="New"> +```javascript + it('some test', async function({ browser }) { + const text = await browser.getText('.selector'); + console.log(text); // some text + }); + ``` +</TabItem> + +</Tabs> + +#### Direct Work with Elements {#feature_commands_of_elements} + +Using the [`browser.$`][browser-dollar] command, you can get an instance of the found element and work with it in the test. This is convenient when you need to interact with an element more than once (the element won't be searched again on the page): + +<Tabs> +<TabItem value="Old" label="Old"> +```javascript + it('some test', async function() { + await this.browser.clearElement('.input'); + await this.browser.setValue('.input', 'text'); + }); + ``` +</TabItem> + +<TabItem value="New" label="New"> +```javascript + it('some test', async function({ browser }) { + const elem = await browser.$('.input'); + + await elem.clearElement(); + await elem.setValue('text'); + +}); + +```` +</TabItem> + +</Tabs> + +Also see the commands: + +* [`browser.$$`][browser-dollar-dollar] +* [`element.$`][element-dollar] +* [`element.$$`][element-dollar-dollar] + +#### Passing Arguments Through an Object {#feature_passing_args_as_object} + +For many commands, arguments are now passed using an object with understandable keys instead of sequentially passing arguments, which could be very confusing. For example, in the [waitForExist][element-wait-for-exist] command, which previously accepted even boolean values as arguments: + +<Tabs> +<TabItem value="Old" label="Old"> +```javascript +it('some test', async function() { + await this.browser.waitForExist('.selector', 1000, true); +}); +```` + +</TabItem> + +<TabItem value="New" label="New"> +```javascript + it('some test', async function({ browser }) { + const elem = await browser.$('.selector'); + + await elem.waitForExist({ + timeout: 1000, + interval: 500, + reverse: true, + timeoutMsg: 'still exists' + }); + +}); + +```` +</TabItem> + +</Tabs> + +#### Special Command for React {#feature_commands_for_react} + +A bonus for those who are already using React — now you can use the [`browser.react$`][browser-react-dollar] and [`browser.react$$`][browser-react-dollar-dollar] commands to find specific react components on the page with certain states. Similar commands are available for elements as well — [`element.react$`][element-react-dollar] and [`element.react$$`][element-react-dollar-dollar]. + +Also read [the article on working with react components][react-selectors] on the _webdriverio_ website. + +Example usage: + +<Tabs> +<TabItem value="Old" label="Old"> +```javascript +it('some test', async function() { + // no special commands for working with react components :( +}); +```` + +</TabItem> + +<TabItem value="New" label="New"> +```javascript + it('some test', async function({ browser }) { + const component = await browser.react$('MyComponent', { + props: { someProp: true }, + state: 'some-state' + }); + + const result = await component.isDisplayed(); + +}); + +```` +</TabItem> + +</Tabs> + +### Up-to-Date Documentation {#feature_actual_docs} + +While hermione used the old version of _webdriverio@4_, users constantly had to be reminded that the documentation for all commands was located at a separate address: [v4.webdriver.io/api.html][webdriverio-v4-api]. Now, descriptions of all _webdriverio_ commands used by hermione can be found at the standard address: [webdriver.io/docs/api][webdriverio-api]. + +In addition, we [translated the descriptions of all commands into Russian][hermione-commands] and adapted all usage examples to hermione, since _webdriverio_ uses its own runner and the examples in its documentation cannot be directly applied in hermione. + +### Tests Run Faster {#feature_running_tests_faster} + +You might not notice this speedup when running several tests locally, but it will be very noticeable with a large number of tests. The new commands work about 15% faster (assuming you have stopped using the old commands). + +### Easy Local Browser Testing {#feature_running_tests_in_local_browser} + +Previously, to run tests locally in your browser, you had to start `selenium-standalone` and specify a magical `gridUrl` to make things work in hermione. Now, it's much simpler: in the config, just specify the `automationProtocol` option with the value `devtools`: + +```javascript +// hermione.conf.js +module.exports = { + browsers: { + chrome: { + automationProtocol: 'devtools', + desiredCapabilities: { + // ... + } + } + }, + + // other hermione settings... +}; +```` + +We also plan to add a separate button in the hermione GUI for switching to CDP mode to make it even easier. + +<Admonition type="warning"> + * Currently, this is fully supported only in the _Chrome_ browser. * Retaking screenshots in + this mode should only be done for debugging, as browsers in the pipeline run under _Linux_, + which means page rendering will differ and tests in the pull request will fail with diffs. +</Admonition> + +### API for Network Request Stubbing {#feature_api_to_mock_network} + +The new version provides the ability to stub or override the responses of your service. This is done using the [mock.respond()][mock] command. You can also block URLs of external services. + +Read more about all the features in the "How to Track and Intercept Network Requests and Responses" guide. + +<Admonition type="warning"> + Currently, this functionality only works in _Chrome DevTools Protocol (CDP)_ mode, which only + works in _Chrome_ and _Firefox Nightly_. +</Admonition> + +### Browser Configuration in the Config {#browsers_config} + +For browsers that support the W3C protocol, instead of the `version` field, you need to specify `browserVersion`. And additional options need to be prefixed with the browser's name: + +<Tabs> +<TabItem value="Old" label="Old"> +```javascript + module.exports = { + browsers: { + 'chrome-desktop': { + desiredCapabilities: { + browserName: 'chrome', + version: '75', + 'chromeOptions': { + // ... + } + } + } + } + }; + ``` +</TabItem> + +<TabItem value="New" label="New"> +```javascript + module.exports = { + browsers: { + 'chrome-desktop': { + desiredCapabilities: { + browserName: 'chrome', + browserVersion: '75', + 'goog:chromeOptions': { // for Chrome browser, prefix with 'goog' + // ... + } + } + } + } + }; + ``` +</TabItem> + +</Tabs> + +Read more about vendor prefixes at [this link](https://w3c.github.io/webdriver/#protocol-extensions). + +The list of all available settings can be found in the [specification](https://w3c.github.io/webdriver/#capabilities). + +## How to Migrate? {#how_to_move} + +We upgraded _webdriverio_ by three major versions at once, so simply updating the hermione version in `package.json` won't suffice. The main issues during migration are the absent _chaining_ in tests and outdated test commands. To help you with these issues, we wrote the following guide. + +### 1. Update hermione to 4+, Install Migrator Plugin and Codemod {#update_hermione_and_install_plugin_and_codmode} + +Specifically: + +- Update hermione to hermione@4.0.0. +- Install the [hermione-wdio-migrator][hermione-wdio-migrator] plugin for smooth command migration. +- Install the [hermione-codemod][hermione-codemod] package to convert existing tests to the new syntax. + +You can do all this with one command: + +```shell +npm install -D hermione@4.0.0 hermione-wdio-migrator hermione-codemod --save-exact +``` + +The versions of all hermione plugins (e.g., [html-reporter][html-reporter]) also need to be updated to the latest versions, as some may not work correctly with the new hermione version. + +### 2. Run the Codemod for async/await {#change_to_async_await_by_codmode} + +The codemod will regenerate your tests from _chaining_ format to _async/await_ format: + +<Tabs> +<TabItem value="zsh" label="zsh"> +If you use the _zsh_ shell, you can pass test files as relative paths and as glob patterns, such as _somefolder/**/*.js_, and so on. + + ```shell + npx jscodeshift -t node_modules/hermione-codemod/transforms/browser-chaining-to-async-await.js path_to_file_mask + ``` + +</TabItem> + +<TabItem value="bash" label="bash"> +If you use the _bash_ shell, globs won't work as easily as in _zsh_, so the command will be more complex if you need to process a group of files. For example: + + ```shell + npx jscodeshift -t node_modules/hermione-codemod/transforms/browser-chaining-to-async-await.js $(find ./somefolder -type f -iname '*.js' | xargs echo) + ``` + +</TabItem> + +</Tabs> + +Upon successful completion, you will see a corresponding message: + +```shell +Results: +0 errors +0 unmodified +0 skipped +251 ok +``` + +However, there can be cases the current codemod cannot handle. For such tests, an error with information about the problematic file will be displayed: + +```shell +WARN: can't correctly transform ConditionalExpression, fix it manually + file: tests/hermione/suites/common/promotion-page/promotion-page.hermione.js + position: {"start":112,"end":116} +``` + +Such tests will need to be fixed manually. We tried to account for most test cases, so there shouldn't be too many of these instances. + +After this, you can **already** merge your changes (this is optional) to perform the migration in parts. The _async/await_ syntax will still work in _wdio@4_. Many services have been writing tests this way for a long time. + +### 3. Run the Codemod to Remove value {#remove_value_by_codmode} + +Since command results now directly return the actual result instead of an object with a `value` key, you need to change result handling in all tests. This codemod is intended for such cases. + +The command is very similar to the previous one, only the path to the next codemod file changes: + +```shell +npx jscodeshift -t node_modules/hermione-codemod/transforms/remove-browser-prop.js path_to_file_mask +``` + +For any warnings, problematic tests need to be fixed manually. For example, if a test uses `value` multiple times through destructuring, the codemod will not handle it properly and generic variable names might be generated. For example: + +```javascript +// test on wdio@4: +it('test', function(){ + return this.browser + ... + .getText('.button') + .then((value) => { + assert.equal(value, 'Button', 'We need a button'); + }) + ... + .getValue('.input') + .then((value) => { + assert.equal(value, 'Hello', 'We were not greeted'); + }); +}); + +// auto-generated for wdio@7 in this format: +it('test', async function() { + const value = await this.browser.getText('.button'); + assert.equal(value, 'Button', 'We need a button'); + ... + // there will be an error due to reuse of the variable name, + // so the codemod will issue a warning about the problematic spot + const value = await this.browser.getValue('.input'); + assert.equal(value, 'Hello', 'We were not greeted'); +}); +``` + +If your project has too many tests that cannot be automatically migrated, please reach out to [github issues][gh-issues] for help. We will analyze these errors and assist with the migration. + +### 4. Add hermione-wdio-migrator to the hermione Config {#add_migrator} + +This plugin "under the hood" simply adds the implementation of old commands using the new API, so during migration, you don't have to update tests yourself. Eventually, you should replace these deprecated commands with new ones in your tests: + +```javascript +module.exports = { + plugins: { + "hermione-wdio-migrator": { + enabled: true, + }, + + // other hermione plugins... + }, + + // other hermione settings... +}; +``` + +### 5. Remove the Codemod {#remove_codemod} + +Uninstall the [hermione-codemod][hermione-codemod] package as you won't need it anymore. + +```shell +npm uninstall hermione-codemod +``` + +### 6. Run Linters {#run_linters} + +Run linters on the modified tests, as the codemod might have violated your project's coding standards. + +### 7. Run the Tests {#run_tests} + +Finally, ensure that all tests pass successfully by creating a pull request and verifying that all tests run successfully in CI. + +## Conclusion {#conclusion} + +Updating _webdriverio_ brings many useful features that can be used right away, while some will appear in a more convenient form later. Therefore, we highly recommend upgrading to the new version to make writing tests more convenient, faster, and enjoyable. + +<Admonition type="warning"> + The old hermione version is in limited support mode, and new features will not appear in it. +</Admonition> + +## Support {#support} + +If you encounter issues during the upgrade or have any questions, come to [github issues][gh-issues] — we will definitely help you! + +[how-to-use-cdp]: ../../guides/how-to-use-cdp +[how-to-intercept-requests-and-responses]: ../../guides/how-to-intercept-requests-and-responses +[webdriverio]: https://webdriver.io +[webdriverio-api]: https://webdriver.io/docs/api +[webdriverio-v4-api]: http://v4.webdriver.io/api.html +[browser-dollar]: ../../commands/browser/_dollar +[browser-dollar-dollar]: ../../commands/browser/_dollardollar +[element-dollar]: ../../commands/element/_dollar +[element-dollar-dollar]: ../../commands/element/_dollardollar +[element-wait-for-exist]: ../../commands/element/waitForExist +[browser-react-dollar]: ../../commands/browser/reactDollar +[browser-react-dollar-dollar]: ../../commands/browser/reactDollarDollar +[element-react-dollar]: ../../commands/element/reactDollar +[element-react-dollar-dollar]: ../../commands/element/reactDollarDollar +[react-selectors]: https://webdriver.io/docs/selectors/#react-selectors +[mock]: ../../commands/mock/respond +[hermione-commands]: ../../commands/overview +[hermione-wdio-migrator]: https://github.com/gemini-testing/hermione-wdio-migrator +[hermione-codemod]: https://github.com/gemini-testing/hermione-codemod +[html-reporter]: ../../html-reporter/html-reporter-setup +[gh-issues]: https://github.com/gemini-testing/testplane/issues/ diff --git a/i18n/en/docusaurus-plugin-content-docs/current/migrations/how-to-upgrade-hermione-to-5.mdx b/i18n/en/docusaurus-plugin-content-docs/current/migrations/how-to-upgrade-hermione-to-5.mdx new file mode 100644 index 0000000..cfae071 --- /dev/null +++ b/i18n/en/docusaurus-plugin-content-docs/current/migrations/how-to-upgrade-hermione-to-5.mdx @@ -0,0 +1,77 @@ +import Admonition from "@theme/Admonition"; + +# How to Upgrade hermione to Version 5.x + +<Admonition type="warning"> +If your project uses hermione version earlier than 4.x, please read “How to Upgrade hermione to Version 4.x” before upgrading to version 5.x. + +We recommend upgrading hermione in stages, ensuring at each stage that all project tests run correctly. + +</Admonition> + +## What Has Changed? {#what_is_new} + +### New Default Configurations {#changes_in_config_options} + +The first thing to note during the upgrade is the change in default values for some configuration options. All changes aim to speed up test execution with basic hermione setup and provide more information about tests after execution. + +<table> +<thead> +<tr><td>**Parameter**</td><td>**Old**</td><td>**New**</td><td>**Description**</td></tr> +</thead> +<tbody> +<tr><td>[antialiasingTolerance][antialiasing-tolerance]</td><td>0</td><td>4</td><td>Sets the sensitivity for detecting antialiasing, which will be ignored when comparing screenshots.</td></tr> +<tr><td>[compositeImage][composite-image]</td><td>false</td><td>true</td><td>Allows testing elements that do not fit into the viewport height.</td></tr> +<tr><td>[saveHistory][save-history]</td><td>false</td><td>true</td><td>Save the history of all executed commands.</td></tr> +<tr><td>[takeScreenshotOnFails.assertViewFail][take-screenshot-on-fails-assert-view-fail]</td><td>false</td><td>true</td><td>Determines whether to take a page screenshot on test failure, including on _assertView_ command failure.</td></tr> +<tr><td>[takeScreenshotOnFailsMode][take-screenshot-on-fails-mode]</td><td>"viewport"</td><td>"fullpage"</td><td>Screenshot mode on test failure. Available values: _viewport_ and _fullpage_.</td></tr> +<tr><td>[takeScreenshotOnFailsTimeout][take-screenshot-on-fails-timeout]</td><td>90000</td><td>5000</td><td>Timeout for taking a page screenshot on test failure, in ms.</td></tr> +<tr><td>[httpTimeout][http-timeout]</td><td>90000</td><td>30000</td><td>Timeout for any requests to the Selenium server, in ms.</td></tr> +<tr><td>[pageLoadTimeout][page-load-timeout]</td><td>300000</td><td>20000</td><td>Timeout for full page loading, in ms.</td></tr> +<tr><td>[sessionQuitTimeout][session-quit-timeout]</td><td>90000</td><td>5000</td><td>Timeout for session termination, in ms.</td></tr> + +</tbody> +</table> + +Additionally, the `screenshotOnReject` and `screenshotOnRejectTimeout` options, previously marked as deprecated, have been removed. + +You can read more about these options [here][config-browsers]. If the new values suit you and there are no apparent reasons to override them, you can skip this section during the migration. + +### CLI Reporters {#changes_in_reporters} + +This section discusses CLI reporters that come out of the box and not the html-reporter. + +- The teamcity reporter has been removed, as it seemed out of place within hermione. If you still use such a report, you can use the [hermione-teamcity-reporter][hermione-teamcity-reporter] plugin. +- The `-r` option, which previously allowed specifying the reporter type, no longer does this. It is more often used for the `--require` option. Many users found this confusing, so we decided to fix it. You can still specify the reporter using the `--reporter` option. +- If you want to add your own reporter, it must have a `create` method for initialization. + +So, if you did not use the teamcity reporter or did not write new reporters, you can skip this section during migration. + +### testParserAPI {#changes_in_test_parser_api} + +The [testParser][test-parser] object, which could be obtained by subscribing to the [BEFORE_FILE_READ][before-file-read] event, is no longer an instance of `EventEmitter`. This means that you can no longer use it to subscribe to the [SUITE_BEGIN][event-suite-begin] and [TEST_BEGIN][event-test-begin] events. + +To be fair, this initially did not work correctly, and no one used it. Therefore, this functionality was removed. You can read about all available events [here][hermione-events]. + +## Support {#support} + +If you encounter any issues during the migration to the new version or have any questions, visit [github issues][gh-issues] — we will definitely help you! + +[how-to-upgrade-hermione-to-4]: ../../migrations/how-to-upgrade-hermione-to-4 +[config-browsers]: ../../config/browsers +[hermione-teamcity-reporter]: https://github.com/gemini-testing/hermione-teamcity-reporter +[before-file-read]: ../../reference/testplane-events#before_file_read +[event-suite-begin]: ../../reference/testplane-events#suite_begin +[event-test-begin]: ../../reference/testplane-events#test_begin +[test-parser]: ../../reference/testplane-events#test_parser +[hermione-events]: ../../reference/testplane-events +[antialiasing-tolerance]: ../../config/browsers#antialiasing_tolerance +[composite-image]: ../../config/browsers#composite_image +[take-screenshot-on-fails-assert-view-fail]: ../../config/browsers#take_screenshot_on_fails +[take-screenshot-on-fails-mode]: ../../config/browsers#take_screenshot_on_fails_mode +[take-screenshot-on-fails-timeout]: ../../config/browsers#take_screenshot_on_fails_timeout +[http-timeout]: ../../config/browsers#http_timeout +[page-load-timeout]: ../../config/browsers#page_load_timeout +[session-quit-timeout]: ../../config/browsers#session_quit_timeout +[save-history]: ../../config/browsers#save_history +[gh-issues]: https://github.com/gemini-testing/testplane/issues diff --git a/i18n/en/docusaurus-plugin-content-docs/current/migrations/how-to-upgrade-hermione-to-6.mdx b/i18n/en/docusaurus-plugin-content-docs/current/migrations/how-to-upgrade-hermione-to-6.mdx new file mode 100644 index 0000000..d5a86a7 --- /dev/null +++ b/i18n/en/docusaurus-plugin-content-docs/current/migrations/how-to-upgrade-hermione-to-6.mdx @@ -0,0 +1,51 @@ +import Admonition from "@theme/Admonition"; + +# How to Upgrade hermione to Version 6.x + +<Admonition type="warning"> +If your project is using a hermione version earlier than 4.x, please first read "How to Upgrade hermione to Version 4.x" and "How to Upgrade hermione to Version 5.x". + +We recommend upgrading hermione in stages, ensuring each step that all project tests run correctly. + +</Admonition> + +## What Has Changed? {#what_is_new} + +### Dropped Support for Node.JS < 14.x {#dropped_nodejs_less_than_14x} + +In this major version, hermione no longer supports versions _Node.JS < 14.x_. + +If your project is already using _Node.JS_ version _14.x_ or higher and this does not apply to you, we still recommend upgrading hermione to version 6 to get [the latest features][hermione-new-features]. + +### browserWSEndpoint {#added_browser_ws_endpoint_setting} + +There is now a [browserWSEndpoint][browser-ws-endpoint] option in the browser settings, which allows you to override the link for accessing browsers via the [Chrome DevTools Protocol (CDP)][how-to-use-cdp]. + +Example configuration: + +```javascript +//.hermione.conf.js +const gridHost = "localhost"; + +module.exports = { + gridUrl: `https://${gridHost}/:4444/wd/hub`, + browserWSEndpoint: `ws:${gridHost}/wd/hub`, + + // other hermione settings... +}; +``` + +<Admonition type="warning"> + This setting will not work if your project is using a very old browser version. It is guaranteed + to work in Chrome, starting from version 101. +</Admonition> + +## Support {#support} + +If you encounter issues during the migration to the new version or have any questions, visit [github issues][gh-issues] — we will definitely help you! + +[how-to-upgrade-hermione-to-4]: ../../migrations/how-to-upgrade-hermione-to-4 +[how-to-upgrade-hermione-to-5]: ../../migrations/how-to-upgrade-hermione-to-5 +[browser-ws-endpoint]: ../../config/browsers#browser_ws_endpoint +[how-to-use-cdp]: ../../guides/how-to-use-cdp +[gh-issues]: https://github.com/gemini-testing/testplane/issues diff --git a/i18n/en/docusaurus-plugin-content-docs/current/migrations/how-to-upgrade-hermione-to-7.mdx b/i18n/en/docusaurus-plugin-content-docs/current/migrations/how-to-upgrade-hermione-to-7.mdx new file mode 100644 index 0000000..16d3046 --- /dev/null +++ b/i18n/en/docusaurus-plugin-content-docs/current/migrations/how-to-upgrade-hermione-to-7.mdx @@ -0,0 +1,137 @@ +import Admonition from "@theme/Admonition"; + +# How to Upgrade hermione to Version 7.x + +<Admonition type="warning"> +If your project is using a hermione version earlier than 4.x, first read "How to Upgrade hermione to Version 4.x," "How to Upgrade hermione to Version 5.x," and "How to Upgrade hermione to Version 6.x." + +We recommend upgrading hermione in stages, ensuring each step that all project tests run correctly. + +</Admonition> + +## What Has Changed? {#what_is_new} + +### Major Changes {#major_changes} + +#### Dropped Support for Node.JS < 16.x {#dropped_nodejs_less_than_16x} + +In this major version, hermione no longer supports versions _Node.JS < 16.x_. + +#### Handling Unhandled Rejections {#handle_unhandled_rejection} + +When an `Unhandled rejection` error occurs in the master or workers, hermione now performs a graceful shutdown. This means all subsequent tests in the queue will end with the error `Browser request was cancelled`, and hermione itself will terminate with `exit code 1`. + +This change is necessary to protect users from writing incorrect tests. Example of a test with an error: + +```typescript +it("test", async ({ browser }) => { + await browser.url("https://yandex.ru/search/?text=aaa"); + expect(browser).toHaveUrlContaining("foo-bar-baz"); +}); +``` + +A keen observer will notice that I forgot to add `await` before calling `expect`. In this case, the test will immediately pass after navigating to the URL. Only then will an `Unhandled rejection` error appear, which hermione@6 would swallow. But hermione@7 will catch and immediately terminate the test run. Unfortunately, it is impossible to determine from which test the `Unhandled rejection` error originated. For example, a test could be written like this: + +```typescript +it("test2", async () => { + new Promise((resolve, reject) => { + setTimeout(() => { + reject("Something goes wrong"); + }, 30000); + }); +}); +``` + +This test will immediately pass, and only after 30 seconds (assuming there are still tests in the queue to run) will an `Unhandled rejection` error appear. + +Therefore, to find the problematic test, you will need to look at the execution log and find the last completed tests before the error information. The error will look something like this: + +```bash +[13:48:57 GMT+3] Unhandled Rejection in hermione:worker:10830: +Promise: {} +Reason: Something goes wrong +[13:48:57 GMT+3] Terminating on critical error: Unhandled Rejection in hermione:worker:10830: +Promise: {} +Reason: Something goes wrong +``` + +#### test.id and suite.id Are Now Properties {#test_id_and_suite_id_are_properties} + +The `id` for a test and suite is now a property, not a method, and should be used as `test.id` or `suite.id`. The `test.id` call is usually used in the hermione config to define the [screenshotsDir][screenshots-dir] option. + +#### Removed the saveHistory Option from the Config {#removed_save_history_option_from_config} + +The [saveHistory][save_history] option has been removed from the config. You now need to use [saveHistoryMode][save_history_mode] with available values: `all`, `none`, `onlyFailed`. The default is `all`, meaning history is saved for all tests. So, you can skip explicitly setting this option. + +### Minor Changes {#minor_changes} + +- Upgraded from [webdriverio@7][webdriverio@7] to [webdriverio@8][webdriverio@8]; +- Increased test reading speed by about 3 times: tests are now read once in the master (previously read separately for each browser); +- Added the ability to write the config in TS. You can create `.hermione.conf.ts`, and hermione will compile and read it (assuming your project includes `ts-node`); +- Added timestamps to hermione logs, looking like: + + ```bash + [13:48:09 GMT+3] ✓ suite test2 [chrome-desktop:SESSION_ID] - 875ms + ``` + +### Patch Changes {#patch_changes} + +- Properly supported taking screenshots on devices with fractional `pixelRatio` (e.g., `Google Pixel`). Previously, screenshots on such devices were incorrect; +- Moving the cursor away using the `resetCursor: true` option no longer triggers an error if the top-left corner of the `body` element has negative coordinates. + +## How to Migrate? {#how_to_move} + +### 1. Update hermione to 7+ and Plugins to the Latest Versions {#update_hermione_and_plugins} + +Specifically: + +- [hermione][hermione] +- [html-reporter][html-reporter] +- [hermione-hide-scrollbars][hermione-hide-scrollbars] +- [hermione-safari-commands][hermione-safari-commands] +- [hermione-wdio-migrator][hermione-wdio-migrator] +- [json-reporter][json-reporter] +- [hermione-passive-browsers][hermione-passive-browsers] + +If you are not using any of these plugins, you do not need to install them. + +### 2. Replace test.id() with test.id in the hermione Config {#replace_test_id_method_on_test_id_property} + +If you are not using `test.id()`, there is nothing to do. + +### 3. Replace the saveHistory Option with saveHistoryMode in the hermione Config {#replace_save_history_on_save_history_mode} + +If you are not using [saveHistory][save_history], there is nothing to do. + +### 4. Replace the Usage of a Custom Mocha Interface {#replace_custom_mocha_interface} + +If you are using a custom mocha interface, replace it with the mocha interface from hermione dependencies: + +```javascript +- const baseBdd = require('@gemini-testing/mocha/lib/interfaces/bdd'); + ++ const { dependencies: hermioneDeps } = require('hermione/package'); ++ const mochaModule = '@gemini-testing/mocha' in hermioneDeps ? '@gemini-testing/mocha' : 'mocha'; ++ const { interfaces: { bdd: baseBdd } } = require(mochaModule); +``` + +## Support {#support} + +If you encounter issues during the migration to the new version or have any questions, visit [github issues][gh-issues] — we will definitely help you! + +[how-to-upgrade-hermione-to-4]: ../../migrations/how-to-upgrade-hermione-to-4 +[how-to-upgrade-hermione-to-5]: ../../migrations/how-to-upgrade-hermione-to-5 +[how-to-upgrade-hermione-to-6]: ../../migrations/how-to-upgrade-hermione-to-6 +[screenshots-dir]: ../../config/browsers#screenshots_dir +[save_history]: ../../config/browsers#save_history +[save_history_mode]: ../../config/browsers#save_history_mode +[webdriverio@7]: https://webdriver.io/ +[webdriverio@8]: https://webdriver.io/ +[gh-issues]: https://github.com/gemini-testing/testplane/issues/ +[hermione]: https://github.com/gemini-testing/hermione +[html-reporter]: https://github.com/gemini-testing/html-reporter +[hermione-hide-scrollbars]: https://github.com/gemini-testing/hermione-hide-scrollbars +[hermione-safari-commands]: https://github.com/gemini-testing/hermione-safari-commands +[hermione-wdio-migrator]: https://github.com/gemini-testing/hermione-wdio-migrator +[json-reporter]: https://github.com/gemini-testing/json-reporter +[hermione-passive-browsers]: https://github.com/gemini-testing/hermione-passive-browsers diff --git a/i18n/en/docusaurus-plugin-content-docs/current/migrations/how-to-upgrade-hermione-to-8.mdx b/i18n/en/docusaurus-plugin-content-docs/current/migrations/how-to-upgrade-hermione-to-8.mdx new file mode 100644 index 0000000..f23efe2 --- /dev/null +++ b/i18n/en/docusaurus-plugin-content-docs/current/migrations/how-to-upgrade-hermione-to-8.mdx @@ -0,0 +1,96 @@ +import Admonition from "@theme/Admonition"; + +# How to Upgrade hermione to Version 8.x + +<Admonition type="warning"> +If your project is using a hermione version earlier than 4.x, first read: +* "How to Upgrade hermione to Version 4.x"; +* "How to Upgrade hermione to Version 5.x"; +* "How to Upgrade hermione to Version 6.x"; +* "How to Upgrade hermione to Version 7.x". + +We recommend upgrading hermione in stages, ensuring each step that all project tests run correctly. + +</Admonition> + +## What Has Changed? {#what_is_new} + +### Major Changes {#major_changes} + +#### Package Exports Changed {#package_exports_changed} + +To use hermione via API in JavaScript, you now need to import `hermione` as follows: + +```javascript +const { default: hermione } = require("hermione"); +``` + +Previously, you could write it like this: + +```javascript +const hermione = require("hermione"); +``` + +For TypeScript: + +```typescript +import hermione from "hermione"; +``` + +#### moveTo Command Changed {#move_command_changed} + +The `moveTo` command now moves the cursor relative to the center of the element rather than the top-left corner. If the element is not visible, it will scroll to it. Previously, calling the command without arguments (`await browser.$('body').moveTo()`) moved the cursor to the top-left corner of the element - now it will move to the center of the element. + +If you don't have time to switch to the new logic of `moveTo`, you can use the `moveCursorTo` command, which works as it did in hermione@7. This command was added in hermione@8.1.0. The `moveCursorTo` command is temporary and will be removed in the next major version. + +#### Dropped Support for Node.JS < 18.x {#dropped_nodejs_less_than_18x} + +In this major version, hermione no longer supports versions _Node.JS < 18.x_. + +### Minor Changes {#minor_changes} + +- Added REPL mode for step-by-step debugging of tests in all browsers (not only CDP) without restarting. +- Added a browser command [clearSession][hermione-clear-session] to clear session state: + - Deletes cookies; + - Clears local storage; + - Clears session storage. +- Added a browser command `openAndWait` with customizable wait options for page loads (by selector, custom predicate, network request, etc.). +- Added a CLI option `--devtools` to simplify switching between the two protocols (`devtools` and `webdriver`). +- Improved stack trace for `unhandled rejection` errors. +- Now [isolation][hermione-isolation] is enabled by default for Chrome >= 94. +- During the execution of the [assertView][hermione-assert-view] command, CSS animations on the page will be disabled by default. +- Implemented generation of a unique `X-Request-ID` header for each request in the browser. The header consists of `${TEST_X_REQ_ID}${DELIMITER}$BROWSER_X_REQ_ID}`, where: + + - `TEST_X_REQ_ID` - a unique UUID for each test run (including retries of the same test). This allows you to find all requests related to a single test run in the logs. + - `DELIMITER` - `__` separator between the test and request UUIDs. + - `BROWSER_X_REQ_ID` - a unique UUID for each browser request. + + A real example of a UUID is `2f31ffb7-369d-41f4-bbb8-77744615d2eb__e8d011d8-bb76-42b9-b80e-02f03b8d6fe1`. + +### Patch Changes {#patch_changes} + +- Fixed disabling animations in iframes for iOS when using [assertView][hermione-assert-view]. +- Eliminated reinitialization of browser sessions in workers. +- Fixed a bug where it was impossible to disable [isolation][hermione-isolation]. + +## How to Migrate? {#how_to_move} + +### 1. Update hermione to 8+ and Plugins to the Latest Versions {#update_hermione_and_plugins} + +### 2. If Using hermione via API in JavaScript, Switch to TypeScript or Change the Import Statement {#replace_hermione_require} + +### 3. If Using moveTo, Switch to moveToCursor or Specify Coordinates from the Center of the Element {#replace_move_to} + +## Support {#support} + +If you encounter issues during the migration to the new version or have any questions, visit [github issues][gh-issues] — we will definitely help you! + +[how-to-upgrade-hermione-to-4]: ../../migrations/how-to-upgrade-hermione-to-4 +[how-to-upgrade-hermione-to-5]: ../../migrations/how-to-upgrade-hermione-to-5 +[how-to-upgrade-hermione-to-6]: ../../migrations/how-to-upgrade-hermione-to-6 +[how-to-upgrade-hermione-to-7]: ../../migrations/how-to-upgrade-hermione-to-7 +[prepare-browser]: ../../config/prepare-browser +[hermione-clear-session]: https://github.com/gemini-testing/hermione?tab=readme-ov-file#clearsession +[hermione-isolation]: https://github.com/gemini-testing/hermione?tab=readme-ov-file#isolation +[hermione-assert-view]: https://github.com/gemini-testing/hermione?tab=readme-ov-file#assertview +[gh-issues]: https://github.com/gemini-testing/testplane/issues diff --git a/i18n/en/docusaurus-plugin-content-docs/current/plugins/hermione-browser-version-changer.mdx b/i18n/en/docusaurus-plugin-content-docs/current/plugins/hermione-browser-version-changer.mdx index bb6ab53..ddd8d55 100644 --- a/i18n/en/docusaurus-plugin-content-docs/current/plugins/hermione-browser-version-changer.mdx +++ b/i18n/en/docusaurus-plugin-content-docs/current/plugins/hermione-browser-version-changer.mdx @@ -33,10 +33,10 @@ module.exports = { } }, - // other testplane plugins... + // other Testplane plugins... }, - // other testplane settings... + // other Testplane settings... }; ``` diff --git a/i18n/en/docusaurus-plugin-content-docs/current/plugins/hermione-hide-scrollbars.mdx b/i18n/en/docusaurus-plugin-content-docs/current/plugins/hermione-hide-scrollbars.mdx index 0cc9c4e..db1658d 100644 --- a/i18n/en/docusaurus-plugin-content-docs/current/plugins/hermione-hide-scrollbars.mdx +++ b/i18n/en/docusaurus-plugin-content-docs/current/plugins/hermione-hide-scrollbars.mdx @@ -36,10 +36,10 @@ module.exports = { `ws://${url.parse(gridUrl).host}/devtools/${sessionId}`, }, - // other testplane plugins... + // other Testplane plugins... }, - // other testplane settings... + // other Testplane settings... }; ``` diff --git a/i18n/en/docusaurus-plugin-content-docs/current/plugins/html-reporter.mdx b/i18n/en/docusaurus-plugin-content-docs/current/plugins/html-reporter.mdx new file mode 100644 index 0000000..e81a53c --- /dev/null +++ b/i18n/en/docusaurus-plugin-content-docs/current/plugins/html-reporter.mdx @@ -0,0 +1,5 @@ +# html-reporter + +Read more about `html-reporter` plugin in the «[Html-Reporter Setup][html-reporter-setup]» section. + +[html-reporter-setup]: ../../html-reporter/html-reporter-setup diff --git a/i18n/en/docusaurus-plugin-content-docs/current/plugins/stat-reporter.mdx b/i18n/en/docusaurus-plugin-content-docs/current/plugins/stat-reporter.mdx index fada716..1b7d40c 100644 --- a/i18n/en/docusaurus-plugin-content-docs/current/plugins/stat-reporter.mdx +++ b/i18n/en/docusaurus-plugin-content-docs/current/plugins/stat-reporter.mdx @@ -74,10 +74,10 @@ Add the plugin to the `plugins` section of the `testplane` config: // the report will only be in the console }, - // other testplane plugins... + // other Testplane plugins... }, - // other testplane settings... + // other Testplane settings... }; ``` @@ -108,10 +108,10 @@ Add the plugin to the `plugins` section of the `testplane` config: } }, - // other testplane plugins... + // other Testplane plugins... }, - // other testplane settings... + // other Testplane settings... }; ``` diff --git a/i18n/en/docusaurus-plugin-content-docs/current/plugins/testplane-chunks.mdx b/i18n/en/docusaurus-plugin-content-docs/current/plugins/testplane-chunks.mdx index caf1ee1..9258b50 100644 --- a/i18n/en/docusaurus-plugin-content-docs/current/plugins/testplane-chunks.mdx +++ b/i18n/en/docusaurus-plugin-content-docs/current/plugins/testplane-chunks.mdx @@ -12,7 +12,7 @@ The plugin allows you to run tests in parallel on multiple servers. Thereby spee ### What does this plugin do then? {#what_does_plugin_do} -The `testplane-chunks` plugin splits **always in the same way** your tests into a given number of chunks and gives testplane to run only the chunk that you specified. The key here is _"always in the same way"_. That is, the operation of splitting into chunks is [idempotent][idempotence]. +The `testplane-chunks` plugin splits **always in the same way** your tests into a given number of chunks and gives Testplane to run only the chunk that you specified. The key here is _"always in the same way"_. That is, the operation of splitting into chunks is [idempotent][idempotence]. ### What gives idempotence? {#what_does_idempotence_give} @@ -60,10 +60,10 @@ module.exports = { run: 1, // Run 1st chunk }, - // other testplane plugins... + // other Testplane plugins... }, - // other testplane settings... + // other Testplane settings... }; ``` diff --git a/i18n/en/docusaurus-plugin-content-docs/current/plugins/testplane-global-hook.mdx b/i18n/en/docusaurus-plugin-content-docs/current/plugins/testplane-global-hook.mdx index 6429c59..b4f30b7 100644 --- a/i18n/en/docusaurus-plugin-content-docs/current/plugins/testplane-global-hook.mdx +++ b/i18n/en/docusaurus-plugin-content-docs/current/plugins/testplane-global-hook.mdx @@ -4,7 +4,7 @@ Use the [testplane-global-hook][testplane-global-hook] plugin to bring out the common logic from your tests into special handlers for `beforeEach` and `afterEach` hooks. -Often, before running the next testplane test, you need to do some preliminary setup, for example: +Often, before running the next Testplane test, you need to do some preliminary setup, for example: - clear all cookies; - clean up your local storage; @@ -12,7 +12,7 @@ Often, before running the next testplane test, you need to do some preliminary s In order not to repeat these actions in each test, you can describe them in the plugin settings as an async-function for the `beforeEach` hook. -Similarly, after completing the basic checks in the testplane test, you may want to always check for errors in the client code, the triggering of the necessary metrics, etc. +Similarly, after completing the basic checks in the Testplane test, you may want to always check for errors in the client code, the triggering of the necessary metrics, etc. In order not to repeat these actions in each test, you can describe them in the plugin settings as an async-function for the `afterEach` hook. @@ -42,10 +42,10 @@ module.exports = { }, }, - // other testplane plugins... + // other Testplane plugins... }, - // other testplane settings... + // other Testplane settings... }; ``` diff --git a/i18n/en/docusaurus-plugin-content-docs/current/plugins/testplane-image-minifier.mdx b/i18n/en/docusaurus-plugin-content-docs/current/plugins/testplane-image-minifier.mdx index 32effbc..1b31c9b 100644 --- a/i18n/en/docusaurus-plugin-content-docs/current/plugins/testplane-image-minifier.mdx +++ b/i18n/en/docusaurus-plugin-content-docs/current/plugins/testplane-image-minifier.mdx @@ -14,7 +14,7 @@ Compression is lossless. At startup, the plugin subscribes to the `UPDATE_REFERENCE` event, which Testplane sends in the following cases: -- if the user started testplane by passing the `--update-refs` option; +- if the user started Testplane by passing the `--update-refs` option; - if the user updates or saves screenshots using the [html-reporter][html-reporter] plugin. When the `UPDATE_REFERENCE` event is received in the `testplane-image-minifier` plugin, it gets a link to the image itself along with the event. Next, the plugin applies a compression algorithm to the received image with the compression level specified in the config. And saves the new image to the file system. After that, the developer can merge the updated files into the main branch of his project. @@ -48,10 +48,10 @@ module.exports = { compressionLevel: 7, // Maximum compression level, compression will take some time }, - // other testplane plugins... + // other Testplane plugins... }, - // other testplane settings... + // other Testplane settings... }; ``` @@ -96,7 +96,7 @@ An example of how it will look in the console: ## Usage {#usage} -After adding the plugin to the project and configuring its parameters, run testplane with the `--update-refs` option: +After adding the plugin to the project and configuring its parameters, run Testplane with the `--update-refs` option: ```bash npx testplane --update-refs diff --git a/i18n/en/docusaurus-plugin-content-docs/current/plugins/testplane-retry-command.mdx b/i18n/en/docusaurus-plugin-content-docs/current/plugins/testplane-retry-command.mdx index 719c21d..3b2d57c 100644 --- a/i18n/en/docusaurus-plugin-content-docs/current/plugins/testplane-retry-command.mdx +++ b/i18n/en/docusaurus-plugin-content-docs/current/plugins/testplane-retry-command.mdx @@ -37,10 +37,10 @@ module.exports = { ], }, - // other testplane plugins... + // other Testplane plugins... }, - // other testplane settings... + // other Testplane settings... }; ``` diff --git a/i18n/en/docusaurus-plugin-content-docs/current/plugins/testplane-retry-progressive.mdx b/i18n/en/docusaurus-plugin-content-docs/current/plugins/testplane-retry-progressive.mdx index 4859efa..50c4cc4 100644 --- a/i18n/en/docusaurus-plugin-content-docs/current/plugins/testplane-retry-progressive.mdx +++ b/i18n/en/docusaurus-plugin-content-docs/current/plugins/testplane-retry-progressive.mdx @@ -42,10 +42,10 @@ module.exports = { ], }, - // other testplane plugins... + // other Testplane plugins... }, - // other testplane settings... + // other Testplane settings... }; ``` diff --git a/i18n/en/docusaurus-plugin-content-docs/current/plugins/testplane-test-repeater.mdx b/i18n/en/docusaurus-plugin-content-docs/current/plugins/testplane-test-repeater.mdx index 7a3b5c4..249eec9 100644 --- a/i18n/en/docusaurus-plugin-content-docs/current/plugins/testplane-test-repeater.mdx +++ b/i18n/en/docusaurus-plugin-content-docs/current/plugins/testplane-test-repeater.mdx @@ -27,10 +27,10 @@ module.exports = { uniqSession: true, }, - // other testplane plugins... + // other Testplane plugins... }, - // other testplane settings... + // other Testplane settings... }; ``` diff --git a/i18n/en/docusaurus-plugin-content-docs/current/plugins/url-decorator.mdx b/i18n/en/docusaurus-plugin-content-docs/current/plugins/url-decorator.mdx index f0ebb61..7940e30 100644 --- a/i18n/en/docusaurus-plugin-content-docs/current/plugins/url-decorator.mdx +++ b/i18n/en/docusaurus-plugin-content-docs/current/plugins/url-decorator.mdx @@ -36,10 +36,10 @@ module.exports = { }, }, - // other testplane plugins... + // other Testplane plugins... }, - // other testplane settings... + // other Testplane settings... }; ``` @@ -91,10 +91,10 @@ The `url` parameter is an object with a `query` field, the value of which can be } }, - // other testplane plugins... + // other Testplane plugins... }, - // other testplane settings... + // other Testplane settings... }; ``` @@ -124,10 +124,10 @@ The `url` parameter is an object with a `query` field, the value of which can be }, }, - // other testplane plugins... + // other Testplane plugins... }, - // other testplane settings... + // other Testplane settings... }; ``` diff --git a/i18n/en/docusaurus-plugin-content-docs/current/quickstart/index.mdx b/i18n/en/docusaurus-plugin-content-docs/current/quickstart/index.mdx index 15dc02d..4d4f5af 100644 --- a/i18n/en/docusaurus-plugin-content-docs/current/quickstart/index.mdx +++ b/i18n/en/docusaurus-plugin-content-docs/current/quickstart/index.mdx @@ -5,7 +5,7 @@ import TabItem from "@theme/TabItem"; ## Installation {#install} -Run the testplane installer using `npm`. +Run the Testplane installer using `npm`. ```bash npm init testplane@latest YOUR_PROJECT_PATH diff --git a/i18n/en/docusaurus-plugin-content-docs/current/reference/testplane-api.mdx b/i18n/en/docusaurus-plugin-content-docs/current/reference/testplane-api.mdx new file mode 100644 index 0000000..81cd2e5 --- /dev/null +++ b/i18n/en/docusaurus-plugin-content-docs/current/reference/testplane-api.mdx @@ -0,0 +1,548 @@ +# API testplane + +With the Testplane API, you can use it in your scripts or build tools. + +To do this, you must include the testplane module and create its instance: + +```javascript +const Testplane = require("testplane"); + +const config = require("./testplane.conf.js"); + +const testplane = new Testplane(config); +``` + +Then you will have access to the following parameters and methods: + +<table> +<thead> +<tr><td>**Name**</td><td>**Type**</td><td>**Description**</td></tr> +</thead> +<tbody> +<tr><td>[config](#testplane_config)</td><td>Object or String</td><td>A testplane config object or a path to the configuration file, relative to the working folder.</td></tr> +<tr><td>[events](#testplane_events)</td><td>Object</td><td>[testplane Events][testplane-events] that you can subscribe to.</td></tr> +<tr><td>[errors](#testplane_errors)</td><td>Object</td><td>Errors that testplane can return.</td></tr> +<tr><td>[intercept](#testplane_intercept)</td><td>Function</td><td>A function that allows you to subscribe to testplane [event interception][testplane-events-interception].</td></tr> +<tr><td>[init](#testplane_init)</td><td>Method</td><td>Initializes the testplane instance, loads all plugins, etc.</td></tr> +<tr><td>[run](#testplane_run)</td><td>Method</td><td>Runs tests located at the specified paths.</td></tr> +<tr><td>[addTestToRun](#testplane_add_test_to_run)</td><td>Method</td><td>Adds a test to the current run.</td></tr> +<tr><td>[readTests](#testplane_read_tests)</td><td>Method</td><td>Returns an object of type [TestCollection](#test_collection).</td></tr> +<tr><td>[isFailed](#testplane_is_failed)</td><td>Method</td><td>Returns `true` or `false` depending on whether there was an error or test failure during the test run.</td></tr> +<tr><td>[isWorker](#testplane_is_worker)</td><td>Method</td><td>Returns `true` if the method was called from a testplane worker.</td></tr> +<tr><td>[halt](#testplane_halt)</td><td>Method</td><td>Forcibly terminates the test run in case of a critical error.</td></tr> + +</tbody> +</table> + +## config {#testplane_config} + +A testplane config object or a path to the configuration file, relative to the working folder: `process.cwd()`. + +## events {#testplane_events} + +[testplane Events][testplane-events] that you can subscribe to. + +Example of using the `testplane.events` object in a testplane plugin: + +```javascript +testplane.on(testplane.events.INIT, async () => { + console.info("Processing INIT event..."); +}); +``` + +## errors {#testplane_errors} + +Testplane can return errors of the following types: + +- [CoreError](#testplane_errors_core_error) +- [CancelledError](#testplane_errors_cancelled_error) +- [ClientBridgeError](#testplane_errors_client_bridge_error) +- [HeightViewportError](#testplane_errors_height_viewport_error) +- [OffsetViewportError](#testplane_errors_offset_viewport_error) +- [AssertViewError](#testplane_errors_assert_view_error) +- [ImageDiffError](#testplane_errors_image_diff_error) +- [NoRefImageError](#testplane_errors_no_ref_image_error) + +### CoreError {#testplane_errors_core_error} + +The `CoreError` is returned in the case of a failed calibration of an empty page (`about:blank`) in the browser. + +The error includes the following message: + +``` +Could not calibrate. This could be due to calibration page has failed to open properly +``` + +### CancelledError {#testplane_errors_cancelled_error} + +The `CancelledError` is returned in case of emergency termination by the [halt](#testplane_halt) command. + +The error includes the following message: + +``` +Browser request was cancelled +``` + +### ClientBridgeError {#testplane_errors_client_bridge_error} + +The `ClientBridgeError` is returned in case of a failed JavaScript code injection on the client side (browser). Testplane performs code injection using the [execute][wdio-execute] command of WebDriverIO. + +The error includes the following message: + +``` +Unable to inject client script +``` + +### HeightViewportError {#testplane_errors_height_viewport_error} + +The `HeightViewportError` is returned when trying to capture a screenshot of an area whose bottom bound does not fit into the viewport. + +The error includes the following message: + +``` +Can not capture the specified region of the viewport. +The region bottom bound is outside of the viewport height. +Alternatively, you can test such cases by setting "true" value to option "compositeImage" in the config file +or setting "false" to "compositeImage" and "true" to option "allowViewportOverflow" in "assertView" command. +Element position: <cropArea.left>, <cropArea.top>; size: <cropArea.width>, <cropArea.height>. +Viewport size: <viewport.width>, <viewport.height>. +``` + +The message advises the testplane user on what settings to configure in testplane to be able to capture the screenshot of the specified area. + +### OffsetViewportError {#testplane_errors_offset_viewport_error} + +The `OffsetViewportError` is returned when trying to capture a screenshot of an area whose left, right, or top bounds go beyond the viewport. + +The error includes the following message: + +``` +Can not capture the specified region of the viewport. +Position of the region is outside of the viewport left, top or right bounds. +Check that elements: +- does not overflow the document +- does not overflow browser viewport +Alternatively, you can increase browser window size using +"setWindowSize" or "windowSize" option in the config file. +But if viewport overflow is expected behavior then you can use +option "allowViewportOverflow" in "assertView" command. +``` + +The message advises the testplane user on what settings to configure in testplane to be able to capture the screenshot of the specified area. + +### AssertViewError {#testplane_errors_assert_view_error} + +The `AssertViewError` is returned in case of a failed attempt to capture a screenshot. + +The error can contain one of the following messages, depending on the cause of the failure: + +``` +duplicate name for "<state>" state +``` + +``` +element ("<selector>") still not existing after <this.options.waitforTimeout> ms +``` + +``` +element ("<this.selector>") still not existing after <this.options.waitforTimeout> ms +``` + +### ImageDiffError {#testplane_errors_image_diff_error} + +The `ImageDiffError` is returned from the `assertView` command if a difference (diff) is found when capturing and comparing the screenshot with the reference screenshot. + +The error includes the following message: + +``` +images are different for "<stateName>" state +``` + +Additionally, the `ImageDiffError` contains the following data: + +<table> +<thead> +<tr><td>**Property**</td><td>**Description**</td></tr> +</thead> +<tbody> +<tr><td>stateName</td><td>the name of the state for which the screenshot was taken</td></tr> +<tr><td>currImg</td><td>link to the actual image</td></tr> +<tr><td>refImg</td><td>link to the reference image</td></tr> +<tr><td>diffOpts</td><td>settings for detecting the diff</td></tr> +<tr><td>diffBounds</td><td>bounds of areas with diffs on the image</td></tr> +<tr><td>diffClusters</td><td>clusters with diffs on the image</td></tr> + +</tbody> +</table> + +Read more about [diffBounds][diff-bounds] and [diffClusters][diff-clusters] in the [looks-same][looks-same] package documentation. + +``` +exports.handleImageDiff = (currImg, refImg, state, opts) => { + const {tolerance, antialiasingTolerance, canHaveCaret, diffAreas, config} = opts; + const {buildDiffOpts, system: {diffColor}} = config; + buildDiffOpts.ignoreCaret = buildDiffOpts.ignoreCaret && canHaveCaret; + + const diffOpts = { + current: currImg.path, reference: refImg.path, + diffColor, tolerance, antialiasingTolerance, ...buildDiffOpts + }; + + return Promise.reject(ImageDiffError.create(state, currImg, refImg, diffOpts, diffAreas)); +}; +``` + +### NoRefImageError {#testplane_errors_no_ref_image_error} + +The `NoRefImageError` is returned from the `assertView` command if testplane does not find the reference screenshot on the filesystem when capturing and comparing the screenshot. + +The error includes the following message: + +``` +can not find reference image at <refImg.path> for "<stateName>" state +``` + +Additionally, the `NoRefImageError` contains the following data: + +<table> +<thead> +<tr><td>**Property**</td><td>**Description**</td></tr> +</thead> +<tbody> +<tr><td>stateName</td><td>the name of the state for which the screenshot was taken</td></tr> +<tr><td>currImg</td><td>link to the actual image</td></tr> +<tr><td>refImg</td><td>link to the reference image</td></tr> + +</tbody> +</table> + +## intercept {#testplane_intercept} + +A function that allows you to subscribe to testplane [event interception][testplane-events-interception]. + +The first argument is the event to intercept, and the second is the event handler. + +For example: + +```javascript +testplane.intercept(testplane.events.TEST_FAIL, ({ event, data }) => { + return {}; +}); +``` + +Read more about event interception in the section "[About Event Interception][testplane-events-interception]". + +## init {#testplane_init} + +Initializes the testplane instance, loads all plugins, etc. + +### Example Call + +```javascript +await testplane.init(); +``` + +## run {#testplane_run} + +Runs tests located at the specified paths. + +Returns `true` if the test run was successful and `false` if it was not. + +### Example Call + +```javascript +const success = await testplane.run(testPaths, options); +``` + +### Call Parameters + +All parameters of the `testplane.run()` method are optional. + +<table> +<thead> +<tr><td>**Parameter Name**</td><td>**Type**</td><td>**Description**</td></tr> +</thead> +<tbody> +<tr><td>testPaths</td><td>String[] or TestCollection</td><td>Paths to tests relative to the working folder _(process.cwd())_ or an object of type _TestCollection_ returned by the _readTests_ method. If paths are not specified, all tests will be run.</td></tr> +<tr><td>options</td><td>Object</td><td>Object with run options.</td></tr> +<tr><td>options.reporters</td><td>String[]</td><td>Reporters for the test run results.</td></tr> +<tr><td>options.browsers</td><td>String[]</td><td>Browsers in which to run the tests.</td></tr> +<tr><td>options.sets</td><td>String[]</td><td>Sets in which to run the tests.</td></tr> +<tr><td>options.grep</td><td>RegExp</td><td>A regular expression pattern that specifies tests to run.</td></tr> + +</tbody> +</table> + +## addTestToRun {#testplane_add_test_to_run} + +Adds a test to the current run. Returns `false` if the current run is already finished or cancelled. Otherwise, returns `true`. + +### Example Call + +```javascript +const success = testplane.addTestToRun(test, browser); +``` + +### Call Parameters + +All parameters are required. + +<table> +<thead> +<tr><td>**Parameter Name**</td><td>**Type**</td><td>**Description**</td></tr> +</thead> +<tbody> +<tr><td>test</td><td>Test</td><td>The test to run.</td></tr> +<tr><td>browserId</td><td>String</td><td>The browser in which to run the test.</td></tr> + +</tbody> +</table> + +## readTests {#testplane_read_tests} + +An asynchronous method that returns an object of type [TestCollection](#test_collection). + +### Example Call + +```javascript +await testplane.readTests(testPaths, options); +``` + +### Call Parameters + +<table> +<thead> +<tr><td>**Parameter Name**</td><td>**Type**</td><td>**Description**</td></tr> +</thead> +<tbody> +<tr><td>testPaths</td><td>String[]</td><td>Paths to tests relative to the working folder _(process.cwd())_.</td></tr> +<tr><td>options</td><td>Object</td><td>Object with test reading mode settings.</td></tr> +<tr><td>options.browsers</td><td>String[]</td><td>Read tests only for the specified browsers.</td></tr> +<tr><td>options.silent</td><td>Boolean</td><td>Disable event generation while reading tests. Default: _false_.</td></tr> +<tr><td>options.ignore</td><td>String or Glob or String[] or Glob[]</td><td>Patterns specifying paths on the filesystem to exclude when searching for tests.</td></tr> +<tr><td>options.sets</td><td>String[]</td><td>Sets in which to read tests.</td></tr> +<tr><td>options.grep</td><td>RegExp</td><td>A regular expression pattern that specifies tests to read.</td></tr> + +</tbody> +</table> + +## isFailed {#testplane_is_failed} + +Returns `true` or `false` depending on whether there was an error or test failure during the test run. Might be useful in plugins to determine the current status of testplane. + +### Example Call + +```javascript +const failed = testplane.isFailed(); +``` + +## isWorker {#testplane_is_worker} + +Returns `true` if the method was called from a testplane worker. +Returns `false` if the method was called from the testplane master process. + +Might be useful in plugins to distinguish the execution context. + +### Example Call + +```javascript +// Implementation of some plugin +module.exports = testplane => { + if (testplane.isWorker()) { + // This code will be executed only in testplane workers + } else { + // This code will be executed only in the testplane master process + } +}; +``` + +## halt {#testplane_halt} + +Forcibly terminates the test run in case of a critical error. If the process cannot gracefully shut down within the specified timeout, it will be forcibly terminated unless the timeout is explicitly set to `0`. + +### Example Call + +```javascript +testplane.halt(error, [timeout=60000ms]); +``` + +### Call Parameters + +<table> +<thead> +<tr><td>**Parameter Name**</td><td>**Type**</td><td>**Description**</td></tr> +</thead> +<tbody> +<tr><td>error</td><td>Error</td><td>The critical error that requires stopping testplane.</td></tr> +<tr><td>timeout</td><td>Number</td><td>Optional parameter. Timeout for shutting down the testplane process. Default: _60000 ms_.</td></tr> + +</tbody> +</table> + +## Test Collection {#test_collection} + +An object of type `TestCollection` is returned by the [testplane.readTests](#testplane_read_tests) method and is also passed as an argument to the handler of the [AFTER_TESTS_READ][after-tests-read] event. + +### getBrowsers {#test_collection_get_browsers} + +Returns a list of browsers for which tests are available in the collection. + +#### Example Call + +```javascript +const browsers = testCollection.getBrowsers(); +``` + +### mapTests {#test_collection_map_tests} + +Builds a mapping of tests for the specified browser. + +#### Example Call + +```javascript +const tests = testCollection.mapTests(browserId, (test, browserId) => { + // ... +}); +``` + +If the browser is not specified (i.e., a callback is passed as the first argument), the mapping will be built for all browsers. + +### sortTests {#test_collection_sort_tests} + +Sorts tests for the specified browser. + +#### Example Call + +```javascript +testCollection.sortTests(browserId, (currentTest, nextTest) => { + // ... +}); +``` + +If the browser is not specified (i.e., a callback is passed as the first argument), the sorting will be applied to all browsers. + +### eachTest {#test_collection_each_test} + +Iterates through the tests for the specified browser. + +#### Example Call + +```javascript +testCollection.eachTest(browserId, (test, browserId) => { + // ... +}); +``` + +If the browser is not specified (i.e., a callback is passed as the first argument), the iteration will occur for tests in all browsers. + +### eachTestByVersions {#test_collection_each_test_by_versions} + +Iterates through the tests and browser versions for the specified browser. + +Utilizes the presence of the `browserVersion` property for each test. + +#### Example Call + +```javascript +testCollection.eachTestByVersions(browserId, (test, browserId, browserVersion) => { + // ... +}); +``` + +### disableAll {#test_collection_disable_all} + +Disables all tests or tests for the specified browser. Returns a reference to the test collection instance. + +#### Example Calls + +```javascript +disableAll(); +``` + +or + +```javascript +disableAll(browserId); +``` + +If the browser is not specified, all tests will be disabled. + +### enableAll {#test_collection_enable_all} + +Enables all tests or tests for the specified browser. Returns a reference to the test collection instance. + +#### Example Calls + +```javascript +enableAll(); +``` + +or + +```javascript +enableAll(browserId); +``` + +If the browser is not specified, all tests will be enabled. + +### disableTest {#test_collection_disable_test} + +Disables the specified test in all browsers or only in the specified browser. Returns a reference to the test collection instance. + +#### Example Calls + +```javascript +disableTest(fullTitle); +``` + +or + +```javascript +disableTest(fullTitle, browserId); +``` + +The test is identified by its full title: `fullTitle`. + +### enableTest {#test_collection_enable_test} + +Enables the specified test in all browsers or only in the specified browser. + +```javascript +enableTest(fullTitle); +``` + +or + +```javascript +enableTest(fullTitle, browserId); +``` + +### getRootSuite {#test_collection_get_root_suite} + +Returns the root suite for the specified browser. Returns undefined if there are no tests for the specified browser in the collection. + +#### Example Call + +```javascript +const rootSuite = getRootSuite(browserId); +``` + +### eachRootSuite {#test_collection_each_root_suite} + +Iterates through all root suites in the collection that have tests. + +#### Example Call + +```javascript +eachRootSuite((root, browserId) => { + // ... +}); +``` + +[after-tests-read]: ../testplane-events#after_tests_read +[testplane-events]: ../testplane-events +[wdio-execute]: ../../commands/browser/execute +[diff-bounds]: https://github.com/gemini-testing/looks-same/blob/master/README.md#getting-diff-bounds +[diff-clusters]: https://github.com/gemini-testing/looks-same/blob/master/README.md#getting-diff-clusters +[looks-same]: https://github.com/gemini-testing/looks-same +[testplane-events-interception]: ../testplane-events#events_interception diff --git a/i18n/en/docusaurus-plugin-content-docs/current/reference/testplane-events.mdx b/i18n/en/docusaurus-plugin-content-docs/current/reference/testplane-events.mdx new file mode 100644 index 0000000..6d479a2 --- /dev/null +++ b/i18n/en/docusaurus-plugin-content-docs/current/reference/testplane-events.mdx @@ -0,0 +1,1814 @@ +import Admonition from "@theme/Admonition"; + +# Testplane Events + +## Overview {#overview} + +### How event descriptions are organized {#how_is_events_description_made} + +Below are described all the Testplane events you can subscribe to in your plugin. + +Each event description begins with tags, which come in the following variants: + +- _sync_ or _async_ denote, respectively, synchronous and asynchronous event handler call modes; +- _master_ indicates that this event is available from the master process of the testplane; +- _worker_ indicates that this event is available from the workers (subprocesses) of the testplane; +- _can be intercepted_ indicates that this event can be intercepted and accordingly, modified. + +Next are: + +- a description of the circumstances under which the event is triggered; +- a code snippet showing how to subscribe to the event; +- the event handler parameters; +- and optionally, examples of using this event in a plugin or plugins. + +### Event generation scheme {#events_scheme} + +[//]: # "<style>" +[//]: # '.yc-root_theme_light img[src*="-dark-mode"] { display: none }' +[//]: # '.yc-root_theme_dark img[src*="-light-mode"] { display: none }' +[//]: # "</style>" + +![Testplane event generation](/img/docs/reference/testplane-events-light-mode.png) + +[//]: # "![Testplane event generation](/img/docs/reference/testplane-events-dark-mode.png)" + +### Event sequence description {#events_scheme_description} + +Testplane can be launched through both the [CLI (command line interface)][cli-wiki] and its API: from a script using the [run][testplane-run] command. + +After launching, Testplane loads all plugins and proceeds to parsing the CLI, if it was launched through the CLI, or directly to the initialization stage if it was launched through its API. + +#### CLI Parsing + +If Testplane was launched through the CLI, it triggers the [CLI](#cli) event. Any plugin can subscribe to this event to add its options and commands to the Testplane before Testplane parses the CLI. + +If Testplane was launched via the API, the CLI parsing stage will be skipped and it will move straight to the initialization stage. + +#### Initialization + +During initialization, Testplane triggers the [INIT](#init) event. This event occurs only once for the entire launch of testplane. Plugins can use this event to perform all necessary preparatory work: open and read files, set up a dev server, initialize data structures, etc. + +Testplane then launches subprocesses (known as workers) in which all tests will be executed. Tests are not executed in the master process of testplane; the master process handles the overall orchestration of the test run process, including generating events upon the completion of individual tests. + +The number of workers that Testplane launches is regulated by the [workers][system-workers] parameter in the [system][system] section of the Testplane config. When a new worker is launched, Testplane triggers the [NEW_WORKER_PROCESS](#new_worker_process) event. + +<Admonition type="info"> + Testplane runs all tests in workers to avoid memory and CPU limitations on the master process. + Once the number of tests completed in a worker reaches the value of + [testsPerWorker][system-tests-per-worker], the worker stops, and a new worker is launched. + Accordingly, the [NEW_WORKER_PROCESS](#new_worker_process) event is triggered again. +</Admonition> + +#### Test reading + +Afterwards, Testplane reads all tests from the file system in the master process. Before reading each file, it sends the [BEFORE_FILE_READ](#before_file_read) event, and after reading it — the [AFTER_FILE_READ](#after_file_read) event. + +After all tests are read, the [AFTER_TESTS_READ](#after_tests_read) event is triggered. + +#### Test execution + +Then Testplane sends the [RUNNER_START](#runner_start) and [BEGIN](#begin) events, and starts a new session (a browser session) in which the tests will be run. At the start of the session, Testplane triggers the [SESSION_START](#session_start) event. + +<Admonition type="info"> +If the number of tests executed within one session reaches the value of the [testsPerSession][browser-tests-per-session] parameter, Testplane will end the session, triggering the [SESSION_END](#session_end) event, and start a new one, sending the [SESSION_START](#session_start) event. + +If a test fails with a critical error, Testplane will: + +- prematurely terminate the session and the browser associated with it; +- create a new session; +- request a new browser and link it to the new session. + +This is to ensure that a failure in a session during one test run does not affect the subsequent test runs. + +</Admonition> + +After creating a new session, Testplane proceeds to run the tests. All tests are executed in workers, but the actual starting and collecting of the test results is done in the master process. The master process triggers the [SUITE_BEGIN](#suite_begin) event for describe-blocks in the test file and [TEST_BEGIN](#test_begin) for it-blocks. If a test is disabled using helpers like [skip.in][skip-in] and others, the [TEST_PENDING](#test_pending) event is triggered. + +Next, the workers receive from the master process the information about specific tests they need to run. Since tests are stored in files, workers read specifically those files where the required tests are located. Before reading each of those files in each worker, the [BEFORE_FILE_READ](#before_file_read) event is triggered, and after reading — the [AFTER_FILE_READ](#after_file_read) event. + +Once the relevant test files are read by the worker, the [AFTER_TESTS_READ](#after_tests_read) event is triggered. + +The listed 3 events — [BEFORE_FILE_READ](#before_file_read), [AFTER_FILE_READ](#after_file_read), and [AFTER_TESTS_READ](#after_tests_read) will be triggered in the workers during the test runs every time workers receive new tests to run from the master process. Except when the corresponding test file has already been read by the worker before. After the first reading of any file, the worker caches it to avoid re-reading the test file next time. + +<Admonition type="info" title="Why may a file be requested multiple times?"> + Because one file may contain many tests. The test run is done per individual test, not per file. + Therefore, at some point in time, a test from a file that already had another test run may be + executed. In such cases, caching protects from unnecessary re-reading of the same files. +</Admonition> + +Before a test is run, the [NEW_BROWSER](#new_browser) event is triggered. However, this event will not be triggered for all tests, as the same browser can be used many times to launch tests (see the [sessionsPerBrowser][browser-sessions-per-browser] parameter). Also, in case of a critical test failure, the browser is recreated to prevent other tests in that browser from failing due to a system crash. In this case, the [NEW_BROWSER](#new_browser) event will be sent again. + +#### Test completion + +After a test completes, the [SESSION_END](#session_end) event can be sent. But this is only if the total number of tests run in the session exceeds the value of the [testsPerSession][browser-tests-per-session] parameter. + +Then everything will depend on the result of the test run. If the test passed successfully, Testplane triggers the [TEST_PASS](#test_pass) event. If the test failed — [TEST_FAIL](#test_fail). If the test failed but needs to be re-run (see the [retry][browsers-retry] and [shouldRetry][browsers-should-retry] settings in the Testplane config), instead of the [TEST_FAIL](#test_fail) event, the [RETRY](#retry) event will be sent. + +If the test does not need to be re-run, and the result is final, Testplane triggers the [TEST_END](#test_end) and [SUITE_END](#suite_end) events if it refers to the completion of a describe-block. + +After all tests have been executed and sessions completed, Testplane triggers the [END](#end) and [RUNNER_END](#runner_end) events. + +#### Updating reference screenshots + +During the test run, reference screenshots may be updated for the following reasons: + +- The developer launched Testplane in a special GUI mode and commanded "accept screenshots"; +- The developer specified the `--update-ref` option when launching testplane; +- The tests did not have reference screenshots. + +In all these cases, the [UPDATE_REFERENCE](#update_reference) event is triggered. + +#### Errors and premature termination + +If during the test run, a critical error occurs in one of the [event interceptors](#events_interception), Testplane triggers the [ERROR](#error) event for this test. However, the other tests will proceed normally. + +If during the test run, Testplane receives a [SIGTERM][sigterm] signal (for example, as a result of pressing `Ctrl + C`), Testplane triggers the [EXIT](#exit) event and prematurely terminates the test run. + +### About event interception {#events_interception} + +Testplane allows the developer to intercept certain events and modify them to other events, ignore them, or delay their processing. + +Events that can be intercepted are tagged with _can be intercepted_ in their description. There are a total of 7 such events: + +- [SUITE_BEGIN](#suite_begin) +- [SUITE_END](#suite_end) +- [TEST_BEGIN](#test_begin) +- [TEST_END](#test_end) +- [TEST_PASS](#test_pass) +- [TEST_FAIL](#test_fail) +- [RETRY](#retry) + +#### Changing one event to another + +For example, the code below shows how to intercept the [TEST_FAIL](#test_fail) event and change it to the [TEST_PENDING](#test_pending) event — thus automatically disabling failing tests, preventing them from bringing down the overall test run: + +```javascript +module.exports = testplane => { + testplane.intercept(testplane.events.TEST_FAIL, ({ event, data }) => { + const test = Object.create(data); + test.pending = true; + test.skipReason = "intercepted failure"; + + return { event: testplane.events.TEST_PENDING, data: test }; + }); + + testplane.on(testplane.events.TEST_FAIL, test => { + // this handler will never be called + }); + + testplane.on(testplane.events.TEST_PENDING, test => { + // this handler will always be called instead of the TEST_FAIL handler + }); +}; +``` + +#### Leaving the event as is {#events_interception_leaving_event_as_is} + +If for some reason the intercepted event needs to be left _as is_, its handler should return exactly the same object or any _falsey_ value: _undefined, null_ or _false._ + +```javascript +module.exports = testplane => { + testplane.intercept(testplane.events.TEST_FAIL, ({ event, data }) => { + return { event, data }; + // or return; + // or return null; + // or return false; + }); + + testplane.on(testplane.events.TEST_FAIL, test => { + // this handler will be called as usual, + // because the interception of the TEST_FAIL event does not change it + }); +}; +``` + +#### Ignoring an event {#events_interception_ignoring_event} + +To ignore an event and prevent it from propagating further, return an empty object from the handler (where the event is intercepted): + +```javascript +module.exports = testplane => { + testplane.intercept(testplane.events.TEST_FAIL, ({ event, data }) => { + return {}; + }); + + testplane.on(testplane.events.TEST_FAIL, test => { + // this handler will never be called + }); +}; +``` + +#### Delaying event processing {#events_interception_delaying_events} + +The approach above with ignoring an event can be used to delay the occurrence of certain events, for example: + +```javascript +module.exports = testplane => { + const intercepted = []; + + testplane.intercept(testplane.events.TEST_FAIL, ({ event, data }) => { + // collect all TEST_FAIL events + intercepted.push({ event, data }); + // and prevent them from propagating further + return {}; + }); + + testplane.on(testplane.events.END, () => { + // after the test run is complete, trigger all accumulated TEST_FAIL events + intercepted.forEach(({ event, data }) => testplane.emit(event, data)); + }); +}; +``` + +### Information sharing between event handlers {#events_handlers_info_interchange} + +Events triggered in the master process and in the Testplane workers cannot exchange information through global variables. + +For example, this approach will not work: + +```javascript +module.exports = testplane => { + let flag = false; + + testplane.on(testplane.events.RUNNER_START, () => { + flag = true; + }); + + testplane.on(testplane.events.NEW_BROWSER, () => { + // the value false will be output, because the NEW_BROWSER event + // is triggered in the Testplane worker, and RUNNER_START – in the master process + console.info(flag); + }); + + testplane.on(testplane.events.RUNNER_END, () => { + // the value true will be output + console.info(flag); + }); +}; +``` + +But the problem can be solved as follows: + +```javascript +module.exports = (testplane, opts) => { + testplane.on(testplane.events.RUNNER_START, () => { + opts.flag = true; + }); + + testplane.on(testplane.events.NEW_BROWSER, () => { + // the value true will be output because properties in the config + // that are primitives (and the "opts" variable is part of the config) + // are automatically passed to workers during the RUNNER_START event + console.info(opts.flag); + }); +}; +``` + +Or as follows: see the [example](#new_worker_process_usage) from the description of the [NEW_WORKER_PROCESS](#new_worker_process) event. + +### Parallel execution of plugin code {#plugin_code_parallel_execution} + +The test runner has a `registerWorkers` method which registers the plugin code for parallel execution in the Testplane workers. The method takes the following parameters: + +<table> +<thead> +<tr><td>**Parameter**</td><td>**Type**</td><td>**Description**</td></tr> +</thead> +<tbody> +<tr><td>workerFilepath</td><td>String</td><td>Absolute path to the worker.</td></tr> +<tr><td>exportedMethods</td><td>String[]</td><td>List of exported methods.</td></tr> + +</tbody> +</table> + +It returns an object which contains asynchronous functions with names from exported methods. + +The file at the `workerFilepath` path should export an object containing asynchronous functions with names from `exportedMethods`. + +#### Example of parallel plugin code execution {#plugin_code_parallel_execution_example} + +**Plugin code: plugin.js** + +```javascript +let workers; + +module.exports = testplane => { + testplane.on(testplane.events.RUNNER_START, async runner => { + const workerFilepath = require.resolve("./worker"); + const exportedMethods = ["foo"]; + + workers = runner.registerWorkers(workerFilepath, exportedMethods); + + // prints FOO_RUNNER_START + console.info(await workers.foo("RUNNER_START")); + }); + + testplane.on(testplane.events.RUNNER_END, async () => { + // prints FOO_RUNNER_END + console.info(await workers.foo("RUNNER_END")); + }); +}; +``` + +**Worker code: worker.js** + +```javascript +module.exports = { + foo: async function (event) { + return "FOO_" + event; + }, +}; +``` + +## CLI {#cli} + +**sync | master** + +The `CLI` event is triggered immediately upon startup, before Testplane parses the [CLI][cli-wiki]. The event handler executes synchronously. With it, you can add new commands or extend testplane's help. + +### Subscribing to the event {#cli_subscription} + +```javascript +testplane.on(testplane.events.CLI, cli => { + console.info("Processing CLI event..."); + + cli.option( + "--some-option <some-value>", + "the full description of the option", + // see more at https://github.com/tj/commander.js#options + ); +}); +``` + +#### Handler parameters {#cli_cb_params} + +An object of the [Commander][commander] type is passed to the event handler. + +### Example of usage {#cli_usage} + +Let's consider an example of the [implementation][testplane-test-repeater-index] of the [testplane-test-repeater][testplane-test-repeater] plugin. + +Using the [CLI](#cli) event, the plugin adds a new `--repeat` option to testplane. With it, you can specify how many times to run tests, regardless of the result of each run. + +<details> + +<summary> +Click to see the code + +</summary> + +```javascript +const parseConfig = require("./config"); + +module.exports = (testplane, opts) => { + const pluginConfig = parseConfig(opts); + + if (!pluginConfig.enabled || testplane.isWorker()) { + // the plugin is either disabled, or we are in the context of a worker – leave + return; + } + + testplane.on(testplane.events.CLI, cli => { + // add the --repeat option + cli.option( + "--repeat <number>", + "how many times tests should be repeated regardless of the result", + value => parseNonNegativeInteger(value, "repeat"), + ); + }); + + // ... +}; +``` + +</details> + +## INIT {#init} + +**async | master** + +The `INIT` event is triggered before the [run][run] or [readTests][read-tests] tasks are performed. The event handler can be asynchronous: in which case, the tasks will only start after the Promise returned by the event handler is resolved. The event triggers only once, regardless of how many times the tasks are performed. + +### Subscribing to the event {#init_subscription} + +```javascript +testplane.on(testplane.events.INIT, async () => { + console.info("Processing INIT event..."); +}); +``` + +#### Handler parameters {#init_cb_params} + +No data is passed to the event handler. + +### Example of usage {#init_usage} + +In the [INIT](#init) event handler, you can organize, for example, the launch of a dev server for your project. + +<Admonition type="info" title="What is a dev server?"> + A dev server is an [express][express]-like application that allows you to develop the frontend + of the project. +</Admonition> + +Below is the shortest implementation. A more detailed example can be found in the section "[Automatic dev server startup](#usage_starting_dev_server)". + +<details> + +<summary> +Click to see the code + +</summary> + +```javascript +const http = require('http'); +const parseConfig = require('./config'); + +module.exports = (testplane, opts) => { + const pluginConfig = parseConfig(opts); + + if (!pluginConfig.enabled</td></tr> testplane.isWorker()) { + // either the plugin is disabled, or we are in the worker context – exit + return; + } + + // ... + + testplane.on(testplane.events.INIT, () => { + // content served by the dev-server + const content = '<h1>Hello, World!</h1>'; + + // create a server and start listening on port 3000 + http + .createServer((req, res) => res.end(content)) + .listen(3000); + + // at http://localhost:3000/index.html it will serve: <h1>Hello, World!</h1> + }); +}; +``` + +</details> + +## BEFORE_FILE_READ {#before_file_read} + +**sync | master | worker** + +The `BEFORE_FILE_READ` event is triggered before the test file is read to parse it. The event handler is executed synchronously. The event is also available in Testplane workers. + +### Subscribing to the event {#before_file_read_subscription} + +```javascript +testplane.on(testplane.events.BEFORE_FILE_READ, ({ file, testplane, testParser }) => { + testParser.setController("<some-command-name>", { + "<some-helper>": function (matcher) { + // ... + }, + }); +}); +``` + +#### Handler parameters {#before_file_read_cb_params} + +The event handler receives an object with the following format: + +```javascript +{ + file, // String: path to the test file + testplane, // Object: same as global.testplane + testParser; // Object: of type TestParserAPI +} +``` + +#### testParser: TestParserAPI {#test_parser} + +The `testParser` object of type `TestParserAPI` is passed to the `BEFORE_FILE_READ` event handler. It allows you to manage the process of parsing test files. The object supports the `setController` method, which allows you to create your own helpers for tests. + +**setController(name, methods)** + +The method adds a controller to the global `testplane` object, available inside the tests. + +- `name` — the name of the helper (or controller); +- `methods` — an object-dictionary where the keys define the names of the corresponding helper methods, and the values define their implementation. Each method will be called on the corresponding test or test suite. + +<Admonition type="info"> + The controller will be removed as soon as the current file parsing is finished. +</Admonition> + +### Usage example {#before_file_read_usage} + +As an example, let's create a special helper `testplane.logger.log()` that allows us to log information about the parsing of the test we are interested in. + +<details> + +<summary> +Click to view the usage example + +</summary> + +**Plugin code** + +```javascript +testplane.on(testplane.events.BEFORE_FILE_READ, ({ file, testParser }) => { + testParser.setController("logger", { + log: function (prefix) { + console.log( + `${prefix}: just parsed ${this.fullTitle()} from file ${file} for browser ${this.browserId}`, + ); + }, + }); +}); +``` + +**Test code** + +```javascript +describe("foo", () => { + testplane.logger.log("some-prefix"); + it("bar", function () { + // ... + }); +}); +``` + +</details> + +Another example of using the [BEFORE_FILE_READ](#before_file_read) event can be found in the section “[Running tests with helpers](#usage_running_tests_with_helpers)”. + +## AFTER_FILE_READ {#after_file_read} + +**sync | master | worker** + +The `AFTER_FILE_READ` event is triggered after the test file is read. The event handler is executed synchronously. The event is also available in Testplane workers. + +### Subscribing to the event {#after_file_read_subscription} + +```javascript +testplane.on(testplane.events.AFTER_FILE_READ, ({ file, testplane }) => { + console.info("Processing AFTER_FILE_READ event..."); +}); +``` + +#### Handler parameters {#after_file_read_cb_params} + +The event handler receives an object with the following format: + +```javascript +{ + file, // String: path to the test file + testplane; // Object: same as global.testplane +} +``` + +## AFTER_TESTS_READ {#after_tests_read} + +**sync | master | worker** + +The `AFTER_TESTS_READ` event is triggered after the methods [readTests][read-tests] or [run][run] of the [TestCollection][test-collection] object are called. The event handler is executed synchronously. The event is also available in Testplane workers. + +By subscribing to this event, you can perform various manipulations on the test collection before they are run. For example, you can exclude certain tests from the runs. + +### Subscribing to the event {#after_tests_read_subscription} + +```javascript +testplane.on(testplane.events.AFTER_TESTS_READ, testCollection => { + console.info("Processing AFTER_TESTS_READ event..."); +}); +``` + +#### Handler parameters {#after_tests_read_cb_params} + +The event handler receives a `testCollection` object of type [TestCollection][test-collection]. + +### Usage example {#after_tests_read_usage} + +Consider the [implementation][testplane-global-hook-index] of the [testplane-global-hook][testplane-global-hook] plugin, which allows you to move actions that repeat before and after each test into separate _beforeEach_ and _afterEach_ handlers. + +Using the [AFTER_TESTS_READ](#after_tests_read) event, the plugin adds _beforeEach_ and _afterEach_ hooks to each root suite. These hooks are defined by the user in the [testplane-global-hook][testplane-global-hook] plugin configuration. + +<details> + +<summary> +Click to view the code + +</summary> + +```javascript +const parseConfig = require("./config"); + +module.exports = (testplane, opts) => { + const pluginConfig = parseConfig(opts); + + // ... + + const { beforeEach, afterEach } = pluginConfig; + + testplane.on(testplane.events.AFTER_TESTS_READ, testCollection => { + testCollection.eachRootSuite(root => { + beforeEach && root.beforeEach(beforeEach); + afterEach && root.afterEach(afterEach); + }); + }); +}; +``` + +</details> + +More examples of using the [AFTER_TESTS_READ](#after_tests_read) event can be found in the sections “[Running tests from a specific list](#usage_running_filtered_tests)” and “[Running tests with helpers](#usage_running_tests_with_helpers)”. + +## RUNNER_START {#runner_start} + +**async | master** + +The `RUNNER_START` event is triggered after all Testplane workers are initialized and before the tests are run. The event handler can be asynchronous: in this case, the tests will start running only after the _Promise_ returned by the event handler is resolved. + +### Subscribing to the event {#runner_start_subscription} + +```javascript +testplane.on(testplane.events.RUNNER_START, async runner => { + console.info("Processing RUNNER_START event..."); +}); +``` + +#### Handler parameters {#runner_start_cb_params} + +The event handler receives a reference to the runner instance. Using this instance, you can trigger various events or subscribe to them. + +### Usage example {#runner_start_usage} + +Suppose we want to automatically set up an SSH tunnel when running tests and redirect all URLs in the tests to the established tunnel. To do this, we can use the [RUNNER_START](#runner_start) and [RUNNER_END](#runner_end) events to open the tunnel when the runner starts and close it after the runner finishes. + +<details> + +<summary> +Click to view the code + +</summary> + +```javascript +const parseConfig = require("./config"); +const Tunnel = require("./lib/tunnel"); + +module.exports = (testplane, opts) => { + const pluginConfig = parseConfig(opts); + + if (!pluginConfig.enabled) { + // plugin is disabled – exit + return; + } + + // plugin config defines tunnel parameters: + // host, ports, localport, retries, etc. + const tunnel = Tunnel.create(testplane.config, pluginConfig); + + testplane.on(testplane.events.RUNNER_START, () => tunnel.open()); + testplane.on(testplane.events.RUNNER_END, () => tunnel.close()); +}; +``` + +</details> + +A similar [implementation][ssh-tunneler-index] can be found in the [ssh-tunneler][ssh-tunneler] plugin. + +## RUNNER_END {#runner_end} + +**async | master** + +The `RUNNER_END` event is triggered after the test is run and before all workers are terminated. The event handler can be asynchronous: in this case, all workers will be terminated only after the _Promise_ returned by the event handler is resolved. + +### Subscribing to the event {#runner_end_subscription} + +```javascript +testplane.on(testplane.events.RUNNER_END, async result => { + console.info("Processing RUNNER_END event..."); +}); +``` + +#### Handler parameters {#runner_end_cb_params} + +The event handler receives an object with the test run statistics in the following format: + +```javascript +{ + passed: 0, // number of successfully completed tests + failed: 0, // number of failed tests + retries: 0, // number of test retries + skipped: 0, // number of skipped tests + total: 0 // total number of tests +}; +``` + +### Usage example {#runner_end_usage} + +See the example [above](#runner_start_usage) about opening and closing the tunnel when the runner starts and stops. + +## NEW_WORKER_PROCESS {#new_worker_process} + +**sync | master** + +The `NEW_WORKER_PROCESS` event is triggered after a new Testplane subprocess (worker) is spawned. The event handler is executed synchronously. + +### Subscribing to the event {#new_worker_process_subscription} + +```javascript +testplane.on(testplane.events.NEW_WORKER_PROCESS, workerProcess => { + console.info("Processing NEW_WORKER_PROCESS event..."); +}); +``` + +#### Handler parameters {#new_worker_process_cb_params} + +The event handler receives a wrapper object over the spawned subprocess, with a single [send][process-send] method for message exchange. + +### Usage example {#new_worker_process_usage} + +The example below shows how to use the [NEW_WORKER_PROCESS](#new_worker_process) event to organize interaction between the master process and all Testplane workers. For example, to update the value of a parameter in all Testplane workers from the master process before the test run starts. + +The example also uses the [BEGIN](#begin) event. + +<details> + +<summary> +Click to view the code + +</summary> + +```javascript +const masterPlugin = (testplane, opts) => { + const workers = []; + + testplane.on(testplane.events.NEW_WORKER_PROCESS, (workerProcess) => { + // store references to all created Testplane workers + workers.push(workerProcess); + }); + + testplane.on(testplane.events.BEGIN, () => { + // send the parameter value to all workers + workers.forEach((worker) => { + worker.send({ + type: PARAM_VALUE_UPDATED_EVENT, + param: 'some-value' + }); + }); + }); +}; + +const workerPlugin = (testplane) => { + process.on('message', ({ type, ...data }) => { + if (type === PARAM_VALUE_UPDATED_EVENT) { + const { param } = data; + console.info(`Received value "${param}" for "param" from the master process`); + } + }); + + ... +}; + +const plugin = (testplane, opts) => { + if (testplane.isWorker()) { + workerPlugin(testplane, opts); + } else { + masterPlugin(testplane, opts); + } +}; + +module.exports = plugin; +``` + +</details> + +## SESSION_START {#session_start} + +**async | master** + +The `SESSION_START` event is triggered after the browser session is initialized. The event handler can be asynchronous: in this case, the tests will start running only after the _Promise_ returned by the event handler is resolved. + +### Subscribing to the event {#session_start_subscription} + +```javascript +testplane.on(testplane.events.SESSION_START, async (browser, { browserId, sessionId }) => { + console.info("Processing SESSION_START event..."); +}); +``` + +#### Handler parameters {#session_start_cb_params} + +The event handler receives 2 arguments: + +- the first argument — an instance of `WebdriverIO`; +- the second argument — an object of the form `{ browserId, sessionId }`, where _browserId_ is the name of the browser, and _sessionId_ is the browser session identifier. + +### Usage example {#session_start_usage} + +Consider an example where the plugin subscribes to the [SESSION_START](#session_start) event to disable scrollbars in browsers using the Chrome DevTools Protocol. + +<details> + +<summary> +Click to view the code + +</summary> + +```javascript +const parseConfig = require("./config"); +const DevTools = require("./dev-tools"); + +module.exports = (testplane, opts) => { + const pluginConfig = parseConfig(opts); + + if (!pluginConfig.enabled) { + // plugin is disabled – exit + return; + } + + testplane.on(testplane.events.SESSION_START, async (browser, { browserId, sessionId }) => { + if (!pluginConfig.browsers.includes(browserId)) { + // the browser is not in the list of browsers for which scrollbars can be disabled + // using the Chrome DevTools Protocol (CDP) – exit + return; + } + + const gridUrl = testplane.config.forBrowser(browserId).gridUrl; + + // pluginConfig.browserWSEndpoint defines a function that should return the URL + // for working with the browser via CDP. To allow the function to compute the URL, + // the function receives the session identifier and the grid URL + const browserWSEndpoint = pluginConfig.browserWSEndpoint({ sessionId, gridUrl }); + + const devtools = await DevTools.create({ browserWSEndpoint }); + + devtools.setScrollbarsHiddenOnNewPage(); + + await devtools.hideScrollbarsOnActivePages(); + }); +}; +``` + +</details> + +A more detailed [implementation][hermione-hide-scrollbars-index] can be found in the [hermione-hide-scrollbars][hermione-hide-scrollbars] plugin. + +## SESSION_END {#session_end} + +**async | master** + +The `SESSION_END` event is triggered after the browser session ends. The event handler can be asynchronous: in this case, the tests will continue running only after the _Promise_ returned by the event handler is resolved. + +### Subscribing to the event {#session_end_subscription} + +```javascript +testplane.on(testplane.events.SESSION_END, async (browser, { browserId, sessionId }) => { + console.info("Processing SESSION_END event..."); +}); +``` + +#### Handler parameters {#session_end_cb_params} + +The event handler receives 2 arguments: + +- the first argument — an instance of `WebdriverIO`; +- the second argument — an object of the form `{ browserId, sessionId }`, where _browserId_ is the name of the browser, and _sessionId_ is the browser session identifier. + +## BEGIN {#begin} + +**sync | master** + +The `BEGIN` event is triggered before the test is run, but after all runners are initialized. The event handler is executed synchronously. + +### Subscribing to the event {#begin_subscription} + +```javascript +testplane.on(testplane.events.BEGIN, () => { + console.info("Processing BEGIN event..."); +}); +``` + +#### Handler parameters {#begin_cb_params} + +No data is passed to the event handler. + +### Usage example {#begin_usage} + +See the example [above](#new_worker_process_usage) about organizing interaction between the Testplane master process and all workers. + +## END {#end} + +**sync | master** + +The `END` event is triggered right before the `RUNNER_END` event. The event handler is executed synchronously. + +### Subscribing to the event {#end_subscription} + +```javascript +testplane.on(testplane.events.END, () => { + console.info("Processing END event..."); +}); +``` + +#### Handler parameters {#end_cb_params} + +No data is passed to the event handler. + +### Usage example {#end_usage} + +For an example of using the [END](#end) event, see the section “[Delaying event processing](#events_interception_delaying_events)”. + +## SUITE_BEGIN {#suite_begin} + +**sync | master | can be intercepted** + +The `SUITE_BEGIN` event is triggered before a test suite is run. The event handler is executed synchronously. + +### Subscribing to the event {#suite_begin_subscription} + +```javascript +testplane.on(testplane.events.SUITE_BEGIN, suite => { + console.info(`Processing SUITE_BEGIN event for "${suite.fullTitle()}"...`); +}); +``` + +#### Handler Parameters {#suite_begin_cb_params} + +An instance of _suite_ is passed to the event handler. + +### Event Interception {#suite_begin_interception} + +```javascript +testplane.intercept(testplane.events.SUITE_BEGIN, ({ event, data: suite }) => { + console.info(`Intercepting SUITE_BEGIN event for "${suite.fullTitle()}"...`); +}); +``` + +## SUITE_END {#suite_end} + +**sync | master | can be intercepted** + +The `SUITE_END` event is triggered after the test suite _(suite)_ has finished executing. The event handler is executed synchronously. + +### Event Subscription {#suite_end_subscription} + +```javascript +testplane.on(testplane.events.SUITE_END, suite => { + console.info(`Handling SUITE_END event for "${suite.fullTitle()}"...`); +}); +``` + +#### Handler Parameters {#suite_end_cb_params} + +An instance of _suite_ is passed to the event handler. + +### Event Interception {#suite_end_interception} + +```javascript +testplane.intercept(testplane.events.SUITE_END, ({ event, data: suite }) => { + console.info(`Intercepting SUITE_END event for "${suite.fullTitle()}"...`); +}); +``` + +## TEST_BEGIN {#test_begin} + +**sync | master | can be intercepted** + +The `TEST_BEGIN` event is triggered before the test is executed. The event handler is executed synchronously. + +### Event Subscription {#test_begin_subscription} + +```javascript +testplane.on(testplane.events.TEST_BEGIN, test => { + if (test.pending) { + // test is disabled, no action needed + return; + } + + console.info( + `Handling TEST_BEGIN event ` + + `for test "${test.fullTitle()}" in browser "${test.browserId}"...`, + ); +}); +``` + +#### Handler Parameters {#test_begin_cb_params} + +An instance of the test is passed to the event handler. + +### Event Interception {#test_begin_interception} + +```javascript +testplane.intercept(testplane.events.TEST_BEGIN, ({ event, data: test }) => { + console.info( + `Intercepting TEST_BEGIN event ` + + `for test "${test.fullTitle()}" in browser "${test.browserId}"...`, + ); +}); +``` + +### Usage Example {#test_begin_usage} + +See the example "Profiling Test Runs" [here](#usage_profiling_tests_runs). + +## TEST_END {#test_end} + +**sync | master | can be intercepted** + +The `TEST_END` event is triggered after the test has finished executing. The event handler is executed synchronously. The event can also be intercepted and modified in a special handler. + +### Event Subscription {#test_end_subscription} + +```javascript +testplane.on(testplane.events.TEST_END, test => { + if (test.pending) { + // test is disabled, no action needed + return; + } + + console.info( + `Handling TEST_END event ` + + `for test "${test.fullTitle()}" in browser "${test.browserId}"...`, + ); +}); +``` + +#### Handler Parameters {#test_end_cb_params} + +An instance of the test is passed to the event handler. + +### Event Interception {#test_end_interception} + +```javascript +testplane.intercept(testplane.events.TEST_END, ({ event, data: test }) => { + console.info( + `Intercepting TEST_END event ` + + `for test "${test.fullTitle()}" in browser "${test.browserId}"...`, + ); +}); +``` + +### Usage Example {#test_end_usage} + +See the example "Profiling Test Runs" [here](#usage_profiling_tests_runs). + +## TEST_PASS {#test_pass} + +**sync | master | can be intercepted** + +The `TEST_PASS` event is triggered if the test passes successfully. The event handler is executed synchronously. The event can also be intercepted and modified in a special handler. + +### Event Subscription {#test_pass_subscription} + +```javascript +testplane.on(testplane.events.TEST_PASS, test => { + console.info( + `Handling TEST_PASS event ` + + `for test "${test.fullTitle()}" in browser "${test.browserId}"...`, + ); +}); +``` + +#### Handler Parameters {#test_pass_cb_params} + +An instance of the test is passed to the event handler. + +### Event Interception {#test_pass_interception} + +```javascript +testplane.intercept(testplane.events.TEST_PASS, ({ event, data: test }) => { + console.info( + `Intercepting TEST_PASS event ` + + `for test "${test.fullTitle()}" in browser "${test.browserId}"...`, + ); +}); +``` + +### Usage Example {#test_pass_usage} + +See the example "Collecting Test Run Statistics" [here](#usage_collecting_stats). + +## TEST_FAIL {#test_fail} + +**sync | master | can be intercepted** + +The `TEST_FAIL` event is triggered if the test fails. The event handler is executed synchronously. The event can also be intercepted and modified in a special handler. + +### Event Subscription {#test_fail_subscription} + +```javascript +testplane.on(testplane.events.TEST_FAIL, test => { + console.info( + `Handling TEST_FAIL event ` + + `for test "${test.fullTitle()}" in browser "${test.browserId}"...`, + ); +}); +``` + +#### Handler Parameters {#test_fail_cb_params} + +An instance of the test is passed to the event handler. + +### Event Interception {#test_fail_interception} + +```javascript +testplane.intercept(testplane.events.TEST_FAIL, ({ event, data }) => { + console.info( + `Intercepting TEST_FAIL event ` + + `for test "${test.fullTitle()}" in browser "${test.browserId}"...`, + ); +}); +``` + +### Usage Example {#test_fail_usage} + +See the example "Collecting Test Run Statistics" [here](#usage_collecting_stats). + +## TEST_PENDING {#test_pending} + +**sync | master** + +The `TEST_PENDING` event is triggered if the test is disabled. The event handler is executed synchronously. + +### Event Subscription {#test_pending_subscription} + +```javascript +testplane.on(testplane.events.TEST_PENDING, test => { + console.info( + `Handling TEST_PENDING event ` + + `for test "${test.fullTitle()}" in browser "${test.browserId}"...`, + ); +}); +``` + +#### Handler Parameters {#test_pending_cb_params} + +An instance of the test is passed to the event handler. + +### Usage Example {#test_pending_usage} + +See the example "Collecting Test Run Statistics" [here](#usage_collecting_stats). + +## RETRY {#retry} + +**sync | master | can be intercepted** + +The `RETRY` event is triggered if the test fails but is retried. The retry capabilities are determined by the [retry][browsers-retry] and [shouldRetry][browsers-should-retry] settings in the Testplane config. Testplane plugins can also influence this by modifying these settings on the fly. See plugins [retry-limiter][retry-limiter] and [testplane-retry-progressive][testplane-retry-progressive] for examples. + +The event handler is executed synchronously. The event can also be intercepted and modified in a special handler. + +### Event Subscription {#retry_subscription} + +```javascript +testplane.on(testplane.events.RETRY, test => { + console.info( + `Handling RETRY event ` + + `for test "${test.fullTitle()}" in browser "${test.browserId}"...`, + ); +}); +``` + +#### Handler Parameters {#retry_cb_params} + +An instance of the test is passed to the event handler. + +### Event Interception {#retry_interception} + +```javascript +testplane.intercept(testplane.events.RETRY, ({ event, data: test }) => { + console.info( + `Intercepting RETRY event ` + + `for test "${test.fullTitle()}" in browser "${test.browserId}"...`, + ); +}); +``` + +### Usage Example {#test_retry_usage} + +See the example "Collecting Test Run Statistics" [here](#usage_collecting_stats). + +## ERROR {#error} + +**sync | master** + +The `ERROR` event is triggered only from [event interceptors](#events_interception) in case of a critical error. The event handler is executed synchronously. + +### Event Subscription {#error_subscription} + +```javascript +testplane.on(testplane.events.ERROR, error => { + console.info("Handling ERROR event..."); +}); +``` + +#### Handler Parameters {#error_cb_params} + +An [error object][error-object] is passed to the event handler. + +### Usage Example {#error_usage} + +See the example "Profiling Test Runs" [here](#usage_profiling_tests_runs). + +## INFO {#info} + +**reserved** + +Reserved. + +## WARNING {#warning} + +**reserved** + +Reserved. + +## EXIT {#exit} + +**async | master** + +The `EXIT` event is triggered upon receiving the [SIGTERM][sigterm] signal (e.g., after pressing `Ctrl + C`). The event handler can be asynchronous. + +### Event Subscription {#exit_subscription} + +```javascript +testplane.on(testplane.events.EXIT, async () => { + console.info("Handling EXIT event..."); +}); +``` + +#### Handler Parameters {#exit_cb_params} + +No data is passed to the event handler. + +## NEW_BROWSER {#new_browser} + +**sync | worker** + +The `NEW_BROWSER` event is triggered after a new browser instance is created. The event handler is executed synchronously. The event is only available in Testplane workers. + +### Event Subscription {#new_browser_subscription} + +```javascript +testplane.on(testplane.events.NEW_BROWSER, (browser, { browserId, browserVersion }) => { + console.info("Handling NEW_BROWSER event..."); +}); +``` + +#### Handler Parameters {#new_browser_cb_params} + +Two arguments are passed to the event handler: + +- The first argument is an instance of `WebdriverIO`; +- The second argument is an object of the form `{ browserId, versionId }`, where _browserId_ is the name of the browser, and _browserVersion_ is the browser version. + +### Implementation Example {#new_browser_usage} + +The [NEW_BROWSER](#new_browser) event is often used to add new commands to the browser or to extend existing commands. For example, some plugins add custom commands to the browser in the [NEW_BROWSER](#new_browser) event handler. + +<details> + +<summary> +Click to view the code + +</summary> + +```javascript +module.exports = (testplane, opts) => { + // ... + + if (testplane.isWorker()) { + testplane.on(testplane.events.NEW_BROWSER, (browser, { browserId }) => { + // ... + + browser.addCommand("vncUrl", vncUrlHandler); + browser.addCommand("openVnc", createOpenVnc(browser, ipcOptions)); + browser.addCommand("waitForEnter", waitForEnter); + }); + } +}; +``` + +</details> + +## UPDATE_REFERENCE {#update_reference} + +**sync | worker** + +The `UPDATE_REFERENCE` event is triggered after the reference screenshots are updated. The event handler is executed synchronously. The event is only available in testplane workers. + +### Event Subscription {#update_reference_subscription} + +```javascript +testplane.on(testplane.events.UPDATE_REFERENCE, ({ state, refImg }) => { + console.info("Handling UPDATE_REFERENCE event..."); +}); +``` + +#### Handler Parameters {#update_reference_cb_params} + +An object of the following format is passed to the event handler: + +```javascript +{ + state, // String: the state that the screenshot reflects, e.g., plain, map-view, scroll-left, etc. + refImg; // Object: of type { path, size: { width, height } }, describing the reference screenshot +} +``` + +The _refImg.path_ parameter stores the path to the reference screenshot on the file system, while _refImg.size.width_ and _refImg.size.height_ store the width and height of the reference screenshot, respectively. + +### Usage Example {#update_reference_usage} + +Consider the example of the [implementation][testplane-image-minifier-index] of the [testplane-image-minifier][testplane-image-minifier] plugin, which automatically compresses reference screenshots with a specified compression level when they are saved. + +<details> + +<summary> +Click to view the code + +</summary> + +```javascript +const parseConfig = require("./config"); +const Minifier = require("./minifier"); + +module.exports = (testplane, opts) => { + const pluginConfig = parseConfig(opts); + + if (!pluginConfig.enabled) { + // plugin is disabled – exit + return; + } + + const minifier = Minifier.create(pluginConfig); + + testplane.on(testplane.events.UPDATE_REFERENCE, ({ refImg }) => { + minifier.minify(refImg.path); + }); +}; +``` + +</details> + +## Event Usage Examples + +### Running Tests from a Specified List {#usage_running_filtered_tests} + +Consider the example of the [implementation][testplane-test-filter-index] of the [testplane-test-filter][testplane-test-filter] plugin, which allows running only the tests specified in a JSON file. + +In this example, the following testplane events and API methods are used: + +- [INIT](#init) +- [AFTER_TESTS_READ](#after_tests_read) +- [TestCollection.disableAll][disable-all] +- [TestCollection.enableTest][enable-test] + +<details> + +<summary> +Click to view the code + +</summary> + +```javascript +const _ = require("lodash"); +const parseConfig = require("./config"); +const utils = require("./utils"); + +module.exports = (testplane, opts) => { + const pluginConfig = parseConfig(opts); + + if (!pluginConfig.enabled) { + // plugin is disabled – exit + return; + } + + if (testplane.isWorker()) { + // nothing to do in testplane workers – exit + return; + } + + let input; + + testplane.on(testplane.events.INIT, async () => { + // read the file with the list of tests to run; + // readFile returns a JSON containing an array like: + // [ + // { "fullTitle": "test-1", "browserId": "bro-1" }, + // { "fullTitle": "test-2", "browserId": "bro-2" } + // ] + input = await utils.readFile(pluginConfig.inputFile); + }); + + testplane.on(testplane.events.AFTER_TESTS_READ, testCollection => { + if (_.isEmpty(input)) { + // test list is empty – run all tests, + // i.e., do not modify the original test collection (testCollection) + return; + } + + // disable all tests + testCollection.disableAll(); + + // enable only those specified in the JSON file + input.forEach(({ fullTitle, browserId }) => { + testCollection.enableTest(fullTitle, browserId); + }); + }); +}; +``` + +</details> + +### Collecting Test Run Statistics {#usage_collecting_stats} + +Consider the example of the [implementation][json-reporter-index] of the [json-reporter][json-reporter] plugin. + +In this example, the following testplane events are used: + +- [TEST_PASS](#test_pass) +- [TEST_FAIL](#test_fail) +- [TEST_PENDING](#test_pending) +- [RETRY](#retry) +- [RUNNER_END](#runner_end) + +<details> + +<summary> +Click to view the code + +</summary> + +```javascript +const Collector = require("./lib/collector"); +const testplaneToolCollector = require("./lib/collector/tool/testplane"); +const parseConfig = require("./lib/config"); + +module.exports = (testplane, opts) => { + const pluginConfig = parseConfig(opts); + + if (!pluginConfig.enabled) { + // plugin is disabled – exit + return; + } + + // collector will accumulate statistics + const collector = Collector.create(testplaneToolCollector, pluginConfig); + + // subscribe to the relevant events + // to ultimately obtain the necessary statistics: + + // - how many tests passed successfully + testplane.on(testplane.events.TEST_PASS, data => collector.addSuccess(data)); + + // - how many tests failed + testplane.on(testplane.events.TEST_FAIL, data => collector.addFail(data)); + + // - how many were skipped + testplane.on(testplane.events.TEST_PENDING, data => collector.addSkipped(data)); + + // - number of retries + testplane.on(testplane.events.RETRY, data => collector.addRetry(data)); + + // after the test run is complete, save the statistics to a JSON file + testplane.on(testplane.events.RUNNER_END, () => collector.saveFile()); +}; +``` + +</details> + +### Automatic Launch of the Dev Server {#usage_starting_dev_server} + +We will schematically implement the `testplane-dev-server` plugin for testplane so that the dev server is automatically started each time testplane is launched. + +Starting the dev server is optional: the plugin adds a special `--dev-server` option to testplane, allowing the developer to specify whether to start the dev server when launching testplane. + +Additionally, the plugin allows setting the `devServer` parameter in its configuration. + +This example uses the following testplane events: + +- [CLI](#cli) +- [INIT](#init) + +<details> + +<summary> +Click to view the code + +</summary> + +**Plugin Code** + +```javascript +const http = require("http"); +const parseConfig = require("./config"); + +module.exports = (testplane, opts) => { + const pluginConfig = parseConfig(opts); + + if (!pluginConfig.enabled || testplane.isWorker()) { + // either the plugin is disabled, or we are in the worker context – exit + return; + } + + let program; + + testplane.on(testplane.events.CLI, cli => { + // need to save a reference to the commander instance (https://github.com/tj/commander.js), + // to later check for the option + program = cli; + // add the --dev-server option to testplane, + // so the user can explicitly specify when to start the dev server + cli.option("--dev-server", "run dev-server"); + }); + + testplane.on(testplane.events.INIT, () => { + // the dev server can be started either by specifying the --dev-server option + // when launching testplane, or in the plugin settings + const devServer = (program && program.devServer) || pluginConfig.devServer; + + if (!devServer) { + // if the dev server doesn't need to be started – exit + return; + } + + // content served by the dev server + const content = "<h1>Hello, World!</h1>"; + + // create the server and start listening on port 3000 + http.createServer((req, res) => res.end(content)).listen(3000); + + // at http://localhost:3000/index.html, the content will be: <h1>Hello, World!</h1> + }); +}; +``` + +**Testplane Configuration** + +```javascript +module.exports = { + // tests will be run in a local browser, + // see selenium-standalone in the "Quick Start" section + gridUrl: "http://localhost:4444/wd/hub", + // specify the path to the dev server + baseUrl: "http://localhost:3000", + + browsers: { + chrome: { + desiredCapabilities: { + browserName: "chrome", + }, + }, + }, + + plugins: { + // add our plugin to the list of plugins + "testplane-dev-server": { + enabled: true, + // by default, the dev server will not be started + devServer: false, + }, + }, +}; +``` + +**Test Code** + +```javascript +const { assert } = require("chai"); + +describe("example", async () => { + it("should find hello world", async ({ browser }) => { + // baseUrl, relative to which index.html is specified, + // is set in the testplane configuration above + await browser.url("index.html"); + + const title = await browser.$("h1").getText(); + assert.equal(title, "Hello, World!"); + }); +}); +``` + +</details> + +### Running Tests with Helpers {#usage_running_tests_with_helpers} + +Let's consider the [implementation][testplane-passive-browsers-index] of the [testplane-passive-browsers][testplane-passive-browsers] plugin. + +Using the [BEFORE_FILE_READ](#before_file_read) and [AFTER_TESTS_READ](#after_tests_read) events, the plugin allows adding a special helper that can run specified tests or test suites in given browsers. This logic can be useful if you don't need to run most tests in certain browsers. However, you might still want to run some tests in these (passive) browsers to check browser-specific things. + +In the example below, we simplified the plugin code a bit by setting the helper name `also` directly in the code, rather than taking it from the plugin configuration. + +This example uses the following testplane events: + +- [BEFORE_FILE_READ](#before_file_read) +- [AFTER_TESTS_READ](#after_tests_read) + +It also uses [testParser](#test_parser) and [testCollection][test-collection]. + +<details> + +<summary> +Click to view the code + +</summary> + +**Plugin Code** + +```javascript +const _ = require("lodash"); + +module.exports = (testplane, opts) => { + const pluginConfig = parseConfig(opts); + + if (!pluginConfig.enabled) { + // plugin is disabled – exit + return; + } + + if (testplane.isWorker()) { + testplane.on(testplane.events.BEFORE_FILE_READ, ({ testParser }) => { + // in workers, the helper will do nothing, + // set it to "no operation" + testParser.setController("also", { in: _.noop }); + }); + + return; + } + + const suitesToRun = {}; + const testsToRun = {}; + + testplane.on(testplane.events.BEFORE_FILE_READ, ({ testParser }) => { + testParser.setController("also", { + // matcher – parameter passed to the also.in() helper; + // can be a string, regular expression, or array of strings/regexps; + // in our case, the matcher defines the passive browsers + // in which the test(s) need to be run + in: function (matcher) { + const storage = this.suites ? suitesToRun : testsToRun; + + if (!shouldRunInBro(this.browserId, matcher)) { + // if the current browser is not in the list + // specified in the helper, do nothing + return; + } + + if (!storage[this.browserId]) { + storage[this.browserId] = []; + } + + // otherwise, collect the IDs of the tests + // that should be run for the current browser + storage[this.browserId].push({ id: this.id() }); + }, + }); + }); + + // use prependListener to initially enable tests only + // in the specified passive browsers, then all other tests + // that should be enabled will be enabled + testplane.prependListener(testplane.events.AFTER_TESTS_READ, testCollection => { + // form the list of passive browsers as the intersection of browsers for tests + // that were read, and browsers from the plugin configuration + const passiveBrowserIds = getPassiveBrowserIds(testCollection, pluginConfig); + + passiveBrowserIds.forEach(passiveBrowserId => { + const shouldRunTest = (runnable, storage = testsToRun) => { + const foundRunnable = + runnable.id && _.find(storage[passiveBrowserId], { id: runnable.id() }); + + return ( + foundRunnable || + (runnable.parent && shouldRunTest(runnable.parent, suitesToRun)) + ); + }; + + // disable all tests except those that should be run + // in the specified passive browsers + testCollection.eachTest(browserId, test => { + test.disabled = !shouldRunTest(test); + }); + }); + }); +}; +``` + +**Test Code** + +```javascript +testplane.also.in("ie6"); +describe("suite", () => { + it("test1", function () { + // ... + }); + + testplane.also.in(["ie7", /ie[89]/]); + it("test2", function () { + // ... + }); +}); +``` + +</details> + +### Profiling Test Runs {#usage_profiling_tests_runs} + +Let's consider a schematic implementation of profiling test runs. Each time a test starts, we will record its start time, and upon completion, the end time. All information will be saved to a stream, which will be closed upon the runner's completion. + +<details> + +<summary> +Click to view the code + +</summary> + +```javascript +const parseConfig = require("./lib/config"); +const StreamWriter = require("./lib/stream-writer"); + +module.exports = (testplane, opts) => { + const pluginConfig = parseConfig(opts); + + if (!pluginConfig.enabled) { + // plugin is disabled – exit + return; + } + + let writeStream; + + testplane.on(testplane.events.RUNNER_START, () => { + // create a stream for writing profiling data + writeStream = StreamWriter.create(pluginConfig.path); + }); + + testplane.on(testplane.events.TEST_BEGIN, test => { + if (test.pending) { + // test is disabled – do nothing + return; + } + + // record the test start time + test.timeStart = Date.now(); + }); + + testplane.on(testplane.events.TEST_END, test => { + if (test.pending) { + // test is disabled – do nothing + return; + } + + // record the test end time + test.timeEnd = Date.now(); + // and save the test information to the stream + writeStream.write(test); + }); + + // in case of an error, close the stream + testplane.on(testplane.events.ERROR, () => writeStream.end()); + + // after the runner completes, close the stream + testplane.on(testplane.events.RUNNER_END, () => writeStream.end()); +}; +``` + +</details> + +A more detailed [implementation][testplane-profiler-index] can be found in the [testplane-profiler][testplane-profiler] plugin. + +[process-send]: https://nodejs.org/docs/latest-v12.x/api/process.html#process_process_send_message_sendhandle_options_callback +[test-collection]: ../testplane-api#test_collection +[disable-all]: ../testplane-api#test_collection_disable_all +[enable-test]: ../testplane-api#test_collection_enable_test +[run]: ../testplane-api#testplane_run +[read-tests]: ../testplane-api#testplane_read_tests +[testplane-passive-browsers-index]: https://github.com/gemini-testing/testplane-passive-browsers/blob/master/lib/index.js +[testplane-passive-browsers]: https://github.com/gemini-testing/testplane-passive-browsers +[testplane-global-hook-index]: https://github.com/gemini-testing/testplane-global-hook/blob/master/index.js +[testplane-global-hook]: https://github.com/gemini-testing/testplane-global-hook +[testplane-test-filter]: https://github.com/gemini-testing/testplane-test-filter +[testplane-test-filter-index]: https://github.com/gemini-testing/testplane-test-filter/blob/master/lib/index.js +[json-reporter]: https://github.com/gemini-testing/json-reporter +[json-reporter-index]: https://github.com/gemini-testing/json-reporter/blob/master/testplane.js +[commander]: https://github.com/gemini-testing/commander.js +[commander-options]: https://github.com/tj/commander.js#options +[cli-wiki]: https://en.wikipedia.org/wiki/Command-line_interface +[testplane-run]: ../testplane-api#testplane_run +[system]: ../../config/system +[system-workers]: ../../config/system#workers +[system-tests-per-worker]: ../../config/system#tests_per_worker +[browser-tests-per-session]: ../../config/browsers#tests_per_session +[browser-sessions-per-browser]: ../../config/browsers#sessions_per_browser +[skip-in]: ../../guides/how-to-skip-test-in-browsers +[sigterm]: https://en.wikipedia.org/wiki/Signal_(Unix) +[testplane-test-repeater-index]: https://github.com/gemini-testing/testplane-test-repeater/blob/master/lib/index.js +[testplane-test-repeater]: https://github.com/gemini-testing/testplane-test-repeater +[express]: https://github.com/expressjs/express +[ssh-tunneler]: https://github.com/gemini-testing/ssh-tunneler +[ssh-tunneler-index]: https://github.com/gemini-testing/ssh-tunneler/blob/master/testplane.js +[testplane-image-minifier]: https://github.com/gemini-testing/testplane-image-minifier +[testplane-image-minifier-index]: https://github.com/gemini-testing/testplane-image-minifier/blob/master/lib/index.js +[testplane-profiler]: https://github.com/gemini-testing/testplane-profiler +[testplane-profiler-index]: https://github.com/gemini-testing/testplane-profiler/blob/master/index.js +[hermione-hide-scrollbars]: https://github.com/gemini-testing/hermione-hide-scrollbars +[hermione-hide-scrollbars-index]: https://github.com/gemini-testing/hermione-hide-scrollbars/blob/master/index.js +[vnc]: https://en.wikipedia.org/wiki/Virtual_Network_Computing +[browsers-retry]: ../../config/browsers#retry +[browsers-should-retry]: ../../config/browsers#should_retry +[retry-limiter]: ../../plugins/retry-limiter +[testplane-retry-progressive]: ../../plugins/testplane-retry-progressive +[error-object]: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Error diff --git a/i18n/en/docusaurus-plugin-content-docs/current/reference/webdriver-vs-cdp.mdx b/i18n/en/docusaurus-plugin-content-docs/current/reference/webdriver-vs-cdp.mdx new file mode 100644 index 0000000..16378ba --- /dev/null +++ b/i18n/en/docusaurus-plugin-content-docs/current/reference/webdriver-vs-cdp.mdx @@ -0,0 +1,90 @@ +# Communication Protocols with the Browser: WebDriver vs Chrome DevTools Protocol + +## Overview + +In the world of automated UI testing, the most popular protocols for controlling the browser are currently [WebDriver][webdriver] and [Chrome DevTools Protocol (CDP)][cdp]. + +## WebDriver + +[WebDriver][webdriver] is a standardized REST API protocol. Browser developers support this protocol in their drivers — [chromedriver][chromedriver], [geckodriver][geckodriver], etc. — which act as intermediaries (proxies) between the client sending requests and the browsers themselves. Such an intermediary is needed because browsers are written completely differently, and communication between the driver and the browser is not standardized. + +Additionally, the WebDriver protocol is used as the base protocol for automating mobile devices on _iOS/Android_ using [Appium][appium]. + +![WebDriver Protocol](/img/docs/reference/webdriver-vs-cdp.webdriver-protocol.png) + +### Pros + +- Official standard supported by all popular browsers; +- Support for automation on mobile devices; +- Can be used both on a local machine and remotely; + +### Cons + +- Out of the box, it does not allow tracking and intercepting network events (mocking requests/responses); +- Limited set of automation capabilities (e.g., no ability to control network bandwidth or CPU performance): the protocol covers only basic user interaction scenarios with the browser; +- No ability to subscribe to browser events (e.g., get information from the browser that a new tab has opened); +- Requires additional setup (installing _selenium-standalone_, necessary browser drivers, etc.). + +## CDP + +[Chrome DevTools Protocol (CDP)][cdp] is essentially [JSON RPC][json-rpc] implemented via [websockets][websockets]. + +_Chrome_ and _Node.js_ implement APIs for this protocol, which allow communication with _DevTools:_ sending commands, subscribing to events, etc. + +This API is used: + +- In _Chrome DevTools_ (the developer panel inside the browser) for debugging and inspecting code; +- In IDEs (e.g., _VSCode_) for similar purposes; +- In various test automation tools: [puppeteer][puppeteer], [cypress][cypress], etc.; +- For communication between [chromedriver][chromedriver] and the Chrome browser (in the image above — `browser protocol`). + +The protocol's API is logically divided into domains, which contain methods and can send events. + +For example, the [Runtime][cdp-runtime] domain allows inspecting the state of JavaScript, and the [Debugger][debugger] domain can be used to debug JavaScript. + +<img + src={ + require("@site/static/img/docs/reference/webdriver-vs-cdp.chrome-devtools-protocol.png") + .default + } + width="434" + height="200" +/> + +### Pros + +- Provides more automation capabilities than WebDriver; with CDP you can: + - [Track and intercept network requests and responses][how-to-intercept-requests-and-responses] + - [Test page accessibility][how-to-check-accessibility] + - [Manage network bandwidth][how-to-manage-network-bandwidth] + - [Manage CPU performance][how-to-manage-cpu-performance] + - [Hide scrollbars][how-to-hide-scrollbars-by-cdp] + - And more. +- No need to set up _selenium-standalone_ or browser drivers: just having a local Chrome browser is enough. + +### Cons + +- Supports a limited list of browsers: _Chrome, Chromium Edge, and Firefox nightly;_ +- By default, works only locally (but it is possible to connect to an already running browser on a remote machine). + +## Useful Links + +- [How to use CDP in Testplane][how-to-use-cdp] + +[webdriver]: https://www.w3.org/TR/webdriver/ +[cdp]: https://chromedevtools.github.io/devtools-protocol/ +[chromedriver]: https://chromedriver.chromium.org/ +[geckodriver]: https://github.com/mozilla/geckodriver +[appium]: https://appium.io/ +[json-rpc]: https://en.wikipedia.org/wiki/JSON-RPC +[websockets]: https://learn.javascript.ru/websocket +[puppeteer]: https://pptr.dev/ +[cypress]: https://cypress.io +[cdp-runtime]: https://chromedevtools.github.io/devtools-protocol/tot/Runtime/ +[debugger]: https://chromedevtools.github.io/devtools-protocol/tot/Debugger/ +[how-to-intercept-requests-and-responses]: ../../guides/how-to-intercept-requests-and-responses +[how-to-check-accessibility]: ../../guides/how-to-check-accessibility +[how-to-manage-network-bandwidth]: ../../guides/how-to-manage-network-bandwidth +[how-to-manage-cpu-performance]: ../../guides/how-to-manage-cpu-performance +[how-to-hide-scrollbars-by-cdp]: ../../guides/how-to-hide-scrollbars-by-cdp +[how-to-use-cdp]: ../../guides/how-to-use-cdp diff --git a/i18n/ru/code.json b/i18n/ru/code.json index 0683872..bde14f3 100644 --- a/i18n/ru/code.json +++ b/i18n/ru/code.json @@ -425,10 +425,10 @@ "message": "Выполняйте тесты в окружении ваших пользователей — не только в последних билдах Chrome." }, "landing.core-features.scale.descr": { - "message": "Сколько бы тестов у вас ни было — testplane отлично с ними справится." + "message": "Сколько бы тестов у вас ни было — Testplane отлично с ними справится." }, "landing.core-features.extend.descr": { - "message": "Благодаря многим точкам расширения testplane можно адаптировать даже к самым специфичным требованиям." + "message": "Благодаря многим точкам расширения Testplane можно адаптировать даже к самым специфичным требованиям." }, "landing.core-features.browsers.modern": { "message": "Все современные браузеры"