diff --git a/CHANGELOG.md b/CHANGELOG.md index 09153af..f99cb8b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,16 @@ +# [1.7.0](https://github.com/alexneo2003/playwright-azure-reporter/compare/v1.7.0-beta.0...v1.7.0) (2023-11-21) + + + +# [1.7.0-beta.0](https://github.com/alexneo2003/playwright-azure-reporter/compare/v1.6.1...v1.7.0-beta.0) (2023-11-15) + + +### Features + +* AZURE_PW_TEST_RUN_ID env variable ([5e51679](https://github.com/alexneo2003/playwright-azure-reporter/commit/5e5167930f21c11ee6a445177b4a922509b38b4a)) + + + ## [1.6.1](https://github.com/alexneo2003/playwright-azure-reporter/compare/v1.6.0...v1.6.1) (2023-10-16) diff --git a/README.md b/README.md index 02b0447..793b473 100644 --- a/README.md +++ b/README.md @@ -164,3 +164,7 @@ Reporter options (\* - required): - `testResult` - Published results of tests, at the end of each test, parallel to test run.. - `testRun` - Published test results to test run, at the end of test run. > **Note:** If you use `testRun` mode and using same test cases in different tests (yes i know it sounds funny), it will be overwritten with last test result. + +## Usefulness + +- **AZURE_PW_TEST_RUN_ID** - Id of current test run. It will be set in environment variables after test run created. Can be accessed by `process.env.AZURE_PW_TEST_RUN_ID`. Pay attention what `publishTestResultsMode` configuration you use. \ No newline at end of file diff --git a/package.json b/package.json index e0f26ea..fe476a3 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@alex_neo/playwright-azure-reporter", - "version": "1.6.1", + "version": "1.7.0", "description": "Playwright Azure Reporter", "main": "./dist/playwright-azure-reporter.js", "types": "./dist/playwright-azure-reporter.d.js", diff --git a/src/playwright-azure-reporter.ts b/src/playwright-azure-reporter.ts index a3cedaf..9c4bc9c 100644 --- a/src/playwright-azure-reporter.ts +++ b/src/playwright-azure-reporter.ts @@ -249,6 +249,8 @@ class AzureDevOpsReporter implements Reporter { if (run?.id) { this._resolveRunId(run.id); this._log(chalk.green(`Using run ${run.id} to publish test results`)); + process.env.AZURE_PW_TEST_RUN_ID = String(run.id); + this._log(chalk.green(`AZURE_PW_TEST_RUN_ID: ${process.env.AZURE_PW_TEST_RUN_ID}`)); } else { this._isDisabled = true; this._rejectRunId('Failed to create test run. Reporting is disabled.'); @@ -327,6 +329,8 @@ class AzureDevOpsReporter implements Reporter { if (runId) { this._resolveRunId(runId); this._log(chalk.green(`Using run ${runId} to publish test results`)); + process.env.AZURE_PW_TEST_RUN_ID = String(runId); + this._log(chalk.green(`AZURE_PW_TEST_RUN_ID: ${process.env.AZURE_PW_TEST_RUN_ID}`)); await this._publishTestResults(runId, this._testResultsToBePublished); } else { this._isDisabled = true; diff --git a/tests/reporter/assets/custom-reporter-testResult.ts b/tests/reporter/assets/custom-reporter-testResult.ts new file mode 100644 index 0000000..4d4213a --- /dev/null +++ b/tests/reporter/assets/custom-reporter-testResult.ts @@ -0,0 +1,24 @@ +import { expect } from '@playwright/test'; +import type { FullConfig, FullResult, Reporter, Suite, TestCase, TestResult } from '@playwright/test/reporter'; + +class MyReporter implements Reporter { + onBegin(config: FullConfig, suite: Suite) { + console.log(`Starting the run with ${suite.allTests().length} tests`); + } + + onTestBegin(test: TestCase) { + console.log(`Starting test ${test.title}`); + } + + onTestEnd(test: TestCase, result: TestResult) { + console.log(`Finished test ${test.title}: ${result.status}`); + } + + onEnd(result: FullResult) { + console.log(`Finished the run: ${result.status}`); + expect(process.env.AZURE_PW_TEST_RUN_ID).toBeDefined(); + expect(process.env.AZURE_PW_TEST_RUN_ID).toBe('150'); + } +} + +export default MyReporter; diff --git a/tests/reporter/assets/custom-reporter-testRun.ts b/tests/reporter/assets/custom-reporter-testRun.ts new file mode 100644 index 0000000..4d4213a --- /dev/null +++ b/tests/reporter/assets/custom-reporter-testRun.ts @@ -0,0 +1,24 @@ +import { expect } from '@playwright/test'; +import type { FullConfig, FullResult, Reporter, Suite, TestCase, TestResult } from '@playwright/test/reporter'; + +class MyReporter implements Reporter { + onBegin(config: FullConfig, suite: Suite) { + console.log(`Starting the run with ${suite.allTests().length} tests`); + } + + onTestBegin(test: TestCase) { + console.log(`Starting test ${test.title}`); + } + + onTestEnd(test: TestCase, result: TestResult) { + console.log(`Finished test ${test.title}: ${result.status}`); + } + + onEnd(result: FullResult) { + console.log(`Finished the run: ${result.status}`); + expect(process.env.AZURE_PW_TEST_RUN_ID).toBeDefined(); + expect(process.env.AZURE_PW_TEST_RUN_ID).toBe('150'); + } +} + +export default MyReporter; diff --git a/tests/reporter/reporter-publish-testResult.spec.ts b/tests/reporter/reporter-publish-testResult.spec.ts index f17dbb7..71db834 100644 --- a/tests/reporter/reporter-publish-testResult.spec.ts +++ b/tests/reporter/reporter-publish-testResult.spec.ts @@ -4,7 +4,7 @@ import { getRequestBody, setHeaders } from '../config/utils'; import azureAreas from './assets/azure-reporter/azureAreas'; import headers from './assets/azure-reporter/azureHeaders'; import location from './assets/azure-reporter/azureLocationOptionsResponse.json'; -import { reporterPath } from './reporterPath'; +import { customReporterTestResultPath, reporterPath } from './reporterPath'; import { expect, test } from './test-fixtures'; const TEST_OPTIONS_RESPONSE_PATH = path.join( @@ -800,4 +800,94 @@ test.describe('Publish results - testResult', () => { expect(result.exitCode).toBe(0); expect(result.passed).toBe(1); }); + + test('should set process.env.AZURE_PW_TEST_RUN_ID for publishTestResultsMode: "testResult"', async ({ + runInlineTest, + server, + }) => { + server.setRoute('/_apis/Location', (_, res) => { + setHeaders(res, headers); + res.end(JSON.stringify(location)); + }); + + server.setRoute('/_apis/ResourceAreas', (_, res) => { + setHeaders(res, headers); + res.end(JSON.stringify(azureAreas(server.PORT))); + }); + + server.setRoute('/_apis/Test', (req, res) => { + setHeaders(res, headers); + server.serveFile(req, res, TEST_OPTIONS_RESPONSE_PATH); + }); + + server.setRoute('/_apis/core', (req, res) => { + setHeaders(res, headers); + server.serveFile(req, res, CORE_OPTIONS_RESPONSE_PATH); + }); + + server.setRoute('/_apis/projects/SampleSample', (req, res) => { + setHeaders(res, headers); + server.serveFile(req, res, PROJECT_VALID_RESPONSE_PATH); + }); + + server.setRoute('/SampleSample/_apis/test/Runs', (req, res) => { + setHeaders(res, headers); + server.serveFile(req, res, CREATE_RUN_VALID_RESPONSE_PATH); + }); + + server.setRoute('/SampleSample/_apis/test/Points', (req, res) => { + setHeaders(res, headers); + server.serveFile(req, res, POINTS_3_VALID_RESPONSE_PATH); + }); + + server.setRoute('/SampleSample/_apis/test/Runs/150/Results', (req, res) => { + setHeaders(res, headers); + server.serveFile(req, res, TEST_RUN_RESULTS_3_VALID_RESPONSE_PATH); + }); + + server.setRoute('/SampleSample/_apis/test/Runs/150', (req, res) => { + setHeaders(res, headers); + server.serveFile(req, res, COMPLETE_RUN_VALID_RESPONSE_PATH); + }); + + const result = await runInlineTest( + { + 'playwright.config.ts': ` + module.exports = { + reporter: [ + ['line'], + ['${reporterPath}', { + orgUrl: 'http://localhost:${server.PORT}', + projectName: 'SampleSample', + planId: 4, + token: 'token', + logging: true, + }], + ['${customReporterTestResultPath}'] + ] + }; + `, + 'a.spec.js': ` + import { test, expect } from '@playwright/test'; + + test('[3] foobar', async () => { + expect(1).toBe(0); + }); + `, + }, + { reporter: '' } + ); + expect(result.output).not.toContain('Failed request: (401)'); + expect(result.output).toMatch(/azure: Using run (\d.*) to publish test results/); + expect(result.output).toContain('azure: AZURE_PW_TEST_RUN_ID: 150'); + expect(result.output).toContain('azure: [3] foobar - failed'); + expect(result.output).toContain('azure: Start publishing: [3] foobar'); + expect(result.output).toContain('azure: Result published: [3] foobar'); + expect(result.output).toMatch(/azure: Run (\d.*) - Completed/); + expect(result.output).not.toContain('Error in reporter'); + expect(result.output).not.toContain('expect(received).toBeDefined()'); + expect(result.output).not.toContain('Expected: "150"\nReceived: undefined'); + expect(result.exitCode).toBe(1); + expect(result.failed).toBe(1); + }); }); diff --git a/tests/reporter/reporter-publish-testRun.spec.ts b/tests/reporter/reporter-publish-testRun.spec.ts index 07def09..a0b8157 100644 --- a/tests/reporter/reporter-publish-testRun.spec.ts +++ b/tests/reporter/reporter-publish-testRun.spec.ts @@ -7,7 +7,7 @@ import location from './assets/azure-reporter/azureLocationOptionsResponse.json' import pointsResponseMapper from './assets/azure-reporter/pointsResponseMapper'; import testResultsByQueryResponseMapping from './assets/azure-reporter/testResultsByQueryResponseMapping'; import testRunResultsMapper from './assets/azure-reporter/testRunResultsMapper'; -import { reporterPath } from './reporterPath'; +import { customReporterTestRunPath, reporterPath } from './reporterPath'; import { expect, test } from './test-fixtures'; const TEST_OPTIONS_RESPONSE_PATH = path.join( @@ -958,4 +958,96 @@ test.describe('Publish results - testRun', () => { expect(result.failed).toBe(12); expect(result.passed).toBe(113); }); + + test('should set process.env.AZURE_PW_TEST_RUN_ID for publishTestResultsMode: "testRun"', async ({ + runInlineTest, + server, + }) => { + server.setRoute('/_apis/Location', (_, res) => { + setHeaders(res, headers); + res.end(JSON.stringify(location)); + }); + + server.setRoute('/_apis/ResourceAreas', (_, res) => { + setHeaders(res, headers); + res.end(JSON.stringify(azureAreas(server.PORT))); + }); + + server.setRoute('/_apis/Test', (req, res) => { + setHeaders(res, headers); + server.serveFile(req, res, TEST_OPTIONS_RESPONSE_PATH); + }); + + server.setRoute('/_apis/core', (req, res) => { + setHeaders(res, headers); + server.serveFile(req, res, CORE_OPTIONS_RESPONSE_PATH); + }); + + server.setRoute('/_apis/projects/SampleSample', (req, res) => { + setHeaders(res, headers); + server.serveFile(req, res, PROJECT_VALID_RESPONSE_PATH); + }); + + server.setRoute('/SampleSample/_apis/test/Runs', (req, res) => { + setHeaders(res, headers); + server.serveFile(req, res, CREATE_RUN_VALID_RESPONSE_PATH); + }); + + server.setRoute('/SampleSample/_apis/test/Points', (req, res) => { + setHeaders(res, headers); + server.serveFile(req, res, POINTS_3_VALID_RESPONSE_PATH); + }); + + server.setRoute('/SampleSample/_apis/test/Runs/150/Results', (req, res) => { + setHeaders(res, headers); + server.serveFile(req, res, TEST_RUN_RESULTS_3_VALID_RESPONSE_PATH); + }); + + server.setRoute('/SampleSample/_apis/test/Runs/150', (req, res) => { + setHeaders(res, headers); + server.serveFile(req, res, COMPLETE_RUN_VALID_RESPONSE_PATH); + }); + + const result = await runInlineTest( + { + 'playwright.config.ts': ` + module.exports = { + reporter: [ + ['line'], + ['${reporterPath}', { + orgUrl: 'http://localhost:${server.PORT}', + projectName: 'SampleSample', + planId: 4, + token: 'token', + logging: true, + publishTestResultsMode: 'testRun', + }], + ['${customReporterTestRunPath}'] + ] + }; + `, + 'a.spec.js': ` + import { test, expect } from '@playwright/test'; + test('[3] foobar', async () => { + expect(1).toBe(0); + }); + `, + }, + { reporter: '' } + ); + + expect(result.output).not.toContain('Failed request: (401)'); + expect(result.output).toMatch(/azure: Using run (\d.*) to publish test results/); + expect(result.output).toContain('azure: AZURE_PW_TEST_RUN_ID: 150'); + expect(result.output).toContain('azure: [3] foobar - failed'); + expect(result.output).toContain('azure: Start publishing test results for 1 test(s)'); + expect(result.output).toContain('azure: Left to publish: 0'); + expect(result.output).toContain('azure: Test results published for 1 test(s)'); + expect(result.output).toMatch(/azure: Run (\d.*) - Completed/); + expect(result.output).not.toContain('Error in reporter'); + expect(result.output).not.toContain('expect(received).toBeDefined()'); + expect(result.output).not.toContain('Expected: "150"\nReceived: undefined'); + expect(result.exitCode).toBe(1); + expect(result.failed).toBe(1); + }); }); diff --git a/tests/reporter/reporterPath.ts b/tests/reporter/reporterPath.ts index 5e6136c..06c5c72 100644 --- a/tests/reporter/reporterPath.ts +++ b/tests/reporter/reporterPath.ts @@ -1,3 +1,5 @@ import path from 'path'; export const reporterPath = path.join(__dirname, '../../dist/playwright-azure-reporter.js'); +export const customReporterTestRunPath = path.join(__dirname, './assets/custom-reporter-testRun.ts'); +export const customReporterTestResultPath = path.join(__dirname, './assets/custom-reporter-testResult.ts');