From 1935dc3c34d640ffd5d182f93e263b2de25b4c5d Mon Sep 17 00:00:00 2001 From: Alex Date: Thu, 15 Feb 2024 20:16:20 +0200 Subject: [PATCH] Specify failed assertion message to BrowserStack/SauceLabs --- CHANGELOG.md | 2 +- .../APIClient/BrowserStackAPIClient.php | 12 +++++-- .../aik099/PHPUnit/APIClient/IAPIClient.php | 7 ++-- .../PHPUnit/APIClient/SauceLabsAPIClient.php | 18 +++++++--- .../ApiBrowserConfiguration.php | 3 +- .../BrowserConfiguration.php | 34 +++++++++++++++++++ .../ApiBrowserConfigurationTestCase.php | 3 +- .../PHPUnit/Fixture/ApiIntegrationFixture.php | 26 +++++++++++++- 8 files changed, 91 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 755b37e..3a427b6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ This project adheres to [Semantic Versioning](http://semver.org/). ## [Unreleased] ### Added -... +- Specify failed PHPUnit assertion text to the BrowserStack ("reason" field)/SauceLabs("custom-data" field) test. ### Changed - Bumped minimum PHP version to 5.6. diff --git a/library/aik099/PHPUnit/APIClient/BrowserStackAPIClient.php b/library/aik099/PHPUnit/APIClient/BrowserStackAPIClient.php index 0ec51f6..9798e7b 100644 --- a/library/aik099/PHPUnit/APIClient/BrowserStackAPIClient.php +++ b/library/aik099/PHPUnit/APIClient/BrowserStackAPIClient.php @@ -68,14 +68,20 @@ public function getInfo($session_id) /** * Update status of the test, that was executed in the given session. * - * @param string $session_id Session ID. - * @param boolean $test_status Test status. + * @param string $session_id Session ID. + * @param boolean $test_status Test status. + * @param string $test_status_message Test status message. * * @return array */ - public function updateStatus($session_id, $test_status) + public function updateStatus($session_id, $test_status, $test_status_message) { $data = array('status' => $test_status ? 'passed' : 'failed'); + + if ( $test_status_message ) { + $data['reason'] = $test_status_message; + } + $result = $this->execute('PUT', 'sessions/' . $session_id . '.json', $data); return $result['automation_session']; diff --git a/library/aik099/PHPUnit/APIClient/IAPIClient.php b/library/aik099/PHPUnit/APIClient/IAPIClient.php index 294ae98..19bc025 100644 --- a/library/aik099/PHPUnit/APIClient/IAPIClient.php +++ b/library/aik099/PHPUnit/APIClient/IAPIClient.php @@ -29,11 +29,12 @@ public function getInfo($session_id); /** * Update status of the test, that was executed in the given session. * - * @param string $session_id Session ID. - * @param boolean $test_status Test status. + * @param string $session_id Session ID. + * @param boolean $test_status Test status. + * @param string $test_status_message Test status message. * * @return array */ - public function updateStatus($session_id, $test_status); + public function updateStatus($session_id, $test_status, $test_status_message); } diff --git a/library/aik099/PHPUnit/APIClient/SauceLabsAPIClient.php b/library/aik099/PHPUnit/APIClient/SauceLabsAPIClient.php index 4875292..448fd32 100644 --- a/library/aik099/PHPUnit/APIClient/SauceLabsAPIClient.php +++ b/library/aik099/PHPUnit/APIClient/SauceLabsAPIClient.php @@ -13,6 +13,9 @@ use WebDriver\SauceLabs\SauceRest; +/** + * @link https://docs.saucelabs.com/dev/api/jobs/ + */ class SauceLabsAPIClient implements IAPIClient { @@ -48,14 +51,21 @@ public function getInfo($session_id) /** * Update status of the test, that was executed in the given session. * - * @param string $session_id Session ID. - * @param boolean $test_status Test status. + * @param string $session_id Session ID. + * @param boolean $test_status Test status. + * @param string $test_status_message Test status message. * * @return array */ - public function updateStatus($session_id, $test_status) + public function updateStatus($session_id, $test_status, $test_status_message) { - return $this->_sauceRest->updateJob($session_id, array('passed' => $test_status)); + $data = array('passed' => $test_status); + + if ( $test_status_message ) { + $data['custom-data'] = array('status_message' => $test_status_message); + } + + return $this->_sauceRest->updateJob($session_id, $data); } } diff --git a/library/aik099/PHPUnit/BrowserConfiguration/ApiBrowserConfiguration.php b/library/aik099/PHPUnit/BrowserConfiguration/ApiBrowserConfiguration.php index 8649091..c8db3e6 100644 --- a/library/aik099/PHPUnit/BrowserConfiguration/ApiBrowserConfiguration.php +++ b/library/aik099/PHPUnit/BrowserConfiguration/ApiBrowserConfiguration.php @@ -206,7 +206,8 @@ public function onTestEnded(TestEndedEvent $event) $this->getAPIClient()->updateStatus( $this->getSessionId($session), - $this->getTestStatus($test_case, $event->getTestResult()) + $this->getTestStatus($test_case, $event->getTestResult()), + $this->getTestStatusMessage($test_case, $event->getTestResult()) ); } diff --git a/library/aik099/PHPUnit/BrowserConfiguration/BrowserConfiguration.php b/library/aik099/PHPUnit/BrowserConfiguration/BrowserConfiguration.php index 0bff67a..2ea425e 100644 --- a/library/aik099/PHPUnit/BrowserConfiguration/BrowserConfiguration.php +++ b/library/aik099/PHPUnit/BrowserConfiguration/BrowserConfiguration.php @@ -529,6 +529,40 @@ public function getTestStatus(BrowserTestCase $test_case, TestResult $test_resul return !$test_case->hasFailed(); } + /** + * Returns test status message based on session strategy requested by browser. + * + * @param BrowserTestCase $test_case Browser test case. + * @param TestResult $test_result Test result. + * + * @return boolean + * @see IsolatedSessionStrategy + * @see SharedSessionStrategy + */ + public function getTestStatusMessage(BrowserTestCase $test_case, TestResult $test_result) + { + if ( $this->isShared() ) { + // All tests in a test case use same session -> failed even if 1 test fails. + $test_failures = array(); + + if ( $test_result->errorCount() > 0 ) { + $test_failures = $test_result->errors(); + } + elseif ( $test_result->failureCount() > 0 ) { + $test_failures = $test_result->failures(); + } + elseif ( $test_result->warningCount() > 0 ) { + $test_failures = $test_result->warnings(); + } + + return $test_failures ? reset($test_failures)->exceptionMessage() : ''; + + } + + // Each test in a test case are using it's own session -> failed if test fails. + return $test_case->getStatusMessage(); + } + /** * Returns checksum from current configuration. * diff --git a/tests/aik099/PHPUnit/BrowserConfiguration/ApiBrowserConfigurationTestCase.php b/tests/aik099/PHPUnit/BrowserConfiguration/ApiBrowserConfigurationTestCase.php index d26b620..4feaf7b 100644 --- a/tests/aik099/PHPUnit/BrowserConfiguration/ApiBrowserConfigurationTestCase.php +++ b/tests/aik099/PHPUnit/BrowserConfiguration/ApiBrowserConfigurationTestCase.php @@ -308,8 +308,9 @@ public function testTestEndedEvent($driver_type) $driver = m::mock('\\Behat\\Mink\\Driver\\Selenium2Driver'); $driver->shouldReceive('getWebDriverSessionId')->once()->andReturn('SID'); - $api_client->shouldReceive('updateStatus')->with('SID', true)->once(); + $api_client->shouldReceive('updateStatus')->with('SID', true, 'test status message')->once(); $test_case->shouldReceive('hasFailed')->once()->andReturn(false); // For shared strategy. + $test_case->shouldReceive('getStatusMessage')->once()->andReturn('test status message'); // For shared strategy. } else { $driver = m::mock('\\Behat\\Mink\\Driver\\DriverInterface'); diff --git a/tests/aik099/PHPUnit/Fixture/ApiIntegrationFixture.php b/tests/aik099/PHPUnit/Fixture/ApiIntegrationFixture.php index 7611a8b..5c58a2a 100644 --- a/tests/aik099/PHPUnit/Fixture/ApiIntegrationFixture.php +++ b/tests/aik099/PHPUnit/Fixture/ApiIntegrationFixture.php @@ -125,6 +125,19 @@ public function verifyRemoteAPICalls(TestEvent $event) $session_info['passed'], 'SauceLabs test status set via API' ); + + $custom_data_mapping = array( + 'testSuccess' => null, + 'testFailure' => array( + 'status_message' => "This test is expected to fail.\nFailed asserting that false is true." + ), + ); + + $this->assertSame( + $custom_data_mapping[$test_name], + $session_info['custom-data'], + 'SauceLabs test status message set via API' + ); } elseif ( $browser instanceof BrowserStackBrowserConfiguration ) { $this->assertEquals( @@ -143,6 +156,17 @@ public function verifyRemoteAPICalls(TestEvent $event) $session_info['status'], 'BrowserStack test status set via API' ); + + $reason_mapping = array( + 'testSuccess' => '', + 'testFailure' => "This test is expected to fail.\nFailed asserting that false is true.", + ); + + $this->assertSame( + $reason_mapping[$test_name], + $session_info['reason'], + 'BrowserStack test status message set via API' + ); } } } @@ -194,7 +218,7 @@ public function testFailure() $session = $this->getSession(); $session->visit('http://www.google.com'); - $this->assertTrue(false); + $this->assertTrue(false, 'This test is expected to fail.'); } /**