Merge pull request #111 from aik099/specify-assertion-error-via-api

Specify failed assertion message to BrowserStack/SauceLabs
minkphp · Feb 15, 2024 · 41afdf4 · 41afdf4
2 parents c9870f3 + 1935dc3
commit 41afdf4
Show file tree

Hide file tree

Showing 8 changed files with 91 additions and 14 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -4,7 +4,7 @@ This project adheres to [Semantic Versioning](http://semver.org/).
 
 ## [Unreleased]
 ### Added
-...
+- Specify failed PHPUnit assertion text to the BrowserStack ("reason" field)/SauceLabs("custom-data" field) test.
 
 ### Changed
 - Bumped minimum PHP version to 5.6.

diff --git a/library/aik099/PHPUnit/APIClient/BrowserStackAPIClient.php b/library/aik099/PHPUnit/APIClient/BrowserStackAPIClient.php
@@ -68,14 +68,20 @@ public function getInfo($session_id)
 	/**
 	 * Update status of the test, that was executed in the given session.
 	 *
-	 * @param string  $session_id  Session ID.
-	 * @param boolean $test_status Test status.
+	 * @param string  $session_id          Session ID.
+	 * @param boolean $test_status         Test status.
+	 * @param string  $test_status_message Test status message.
 	 *
 	 * @return array
 	 */
-	public function updateStatus($session_id, $test_status)
+	public function updateStatus($session_id, $test_status, $test_status_message)
 	{
 		$data = array('status' => $test_status ? 'passed' : 'failed');
+
+		if ( $test_status_message ) {
+			$data['reason'] = $test_status_message;
+		}
+
 		$result = $this->execute('PUT', 'sessions/' . $session_id . '.json', $data);
 
 		return $result['automation_session'];

diff --git a/library/aik099/PHPUnit/APIClient/IAPIClient.php b/library/aik099/PHPUnit/APIClient/IAPIClient.php
@@ -29,11 +29,12 @@ public function getInfo($session_id);
 	/**
 	 * Update status of the test, that was executed in the given session.
 	 *
-	 * @param string  $session_id  Session ID.
-	 * @param boolean $test_status Test status.
+	 * @param string  $session_id          Session ID.
+	 * @param boolean $test_status         Test status.
+	 * @param string  $test_status_message Test status message.
 	 *
 	 * @return array
 	 */
-	public function updateStatus($session_id, $test_status);
+	public function updateStatus($session_id, $test_status, $test_status_message);
 
 }
diff --git a/library/aik099/PHPUnit/APIClient/SauceLabsAPIClient.php b/library/aik099/PHPUnit/APIClient/SauceLabsAPIClient.php
@@ -13,6 +13,9 @@
 
 use WebDriver\SauceLabs\SauceRest;
 
+/**
+ * @link https://docs.saucelabs.com/dev/api/jobs/
+ */
 class SauceLabsAPIClient implements IAPIClient
 {
 
@@ -48,14 +51,21 @@ public function getInfo($session_id)
 	/**
 	 * Update status of the test, that was executed in the given session.
 	 *
-	 * @param string  $session_id  Session ID.
-	 * @param boolean $test_status Test status.
+	 * @param string  $session_id          Session ID.
+	 * @param boolean $test_status         Test status.
+	 * @param string  $test_status_message Test status message.
 	 *
 	 * @return array
 	 */
-	public function updateStatus($session_id, $test_status)
+	public function updateStatus($session_id, $test_status, $test_status_message)
 	{
-		return $this->_sauceRest->updateJob($session_id, array('passed' => $test_status));
+		$data = array('passed' => $test_status);
+
+		if ( $test_status_message ) {
+			$data['custom-data'] = array('status_message' => $test_status_message);
+		}
+
+		return $this->_sauceRest->updateJob($session_id, $data);
 	}
 
 }
diff --git a/library/aik099/PHPUnit/BrowserConfiguration/ApiBrowserConfiguration.php b/library/aik099/PHPUnit/BrowserConfiguration/ApiBrowserConfiguration.php
@@ -206,7 +206,8 @@ public function onTestEnded(TestEndedEvent $event)
 
 		$this->getAPIClient()->updateStatus(
 			$this->getSessionId($session),
-			$this->getTestStatus($test_case, $event->getTestResult())
+			$this->getTestStatus($test_case, $event->getTestResult()),
+			$this->getTestStatusMessage($test_case, $event->getTestResult())
 		);
 	}
 

diff --git a/library/aik099/PHPUnit/BrowserConfiguration/BrowserConfiguration.php b/library/aik099/PHPUnit/BrowserConfiguration/BrowserConfiguration.php
@@ -529,6 +529,40 @@ public function getTestStatus(BrowserTestCase $test_case, TestResult $test_resul
 		return !$test_case->hasFailed();
 	}
 
+	/**
+	 * Returns test status message based on session strategy requested by browser.
+	 *
+	 * @param BrowserTestCase $test_case   Browser test case.
+	 * @param TestResult      $test_result Test result.
+	 *
+	 * @return boolean
+	 * @see    IsolatedSessionStrategy
+	 * @see    SharedSessionStrategy
+	 */
+	public function getTestStatusMessage(BrowserTestCase $test_case, TestResult $test_result)
+	{
+		if ( $this->isShared() ) {
+			// All tests in a test case use same session -> failed even if 1 test fails.
+			$test_failures = array();
+
+			if ( $test_result->errorCount() > 0 ) {
+				$test_failures = $test_result->errors();
+			}
+			elseif ( $test_result->failureCount() > 0 ) {
+				$test_failures = $test_result->failures();
+			}
+			elseif ( $test_result->warningCount() > 0 ) {
+				$test_failures = $test_result->warnings();
+			}
+
+			return $test_failures ? reset($test_failures)->exceptionMessage() : '';
+
+		}
+
+		// Each test in a test case are using it's own session -> failed if test fails.
+		return $test_case->getStatusMessage();
+	}
+
 	/**
 	 * Returns checksum from current configuration.
 	 *

diff --git a/tests/aik099/PHPUnit/BrowserConfiguration/ApiBrowserConfigurationTestCase.php b/tests/aik099/PHPUnit/BrowserConfiguration/ApiBrowserConfigurationTestCase.php
@@ -308,8 +308,9 @@ public function testTestEndedEvent($driver_type)
 			$driver = m::mock('\\Behat\\Mink\\Driver\\Selenium2Driver');
 			$driver->shouldReceive('getWebDriverSessionId')->once()->andReturn('SID');
 
-			$api_client->shouldReceive('updateStatus')->with('SID', true)->once();
+			$api_client->shouldReceive('updateStatus')->with('SID', true, 'test status message')->once();
 			$test_case->shouldReceive('hasFailed')->once()->andReturn(false); // For shared strategy.
+			$test_case->shouldReceive('getStatusMessage')->once()->andReturn('test status message'); // For shared strategy.
 		}
 		else {
 			$driver = m::mock('\\Behat\\Mink\\Driver\\DriverInterface');

diff --git a/tests/aik099/PHPUnit/Fixture/ApiIntegrationFixture.php b/tests/aik099/PHPUnit/Fixture/ApiIntegrationFixture.php
@@ -125,6 +125,19 @@ public function verifyRemoteAPICalls(TestEvent $event)
 					$session_info['passed'],
 					'SauceLabs test status set via API'
 				);
+
+				$custom_data_mapping = array(
+					'testSuccess' => null,
+					'testFailure' => array(
+						'status_message' => "This test is expected to fail.\nFailed asserting that false is true."
+					),
+				);
+
+				$this->assertSame(
+					$custom_data_mapping[$test_name],
+					$session_info['custom-data'],
+					'SauceLabs test status message set via API'
+				);
 			}
 			elseif ( $browser instanceof BrowserStackBrowserConfiguration ) {
 				$this->assertEquals(
@@ -143,6 +156,17 @@ public function verifyRemoteAPICalls(TestEvent $event)
 					$session_info['status'],
 					'BrowserStack test status set via API'
 				);
+
+				$reason_mapping = array(
+					'testSuccess' => '',
+					'testFailure' => "This test is expected to fail.\nFailed asserting that false is true.",
+				);
+
+				$this->assertSame(
+					$reason_mapping[$test_name],
+					$session_info['reason'],
+					'BrowserStack test status message set via API'
+				);
 			}
 		}
 	}
@@ -194,7 +218,7 @@ public function testFailure()
 		$session = $this->getSession();
 		$session->visit('http://www.google.com');
 
-		$this->assertTrue(false);
+		$this->assertTrue(false, 'This test is expected to fail.');
 	}
 
 	/**