diff --git a/.github/workflows/pr_android.yml b/.github/workflows/pr_android.yml index 984d76004..3b9ab26b8 100644 --- a/.github/workflows/pr_android.yml +++ b/.github/workflows/pr_android.yml @@ -82,6 +82,5 @@ jobs: disable-animations: true script: | npm run test:e2e:android - echo "### E2E Test Summary" >> $GITHUB_STEP_SUMMARY - echo $(adb logcat -d -v raw | grep "E2E_RESULTS" | sed "s/'E2E_RESULTS', '//") >> $GITHUB_STEP_SUMMARY + cat cavy_results.md >> $GITHUB_STEP_SUMMARY working-directory: e2e diff --git a/.github/workflows/pr_ios.yml b/.github/workflows/pr_ios.yml index 1b48f4e54..2b4eda4a5 100644 --- a/.github/workflows/pr_ios.yml +++ b/.github/workflows/pr_ios.yml @@ -57,5 +57,7 @@ jobs: - name: Run e2e tests working-directory: e2e - run: npm run test:e2e:ios + run: | + npm run test:e2e:ios + cat cavy_results.md >> $GITHUB_STEP_SUMMARY diff --git a/e2e/cavy_results.md b/e2e/cavy_results.md new file mode 100644 index 000000000..d845cd432 --- /dev/null +++ b/e2e/cavy_results.md @@ -0,0 +1,15 @@ +### E2E Test Summary|Description ๐Ÿ“|Test results ๐Ÿงช|Duration โฐ| +|---|---|---| +|Set HLS source and auto-play: dispatches sourcechange event on setting a source without autoplay|โœ…|0.732s| +|Set HLS source and auto-play: dispatches sourcechange, play and playing events in order on setting a source with autoplay|โœ…|1.698s| +|Set HLS source and auto-play: dispatches a seeked event after seeking|โœ…|1.786s| +|Set HLS source and auto-play: dispatches paused, play and playing events after pausing & resuming playback|โœ…|0.836s| +|Set mp4 source and auto-play: dispatches sourcechange event on setting a source without autoplay|โœ…|0.176s| +|Set mp4 source and auto-play: dispatches sourcechange, play and playing events in order on setting a source with autoplay|โœ…|1.586s| +|Set mp4 source and auto-play: dispatches a seeked event after seeking|โœ…|1.961s| +|Set mp4 source and auto-play: dispatches paused, play and playing events after pausing & resuming playback|โœ…|2.042s| +|Set source with ads and auto-play: dispatches sourcechange, play, playing and ad events|โŒ|10.171s| +|Setup Conviva connector: successfully creates the connector, connects to the player, uses API, and cleans up and destroys.|โœ…|0.712s| +|Setup Nielsen connector: successfully creates the connector, connects to the player, uses API, and cleans up and destroys.|โœ…|0.884s| +|Setup Adobe connector: successfully creates the connector, connects to the player, uses API, and cleans up and destroys.|โœ…|0.916s| +|Switch between presentation modes: dispatches presentationmodechange events between inline and fullscreen.|โœ…|1.352s| \ No newline at end of file diff --git a/e2e/package.json b/e2e/package.json index b9e091ec1..f07baeee1 100644 --- a/e2e/package.json +++ b/e2e/package.json @@ -5,8 +5,8 @@ "private": true, "scripts": { "start": "npx react-native start", - "test:e2e:android": "npx cavy run-android --terminal bash --mode release", - "test:e2e:ios": "npx cavy run-ios --terminal bash", + "test:e2e:android": "npx cavy run-android --md --terminal bash --mode release", + "test:e2e:ios": "npx cavy run-ios --md --terminal bash", "lint": "eslint \"**/*.{ts,tsx}\"", "postinstall": "patch-package" }, diff --git a/e2e/patches/@types+cavy+3.2.9.patch b/e2e/patches/@types+cavy+3.2.9.patch deleted file mode 100644 index 40c091ce9..000000000 --- a/e2e/patches/@types+cavy+3.2.9.patch +++ /dev/null @@ -1,27 +0,0 @@ -diff --git a/node_modules/@types/cavy/index.d.ts b/node_modules/@types/cavy/index.d.ts -index 9e0fb3f..1857974 100644 ---- a/node_modules/@types/cavy/index.d.ts -+++ b/node_modules/@types/cavy/index.d.ts -@@ -19,6 +19,13 @@ export function wrap

( - WrappedComponent: React.ComponentClass

| React.FunctionComponent

, - ): React.ComponentClass

; - -+export interface TestReporter { -+ type: string; -+ onStart: () => void; -+ send: (result: TestReport) => void; -+ onFinish: (report: TestReport) => void; -+} -+ - export interface TesterProps { - children: React.ReactElement; - store: TestHookStore; -@@ -26,7 +33,7 @@ export interface TesterProps { - waitTime?: number | undefined; - startDelay?: number | undefined; - clearAsyncStorage?: boolean | undefined; -- reporter?: ((report: TestReport) => void) | undefined; -+ reporter?: typeof TestReporter | undefined; - - // Deprecated - sendReport?: boolean | undefined; diff --git a/e2e/patches/cavy+4.0.2.patch b/e2e/patches/cavy+4.0.2.patch deleted file mode 100644 index e5f4fbdd5..000000000 --- a/e2e/patches/cavy+4.0.2.patch +++ /dev/null @@ -1,27 +0,0 @@ -diff --git a/node_modules/cavy/src/Tester.js b/node_modules/cavy/src/Tester.js -index c61e31a..8d222ae 100644 ---- a/node_modules/cavy/src/Tester.js -+++ b/node_modules/cavy/src/Tester.js -@@ -57,20 +57,8 @@ export default class Tester extends Component { - key: Math.random() - }; - this.testHookStore = props.store; -- // Default to sending a test report to cavy-cli if no custom reporter is -- // supplied. -- if (props.reporter instanceof Function) { -- const message = 'Deprecation warning: support for custom function' + -- 'reporters will soon be deprecated. Cavy supports custom ' + -- 'class based reporters. For more info, see the ' + -- 'documentation here: ' + -- 'https://cavy.app/docs/guides/writing-custom-reporters'; -- console.warn(message); -- this.reporter = props.reporter; -- } else { -- reporterClass = props.reporter || Reporter; -- this.reporter = new reporterClass; -- } -+ reporterClass = props.reporter || Reporter; -+ this.reporter = new reporterClass; - } - - componentDidMount() { diff --git a/e2e/patches/cavy-cli+3.0.0.patch b/e2e/patches/cavy-cli+3.0.0.patch new file mode 100644 index 000000000..3c0d4bff7 --- /dev/null +++ b/e2e/patches/cavy-cli+3.0.0.patch @@ -0,0 +1,118 @@ +diff --git a/node_modules/cavy-cli/cavy.js b/node_modules/cavy-cli/cavy.js +index da9fc72..c3d3b41 100755 +--- a/node_modules/cavy-cli/cavy.js ++++ b/node_modules/cavy-cli/cavy.js +@@ -32,9 +32,10 @@ function test(cmd) { + const entryFile = cmd.file; + const skipbuild = cmd.skipbuild; + const outputAsXml = cmd.xml; ++ const outputAsMarkdown = cmd.md; + const dev = cmd.dev; + const bootTimeout = cmd.bootTimeout; +- runTests(commandName, entryFile, skipbuild, dev, outputAsXml, bootTimeout, args); ++ runTests(commandName, entryFile, skipbuild, dev, outputAsXml, outputAsMarkdown, bootTimeout, args); + } + + // Stop quitting unless we want to +@@ -64,6 +65,7 @@ program + + '(is ignored if used with --skipbuild, defaults to 2 minutes, requires Cavy 4.0.0)' + ) + .option('--xml', 'Write out test results to cavy_results.xml (requires Cavy 3.3.0)') ++ .option('--md', 'Write out test results to cavy_results.md') + .allowUnknownOption() + .action(cmd => test(cmd)); + +@@ -82,6 +84,7 @@ program + + '(is ignored if used with --skipbuild, defaults to 2 minutes, requires Cavy 4.0.0)' + ) + .option('--xml', 'Write out test results to cavy_results.xml (requires Cavy 3.3.0)') ++ .option('--md', 'Write out test results to cavy_results.md') + .allowUnknownOption() + .action(cmd => test(cmd)); + +diff --git a/node_modules/cavy-cli/server.js b/node_modules/cavy-cli/server.js +index 64faf22..8ecf8b3 100644 +--- a/node_modules/cavy-cli/server.js ++++ b/node_modules/cavy-cli/server.js +@@ -2,6 +2,7 @@ const http = require('http'); + const WebSocket = require('ws'); + const chalk = require('chalk'); + const constructXML = require('./src/junitFormatter'); ++const { writeFileSync } = require('fs'); + + // Initialize a server + const server = http.createServer(); +@@ -62,6 +63,18 @@ function logTestResult(testResultJson) { + } + }; + ++function constructMarkdown(results) { ++ const filename = 'cavy_results.md'; ++ console.log(`Writing results to ${filename}`); ++ const data = ++ `### E2E Test Summary\n` + ++ `|Description ๐Ÿ“|Test results ๐Ÿงช|Duration โฐ|\n` + ++ `|---|---|---|\n` + ++ results.testCases.map((result) => `|${result.description}|${result.passed ? `โœ…` : `โŒ`}|${result.time}s|`).join('\n'); ++ ++ writeFileSync(filename, data); ++}; ++ + // Internal: Accepts a json report object, console.logs the overall result of + // the test suite and quits the process with either exit code 1 or 0 depending + // on whether any tests failed. +@@ -81,6 +94,11 @@ function finishTesting(reportJson) { + constructXML(fullResults); + } + ++ // If requested, construct XML report. ++ if (server.locals.outputAsMarkdown) { ++ constructMarkdown(fullResults); ++ } ++ + // If all tests pass, exit with code 0, else code 42. + // Code 42 chosen at random so that a test failure can be distinuguished from + // a build failure (in which case the React Native CLI would exit with code 1). +diff --git a/node_modules/cavy-cli/src/runTests.js b/node_modules/cavy-cli/src/runTests.js +index 69ef62c..9eb750e 100644 +--- a/node_modules/cavy-cli/src/runTests.js ++++ b/node_modules/cavy-cli/src/runTests.js +@@ -57,9 +57,10 @@ function getAdbPath() { + } + + // Start test server, listening for test results to be posted. +-function runServer({ command, dev, outputAsXml, skipbuild, bootTimeout }) { ++function runServer({ command, dev, outputAsXml, outputAsMarkdown, skipbuild, bootTimeout }) { + server.locals.dev = dev; + server.locals.outputAsXml = outputAsXml; ++ server.locals.outputAsMarkdown = outputAsMarkdown; + server.listen(8082, () => { + if (command == 'run-android') { + runAdbReverse(); +@@ -93,7 +94,7 @@ function runServer({ command, dev, outputAsXml, skipbuild, bootTimeout }) { + // outputAsXml: whether to write and save the results to XML file + // bootTimeout: how long the CLI should wait for the RN app to boot. + // args: any extra arguments the user would usually to pass to `react native run...` +-function runTests(command, file, skipbuild, dev, outputAsXml, bootTimeout, args) { ++function runTests(command, file, skipbuild, dev, outputAsXml, outputAsMarkdown, bootTimeout, args) { + + // Assume entry file is 'index.js' if user doesn't supply one. + const entryFile = file || 'index.js'; +@@ -134,7 +135,7 @@ function runTests(command, file, skipbuild, dev, outputAsXml, bootTimeout, args) + }); + + if (skipbuild) { +- runServer({ command, dev, outputAsXml, skipbuild, bootTimeout }); ++ runServer({ command, dev, outputAsXml, outputAsMarkdown, skipbuild, bootTimeout }); + } else { + // Build the app, start the test server and wait for results. + console.log(`cavy: Running \`npx react-native ${command}\`...`); +@@ -151,7 +152,7 @@ function runTests(command, file, skipbuild, dev, outputAsXml, bootTimeout, args) + if (code) { + return process.exit(code); + } +- runServer({ command, dev, outputAsXml, skipbuild, bootTimeout }); ++ runServer({ command, dev, outputAsXml, outputAsMarkdown, skipbuild, bootTimeout }); + }); + } + } diff --git a/e2e/src/TestableApp.tsx b/e2e/src/TestableApp.tsx index c7bbd11d5..8f1b53fa5 100644 --- a/e2e/src/TestableApp.tsx +++ b/e2e/src/TestableApp.tsx @@ -4,7 +4,6 @@ import { Platform, SafeAreaView, StyleSheet, View, ViewStyle } from 'react-nativ import { TestableTHEOplayerView } from './components/TestableTHEOplayerView'; import Specs from './tests'; import { getStatusBarHeight } from 'react-native-status-bar-height'; -import { Reporter } from './utils/TestReporter'; const testHookStore = new TestHookStore(); @@ -23,7 +22,7 @@ const PLAYER_CONTAINER_STYLE: ViewStyle = { export class TestableApp extends Component { render() { return ( - + diff --git a/e2e/src/utils/TestReporter.ts b/e2e/src/utils/TestReporter.ts deleted file mode 100644 index a3d848fb9..000000000 --- a/e2e/src/utils/TestReporter.ts +++ /dev/null @@ -1,74 +0,0 @@ -import { TestReport, TestReporter } from 'cavy'; - -interface TestData { - event: string; - data: TestReport; -} - -export class Reporter implements TestReporter { - private ws: WebSocket | undefined = undefined; - - get type(): string { - return 'realtime'; - } - - // Internal: Creates a websocket connection to the cavy-cli server. - onStart() { - const url = 'ws://127.0.0.1:8082/'; - this.ws = new WebSocket(url); - } - - // Internal: Send a single test result to cavy-cli over the websocket connection. - send(result: TestReport) { - if (this.websocketReady()) { - const testData = { event: 'singleResult', data: result }; - this.sendData(testData); - } - } - - // Internal: Send report to cavy-cli over the websocket connection. - onFinish(report: TestReport) { - if (this.websocketReady()) { - const testData = { event: 'testingComplete', data: report }; - this.sendData(testData); - console.log(`E2E_RESULTS`, this.generateTable(report)); - } else { - // If cavy-cli is not running, let people know in a friendly way - const message = - "Skipping sending test report to cavy-cli - if you'd " + - 'like information on how to set up cavy-cli, check out the README ' + - 'https://github.com/pixielabs/cavy-cli'; - - console.log(message); - } - } - - // Private: Determines whether data can be sent over the websocket. - websocketReady() { - // WebSocket.readyState 1 means the web socket connection is OPEN. - return this.ws?.readyState == 1; - } - - // Private: Sends data over the websocket and console logs any errors. - sendData(testData: TestData) { - try { - this.ws?.send(JSON.stringify(testData)); - if (testData?.event == 'testingComplete') { - console.log('Cavy test report successfully sent to cavy-cli'); - } - } catch (e: any) { - console.group('Error sending test data'); - console.warn(e.message); - console.groupEnd(); - } - } - - generateTable(report: TestReport): string { - return ( - `|Description ๐Ÿ“|Test results ๐Ÿงช|Duration โฐ|\n` + - `|---|---|---|\n` + - // @ts-ignore - report.results.map((result) => `|${result.description}|${result.passed ? `โœ…` : `โŒ`}|${result.time}s|`).join('\n') - ); - } -}