From 5690f047af42e72d5371454890c0978d91cfe666 Mon Sep 17 00:00:00 2001 From: Guillaume Egret Date: Thu, 18 Jan 2024 17:33:53 +0100 Subject: [PATCH 01/11] feat(instruments profiler): add new profiler --- .../platforms/ios-instruments/src/index.ts | 39 +++++++++++++++++++ packages/platforms/profiler/package.json | 1 + packages/platforms/profiler/src/index.ts | 3 ++ 3 files changed, 43 insertions(+) diff --git a/packages/platforms/ios-instruments/src/index.ts b/packages/platforms/ios-instruments/src/index.ts index 92b13684..9eab5c9d 100644 --- a/packages/platforms/ios-instruments/src/index.ts +++ b/packages/platforms/ios-instruments/src/index.ts @@ -1 +1,40 @@ +import { Measure, Profiler, ProfilerPollingOptions, ScreenRecorder } from "@perf-profiler/types"; export { killApp } from "./utils/DeviceManager"; + +export class IOSInstrumentsProfiler implements Profiler { + connectedDevice: IdbDevice | undefined; + recordingProcess: ChildProcess | undefined; + traceFile: string | undefined; + pid: number | undefined; + bundleId: string | undefined; + onMeasure: ((measure: Measure) => void) | undefined; + pollPerformanceMeasures(bundleId: string, options: ProfilerPollingOptions): { stop: () => void } { + if (!this.pid) throw new Error("Profiler is not ready, app is not running"); + this.onMeasure = options.onMeasure; + return { + stop: () => { + return; + }, + }; + } + + detectCurrentBundleId(): string { + throw new Error("App Id detection is not implemented on iOS with Instruments"); + } + + installProfilerOnDevice() { + // Do we need anything here? + } + + getScreenRecorder(videoPath: string): ScreenRecorder | undefined { + return undefined; + } + + cleanup: () => void = () => { + // Do we need anything here? + }; + async stopApp(bundleId: string): Promise { + killApp(bundleId); + return new Promise((resolve) => resolve()); + } +} diff --git a/packages/platforms/profiler/package.json b/packages/platforms/profiler/package.json index 7b923267..c6d3a296 100644 --- a/packages/platforms/profiler/package.json +++ b/packages/platforms/profiler/package.json @@ -16,6 +16,7 @@ "dependencies": { "@perf-profiler/android": "^0.13.0", "@perf-profiler/ios": "^0.3.3", + "@perf-profiler/ios-instruments": "^0.3.3", "@perf-profiler/types": "^0.8.0" } } diff --git a/packages/platforms/profiler/src/index.ts b/packages/platforms/profiler/src/index.ts index ae985021..39504d5d 100644 --- a/packages/platforms/profiler/src/index.ts +++ b/packages/platforms/profiler/src/index.ts @@ -1,11 +1,14 @@ import { AndroidProfiler, FlashlightSelfProfiler } from "@perf-profiler/android"; import { IOSProfiler } from "@perf-profiler/ios"; +import { IOSInstrumentsProfiler } from "@perf-profiler/ios-instruments"; import { Profiler } from "@perf-profiler/types"; const getProfiler = (): Profiler => { switch (process.env.PLATFORM) { case "ios": return new IOSProfiler(); + case "ios-instruments": + return new IOSInstrumentsProfiler(); case "flashlight": return new FlashlightSelfProfiler(); default: From ba0e23622ad8bc4b9305a7fdd235624e77e3b470 Mon Sep 17 00:00:00 2001 From: Guillaume Egret Date: Thu, 18 Jan 2024 17:36:32 +0100 Subject: [PATCH 02/11] feat(launchIOS): remove custom command --- .../ios-instruments/src/launchIOS.ts | 134 ------------------ 1 file changed, 134 deletions(-) delete mode 100644 packages/platforms/ios-instruments/src/launchIOS.ts diff --git a/packages/platforms/ios-instruments/src/launchIOS.ts b/packages/platforms/ios-instruments/src/launchIOS.ts deleted file mode 100644 index 8607a4bf..00000000 --- a/packages/platforms/ios-instruments/src/launchIOS.ts +++ /dev/null @@ -1,134 +0,0 @@ -#!/usr/bin/env node - -// TODO: refactor so that these functions are not in android -// eslint-disable-next-line import/no-extraneous-dependencies -import { executeAsync, executeCommand } from "@perf-profiler/android/dist/src/commands/shell"; -import fs from "fs"; -import { writeReport } from "./writeReport"; -import { program } from "commander"; -import { execSync, ChildProcess } from "child_process"; -import os from "os"; - -const tmpFiles: string[] = []; -const removeTmpFiles = () => { - for (const tmpFile of tmpFiles) { - fs.rmSync(tmpFile, { recursive: true }); - } -}; - -const getTmpFilePath = (fileName: string) => { - const filePath = `${os.tmpdir()}/${fileName}`; - tmpFiles.push(filePath); - - return filePath; -}; - -const writeTmpFile = (fileName: string, content: string): string => { - const tmpPath = getTmpFilePath(fileName); - fs.writeFileSync(tmpPath, content); - return tmpPath; -}; - -const startRecord = (simulatorId: string, traceFile: string): ChildProcess => { - const templateFilePath = `${__dirname}/../Flashlight.tracetemplate`; - return executeAsync( - `xcrun xctrace record --device ${simulatorId} --template ${templateFilePath} --attach fakeStore --output ${traceFile}` - ); -}; - -const save = (traceFile: string, resultsFilePath: string) => { - const xmlOutputFile = getTmpFilePath("report.xml"); - executeCommand( - `xctrace export --input ${traceFile} --xpath '/trace-toc/run[@number="1"]/data/table[@schema="time-profile"]' --output ${xmlOutputFile}` - ); - writeReport(xmlOutputFile, resultsFilePath); -}; - -const launchTest = async ({ - testCommand, - appId, - simulatorId, - resultsFilePath, -}: { - testCommand: string; - appId: string; - simulatorId: string; - resultsFilePath: string; -}) => { - const traceFile = `report_${new Date().getTime()}.trace`; - const lauchAppFile = writeTmpFile( - "./launch.yaml", - `appId: ${appId} ---- -- launchApp -` - ); - execSync(`maestro test ${lauchAppFile} --no-ansi`, { - stdio: "inherit", - }); - const recordingProcess = startRecord(simulatorId, traceFile); - await new Promise((resolve) => { - recordingProcess.stdout?.on("data", (data) => { - if (data.toString().includes("Ctrl-C to stop")) { - resolve(); - } - }); - }); - execSync(`${testCommand} --no-ansi`, { - stdio: "inherit", - }); - const stopAppFile = writeTmpFile( - "./stop.yaml", - `appId: ${appId} ---- -- stopApp -` - ); - execSync(`maestro test ${stopAppFile} --no-ansi`, { - stdio: "inherit", - }); - try { - await new Promise((resolve) => { - recordingProcess.stdout?.on("data", (data) => { - console.log(data.toString()); - if (data.toString().includes("Output file saved as")) { - resolve(); - } - }); - }); - } catch (e) { - console.log("Error while recording: ", e); - } - save(traceFile, resultsFilePath); - - removeTmpFiles(); -}; - -program - .command("ios-test") - .requiredOption("--appId ", "App ID (e.g. com.monapp)") - .requiredOption( - "--simulatorId ", - "Simulator ID (e.g. 12345678-1234-1234-1234-123456789012)" - ) - .requiredOption( - "--testCommand ", - "Test command (e.g. `maestro test flow.yml`). App performance during execution of this script will be measured over several iterations." - ) - .requiredOption( - "--resultsFilePath ", - "Path where the JSON of results will be written" - ) - .summary("Generate web report from performance measures for iOS.") - .description( - `Generate web report from performance measures. - -Examples: -flashlight ios-test --appId com.monapp --simulatorId 12345678-1234-1234-1234-123456789012 --testCommand "maestro test flow.yml" --resultsFilePath report.json -` - ) - .action((options) => { - launchTest(options); - }); - -program.parse(); From 8dcdff70d3958368a734118f1230d405bec66e67 Mon Sep 17 00:00:00 2001 From: Guillaume Egret Date: Thu, 18 Jan 2024 17:37:25 +0100 Subject: [PATCH 03/11] refacto(writeReport): parse and return data instead of writting report --- .../ios-instruments/src/XcodePerfParser.ts | 112 +++++++++++++++ .../ios-instruments/src/writeReport.ts | 128 ------------------ 2 files changed, 112 insertions(+), 128 deletions(-) create mode 100644 packages/platforms/ios-instruments/src/XcodePerfParser.ts delete mode 100644 packages/platforms/ios-instruments/src/writeReport.ts diff --git a/packages/platforms/ios-instruments/src/XcodePerfParser.ts b/packages/platforms/ios-instruments/src/XcodePerfParser.ts new file mode 100644 index 00000000..57d44361 --- /dev/null +++ b/packages/platforms/ios-instruments/src/XcodePerfParser.ts @@ -0,0 +1,112 @@ +import { XMLParser } from "fast-xml-parser"; +import { Result, Row, Thread, isRefField } from "./utils/xmlTypes"; +import fs from "fs"; +import { CpuMeasure, Measure } from "@perf-profiler/types"; + +const FAKE_RAM = 200; +const FAKE_FPS = 60; +const TIME_INTERVAL = 500; +const NANOSEC_TO_MILLISEC = 1_000_000; +const CPU_TIME_INTERVAL = 10; + +const initThreadMap = (row: Row[]): { [id: number]: string } => { + const threadRef: { [id: number]: Thread } = {}; + row.forEach((row: Row) => { + if (!isRefField(row.thread)) { + threadRef[row.thread.id] = row.thread; + } + }); + return Object.values(threadRef).reduce((acc: { [id: number]: string }, thread) => { + const currentThreadName = thread.fmt + .split(" ") + .slice(0, thread.fmt.split(" ").indexOf("")) + .join(" "); + const currentTid = thread.tid.value; + const numberOfThread = Object.values(threadRef).filter((thread: Thread) => { + return thread.fmt.includes(currentThreadName) && thread.tid.value < currentTid; + }).length; + acc[thread.id] = + numberOfThread > 0 ? `${currentThreadName} (${numberOfThread})` : currentThreadName; + return acc; + }, {}); +}; + +const getMeasures = (row: Row[]): Map> => { + const sampleTimeRef: { [id: number]: number } = {}; + const threadRef: { [id: number]: string } = initThreadMap(row); + const classifiedMeasures = row.reduce((acc: Map>, row: Row) => { + const sampleTime = isRefField(row.sampleTime) + ? sampleTimeRef[row.sampleTime.ref] + : row.sampleTime.value / NANOSEC_TO_MILLISEC; + if (!isRefField(row.sampleTime)) { + sampleTimeRef[row.sampleTime.id] = sampleTime; + } + + const threadName = isRefField(row.thread) + ? threadRef[row.thread.ref] + : threadRef[row.thread.id]; + + const correspondingTimeInterval = + parseInt((sampleTime / TIME_INTERVAL).toFixed(0), 10) * TIME_INTERVAL; + + const timeIntervalMap = acc.get(correspondingTimeInterval) ?? new Map(); + + const numberOfPointsIn = timeIntervalMap.get(threadName) ?? 0; + + timeIntervalMap.set(threadName, numberOfPointsIn + 1); + + acc.set(correspondingTimeInterval, timeIntervalMap); + + return acc; + }, new Map>()); + return classifiedMeasures; +}; + +export const computeMeasures = (inputFileName: string) => { + const xml = fs.readFileSync(inputFileName, "utf8"); + const options = { + attributeNamePrefix: "", + ignoreAttributes: false, + parseAttributeValue: true, + textNodeName: "value", + updateTag(tagName: string, jPath: string, attrs: { [x: string]: string | number }) { + switch (tagName) { + case "trace-query-result": { + return "result"; + } + case "sample-time": { + return "sampleTime"; + } + default: { + return tagName; + } + } + }, + }; + const parser = new XMLParser(options); + const jsonObject: Result = parser.parse(xml); + if (!jsonObject.result.node.row) { + throw new Error("No rows in the xml file"); + } + const measures: Map> = getMeasures(jsonObject.result.node.row); + const formattedMeasures: Measure[] = Array.from(measures.entries()).map( + (classifiedMeasures: [number, Map]) => { + const timeInterval = classifiedMeasures[0]; + const timeIntervalMap = classifiedMeasures[1]; + const cpuMeasure: CpuMeasure = { + perName: {}, + perCore: {}, + }; + timeIntervalMap.forEach((value: number, key: string) => { + cpuMeasure.perName[key] = (value * 10) / (TIME_INTERVAL / CPU_TIME_INTERVAL); + }); + return { + cpu: cpuMeasure, + ram: FAKE_RAM, + fps: FAKE_FPS, + time: timeInterval, + }; + } + ); + return formattedMeasures; +}; diff --git a/packages/platforms/ios-instruments/src/writeReport.ts b/packages/platforms/ios-instruments/src/writeReport.ts deleted file mode 100644 index 17ffa701..00000000 --- a/packages/platforms/ios-instruments/src/writeReport.ts +++ /dev/null @@ -1,128 +0,0 @@ -import { XMLParser } from "fast-xml-parser"; -import fs from "fs"; -import { CpuMeasure, Measure, TestCaseIterationResult, TestCaseResult } from "@perf-profiler/types"; -import { Result, Row, Thread, isRefField } from "./utils/xmlTypes"; - -export const writeReport = (inputFileName: string, outputFileName: string) => { - const xml = fs.readFileSync(inputFileName, "utf8"); - const iterations: TestCaseIterationResult[] = []; - const FAKE_RAM = 200; - const FAKE_FPS = 60; - const TIME_INTERVAL = 500; - const NANOSEC_TO_MILLISEC = 1_000_000; - const CPU_TIME_INTERVAL = 10; - - const initThreadMap = (row: Row[]): { [id: number]: string } => { - const threadRef: { [id: number]: Thread } = {}; - row.forEach((row: Row) => { - if (!isRefField(row.thread)) { - threadRef[row.thread.id] = row.thread; - } - }); - return Object.values(threadRef).reduce((acc: { [id: number]: string }, thread) => { - const currentThreadName = thread.fmt - .split(" ") - .slice(0, thread.fmt.split(" ").indexOf("")) - .join(" "); - const currentTid = thread.tid.value; - const numberOfThread = Object.values(threadRef).filter((thread: Thread) => { - return thread.fmt.includes(currentThreadName) && thread.tid.value < currentTid; - }).length; - acc[thread.id] = - numberOfThread > 0 ? `${currentThreadName} (${numberOfThread})` : currentThreadName; - return acc; - }, {}); - }; - - const getMeasures = (row: Row[]): Map> => { - const sampleTimeRef: { [id: number]: number } = {}; - const threadRef: { [id: number]: string } = initThreadMap(row); - const classifiedMeasures = row.reduce((acc: Map>, row: Row) => { - const sampleTime = isRefField(row.sampleTime) - ? sampleTimeRef[row.sampleTime.ref] - : row.sampleTime.value / NANOSEC_TO_MILLISEC; - if (!isRefField(row.sampleTime)) { - sampleTimeRef[row.sampleTime.id] = sampleTime; - } - - const threadName = isRefField(row.thread) - ? threadRef[row.thread.ref] - : threadRef[row.thread.id]; - - const correspondingTimeInterval = - parseInt((sampleTime / TIME_INTERVAL).toFixed(0), 10) * TIME_INTERVAL; - - const timeIntervalMap = acc.get(correspondingTimeInterval) ?? new Map(); - - const numberOfPointsIn = timeIntervalMap.get(threadName) ?? 0; - - timeIntervalMap.set(threadName, numberOfPointsIn + 1); - - acc.set(correspondingTimeInterval, timeIntervalMap); - - return acc; - }, new Map>()); - return classifiedMeasures; - }; - - const options = { - attributeNamePrefix: "", - ignoreAttributes: false, - parseAttributeValue: true, - textNodeName: "value", - updateTag(tagName: string) { - switch (tagName) { - case "trace-query-result": { - return "result"; - } - case "sample-time": { - return "sampleTime"; - } - default: { - return tagName; - } - } - }, - }; - const parser = new XMLParser(options); - const jsonObject: Result = parser.parse(xml); - if (!jsonObject.result.node.row) { - throw new Error("No rows in the xml file"); - } - - const measures: Map> = getMeasures(jsonObject.result.node.row); - const formattedMeasures: Measure[] = Array.from(measures.entries()).map( - (classifiedMeasures: [number, Map]) => { - const timeInterval = classifiedMeasures[0]; - const timeIntervalMap = classifiedMeasures[1]; - const cpuMeasure: CpuMeasure = { - perName: {}, - perCore: {}, - }; - timeIntervalMap.forEach((value: number, key: string) => { - cpuMeasure.perName[key] = (value * 10) / (TIME_INTERVAL / CPU_TIME_INTERVAL); - }); - return { - cpu: cpuMeasure, - ram: FAKE_RAM, - fps: FAKE_FPS, - time: timeInterval, - }; - } - ); - - iterations.push({ - time: formattedMeasures[formattedMeasures.length - 1].time, - measures: formattedMeasures, - status: "SUCCESS", - }); - - const results: TestCaseResult = { - name: "iOS Measures", - status: "SUCCESS", - iterations, - type: "IOS_EXPERIMENTAL", - }; - - fs.writeFileSync(outputFileName, JSON.stringify(results, null, 2)); -}; From 4ed5d1d78c9bd2cd15ed557b2f7e1175bfab61ab Mon Sep 17 00:00:00 2001 From: Guillaume Egret Date: Thu, 18 Jan 2024 17:39:01 +0100 Subject: [PATCH 04/11] feat(profiler): add 'hack' functions for instruments profiler --- packages/core/types/index.ts | 2 + .../src/commands/platforms/UnixProfiler.ts | 8 ++ .../platforms/ios-instruments/src/index.ts | 80 +++++++++++++++++++ packages/platforms/ios/src/index.ts | 8 ++ 4 files changed, 98 insertions(+) diff --git a/packages/core/types/index.ts b/packages/core/types/index.ts index 66a75dc6..df2588f3 100644 --- a/packages/core/types/index.ts +++ b/packages/core/types/index.ts @@ -97,4 +97,6 @@ export interface Profiler { cleanup: () => void; getScreenRecorder: (videoPath: string) => ScreenRecorder | undefined; stopApp: (bundleId: string) => Promise; + waitUntilReady: (bundleId: string) => Promise; + getMeasures: () => Promise; } diff --git a/packages/platforms/android/src/commands/platforms/UnixProfiler.ts b/packages/platforms/android/src/commands/platforms/UnixProfiler.ts index be26fe66..5584f8ad 100644 --- a/packages/platforms/android/src/commands/platforms/UnixProfiler.ts +++ b/packages/platforms/android/src/commands/platforms/UnixProfiler.ts @@ -27,6 +27,14 @@ const defaultBinaryFolder = `${__dirname}/../../..${__dirname.includes("dist") ? const binaryFolder = process.env.FLASHLIGHT_BINARY_PATH || defaultBinaryFolder; export abstract class UnixProfiler implements Profiler { + // eslint-disable-next-line @typescript-eslint/no-unused-vars + waitUntilReady = (bundleId: string) => { + return new Promise((resolve) => resolve()); + }; + getMeasures = () => { + return new Promise((resolve) => resolve()); + }; + stop(): void { throw new Error("Method not implemented."); } diff --git a/packages/platforms/ios-instruments/src/index.ts b/packages/platforms/ios-instruments/src/index.ts index 9eab5c9d..ac04babf 100644 --- a/packages/platforms/ios-instruments/src/index.ts +++ b/packages/platforms/ios-instruments/src/index.ts @@ -1,6 +1,64 @@ import { Measure, Profiler, ProfilerPollingOptions, ScreenRecorder } from "@perf-profiler/types"; +import { ChildProcess } from "child_process"; +// TODO: refactor so that these functions are not in android +// eslint-disable-next-line import/no-extraneous-dependencies +import { executeAsync, executeCommand } from "@perf-profiler/android"; +import { IdbDevice, getConnectedDevice, killApp, launchApp } from "./utils/DeviceManager"; +import { computeMeasures } from "./XcodePerfParser"; +import { getTmpFilePath, removeTmpFiles } from "./utils/tmpFileManager"; export { killApp } from "./utils/DeviceManager"; +const startRecord = async ( + deviceUdid: string, + appPid: number, + traceFile: string +): Promise => { + const templateFilePath = `${__dirname}/../Flashlight.tracetemplate`; + const recordingProcess = executeAsync( + `xcrun xctrace record --device ${deviceUdid} --template ${templateFilePath} --attach ${appPid} --output ${traceFile}` + ); + await new Promise((resolve) => { + recordingProcess.stdout?.on("data", (data) => { + if (data.toString().includes("Ctrl-C to stop")) { + resolve(); + } + }); + }); + return recordingProcess; +}; + +const saveTraceFile = (traceFile: string): string => { + const xmlOutputFile = getTmpFilePath("report.xml"); + executeCommand( + `xctrace export --input ${traceFile} --xpath '/trace-toc/run[@number="1"]/data/table[@schema="time-profile"]' --output ${xmlOutputFile}` + ); + return xmlOutputFile; +}; + +const stopPerfRecord = async ( + recordingProcess: ChildProcess, + traceFile: string, + onMeasure: (measure: Measure) => void +) => { + try { + await new Promise((resolve) => { + recordingProcess.stdout?.on("data", (data) => { + if (data.toString().includes("Output file saved as")) { + resolve(); + } + }); + }); + } catch (e) { + console.log("Error while recording: ", e); + } + const xmlFile = saveTraceFile(traceFile); + const measures = computeMeasures(xmlFile); + measures.forEach((measure) => { + onMeasure(measure); + }); + removeTmpFiles(); +}; + export class IOSInstrumentsProfiler implements Profiler { connectedDevice: IdbDevice | undefined; recordingProcess: ChildProcess | undefined; @@ -33,6 +91,28 @@ export class IOSInstrumentsProfiler implements Profiler { cleanup: () => void = () => { // Do we need anything here? }; + + async waitUntilReady(bundleId: string): Promise { + this.connectedDevice = getConnectedDevice(); + if (!this.connectedDevice) { + throw new Error("No device connected"); + } + this.bundleId = bundleId; + this.pid = launchApp(bundleId); + const traceFile = `report_${new Date().getTime()}.trace`; + this.traceFile = traceFile; + this.recordingProcess = await startRecord(this.connectedDevice.udid, this.pid, traceFile); + } + + async getMeasures(): Promise { + if (!this.recordingProcess || !this.traceFile || !this.pid || !this.onMeasure || !this.bundleId) + throw new Error("Profiler is not ready to get measures"); + const recordingProcess = this.recordingProcess; + const traceFile = this.traceFile; + killApp(this.bundleId); + await stopPerfRecord(recordingProcess, traceFile, this.onMeasure); + } + async stopApp(bundleId: string): Promise { killApp(bundleId); return new Promise((resolve) => resolve()); diff --git a/packages/platforms/ios/src/index.ts b/packages/platforms/ios/src/index.ts index adadcbb0..0a2efd16 100644 --- a/packages/platforms/ios/src/index.ts +++ b/packages/platforms/ios/src/index.ts @@ -104,4 +104,12 @@ export class IOSProfiler implements Profiler { killApp(bundleId); return new Promise((resolve) => resolve()); } + + async waitUntilReady(bundleId: string): Promise { + return new Promise((resolve) => resolve()); + } + + async getMeasures(): Promise { + return new Promise((resolve) => resolve()); + } } From 3df4a6d3f8aae5f0860b6254980606a7cd36811b Mon Sep 17 00:00:00 2001 From: Guillaume Egret Date: Thu, 18 Jan 2024 17:39:59 +0100 Subject: [PATCH 05/11] feat(PerformanceMeasurer): setup and get measures from instruments profiler --- packages/commands/test/src/PerformanceMeasurer.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/packages/commands/test/src/PerformanceMeasurer.ts b/packages/commands/test/src/PerformanceMeasurer.ts index 834bf3d0..d3184049 100644 --- a/packages/commands/test/src/PerformanceMeasurer.ts +++ b/packages/commands/test/src/PerformanceMeasurer.ts @@ -35,9 +35,12 @@ export class PerformanceMeasurer { ) { await this.maybeStartRecording(); + // Hack to make sure the profiler is ready to receive measures + await profiler.waitUntilReady(this.bundleId); this.polling = profiler.pollPerformanceMeasures(this.bundleId, { onMeasure: (measure) => { - if (this.shouldStop) { + // The ios-instruments profiler yields measures at the end of the test when the polling is already stopped + if (this.shouldStop && process.env.PLATFORM !== "ios-instruments") { this.polling?.stop(); } @@ -76,6 +79,8 @@ export class PerformanceMeasurer { // Ensure polling has stopped this.polling?.stop(); + // Hack for ios-instruments to get the measures at the end of the test + await profiler.getMeasures(); await this.maybeStopRecording(); From a2e183aedf70435222f316d2576989517b7bc1f0 Mon Sep 17 00:00:00 2001 From: Guillaume Egret Date: Thu, 18 Jan 2024 17:41:40 +0100 Subject: [PATCH 06/11] fix(tmpFileManager): prevent from adding twice a file in the tmp file list --- .../platforms/ios-instruments/src/utils/tmpFileManager.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/platforms/ios-instruments/src/utils/tmpFileManager.ts b/packages/platforms/ios-instruments/src/utils/tmpFileManager.ts index 3ab96330..4693ee5e 100644 --- a/packages/platforms/ios-instruments/src/utils/tmpFileManager.ts +++ b/packages/platforms/ios-instruments/src/utils/tmpFileManager.ts @@ -11,7 +11,9 @@ export const writeTmpFile = (fileName: string, content: string): string => { export const getTmpFilePath = (fileName: string) => { const filePath = `${os.tmpdir()}/${fileName}`; - tmpFiles.push(filePath); + if (!tmpFiles.includes(filePath)) { + tmpFiles.push(filePath); + } return filePath; }; From 2c62578b509ccb1f1f380d1139d2ef762d24357a Mon Sep 17 00:00:00 2001 From: Guillaume Egret Date: Thu, 18 Jan 2024 17:58:48 +0100 Subject: [PATCH 07/11] chore(README): update readme --- packages/platforms/ios-instruments/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/platforms/ios-instruments/README.md b/packages/platforms/ios-instruments/README.md index b0d531bd..42e169c2 100644 --- a/packages/platforms/ios-instruments/README.md +++ b/packages/platforms/ios-instruments/README.md @@ -10,7 +10,7 @@ Requirements: - Get a running simulator id with `xcrun simctl list devices` - Create template Flashlight in Xcode Instruments (with cpu-profile and memory usage) - Add your own test in `test.yaml` -- `flashlight-ios-poc ios-test --appId --simulatorId 9F852910-03AD-495A-8E16-7356B764284 --testCommand "maestro test test.yaml" --resultsFilePath "./result.json"` +- `PLATFORM=ios-instruments node packages/commands/test/dist/bin.js test --bundleId --simulatorId 9F852910-03AD-495A-8E16-7356B764284 --testCommand "maestro test test.yaml" --resultsFilePath "./result.json"` - Check the results in the web-reporter `yarn workspace @perf-profiler/web-reporter build` @@ -19,5 +19,5 @@ Requirements: ## Next steps - run several iterations -- add more metrics (RAM, FPS, CPU per thread) +- add more metrics (RAM, FPS) - Unify API with flashlight test From 24e4232fe7b5573267aadcb17e43b60e737d1b61 Mon Sep 17 00:00:00 2001 From: almouro Date: Thu, 25 Jan 2024 17:45:59 +0100 Subject: [PATCH 08/11] chore: fix ios ci --- .github/workflows/ios_e2e.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ios_e2e.sh b/.github/workflows/ios_e2e.sh index a2a16550..c3627d3d 100755 --- a/.github/workflows/ios_e2e.sh +++ b/.github/workflows/ios_e2e.sh @@ -22,4 +22,4 @@ xcrun simctl install $UDID ./.github/workflows/fakeStore.app xcrun simctl launch $UDID $APPID mkdir -p report -npx flashlight-ios-poc ios-test --appId $APPID --simulatorId $UDID --testCommand 'maestro test ./packages/platforms/ios-instruments/test.yaml' --resultsFilePath './report/result.json' \ No newline at end of file +PLATFORM=ios-instruments node packages/commands/test/dist/bin.js test --bundleId $APPID --testCommand 'maestro test ./packages/platforms/ios-instruments/test.yaml' --resultsFilePath './report/result.json' --iterationCount 2 From 293379a0bc33a16893d63b790aee89ad36d742ee Mon Sep 17 00:00:00 2001 From: almouro Date: Fri, 10 May 2024 17:34:19 +0200 Subject: [PATCH 09/11] chore(ios): fix ci install --- .github/workflows/ios_e2e.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ios_e2e.sh b/.github/workflows/ios_e2e.sh index c3627d3d..b3baae10 100755 --- a/.github/workflows/ios_e2e.sh +++ b/.github/workflows/ios_e2e.sh @@ -3,6 +3,7 @@ export MAESTRO_VERSION={1.29.0}; curl -Ls "https://get.maestro.mobile.dev" | bas export PATH="$PATH":"$HOME/.maestro/bin" brew tap facebook/fb brew install facebook/fb/idb-companion +pip install fb-idb APPID="org.reactjs.native.example.fakeStore" From b1e6dea7435efe65ec3ffe9c44e163991a38cebc Mon Sep 17 00:00:00 2001 From: almouro Date: Fri, 10 May 2024 17:39:04 +0200 Subject: [PATCH 10/11] chore(ios): fix eslint errors --- packages/platforms/ios-instruments/src/XcodePerfParser.ts | 1 + packages/platforms/ios-instruments/src/index.ts | 2 +- packages/platforms/ios/src/index.ts | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/platforms/ios-instruments/src/XcodePerfParser.ts b/packages/platforms/ios-instruments/src/XcodePerfParser.ts index 57d44361..415e89da 100644 --- a/packages/platforms/ios-instruments/src/XcodePerfParser.ts +++ b/packages/platforms/ios-instruments/src/XcodePerfParser.ts @@ -69,6 +69,7 @@ export const computeMeasures = (inputFileName: string) => { ignoreAttributes: false, parseAttributeValue: true, textNodeName: "value", + // eslint-disable-next-line @typescript-eslint/no-unused-vars updateTag(tagName: string, jPath: string, attrs: { [x: string]: string | number }) { switch (tagName) { case "trace-query-result": { diff --git a/packages/platforms/ios-instruments/src/index.ts b/packages/platforms/ios-instruments/src/index.ts index ac04babf..f254cddd 100644 --- a/packages/platforms/ios-instruments/src/index.ts +++ b/packages/platforms/ios-instruments/src/index.ts @@ -84,7 +84,7 @@ export class IOSInstrumentsProfiler implements Profiler { // Do we need anything here? } - getScreenRecorder(videoPath: string): ScreenRecorder | undefined { + getScreenRecorder(): ScreenRecorder | undefined { return undefined; } diff --git a/packages/platforms/ios/src/index.ts b/packages/platforms/ios/src/index.ts index 0a2efd16..bc700712 100644 --- a/packages/platforms/ios/src/index.ts +++ b/packages/platforms/ios/src/index.ts @@ -105,7 +105,7 @@ export class IOSProfiler implements Profiler { return new Promise((resolve) => resolve()); } - async waitUntilReady(bundleId: string): Promise { + async waitUntilReady(): Promise { return new Promise((resolve) => resolve()); } From 711101a4368ae124600473752f9199eaed9986bb Mon Sep 17 00:00:00 2001 From: almouro Date: Mon, 13 May 2024 10:44:41 +0200 Subject: [PATCH 11/11] wip --- packages/commands/test/src/PerformanceMeasurer.ts | 10 +++++----- packages/platforms/ios-instruments/README.md | 6 ++---- packages/platforms/ios-instruments/src/index.ts | 4 ++-- 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/packages/commands/test/src/PerformanceMeasurer.ts b/packages/commands/test/src/PerformanceMeasurer.ts index d3184049..4f2c6121 100644 --- a/packages/commands/test/src/PerformanceMeasurer.ts +++ b/packages/commands/test/src/PerformanceMeasurer.ts @@ -55,8 +55,11 @@ export class PerformanceMeasurer { }); } - forceStop() { + async forceStop() { + // Ensure polling has stopped this.polling?.stop(); + // Hack for ios-instruments to get the measures at the end of the test + await profiler.getMeasures(); } async stop(duration?: number): Promise { @@ -77,10 +80,7 @@ export class PerformanceMeasurer { await new Promise((resolve) => setTimeout(resolve, POLLING_INTERVAL * 2)); } - // Ensure polling has stopped - this.polling?.stop(); - // Hack for ios-instruments to get the measures at the end of the test - await profiler.getMeasures(); + await this.forceStop(); await this.maybeStopRecording(); diff --git a/packages/platforms/ios-instruments/README.md b/packages/platforms/ios-instruments/README.md index 42e169c2..1bb5f85c 100644 --- a/packages/platforms/ios-instruments/README.md +++ b/packages/platforms/ios-instruments/README.md @@ -4,13 +4,11 @@ Requirements: - `maestro` installed - `node` installed +- `idb` installed ## Steps -- Get a running simulator id with `xcrun simctl list devices` -- Create template Flashlight in Xcode Instruments (with cpu-profile and memory usage) -- Add your own test in `test.yaml` -- `PLATFORM=ios-instruments node packages/commands/test/dist/bin.js test --bundleId --simulatorId 9F852910-03AD-495A-8E16-7356B764284 --testCommand "maestro test test.yaml" --resultsFilePath "./result.json"` +PLATFORM=ios-instruments node packages/commands/test/dist/bin.js test --bundleId --testCommand "maestro test test.yaml" --resultsFilePath "./result.json"` - Check the results in the web-reporter `yarn workspace @perf-profiler/web-reporter build` diff --git a/packages/platforms/ios-instruments/src/index.ts b/packages/platforms/ios-instruments/src/index.ts index f254cddd..27aa9af0 100644 --- a/packages/platforms/ios-instruments/src/index.ts +++ b/packages/platforms/ios-instruments/src/index.ts @@ -15,7 +15,7 @@ const startRecord = async ( ): Promise => { const templateFilePath = `${__dirname}/../Flashlight.tracetemplate`; const recordingProcess = executeAsync( - `xcrun xctrace record --device ${deviceUdid} --template ${templateFilePath} --attach ${appPid} --output ${traceFile}` + `arch -arm64 xcrun xctrace record --device ${deviceUdid} --template ${templateFilePath} --attach ${appPid} --output ${traceFile}` ); await new Promise((resolve) => { recordingProcess.stdout?.on("data", (data) => { @@ -30,7 +30,7 @@ const startRecord = async ( const saveTraceFile = (traceFile: string): string => { const xmlOutputFile = getTmpFilePath("report.xml"); executeCommand( - `xctrace export --input ${traceFile} --xpath '/trace-toc/run[@number="1"]/data/table[@schema="time-profile"]' --output ${xmlOutputFile}` + `arch -arm64 xctrace export --input ${traceFile} --xpath '/trace-toc/run[@number="1"]/data/table[@schema="time-profile"]' --output ${xmlOutputFile}` ); return xmlOutputFile; };