test: add efps to CI (#7556)

* test: revamp eFPS suite * chore: add efps github actions * chore: use different token for efps tests * test: add comment with perf report result * test: keep playwright install script * feat: add workflow dispatch inputs * fix: use `requestAnimationFrame` instead --------- Co-authored-by: Binoy Patel <[email protected]>
sanity-io · Oct 1, 2024 · aeb9e71 · aeb9e71
1 parent 30c046b
commit aeb9e71
Show file tree

Hide file tree

Showing 12 changed files with 624 additions and 161 deletions.
diff --git a/.github/workflows/efps.yml b/.github/workflows/efps.yml
@@ -0,0 +1,95 @@
+name: eFPS Test
+on:
+  pull_request:
+  workflow_dispatch:
+    inputs:
+      reference_tag:
+        description: "Reference tag for comparison"
+        required: true
+        default: "latest"
+      enable_profiler:
+        description: "Enable profiler"
+        required: true
+        type: boolean
+        default: false
+
+jobs:
+  install:
+    timeout-minutes: 30
+    runs-on: ubuntu-latest
+    env:
+      TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }}
+      TURBO_TEAM: ${{ vars.TURBO_TEAM }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 18
+
+      - uses: pnpm/action-setup@v4
+        name: Install pnpm
+        id: pnpm-install
+        with:
+          run_install: false
+
+      - name: Get pnpm store directory
+        id: pnpm-cache
+        shell: bash
+        run: |
+          echo "STORE_PATH=$(pnpm store path)" >> $GITHUB_OUTPUT
+
+      - name: Cache node modules
+        id: cache-node-modules
+        uses: actions/cache@v4
+        env:
+          cache-name: cache-node-modules
+        with:
+          path: ${{ steps.pnpm-cache.outputs.STORE_PATH }}
+          key: ${{ runner.os }}-pnpm-store-${{ env.cache-name }}-${{ hashFiles('**/pnpm-lock.yaml') }}
+          restore-keys: |
+            v1-${{ runner.os }}-pnpm-store-${{ env.cache-name }}-
+            v1-${{ runner.os }}-pnpm-store-
+            v1-${{ runner.os }}-
+
+      - name: Install project dependencies
+        run: pnpm install
+
+      - name: Store Playwright's Version
+        run: |
+          PLAYWRIGHT_VERSION=$(npx playwright --version | sed 's/Version //')
+          echo "Playwright's Version: $PLAYWRIGHT_VERSION"
+          echo "PLAYWRIGHT_VERSION=$PLAYWRIGHT_VERSION" >> $GITHUB_ENV
+
+      - name: Cache Playwright Browsers for Playwright's Version
+        id: cache-playwright-browsers
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/ms-playwright
+          key: playwright-browsers-${{ env.PLAYWRIGHT_VERSION }}
+
+      - name: Install Playwright Browsers
+        if: steps.cache-playwright-browsers.outputs.cache-hit != 'true'
+        run: npx playwright install --with-deps
+
+      - name: Run eFPS tests
+        env:
+          VITE_PERF_EFPS_PROJECT_ID: ${{ secrets.PERF_EFPS_PROJECT_ID }}
+          VITE_PERF_EFPS_DATASET: ${{ secrets.PERF_EFPS_DATASET }}
+          PERF_EFPS_SANITY_TOKEN: ${{ secrets.PERF_EFPS_SANITY_TOKEN }}
+          REFERENCE_TAG: ${{ github.event.inputs.reference_tag || 'latest' }}
+          ENABLE_PROFILER: ${{ github.event.inputs.enable_profiler || false }}
+        run: pnpm efps:test
+
+      - name: PR comment with report
+        uses: thollander/actions-comment-pull-request@fabd468d3a1a0b97feee5f6b9e499eab0dd903f6 # v2
+        if: ${{ github.event_name == 'pull_request' }}
+        with:
+          comment_tag: "efps-report"
+          filePath: ${{ github.workspace }}/perf/efps/results/benchmark-results.md
+
+      - uses: actions/upload-artifact@v3
+        if: always()
+        with:
+          name: efps-report
+          path: perf/efps/results
+          retention-days: 30
diff --git a/package.json b/package.json
@@ -54,6 +54,7 @@
     "e2e:setup": "node -r dotenv-flow/config -r esbuild-register scripts/e2e/setup",
     "e2e:start": "pnpm --filter studio-e2e-testing preview",
     "etl": "node -r dotenv-flow/config -r esbuild-register scripts/etl",
+    "efps:test": "cd perf/efps && pnpm test",
     "example:blog-studio": "cd examples/blog-studio && pnpm start",
     "example:clean-studio": "cd examples/blog-studio && pnpm start",
     "example:ecommerce-studio": "cd examples/blog-studio && pnpm start",

diff --git a/perf/efps/helpers/calculatePercentile.ts → perf/efps/helpers/aggregateLatencies.ts b/perf/efps/helpers/calculatePercentile.ts → perf/efps/helpers/aggregateLatencies.ts
@@ -1,4 +1,6 @@
-export function calculatePercentile(numbers: number[], percentile: number): number {
+import {type EfpsResult} from '../types'
+
+function calculatePercentile(numbers: number[], percentile: number): number {
   // Sort the array in ascending order
   const sorted = numbers.slice().sort((a, b) => a - b)
 
@@ -19,3 +21,12 @@ export function calculatePercentile(numbers: number[], percentile: number): numb
   const fraction = index - lowerIndex
   return lowerValue + (upperValue - lowerValue) * fraction
 }
+
+export function aggregateLatencies(values: number[]): EfpsResult['latency'] {
+  return {
+    p50: calculatePercentile(values, 0.5),
+    p75: calculatePercentile(values, 0.75),
+    p90: calculatePercentile(values, 0.9),
+    p99: calculatePercentile(values, 0.99),
+  }
+}
diff --git a/perf/efps/helpers/measureBlockingTime.ts b/perf/efps/helpers/measureBlockingTime.ts
@@ -0,0 +1,47 @@
+import {type Page} from 'playwright'
+
+const BLOCKING_TASK_THRESHOLD = 50
+
+export function measureBlockingTime(page: Page): () => Promise<number> {
+  const idleGapLengthsPromise = page.evaluate(async () => {
+    const idleGapLengths: number[] = []
+    const done = false
+    let last = performance.now()
+
+    const handler = () => {
+      const current = performance.now()
+      idleGapLengths.push(current - last)
+      last = current
+
+      if (done) return
+      requestAnimationFrame(handler)
+    }
+
+    requestAnimationFrame(handler)
+
+    await new Promise((resolve) => {
+      document.addEventListener('__blockingTimeFinish', resolve, {once: true})
+    })
+
+    return idleGapLengths
+  })
+
+  async function getBlockingTime() {
+    await page.evaluate(() => {
+      document.dispatchEvent(new CustomEvent('__blockingTimeFinish'))
+    })
+
+    const idleGapLengths = await idleGapLengthsPromise
+
+    const blockingTime = idleGapLengths
+      // only consider the gap lengths that are blocking
+      .filter((idleGapLength) => idleGapLength > BLOCKING_TASK_THRESHOLD)
+      // subtract the allowed time so we're only left with blocking time
+      .map((idleGapLength) => idleGapLength - BLOCKING_TASK_THRESHOLD)
+      .reduce((sum, next) => sum + next, 0)
+
+    return blockingTime
+  }
+
+  return getBlockingTime
+}
diff --git a/perf/efps/helpers/measureFpsForInput.ts b/perf/efps/helpers/measureFpsForInput.ts
@@ -1,9 +1,28 @@
-import {type Locator} from 'playwright'
+import {type Page} from 'playwright'
 
 import {type EfpsResult} from '../types'
-import {calculatePercentile} from './calculatePercentile'
+import {aggregateLatencies} from './aggregateLatencies'
+import {measureBlockingTime} from './measureBlockingTime'
 
-export async function measureFpsForInput(input: Locator): Promise<EfpsResult> {
+interface MeasureFpsForInputOptions {
+  label?: string
+  page: Page
+  fieldName: string
+}
+
+export async function measureFpsForInput({
+  label,
+  fieldName,
+  page,
+}: MeasureFpsForInputOptions): Promise<EfpsResult> {
+  const start = Date.now()
+
+  const input = page
+    .locator(
+      `[data-testid="field-${fieldName}"] input[type="text"], ` +
+        `[data-testid="field-${fieldName}"] textarea`,
+    )
+    .first()
   await input.waitFor({state: 'visible'})
   const characters = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'
 
@@ -49,6 +68,8 @@ export async function measureFpsForInput(input: Locator): Promise<EfpsResult> {
   await input.pressSequentially(startingMarker)
   await new Promise((resolve) => setTimeout(resolve, 500))
 
+  const getBlockingTime = measureBlockingTime(page)
+
   for (const character of characters) {
     inputEvents.push({character, timestamp: Date.now()})
     await input.press(character)
@@ -57,6 +78,9 @@ export async function measureFpsForInput(input: Locator): Promise<EfpsResult> {
 
   await input.blur()
 
+  await page.evaluate(() => window.document.dispatchEvent(new CustomEvent('__finish')))
+
+  const blockingTime = await getBlockingTime()
   const renderEvents = await rendersPromise
 
   await new Promise((resolve) => setTimeout(resolve, 500))
@@ -74,9 +98,10 @@ export async function measureFpsForInput(input: Locator): Promise<EfpsResult> {
     return matchingEvent.timestamp - inputEvent.timestamp
   })
 
-  const p50 = 1000 / calculatePercentile(latencies, 0.5)
-  const p75 = 1000 / calculatePercentile(latencies, 0.75)
-  const p90 = 1000 / calculatePercentile(latencies, 0.9)
-
-  return {p50, p75, p90, latencies}
+  return {
+    latency: aggregateLatencies(latencies),
+    blockingTime,
+    label: label || fieldName,
+    runDuration: Date.now() - start,
+  }
 }
diff --git a/perf/efps/helpers/measureFpsForPte.ts b/perf/efps/helpers/measureFpsForPte.ts
@@ -1,9 +1,22 @@
-import {type Locator} from 'playwright'
+import {type Page} from 'playwright'
 
 import {type EfpsResult} from '../types'
-import {calculatePercentile} from './calculatePercentile'
+import {aggregateLatencies} from './aggregateLatencies'
+import {measureBlockingTime} from './measureBlockingTime'
 
-export async function measureFpsForPte(pteField: Locator): Promise<EfpsResult> {
+interface MeasureFpsForPteOptions {
+  fieldName: string
+  label?: string
+  page: Page
+}
+
+export async function measureFpsForPte({
+  fieldName,
+  page,
+  label,
+}: MeasureFpsForPteOptions): Promise<EfpsResult> {
+  const start = Date.now()
+  const pteField = page.locator(`[data-testid="field-${fieldName}"]`)
   const characters = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'
 
   await pteField.waitFor({state: 'visible'})
@@ -24,14 +37,14 @@ export async function measureFpsForPte(pteField: Locator): Promise<EfpsResult> {
     }[] = []
 
     const mutationObserver = new MutationObserver(() => {
-      const start = performance.now()
+      const textStart = performance.now()
       const textContent = el.textContent || ''
-      const end = performance.now()
+      const textEnd = performance.now()
 
       updates.push({
         value: textContent,
         timestamp: Date.now(),
-        textContentProcessingTime: end - start,
+        textContentProcessingTime: textEnd - textStart,
       })
     })
 
@@ -63,6 +76,7 @@ export async function measureFpsForPte(pteField: Locator): Promise<EfpsResult> {
   await contentEditable.pressSequentially(startingMarker)
   await new Promise((resolve) => setTimeout(resolve, 500))
 
+  const getBlockingTime = measureBlockingTime(page)
   for (const character of characters) {
     inputEvents.push({character, timestamp: Date.now()})
     await contentEditable.press(character)
@@ -71,6 +85,7 @@ export async function measureFpsForPte(pteField: Locator): Promise<EfpsResult> {
 
   await contentEditable.blur()
 
+  const blockingTime = await getBlockingTime()
   const renderEvents = await rendersPromise
 
   const latencies = inputEvents.map((inputEvent) => {
@@ -86,9 +101,10 @@ export async function measureFpsForPte(pteField: Locator): Promise<EfpsResult> {
     return matchingEvent.timestamp - inputEvent.timestamp - matchingEvent.textContentProcessingTime
   })
 
-  const p50 = 1000 / calculatePercentile(latencies, 0.5)
-  const p75 = 1000 / calculatePercentile(latencies, 0.75)
-  const p90 = 1000 / calculatePercentile(latencies, 0.9)
-
-  return {p50, p75, p90, latencies}
+  return {
+    latency: aggregateLatencies(latencies),
+    blockingTime,
+    label: label || fieldName,
+    runDuration: Date.now() - start,
+  }
 }