Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

chore: slow operations on deep tests and improve stability #9979

Merged
merged 1 commit into from
Dec 14, 2024
Merged

chore: slow operations on deep tests and improve stability #9979

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions projects/demo-playwright/tests/addon-mobile/mobile-calendar/input-date.pw.spec.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ test.describe('InputDate and mobile user agent', () => {
.click();

await page.waitForSelector('tui-mobile-calendar-dropdown', {state: 'visible'});
await page.waitForTimeout(300); // safari flaky

await expect(page).toHaveScreenshot('01-input-date-range-mobile-1.png');

Expand Down Expand Up @@ -49,6 +50,7 @@ test.describe('InputDate and mobile user agent', () => {
.click();

await page.waitForSelector('tui-mobile-calendar-dropdown', {state: 'visible'});
await page.waitForTimeout(300); // safari flaky

await expect(page).toHaveScreenshot('02-input-date-range-mobile-1.png');

Expand Down Expand Up @@ -84,6 +86,7 @@ test.describe('InputDate and mobile user agent', () => {
.click();

await page.waitForSelector('tui-mobile-calendar-dropdown', {state: 'visible'});
await page.waitForTimeout(300); // safari flaky

await expect(page).toHaveScreenshot('03-input-date-range-mobile-1.png');

Expand Down
8 changes: 8 additions & 0 deletions projects/demo-playwright/tests/core/dropdown/dropdown.pw.spec.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ test.describe('Dropdown', () => {

await example.scrollIntoViewIfNeeded();
await example.locator('button').click();
await page.waitForTimeout(300);

await expect(page).toHaveScreenshot('01-dropdown.png');
});
Expand All @@ -21,6 +22,7 @@ test.describe('Dropdown', () => {

await example.scrollIntoViewIfNeeded();
await example.locator('input').click();
await page.waitForTimeout(300);

await expect(page).toHaveScreenshot('02-dropdown.png');
});
Expand All @@ -31,6 +33,7 @@ test.describe('Dropdown', () => {

await example.scrollIntoViewIfNeeded();
await example.locator('input').click();
await page.waitForTimeout(300);

await expect(page).toHaveScreenshot('03-dropdown.png');
});
Expand All @@ -41,6 +44,7 @@ test.describe('Dropdown', () => {

await example.scrollIntoViewIfNeeded();
await example.locator('button').click();
await page.waitForTimeout(300);

await expect(page).toHaveScreenshot('04-dropdown.png');
});
Expand All @@ -51,6 +55,7 @@ test.describe('Dropdown', () => {

await example.scrollIntoViewIfNeeded();
await example.locator('button').click();
await page.waitForTimeout(300);

await expect(page).toHaveScreenshot('05-dropdown.png');
});
Expand All @@ -60,7 +65,9 @@ test.describe('Dropdown', () => {
const example = new TuiDocumentationPagePO(page).getExample('#portal');

await example.scrollIntoViewIfNeeded();
// eslint-disable-next-line playwright/no-force-option
await example.locator('.t2').click({force: true});
await page.waitForTimeout(300);

await expect(page).toHaveScreenshot('13-dropdown.png');
});
Expand Down Expand Up @@ -123,6 +130,7 @@ test.describe('Dropdown', () => {

await example.scrollIntoViewIfNeeded();
await example.locator('button').click();
await page.waitForTimeout(300);

expect(page.locator('tui-dropdown')).toBeDefined();

Expand Down
4 changes: 2 additions & 2 deletions projects/demo-playwright/tests/core/hint/hint.pw.spec.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -91,10 +91,10 @@ test.describe('TuiHint', () => {
`03-hint-mode-${mode}-tuiHintShowDelay-1000-wait-500.png`,
);

await page.waitForTimeout(500);
await page.waitForTimeout(1500);

await expect(page).toHaveScreenshot(
`03-hint-mode-${mode}-tuiHintShowDelay-1000-wait-1000.png`,
`03-hint-mode-${mode}-tuiHintShowDelay-1000-wait-more-1000.png`,
);
});
});
Expand Down
29 changes: 22 additions & 7 deletions projects/demo-playwright/tests/deep/deep-select.pw.spec.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
/* eslint-disable playwright/no-conditional-in-test */
import {DemoRoute} from '@demo/routes';
import {TuiDocumentationApiPagePO, tuiGoto, tuiMockImages} from '@demo-playwright/utils';
import {expect, test} from '@playwright/test';
Expand All @@ -10,7 +11,7 @@ test.describe('Deep / Select', () => {
);

deepPaths.forEach((path) =>
test(`${path}`, async ({page}) => {
test(`${path}`, async ({page, browserName}) => {
await tuiMockImages(page);
await tuiGoto(page, `${path}/API`);

Expand All @@ -25,7 +26,6 @@ test.describe('Deep / Select', () => {
const select = await api.getSelect(row);
const name = await api.getNameProperty(row);

// eslint-disable-next-line playwright/no-conditional-in-test
if (!select) {
continue;
}
Expand All @@ -37,7 +37,11 @@ test.describe('Deep / Select', () => {
await expect(select).toBeVisible();

await select.click();
await api.waitTuiIcons();

// note: hello Safari
if (browserName === 'webkit') {
await page.waitForTimeout(200);
}

const options = await api.getOptions();

Expand All @@ -47,19 +51,26 @@ test.describe('Deep / Select', () => {
await api.focusOnBody();
await api.hideNotifications();
await api.waitStableState();
await api.waitTuiIcons();

// note: hello Safari
if (browserName === 'webkit') {
await page.waitForTimeout(200);
}

await expect(api.apiPageExample).toHaveScreenshot(
`deep-${path}-${name}-row—${rowIndex}-select-option-${index}.png`,
);

await select.click();
await api.waitTuiIcons();

// note: hello Safari
if (browserName === 'webkit') {
await page.waitForTimeout(200);
}
}

const cleaner = await api.getCleaner(select);

// eslint-disable-next-line playwright/no-conditional-in-test
if (cleaner) {
await cleaner.click();
} else {
Expand All @@ -69,7 +80,11 @@ test.describe('Deep / Select', () => {

await api.waitStableState();
await api.focusOnBody();
await api.waitTuiIcons();

// note: hello Safari
if (browserName === 'webkit') {
await page.waitForTimeout(200);
}
}
}),
);
Expand Down
17 changes: 12 additions & 5 deletions projects/demo-playwright/tests/deep/deep-toggle.pw.spec.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,11 +1,12 @@
/* eslint-disable playwright/no-conditional-in-test */
import {TuiDocumentationApiPagePO, tuiGoto, tuiMockImages} from '@demo-playwright/utils';
import {expect, test} from '@playwright/test';

test.describe('Deep / Toggle', () => {
const deepPaths: string[] = JSON.parse(process.env['DEMO_PATHS']!);

deepPaths.forEach((path) =>
test(`${path}`, async ({page}) => {
test(`${path}`, async ({page, browserName}) => {
await tuiMockImages(page);
await tuiGoto(page, `${path}/API`);

Expand All @@ -20,7 +21,6 @@ test.describe('Deep / Toggle', () => {
const toggle = await api.getToggle(row);
const name = await api.getNameProperty(row);

// eslint-disable-next-line playwright/no-conditional-in-test
if (!toggle) {
continue;
}
Expand All @@ -31,17 +31,24 @@ test.describe('Deep / Toggle', () => {
await expect(toggle).toBeVisible();

await toggle.click();
await api.waitTuiIcons();
await api.hideNotifications();
await api.waitStableState();
await page.waitForTimeout(100);

// note: hello Safari
if (browserName === 'webkit') {
await page.waitForTimeout(300);
}

await expect(api.apiPageExample).toHaveScreenshot(
`deep-${path}-${name}-row-${rowIndex}-toggled.png`,
);

await toggle.click();
await api.waitTuiIcons();

// note: hello Safari
if (browserName === 'webkit') {
await page.waitForTimeout(200);
}
}
}),
);
Expand Down
4 changes: 2 additions & 2 deletions projects/demo-playwright/tests/demo/demo.pw.spec.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ test.describe('Demo', () => {
const demoPaths: string[] = JSON.parse(process.env['DEMO_PATHS']!);

demoPaths.forEach((path) => {
test(`${path}`, async ({page}) => {
test(`${path}`, async ({page, browserName}) => {
const documentation = new TuiDocumentationPagePO(page);

await tuiMockImages(page);
Expand All @@ -29,7 +29,7 @@ test.describe('Demo', () => {

for (const [i, example] of examples.entries()) {
// eslint-disable-next-line playwright/no-conditional-in-test
if (tuiIsFlakyExample(path, i)) {
if (tuiIsFlakyExample(path, i, browserName)) {
continue;
}

Expand Down
4 changes: 4 additions & 0 deletions projects/demo-playwright/tests/kit/dropdown-selection/dropdown-selection.pw.spec.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ test.describe('DropdownSelection', () => {
await example.locator('textarea').focus();
await page.keyboard.press('Control+ArrowLeft');
await api.waitStableState();
await page.waitForTimeout(300); // flaky in Safari

await expect(page).toHaveScreenshot('01-dropdown-selection.png');

Expand All @@ -24,17 +25,20 @@ test.describe('DropdownSelection', () => {
await page.keyboard.press('ArrowRight');
await page.keyboard.press('ArrowRight');
await api.waitStableState();
await page.waitForTimeout(300); // flaky in Safari

await expect(page).toHaveScreenshot('02-dropdown-selection.png');

await example.locator('textarea').fill('');
await example.locator('textarea').fill('@');
await api.waitStableState();
await page.waitForTimeout(300); // flaky in Safari

await expect(page).toHaveScreenshot('03-dropdown-selection.png');

await page.locator('button[tuiOption]').first().click();
await api.waitStableState();
await page.waitForTimeout(300); // flaky in Safari

await expect(page).toHaveScreenshot('04-dropdown-selection.png');
});
Expand Down
39 changes: 27 additions & 12 deletions projects/demo-playwright/utils/is-flaky-examples.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,26 +1,41 @@
import {DemoRoute} from '@demo/routes';

const FLAKY_EXAMPLES = new Map<string, number[]>([
const FLAKY_EXAMPLES = new Map<
string,
Array<{exampleIndex: number; browserName?: string}>
>([
[DemoRoute.AppBar, [{exampleIndex: 0, browserName: 'webkit'}]], // Flaky in safari, need to investigate a problem
[DemoRoute.Carousel, [{exampleIndex: 2, browserName: 'webkit'}]], // Flaky in safari, need to investigate a problem
[
DemoRoute.Carousel,
[
0, // [duration]="4000"
3, // just button (to open dialog)
{exampleIndex: 0}, // [duration]="4000"
{exampleIndex: 3}, // just button (to open dialog)
],
],
[DemoRoute.IconsCustomization, [0]], // TODO: investigate flaky test
[DemoRoute.MultiSelect, [3]], // Imitating server response (timer(5000))
[DemoRoute.Select, [4]], // Imitating server response (delay(3000))
[DemoRoute.Stepper, [2]], // TODO: flaky test for proprietary demo (autoscroll problems)
[DemoRoute.TabBar, [3]], // Imitating server response (timer(3000))
[DemoRoute.Table, [3, 4]], // Imitating server response (delay(3000)) and virtual scroll
[DemoRoute.Tiles, [0]], // YouTube iframe player
[DemoRoute.IconsCustomization, [{exampleIndex: 0}]], // TODO: investigate flaky test
[DemoRoute.LegendItem, [{exampleIndex: 0, browserName: 'webkit'}]], // Flaky in safari, need to investigate a problem
[DemoRoute.MultiSelect, [{exampleIndex: 3}]], // Imitating server response (timer(5000))
[DemoRoute.RingChart, [{exampleIndex: 0, browserName: 'webkit'}]], // Flaky in safari, need to investigate a problem
[DemoRoute.Select, [{exampleIndex: 4}]], // Imitating server response (delay(3000))
[DemoRoute.Stepper, [{exampleIndex: 2}]], // TODO: flaky test for proprietary demo (autoscroll problems)
[DemoRoute.TabBar, [{exampleIndex: 3}]], // Imitating server response (timer(3000))
[DemoRoute.Table, [{exampleIndex: 3}, {exampleIndex: 4}]], // Imitating server response (delay(3000)) and virtual scroll
[DemoRoute.Tiles, [{exampleIndex: 0}]], // YouTube iframe player
]);

export function tuiIsFlakyExample(path: string, exampleIndex: number): boolean {
export function tuiIsFlakyExample(
path: string,
exampleIndex: number,
browserName: string,
): boolean {
const exclusions = FLAKY_EXAMPLES.get(path) ?? [];

const excluded = !!exclusions?.includes(exampleIndex);
const excluded = !!exclusions.find(
(exclusion) =>
exclusion.exampleIndex === exampleIndex &&
(exclusion.browserName ? exclusion.browserName === browserName : true),
);

if (excluded) {
console.info(`skip test for: ${path}[${exampleIndex}]`);
Expand Down
Loading