Skip to content

Commit

Permalink
feat: wait page load
Browse files Browse the repository at this point in the history
  • Loading branch information
KuznetsovRoman committed Jul 26, 2023
1 parent e0e698f commit b3945ba
Show file tree
Hide file tree
Showing 10 changed files with 1,974 additions and 1,850 deletions.
2,923 changes: 1,077 additions & 1,846 deletions package-lock.json

Large diffs are not rendered by default.

8 changes: 5 additions & 3 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -45,8 +45,8 @@
"dependencies": {
"@gemini-testing/commander": "2.15.3",
"@types/mocha": "^10.0.1",
"@wdio/globals": "^8.5.7",
"@wdio/types": "^8.4.0",
"@wdio/globals": "^8.10.7",
"@wdio/types": "^8.10.4",
"@wdio/utils": "^7.26.0",
"aliasify": "^1.9.0",
"bluebird": "^3.5.1",
Expand All @@ -73,13 +73,14 @@
"uglifyify": "^3.0.4",
"urijs": "^1.19.11",
"url-join": "^4.0.1",
"webdriverio": "^8.8.2",
"webdriverio": "^8.10.7",
"worker-farm": "^1.7.0",
"yallist": "^3.1.1"
},
"devDependencies": {
"@commitlint/cli": "^17.1.2",
"@commitlint/config-conventional": "^17.1.0",
"@sinonjs/fake-timers": "^10.3.0",
"@swc/core": "^1.3.40",
"@types/bluebird": "^3.5.38",
"@types/chai": "^4.3.4",
Expand All @@ -89,6 +90,7 @@
"@types/proxyquire": "^1.3.28",
"@types/sharp": "^0.31.1",
"@types/sinon": "^4.3.3",
"@types/sinonjs__fake-timers": "^8.1.2",
"@typescript-eslint/eslint-plugin": "^5.54.1",
"@typescript-eslint/parser": "^5.54.1",
"app-module-path": "^2.2.0",
Expand Down
2 changes: 1 addition & 1 deletion src/browser/commands/index.js
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
"use strict";

module.exports = ["assert-view", "getConfig", "getPuppeteer", "setOrientation", "scrollIntoView"];
module.exports = ["assert-view", "getConfig", "getPuppeteer", "setOrientation", "scrollIntoView", "openAndWait"];
139 changes: 139 additions & 0 deletions src/browser/commands/openAndWait.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,139 @@
import _ from "lodash";
import { Matches } from "webdriverio";
import PageLoader from "../../utils/page-loader";

interface Browser {
publicAPI: WebdriverIO.Browser;
config: {
desiredCapabilities: {
browserName: string;
};
automationProtocol: "webdriver" | "devtools";
pageLoadTimeout: number;
openAndWaitOpts: {
timeout?: number;
waitNetworkIdle: boolean;
waitNetworkIdleTimeout: number;
rejectOnNetworkError: boolean;
ignoreNetworkErrorsPatterns: Array<RegExp | string>;
};
};
}

interface WaitOpts {
selector?: string | string[];
predicate?: () => boolean;
waitNetworkIdle?: boolean;
waitNetworkIdleTimeout?: number;
rejectOnNetworkError?: boolean;
shouldThrowError?: (match: Matches) => boolean;
ignoreNetworkErrorsPatterns?: Array<RegExp | string>;
timeout?: number;
}

const emptyPageUrl = "about:blank";

const is: Record<string, (match: Matches) => boolean> = {
image: match => match.headers?.Accept?.includes("image"),
stylesheet: match => match.headers?.Accept?.includes("text/css"),
font: match => _.isString(match.url) && [".ttf", ".woff", ".woff2"].some(ext => match.url.endsWith(ext)),
favicon: match => _.isString(match.url) && match.url.endsWith("/favicon.ico"),
};

export = (browser: Browser): void => {
const { publicAPI: session, config } = browser;
const { openAndWaitOpts } = config;
const isChrome = config.desiredCapabilities.browserName === "chrome";
const isCDP = config.automationProtocol === "devtools";

function openAndWait(
uri: string,
{
selector = [],
predicate,
waitNetworkIdle = openAndWaitOpts?.waitNetworkIdle,
waitNetworkIdleTimeout = openAndWaitOpts?.waitNetworkIdleTimeout,
rejectOnNetworkError = openAndWaitOpts?.rejectOnNetworkError,
shouldThrowError = shouldThrowErrorDefault,
ignoreNetworkErrorsPatterns = openAndWaitOpts?.ignoreNetworkErrorsPatterns,
timeout = openAndWaitOpts?.timeout || config?.pageLoadTimeout,
}: WaitOpts = {},
): Promise<string | void> {
waitNetworkIdle &&= isChrome || isCDP;

if (!uri || uri === emptyPageUrl) {
return session.url(uri);
}

const selectors = typeof selector === "string" ? [selector] : selector;

const pageLoader = new PageLoader(session, {
selectors,
predicate,
timeout,
waitNetworkIdle,
waitNetworkIdleTimeout,
});

let selectorsResolved = !selectors.length;
let predicateResolved = !predicate;
let networkResolved = !waitNetworkIdle;

return new Promise<void>((resolve, reject) => {
const handleError = (err: Error): void => {
reject(new Error(`url: ${err.message}`));
};

const checkLoaded = (): void => {
if (selectorsResolved && predicateResolved && networkResolved) {
resolve();
}
};

pageLoader.on("pageLoadError", handleError);
pageLoader.on("selectorsError", handleError);
pageLoader.on("predicateError", handleError);
pageLoader.on("networkError", match => {
if (!rejectOnNetworkError) {
return;
}

const shouldIgnore = isMatchPatterns(ignoreNetworkErrorsPatterns, match.url);

if (!shouldIgnore && shouldThrowError(match)) {
reject(new Error(`url: couldn't get content from ${match.url}: ${match.statusCode}`));
}
});
pageLoader.on("selectorsExist", () => {
selectorsResolved = true;
checkLoaded();
});

pageLoader.on("predicateResolved", () => {
predicateResolved = true;
checkLoaded();
});

pageLoader.on("networkResolved", () => {
networkResolved = true;
checkLoaded();
});

pageLoader.load(() => session.url(uri)).then(checkLoaded);
}).finally(() => pageLoader.unsubscribe());
}

session.addCommand("openAndWait", openAndWait);
};

function isMatchPatterns(patterns: Array<RegExp | string> = [], str: string): boolean {
return patterns.some(pattern => (_.isString(pattern) ? str.includes(pattern) : pattern.exec(str)));
}

function shouldThrowErrorDefault(match: Matches): boolean {
if (is.favicon(match)) {
return false;
}

return is.image(match) || is.stylesheet(match) || is.font(match);
}
10 changes: 10 additions & 0 deletions src/config/browser-options.js
Original file line number Diff line number Diff line change
Expand Up @@ -240,6 +240,16 @@ function buildBrowserOptions(defaultFactory, extra) {
},
}),

openAndWaitOpts: option({
defaultValue: defaultFactory("openAndWaitOpts"),
parseEnv: JSON.parse,
parseCli: JSON.parse,
validate: value => utils.assertOptionalObject(value, "openAndWaitOpts"),
map: value => {
return value === defaults.openAndWaitOpts ? value : { ...defaults.openAndWaitOpts, ...value };
},
}),

meta: options.optionalObject("meta"),

windowSize: option({
Expand Down
6 changes: 6 additions & 0 deletions src/config/defaults.js
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,12 @@ module.exports = {
captureElementFromTop: true,
allowViewportOverflow: false,
},
openAndWaitOpts: {
waitNetworkIdle: true,
waitNetworkIdleTimeout: 500,
rejectOnNetworkError: true,
ignoreNetworkErrorsPatterns: [],
},
calibrate: false,
screenshotMode: "auto",
screenshotDelay: 0,
Expand Down
157 changes: 157 additions & 0 deletions src/utils/page-loader.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,157 @@
import EventEmitter from "events";
import type { Matches, Mock } from "webdriverio";
import logger from "./logger";

export interface PageLoaderOpts {
selectors: string[];
predicate?: () => boolean | Promise<boolean>;
timeout: number;
waitNetworkIdle: boolean;
waitNetworkIdleTimeout: number;
}

export default class PageLoader extends EventEmitter {
private session: WebdriverIO.Browser;
private mock?: Mock | null;
private selectors: string[];
private predicate?: () => boolean | Promise<boolean>;
private timeout: number;
private waitNetworkIdle: boolean;
private waitNetworkIdleTimeout: number;
private totalRequests = 0;
private networkResolved = false;

constructor(
session: WebdriverIO.Browser,
{ selectors, predicate, timeout, waitNetworkIdle, waitNetworkIdleTimeout }: PageLoaderOpts,
) {
super();

this.session = session;
this.selectors = selectors;
this.predicate = predicate;
this.timeout = timeout;
this.waitNetworkIdle = waitNetworkIdle;
this.waitNetworkIdleTimeout = waitNetworkIdleTimeout;
}

public async load(goToPage: () => Promise<string | void>): Promise<void> {
await this.initMock();

await goToPage().catch(err => {
this.emit("pageLoadError", err);
});

this.launchSelectorsPromise();
this.launchPredicatePromise();
this.launchNetworkPromise();
}

public unsubscribe(): Promise<void> | undefined {
return this.mock?.restore().catch(() => {
logger.warn("PageLoader: Got err while unsubscribing");
});
}

private launchSelectorsPromise(): void {
const selectorPromises = this.selectors.map(async selector => {
const element = await this.session.$(selector);
await element.waitForExist({ timeout: this.timeout });
});

Promise.all(selectorPromises)
.then(() => {
this.emit("selectorsExist");
})
.catch(err => {
this.emit("selectorsError", err);
});
}

private launchPredicatePromise(): void {
if (!this.predicate) {
return;
}

this.session
.waitUntil(this.predicate, { timeout: this.timeout })
.then(() => {
this.emit("predicateResolved");
})
.catch(() => {
this.emit("predicateError", new Error(`predicate was never truthy in ${this.timeout}ms`));
});
}

private launchNetworkPromise(): void {
if (!this.waitNetworkIdle) {
return;
}

setTimeout(() => {
const markSuccess = this.markNetworkIdle();
if (markSuccess) {
logger.warn(`PageLoader: Network idle timeout`);
}
}, this.timeout);
setTimeout(() => {
if (!this.totalRequests) {
this.markNetworkIdle();
}
}, this.waitNetworkIdleTimeout);
}

private async initMock(): Promise<void> {
if (!this.waitNetworkIdle) {
return;
}

this.mock = await this.session.mock("**").catch(() => {
logger.warn(`PageLoader: Could not create CDP interceptor`);

return null;
});

if (!this.mock) {
this.markNetworkIdle();

return;
}

let pendingRequests = 0;
let pendingIdleTimeout: NodeJS.Timeout;
this.mock.on("request", () => {
this.totalRequests++;
pendingRequests++;
clearTimeout(pendingIdleTimeout);
});

this.mock.on("continue", () => {
pendingRequests--;

if (!pendingRequests) {
pendingIdleTimeout = setTimeout(() => this.markNetworkIdle(), this.waitNetworkIdleTimeout);
}
});

this.mock.on("match", (match: Matches) => {
if (this.isMatchError(match)) {
this.emit("networkError", match);
}
});
}

private isMatchError(match: Matches): boolean {
return match.statusCode >= 400 && match.statusCode < 600;
}

private markNetworkIdle(): boolean {
if (this.networkResolved) {
return false;
}

this.networkResolved = true;
this.emit("networkResolved");
return true;
}
}
Loading

0 comments on commit b3945ba

Please sign in to comment.