feat: add Cloudflare blocking benchmarking (#114)

package.json

@@ -53,7 +53,9 @@
         "prepublishOnly": "npm run build",
         "lint": "eslint src test",
         "lint:fix": "eslint src test --fix",
-        "test": "node --experimental-vm-modules ./node_modules/jest/bin/jest.js --coverage"
+        "test": "node --experimental-vm-modules ./node_modules/jest/bin/jest.js --coverage",
+        "pretest:blocking": "npm run build",
+        "test:blocking": "ts-node -T ./test/live-testing/index.js"
     },
     "author": {
         "name": "Apify",

test/live-testing/index.js

@@ -0,0 +1,74 @@
+import { setTimeout } from 'timers/promises';
+import got from 'got';
+import { gotScraping } from '../../dist/index.js';
+
+async function processUrls(gotImplementation, urls) {
+    let passed = 0;
+    let blocked = 0;
+    let failed = 0;
+
+    let url = urls.shift();
+    while (url) {
+        try {
+            // console.log(`crawling ${url}`);
+            const request = gotImplementation.get(url);
+
+            const result = await Promise.race([
+                request,
+                setTimeout(5000),
+            ]);
+
+            if (!result?.body) {
+                request.cancel();
+                throw new Error('timeout');
+            }
+
+            // console.log(`crawled ${url}`);
+
+            if (result.body.includes('Just a moment...')) {
+                blocked++;
+            } else {
+                passed++;
+            }
+        } catch (e) {
+            failed++;
+            // console.error(e.message);
+        }
+
+        url = urls.shift();
+    }
+
+    // console.log('done!');
+    return { passed, blocked, failed };
+}
+
+async function runInParallel(implementation, urls) {
+    const localUrls = [...urls];
+    const partialResults = await Promise.all(Array.from({ length: 5 }, () => processUrls(implementation, localUrls)));
+
+    return partialResults.reduce((acc, { passed, blocked, failed }) => {
+        acc.passed += passed;
+        acc.blocked += blocked;
+        acc.failed += failed;
+        return acc;
+    }, { passed: 0, blocked: 0, failed: 0 });
+}
+
+(async () => {
+    const { body } = await got.get('https://raw.githubusercontent.com/apify/fingerprint-suite/master/test/antibot-services/live-testing/cloudflare-websites.csv');
+    const urls = body.split('\n');
+
+    const [gotScrapingResults, gotResults] = await Promise.all([
+        runInParallel(gotScraping, urls),
+        runInParallel(got, urls),
+    ]);
+
+    console.log('got-scraping');
+    console.log(gotScrapingResults);
+
+    console.log('---');
+    console.log('got');
+    console.log(gotResults);
+
+    process.exit(0);
+})();