From b9b0dd29878b344b62995f6fe087b1718c01da43 Mon Sep 17 00:00:00 2001 From: d3or Date: Sat, 7 Dec 2024 23:12:00 -0500 Subject: [PATCH 1/2] fix: use correct ProxyAgent for global fetch --- src/test-utils.ts | 31 ++++++++++++++++++++++++++----- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/src/test-utils.ts b/src/test-utils.ts index 071e83d..630f950 100644 --- a/src/test-utils.ts +++ b/src/test-utils.ts @@ -1,4 +1,4 @@ -import { HttpsProxyAgent } from 'https-proxy-agent'; +import { ProxyAgent } from 'undici'; import { Scraper } from './scraper'; import fs from 'fs'; @@ -72,16 +72,37 @@ export async function getScraper( } if (proxyUrl) { - agent = new HttpsProxyAgent(proxyUrl, { - rejectUnauthorized: false, - }); + // Parse the proxy URL + const url = new URL(proxyUrl); + const username = url.username; + const password = url.password; + + // Strip auth from URL if present + url.username = ''; + url.password = ''; + + const agentOptions: any = { + uri: url.toString(), + requestTls: { + rejectUnauthorized: false, + }, + }; + + // Add Basic auth if credentials exist + if (username && password) { + agentOptions.token = `Basic ${Buffer.from( + `${username}:${password}`, + ).toString('base64')}`; + } + + agent = new ProxyAgent(agentOptions); } const scraper = new Scraper({ transform: { request: (input, init) => { if (agent) { - return [input, { ...init, agent }]; + return [input, { ...init, dispatcher: agent }]; } return [input, init]; }, From f3bef5c55a97eb8c964d46032534d708cb7119d0 Mon Sep 17 00:00:00 2001 From: d3or Date: Sat, 7 Dec 2024 23:21:19 -0500 Subject: [PATCH 2/2] fix: remove https-proxy-agent --- package-lock.json | 121 +++++++++++++++++++++++++++++++++++----------- package.json | 1 - 2 files changed, 92 insertions(+), 30 deletions(-) diff --git a/package-lock.json b/package-lock.json index b1b6552..42d8dc7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,6 +12,7 @@ "@sinclair/typebox": "^0.32.20", "headers-polyfill": "^3.1.2", "json-stable-stringify": "^1.0.2", + "node-fetch": "^3.3.2", "otpauth": "^9.2.2", "set-cookie-parser": "^2.6.0", "tough-cookie": "^4.1.2", @@ -24,7 +25,7 @@ "@tsconfig/node16": "^16.1.3", "@types/jest": "^29.5.1", "@types/json-stable-stringify": "^1.0.34", - "@types/node": "^22.9.0", + "@types/node": "^22.9.1", "@types/set-cookie-parser": "^2.4.2", "@types/tough-cookie": "^4.0.2", "@typescript-eslint/eslint-plugin": "^5.59.7", @@ -35,7 +36,6 @@ "eslint-config-prettier": "^8.8.0", "eslint-plugin-prettier": "^4.2.1", "gh-pages": "^5.0.0", - "https-proxy-agent": "^7.0.2", "jest": "^29.7.0", "lint-staged": "^13.2.2", "prettier": "^2.8.8", @@ -2736,19 +2736,6 @@ "node": ">=0.4.0" } }, - "node_modules/agent-base": { - "version": "7.1.1", - "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.1.tgz", - "integrity": "sha512-H0TSyFNDMomMNJQBn8wFV5YC/2eJ+VXECwOadZJT554xP6cODZHPX3H9QMQECxvrgiSOP1pHjy1sMWQVYJOUOA==", - "dev": true, - "license": "MIT", - "dependencies": { - "debug": "^4.3.4" - }, - "engines": { - "node": ">= 14" - } - }, "node_modules/ajv": { "version": "8.17.1", "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.17.1.tgz", @@ -3572,6 +3559,15 @@ "node": ">=8" } }, + "node_modules/data-uri-to-buffer": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz", + "integrity": "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==", + "license": "MIT", + "engines": { + "node": ">= 12" + } + }, "node_modules/debug": { "version": "4.3.7", "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.7.tgz", @@ -4332,6 +4328,29 @@ "bser": "2.1.1" } }, + "node_modules/fetch-blob": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz", + "integrity": "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "paypal", + "url": "https://paypal.me/jimmywarting" + } + ], + "license": "MIT", + "dependencies": { + "node-domexception": "^1.0.0", + "web-streams-polyfill": "^3.0.3" + }, + "engines": { + "node": "^12.20 || >= 14.13" + } + }, "node_modules/file-entry-cache": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-6.0.1.tgz", @@ -4523,6 +4542,18 @@ "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/formdata-polyfill": { + "version": "4.0.10", + "resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz", + "integrity": "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==", + "license": "MIT", + "dependencies": { + "fetch-blob": "^3.1.2" + }, + "engines": { + "node": ">=12.20.0" + } + }, "node_modules/fs-extra": { "version": "11.2.0", "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.2.0.tgz", @@ -4977,20 +5008,6 @@ "dev": true, "license": "MIT" }, - "node_modules/https-proxy-agent": { - "version": "7.0.5", - "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.5.tgz", - "integrity": "sha512-1e4Wqeblerz+tMKPIq2EMGiiWW1dIjZOksyHWSUm1rmuvw/how9hBHZ38lAGj5ID4Ik6EdkOw7NmWPy6LAwalw==", - "dev": true, - "license": "MIT", - "dependencies": { - "agent-base": "^7.0.2", - "debug": "4" - }, - "engines": { - "node": ">= 14" - } - }, "node_modules/human-signals": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/human-signals/-/human-signals-2.1.0.tgz", @@ -6843,6 +6860,43 @@ "dev": true, "license": "MIT" }, + "node_modules/node-domexception": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", + "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "github", + "url": "https://paypal.me/jimmywarting" + } + ], + "license": "MIT", + "engines": { + "node": ">=10.5.0" + } + }, + "node_modules/node-fetch": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz", + "integrity": "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==", + "license": "MIT", + "dependencies": { + "data-uri-to-buffer": "^4.0.0", + "fetch-blob": "^3.1.4", + "formdata-polyfill": "^4.0.10" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/node-fetch" + } + }, "node_modules/node-int64": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/node-int64/-/node-int64-0.4.0.tgz", @@ -8917,6 +8971,15 @@ "makeerror": "1.0.12" } }, + "node_modules/web-streams-polyfill": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.3.3.tgz", + "integrity": "sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==", + "license": "MIT", + "engines": { + "node": ">= 8" + } + }, "node_modules/which": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", diff --git a/package.json b/package.json index 6141653..33f6595 100644 --- a/package.json +++ b/package.json @@ -53,7 +53,6 @@ "eslint-config-prettier": "^8.8.0", "eslint-plugin-prettier": "^4.2.1", "gh-pages": "^5.0.0", - "https-proxy-agent": "^7.0.2", "jest": "^29.7.0", "lint-staged": "^13.2.2", "prettier": "^2.8.8",