From a8a5dfec45ab2b5f818599d2c4462a967ab64e37 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jind=C5=99ich=20B=C3=A4r?= Date: Wed, 22 May 2024 13:32:13 +0200 Subject: [PATCH] feat: log CONNECT error response body (#139) Connected to https://github.com/apify/apify-proxy/issues/823. On an internal proxy error (such as trying to access an invalid or unresolvable URL - e.g. `https://example.comundefined`), the proxy returns a response with the error message. `got-scraping` only logs the length of the message, which is quite useless for most applications. --- package-lock.json | 214 ++++++++++++++++++++-------------------- package.json | 2 +- src/hooks/proxy.ts | 4 +- src/resolve-protocol.ts | 5 +- tsconfig.json | 1 + 5 files changed, 117 insertions(+), 109 deletions(-) diff --git a/package-lock.json b/package-lock.json index 8d87821..7b05d97 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,7 +9,7 @@ "version": "4.0.5", "license": "Apache-2.0", "dependencies": { - "got": "^13.0.0", + "got": "^14.2.1", "header-generator": "^2.1.41", "http2-wrapper": "^2.2.0", "mimic-response": "^4.0.0", @@ -1872,9 +1872,9 @@ } }, "node_modules/@types/http-cache-semantics": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/@types/http-cache-semantics/-/http-cache-semantics-4.0.2.tgz", - "integrity": "sha512-FD+nQWA2zJjh4L9+pFXqWOi0Hs1ryBCfI+985NjluQ1p8EYtoLvjLOKidXBtZ4/IcxDX4o8/E8qDS3540tNliw==" + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/@types/http-cache-semantics/-/http-cache-semantics-4.0.4.tgz", + "integrity": "sha512-1m0bIFVc7eJWyve9S0RnuRgcQqF/Xd5QsUZAZeQFr1Q3/p9JWoQQEqmVy+DPTNpGXwhgIetAoYF8JSc33q29QA==" }, "node_modules/@types/istanbul-lib-coverage": { "version": "2.0.4", @@ -2849,6 +2849,34 @@ "node": ">=8" } }, + "node_modules/cacheable-request": { + "version": "10.2.14", + "resolved": "https://registry.npmjs.org/cacheable-request/-/cacheable-request-10.2.14.tgz", + "integrity": "sha512-zkDT5WAF4hSSoUgyfg5tFIxz8XQK+25W/TLVojJTMKBaxevLBBtLxgqguAuVQB8PVW79FVjHcU+GJ9tVbDZ9mQ==", + "dependencies": { + "@types/http-cache-semantics": "^4.0.2", + "get-stream": "^6.0.1", + "http-cache-semantics": "^4.1.1", + "keyv": "^4.5.3", + "mimic-response": "^4.0.0", + "normalize-url": "^8.0.0", + "responselike": "^3.0.0" + }, + "engines": { + "node": ">=14.16" + } + }, + "node_modules/cacheable-request/node_modules/get-stream": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-6.0.1.tgz", + "integrity": "sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg==", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/call-bind": { "version": "1.0.2", "dev": true, @@ -4659,6 +4687,14 @@ "node": ">=0.10.0" } }, + "node_modules/form-data-encoder": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/form-data-encoder/-/form-data-encoder-4.0.2.tgz", + "integrity": "sha512-KQVhvhK8ZkWzxKxOr56CPulAhH3dobtuQ4+hNQ+HekH/Wp5gSOafqRAeTphQUJAIk0GBvHZgJ2ZGRWd5kphMuw==", + "engines": { + "node": ">= 18" + } + }, "node_modules/forwarded": { "version": "0.2.0", "dev": true, @@ -4803,7 +4839,6 @@ "version": "8.0.1", "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-8.0.1.tgz", "integrity": "sha512-VaUJspBffn/LMCJVoMvSAdmscJyS1auj5Zulnn5UoYcY531UWmdwhRWkcGKnGU93m5HSXP9LP2usOryrBtQowA==", - "dev": true, "engines": { "node": ">=16" }, @@ -4933,35 +4968,35 @@ } }, "node_modules/got": { - "version": "13.0.0", - "resolved": "https://registry.npmjs.org/got/-/got-13.0.0.tgz", - "integrity": "sha512-XfBk1CxOOScDcMr9O1yKkNaQyy865NbYs+F7dr4H0LZMVgCj2Le59k6PqbNHoL5ToeaEQUYh6c6yMfVcc6SJxA==", + "version": "14.2.1", + "resolved": "https://registry.npmjs.org/got/-/got-14.2.1.tgz", + "integrity": "sha512-KOaPMremmsvx6l9BLC04LYE6ZFW4x7e4HkTe3LwBmtuYYQwpeS4XKqzhubTIkaQ1Nr+eXxeori0zuwupXMovBQ==", "dependencies": { - "@sindresorhus/is": "^5.2.0", + "@sindresorhus/is": "^6.1.0", "@szmarczak/http-timer": "^5.0.1", "cacheable-lookup": "^7.0.0", - "cacheable-request": "^10.2.8", + "cacheable-request": "^10.2.14", "decompress-response": "^6.0.0", - "form-data-encoder": "^2.1.2", - "get-stream": "^6.0.1", - "http2-wrapper": "^2.1.10", + "form-data-encoder": "^4.0.2", + "get-stream": "^8.0.1", + "http2-wrapper": "^2.2.1", "lowercase-keys": "^3.0.0", - "p-cancelable": "^3.0.0", + "p-cancelable": "^4.0.1", "responselike": "^3.0.0" }, "engines": { - "node": ">=16" + "node": ">=20" }, "funding": { "url": "https://github.com/sindresorhus/got?sponsor=1" } }, "node_modules/got/node_modules/@sindresorhus/is": { - "version": "5.6.0", - "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-5.6.0.tgz", - "integrity": "sha512-TV7t8GKYaJWsn00tFDqBw8+Uqmr8A0fRU1tvTQhyZzGv0sJCGRQL3JGMI3ucuKo3XIZdUP+Lx7/gh2t3lewy7g==", + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-6.3.1.tgz", + "integrity": "sha512-FX4MfcifwJyFOI2lPoX7PQxCqx8BG1HCho7WdiXwpEQx1Ycij0JxkfYtGK7yqNScrZGSlt6RE6sw8QYoH7eKnQ==", "engines": { - "node": ">=14.16" + "node": ">=16" }, "funding": { "url": "https://github.com/sindresorhus/is?sponsor=1" @@ -4986,86 +5021,6 @@ "node": ">=14.16" } }, - "node_modules/got/node_modules/cacheable-request": { - "version": "10.2.13", - "resolved": "https://registry.npmjs.org/cacheable-request/-/cacheable-request-10.2.13.tgz", - "integrity": "sha512-3SD4rrMu1msNGEtNSt8Od6enwdo//U9s4ykmXfA2TD58kcLkCobtCDiby7kNyj7a/Q7lz/mAesAFI54rTdnvBA==", - "dependencies": { - "@types/http-cache-semantics": "^4.0.1", - "get-stream": "^6.0.1", - "http-cache-semantics": "^4.1.1", - "keyv": "^4.5.3", - "mimic-response": "^4.0.0", - "normalize-url": "^8.0.0", - "responselike": "^3.0.0" - }, - "engines": { - "node": ">=14.16" - } - }, - "node_modules/got/node_modules/form-data-encoder": { - "version": "2.1.4", - "resolved": "https://registry.npmjs.org/form-data-encoder/-/form-data-encoder-2.1.4.tgz", - "integrity": "sha512-yDYSgNMraqvnxiEXO4hi88+YZxaHC6QKzb5N84iRCTDeRO7ZALpir/lVmf/uXUhnwUr2O4HU8s/n6x+yNjQkHw==", - "engines": { - "node": ">= 14.17" - } - }, - "node_modules/got/node_modules/get-stream": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-6.0.1.tgz", - "integrity": "sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg==", - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/got/node_modules/lowercase-keys": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/lowercase-keys/-/lowercase-keys-3.0.0.tgz", - "integrity": "sha512-ozCC6gdQ+glXOQsveKD0YsDy8DSQFjDTz4zyzEHNV5+JP5D62LmfDZ6o1cycFx9ouG940M5dE8C8CTewdj2YWQ==", - "engines": { - "node": "^12.20.0 || ^14.13.1 || >=16.0.0" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/got/node_modules/normalize-url": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/normalize-url/-/normalize-url-8.0.0.tgz", - "integrity": "sha512-uVFpKhj5MheNBJRTiMZ9pE/7hD1QTeEvugSJW/OmLzAp78PB5O6adfMNTvmfKhXBkvCzC+rqifWcVYpGFwTjnw==", - "engines": { - "node": ">=14.16" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/got/node_modules/p-cancelable": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/p-cancelable/-/p-cancelable-3.0.0.tgz", - "integrity": "sha512-mlVgR3PGuzlo0MmTdk4cXqXWlwQDLnONTAg6sm62XkMJEiRxN3GL3SffkYvqwonbkJBcrI7Uvv5Zh9yjvn2iUw==", - "engines": { - "node": ">=12.20" - } - }, - "node_modules/got/node_modules/responselike": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/responselike/-/responselike-3.0.0.tgz", - "integrity": "sha512-40yHxbNcl2+rzXvZuVkrYohathsSJlMTXKryG5y8uciHv1+xDLHQpgjG64JUO9nrEq2jGLH6IZ8BcZyw3wrweg==", - "dependencies": { - "lowercase-keys": "^3.0.0" - }, - "engines": { - "node": ">=14.16" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/graceful-fs": { "version": "4.2.10", "dev": true, @@ -5254,7 +5209,8 @@ }, "node_modules/http-cache-semantics": { "version": "4.1.1", - "license": "BSD-2-Clause" + "resolved": "https://registry.npmjs.org/http-cache-semantics/-/http-cache-semantics-4.1.1.tgz", + "integrity": "sha512-er295DKPVsV82j5kw1Gjt+ADA/XYHsajl82cGNQG2eyoPkvgUhX+nDIyelzhIWbbsXP39EHcI6l5tYs2FYqYXQ==" }, "node_modules/http-errors": { "version": "2.0.0", @@ -5272,8 +5228,9 @@ } }, "node_modules/http2-wrapper": { - "version": "2.2.0", - "license": "MIT", + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/http2-wrapper/-/http2-wrapper-2.2.1.tgz", + "integrity": "sha512-V5nVw1PAOgfI3Lmeaj2Exmeg7fenjhRUgz1lPSezy1CuhPYbgQtbQj4jZfEAEMlaL+vupsvhjqCyjzob0yxsmQ==", "dependencies": { "quick-lru": "^5.1.1", "resolve-alpn": "^1.2.0" @@ -6631,7 +6588,8 @@ }, "node_modules/json-buffer": { "version": "3.0.1", - "license": "MIT" + "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz", + "integrity": "sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ==" }, "node_modules/json-parse-even-better-errors": { "version": "2.3.1", @@ -6688,9 +6646,9 @@ } }, "node_modules/keyv": { - "version": "4.5.3", - "resolved": "https://registry.npmjs.org/keyv/-/keyv-4.5.3.tgz", - "integrity": "sha512-QCiSav9WaX1PgETJ+SpNnx2PRRapJ/oRSXM4VO5OGYGSjrxbKPVFVhB3l2OCbLCk329N8qyAtsJjSjvVBWzEug==", + "version": "4.5.4", + "resolved": "https://registry.npmjs.org/keyv/-/keyv-4.5.4.tgz", + "integrity": "sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==", "dependencies": { "json-buffer": "3.0.1" } @@ -6914,6 +6872,17 @@ "loose-envify": "cli.js" } }, + "node_modules/lowercase-keys": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/lowercase-keys/-/lowercase-keys-3.0.0.tgz", + "integrity": "sha512-ozCC6gdQ+glXOQsveKD0YsDy8DSQFjDTz4zyzEHNV5+JP5D62LmfDZ6o1cycFx9ouG940M5dE8C8CTewdj2YWQ==", + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/lru-cache": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz", @@ -7251,6 +7220,17 @@ "node": ">=0.10.0" } }, + "node_modules/normalize-url": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/normalize-url/-/normalize-url-8.0.1.tgz", + "integrity": "sha512-IO9QvjUMWxPQQhs60oOu10CRkWCiZzSUkzbXGGV9pviYl1fXYcvkzQ5jV9z8Y6un8ARoVRl4EtC6v6jNqbaJ/w==", + "engines": { + "node": ">=14.16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/npm-run-path": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-4.0.1.tgz", @@ -7504,6 +7484,14 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/p-cancelable": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/p-cancelable/-/p-cancelable-4.0.1.tgz", + "integrity": "sha512-wBowNApzd45EIKdO1LaU+LrMBwAcjfPaYtVzV3lmfM3gf8Z4CHZsiIqlM8TZZ8okYvh5A1cP6gTfCRQtwUpaUg==", + "engines": { + "node": ">=14.16" + } + }, "node_modules/p-limit": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", @@ -8221,6 +8209,20 @@ "node": ">=10" } }, + "node_modules/responselike": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/responselike/-/responselike-3.0.0.tgz", + "integrity": "sha512-40yHxbNcl2+rzXvZuVkrYohathsSJlMTXKryG5y8uciHv1+xDLHQpgjG64JUO9nrEq2jGLH6IZ8BcZyw3wrweg==", + "dependencies": { + "lowercase-keys": "^3.0.0" + }, + "engines": { + "node": ">=14.16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/reusify": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.0.4.tgz", diff --git a/package.json b/package.json index baf0a97..c329d3b 100644 --- a/package.json +++ b/package.json @@ -16,7 +16,7 @@ "types": "./dist/index.d.ts", "type": "module", "dependencies": { - "got": "^13.0.0", + "got": "^14.2.1", "header-generator": "^2.1.41", "http2-wrapper": "^2.2.0", "mimic-response": "^4.0.0", diff --git a/src/hooks/proxy.ts b/src/hooks/proxy.ts index e16f98a..3ca15bd 100644 --- a/src/hooks/proxy.ts +++ b/src/hooks/proxy.ts @@ -24,11 +24,13 @@ export async function proxyHook(options: Options): Promise { } } +export class ProxyError extends Error {} + function validateProxyProtocol(protocol: string) { const isSupported = protocol === 'http:' || protocol === 'https:'; if (!isSupported) { - throw new Error(`Proxy URL protocol "${protocol}" is not supported. Please use HTTP or HTTPS.`); + throw new ProxyError(`Proxy URL protocol "${protocol}" is not supported. Please use HTTP or HTTPS.`); } } diff --git a/src/resolve-protocol.ts b/src/resolve-protocol.ts index 6c8ed95..c6c5bab 100644 --- a/src/resolve-protocol.ts +++ b/src/resolve-protocol.ts @@ -4,6 +4,7 @@ import { URL } from 'node:url'; import { type Headers } from 'got'; import { auto, type ResolveProtocolConnectFunction, type ResolveProtocolFunction } from 'http2-wrapper'; import QuickLRU from 'quick-lru'; +import { ProxyError } from './hooks/proxy.js'; const connect = async (proxyUrl: string, options: tls.ConnectionOptions, callback: () => void) => new Promise((resolve, reject) => { let host = `${options.host}:${options.port}`; @@ -42,8 +43,10 @@ const connect = async (proxyUrl: string, options: tls.ConnectionOptions, callbac request.once('connect', (response, socket, head) => { if (response.statusCode !== 200 || head.length > 0) { - reject(new Error(`Proxy responded with ${response.statusCode} ${response.statusMessage}: ${head.length} bytes`)); + reject(new ProxyError(`Proxy responded with ${response.statusCode} ${response.statusMessage}: ${head.length} bytes. +Below is the first 100 bytes of the proxy response body: +${head.toString('utf8', 0, 100)}`, { cause: head.toString('utf8') })); socket.destroy(); return; } diff --git a/tsconfig.json b/tsconfig.json index ad151b7..f8bf6ed 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -6,6 +6,7 @@ "target": "ES2022", "verbatimModuleSyntax": true, "incremental": false, + "lib": ["DOM", "ESNext"], }, "include": [ "src"