From a0ff98ee2a268b065b71041126c93c32b20c0077 Mon Sep 17 00:00:00 2001 From: Vlad Frangu Date: Mon, 11 Dec 2023 14:35:02 +0200 Subject: [PATCH] fix: correct types for got scraping chore: fixes chore: bump lock file --- CHANGELOG.md | 5 +++ package-lock.json | 4 +- package.json | 2 +- src/context.ts | 8 +++- src/resolve-protocol.ts | 2 +- src/types.ts | 88 +++++++++++++++++++++++++++++++++++++---- test/main.test.ts | 4 -- 7 files changed, 95 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8b86ec1..7e7aa4d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +4.0.3 / 2023/12/11 +==================== +- Fixed missing extended types for `gotScraping.stream` and `gotScraping.paginate` +- Fixed general type issues with `got-scraping`, including not reporting incorrect types for known properties like `proxyUrl` + 4.0.2 / 2023/11/29 ==================== - Fixed runtime exceptions when using `got-scraping` in a project with older versions of node.js 16 diff --git a/package-lock.json b/package-lock.json index 8a71a07..79d8cd2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "got-scraping", - "version": "4.0.2", + "version": "4.0.3", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "got-scraping", - "version": "4.0.2", + "version": "4.0.3", "license": "Apache-2.0", "dependencies": { "got": "^13.0.0", diff --git a/package.json b/package.json index 75b3a06..b03de03 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "got-scraping", - "version": "4.0.2", + "version": "4.0.3", "description": "HTTP client made for scraping based on got.", "engines": { "node": ">=16" diff --git a/src/context.ts b/src/context.ts index bfcf7be..606cd90 100644 --- a/src/context.ts +++ b/src/context.ts @@ -2,13 +2,17 @@ import type { OptionsInit as GotOptionsInit } from 'got'; export type { GotOptionsInit }; -export interface Context extends Record { +export interface Context { proxyUrl?: string; headerGeneratorOptions?: Record; useHeaderGenerator?: boolean; headerGenerator?: { getHeaders: (options: Record) => Record }; insecureHTTPParser?: boolean; sessionToken?: object; + /** @private */ + sessionData?: unknown; + /** @private */ + resolveProtocol?: (data: unknown) => { alpnProtocol: string } | Promise<{ alpnProtocol: string }>; } -export interface OptionsInit extends Context, GotOptionsInit {} +export type OptionsInit = GotOptionsInit & Context; diff --git a/src/resolve-protocol.ts b/src/resolve-protocol.ts index 6d31fb3..1e03998 100644 --- a/src/resolve-protocol.ts +++ b/src/resolve-protocol.ts @@ -68,7 +68,7 @@ const createCaches = () => ({ const defaults = createCaches(); -interface ProtocolCache { +export interface ProtocolCache { protocolCache?: typeof defaults.protocolCache; resolveAlpnQueue?: typeof defaults.resolveAlpnQueue; } diff --git a/src/types.ts b/src/types.ts index 4e7fe61..8c5485b 100644 --- a/src/types.ts +++ b/src/types.ts @@ -1,10 +1,10 @@ -import type { CancelableRequest, ExtendOptions, Got, HTTPAlias, Options, Request, Response } from 'got'; +import type { CancelableRequest, ExtendOptions, Got, HTTPAlias, Options, PaginateData, PaginationOptions, Request, Response } from 'got'; import type { OptionsInit } from './context.js'; type Except = Pick>; type Merge = Except> & SecondType; -export interface ExtendedGotRequestFunction { +export type ExtendedGotRequestFunction = { (url: string | URL, options?: ExtendedOptionsOfTextResponseBody): CancelableRequest>; (url: string | URL, options?: ExtendedOptionsOfJSONResponseBody): CancelableRequest>; (url: string | URL, options?: ExtendedOptionsOfBufferResponseBody): CancelableRequest>; @@ -28,13 +28,13 @@ export interface ExtendedGotRequestFunction { (url: string | URL, options?: OptionsInit): CancelableRequest | Request; (options: OptionsInit): CancelableRequest | Request; (url: undefined, options: undefined, defaults: Options): CancelableRequest | Request; -} +}; export type ExtendedOptionsOfTextResponseBody = Merge +}>; export type ExtendedOptionsOfJSONResponseBody = Merge) => Request) & ((options?: Merge) => Request); + export type ExtendedExtendOptions = ExtendOptions & OptionsInit; -export interface GotScraping extends Record, ExtendedGotRequestFunction { - stream: Got['stream']; - paginate: Got['paginate']; +export type ExtendedGotStream = ExtendedGotStreamFunction & Record; + +export type ExtendedPaginationOptions = PaginationOptions & { + paginate?: (data: PaginateData) => OptionsInit | false; +}; + +export type ExtendedOptionsWithPagination = Merge; +}>; + +export type ExtendedGotPaginate = { + /** + Same as `GotPaginate.each`. + */ + (url: string | URL, options?: ExtendedOptionsWithPagination): AsyncIterableIterator; + /** + Same as `GotPaginate.each`. + */ + (options?: ExtendedOptionsWithPagination): AsyncIterableIterator; + /** + Returns an async iterator. + + See pagination.options for more pagination options. + + @example + ``` + import { gotScraping } from 'got-scraping'; + + const countLimit = 10; + + const pagination = gotScraping.paginate('https://api.github.com/repos/sindresorhus/got/commits', { + pagination: { countLimit } + }); + + console.log(`Printing latest ${countLimit} Got commits (newest to oldest):`); + + for await (const commitData of pagination) { + console.log(commitData.commit.message); + } + ``` + */ + each: ((url: string | URL, options?: ExtendedOptionsWithPagination) => AsyncIterableIterator) + & ((options?: ExtendedOptionsWithPagination) => AsyncIterableIterator); + /** + Returns a Promise for an array of all results. + + See pagination.options for more pagination options. + + @example + ``` + import { gotScraping } from 'got-scraping'; + + const countLimit = 10; + + const results = await gotScraping.paginate.all('https://api.github.com/repos/sindresorhus/got/commits', { + pagination: { countLimit } + }); + + console.log(`Printing latest ${countLimit} Got commits (newest to oldest):`); + console.log(results); + ``` + */ + all: ((url: string | URL, options?: ExtendedOptionsWithPagination) => Promise) + & ((options?: ExtendedOptionsWithPagination) => Promise); +}; + +export type GotScraping = { + stream: ExtendedGotStream; + paginate: ExtendedGotPaginate; defaults: Got['defaults']; extend: (...instancesOrOptions: Array) => GotScraping; -} +} & Record & ExtendedGotRequestFunction; diff --git a/test/main.test.ts b/test/main.test.ts index f9dd21b..37fc014 100644 --- a/test/main.test.ts +++ b/test/main.test.ts @@ -493,7 +493,6 @@ describe('GotScraping', () => { // FIXME: this should be using a local server instead test.skip('should order headers with proxyUrl and http1', async () => { - // @ts-expect-error FIXME const body = await getStream(gotScraping.stream({ url: 'https://api.apify.com/v2/browser-info?rawHeaders=1', proxyUrl: `http://groups-SHADER:${process.env.APIFY_PROXY_PASSWORD}@proxy.apify.com:8000`, @@ -522,7 +521,6 @@ describe('GotScraping', () => { // FIXME: this should use a local server instead test.skip('Should allow https target via http proxy when auto downgrading', async () => { - // @ts-expect-error FIXME const stream = gotScraping.stream({ url: 'https://eshop.coop-box.cz/', proxyUrl: `http://groups-SHADER:${process.env.APIFY_PROXY_PASSWORD}@proxy.apify.com:8000`, @@ -546,7 +544,6 @@ describe('GotScraping', () => { } const responseBody = chunks.join(); - // @ts-expect-error FIXME const proxyStream = gotScraping.stream({ url: 'https://api.apify.com/v2/browser-info', proxyUrl: `http://groups-SHADER:${process.env.APIFY_PROXY_PASSWORD}@proxy.apify.com:8000`, @@ -584,7 +581,6 @@ describe('GotScraping', () => { expect(response.statusCode).toBe(200); expect(response.request.options).toMatchObject({ http2: true }); - // @ts-expect-error FIXME const proxyStream = gotScraping.stream({ url: 'https://api.apify.com/v2/browser-info', proxyUrl: `http://groups-SHADER:${process.env.APIFY_PROXY_PASSWORD}@proxy.apify.com:8000`,