fix: correct types for got scraping

chore: fixes

chore: bump lock file

CHANGELOG.md

@@ -1,3 +1,8 @@
+4.0.3 / 2023/12/11
+====================
+- Fixed missing extended types for `gotScraping.stream` and `gotScraping.paginate`
+- Fixed general type issues with `got-scraping`, including not reporting incorrect types for known properties like `proxyUrl`
+
 4.0.2 / 2023/11/29
 ====================
 - Fixed runtime exceptions when using `got-scraping` in a project with older versions of node.js 16

package.json

@@ -1,6 +1,6 @@
 {
     "name": "got-scraping",
-    "version": "4.0.2",
+    "version": "4.0.3",
     "description": "HTTP client made for scraping based on got.",
     "engines": {
         "node": ">=16"

src/context.ts

@@ -2,13 +2,17 @@ import type { OptionsInit as GotOptionsInit } from 'got';
 
 export type { GotOptionsInit };
 
-export interface Context extends Record<string, unknown> {
+export interface Context {
     proxyUrl?: string;
     headerGeneratorOptions?: Record<string, unknown>;
     useHeaderGenerator?: boolean;
     headerGenerator?: { getHeaders: (options: Record<string, unknown>) => Record<string, string> };
     insecureHTTPParser?: boolean;
     sessionToken?: object;
+    /** @private */
+    sessionData?: unknown;
+    /** @private */
+    resolveProtocol?: (data: unknown) => { alpnProtocol: string } | Promise<{ alpnProtocol: string }>;
 }
 
-export interface OptionsInit extends Context, GotOptionsInit {}
+export type OptionsInit = GotOptionsInit & Context;

src/resolve-protocol.ts

@@ -68,7 +68,7 @@ const createCaches = () => ({
 
 const defaults = createCaches();
 
-interface ProtocolCache {
+export interface ProtocolCache {
     protocolCache?: typeof defaults.protocolCache;
     resolveAlpnQueue?: typeof defaults.resolveAlpnQueue;
 }

src/types.ts

@@ -1,10 +1,10 @@
-import type { CancelableRequest, ExtendOptions, Got, HTTPAlias, Options, Request, Response } from 'got';
+import type { CancelableRequest, ExtendOptions, Got, HTTPAlias, Options, PaginateData, PaginationOptions, Request, Response } from 'got';
 import type { OptionsInit } from './context.js';
 
 type Except<ObjectType, KeysType extends keyof ObjectType> = Pick<ObjectType, Exclude<keyof ObjectType, KeysType>>;
 type Merge<FirstType, SecondType> = Except<FirstType, Extract<keyof FirstType, keyof SecondType>> & SecondType;
 
-export interface ExtendedGotRequestFunction {
+export type ExtendedGotRequestFunction = {
     (url: string | URL, options?: ExtendedOptionsOfTextResponseBody): CancelableRequest<Response<string>>;
     <T>(url: string | URL, options?: ExtendedOptionsOfJSONResponseBody): CancelableRequest<Response<T>>;
     (url: string | URL, options?: ExtendedOptionsOfBufferResponseBody): CancelableRequest<Response<Buffer>>;
@@ -28,13 +28,13 @@ export interface ExtendedGotRequestFunction {
     (url: string | URL, options?: OptionsInit): CancelableRequest | Request;
     (options: OptionsInit): CancelableRequest | Request;
     (url: undefined, options: undefined, defaults: Options): CancelableRequest | Request;
-}
+};
 
 export type ExtendedOptionsOfTextResponseBody = Merge<OptionsInit, {
     isStream?: false;
     resolveBodyOnly?: false;
     responseType?: 'text';
-}>
+}>;
 
 export type ExtendedOptionsOfJSONResponseBody = Merge<OptionsInit, {
     isStream?: false;
@@ -57,11 +57,83 @@ export type ResponseBodyOnly = {
     resolveBodyOnly: true;
 };
 
+export type ExtendedGotStreamFunction = ((url?: string | URL, options?: Merge<OptionsInit, {
+    isStream?: true;
+}>) => Request) & ((options?: Merge<OptionsInit, {
+    isStream?: true;
+}>) => Request);
+
 export type ExtendedExtendOptions = ExtendOptions & OptionsInit;
 
-export interface GotScraping extends Record<HTTPAlias, ExtendedGotRequestFunction>, ExtendedGotRequestFunction {
-    stream: Got['stream'];
-    paginate: Got['paginate'];
+export type ExtendedGotStream = ExtendedGotStreamFunction & Record<HTTPAlias, ExtendedGotStreamFunction>;
+
+export type ExtendedPaginationOptions<ElementType, BodyType> = PaginationOptions<ElementType, BodyType> & {
+    paginate?: (data: PaginateData<BodyType, ElementType>) => OptionsInit | false;
+};
+
+export type ExtendedOptionsWithPagination<T = unknown, R = unknown> = Merge<OptionsInit, {
+    pagination?: ExtendedPaginationOptions<T, R>;
+}>;
+
+export type ExtendedGotPaginate = {
+    /**
+    Same as `GotPaginate.each`.
+    */
+    <T, R = unknown>(url: string | URL, options?: ExtendedOptionsWithPagination<T, R>): AsyncIterableIterator<T>;
+    /**
+    Same as `GotPaginate.each`.
+    */
+    <T, R = unknown>(options?: ExtendedOptionsWithPagination<T, R>): AsyncIterableIterator<T>;
+    /**
+    Returns an async iterator.
+
+    See pagination.options for more pagination options.
+
+    @example
+    ```
+    import { gotScraping } from 'got-scraping';
+
+    const countLimit = 10;
+
+    const pagination = gotScraping.paginate('https://api.github.com/repos/sindresorhus/got/commits', {
+        pagination: { countLimit }
+    });
+
+    console.log(`Printing latest ${countLimit} Got commits (newest to oldest):`);
+
+    for await (const commitData of pagination) {
+        console.log(commitData.commit.message);
+    }
+    ```
+    */
+    each: (<T, R = unknown>(url: string | URL, options?: ExtendedOptionsWithPagination<T, R>) => AsyncIterableIterator<T>)
+        & (<T, R = unknown>(options?: ExtendedOptionsWithPagination<T, R>) => AsyncIterableIterator<T>);
+    /**
+    Returns a Promise for an array of all results.
+
+    See pagination.options for more pagination options.
+
+    @example
+    ```
+    import { gotScraping } from 'got-scraping';
+
+    const countLimit = 10;
+
+    const results = await gotScraping.paginate.all('https://api.github.com/repos/sindresorhus/got/commits', {
+        pagination: { countLimit }
+    });
+
+    console.log(`Printing latest ${countLimit} Got commits (newest to oldest):`);
+    console.log(results);
+    ```
+    */
+    all: (<T, R = unknown>(url: string | URL, options?: ExtendedOptionsWithPagination<T, R>) => Promise<T[]>)
+        & (<T, R = unknown>(options?: ExtendedOptionsWithPagination<T, R>) => Promise<T[]>);
+};
+
+export type GotScraping = {
+    stream: ExtendedGotStream;
+    paginate: ExtendedGotPaginate;
     defaults: Got['defaults'];
     extend: (...instancesOrOptions: Array<GotScraping | ExtendedExtendOptions>) => GotScraping;
-}
+} & Record<HTTPAlias, ExtendedGotRequestFunction> & ExtendedGotRequestFunction;

test/main.test.ts

@@ -493,7 +493,6 @@ describe('GotScraping', () => {
 
         // FIXME: this should be using a local server instead
         test.skip('should order headers with proxyUrl and http1', async () => {
-            // @ts-expect-error FIXME
             const body = await getStream(gotScraping.stream({
                 url: 'https://api.apify.com/v2/browser-info?rawHeaders=1',
                 proxyUrl: `http://groups-SHADER:${process.env.APIFY_PROXY_PASSWORD}@proxy.apify.com:8000`,
@@ -522,7 +521,6 @@ describe('GotScraping', () => {
 
             // FIXME: this should use a local server instead
             test.skip('Should allow https target via http proxy when auto downgrading', async () => {
-                // @ts-expect-error FIXME
                 const stream = gotScraping.stream({
                     url: 'https://eshop.coop-box.cz/',
                     proxyUrl: `http://groups-SHADER:${process.env.APIFY_PROXY_PASSWORD}@proxy.apify.com:8000`,
@@ -546,7 +544,6 @@ describe('GotScraping', () => {
                 }
                 const responseBody = chunks.join();
 
-                // @ts-expect-error FIXME
                 const proxyStream = gotScraping.stream({
                     url: 'https://api.apify.com/v2/browser-info',
                     proxyUrl: `http://groups-SHADER:${process.env.APIFY_PROXY_PASSWORD}@proxy.apify.com:8000`,
@@ -584,7 +581,6 @@ describe('GotScraping', () => {
                 expect(response.statusCode).toBe(200);
                 expect(response.request.options).toMatchObject({ http2: true });
 
-                // @ts-expect-error FIXME
                 const proxyStream = gotScraping.stream({
                     url: 'https://api.apify.com/v2/browser-info',
                     proxyUrl: `http://groups-SHADER:${process.env.APIFY_PROXY_PASSWORD}@proxy.apify.com:8000`,