More advanced type hints for JSONEachRowWithProgress

examples/node/select_json_each_row_with_progress.ts

@@ -1,9 +1,10 @@
-import { createClient, isProgressRow, type Progress } from '@clickhouse/client'
+import { createClient } from '@clickhouse/client'
+import { isProgress } from '@clickhouse/client-common'
 
-/** See the format spec - https://clickhouse.com/docs/en/interfaces/formats#jsoneachrowwithprogress */
-type Row = {
-  row: { number: string }
-}
+/** See the format spec - https://clickhouse.com/docs/en/interfaces/formats#jsoneachrowwithprogress
+ *  When JSONEachRowWithProgress format is used in TypeScript,
+ *  the ResultSet should infer the final row type as `{ row: Data } | Progress`. */
+type Data = { number: string }
 
 void (async () => {
   const client = createClient()
@@ -15,15 +16,18 @@ void (async () => {
   let totalRows = 0
   let totalProgressRows = 0
 
-  const stream = rs.stream<Row | Progress>()
+  const stream = rs.stream<Data>()
   for await (const rows of stream) {
     for (const row of rows) {
       const decodedRow = row.json()
-      if (isProgressRow(decodedRow)) {
+      if (isProgress(decodedRow)) {
         console.log('Got a progress row:', decodedRow)
         totalProgressRows++
       } else {
         totalRows++
+        if (totalRows % 100 === 0) {
+          console.log('Sample row:', decodedRow)
+        }
       }
     }
   }

packages/client-common/__tests__/unit/clickhouse_types.test.ts

@@ -1,4 +1,4 @@
-import { isProgressRow } from '@clickhouse/client-common'
+import { isProgress } from '@clickhouse/client-common'
 
 describe('ClickHouse types', () => {
   it('should check if a row is progress row', async () => {
@@ -14,14 +14,14 @@ describe('ClickHouse types', () => {
         elapsed_ns: '1',
       },
     }
-    expect(isProgressRow(row)).toBeTruthy()
-    expect(isProgressRow({})).toBeFalsy()
+    expect(isProgress(row)).toBeTruthy()
+    expect(isProgress({})).toBeFalsy()
     expect(
-      isProgressRow({
+      isProgress({
         ...row,
         extra: 'extra',
       }),
     ).toBeFalsy()
-    expect(isProgressRow(null)).toBeFalsy()
+    expect(isProgress(null)).toBeFalsy()
   })
 })

packages/client-common/src/clickhouse_types.ts

@@ -48,7 +48,7 @@ export interface Progress {
 
 /** Type guard to use with JSONEachRowWithProgress, checking if the emitted row is a progress row.
  *  @see https://clickhouse.com/docs/en/interfaces/formats#jsoneachrowwithprogress */
-export function isProgressRow(row: unknown): row is Progress {
+export function isProgress(row: unknown): row is Progress {
   return (
     row !== null &&
     typeof row === 'object' &&

packages/client-common/src/index.ts

@@ -16,6 +16,7 @@ export {
 export { type BaseClickHouseClientConfigOptions } from './config'
 export type {
   Row,
+  RowOrProgress,
   BaseResultSet,
   ResultJSONType,
   RowJSONType,
@@ -53,7 +54,7 @@ export type {
   WithResponseHeaders,
   Progress,
 } from './clickhouse_types'
-export { isProgressRow } from './clickhouse_types'
+export { isProgress } from './clickhouse_types'
 export {
   type ClickHouseSettings,
   type MergeTreeSettings,

packages/client-common/src/result.ts

@@ -1,4 +1,8 @@
-import type { ResponseHeaders, ResponseJSON } from './clickhouse_types'
+import type {
+  Progress,
+  ResponseHeaders,
+  ResponseJSON,
+} from './clickhouse_types'
 import type {
   DataFormat,
   RawDataFormat,
@@ -8,6 +12,8 @@ import type {
   StreamableJSONDataFormat,
 } from './data_formatter'
 
+export type RowOrProgress<T> = { row: T } | Progress
+
 export type ResultStream<Format extends DataFormat | unknown, Stream> =
   // JSON*EachRow (except JSONObjectEachRow), CSV, TSV etc.
   Format extends StreamableDataFormat
@@ -22,29 +28,35 @@ export type ResultStream<Format extends DataFormat | unknown, Stream> =
           Stream
 
 export type ResultJSONType<T, F extends DataFormat | unknown> =
-  // JSON*EachRow formats except JSONObjectEachRow
-  F extends StreamableJSONDataFormat
-    ? T[]
-    : // JSON formats with known layout { data, meta, statistics, ... }
-      F extends SingleDocumentJSONFormat
-      ? ResponseJSON<T>
-      : // JSON formats represented as a Record<string, T>
-        F extends RecordsJSONFormat
-        ? Record<string, T>
-        : // CSV, TSV etc. - cannot be represented as JSON
-          F extends RawDataFormat
-          ? never
-          : // happens only when Format could not be inferred from a literal
-            T[] | Record<string, T> | ResponseJSON<T>
+  // Emits either a { row: T } or an object with progress
+  F extends 'JSONEachRowWithProgress'
+    ? RowOrProgress<T>[]
+    : // JSON*EachRow formats except JSONObjectEachRow
+      F extends StreamableJSONDataFormat
+      ? T[]
+      : // JSON formats with known layout { data, meta, statistics, ... }
+        F extends SingleDocumentJSONFormat
+        ? ResponseJSON<T>
+        : // JSON formats represented as a Record<string, T>
+          F extends RecordsJSONFormat
+          ? Record<string, T>
+          : // CSV, TSV etc. - cannot be represented as JSON
+            F extends RawDataFormat
+            ? never
+            : // happens only when Format could not be inferred from a literal
+              T[] | Record<string, T> | ResponseJSON<T>
 
 export type RowJSONType<T, F extends DataFormat | unknown> =
-  // JSON*EachRow formats
-  F extends StreamableJSONDataFormat
-    ? T
-    : // CSV, TSV, non-streamable JSON formats - cannot be streamed as JSON
-      F extends RawDataFormat | SingleDocumentJSONFormat | RecordsJSONFormat
-      ? never
-      : T // happens only when Format could not be inferred from a literal
+  // Emits either a { row: T } or an object with progress
+  F extends 'JSONEachRowWithProgress'
+    ? RowOrProgress<T>
+    : // JSON*EachRow formats
+      F extends StreamableJSONDataFormat
+      ? T
+      : // CSV, TSV, non-streamable JSON formats - cannot be streamed as JSON
+        F extends RawDataFormat | SingleDocumentJSONFormat | RecordsJSONFormat
+        ? never
+        : T // happens only when Format could not be inferred from a literal
 
 export interface Row<
   JSONType = unknown,

packages/client-node/__tests__/integration/node_stream_json_formats.test.ts

@@ -1,5 +1,4 @@
-import { type ClickHouseClient, isProgressRow } from '@clickhouse/client-common'
-import type { Progress } from '@clickhouse/client-common/src/clickhouse_types'
+import { type ClickHouseClient, isProgress } from '@clickhouse/client-common'
 import { createSimpleTable } from '@test/fixtures/simple_table'
 import { assertJsonValues, jsonValues } from '@test/fixtures/test_data'
 import { createTestClient, guid } from '@test/utils'
@@ -233,8 +232,6 @@ describe('[Node.js] stream JSON formats', () => {
   })
 
   describe('JSONEachRowWithProgress', () => {
-    type Row = { row: { number: string } }
-
     it('should work', async () => {
       const limit = 2
       const expectedProgressRowsCount = 4
@@ -245,9 +242,9 @@ describe('[Node.js] stream JSON formats', () => {
           max_block_size: '1', // reduce the block size, so the progress is reported more frequently
         },
       })
-      const rows = await rs.json<Row | Progress>()
+      const rows = await rs.json<{ number: 'string' }>()
       expect(rows.length).toEqual(limit + expectedProgressRowsCount)
-      expect(rows.filter((r) => !isProgressRow(r))).toEqual([
+      expect(rows.filter((r) => !isProgress(r)) as unknown[]).toEqual([
         { row: { number: '0' } },
         { row: { number: '1' } },
       ])

packages/client-node/src/index.ts

@@ -61,5 +61,6 @@ export {
   parseColumnType,
   SimpleColumnTypes,
   type Progress,
-  isProgressRow,
+  isProgress,
+  type RowOrProgress,
 } from '@clickhouse/client-common'

packages/client-node/src/result_set.ts

@@ -85,7 +85,7 @@ export class ResultSet<Format extends DataFormat | unknown>
       const stream = this.stream<T>()
       for await (const rows of stream) {
         for (const row of rows) {
-          result.push(row.json())
+          result.push(row.json() as T)
         }
       }
       return result as any

packages/client-web/__tests__/integration/web_select_streaming.test.ts

@@ -1,5 +1,5 @@
-import type { ClickHouseClient, Progress, Row } from '@clickhouse/client-common'
-import { isProgressRow } from '@clickhouse/client-common'
+import type { ClickHouseClient, Row } from '@clickhouse/client-common'
+import { isProgress } from '@clickhouse/client-common'
 import { createTestClient } from '@test/utils'
 
 describe('[Web] SELECT streaming', () => {
@@ -119,7 +119,6 @@ describe('[Web] SELECT streaming', () => {
     })
 
     it('should return objects in JSONEachRowWithProgress format', async () => {
-      type Row = { row: { number: string } }
       const limit = 2
       const expectedProgressRowsCount = 4
       const rs = await client.query({
@@ -129,9 +128,9 @@ describe('[Web] SELECT streaming', () => {
           max_block_size: '1', // reduce the block size, so the progress is reported more frequently
         },
       })
-      const rows = await rs.json<Row | Progress>()
+      const rows = await rs.json<{ number: string }>()
       expect(rows.length).toEqual(limit + expectedProgressRowsCount)
-      expect(rows.filter((r) => !isProgressRow(r))).toEqual([
+      expect(rows.filter((r) => !isProgress(r)) as unknown[]).toEqual([
         { row: { number: '0' } },
         { row: { number: '1' } },
       ])

packages/client-web/src/index.ts

@@ -60,5 +60,6 @@ export {
   parseColumnType,
   SimpleColumnTypes,
   type Progress,
-  isProgressRow,
+  isProgress,
+  type RowOrProgress,
 } from '@clickhouse/client-common'

packages/client-web/src/result_set.ts

@@ -48,7 +48,7 @@ export class ResultSet<Format extends DataFormat | unknown>
           break
         }
         for (const row of value) {
-          result.push(row.json())
+          result.push(row.json() as T)
         }
       }
       return result as any