From 62dc9cde5a8564ad4cdf2c1eff9592ca32351ec3 Mon Sep 17 00:00:00 2001 From: slvrtrn Date: Mon, 30 Sep 2024 23:22:17 +0200 Subject: [PATCH] Reorganize the tests a bit --- .../client-common/__tests__/utils/datasets.ts | 37 ++++++++++++ .../integration/node_streaming_e2e.test.ts | 42 ++------------ .../integration/web_select_streaming.test.ts | 56 +++---------------- 3 files changed, 49 insertions(+), 86 deletions(-) create mode 100644 packages/client-common/__tests__/utils/datasets.ts diff --git a/packages/client-common/__tests__/utils/datasets.ts b/packages/client-common/__tests__/utils/datasets.ts new file mode 100644 index 00000000..1966ccb4 --- /dev/null +++ b/packages/client-common/__tests__/utils/datasets.ts @@ -0,0 +1,37 @@ +import type { ClickHouseClient } from '@clickhouse/client-common' +import { fakerRU } from '@faker-js/faker' +import { createTableWithFields } from '@test/fixtures/table_with_fields' + +export async function genLargeStringsDataset( + client: ClickHouseClient, + { + rows, + words, + }: { + rows: number + words: number + }, +): Promise<{ + table: string + values: { id: number; sentence: string; timestamp: string }[] +}> { + const table = await createTableWithFields( + client as ClickHouseClient, + `sentence String, timestamp String`, + ) + const values = [...new Array(rows)].map((_, id) => ({ + id, + // it seems that it is easier to trigger an incorrect behavior with non-ASCII symbols + sentence: fakerRU.lorem.sentence(words), + timestamp: new Date().toISOString(), + })) + await client.insert({ + table, + values, + format: 'JSONEachRow', + }) + return { + table, + values, + } +} diff --git a/packages/client-node/__tests__/integration/node_streaming_e2e.test.ts b/packages/client-node/__tests__/integration/node_streaming_e2e.test.ts index 38539f6f..a9f7184f 100644 --- a/packages/client-node/__tests__/integration/node_streaming_e2e.test.ts +++ b/packages/client-node/__tests__/integration/node_streaming_e2e.test.ts @@ -3,10 +3,9 @@ import { type ClickHouseClient, type ClickHouseSettings, } from '@clickhouse/client-common' -import { fakerRU } from '@faker-js/faker' import { createSimpleTable } from '@test/fixtures/simple_table' -import { createTableWithFields } from '@test/fixtures/table_with_fields' import { createTestClient, guid } from '@test/utils' +import { genLargeStringsDataset } from '@test/utils/datasets' import { tableFromIPC } from 'apache-arrow' import { Buffer } from 'buffer' import Fs from 'fs' @@ -152,40 +151,9 @@ describe('[Node.js] streaming e2e', () => { // Here we generate a large enough dataset to break into multiple chunks while streaming, // effectively testing the implementation of incomplete rows handling describe('should correctly process multiple chunks', () => { - async function generateData({ - rows, - words, - }: { - rows: number - words: number - }): Promise<{ - table: string - values: { id: number; sentence: string; timestamp: string }[] - }> { - const table = await createTableWithFields( - client as ClickHouseClient, - `sentence String, timestamp String`, - ) - const values = [...new Array(rows)].map((_, id) => ({ - id, - // it seems that it is easier to trigger an incorrect behavior with non-ASCII symbols - sentence: fakerRU.lorem.sentence(words), - timestamp: new Date().toISOString(), - })) - await client.insert({ - table, - values, - format: 'JSONEachRow', - }) - return { - table, - values, - } - } - describe('large amount of rows', () => { it('should work with .json()', async () => { - const { table, values } = await generateData({ + const { table, values } = await genLargeStringsDataset(client, { rows: 10000, words: 10, }) @@ -199,7 +167,7 @@ describe('[Node.js] streaming e2e', () => { }) it('should work with .stream()', async () => { - const { table, values } = await generateData({ + const { table, values } = await genLargeStringsDataset(client, { rows: 10000, words: 10, }) @@ -222,7 +190,7 @@ describe('[Node.js] streaming e2e', () => { describe("rows that don't fit into a single chunk", () => { it('should work with .json()', async () => { - const { table, values } = await generateData({ + const { table, values } = await genLargeStringsDataset(client, { rows: 5, words: 10000, }) @@ -236,7 +204,7 @@ describe('[Node.js] streaming e2e', () => { }) it('should work with .stream()', async () => { - const { table, values } = await generateData({ + const { table, values } = await genLargeStringsDataset(client, { rows: 5, words: 10000, }) diff --git a/packages/client-web/__tests__/integration/web_select_streaming.test.ts b/packages/client-web/__tests__/integration/web_select_streaming.test.ts index 62553620..fcbfe8e8 100644 --- a/packages/client-web/__tests__/integration/web_select_streaming.test.ts +++ b/packages/client-web/__tests__/integration/web_select_streaming.test.ts @@ -1,7 +1,6 @@ import type { ClickHouseClient, Row } from '@clickhouse/client-common' -import { fakerRU } from '@faker-js/faker' -import { createTableWithFields } from '@test/fixtures/table_with_fields' import { createTestClient } from '@test/utils' +import { genLargeStringsDataset } from '@test/utils/datasets' describe('[Web] SELECT streaming', () => { let client: ClickHouseClient> @@ -206,40 +205,9 @@ describe('[Web] SELECT streaming', () => { // Here we generate a large enough dataset to break into multiple chunks while streaming, // effectively testing the implementation of incomplete rows handling describe('should correctly process multiple chunks', () => { - async function generateData({ - rows, - words, - }: { - rows: number - words: number - }): Promise<{ - table: string - values: { id: number; sentence: string; timestamp: string }[] - }> { - const table = await createTableWithFields( - client as ClickHouseClient, - `sentence String, timestamp String`, - ) - const values = [...new Array(rows)].map((_, id) => ({ - id, - // it seems that it is easier to trigger an incorrect behavior with non-ASCII symbols - sentence: fakerRU.lorem.sentence(words), - timestamp: new Date().toISOString(), - })) - await client.insert({ - table, - values, - format: 'JSONEachRow', - }) - return { - table, - values, - } - } - describe('large amount of rows', () => { it('should work with .json()', async () => { - const { table, values } = await generateData({ + const { table, values } = await genLargeStringsDataset(client, { rows: 10000, words: 10, }) @@ -253,7 +221,7 @@ describe('[Web] SELECT streaming', () => { }) it('should work with .stream()', async () => { - const { table, values } = await generateData({ + const { table, values } = await genLargeStringsDataset(client, { rows: 10000, words: 10, }) @@ -264,19 +232,14 @@ describe('[Web] SELECT streaming', () => { }) .then((r) => r.stream()) - const result = [] - for await (const rows of stream) { - for (const row of rows) { - result.push(await row.json()) - } - } + const result = await rowsJsonValues(stream) expect(result).toEqual(values) }) }) describe("rows that don't fit into a single chunk", () => { it('should work with .json()', async () => { - const { table, values } = await generateData({ + const { table, values } = await genLargeStringsDataset(client, { rows: 5, words: 10000, }) @@ -290,7 +253,7 @@ describe('[Web] SELECT streaming', () => { }) it('should work with .stream()', async () => { - const { table, values } = await generateData({ + const { table, values } = await genLargeStringsDataset(client, { rows: 5, words: 10000, }) @@ -301,12 +264,7 @@ describe('[Web] SELECT streaming', () => { }) .then((r) => r.stream()) - const result = [] - for await (const rows of stream) { - for (const row of rows) { - result.push(await row.json()) - } - } + const result = await rowsJsonValues(stream) expect(result).toEqual(values) }) })