From b129f8722c7fca4fd88d2904368f2985946fe373 Mon Sep 17 00:00:00 2001 From: Duyet Le Date: Fri, 29 Nov 2024 12:45:48 +0700 Subject: [PATCH] fix(ci): add beforeAll to force create needed system tables --- .github/workflows/test.yml | 24 +++++++++---- app/[host]/[query]/more/backups.ts | 2 ++ app/[host]/[query]/more/errors.ts | 1 + app/[host]/[query]/more/zookeeper.ts | 2 ++ app/[host]/[query]/query-config.test.ts | 46 +++++++++++++++++++++++-- package.json | 5 +-- types/query-config.ts | 7 ++++ 7 files changed, 75 insertions(+), 12 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 05c00d11..7d6d65c3 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -27,7 +27,13 @@ jobs: matrix: containers: [1] browser: [chrome, firefox, edge] - clickhouse: [24.5, 24.6, 24.7, 24.8] + clickhouse: + - "24.5" + - "24.6" + - "24.7" + - "24.8" + - "24.9" + - "24.10" services: clickhouse: @@ -228,7 +234,13 @@ jobs: fail-fast: false matrix: node: [21] - clickhouse: [24.5, 24.6, 24.7, 24.8] + clickhouse: + - "24.5" + - "24.6" + - "24.7" + - "24.8" + - "24.9" + - "24.10" services: clickhouse: @@ -282,9 +294,7 @@ jobs: ${{ runner.os }}-nextjs-${{ hashFiles('**/package-lock.json', '**/yarn.lock') }}- - name: Install dependencies - run: | - yarn install + run: yarn install - - name: Jest - run: | - yarn jest -t 'query config' \ No newline at end of file + - name: yarn test-queries-config + run: yarn test-queries-config \ No newline at end of file diff --git a/app/[host]/[query]/more/backups.ts b/app/[host]/[query]/more/backups.ts index 2ee54421..84c73143 100644 --- a/app/[host]/[query]/more/backups.ts +++ b/app/[host]/[query]/more/backups.ts @@ -7,6 +7,8 @@ export const backupsConfig: QueryConfig = { description: `To restore a backup: RESTORE TABLE data_lake.events AS data_lake.events_restore FROM Disk('s3_backup', 'data_lake.events_20231212')`, docs: BACKUP_LOG, + // system.backup_log can be not exist if no backups were made + optional: true, sql: ` SELECT *, formatReadableSize(total_size) as readable_total_size, diff --git a/app/[host]/[query]/more/errors.ts b/app/[host]/[query]/more/errors.ts index b5e1b684..824d78f7 100644 --- a/app/[host]/[query]/more/errors.ts +++ b/app/[host]/[query]/more/errors.ts @@ -4,6 +4,7 @@ import { type QueryConfig } from '@/types/query-config' export const errorsConfig: QueryConfig = { name: 'errors', description: 'System error logs and history', + optional: true, sql: ` SELECT * FROM system.error_log diff --git a/app/[host]/[query]/more/zookeeper.ts b/app/[host]/[query]/more/zookeeper.ts index 7ec60690..263cd48d 100644 --- a/app/[host]/[query]/more/zookeeper.ts +++ b/app/[host]/[query]/more/zookeeper.ts @@ -7,6 +7,8 @@ export const zookeeperConfig: QueryConfig = { description: 'Exposes data from the Keeper cluster defined in the config. https://clickhouse.com/docs/en/operations/system-tables/zookeeper', docs: ZOOKEEPER, + // system.zookeeper can be not exist if no zookeeper is configured + optional: true, sql: ` SELECT replaceOne(format('{}/{}', path, name), '//', '/') AS _path, diff --git a/app/[host]/[query]/query-config.test.ts b/app/[host]/[query]/query-config.test.ts index 1fd02941..0d23daf9 100644 --- a/app/[host]/[query]/query-config.test.ts +++ b/app/[host]/[query]/query-config.test.ts @@ -1,5 +1,6 @@ +import { beforeAll, expect, test } from '@jest/globals' + import { fetchData } from '@/lib/clickhouse' -import { expect, test } from '@jest/globals' import { queries } from './clickhouse-queries' describe('query config', () => { @@ -11,23 +12,62 @@ describe('query config', () => { return { name: config.name, config } }) + beforeAll(async () => { + try { + console.log('prepare data for system.error_log') + await fetchData({ + query: 'SELECT * FROM not_found_table_will_fail', + }) + await fetchData({ + query: 'INSERT INTO not_found', + }) + } catch (e) { + console.log('generated a record in system.error_log', e) + } + + try { + console.log('prepare data for system.backup_log') + await fetchData({ + query: "BACKUP DATABASE default TO File('/tmp/backup')", + }) + console.log('generated a record in system.backup_log') + } catch (e) { + console.log('generated a record in system.backup_log', e) + console.log(` + Although the backup can be failed, it will generate a record in system.backup_log + DB::Exception: Path '/tmp/backup' is not allowed for backups, + see the 'backups.allowed_path' configuration parameter`) + } + }) + test.each(namedConfig)( 'check if valid sql for $name config', async ({ name, config }) => { expect(config.sql).toBeDefined() console.log(`Testing config ${name} query:`, config.sql) - console.log('with default params:', config.defaultParams) + console.log('with default params:', config.defaultParams || {}) try { const { data, metadata } = await fetchData({ query: config.sql, - query_params: config.defaultParams, + query_params: config.defaultParams || {}, format: 'JSONEachRow', }) + console.log('Response:', data) + console.log('Metadata:', metadata) + expect(data).toBeDefined() expect(metadata).toBeDefined() } catch (e) { + if (config.optional) { + console.log( + 'Query is marked optional, that mean can be failed due to missing table for example' + ) + expect(e).toHaveProperty('type', 'UNKNOWN_TABLE') + return + } + console.error(e) throw e } diff --git a/package.json b/package.json index af26c63c..dd16886b 100644 --- a/package.json +++ b/package.json @@ -19,8 +19,9 @@ "component": "cypress open --component", "component:headless": "cypress run --component", "fmt": "prettier --write \"**/*.{ts,tsx,md,mdx,json}\" --cache", - "test": "jest --coverage", - "jest": "jest --coverage" + "test": "yarn jest", + "jest": "jest --coverage --testPathIgnorePatterns=query-config", + "test-queries-config": "jest --coverage --testPathPattern=query-config" }, "dependencies": { "@clickhouse/client": "^0.3.0", diff --git a/types/query-config.ts b/types/query-config.ts index 6731697f..b2c8263b 100644 --- a/types/query-config.ts +++ b/types/query-config.ts @@ -85,6 +85,13 @@ export interface QueryConfig { * The documents or url to be used when query is errors. e.g. log table missing due to cluster configuration. */ docs?: string + /** + * Whether the query is optional or not. If the query is optional, it can be raised as a error due to missing table or view. + * e.g. system.error_log (when there is no error), system.zookeeper (when zookeeper is not configured), system.backup_log (when there is no backup). + * + * Default: false + */ + optional?: boolean } export type QueryConfigNoName = PartialBy