feat(ci): testing configured queries against ClickHouse versions

.github/workflows/test.yml

@@ -27,7 +27,13 @@ jobs:
       matrix:
         containers: [1]
         browser: [chrome, firefox, edge]
-        clickhouse: [24.5, 24.6, 24.7, 24.8]
+        clickhouse:
+          - '24.5'
+          - '24.6'
+          - '24.7'
+          - '24.8'
+          - '24.9'
+          - '24.10'
 
     services:
       clickhouse:
@@ -219,4 +225,86 @@ jobs:
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
           directory: jest-reports
-          flags: jest,node-${{ matrix.node }}
+          flags: jest,node-${{ matrix.node }}
+
+  test-queries-config:
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        clickhouse:
+          - '24.5'
+          - '24.6'
+          - '24.7'
+          - '24.8'
+          - '24.9'
+          - '24.10'
+          - '24.11'
+
+    services:
+      clickhouse:
+        image: ghcr.io/duyet/docker-images:clickhouse_${{ matrix.clickhouse}}
+        ports:
+          - 8123:8123
+          - 9000:9000
+        options: >-
+          --health-cmd "wget --no-verbose --tries=1 --spider http://localhost:8123/?query=SELECT%201 || exit 1"
+          --health-interval 5s
+          --health-timeout 10s
+          --health-retries 20
+          --health-start-period 5s
+
+      keeper:
+        image: ghcr.io/duyet/docker-images:clickhouse_${{ matrix.clickhouse}}
+        options: >-
+          --entrypoint /keeper/entrypoint.sh
+          --health-cmd /keeper/healthcheck.sh
+          --health-interval 5s
+          --health-timeout 10s
+          --health-retries 20
+          --health-start-period 5s
+
+    steps:
+      - run: |
+          echo Testing queries configured from /app/[host]/[query]/**.ts on ClickHouse ${{ matrix.clickhouse }}
+
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Node
+        uses: actions/setup-node@v4
+        with:
+          node-version: 21
+          cache: yarn
+
+      - name: Get yarn cache directory path
+        id: yarn-cache-dir-path
+        run: echo "dir=$(yarn cache dir)" >> $GITHUB_OUTPUT
+
+      - name: Yarn Cache
+        uses: actions/cache@v4
+        with:
+          path: |
+            ${{ steps.yarn-cache-dir-path.outputs.dir }}
+            ${{ github.workspace }}/.cache
+            ~/.cache
+          key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-yarn-
+
+      - name: Next Cache
+        uses: actions/cache@v4
+        with:
+          path: .next/cache
+          # Generate a new cache whenever packages or source files change.
+          key: ${{ runner.os }}-nextjs-${{ hashFiles('**/package-lock.json', '**/yarn.lock') }}-${{ hashFiles('**.[jt]s', '**.[jt]sx') }}
+          # If source files changed but packages didn't, rebuild from a prior cache.
+          restore-keys: |
+            ${{ runner.os }}-nextjs-${{ hashFiles('**/package-lock.json', '**/yarn.lock') }}-
+
+      - name: Install dependencies
+        run: yarn install
+
+      - name: yarn test-queries-config
+        run: yarn test-queries-config

app/[host]/[query]/more/backups.ts

@@ -7,6 +7,8 @@ export const backupsConfig: QueryConfig = {
   description: `To restore a backup:
       RESTORE TABLE data_lake.events AS data_lake.events_restore FROM Disk('s3_backup', 'data_lake.events_20231212')`,
   docs: BACKUP_LOG,
+  // system.backup_log can be not exist if no backups were made
+  optional: true,
   sql: `
       SELECT *,
         formatReadableSize(total_size) as readable_total_size,

app/[host]/[query]/more/errors.ts

@@ -4,6 +4,7 @@ import { type QueryConfig } from '@/types/query-config'
 export const errorsConfig: QueryConfig = {
   name: 'errors',
   description: 'System error logs and history',
+  optional: true,
   sql: `
       SELECT *
       FROM system.error_log

app/[host]/[query]/more/zookeeper.ts

@@ -7,6 +7,8 @@ export const zookeeperConfig: QueryConfig = {
   description:
     'Exposes data from the Keeper cluster defined in the config. https://clickhouse.com/docs/en/operations/system-tables/zookeeper',
   docs: ZOOKEEPER,
+  // system.zookeeper can be not exist if no zookeeper is configured
+  optional: true,
   sql: `
       SELECT
           replaceOne(format('{}/{}', path, name), '//', '/') AS _path,

app/[host]/[query]/query-config.test.ts

@@ -0,0 +1,76 @@
+import { beforeAll, expect, test } from '@jest/globals'
+
+import { fetchData } from '@/lib/clickhouse'
+import { queries } from './clickhouse-queries'
+
+describe('query config', () => {
+  it('should have more than 1 config', () => {
+    expect(queries.length).toBeGreaterThan(0)
+  })
+
+  const namedConfig = queries.map((config) => {
+    return { name: config.name, config }
+  })
+
+  beforeAll(async () => {
+    try {
+      console.log('prepare data for system.error_log')
+      await fetchData({
+        query: 'SELECT * FROM not_found_table_will_fail',
+      })
+      await fetchData({
+        query: 'INSERT INTO not_found',
+      })
+    } catch (e) {
+      console.log('generated a record in system.error_log', e)
+    }
+
+    try {
+      console.log('prepare data for system.backup_log')
+      await fetchData({
+        query: "BACKUP DATABASE default TO File('/tmp/backup')",
+      })
+      console.log('generated a record in system.backup_log')
+    } catch (e) {
+      console.log('generated a record in system.backup_log', e)
+      console.log(`
+        Although the backup can be failed, it will generate a record in system.backup_log
+        DB::Exception: Path '/tmp/backup' is not allowed for backups,
+        see the 'backups.allowed_path' configuration parameter`)
+    }
+  })
+
+  test.each(namedConfig)(
+    'check if valid sql for $name config',
+    async ({ name, config }) => {
+      expect(config.sql).toBeDefined()
+      console.log(`Testing config ${name} query:`, config.sql)
+      console.log('with default params:', config.defaultParams || {})
+
+      try {
+        const { data, metadata } = await fetchData({
+          query: config.sql,
+          query_params: config.defaultParams || {},
+          format: 'JSONEachRow',
+        })
+
+        console.log('Response:', data)
+        console.log('Metadata:', metadata)
+
+        expect(data).toBeDefined()
+        expect(metadata).toBeDefined()
+      } catch (e) {
+        if (config.optional) {
+          console.log(
+            'Query is marked optional, that mean can be failed due to missing table for example'
+          )
+          expect(e).toHaveProperty('type', 'UNKNOWN_TABLE')
+          return
+        }
+
+        console.error(e)
+        throw e
+      }
+    }
+  )
+})

app/[host]/query/[query_id]/config.ts

@@ -161,6 +161,7 @@ export const config: QueryConfig = {
     'thread_ids_count',
     'peak_threads_usage',
     'file_open',
+    'query_id',
   ],
   columnFormats: {
     type: ColumnFormat.ColoredBadge,
@@ -184,6 +185,12 @@ export const config: QueryConfig = {
     progress: ColumnFormat.BackgroundBar,
     file_open: ColumnFormat.Number,
     query_cache_usage: ColumnFormat.ColoredBadge,
+    query_id: [
+      ColumnFormat.Link,
+      {
+        href: '/[ctx.hostId]/query/[query_id]',
+      },
+    ],
   },
   defaultParams: {
     query_id: '',

app/[host]/query/[query_id]/query-detail-badge.tsx

@@ -30,7 +30,7 @@ export async function QueryDetailBadge({
     })
 
     if (!data.length) {
-      return <div className="text-xs text-muted-foreground">No data</div>
+      return null
     }
 
     const { user } = data[0]