diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 00041cbc..25e3251a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -27,7 +27,13 @@ jobs: matrix: containers: [1] browser: [chrome, firefox, edge] - clickhouse: [24.5, 24.6, 24.7, 24.8] + clickhouse: + - '24.5' + - '24.6' + - '24.7' + - '24.8' + - '24.9' + - '24.10' services: clickhouse: @@ -219,4 +225,86 @@ jobs: with: token: ${{ secrets.CODECOV_TOKEN }} directory: jest-reports - flags: jest,node-${{ matrix.node }} \ No newline at end of file + flags: jest,node-${{ matrix.node }} + + test-queries-config: + runs-on: ubuntu-latest + + strategy: + fail-fast: false + matrix: + clickhouse: + - '24.5' + - '24.6' + - '24.7' + - '24.8' + - '24.9' + - '24.10' + - '24.11' + + services: + clickhouse: + image: ghcr.io/duyet/docker-images:clickhouse_${{ matrix.clickhouse}} + ports: + - 8123:8123 + - 9000:9000 + options: >- + --health-cmd "wget --no-verbose --tries=1 --spider http://localhost:8123/?query=SELECT%201 || exit 1" + --health-interval 5s + --health-timeout 10s + --health-retries 20 + --health-start-period 5s + + keeper: + image: ghcr.io/duyet/docker-images:clickhouse_${{ matrix.clickhouse}} + options: >- + --entrypoint /keeper/entrypoint.sh + --health-cmd /keeper/healthcheck.sh + --health-interval 5s + --health-timeout 10s + --health-retries 20 + --health-start-period 5s + + steps: + - run: | + echo Testing queries configured from /app/[host]/[query]/**.ts on ClickHouse ${{ matrix.clickhouse }} + + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Node + uses: actions/setup-node@v4 + with: + node-version: 21 + cache: yarn + + - name: Get yarn cache directory path + id: yarn-cache-dir-path + run: echo "dir=$(yarn cache dir)" >> $GITHUB_OUTPUT + + - name: Yarn Cache + uses: actions/cache@v4 + with: + path: | + ${{ steps.yarn-cache-dir-path.outputs.dir }} + ${{ github.workspace }}/.cache + ~/.cache + key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }} + restore-keys: | + ${{ runner.os }}-yarn- + + - name: Next Cache + uses: actions/cache@v4 + with: + path: .next/cache + # Generate a new cache whenever packages or source files change. + key: ${{ runner.os }}-nextjs-${{ hashFiles('**/package-lock.json', '**/yarn.lock') }}-${{ hashFiles('**.[jt]s', '**.[jt]sx') }} + # If source files changed but packages didn't, rebuild from a prior cache. + restore-keys: | + ${{ runner.os }}-nextjs-${{ hashFiles('**/package-lock.json', '**/yarn.lock') }}- + + - name: Install dependencies + run: yarn install + + - name: yarn test-queries-config + run: yarn test-queries-config diff --git a/app/[host]/[query]/more/backups.ts b/app/[host]/[query]/more/backups.ts index 2ee54421..84c73143 100644 --- a/app/[host]/[query]/more/backups.ts +++ b/app/[host]/[query]/more/backups.ts @@ -7,6 +7,8 @@ export const backupsConfig: QueryConfig = { description: `To restore a backup: RESTORE TABLE data_lake.events AS data_lake.events_restore FROM Disk('s3_backup', 'data_lake.events_20231212')`, docs: BACKUP_LOG, + // system.backup_log can be not exist if no backups were made + optional: true, sql: ` SELECT *, formatReadableSize(total_size) as readable_total_size, diff --git a/app/[host]/[query]/more/errors.ts b/app/[host]/[query]/more/errors.ts index b5e1b684..824d78f7 100644 --- a/app/[host]/[query]/more/errors.ts +++ b/app/[host]/[query]/more/errors.ts @@ -4,6 +4,7 @@ import { type QueryConfig } from '@/types/query-config' export const errorsConfig: QueryConfig = { name: 'errors', description: 'System error logs and history', + optional: true, sql: ` SELECT * FROM system.error_log diff --git a/app/[host]/[query]/more/zookeeper.ts b/app/[host]/[query]/more/zookeeper.ts index 7ec60690..263cd48d 100644 --- a/app/[host]/[query]/more/zookeeper.ts +++ b/app/[host]/[query]/more/zookeeper.ts @@ -7,6 +7,8 @@ export const zookeeperConfig: QueryConfig = { description: 'Exposes data from the Keeper cluster defined in the config. https://clickhouse.com/docs/en/operations/system-tables/zookeeper', docs: ZOOKEEPER, + // system.zookeeper can be not exist if no zookeeper is configured + optional: true, sql: ` SELECT replaceOne(format('{}/{}', path, name), '//', '/') AS _path, diff --git a/app/[host]/[query]/query-config.test.ts b/app/[host]/[query]/query-config.test.ts new file mode 100644 index 00000000..0d23daf9 --- /dev/null +++ b/app/[host]/[query]/query-config.test.ts @@ -0,0 +1,76 @@ +import { beforeAll, expect, test } from '@jest/globals' + +import { fetchData } from '@/lib/clickhouse' +import { queries } from './clickhouse-queries' + +describe('query config', () => { + it('should have more than 1 config', () => { + expect(queries.length).toBeGreaterThan(0) + }) + + const namedConfig = queries.map((config) => { + return { name: config.name, config } + }) + + beforeAll(async () => { + try { + console.log('prepare data for system.error_log') + await fetchData({ + query: 'SELECT * FROM not_found_table_will_fail', + }) + await fetchData({ + query: 'INSERT INTO not_found', + }) + } catch (e) { + console.log('generated a record in system.error_log', e) + } + + try { + console.log('prepare data for system.backup_log') + await fetchData({ + query: "BACKUP DATABASE default TO File('/tmp/backup')", + }) + console.log('generated a record in system.backup_log') + } catch (e) { + console.log('generated a record in system.backup_log', e) + console.log(` + Although the backup can be failed, it will generate a record in system.backup_log + DB::Exception: Path '/tmp/backup' is not allowed for backups, + see the 'backups.allowed_path' configuration parameter`) + } + }) + + test.each(namedConfig)( + 'check if valid sql for $name config', + async ({ name, config }) => { + expect(config.sql).toBeDefined() + console.log(`Testing config ${name} query:`, config.sql) + console.log('with default params:', config.defaultParams || {}) + + try { + const { data, metadata } = await fetchData({ + query: config.sql, + query_params: config.defaultParams || {}, + format: 'JSONEachRow', + }) + + console.log('Response:', data) + console.log('Metadata:', metadata) + + expect(data).toBeDefined() + expect(metadata).toBeDefined() + } catch (e) { + if (config.optional) { + console.log( + 'Query is marked optional, that mean can be failed due to missing table for example' + ) + expect(e).toHaveProperty('type', 'UNKNOWN_TABLE') + return + } + + console.error(e) + throw e + } + } + ) +}) diff --git a/app/[host]/query/[query_id]/config.ts b/app/[host]/query/[query_id]/config.ts index dfe9e03a..f8e5841b 100644 --- a/app/[host]/query/[query_id]/config.ts +++ b/app/[host]/query/[query_id]/config.ts @@ -161,6 +161,7 @@ export const config: QueryConfig = { 'thread_ids_count', 'peak_threads_usage', 'file_open', + 'query_id', ], columnFormats: { type: ColumnFormat.ColoredBadge, @@ -184,6 +185,12 @@ export const config: QueryConfig = { progress: ColumnFormat.BackgroundBar, file_open: ColumnFormat.Number, query_cache_usage: ColumnFormat.ColoredBadge, + query_id: [ + ColumnFormat.Link, + { + href: '/[ctx.hostId]/query/[query_id]', + }, + ], }, defaultParams: { query_id: '', diff --git a/app/[host]/query/[query_id]/query-detail-badge.tsx b/app/[host]/query/[query_id]/query-detail-badge.tsx index 7aca6d32..6ca2977c 100644 --- a/app/[host]/query/[query_id]/query-detail-badge.tsx +++ b/app/[host]/query/[query_id]/query-detail-badge.tsx @@ -30,7 +30,7 @@ export async function QueryDetailBadge({ }) if (!data.length) { - return
No data
+ return null } const { user } = data[0] diff --git a/app/[host]/query/[query_id]/query-detail-card.tsx b/app/[host]/query/[query_id]/query-detail-card.tsx index c0998fb3..ee136f83 100644 --- a/app/[host]/query/[query_id]/query-detail-card.tsx +++ b/app/[host]/query/[query_id]/query-detail-card.tsx @@ -1,3 +1,11 @@ +import { + ExternalLinkIcon, + FilterIcon, + InfoIcon, + MoveRightIcon, +} from 'lucide-react' +import Link from 'next/link' + import { ErrorAlert } from '@/components/error-alert' import { TruncatedList } from '@/components/truncated-list' import { TruncatedParagraph } from '@/components/truncated-paragraph' @@ -12,8 +20,7 @@ import { formatQuery } from '@/lib/format-readable' import { getHostIdCookie, getScopedLink } from '@/lib/scoped-link' import { dedent } from '@/lib/utils' import { QueryConfig } from '@/types/query-config' -import { ExternalLinkIcon } from 'lucide-react' -import Link from 'next/link' + import { type RowData } from './config' import { PageProps } from './types' @@ -42,7 +49,7 @@ export async function QueryDetailCard({ }) if (!data.length) { - return
No data
+ return
No data
} const { query } = data[0] @@ -139,7 +146,7 @@ export async function QueryDetailCard({ key="initial_user" > {initial_user} - + ) : ( '' @@ -154,7 +161,7 @@ export async function QueryDetailCard({ key="initial_query_id" > {initial_query_id} - + , ], [ @@ -224,13 +231,7 @@ export async function QueryDetailCard({ JSON.stringify(used_data_type_families, null, 2), ], ['Used dictionaries', JSON.stringify(used_dictionaries, null, 2)], - [ - { - key: 'Used formats', - link: 'https://clickhouse.com/docs/en/chdb/data-formats', - }, - JSON.stringify(used_formats, null, 2), - ], + ['Used formats', bindingDataFormat(used_formats)], ['Used functions', bindingReference(used_functions)], ['Used storages', JSON.stringify(used_storages, null, 2)], [ @@ -285,6 +286,17 @@ export async function QueryDetailCard({ ))} + +
+ + See the detailed explanation at the{' '} + + system.query_log document + +
) } catch (error) { @@ -299,36 +311,70 @@ export async function QueryDetailCard({ } } -function bindingDatabaseLink(databases: Array): React.ReactNode[] { +function bindingDatabaseLink( + databases: Array +): React.ReactNode[] | null { + if (!databases.length) { + return null + } + return databases.map(async (database) => { + if (database.startsWith('_table_function')) { + return database + } + return ( - {database} + {database} ) }) } -function bindingTableLink(tables: Array): React.ReactNode[] { +function bindingTableLink(tables: Array): React.ReactNode[] | null { + if (!tables.length) { + return null + } + return tables.map(async (databaseTable) => { const [database, table] = databaseTable.split('.') + + // Link to ClickHouse docs + if (database.startsWith('_table_function')) { + return ( + + {databaseTable} + + ) + } + return ( - {databaseTable} + {databaseTable} ) }) } -function bindingReference(value: Array): React.ReactNode[] { +function bindingReference(value: Array): React.ReactNode[] | null { + if (!value.length) { + return null + } + const getSearchLink = (item: string) => { const searchParams = new URLSearchParams({ q: `repo:ClickHouse/ClickHouse path:docs/en/sql-reference path:*.md "# ${item}"`, @@ -352,3 +398,22 @@ function bindingReference(value: Array): React.ReactNode[] { ) }) } + +function bindingDataFormat(value: Array): React.ReactNode[] | null { + if (!value.length) { + return null + } + + return value.map((item) => { + return ( + + {item} + + ) + }) +} diff --git a/components/truncated-list.cy.tsx b/components/truncated-list.cy.tsx index e499ebe0..6e4af326 100644 --- a/components/truncated-list.cy.tsx +++ b/components/truncated-list.cy.tsx @@ -20,15 +20,15 @@ describe('TruncatedList', () => { }) it('shows "Show more" button when items exceed limit', () => { - cy.mount({items}) + cy.mount({items}) - cy.contains('Show more').should('be.visible') + cy.contains('Show 2 more').should('be.visible') }) it('expands to show all items when "Show more" is clicked', () => { - cy.mount({items}) + cy.mount({items}) - cy.contains('Show more').click() + cy.contains('Show 2 more').click() cy.contains('Item 4').should('be.visible') cy.contains('Item 5').should('be.visible') @@ -36,19 +36,20 @@ describe('TruncatedList', () => { }) it('collapses back when "Show less" is clicked', () => { - cy.mount({items}) + cy.mount({items}) - cy.contains('Show more').click() + cy.contains('Show 2 more').click() cy.contains('Show less').click() cy.contains('Item 4').should('not.exist') cy.contains('Item 5').should('not.exist') - cy.contains('Show more').should('be.visible') + cy.contains('Show 2 more').should('be.visible') }) it('respects custom items prop', () => { cy.mount({items}) + cy.contains('Show 3 more').should('be.visible') cy.contains('Item 1').should('be.visible') cy.contains('Item 2').should('be.visible') cy.contains('Item 3').should('not.exist') diff --git a/components/truncated-list.tsx b/components/truncated-list.tsx index fd05f205..f129f150 100644 --- a/components/truncated-list.tsx +++ b/components/truncated-list.tsx @@ -16,6 +16,7 @@ export function TruncatedList({ }: TruncatedListProps) { const [isExpanded, setIsExpanded] = useState(false) const isClamped = Children.count(children) > items + const length = Children.count(children) return (
@@ -26,7 +27,7 @@ export function TruncatedList({ onClick={() => setIsExpanded(!isExpanded)} className="mt-2 text-sm text-blue-500 hover:text-blue-700 focus:outline-none" > - {isExpanded ? 'Show less' : 'Show more'} + {isExpanded ? 'Show less' : `Show ${length - items} more`} )}
diff --git a/package.json b/package.json index af26c63c..dd16886b 100644 --- a/package.json +++ b/package.json @@ -19,8 +19,9 @@ "component": "cypress open --component", "component:headless": "cypress run --component", "fmt": "prettier --write \"**/*.{ts,tsx,md,mdx,json}\" --cache", - "test": "jest --coverage", - "jest": "jest --coverage" + "test": "yarn jest", + "jest": "jest --coverage --testPathIgnorePatterns=query-config", + "test-queries-config": "jest --coverage --testPathPattern=query-config" }, "dependencies": { "@clickhouse/client": "^0.3.0", diff --git a/types/query-config.ts b/types/query-config.ts index 6731697f..b2c8263b 100644 --- a/types/query-config.ts +++ b/types/query-config.ts @@ -85,6 +85,13 @@ export interface QueryConfig { * The documents or url to be used when query is errors. e.g. log table missing due to cluster configuration. */ docs?: string + /** + * Whether the query is optional or not. If the query is optional, it can be raised as a error due to missing table or view. + * e.g. system.error_log (when there is no error), system.zookeeper (when zookeeper is not configured), system.backup_log (when there is no backup). + * + * Default: false + */ + optional?: boolean } export type QueryConfigNoName = PartialBy