From f3ed6adeacf3c9fa119e3fc6c1d25ccfd761cddb Mon Sep 17 00:00:00 2001 From: Duyet Le Date: Wed, 22 Nov 2023 12:31:41 +0700 Subject: [PATCH] feat: add expensive-queries, format number, refactor menu and data-table --- app/[name]/clickhouse-queries.ts | 175 +++++++++++++++++++++++++++ app/tables/page.tsx | 4 + components/count-badge.tsx | 32 +++++ components/data-table/cell.tsx | 14 ++- components/data-table/columns.tsx | 3 + components/data-table/data-table.tsx | 5 +- components/menu.tsx | 49 ++++---- cypress/e2e/navigation.cy.js | 4 - lib/format-readable.ts | 30 ++++- lib/types/query-config.ts | 1 + 10 files changed, 285 insertions(+), 32 deletions(-) create mode 100644 components/count-badge.tsx diff --git a/app/[name]/clickhouse-queries.ts b/app/[name]/clickhouse-queries.ts index 9e05e785..aebac5cc 100644 --- a/app/[name]/clickhouse-queries.ts +++ b/app/[name]/clickhouse-queries.ts @@ -48,6 +48,141 @@ export const queries: Array = [ ], ], }, + { + name: 'history-queries', + sql: ` + SELECT + type, + query_id, + query_duration_ms, + query_duration_ms as query_duration, + event_time, + query, + formatted_query AS readable_query, + user, + read_rows, + formatReadableQuantity(read_rows) AS readable_read_rows, + written_rows, + formatReadableQuantity(written_rows) AS readable_written_rows, + result_rows, + formatReadableQuantity(result_rows) AS readable_result_rows, + memory_usage, + formatReadableSize(memory_usage) AS readable_memory_usage, + query_kind, + client_name + FROM system.query_log + WHERE type != 'QueryStart' + ORDER BY event_time DESC + LIMIT 1000 + `, + columns: [ + 'user', + 'query', + 'event_time', + 'query_id', + 'query_duration', + 'user', + 'readable_read_rows', + 'readable_written_rows', + 'readable_result_rows', + 'readable_memory_usage', + 'query_kind', + 'client_name', + ], + columnFormats: { + query_duration: ColumnFormat.Duration, + readable_query: ColumnFormat.Code, + query: ColumnFormat.Code, + event_time: ColumnFormat.RelatedTime, + }, + + relatedCharts: [ + [ + 'query-count', + { + title: 'Total Running Queries over last 14 days (query / day)', + interval: 'toStartOfDay', + lastHours: 24 * 14, + }, + ], + ], + }, + { + name: 'expensive-queries', + description: 'Most expensive queries finished over last 24 hours', + sql: ` + SELECT + normalized_query_hash, + replace(substr(argMax(query, utime), 1, 200), '\n', ' ') AS query, + count() AS cnt, + sum(query_duration_ms) / 1000 AS queries_duration, + sum(ProfileEvents.Values[indexOf(ProfileEvents.Names, 'RealTimeMicroseconds')]) / 1000000 AS real_time, + sum(ProfileEvents.Values[indexOf(ProfileEvents.Names, 'UserTimeMicroseconds')] AS utime) / 1000000 AS user_time, + sum(ProfileEvents.Values[indexOf(ProfileEvents.Names, 'SystemTimeMicroseconds')]) / 1000000 AS system_time, + sum(ProfileEvents.Values[indexOf(ProfileEvents.Names, 'DiskReadElapsedMicroseconds')]) / 1000000 AS disk_read_time, + sum(ProfileEvents.Values[indexOf(ProfileEvents.Names, 'DiskWriteElapsedMicroseconds')]) / 1000000 AS disk_write_time, + sum(ProfileEvents.Values[indexOf(ProfileEvents.Names, 'NetworkSendElapsedMicroseconds')]) / 1000000 AS network_send_time, + sum(ProfileEvents.Values[indexOf(ProfileEvents.Names, 'NetworkReceiveElapsedMicroseconds')]) / 1000000 AS network_receive_time, + sum(ProfileEvents.Values[indexOf(ProfileEvents.Names, 'ZooKeeperWaitMicroseconds')]) / 1000000 AS zookeeper_wait_time, + sum(ProfileEvents.Values[indexOf(ProfileEvents.Names, 'OSIOWaitMicroseconds')]) / 1000000 AS os_io_wait_time, + sum(ProfileEvents.Values[indexOf(ProfileEvents.Names, 'OSCPUWaitMicroseconds')]) / 1000000 AS os_cpu_wait_time, + sum(ProfileEvents.Values[indexOf(ProfileEvents.Names, 'OSCPUVirtualTimeMicroseconds')]) / 1000000 AS os_cpu_virtual_time, + sum(read_rows) AS read_rows, + formatReadableSize(sum(read_bytes)) AS read_bytes, + sum(written_rows) AS written_rows, + formatReadableSize(sum(written_bytes)) AS written_bytes, + sum(result_rows) AS result_rows, + formatReadableSize(sum(result_bytes)) AS result_bytes + FROM system.query_log + WHERE (event_time > (now() - interval 24 hours)) AND (type IN (2, 4)) + GROUP BY + GROUPING SETS ( + (normalized_query_hash), + ()) + ORDER BY user_time DESC + LIMIT 200 + `, + columns: [ + 'query', + 'cnt', + 'queries_duration', + 'real_time', + 'user_time', + 'system_time', + 'disk_read_time', + 'disk_write_time', + 'network_send_time', + 'network_receive_time', + 'zookeeper_wait_time', + 'os_io_wait_time', + 'os_cpu_wait_time', + 'os_cpu_virtual_time', + 'read_rows', + 'read_bytes', + 'written_rows', + 'written_bytes', + 'result_rows', + 'result_bytes', + ], + columnFormats: { + queries_duration: ColumnFormat.Duration, + real_time: ColumnFormat.Duration, + user_time: ColumnFormat.Duration, + system_time: ColumnFormat.Duration, + disk_read_time: ColumnFormat.Duration, + disk_write_time: ColumnFormat.Duration, + network_send_time: ColumnFormat.Duration, + network_receive_time: ColumnFormat.Duration, + zookeeper_wait_time: ColumnFormat.Duration, + os_io_wait_time: ColumnFormat.Duration, + os_cpu_wait_time: ColumnFormat.Duration, + os_cpu_virtual_time: ColumnFormat.Duration, + read_rows: ColumnFormat.Number, + written_rows: ColumnFormat.Number, + result_rows: ColumnFormat.Number, + }, + relatedCharts: [], + }, { name: 'merges', sql: ` @@ -103,6 +238,46 @@ export const queries: Array = [ default: ColumnFormat.Code, }, }, + { + name: 'settings', + sql: ` + SELECT * + FROM system.settings + ORDER BY name + `, + columns: ['name', 'value', 'changed', 'description', 'default'], + columnFormats: { + name: ColumnFormat.Code, + changed: ColumnFormat.Boolean, + value: ColumnFormat.Code, + default: ColumnFormat.Code, + }, + }, + { + name: 'mergetree-settings', + sql: ` + SELECT name, value, changed, description, readonly, min, max, type, is_obsolete + FROM system.merge_tree_settings + ORDER BY name + `, + columns: [ + 'name', + 'value', + 'changed', + 'description', + 'readonly', + 'min', + 'max', + 'type', + 'is_obsolete', + ], + columnFormats: { + changed: ColumnFormat.Boolean, + readonly: ColumnFormat.Boolean, + is_obsolete: ColumnFormat.Boolean, + value: ColumnFormat.Code, + }, + }, ] export const getQueryConfigByName = (name: string) => { diff --git a/app/tables/page.tsx b/app/tables/page.tsx index b2e5e5be..5ae3c2f8 100644 --- a/app/tables/page.tsx +++ b/app/tables/page.tsx @@ -1,6 +1,7 @@ import { fetchData } from '@/lib/clickhouse' import type { QueryConfig } from '@/lib/types/query-config' import { Tabs, TabsContent, TabsList, TabsTrigger } from '@/components/ui/tabs' +import { ColumnFormat } from '@/components/data-table/columns' import { DataTable } from '@/components/data-table/data-table' export const dynamic = 'force-dynamic' @@ -33,6 +34,9 @@ const config: QueryConfig = { 'readable_total_rows', 'part_count', ], + columnFormats: { + part_count: ColumnFormat.Number, + }, } export default async function TablePage() { diff --git a/components/count-badge.tsx b/components/count-badge.tsx new file mode 100644 index 00000000..4eb22765 --- /dev/null +++ b/components/count-badge.tsx @@ -0,0 +1,32 @@ +import { fetchData } from '@/lib/clickhouse' +import { Badge } from '@/components/ui/badge' + +interface CountBadgeProps { + sql?: string +} + +export async function CountBadge({ + sql, +}: CountBadgeProps): Promise { + if (!sql) return null + + let data: any[] = [] + + try { + data = await fetchData(sql) + } catch (e: any) { + console.error(`Could not get count for sql: ${sql}, error: ${e}`) + return null + } + + if (!data || !data.length || !data?.[0]?.['count()']) return null + + const count = data[0]['count()'] || data[0]['count'] || 0 + if (count === 0) return null + + return ( + + {count} + + ) +} diff --git a/components/data-table/cell.tsx b/components/data-table/cell.tsx index 320405a1..f8a90a46 100644 --- a/components/data-table/cell.tsx +++ b/components/data-table/cell.tsx @@ -2,6 +2,7 @@ import { CheckCircledIcon, CrossCircledIcon } from '@radix-ui/react-icons' import { MoreHorizontal } from 'lucide-react' import dayjs from '@/lib/dayjs' +import { formatReadableQuantity } from '@/lib/format-readable' import { Accordion, AccordionContent, @@ -26,6 +27,12 @@ export const formatCell = (row: any, value: any, format: ColumnFormat) => { case ColumnFormat.Code: return {value} + case ColumnFormat.Number: + return formatReadableQuantity(value, 'long') + + case ColumnFormat.NumberShort: + return formatReadableQuantity(value, 'short') + case ColumnFormat.CodeToggle: if (value.length < CODE_TRUNCATE_LENGTH) { return {value} @@ -48,8 +55,13 @@ export const formatCell = (row: any, value: any, format: ColumnFormat) => { ) + case ColumnFormat.RelatedTime: + let fromNow = dayjs(value).fromNow() + return {fromNow} + case ColumnFormat.Duration: - return dayjs.duration({ seconds: parseFloat(value) }).humanize() + let humanized = dayjs.duration({ seconds: parseFloat(value) }).humanize() + return {humanized} case ColumnFormat.Boolean: return value ? ( diff --git a/components/data-table/columns.tsx b/components/data-table/columns.tsx index 4ee58510..df686ffb 100644 --- a/components/data-table/columns.tsx +++ b/components/data-table/columns.tsx @@ -11,7 +11,10 @@ import { formatCell } from '@/components/data-table/cell' export enum ColumnFormat { Code = 'code', + Number = 'number', + NumberShort = 'number-short', CodeToggle = 'code-toggle', + RelatedTime = 'related-time', Duration = 'duration', Boolean = 'boolean', Action = 'action', diff --git a/components/data-table/data-table.tsx b/components/data-table/data-table.tsx index 6f8124c1..f17c9d86 100644 --- a/components/data-table/data-table.tsx +++ b/components/data-table/data-table.tsx @@ -101,7 +101,10 @@ export function DataTable({ return (
-

{title}

+
+

{title}

+

{config.description}

+
{showSQL ? : null} diff --git a/components/menu.tsx b/components/menu.tsx index 80ac6a23..a3594140 100644 --- a/components/menu.tsx +++ b/components/menu.tsx @@ -1,9 +1,8 @@ import React from 'react' import Link from 'next/link' -import { fetchData, QUERY_COMMENT } from '@/lib/clickhouse' +import { QUERY_COMMENT } from '@/lib/clickhouse' import { cn } from '@/lib/utils' -import { Badge } from '@/components/ui/badge' import { NavigationMenu, NavigationMenuContent, @@ -13,6 +12,7 @@ import { NavigationMenuTrigger, navigationMenuTriggerClasses, } from '@/components/ui/navigation-menu' +import { CountBadge } from '@/components/count-badge' interface MenuProps { items?: MenuItem[] @@ -48,6 +48,11 @@ const defaultItems = [ description: 'Queries that have been run including successed, failed queries with resourses usage details', }, + { + title: 'Most Expensive Queries', + href: '/expensive-queries', + description: 'Most expensive queries in my ClickHouse', + }, ], }, { @@ -62,7 +67,21 @@ const defaultItems = [ }, { title: 'Settings', - href: '/settings', + href: '', + items: [ + { + title: 'Settings', + href: '/settings', + description: + 'The values of global server settings which can be viewed in the table `system.settings`', + }, + { + title: 'MergeTree Settings', + href: '/settings', + description: + 'The values of merge_tree settings (for all MergeTree tables) which can be viewed in the table `system.merge_tree_settings`', + }, + ], }, ] @@ -78,22 +97,6 @@ export function Menu({ items = defaultItems }: MenuProps) { ) } -async function MenuCounter({ sql }: { sql?: string }) { - if (!sql) return null - - const data = await fetchData(sql) - if (!data || !data.length || !data?.[0]?.['count()']) return null - - const count = data[0]['count()'] || data[0]['count'] || 0 - if (count === 0) return null - - return ( - - {count} - - ) -} - function MenuItem({ item }: { item: MenuItem }) { if (item.items) { return @@ -108,7 +111,7 @@ function SingleItem({ item }: { item: MenuItem }) { {item.title} - + @@ -119,16 +122,16 @@ function HasChildItems({ item }: { item: MenuItem }) { return ( - {item.title} + {item.title} -
    +
      {item.items?.map((childItem) => ( - {childItem.title} + {childItem.title} } href={childItem.href} diff --git a/cypress/e2e/navigation.cy.js b/cypress/e2e/navigation.cy.js index b26e4edd..cbd1b929 100644 --- a/cypress/e2e/navigation.cy.js +++ b/cypress/e2e/navigation.cy.js @@ -2,8 +2,4 @@ describe('Navigation', () => { it('should navigate to the /tables page', () => { cy.visitAndValidate('/overview', 'a[href*="merges"]', '/merges', 'Merges') }) - - it('should navigate to the /settings page', () => { - cy.visitAndValidate('/overview', 'a[href*="settings"]', '/settings', 'Settings') - }) }) diff --git a/lib/format-readable.ts b/lib/format-readable.ts index 526d1db4..4558b650 100644 --- a/lib/format-readable.ts +++ b/lib/format-readable.ts @@ -21,10 +21,34 @@ export function formatReadableSize(bytes: number, decimals = 1) { return `${parseFloat((bytes / Math.pow(k, i)).toFixed(dm))} ${sizes[i]}` } -export function formatReadableQuantity(quantity: number) { +/** + * Format a number to a human readable quantity. + * For example: + * formatReadableQuantity(123456789) => 123.5M + * formatReadableQuantity(123456789, 'short') => 123.5M + * formatReadableQuantity(123456789, 'long') => 123,456,789 + * + * @param quantity + * @param preset 'short' or 'long', defaults to 'short' + * @returns + */ +export function formatReadableQuantity( + quantity: number, + preset: string = 'short' +) { + const options = + preset === 'short' + ? { + notation: 'compact', + compactDisplay: 'short', + } + : { + notation: 'standard', + compactDisplay: 'long', + } + return new Intl.NumberFormat('en-US', { style: 'decimal', - notation: 'compact', - compactDisplay: 'short', + ...options, }).format(quantity) } diff --git a/lib/types/query-config.ts b/lib/types/query-config.ts index b4e036e6..820c6ffd 100644 --- a/lib/types/query-config.ts +++ b/lib/types/query-config.ts @@ -3,6 +3,7 @@ import type { ColumnFormat } from '@/components/data-table/columns' export interface QueryConfig { name: string + description?: string sql: string columns: string[] columnFormats?: { [key: string]: ColumnFormat }