diff --git a/.env.example b/.env.example index 13bf1fdf..a9f92080 100644 --- a/.env.example +++ b/.env.example @@ -9,4 +9,5 @@ CLICKHOUSE_PASSWORD= CLICKHOUSE_NAME= CLICKHOUSE_TZ= CLICKHOUSE_MAX_EXECUTION_TIME=60 -NEXT_QUERY_CACHE_TTL=86400 \ No newline at end of file +NEXT_QUERY_CACHE_TTL=86400 +CLICKHOUSE_EXCLUDE_USER_DEFAULT= \ No newline at end of file diff --git a/README.md b/README.md index 50456176..f0e27e83 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,15 @@ Features: **Documentation**: - https://duyet.github.io/clickhouse-monitoring/ + - [Getting Started](https://duyet.github.io/clickhouse-monitoring/getting-started) + - [Local Development](https://duyet.github.io/clickhouse-monitoring/getting-started/local) + - [User Role and Profile](https://duyet.github.io/clickhouse-monitoring/getting-started/clickhouse-requirements) + - [Enable System Tables](https://duyet.github.io/clickhouse-monitoring/getting-started/clickhouse-enable-system-tables) + - [Deployments](https://duyet.github.io/clickhouse-monitoring/deploy) + - [Vercel](https://duyet.github.io/clickhouse-monitoring/deploy/vercel) + - [Docker](https://duyet.github.io/clickhouse-monitoring/deploy/docker) + - [Kubernetes Helm Chart](https://duyet.github.io/clickhouse-monitoring/deploy/k8s) + - [Advanced](https://duyet.github.io/clickhouse-monitoring/advanced) **Screenshots**: @@ -33,130 +42,6 @@ Features: ![](.github/screenshots/screenshot_7.png) ![](.github/screenshots/screenshot_8.png) -## Getting Started - -To get the project up and running on your local machine, follow these steps: - -1. Clone the repository -2. Install dependencies using `npm install` or `yarn install` -3. Create a `.env.local` file by copying the `.env.example` file and filling in the required environment variables: - - - `CLICKHOUSE_HOST`: ClickHouse host(s), for example `http://localhost:8123` or `http://ch-1:8123,http://ch-2:8123` - - `CLICKHOUSE_NAME`: (Optional) Name of ClickHouse instance, must match the number of hosts in `CLICKHOUSE_HOST`, for example `localhost` or `ch-1,ch-2`. - - `CLICKHOUSE_USER`: ClickHouse user with permission to query the `system` database. - - `CLICKHOUSE_PASSWORD`: ClickHouse password for the specified user. - - `CLICKHOUSE_MAX_EXECUTION_TIME`: [`max_execution_time`](https://clickhouse.com/docs/en/operations/settings/query-complexity#max-execution-time) timeout in seconds. Default is `10`. - - `CLICKHOUSE_TZ`: ClickHouse server timezone. Default: `''`. - - `NEXT_QUERY_CACHE_TTL`: TTL of [`unstable_cache`](https://nextjs.org/docs/app/api-reference/functions/unstable_cache) - cache the results of most charts to speed up and reuse them across multiple requests. Default: `86400`. - - `NEXT_PUBLIC_LOGO`: (Optional) HTTP path to logo image. - - `EVENTS_TABLE_NAME`: The table name for storing dashboard self-tracking events. Default: `system.monitoring_events` - -4. Run the development server with `npm run dev` or `yarn dev` -5. Open [http://localhost:3000](http://localhost:3000) in your browser to see the dashboard. - -## ClickHouse Requirements - -### 1. Monitoring user role - -Suggested role for "monitoring" user must have these privileges on `system` database: - -```xml -# File: users.d/monitoring_role.xml - - - - - monitoring_profile - ::/0 - - GRANT monitoring_role - - - - - - - - REVOKE ALL ON *.* - GRANT SELECT,SHOW,dictGet,REMOTE ON *.* - GRANT SELECT,INSERT,ALTER,CREATE,DROP,TRUNCATE,OPTIMIZE,SHOW,dictGet ON system.* - GRANT CREATE TEMPORARY TABLE ON *.* - - - - -``` - -`CREATE TEMPORARY TABLE` is needed because the UI using `FROM merge(system, '^query_log')` allows retrieving all the data from old tables that were renamed during the upgrade. - -### 2. Monitoring user profile - -```xml -# File: users.d/monitoring_profile.xml - - - - 1 - - - 1 - 50 - 0 - save - save - - - -``` - -## Deployment - -### 1. Vercel - -For easy deployment, use the [Vercel Platform](https://vercel.com/new?utm_medium=default-template&filter=next.js&utm_source=create-next-app&utm_campaign=create-next-app-readme), created by the makers of Next.js. Refer to the [Next.js deployment documentation](https://nextjs.org/docs/deployment) for more details. - -### 2. Docker - -Using the latest image here: https://github.com/duyet/clickhouse-monitoring/pkgs/container/clickhouse-monitoring - -```bash -docker run -it \ - -e CLICKHOUSE_HOST='http://localhost' \ - -e CLICKHOUSE_USER='default' \ - -e CLICKHOUSE_PASSWORD='' \ - -e CLICKHOUSE_TZ='Asia/Ho_Chi_Minh' \ - -e CLICKHOUSE_MAX_EXECUTION_TIME='15' \ - -e NEXT_QUERY_CACHE_TTL='86400' \ - --name clickhouse-monitoring \ - ghcr.io/duyet/clickhouse-monitoring:main -``` - -### 3. Kubernetes Helm Chart - -Using the latest helm chart here: https://github.com/duyet/charts/tree/master/clickhouse-monitoring - -```bash -helm repo add duyet https://duyet.github.io/charts - -cat <> values.yaml -env: - - name: CLICKHOUSE_HOST - value: http://localhost:8123 - - name: CLICKHOUSE_USER - value: default - - name: CLICKHOUSE_PASSWORD - value: '' - - name: CLICKHOUSE_TZ - value: 'Asia/Ho_Chi_Minh' - - name: CLICKHOUSE_MAX_EXECUTION_TIME - value: '15' - - name: NEXT_QUERY_CACHE_TTL - value: '86400' -EOF - -helm install -f values.yaml clickhouse-monitoring-release duyet/clickhouse-monitoring -``` - ## Feedback and Contributions Feedback and contributions are welcome! Feel free to open issues or submit pull requests. diff --git a/app/[host]/[query]/queries/history-queries.ts b/app/[host]/[query]/queries/history-queries.ts index 6cc0f2e2..f5b228fa 100644 --- a/app/[host]/[query]/queries/history-queries.ts +++ b/app/[host]/[query]/queries/history-queries.ts @@ -37,6 +37,7 @@ export const historyQueriesConfig: QueryConfig = { AND if (notEmpty({event_time: String}), toDate(event_time) = {event_time: String}, true) AND if ({database: String} != '' AND {table: String} != '', has(tables, format('{}.{}', {database: String}, {table: String})), true) AND if ({user: String} != '', user = {user: String}, true) + AND if ({excluded_users: String} != '', not(has(splitByChar(',', {excluded_users: String}), user)), true) ORDER BY event_time DESC LIMIT 1000 `, @@ -86,6 +87,7 @@ export const historyQueriesConfig: QueryConfig = { database: '', table: '', user: '', + excluded_users: process.env.CLICKHOUSE_EXCLUDE_USER_DEFAULT || '', }, filterParamPresets: [ @@ -114,6 +116,11 @@ export const historyQueriesConfig: QueryConfig = { key: 'duration_1m', value: '1', }, + { + name: `user != ${process.env.CLICKHOUSE_EXCLUDE_USER_DEFAULT || ''}`, + key: 'excluded_users', + value: process.env.CLICKHOUSE_EXCLUDE_USER_DEFAULT || '', + }, ], relatedCharts: [ diff --git a/components/data-table/data-table-faceted-filter.tsx b/components/data-table/data-table-faceted-filter.tsx index 3147fc57..3efaf075 100644 --- a/components/data-table/data-table-faceted-filter.tsx +++ b/components/data-table/data-table-faceted-filter.tsx @@ -30,16 +30,25 @@ export function DataTableFacetedFilter({ const pathname = usePathname() const { filterParamPresets = [], defaultParams = {} } = queryConfig - const selected = useMemo( - () => new URLSearchParams(searchParams), - [searchParams] - ) + const selected = useMemo(() => { + const params = new URLSearchParams(searchParams) + + // Add default params have not null value + Object.entries(defaultParams).forEach(([key, value]) => { + if (value !== '' && !params.has(key)) { + params.set(key, value as string) + } + }) + + return params + }, [searchParams, defaultParams]) + console.log('selected', selected.toString()) const filters = useMemo< NonNullable >(() => { const filterNotFromPreset = Object.keys(defaultParams) - // key in URL Params + // Key in URL Params .filter((key) => selected.has(key)) // And custom that value is not in presets .filter( @@ -50,8 +59,10 @@ export function DataTableFacetedFilter({ ) ) .map((key) => ({ - name: `${key} = ${selected.get(key)} *`, key, + name: selected.get(key) + ? filterParamPresets.find((preset) => preset.key === key)?.name + : `${key} = N/A`, value: selected.get(key), })) as NonNullable console.log('filterNotFromPreset', filterNotFromPreset) @@ -90,7 +101,13 @@ export function DataTableFacetedFilter({ name={name} value={value} isSelected={selected.get(key) === value} - href={getUpdatedHref(pathname, searchParams, key, value)} + href={getUpdatedHref( + pathname, + searchParams, + key, + value, + defaultParams + )} icon={preset?.icon} filterKey={key} filterValue={value} @@ -133,9 +150,8 @@ function FilterMenuItem({ }: FilterMenuItemProps) { return ( - {Icon && } {name} - + ) } @@ -154,12 +170,34 @@ function getUpdatedHref( pathname: string, searchParams: URLSearchParams, key: string, - value: string + value: string, + defaultParams: QueryConfig['defaultParams'] ) { const newParams = new URLSearchParams(searchParams) if (newParams.get(key) === value) { - newParams.delete(key) + /** + * For example the query config has defaultParams = { type: 'abc' } + * and the URL has ?type=abc. If the user clicks on the filter to toggle it off, + * We should set the URL to ?type= instead of removing the key completely, + * as the default value is still 'abc'. + */ + if (defaultParams?.[key] !== '') { + newParams.set(key, '') + } else { + /** + * If the default value is an empty string, we can just remove the key from the URL + * as there is no default value to fall back to. + * + * For example, if the query config has defaultParams = { type: '' } + * and the URL has ?type=abc. If the user clicks on the filter to toggle it off, + * We can set the URL to ? instead of ?type=abc. + */ + newParams.delete(key) + } } else { + /** + * If the filter is not selected, we should set the URL to ?key=value + */ newParams.set(key, value) } return `${pathname}?${newParams.toString()}` diff --git a/docs/pages/advanced/_meta.ts b/docs/pages/advanced/_meta.ts index 3f38a055..bd3b289d 100644 --- a/docs/pages/advanced/_meta.ts +++ b/docs/pages/advanced/_meta.ts @@ -1,6 +1,8 @@ export const meta = { 'multiple-hosts': 'Multiple Hosts', - 'custom-name': 'Custom ClickHouse Name', + 'custom-name': 'Custom Host Name', + 'queries-history': 'Queries History', + 'self-tracking': 'Self-Tracking', } export default meta diff --git a/docs/pages/advanced/multiple-hosts.mdx b/docs/pages/advanced/multiple-hosts.mdx index 13787f39..c763db3e 100644 --- a/docs/pages/advanced/multiple-hosts.mdx +++ b/docs/pages/advanced/multiple-hosts.mdx @@ -18,18 +18,26 @@ All these hosts will share the same `CLICKHOUSE_USER` and `CLICKHOUSE_PASSWORD`. - ```bash docker run -it \ -e - CLICKHOUSE_HOST='http://ch-1:8123,http://ch-2:8123' \ -e - CLICKHOUSE_NAME='ch-1,ch-2' \ -e CLICKHOUSE_USER='default' \ -e - CLICKHOUSE_PASSWORD='' \ --name clickhouse-monitoring \ - ghcr.io/duyet/clickhouse-monitoring:main ``` + ```bash + docker run -it \ + -e CLICKHOUSE_HOST='http://ch-1:8123,http://ch-2:8123' \ + -e CLICKHOUSE_NAME='ch-1,ch-2' \ + -e CLICKHOUSE_USER='default' \ + -e CLICKHOUSE_PASSWORD='' \ + --name clickhouse-monitoring \ + ghcr.io/duyet/clickhouse-monitoring:main + ``` - ```bash docker run -it \ -e - CLICKHOUSE_HOST='http://ch-1:8123,http://ch-2:8123' \ -e - CLICKHOUSE_NAME='ch-1,ch-2' \ -e CLICKHOUSE_USER='user1,user2' \ -e - CLICKHOUSE_PASSWORD='password1,password2' \ --name clickhouse-monitoring \ - ghcr.io/duyet/clickhouse-monitoring:main ``` + ```bash + docker run -it \ + -e CLICKHOUSE_HOST='http://ch-1:8123,http://ch-2:8123' \ + -e CLICKHOUSE_NAME='ch-1,ch-2' \ + -e CLICKHOUSE_USER='user1,user2' \ + -e CLICKHOUSE_PASSWORD='password1,password2' \ + --name clickhouse-monitoring \ + ghcr.io/duyet/clickhouse-monitoring:main + ``` diff --git a/docs/pages/advanced/queries-history.mdx b/docs/pages/advanced/queries-history.mdx new file mode 100644 index 00000000..66d1d49e --- /dev/null +++ b/docs/pages/advanced/queries-history.mdx @@ -0,0 +1,50 @@ +import { Tabs } from 'nextra/components' + +# Query History + +## 1. Excluding monitoring users by default + +![](/excluding-users.png) + +You can configured the [`/history-queries`](http://clickhouse-monitoring.vercel.app/0/history-queries) page to filter out the `monitoring` users by default. +To do this, using these environment variables: + +- `CLICKHOUSE_EXCLUDE_USER_DEFAULT`: for example `monitoring,healthcheck`. + +Examples + + + + ```bash + docker run -it \ + -e CLICKHOUSE_HOST='http://ch-1:8123' \ + -e CLICKHOUSE_NAME='ch-1' \ + -e CLICKHOUSE_USER='monitoring' \ + -e CLICKHOUSE_PASSWORD='' \ + -e CLICKHOUSE_EXCLUDE_USER_DEFAULT='monitoring,healthcheck' \ + --name clickhouse-monitoring \ + ghcr.io/duyet/clickhouse-monitoring:main + ``` + + + ```bash + helm repo add duyet https://duyet.github.io/charts + + cat <> values.yaml + env: + - name: CLICKHOUSE_HOST + value: http://ch-1:8123 + - name: CLICKHOUSE_NAME + value: ch-1 + - name: CLICKHOUSE_USER + value: monitoring + - name: CLICKHOUSE_PASSWORD + value: '' + - name: CLICKHOUSE_EXCLUDE_USER_DEFAULT + value: monitoring,healthcheck + EOF + + helm install -f values.yaml clickhouse-monitoring-release duyet/clickhouse-monitoring + ``` + + diff --git a/docs/pages/advanced/self-tracking.mdx b/docs/pages/advanced/self-tracking.mdx new file mode 100644 index 00000000..93b321a0 --- /dev/null +++ b/docs/pages/advanced/self-tracking.mdx @@ -0,0 +1,12 @@ +# Dashboard Self Tracking + +The monitoring dashboard tracks user behavior by default. It's useful for debugging and improving the dashboard. + +Use these environment variables to configure the dashboard's self-tracking: + +- `EVENTS_TABLE_NAME`: The table name for storing dashboard self-tracking events. + Default: [`system.monitoring_events`](https://clickhouse-monitoring.vercel.app/0/database/system/monitoring_events) + +![](/self-tracking-1.png) + +![](/self-tracking-2.png) \ No newline at end of file diff --git a/docs/public/excluding-users.png b/docs/public/excluding-users.png new file mode 100644 index 00000000..4479fdd2 Binary files /dev/null and b/docs/public/excluding-users.png differ diff --git a/docs/public/self-tracking-1.png b/docs/public/self-tracking-1.png new file mode 100644 index 00000000..c808091d Binary files /dev/null and b/docs/public/self-tracking-1.png differ diff --git a/docs/public/self-tracking-2.png b/docs/public/self-tracking-2.png new file mode 100644 index 00000000..11455aff Binary files /dev/null and b/docs/public/self-tracking-2.png differ diff --git a/package.json b/package.json index dd16886b..2d7efe12 100644 --- a/package.json +++ b/package.json @@ -18,7 +18,7 @@ "e2e:headless": "start-server-and-test dev http://localhost:3000 \"cypress run --e2e\"", "component": "cypress open --component", "component:headless": "cypress run --component", - "fmt": "prettier --write \"**/*.{ts,tsx,md,mdx,json}\" --cache", + "fmt": "prettier --write \"**/*.{ts,tsx,md,json}\" --cache", "test": "yarn jest", "jest": "jest --coverage --testPathIgnorePatterns=query-config", "test-queries-config": "jest --coverage --testPathPattern=query-config"