From 662ebd871a2f4236e108a5a6bb1e52f2a0375058 Mon Sep 17 00:00:00 2001 From: WUJingdi Date: Thu, 4 Jan 2024 16:33:04 +0800 Subject: [PATCH 1/3] docs: export metrics --- .../en/user-guide/operations/configuration.md | 36 ++++++++++++++++++ .../zh/user-guide/operations/configuration.md | 37 +++++++++++++++++++ 2 files changed, 73 insertions(+) diff --git a/docs/v0.5/en/user-guide/operations/configuration.md b/docs/v0.5/en/user-guide/operations/configuration.md index 424f0d577..8eea738e1 100644 --- a/docs/v0.5/en/user-guide/operations/configuration.md +++ b/docs/v0.5/en/user-guide/operations/configuration.md @@ -451,6 +451,42 @@ append_stdout = true How to use distributed tracing, please reference [Tracing](./tracing.md) +### Export metrics options + +`frontend`, `metasrv`, `datanode` and `standalone` support collecting metrics generated by themselves and sending the collected metrics to [Prometheus RemoteWrite protocol](https://prometheus.io/docs/concepts/remote_write_spec/) compatible receiver, such as sending to `greptimedb` itself. **Note: Export metrics has nothing to do with using Prometheus to scape metrics generated by `greptimedb`. Export metrics is only used as a way for `greptimedb` to export the metrics generated by itself.** Users can configure related export parameters through the `[export_metrics]` section. + +There are two ways to export metrics data: +1. `remote_write` method: All components can be exported using this method. The user specifies a URL that supports the Prometheus RemoteWrite protocol through the relevant configuration under `[export_metrics.remote_write]`, and sends the metrics to the receiver through the URL. +2. `self_import` method: Only `frontend` and `standalone` support exporting metrics using `self_import` method. Choosing the `self_import` method means that the user specifies that the collected metrics will be sent back to `greptimedb` itself. Users can export metrics by specifying the database name to which the metrics is exported through the relevant configuration under `[export_metrics.self_import]`. + +A configuration file exported using `remote_write` method: + +```toml +[export_metrics] +# Whether to enable export_metrics +enable=true +# Export time interval +write_interval = "30s" +[export_metrics.remote_write] +# URL specified by Prometheus RemoteWrite protocol +url = "http://127.0.0.1:4000/v1/prometheus/write?db=information_schema" +# Some optional HTTP parameters, such as authentication information +headers = { Authorization = "Basic Z3JlcHRpbWVfdXNlcjpncmVwdGltZV9wd2Q=" } +``` + +A configuration file exported using `self_import`: + +```toml +[export_metrics] +# Whether to enable export_metrics +enable=true +# Export time interval +write_interval = "30s" +[export_metrics.self_import] +#Select the imported database name +db = "information_schema" +``` + ## Environment variable Every item in the configuration file can be mapped into environment variables. For example, if we want to set the configuration item `max_inflight_tasks` of datanode by environment variable: diff --git a/docs/v0.5/zh/user-guide/operations/configuration.md b/docs/v0.5/zh/user-guide/operations/configuration.md index 2f7279ce5..b6c43945b 100644 --- a/docs/v0.5/zh/user-guide/operations/configuration.md +++ b/docs/v0.5/zh/user-guide/operations/configuration.md @@ -450,6 +450,43 @@ append_stdout = true 如何使用分布式追踪，请参考 [Tracing](./tracing.md) +### Export metrics 选项 + +`frontend`、`metasrv`、`datanode` 和 `standalone` 支持收集自身产生的 metrics 数据，并将采集到的 metrics 数据发送到[Prometheus RemoteWrite 协议](https://prometheus.io/docs/concepts/remote_write_spec/)兼容的接收端，比如发送到 `greptimedb` 自身。**注意：该功能和使用 Prometheus 收集 `greptimedb` 产生的 metrics 没有关系，该功能仅作为 `greptimedb` 导出自身产生 metrics 的一种方式。** 用户可以通过 `[export_metrics]` 部分配置相关导出参数。 + +总共有两种方式用于导出 metrics 数据： +1. `remote_write` 方式：所有组件均可使用该方式导出。用户通过 `[export_metrics.remote_write]` 下的相关配置，指定一个支持 Prometheus RemoteWrite 协议的 URL，将产生的数据通过 URL 发送到接收端。 +2. `self_import` 方式：只有 `frontend`、`standalone` 支持使用 `self_import` 方式导出数据。选择 `self_import` 方式意味着用户指定了将收集到的 metrics 数据又送回到 `greptimedb` 自身。用户通过 `[export_metrics.self_import]` 下的相关配置，指定数据导出到的数据库库名，即可将数据导出到自身。 + +一份使用 `remote_write` 方式导出的配置文件： + +```toml +[export_metrics] +# 是否启用 export_metrics +enable = true +# 导出时间间隔 +write_interval = "30s" +[export_metrics.remote_write] +# Prometheus RemoteWrite 协议规定的 URL +url = "http://127.0.0.1:4000/v1/prometheus/write?db=information_schema" +# 一些可选的 HTTP 参数，如鉴权信息 +headers = { Authorization = "Basic Z3JlcHRpbWVfdXNlcjpncmVwdGltZV9wd2Q=" } +``` + + +一份使用 `self_import` 方式导出的配置文件： + +```toml +[export_metrics] +# 是否启用 export_metrics +enable = true +# 导出时间间隔 +write_interval = "30s" +[export_metrics.self_import] +# 选择导入的数据库库名 +db = "information_schema" +``` + ## 环境变量配置 配置文件中的每一项都可以映射到环境变量。例如，如果我们想通过环境变量设置 datanode 的配置项 `data_home`： From 2efcf4434633f86b198f74198890e9f2b22451f4 Mon Sep 17 00:00:00 2001 From: WUJingdi Date: Fri, 5 Jan 2024 10:46:04 +0800 Subject: [PATCH 2/3] chore: fix code advice --- .../en/user-guide/operations/configuration.md | 24 +++++++++++++++---- .../zh/user-guide/operations/configuration.md | 19 ++++++++++++--- 2 files changed, 35 insertions(+), 8 deletions(-) diff --git a/docs/v0.5/en/user-guide/operations/configuration.md b/docs/v0.5/en/user-guide/operations/configuration.md index 8eea738e1..fec1a1d15 100644 --- a/docs/v0.5/en/user-guide/operations/configuration.md +++ b/docs/v0.5/en/user-guide/operations/configuration.md @@ -453,11 +453,17 @@ How to use distributed tracing, please reference [Tracing](./tracing.md) ### Export metrics options -`frontend`, `metasrv`, `datanode` and `standalone` support collecting metrics generated by themselves and sending the collected metrics to [Prometheus RemoteWrite protocol](https://prometheus.io/docs/concepts/remote_write_spec/) compatible receiver, such as sending to `greptimedb` itself. **Note: Export metrics has nothing to do with using Prometheus to scape metrics generated by `greptimedb`. Export metrics is only used as a way for `greptimedb` to export the metrics generated by itself.** Users can configure related export parameters through the `[export_metrics]` section. +`frontend`, `metasrv`, `datanode` and `standalone` support collecting metrics generated by themselves and sending to [Prometheus RemoteWrite protocol](https://prometheus.io/docs/concepts/remote_write_spec/) compatible receiver, such as sending to `greptimedb` itself. You can configure related export parameters through the `[export_metrics]` section. -There are two ways to export metrics data: -1. `remote_write` method: All components can be exported using this method. The user specifies a URL that supports the Prometheus RemoteWrite protocol through the relevant configuration under `[export_metrics.remote_write]`, and sends the metrics to the receiver through the URL. -2. `self_import` method: Only `frontend` and `standalone` support exporting metrics using `self_import` method. Choosing the `self_import` method means that the user specifies that the collected metrics will be sent back to `greptimedb` itself. Users can export metrics by specifying the database name to which the metrics is exported through the relevant configuration under `[export_metrics.self_import]`. +:::tip NOTE +Note: Export metrics has nothing to do with using Prometheus to export metrics generated by `greptimedb`. Learn how to export `greptimedb` metrics to Prometheus in [monitoring](./monitoring.md). Export metrics is only used as a way for `greptimedb` to export the metrics generated by itself. +::: + +There are two ways to export metrics: + +#### `remote_write` method + +All components can be exported using this method. An URL is needed to indicate the receiver through the relevant configuration under `[export_metrics.remote_write]`, metrics will be send to the receiver through the URL. A configuration file exported using `remote_write` method: @@ -474,6 +480,14 @@ url = "http://127.0.0.1:4000/v1/prometheus/write?db=information_schema" headers = { Authorization = "Basic Z3JlcHRpbWVfdXNlcjpncmVwdGltZV9wd2Q=" } ``` +#### `self_import` method + +:::tip NOTE +Only `frontend` and `standalone` support exporting metrics using `self_import` method. +::: + +Choosing the `self_import` method means that the collected metrics will be sent back to `greptimedb` itself. You can export metrics by specifying the database name to which the metrics is exported through the relevant configuration under `[export_metrics.self_import]`. + A configuration file exported using `self_import`: ```toml @@ -483,7 +497,7 @@ enable=true # Export time interval write_interval = "30s" [export_metrics.self_import] -#Select the imported database name +# Select the imported database name db = "information_schema" ``` diff --git a/docs/v0.5/zh/user-guide/operations/configuration.md b/docs/v0.5/zh/user-guide/operations/configuration.md index b6c43945b..0521d413e 100644 --- a/docs/v0.5/zh/user-guide/operations/configuration.md +++ b/docs/v0.5/zh/user-guide/operations/configuration.md @@ -452,11 +452,17 @@ append_stdout = true ### Export metrics 选项 -`frontend`、`metasrv`、`datanode` 和 `standalone` 支持收集自身产生的 metrics 数据，并将采集到的 metrics 数据发送到[Prometheus RemoteWrite 协议](https://prometheus.io/docs/concepts/remote_write_spec/)兼容的接收端，比如发送到 `greptimedb` 自身。**注意：该功能和使用 Prometheus 收集 `greptimedb` 产生的 metrics 没有关系，该功能仅作为 `greptimedb` 导出自身产生 metrics 的一种方式。** 用户可以通过 `[export_metrics]` 部分配置相关导出参数。 +`frontend`、`metasrv`、`datanode` 和 `standalone` 支持收集自身产生的 metrics 数据，并将采集到的 metrics 数据发送到[Prometheus RemoteWrite 协议](https://prometheus.io/docs/concepts/remote_write_spec/)兼容的接收端，比如发送到 `greptimedb` 自身。 用户可以通过 `[export_metrics]` 配置相关导出参数。 + +:::tip NOTE +注意：该功能和使用 Prometheus 收集 `greptimedb` 产生的 metrics 没有关系，该功能仅作为 `greptimedb` 导出自身产生 metrics 的一种方式。 +::: 总共有两种方式用于导出 metrics 数据： -1. `remote_write` 方式：所有组件均可使用该方式导出。用户通过 `[export_metrics.remote_write]` 下的相关配置，指定一个支持 Prometheus RemoteWrite 协议的 URL，将产生的数据通过 URL 发送到接收端。 -2. `self_import` 方式：只有 `frontend`、`standalone` 支持使用 `self_import` 方式导出数据。选择 `self_import` 方式意味着用户指定了将收集到的 metrics 数据又送回到 `greptimedb` 自身。用户通过 `[export_metrics.self_import]` 下的相关配置，指定数据导出到的数据库库名，即可将数据导出到自身。 + +#### `remote_write` 方式 + +所有组件均可使用该方式导出。用户通过 `[export_metrics.remote_write]` 下的相关配置，指定一个支持 Prometheus RemoteWrite 协议的 URL，将产生的数据通过 URL 发送到接收端。 一份使用 `remote_write` 方式导出的配置文件： @@ -473,6 +479,13 @@ url = "http://127.0.0.1:4000/v1/prometheus/write?db=information_schema" headers = { Authorization = "Basic Z3JlcHRpbWVfdXNlcjpncmVwdGltZV9wd2Q=" } ``` +#### `self_import` 方式 + +:::tip NOTE +只有 `frontend`、`standalone` 支持使用 `self_import` 方式导出数据。 +::: + +选择 `self_import` 方式意味着用户指定了将收集到的 metrics 数据又送回到 `greptimedb` 自身。用户通过 `[export_metrics.self_import]` 下的相关配置，指定数据导出到的数据库库名，即可将数据导出到自身。 一份使用 `self_import` 方式导出的配置文件： From c0c7b2a67a1bde85c49f98bad768bfffbbd6e608 Mon Sep 17 00:00:00 2001 From: WUJingdi Date: Thu, 11 Jan 2024 10:25:04 +0800 Subject: [PATCH 3/3] docs: add describe in monitor --- .../en/user-guide/operations/configuration.md | 2 +- .../en/user-guide/operations/monitoring.md | 51 ++++++++++++++++++ .../zh/user-guide/operations/configuration.md | 2 +- .../zh/user-guide/operations/monitoring.md | 52 +++++++++++++++++++ 4 files changed, 105 insertions(+), 2 deletions(-) diff --git a/docs/v0.5/en/user-guide/operations/configuration.md b/docs/v0.5/en/user-guide/operations/configuration.md index fec1a1d15..1d51349a7 100644 --- a/docs/v0.5/en/user-guide/operations/configuration.md +++ b/docs/v0.5/en/user-guide/operations/configuration.md @@ -456,7 +456,7 @@ How to use distributed tracing, please reference [Tracing](./tracing.md) `frontend`, `metasrv`, `datanode` and `standalone` support collecting metrics generated by themselves and sending to [Prometheus RemoteWrite protocol](https://prometheus.io/docs/concepts/remote_write_spec/) compatible receiver, such as sending to `greptimedb` itself. You can configure related export parameters through the `[export_metrics]` section. :::tip NOTE -Note: Export metrics has nothing to do with using Prometheus to export metrics generated by `greptimedb`. Learn how to export `greptimedb` metrics to Prometheus in [monitoring](./monitoring.md). Export metrics is only used as a way for `greptimedb` to export the metrics generated by itself. +Please refer to [Monitoring](./monitoring.md#use-greptimedb-to-monitor-greptimedb) to learn how to use export metrics to monitor `greptimedb` ::: There are two ways to export metrics: diff --git a/docs/v0.5/en/user-guide/operations/monitoring.md b/docs/v0.5/en/user-guide/operations/monitoring.md index c3823c2c6..efea54408 100644 --- a/docs/v0.5/en/user-guide/operations/monitoring.md +++ b/docs/v0.5/en/user-guide/operations/monitoring.md @@ -65,6 +65,57 @@ gtctl cluster create mycluster -n default \ --set cluster.prometheusMonitor.labelsSelector.release="prometheus" ``` +## Use GreptimeDB to monitor GreptimeDB + +In most cases, using Prometheus can provide good observation of metrics in GreptimeDB. But if the user does not want to rely on Prometheus to process various metrics generated by GreptimeDB. GreptimeDB provides `export metrics`, which is used to store the metrics generated by GreptimeDB directly into GreptimeDB, thereby avoiding dependence on Prometheus. For configuration of `export metrics`, please see [export metrics configuration](./configuration.md#export-metrics-options) + +### Standalone + +In standalone mode, it is recommended to use `self_import` to export metrics. The default database name imported by `self_import` is `information_schema`. In order to avoid mixing metrics with other tables in the `information_schema`, users can create a new database first, such as `system`. + +```sql +CREATE DATABASE system; +``` + +The final configuration looks like this: + +```toml +[export_metrics] +enable=true +write_interval = "30s" +[export_metrics.self_import] +db = "system" +``` + +### Cluster + +In cluster mode, it is recommended that `frontend` be exported using the `self_import` method, and `metasrv` and `datanode` be exported using the `remote_write` method. + +Same as above, first create the database `system`. The final relevant configuration of each component is as follows: + +Configuration of `frontend`, assuming that `frontend` uses HTTP protocol to listen on `127.0.0.1:4000` + +```toml +[http] +addr = "127.0.0.1:4000" +... + +[export_metrics] +enable=true +write_interval = "30s" +[export_metrics.self_import] +db = "system" +``` + +The configuration of `metasrv` and `datanode` is as follows, use `remote_write` to send data to `frontend` + +```toml +[export_metrics] +enable=true +write_interval = "30s" +[export_metrics.remote_write] +url = "http://127.0.0.1:4000/v1/prometheus/write?db=system" +``` ## Metrics Detail You can check the output of `curl http://:/metrics` by getting the latest metrics of GreptimeDB. We will add more documents of the metrics sooner. diff --git a/docs/v0.5/zh/user-guide/operations/configuration.md b/docs/v0.5/zh/user-guide/operations/configuration.md index 0521d413e..63f603a7e 100644 --- a/docs/v0.5/zh/user-guide/operations/configuration.md +++ b/docs/v0.5/zh/user-guide/operations/configuration.md @@ -455,7 +455,7 @@ append_stdout = true `frontend`、`metasrv`、`datanode` 和 `standalone` 支持收集自身产生的 metrics 数据，并将采集到的 metrics 数据发送到[Prometheus RemoteWrite 协议](https://prometheus.io/docs/concepts/remote_write_spec/)兼容的接收端，比如发送到 `greptimedb` 自身。 用户可以通过 `[export_metrics]` 配置相关导出参数。 :::tip NOTE -注意：该功能和使用 Prometheus 收集 `greptimedb` 产生的 metrics 没有关系，该功能仅作为 `greptimedb` 导出自身产生 metrics 的一种方式。 +请参考 [Monitoring](./monitoring.md#使用-greptimedb-观测-greptimedb)，了解如何使用 export metrics 监控 `greptimedb` ::: 总共有两种方式用于导出 metrics 数据： diff --git a/docs/v0.5/zh/user-guide/operations/monitoring.md b/docs/v0.5/zh/user-guide/operations/monitoring.md index 927dd91a4..c6519106f 100644 --- a/docs/v0.5/zh/user-guide/operations/monitoring.md +++ b/docs/v0.5/zh/user-guide/operations/monitoring.md @@ -64,6 +64,58 @@ gtctl cluster create mycluster -n default \ --set cluster.prometheusMonitor.labelsSelector.release="prometheus" ``` +## 使用 GreptimeDB 观测 GreptimeDB + +在大部分情况下，使用 Prometheus 可以对 GreptimeDB 的各项指标进行很好的观测。但如果用户不想依赖于Prometheus 去处理 GreptimeDB 产生的各种 metrics 数据。GreptimeDB 提供了 export metrics 功能，用于将 GreptimeDB 产生的 metrics 直接存储到 GreptimeDB 内部，从而避免依赖 Prometheus。关于 export metrics 的配置说明，请看 [export metrics 配置](./configuration.md#export-metrics-选项) + +### Standalone + +在 standalone 模式下建议使用 `self_import` 方式导出数据，默认 `self_import` 方式导入到的数据库库名是 `information_schema`，为了避免 metrics 与 `information_schema` 库中其他数据表混杂，用户可以先创建一个新的数据库，比如 `system`。 + +```sql +CREATE DATABASE system; +``` + +最后配置如下所示: + +```toml +[export_metrics] +enable = true +write_interval = "30s" +[export_metrics.self_import] +db = "system" +``` + +### Cluster + +在集群模式下，建议 `frontend` 使用 `self_import` 方式导出、`metasrv`、`datanode` 使用 `remote_write` 方式导出。 + +同上，首先创建数据库 `system`。最后各组件的相关配置如下所示： + +`frontend` 的配置，假设 `frontend` 使用 HTTP 协议监听 `127.0.0.1:4000` + +```toml +[http] +addr = "127.0.0.1:4000" +... + +[export_metrics] +enable = true +write_interval = "30s" +[export_metrics.self_import] +db = "system" +``` + +`metasrv`、`datanode` 的配置如下所示，使用 `remote_write` 将数据发送到 `frontend` + +```toml +[export_metrics] +enable = true +write_interval = "30s" +[export_metrics.remote_write] +url = "http://127.0.0.1:4000/v1/prometheus/write?db=system" +``` + ## Metrics Detail 可以通过执行`curl http://:/metrics`的输出来获取 GreptimeDB 的最新指标。