From 1db9075ce92160ccb39ead48ab8fbfe35ce4d6d3 Mon Sep 17 00:00:00 2001 From: hanguobiao Date: Tue, 24 Oct 2023 10:48:23 +0800 Subject: [PATCH] chore: update cue template --- .../builder/cue/engine/engine_template.cue | 34 ++++++++++++++ .../monitor/builder/cue/engine/infra.cue | 11 +++++ .../builder/cue/extension/extensions.cue | 5 -- .../builder/cue/processor/processors.cue | 12 ++++- ..._attributes.cue => logs_creator_infra.cue} | 22 ++++++++- .../cue/receiver/metrics/app/mongodb.cue | 36 ++++++++++++++ .../{apecloudmysql.cue => app/mysql.cue} | 7 +-- .../cue/receiver/metrics/app/postgresql.cue | 47 +++++++++++++++++++ .../cue/receiver/metrics/app/pulsar.cue | 41 ++++++++++++++++ .../cue/receiver/metrics/app/qdrant.cue | 27 +++++++++++ .../cue/receiver/metrics/app/redis.cue | 45 ++++++++++++++++++ .../cue/receiver/metrics/app/wescale.cue | 34 ++++++++++++++ .../metrics/{ => system}/k8s_cluster.cue | 3 +- .../kubeletstats.cue} | 7 ++- .../{apecloudnode.cue => system/node.cue} | 1 + ...tributes.cue => metrics_creator_infra.cue} | 6 ++- 16 files changed, 319 insertions(+), 19 deletions(-) create mode 100644 controllers/monitor/builder/cue/engine/engine_template.cue create mode 100644 controllers/monitor/builder/cue/engine/infra.cue rename controllers/monitor/builder/cue/receiver/{logs_resource_attributes.cue => logs_creator_infra.cue} (71%) create mode 100644 controllers/monitor/builder/cue/receiver/metrics/app/mongodb.cue rename controllers/monitor/builder/cue/receiver/metrics/{apecloudmysql.cue => app/mysql.cue} (84%) create mode 100644 controllers/monitor/builder/cue/receiver/metrics/app/postgresql.cue create mode 100644 controllers/monitor/builder/cue/receiver/metrics/app/pulsar.cue create mode 100644 controllers/monitor/builder/cue/receiver/metrics/app/qdrant.cue create mode 100644 controllers/monitor/builder/cue/receiver/metrics/app/redis.cue create mode 100644 controllers/monitor/builder/cue/receiver/metrics/app/wescale.cue rename controllers/monitor/builder/cue/receiver/metrics/{ => system}/k8s_cluster.cue (93%) rename controllers/monitor/builder/cue/receiver/metrics/{apecloudkubeletstats.cue => system/kubeletstats.cue} (90%) rename controllers/monitor/builder/cue/receiver/metrics/{apecloudnode.cue => system/node.cue} (96%) rename controllers/monitor/builder/cue/receiver/{metrics_resource_attributes.cue => metrics_creator_infra.cue} (96%) diff --git a/controllers/monitor/builder/cue/engine/engine_template.cue b/controllers/monitor/builder/cue/engine/engine_template.cue new file mode 100644 index 000000000000..583f208cf5da --- /dev/null +++ b/controllers/monitor/builder/cue/engine/engine_template.cue @@ -0,0 +1,34 @@ +logs: { + include:[string] +} + +parameters: { + clusterName: string + componentName: string + containerName: string + metrics: { + enabled: *true | bool + collectionInterval: *"30s" | string + enabled_metrics?: [string] + } + logs: { + enabled: *true | bool + logs_collector?: [logs] + } +} + +output: { + "\(parameters.clusterName)/\(parameters.componentName)/\(parameters.containerName)/\(parameters.containerName)": { + enabled_metrics: parameters.metrics.enabled + enabled_logs: parameters.logs.enabled + metrics_collector: { + collection_interval: parameters.collection_interval + if parameters.enabled_metrics != _|_ { + enable_metrics: parameters.enabled_metrics + } + } + if logs.logs_collector != _|_ { + logs_collector: logs.logs_collector + } + } +} \ No newline at end of file diff --git a/controllers/monitor/builder/cue/engine/infra.cue b/controllers/monitor/builder/cue/engine/infra.cue new file mode 100644 index 000000000000..8cd0bff626a9 --- /dev/null +++ b/controllers/monitor/builder/cue/engine/infra.cue @@ -0,0 +1,11 @@ +parameters: { + collection_interval: *"15s" | string, + initial_delay: *"10s" | string, + timeout: *"5s" | string, +} + +output: { + collection_interval: parameters.collection_interval, + initial_delay: parameters.initial_delay, + timeout: parameters.timeout, +} diff --git a/controllers/monitor/builder/cue/extension/extensions.cue b/controllers/monitor/builder/cue/extension/extensions.cue index ad46101bcc82..a48caeecff1c 100644 --- a/controllers/monitor/builder/cue/extension/extensions.cue +++ b/controllers/monitor/builder/cue/extension/extensions.cue @@ -15,11 +15,6 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -parameters: { - enable: *true | bool - container_runtime_type: *"containerd" | string -} - output: { extensions: { memory_ballast: diff --git a/controllers/monitor/builder/cue/processor/processors.cue b/controllers/monitor/builder/cue/processor/processors.cue index 08e6ba105b76..090e79706fde 100644 --- a/controllers/monitor/builder/cue/processor/processors.cue +++ b/controllers/monitor/builder/cue/processor/processors.cue @@ -15,9 +15,17 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -output: - memory_limiter:{ +output: { + memory_limiter:{ limit_mib: 512 spike_limit_mib: 128 check_interval: "10s" } + batch: { + send_batch_max_size: 200 + send_batch_size: 200 + timeout: "5s" + } +} + + diff --git a/controllers/monitor/builder/cue/receiver/logs_resource_attributes.cue b/controllers/monitor/builder/cue/receiver/logs_creator_infra.cue similarity index 71% rename from controllers/monitor/builder/cue/receiver/logs_resource_attributes.cue rename to controllers/monitor/builder/cue/receiver/logs_creator_infra.cue index cfd7abec4b88..ecd4384e47f2 100644 --- a/controllers/monitor/builder/cue/receiver/logs_resource_attributes.cue +++ b/controllers/monitor/builder/cue/receiver/logs_creator_infra.cue @@ -15,8 +15,24 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -output: - resource_attributes: { +output: { + watch_observers: ["apecloud_engine_observer"] + receivers:{ + container_filelog: { + rule: "type == \"container\" && config != nil && config.EnabledLogs" + config: { + input_configs: "`config.LogsCollector`" + container_id: "`container_id`" + pod_ip: "`endpoint`" + storage: "file_storage/oteld" + cluster_name: "`config.ClusterName`" + component_name: "`config.ComponentName`" + character_type: "`config.CharacterType`" + } + } + + } + resource_attributes: { container: { app_kubernetes_io_component: "`labels[\"app.kubernetes.io/component\"]`" app_kubernetes_io_instance: "`labels[\"app.kubernetes.io/instance\"]`" @@ -30,3 +46,5 @@ output: job: "oteld-app-metrics" } } +} + diff --git a/controllers/monitor/builder/cue/receiver/metrics/app/mongodb.cue b/controllers/monitor/builder/cue/receiver/metrics/app/mongodb.cue new file mode 100644 index 000000000000..bb714b55ddb6 --- /dev/null +++ b/controllers/monitor/builder/cue/receiver/metrics/app/mongodb.cue @@ -0,0 +1,36 @@ +// Copyright (C) 2022-2023 ApeCloud Co., Ltd +// +// This file is part of KubeBlocks project +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +output: + apecloudmongo: { + rule: "type == \"container\" && monitor_type == \"mongodb\" && config != nil && config.EnabledMetrics" + config: { + endpoint: "`endpoint`:27017" + username: "`envs[\"MONGODB_ROOT_USER\"]`" + password: "`envs[\"MONGODB_ROOT_PASSWORD\"]`" + connect_params: "admin?ssl=false&authSource=admin" + collect_all: true + collection_interval: "`settings.CollectionInterval`" + direct_connect: true + global_conn_pool: false + compatible_mode: true + } + resource_attributes: { + job: oteld-app-metrics + receiver: apecloudmongodb + } + } \ No newline at end of file diff --git a/controllers/monitor/builder/cue/receiver/metrics/apecloudmysql.cue b/controllers/monitor/builder/cue/receiver/metrics/app/mysql.cue similarity index 84% rename from controllers/monitor/builder/cue/receiver/metrics/apecloudmysql.cue rename to controllers/monitor/builder/cue/receiver/metrics/app/mysql.cue index 52e8847365c5..050761cb8e33 100644 --- a/controllers/monitor/builder/cue/receiver/metrics/apecloudmysql.cue +++ b/controllers/monitor/builder/cue/receiver/metrics/app/mysql.cue @@ -19,9 +19,6 @@ parameters: { endpoint: *"`endpoint`:3306" | string username: *"`envs[\"MYSQL_ROOT_USER\"]`" | string password: *"`envs[\"MYSQL_ROOT_PASSWORD\"]`" | string - allow_native_passwords: *true | bool - transport: *"tcp" | string - collection_interval: *"`settings.CollectionInterval`" | string } output: @@ -31,8 +28,8 @@ output: endpoint: parameters.endpoint username: parameters.username password: parameters.password - allow_native_passwords: parameters.allow_native_passwords - transport: parameters.transport + allow_native_passwords: true + transport: "tcp" collection_interval: "`settings.CollectionInterval`" } resource_attributes: diff --git a/controllers/monitor/builder/cue/receiver/metrics/app/postgresql.cue b/controllers/monitor/builder/cue/receiver/metrics/app/postgresql.cue new file mode 100644 index 000000000000..45109d693a90 --- /dev/null +++ b/controllers/monitor/builder/cue/receiver/metrics/app/postgresql.cue @@ -0,0 +1,47 @@ +// Copyright (C) 2022-2023 ApeCloud Co., Ltd +// +// This file is part of KubeBlocks project +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +parameters: { + enpoint: *"`endpoint`:5432" | string + username: *"`envs[\"PGUSER_SUPERUSER\"]`" | string + password: *"`envs[\"PGPASSWORD_SUPERUSER\"]`" | string + job: *"oteld-app-metrics" | string + databases: *["postgres"] | [string] + exclude_databases: *["template0", "template1"] | [string] +} + +output: + apecloudpostgresql: { + rule: "type == \"container\" && monitor_type == \"postgresql\" && config != nil && config.EnabledMetrics" + config: { + endpoint: parameters.enpoint + username: parameters.username + password: parameters.password + databases: parameters.databases + exclude_databases: parameters.exclude_databases + collection_interval: "`settings.CollectionInterval`" + transport: "tcp" + tls: { + insecure: true + insecure_skip_verify: true + } + } + resource_attributes: { + job: parameters.job + receiver: "apecloudpostgresql" + } + } \ No newline at end of file diff --git a/controllers/monitor/builder/cue/receiver/metrics/app/pulsar.cue b/controllers/monitor/builder/cue/receiver/metrics/app/pulsar.cue new file mode 100644 index 000000000000..5cadb0df24bc --- /dev/null +++ b/controllers/monitor/builder/cue/receiver/metrics/app/pulsar.cue @@ -0,0 +1,41 @@ +// Copyright (C) 2022-2023 ApeCloud Co., Ltd +// +// This file is part of KubeBlocks project +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +parameters: { + endpoint: *"`endpoint`:8000" | string +} + +output: { + "prometheus_simple/pulsar": { + rule: "type == \"container\" && config != nil && config.EnabledMetrics && config.ClusterDefName == \"pulsar\" && ( config.ComponentDefName == \"bookies-recovery\" || config.ComponentDefName == \"zookeeper\" || config.ComponentDefName == \"bookies\" )" + config: { + collection_interval: "settings.CollectionInterval" + use_service_account: false + endpoint: parameters.endpoint + disable_keep_alives: false + } + } + "prometheus_simple/pulsar2": { + rule: "type == \"container\" && config != nil && config.EnabledMetrics && config.ClusterDefName == \"pulsar\" && ( config.ComponentDefName == \"bookies-recovery\" || config.ComponentDefName == \"zookeeper\" || config.ComponentDefName == \"bookies\" )" + config: { + collection_interval: "settings.CollectionInterval" + use_service_account: false + endpoint: parameters.endpoint + disable_keep_alives: false + } + } +} diff --git a/controllers/monitor/builder/cue/receiver/metrics/app/qdrant.cue b/controllers/monitor/builder/cue/receiver/metrics/app/qdrant.cue new file mode 100644 index 000000000000..6cf56cac5292 --- /dev/null +++ b/controllers/monitor/builder/cue/receiver/metrics/app/qdrant.cue @@ -0,0 +1,27 @@ +// Copyright (C) 2022-2023 ApeCloud Co., Ltd +// +// This file is part of KubeBlocks project +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +output: + "prometheus_simple/qdrant": { + rule: "type == \"container\" && config != nil && config.EnabledMetrics && config.ClusterDefName == \"qdrant\" && config.ComponentDefName == \"qdrant\"" + config: { + collection_interval: "`settings.CollectionInterval`" + endpoint: "`endpoint`:6333" + disable_keep_alives: false + use_service_account: false + } + } diff --git a/controllers/monitor/builder/cue/receiver/metrics/app/redis.cue b/controllers/monitor/builder/cue/receiver/metrics/app/redis.cue new file mode 100644 index 000000000000..7995652874ed --- /dev/null +++ b/controllers/monitor/builder/cue/receiver/metrics/app/redis.cue @@ -0,0 +1,45 @@ +// Copyright (C) 2022-2023 ApeCloud Co., Ltd +// +// This file is part of KubeBlocks project +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +parameters: { + job: *"oteld-app-metrics" | string + endpoint: *"`endpoint`:6379" | string + username: *"envs[\"REDIS_REPL_USER\"]" | string + password: *"envs[\"REDIS_REPL_PASSWORD\"]" | string +} + +output: + apecloudredis: { + rule: "type == \"container\" && monitor_type == \"redis\" && config != nil && config.EnabledMetrics" + config: { + endpoint: parameters.endpoint + username: parameters.username + password: parameters.password + password_file: "" + lua_script: "" + tls: { + insecure: true + insecure_skip_verify: true + } + collection_interval: "`settings.CollectionInterval`" + } + + resource_attributes: { + job: parameters.job + receiver: "apecloudredis" + } + } \ No newline at end of file diff --git a/controllers/monitor/builder/cue/receiver/metrics/app/wescale.cue b/controllers/monitor/builder/cue/receiver/metrics/app/wescale.cue new file mode 100644 index 000000000000..df7f4bea9b07 --- /dev/null +++ b/controllers/monitor/builder/cue/receiver/metrics/app/wescale.cue @@ -0,0 +1,34 @@ +// Copyright (C) 2022-2023 ApeCloud Co., Ltd +// +// This file is part of KubeBlocks project +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +parameters: { + metrics_path: *"/metrics" | string, + endpoint: *"`endpoint`:`envs[\"VTTABLET_PORT\"]`" | string, + disable_keep_alives: *false | bool, +} + +output: + "prometheus_simple/vttablet": { + rule: "type == \"container\" && config != nil && config.EnabledMetrics && config.ComponentDefName == \"mysql\" && config.ClusterDefName == \"apecloud-mysql\" && config.ContainerName == \"vttablet\"" + config: { + collection_interval: "`settings.CollectionInterval`" + use_service_account: false + metrics_path: parameters.metrics_path + endpoint: parameters.endpoint + disable_keep_alives: parameters.disable_keep_alives + } + } diff --git a/controllers/monitor/builder/cue/receiver/metrics/k8s_cluster.cue b/controllers/monitor/builder/cue/receiver/metrics/system/k8s_cluster.cue similarity index 93% rename from controllers/monitor/builder/cue/receiver/metrics/k8s_cluster.cue rename to controllers/monitor/builder/cue/receiver/metrics/system/k8s_cluster.cue index 8b40b01ee083..90b7f4b55cef 100644 --- a/controllers/monitor/builder/cue/receiver/metrics/k8s_cluster.cue +++ b/controllers/monitor/builder/cue/receiver/metrics/system/k8s_cluster.cue @@ -17,9 +17,9 @@ parameters: { collection_interval: *"15s" | string + job: *"oteld-k8s-cluster-metrics" | string } - output: k8s_cluster: { rule: "type == \"k8s.node\"" @@ -28,6 +28,7 @@ output: } resource_attributes: { receiver: "k8s_cluster" + job: parameters.job } } diff --git a/controllers/monitor/builder/cue/receiver/metrics/apecloudkubeletstats.cue b/controllers/monitor/builder/cue/receiver/metrics/system/kubeletstats.cue similarity index 90% rename from controllers/monitor/builder/cue/receiver/metrics/apecloudkubeletstats.cue rename to controllers/monitor/builder/cue/receiver/metrics/system/kubeletstats.cue index f448af256d0a..0026e17a4155 100644 --- a/controllers/monitor/builder/cue/receiver/metrics/apecloudkubeletstats.cue +++ b/controllers/monitor/builder/cue/receiver/metrics/system/kubeletstats.cue @@ -19,6 +19,7 @@ parameters: { auth_type: *"serviceAccount" | string collection_interval: *"15s" | string endpoint: *"`endpoint`:`kubelet_endpoint_port`" | string + job: *"oteld-kubeletstats-metrics" | string } output: @@ -31,8 +32,10 @@ output: extra_metadata_labels: ["k8s.volume.type", "kubeblocks"] metric_groups: ["container", "pod", "volume"] } - resource_attributes: - receiver: "apecloudkubeletstats" + resource_attributes:{ + receiver: "apecloudkubeletstats" + job: parameters.job + } } diff --git a/controllers/monitor/builder/cue/receiver/metrics/apecloudnode.cue b/controllers/monitor/builder/cue/receiver/metrics/system/node.cue similarity index 96% rename from controllers/monitor/builder/cue/receiver/metrics/apecloudnode.cue rename to controllers/monitor/builder/cue/receiver/metrics/system/node.cue index 05606294925b..7cd19cd1b418 100644 --- a/controllers/monitor/builder/cue/receiver/metrics/apecloudnode.cue +++ b/controllers/monitor/builder/cue/receiver/metrics/system/node.cue @@ -17,6 +17,7 @@ parameters: { collection_interval: *"15s" | string + job: *"oteld-node-metrics" | string } diff --git a/controllers/monitor/builder/cue/receiver/metrics_resource_attributes.cue b/controllers/monitor/builder/cue/receiver/metrics_creator_infra.cue similarity index 96% rename from controllers/monitor/builder/cue/receiver/metrics_resource_attributes.cue rename to controllers/monitor/builder/cue/receiver/metrics_creator_infra.cue index 2bd1d2860589..bf2a9ce2514b 100644 --- a/controllers/monitor/builder/cue/receiver/metrics_resource_attributes.cue +++ b/controllers/monitor/builder/cue/receiver/metrics_creator_infra.cue @@ -15,8 +15,9 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -output: - resource_attributes: { +output: { + watch_observers: ["apecloud_engine_observer"] + resource_attributes: { container: { app_kubernetes_io_component: "`labels[\"app.kubernetes.io/component\"]`" app_kubernetes_io_instance: "`labels[\"app.kubernetes.io/instance\"]`" @@ -50,4 +51,5 @@ output: job: "oteld-system" } } +}