diff --git a/distributed-databases/pinot/addons.tf b/distributed-databases/pinot/addons.tf new file mode 100644 index 000000000..b3c741f74 --- /dev/null +++ b/distributed-databases/pinot/addons.tf @@ -0,0 +1,203 @@ +#--------------------------------------------------------------- +# IRSA for EBS CSI Driver +#--------------------------------------------------------------- +module "ebs_csi_driver_irsa" { + source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks" + version = "~> 5.14" + role_name = format("%s-%s", local.name, "ebs-csi-driver") + attach_ebs_csi_policy = true + oidc_providers = { + main = { + provider_arn = module.eks.oidc_provider_arn + namespace_service_accounts = ["kube-system:ebs-csi-controller-sa"] + } + } + tags = local.tags +} + +#--------------------------------------------------------------- +# GP3 Encrypted Storage Class +#--------------------------------------------------------------- +resource "kubernetes_annotations" "disable_gp2" { + annotations = { + "storageclass.kubernetes.io/is-default-class" : "false" + } + api_version = "storage.k8s.io/v1" + kind = "StorageClass" + metadata { + name = "gp2" + } + force = true + + depends_on = [module.eks] +} + +resource "kubernetes_storage_class" "ebs_csi_encrypted_gp3_storage_class" { + metadata { + name = "gp3" + annotations = { + "storageclass.kubernetes.io/is-default-class" : "true" + } + } + + storage_provisioner = "ebs.csi.aws.com" + reclaim_policy = "Delete" + allow_volume_expansion = true + volume_binding_mode = "WaitForFirstConsumer" + parameters = { + fsType = "xfs" + encrypted = true + type = "gp3" + } + + depends_on = [kubernetes_annotations.disable_gp2] +} + +#--------------------------------------------------------------- +# EKS Blueprints Kubernetes Addons +#--------------------------------------------------------------- +module "eks_blueprints_kubernetes_addons" { + source = "aws-ia/eks-blueprints-addons/aws" + version = "~> 1.0" + + cluster_name = module.eks.cluster_name + cluster_endpoint = module.eks.cluster_endpoint + cluster_version = module.eks.cluster_version + oidc_provider_arn = module.eks.oidc_provider_arn + + #--------------------------------------------------------------- + # Amazon EKS Managed Add-ons + #--------------------------------------------------------------- + eks_addons = { + aws-ebs-csi-driver = { + service_account_role_arn = module.ebs_csi_driver_irsa.iam_role_arn + } + coredns = { + preserve = true + } + vpc-cni = { + preserve = true + } + kube-proxy = { + preserve = true + } + } + #--------------------------------------- + # Kubernetes Add-ons + #--------------------------------------- + + enable_kube_prometheus_stack = true + kube_prometheus_stack = { + namespace = "monitoring" + name = "prometheus" + chart_version = "48.1.1" + set_sensitive = [ + { + name = "grafana.adminPassword" + value = data.aws_secretsmanager_secret_version.admin_password_version.secret_string + }] + + values = [ + templatefile("${path.module}/helm/kube-prometheus-stack-values.yaml", { + storage_class_type = kubernetes_storage_class.ebs_csi_encrypted_gp3_storage_class.id, + }) + ] + } + + #--------------------------------------- + # AWS for FluentBit - DaemonSet + #--------------------------------------- + enable_aws_for_fluentbit = true + aws_for_fluentbit_cw_log_group = { + use_name_prefix = false + name = "/${local.name}/aws-fluentbit-logs" # Add-on creates this log group + retention_in_days = 30 + } + aws_for_fluentbit = { + values = [templatefile("${path.module}/helm/aws-for-fluentbit-values.yaml", { + region = local.region, + cloudwatch_log_group = "/${local.name}/aws-fluentbit-logs" + cluster_name = module.eks.cluster_name + })] + } + #--------------------------------------- + # AWS Load Balancer Controller + #--------------------------------------- + enable_aws_load_balancer_controller = false + + + tags = local.tags +} + +#--------------------------------------------------------------- +# Apache Pinot +#--------------------------------------------------------------- + +resource "random_string" "random_suffix" { + length = 10 + special = false + upper = false +} + +resource "random_password" "sensitive_key" { + length = 16 + special = false +} + +module "eks_data_addons" { + source = "aws-ia/eks-data-addons/aws" + version = "~> 1.2.6" + + oidc_provider_arn = module.eks.oidc_provider_arn + + #--------------------------------------------------------------- + # Apache Pinot Add-on + #--------------------------------------------------------------- + enable_pinot = true + pinot_helm_config = { + namespace = "pinot" + values = [templatefile("${path.module}/helm/pinot-values.yaml", {})] + set = [ + { + name = "cluster.name" + value = local.cluster_name + }, + { + name = "controller.replicaCount" + value = 3 + }, + { + name = "controller.persistence.storageClass" + value = "gp3" + }, + { + name = "broker.replicaCount" + value = 3 + }, + { + name = "server.replicaCount" + value = 3 + }, + { + name = "server.persistence.storageClass" + value = "gp3" + }, + { + name = "minionStateless.replicaCount" + value = 3 + }, + { + name = "minionStateless.persistence.storageClass" + value = "gp3" + }, + { + name = "zookeeper.replicaCount" + value = 3 + }, + { + name = "zookeeper.persistence.storageClass" + value = "gp3" + } + ] + } +} diff --git a/distributed-databases/pinot/cleanup.sh b/distributed-databases/pinot/cleanup.sh new file mode 100755 index 000000000..4b3dd1fcd --- /dev/null +++ b/distributed-databases/pinot/cleanup.sh @@ -0,0 +1,28 @@ +#!/bin/bash +set -o errexit +set -o pipefail + +# Make sure you have `terraform.tfvars` file with the desired region. +# Otherwise script will ask you to input your region each time it runs `terraform destroy` + +targets=( + "module.eks_data_addons" + "module.eks_blueprints_kubernetes_addons" + "module.vpc_cni_irsa" + "module.ebs_csi_driver_irsa" + "module.eks" + "module.vpc" +) + +#------------------------------------------- +# Terraform destroy per module target +#------------------------------------------- +for target in "${targets[@]}" +do + terraform destroy -target="$target" -auto-approve +done + +#------------------------------------------- +# Terraform destroy full +#------------------------------------------- +terraform destroy -auto-approve diff --git a/distributed-databases/pinot/dashboard/pinot.json b/distributed-databases/pinot/dashboard/pinot.json new file mode 100644 index 000000000..8410676f9 --- /dev/null +++ b/distributed-databases/pinot/dashboard/pinot.json @@ -0,0 +1,2673 @@ +{ + "__inputs": [ + { + "name": "DS_PROMETHEUS", + "label": "Prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "7.1.1" + }, + { + "type": "panel", + "id": "graph", + "name": "Graph", + "version": "" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (kubernetes_pod_name) (pinot_controller_LeaderPartitionCount_Value)", + "interval": "", + "legendFormat": "{{kubernetes_pod_name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Controller Leader", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (kubernetes_namespace) (pinot_controller_realtimeTableCount_Value)", + "interval": "", + "legendFormat": "total table count", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Realtime Table Count", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "connected", + "percentage": false, + "pluginVersion": "7.1.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (table) (rate(pinot_broker_queries_Count[10m]))", + "interval": "", + "legendFormat": "{{table}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Table Query QPS", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "connected", + "percentage": false, + "pluginVersion": "7.1.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg by (table) (pinot_broker_queryExecution_50thPercentile) ", + "interval": "", + "legendFormat": "{{table}}-p50", + "refId": "A" + }, + { + "expr": "avg by (table) (pinot_broker_queryExecution_75thPercentile)", + "interval": "", + "legendFormat": "{{table}}-p75", + "refId": "B" + }, + { + "expr": "avg by (table) (pinot_broker_queryExecution_95thPercentile)", + "interval": "", + "legendFormat": "{{table}}-p95", + "refId": "C" + }, + { + "expr": "avg by (table) (pinot_broker_queryExecution_99thPercentile)", + "interval": "", + "legendFormat": "{{table}}-p99", + "refId": "D" + }, + { + "expr": "avg by (table) (pinot_broker_queryExecution_999thPercentile)", + "interval": "", + "legendFormat": "{{table}}-p999", + "refId": "E" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Table Query Latency", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 10, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 16 + }, + "hiddenSeries": false, + "id": 10, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "connected", + "percentage": false, + "pluginVersion": "7.1.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (topic) (avg by (partition) (rate(pinot_server_realtimeRowsConsumed_Count[5m])))", + "interval": "", + "legendFormat": "Received", + "refId": "A" + }, + { + "expr": "rate(kafka_events_published_total[5m])", + "interval": "", + "legendFormat": "Published", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Table Ingestion Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 23 + }, + "hiddenSeries": false, + "id": 20, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "connected", + "percentage": false, + "pluginVersion": "7.1.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg by (table) (pinot_server_freshnessLagMs_50thPercentile)", + "interval": "", + "legendFormat": "{{table}}-p50", + "refId": "A" + }, + { + "expr": "avg by (table) (pinot_server_freshnessLagMs_75thPercentile)", + "interval": "", + "legendFormat": "{{table}}-p75", + "refId": "B" + }, + { + "expr": "avg by (table) (pinot_server_freshnessLagMs_95thPercentile)", + "interval": "", + "legendFormat": "{{table}}-p95", + "refId": "C" + }, + { + "expr": "avg by (table) (pinot_server_freshnessLagMs_99thPercentile)", + "interval": "", + "legendFormat": "{{table}}-p99", + "refId": "D" + }, + { + "expr": "avg by (table) (pinot_server_freshnessLagMs_999thPercentile)", + "interval": "", + "legendFormat": "{{table}}-p999", + "refId": "E" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Table Consuming Latency", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 10, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 23 + }, + "hiddenSeries": false, + "id": 32, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "connected", + "percentage": false, + "pluginVersion": "7.1.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (statefulset_kubernetes_io_pod_name) (pinot_server_llcSimultaneousSegmentBuilds_Value)", + "interval": "", + "legendFormat": "{{statefulset_kubernetes_io_pod_name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Server Segment Build", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 31 + }, + "hiddenSeries": false, + "id": 15, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "connected", + "percentage": false, + "pluginVersion": "7.1.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg by (pod)(rate(container_cpu_user_seconds_total{pod=~\"pinot-controller-[0-9]*\"}[5m])) ", + "interval": "", + "legendFormat": "{{pod}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Pinot Controller CPU User", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 31 + }, + "hiddenSeries": false, + "id": 26, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "connected", + "percentage": false, + "pluginVersion": "7.1.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "max by (statefulset_kubernetes_io_pod_name)(jvm_memory_bytes_used{statefulset_kubernetes_io_pod_name=~\"pinot-controller-[0-9]*\"}) ", + "interval": "", + "legendFormat": "{{statefulset_kubernetes_io_pod_name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Pinot Controller JVM Used", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 39 + }, + "hiddenSeries": false, + "id": 24, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "connected", + "percentage": false, + "pluginVersion": "7.1.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg by (pod)(rate(container_cpu_user_seconds_total{pod=~\"pinot-broker-[0-9]*\"}[5m])) ", + "interval": "", + "legendFormat": "{{pod}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Pinot Broker CPU User", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 39 + }, + "hiddenSeries": false, + "id": 27, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "connected", + "percentage": false, + "pluginVersion": "7.1.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "max by (statefulset_kubernetes_io_pod_name)(jvm_memory_bytes_used{statefulset_kubernetes_io_pod_name=~\"pinot-broker-[0-9]*\"}) ", + "interval": "", + "legendFormat": "{{statefulset_kubernetes_io_pod_name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Pinot Broker JVM Used", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 47 + }, + "hiddenSeries": false, + "id": 25, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "connected", + "percentage": false, + "pluginVersion": "7.1.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg by (pod)(rate(container_cpu_user_seconds_total{pod=~\"pinot-server-[0-9]*\"}[5m])) ", + "interval": "", + "legendFormat": "{{pod}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Pinot Server CPU User", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 47 + }, + "hiddenSeries": false, + "id": 28, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "connected", + "percentage": false, + "pluginVersion": "7.1.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "max by (statefulset_kubernetes_io_pod_name)(jvm_memory_bytes_used{statefulset_kubernetes_io_pod_name=~\"pinot-server-[0-9]*\"}) ", + "interval": "", + "legendFormat": "{{statefulset_kubernetes_io_pod_name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Pinot Server JVM Used", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 55 + }, + "hiddenSeries": false, + "id": 23, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "connected", + "percentage": false, + "pluginVersion": "7.1.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg by (table) (pinot_server_queryProcessing_50thPercentile)", + "interval": "", + "legendFormat": "{{table}}-p50", + "refId": "A" + }, + { + "expr": "avg by (table) (pinot_server_queryProcessing_75thPercentile)", + "interval": "", + "legendFormat": "{{table}}-p75", + "refId": "E" + }, + { + "expr": "avg by (table) (pinot_server_queryProcessing_95thPercentile)", + "interval": "", + "legendFormat": "{{table}}-p95", + "refId": "D" + }, + { + "expr": "avg by (table) (pinot_server_queryProcessing_99thPercentile)", + "interval": "", + "legendFormat": "{{table}}-p99", + "refId": "B" + }, + { + "expr": "avg by (table) (pinot_server_queryProcessing_999thPercentile)", + "interval": "", + "legendFormat": "{{table}}-p999", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Server Query Processing Per Table", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 10, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 55 + }, + "hiddenSeries": false, + "id": 16, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "connected", + "percentage": false, + "pluginVersion": "7.1.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg by (table) (pinot_server_totalQueryTime_50thPercentile)", + "interval": "", + "legendFormat": "{{table}}-p50", + "refId": "A" + }, + { + "expr": "avg by (table) (pinot_server_totalQueryTime_75thPercentile)", + "interval": "", + "legendFormat": "{{table}}-p75", + "refId": "D" + }, + { + "expr": "avg by (table) (pinot_server_totalQueryTime_95thPercentile)", + "interval": "", + "legendFormat": "{{table}}-p95", + "refId": "E" + }, + { + "expr": "avg by (table) (pinot_server_totalQueryTime_99thPercentile)", + "interval": "", + "legendFormat": "{{table}}-p99", + "refId": "B" + }, + { + "expr": "avg by (table) (pinot_server_totalQueryTime_999thPercentile)", + "interval": "", + "legendFormat": "{{table}}-p999", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Server Total Query Time Per Table", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 10, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 10, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 63 + }, + "hiddenSeries": false, + "id": 14, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "connected", + "percentage": false, + "pluginVersion": "7.1.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (statefulset_kubernetes_io_pod_name)(pinot_server_realtime_offheapMemoryUsed_Value) ", + "interval": "", + "legendFormat": "{{statefulset_kubernetes_io_pod_name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Pinot Server Off-Heap Memory Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 63 + }, + "hiddenSeries": false, + "id": 29, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "connected", + "percentage": false, + "pluginVersion": "7.1.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg by (statefulset_kubernetes_io_pod_name)(pinot_server_memory_mmapBufferUsage_Value) ", + "interval": "", + "legendFormat": "{{statefulset_kubernetes_io_pod_name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Pinot Server Memory Mapped Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 71 + }, + "hiddenSeries": false, + "id": 11, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "connected", + "percentage": false, + "pluginVersion": "7.1.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (table) (pinot_server_documentCount_Value)", + "interval": "", + "legendFormat": "{{table}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Server Total Documents Per Table", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 71 + }, + "hiddenSeries": false, + "id": 12, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "connected", + "percentage": false, + "pluginVersion": "7.1.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (table) (pinot_server_segmentCount_Value)", + "interval": "", + "legendFormat": "{{table}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Server Total Segment Per Table", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 79 + }, + "hiddenSeries": false, + "id": 33, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "connected", + "percentage": false, + "pluginVersion": "7.1.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (statefulset_kubernetes_io_pod_name) (pinot_server_documentCount_Value)", + "interval": "", + "legendFormat": "{{statefulset_kubernetes_io_pod_name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Pinot Server Documents Count", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 79 + }, + "hiddenSeries": false, + "id": 34, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "connected", + "percentage": false, + "pluginVersion": "7.1.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (statefulset_kubernetes_io_pod_name) (pinot_server_segmentCount_Value)", + "interval": "", + "legendFormat": "{{statefulset_kubernetes_io_pod_name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Pinot Server Segments Count", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 87 + }, + "hiddenSeries": false, + "id": 18, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "connected", + "percentage": false, + "pluginVersion": "7.1.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (table) (rate(pinot_server_numDocsScanned_Count[10m]))", + "interval": "", + "legendFormat": "{{table}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Server Query Doc Scanned Per Table", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 87 + }, + "hiddenSeries": false, + "id": 36, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "connected", + "percentage": false, + "pluginVersion": "7.1.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "avg by (statefulset_kubernetes_io_pod_name) (rate(pinot_server_numDocsScanned_Count[10m]))", + "interval": "", + "legendFormat": "{{statefulset_kubernetes_io_pod_name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Server Query Doc Scanned", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 95 + }, + "hiddenSeries": false, + "id": 19, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "connected", + "percentage": false, + "pluginVersion": "7.1.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (table) (rate(pinot_server_numSegmentsProcessed_Count[5m]))", + "interval": "", + "legendFormat": "{{table}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Server Query Segment Processed Per Table", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 95 + }, + "hiddenSeries": false, + "id": 35, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "connected", + "percentage": false, + "pluginVersion": "7.1.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "avg by (statefulset_kubernetes_io_pod_name) (rate(pinot_server_numSegmentsProcessed_Count[5m]))", + "interval": "", + "legendFormat": "{{statefulset_kubernetes_io_pod_name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Server Query Segment Processed", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 103 + }, + "hiddenSeries": false, + "id": 13, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "connected", + "percentage": false, + "pluginVersion": "7.1.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (kubernetes_pod_name)(pinot_server_memory_directBufferUsage_Value) ", + "interval": "", + "legendFormat": "{{kubernetes_pod_name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Server DirectBuffer Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": "1m", + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-3h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Pinot", + "uid": "a2HnNdCWk", + "version": 48 +} diff --git a/distributed-databases/pinot/example/pinot-realtime-quickstart.yml b/distributed-databases/pinot/example/pinot-realtime-quickstart.yml new file mode 100644 index 000000000..3a92f82ac --- /dev/null +++ b/distributed-databases/pinot/example/pinot-realtime-quickstart.yml @@ -0,0 +1,875 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +apiVersion: v1 +kind: ConfigMap +metadata: + name: examples + namespace: pinot +data: + airlineStats_realtime_table_config.json: |- + { + "tableName": "airlineStats", + "tableType": "REALTIME", + "segmentsConfig": { + "timeColumnName": "DaysSinceEpoch", + "timeType": "DAYS", + "retentionTimeUnit": "DAYS", + "retentionTimeValue": "3650", + "segmentPushType": "APPEND", + "segmentAssignmentStrategy": "BalanceNumSegmentAssignmentStrategy", + "schemaName": "airlineStats", + "replication": "1", + "replicasPerPartition": "1" + }, + "tenants": {}, + "tableIndexConfig": { + "loadMode": "MMAP", + "streamConfigs": { + "streamType": "kafka", + "stream.kafka.consumer.type": "simple", + "stream.kafka.topic.name": "flights-realtime", + "stream.kafka.decoder.class.name": "org.apache.pinot.plugin.stream.kafka.KafkaJSONMessageDecoder", + "stream.kafka.consumer.factory.class.name": "org.apache.pinot.plugin.stream.kafka20.KafkaConsumerFactory", + "stream.kafka.hlc.zk.connect.string": "kafka-zookeeper:2181", + "stream.kafka.zk.broker.url": "kafka-zookeeper:2181", + "stream.kafka.broker.list": "pinot-kafka-controller-0.pinot-kafka-controller-headless.pinot.svc.cluster.local:9092,pinot-kafka-controller-1.pinot-kafka-controller-headless.pinot.svc.cluster.local:9092,pinot-kafka-controller-2.pinot-kafka-controller-headless.pinot.svc.cluster.local:9092", + "realtime.segment.flush.threshold.time": "3600000", + "realtime.segment.flush.threshold.size": "50000", + "stream.kafka.consumer.prop.auto.offset.reset": "smallest" + } + }, + "metadata": { + "customConfigs": {} + } + } + + airlineStatsAvro_realtime_table_config.json: |- + { + "tableName": "airlineStatsAvro", + "tableType": "REALTIME", + "segmentsConfig": { + "timeColumnName": "DaysSinceEpoch", + "timeType": "DAYS", + "retentionTimeUnit": "DAYS", + "retentionTimeValue": "3650", + "segmentPushType": "APPEND", + "segmentAssignmentStrategy": "BalanceNumSegmentAssignmentStrategy", + "schemaName": "airlineStatsAvro", + "replication": "1", + "replicasPerPartition": "1" + }, + "tenants": {}, + "tableIndexConfig": { + "loadMode": "MMAP", + "streamConfigs": { + "streamType": "kafka", + "stream.kafka.consumer.type": "simple", + "stream.kafka.topic.name": "flights-realtime-avro", + "stream.kafka.decoder.class.name": "org.apache.pinot.core.realtime.stream.SimpleAvroMessageDecoder", + "stream.kafka.decoder.prop.schema": "{\"type\":\"record\",\"name\":\"Flight\",\"namespace\":\"pinot\",\"fields\":[{\"name\":\"DaysSinceEpoch\",\"type\":[\"int\"]},{\"name\":\"Year\",\"type\":[\"int\"]},{\"name\":\"Quarter\",\"type\":[\"int\"]},{\"name\":\"Month\",\"type\":[\"int\"]},{\"name\":\"DayofMonth\",\"type\":[\"int\"]},{\"name\":\"DayOfWeek\",\"type\":[\"int\"]},{\"name\":\"FlightDate\",\"type\":[\"string\"]},{\"name\":\"UniqueCarrier\",\"type\":[\"string\"]},{\"name\":\"AirlineID\",\"type\":[\"int\"]},{\"name\":\"Carrier\",\"type\":[\"string\"]},{\"name\":\"TailNum\",\"type\":[\"string\",\"null\"]},{\"name\":\"FlightNum\",\"type\":[\"int\"]},{\"name\":\"OriginAirportID\",\"type\":[\"int\"]},{\"name\":\"OriginAirportSeqID\",\"type\":[\"int\"]},{\"name\":\"OriginCityMarketID\",\"type\":[\"int\"]},{\"name\":\"Origin\",\"type\":[\"string\"]},{\"name\":\"OriginCityName\",\"type\":[\"string\"]},{\"name\":\"OriginState\",\"type\":[\"string\"]},{\"name\":\"OriginStateFips\",\"type\":[\"int\"]},{\"name\":\"OriginStateName\",\"type\":[\"string\"]},{\"name\":\"OriginWac\",\"type\":[\"int\"]},{\"name\":\"DestAirportID\",\"type\":[\"int\"]},{\"name\":\"DestAirportSeqID\",\"type\":[\"int\"]},{\"name\":\"DestCityMarketID\",\"type\":[\"int\"]},{\"name\":\"Dest\",\"type\":[\"string\"]},{\"name\":\"DestCityName\",\"type\":[\"string\"]},{\"name\":\"DestState\",\"type\":[\"string\"]},{\"name\":\"DestStateFips\",\"type\":[\"int\"]},{\"name\":\"DestStateName\",\"type\":[\"string\"]},{\"name\":\"DestWac\",\"type\":[\"int\"]},{\"name\":\"CRSDepTime\",\"type\":[\"int\"]},{\"name\":\"DepTime\",\"type\":[\"int\",\"null\"]},{\"name\":\"DepDelay\",\"type\":[\"int\",\"null\"]},{\"name\":\"DepDelayMinutes\",\"type\":[\"int\",\"null\"]},{\"name\":\"DepDel15\",\"type\":[\"int\",\"null\"]},{\"name\":\"DepartureDelayGroups\",\"type\":[\"int\",\"null\"]},{\"name\":\"DepTimeBlk\",\"type\":[\"string\"]},{\"name\":\"TaxiOut\",\"type\":[\"int\",\"null\"]},{\"name\":\"WheelsOff\",\"type\":[\"int\",\"null\"]},{\"name\":\"WheelsOn\",\"type\":[\"int\",\"null\"]},{\"name\":\"TaxiIn\",\"type\":[\"int\",\"null\"]},{\"name\":\"CRSArrTime\",\"type\":[\"int\"]},{\"name\":\"ArrTime\",\"type\":[\"int\",\"null\"]},{\"name\":\"ArrDelay\",\"type\":[\"int\",\"null\"]},{\"name\":\"ArrDelayMinutes\",\"type\":[\"int\",\"null\"]},{\"name\":\"ArrDel15\",\"type\":[\"int\",\"null\"]},{\"name\":\"ArrivalDelayGroups\",\"type\":[\"int\",\"null\"]},{\"name\":\"ArrTimeBlk\",\"type\":[\"string\"]},{\"name\":\"Cancelled\",\"type\":[\"int\"]},{\"name\":\"CancellationCode\",\"type\":[\"string\",\"null\"]},{\"name\":\"Diverted\",\"type\":[\"int\"]},{\"name\":\"CRSElapsedTime\",\"type\":[\"int\",\"null\"]},{\"name\":\"ActualElapsedTime\",\"type\":[\"int\",\"null\"]},{\"name\":\"AirTime\",\"type\":[\"int\",\"null\"]},{\"name\":\"Flights\",\"type\":[\"int\"]},{\"name\":\"Distance\",\"type\":[\"int\"]},{\"name\":\"DistanceGroup\",\"type\":[\"int\"]},{\"name\":\"CarrierDelay\",\"type\":[\"int\",\"null\"]},{\"name\":\"WeatherDelay\",\"type\":[\"int\",\"null\"]},{\"name\":\"NASDelay\",\"type\":[\"int\",\"null\"]},{\"name\":\"SecurityDelay\",\"type\":[\"int\",\"null\"]},{\"name\":\"LateAircraftDelay\",\"type\":[\"int\",\"null\"]},{\"name\":\"FirstDepTime\",\"type\":[\"int\",\"null\"]},{\"name\":\"TotalAddGTime\",\"type\":[\"int\",\"null\"]},{\"name\":\"LongestAddGTime\",\"type\":[\"int\",\"null\"]},{\"name\":\"DivAirportLandings\",\"type\":[\"int\"]},{\"name\":\"DivReachedDest\",\"type\":[\"int\",\"null\"]},{\"name\":\"DivActualElapsedTime\",\"type\":[\"int\",\"null\"]},{\"name\":\"DivArrDelay\",\"type\":[\"int\",\"null\"]},{\"name\":\"DivDistance\",\"type\":[\"int\",\"null\"]},{\"name\":\"DivAirports\",\"type\":{\"type\":\"array\",\"items\":\"string\"}},{\"name\":\"DivAirportIDs\",\"type\":{\"type\":\"array\",\"items\":\"int\"}},{\"name\":\"DivAirportSeqIDs\",\"type\":{\"type\":\"array\",\"items\":\"int\"}},{\"name\":\"DivWheelsOns\",\"type\":{\"type\":\"array\",\"items\":\"int\"}},{\"name\":\"DivTotalGTimes\",\"type\":{\"type\":\"array\",\"items\":\"int\"}},{\"name\":\"DivLongestGTimes\",\"type\":{\"type\":\"array\",\"items\":\"int\"}},{\"name\":\"DivWheelsOffs\",\"type\":{\"type\":\"array\",\"items\":\"int\"}},{\"name\":\"DivTailNums\",\"type\":{\"type\":\"array\",\"items\":\"string\"}},{\"name\":\"RandomAirports\",\"type\":{\"type\":\"array\",\"items\":\"string\"}}]}", + "stream.kafka.consumer.factory.class.name": "org.apache.pinot.plugin.stream.kafka20.KafkaConsumerFactory", + "stream.kafka.hlc.zk.connect.string": "kafka-zookeeper:2181", + "stream.kafka.zk.broker.url": "kafka-zookeeper:2181", + "stream.kafka.broker.list": "pinot-kafka-controller-0.pinot-kafka-controller-headless.pinot.svc.cluster.local:9092,pinot-kafka-controller-1.pinot-kafka-controller-headless.pinot.svc.cluster.local:9092,pinot-kafka-controller-2.pinot-kafka-controller-headless.pinot.svc.cluster.local:9092", + "realtime.segment.flush.threshold.time": "3600000", + "realtime.segment.flush.threshold.size": "50000", + "stream.kafka.consumer.prop.auto.offset.reset": "smallest" + } + }, + "metadata": { + "customConfigs": {} + } + } + + airlineStats_schema.json: |- + { + "metricFieldSpecs": [ + ], + "dimensionFieldSpecs": [ + { + "dataType": "INT", + "name": "ActualElapsedTime" + }, + { + "dataType": "INT", + "name": "AirTime" + }, + { + "dataType": "INT", + "name": "AirlineID" + }, + { + "dataType": "INT", + "name": "ArrDel15" + }, + { + "dataType": "INT", + "name": "ArrDelay" + }, + { + "dataType": "INT", + "name": "ArrDelayMinutes" + }, + { + "dataType": "INT", + "name": "ArrTime" + }, + { + "dataType": "STRING", + "name": "ArrTimeBlk" + }, + { + "dataType": "INT", + "name": "ArrivalDelayGroups" + }, + { + "dataType": "INT", + "name": "CRSArrTime" + }, + { + "dataType": "INT", + "name": "CRSDepTime" + }, + { + "dataType": "INT", + "name": "CRSElapsedTime" + }, + { + "dataType": "STRING", + "name": "CancellationCode" + }, + { + "dataType": "INT", + "name": "Cancelled" + }, + { + "dataType": "STRING", + "name": "Carrier" + }, + { + "dataType": "INT", + "name": "CarrierDelay" + }, + { + "dataType": "INT", + "name": "DayOfWeek" + }, + { + "dataType": "INT", + "name": "DayofMonth" + }, + { + "dataType": "INT", + "name": "DepDel15" + }, + { + "dataType": "INT", + "name": "DepDelay" + }, + { + "dataType": "INT", + "name": "DepDelayMinutes" + }, + { + "dataType": "INT", + "name": "DepTime" + }, + { + "dataType": "STRING", + "name": "DepTimeBlk" + }, + { + "dataType": "INT", + "name": "DepartureDelayGroups" + }, + { + "dataType": "STRING", + "name": "Dest" + }, + { + "dataType": "INT", + "name": "DestAirportID" + }, + { + "dataType": "INT", + "name": "DestAirportSeqID" + }, + { + "dataType": "INT", + "name": "DestCityMarketID" + }, + { + "dataType": "STRING", + "name": "DestCityName" + }, + { + "dataType": "STRING", + "name": "DestState" + }, + { + "dataType": "INT", + "name": "DestStateFips" + }, + { + "dataType": "STRING", + "name": "DestStateName" + }, + { + "dataType": "INT", + "name": "DestWac" + }, + { + "dataType": "INT", + "name": "Distance" + }, + { + "dataType": "INT", + "name": "DistanceGroup" + }, + { + "dataType": "INT", + "name": "DivActualElapsedTime" + }, + { + "dataType": "INT", + "name": "DivAirportIDs", + "singleValueField": false + }, + { + "dataType": "INT", + "name": "DivAirportLandings" + }, + { + "dataType": "INT", + "name": "DivAirportSeqIDs", + "singleValueField": false + }, + { + "dataType": "STRING", + "name": "DivAirports", + "singleValueField": false + }, + { + "dataType": "INT", + "name": "DivArrDelay" + }, + { + "dataType": "INT", + "name": "DivDistance" + }, + { + "dataType": "INT", + "name": "DivLongestGTimes", + "singleValueField": false + }, + { + "dataType": "INT", + "name": "DivReachedDest" + }, + { + "dataType": "STRING", + "name": "DivTailNums", + "singleValueField": false + }, + { + "dataType": "INT", + "name": "DivTotalGTimes", + "singleValueField": false + }, + { + "dataType": "INT", + "name": "DivWheelsOffs", + "singleValueField": false + }, + { + "dataType": "INT", + "name": "DivWheelsOns", + "singleValueField": false + }, + { + "dataType": "INT", + "name": "Diverted" + }, + { + "dataType": "INT", + "name": "FirstDepTime" + }, + { + "dataType": "STRING", + "name": "FlightDate" + }, + { + "dataType": "INT", + "name": "FlightNum" + }, + { + "dataType": "INT", + "name": "Flights" + }, + { + "dataType": "INT", + "name": "LateAircraftDelay" + }, + { + "dataType": "INT", + "name": "LongestAddGTime" + }, + { + "dataType": "INT", + "name": "Month" + }, + { + "dataType": "INT", + "name": "NASDelay" + }, + { + "dataType": "STRING", + "name": "Origin" + }, + { + "dataType": "INT", + "name": "OriginAirportID" + }, + { + "dataType": "INT", + "name": "OriginAirportSeqID" + }, + { + "dataType": "INT", + "name": "OriginCityMarketID" + }, + { + "dataType": "STRING", + "name": "OriginCityName" + }, + { + "dataType": "STRING", + "name": "OriginState" + }, + { + "dataType": "INT", + "name": "OriginStateFips" + }, + { + "dataType": "STRING", + "name": "OriginStateName" + }, + { + "dataType": "INT", + "name": "OriginWac" + }, + { + "dataType": "INT", + "name": "Quarter" + }, + { + "dataType": "STRING", + "name": "RandomAirports", + "singleValueField": false + }, + { + "dataType": "INT", + "name": "SecurityDelay" + }, + { + "dataType": "STRING", + "name": "TailNum" + }, + { + "dataType": "INT", + "name": "TaxiIn" + }, + { + "dataType": "INT", + "name": "TaxiOut" + }, + { + "dataType": "INT", + "name": "Year" + }, + { + "dataType": "INT", + "name": "WheelsOn" + }, + { + "dataType": "INT", + "name": "WheelsOff" + }, + { + "dataType": "INT", + "name": "WeatherDelay" + }, + { + "dataType": "STRING", + "name": "UniqueCarrier" + }, + { + "dataType": "INT", + "name": "TotalAddGTime" + } + ], + "dateTimeFieldSpecs": [ + { + "name": "DaysSinceEpoch", + "dataType": "INT", + "format": "1:DAYS:EPOCH", + "granularity": "1:DAYS" + } + ], + "schemaName": "airlineStats" + } + + airlineStatsAvro_schema.json: |- + { + "metricFieldSpecs": [ + ], + "dimensionFieldSpecs": [ + { + "dataType": "INT", + "name": "ActualElapsedTime" + }, + { + "dataType": "INT", + "name": "AirTime" + }, + { + "dataType": "INT", + "name": "AirlineID" + }, + { + "dataType": "INT", + "name": "ArrDel15" + }, + { + "dataType": "INT", + "name": "ArrDelay" + }, + { + "dataType": "INT", + "name": "ArrDelayMinutes" + }, + { + "dataType": "INT", + "name": "ArrTime" + }, + { + "dataType": "STRING", + "name": "ArrTimeBlk" + }, + { + "dataType": "INT", + "name": "ArrivalDelayGroups" + }, + { + "dataType": "INT", + "name": "CRSArrTime" + }, + { + "dataType": "INT", + "name": "CRSDepTime" + }, + { + "dataType": "INT", + "name": "CRSElapsedTime" + }, + { + "dataType": "STRING", + "name": "CancellationCode" + }, + { + "dataType": "INT", + "name": "Cancelled" + }, + { + "dataType": "STRING", + "name": "Carrier" + }, + { + "dataType": "INT", + "name": "CarrierDelay" + }, + { + "dataType": "INT", + "name": "DayOfWeek" + }, + { + "dataType": "INT", + "name": "DayofMonth" + }, + { + "dataType": "INT", + "name": "DepDel15" + }, + { + "dataType": "INT", + "name": "DepDelay" + }, + { + "dataType": "INT", + "name": "DepDelayMinutes" + }, + { + "dataType": "INT", + "name": "DepTime" + }, + { + "dataType": "STRING", + "name": "DepTimeBlk" + }, + { + "dataType": "INT", + "name": "DepartureDelayGroups" + }, + { + "dataType": "STRING", + "name": "Dest" + }, + { + "dataType": "INT", + "name": "DestAirportID" + }, + { + "dataType": "INT", + "name": "DestAirportSeqID" + }, + { + "dataType": "INT", + "name": "DestCityMarketID" + }, + { + "dataType": "STRING", + "name": "DestCityName" + }, + { + "dataType": "STRING", + "name": "DestState" + }, + { + "dataType": "INT", + "name": "DestStateFips" + }, + { + "dataType": "STRING", + "name": "DestStateName" + }, + { + "dataType": "INT", + "name": "DestWac" + }, + { + "dataType": "INT", + "name": "Distance" + }, + { + "dataType": "INT", + "name": "DistanceGroup" + }, + { + "dataType": "INT", + "name": "DivActualElapsedTime" + }, + { + "dataType": "INT", + "name": "DivAirportIDs", + "singleValueField": false + }, + { + "dataType": "INT", + "name": "DivAirportLandings" + }, + { + "dataType": "INT", + "name": "DivAirportSeqIDs", + "singleValueField": false + }, + { + "dataType": "STRING", + "name": "DivAirports", + "singleValueField": false + }, + { + "dataType": "INT", + "name": "DivArrDelay" + }, + { + "dataType": "INT", + "name": "DivDistance" + }, + { + "dataType": "INT", + "name": "DivLongestGTimes", + "singleValueField": false + }, + { + "dataType": "INT", + "name": "DivReachedDest" + }, + { + "dataType": "STRING", + "name": "DivTailNums", + "singleValueField": false + }, + { + "dataType": "INT", + "name": "DivTotalGTimes", + "singleValueField": false + }, + { + "dataType": "INT", + "name": "DivWheelsOffs", + "singleValueField": false + }, + { + "dataType": "INT", + "name": "DivWheelsOns", + "singleValueField": false + }, + { + "dataType": "INT", + "name": "Diverted" + }, + { + "dataType": "INT", + "name": "FirstDepTime" + }, + { + "dataType": "STRING", + "name": "FlightDate" + }, + { + "dataType": "INT", + "name": "FlightNum" + }, + { + "dataType": "INT", + "name": "Flights" + }, + { + "dataType": "INT", + "name": "LateAircraftDelay" + }, + { + "dataType": "INT", + "name": "LongestAddGTime" + }, + { + "dataType": "INT", + "name": "Month" + }, + { + "dataType": "INT", + "name": "NASDelay" + }, + { + "dataType": "STRING", + "name": "Origin" + }, + { + "dataType": "INT", + "name": "OriginAirportID" + }, + { + "dataType": "INT", + "name": "OriginAirportSeqID" + }, + { + "dataType": "INT", + "name": "OriginCityMarketID" + }, + { + "dataType": "STRING", + "name": "OriginCityName" + }, + { + "dataType": "STRING", + "name": "OriginState" + }, + { + "dataType": "INT", + "name": "OriginStateFips" + }, + { + "dataType": "STRING", + "name": "OriginStateName" + }, + { + "dataType": "INT", + "name": "OriginWac" + }, + { + "dataType": "INT", + "name": "Quarter" + }, + { + "dataType": "STRING", + "name": "RandomAirports", + "singleValueField": false + }, + { + "dataType": "INT", + "name": "SecurityDelay" + }, + { + "dataType": "STRING", + "name": "TailNum" + }, + { + "dataType": "INT", + "name": "TaxiIn" + }, + { + "dataType": "INT", + "name": "TaxiOut" + }, + { + "dataType": "INT", + "name": "Year" + }, + { + "dataType": "INT", + "name": "WheelsOn" + }, + { + "dataType": "INT", + "name": "WheelsOff" + }, + { + "dataType": "INT", + "name": "WeatherDelay" + }, + { + "dataType": "STRING", + "name": "UniqueCarrier" + }, + { + "dataType": "INT", + "name": "TotalAddGTime" + } + ], + "dateTimeFieldSpecs": [ + { + "name": "DaysSinceEpoch", + "dataType": "INT", + "format": "1:DAYS:EPOCH", + "granularity": "1:DAYS" + } + ], + "schemaName": "airlineStatsAvro" + } +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: pinot-realtime-quickstart-pinot-table-creation + namespace: pinot +spec: + template: + spec: + containers: + - name: pinot-add-example-realtime-table-json + image: apachepinot/pinot:latest + args: + [ + "AddTable", + "-schemaFile", + "/var/pinot/examples/airlineStats_schema.json", + "-tableConfigFile", + "/var/pinot/examples/airlineStats_realtime_table_config.json", + "-controllerHost", + "pinot-controller", + "-controllerPort", + "9000", + "-exec", + ] + env: + - name: JAVA_OPTS + value: "-Xms4G -Xmx4G -Dpinot.admin.system.exit=true" + volumeMounts: + - name: examples + mountPath: /var/pinot/examples + - name: pinot-add-example-realtime-table-avro + image: apachepinot/pinot:latest + args: + [ + "AddTable", + "-schemaFile", + "/var/pinot/examples/airlineStatsAvro_schema.json", + "-tableConfigFile", + "/var/pinot/examples/airlineStatsAvro_realtime_table_config.json", + "-controllerHost", + "pinot-controller", + "-controllerPort", + "9000", + "-exec", + ] + env: + - name: JAVA_OPTS + value: "-Xms4G -Xmx4G -Dpinot.admin.system.exit=true" + volumeMounts: + - name: examples + mountPath: /var/pinot/examples + restartPolicy: OnFailure + volumes: + - name: examples + configMap: + name: examples + backoffLimit: 100 +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: pinot-realtime-quickstart-load-data-into-kafka + namespace: pinot +spec: + template: + spec: + containers: + - name: loading-json-data-to-kafka + image: apachepinot/pinot:latest + args: + [ + "StreamAvroIntoKafka", + "-avroFile", + "examples/stream/airlineStats/rawdata/airlineStats_data.avro", + "-kafkaTopic", + "flights-realtime", + "-kafkaBrokerList", + "pinot-kafka-controller-0.pinot-kafka-controller-headless.pinot.svc.cluster.local:9092,pinot-kafka-controller-1.pinot-kafka-controller-headless.pinot.svc.cluster.local:9092,pinot-kafka-controller-2.pinot-kafka-controller-headless.pinot.svc.cluster.local:9092", + "-zkAddress", + "kafka-zookeeper:2181", + ] + - name: loading-avro-data-to-kafka + image: apachepinot/pinot:latest + args: + [ + "StreamAvroIntoKafka", + "-avroFile", + "examples/stream/airlineStats/rawdata/airlineStats_data.avro", + "-kafkaTopic", + "flights-realtime-avro", + "-kafkaBrokerList", + "pinot-kafka-controller-0.pinot-kafka-controller-headless.pinot.svc.cluster.local:9092,pinot-kafka-controller-1.pinot-kafka-controller-headless.pinot.svc.cluster.local:9092,pinot-kafka-controller-2.pinot-kafka-controller-headless.pinot.svc.cluster.local:9092", + "-zkAddress", + "kafka-zookeeper:2181", + "-outputFormat", + "avro", + ] + restartPolicy: OnFailure + backoffLimit: 3 diff --git a/distributed-databases/pinot/helm/aws-for-fluentbit-values.yaml b/distributed-databases/pinot/helm/aws-for-fluentbit-values.yaml new file mode 100644 index 000000000..5e5cca95c --- /dev/null +++ b/distributed-databases/pinot/helm/aws-for-fluentbit-values.yaml @@ -0,0 +1,79 @@ +global: + +#hostNetwork and dnsPolicy are critical for enabling large clusters to avoid making calls to API server +# see this link https://docs.fluentbit.io/manual/pipeline/filters/kubernetes#optional-feature-using-kubelet-to-get-metadata +hostNetwork: true +dnsPolicy: ClusterFirstWithHostNet + +service: + parsersFiles: + - /fluent-bit/parsers/parsers.conf + extraParsers: | + [PARSER] + Name kubernetes + Format regex + Regex ^(?[^_]+)\.(?.+)\.(?[a-z0-9](?:[-a-z0-9]*[a-z0-9])?(?:\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)\.(?[a-z0-9]{64})-$ + +input: + name: "tail" + enabled: true + tag: "systempods....-" + path: "/var/log/containers/*.log" + db: "/var/log/flb_kube.db" + memBufLimit: 5MB + skipLongLines: "On" + refreshInterval: 10 + extraInputs: | + multiline.parser docker, cri + Tag_Regex (?[a-z0-9](?:[-a-z0-9]*[a-z0-9])?(?:\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)_(?[^_]+)_(?.+)-(?[a-z0-9]{64})\.log$ + +# NOTE: extraFilters config for using Kubelet to get the Metadata instead of talking to API server for large clusters +filter: + name: "kubernetes" + match: "systempods.*" + kubeURL: "https://kubernetes.default.svc.cluster.local:443" + mergeLog: "On" + mergeLogKey: "log_processed" + keepLog: "On" + k8sLoggingParser: "On" + k8sLoggingExclude: "Off" + bufferSize: "0" + extraFilters: | + Kube_Tag_Prefix systempods. + Regex_Parser kubernetes + Labels On + Annotations Off + Use_Kubelet true + Kubelet_Port 10250 + Kube_CA_File /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + Kube_Token_File /var/run/secrets/kubernetes.io/serviceaccount/token + +# CATION: Do not use `cloudwatch` plugin. This Golang Plugin is not recommended by AWS anymore instead use C plugin(`cloudWatchLogs`) for better performance. +# cloudWatch: +# enabled: false + +#ย This is a new high performance C Plugin for CloudWatchLogs. See docs here https://docs.fluentbit.io/manual/pipeline/outputs/cloudwatch +cloudWatchLogs: + enabled: true + match: "systempods.*" + region: ${region} + logGroupName: ${cloudwatch_log_group} + autoCreateGroup: false + extraOutputs: | + log_key log + +# Resource config for large clusters +resources: + limits: + cpu: 1000m + memory: 1500Mi + requests: + cpu: 500m + memory: 500Mi + +## Assign a PriorityClassName to pods if set +priorityClassName: system-node-critical + +# This toleration allows Daemonset pod to be scheduled on any node, regardless of their Taints. +tolerations: + - operator: Exists diff --git a/distributed-databases/pinot/helm/kafka-values.yaml b/distributed-databases/pinot/helm/kafka-values.yaml new file mode 100644 index 000000000..8ed1e5787 --- /dev/null +++ b/distributed-databases/pinot/helm/kafka-values.yaml @@ -0,0 +1,9 @@ +listeners: + client: + protocol: PLAINTEXT + controller: + protocol: PLAINTEXT + interbroker: + protocol: PLAINTEXT + external: + protocol: PLAINTEXT diff --git a/distributed-databases/pinot/helm/kube-prometheus-stack-values.yaml b/distributed-databases/pinot/helm/kube-prometheus-stack-values.yaml new file mode 100644 index 000000000..9acf4aabb --- /dev/null +++ b/distributed-databases/pinot/helm/kube-prometheus-stack-values.yaml @@ -0,0 +1,74 @@ +prometheus: + prometheusSpec: + retention: 5h + scrapeInterval: 5s + # evaluationInterval: 30s + scrapeTimeout: 2s + storageSpec: + volumeClaimTemplate: + metadata: + name: data + spec: + storageClassName: ${storage_class_type} + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 50Gi + # Scrape metrics for Apache Pinot add-on + additionalScrapeConfigs: + - job_name: "kubernetes-service-endpoints" + honor_labels: true + + kubernetes_sd_configs: + - role: endpoints + + relabel_configs: + - source_labels: + [__meta_kubernetes_service_annotation_prometheus_io_scrape] + action: keep + regex: true + - source_labels: + [__meta_kubernetes_service_annotation_prometheus_io_scrape_slow] + action: drop + regex: true + - source_labels: + [__meta_kubernetes_service_annotation_prometheus_io_scheme] + action: replace + target_label: __scheme__ + regex: (https?) + - source_labels: + [__meta_kubernetes_service_annotation_prometheus_io_path] + action: replace + target_label: __metrics_path__ + regex: (.+) + - source_labels: + [ + __address__, + __meta_kubernetes_service_annotation_prometheus_io_port, + ] + action: replace + target_label: __address__ + regex: (.+?)(?::\d+)?;(\d+) + replacement: $1:$2 + - action: labelmap + regex: __meta_kubernetes_service_annotation_prometheus_io_param_(.+) + replacement: __param_$1 + - action: labelmap + regex: __meta_kubernetes_service_label_(.+) + - source_labels: [__meta_kubernetes_namespace] + action: replace + target_label: namespace + - source_labels: [__meta_kubernetes_service_name] + action: replace + target_label: service + - source_labels: [__meta_kubernetes_pod_node_name] + action: replace + target_label: node + +alertmanager: + enabled: false + +grafana: + enabled: true + defaultDashboardsEnabled: true diff --git a/distributed-databases/pinot/helm/pinot-values.yaml b/distributed-databases/pinot/helm/pinot-values.yaml new file mode 100644 index 000000000..a78faee36 --- /dev/null +++ b/distributed-databases/pinot/helm/pinot-values.yaml @@ -0,0 +1,566 @@ +# ------------------------------------------------------------------------------ +# Pinot Controller: +# ------------------------------------------------------------------------------ +controller: + name: controller + replicaCount: 1 + podManagementPolicy: Parallel + podSecurityContext: + {} + # fsGroup: 2000 + securityContext: {} + startCommand: "StartController" + + probes: + endpoint: "/health" + livenessEnabled: false + readinessEnabled: false + + persistence: + enabled: true + accessMode: ReadWriteOnce + size: 1G + mountPath: /var/pinot/controller/data + storageClass: "" + extraVolumes: [] + extraVolumeMounts: [] + + data: + dir: /var/pinot/controller/data + + vip: + enabled: false + host: pinot-controller + port: 9000 + + #jvmOpts: "-XX:ActiveProcessorCount=2 -Xms256M -Xmx1G -XX:+UseG1GC -XX:MaxGCPauseMillis=200 -Xlog:gc*:file=/opt/pinot/gc-pinot-controller.log" + jvmOpts: "-javaagent:/opt/pinot/etc/jmx_prometheus_javaagent/jmx_prometheus_javaagent.jar=8008:/opt/pinot/etc/jmx_prometheus_javaagent/configs/pinot.yml -Xms256M -Xmx1G" + + log4j2ConfFile: /opt/pinot/conf/log4j2.xml + pluginsDir: /opt/pinot/plugins + + service: + annotations: + "prometheus.io/scrape": "true" + "prometheus.io/port": "8008" + clusterIP: "None" + externalIPs: [] + loadBalancerIP: "" + loadBalancerSourceRanges: [] + type: ClusterIP + port: 9000 + nodePort: "" + protocol: TCP + name: controller + + external: + enabled: false + type: LoadBalancer + port: 9000 + annotations: {} + + ingress: + v1beta1: + enabled: false + annotations: {} + tls: {} + path: / + hosts: [] + v1: + enabled: false + ingressClassName: "" + annotations: {} + tls: [] + path: / + hosts: [] + + resources: + requests: + memory: "1.25Gi" + + nodeSelector: + NodeGroupType: controller + + tolerations: + - key: "dedicated" + operator: "Equal" + value: "pinot" + effect: "NoSchedule" + + affinity: {} + + podAnnotations: + "prometheus.io/scrape": "true" + "prometheus.io/port": "8008" + + updateStrategy: + type: RollingUpdate + + # Use envFrom to define all of the ConfigMap or Secret data as container environment variables. + # ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-pod-configmap/#configure-all-key-value-pairs-in-a-configmap-as-container-environment-variables + # ref: https://kubernetes.io/docs/tasks/inject-data-application/distribute-credentials-secure/#configure-all-key-value-pairs-in-a-secret-as-container-environment-variables + envFrom: [] + # - configMapRef: + # name: special-config + # - secretRef: + # name: test-secret + + # Use extraEnv to add individual key value pairs as container environment variables. + # ref: https://kubernetes.io/docs/tasks/inject-data-application/define-environment-variable-container/ + extraEnv: + - name: LOG4J_CONSOLE_LEVEL + value: info + # - name: PINOT_CUSTOM_ENV + # value: custom-value + + # Extra configs will be appended to pinot-controller.conf file + extra: + configs: |- + pinot.set.instance.id.to.hostname=true + controller.task.scheduler.enabled=true + +# ------------------------------------------------------------------------------ +# Pinot Broker: +# ------------------------------------------------------------------------------ +broker: + name: broker + replicaCount: 1 + podManagementPolicy: Parallel + podSecurityContext: + {} + # fsGroup: 2000 + securityContext: {} + startCommand: "StartBroker" + + #jvmOpts: "-XX:ActiveProcessorCount=2 -Xms256M -Xmx1G -XX:+UseG1GC -XX:MaxGCPauseMillis=200 -Xlog:gc*:file=/opt/pinot/gc-pinot-broker.log" + jvmOpts: "-javaagent:/opt/pinot/etc/jmx_prometheus_javaagent/jmx_prometheus_javaagent.jar=8008:/opt/pinot/etc/jmx_prometheus_javaagent/configs/pinot.yml -Xms256M -Xmx1G" + + log4j2ConfFile: /opt/pinot/conf/log4j2.xml + pluginsDir: /opt/pinot/plugins + + routingTable: + builderClass: random + + probes: + endpoint: "/health" + livenessEnabled: true + readinessEnabled: true + + persistence: + extraVolumes: [] + extraVolumeMounts: [] + + service: + annotations: + "prometheus.io/scrape": "true" + "prometheus.io/port": "8008" + clusterIP: "None" + externalIPs: [] + loadBalancerIP: "" + loadBalancerSourceRanges: [] + type: ClusterIP + protocol: TCP + port: 8099 + name: broker + nodePort: "" + + external: + enabled: false + type: LoadBalancer + port: 8099 + # For example, in private GKE cluster, you might add cloud.google.com/load-balancer-type: Internal + annotations: {} + + ingress: + v1beta1: + enabled: false + annotations: {} + tls: {} + path: / + hosts: [] + v1: + enabled: false + ingressClassName: "" + annotations: {} + tls: [] + path: / + hosts: [] + + resources: + requests: + memory: "1.25Gi" + + nodeSelector: + NodeGroupType: broker + + affinity: {} + + tolerations: + - key: "dedicated" + operator: "Equal" + value: "pinot" + effect: "NoSchedule" + + podAnnotations: + "prometheus.io/scrape": "true" + "prometheus.io/port": "8008" + + updateStrategy: + type: RollingUpdate + + # Use envFrom to define all of the ConfigMap or Secret data as container environment variables. + # ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-pod-configmap/#configure-all-key-value-pairs-in-a-configmap-as-container-environment-variables + # ref: https://kubernetes.io/docs/tasks/inject-data-application/distribute-credentials-secure/#configure-all-key-value-pairs-in-a-secret-as-container-environment-variables + envFrom: [] + # - configMapRef: + # name: special-config + # - secretRef: + # name: test-secret + + # Use extraEnv to add individual key value pairs as container environment variables. + # ref: https://kubernetes.io/docs/tasks/inject-data-application/define-environment-variable-container/ + extraEnv: + - name: LOG4J_CONSOLE_LEVEL + value: info + # - name: PINOT_CUSTOM_ENV + # value: custom-value + + # Extra configs will be appended to pinot-broker.conf file + extra: + configs: |- + pinot.set.instance.id.to.hostname=true + +# ------------------------------------------------------------------------------ +# Pinot Server: +# ------------------------------------------------------------------------------ +server: + name: server + replicaCount: 1 + podManagementPolicy: Parallel + podSecurityContext: + {} + # fsGroup: 2000 + securityContext: {} + startCommand: "StartServer" + + probes: + endpoint: "/health" + livenessEnabled: false + readinessEnabled: false + + dataDir: /var/pinot/server/data/index + segmentTarDir: /var/pinot/server/data/segment + + persistence: + enabled: true + accessMode: ReadWriteOnce + size: 4G + mountPath: /var/pinot/server/data + storageClass: "" + #storageClass: "ssd" + extraVolumes: [] + extraVolumeMounts: [] + + #jvmOpts: "-Xms512M -Xmx1G -XX:+UseG1GC -XX:MaxGCPauseMillis=200 -Xlog:gc*:file=/opt/pinot/gc-pinot-server.log" + jvmOpts: "-javaagent:/opt/pinot/etc/jmx_prometheus_javaagent/jmx_prometheus_javaagent.jar=8008:/opt/pinot/etc/jmx_prometheus_javaagent/configs/pinot.yml -Xms256M -Xmx1G" + + log4j2ConfFile: /opt/pinot/conf/log4j2.xml + pluginsDir: /opt/pinot/plugins + + service: + annotations: + "prometheus.io/scrape": "true" + "prometheus.io/port": "8008" + clusterIP: "" + externalIPs: [] + loadBalancerIP: "" + loadBalancerSourceRanges: [] + type: ClusterIP + nettyPort: 8098 + nettyPortName: netty + adminPort: 8097 + adminExposePort: 80 + adminPortName: admin + nodePort: "" + protocol: TCP + + resources: + requests: + memory: "1.25Gi" + + nodeSelector: + NodeGroupType: server + + affinity: {} + + tolerations: + - key: "dedicated" + operator: "Equal" + value: "pinot" + effect: "NoSchedule" + + podAnnotations: + "prometheus.io/scrape": "true" + "prometheus.io/port": "8008" + + updateStrategy: + type: RollingUpdate + + # Use envFrom to define all of the ConfigMap or Secret data as container environment variables. + # ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-pod-configmap/#configure-all-key-value-pairs-in-a-configmap-as-container-environment-variables + # ref: https://kubernetes.io/docs/tasks/inject-data-application/distribute-credentials-secure/#configure-all-key-value-pairs-in-a-secret-as-container-environment-variables + envFrom: [] + # - configMapRef: + # name: special-config + # - secretRef: + # name: test-secret + + # Use extraEnv to add individual key value pairs as container environment variables. + # ref: https://kubernetes.io/docs/tasks/inject-data-application/define-environment-variable-container/ + extraEnv: + - name: LOG4J_CONSOLE_LEVEL + value: info + # - name: PINOT_CUSTOM_ENV + # value: custom-value + + # Extra configs will be appended to pinot-server.conf file + extra: + configs: |- + pinot.set.instance.id.to.hostname=true + pinot.server.instance.realtime.alloc.offheap=true + +# ------------------------------------------------------------------------------ +# Pinot Minion: +# ------------------------------------------------------------------------------ +minion: + enabled: false + name: minion + replicaCount: 0 + podManagementPolicy: Parallel + podSecurityContext: + {} + # fsGroup: 2000 + securityContext: {} + startCommand: "StartMinion" + + probes: + endpoint: "/health" + livenessEnabled: true + readinessEnabled: true + + dataDir: /var/pinot/minion/data + jvmOpts: "-XX:ActiveProcessorCount=2 -Xms256M -Xmx1G -XX:+UseG1GC -XX:MaxGCPauseMillis=200 -Xlog:gc*:file=/opt/pinot/gc-pinot-minion.log" + + log4j2ConfFile: /opt/pinot/conf/log4j2.xml + pluginsDir: /opt/pinot/plugins + + persistence: + enabled: true + accessMode: ReadWriteOnce + size: 4G + mountPath: /var/pinot/minion/data + storageClass: "" + #storageClass: "ssd" + extraVolumes: [] + extraVolumeMounts: [] + + service: + annotations: {} + clusterIP: "" + externalIPs: [] + loadBalancerIP: "" + loadBalancerSourceRanges: [] + type: ClusterIP + port: 9514 + nodePort: "" + protocol: TCP + name: minion + + resources: + requests: + memory: "1.25Gi" + + nodeSelector: + NodeGroupType: minion + + affinity: {} + + tolerations: + - key: "dedicated" + operator: "Equal" + value: "pinot" + effect: "NoSchedule" + + podAnnotations: {} + + updateStrategy: + type: RollingUpdate + + # Use envFrom to define all of the ConfigMap or Secret data as container environment variables. + # ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-pod-configmap/#configure-all-key-value-pairs-in-a-configmap-as-container-environment-variables + # ref: https://kubernetes.io/docs/tasks/inject-data-application/distribute-credentials-secure/#configure-all-key-value-pairs-in-a-secret-as-container-environment-variables + envFrom: [] + # - configMapRef: + # name: special-config + # - secretRef: + # name: test-secret + + # Use extraEnv to add individual key value pairs as container environment variables. + # ref: https://kubernetes.io/docs/tasks/inject-data-application/define-environment-variable-container/ + extraEnv: + - name: LOG4J_CONSOLE_LEVEL + value: info + # - name: PINOT_CUSTOM_ENV + # value: custom-value + + # Extra configs will be appended to pinot-minion.conf file + extra: + configs: |- + pinot.set.instance.id.to.hostname=true + +# ------------------------------------------------------------------------------ +# Pinot Minion Stateless: +# ------------------------------------------------------------------------------ +minionStateless: + enabled: true + name: minion-stateless + replicaCount: 1 + podSecurityContext: + {} + # fsGroup: 2000 + securityContext: {} + startCommand: "StartMinion" + + probes: + endpoint: "/health" + livenessEnabled: true + readinessEnabled: true + + dataDir: /var/pinot/minion/data + jvmOpts: "-XX:ActiveProcessorCount=2 -Xms256M -Xmx1G -XX:+UseG1GC -XX:MaxGCPauseMillis=200 -Xlog:gc*:file=/opt/pinot/gc-pinot-minion.log" + + log4j2ConfFile: /opt/pinot/conf/log4j2.xml + pluginsDir: /opt/pinot/plugins + + persistence: + enabled: false + pvcName: minion-data-vol + accessMode: ReadWriteOnce + size: 4G + mountPath: /var/pinot/minion/data + storageClass: "" + #storageClass: "ssd" + extraVolumes: [] + extraVolumeMounts: [] + + service: + port: 9514 + protocol: TCP + name: minion + + resources: + requests: + memory: "1.25Gi" + + nodeSelector: + NodeGroupType: minion + + affinity: {} + + tolerations: + - key: "dedicated" + operator: "Equal" + value: "pinot" + effect: "NoSchedule" + + podAnnotations: {} + + # Use envFrom to define all of the ConfigMap or Secret data as container environment variables. + # ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-pod-configmap/#configure-all-key-value-pairs-in-a-configmap-as-container-environment-variables + # ref: https://kubernetes.io/docs/tasks/inject-data-application/distribute-credentials-secure/#configure-all-key-value-pairs-in-a-secret-as-container-environment-variables + envFrom: [] + # - configMapRef: + # name: special-config + # - secretRef: + # name: test-secret + + # Use extraEnv to add individual key value pairs as container environment variables. + # ref: https://kubernetes.io/docs/tasks/inject-data-application/define-environment-variable-container/ + extraEnv: + - name: LOG4J_CONSOLE_LEVEL + value: info + # - name: PINOT_CUSTOM_ENV + # value: custom-value + + # Extra configs will be appended to pinot-minion.conf file + extra: + configs: |- + pinot.set.instance.id.to.hostname=true + +# ------------------------------------------------------------------------------ +# Zookeeper: +# NOTE: IN PRODUCTION USE CASES, IT's BEST TO USE ZOOKEEPER K8S OPERATOR +# ref: https://github.com/pravega/zookeeper-operator#install-the-operator +# ------------------------------------------------------------------------------ + +zookeeper: + ## If true, install the Zookeeper chart alongside Pinot + ## ref: https://github.com/bitnami/charts/tree/master/bitnami/zookeeper + enabled: true + + ## If the Zookeeper Chart is disabled a URL override is required to connect + urlOverride: "my-zookeeper:2181/my-pinot" + + ## Zookeeper port + port: 2181 + + ## Configure Zookeeper resource requests and limits + ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ + resources: + requests: + memory: "1.25Gi" + + ## Replicas + replicaCount: 1 + + ## Ongoing data directory cleanup configuration + autopurge: + ## The time interval (in hours) for which the purge task has to be triggered + ## Set to a positive integer to enable the auto purging + purgeInterval: 1 + + ## The most recent snapshots amount (and corresponding transaction logs) to retain + snapRetainCount: 5 + + ## Size (in MB) for the Java Heap options (Xmx and Xms) + ## This env var is ignored if Xmx an Xms are configured via `zookeeper.jvmFlags` + heapSize: "1024" + + persistence: + enabled: true + ## The amount of PV storage allocated to each Zookeeper pod in the statefulset + # size: "2Gi" + + ## Specify a Zookeeper imagePullPolicy + ## ref: http://kubernetes.io/docs/user-guide/images/#pre-pulling-images + image: + PullPolicy: "IfNotPresent" + + ## Pod scheduling preferences (by default keep pods within a release on separate nodes). + ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity + ## By default we don't set affinity: + affinity: {} # Criteria by which pod label-values influence scheduling for zookeeper pods. + # podAntiAffinity: + # requiredDuringSchedulingIgnoredDuringExecution: + # - topologyKey: "kubernetes.io/hostname" + # labelSelector: + # matchLabels: + # release: zookeeper + nodeSelector: + NodeGroupType: zookeeper + + tolerations: + - key: "dedicated" + operator: "Equal" + value: "pinot" + effect: "NoSchedule" diff --git a/distributed-databases/pinot/install.sh b/distributed-databases/pinot/install.sh new file mode 100755 index 000000000..463719edd --- /dev/null +++ b/distributed-databases/pinot/install.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +echo "Initializing ..." +terraform init || echo "\"terraform init\" failed" + +# List of Terraform modules to apply in sequence +targets=( + "module.vpc" + "module.eks" + "module.ebs_csi_driver_irsa" + "module.vpc_cni_irsa" + "module.eks_blueprints_kubernetes_addons" +) + +# Apply modules in sequence +for target in "${targets[@]}" +do + echo "Applying module $target..." + apply_output=$(terraform apply -target="$target" -auto-approve 2>&1 | tee /dev/tty) + if [[ ${PIPESTATUS[0]} -eq 0 && $apply_output == *"Apply complete"* ]]; then + echo "SUCCESS: Terraform apply of $target completed successfully" + else + echo "FAILED: Terraform apply of $target failed" + exit 1 + fi +done + +# Final apply to catch any remaining resources +echo "Applying remaining resources..." +apply_output=$(terraform apply -auto-approve 2>&1 | tee /dev/tty) +if [[ ${PIPESTATUS[0]} -eq 0 && $apply_output == *"Apply complete"* ]]; then + echo "SUCCESS: Terraform apply of all modules completed successfully" +else + echo "FAILED: Terraform apply of all modules failed" + exit 1 +fi diff --git a/distributed-databases/pinot/main.tf b/distributed-databases/pinot/main.tf new file mode 100644 index 000000000..5e0aae1a3 --- /dev/null +++ b/distributed-databases/pinot/main.tf @@ -0,0 +1,211 @@ +#--------------------------------------------------------------- +# Local variables +#--------------------------------------------------------------- +locals { + name = var.name + region = var.region + cluster_name = format("%s-%s", local.name, "cluster") + + azs = slice(data.aws_availability_zones.available.names, 0, 3) + + tags = { + Blueprint = local.name + GithubRepo = "github.com/awslabs/data-on-eks" + } +} + +#--------------------------------------------------------------- +# Grafana Admin credentials resources +#--------------------------------------------------------------- +data "aws_secretsmanager_secret_version" "admin_password_version" { + secret_id = aws_secretsmanager_secret.grafana.id + depends_on = [aws_secretsmanager_secret_version.grafana] +} + +resource "random_password" "grafana" { + length = 16 + special = true + override_special = "@_" +} + +#tfsec:ignore:aws-ssm-secret-use-customer-key +resource "aws_secretsmanager_secret" "grafana" { + name = "${local.name}-grafana" + recovery_window_in_days = 0 # Set to zero for this example to force delete during Terraform destroy +} + +resource "aws_secretsmanager_secret_version" "grafana" { + secret_id = aws_secretsmanager_secret.grafana.id + secret_string = random_password.grafana.result +} + +#--------------------------------------------------------------- +# EKS Cluster +#--------------------------------------------------------------- +data "aws_eks_cluster_auth" "this" { + name = module.eks.cluster_name +} + +data "aws_availability_zones" "available" {} + +#tfsec:ignore:aws-eks-enable-control-plane-logging +module "eks" { + source = "terraform-aws-modules/eks/aws" + version = "~> 19.9" + + cluster_name = local.name + cluster_version = var.eks_cluster_version + + vpc_id = module.vpc.vpc_id + subnet_ids = module.vpc.private_subnets + + cluster_endpoint_private_access = true # if true, Kubernetes API requests within your cluster's VPC (such as node to control plane communication) use the private VPC endpoint + cluster_endpoint_public_access = true # if true, Your cluster API server is accessible from the internet. You can, optionally, limit the CIDR blocks that can access the public endpoint. + + eks_managed_node_groups = { + + core_node_group = { + name = "core-node-group" + description = "EKS Core node group for hosting system add-ons" + subnet_ids = module.vpc.private_subnets + + min_size = 3 + max_size = 8 + desired_size = 3 + + instance_types = ["m5.xlarge"] + + ebs_optimized = true + block_device_mappings = { + xvda = { + device_name = "/dev/xvda" + ebs = { + volume_size = 100 + volume_type = "gp3" + } + } + } + + labels = { + WorkerType = "ON_DEMAND" + NodeGroupType = "core" + } + } + + # cluster configuration based on + # https://startree.ai/blog/capacity-planning-in-apache-pinot-part-1 + # https://startree.ai/blog/capacity-planning-in-apache-pinot-part-2 + + controller_node_group = { + instance_types = ["m5.xlarge"] + capacity_type = "ON_DEMAND" + max_size = 5 + min_size = 3 + desired_size = 3 + + labels = { + WorkerType = "ON_DEMAND" + NodeGroupType = "controller" + } + taints = [ + { + key = "dedicated" + value = "pinot" + effect = "NO_SCHEDULE" + } + ] + } + + zookeeper_node_group = { + instance_types = ["m5.xlarge"] + capacity_type = "ON_DEMAND" + max_size = 5 + min_size = 3 + desired_size = 3 + + labels = { + WorkerType = "ON_DEMAND" + NodeGroupType = "zookeeper" + } + taints = [ + { + key = "dedicated" + value = "pinot" + effect = "NO_SCHEDULE" + } + ] + } + + broker_node_group = { + instance_types = ["m5.xlarge"] + capacity_type = "ON_DEMAND" + max_size = 5 + min_size = 3 + desired_size = 3 + + labels = { + WorkerType = "ON_DEMAND" + NodeGroupType = "broker" + } + taints = [ + { + key = "dedicated" + value = "pinot" + effect = "NO_SCHEDULE" + } + ] + } + + server_node_group = { + instance_types = ["r5.xlarge"] + capacity_type = "ON_DEMAND" + max_size = 9 + min_size = 6 + desired_size = 6 + + ebs_optimized = true + block_device_mappings = { + xvda = { + device_name = "/dev/xvda" + ebs = { + volume_size = 100 + volume_type = "gp3" + } + } + } + + labels = { + WorkerType = "ON_DEMAND" + NodeGroupType = "server" + } + taints = [ + { + key = "dedicated" + value = "pinot" + effect = "NO_SCHEDULE" + } + ] + } + + minion_node_group = { + instance_types = ["m5.xlarge"] + capacity_type = "ON_DEMAND" + max_size = 3 + min_size = 1 + desired_size = 1 + + labels = { + WorkerType = "ON_DEMAND" + NodeGroupType = "minion" + } + taints = [ + { + key = "dedicated" + value = "pinot" + effect = "NO_SCHEDULE" + } + ] + } + } + tags = local.tags +} diff --git a/distributed-databases/pinot/outputs.tf b/distributed-databases/pinot/outputs.tf new file mode 100644 index 000000000..f6444daab --- /dev/null +++ b/distributed-databases/pinot/outputs.tf @@ -0,0 +1,4 @@ +output "configure_kubectl" { + description = "Configure kubectl: make sure you're logged in with the correct AWS profile and run the following command to update your kubeconfig" + value = "aws eks --region ${var.region} update-kubeconfig --name ${var.name}" +} diff --git a/distributed-databases/pinot/providers.tf b/distributed-databases/pinot/providers.tf new file mode 100644 index 000000000..f5cf1302b --- /dev/null +++ b/distributed-databases/pinot/providers.tf @@ -0,0 +1,25 @@ +provider "aws" { + region = local.region +} + +provider "kubernetes" { + host = module.eks.cluster_endpoint + cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) + token = data.aws_eks_cluster_auth.this.token +} + +provider "helm" { + kubernetes { + host = module.eks.cluster_endpoint + cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) + token = data.aws_eks_cluster_auth.this.token + } +} + +provider "kubectl" { + apply_retry_count = 10 + host = module.eks.cluster_endpoint + cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) + token = data.aws_eks_cluster_auth.this.token + load_config_file = false +} diff --git a/distributed-databases/pinot/variables.tf b/distributed-databases/pinot/variables.tf new file mode 100644 index 000000000..99d71b35a --- /dev/null +++ b/distributed-databases/pinot/variables.tf @@ -0,0 +1,35 @@ +variable "name" { + description = "Name of the VPC and EKS Cluster" + type = string + default = "pinot-on-eks" +} + +variable "region" { + description = "region" + type = string + default = "us-west-2" +} + +variable "eks_cluster_version" { + description = "EKS Cluster version" + type = string + default = "1.28" +} + +variable "vpc_cidr" { + description = "VPC CIDR" + default = "10.1.0.0/16" + type = string +} + +variable "public_subnets" { + description = "Public Subnets CIDRs. 4094 IPs per Subnet" + default = ["10.1.192.0/20", "10.1.208.0/20", "10.1.224.0/20"] + type = list(string) +} + +variable "private_subnets" { + description = "Private Subnets CIDRs. 16382 IPs per Subnet" + default = ["10.1.0.0/18", "10.1.64.0/18", "10.1.128.0/18"] + type = list(string) +} diff --git a/distributed-databases/pinot/versions.tf b/distributed-databases/pinot/versions.tf new file mode 100644 index 000000000..f1fc006f4 --- /dev/null +++ b/distributed-databases/pinot/versions.tf @@ -0,0 +1,26 @@ +terraform { + required_version = ">= 1.0.0" + + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 3.72" + } + kubernetes = { + source = "hashicorp/kubernetes" + version = ">= 2.10" + } + helm = { + source = "hashicorp/helm" + version = ">= 2.4.1" + } + kubectl = { + source = "gavinbunney/kubectl" + version = ">= 1.14" + } + random = { + source = "hashicorp/random" + version = ">= 3.1" + } + } +} diff --git a/distributed-databases/pinot/vpc.tf b/distributed-databases/pinot/vpc.tf new file mode 100644 index 000000000..26978da3a --- /dev/null +++ b/distributed-databases/pinot/vpc.tf @@ -0,0 +1,30 @@ +#--------------------------------------------------------------- +# VPC +#--------------------------------------------------------------- +# WARNING: This VPC module includes the creation of an Internet Gateway and NAT Gateway, which simplifies cluster deployment and testing, primarily intended for sandbox accounts. +# IMPORTANT: For preprod and prod use cases, it is crucial to consult with your security team and AWS architects to design a private infrastructure solution that aligns with your security requirements +module "vpc" { + source = "terraform-aws-modules/vpc/aws" + version = "~> 5.0" + + name = local.name + cidr = var.vpc_cidr + + azs = local.azs + public_subnets = var.public_subnets # Two Subnets. 4094 IPs per Subnet + private_subnets = var.private_subnets # Three Subnets. 16382 IPs per Subnet + + enable_nat_gateway = true + single_nat_gateway = true + enable_dns_hostnames = true + + public_subnet_tags = { + "kubernetes.io/role/elb" = 1 + } + + private_subnet_tags = { + "kubernetes.io/role/internal-elb" = 1 + } + + tags = local.tags +} diff --git a/website/docs/blueprints/distributed-databases/img/grafana.png b/website/docs/blueprints/distributed-databases/img/grafana.png new file mode 100644 index 000000000..b05f6f927 Binary files /dev/null and b/website/docs/blueprints/distributed-databases/img/grafana.png differ diff --git a/website/docs/blueprints/distributed-databases/img/pinot-console.png b/website/docs/blueprints/distributed-databases/img/pinot-console.png new file mode 100644 index 000000000..cfe1c716d Binary files /dev/null and b/website/docs/blueprints/distributed-databases/img/pinot-console.png differ diff --git a/website/docs/blueprints/distributed-databases/img/pinot-example.png b/website/docs/blueprints/distributed-databases/img/pinot-example.png new file mode 100644 index 000000000..0b0db79e4 Binary files /dev/null and b/website/docs/blueprints/distributed-databases/img/pinot-example.png differ diff --git a/website/docs/blueprints/distributed-databases/img/prometheus.png b/website/docs/blueprints/distributed-databases/img/prometheus.png new file mode 100644 index 000000000..9ac5b7361 Binary files /dev/null and b/website/docs/blueprints/distributed-databases/img/prometheus.png differ diff --git a/website/docs/blueprints/distributed-databases/pinot.md b/website/docs/blueprints/distributed-databases/pinot.md new file mode 100644 index 000000000..5ff59fb43 --- /dev/null +++ b/website/docs/blueprints/distributed-databases/pinot.md @@ -0,0 +1,272 @@ +--- +sidebar_position: 2 +sidebar_label: Apache Pinot +--- +# Deploying Apache Pinot (๐Ÿท) on EKS (Experimental) + +[Apache Pinot](https://pinot.apache.org/) is real-time distributed OLAP datastore, purpose built for low-latency and high-throughput analytics. You can use pinot to ingest and immediately query data from streaming or batch data sources e.g. Apache Kafka, Amazon Kinesis Data Streams, Amazon S3, etc). + +Apache Pinot includes the following characteristics: + +- **Ultra low-latency** analytics even at extremely high throughput. +- **Columnar data store** with several smart indexing and pre-aggregation techniques. +- **Scaling up** and **out** with no upper bound. +- **Consistent performance** based on the size of your cluster and an expected query per second (QPS) threshold. + +It's a perfect solution for user-facing real-time analytics and other analytical use cases, including internal dashboards, anomaly detection, and ad hoc data exploration. You can learn more about Apache Pinot and its components in its [documentation](https://docs.pinot.apache.org/). + +In this blueprint, we will deploy Apache Pinot on Kubernetes cluster managed by Elastic Kubernetes Service (EKS). Some of the benefits of deploying Apache Pinot on EKS cluster are + +- Manage Apache Pinot Cluster using Kubernetes +- Scale each layer independently +- No single point of failure +- Auto recovery + +> Note: All Apache Pinot components run on `StatefulSet` including **Zookeeper** + +> Note: This blueprint doesn't leverage [DeepStore](#link) currently and uses EBS volumes to store table segments. + +> Future: Currently all Apache Pinot components run on m5.xlarge nodes that are provisioned as part of node groups. In future, we would leverage separate node groups for each components so that we can provide each Apache Pinot component specialized underlying hardware that improves overall performance of Apache Pinot. + +## Prerequisites ๐Ÿ“ + +Ensure that you have following tools installed on your machine. + +1. [aws cli](https://docs.aws.amazon.com/cli/latest/userguide/install-cliv2.html) +2. [kubectl](https://Kubernetes.io/docs/tasks/tools/) +3. [terraform](https://learn.hashicorp.com/tutorials/terraform/install-cli) + +## Deployment โš™๏ธ + +### Deploy the EKS Cluster with Apache Pinot + +First, clone the repository. + +```bash +git clone https://github.com/awslabs/data-on-eks.git +``` + + +Navigate to apache pinot folder and create `terraform.tfvars` to provide desired values for all the variables. This is also the time to update any other input variables or make any other changes to the terraform template. + +```bash +cd data-on-eks/distributed-databases/pinot +touch terraform.tfvars +``` + +#### Sample `terraform.tfvars` +```terraform +name = "pinot-on-eks" +region = "us-west-2" +eks_cluster_version = "1.25" +... +``` + +### Verify Deployment + +Verify the Amazon EKS Cluster + +```bash +aws eks describe-cluster --name pinot-on-eks +``` + +Update local kubeconfig so we can access kubernetes cluster. + +```bash +aws eks update-kubeconfig --name pinot-on-eks --region us-west-2 +``` + +First, lets verify that we have worker nodes running in the cluster. + +```bash +kubectl get nodes +``` +#### Output +```bash +NAME STATUS ROLES AGE VERSION +ip-10-1-189-200.us-west-2.compute.internal Ready 12d v1.24.17-eks-43840fb +ip-10-1-46-117.us-west-2.compute.internal Ready 12d v1.24.17-eks-43840fb +ip-10-1-84-80.us-west-2.compute.internal Ready 12d v1.24.17-eks-43840fb +``` + +Next, lets verify all the pods are running. + +```bash +kubectl get pods -n pinot +``` +#### Output +```bash +NAME READY STATUS RESTARTS AGE +pinot-broker-0 1/1 Running 0 11d +pinot-broker-1 1/1 Running 0 11d +pinot-broker-2 1/1 Running 0 11d +pinot-controller-0 1/1 Running 0 11d +pinot-controller-1 1/1 Running 0 11d +pinot-controller-2 1/1 Running 0 11d +pinot-minion-stateless-86cf65f89-rlpwn 1/1 Running 0 12d +pinot-minion-stateless-86cf65f89-tkbjf 1/1 Running 0 12d +pinot-minion-stateless-86cf65f89-twp8n 1/1 Running 0 12d +pinot-server-0 1/1 Running 0 11d +pinot-server-1 1/1 Running 0 11d +pinot-server-2 1/1 Running 0 11d +pinot-zookeeper-0 1/1 Running 0 12d +pinot-zookeeper-1 1/1 Running 0 12d +pinot-zookeeper-2 1/1 Running 0 12d +``` + +We have also deployed `prometheus` and `grafana` under `monitoring` namespace. So also make sure all the pods for `monitoring` are also running. + +```bash +kubectl get pods -n monitoring +``` +#### Output +```bash +prometheus-grafana-85b4584dbf-4l72l 3/3 Running 0 12d +prometheus-kube-prometheus-operator-84dcddccfc-pv8nv 1/1 Running 0 12d +prometheus-kube-state-metrics-57f6b6b4fd-txjtb 1/1 Running 0 12d +prometheus-prometheus-kube-prometheus-prometheus-0 2/2 Running 0 4d3h +prometheus-prometheus-node-exporter-4jh8q 1/1 Running 0 12d +prometheus-prometheus-node-exporter-f5znb 1/1 Running 0 12d +prometheus-prometheus-node-exporter-f9xrz 1/1 Running 0 12d +``` + + +Now lets access Apache Pinot Console using the below command. Console consist of **Cluster Manager**, **Query Explorer**, **Zookeeper Browser** and **Swagger REST API Explorer**. + +```bash +kubectl port-forward service/pinot-controller 9000:9000 -n pinot +``` + +This will allow you to access Apache Pinot Console like the one shown below using `http://localhost:9000` + +![Apache Pinot Web Console](./img/pinot-console.png) + + +Apache Pinot supports exporting metrics using Prometheus JMX exporter that is packaged within the Apache Pinot docker image. Lets ensure metrics from all Apache Pinot components are getting published to `prometheus`. + +```bash +kubectl port-forward service/prometheus-kube-prometheus-prometheus 9090:9090 -n monitoring +``` + +Navigate to the prometheus UI at `http://localhost:9090`, type `pinot` in the search box and you should be able to see all the metrics. + +![Prometheus](./img/prometheus.png) + +Next, Let's use Grafana to visualize the Apache Pinot metrics. In order to access Grafana, we need to get the grafana password from AWS Secrets Manager. + +```bash +aws secretsmanager get-secret-value --secret-id pinot-on-eks-grafana | jq '.SecretString' --raw-output +``` + +Now use the port-forwarding to access Grafana at port `8080` + +```bash +kubectl port-forward service/prometheus-grafana 8080:80 -n monitoring +``` + +Login to grafana dashboard using `admin` and password retrieved in the previous step and then navigate to Dashboard and click New and then Import. Use the file `pinot.json` under `data-on-eks/distributed-database/pinot/dashboard` to create a pinot dashboard. + +![Grafana Dashboard for Pinot](./img/grafana.png) + +To learn more about the monitoring of Apache Pinot using Prometheus and Grafana use the [official guide](https://docs.pinot.apache.org/operators/tutorials/monitor-pinot-using-prometheus-and-grafana). + +## Additional Deployment (Optional) ๐Ÿ† + +### Deploy Apache Kafka for Streaming Data + +Apache Pinot can ingest data from streaming data sources (real-time) as well as batch data sources (offline). In this example, we will leverage [Apache Kafka](https://kafka.apache.org/) to push real-time data to a topic. + +If you already have Apache Kafka running in your EKS cluster or you are leveraging Amazon Managed Streaming for Apache Kafka (MSK) you can skip this step. Otherwise, follow the steps below to install Kafka in your EKS cluster. + +> Note: Following deployment configure Kafka Brokers with PLAINTEXT listeners for simplified deployment. Modify the `kafka-values.yaml` file for production deployment + +```bash +helm repo add bitnami https://charts.bitnami.com/bitnami +helm install -n pinot pinot-kafka bitnami/kafka --values ./helm/kafka-values.yaml +``` + +#### Output +```bash +NAME: pinot-kafka +LAST DEPLOYED: Tue Oct 24 01:10:25 2023 +NAMESPACE: pinot +STATUS: deployed +REVISION: 1 +TEST SUITE: None +NOTES: +CHART NAME: kafka +CHART VERSION: 26.2.0 +APP VERSION: 3.6.0 + +** Please be patient while the chart is being deployed ** + +Kafka can be accessed by consumers via port 9092 on the following DNS name from within your cluster: + + pinot-kafka.pinot.svc.cluster.local + +Each Kafka broker can be accessed by producers via port 9092 on the following DNS name(s) from within your cluster: + + pinot-kafka-controller-0.pinot-kafka-controller-headless.pinot.svc.cluster.local:9092 + pinot-kafka-controller-1.pinot-kafka-controller-headless.pinot.svc.cluster.local:9092 + pinot-kafka-controller-2.pinot-kafka-controller-headless.pinot.svc.cluster.local:9092 + +To create a pod that you can use as a Kafka client run the following commands: + + kubectl run pinot-kafka-client --restart='Never' --image docker.io/bitnami/kafka:3.6.0-debian-11-r0 --namespace pinot --command -- sleep infinity + kubectl exec --tty -i pinot-kafka-client --namespace pinot -- bash + + PRODUCER: + kafka-console-producer.sh \ + --broker-list pinot-kafka-controller-0.pinot-kafka-controller-headless.pinot.svc.cluster.local:9092,pinot-kafka-controller-1.pinot-kafka-controller-headless.pinot.svc.cluster.local:9092,pinot-kafka-controller-2.pinot-kafka-controller-headless.pinot.svc.cluster.local:9092 \ + --topic test + + CONSUMER: + kafka-console-consumer.sh \ + --bootstrap-server pinot-kafka.pinot.svc.cluster.local:9092 \ + --topic test \ + --from-beginning +``` + +Use the command mentioned above to create **Kafka Client** pod within your namespace. + +```bash +kubectl run pinot-kafka-client --restart='Never' --image docker.io/bitnami/kafka:3.6.0-debian-11-r0 --namespace pinot --command -- sleep infinity +kubectl exec --tty -i pinot-kafka-client --namespace pinot -- bash +``` + +Create Kafka topics using the below commands, which will then be used to publish messages. + +```bash +kafka-topics.sh --bootstrap-server pinot-kafka-controller-0.pinot-kafka-controller-headless.pinot.svc.cluster.local:9092 --topic flights-realtime --create --partitions 1 --replication-factor 1 + +kafka-topics.sh --bootstrap-server pinot-kafka-controller-0.pinot-kafka-controller-headless.pinot.svc.cluster.local:9092 --topic flights-realtime-avro --create --partitions 1 --replication-factor 1 +``` + +Use provided `example/pinot-realtime-quickstart.yml` to create tables and publish sample data to the above topics, which will then get ingested into tables. + +```bash +kubectl apply -f example/pinot-realtime-quickstart.yml +``` + +Now, let navigate back to **Query Console** and then click one of the tables. You should be able to see the newly created tables and data coming into tables. +```bash +kubectl port-forward service/pinot-controller 9000:9000 -n pinot +``` + +![Pinot Example](./img/pinot-example.png) + + +## Cleanup ๐Ÿงน + +To delete all the components provisioned as part of this blueprint, using the following command to destroy all the resources. + +```bash +./cleanup.sh +``` + +:::caution + +To avoid unwanted charges to your AWS account, delete all the AWS resources created during this deployment + +ex. Delete kafka-on-eks EBS volumes +:::