-
-
Notifications
You must be signed in to change notification settings - Fork 47
/
prometheus-grafana.sh
171 lines (151 loc) · 8.33 KB
/
prometheus-grafana.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
#!/bin/bash
# https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack
# https://prometheus.io/docs/visualization/grafana/#using
# https://blog.marcnuri.com/prometheus-grafana-setup-minikube
cd ~/
# Determine CPU Architecture
arch=$(lscpu | grep "Architecture" | awk '{print $NF}')
if [[ $arch == x86_64* ]]; then
ARCH="amd64"
elif [[ $arch == aarch64 ]]; then
ARCH="arm64"
fi
echo -e '\e[38;5;198m'"CPU is $ARCH"
echo -e '\e[38;5;198m'"++++ "
echo -e '\e[38;5;198m'"++++ Cleanup"
echo -e '\e[38;5;198m'"++++ "
sudo docker stop grafana prometheus
sudo docker rm grafana prometheus
yes | sudo docker system prune -a
yes | sudo docker system prune --volumes
for i in $(ps aux | grep kubectl | grep -ve sudo -ve grep -ve bin | grep -e grafana -e prometheus -e alertmanager | tr -s " " | cut -d " " -f2); do kill -9 $i; done
sudo --preserve-env=PATH -u vagrant helm list
sudo --preserve-env=PATH -u vagrant helm uninstall prometheus
sudo --preserve-env=PATH -u vagrant helm uninstall grafana
sudo --preserve-env=PATH -u vagrant helm list
echo -e '\e[38;5;198m'"++++ "
echo -e '\e[38;5;198m'"++++ helm version"
echo -e '\e[38;5;198m'"++++ "
sudo --preserve-env=PATH -u vagrant helm version
# https://helm.sh/docs/intro/quickstart/#initialize-a-helm-chart-repository
echo -e '\e[38;5;198m'"++++ "
echo -e '\e[38;5;198m'"++++ Helm add Prometheus repo"
echo -e '\e[38;5;198m'"++++ helm repo add prometheus-community https://prometheus-community.github.io/helm-charts"
echo -e '\e[38;5;198m'"++++ "
sudo --preserve-env=PATH -u vagrant helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
echo -e '\e[38;5;198m'"++++ "
echo -e '\e[38;5;198m'"++++ helm repo update"
echo -e '\e[38;5;198m'"++++ "
sudo --preserve-env=PATH -u vagrant helm repo update
echo -e '\e[38;5;198m'"++++ "
echo -e '\e[38;5;198m'"++++ helm search repo prometheus-community"
echo -e '\e[38;5;198m'"++++ "
sudo --preserve-env=PATH -u vagrant helm search repo prometheus-community
# https://developer.hashicorp.com/vault/docs/configuration/telemetry#prometheus
echo -e '\e[38;5;198m'"++++ "
echo -e '\e[38;5;198m'"++++ Set Vault token in prometheus-values.yaml for prometheus for monitoring Vault"
echo -e '\e[38;5;198m'"++++ "
export VAULT_TOKEN=$(grep 'Initial Root Token' /etc/vault/init.file | cut -d ':' -f2 | tr -d ' ')
sed -i "s/bearer_token: .*/bearer_token: \"$VAULT_TOKEN\"/g" /vagrant/prometheus-grafana/prometheus-values.yaml
cat /vagrant/prometheus-grafana/prometheus-values.yaml
echo -e '\e[38;5;198m'"++++ "
echo -e '\e[38;5;198m'"++++ helm install prometheus prometheus-community/prometheus"
echo -e '\e[38;5;198m'"++++ "
sudo --preserve-env=PATH -u vagrant helm install prometheus prometheus-community/prometheus -f /vagrant/prometheus-grafana/prometheus-values.yaml
echo -e '\e[38;5;198m'"++++ "
echo -e '\e[38;5;198m'"++++ Helm add Grafana repo"
echo -e '\e[38;5;198m'"++++ helm repo add grafana https://grafana.github.io/helm-charts"
echo -e '\e[38;5;198m'"++++ "
sudo --preserve-env=PATH -u vagrant helm repo add grafana https://grafana.github.io/helm-charts
echo -e '\e[38;5;198m'"++++ "
echo -e '\e[38;5;198m'"++++ helm repo update"
echo -e '\e[38;5;198m'"++++ "
sudo --preserve-env=PATH -u vagrant helm repo update
echo -e '\e[38;5;198m'"++++ "
echo -e '\e[38;5;198m'"++++ helm search repo grafana"
echo -e '\e[38;5;198m'"++++ "
sudo --preserve-env=PATH -u vagrant helm search repo grafana
echo -e '\e[38;5;198m'"++++ "
echo -e '\e[38;5;198m'"++++ helm install grafana grafana/grafana"
echo -e '\e[38;5;198m'"++++ "
sudo --preserve-env=PATH -u vagrant helm install grafana grafana/grafana -f /vagrant/prometheus-grafana/grafana-values.yaml
echo -e '\e[38;5;198m'"++++ "
echo -e '\e[38;5;198m'"++++ Waiting for Prometheus and Alertmanager and Grafana to become available.."
echo -e '\e[38;5;198m'"++++ See Minikube Dashboard for details: http://localhost:10888"
echo -e '\e[38;5;198m'"++++ "
attempts=0
max_attempts=15
while ! ( sudo --preserve-env=PATH -u vagrant kubectl get po | grep prometheus | tr -s " " | cut -d " " -f3 | grep Running ) && (( $attempts < $max_attempts )); do
attempts=$((attempts+1))
sleep 60;
echo -e '\e[38;5;198m'"++++ Waiting for Prometheus to become available, (${attempts}/${max_attempts}) sleep 60s"
sudo --preserve-env=PATH -u vagrant kubectl get po
sudo --preserve-env=PATH -u vagrant kubectl get events | grep -e Memory -e OOM
done
echo -e '\e[38;5;198m'"++++ "
echo -e '\e[38;5;198m'"++++ Waiting for Prometheus to stabalize, sleep 30s"
echo -e '\e[38;5;198m'"++++ "
sleep 30;
sudo --preserve-env=PATH -u vagrant kubectl get po
sudo --preserve-env=PATH -u vagrant kubectl get events | grep -e Memory -e OOM
# https://gitlab.com/gitlab-org/charts/gitlab/-/issues/2572 Error 422
echo -e '\e[38;5;198m'"The easiest way to access this service is to let kubectl to forward the port:"
echo -e '\e[38;5;198m'"kubectl port-forward service/prometheus-server 9090:9090"
# https://stackoverflow.com/questions/67084554/how-to-kubectl-port-forward-gitlab-webservice
# https://github.com/kubernetes/kubernetes/issues/44371
attempts=0
max_attempts=20
while ! ( sudo netstat -nlp | grep 9090 ) && (( $attempts < $max_attempts )); do
attempts=$((attempts+1))
sleep 60;
echo -e '\e[38;5;198m'"++++ "
echo -e '\e[38;5;198m'"++++ kubectl port-forward -n default prometheus-server 9090 --address=\"0.0.0.0\", (${attempts}/${max_attempts}) sleep 60s"
echo -e '\e[38;5;198m'"++++ "
sudo --preserve-env=PATH -u vagrant kubectl port-forward --namespace default $(sudo --preserve-env=PATH -u vagrant kubectl get po -n default | grep prometheus-server | tr -s " " | cut -d " " -f1) 9090 --address="0.0.0.0" > /dev/null 2>&1 &
done
attempts=0
max_attempts=20
while ! ( sudo netstat -nlp | grep 9093 ) && (( $attempts < $max_attempts )); do
attempts=$((attempts+1))
sleep 60;
echo -e '\e[38;5;198m'"++++ "
echo -e '\e[38;5;198m'"++++ kubectl port-forward -n default prometheus-alertmanager 9093 --address=\"0.0.0.0\", (${attempts}/${max_attempts}) sleep 60s"
echo -e '\e[38;5;198m'"++++ "
sudo --preserve-env=PATH -u vagrant kubectl port-forward --namespace default $(sudo --preserve-env=PATH -u vagrant kubectl get po -n default | grep prometheus-alertmanager | tr -s " " | cut -d " " -f1) 9093 --address="0.0.0.0" > /dev/null 2>&1 &
done
attempts=0
max_attempts=20
while ! ( sudo netstat -nlp | grep 3000 ) && (( $attempts < $max_attempts )); do
attempts=$((attempts+1))
sleep 60;
echo -e '\e[38;5;198m'"++++ "
echo -e '\e[38;5;198m'"++++ kubectl port-forward -n default grafana 3000 --address=\"0.0.0.0\", (${attempts}/${max_attempts}) sleep 60s"
echo -e '\e[38;5;198m'"++++ "
sudo --preserve-env=PATH -u vagrant kubectl port-forward --namespace default $(sudo --preserve-env=PATH -u vagrant kubectl get po -n default | grep grafana | tr -s " " | cut -d " " -f1) 3000 --address="0.0.0.0" > /dev/null 2>&1 &
done
ps aux | grep kubectl | grep -ve sudo -ve grep -ve bin
# https://developer.hashicorp.com/vault/tutorials/monitoring/monitor-telemetry-grafana-prometheus
# https://developer.hashicorp.com/vault/docs/configuration/telemetry#prometheus
echo -e '\e[38;5;198m'"++++ "
echo -e '\e[38;5;198m'"++++ Vault policy write prometheus-metrics path /sys/metrics"
echo -e '\e[38;5;198m'"++++ "
export VAULT_ADDR=http://127.0.0.1:8200
env | grep VAULT_ADDR
vault policy write prometheus-metrics - << EOF
path "/sys/metrics*" {
capabilities = ["read", "list"]
}
EOF
# https://developer.hashicorp.com/vault/docs/configuration/telemetry#prometheus
echo -e '\e[38;5;198m'"++++ "
echo -e '\e[38;5;198m'"++++ Reset Vault token in prometheus-values.yaml"
echo -e '\e[38;5;198m'"++++ "
sed -i "s/bearer_token: .*/bearer_token: \"VAULT_TOKEN\"/g" /vagrant/prometheus-grafana/prometheus-values.yaml
# https://github.com/grafana/grafana/issues/29296
echo -e '\e[38;5;198m'"++++ Prometheus http://localhost:9090"
echo -e '\e[38;5;198m'"++++ Alertmanager http://localhost:9093"
echo -e '\e[38;5;198m'"++++ Grafana http://localhost:3000 and login with Username: admin Password:"
sudo --preserve-env=PATH -u vagrant kubectl get secret --namespace default grafana -o jsonpath="{.data.admin-password}" | base64 --decode ; echo
echo -e '\e[38;5;198m'"++++ You should now be able to access Prometheus http://localhost:9090 \
and Grafana http://localhost:3000 Please login to Grafana and add Prometheus (http://10.9.99.10:9090) as a Datasource, next \
please click on the + symbol in Grafana and import 6417 dashboard."