-
-
Notifications
You must be signed in to change notification settings - Fork 47
/
nomad.sh
296 lines (260 loc) · 10.7 KB
/
nomad.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
#!/bin/bash
VERSION=latest
arch=$(lscpu | grep "Architecture" | awk '{print $NF}')
if [[ $arch == x86_64* ]]; then
ARCH="amd64"
elif [[ $arch == aarch64 ]]; then
ARCH="arm64"
fi
echo -e '\e[38;5;198m'"CPU is $ARCH"
# https://github.com/hashicorp/nomad/issues/19343 nomad needs dmidecode
sudo DEBIAN_FRONTEND=noninteractive apt-get --assume-yes install -qq dmidecode curl unzip jq < /dev/null > /dev/null
echo -e '\e[38;5;198m'"++++ "
echo -e '\e[38;5;198m'"++++ Cleanup any Nomad if found"
echo -e '\e[38;5;198m'"++++ "
sudo systemctl stop nomad
sudo rm -rf /etc/nomad
sudo rm -rf /var/lib/nomad
sudo rm -rf /opt/nomad
sudo rm -rf /tmp/nomad.zip
sudo rm -rf /opt/cni
sudo rm -rf /tmp/cni-plugins.tgz
yes | sudo docker system prune -a
yes | sudo docker system prune --volumes
echo -e '\e[38;5;198m'"++++ "
echo -e '\e[38;5;198m'"++++ Ensure Consul is running (Dependency)"
echo -e '\e[38;5;198m'"++++ "
if pgrep -x "consul" >/dev/null
then
echo -e '\e[38;5;198m'"++++ Consul is running"
else
echo -e '\e[38;5;198m'"++++ Ensure Consul is running.."
sudo bash /vagrant/consul/consul.sh
fi
echo -e '\e[38;5;198m'"++++ "
echo -e '\e[38;5;198m'"++++ Ensure Docker Daemon is running (Dependency)"
echo -e '\e[38;5;198m'"++++ "
if pgrep -x "dockerd" >/dev/null
then
echo -e '\e[38;5;198m'"++++ Docker is running"
else
echo -e '\e[38;5;198m'"++++ Ensure Docker is running.."
sudo bash /vagrant/docker/docker.sh
fi
if [ -f /vagrant/nomad/license.hclic ]; then
# https://developer.hashicorp.com/nomad/tutorials/enterprise/hashicorp-enterprise-license
echo -e '\e[38;5;198m'"++++ "
echo -e '\e[38;5;198m'"++++ Found license.hclic Installing Enterprise Edition version: $VERSION"
echo -e '\e[38;5;198m'"++++ "
export NOMAD_LICENSE_PATH=/vagrant/nomad/license.hclic
export NOMAD_LICENSE=$(cat /vagrant/nomad/license.hclic)
if [[ $VERSION == "latest" ]]; then
LATEST_URL=$(curl -sL https://releases.hashicorp.com/nomad/index.json | jq -r '.versions[].builds[].url' | sort -t. -k 1,1n -k 2,2n -k 3,3n -k 4,4n | egrep 'ent' | egrep "linux.*$ARCH" | sort -V | tail -n 1)
else
LATEST_URL=$(curl -sL https://releases.hashicorp.com/nomad/index.json | jq -r '.versions[].builds[].url' | sort -t. -k 1,1n -k 2,2n -k 3,3n -k 4,4n | egrep 'ent' | egrep "linux.*$ARCH" | sort -V | grep $VERSION | tail -1)
fi
else
echo -e '\e[38;5;198m'"++++ "
echo -e '\e[38;5;198m'"++++ Installing Community Edition version: $VERSION"
echo -e '\e[38;5;198m'"++++ "
if [[ $VERSION == "latest" ]]; then
LATEST_URL=$(curl -sL https://releases.hashicorp.com/nomad/index.json | jq -r '.versions[].builds[].url' | sort -t. -k 1,1n -k 2,2n -k 3,3n -k 4,4n | egrep -v 'rc|ent|beta' | egrep "linux.*$ARCH" | sort -V | tail -n 1)
else
LATEST_URL=$(curl -sL https://releases.hashicorp.com/nomad/index.json | jq -r '.versions[].builds[].url' | sort -t. -k 1,1n -k 2,2n -k 3,3n -k 4,4n | egrep -v 'rc|ent|beta' | egrep "linux.*$ARCH" | sort -V | grep $VERSION | tail -1)
fi
fi
wget -q $LATEST_URL -O /tmp/nomad.zip
mkdir -p /usr/local/bin
(cd /usr/local/bin && unzip -o /tmp/nomad.zip)
echo -e '\e[38;5;198m'"++++ "
echo -e '\e[38;5;198m'"++++ Installed `/usr/local/bin/nomad --version`"
echo -e '\e[38;5;198m'"++++ "
# create /var/log/nomad.log
sudo touch /var/log/nomad.log
# create Nomad data directories
sudo mkdir -p /etc/nomad
echo -e '\e[38;5;198m'"++++ "
echo -e '\e[38;5;198m'"++++ Create Nomad Systemd service file"
echo -e '\e[38;5;198m'"++++ "
# create a Nomad service file at /etc/systemd/system/nomad.service
cat <<EOF | sudo tee /etc/systemd/system/nomad.service
[Unit]
Description=Nomad
Documentation=https://nomadproject.io/docs/
Wants=network-online.target
After=network-online.target
# When using Nomad with Consul it is not necessary to start Consul first. These
# lines start Consul before Nomad as an optimization to avoid Nomad logging
# that Consul is unavailable at startup.
#Wants=consul.service
#After=consul.service
[Service]
# EnvironmentFile=/etc/nomad.d/nomad.env
ExecReload=/bin/kill -HUP $MAINPID
ExecStart=/usr/local/bin/nomad agent -config=/etc/nomad/server.conf -dev-connect
KillMode=process
KillSignal=SIGINT
LimitNOFILE=65536
LimitNPROC=infinity
Restart=on-failure
RestartSec=2
LogsDirectory=nomad
StandardOutput=append:/var/log/nomad.log
StandardError=append:/var/log/nomad.log
StartLimitBurst=3
## Configure unit start rate limiting. Units which are started more than
## *burst* times within an *interval* time span are not permitted to start any
## more. Use StartLimitIntervalSec or StartLimitInterval (depending on
## systemd version) to configure the checking interval and StartLimitBurst
## to configure how many starts per interval are allowed. The values in the
## commented lines are defaults.
TasksMax=infinity
OOMScoreAdjust=-1000
[Install]
WantedBy=multi-user.target
EOF
echo -e '\e[38;5;198m'"++++ "
echo -e '\e[38;5;198m'"++++ Create Nomad config file /etc/nomad/server.conf"
echo -e '\e[38;5;198m'"++++ "
cat <<EOF | sudo tee /etc/nomad/server.conf
data_dir = "/var/lib/nomad"
bind_addr = "0.0.0.0" # the default
datacenter = "dc1"
advertise {
# Defaults to the first private IP address.
http = "{{ GetInterfaceIP \"eth0\" }}"
rpc = "{{ GetInterfaceIP \"eth0\" }}"
serf = "{{ GetInterfaceIP \"eth0\" }}:5648" # non-default ports may be specified
}
server {
enabled = true
bootstrap_expect = 1
}
autopilot {
cleanup_dead_servers = true
last_contact_threshold = "400ms"
max_trailing_logs = 250
server_stabilization_time = "30s"
enable_redundancy_zones = false
disable_upgrade_migration = false
enable_custom_upgrades = false
}
client {
enabled = true
# https://github.com/hashicorp/nomad/issues/1282
network_speed = 100
# https://developer.hashicorp.com/nomad/docs/configuration/client#cpu_total_compute
# BUG: CPU fingerprint with Docker Desktop on Apple Silicon never really worked because the CPU speed is not made available anywhere, so its impossible for Nomad to detect it
# If you run previous versions of Nomad you will notice that the fingerprinted capacity is always 1000MHz. This is a value we used to hardcode as a fallback but we dont anymore on 1.7.x (https://github.com/hashicorp/nomad/blob/release/1.6.x/client/fingerprint/cpu.go#L23) because its just wrong.
# The only option for now is to pass their own hardcoded value using client.cpu_total_compute (https://developer.hashicorp.com/nomad/docs/configuration/client#cpu_total_compute)
cpu_total_compute = 8000
servers = ["{{ GetInterfaceIP \"eth0\" }}:4647"]
# network_interface = "enp0s8"
# https://www.nomadproject.io/docs/drivers/docker.html#volumes
# https://github.com/hashicorp/nomad/issues/5562
options = {
"docker.volumes.enabled" = true
"docker.auth.config" = "/etc/docker/auth.json"
}
host_volume "waypoint" {
path = "/opt/nomad/data/volume/waypoint"
read_only = false
}
}
plugin "docker" {
config {
endpoint = "unix:///var/run/docker.sock"
volumes {
enabled = true
selinuxlabel = "z"
}
allow_privileged = true
allow_caps = ["chown", "net_raw"]
}
}
plugin "raw_exec" {
config {
enabled = true
}
}
# https://developer.hashicorp.com/nomad/docs/configuration/telemetry
# https://developer.hashicorp.com/nomad/docs/configuration/telemetry#prometheus
# https://developer.hashicorp.com/nomad/docs/operations/monitoring-nomad
telemetry {
collection_interval = "1s"
disable_hostname = true
prometheus_metrics = true
publish_allocation_metrics = true
publish_node_metrics = true
}
consul {
address = "{{ GetInterfaceIP \"eth0\" }}:8500"
}
EOF
if [ -f /vagrant/nomad/license.hclic ]; then
sed -i -e 's;bootstrap_expect = 1;bootstrap_expect = 1\n license_path = "/vagrant/nomad/license.hclic";' /etc/nomad/server.conf
fi
echo -e '\e[38;5;198m'"++++ "
echo -e '\e[38;5;198m'"++++ Creating Waypoint host volume /opt/nomad/data/volume/waypoint"
echo -e '\e[38;5;198m'"++++ "
sudo mkdir -p /opt/nomad/data/volume/waypoint
sudo chmod -R 777 /opt/nomad
echo -e '\e[38;5;198m'"++++ "
echo -e '\e[38;5;198m'"++++ Check if CNI Plugins are installed"
echo -e '\e[38;5;198m'"++++ "
if [ -f /opt/cni/bin/bridge ]; then
echo -e '\e[38;5;198m'"++++ CNI Plugins already installed"
else
echo -e '\e[38;5;198m'"++++ CNI Plugins not found, installing.."
wget -q https://github.com/containernetworking/plugins/releases/download/v1.4.1/cni-plugins-linux-$ARCH-v1.4.1.tgz -O /tmp/cni-plugins.tgz
mkdir -p /opt/cni/bin
tar -C /opt/cni/bin -xzf /tmp/cni-plugins.tgz
echo 1 > /proc/sys/net/bridge/bridge-nf-call-arptables
echo 1 > /proc/sys/net/bridge/bridge-nf-call-ip6tables
echo 1 > /proc/sys/net/bridge/bridge-nf-call-iptables
fi
# start and enable nomad service to start on system boot
echo -e '\e[38;5;198m'"++++ "
echo -e '\e[38;5;198m'"++++ Start Nomad Service"
echo -e '\e[38;5;198m'"++++ "
sudo systemctl daemon-reload
sudo service nomad start
sh -c 'sudo tail -f /var/log/nomad.log | { sed "/node registration complete/ q" && kill $$ ;}'
sleep 2
echo -e '\e[38;5;198m'"++++ "
echo -e '\e[38;5;198m'"++++ Get Nomad Members and Status"
echo -e '\e[38;5;198m'"++++ "
sleep 10
nomad server members
nomad node status
cd /vagrant/nomad/nomad/jobs;
echo -e '\e[38;5;198m'"++++ "
echo -e '\e[38;5;198m'"++++ Start Nomad Fabio job"
echo -e '\e[38;5;198m'"++++ "
nomad plan --address=http://localhost:4646 fabio.nomad
nomad run --address=http://localhost:4646 fabio.nomad
# # curl -v -H 'Host: fabio.service.consul' http://${VAGRANT_IP}:9999/
echo -e '\e[38;5;198m'"++++ "
echo -e '\e[38;5;198m'"++++ Start Nomad Traefik job"
echo -e '\e[38;5;198m'"++++ "
nomad plan --address=http://localhost:4646 traefik.nomad
nomad run --address=http://localhost:4646 traefik.nomad
# nomad plan --address=http://localhost:4646 traefik-whoami.nomad
# nomad run --address=http://localhost:4646 traefik-whoami.nomad
if [ -f /vagrant/nomad/license.hclic ]; then
echo -e '\e[38;5;198m'"++++ "
echo -e '\e[38;5;198m'"++++ Nomad License Inspect"
echo -e '\e[38;5;198m'"++++ "
nomad license inspect /vagrant/nomad/license.hclic
fi
echo -e '\e[38;5;198m'"++++ "
echo -e '\e[38;5;198m'"++++ Access Nomad"
echo -e '\e[38;5;198m'"++++ "
echo -e '\e[38;5;198m'"++++ Nomad http://localhost:4646"
echo -e '\e[38;5;198m'"++++ Nomad Documentation http://localhost:3333/#/nomad/README?id=nomad"
echo -e '\e[38;5;198m'"++++ Fabio Dashboard http://localhost:9998"
echo -e '\e[38;5;198m'"++++ Fabio Loadbalancer http://localhost:9998"
echo -e '\e[38;5;198m'"++++ Fabio Documentation http://localhost:3333/#/nomad/README?id=fabio-load-balancer-for-nomad"
echo -e '\e[38;5;198m'"++++ Treafik Dashboard http://localhost:38081"
echo -e '\e[38;5;198m'"++++ Traefik Loadbalancer: http://localhost:38080"
echo -e '\e[38;5;198m'"++++ Traefik Documentation: http://localhost:3333/#/nomad/README?id=traefik-load-balancer-for-nomad"