fix #58 change order of init containers to prevent zerotier races

replaces #94 There are two relevant init containers, get-zerotier-ip and wait-for-bootstrap. get-zerotier-ip used to run first. In addition to getting an ip from zerotier, get-zerotier-ip also downloads all interfaces for the config generator to generate a list of bootstrap ips. When get-zerotier-ip runs first, it may download this list before the bootstrap node has registered its ip as a bootstrap ip, sometimes causing a race condition where the non-bootstrap nodes do not have a bootstrap ip configured. With this PR, wait-for-bootstrap comes first. It waits for bootstrap node to be up (by probing it internally, not through zerotier). Only then will it create an ip and download the bootstrap peer ip from zerotier. Since at this point the init container for the bootstrap node has already ran, the bootstrap ip will for sure be registered in zerotier. I tried several times in zerotier and non zerotier modes, it always works for me. See PR 94 for a better explanation. Also make the zerotier script fail on error, so any error with zerotier api will fail the pod and trigger a retry.
oxheadalpha · Feb 11, 2021 · 908ca91 · 908ca91
1 parent f4aa12e
commit 908ca91
Show file tree

Hide file tree

Showing 3 changed files with 61 additions and 60 deletions.
diff --git a/charts/tezos/templates/baker.yaml b/charts/tezos/templates/baker.yaml
@@ -159,6 +159,36 @@ spec:
               name: var-volume
 {{- end }}
       initContainers:
+        - image: {{ .Values.tezos_k8s_images.key_importer }}
+          name: key-importer
+          imagePullPolicy: IfNotPresent
+          envFrom:
+            - secretRef:
+                name: tezos-secret
+            - configMapRef:
+                name: tezos-config
+          volumeMounts:
+            - mountPath: /var/tezos
+              name: var-volume
+        - args:
+            - "-c"
+            - "[ -f /var/tezos/node/identity.json ] || (mkdir -p /var/tezos/node && /usr/local/bin/tezos-node identity generate 0 --data-dir /var/tezos/node --config-file /etc/tezos/config.json)"
+          command:
+            - /bin/sh
+          image: {{ .Values.images.tezos | quote }}
+          name: identity-job
+          imagePullPolicy: IfNotPresent
+          volumeMounts:
+            - mountPath: /etc/tezos
+              name: config-volume
+            - mountPath: /var/tezos
+              name: var-volume
+        - image: {{ .Values.tezos_k8s_images.wait_for_bootstrap }}
+          imagePullPolicy: IfNotPresent
+          name: wait-for-bootstrap
+          volumeMounts:
+            - mountPath: /var/tezos
+              name: var-volume
 {{- if .Values.zerotier_in_use }}
         - envFrom:
             - configMapRef:
@@ -184,17 +214,6 @@ spec:
               fieldRef:
                 fieldPath: metadata.name
 {{- end }}
-        - image: {{ .Values.tezos_k8s_images.key_importer }}
-          name: key-importer
-          imagePullPolicy: IfNotPresent
-          envFrom:
-            - secretRef:
-                name: tezos-secret
-            - configMapRef:
-                name: tezos-config
-          volumeMounts:
-            - mountPath: /var/tezos
-              name: var-volume
         - image: {{ .Values.tezos_k8s_images.config_generator }}
           imagePullPolicy: IfNotPresent
           name: config-generator
@@ -213,25 +232,6 @@ spec:
               name: config-volume
             - mountPath: /var/tezos
               name: var-volume
-        - args:
-            - "-c"
-            - "[ -f /var/tezos/node/identity.json ] || (mkdir -p /var/tezos/node && /usr/local/bin/tezos-node identity generate 0 --data-dir /var/tezos/node --config-file /etc/tezos/config.json)"
-          command:
-            - /bin/sh
-          image: {{ .Values.images.tezos | quote }}
-          name: identity-job
-          imagePullPolicy: IfNotPresent
-          volumeMounts:
-            - mountPath: /etc/tezos
-              name: config-volume
-            - mountPath: /var/tezos
-              name: var-volume
-        - image: {{ .Values.tezos_k8s_images.wait_for_bootstrap }}
-          imagePullPolicy: IfNotPresent
-          name: wait-for-bootstrap
-          volumeMounts:
-            - mountPath: /var/tezos
-              name: var-volume
       securityContext:
         fsGroup: 100
       volumes:

diff --git a/charts/tezos/templates/node.yaml b/charts/tezos/templates/node.yaml
@@ -63,6 +63,36 @@ spec:
               name: var-volume
 {{- end }}
       initContainers:
+        - image: {{ .Values.tezos_k8s_images.key_importer }}
+          name: key-importer
+          imagePullPolicy: IfNotPresent
+          envFrom:
+            - secretRef:
+                name: tezos-secret
+            - configMapRef:
+                name: tezos-config
+          volumeMounts:
+            - mountPath: /var/tezos
+              name: var-volume
+        - args:
+            - "-c"
+            - "[ -f /var/tezos/node/identity.json ] || (mkdir -p /var/tezos/node && /usr/local/bin/tezos-node identity generate 0 --data-dir /var/tezos/node --config-file /etc/tezos/config.json)"
+          command:
+            - /bin/sh
+          image: {{ .Values.images.tezos | quote }}
+          name: identity-job
+          imagePullPolicy: IfNotPresent
+          volumeMounts:
+            - mountPath: /etc/tezos
+              name: config-volume
+            - mountPath: /var/tezos
+              name: var-volume
+        - image: {{ .Values.tezos_k8s_images.wait_for_bootstrap }}
+          imagePullPolicy: IfNotPresent
+          name: wait-for-bootstrap
+          volumeMounts:
+            - mountPath: /var/tezos
+              name: var-volume
 {{- if .Values.zerotier_in_use }}
         - envFrom:
             - configMapRef:
@@ -88,17 +118,6 @@ spec:
               fieldRef:
                 fieldPath: metadata.name
 {{- end }}
-        - image: {{ .Values.tezos_k8s_images.key_importer }}
-          name: key-importer
-          imagePullPolicy: IfNotPresent
-          envFrom:
-            - secretRef:
-                name: tezos-secret
-            - configMapRef:
-                name: tezos-config
-          volumeMounts:
-            - mountPath: /var/tezos
-              name: var-volume
         - image: {{ .Values.tezos_k8s_images.config_generator }}
           imagePullPolicy: IfNotPresent
           name: config-generator
@@ -117,25 +136,6 @@ spec:
               name: config-volume
             - mountPath: /var/tezos
               name: var-volume
-        - args:
-            - "-c"
-            - "[ -f /var/tezos/node/identity.json ] || (mkdir -p /var/tezos/node && /usr/local/bin/tezos-node identity generate 0 --data-dir /var/tezos/node --config-file /etc/tezos/config.json)"
-          command:
-            - /bin/sh
-          image: {{ .Values.images.tezos | quote }}
-          name: identity-job
-          imagePullPolicy: IfNotPresent
-          volumeMounts:
-            - mountPath: /etc/tezos
-              name: config-volume
-            - mountPath: /var/tezos
-              name: var-volume
-        - image: {{ .Values.tezos_k8s_images.wait_for_bootstrap }}
-          imagePullPolicy: IfNotPresent
-          name: wait-for-bootstrap
-          volumeMounts:
-            - mountPath: /var/tezos
-              name: var-volume
       securityContext:
         fsGroup: 100
       volumes:

diff --git a/zerotier/entrypoint.sh b/zerotier/entrypoint.sh
@@ -4,6 +4,7 @@
 # The IP is meant to be passed to the tezos container.
 # Then, this container should be restarted with a different command: `zerotier-one/var/tezos/zerotier`
 set -x
+set -e
 
 supervisord -c /etc/supervisor/supervisord.conf
-Original file line number
+Diff line change
@@ Expand Up / @@ -4,6 +4,7 @@ @@
     # The IP is meant to be passed to the tezos container.
     # Then, this container should be restarted with a different command: `zerotier-one/var/tezos/zerotier`
     set -x
+    set -e
     supervisord -c /etc/supervisor/supervisord.conf
@@ Expand Down @@