Restore archive node filesystem archive from s3 (#277)

* add new image * break out download and extract steps * nice messages * nice messages * use teztnets v10 snapshot * cleanup * revert comment * remove testing if statement * use mainnet filesystem snapshot * move tarbal download to separate script * restore default snapshot-downloader behavior * set up helpers and containers * copy new tarball script into utils container * typo in script name in dockerfile * add to entrypoint * add tarball-downloader command * fix helpers var, add init container * remove /var/tezos/node and remove version * chmod script * document snapshot download feature usage in values * rearrange doc * fix doc * exit on error and change doc * remove alphanet copy * put alphanet back * take out cp * add storageclass for node volumes * new curl and rm identity.json for helpful logging * cannot use both snapshot_url and tarball_url * extra dash * remove alphanet_version file copy since it was failing Co-authored-by: Nicolas Ochem <[email protected]>
oxheadalpha · Sep 30, 2021 · 2f118ec · 2f118ec
1 parent 667a197
commit 2f118ec
Show file tree

Hide file tree

Showing 9 changed files with 81 additions and 3 deletions.
diff --git a/charts/tezos/templates/_containers.tpl b/charts/tezos/templates/_containers.tpl
@@ -76,6 +76,24 @@
 {{- end }}
 {{- end }}
 
+{{- define "tezos.init_container.tarball_downloader" }}
+{{- if include "tezos.shouldDownloadTarball" . }}
+- image: "{{ .Values.tezos_k8s_images.utils }}"
+  imagePullPolicy: IfNotPresent
+  name: tarball-downloader
+  args:
+    - tarball-downloader
+  volumeMounts:
+    - mountPath: /var/tezos
+      name: var-volume
+  envFrom:
+    - configMapRef:
+        name: tezos-config
+  env:
+{{- include "tezos.localvars.pod_envvars" . | indent 4 }}
+{{- end }}
+{{- end }}
+
 {{- define "tezos.init_container.snapshot_downloader" }}
 {{- if include "tezos.shouldDownloadSnapshot" . }}
 - image: "{{ .Values.tezos_k8s_images.utils }}"

diff --git a/charts/tezos/templates/_helpers.tpl b/charts/tezos/templates/_helpers.tpl
@@ -63,14 +63,33 @@
 {{- end }}
 {{- end }}
 
+{{/*
+  Checks if filesystem archive should be downloaded.
+*/}}
+{{- define "tezos.shouldDownloadTarball" -}}
+{{- if (.Values.tarball_url)}}
+  {{- if or (.Values.full_snapshot_url)  (.Values.rolling_snapshot_url) }}
+    {{- fail ".Values.tarball_url cannot be defined with .Values.full_snapshot_url or .Values.rolling_snapshot_url" }}
+  {{- else}}
+  {{- "true" }}
+  {{- end }}
+{{- else }}
+{{- "" }}
+{{- end }}
+{{- end }}
+
 {{/*
   Checks if a snapshot should be downloaded. Either full_snapshot_url or
   rolling_snapshot_url must not be null.
   Returns a string "true" or empty string which is falsey.
 */}}
 {{- define "tezos.shouldDownloadSnapshot" -}}
 {{- if or (.Values.full_snapshot_url)  (.Values.rolling_snapshot_url) }}
-{{- "true" }}
+  {{- if (.Values.tarball_url)}}
+  {{- fail ".Values.full_snapshot_url or .Values.rolling_snapshot_url cannot be defined with .Values.tarball_url" }}
+  {{- else }}
+  {{- "true" }}
+  {{- end }}
 {{- else }}
 {{- "" }}
 {{- end }}

diff --git a/charts/tezos/templates/configs.yaml b/charts/tezos/templates/configs.yaml
@@ -13,6 +13,7 @@ data:
     }
   FULL_SNAPSHOT_URL: "{{ .Values.full_snapshot_url }}"
   ROLLING_SNAPSHOT_URL: "{{ .Values.rolling_snapshot_url }}"
+  TARBALL_URL: "{{ .Values.tarball_url }}"
   NODES: |
 {{ .Values.nodes | mustToPrettyJson | indent 4 }}
   SIGNERS: |

diff --git a/charts/tezos/templates/nodes.yaml b/charts/tezos/templates/nodes.yaml
@@ -36,6 +36,7 @@ spec:
 {{- include "tezos.init_container.wait_for_dns"        $ | indent 8 }}
 {{- include "tezos.init_container.snapshot_downloader" $ | indent 8 }}
 {{- include "tezos.init_container.snapshot_importer"   $ | indent 8 }}
+{{- include "tezos.init_container.tarball_downloader"  $ | indent 8 }}
       securityContext:
         fsGroup: 100
 {{- include "tezos.nodeSelectorConfig" $ | indent 6 }}
@@ -56,6 +57,9 @@ spec:
       spec:
         accessModes:
           - ReadWriteOnce
+        {{- if $val.storageClassName }}
+        storageClassName: {{ $val.storageClassName }}
+        {{- end }}
         resources:
           requests:
             storage: {{ default "15Gi" $val.storage_size }}

diff --git a/charts/tezos/values.yaml b/charts/tezos/values.yaml
@@ -148,6 +148,12 @@ signers: {}
 full_snapshot_url: https://mainnet.xtz-shots.io/full
 rolling_snapshot_url: https://mainnet.xtz-shots.io/rolling
 
+# Alternatively to a snapshot, you can download a public LZ4-compressed filesystem tar
+# of a node's /var/tezos/node folder by putting the URL to the archive file in tarball_url:
+# NOTE: tarballl_url and *_snapshot_url are mutually exclusive. When tarball_url is 
+# configured,full_snapshot_url and rolling_snapshot_url must be null.
+# tarball_url: 
+
 # List of peers for nodes to connect to. Gets set under config.json `p2p` field
 bootstrap_peers: []
 

diff --git a/utils/Dockerfile b/utils/Dockerfile
@@ -17,14 +17,16 @@ RUN     apk add --no-cache --virtual .build-deps gcc g++ python3-dev	\
      && pip install pyblake2 pysodium numpy flask \
      && pip install -e 'git+https://github.com/vbuterin/pybitcointools.git@aeb0a2bbb8bbfe421432d776c649650eaeb882a5#egg=bitcoin' \
      && apk del .build-deps git \
-     && apk add jq netcat-openbsd curl binutils
+     && apk add jq netcat-openbsd curl binutils \ 
+     && apk add lz4
 
 COPY config-generator.py /
 COPY faucet-gen.py /
 COPY config-generator.sh /
 COPY entrypoint.sh /
 COPY logger.sh /
 COPY snapshot-downloader.sh /
+COPY tarball-downloader.sh /
 COPY wait-for-dns.sh /
 ENTRYPOINT ["/entrypoint.sh"]
 CMD []
diff --git a/utils/entrypoint.sh b/utils/entrypoint.sh
@@ -7,6 +7,7 @@ case "$CMD" in
 	config-generator)	exec /config-generator.sh	"$@"	;;
 	logger)			exec /logger.sh			"$@"	;;
 	snapshot-downloader)	exec /snapshot-downloader.sh	"$@"	;;
+	tarball-downloader)	exec /tarball-downloader.sh	"$@"	;;
 	wait-for-dns)		exec /wait-for-dns.sh		"$@"	;;
 	faucet-gen)	        exec /faucet-gen.py     	"$@"	;;
 esac
@@ -21,6 +22,7 @@ echo "Valid options are:"
 echo "	config-generator"
 echo "	logger"
 echo "	snapshot-downloader"
+echo "	tarball-downloader"
 echo "	wait-for-dns"
 echo "	faucet-gen"
 

diff --git a/utils/snapshot-downloader.sh b/utils/snapshot-downloader.sh
@@ -1,5 +1,7 @@
 #!/bin/sh
 
+set -e
+
 data_dir="/var/tezos"
 node_dir="$data_dir/node"
 node_data_dir="$node_dir/data"
@@ -23,7 +25,6 @@ if [ ! -d $node_data_dir/context ]; then
 	echo "Did not find pre-existing data, importing blockchain"
 	mkdir -p $node_data_dir
 	echo '{ "version": "0.0.4" }' > $node_dir/version.json
-	cp -v /usr/local/share/tezos/alphanet_version $node_dir
 	curl -Lf -o $snapshot_file $snapshot_url
 fi
 

diff --git a/utils/tarball-downloader.sh b/utils/tarball-downloader.sh
@@ -0,0 +1,25 @@
+#!/bin/sh
+
+set -e
+
+data_dir="/var/tezos"
+node_dir="$data_dir/node"
+node_data_dir="$node_dir/data"
+
+
+if [ -d /var/tezos ] ; then
+  if [ ! -d $node_data_dir/context ]; then
+    echo "Did not find pre-existing data, importing blockchain"
+    rm -rf $node_data_dir 
+    mkdir -p $node_data_dir
+    curl -LfsS "$TARBALL_URL" | lz4 -d | tar -x -C /var/tezos
+    rm -fv $node_data_dir/identity.json
+  fi
+else
+  echo "/var/tezos does not exist."
+  echo "Error!" 1>&2
+  exit 1
+fi
+
+chown -R 100 /var/tezos
+ls -lR /var/tezos