From 2f118ecb125bc8a9e594621c702d1cc037e85535 Mon Sep 17 00:00:00 2001 From: Jimmy <5608027+orcutt989@users.noreply.github.com> Date: Thu, 30 Sep 2021 17:46:31 -0400 Subject: [PATCH] Restore archive node filesystem archive from s3 (#277) * add new image * break out download and extract steps * nice messages * nice messages * use teztnets v10 snapshot * cleanup * revert comment * remove testing if statement * use mainnet filesystem snapshot * move tarbal download to separate script * restore default snapshot-downloader behavior * set up helpers and containers * copy new tarball script into utils container * typo in script name in dockerfile * add to entrypoint * add tarball-downloader command * fix helpers var, add init container * remove /var/tezos/node and remove version * chmod script * document snapshot download feature usage in values * rearrange doc * fix doc * exit on error and change doc * remove alphanet copy * put alphanet back * take out cp * add storageclass for node volumes * new curl and rm identity.json for helpful logging * cannot use both snapshot_url and tarball_url * extra dash * remove alphanet_version file copy since it was failing Co-authored-by: Nicolas Ochem --- charts/tezos/templates/_containers.tpl | 18 ++++++++++++++++++ charts/tezos/templates/_helpers.tpl | 21 ++++++++++++++++++++- charts/tezos/templates/configs.yaml | 1 + charts/tezos/templates/nodes.yaml | 4 ++++ charts/tezos/values.yaml | 6 ++++++ utils/Dockerfile | 4 +++- utils/entrypoint.sh | 2 ++ utils/snapshot-downloader.sh | 3 ++- utils/tarball-downloader.sh | 25 +++++++++++++++++++++++++ 9 files changed, 81 insertions(+), 3 deletions(-) create mode 100755 utils/tarball-downloader.sh diff --git a/charts/tezos/templates/_containers.tpl b/charts/tezos/templates/_containers.tpl index dd47df575..d6d73f63d 100644 --- a/charts/tezos/templates/_containers.tpl +++ b/charts/tezos/templates/_containers.tpl @@ -76,6 +76,24 @@ {{- end }} {{- end }} +{{- define "tezos.init_container.tarball_downloader" }} +{{- if include "tezos.shouldDownloadTarball" . }} +- image: "{{ .Values.tezos_k8s_images.utils }}" + imagePullPolicy: IfNotPresent + name: tarball-downloader + args: + - tarball-downloader + volumeMounts: + - mountPath: /var/tezos + name: var-volume + envFrom: + - configMapRef: + name: tezos-config + env: +{{- include "tezos.localvars.pod_envvars" . | indent 4 }} +{{- end }} +{{- end }} + {{- define "tezos.init_container.snapshot_downloader" }} {{- if include "tezos.shouldDownloadSnapshot" . }} - image: "{{ .Values.tezos_k8s_images.utils }}" diff --git a/charts/tezos/templates/_helpers.tpl b/charts/tezos/templates/_helpers.tpl index 3683cdb19..9d858fc06 100644 --- a/charts/tezos/templates/_helpers.tpl +++ b/charts/tezos/templates/_helpers.tpl @@ -63,6 +63,21 @@ {{- end }} {{- end }} +{{/* + Checks if filesystem archive should be downloaded. +*/}} +{{- define "tezos.shouldDownloadTarball" -}} +{{- if (.Values.tarball_url)}} + {{- if or (.Values.full_snapshot_url) (.Values.rolling_snapshot_url) }} + {{- fail ".Values.tarball_url cannot be defined with .Values.full_snapshot_url or .Values.rolling_snapshot_url" }} + {{- else}} + {{- "true" }} + {{- end }} +{{- else }} +{{- "" }} +{{- end }} +{{- end }} + {{/* Checks if a snapshot should be downloaded. Either full_snapshot_url or rolling_snapshot_url must not be null. @@ -70,7 +85,11 @@ */}} {{- define "tezos.shouldDownloadSnapshot" -}} {{- if or (.Values.full_snapshot_url) (.Values.rolling_snapshot_url) }} -{{- "true" }} + {{- if (.Values.tarball_url)}} + {{- fail ".Values.full_snapshot_url or .Values.rolling_snapshot_url cannot be defined with .Values.tarball_url" }} + {{- else }} + {{- "true" }} + {{- end }} {{- else }} {{- "" }} {{- end }} diff --git a/charts/tezos/templates/configs.yaml b/charts/tezos/templates/configs.yaml index de0284657..e1dd1d4f3 100644 --- a/charts/tezos/templates/configs.yaml +++ b/charts/tezos/templates/configs.yaml @@ -13,6 +13,7 @@ data: } FULL_SNAPSHOT_URL: "{{ .Values.full_snapshot_url }}" ROLLING_SNAPSHOT_URL: "{{ .Values.rolling_snapshot_url }}" + TARBALL_URL: "{{ .Values.tarball_url }}" NODES: | {{ .Values.nodes | mustToPrettyJson | indent 4 }} SIGNERS: | diff --git a/charts/tezos/templates/nodes.yaml b/charts/tezos/templates/nodes.yaml index c8098030c..9c09dff93 100644 --- a/charts/tezos/templates/nodes.yaml +++ b/charts/tezos/templates/nodes.yaml @@ -36,6 +36,7 @@ spec: {{- include "tezos.init_container.wait_for_dns" $ | indent 8 }} {{- include "tezos.init_container.snapshot_downloader" $ | indent 8 }} {{- include "tezos.init_container.snapshot_importer" $ | indent 8 }} +{{- include "tezos.init_container.tarball_downloader" $ | indent 8 }} securityContext: fsGroup: 100 {{- include "tezos.nodeSelectorConfig" $ | indent 6 }} @@ -56,6 +57,9 @@ spec: spec: accessModes: - ReadWriteOnce + {{- if $val.storageClassName }} + storageClassName: {{ $val.storageClassName }} + {{- end }} resources: requests: storage: {{ default "15Gi" $val.storage_size }} diff --git a/charts/tezos/values.yaml b/charts/tezos/values.yaml index 66c92cecb..2d6771045 100644 --- a/charts/tezos/values.yaml +++ b/charts/tezos/values.yaml @@ -148,6 +148,12 @@ signers: {} full_snapshot_url: https://mainnet.xtz-shots.io/full rolling_snapshot_url: https://mainnet.xtz-shots.io/rolling +# Alternatively to a snapshot, you can download a public LZ4-compressed filesystem tar +# of a node's /var/tezos/node folder by putting the URL to the archive file in tarball_url: +# NOTE: tarballl_url and *_snapshot_url are mutually exclusive. When tarball_url is +# configured,full_snapshot_url and rolling_snapshot_url must be null. +# tarball_url: + # List of peers for nodes to connect to. Gets set under config.json `p2p` field bootstrap_peers: [] diff --git a/utils/Dockerfile b/utils/Dockerfile index 61c80da52..dc343c7cf 100644 --- a/utils/Dockerfile +++ b/utils/Dockerfile @@ -17,7 +17,8 @@ RUN apk add --no-cache --virtual .build-deps gcc g++ python3-dev \ && pip install pyblake2 pysodium numpy flask \ && pip install -e 'git+https://github.com/vbuterin/pybitcointools.git@aeb0a2bbb8bbfe421432d776c649650eaeb882a5#egg=bitcoin' \ && apk del .build-deps git \ - && apk add jq netcat-openbsd curl binutils + && apk add jq netcat-openbsd curl binutils \ + && apk add lz4 COPY config-generator.py / COPY faucet-gen.py / @@ -25,6 +26,7 @@ COPY config-generator.sh / COPY entrypoint.sh / COPY logger.sh / COPY snapshot-downloader.sh / +COPY tarball-downloader.sh / COPY wait-for-dns.sh / ENTRYPOINT ["/entrypoint.sh"] CMD [] diff --git a/utils/entrypoint.sh b/utils/entrypoint.sh index 7c5a80782..aae645e05 100755 --- a/utils/entrypoint.sh +++ b/utils/entrypoint.sh @@ -7,6 +7,7 @@ case "$CMD" in config-generator) exec /config-generator.sh "$@" ;; logger) exec /logger.sh "$@" ;; snapshot-downloader) exec /snapshot-downloader.sh "$@" ;; + tarball-downloader) exec /tarball-downloader.sh "$@" ;; wait-for-dns) exec /wait-for-dns.sh "$@" ;; faucet-gen) exec /faucet-gen.py "$@" ;; esac @@ -21,6 +22,7 @@ echo "Valid options are:" echo " config-generator" echo " logger" echo " snapshot-downloader" +echo " tarball-downloader" echo " wait-for-dns" echo " faucet-gen" diff --git a/utils/snapshot-downloader.sh b/utils/snapshot-downloader.sh index b5ad66f58..13e2d3739 100755 --- a/utils/snapshot-downloader.sh +++ b/utils/snapshot-downloader.sh @@ -1,5 +1,7 @@ #!/bin/sh +set -e + data_dir="/var/tezos" node_dir="$data_dir/node" node_data_dir="$node_dir/data" @@ -23,7 +25,6 @@ if [ ! -d $node_data_dir/context ]; then echo "Did not find pre-existing data, importing blockchain" mkdir -p $node_data_dir echo '{ "version": "0.0.4" }' > $node_dir/version.json - cp -v /usr/local/share/tezos/alphanet_version $node_dir curl -Lf -o $snapshot_file $snapshot_url fi diff --git a/utils/tarball-downloader.sh b/utils/tarball-downloader.sh new file mode 100755 index 000000000..412acd45a --- /dev/null +++ b/utils/tarball-downloader.sh @@ -0,0 +1,25 @@ +#!/bin/sh + +set -e + +data_dir="/var/tezos" +node_dir="$data_dir/node" +node_data_dir="$node_dir/data" + + +if [ -d /var/tezos ] ; then + if [ ! -d $node_data_dir/context ]; then + echo "Did not find pre-existing data, importing blockchain" + rm -rf $node_data_dir + mkdir -p $node_data_dir + curl -LfsS "$TARBALL_URL" | lz4 -d | tar -x -C /var/tezos + rm -fv $node_data_dir/identity.json + fi +else + echo "/var/tezos does not exist." + echo "Error!" 1>&2 + exit 1 +fi + +chown -R 100 /var/tezos +ls -lR /var/tezos \ No newline at end of file