tezos-proto-cruncher chart (#532)

* tezos-proto-cruncher chart

This chart deploys a daemonset in your cluster. the daemonset runs a
parallel search for a vanity protocol hash.

It takes s3 bucket credentials. Protocol is downloaded from bucket then
results are written to the bucket.

It does not terminate: it keeps running until the cluster is torn down.

* better variable names

* Update charts/tezos-proto-cruncher/scripts/proto-downloader.py

Co-authored-by: Aryeh Harris <[email protected]>

* address review comments

* lint python scripts with black

* catch exception when s3 upload fails

* fix manual method

* fix manual method, better way per Aryeh

* fail fast when env vars not set in proto downloader

Co-authored-by: Aryeh Harris <[email protected]>

charts/tezos-proto-cruncher/.helmignore

@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

charts/tezos-proto-cruncher/Chart.yaml

@@ -0,0 +1,24 @@
+apiVersion: v2
+name: tezos-proto-cruncher
+description: A Helm chart for Kubernetes
+
+# A chart can be either an 'application' or a 'library' chart.
+#
+# Application charts are a collection of templates that can be packaged into versioned archives
+# to be deployed.
+#
+# Library charts provide useful utilities or functions for the chart developer. They're included as
+# a dependency of application charts to inject those utilities and functions into the rendering
+# pipeline. Library charts do not define any templates and therefore cannot be deployed.
+type: application
+
+# This is the chart version. This version number should be incremented each time you make changes
+# to the chart and its templates, including the app version.
+# Versions are expected to follow Semantic Versioning (https://semver.org/)
+version: 0.1.0
+
+# This is the version number of the application being deployed. This version number should be
+# incremented each time you make changes to the application. Versions are not expected to
+# follow Semantic Versioning. They should reflect the version the application is using.
+# It is recommended to use it with quotes.
+appVersion: "1.16.0"

charts/tezos-proto-cruncher/README.md

@@ -0,0 +1,6 @@
+# Tezos-proto-cruncher
+
+This chart deploys a daemonset to perform a brute-force search
+of a vanity protocol name.
+
+See values.yaml for details.

charts/tezos-proto-cruncher/scripts/proto-cruncher.py

@@ -0,0 +1,79 @@
+import hashlib
+import random
+import base58
+import string
+import os
+import sys
+import re
+
+
+proto_file = sys.argv[1]
+
+
+def tb(l):
+    return b"".join(map(lambda x: x.to_bytes(1, "big"), l))
+
+
+proto_prefix = tb([2, 170])
+
+PROTO_NAME = os.getenv("PROTO_NAME")
+VANITY_STRING = os.getenv("VANITY_STRING")
+NUM_NONCE_DIGITS = int(os.getenv("NUM_NONCE_DIGITS", 16))
+BUCKET_NAME = os.getenv("BUCKET_NAME")
+BUCKET_ENDPOINT_URL = os.getenv("BUCKET_ENDPOINT_URL")
+BUCKET_REGION = os.getenv("BUCKET_REGION")
+
+if not VANITY_STRING:
+    raise ValueError("VANITY_STRING env var must be set")
+
+if BUCKET_NAME:
+    import boto3
+
+    s3 = boto3.resource(
+        "s3", region_name=BUCKET_REGION, endpoint_url=f"https://{BUCKET_ENDPOINT_URL}"
+    )
+
+with open(proto_file, "rb") as f:
+    proto_bytes = f.read()
+
+with open(proto_file, "rb") as f:
+    proto_lines = f.readlines()
+
+
+# For speed, we precompute the hash of the proto without the last line
+# containing the vanity nonce.
+# Later on, we add the last line and recompute.
+# This speeds up the brute force by a large factor, compared to hashing
+# in full at every try.
+original_nonce = proto_lines[-1]
+# First 4 bytes are truncated (not used in proto hashing)
+proto_hash = hashlib.blake2b(proto_bytes[4:][: -len(original_nonce)], digest_size=32)
+
+
+def get_hash(vanity_nonce, proto_hash):
+    proto_hash.update(vanity_nonce)
+    return base58.b58encode_check(proto_prefix + proto_hash.digest()).decode("utf-8")
+
+
+print(
+    f"Original proto nonce: {original_nonce} and hash: {get_hash(original_nonce, proto_hash.copy())}"
+)
+
+while True:
+    # Warning - assuming the nonce is 16 chars in the original proto.
+    # If it is not, make sure to set NUM_NONCE_DIGITS to the right number
+    # otherwise you will get bad nonces.
+    new_nonce_digits = "".join(
+        random.choice(string.digits) for _ in range(NUM_NONCE_DIGITS)
+    )
+    new_nonce = b"(* Vanity nonce: " + bytes(new_nonce_digits, "utf-8") + b" *)\n"
+
+    new_hash = get_hash(new_nonce, proto_hash.copy())
+    if re.match(f"^{VANITY_STRING}.*", new_hash):
+        print(f"Found vanity nonce: {new_nonce} and hash: {new_hash}")
+        if BUCKET_NAME:
+            try:
+                s3.Object(BUCKET_NAME, f"{PROTO_NAME}_{new_hash}").put(Body=new_nonce)
+            except:
+                print("ERROR: upload of the nonce and hash to s3 failed.")
+                continue

charts/tezos-proto-cruncher/scripts/proto-downloader.py

@@ -0,0 +1,24 @@
+import os
+
+BUCKET_NAME = os.environ["BUCKET_NAME"]
+BUCKET_ENDPOINT_URL = os.environ["BUCKET_ENDPOINT_URL"]
+BUCKET_REGION = os.environ["BUCKET_REGION"]
+
+PROTO_NAME = os.environ["PROTO_NAME"]
+
+import boto3
+
+s3 = boto3.resource(
+    "s3", region_name=BUCKET_REGION, endpoint_url=f"https://{BUCKET_ENDPOINT_URL}"
+)
+
+print(f"Downloading {PROTO_NAME}")
+proto_file = f"/{PROTO_NAME}"
+s3_bucket = s3.Bucket(BUCKET_NAME)
+try:
+    s3_bucket.download_file(PROTO_NAME, proto_file)
+except botocore.exceptions.ClientError as e:
+    if e.response["Error"]["Code"] == "404":
+        print("The object does not exist.")
+    else:
+        raise

charts/tezos-proto-cruncher/scripts/tezos-proto-cruncher.sh

@@ -0,0 +1,11 @@
+set -eo pipefail
+apk add parallel
+pip install boto3
+
+python /proto-downloader.py
+
+if [ -z "${NUM_PARALLEL_PROCESSES}" ]; then
+    # Launch one process per CPU core to maximize utilization
+    NUM_PARALLEL_PROCESSES=$(grep -c ^processor /proc/cpuinfo)
+fi
+seq $NUM_PARALLEL_PROCESSES | parallel --ungroup python /proto-cruncher.py /${PROTO_NAME}

charts/tezos-proto-cruncher/templates/_helpers.tpl

@@ -0,0 +1,62 @@
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "tezos-proto-cruncher.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "tezos-proto-cruncher.fullname" -}}
+{{- if .Values.fullnameOverride }}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- $name := default .Chart.Name .Values.nameOverride }}
+{{- if contains $name .Release.Name }}
+{{- .Release.Name | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
+{{- end }}
+{{- end }}
+{{- end }}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "tezos-proto-cruncher.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Common labels
+*/}}
+{{- define "tezos-proto-cruncher.labels" -}}
+helm.sh/chart: {{ include "tezos-proto-cruncher.chart" . }}
+{{ include "tezos-proto-cruncher.selectorLabels" . }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end }}
+
+{{/*
+Selector labels
+*/}}
+{{- define "tezos-proto-cruncher.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "tezos-proto-cruncher.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end }}
+
+{{/*
+Create the name of the service account to use
+*/}}
+{{- define "tezos-proto-cruncher.serviceAccountName" -}}
+{{- if .Values.serviceAccount.create }}
+{{- default (include "tezos-proto-cruncher.fullname" .) .Values.serviceAccount.name }}
+{{- else }}
+{{- default "default" .Values.serviceAccount.name }}
+{{- end }}
+{{- end }}

charts/tezos-proto-cruncher/templates/config.yaml

@@ -0,0 +1,30 @@
+kind: Secret
+apiVersion: v1
+metadata:
+  name: s3-secrets
+data:
+  AWS_SECRET_ACCESS_KEY: "{{ .Values.s3SecretAccessKey | b64enc }}"
+---
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: s3-env
+data:
+  AWS_ACCESS_KEY_ID: "{{ .Values.s3AccessKeyId }}"
+  BUCKET_ENDPOINT_URL: "{{ .Values.bucketEndpointUrl }}"
+  BUCKET_NAME: "{{ .Values.bucketName}}"
+  BUCKET_REGION: "{{ .Values.bucketRegion}}"
+  PROTO_NAME: "{{ .Values.protoName}}"
+  VANITY_STRING: "{{ .Values.vanityString}}"
+  NUM_NONCE_DIGITS: "{{ .Values.numNonceDigits }}"
+  NUM_PARALLEL_PROCESSES: "{{ .Values.numParallelProcesses }}"
+---
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: scripts
+data:
+  proto-cruncher.py: |
+{{ tpl ($.Files.Get (print "scripts/proto-cruncher.py")) $ | indent 4 }}
+  proto-downloader.py: |
+{{ tpl ($.Files.Get (print "scripts/proto-downloader.py")) $ | indent 4 }}

charts/tezos-proto-cruncher/templates/daemonset.yaml

@@ -0,0 +1,66 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: {{ include "tezos-proto-cruncher.fullname" . }}
+  labels:
+    {{- include "tezos-proto-cruncher.labels" . | nindent 4 }}
+spec:
+  selector:
+    matchLabels:
+      {{- include "tezos-proto-cruncher.selectorLabels" . | nindent 6 }}
+  template:
+    metadata:
+      {{- with .Values.podAnnotations }}
+      annotations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      labels:
+        {{- include "tezos-proto-cruncher.selectorLabels" . | nindent 8 }}
+    spec:
+      {{- with .Values.imagePullSecrets }}
+      imagePullSecrets:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      securityContext:
+        {{- toYaml .Values.podSecurityContext | nindent 8 }}
+      containers:
+        - name: {{ .Chart.Name }}
+          securityContext:
+            {{- toYaml .Values.securityContext | nindent 12 }}
+          image: "{{ .Values.tezos_k8s_images.utils }}"
+          volumeMounts:
+            - mountPath: /proto-cruncher.py
+              name: scripts
+              subPath: proto-cruncher.py
+            - mountPath: /proto-downloader.py
+              name: scripts
+              subPath: proto-downloader.py
+          envFrom:
+          - secretRef:
+              name: s3-secrets
+          - configMapRef:
+              name: s3-env
+          command:
+            - /bin/sh
+          args:
+            - "-c"
+            - |
+{{ tpl ($.Files.Get (print "scripts/tezos-proto-cruncher.sh")) $ | indent 14 }}
+          resources:
+            {{- toYaml .Values.resources | nindent 12 }}
+      volumes:
+        - name: scripts
+          configMap:
+             name: scripts
+      {{- with .Values.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}

charts/tezos-proto-cruncher/values.yaml

@@ -0,0 +1,43 @@
+# tezos-proto-cruncher downloads a protocol in binary format
+# from a S3 bucket, and writes the results in the same bucket.
+# Put the credentials for access to your bucket below.
+s3AccessKeyId: ''
+s3SecretAccessKey: ''
+bucketEndpointUrl: 'ams3.digitaloceanspaces.com'
+bucketRegion: 'ams'
+
+# name of the bucket
+bucketName: "tezos-proto-cruncher"
+
+# name of the protocol file in the bucket
+protoName: "mumbai-f"
+#
+# vanity string we are looking for.
+# Must start with Pr, Ps, or Pt, but probably Pt as it is the convention.
+vanityString: "PtMumbai"
+
+# number of nonce digits in original proto - normally 16
+numNonceDigits: 16
+
+# number of parallel Python processes per k8s node
+# When unset, defaults to number of cores
+# numParallelProcesses: 8
+
+tezos_k8s_images:
+  utils: ghcr.io/oxheadalpha/tezos-k8s-utils:master
+
+nameOverride: ""
+fullnameOverride: ""
+
+podAnnotations: {}
+
+podSecurityContext: {}
+  # fsGroup: 2000
+
+securityContext: {}
+
+nodeSelector: {}
+
+tolerations: []
+
+affinity: {}

docs/04-other-helm-charts.md

@@ -7,3 +7,4 @@ The Tezos-k8s project also maintains several other helm charts:
 * **[tezos-faucet](https://github.com/oxheadalpha/tezos-k8s/tree/master/charts/tezos-faucet)**: a faucet to dispense testnet funds
 * **[snapshotEngine](https://github.com/oxheadalpha/tezos-k8s/tree/master/charts/snapshotEngine)**: an utility to generate Tezos snapshots from running nodes, using Kubernetes CSI driver
 * **[tezos-reward-distributor](https://github.com/oxheadalpha/tezos-k8s/tree/master/charts/tezos-reward-distributor)**: a chart to deploy [TRD](https://tezos-reward-distributor-organization.github.io/tezos-reward-distributor/)
+* **[tezos-proto-cruncher](https://github.com/oxheadalpha/tezos-k8s/tree/master/charts/tezos-proto-cruncher)**: a chart to deploy a daemonset in your cluster to find vanity hashes for protocols in parallel