From 41d8908ed6265d8aad1a37b338f81718477da4b3 Mon Sep 17 00:00:00 2001
From: Alexander Cristurean <alexander.cristurean@yahoo.com>
Date: Tue, 27 Feb 2024 17:57:48 +0200
Subject: [PATCH] added script to create local-testnet with docker.

---
 .github/workflows/build_local_tesnet.yml |  54 ++++++
 docker/node/Dockerfile                   |   4 +-
 scripts/docker-testnet/clean.sh          |  16 ++
 scripts/docker-testnet/helpers.sh        | 153 +++++++++++++++++
 scripts/docker-testnet/start.sh          |  38 +++++
 scripts/docker-testnet/variables.sh      | 199 +++++++++++++++++++++++
 6 files changed, 461 insertions(+), 3 deletions(-)
 create mode 100644 .github/workflows/build_local_tesnet.yml
 create mode 100755 scripts/docker-testnet/clean.sh
 create mode 100755 scripts/docker-testnet/helpers.sh
 create mode 100755 scripts/docker-testnet/start.sh
 create mode 100644 scripts/docker-testnet/variables.sh

diff --git a/.github/workflows/build_local_tesnet.yml b/.github/workflows/build_local_tesnet.yml
new file mode 100644
index 00000000000..edd77fe9026
--- /dev/null
+++ b/.github/workflows/build_local_tesnet.yml
@@ -0,0 +1,54 @@
+name: Build local testnet
+
+on:
+  pull_request:
+    branches: [ master, rc/* ]
+    types: [opened, ready_for_review]
+  push:
+  workflow_dispatch:
+
+jobs:
+  build:
+    strategy:
+      matrix:
+        runs-on: [ubuntu-latest]
+    runs-on: ${{ matrix.runs-on }}
+    name: Build
+    steps:
+      - name: Set up Go 1.20.7
+        uses: actions/setup-go@v3
+        with:
+          go-version: 1.20.7
+        id: go
+
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v4
+
+      - name: Check out mx-deploy-go
+        uses: actions/checkout@v4
+        with:
+          repository: multiversx/mx-chain-deploy-go
+          path: mx-chain-deploy-go
+
+      - name: Check out mx-chain-proxy-go
+        uses: actions/checkout@v4
+        with:
+          repository: multiversx/mx-chain-proxy-go
+          path: mx-chain-proxy-go
+
+      - name: Build images
+        run: |
+          docker build -f docker/node/Dockerfile . -t node:dev
+          docker build -f docker/seednode/Dockerfile . -t seednode:dev
+
+      - name: Start localnet
+        id: generate-config
+        run: |
+          cd ${GITHUB_WORKSPACE}/scripts/docker-testnet
+          export TESTNETDIR=${GITHUB_WORKSPACE}/docker-testnet
+          export CI_RUN=1
+          ./start.sh
+          echo "Check everything is alright. Remove once confirmed"
+          docker ps
+          sleep 1m
+          curl http://localhost:7950
diff --git a/docker/node/Dockerfile b/docker/node/Dockerfile
index cf6a8955c76..2513f789dc8 100644
--- a/docker/node/Dockerfile
+++ b/docker/node/Dockerfile
@@ -10,12 +10,10 @@ RUN go build -v -ldflags="-X main.appVersion=$(git describe --tags --long --dirt
 RUN cp /go/pkg/mod/github.com/multiversx/$(cat /go/mx-chain-go/go.mod | grep mx-chain-vm-v | sort -n | tail -n -1| awk -F '/' '{print$3}'| sed 's/ /@/g')/wasmer/libwasmer_linux_amd64.so /lib/libwasmer_linux_amd64.so
 RUN cp /go/pkg/mod/github.com/multiversx/$(cat /go/mx-chain-go/go.mod | grep mx-chain-vm-go | sort -n | tail -n -1| awk -F '/' '{print$3}'| sed 's/ /@/g')/wasmer2/libvmexeccapi.so /lib/libvmexeccapi.so
 
-WORKDIR /go/mx-chain-go/cmd/node
-
 # ===== SECOND STAGE ======
 FROM ubuntu:22.04
 RUN apt-get update && apt-get upgrade -y
-COPY --from=builder "/go/mx-chain-go/cmd/node" "/go/mx-chain-go/cmd/node/"
+COPY --from=builder "/go/mx-chain-go/cmd/node/node" "/go/mx-chain-go/cmd/node/"
 COPY --from=builder "/lib/libwasmer_linux_amd64.so" "/lib/libwasmer_linux_amd64.so"
 COPY --from=builder "/lib/libvmexeccapi.so" "/lib/libvmexeccapi.so"
 WORKDIR /go/mx-chain-go/cmd/node/
diff --git a/scripts/docker-testnet/clean.sh b/scripts/docker-testnet/clean.sh
new file mode 100755
index 00000000000..a872ed57f13
--- /dev/null
+++ b/scripts/docker-testnet/clean.sh
@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+# Delete the entire testnet folder, which includes configuration, executables and logs.
+
+export MULTIVERSXTESTNETSCRIPTSDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+
+source "$MULTIVERSXTESTNETSCRIPTSDIR/variables.sh"
+
+echo "Stopping all containers..."
+docker stop $(docker ps -a -q)
+
+echo "Removing all containers..."
+docker container prune -f
+
+echo "Removing $TESTNETDIR..."
+rm -rf $TESTNETDIR
diff --git a/scripts/docker-testnet/helpers.sh b/scripts/docker-testnet/helpers.sh
new file mode 100755
index 00000000000..c7967c8c408
--- /dev/null
+++ b/scripts/docker-testnet/helpers.sh
@@ -0,0 +1,153 @@
+#!/usr/bin/env bash
+
+startSeedNode() {
+  docker run -d --name seednode -v ${TESTNETDIR}/seednode/config:/go/mx-chain-go/cmd/seednode/config seednode:dev \
+  --rest-api-interface=0.0.0.0:10000
+}
+
+startObservers() {
+ local observerIdx=0
+ # Example for loop with injected variables in Bash
+ for ((i = 0; i < SHARDCOUNT; i++)); do
+   for ((j = 0; j < SHARD_OBSERVERCOUNT; j++)); do
+     # Your commands or code to be executed in each iteration
+     KEY_INDEX=$((TOTAL_NODECOUNT - observerIdx - 1))
+
+     docker run -d --name "observer${observerIdx}" \
+     -v $TESTNETDIR/node/config:/go/mx-chain-go/cmd/node/config \
+     node:dev \
+     --destination-shard-as-observer $i \
+     --rest-api-interface=0.0.0.0:10200 \
+     --config ./config/config_observer.toml \
+     --sk-index=${KEY_INDEX} \
+
+     ((observerIdx++)) || true
+   done
+ done
+
+ for ((i = 0; i < META_OBSERVERCOUNT; i++)); do
+    KEY_INDEX=$((TOTAL_NODECOUNT - observerIdx - 1))
+
+    docker run -d --name "observer${observerIdx}" \
+        -v $TESTNETDIR/node/config:/go/mx-chain-go/cmd/node/config \
+        node:dev \
+        --destination-shard-as-observer "metachain" \
+        --rest-api-interface=0.0.0.0:10200 \
+        --config ./config/config_observer.toml \
+        --sk-index=${KEY_INDEX} \
+
+    ((observerIdx++)) || true
+ done
+}
+
+startValidators() {
+ validatorIdx=0
+ # Example for loop with injected variables in Bash
+ for ((i = 0; i < SHARDCOUNT; i++)); do
+   for ((j = 0; j < SHARD_VALIDATORCOUNT; j++)); do
+
+     docker run -d --name "validator${validatorIdx}" \
+     -v $TESTNETDIR/node/config:/go/mx-chain-go/cmd/node/config \
+     node:dev \
+     --rest-api-interface=0.0.0.0:10200 \
+     --config ./config/config_validator.toml \
+     --sk-index=${validatorIdx} \
+
+     ((validatorIdx++)) || true
+   done
+ done
+
+  for ((i = 0; i < META_VALIDATORCOUNT; i++)); do
+     docker run -d --name "validator${validatorIdx}" \
+         -v $TESTNETDIR/node/config:/go/mx-chain-go/cmd/node/config \
+         node:dev \
+         --rest-api-interface=0.0.0.0:10200 \
+         --config ./config/config_observer.toml \
+         --sk-index=${validatorIdx} \
+
+     ((validatorIdx++)) || true
+  done
+}
+
+updateProxyConfigDocker() {
+    pushd $TESTNETDIR/proxy/config
+    cp config.toml config_edit.toml
+
+    # Truncate config.toml before the [[Observers]] list
+    sed -i -n '/\[\[Observers\]\]/q;p' config_edit.toml
+
+    if [ "$SHARD_OBSERVERCOUNT" -le 0 ]; then
+        generateProxyValidatorListDocker config_edit.toml
+    else
+        generateProxyObserverListDocker config_edit.toml
+    fi
+
+    cp config_edit.toml config.toml
+    rm config_edit.toml
+
+    echo "Updated configuration for the Proxy."
+    popd
+}
+
+generateProxyObserverListDocker() {
+  IP_BIT=3
+  OUTPUTFILE=$!
+
+
+  for ((i = 0; i < SHARDCOUNT; i++)); do
+     for ((j = 0; j < SHARD_OBSERVERCOUNT; j++)); do
+
+        echo "[[Observers]]" >> config_edit.toml
+        echo "   ShardId = $i" >> config_edit.toml
+        echo "   Address = \"http://172.17.0.${IP_BIT}:10200\"" >> config_edit.toml
+        echo ""$'\n' >> config_edit.toml
+
+        (( IP_BIT++ ))
+     done
+  done
+
+  for META_OBSERVER in $(seq $META_OBSERVERCOUNT); do
+        echo "[[Observers]]" >> config_edit.toml
+        echo "   ShardId = $METASHARD_ID" >> config_edit.toml
+        echo "   Address = \"http://172.17.0.${IP_BIT}:10200\"" >> config_edit.toml
+        echo ""$'\n' >> config_edit.toml
+
+        (( IP_BIT++ ))
+  done
+}
+
+generateProxyValidatorListDocker() {
+  IP_BIT=3
+  OUTPUTFILE=$!
+
+
+  for ((i = 0; i < SHARDCOUNT; i++)); do
+     for ((j = 0; j < SHARD_VALIDATORCOUNT; j++)); do
+
+        echo "[[Observers]]" >> config_edit.toml
+        echo "   ShardId = $i" >> config_edit.toml
+        echo "   Address = \"http://172.17.0.${IP_BIT}:10200\"" >> config_edit.toml
+        echo "   Type = \"Validator\"" >> config_edit.toml
+        echo ""$'\n' >> config_edit.toml
+
+        (( IP_BIT++ ))
+     done
+  done
+
+  for META_OBSERVER in $(seq $META_VALIDATORCOUNT); do
+        echo "[[Observers]]" >> config_edit.toml
+        echo "   ShardId = $METASHARD_ID" >> config_edit.toml
+        echo "   Address = \"http://172.17.0.${IP_BIT}:10200\"" >> config_edit.toml
+        echo "   Type = \"Validator\"" >> config_edit.toml
+        echo ""$'\n' >> config_edit.toml
+
+        (( IP_BIT++ ))
+  done
+}
+
+startProxyDocker() {
+  docker run -d --name "proxy" \
+           -p $PORT_PROXY:8080 \
+           -v $TESTNETDIR/proxy/config:/mx-chain-proxy-go/cmd/proxy/config \
+           multiversx/chain-proxy:v1.1.45-sp4
+}
diff --git a/scripts/docker-testnet/start.sh b/scripts/docker-testnet/start.sh
new file mode 100755
index 00000000000..0181ac3c3a9
--- /dev/null
+++ b/scripts/docker-testnet/start.sh
@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+
+set -e
+
+export DOCKERTESTNETDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+
+MULTIVERSXTESTNETSCRIPTSDIR="$(dirname "$DOCKERTESTNETDIR")/testnet"
+
+source "$DOCKERTESTNETDIR/variables.sh"
+source "$DOCKERTESTNETDIR/helpers.sh"
+source "$MULTIVERSXTESTNETSCRIPTSDIR/include/config.sh"
+source "$MULTIVERSXTESTNETSCRIPTSDIR/include/build.sh"
+
+prepareFolders
+
+buildConfigGenerator
+
+generateConfig
+
+copyConfig
+
+copySeednodeConfig
+updateSeednodeConfig
+
+copyNodeConfig
+updateNodeConfig
+
+startSeedNode
+startObservers
+startValidators
+
+if [ $USE_PROXY -eq 1 ]; then
+  prepareFolders_Proxy
+  copyProxyConfig
+  updateProxyConfigDocker
+  startProxyDocker
+fi
+
diff --git a/scripts/docker-testnet/variables.sh b/scripts/docker-testnet/variables.sh
new file mode 100644
index 00000000000..f4afd395c41
--- /dev/null
+++ b/scripts/docker-testnet/variables.sh
@@ -0,0 +1,199 @@
+# These paths must be absolute
+
+# METASHARD_ID will be used to identify a shard ID as metachain
+export METASHARD_ID=4294967295
+
+# Path to mx-chain-go. Determined automatically. Do not change.
+export MULTIVERSXDIR=$(dirname $(dirname $MULTIVERSXTESTNETSCRIPTSDIR))
+
+# Enable the MultiversX Proxy. Note that this is a private repository
+# (mx-chain-proxy-go).
+export USE_PROXY=1
+
+# Enable the MultiversX Transaction Generator. Note that this is a private
+# repository (mx-chain-txgen-go).
+export USE_TXGEN=0
+
+# Path where the testnet will be instantiated. This folder is assumed to not
+# exist, but it doesn't matter if it already does. It will be created if not,
+# anyway.
+export TESTNETDIR="$HOME/MultiversX/testnet"
+
+
+# Path to mx-chain-deploy-go, branch: master. Default: near mx-chain-go.
+
+if [[ -n $CI_RUN ]]; then
+  export CONFIGGENERATORDIR="$(dirname $MULTIVERSXDIR)/mx-chain-go/mx-chain-deploy-go/cmd/filegen"
+else
+  export CONFIGGENERATORDIR="$(dirname $MULTIVERSXDIR)/mx-chain-deploy-go/cmd/filegen"
+fi
+
+export CONFIGGENERATOR="$CONFIGGENERATORDIR/filegen"    # Leave unchanged.
+export CONFIGGENERATOROUTPUTDIR="output"
+
+# Path to the executable node. Leave unchanged unless well justified.
+export NODEDIR="$MULTIVERSXDIR/cmd/node"
+export NODE="$NODEDIR/node"     # Leave unchanged
+
+# Path to the executable seednode. Leave unchanged unless well justified.
+export SEEDNODEDIR="$MULTIVERSXDIR/cmd/seednode"
+export SEEDNODE="$SEEDNODEDIR/seednode"   # Leave unchanged.
+
+# Niceness value of the Seednode, Observer Nodes and Validator Nodes. Leave
+# blank to not adjust niceness.
+export NODE_NICENESS=10
+
+# Start a watcher daemon for each validator node, which restarts the node if it
+# is suffled out of its shard.
+export NODE_WATCHER=0
+
+# Delays after running executables.
+export SEEDNODE_DELAY=5
+export GENESIS_DELAY=30
+export HARDFORK_DELAY=900 #15 minutes enough to take export and gracefully close
+export NODE_DELAY=60
+
+export GENESIS_STAKE_TYPE="direct" #'delegated' or 'direct' as in direct stake
+
+#if set to 1, each observer will turn off the antiflooding capability, allowing spam in our network
+export OBSERVERS_ANTIFLOOD_DISABLE=0
+
+# Shard structure
+export SHARDCOUNT=2
+export SHARD_VALIDATORCOUNT=3
+export SHARD_OBSERVERCOUNT=1
+export SHARD_CONSENSUS_SIZE=3
+
+# Metashard structure
+export META_VALIDATORCOUNT=3
+export META_OBSERVERCOUNT=1
+export META_CONSENSUS_SIZE=$META_VALIDATORCOUNT
+
+# MULTI_KEY_NODES if set to 1, one observer will be generated on each shard that will handle all generated keys
+export MULTI_KEY_NODES=0
+
+# EXTRA_KEYS if set to 1, extra keys will be added to the generated keys
+export EXTRA_KEYS=1
+
+# ALWAYS_NEW_CHAINID will generate a fresh new chain ID each time start.sh/config.sh is called
+export ALWAYS_NEW_CHAINID=1
+
+# ROUNDS_PER_EPOCH represents the number of rounds per epoch. If set to 0, it won't override the node's config
+export ROUNDS_PER_EPOCH=0
+
+# HYSTERESIS defines the hysteresis value for number of nodes in shard
+export HYSTERESIS=0.0
+
+# ALWAYS_NEW_APP_VERSION will set a new version each time the node will be compiled
+export ALWAYS_NEW_APP_VERSION=0
+
+# ALWAYS_UPDATE_CONFIGS will re-generate configs (toml + json) each time ./start.sh
+# Set this variable to 0 when testing bootstrap from storage or other edge cases where you do not want a fresh new config
+# each time.
+export ALWAYS_UPDATE_CONFIGS=1
+
+# IP of the seednode
+export SEEDNODE_IP="172.17.0.2"
+
+# Ports used by the Nodes
+export PORT_SEEDNODE="9999"
+export PORT_ORIGIN_OBSERVER="21100"
+export PORT_ORIGIN_OBSERVER_REST="10000"
+export PORT_ORIGIN_VALIDATOR="21500"
+export PORT_ORIGIN_VALIDATOR_REST="9500"
+
+# UI configuration profiles
+
+# Use tmux or not. If set to 1, only 2 terminal windows will be opened, and
+# tmux will be used to display the running executables using split windows.
+# Recommended. Tmux needs to be installed.
+export USETMUX=1
+
+# Log level for the logger in the Node.
+export LOGLEVEL="*:INFO"
+
+
+if [ "$TESTNETMODE" == "debug" ]; then
+  LOGLEVEL="*:DEBUG,api:INFO"
+fi
+
+if [ "$TESTNETMODE" == "trace" ]; then
+  LOGLEVEL="*:TRACE"
+fi
+
+########################################################################
+# Proxy configuration
+
+# Path to mx-chain-proxy-go, branch: master. Default: near mx-chain-go.
+if [[ -n $CI_RUN ]]; then
+  export PROXYDIR="$(dirname $MULTIVERSXDIR)/mx-chain-go/mx-chain-proxy-go/cmd/proxy"
+else
+   export PROXYDIR="$(dirname $MULTIVERSXDIR)/mx-chain-proxy-go/cmd/proxy"
+fi
+export PROXY=$PROXYDIR/proxy    # Leave unchanged.
+
+export PORT_PROXY="7950"
+export PROXY_DELAY=10
+
+
+
+########################################################################
+# TxGen configuration
+
+# Path to mx-chain-txgen-go. Default: near mx-chain-go.
+export TXGENDIR="$(dirname $MULTIVERSXDIR)/mx-chain-txgen-go/cmd/txgen"
+export TXGEN=$TXGENDIR/txgen    # Leave unchanged.
+
+export PORT_TXGEN="7951"
+
+export TXGEN_SCENARIOS_LINE='Scenarios = ["basic", "erc20", "esdt"]'
+
+# Number of accounts to be generated by txgen
+export NUMACCOUNTS="250"
+
+# Whether txgen should regenerate its accounts when starting, or not.
+# Recommended value is 1, but 0 is useful to run the txgen a second time, to
+# continue a testing session on the same accounts.
+export TXGEN_REGENERATE_ACCOUNTS=0
+
+# COPY_BACK_CONFIGS when set to 1 will copy back the configs and keys to the ./cmd/node/config directory
+# in order to have a node in the IDE that can run a node in debug mode but in the same network with the rest of the nodes
+# this option greatly helps the debugging process when running a small system test
+export COPY_BACK_CONFIGS=0
+# SKIP_VALIDATOR_IDX when setting a value greater than -1 will not launch the validator with the provided index
+export SKIP_VALIDATOR_IDX=-1
+# SKIP_OBSERVER_IDX when setting a value greater than -1 will not launch the observer with the provided index
+export SKIP_OBSERVER_IDX=-1
+
+# USE_HARDFORK will prepare the nodes to run the hardfork process, if needed
+export USE_HARDFORK=1
+
+# Load local overrides, .gitignored
+LOCAL_OVERRIDES="$MULTIVERSXTESTNETSCRIPTSDIR/local.sh"
+if [ -f "$LOCAL_OVERRIDES" ]; then
+  source "$MULTIVERSXTESTNETSCRIPTSDIR/local.sh"
+fi
+
+# Leave unchanged.
+let "total_observer_count = $SHARD_OBSERVERCOUNT * $SHARDCOUNT + $META_OBSERVERCOUNT"
+export TOTAL_OBSERVERCOUNT=$total_observer_count
+
+# to enable the full archive feature on the observers, please use the --full-archive flag
+export EXTRA_OBSERVERS_FLAGS="-operation-mode db-lookup-extension"
+
+if [[ $MULTI_KEY_NODES -eq 1 ]]; then
+  EXTRA_OBSERVERS_FLAGS="--no-key"
+fi
+
+# Leave unchanged.
+let "total_node_count = $SHARD_VALIDATORCOUNT * $SHARDCOUNT + $META_VALIDATORCOUNT + $TOTAL_OBSERVERCOUNT"
+export TOTAL_NODECOUNT=$total_node_count
+
+# VALIDATOR_KEY_PEM_FILE is the pem file name when running single key mode, with all nodes' keys
+export VALIDATOR_KEY_PEM_FILE="validatorKey.pem"
+
+# MULTI_KEY_PEM_FILE is the pem file name when running multi key mode, with all managed
+export MULTI_KEY_PEM_FILE="allValidatorsKeys.pem"
+
+# EXTRA_KEY_PEM_FILE is the pem file name when running multi key mode, with all extra managed
+export EXTRA_KEY_PEM_FILE="extraValidatorsKeys.pem"