Merge #25

25: OP-1964: Adding network_configs to change download location source for protocol versions. r=sacherjj a=sacherjj

Using network_configs folder in debian for allowing multiple network names and source URLs.  This will allow others to submit PRs to allow using the launcher package to pull configs and binaries from other networks.

Also updated config_from_example.sh to allow a second argument to specify IP for NAT installs or when external auto-detection fails.

Added check that these are run as `casper` user to eliminate permissions issues when this is not done. 

Small fix from clippy update.

Co-authored-by: Joe Sacher <[email protected]>

Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "casper-node-launcher"
-version = "0.3.2"
-authors = ["Fraser Hutchison <[email protected]>"]
+version = "0.3.3"
+authors = ["Fraser Hutchison <[email protected]>", "Joe Sacher <[email protected]>"]
 edition = "2018"
 description = "A binary which runs and upgrades the casper-node of the Casper network"
 readme = "README.md"
@@ -36,6 +36,7 @@ assets = [
     ["./resources/BIN_README.md", "/var/lib/casper/bin/README.md", "755"],
     ["./resources/maintainer_scripts/logrotate.d/casper-node", "/etc/logrotate.d/casper-node", "644"],
     ["./resources/maintainer_scripts/pull_casper_node_version.sh", "/etc/casper/pull_casper_node_version.sh", "755"],
+    ["./resources/maintainer_scripts/network_configs/*", "/etc/casper/network_configs/", "644"],
     ["./resources/maintainer_scripts/delete_local_db.sh", "/etc/casper/delete_local_db.sh", "755"],
     ["./resources/maintainer_scripts/config_from_example.sh", "/etc/casper/config_from_example.sh", "755"],
     ["./resources/DEB_README.md", "/etc/casper/README.md", "644"],

resources/maintainer_scripts/config_from_example.sh

@@ -3,12 +3,24 @@ set -e
 
 # This script will generate a CONFIG file appropriate to installation machine.
 
+USERNAME=casper
+
+if [ "$(whoami)" != "$USERNAME" ]; then
+  echo
+  echo "Script must be run as user: $USERNAME"
+  echo "Do this with 'sudo -u $USERNAME $0 <protocol_version> [optional external IP]'"
+  echo
+  exit 1
+fi
+
 if [ -z "$1" ]; then
+  echo
   echo "Error: version argument missing."
   echo "config-example.toml should exist in a given /etc/casper/[version] folder."
   echo ""
   echo "Ex: for version 1.0.1 of casper-node, /etc/casper/1_0_1/config-example.toml should exist."
   echo "    Should be called with '${0} 1_0_1'"
+  echo
   exit 1
 fi
 
@@ -18,46 +30,76 @@ CONFIG_EXAMPLE="$CONFIG_PATH/config-example.toml"
 CONFIG_NEW="$CONFIG_PATH/config.toml.new"
 
 if [ ! -f "$CONFIG_EXAMPLE" ]; then
+  echo
   echo "Error: $CONFIG_EXAMPLE not found."
+  echo
   exit 2
 fi
 
-IPv4_STRING='(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)'
-
-re='^(0*(1?[0-9]{1,2}|2([0-4][0-9]|5[0-5]))\.){3}'
-
-function curl_ext_ip()
+function valid_ip()
 {
-  result=$(curl -s --max-time 10 --connect-timeout 10 "$1") || result='dead pipe'
+  local  ip=$1
+  local  stat=1
+
+  if [[ $ip =~ ^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$ ]]; then
+    OIFS=$IFS
+    IFS='.'
+    ip=($ip)
+    IFS=$OIFS
+    [[ ${ip[0]} -le 255 && ${ip[1]} -le 255 \
+        && ${ip[2]} -le 255 && ${ip[3]} -le 255 ]]
+    stat=$?
+  fi
+  return $stat
 }
 
-URLS=("https://checkip.amazonaws.com" "https://ifconfig.me" "https://ident.me")
-NAMES=("amazonaws.com" "ifconfig.me" "ident.me")
-RESULTS=()
-array_len=${#URLS[@]}
+if [ -z "$2" ]; then
+  # IP to be detected
+  function curl_ext_ip()
+  {
+    result=$(curl -s --max-time 10 --connect-timeout 10 "$1") || result='dead pipe'
+  }
+
+  URLS=("https://checkip.amazonaws.com" "https://ifconfig.me" "https://ident.me")
+  NAMES=("amazonaws.com" "ifconfig.me" "ident.me")
+  RESULTS=()
+  array_len=${#URLS[@]}
+
+  echo "Trying to get external IP from couple of services ..."
+
+  for (( i=0; i<$array_len; i++ )); do
+    curl_ext_ip "${URLS[$i]}"
+    if [[ $result != "dead pipe" ]]; then
+      RESULTS+=($result)
+    fi
+    echo "${NAMES[$i]} report: $result"
+  done
 
-echo && echo -e "Trying to get external IP from couple of services ..."
+  IPv4_STRING='(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)'
+  EXTERNAL_IP=$(echo "${RESULTS[@]}" | awk '{for(i=1;i<=NF;i++) print $i}' | awk '!x[$0]++' | grep -E -o "$IPv4_STRING" | head -n 1)
 
-for (( i=0; i<$array_len; i++ )); do
-  curl_ext_ip "${URLS[$i]}"
-  if [[ $result != "dead pipe" ]]; then
-    RESULTS+=($result)
+  if valid_ip $EXTERNAL_IP; then
+   echo "Using External IP: $EXTERNAL_IP"
+  else
+   echo
+   echo "WARNING: Can't get external VPS IP automatically."
+   echo "Run script again with '$0 $1 <external ip address>'"
+   echo
+   exit 3
   fi
-  echo -e "${NAMES[$i]} report: $result"
-done
-
-EXTERNAL_IP=$(echo "${RESULTS[@]}" | awk '{for(i=1;i<=NF;i++) print $i}' | awk '!x[$0]++' | grep -E -o "$IPv4_STRING" | head -n 1)
-
-if ! [[ $EXTERNAL_IP =~ $re ]]; then
- echo -e
- echo -e "WARNING: Can't get external VPS IP automatically."
- echo -e "Please manually create $CONFIG from $CONFIG_EXAMPLE"
- echo -e "by replacing <IP_ADDRESS> with your external IP after installation."
- echo
- sleep 2
- exit 0
+
 else
- echo && echo -e "Using External IP: $EXTERNAL_IP" && echo
+  # IP passed into script
+  EXTERNAL_IP=$2
+  if valid_ip $EXTERNAL_IP; then
+    echo "Using provided IP: $EXTERNAL_IP"
+  else
+    echo
+    echo "Error: Provided IP: $EXTERNAL_IP is invalid. Expected IPv4 address."
+    echo
+    exit 4
+  fi
+
 fi
 
 OUTFILE=$CONFIG

resources/maintainer_scripts/network_configs/README.md

@@ -0,0 +1,64 @@
+# Network Configurations
+
+Network configurations should be of the format:
+```
+SOURCE_URL=<url for packages>
+NETWORK_NAME=<name of network>
+```
+It is recommended that network_name used is the same as the <network_name>.conf.  This will be executed as 
+`source "$DIR/network_configs/<config_file>.conf` to load these variables.
+
+## Usage 
+
+These configurations will be sent to `pull_casper_node_version.sh` as an argument.
+
+The target URL is expected to serve HTTP access to `<url>/<network_name>/<protocol_version>/[bin.tar.gz|config.tar.gz]`
+
+Example:
+`sudo -u casper pull_casper_node_version.sh casper.conf 1_0_0`
+
+With `casper.conf` of:
+```
+SOURCE_URL=genesis.casperlabs.io
+NETWORK_NAME=casper
+```
+
+Will perform:
+```
+curl -JLO genesis.casperlabs.io/casper/1_0_0/bin.tar.gz
+curl -JLO genesis.casperlabs.io/casper/1_0_0/config.tar.gz
+```
+
+`config.tar.gz` is decompressed into `/etc/casper/<protocol_version>`.
+`bin.tar.gz` is decompressed into `/var/lib/casper/<protocol_version>`.
+
+The script will error if protocol versions already exist.
+
+## Packaging
+
+With merges to `master`, `release-*` and `dev` branches in the `casper-node` repo, the artifacts are created in
+`genesis.casperlabs.io/drone/<git_hash>/<protocol version>/[bin.tar.gz|config.tar.gz]`.
+
+You may also pull down the artifacts for a given network and modify to stage a new network.  
+If you want to launch a network with the same software version of `casper`, you could pull down the `bin.tar.gz` and 
+use as is.  However, your `config.tar.gz` would need modified.
+
+```
+mkdir config
+curl -JLO genesis.casperlabs.io/casper/1_0_0/bin.tar.gz
+curl -JLO genesis.casperlabs.io/casper/1_0_0/config.tar.gz
+mv config.tar.gz config_old.tar.gz
+cd config
+tar -xzvf ../config_old.tar.gz
+```
+
+You would need to customize `chainspec.toml` with a new network name and activation_point timestamp.
+You would need to customize `config-example.toml` with new known_addresses.
+
+Once all the configuration changes are done.  Create a new config.tar.gz from within the config directory.
+
+```
+tar -czvf ../config.tar.gz .
+```
+
+Now upload `bin.tar.gz` and `config.tar.gz` to your `<url>/<network>/<protocol_version>` location.

resources/maintainer_scripts/network_configs/casper.conf

@@ -0,0 +1,2 @@
+SOURCE_URL=genesis.casperlabs.io
+NETWORK_NAME=casper

resources/maintainer_scripts/pull_casper_node_version.sh

@@ -3,40 +3,89 @@
 # This script will pull casper-node software and associated files required to run or upgrade
 # casper-node.
 
+USERNAME=casper
+ARGUMENT_EXAMPLES="<config filename in network_configs dir> <protocol version>"
+
+if [ "$(whoami)" != "$USERNAME" ]; then
+  echo
+  echo "Script must be run as user: $USERNAME"
+  echo "Do this with 'sudo -u $USERNAME $0' $ARGUMENT_EXAMPLES"
+  echo
+  exit 1
+fi
 
 if [ -z "$1" ]; then
+  echo
   echo "Error: arguments missing"
-  echo "Expected $0 <Semantic Version> <Network Name>"
-  echo "Example: $0 1_0_0 mainnet"
-  exit 1
+  echo "Expected $0 $ARGUMENT_EXAMPLES"
+  echo "Example: $0 casper.conf 1_0_0"
+  echo
+  exit 2
 fi
 
 if [ -z "$2" ]; then
+  echo
   echo "Error: arguments missing"
-  echo "Expected $0 <Semantic Version> <Network Name>"
-  echo "Example: $0 1_0_0 mainnet"
-  exit 10
+  echo "Expected $0 $ARGUMENT_EXAMPLES"
+  echo "Example: $0 casper.conf 1_0_0"
+  echo
+  exit 3
+fi
+
+SEMVER=$2
+if [[ ! $SEMVER =~ ^[0-9]+_[0-9]+_[0-9]+ ]]; then
+  echo
+  echo "Error: Illegal semver format. Please use <major>_<minor>_<patch> such as 1_0_0."
+  echo
+  exit 4
+fi
+CONFIG="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )/network_configs/$1"
+
+if [ ! -f "$CONFIG" ]; then
+  echo
+  echo "Config file given: $CONFIG does not exist."
+  echo
+  exit 5
+fi
+
+# This should set SOURCE_URL and NETWORK_NAME vars
+source "$CONFIG"
+
+if [ "$SOURCE_URL" == "" ]; then
+  echo
+  echo "Error: source_url not set and expected from '$CONFIG'."
+  echo
+  exit 6
+fi
+
+if [ "$NETWORK_NAME" == "" ]; then
+  echo
+  echo "Error: network_name not set and expected from '$CONFIG'."
+  echo
+  exit 7
 fi
 
-SEMVER=$1
-NETWORK=$2
 ETC_PATH="/etc/casper"
 BIN_PATH="/var/lib/casper/bin"
 
 if [ ! -d "$ETC_PATH" ]; then
+  echo
   echo "Error: expected config file location $ETC_PATH not found."
-  exit 2
+  echo
+  exit 8
 fi
 
 if [ ! -d "$BIN_PATH" ]; then
+  echo
   echo "Error: expected bin file location $BIN_PATH not found."
-  exit 3
+  echo
+  exit 9
 fi
 
 ETC_FULL_PATH="$ETC_PATH/$SEMVER"
 BIN_FULL_PATH="$BIN_PATH/$SEMVER"
 
-BASE_URL="http://genesis.casperlabs.io/$NETWORK/$SEMVER"
+BASE_URL="http://$SOURCE_URL/$NETWORK_NAME/$SEMVER"
 CONFIG_ARCHIVE="config.tar.gz"
 CONFIG_URL="$BASE_URL/$CONFIG_ARCHIVE"
 BIN_ARCHIVE="bin.tar.gz"
@@ -47,23 +96,31 @@ cd $ETC_PATH
 echo "Verifying semver Path"
 curl -I 2>/dev/null "$CONFIG_URL" | head -1 | grep 404 >/dev/null
 if [ $? == 0 ]; then
+  echo
   echo "$CONFIG_URL not found.  Please verify provided arguments"
-  exit 4
+  echo
+  exit 10
 fi
 curl -I 2>/dev/null "$BIN_URL" | head -1 | grep 404 >/dev/null
 if [ $? == 0 ]; then
+  echo
   echo "$BIN_URL not found.  Please verify provided arguments"
-  exit 5
+  echo
+  exit 11
 fi
 
 if [ -d "$ETC_FULL_PATH" ]; then
+  echo
   echo "Error: config version path $ETC_FULL_PATH already exists. Aborting."
-  exit 6
+  echo
+  exit 12
 fi
 
 if [ -d "$BIN_FULL_PATH" ]; then
+  echo
   echo "Error: bin version path $BIN_FULL_PATH already exists. Aborting."
-  exit 7
+  echo
+  exit 13
 fi
 
 echo "Downloading $CONFIG_ARCHIVE from $CONFIG_URL"
@@ -72,7 +129,7 @@ if curl -JLO "$CONFIG_URL"; then
 else
   echo "Error: unable to pull $CONFIG_ARCHIVE from $CONFIG_URL."
   echo "File probably doesn't exist.  Please verify provided arguments"
-  exit 8
+  exit 14
 fi
 CONFIG_ARCHIVE_PATH="$ETC_PATH/$CONFIG_ARCHIVE"
 
@@ -82,7 +139,7 @@ if curl -JLO "$BIN_URL"; then
 else
   echo "Error: unable to pull $BIN_ARCHIVE from $BIN_URL"
   echo "File probably doesn't exist.  Please verify provided arguments"
-  exit 9
+  exit 15
 fi
 BIN_ARCHIVE_PATH="$ETC_PATH/$BIN_ARCHIVE"
 

src/logging.rs

@@ -14,10 +14,10 @@ pub fn init() -> Result<()> {
             .unwrap_or(DEFAULT_LOG_LEVEL),
     );
 
-    Ok(tracing_subscriber::fmt()
+    tracing_subscriber::fmt()
         .with_writer(io::stdout)
         .with_env_filter(filter)
         .json()
         .try_init()
-        .map_err(Error::msg)?)
+        .map_err(Error::msg)
 }