Merge pull request #146 from aws-samples/solana

Solana. New instance configs after testing and dashboard fixes

lib/solana/README.md

@@ -84,9 +84,9 @@ This is the Well-Architected checklist for Solana nodes implementation of the AW
 
 | Usage pattern  | Ideal configuration  | Primary option on AWS  | Data Transfer Estimates | Config reference |
 |---|---|---|---|---|
-| 1/ Consensus node | 32 vCPU, 256 GB RAM, Accounts volume: 1TB, 5K IOPS, 700 MB/s throughput, Data volume: 3TB, 10K IOPS, 700 MB/s throughput   | r6a.8xlarge, Accounts volume: EBS gp3 1TB, 5K IOPS, 700 MB/s throughput, Data volume: EBS gp3 10K IOPS, 700 MB/s throughput | Proportional to the amount at stake. Between 200TB to 400TB/month  | [.env-sample-consensus](./sample-configs/.env-sample-consensus) |
-| 2/ Base RPC node (no secondary indexes) | 32 vCPU, 256 GB RAM, Accounts volume: 1TB, 5K IOPS, 700 MB/s throughput, Data volume: 3TB, 12K IOPS, 700 MB/s throughput   | r6a.8xlarge, Accounts volume: EBS gp3 1TB, 5K IOPS, 700 MB/s throughput Data volume: EBS gp3 12K IOPS, 700 MB/s throughput | 150-200TB/month (no staking) | [.env-sample-baserpc](./sample-configs/.env-sample-baserpc) |
-| 3/ Extended RPC node (with all secondary indexes) | 64 vCPU, 1 TB RAM, Accounts volume: 1TB, 7K IOPS, 700 MB/s throughput, Data volume: 3TB, 16K IOPS, 700 MB/s throughput    | x2idn.16xlarge, Accounts: instance storage (ephemeral NVMe volumes) 1.9 TB, Data volume: 3TB, 12K IOPS, 700 MB/s throughput | 150-200TB/month (no staking) | [.env-sample-extendedrpc](./sample-configs/.env-sample-extendedrpc) |
+| 1/ Consensus node | 48 vCPU, 384 GiB RAM, Accounts volume: 500GiB, 7K IOPS, 700 MB/s throughput, Data volume: 2TB, 9K IOPS, 700 MB/s throughput   | r7a.12xlarge, Accounts volume: 500GiB, 7K IOPS, 700 MB/s throughput, Data volume: 2TB, 9K IOPS, 700 MB/s throughput | Proportional to the amount at stake. Between 200TB to 400TB/month  | [.env-sample-consensus](./sample-configs/.env-sample-consensus) |
+| 2/ Base RPC node (no secondary indexes) | 48 vCPU, 384 GiB RAM, Accounts volume: 500GiB, 7K IOPS, 700 MB/s throughput, Data volume: 2TB, 9K IOPS, 700 MB/s throughput   | r7a.12xlarge, Accounts volume: 500GiB, 7K IOPS, 700 MB/s throughput, Data volume: 2TB, 9K IOPS, 700 MB/s throughput | 150-200TB/month (no staking) | [.env-sample-baserpc](./sample-configs/.env-sample-baserpc) |
+| 3/ Extended RPC node (with all secondary indexes) | 96 vCPU, 768 GiB RAM, Accounts volume: 500GiB, 7K IOPS, 700 MB/s throughput, Data volume: 2TB, 9K IOPS, 700 MB/s throughput  | r7a.24xlarge, Accounts volume: 500GiB, 7K IOPS, 700 MB/s throughput, Data volume: 2TB, 9K IOPS, 700 MB/s throughput | 150-200TB/month (no staking) | [.env-sample-extendedrpc](./sample-configs/.env-sample-extendedrpc) |
 </details>
 
 ## Setup Instructions
@@ -248,7 +248,7 @@ export INSTANCE_ID=$(cat single-node-deploy.json | jq -r '..|.node-instance-id?
 echo "INSTANCE_ID=" $INSTANCE_ID
 aws ssm start-session --target $INSTANCE_ID --region $AWS_REGION
 sudo su bcuser
-sudo journalctl -o cat -fu sol
+sudo journalctl -o cat -fu node
 ```
 
 2. How to check the logs from the EC2 user-data script?
@@ -268,7 +268,7 @@ sudo cat /var/log/cloud-init-output.log
 export INSTANCE_ID=$(cat single-node-deploy.json | jq -r '..|.node-instance-id? | select(. != null)')
 echo "INSTANCE_ID=" $INSTANCE_ID
 aws ssm start-session --target $INSTANCE_ID --region $AWS_REGION
-sudo systemctl status sol
+sudo systemctl status node
 ```
 
 4. How to upload a secret to AWS Secrets Manager?
@@ -298,11 +298,11 @@ free -g
 - Option 2: Existing volume (using Data directory as example):
 
 ```bash
-sudo mkdir /var/solana/data/swapfile
-sudo dd if=/dev/zero of=/var/solana/data/swapfile bs=1MiB count=250KiB
-sudo chmod 0600 /var/solana/data/swapfile
-sudo mkswap /var/solana/data/swapfile
-sudo swapon /var/solana/data/swapfile
+sudo mkdir /data/solana/data/swapfile
+sudo dd if=/dev/zero of=/data/solana/data/swapfile bs=1MiB count=250KiB
+sudo chmod 0600 /data/solana/data/swapfile
+sudo mkswap /data/solana/data/swapfile
+sudo swapon /data/solana/data/swapfile
 free -g
 sudo sysctl vm.swappiness=10
 ```

lib/solana/lib/assets/setup-instance-store-volumes.sh

@@ -18,20 +18,20 @@ fi
 
 if [ -n "$DATA_VOLUME_ID" ]; then
   echo "If Data volume is mounted, dont do anything"
-  if [ $(df --output=target | grep -c "/var/solana/data") -lt 1 ]; then
+  if [ $(df --output=target | grep -c "/data/solana/data") -lt 1 ]; then
     echo "Checking fstab for Data volume"
 
     sudo mkfs.xfs -f $DATA_VOLUME_ID
     sleep 10
     DATA_VOLUME_UUID=$(lsblk -fn -o UUID  $DATA_VOLUME_ID)
-    DATA_VOLUME_FSTAB_CONF="UUID=$DATA_VOLUME_UUID /var/solana/data xfs defaults 0 2"
+    DATA_VOLUME_FSTAB_CONF="UUID=$DATA_VOLUME_UUID /data/solana/data xfs defaults 0 2"
     echo "DATA_VOLUME_ID="$DATA_VOLUME_ID
     echo "DATA_VOLUME_UUID="$DATA_VOLUME_UUID
     echo "DATA_VOLUME_FSTAB_CONF="$DATA_VOLUME_FSTAB_CONF
 
     # Check if data disc is already in fstab and replace the line if it is with the new disc UUID
     if [ $(grep -c "data" /etc/fstab) -gt 0 ]; then
-      SED_REPLACEMENT_STRING="$(grep -n "/var/solana/data" /etc/fstab | cut -d: -f1)s#.*#$DATA_VOLUME_FSTAB_CONF#"
+      SED_REPLACEMENT_STRING="$(grep -n "/data/solana/data" /etc/fstab | cut -d: -f1)s#.*#$DATA_VOLUME_FSTAB_CONF#"
       sudo cp /etc/fstab /etc/fstab.bak
       sudo sed -i "$SED_REPLACEMENT_STRING" /etc/fstab
     else
@@ -40,29 +40,29 @@ if [ -n "$DATA_VOLUME_ID" ]; then
 
     sudo mount -a
 
-    sudo mkdir /var/solana/data/ledger
-    sudo chown -R solana:solana /var/solana
+    sudo mkdir /data/solana/data/ledger
+    sudo chown -R solana:solana /data/solana
   else
     echo "Data volume is mounted, nothing changed"
   fi
 fi
 
 if [ -n "$ACCOUNTS_VOLUME_ID" ]; then
   echo "If Accounts volume is mounted, dont do anything"
-  if [ $(df --output=target | grep -c "/var/solana/accounts") -lt 1 ]; then
+  if [ $(df --output=target | grep -c "/data/solana/accounts") -lt 1 ]; then
     echo "Checking fstab for Accounts volume"
 
     sudo mkfs.xfs -f $ACCOUNTS_VOLUME_ID
     sleep 10
     ACCOUNTS_VOLUME_UUID=$(lsblk -fn -o UUID $ACCOUNTS_VOLUME_ID)
-    ACCOUNTS_VOLUME_FSTAB_CONF="UUID=$ACCOUNTS_VOLUME_UUID /var/solana/accounts xfs defaults 0 2"
+    ACCOUNTS_VOLUME_FSTAB_CONF="UUID=$ACCOUNTS_VOLUME_UUID /data/solana/accounts xfs defaults 0 2"
     echo "ACCOUNTS_VOLUME_ID="$ACCOUNTS_VOLUME_ID
     echo "ACCOUNTS_VOLUME_UUID="$ACCOUNTS_VOLUME_UUID
     echo "ACCOUNTS_VOLUME_FSTAB_CONF="$ACCOUNTS_VOLUME_FSTAB_CONF
 
     # Check if accounts disc is already in fstab and replace the line if it is with the new disc UUID
-    if [ $(grep -c "/var/solana/accounts" /etc/fstab) -gt 0 ]; then
-      SED_REPLACEMENT_STRING="$(grep -n "/var/solana/accounts" /etc/fstab | cut -d: -f1)s#.*#$ACCOUNTS_VOLUME_FSTAB_CONF#"
+    if [ $(grep -c "/data/solana/accounts" /etc/fstab) -gt 0 ]; then
+      SED_REPLACEMENT_STRING="$(grep -n "/data/solana/accounts" /etc/fstab | cut -d: -f1)s#.*#$ACCOUNTS_VOLUME_FSTAB_CONF#"
       sudo cp /etc/fstab /etc/fstab.bak
       sudo sed -i "$SED_REPLACEMENT_STRING" /etc/fstab
     else
@@ -71,7 +71,7 @@ if [ -n "$ACCOUNTS_VOLUME_ID" ]; then
 
     sudo mount -a
 
-    sudo chown -R solana:solana /var/solana
+    sudo chown -R solana:solana /data/solana
   else
     echo "Accounts volume is mounted, nothing changed"
   fi

lib/solana/lib/assets/solana/node-consensus-template.sh

@@ -3,7 +3,7 @@ set -o errexit
 set -o nounset
 set -o pipefail
 # Remove empty snapshots
-find "/var/solana/data/ledger" -name "snapshot-*" -size 0 -print -exec rm {} \; || true
+find "/data/solana/data/ledger" -name "snapshot-*" -size 0 -print -exec rm {} \; || true
 export RUST_LOG=error
 export RUST_BACKTRACE=full
 export SOLANA_METRICS_CONFIG=__SOLANA_METRICS_CONFIG__
@@ -15,7 +15,7 @@ TOKEN=$(curl -s -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-m
 export EC2_INTERNAL_IP=$(curl -H "X-aws-ec2-metadata-token: $TOKEN" -s http://169.254.169.254/latest/meta-data/local-ipv4)
 
 /home/solana/bin/solana-validator \
---ledger /var/solana/data/ledger \
+--ledger /data/solana/data/ledger \
 --identity /home/solana/config/validator-keypair.json \
 --vote-account /home/solana/config/vote-account-keypair.json \
 __KNOWN_VALIDATORS__ \
@@ -25,7 +25,8 @@ __ENTRY_POINTS__ \
 --private-rpc \
 --rpc-bind-address $EC2_INTERNAL_IP \
 --wal-recovery-mode skip_any_corrupted_record \
---init-complete-file /var/solana/data/init-completed \
+--init-complete-file /data/solana/data/init-completed \
 --limit-ledger-size \
---accounts /var/solana/accounts \
+--accounts /data/solana/accounts \
+--incremental-snapshot-interval-slots 0 \
 --log -

lib/solana/lib/assets/solana/node-heavy-rpc-template.sh

@@ -3,7 +3,7 @@ set -o errexit
 set -o nounset
 set -o pipefail
 # Remove empty snapshots
-find "/var/solana/data/ledger" -name "snapshot-*" -size 0 -print -exec rm {} \; || true
+find "/data/solana/data/ledger" -name "snapshot-*" -size 0 -print -exec rm {} \; || true
 export RUST_LOG=error
 export RUST_BACKTRACE=full
 export SOLANA_METRICS_CONFIG=__SOLANA_METRICS_CONFIG__
@@ -15,7 +15,7 @@ TOKEN=$(curl -s -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-m
 export EC2_INTERNAL_IP=$(curl -H "X-aws-ec2-metadata-token: $TOKEN" -s http://169.254.169.254/latest/meta-data/local-ipv4)
 
 /home/solana/bin/solana-validator \
---ledger /var/solana/data/ledger \
+--ledger /data/solana/data/ledger \
 --identity /home/solana/config/validator-keypair.json \
 __KNOWN_VALIDATORS__ \
 --expected-genesis-hash __EXPECTED_GENESIS_HASH__ \
@@ -30,14 +30,15 @@ __ENTRY_POINTS__ \
 --wal-recovery-mode skip_any_corrupted_record \
 --enable-rpc-transaction-history \
 --enable-cpi-and-log-storage \
---init-complete-file /var/solana/data/init-completed \
+--init-complete-file /data/solana/data/init-completed \
 --require-tower \
 --no-wait-for-vote-to-start-leader \
 --limit-ledger-size \
---accounts /var/solana/accounts \
+--accounts /data/solana/accounts \
 --account-index spl-token-owner \
 --account-index program-id \
 --account-index spl-token-mint \
 --account-index-exclude-key kinXdEcpDQeHPEuQnqmUgtYykqKGVFq6CeVX5iAHJq6 \
 --account-index-exclude-key TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA \
+--incremental-snapshot-interval-slots 0 \
 --log -

lib/solana/lib/assets/solana/node-light-rpc-template.sh

@@ -3,7 +3,7 @@ set -o errexit
 set -o nounset
 set -o pipefail
 # Remove empty snapshots
-find "/var/solana/data/ledger" -name "snapshot-*" -size 0 -print -exec rm {} \; || true
+find "/data/solana/data/ledger" -name "snapshot-*" -size 0 -print -exec rm {} \; || true
 export RUST_LOG=error
 export RUST_BACKTRACE=full
 export SOLANA_METRICS_CONFIG=__SOLANA_METRICS_CONFIG__
@@ -15,7 +15,7 @@ TOKEN=$(curl -s -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-m
 export EC2_INTERNAL_IP=$(curl -H "X-aws-ec2-metadata-token: $TOKEN" -s http://169.254.169.254/latest/meta-data/local-ipv4)
 
 /home/solana/bin/solana-validator \
---ledger /var/solana/data/ledger \
+--ledger /data/solana/data/ledger \
 --identity /home/solana/config/validator-keypair.json \
 __KNOWN_VALIDATORS__ \
 --expected-genesis-hash __EXPECTED_GENESIS_HASH__ \
@@ -30,9 +30,10 @@ __ENTRY_POINTS__ \
 --wal-recovery-mode skip_any_corrupted_record \
 --enable-rpc-transaction-history \
 --enable-cpi-and-log-storage \
---init-complete-file /var/solana/data/init-completed \
+--init-complete-file /data/solana/data/init-completed \
 --require-tower \
 --no-wait-for-vote-to-start-leader \
 --limit-ledger-size \
---accounts /var/solana/accounts \
+--accounts /data/solana/accounts \
+--incremental-snapshot-interval-slots 0 \
 --log -

lib/solana/lib/assets/sync-checker/syncchecker-solana.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-INIT_COMPLETED_FILE=/var/solana/data/init-completed
+INIT_COMPLETED_FILE=/data/solana/data/init-completed
 
 TOKEN=$(curl -s -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 21600")
 INSTANCE_ID=$(curl -H "X-aws-ec2-metadata-token: $TOKEN" -s http://169.254.169.254/latest/meta-data/instance-id)

lib/solana/lib/assets/systemd/node.service

@@ -0,0 +1,15 @@
+[Unit]
+Description=Solana Validator
+After=network.target
+StartLimitIntervalSec=0
+[Service]
+Type=simple
+Restart=always
+RestartSec=1
+User=solana
+LimitNOFILE=1000000
+LogRateLimitIntervalSec=0
+Environment="PATH=/bin:/usr/bin:/home/solana/bin"
+ExecStart=/home/solana/bin/validator.sh
+[Install]
+WantedBy=multi-user.target

lib/solana/lib/assets/systemd/sync-checker.service

@@ -0,0 +1,5 @@
+[Unit]
+Description="Sync checker for the tezos node"
+
+[Service]
+ExecStart=/opt/syncchecker.sh

lib/solana/lib/assets/systemd/sync-checker.timer

@@ -0,0 +1,9 @@
+[Unit]
+Description="Run Sync checker service every 5 min"
+
+[Timer]
+OnCalendar=*:*:0/5
+Unit=sync-checker.service
+
+[Install]
+WantedBy=multi-user.target