diff --git a/lib/solana/README.md b/lib/solana/README.md index 6f7fb52b..35d1d39c 100644 --- a/lib/solana/README.md +++ b/lib/solana/README.md @@ -84,9 +84,9 @@ This is the Well-Architected checklist for Solana nodes implementation of the AW | Usage pattern | Ideal configuration | Primary option on AWS | Data Transfer Estimates | Config reference | |---|---|---|---|---| -| 1/ Consensus node | 32 vCPU, 256 GB RAM, Accounts volume: 1TB, 5K IOPS, 700 MB/s throughput, Data volume: 3TB, 10K IOPS, 700 MB/s throughput | r6a.8xlarge, Accounts volume: EBS gp3 1TB, 5K IOPS, 700 MB/s throughput, Data volume: EBS gp3 10K IOPS, 700 MB/s throughput | Proportional to the amount at stake. Between 200TB to 400TB/month | [.env-sample-consensus](./sample-configs/.env-sample-consensus) | -| 2/ Base RPC node (no secondary indexes) | 32 vCPU, 256 GB RAM, Accounts volume: 1TB, 5K IOPS, 700 MB/s throughput, Data volume: 3TB, 12K IOPS, 700 MB/s throughput | r6a.8xlarge, Accounts volume: EBS gp3 1TB, 5K IOPS, 700 MB/s throughput Data volume: EBS gp3 12K IOPS, 700 MB/s throughput | 150-200TB/month (no staking) | [.env-sample-baserpc](./sample-configs/.env-sample-baserpc) | -| 3/ Extended RPC node (with all secondary indexes) | 64 vCPU, 1 TB RAM, Accounts volume: 1TB, 7K IOPS, 700 MB/s throughput, Data volume: 3TB, 16K IOPS, 700 MB/s throughput | x2idn.16xlarge, Accounts: instance storage (ephemeral NVMe volumes) 1.9 TB, Data volume: 3TB, 12K IOPS, 700 MB/s throughput | 150-200TB/month (no staking) | [.env-sample-extendedrpc](./sample-configs/.env-sample-extendedrpc) | +| 1/ Consensus node | 48 vCPU, 384 GiB RAM, Accounts volume: 500GiB, 7K IOPS, 700 MB/s throughput, Data volume: 2TB, 9K IOPS, 700 MB/s throughput | r7a.12xlarge, Accounts volume: 500GiB, 7K IOPS, 700 MB/s throughput, Data volume: 2TB, 9K IOPS, 700 MB/s throughput | Proportional to the amount at stake. Between 200TB to 400TB/month | [.env-sample-consensus](./sample-configs/.env-sample-consensus) | +| 2/ Base RPC node (no secondary indexes) | 48 vCPU, 384 GiB RAM, Accounts volume: 500GiB, 7K IOPS, 700 MB/s throughput, Data volume: 2TB, 9K IOPS, 700 MB/s throughput | r7a.12xlarge, Accounts volume: 500GiB, 7K IOPS, 700 MB/s throughput, Data volume: 2TB, 9K IOPS, 700 MB/s throughput | 150-200TB/month (no staking) | [.env-sample-baserpc](./sample-configs/.env-sample-baserpc) | +| 3/ Extended RPC node (with all secondary indexes) | 96 vCPU, 768 GiB RAM, Accounts volume: 500GiB, 7K IOPS, 700 MB/s throughput, Data volume: 2TB, 9K IOPS, 700 MB/s throughput | r7a.24xlarge, Accounts volume: 500GiB, 7K IOPS, 700 MB/s throughput, Data volume: 2TB, 9K IOPS, 700 MB/s throughput | 150-200TB/month (no staking) | [.env-sample-extendedrpc](./sample-configs/.env-sample-extendedrpc) | ## Setup Instructions @@ -248,7 +248,7 @@ export INSTANCE_ID=$(cat single-node-deploy.json | jq -r '..|.node-instance-id? echo "INSTANCE_ID=" $INSTANCE_ID aws ssm start-session --target $INSTANCE_ID --region $AWS_REGION sudo su bcuser -sudo journalctl -o cat -fu sol +sudo journalctl -o cat -fu node ``` 2. How to check the logs from the EC2 user-data script? @@ -268,7 +268,7 @@ sudo cat /var/log/cloud-init-output.log export INSTANCE_ID=$(cat single-node-deploy.json | jq -r '..|.node-instance-id? | select(. != null)') echo "INSTANCE_ID=" $INSTANCE_ID aws ssm start-session --target $INSTANCE_ID --region $AWS_REGION -sudo systemctl status sol +sudo systemctl status node ``` 4. How to upload a secret to AWS Secrets Manager? @@ -298,11 +298,11 @@ free -g - Option 2: Existing volume (using Data directory as example): ```bash -sudo mkdir /var/solana/data/swapfile -sudo dd if=/dev/zero of=/var/solana/data/swapfile bs=1MiB count=250KiB -sudo chmod 0600 /var/solana/data/swapfile -sudo mkswap /var/solana/data/swapfile -sudo swapon /var/solana/data/swapfile +sudo mkdir /data/solana/data/swapfile +sudo dd if=/dev/zero of=/data/solana/data/swapfile bs=1MiB count=250KiB +sudo chmod 0600 /data/solana/data/swapfile +sudo mkswap /data/solana/data/swapfile +sudo swapon /data/solana/data/swapfile free -g sudo sysctl vm.swappiness=10 ``` diff --git a/lib/solana/lib/assets/setup-instance-store-volumes.sh b/lib/solana/lib/assets/setup-instance-store-volumes.sh index 2ed0f85d..8b40e912 100644 --- a/lib/solana/lib/assets/setup-instance-store-volumes.sh +++ b/lib/solana/lib/assets/setup-instance-store-volumes.sh @@ -18,20 +18,20 @@ fi if [ -n "$DATA_VOLUME_ID" ]; then echo "If Data volume is mounted, dont do anything" - if [ $(df --output=target | grep -c "/var/solana/data") -lt 1 ]; then + if [ $(df --output=target | grep -c "/data/solana/data") -lt 1 ]; then echo "Checking fstab for Data volume" sudo mkfs.xfs -f $DATA_VOLUME_ID sleep 10 DATA_VOLUME_UUID=$(lsblk -fn -o UUID $DATA_VOLUME_ID) - DATA_VOLUME_FSTAB_CONF="UUID=$DATA_VOLUME_UUID /var/solana/data xfs defaults 0 2" + DATA_VOLUME_FSTAB_CONF="UUID=$DATA_VOLUME_UUID /data/solana/data xfs defaults 0 2" echo "DATA_VOLUME_ID="$DATA_VOLUME_ID echo "DATA_VOLUME_UUID="$DATA_VOLUME_UUID echo "DATA_VOLUME_FSTAB_CONF="$DATA_VOLUME_FSTAB_CONF # Check if data disc is already in fstab and replace the line if it is with the new disc UUID if [ $(grep -c "data" /etc/fstab) -gt 0 ]; then - SED_REPLACEMENT_STRING="$(grep -n "/var/solana/data" /etc/fstab | cut -d: -f1)s#.*#$DATA_VOLUME_FSTAB_CONF#" + SED_REPLACEMENT_STRING="$(grep -n "/data/solana/data" /etc/fstab | cut -d: -f1)s#.*#$DATA_VOLUME_FSTAB_CONF#" sudo cp /etc/fstab /etc/fstab.bak sudo sed -i "$SED_REPLACEMENT_STRING" /etc/fstab else @@ -40,8 +40,8 @@ if [ -n "$DATA_VOLUME_ID" ]; then sudo mount -a - sudo mkdir /var/solana/data/ledger - sudo chown -R solana:solana /var/solana + sudo mkdir /data/solana/data/ledger + sudo chown -R solana:solana /data/solana else echo "Data volume is mounted, nothing changed" fi @@ -49,20 +49,20 @@ fi if [ -n "$ACCOUNTS_VOLUME_ID" ]; then echo "If Accounts volume is mounted, dont do anything" - if [ $(df --output=target | grep -c "/var/solana/accounts") -lt 1 ]; then + if [ $(df --output=target | grep -c "/data/solana/accounts") -lt 1 ]; then echo "Checking fstab for Accounts volume" sudo mkfs.xfs -f $ACCOUNTS_VOLUME_ID sleep 10 ACCOUNTS_VOLUME_UUID=$(lsblk -fn -o UUID $ACCOUNTS_VOLUME_ID) - ACCOUNTS_VOLUME_FSTAB_CONF="UUID=$ACCOUNTS_VOLUME_UUID /var/solana/accounts xfs defaults 0 2" + ACCOUNTS_VOLUME_FSTAB_CONF="UUID=$ACCOUNTS_VOLUME_UUID /data/solana/accounts xfs defaults 0 2" echo "ACCOUNTS_VOLUME_ID="$ACCOUNTS_VOLUME_ID echo "ACCOUNTS_VOLUME_UUID="$ACCOUNTS_VOLUME_UUID echo "ACCOUNTS_VOLUME_FSTAB_CONF="$ACCOUNTS_VOLUME_FSTAB_CONF # Check if accounts disc is already in fstab and replace the line if it is with the new disc UUID - if [ $(grep -c "/var/solana/accounts" /etc/fstab) -gt 0 ]; then - SED_REPLACEMENT_STRING="$(grep -n "/var/solana/accounts" /etc/fstab | cut -d: -f1)s#.*#$ACCOUNTS_VOLUME_FSTAB_CONF#" + if [ $(grep -c "/data/solana/accounts" /etc/fstab) -gt 0 ]; then + SED_REPLACEMENT_STRING="$(grep -n "/data/solana/accounts" /etc/fstab | cut -d: -f1)s#.*#$ACCOUNTS_VOLUME_FSTAB_CONF#" sudo cp /etc/fstab /etc/fstab.bak sudo sed -i "$SED_REPLACEMENT_STRING" /etc/fstab else @@ -71,7 +71,7 @@ if [ -n "$ACCOUNTS_VOLUME_ID" ]; then sudo mount -a - sudo chown -R solana:solana /var/solana + sudo chown -R solana:solana /data/solana else echo "Accounts volume is mounted, nothing changed" fi diff --git a/lib/solana/lib/assets/solana/node-consensus-template.sh b/lib/solana/lib/assets/solana/node-consensus-template.sh index 8d8b0383..d186a0a3 100644 --- a/lib/solana/lib/assets/solana/node-consensus-template.sh +++ b/lib/solana/lib/assets/solana/node-consensus-template.sh @@ -3,7 +3,7 @@ set -o errexit set -o nounset set -o pipefail # Remove empty snapshots -find "/var/solana/data/ledger" -name "snapshot-*" -size 0 -print -exec rm {} \; || true +find "/data/solana/data/ledger" -name "snapshot-*" -size 0 -print -exec rm {} \; || true export RUST_LOG=error export RUST_BACKTRACE=full export SOLANA_METRICS_CONFIG=__SOLANA_METRICS_CONFIG__ @@ -15,7 +15,7 @@ TOKEN=$(curl -s -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-m export EC2_INTERNAL_IP=$(curl -H "X-aws-ec2-metadata-token: $TOKEN" -s http://169.254.169.254/latest/meta-data/local-ipv4) /home/solana/bin/solana-validator \ ---ledger /var/solana/data/ledger \ +--ledger /data/solana/data/ledger \ --identity /home/solana/config/validator-keypair.json \ --vote-account /home/solana/config/vote-account-keypair.json \ __KNOWN_VALIDATORS__ \ @@ -25,7 +25,8 @@ __ENTRY_POINTS__ \ --private-rpc \ --rpc-bind-address $EC2_INTERNAL_IP \ --wal-recovery-mode skip_any_corrupted_record \ ---init-complete-file /var/solana/data/init-completed \ +--init-complete-file /data/solana/data/init-completed \ --limit-ledger-size \ ---accounts /var/solana/accounts \ +--accounts /data/solana/accounts \ +--incremental-snapshot-interval-slots 0 \ --log - diff --git a/lib/solana/lib/assets/solana/node-heavy-rpc-template.sh b/lib/solana/lib/assets/solana/node-heavy-rpc-template.sh index b7b5e670..7129404f 100644 --- a/lib/solana/lib/assets/solana/node-heavy-rpc-template.sh +++ b/lib/solana/lib/assets/solana/node-heavy-rpc-template.sh @@ -3,7 +3,7 @@ set -o errexit set -o nounset set -o pipefail # Remove empty snapshots -find "/var/solana/data/ledger" -name "snapshot-*" -size 0 -print -exec rm {} \; || true +find "/data/solana/data/ledger" -name "snapshot-*" -size 0 -print -exec rm {} \; || true export RUST_LOG=error export RUST_BACKTRACE=full export SOLANA_METRICS_CONFIG=__SOLANA_METRICS_CONFIG__ @@ -15,7 +15,7 @@ TOKEN=$(curl -s -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-m export EC2_INTERNAL_IP=$(curl -H "X-aws-ec2-metadata-token: $TOKEN" -s http://169.254.169.254/latest/meta-data/local-ipv4) /home/solana/bin/solana-validator \ ---ledger /var/solana/data/ledger \ +--ledger /data/solana/data/ledger \ --identity /home/solana/config/validator-keypair.json \ __KNOWN_VALIDATORS__ \ --expected-genesis-hash __EXPECTED_GENESIS_HASH__ \ @@ -30,14 +30,15 @@ __ENTRY_POINTS__ \ --wal-recovery-mode skip_any_corrupted_record \ --enable-rpc-transaction-history \ --enable-cpi-and-log-storage \ ---init-complete-file /var/solana/data/init-completed \ +--init-complete-file /data/solana/data/init-completed \ --require-tower \ --no-wait-for-vote-to-start-leader \ --limit-ledger-size \ ---accounts /var/solana/accounts \ +--accounts /data/solana/accounts \ --account-index spl-token-owner \ --account-index program-id \ --account-index spl-token-mint \ --account-index-exclude-key kinXdEcpDQeHPEuQnqmUgtYykqKGVFq6CeVX5iAHJq6 \ --account-index-exclude-key TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA \ +--incremental-snapshot-interval-slots 0 \ --log - diff --git a/lib/solana/lib/assets/solana/node-light-rpc-template.sh b/lib/solana/lib/assets/solana/node-light-rpc-template.sh index 8b479dd5..d635affe 100644 --- a/lib/solana/lib/assets/solana/node-light-rpc-template.sh +++ b/lib/solana/lib/assets/solana/node-light-rpc-template.sh @@ -3,7 +3,7 @@ set -o errexit set -o nounset set -o pipefail # Remove empty snapshots -find "/var/solana/data/ledger" -name "snapshot-*" -size 0 -print -exec rm {} \; || true +find "/data/solana/data/ledger" -name "snapshot-*" -size 0 -print -exec rm {} \; || true export RUST_LOG=error export RUST_BACKTRACE=full export SOLANA_METRICS_CONFIG=__SOLANA_METRICS_CONFIG__ @@ -15,7 +15,7 @@ TOKEN=$(curl -s -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-m export EC2_INTERNAL_IP=$(curl -H "X-aws-ec2-metadata-token: $TOKEN" -s http://169.254.169.254/latest/meta-data/local-ipv4) /home/solana/bin/solana-validator \ ---ledger /var/solana/data/ledger \ +--ledger /data/solana/data/ledger \ --identity /home/solana/config/validator-keypair.json \ __KNOWN_VALIDATORS__ \ --expected-genesis-hash __EXPECTED_GENESIS_HASH__ \ @@ -30,9 +30,10 @@ __ENTRY_POINTS__ \ --wal-recovery-mode skip_any_corrupted_record \ --enable-rpc-transaction-history \ --enable-cpi-and-log-storage \ ---init-complete-file /var/solana/data/init-completed \ +--init-complete-file /data/solana/data/init-completed \ --require-tower \ --no-wait-for-vote-to-start-leader \ --limit-ledger-size \ ---accounts /var/solana/accounts \ +--accounts /data/solana/accounts \ +--incremental-snapshot-interval-slots 0 \ --log - diff --git a/lib/solana/lib/assets/sync-checker/syncchecker-solana.sh b/lib/solana/lib/assets/sync-checker/syncchecker-solana.sh index 2bb0ee1a..b5bf081a 100644 --- a/lib/solana/lib/assets/sync-checker/syncchecker-solana.sh +++ b/lib/solana/lib/assets/sync-checker/syncchecker-solana.sh @@ -1,6 +1,6 @@ #!/bin/bash -INIT_COMPLETED_FILE=/var/solana/data/init-completed +INIT_COMPLETED_FILE=/data/solana/data/init-completed TOKEN=$(curl -s -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 21600") INSTANCE_ID=$(curl -H "X-aws-ec2-metadata-token: $TOKEN" -s http://169.254.169.254/latest/meta-data/instance-id) diff --git a/lib/solana/lib/assets/systemd/node.service b/lib/solana/lib/assets/systemd/node.service new file mode 100644 index 00000000..6194889f --- /dev/null +++ b/lib/solana/lib/assets/systemd/node.service @@ -0,0 +1,15 @@ +[Unit] +Description=Solana Validator +After=network.target +StartLimitIntervalSec=0 +[Service] +Type=simple +Restart=always +RestartSec=1 +User=solana +LimitNOFILE=1000000 +LogRateLimitIntervalSec=0 +Environment="PATH=/bin:/usr/bin:/home/solana/bin" +ExecStart=/home/solana/bin/validator.sh +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/lib/solana/lib/assets/systemd/sync-checker.service b/lib/solana/lib/assets/systemd/sync-checker.service new file mode 100644 index 00000000..7c9eee35 --- /dev/null +++ b/lib/solana/lib/assets/systemd/sync-checker.service @@ -0,0 +1,5 @@ +[Unit] +Description="Sync checker for the tezos node" + +[Service] +ExecStart=/opt/syncchecker.sh \ No newline at end of file diff --git a/lib/solana/lib/assets/systemd/sync-checker.timer b/lib/solana/lib/assets/systemd/sync-checker.timer new file mode 100644 index 00000000..b531147b --- /dev/null +++ b/lib/solana/lib/assets/systemd/sync-checker.timer @@ -0,0 +1,9 @@ +[Unit] +Description="Run Sync checker service every 5 min" + +[Timer] +OnCalendar=*:*:0/5 +Unit=sync-checker.service + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/lib/solana/lib/assets/user-data/node.sh b/lib/solana/lib/assets/user-data/node.sh index 9d27c4c0..e923f86a 100644 --- a/lib/solana/lib/assets/user-data/node.sh +++ b/lib/solana/lib/assets/user-data/node.sh @@ -1,28 +1,29 @@ #!/bin/bash set +e - -echo "AWS_REGION=${_AWS_REGION_}" >> /etc/environment -echo "ASSETS_S3_PATH=${_ASSETS_S3_PATH_}" >> /etc/environment -echo "STACK_NAME=${_STACK_NAME_}" >> /etc/environment -echo "STACK_ID=${_STACK_ID_}" >> /etc/environment -echo "RESOURCE_ID=${_NODE_CF_LOGICAL_ID_}" >> /etc/environment -echo "ACCOUNTS_VOLUME_TYPE=${_ACCOUNTS_VOLUME_TYPE_}" >> /etc/environment -echo "ACCOUNTS_VOLUME_SIZE=${_ACCOUNTS_VOLUME_SIZE_}" >> /etc/environment -echo "DATA_VOLUME_TYPE=${_DATA_VOLUME_TYPE_}" >> /etc/environment -echo "DATA_VOLUME_SIZE=${_DATA_VOLUME_SIZE_}" >> /etc/environment -echo "SOLANA_VERSION=${_SOLANA_VERSION_}" >> /etc/environment -echo "SOLANA_NODE_TYPE=${_SOLANA_NODE_TYPE_}" >> /etc/environment -echo "NODE_IDENTITY_SECRET_ARN=${_NODE_IDENTITY_SECRET_ARN_}" >> /etc/environment -echo "VOTE_ACCOUNT_SECRET_ARN=${_VOTE_ACCOUNT_SECRET_ARN_}" >> /etc/environment -echo "AUTHORIZED_WITHDRAWER_ACCOUNT_SECRET_ARN=${_AUTHORIZED_WITHDRAWER_ACCOUNT_SECRET_ARN_}" >> /etc/environment -echo "REGISTRATION_TRANSACTION_FUNDING_ACCOUNT_SECRET_ARN=${_REGISTRATION_TRANSACTION_FUNDING_ACCOUNT_SECRET_ARN_}" >> /etc/environment -echo "SOLANA_CLUSTER=${_SOLANA_CLUSTER_}" >> /etc/environment -echo "LIFECYCLE_HOOK_NAME=${_LIFECYCLE_HOOK_NAME_}" >> /etc/environment -echo "ASG_NAME=${_ASG_NAME_}" >> /etc/environment +{ + echo "AWS_REGION=${_AWS_REGION_}" + echo "ASSETS_S3_PATH=${_ASSETS_S3_PATH_}" + echo "STACK_NAME=${_STACK_NAME_}" + echo "STACK_ID=${_STACK_ID_}" + echo "RESOURCE_ID=${_NODE_CF_LOGICAL_ID_}" + echo "ACCOUNTS_VOLUME_TYPE=${_ACCOUNTS_VOLUME_TYPE_}" + echo "ACCOUNTS_VOLUME_SIZE=${_ACCOUNTS_VOLUME_SIZE_}" + echo "DATA_VOLUME_TYPE=${_DATA_VOLUME_TYPE_}" + echo "DATA_VOLUME_SIZE=${_DATA_VOLUME_SIZE_}" + echo "SOLANA_VERSION=${_SOLANA_VERSION_}" + echo "SOLANA_NODE_TYPE=${_SOLANA_NODE_TYPE_}" + echo "NODE_IDENTITY_SECRET_ARN=${_NODE_IDENTITY_SECRET_ARN_}" + echo "VOTE_ACCOUNT_SECRET_ARN=${_VOTE_ACCOUNT_SECRET_ARN_}" + echo "AUTHORIZED_WITHDRAWER_ACCOUNT_SECRET_ARN=${_AUTHORIZED_WITHDRAWER_ACCOUNT_SECRET_ARN_}" + echo "REGISTRATION_TRANSACTION_FUNDING_ACCOUNT_SECRET_ARN=${_REGISTRATION_TRANSACTION_FUNDING_ACCOUNT_SECRET_ARN_}" + echo "SOLANA_CLUSTER=${_SOLANA_CLUSTER_}" + echo "LIFECYCLE_HOOK_NAME=${_LIFECYCLE_HOOK_NAME_}" + echo "ASG_NAME=${_ASG_NAME_}" +} >> /etc/environment source /etc/environment apt-get -yqq update -apt-get -yqq install awscli jq unzip python3-pip +apt-get -yqq install awscli jq unzip python3-pip chrony apt install unzip cd /opt @@ -111,9 +112,14 @@ sudo bash -c "cat >/etc/security/limits.d/90-solana-nofiles.conf </tmp/setup-instance-store-volumes.log 2>&1") | crontab - crontab -l - sudo /opt/setup-instance-store-volumes.sh - else - echo "Data and Accounts volumes are instance stores and should be both configured by now" + echo "Data and Accounts are configured by now" fi - else echo "Accounts volume type is EBS" ACCOUNTS_VOLUME_ID=/dev/$(lsblk -lnb | awk -v VOLUME_SIZE_BYTES="$ACCOUNTS_VOLUME_SIZE" '{if ($4== VOLUME_SIZE_BYTES) {print $1}}') sudo mkfs -t xfs $ACCOUNTS_VOLUME_ID sleep 10 ACCOUNTS_VOLUME_UUID=$(lsblk -fn -o UUID $ACCOUNTS_VOLUME_ID) - ACCOUNTS_VOLUME_FSTAB_CONF="UUID=$ACCOUNTS_VOLUME_UUID /var/solana/accounts xfs defaults 0 2" + ACCOUNTS_VOLUME_FSTAB_CONF="UUID=$ACCOUNTS_VOLUME_UUID /data/solana/accounts xfs defaults 0 2" echo "ACCOUNTS_VOLUME_ID="$ACCOUNTS_VOLUME_ID echo "ACCOUNTS_VOLUME_UUID="$ACCOUNTS_VOLUME_UUID echo "ACCOUNTS_VOLUME_FSTAB_CONF="$ACCOUNTS_VOLUME_FSTAB_CONF @@ -203,17 +203,13 @@ else sudo mount -a fi -sudo mkdir /var/solana/data/ledger - -echo 'Adding solana user and group' -sudo groupadd -g 1002 solana -sudo useradd -u 1002 -g 1002 -m -s /bin/bash solana +sudo mkdir /data/solana/data/ledger sudo usermod -aG sudo solana cd /home/solana sudo mkdir ./bin -ln -s /var/solana/data/ledger /home/solana +ln -s /data/solana/data/ledger /home/solana echo "Downloading x86 binaries for version v$SOLANA_VERSION" sudo wget -q https://github.com/solana-labs/solana/releases/download/v$SOLANA_VERSION/solana-release-x86_64-unknown-linux-gnu.tar.bz2 @@ -244,20 +240,17 @@ if [[ "$SOLANA_NODE_TYPE" == "consensus" ]]; then NODE_IDENTITY=$(sudo ./solana-keygen pubkey /home/solana/config/vote-account-keypair.json) echo "Store Vote Account Secret to AWS Secrets Manager" sudo aws secretsmanager create-secret --name "solana-node/"$NODE_IDENTITY --description "Solana Vote Account Secret created for stack $CF_STACK_NAME" --secret-string file:///home/solana/config/vote-account-keypair.json --region $AWS_REGION - if [[ $AUTHORIZED_WITHDRAWER_ACCOUNT_SECRET_ARN == "none" ]]; then echo "Create Authorized Withdrawer Account Secret" sudo ./solana-keygen new --no-passphrase -o /home/solana/config/authorized-withdrawer-keypair.json NODE_IDENTITY=$(sudo ./solana-keygen pubkey /home/solana/config/authorized-withdrawer-keypair.json) echo "Store Authorized Withdrawer Account to AWS Secrets Manager" sudo aws secretsmanager create-secret --name "solana-node/"$NODE_IDENTITY --description "Authorized Withdrawer Account Secret created for stack $CF_STACK_NAME" --secret-string file:///home/solana/config/authorized-withdrawer-keypair.json --region $AWS_REGION - else echo "Get Authorized Withdrawer Account Secret from AWS Secrets Manager" sudo aws secretsmanager get-secret-value --secret-id $AUTHORIZED_WITHDRAWER_ACCOUNT_SECRET_ARN --query SecretString --output text --region $AWS_REGION > ~/authorized-withdrawer-keypair.json sudo mv ~/authorized-withdrawer-keypair.json /home/solana/config/authorized-withdrawer-keypair.json fi - if [[ $REGISTRATION_TRANSACTION_FUNDING_ACCOUNT_SECRET_ARN != "none" ]]; then echo "Get Registration Transaction Funding Account Secret from AWS Secrets Manager" sudo aws secretsmanager get-secret-value --secret-id $REGISTRATION_TRANSACTION_FUNDING_ACCOUNT_SECRET_ARN --query SecretString --output text --region $AWS_REGION > ~/id.json @@ -265,13 +258,11 @@ if [[ "$SOLANA_NODE_TYPE" == "consensus" ]]; then sudo mv ~/id.json /root/.config/solana/id.json echo "Creating Vote Account on-chain" sudo ./solana create-vote-account /home/solana/config/vote-account-keypair.json /home/solana/config/validator-keypair.json /home/solana/config/authorized-withdrawer-keypair.json - echo "Delete Transaction Funding Account Secret from the local disc" sudo rm /root/.config/solana/id.json else echo "Vote Account not created. Please create it manually: https://docs.solana.com/running-validator/validator-start#create-vote-account" fi - echo "Delete Authorized Withdrawer Account from the local disc" sudo rm /home/solana/config/authorized-withdrawer-keypair.json else @@ -279,7 +270,6 @@ if [[ "$SOLANA_NODE_TYPE" == "consensus" ]]; then sudo aws secretsmanager get-secret-value --secret-id $VOTE_ACCOUNT_SECRET_ARN --query SecretString --output text --region $AWS_REGION > ~/vote-account-keypair.json sudo mv ~/vote-account-keypair.json /home/solana/config/vote-account-keypair.json fi - mv /opt/solana/node-consensus-template.sh /home/solana/bin/validator.sh fi @@ -297,54 +287,29 @@ sed -i "s/__KNOWN_VALIDATORS__/$KNOWN_VALIDATORS/g" /home/solana/bin/validator.s sed -i "s/__ENTRY_POINTS__/$ENTRY_POINTS/g" /home/solana/bin/validator.sh sudo chmod +x /home/solana/bin/validator.sh -sudo chown -R solana:solana /var/solana +sudo chown -R solana:solana /data/solana sudo chown -R solana:solana /home/solana echo "Starting solana as a service" -sudo bash -c 'cat > /etc/systemd/system/sol.service < logrotate.sol </tmp/syncchecker.log 2>&1") | crontab - -crontab -l +echo "Setting up sync-checker service" +sudo mv /opt/systemd/sync-checker.service /etc/systemd/system/sync-checker.service + +# Run every 5 minutes +echo "Setting up sync-checker timer" +sudo mv /opt/systemd/sync-checker.timer /etc/systemd/system/sync-checker.timer + +echo "Starting sync checker timer" +systemctl start sync-checker.timer +systemctl enable sync-checker.timer if [[ "$LIFECYCLE_HOOK_NAME" != "none" ]]; then echo "Signaling ASG lifecycle hook to complete" diff --git a/lib/solana/lib/assets/node-cw-dashboard.ts b/lib/solana/lib/constructs/node-cw-dashboard.ts similarity index 97% rename from lib/solana/lib/assets/node-cw-dashboard.ts rename to lib/solana/lib/constructs/node-cw-dashboard.ts index c4ee4b85..f9327486 100644 --- a/lib/solana/lib/assets/node-cw-dashboard.ts +++ b/lib/solana/lib/constructs/node-cw-dashboard.ts @@ -292,7 +292,7 @@ export const SingleNodeCWDashboardJSON = { "type": "metric", "properties": { "metrics": [ - [ "CWAgent", "disk_used_percent", "InstanceId", "${INSTANCE_ID}", "device", "nvme1n1", "path", "/var/solana/data", "fstype", "xfs", { "region": "${REGION}", "label": "/home/ubuntu/l2geth-source/l2geth-datadir" } ] + [ "CWAgent", "disk_used_percent", "InstanceId", "${INSTANCE_ID}", "device", "nvme1n1", "path", "/data/solana/data", "fstype", "xfs", { "region": "${REGION}", "label": "/data/solana/data" } ] ], "sparkline": true, "view": "singleValue", @@ -310,7 +310,7 @@ export const SingleNodeCWDashboardJSON = { "type": "metric", "properties": { "metrics": [ - [ "CWAgent", "disk_used_percent", "InstanceId", "${INSTANCE_ID}", "device", "nvme2n1", "path", "/var/solana/accounts", "fstype", "xfs", { "region": "${REGION}", "label": "/home/ubuntu/l2geth-source/l2geth-datadir" } ] + [ "CWAgent", "disk_used_percent", "InstanceId", "${INSTANCE_ID}", "device", "nvme2n1", "path", "/data/solana/accounts", "fstype", "xfs", { "region": "${REGION}", "label": "/data/solana/accounts" } ] ], "sparkline": true, "view": "singleValue", @@ -321,4 +321,4 @@ export const SingleNodeCWDashboardJSON = { } } ] -} +} \ No newline at end of file diff --git a/lib/solana/lib/ha-nodes-stack.ts b/lib/solana/lib/ha-nodes-stack.ts index ccecf8f6..0b5abd4e 100644 --- a/lib/solana/lib/ha-nodes-stack.ts +++ b/lib/solana/lib/ha-nodes-stack.ts @@ -112,7 +112,7 @@ export class SolanaHANodesStack extends cdk.Stack { const healthCheckPath = "/health"; const rpcNodes = new HANodesConstruct (this, "rpc-nodes", { instanceType, - dataVolumes: [dataVolume, accountsVolume], + dataVolumes: [accountsVolume, dataVolume], rootDataVolumeDeviceName: "/dev/sda1", machineImage: ec2.MachineImage.fromSsmParameter(ubuntu204stableImageSsmName), role: instanceRole, diff --git a/lib/solana/lib/single-node-stack.ts b/lib/solana/lib/single-node-stack.ts index 95025374..256c8d82 100644 --- a/lib/solana/lib/single-node-stack.ts +++ b/lib/solana/lib/single-node-stack.ts @@ -5,13 +5,13 @@ import * as iam from "aws-cdk-lib/aws-iam"; import * as s3Assets from "aws-cdk-lib/aws-s3-assets"; import * as path from "path"; import * as fs from "fs"; -import * as nodeCwDashboard from "./assets/node-cw-dashboard" import * as cw from 'aws-cdk-lib/aws-cloudwatch'; import * as nag from "cdk-nag"; import { SingleNodeConstruct } from "../../constructs/single-node" import * as configTypes from "./config/solanaConfig.interface"; import * as constants from "../../constructs/constants"; import { SolanaNodeSecurityGroupConstruct } from "./constructs/solana-node-security-group" +import { SingleNodeCWDashboardJSON } from "./constructs/node-cw-dashboard" export interface SolanaSingleNodeStackProps extends cdk.StackProps { instanceType: ec2.InstanceType; @@ -85,7 +85,7 @@ export class SolanaSingleNodeStack extends cdk.Stack { const node = new SingleNodeConstruct(this, "sync-node", { instanceName: STACK_NAME, instanceType, - dataVolumes: [dataVolume, accountsVolume], + dataVolumes: [accountsVolume, dataVolume], rootDataVolumeDeviceName: "/dev/sda1", machineImage: ec2.MachineImage.fromSsmParameter(ubuntu204stableImageSsmName), vpc, @@ -125,7 +125,7 @@ export class SolanaSingleNodeStack extends cdk.Stack { node.instance.addUserData(modifiedInitNodeScript); // Adding CloudWatch dashboard to the node - const dashboardString = cdk.Fn.sub(JSON.stringify(nodeCwDashboard.SingleNodeCWDashboardJSON), { + const dashboardString = cdk.Fn.sub(JSON.stringify(SingleNodeCWDashboardJSON), { INSTANCE_ID:node.instanceId, INSTANCE_NAME: STACK_NAME, REGION: REGION, diff --git a/lib/solana/package.json b/lib/solana/package.json index ceeebb54..ca5ce52e 100644 --- a/lib/solana/package.json +++ b/lib/solana/package.json @@ -4,7 +4,7 @@ "scripts": { "build": "npx tsc", "watch": "npx tsc -w", - "test": "npx jest", + "test": "npx jest --detectOpenHandles", "cdk": "npx cdk", "scan-cdk": "npx cdk synth" } diff --git a/lib/solana/sample-configs/.env-sample-baserpc b/lib/solana/sample-configs/.env-sample-baserpc index aefebf35..49b7282e 100644 --- a/lib/solana/sample-configs/.env-sample-baserpc +++ b/lib/solana/sample-configs/.env-sample-baserpc @@ -9,14 +9,14 @@ AWS_REGION="us-east-2" ## Common configuration parameters ## SOLANA_CLUSTER="mainnet-beta" # All options: "mainnet-beta", "testnet", "devnet" SOLANA_NODE_CONFIGURATION="baserpc" # All options: "consensus", "baserpc", "extendedrpc" -SOLANA_VERSION="1.18.22" # Latest required version of Solana. Check for latest Mainnet version https://github.com/solana-labs/solana/releases +SOLANA_VERSION="1.18.26" # Latest required version of Solana. Check for latest Mainnet version https://github.com/solana-labs/solana/releases -SOLANA_INSTANCE_TYPE="r6a.8xlarge" +SOLANA_INSTANCE_TYPE="r7a.12xlarge" SOLANA_CPU_TYPE="x86_64" # All options: "x86_64", "ARM_64". IMPORTANT: Make sure the CPU type matches the instance type used # Data volume configuration SOLANA_DATA_VOL_TYPE="gp3" # Other options: "io1" | "io2" | "gp3" | "instance-store" . IMPORTANT: Use "instance-store" option only with instance types that support that feature, like popular for node im4gn, d3, i3en, and i4i instance families SOLANA_DATA_VOL_SIZE="2000" # Current required data size to keep both smapshot archive and unarchived version of it -SOLANA_DATA_VOL_IOPS="12000" # Max IOPS for EBS volumes (not applicable for "instance-store") +SOLANA_DATA_VOL_IOPS="9000" # Max IOPS for EBS volumes (not applicable for "instance-store") SOLANA_DATA_VOL_THROUGHPUT="700" # Max throughput for EBS gp3 volumes (not applicable for "io1" | "io2" | "instance-store") # Accounts volume configuration SOLANA_ACCOUNTS_VOL_TYPE="gp3" # Other options: "io1" | "io2" | "gp3" | "instance-store" . IMPORTANT: Use "instance-store" option only with instance types that support that feature, like popular for node im4gn, d3, i3en, and i4i instance families diff --git a/lib/solana/sample-configs/.env-sample-consensus b/lib/solana/sample-configs/.env-sample-consensus index e7f26b4b..955783f3 100644 --- a/lib/solana/sample-configs/.env-sample-consensus +++ b/lib/solana/sample-configs/.env-sample-consensus @@ -9,14 +9,14 @@ AWS_REGION="us-east-2" ## Common configuration parameters ## SOLANA_CLUSTER="mainnet-beta" # All options: "mainnet-beta", "testnet", "devnet" SOLANA_NODE_CONFIGURATION="consensus" # All options: "consensus", "baserpc", "extendedrpc" -SOLANA_VERSION="1.18.22" # Latest required version of Solana. Check for latest Mainnet version https://github.com/solana-labs/solana/releases +SOLANA_VERSION="1.18.26" # Latest required version of Solana. Check for latest Mainnet version https://github.com/solana-labs/solana/releases -SOLANA_INSTANCE_TYPE="r6a.8xlarge" +SOLANA_INSTANCE_TYPE="r7a.12xlarge" SOLANA_CPU_TYPE="x86_64" # All options: "x86_64", "ARM_64". IMPORTANT: Make sure the CPU type matches the instance type used # Data volume configuration SOLANA_DATA_VOL_TYPE="gp3" # Other options: "io1" | "io2" | "gp3" | "instance-store" . IMPORTANT: Use "instance-store" option only with instance types that support that feature, like popular for node im4gn, d3, i3en, and i4i instance families SOLANA_DATA_VOL_SIZE="2000" # Current required data size to keep both smapshot archive and unarchived version of it -SOLANA_DATA_VOL_IOPS="12000" # Max IOPS for EBS volumes (not applicable for "instance-store") +SOLANA_DATA_VOL_IOPS="9000" # Max IOPS for EBS volumes (not applicable for "instance-store") SOLANA_DATA_VOL_THROUGHPUT="700" # Max throughput for EBS gp3 volumes (not applicable for "io1" | "io2" | "instance-store") # Accounts volume configuration SOLANA_ACCOUNTS_VOL_TYPE="gp3" # Other options: "io1" | "io2" | "gp3" | "instance-store" . IMPORTANT: Use "instance-store" option only with instance types that support that feature, like popular for node im4gn, d3, i3en, and i4i instance families diff --git a/lib/solana/sample-configs/.env-sample-extendedrpc b/lib/solana/sample-configs/.env-sample-extendedrpc index 0b7c24cb..c03d4e82 100644 --- a/lib/solana/sample-configs/.env-sample-extendedrpc +++ b/lib/solana/sample-configs/.env-sample-extendedrpc @@ -9,20 +9,20 @@ AWS_REGION="us-east-2" ## Common configuration parameters ## SOLANA_CLUSTER="mainnet-beta" # All options: "mainnet-beta", "testnet", "devnet" SOLANA_NODE_CONFIGURATION="extendedrpc" # All options: "consensus", "baserpc", "extendedrpc" -SOLANA_VERSION="1.18.22" # Latest required version of Solana. Check for latest Mainnet version https://github.com/solana-labs/solana/releases +SOLANA_VERSION="1.18.26" # Latest required version of Solana. Check for latest Mainnet version https://github.com/solana-labs/solana/releases -SOLANA_INSTANCE_TYPE="x2idn.16xlarge" +SOLANA_INSTANCE_TYPE="r7a.24xlarge" SOLANA_CPU_TYPE="x86_64" # All options: "x86_64", "ARM_64". IMPORTANT: Make sure the CPU type matches the instance type used # Data volume configuration SOLANA_DATA_VOL_TYPE="gp3" # Other options: "io1" | "io2" | "gp3" | "instance-store" . IMPORTANT: Use "instance-store" option only with instance types that support that feature, like popular for node im4gn, d3, i3en, and i4i instance families SOLANA_DATA_VOL_SIZE="2000" # Current required data size to keep both smapshot archive and unarchived version of it -SOLANA_DATA_VOL_IOPS="12000" # Max IOPS for EBS volumes (not applicable for "instance-store") +SOLANA_DATA_VOL_IOPS="9000" # Max IOPS for EBS volumes (not applicable for "instance-store") SOLANA_DATA_VOL_THROUGHPUT="700" # Max throughput for EBS gp3 volumes (not applicable for "io1" | "io2" | "instance-store") # Accounts volume configuration -SOLANA_ACCOUNTS_VOL_TYPE="instance-store" # Other options: "io1" | "io2" | "gp3" | "instance-store" . IMPORTANT: Use "instance-store" option only with instance types that support that feature, like popular for node im4gn, d3, i3en, and i4i instance families -SOLANA_ACCOUNTS_VOL_SIZE="0" # Current required data size to keep both smapshot archive and unarchived version of it -SOLANA_ACCOUNTS_VOL_IOPS="0" # Max IOPS for EBS volumes (not applicable for "instance-store") -SOLANA_ACCOUNTS_VOL_THROUGHPUT="0" # Max throughput for EBS gp3 volumes (not applicable for "io1" | "io2" | "instance-store") +SOLANA_ACCOUNTS_VOL_TYPE="gp3" # Other options: "io1" | "io2" | "gp3" | "instance-store" . IMPORTANT: Use "instance-store" option only with instance types that support that feature, like popular for node im4gn, d3, i3en, and i4i instance families +SOLANA_ACCOUNTS_VOL_SIZE="500" # Current required data size to keep both smapshot archive and unarchived version of it +SOLANA_ACCOUNTS_VOL_IOPS="4000" # Max IOPS for EBS volumes (not applicable for "instance-store") +SOLANA_ACCOUNTS_VOL_THROUGHPUT="700" # Max throughput for EBS gp3 volumes (not applicable for "io1" | "io2" | "instance-store") # Secrets configuration SOLANA_NODE_IDENTITY_SECRET_ARN="none" # Optional. The ARN of the secret in AWS Secrets Manager with the node identity key pair. In case you want to provision a node but keep the old identity. SOLANA_VOTE_ACCOUNT_SECRET_ARN="none" # Optional for consensus node type. ARN of the secret in AWS Secrets Manager with the Vote Account's key pair.