populate records_by_bucket during restart #10424
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Check before merge | |
| on: | |
| # tests must run for a PR to be valid and pass merge queue muster | |
| # on main, we want to know that all commits are passing at a glance, any deviation should help bisecting errors | |
| # the merge run checks should show on master and enable this clear test/passing history | |
| merge_group: | |
| branches: [main, alpha*, beta*, rc*] | |
| pull_request: | |
| branches: ["*"] | |
| env: | |
| CARGO_INCREMENTAL: 0 # bookkeeping for incremental builds has overhead, not useful in CI. | |
| WINSW_URL: https://github.com/winsw/winsw/releases/download/v3.0.0-alpha.11/WinSW-x64.exe | |
| GENESIS_PK: 9377ab39708a59d02d09bfd3c9bc7548faab9e0c2a2700b9ac7d5c14f0842f0b4bb0df411b6abd3f1a92b9aa1ebf5c3d | |
| GENESIS_SK: 5ec88891c1098a0fede5b98b07f8abc931d7247b7aa310d21ab430cc957f9f02 | |
| jobs: | |
| build: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.x' | |
| - name: Display Python version | |
| run: python --version | |
| cargo-udeps: | |
| if: "!startsWith(github.event.head_commit.message, 'chore(release):')" | |
| name: Unused dependency check | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Install Rust | |
| uses: dtolnay/rust-toolchain@nightly # Needs nightly to distinguish between deps of different versions | |
| with: | |
| # we need rustfmt here while we have a build step | |
| components: rustfmt | |
| - name: Install cargo-udeps | |
| run: cargo install cargo-udeps --locked | |
| - name: Run cargo-udeps | |
| run: cargo +nightly udeps --all-targets | |
| # ignore the error cause by the latest nightly changes. | |
| # should be fixed by https://github.com/dalek-cryptography/curve25519-dalek/pull/619 | |
| continue-on-error: true | |
| lint: | |
| runs-on: ubuntu-latest | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - uses: wagoid/commitlint-github-action@7f0a61df502599e1f1f50880aaa7ec1e2c0592f2 | |
| checks: | |
| if: "!startsWith(github.event.head_commit.message, 'chore(release):')" | |
| name: various checks | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Install Rust | |
| uses: dtolnay/rust-toolchain@stable | |
| with: | |
| components: rustfmt, clippy | |
| - uses: Swatinem/rust-cache@v2 | |
| - name: Check formatting | |
| run: cargo fmt --all -- --check | |
| - shell: bash | |
| run: cargo clippy --all-targets --all-features -- -Dwarnings | |
| - name: Check documentation | |
| # Deny certain `rustdoc` lints that are unwanted. | |
| # See https://doc.rust-lang.org/rustdoc/lints.html for lints that are 'warning' by default. | |
| run: RUSTDOCFLAGS="--deny=warnings" cargo doc --no-deps | |
| - name: Check local is not a default feature | |
| shell: bash | |
| run: if [[ ! $(cargo metadata --no-deps --format-version 1 | jq -r '.packages[].features.default[]? | select(. == "local")') ]]; then echo "local is not a default feature in any package."; else echo "local is a default feature in at least one package." && exit 1; fi | |
| - name: Clean out the target directory | |
| run: cargo clean | |
| # In a cargo workspace, feature unification can occur, allowing a crate to be built successfully even if it | |
| # doesn't explicitly specify a feature it uses, provided another crate in the workspace enables that feature. | |
| # To detect such cases, we must build each crate using `--package` flag, building all packages at once does not work. | |
| - name: Check the whole workspace can build | |
| shell: bash | |
| run: | | |
| for package in $(cargo metadata --no-deps --format-version=1 | jq -r '.packages[].name'); do | |
| cargo build -p "$package" --all-targets --all-features | |
| done | |
| echo "All packages built successfully. Cleaning up..." | |
| cargo clean | |
| unit: | |
| if: "!startsWith(github.event.head_commit.message, 'chore(release):')" | |
| name: Unit Tests | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| matrix: | |
| os: [ubuntu-latest, windows-latest, macos-latest] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Check we're on the right commit | |
| run: git log -1 --oneline | |
| - name: Install Rust | |
| uses: dtolnay/rust-toolchain@stable | |
| - uses: Swatinem/rust-cache@v2 | |
| - name: Run autonomi tests | |
| timeout-minutes: 25 | |
| run: cargo test --release --package autonomi --lib --features="full,fs" | |
| - name: Run node tests | |
| timeout-minutes: 25 | |
| run: cargo test --release --package sn_node --lib | |
| - name: Run network tests (with encrypt-records) | |
| timeout-minutes: 25 | |
| run: cargo test --release --package sn_networking --features="open-metrics, encrypt-records" | |
| - name: Run network tests (without encrypt-records) | |
| timeout-minutes: 25 | |
| run: cargo test --release --package sn_networking --features="open-metrics" | |
| - name: Run protocol tests | |
| timeout-minutes: 25 | |
| run: cargo test --release --package sn_protocol | |
| - name: Run transfers tests | |
| timeout-minutes: 25 | |
| run: cargo test --release --package sn_transfers | |
| - name: Run logging tests | |
| timeout-minutes: 25 | |
| run: cargo test --release --package sn_logging | |
| - name: Run register tests | |
| timeout-minutes: 25 | |
| run: cargo test --release --package sn_registers | |
| env: | |
| # this will speed up PR merge flows, while giving us a modicum | |
| # of proptesting | |
| # we do many more runs on the nightly run | |
| PROPTEST_CASES: 50 | |
| e2e: | |
| if: "!startsWith(github.event.head_commit.message, 'chore(release):')" | |
| name: E2E tests | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| matrix: | |
| include: | |
| - os: ubuntu-latest | |
| safe_path: /home/runner/.local/share/safe | |
| - os: windows-latest | |
| safe_path: C:\\Users\\runneradmin\\AppData\\Roaming\\safe | |
| - os: macos-latest | |
| safe_path: /Users/runner/Library/Application\ Support/safe | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Install Rust | |
| uses: dtolnay/rust-toolchain@stable | |
| - uses: Swatinem/rust-cache@v2 | |
| - name: Build binaries | |
| run: cargo build --release --features local --bin safenode --bin autonomi | |
| timeout-minutes: 30 | |
| - name: Start a local network | |
| uses: maidsafe/sn-local-testnet-action@main | |
| with: | |
| action: start | |
| enable-evm-testnet: true | |
| node-path: target/release/safenode | |
| platform: ${{ matrix.os }} | |
| build: true | |
| - name: Check if SAFE_PEERS and EVM_NETWORK are set | |
| shell: bash | |
| run: | | |
| if [[ -z "$SAFE_PEERS" ]]; then | |
| echo "The SAFE_PEERS variable has not been set" | |
| exit 1 | |
| elif [[ -z "$EVM_NETWORK" ]]; then | |
| echo "The EVM_NETWORK variable has not been set" | |
| exit 1 | |
| else | |
| echo "SAFE_PEERS has been set to $SAFE_PEERS" | |
| echo "EVM_NETWORK has been set to $EVM_NETWORK" | |
| fi | |
| # only these unit tests require a network, the rest are run above in unit test section | |
| - name: Run autonomi --tests | |
| run: cargo test --package autonomi --tests -- --nocapture | |
| env: | |
| SN_LOG: "v" | |
| # only set the target dir for windows to bypass the linker issue. | |
| # happens if we build the node manager via testnet action | |
| CARGO_TARGET_DIR: ${{ matrix.os == 'windows-latest' && './test-target' || '.' }} | |
| timeout-minutes: 15 | |
| # FIXME: do this in a generic way for localtestnets | |
| - name: export default secret key | |
| if: matrix.os != 'windows-latest' | |
| run: echo "SECRET_KEY=0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80" >> $GITHUB_ENV | |
| shell: bash | |
| - name: Set secret key for Windows | |
| if: matrix.os == 'windows-latest' | |
| run: echo "SECRET_KEY=0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append | |
| shell: pwsh | |
| - name: Get file cost | |
| run: ./target/release/autonomi --log-output-dest=data-dir file cost "./resources" | |
| env: | |
| SN_LOG: "v" | |
| timeout-minutes: 15 | |
| - name: File upload | |
| run: ./target/release/autonomi --log-output-dest=data-dir file upload "./resources" > ./upload_output 2>&1 | |
| env: | |
| SN_LOG: "v" | |
| timeout-minutes: 15 | |
| - name: parse address (unix) | |
| if: matrix.os != 'windows-latest' | |
| run: | | |
| UPLOAD_ADDRESS=$(rg "At address: ([0-9a-f]*)" -o -r '$1' ./upload_output) | |
| echo "UPLOAD_ADDRESS=$UPLOAD_ADDRESS" >> $GITHUB_ENV | |
| shell: bash | |
| - name: parse address (win) | |
| if: matrix.os == 'windows-latest' | |
| run: | | |
| $UPLOAD_ADDRESS = rg "At address: ([0-9a-f]*)" -o -r '$1' ./upload_output | |
| echo "UPLOAD_ADDRESS=$UPLOAD_ADDRESS" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append | |
| shell: pwsh | |
| - name: File Download | |
| run: ./target/release/autonomi --log-output-dest=data-dir file download ${{ env.UPLOAD_ADDRESS }} ./downloaded_resources | |
| env: | |
| SN_LOG: "v" | |
| timeout-minutes: 5 | |
| - name: Generate register signing key | |
| run: ./target/release/autonomi --log-output-dest=data-dir register generate-key | |
| - name: Create register (writeable by owner) | |
| run: ./target/release/autonomi --log-output-dest=data-dir register create baobao 123 > ./register_create_output 2>&1 | |
| env: | |
| SN_LOG: "v" | |
| timeout-minutes: 10 | |
| - name: parse register address (unix) | |
| if: matrix.os != 'windows-latest' | |
| run: | | |
| REGISTER_ADDRESS=$(rg "Register created at address: ([0-9a-f]*)" -o -r '$1' ./register_create_output) | |
| echo "REGISTER_ADDRESS=$REGISTER_ADDRESS" >> $GITHUB_ENV | |
| shell: bash | |
| - name: parse register address (win) | |
| if: matrix.os == 'windows-latest' | |
| run: | | |
| $REGISTER_ADDRESS = rg "Register created at address: ([0-9a-f]*)" -o -r '$1' ./register_create_output | |
| echo "REGISTER_ADDRESS=$REGISTER_ADDRESS" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append | |
| shell: pwsh | |
| - name: Get register | |
| run: ./target/release/autonomi --log-output-dest=data-dir register get ${{ env.REGISTER_ADDRESS }} | |
| env: | |
| SN_LOG: "v" | |
| timeout-minutes: 5 | |
| - name: Edit register | |
| run: ./target/release/autonomi --log-output-dest=data-dir register edit ${{ env.REGISTER_ADDRESS }} 456 | |
| env: | |
| SN_LOG: "v" | |
| timeout-minutes: 10 | |
| - name: Get register (after edit) | |
| run: ./target/release/autonomi --log-output-dest=data-dir register get ${{ env.REGISTER_ADDRESS }} | |
| env: | |
| SN_LOG: "v" | |
| timeout-minutes: 5 | |
| - name: Create Public Register (writeable by anyone) | |
| run: ./target/release/autonomi --log-output-dest=data-dir register create bao 111 --public > ./register_public_create_output 2>&1 | |
| env: | |
| SN_LOG: "v" | |
| timeout-minutes: 5 | |
| - name: parse public register address (unix) | |
| if: matrix.os != 'windows-latest' | |
| run: | | |
| PUBLIC_REGISTER_ADDRESS=$(rg "Register created at address: ([0-9a-f]*)" -o -r '$1' ./register_public_create_output) | |
| echo "PUBLIC_REGISTER_ADDRESS=$PUBLIC_REGISTER_ADDRESS" >> $GITHUB_ENV | |
| shell: bash | |
| - name: parse public register address (win) | |
| if: matrix.os == 'windows-latest' | |
| run: | | |
| $PUBLIC_REGISTER_ADDRESS = rg "Register created at address: ([0-9a-f]*)" -o -r '$1' ./register_public_create_output | |
| echo "PUBLIC_REGISTER_ADDRESS=$PUBLIC_REGISTER_ADDRESS" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append | |
| shell: pwsh | |
| - name: Get Public Register (current key is the owner) | |
| run: ./target/release/autonomi --log-output-dest=data-dir register get ${{ env.PUBLIC_REGISTER_ADDRESS }} | |
| env: | |
| SN_LOG: "v" | |
| timeout-minutes: 5 | |
| - name: Edit Public Register (current key is the owner) | |
| run: ./target/release/autonomi --log-output-dest=data-dir register edit ${{ env.PUBLIC_REGISTER_ADDRESS }} 222 | |
| env: | |
| SN_LOG: "v" | |
| timeout-minutes: 10 | |
| - name: Delete current register signing key | |
| shell: bash | |
| run: rm -rf ${{ matrix.safe_path }}/autonomi | |
| - name: Generate new register signing key | |
| run: ./target/release/autonomi --log-output-dest=data-dir register generate-key | |
| - name: Get Public Register (new signing key is not the owner) | |
| run: ./target/release/autonomi --log-output-dest=data-dir register get ${{ env.PUBLIC_REGISTER_ADDRESS }} | |
| env: | |
| SN_LOG: "v" | |
| timeout-minutes: 2 | |
| - name: Edit Public Register (new signing key is not the owner) | |
| run: ./target/release/autonomi --log-output-dest=data-dir register edit ${{ env.PUBLIC_REGISTER_ADDRESS }} 333 | |
| env: | |
| SN_LOG: "v" | |
| timeout-minutes: 10 | |
| - name: Get Public Register (new signing key is not the owner) | |
| run: ./target/release/autonomi --log-output-dest=data-dir register get ${{ env.PUBLIC_REGISTER_ADDRESS }} | |
| env: | |
| SN_LOG: "v" | |
| timeout-minutes: 2 | |
| - name: Estimate cost to create a vault | |
| if: matrix.os != 'windows-latest' | |
| run: | | |
| echo "test-file" > upload-test.txt | |
| ./target/release/autonomi --log-output-dest=data-dir file upload ./upload-test.txt | |
| ./target/release/autonomi --log-output-dest=data-dir register create sample_new_register 1234 | |
| ./target/release/autonomi --log-output-dest=data-dir vault cost | |
| ./target/release/autonomi --log-output-dest=data-dir file list 2>&1 | tee file_list.txt | |
| ./target/release/autonomi --log-output-dest=data-dir register list 2>&1 | tee register_list.txt | |
| env: | |
| SN_LOG: "v" | |
| timeout-minutes: 2 | |
| - name: create a vault with existing user data as above | |
| if: matrix.os != 'windows-latest' | |
| run: ./target/release/autonomi --log-output-dest=data-dir vault create | |
| env: | |
| SN_LOG: "v" | |
| timeout-minutes: 2 | |
| - name: add more files | |
| if: matrix.os != 'windows-latest' | |
| run: | | |
| for i in {1..100}; do | |
| dd if=/dev/urandom of=random_file_$i.bin bs=1M count=1 status=none | |
| ./target/release/autonomi --log-output-dest=data-dir file upload random_file_$i.bin --public | |
| ./target/release/autonomi --log-output-dest=data-dir file upload random_file_$i.bin | |
| ./target/release/autonomi --log-output-dest=data-dir register create $i random_file_$i.bin | |
| done | |
| env: | |
| SN_LOG: "v" | |
| timeout-minutes: 25 | |
| - name: sync the vault | |
| if: matrix.os != 'windows-latest' | |
| run: ./target/release/autonomi --log-output-dest=data-dir vault sync | |
| env: | |
| SN_LOG: "v" | |
| timeout-minutes: 2 | |
| - name: vault sync validation | |
| if: matrix.os != 'windows-latest' | |
| run: | | |
| NUM_OF_PUBLIC_FILES="" | |
| NUM_OF_PRIVATE_FILES="" | |
| NUM_OF_REGISTERS="" | |
| NUM_OF_PUBLIC_FILES_IN_VAULT="" | |
| NUM_OF_PRIVATE_FILES_IN_VAULT="" | |
| NUM_OF_REGISTERS_IN_VAULT="" | |
| ./target/release/autonomi --log-output-dest=data-dir file list 2>&1 > file_list.txt | |
| # ./target/release/autonomi --log-output-dest=data-dir register list | grep archives > register_list.txt | |
| NUM_OF_PUBLIC_FILES=`cat file_list.txt | grep "public" | grep -o '[0-9]\+'` | |
| NUM_OF_PRIVATE_FILES=`cat file_list.txt | grep "private" | grep -o '[0-9]\+'` | |
| NUM_OF_REGISTERS=`cat register_list.txt | grep "register" | grep -o '[0-9]\+'` | |
| ./target/release/autonomi --log-output-dest=data-dir vault load 2>&1 > vault_data.txt | |
| NUM_OF_PUBLIC_FILES_IN_VAULT=`cat vault_data.txt | grep "public" | grep -o '[0-9]\+'` | |
| NUM_OF_PRIVATE_FILES_IN_VAULT=`cat vault_data.txt| grep "private" | grep -o '[0-9]\+'` | |
| # NUM_OF_REGISTERS_IN_VAULT=`cat vault_data.txt | grep "register" | grep -o '[0-9]\+'` | |
| echo "Total Num of local public files is $NUM_OF_PUBLIC_FILES and in vault is $NUM_OF_PUBLIC_FILES_IN_VAULT" | |
| echo "Total Num of local private files is $NUM_OF_PRIVATE_FILES and in vault is $NUM_OF_PRIVATE_FILES_IN_VAULT" | |
| # echo "Total Num of local registers is $NUM_OF_REGISTERS and in vault is $NUM_OF_REGISTERS_IN_VAULT" | |
| rm -rf file_list.txt register_list.txt vault_data.txt | |
| python3 -c 'import sys; assert sys.argv[1] == sys.argv[2], f"Error: Local public Files: {sys.argv[1]} and vault public files: {sys.argv[2]} are Not Equal"' $NUM_OF_PUBLIC_FILES $NUM_OF_PUBLIC_FILES_IN_VAULT | |
| python3 -c 'import sys; assert sys.argv[1] == sys.argv[2], f"Error: Local private Files: {sys.argv[1]} and vault private files: {sys.argv[2]} are Not Equal"' $NUM_OF_PRIVATE_FILES $NUM_OF_PRIVATE_FILES_IN_VAULT | |
| # python3 -c 'import sys; assert sys.argv[1] == sys.argv[2], f"Error: Local registers: {sys.argv[1]} and vault registers: {sys.argv[2]} are Not Equal"' $NUM_OF_REGISTERS $NUM_OF_REGISTERS_IN_VAULT | |
| echo "vault synced successfully!" | |
| env: | |
| SN_LOG: "v" | |
| timeout-minutes: 15 | |
| - name: load an existing vault from the network | |
| if: matrix.os != 'windows-latest' | |
| run: ./target/release/autonomi --log-output-dest=data-dir vault load | |
| env: | |
| SN_LOG: "v" | |
| timeout-minutes: 2 | |
| - name: Time profiling for Different files | |
| if: matrix.os != 'windows-latest' | |
| run: | | |
| # 1 MB | |
| python3 -c "with open('random_1MB.bin', 'wb') as f: f.write(bytearray([0xff] * 1 * 1024 * 1024))" | |
| # 10 MB | |
| python3 -c "with open('random_10MB.bin', 'wb') as f: f.write(bytearray([0xff] * 10 * 1024 * 1024))" | |
| # 100 MB | |
| python3 -c "with open('random_100MB.bin', 'wb') as f: f.write(bytearray([0xff] * 100 * 1024 * 1024))" | |
| # 1 GB | |
| python3 -c "with open('random_1GB.bin', 'wb') as f: f.write(bytearray([0xff] * 1000 * 1024 * 1024))" | |
| ./target/release/autonomi --log-output-dest=data-dir file list | |
| time ./target/release/autonomi --log-output-dest=data-dir file upload random_1MB.bin | |
| time ./target/release/autonomi --log-output-dest=data-dir file upload random_10MB.bin | |
| time ./target/release/autonomi --log-output-dest=data-dir file upload random_100MB.bin | |
| time ./target/release/autonomi --log-output-dest=data-dir file upload random_1GB.bin | |
| ./target/release/autonomi --log-output-dest=data-dir vault sync | |
| rm -rf random*.bin | |
| rm -rf ${{ matrix.safe_path }}/autonomi | |
| env: | |
| SN_LOG: "v" | |
| timeout-minutes: 15 | |
| - name: Stop the local network and upload logs | |
| if: always() | |
| uses: maidsafe/sn-local-testnet-action@main | |
| with: | |
| action: stop | |
| log_file_prefix: safe_test_logs_e2e | |
| platform: ${{ matrix.os }} | |
| # spend_test: | |
| # if: "!startsWith(github.event.head_commit.message, 'chore(release):')" | |
| # name: spend tests against network | |
| # runs-on: ${{ matrix.os }} | |
| # strategy: | |
| # matrix: | |
| # os: [ubuntu-latest, windows-latest, macos-latest] | |
| # steps: | |
| # - uses: actions/checkout@v4 | |
| # - name: Install Rust | |
| # uses: dtolnay/rust-toolchain@stable | |
| # - uses: Swatinem/rust-cache@v2 | |
| # - name: Build binaries | |
| # run: cargo build --release --features=local --bin safenode | |
| # timeout-minutes: 30 | |
| # - name: Build faucet binary | |
| # run: cargo build --release --bin faucet --features="local,gifting" | |
| # timeout-minutes: 30 | |
| # - name: Build testing executable | |
| # run: cargo test --release -p sn_node --features=local --test sequential_transfers --test storage_payments --test double_spend --no-run | |
| # env: | |
| # # only set the target dir for windows to bypass the linker issue. | |
| # # happens if we build the node manager via testnet action | |
| # CARGO_TARGET_DIR: ${{ matrix.os == 'windows-latest' && './test-target' || '.' }} | |
| # timeout-minutes: 30 | |
| # - name: Start a local network | |
| # uses: maidsafe/sn-local-testnet-action@main | |
| # with: | |
| # action: start | |
| # interval: 2000 | |
| # node-path: target/release/safenode | |
| # faucet-path: target/release/faucet | |
| # platform: ${{ matrix.os }} | |
| # build: true | |
| # - name: Check SAFE_PEERS was set | |
| # shell: bash | |
| # run: | | |
| # if [[ -z "$SAFE_PEERS" ]]; then | |
| # echo "The SAFE_PEERS variable has not been set" | |
| # exit 1 | |
| # else | |
| # echo "SAFE_PEERS has been set to $SAFE_PEERS" | |
| # fi | |
| # - name: execute the sequential transfers tests | |
| # run: cargo test --release -p sn_node --features="local" --test sequential_transfers -- --nocapture --test-threads=1 | |
| # env: | |
| # SN_LOG: "all" | |
| # CARGO_TARGET_DIR: ${{ matrix.os == 'windows-latest' && './test-target' || '.' }} | |
| # timeout-minutes: 25 | |
| # - name: execute the storage payment tests | |
| # run: cargo test --release -p sn_node --features="local" --test storage_payments -- --nocapture --test-threads=1 | |
| # env: | |
| # SN_LOG: "all" | |
| # CARGO_TARGET_DIR: ${{ matrix.os == 'windows-latest' && './test-target' || '.' }} | |
| # timeout-minutes: 25 | |
| # - name: execute the double spend tests | |
| # run: cargo test --release -p sn_node --features="local" --test double_spend -- --nocapture --test-threads=1 | |
| # env: | |
| # CARGO_TARGET_DIR: ${{ matrix.os == 'windows-latest' && './test-target' || '.' }} | |
| # timeout-minutes: 25 | |
| # - name: Stop the local network and upload logs | |
| # if: always() | |
| # uses: maidsafe/sn-local-testnet-action@main | |
| # with: | |
| # action: stop | |
| # log_file_prefix: safe_test_logs_spend | |
| # platform: ${{ matrix.os }} | |
| # # runs with increased node count | |
| # spend_simulation: | |
| # if: "!startsWith(github.event.head_commit.message, 'chore(release):')" | |
| # name: spend simulation | |
| # runs-on: ${{ matrix.os }} | |
| # strategy: | |
| # matrix: | |
| # os: [ ubuntu-latest, windows-latest, macos-latest ] | |
| # steps: | |
| # - uses: actions/checkout@v4 | |
| # - name: Install Rust | |
| # uses: dtolnay/rust-toolchain@stable | |
| # - uses: Swatinem/rust-cache@v2 | |
| # - name: Build binaries | |
| # run: cargo build --release --features=local --bin safenode | |
| # timeout-minutes: 30 | |
| # - name: Build faucet binary | |
| # run: cargo build --release --bin faucet --features="local,gifting" | |
| # timeout-minutes: 30 | |
| # - name: Build testing executable | |
| # run: cargo test --release -p sn_node --features=local --test spend_simulation --no-run | |
| # env: | |
| # # only set the target dir for windows to bypass the linker issue. | |
| # # happens if we build the node manager via testnet action | |
| # CARGO_TARGET_DIR: ${{ matrix.os == 'windows-latest' && './test-target' || '.' }} | |
| # timeout-minutes: 30 | |
| # - name: Start a local network | |
| # uses: maidsafe/sn-local-testnet-action@main | |
| # with: | |
| # action: start | |
| # interval: 2000 | |
| # node-count: 50 | |
| # node-path: target/release/safenode | |
| # faucet-path: target/release/faucet | |
| # platform: ${{ matrix.os }} | |
| # build: true | |
| # - name: Check SAFE_PEERS was set | |
| # shell: bash | |
| # run: | | |
| # if [[ -z "$SAFE_PEERS" ]]; then | |
| # echo "The SAFE_PEERS variable has not been set" | |
| # exit 1 | |
| # else | |
| # echo "SAFE_PEERS has been set to $SAFE_PEERS" | |
| # fi | |
| # - name: execute the spend simulation | |
| # run: cargo test --release -p sn_node --features="local" --test spend_simulation -- --nocapture | |
| # env: | |
| # CARGO_TARGET_DIR: ${{ matrix.os == 'windows-latest' && './test-target' || '.' }} | |
| # timeout-minutes: 25 | |
| # - name: Stop the local network and upload logs | |
| # if: always() | |
| # uses: maidsafe/sn-local-testnet-action@main | |
| # with: | |
| # action: stop | |
| # log_file_prefix: safe_test_logs_spend_simulation | |
| # platform: ${{ matrix.os }} | |
| # token_distribution_test: | |
| # if: "!startsWith(github.event.head_commit.message, 'chore(release):')" | |
| # name: token distribution test | |
| # runs-on: ${{ matrix.os }} | |
| # strategy: | |
| # matrix: | |
| # os: [ubuntu-latest, windows-latest, macos-latest] | |
| # steps: | |
| # - uses: actions/checkout@v4 | |
| # - name: Install Rust | |
| # uses: dtolnay/rust-toolchain@stable | |
| # - uses: Swatinem/rust-cache@v2 | |
| # - name: Build binaries | |
| # run: cargo build --release --features=local,distribution --bin safenode | |
| # timeout-minutes: 35 | |
| # - name: Build faucet binary | |
| # run: cargo build --release --features=local,distribution,gifting --bin faucet | |
| # timeout-minutes: 35 | |
| # - name: Build testing executable | |
| # run: cargo test --release --features=local,distribution --no-run | |
| # env: | |
| # # only set the target dir for windows to bypass the linker issue. | |
| # # happens if we build the node manager via testnet action | |
| # CARGO_TARGET_DIR: ${{ matrix.os == 'windows-latest' && './test-target' || '.' }} | |
| # timeout-minutes: 35 | |
| # - name: Start a local network | |
| # uses: maidsafe/sn-local-testnet-action@main | |
| # with: | |
| # action: start | |
| # interval: 2000 | |
| # node-path: target/release/safenode | |
| # faucet-path: target/release/faucet | |
| # platform: ${{ matrix.os }} | |
| # build: true | |
| # - name: Check SAFE_PEERS was set | |
| # shell: bash | |
| # run: | | |
| # if [[ -z "$SAFE_PEERS" ]]; then | |
| # echo "The SAFE_PEERS variable has not been set" | |
| # exit 1 | |
| # else | |
| # echo "SAFE_PEERS has been set to $SAFE_PEERS" | |
| # fi | |
| # - name: execute token_distribution tests | |
| # run: cargo test --release --features=local,distribution token_distribution -- --nocapture --test-threads=1 | |
| # env: | |
| # SN_LOG: "all" | |
| # CARGO_TARGET_DIR: ${{ matrix.os == 'windows-latest' && './test-target' || '.' }} | |
| # timeout-minutes: 25 | |
| # - name: Stop the local network and upload logs | |
| # if: always() | |
| # uses: maidsafe/sn-local-testnet-action@main | |
| # with: | |
| # action: stop | |
| # log_file_prefix: safe_test_logs_token_distribution | |
| # platform: ${{ matrix.os }} | |
| churn: | |
| if: "!startsWith(github.event.head_commit.message, 'chore(release):')" | |
| name: Network churning tests | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| matrix: | |
| include: | |
| - os: ubuntu-latest | |
| node_data_path: /home/runner/.local/share/safe/node | |
| safe_path: /home/runner/.local/share/safe | |
| - os: windows-latest | |
| node_data_path: C:\\Users\\runneradmin\\AppData\\Roaming\\safe\\node | |
| safe_path: C:\\Users\\runneradmin\\AppData\\Roaming\\safe | |
| - os: macos-latest | |
| node_data_path: /Users/runner/Library/Application Support/safe/node | |
| safe_path: /Users/runner/Library/Application Support/safe | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: dtolnay/rust-toolchain@stable | |
| - uses: Swatinem/rust-cache@v2 | |
| - name: Build binaries | |
| run: cargo build --release --features local --bin safenode | |
| timeout-minutes: 30 | |
| - name: Build churn tests | |
| run: cargo test --release -p sn_node --features=local --test data_with_churn --no-run | |
| env: | |
| # only set the target dir for windows to bypass the linker issue. | |
| # happens if we build the node manager via testnet action | |
| CARGO_TARGET_DIR: ${{ matrix.os == 'windows-latest' && './test-target' || '.' }} | |
| timeout-minutes: 30 | |
| - name: Start a local network | |
| uses: maidsafe/sn-local-testnet-action@main | |
| with: | |
| action: start | |
| enable-evm-testnet: true | |
| node-path: target/release/safenode | |
| platform: ${{ matrix.os }} | |
| build: true | |
| - name: Check if SAFE_PEERS and EVM_NETWORK are set | |
| shell: bash | |
| run: | | |
| if [[ -z "$SAFE_PEERS" ]]; then | |
| echo "The SAFE_PEERS variable has not been set" | |
| exit 1 | |
| elif [[ -z "$EVM_NETWORK" ]]; then | |
| echo "The EVM_NETWORK variable has not been set" | |
| exit 1 | |
| else | |
| echo "SAFE_PEERS has been set to $SAFE_PEERS" | |
| echo "EVM_NETWORK has been set to $EVM_NETWORK" | |
| fi | |
| - name: Chunks data integrity during nodes churn | |
| run: cargo test --release -p sn_node --features=local --test data_with_churn -- --nocapture | |
| env: | |
| TEST_DURATION_MINS: 5 | |
| TEST_TOTAL_CHURN_CYCLES: 15 | |
| SN_LOG: "all" | |
| CARGO_TARGET_DIR: ${{ matrix.os == 'windows-latest' && './test-target' || '.' }} | |
| timeout-minutes: 30 | |
| # Sleep for a while to allow restarted nodes can be detected by others | |
| - name: Sleep a while | |
| run: sleep 300 | |
| - name: Stop the local network and upload logs | |
| if: always() | |
| uses: maidsafe/sn-local-testnet-action@main | |
| with: | |
| action: stop | |
| log_file_prefix: safe_test_logs_churn | |
| platform: ${{ matrix.os }} | |
| - name: Get total node count | |
| shell: bash | |
| timeout-minutes: 1 | |
| run: | | |
| node_count=$(ls "${{ matrix.node_data_path }}" | wc -l) | |
| echo "Node dir count is $node_count" | |
| - name: Get restart of nodes using rg | |
| shell: bash | |
| timeout-minutes: 1 | |
| # get the counts, then the specific line, and then the digit count only | |
| # then check we have an expected level of restarts | |
| # TODO: make this use an env var, or relate to testnet size | |
| run: | | |
| restart_count=$(rg "Node is restarting in" "${{ matrix.node_data_path }}" -c --stats | \ | |
| rg "(\d+) matches" | rg "\d+" -o) | |
| echo "Restarted $restart_count nodes" | |
| # `PeerRemovedFromRoutingTable` now only happens when a peer reported as `BadNode`. | |
| # Otherwise kad will remove a `dropped out node` directly from RT. | |
| # So, the detection of the removal explicity will now have much less chance, | |
| # due to the removal of connection_issue tracking. | |
| - name: Get peers removed from nodes using rg | |
| shell: bash | |
| timeout-minutes: 1 | |
| run: | | |
| peer_removed=$(rg "PeerRemovedFromRoutingTable" "${{ matrix.node_data_path }}" -c --stats | \ | |
| rg "(\d+) matches" | rg "\d+" -o) || { echo "Failed to extract peer removal count"; exit 0; } | |
| if [ -z "$peer_removed" ]; then | |
| echo "No peer removal count found" | |
| exit 1 | |
| fi | |
| echo "PeerRemovedFromRoutingTable $peer_removed times" | |
| # TODO: reenable this once the testnet dir creation is tidied up to avoid a large count here | |
| # if [ $restart_count -lt $node_count ]; then | |
| # echo "Restart count of: $restart_count is less than the node count of: $node_count" | |
| # exit 1 | |
| # fi | |
| - name: Verify data replication using rg | |
| shell: bash | |
| timeout-minutes: 1 | |
| # get the counts, then the specific line, and then the digit count only | |
| # then check we have an expected level of replication | |
| # TODO: make this use an env var, or relate to testnet size | |
| run: | | |
| fetching_attempt_count=$(rg "FetchingKeysForReplication" "${{ matrix.node_data_path }}" -c --stats | \ | |
| rg "(\d+) matches" | rg "\d+" -o) | |
| echo "Carried out $fetching_attempt_count fetching attempts" | |
| node_count=$(ls "${{ matrix.node_data_path }}" | wc -l) | |
| if [ $fetching_attempt_count -lt $node_count ]; then | |
| echo "Replication fetching attempts of: $fetching_attempt_count is less than the node count of: $node_count" | |
| exit 1 | |
| fi | |
| # Only error out after uploading the logs | |
| - name: Don't log raw data | |
| if: matrix.os != 'windows-latest' # causes error | |
| shell: bash | |
| timeout-minutes: 10 | |
| run: | | |
| if ! rg '^' "${{ matrix.safe_path }}"/*/*/logs | awk 'length($0) > 15000 { print; exit 1 }' | |
| then | |
| echo "We are logging an extremely large data" | |
| exit 1 | |
| fi | |
| verify_data_location_routing_table: | |
| if: "!startsWith(github.event.head_commit.message, 'chore(release):')" | |
| name: Verify data location and Routing Table | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| matrix: | |
| include: | |
| - os: ubuntu-latest | |
| node_data_path: /home/runner/.local/share/safe/node | |
| safe_path: /home/runner/.local/share/safe | |
| - os: windows-latest | |
| node_data_path: C:\\Users\\runneradmin\\AppData\\Roaming\\safe\\node | |
| safe_path: C:\\Users\\runneradmin\\AppData\\Roaming\\safe | |
| - os: macos-latest | |
| node_data_path: /Users/runner/Library/Application Support/safe/node | |
| safe_path: /Users/runner/Library/Application Support/safe | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: dtolnay/rust-toolchain@stable | |
| - uses: Swatinem/rust-cache@v2 | |
| - name: Build binaries | |
| run: cargo build --release --features local --bin safenode | |
| timeout-minutes: 30 | |
| - name: Build data location and routing table tests | |
| run: cargo test --release -p sn_node --features=local --test verify_data_location --test verify_routing_table --no-run | |
| env: | |
| # only set the target dir for windows to bypass the linker issue. | |
| # happens if we build the node manager via testnet action | |
| CARGO_TARGET_DIR: ${{ matrix.os == 'windows-latest' && './test-target' || '.' }} | |
| timeout-minutes: 30 | |
| - name: Start a local network | |
| uses: maidsafe/sn-local-testnet-action@main | |
| with: | |
| action: start | |
| enable-evm-testnet: true | |
| node-path: target/release/safenode | |
| platform: ${{ matrix.os }} | |
| build: true | |
| - name: Check if SAFE_PEERS and EVM_NETWORK are set | |
| shell: bash | |
| run: | | |
| if [[ -z "$SAFE_PEERS" ]]; then | |
| echo "The SAFE_PEERS variable has not been set" | |
| exit 1 | |
| elif [[ -z "$EVM_NETWORK" ]]; then | |
| echo "The EVM_NETWORK variable has not been set" | |
| exit 1 | |
| else | |
| echo "SAFE_PEERS has been set to $SAFE_PEERS" | |
| echo "EVM_NETWORK has been set to $EVM_NETWORK" | |
| fi | |
| - name: Verify the routing tables of the nodes | |
| run: cargo test --release -p sn_node --features="local" --test verify_routing_table -- --nocapture | |
| env: | |
| CARGO_TARGET_DIR: ${{ matrix.os == 'windows-latest' && './test-target' || '.' }} | |
| timeout-minutes: 5 | |
| - name: Verify the location of the data on the network | |
| run: cargo test --release -p sn_node --features="local" --test verify_data_location -- --nocapture | |
| env: | |
| CHURN_COUNT: 6 | |
| SN_LOG: "all" | |
| CARGO_TARGET_DIR: ${{ matrix.os == 'windows-latest' && './test-target' || '.' }} | |
| timeout-minutes: 25 | |
| - name: Verify the routing tables of the nodes | |
| run: cargo test --release -p sn_node --features="local" --test verify_routing_table -- --nocapture | |
| env: | |
| CARGO_TARGET_DIR: ${{ matrix.os == 'windows-latest' && './test-target' || '.' }} | |
| timeout-minutes: 5 | |
| # Sleep for a while to allow restarted nodes can be detected by others | |
| - name: Sleep a while | |
| run: sleep 300 | |
| - name: Stop the local network and upload logs | |
| if: always() | |
| uses: maidsafe/sn-local-testnet-action@main | |
| with: | |
| action: stop | |
| log_file_prefix: safe_test_logs_data_location | |
| platform: ${{ matrix.os }} | |
| - name: Verify restart of nodes using rg | |
| shell: bash | |
| timeout-minutes: 1 | |
| # get the counts, then the specific line, and then the digit count only | |
| # then check we have an expected level of restarts | |
| # | |
| # `PeerRemovedFromRoutingTable` now only happens when a peer reported as `BadNode`. | |
| # Otherwise kad will remove a `dropped out node` directly from RT. | |
| # So, the detection of the removal explicity will now have much less chance, | |
| # due to the removal of connection_issue tracking. | |
| # | |
| # With the further reduction of replication frequency, | |
| # it now becomes harder to detect a `dropped out node` as a `failed to replicate` node. | |
| # Hence now remove the assertion check and replace with a print out only. | |
| run: | | |
| node_count=$(ls "${{ matrix.node_data_path }}" | wc -l) | |
| echo "Node dir count is $node_count" | |
| restart_count=$(rg "Node is restarting in" "${{ matrix.node_data_path }}" -c --stats | \ | |
| rg "(\d+) matches" | rg "\d+" -o) | |
| echo "Restart $restart_count nodes" | |
| if ! rg "PeerRemovedFromRoutingTable" "${{ matrix.node_data_path }}" -c --stats | |
| then | |
| echo "No peer removal count found" | |
| exit 0 | |
| fi | |
| peer_removed=$(rg "PeerRemovedFromRoutingTable" "${{ matrix.node_data_path }}" -c --stats | \ | |
| rg "(\d+) matches" | rg "\d+" -o) | |
| echo "PeerRemovedFromRoutingTable $peer_removed times" | |
| # Only error out after uploading the logs | |
| - name: Don't log raw data | |
| if: matrix.os != 'windows-latest' # causes error | |
| shell: bash | |
| timeout-minutes: 10 | |
| run: | | |
| if ! rg '^' "${{ matrix.safe_path }}"/*/*/logs | awk 'length($0) > 15000 { print; exit 1 }' | |
| then | |
| echo "We are logging an extremely large data" | |
| exit 1 | |
| fi | |
| # faucet_test: | |
| # if: "!startsWith(github.event.head_commit.message, 'chore(release):')" | |
| # name: Faucet test | |
| # runs-on: ubuntu-latest | |
| # steps: | |
| # - uses: actions/checkout@v4 | |
| # - name: Install Rust | |
| # uses: dtolnay/rust-toolchain@stable | |
| # - uses: Swatinem/rust-cache@v2 | |
| # - name: install ripgrep | |
| # shell: bash | |
| # run: sudo apt-get install -y ripgrep | |
| # - name: Build binaries | |
| # run: cargo build --release --bin safenode --bin safe | |
| # timeout-minutes: 30 | |
| # - name: Build faucet binary | |
| # run: cargo build --release --bin faucet --features gifting | |
| # timeout-minutes: 30 | |
| # - name: Start a local network | |
| # uses: maidsafe/sn-local-testnet-action@main | |
| # with: | |
| # action: start | |
| # interval: 2000 | |
| # node-path: target/release/safenode | |
| # faucet-path: target/release/faucet | |
| # platform: ubuntu-latest | |
| # build: true | |
| # - name: Check we're _not_ warned about using default genesis | |
| # run: | | |
| # if rg "USING DEFAULT" "${{ matrix.safe_path }}"/*/*/logs; then | |
| # exit 1 | |
| # fi | |
| # shell: bash | |
| # - name: Move built binaries and clear out target dir | |
| # shell: bash | |
| # run: | | |
| # mv target/release/faucet ~/faucet | |
| # mv target/release/safe ~/safe | |
| # rm -rf target | |
| # - name: Check SAFE_PEERS was set | |
| # shell: bash | |
| # run: | | |
| # if [[ -z "$SAFE_PEERS" ]]; then | |
| # echo "The SAFE_PEERS variable has not been set" | |
| # exit 1 | |
| # else | |
| # echo "SAFE_PEERS has been set to $SAFE_PEERS" | |
| # fi | |
| # - name: Create and fund a wallet first time | |
| # run: | | |
| # ~/safe --log-output-dest=data-dir wallet create --no-password | |
| # ~/faucet --log-output-dest=data-dir send 100000000 $(~/safe --log-output-dest=data-dir wallet address | tail -n 1) | tail -n 1 1>first.txt | |
| # echo "----------" | |
| # cat first.txt | |
| # env: | |
| # SN_LOG: "all" | |
| # timeout-minutes: 5 | |
| # - name: Move faucet log to the working folder | |
| # run: | | |
| # echo "SAFE_DATA_PATH has: " | |
| # ls -l $SAFE_DATA_PATH | |
| # echo "test_faucet foder has: " | |
| # ls -l $SAFE_DATA_PATH/test_faucet | |
| # echo "logs folder has: " | |
| # ls -l $SAFE_DATA_PATH/test_faucet/logs | |
| # mv $SAFE_DATA_PATH/test_faucet/logs/faucet.log ./faucet_log.log | |
| # env: | |
| # SN_LOG: "all" | |
| # SAFE_DATA_PATH: /home/runner/.local/share/safe | |
| # continue-on-error: true | |
| # if: always() | |
| # timeout-minutes: 1 | |
| # - name: Upload faucet log | |
| # uses: actions/upload-artifact@main | |
| # with: | |
| # name: faucet_test_first_faucet_log | |
| # path: faucet_log.log | |
| # continue-on-error: true | |
| # if: always() | |
| # - name: Cleanup prior faucet and cashnotes | |
| # run: | | |
| # ls -l /home/runner/.local/share | |
| # ls -l /home/runner/.local/share/safe | |
| # rm -rf /home/runner/.local/share/safe/test_faucet | |
| # rm -rf /home/runner/.local/share/safe/test_genesis | |
| # rm -rf /home/runner/.local/share/safe/autonomi | |
| # env: | |
| # SN_LOG: "all" | |
| # timeout-minutes: 5 | |
| # - name: Create a new wallet | |
| # run: ~/safe --log-output-dest=data-dir wallet create --no-password | |
| # env: | |
| # SN_LOG: "all" | |
| # timeout-minutes: 5 | |
| # - name: Attempt second faucet genesis disbursement | |
| # run: ~/faucet --log-output-dest=data-dir send 100000000 $(~/safe --log-output-dest=data-dir wallet address | tail -n 1) > second.txt 2>&1 || true | |
| # env: | |
| # SN_LOG: "all" | |
| # timeout-minutes: 5 | |
| # - name: cat second.txt | |
| # run: cat second.txt | |
| # env: | |
| # SN_LOG: "all" | |
| # timeout-minutes: 5 | |
| # - name: Verify a second disbursement is rejected | |
| # run: | | |
| # if grep "Faucet disbursement has already occured" second.txt; then | |
| # echo "Duplicated faucet rejected" | |
| # else | |
| # echo "Duplicated faucet not rejected!" | |
| # exit 1 | |
| # fi | |
| # env: | |
| # SN_LOG: "all" | |
| # timeout-minutes: 5 | |
| # - name: Create and fund a wallet with different keypair | |
| # run: | | |
| # ls -l /home/runner/.local/share | |
| # ls -l /home/runner/.local/share/safe | |
| # rm -rf /home/runner/.local/share/safe/test_faucet | |
| # rm -rf /home/runner/.local/share/safe/test_genesis | |
| # rm -rf /home/runner/.local/share/safe/autonomi | |
| # ~/safe --log-output-dest=data-dir wallet create --no-password | |
| # if GENESIS_PK=a9925296499299fdbf4412509d342a92e015f5b996e9acd1d2ab7f2326e3ad05934326efdc345345a95e973ac1bb6637 GENESIS_SK=40f6bbc870355c68138ac70b450b6425af02b49874df3f141b7018378ceaac66 nohup ~/faucet --log-output-dest=data-dir send 100000000 $(~/safe --log-output-dest=data-dir wallet address | tail -n 1); then | |
| # echo "Faucet with different genesis key not rejected!" | |
| # exit 1 | |
| # else | |
| # echo "Faucet with different genesis key rejected" | |
| # fi | |
| # env: | |
| # SN_LOG: "all" | |
| # timeout-minutes: 5 | |
| # - name: Build faucet binary again without the gifting feature | |
| # run: cargo build --release --bin faucet | |
| # timeout-minutes: 30 | |
| # - name: Start up a faucet in server mode | |
| # run: | | |
| # ls -l /home/runner/.local/share | |
| # ls -l /home/runner/.local/share/safe | |
| # rm -rf /home/runner/.local/share/safe/test_faucet | |
| # rm -rf /home/runner/.local/share/safe/test_genesis | |
| # rm -rf /home/runner/.local/share/safe/autonomi | |
| # target/release/faucet server & | |
| # sleep 60 | |
| # env: | |
| # SN_LOG: "all" | |
| # timeout-minutes: 5 | |
| # - name: check there is no upload happens | |
| # shell: bash | |
| # run: | | |
| # if grep -r "NanoTokens(10) }, Output" $NODE_DATA_PATH | |
| # then | |
| # echo "We find ongoing upload !" | |
| # exit 1 | |
| # fi | |
| # env: | |
| # NODE_DATA_PATH: /home/runner/.local/share/safe/node | |
| # timeout-minutes: 1 | |
| # - name: Stop the local network and upload logs | |
| # if: always() | |
| # uses: maidsafe/sn-local-testnet-action@main | |
| # with: | |
| # action: stop | |
| # platform: ubuntu-latest | |
| # log_file_prefix: safe_test_logs_faucet | |
| large_file_upload_test: | |
| if: "!startsWith(github.event.head_commit.message, 'chore(release):')" | |
| name: Large file upload | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Install Rust | |
| uses: dtolnay/rust-toolchain@stable | |
| - uses: Swatinem/rust-cache@v2 | |
| - name: install ripgrep | |
| shell: bash | |
| run: sudo apt-get install -y ripgrep | |
| - name: Check the available space | |
| run: | | |
| df | |
| echo "Home dir:" | |
| du -sh /home/runner/ | |
| echo "Home subdirs:" | |
| du -sh /home/runner/*/ | |
| echo "PWD:" | |
| du -sh . | |
| echo "PWD subdirs:" | |
| du -sh */ | |
| - name: Download material (135MB) | |
| shell: bash | |
| run: | | |
| mkdir test_data_1 | |
| cd test_data_1 | |
| wget https://sn-node.s3.eu-west-2.amazonaws.com/joshuef/Qi930/safe-qiWithListeners-x86_64.tar.gz | |
| wget https://sn-node.s3.eu-west-2.amazonaws.com/joshuef/Qi930/safenode-qiWithListeners-x86_64.tar.gz | |
| ls -l | |
| cd .. | |
| tar -cvzf test_data_1.tar.gz test_data_1 | |
| ls -l | |
| - name: Build binaries | |
| run: cargo build --release --features local --bin safenode --bin autonomi | |
| timeout-minutes: 30 | |
| - name: Start a local network | |
| uses: maidsafe/sn-local-testnet-action@main | |
| with: | |
| action: start | |
| enable-evm-testnet: true | |
| node-path: target/release/safenode | |
| platform: ubuntu-latest | |
| build: true | |
| - name: Check if SAFE_PEERS and EVM_NETWORK are set | |
| shell: bash | |
| run: | | |
| if [[ -z "$SAFE_PEERS" ]]; then | |
| echo "The SAFE_PEERS variable has not been set" | |
| exit 1 | |
| elif [[ -z "$EVM_NETWORK" ]]; then | |
| echo "The EVM_NETWORK variable has not been set" | |
| exit 1 | |
| else | |
| echo "SAFE_PEERS has been set to $SAFE_PEERS" | |
| echo "EVM_NETWORK has been set to $EVM_NETWORK" | |
| fi | |
| - name: Check the available space post download | |
| run: | | |
| df | |
| echo "Home dir:" | |
| du -sh /home/runner/ | |
| echo "Home subdirs:" | |
| du -sh /home/runner/*/ | |
| echo "PWD:" | |
| du -sh . | |
| echo "PWD subdirs:" | |
| du -sh */ | |
| - name: export default secret key | |
| run: echo "SECRET_KEY=0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80" >> $GITHUB_ENV | |
| shell: bash | |
| - name: File upload | |
| run: ./target/release/autonomi --log-output-dest=data-dir file upload "./test_data_1.tar.gz" > ./upload_output 2>&1 | |
| env: | |
| SN_LOG: "v" | |
| timeout-minutes: 5 | |
| - name: showing the upload terminal output | |
| run: cat upload_output | |
| shell: bash | |
| if: always() | |
| - name: parse address | |
| run: | | |
| UPLOAD_ADDRESS=$(rg "At address: ([0-9a-f]*)" -o -r '$1' ./upload_output) | |
| echo "UPLOAD_ADDRESS=$UPLOAD_ADDRESS" >> $GITHUB_ENV | |
| shell: bash | |
| - name: File Download | |
| run: ./target/release/autonomi --log-output-dest=data-dir file download ${{ env.UPLOAD_ADDRESS }} ./downloaded_resources > ./download_output 2>&1 | |
| env: | |
| SN_LOG: "v" | |
| timeout-minutes: 5 | |
| - name: showing the download terminal output | |
| run: | | |
| cat download_output | |
| ls -l | |
| cd downloaded_resources | |
| ls -l | |
| shell: bash | |
| if: always() | |
| - name: Confirming connection errors | |
| shell: bash | |
| timeout-minutes: 1 | |
| env: | |
| NODE_DATA_PATH: /home/runner/.local/share/safe/node | |
| run: | | |
| incoming_connection_errors=$(rg "IncomingConnectionError" $NODE_DATA_PATH -c --stats | \ | |
| rg "(\d+) matches" | rg "\d+" -o) || { echo "Failed to find IncomingConnectionError error"; exit 0; } | |
| if [ -z "$incoming_connection_errors" ]; then | |
| echo "Doesn't find any IncomingConnectionError error !" | |
| else | |
| echo "Found $incoming_connection_errors IncomingConnectionError errors." | |
| fi | |
| if ! rg "UnexpectedEof" $NODE_DATA_PATH -c --stats; then | |
| echo "Doesn't find any UnexpectedEof error !" | |
| else | |
| echo "Found errors." | |
| exit 1 | |
| fi | |
| - name: Stop the local network and upload logs | |
| if: always() | |
| uses: maidsafe/sn-local-testnet-action@main | |
| with: | |
| action: stop | |
| platform: ubuntu-latest | |
| log_file_prefix: safe_test_logs_large_file_upload_no_ws | |
| build: true | |
| # replication_bench_with_heavy_upload: | |
| # if: "!startsWith(github.event.head_commit.message, 'chore(release):')" | |
| # name: Replication bench with heavy upload | |
| # runs-on: ubuntu-latest | |
| # env: | |
| # CLIENT_DATA_PATH: /home/runner/.local/share/safe/autonomi | |
| # steps: | |
| # - uses: actions/checkout@v4 | |
| # - name: Install Rust | |
| # uses: dtolnay/rust-toolchain@stable | |
| # - uses: Swatinem/rust-cache@v2 | |
| # - name: install ripgrep | |
| # shell: bash | |
| # run: sudo apt-get install -y ripgrep | |
| # - name: Download materials to create two 300MB test_files to be uploaded by client | |
| # shell: bash | |
| # run: | | |
| # mkdir test_data_1 | |
| # cd test_data_1 | |
| # wget https://sn-node.s3.eu-west-2.amazonaws.com/joshuef/Qi930/safe-qiWithListeners-x86_64.tar.gz | |
| # wget https://sn-node.s3.eu-west-2.amazonaws.com/joshuef/Qi930/safenode-qiWithListeners-x86_64.tar.gz | |
| # wget https://sn-node.s3.eu-west-2.amazonaws.com/joshuef/Qi930/safenode_rpc_client-qiWithListeners-x86_64.tar.gz | |
| # wget https://sn-node.s3.eu-west-2.amazonaws.com/joshuef/QiSubdivisionBranch/faucet-qilesssubs-x86_64.tar.gz | |
| # wget https://sn-node.s3.eu-west-2.amazonaws.com/joshuef/QiSubdivisionBranch/safe-qilesssubs-x86_64.tar.gz | |
| # ls -l | |
| # cd .. | |
| # tar -cvzf test_data_1.tar.gz test_data_1 | |
| # mkdir test_data_2 | |
| # cd test_data_2 | |
| # wget https://sn-node.s3.eu-west-2.amazonaws.com/joshuef/QiSubdivisionBranch/safenode-qilesssubs-x86_64.tar.gz | |
| # wget https://sn-node.s3.eu-west-2.amazonaws.com/joshuef/QiSubdivisionBranch/safenode_rpc_client-qilesssubs-x86_64.tar.gz | |
| # wget https://sn-node.s3.eu-west-2.amazonaws.com/joshuef/RemoveArtificalReplPush/faucet-DebugMem-x86_64.tar.gz | |
| # wget https://sn-node.s3.eu-west-2.amazonaws.com/joshuef/RemoveArtificalReplPush/safe-DebugMem-x86_64.tar.gz | |
| # wget https://sn-node.s3.eu-west-2.amazonaws.com/joshuef/RemoveArtificalReplPush/safenode-DebugMem-x86_64.tar.gz | |
| # ls -l | |
| # cd .. | |
| # tar -cvzf test_data_2.tar.gz test_data_2 | |
| # ls -l | |
| # mkdir test_data_3 | |
| # cd test_data_3 | |
| # wget https://sn-node.s3.eu-west-2.amazonaws.com/joshuef/RemoveArtificalReplPush/safenode_rpc_client-DebugMem-x86_64.tar.gz | |
| # wget https://sn-node.s3.eu-west-2.amazonaws.com/joshuef/RemoveArtificalReplPush2/faucet-DebugMem-x86_64.tar.gz | |
| # wget https://sn-node.s3.eu-west-2.amazonaws.com/joshuef/RemoveArtificalReplPush2/safe-DebugMem-x86_64.tar.gz | |
| # wget https://sn-node.s3.eu-west-2.amazonaws.com/joshuef/RemoveArtificalReplPush2/safenode-DebugMem-x86_64.tar.gz | |
| # wget https://sn-node.s3.eu-west-2.amazonaws.com/joshuef/RemoveArtificalReplPush2/safenode_rpc_client-DebugMem-x86_64.tar.gz | |
| # ls -l | |
| # cd .. | |
| # tar -cvzf test_data_3.tar.gz test_data_3 | |
| # ls -l | |
| # df | |
| # - name: Build binaries | |
| # run: cargo build --release --bin safenode --bin safe | |
| # timeout-minutes: 30 | |
| # - name: Build faucet binary | |
| # run: cargo build --release --bin faucet --features gifting | |
| # timeout-minutes: 30 | |
| # - name: Start a local network | |
| # uses: maidsafe/sn-local-testnet-action@main | |
| # with: | |
| # action: start | |
| # interval: 2000 | |
| # node-path: target/release/safenode | |
| # faucet-path: target/release/faucet | |
| # platform: ubuntu-latest | |
| # build: true | |
| # - name: Check SAFE_PEERS was set | |
| # shell: bash | |
| # run: | | |
| # if [[ -z "$SAFE_PEERS" ]]; then | |
| # echo "The SAFE_PEERS variable has not been set" | |
| # exit 1 | |
| # else | |
| # echo "SAFE_PEERS has been set to $SAFE_PEERS" | |
| # fi | |
| # - name: Create and fund a wallet to pay for files storage | |
| # run: | | |
| # ./target/release/safe --log-output-dest=data-dir wallet create --no-password | |
| # ./target/release/faucet --log-output-dest=data-dir send 100000000 $(./target/release/safe --log-output-dest=data-dir wallet address | tail -n 1) | tail -n 1 > transfer_hex | |
| # ./target/release/safe --log-output-dest=data-dir wallet receive --file transfer_hex | |
| # env: | |
| # SN_LOG: "all" | |
| # timeout-minutes: 5 | |
| # - name: Start a client to upload first file | |
| # run: ./target/release/safe --log-output-dest=data-dir files upload "./test_data_1.tar.gz" --retry-strategy quick | |
| # env: | |
| # SN_LOG: "all" | |
| # timeout-minutes: 5 | |
| # - name: Ensure no leftover cash_notes and payment files | |
| # run: | | |
| # expected_cash_notes_files="1" | |
| # expected_payment_files="0" | |
| # pwd | |
| # ls $CLIENT_DATA_PATH/ -l | |
| # ls $CLIENT_DATA_PATH/wallet -l | |
| # ls $CLIENT_DATA_PATH/wallet/cash_notes -l | |
| # cash_note_files=$(ls $CLIENT_DATA_PATH/wallet/cash_notes | wc -l) | |
| # echo "Find $cash_note_files cash_note files" | |
| # if [ $expected_cash_notes_files -lt $cash_note_files ]; then | |
| # echo "Got too many cash_note files leftover: $cash_note_files" | |
| # exit 1 | |
| # fi | |
| # ls $CLIENT_DATA_PATH/wallet/payments -l | |
| # payment_files=$(ls $CLIENT_DATA_PATH/wallet/payments | wc -l) | |
| # if [ $expected_payment_files -lt $payment_files ]; then | |
| # echo "Got too many payment files leftover: $payment_files" | |
| # exit 1 | |
| # fi | |
| # env: | |
| # CLIENT_DATA_PATH: /home/runner/.local/share/safe/client | |
| # timeout-minutes: 10 | |
| # - name: Wait for certain period | |
| # run: sleep 300 | |
| # timeout-minutes: 6 | |
| # - name: Use same client to upload second file | |
| # run: ./target/release/safe --log-output-dest=data-dir files upload "./test_data_2.tar.gz" --retry-strategy quick | |
| # env: | |
| # SN_LOG: "all" | |
| # timeout-minutes: 10 | |
| # - name: Ensure no leftover cash_notes and payment files | |
| # run: | | |
| # expected_cash_notes_files="1" | |
| # expected_payment_files="0" | |
| # pwd | |
| # ls $CLIENT_DATA_PATH/ -l | |
| # ls $CLIENT_DATA_PATH/wallet -l | |
| # ls $CLIENT_DATA_PATH/wallet/cash_notes -l | |
| # cash_note_files=$(find $CLIENT_DATA_PATH/wallet/cash_notes -type f | wc -l) | |
| # if (( $(echo "$cash_note_files > $expected_cash_notes_files" | bc -l) )); then | |
| # echo "Got too many cash_note files leftover: $cash_note_files when we expected $expected_cash_notes_files" | |
| # exit 1 | |
| # fi | |
| # ls $CLIENT_DATA_PATH/wallet/payments -l | |
| # payment_files=$(find $CLIENT_DATA_PATH/wallet/payments -type f | wc -l) | |
| # if (( $(echo "$payment_files > $expected_payment_files" | bc -l) )); then | |
| # echo "Got too many payment files leftover: $payment_files" | |
| # exit 1 | |
| # fi | |
| # env: | |
| # CLIENT_DATA_PATH: /home/runner/.local/share/safe/client | |
| # timeout-minutes: 10 | |
| # - name: Wait for certain period | |
| # run: sleep 300 | |
| # timeout-minutes: 6 | |
| # # Start a different client to avoid local wallet slow down with more payments handled. | |
| # - name: Start a different client | |
| # run: | | |
| # pwd | |
| # mv $CLIENT_DATA_PATH $SAFE_DATA_PATH/client_first | |
| # ls -l $SAFE_DATA_PATH | |
| # ls -l $SAFE_DATA_PATH/client_first | |
| # mkdir $SAFE_DATA_PATH/client | |
| # ls -l $SAFE_DATA_PATH | |
| # mv $SAFE_DATA_PATH/client_first/logs $CLIENT_DATA_PATH/logs | |
| # ls -l $CLIENT_DATA_PATH | |
| # ./target/release/safe --log-output-dest=data-dir wallet create --no-password | |
| # ./target/release/faucet --log-output-dest=data-dir send 100000000 $(./target/release/safe --log-output-dest=data-dir wallet address | tail -n 1) | tail -n 1 > transfer_hex | |
| # ./target/release/safe --log-output-dest=data-dir wallet receive --file transfer_hex | |
| # env: | |
| # SN_LOG: "all" | |
| # SAFE_DATA_PATH: /home/runner/.local/share/safe | |
| # CLIENT_DATA_PATH: /home/runner/.local/share/safe/client | |
| # timeout-minutes: 25 | |
| # - name: Use second client to upload third file | |
| # run: ./target/release/safe --log-output-dest=data-dir files upload "./test_data_3.tar.gz" --retry-strategy quick | |
| # env: | |
| # SN_LOG: "all" | |
| # timeout-minutes: 10 | |
| # - name: Ensure no leftover cash_notes and payment files | |
| # run: | | |
| # expected_cash_notes_files="1" | |
| # expected_payment_files="0" | |
| # pwd | |
| # ls $CLIENT_DATA_PATH/ -l | |
| # ls $CLIENT_DATA_PATH/wallet -l | |
| # ls $CLIENT_DATA_PATH/wallet/cash_notes -l | |
| # cash_note_files=$(ls $CLIENT_DATA_PATH/wallet/cash_notes | wc -l) | |
| # echo "Find $cash_note_files cash_note files" | |
| # if [ $expected_cash_notes_files -lt $cash_note_files ]; then | |
| # echo "Got too many cash_note files leftover: $cash_note_files" | |
| # exit 1 | |
| # fi | |
| # ls $CLIENT_DATA_PATH/wallet/payments -l | |
| # payment_files=$(ls $CLIENT_DATA_PATH/wallet/payments | wc -l) | |
| # if [ $expected_payment_files -lt $payment_files ]; then | |
| # echo "Got too many payment files leftover: $payment_files" | |
| # exit 1 | |
| # fi | |
| # env: | |
| # CLIENT_DATA_PATH: /home/runner/.local/share/safe/client | |
| # timeout-minutes: 10 | |
| # - name: Stop the local network and upload logs | |
| # if: always() | |
| # uses: maidsafe/sn-local-testnet-action@main | |
| # with: | |
| # action: stop | |
| # log_file_prefix: safe_test_logs_heavy_replicate_bench | |
| # platform: ubuntu-latest |