Merge branch 'main' into main

.cursorrules

@@ -0,0 +1,9 @@
+You are an AI assistant specialized in Python and Rust  development. 
+
+For python
+
+Your approach emphasizes:Clear project structure with separate directories for source code, tests, docs, and config.Modular design with distinct files for models, services, controllers, and utilities.Configuration management using environment variables.Robust error handling and logging, including context capture.Comprehensive testing with pytest.Detailed documentation using docstrings and README files.Dependency management via https://github.com/astral-sh/uv and virtual environments.Code style consistency using Ruff.CI/CD implementation with GitHub Actions or GitLab CI.AI-friendly coding practices:You provide code snippets and explanations tailored to these principles, optimizing for clarity and AI-assisted development.Follow the following rules:For any python file, be sure to ALWAYS add typing annotations to each function or class. Be sure to include return types when necessary. Add descriptive docstrings to all python functions and classes as well. Please use pep257 convention for python. Update existing docstrings if need be.Make sure you keep any comments that exist in a file.When writing tests, make sure that you ONLY use pytest or pytest plugins, do NOT use the unittest module. All tests should have typing annotations as well. All tests should be in ./tests. Be sure to create all necessary files and folders. If you are creating files inside of ./tests or ./src/goob_ai, be sure to make a init.py file if one does not exist.All tests should be fully annotated and should contain docstrings. Be sure to import  the following if TYPE_CHECKING:from _pytest.capture import CaptureFixturefrom _pytest.fixtures import FixtureRequestfrom _pytest.logging import LogCaptureFixturefrom _pytest.monkeypatch import MonkeyPatchfrom pytest_mock.plugin import MockerFixture
+
+For Rust 
+
+Please do not use unwraps or panics. Please ensure all methods are fully tested and annotated. 

.github/workflows/cross-platform.yml

@@ -34,6 +34,11 @@ jobs:
         run: wasm-pack build --dev --target=web autonomi
         timeout-minutes: 30
 
+      - name: Cargo check for WASM
+        # Allow clippy lints (these can be pedantic on WASM), but deny regular Rust warnings
+        run: cargo clippy --target=wasm32-unknown-unknown --package=autonomi --all-targets -- --allow=clippy::all --deny=warnings
+        timeout-minutes: 30
+
   websocket:
     if: "!startsWith(github.event.head_commit.message, 'chore(release):')"
     name: Standard Websocket builds

.github/workflows/merge.yml

@@ -79,9 +79,12 @@ jobs:
         run: cargo clippy --all-targets --all-features -- -Dwarnings
 
       - name: Check documentation
-        # Deny certain `rustdoc` lints that are unwanted.
-        # See https://doc.rust-lang.org/rustdoc/lints.html for lints that are 'warning' by default.
-        run: RUSTDOCFLAGS="--deny=warnings" cargo doc --no-deps
+        # Deny certain `rustdoc` lints that are unwanted with `RUSTDOCFLAGS`. See
+        # https://doc.rust-lang.org/rustdoc/lints.html for lints that are 'warning' by default.
+        # 
+        # We exclude autonomi-cli because it is not published and conflicts with the `autonomi` crate name,
+        # resulting in an error when building docs.
+        run: RUSTDOCFLAGS="--deny=warnings" cargo doc --no-deps --workspace --exclude=autonomi-cli
 
       - name: Check local is not a default feature
         shell: bash
@@ -128,7 +131,11 @@ jobs:
         timeout-minutes: 25
         run: cargo test --release --package sn_node --lib
 
-      - name: Run network tests
+      - name: Run network tests (with encrypt-records)
+        timeout-minutes: 25
+        run: cargo test --release --package sn_networking --features="open-metrics, encrypt-records"
+
+      - name: Run network tests (without encrypt-records)
         timeout-minutes: 25
         run: cargo test --release --package sn_networking --features="open-metrics"
 
@@ -654,19 +661,15 @@ jobs:
   #         platform: ${{ matrix.os }}
   #         build: true
 
-  # # incase the faucet is not ready yet
-  # - name: 30s sleep for faucet completion
-  #   run: sleep 30
-
-  # - name: Check SAFE_PEERS was set
-  #   shell: bash
-  #   run: |
-  #     if [[ -z "$SAFE_PEERS" ]]; then
-  #       echo "The SAFE_PEERS variable has not been set"
-  #       exit 1
-  #     else
-  #       echo "SAFE_PEERS has been set to $SAFE_PEERS"
-  #     fi
+  #     - name: Check SAFE_PEERS was set
+  #       shell: bash
+  #       run: |
+  #         if [[ -z "$SAFE_PEERS" ]]; then
+  #           echo "The SAFE_PEERS variable has not been set"
+  #           exit 1
+  #         else
+  #           echo "SAFE_PEERS has been set to $SAFE_PEERS"
+  #         fi
 
   #     - name: execute token_distribution tests
   #       run: cargo test --release --features=local,distribution token_distribution -- --nocapture --test-threads=1
@@ -917,7 +920,7 @@ jobs:
         uses: maidsafe/sn-local-testnet-action@main
         with:
           action: stop
-          log_file_prefix: safe_test_logs_data_location_routing_table
+          log_file_prefix: safe_test_logs_data_location
           platform: ${{ matrix.os }}
 
       - name: Verify restart of nodes using rg
@@ -1018,15 +1021,15 @@ jobs:
   #           echo "SAFE_PEERS has been set to $SAFE_PEERS"
   #         fi
 
-  # - name: Create and fund a wallet first time
-  #   run: |
-  #     ~/safe --log-output-dest=data-dir wallet create --no-password
-  #     ~/faucet --log-output-dest=data-dir send 100000000 $(~/safe --log-output-dest=data-dir wallet address | tail -n 1) | tail -n 1 1>first.txt
-  #     echo "----------"
-  #     cat first.txt
-  #   env:
-  #     SN_LOG: "all"
-  #   timeout-minutes: 5
+  #     - name: Create and fund a wallet first time
+  #       run: |
+  #         ~/safe --log-output-dest=data-dir wallet create --no-password
+  #         ~/faucet --log-output-dest=data-dir send 100000000 $(~/safe --log-output-dest=data-dir wallet address | tail -n 1) | tail -n 1 1>first.txt
+  #         echo "----------"
+  #         cat first.txt
+  #       env:
+  #         SN_LOG: "all"
+  #       timeout-minutes: 5
 
   #     - name: Move faucet log to the working folder
   #       run: |
@@ -1277,6 +1280,7 @@ jobs:
   #   runs-on: ubuntu-latest
   #   env:
   #     CLIENT_DATA_PATH: /home/runner/.local/share/safe/autonomi
+
   #   steps:
   #     - uses: actions/checkout@v4
 
@@ -1353,61 +1357,44 @@ jobs:
   #           echo "SAFE_PEERS has been set to $SAFE_PEERS"
   #         fi
 
-  # - name: Sleep 15s
-  #   shell: bash
-  #   run: sleep 15
-
-  # - name: Check faucet has been funded
-  #   shell: bash
-  #   run: |
-  #     cash_note_count=$(ls -l /home/runner/.local/share/safe/test_faucet/wallet/cash_notes/ | wc -l)
-  #     echo $cash_note_count
-  #     if [ "$cash_note_count" -eq 0 ]; then
-  #       echo "Error: Expected at least 1 cash note, but found $cash_note_count"
-  #       exit 1
-  #     fi
-
-  # - name: Create and fund a wallet to pay for files storage
-  #   run: |
-  #     ./target/release/safe --log-output-dest=data-dir wallet create --no-password
-  #     ./target/release/faucet --log-output-dest=data-dir send 100000000 $(./target/release/safe --log-output-dest=data-dir wallet address | tail -n 1) | tail -n 1 > transfer_hex
-  #     ./target/release/safe --log-output-dest=data-dir wallet receive --file transfer_hex
-  #   env:
-  #     SN_LOG: "all"
-  #   timeout-minutes: 5
+  #     - name: Create and fund a wallet to pay for files storage
+  #       run: |
+  #         ./target/release/safe --log-output-dest=data-dir wallet create --no-password
+  #         ./target/release/faucet --log-output-dest=data-dir send 100000000 $(./target/release/safe --log-output-dest=data-dir wallet address | tail -n 1) | tail -n 1 > transfer_hex
+  #         ./target/release/safe --log-output-dest=data-dir wallet receive --file transfer_hex
+  #       env:
+  #         SN_LOG: "all"
+  #       timeout-minutes: 5
 
   #     - name: Start a client to upload first file
   #       run: ./target/release/safe --log-output-dest=data-dir files upload "./test_data_1.tar.gz" --retry-strategy quick
   #       env:
   #         SN_LOG: "all"
   #       timeout-minutes: 5
 
-  # - name: Check current directories
-  #   run: |
-  #     pwd
-  #     ls $CLIENT_DATA_PATH/ -l
-  #     ls $CLIENT_DATA_PATH/wallet -l
-  #     ls $CLIENT_DATA_PATH/wallet/cash_notes -l
-  #   timeout-minutes: 1
-
-  # - name: Ensure no leftover cash_notes and payment files
-  #   run: |
-  #     expected_cash_notes_files="1"
-  #     expected_payment_files="0"
-  #     cash_note_files=$(ls $CLIENT_DATA_PATH/wallet/cash_notes | wc -l)
-  #     echo "Find $cash_note_files cash_note files"
-  #     if [ $expected_cash_notes_files -lt $cash_note_files ]; then
-  #       echo "Got too many cash_note files leftover: $cash_note_files"
-  #       exit 1
-  #     fi
-  #     ls $CLIENT_DATA_PATH/wallet/payments -l
-  #     payment_files=$(ls $CLIENT_DATA_PATH/wallet/payments | wc -l)
-  #     if [ $expected_payment_files -lt $payment_files ]; then
-  #       echo "Got too many payment files leftover: $payment_files"
-  #       exit 1
-  #     fi
-
-  #   timeout-minutes: 10
+  #     - name: Ensure no leftover cash_notes and payment files
+  #       run: |
+  #         expected_cash_notes_files="1"
+  #         expected_payment_files="0"
+  #         pwd
+  #         ls $CLIENT_DATA_PATH/ -l
+  #         ls $CLIENT_DATA_PATH/wallet -l
+  #         ls $CLIENT_DATA_PATH/wallet/cash_notes -l
+  #         cash_note_files=$(ls $CLIENT_DATA_PATH/wallet/cash_notes | wc -l)
+  #         echo "Find $cash_note_files cash_note files"
+  #         if [ $expected_cash_notes_files -lt $cash_note_files ]; then
+  #           echo "Got too many cash_note files leftover: $cash_note_files"
+  #           exit 1
+  #         fi
+  #         ls $CLIENT_DATA_PATH/wallet/payments -l
+  #         payment_files=$(ls $CLIENT_DATA_PATH/wallet/payments | wc -l)
+  #         if [ $expected_payment_files -lt $payment_files ]; then
+  #           echo "Got too many payment files leftover: $payment_files"
+  #           exit 1
+  #         fi
+  #       env:
+  #         CLIENT_DATA_PATH: /home/runner/.local/share/safe/client
+  #       timeout-minutes: 10
 
   #     - name: Wait for certain period
   #       run: sleep 300
@@ -1419,77 +1406,82 @@ jobs:
   #         SN_LOG: "all"
   #       timeout-minutes: 10
 
-  # - name: Ensure no leftover cash_notes and payment files
-  #   run: |
-  #     expected_cash_notes_files="1"
-  #     expected_payment_files="0"
-  #     pwd
-  #     ls $CLIENT_DATA_PATH/ -l
-  #     ls $CLIENT_DATA_PATH/wallet -l
-  #     ls $CLIENT_DATA_PATH/wallet/cash_notes -l
-  #     cash_note_files=$(find $CLIENT_DATA_PATH/wallet/cash_notes -type f | wc -l)
-  #     if (( $(echo "$cash_note_files > $expected_cash_notes_files" | bc -l) )); then
-  #       echo "Got too many cash_note files leftover: $cash_note_files when we expected $expected_cash_notes_files"
-  #       exit 1
-  #     fi
-  #     ls $CLIENT_DATA_PATH/wallet/payments -l
-  #     payment_files=$(find $CLIENT_DATA_PATH/wallet/payments -type f | wc -l)
-  #     if (( $(echo "$payment_files > $expected_payment_files" | bc -l) )); then
-  #       echo "Got too many payment files leftover: $payment_files"
-  #       exit 1
-  #     fi
-  #   timeout-minutes: 10
+  #     - name: Ensure no leftover cash_notes and payment files
+  #       run: |
+  #         expected_cash_notes_files="1"
+  #         expected_payment_files="0"
+  #         pwd
+  #         ls $CLIENT_DATA_PATH/ -l
+  #         ls $CLIENT_DATA_PATH/wallet -l
+  #         ls $CLIENT_DATA_PATH/wallet/cash_notes -l
+  #         cash_note_files=$(find $CLIENT_DATA_PATH/wallet/cash_notes -type f | wc -l)
+  #         if (( $(echo "$cash_note_files > $expected_cash_notes_files" | bc -l) )); then
+  #           echo "Got too many cash_note files leftover: $cash_note_files when we expected $expected_cash_notes_files"
+  #           exit 1
+  #         fi
+  #         ls $CLIENT_DATA_PATH/wallet/payments -l
+  #         payment_files=$(find $CLIENT_DATA_PATH/wallet/payments -type f | wc -l)
+  #         if (( $(echo "$payment_files > $expected_payment_files" | bc -l) )); then
+  #           echo "Got too many payment files leftover: $payment_files"
+  #           exit 1
+  #         fi
+  #       env:
+  #         CLIENT_DATA_PATH: /home/runner/.local/share/safe/client
+  #       timeout-minutes: 10
 
   #     - name: Wait for certain period
   #       run: sleep 300
   #       timeout-minutes: 6
 
-  # # Start a different client to avoid local wallet slow down with more payments handled.
-  # - name: Start a different client
-  #   run: |
-  #     pwd
-  #     mv $CLIENT_DATA_PATH $SAFE_DATA_PATH/client_first
-  #     ls -l $SAFE_DATA_PATH
-  #     ls -l $SAFE_DATA_PATH/client_first
-  #     mkdir $SAFE_DATA_PATH/client
-  #     ls -l $SAFE_DATA_PATH
-  #     mv $SAFE_DATA_PATH/client_first/logs $CLIENT_DATA_PATH/logs
-  #     ls -l $CLIENT_DATA_PATH
-  #     ./target/release/safe --log-output-dest=data-dir wallet create --no-password
-  #     ./target/release/faucet --log-output-dest=data-dir send 100000000 $(./target/release/safe --log-output-dest=data-dir wallet address | tail -n 1) | tail -n 1 > transfer_hex
-  #     ./target/release/safe --log-output-dest=data-dir wallet receive --file transfer_hex
-  #   env:
-  #     SN_LOG: "all"
-  #     SAFE_DATA_PATH: /home/runner/.local/share/safe
-  #   timeout-minutes: 25
+  #     # Start a different client to avoid local wallet slow down with more payments handled.
+  #     - name: Start a different client
+  #       run: |
+  #         pwd
+  #         mv $CLIENT_DATA_PATH $SAFE_DATA_PATH/client_first
+  #         ls -l $SAFE_DATA_PATH
+  #         ls -l $SAFE_DATA_PATH/client_first
+  #         mkdir $SAFE_DATA_PATH/client
+  #         ls -l $SAFE_DATA_PATH
+  #         mv $SAFE_DATA_PATH/client_first/logs $CLIENT_DATA_PATH/logs
+  #         ls -l $CLIENT_DATA_PATH
+  #         ./target/release/safe --log-output-dest=data-dir wallet create --no-password
+  #         ./target/release/faucet --log-output-dest=data-dir send 100000000 $(./target/release/safe --log-output-dest=data-dir wallet address | tail -n 1) | tail -n 1 > transfer_hex
+  #         ./target/release/safe --log-output-dest=data-dir wallet receive --file transfer_hex
+  #       env:
+  #         SN_LOG: "all"
+  #         SAFE_DATA_PATH: /home/runner/.local/share/safe
+  #         CLIENT_DATA_PATH: /home/runner/.local/share/safe/client
+  #       timeout-minutes: 25
 
   #     - name: Use second client to upload third file
   #       run: ./target/release/safe --log-output-dest=data-dir files upload "./test_data_3.tar.gz" --retry-strategy quick
   #       env:
   #         SN_LOG: "all"
   #       timeout-minutes: 10
 
-  # - name: Ensure no leftover cash_notes and payment files
-  #   run: |
-  #     expected_cash_notes_files="1"
-  #     expected_payment_files="0"
-  #     pwd
-  #     ls $CLIENT_DATA_PATH/ -l
-  #     ls $CLIENT_DATA_PATH/wallet -l
-  #     ls $CLIENT_DATA_PATH/wallet/cash_notes -l
-  #     cash_note_files=$(ls $CLIENT_DATA_PATH/wallet/cash_notes | wc -l)
-  #     echo "Find $cash_note_files cash_note files"
-  #     if [ $expected_cash_notes_files -lt $cash_note_files ]; then
-  #       echo "Got too many cash_note files leftover: $cash_note_files"
-  #       exit 1
-  #     fi
-  #     ls $CLIENT_DATA_PATH/wallet/payments -l
-  #     payment_files=$(ls $CLIENT_DATA_PATH/wallet/payments | wc -l)
-  #     if [ $expected_payment_files -lt $payment_files ]; then
-  #       echo "Got too many payment files leftover: $payment_files"
-  #       exit 1
-  #     fi
-  #   timeout-minutes: 10
+  #     - name: Ensure no leftover cash_notes and payment files
+  #       run: |
+  #         expected_cash_notes_files="1"
+  #         expected_payment_files="0"
+  #         pwd
+  #         ls $CLIENT_DATA_PATH/ -l
+  #         ls $CLIENT_DATA_PATH/wallet -l
+  #         ls $CLIENT_DATA_PATH/wallet/cash_notes -l
+  #         cash_note_files=$(ls $CLIENT_DATA_PATH/wallet/cash_notes | wc -l)
+  #         echo "Find $cash_note_files cash_note files"
+  #         if [ $expected_cash_notes_files -lt $cash_note_files ]; then
+  #           echo "Got too many cash_note files leftover: $cash_note_files"
+  #           exit 1
+  #         fi
+  #         ls $CLIENT_DATA_PATH/wallet/payments -l
+  #         payment_files=$(ls $CLIENT_DATA_PATH/wallet/payments | wc -l)
+  #         if [ $expected_payment_files -lt $payment_files ]; then
+  #           echo "Got too many payment files leftover: $payment_files"
+  #           exit 1
+  #         fi
+  #       env:
+  #         CLIENT_DATA_PATH: /home/runner/.local/share/safe/client
+  #       timeout-minutes: 10
 
   #     - name: Stop the local network and upload logs
   #       if: always()

.github/workflows/nightly.yml

@@ -250,7 +250,7 @@ jobs:
 
       - name: Run network tests
         timeout-minutes: 25
-        run: cargo test --release --package sn_networking --features="open-metrics"
+        run: cargo test --release --package sn_networking --features="open-metrics, encrypt-records"
 
       - name: Run protocol tests
         timeout-minutes: 25

.github/workflows/nightly_wan.yml

@@ -1,8 +1,9 @@
 name: Nightly -- Full WAN Network Tests
 
 on:
-  schedule:
-    - cron: "0 0 * * *"
+  # To do: this is broken, need to fix and enable later
+  # schedule:
+  #   - cron: "0 0 * * *"
   # enable as below for testing purpose.
   # pull_request:
   #   branches: ["*"]