From f62d02d34cb218563c9ac48431c13c5487d370b1 Mon Sep 17 00:00:00 2001 From: Ermine Jose Date: Mon, 4 Nov 2024 14:19:17 +0530 Subject: [PATCH 1/5] test: add vault register check --- .github/workflows/merge.yml | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/.github/workflows/merge.yml b/.github/workflows/merge.yml index afbf008f8c..85d932d1c2 100644 --- a/.github/workflows/merge.yml +++ b/.github/workflows/merge.yml @@ -399,6 +399,7 @@ jobs: - name: vault sync validation if: matrix.os != 'windows-latest' + shell: bash run: | NUM_OF_PUBLIC_FILES="" NUM_OF_PRIVATE_FILES="" @@ -409,27 +410,29 @@ jobs: ./target/release/autonomi --log-output-dest=data-dir file list 2>&1 > file_list.txt - # ./target/release/autonomi --log-output-dest=data-dir register list | grep archives > register_list.txt + ./target/release/autonomi register list | grep register > register_list.txt NUM_OF_PUBLIC_FILES=`cat file_list.txt | grep "public" | grep -o '[0-9]\+'` NUM_OF_PRIVATE_FILES=`cat file_list.txt | grep "private" | grep -o '[0-9]\+'` NUM_OF_REGISTERS=`cat register_list.txt | grep "register" | grep -o '[0-9]\+'` - + # when obtaining registers we get random garbage, this is the only hack that works. + NUM_OF_REGISTERS_first=${NUM_OF_REGISTERS%%[ $'\n']*} + echo "NUM_OF_REGISTERS is $NUM_OF_REGISTERS_first" ./target/release/autonomi --log-output-dest=data-dir vault load 2>&1 > vault_data.txt NUM_OF_PUBLIC_FILES_IN_VAULT=`cat vault_data.txt | grep "public" | grep -o '[0-9]\+'` NUM_OF_PRIVATE_FILES_IN_VAULT=`cat vault_data.txt| grep "private" | grep -o '[0-9]\+'` - # NUM_OF_REGISTERS_IN_VAULT=`cat vault_data.txt | grep "register" | grep -o '[0-9]\+'` + NUM_OF_REGISTERS_IN_VAULT=`cat vault_data.txt | grep "register" | grep -o '[0-9]\+'` echo "Total Num of local public files is $NUM_OF_PUBLIC_FILES and in vault is $NUM_OF_PUBLIC_FILES_IN_VAULT" echo "Total Num of local private files is $NUM_OF_PRIVATE_FILES and in vault is $NUM_OF_PRIVATE_FILES_IN_VAULT" - # echo "Total Num of local registers is $NUM_OF_REGISTERS and in vault is $NUM_OF_REGISTERS_IN_VAULT" + echo "Total Num of local registers is $NUM_OF_REGISTERS_first and in vault is $NUM_OF_REGISTERS_IN_VAULT" rm -rf file_list.txt register_list.txt vault_data.txt python3 -c 'import sys; assert sys.argv[1] == sys.argv[2], f"Error: Local public Files: {sys.argv[1]} and vault public files: {sys.argv[2]} are Not Equal"' $NUM_OF_PUBLIC_FILES $NUM_OF_PUBLIC_FILES_IN_VAULT python3 -c 'import sys; assert sys.argv[1] == sys.argv[2], f"Error: Local private Files: {sys.argv[1]} and vault private files: {sys.argv[2]} are Not Equal"' $NUM_OF_PRIVATE_FILES $NUM_OF_PRIVATE_FILES_IN_VAULT - # python3 -c 'import sys; assert sys.argv[1] == sys.argv[2], f"Error: Local registers: {sys.argv[1]} and vault registers: {sys.argv[2]} are Not Equal"' $NUM_OF_REGISTERS $NUM_OF_REGISTERS_IN_VAULT + python3 -c 'import sys; assert sys.argv[1] == sys.argv[2], f"Error: Local registers: {sys.argv[1]} and vault registers: {sys.argv[2]} are Not Equal"' $NUM_OF_REGISTERS_first $NUM_OF_REGISTERS_IN_VAULT echo "vault synced successfully!" env: SN_LOG: "v" From 1ffd45be102f629901f9de8a6465202ebb2af518 Mon Sep 17 00:00:00 2001 From: Ermine Jose Date: Mon, 4 Nov 2024 16:56:18 +0530 Subject: [PATCH 2/5] test: add windows support to vault CLI --- .github/workflows/merge.yml | 84 ++++++++++++++++++++++++++++++++++++- 1 file changed, 82 insertions(+), 2 deletions(-) diff --git a/.github/workflows/merge.yml b/.github/workflows/merge.yml index 85d932d1c2..428f42fa80 100644 --- a/.github/workflows/merge.yml +++ b/.github/workflows/merge.yml @@ -390,8 +390,28 @@ jobs: SN_LOG: "v" timeout-minutes: 25 + - name: add more files + if: matrix.os == 'windows-latest' + shell: pwsh + run: | + for ($i = 1; $i -le 100; $i++) { + # Generate a random file with PowerShell + $randomData = [System.IO.File]::OpenWrite("random_file_$i.bin") + $buffer = New-Object byte[](1024 * 1024) # 1MB buffer + [System.Security.Cryptography.RNGCryptoServiceProvider]::Create().GetBytes($buffer) + $randomData.Write($buffer, 0, $buffer.Length) + $randomData.Close() + + # Run autonomi commands + ./target/release/autonomi --log-output-dest=data-dir file upload "random_file_$i.bin" --public + ./target/release/autonomi --log-output-dest=data-dir file upload "random_file_$i.bin" + ./target/release/autonomi --log-output-dest=data-dir register create $i "random_file_$i.bin" + } + env: + SN_LOG: "v" + timeout-minutes: 25 + - name: sync the vault - if: matrix.os != 'windows-latest' run: ./target/release/autonomi --log-output-dest=data-dir vault sync env: SN_LOG: "v" @@ -438,6 +458,67 @@ jobs: SN_LOG: "v" timeout-minutes: 15 + - name: Set up variables - vault sync + if: matrix.os == 'windows-latest' + shell: pwsh + run: | + # Initialize variables to empty + $NUM_OF_PUBLIC_FILES = "" + $NUM_OF_PRIVATE_FILES = "" + $NUM_OF_REGISTERS = "" + $NUM_OF_PUBLIC_FILES_IN_VAULT = "" + $NUM_OF_PRIVATE_FILES_IN_VAULT = "" + $NUM_OF_REGISTERS_IN_VAULT = "" + + # Execute commands and save outputs to files + ./target/release/autonomi --log-output-dest=data-dir file list > file_list.txt 2>&1 + ./target/release/autonomi register list | Select-String "register" > register_list.txt + ./target/release/autonomi --log-output-dest=data-dir vault load > vault_data.txt 2>&1 + + # Parse the files and extract numbers + $NUM_OF_PUBLIC_FILES = (Select-String "public" file_list.txt | ForEach-Object { $_ -match "\d+"; $matches[0] })[0] + $NUM_OF_PRIVATE_FILES = (Select-String "private" file_list.txt | ForEach-Object { $_ -match "\d+"; $matches[0] })[0] + $NUM_OF_REGISTERS = (Select-String "register" register_list.txt | ForEach-Object { $_ -match "\d+"; $matches[0] })[0] + + # Get the first word only (PowerShell handles this without additional parsing) + $NUM_OF_REGISTERS_first = $NUM_OF_REGISTERS -split '\s+' | Select-Object -First 1 + + Write-Output "NUM_OF_REGISTERS is $NUM_OF_REGISTERS_first" + + # Continue with vault data parsing + $NUM_OF_PUBLIC_FILES_IN_VAULT = (Select-String "public" vault_data.txt | ForEach-Object { $_ -match "\d+"; $matches[0] })[0] + $NUM_OF_PRIVATE_FILES_IN_VAULT = (Select-String "private" vault_data.txt | ForEach-Object { $_ -match "\d+"; $matches[0] })[0] + $NUM_OF_REGISTERS_IN_VAULT = (Select-String "register" vault_data.txt | ForEach-Object { $_ -match "\d+"; $matches[0] })[0] + + # Output summary + Write-Output "Total Num of local public files is $NUM_OF_PUBLIC_FILES and in vault is $NUM_OF_PUBLIC_FILES_IN_VAULT" + Write-Output "Total Num of local private files is $NUM_OF_PRIVATE_FILES and in vault is $NUM_OF_PRIVATE_FILES_IN_VAULT" + Write-Output "Total Num of local registers is $NUM_OF_REGISTERS_first and in vault is $NUM_OF_REGISTERS_IN_VAULT" + + # Clean up temporary files + Remove-Item -Force file_list.txt, register_list.txt, vault_data.txt + + - name: Vault sync validation + if: matrix.os == 'windows-latest' + shell: python + run: | + import sys + + # Define the values as environment variables + NUM_OF_PUBLIC_FILES = int("$env:NUM_OF_PUBLIC_FILES") + NUM_OF_PUBLIC_FILES_IN_VAULT = int("$env:NUM_OF_PUBLIC_FILES_IN_VAULT") + NUM_OF_PRIVATE_FILES = int("$env:NUM_OF_PRIVATE_FILES") + NUM_OF_PRIVATE_FILES_IN_VAULT = int("$env:NUM_OF_PRIVATE_FILES_IN_VAULT") + NUM_OF_REGISTERS_FIRST = int("$env:NUM_OF_REGISTERS_first") + NUM_OF_REGISTERS_IN_VAULT = int("$env:NUM_OF_REGISTERS_IN_VAULT") + + # Assertions + assert NUM_OF_PUBLIC_FILES == NUM_OF_PUBLIC_FILES_IN_VAULT, f"Error: Local public Files: {NUM_OF_PUBLIC_FILES} and vault public files: {NUM_OF_PUBLIC_FILES_IN_VAULT} are Not Equal" + assert NUM_OF_PRIVATE_FILES == NUM_OF_PRIVATE_FILES_IN_VAULT, f"Error: Local private Files: {NUM_OF_PRIVATE_FILES} and vault private files: {NUM_OF_PRIVATE_FILES_IN_VAULT} are Not Equal" + assert NUM_OF_REGISTERS_FIRST == NUM_OF_REGISTERS_IN_VAULT, f"Error: Local registers: {NUM_OF_REGISTERS_FIRST} and vault registers: {NUM_OF_REGISTERS_IN_VAULT} are Not Equal" + + print("Vault synced successfully!") + - name: load an existing vault from the network if: matrix.os != 'windows-latest' run: ./target/release/autonomi --log-output-dest=data-dir vault load @@ -446,7 +527,6 @@ jobs: timeout-minutes: 2 - name: Time profiling for Different files - if: matrix.os != 'windows-latest' run: | # 1 MB python3 -c "with open('random_1MB.bin', 'wb') as f: f.write(bytearray([0xff] * 1 * 1024 * 1024))" From d0ead2432e47ab900b45c42a730f06caa0059426 Mon Sep 17 00:00:00 2001 From: Ermine Jose Date: Tue, 5 Nov 2024 14:05:27 +0530 Subject: [PATCH 3/5] fix: errors related to windows/linux --- .github/workflows/merge.yml | 142 +++++++++++++++++------------------- 1 file changed, 67 insertions(+), 75 deletions(-) diff --git a/.github/workflows/merge.yml b/.github/workflows/merge.yml index 428f42fa80..f51d959db8 100644 --- a/.github/workflows/merge.yml +++ b/.github/workflows/merge.yml @@ -358,26 +358,23 @@ jobs: timeout-minutes: 2 - name: Estimate cost to create a vault - if: matrix.os != 'windows-latest' run: | - echo "test-file" > upload-test.txt - ./target/release/autonomi --log-output-dest=data-dir file upload ./upload-test.txt + # 1 MB + python3 -c "with open('random.txt', 'wb') as f: f.write(bytearray([0xff] * 1 * 1024 * 1024))" + ./target/release/autonomi --log-output-dest=data-dir file upload random.txt ./target/release/autonomi --log-output-dest=data-dir register create sample_new_register 1234 ./target/release/autonomi --log-output-dest=data-dir vault cost - ./target/release/autonomi --log-output-dest=data-dir file list 2>&1 | tee file_list.txt - ./target/release/autonomi --log-output-dest=data-dir register list 2>&1 | tee register_list.txt env: SN_LOG: "v" timeout-minutes: 2 - name: create a vault with existing user data as above - if: matrix.os != 'windows-latest' run: ./target/release/autonomi --log-output-dest=data-dir vault create env: SN_LOG: "v" timeout-minutes: 2 - - name: add more files + - name: add more files - linux/macos if: matrix.os != 'windows-latest' run: | for i in {1..100}; do @@ -390,18 +387,13 @@ jobs: SN_LOG: "v" timeout-minutes: 25 - - name: add more files + - name: add more files - windows if: matrix.os == 'windows-latest' shell: pwsh run: | for ($i = 1; $i -le 100; $i++) { - # Generate a random file with PowerShell - $randomData = [System.IO.File]::OpenWrite("random_file_$i.bin") - $buffer = New-Object byte[](1024 * 1024) # 1MB buffer - [System.Security.Cryptography.RNGCryptoServiceProvider]::Create().GetBytes($buffer) - $randomData.Write($buffer, 0, $buffer.Length) - $randomData.Close() + python3 -c "import sys; with open(sys.argv[1], 'wb') as f: f.write(bytearray([0xff] * 1 * 1024 * 1024))" "random_file_$i.bin" # Run autonomi commands ./target/release/autonomi --log-output-dest=data-dir file upload "random_file_$i.bin" --public ./target/release/autonomi --log-output-dest=data-dir file upload "random_file_$i.bin" @@ -458,75 +450,75 @@ jobs: SN_LOG: "v" timeout-minutes: 15 - - name: Set up variables - vault sync - if: matrix.os == 'windows-latest' - shell: pwsh - run: | - # Initialize variables to empty - $NUM_OF_PUBLIC_FILES = "" - $NUM_OF_PRIVATE_FILES = "" - $NUM_OF_REGISTERS = "" - $NUM_OF_PUBLIC_FILES_IN_VAULT = "" - $NUM_OF_PRIVATE_FILES_IN_VAULT = "" - $NUM_OF_REGISTERS_IN_VAULT = "" - - # Execute commands and save outputs to files - ./target/release/autonomi --log-output-dest=data-dir file list > file_list.txt 2>&1 - ./target/release/autonomi register list | Select-String "register" > register_list.txt - ./target/release/autonomi --log-output-dest=data-dir vault load > vault_data.txt 2>&1 - - # Parse the files and extract numbers - $NUM_OF_PUBLIC_FILES = (Select-String "public" file_list.txt | ForEach-Object { $_ -match "\d+"; $matches[0] })[0] - $NUM_OF_PRIVATE_FILES = (Select-String "private" file_list.txt | ForEach-Object { $_ -match "\d+"; $matches[0] })[0] - $NUM_OF_REGISTERS = (Select-String "register" register_list.txt | ForEach-Object { $_ -match "\d+"; $matches[0] })[0] - - # Get the first word only (PowerShell handles this without additional parsing) - $NUM_OF_REGISTERS_first = $NUM_OF_REGISTERS -split '\s+' | Select-Object -First 1 - - Write-Output "NUM_OF_REGISTERS is $NUM_OF_REGISTERS_first" - - # Continue with vault data parsing - $NUM_OF_PUBLIC_FILES_IN_VAULT = (Select-String "public" vault_data.txt | ForEach-Object { $_ -match "\d+"; $matches[0] })[0] - $NUM_OF_PRIVATE_FILES_IN_VAULT = (Select-String "private" vault_data.txt | ForEach-Object { $_ -match "\d+"; $matches[0] })[0] - $NUM_OF_REGISTERS_IN_VAULT = (Select-String "register" vault_data.txt | ForEach-Object { $_ -match "\d+"; $matches[0] })[0] - - # Output summary - Write-Output "Total Num of local public files is $NUM_OF_PUBLIC_FILES and in vault is $NUM_OF_PUBLIC_FILES_IN_VAULT" - Write-Output "Total Num of local private files is $NUM_OF_PRIVATE_FILES and in vault is $NUM_OF_PRIVATE_FILES_IN_VAULT" - Write-Output "Total Num of local registers is $NUM_OF_REGISTERS_first and in vault is $NUM_OF_REGISTERS_IN_VAULT" - - # Clean up temporary files - Remove-Item -Force file_list.txt, register_list.txt, vault_data.txt - - - name: Vault sync validation - if: matrix.os == 'windows-latest' - shell: python - run: | - import sys - - # Define the values as environment variables - NUM_OF_PUBLIC_FILES = int("$env:NUM_OF_PUBLIC_FILES") - NUM_OF_PUBLIC_FILES_IN_VAULT = int("$env:NUM_OF_PUBLIC_FILES_IN_VAULT") - NUM_OF_PRIVATE_FILES = int("$env:NUM_OF_PRIVATE_FILES") - NUM_OF_PRIVATE_FILES_IN_VAULT = int("$env:NUM_OF_PRIVATE_FILES_IN_VAULT") - NUM_OF_REGISTERS_FIRST = int("$env:NUM_OF_REGISTERS_first") - NUM_OF_REGISTERS_IN_VAULT = int("$env:NUM_OF_REGISTERS_IN_VAULT") - - # Assertions - assert NUM_OF_PUBLIC_FILES == NUM_OF_PUBLIC_FILES_IN_VAULT, f"Error: Local public Files: {NUM_OF_PUBLIC_FILES} and vault public files: {NUM_OF_PUBLIC_FILES_IN_VAULT} are Not Equal" - assert NUM_OF_PRIVATE_FILES == NUM_OF_PRIVATE_FILES_IN_VAULT, f"Error: Local private Files: {NUM_OF_PRIVATE_FILES} and vault private files: {NUM_OF_PRIVATE_FILES_IN_VAULT} are Not Equal" - assert NUM_OF_REGISTERS_FIRST == NUM_OF_REGISTERS_IN_VAULT, f"Error: Local registers: {NUM_OF_REGISTERS_FIRST} and vault registers: {NUM_OF_REGISTERS_IN_VAULT} are Not Equal" - - print("Vault synced successfully!") + # - name: Set up variables - vault sync - windows + # if: matrix.os == 'windows-latest' + # shell: pwsh + # run: | + # # Initialize variables to empty + # $NUM_OF_PUBLIC_FILES = "" + # $NUM_OF_PRIVATE_FILES = "" + # $NUM_OF_REGISTERS = "" + # $NUM_OF_PUBLIC_FILES_IN_VAULT = "" + # $NUM_OF_PRIVATE_FILES_IN_VAULT = "" + # $NUM_OF_REGISTERS_IN_VAULT = "" + + # # Execute commands and save outputs to files + # ./target/release/autonomi --log-output-dest=data-dir file list > file_list.txt 2>&1 + # ./target/release/autonomi register list | Select-String "register" > register_list.txt + # ./target/release/autonomi --log-output-dest=data-dir vault load > vault_data.txt 2>&1 + + # # Parse the files and extract numbers + # $NUM_OF_PUBLIC_FILES = (Select-String "public" file_list.txt | ForEach-Object { $_ -match "\d+"; $matches[0] })[0] + # $NUM_OF_PRIVATE_FILES = (Select-String "private" file_list.txt | ForEach-Object { $_ -match "\d+"; $matches[0] })[0] + # $NUM_OF_REGISTERS = (Select-String "register" register_list.txt | ForEach-Object { $_ -match "\d+"; $matches[0] })[0] + + # # Get the first word only (PowerShell handles this without additional parsing) + # $NUM_OF_REGISTERS_first = $NUM_OF_REGISTERS -split '\s+' | Select-Object -First 1 + + # Write-Output "NUM_OF_REGISTERS is $NUM_OF_REGISTERS_first" + + # # Continue with vault data parsing + # $NUM_OF_PUBLIC_FILES_IN_VAULT = (Select-String "public" vault_data.txt | ForEach-Object { $_ -match "\d+"; $matches[0] })[0] + # $NUM_OF_PRIVATE_FILES_IN_VAULT = (Select-String "private" vault_data.txt | ForEach-Object { $_ -match "\d+"; $matches[0] })[0] + # $NUM_OF_REGISTERS_IN_VAULT = (Select-String "register" vault_data.txt | ForEach-Object { $_ -match "\d+"; $matches[0] })[0] + + # # Output summary + # Write-Output "Total Num of local public files is $NUM_OF_PUBLIC_FILES and in vault is $NUM_OF_PUBLIC_FILES_IN_VAULT" + # Write-Output "Total Num of local private files is $NUM_OF_PRIVATE_FILES and in vault is $NUM_OF_PRIVATE_FILES_IN_VAULT" + # Write-Output "Total Num of local registers is $NUM_OF_REGISTERS_first and in vault is $NUM_OF_REGISTERS_IN_VAULT" + + # # Clean up temporary files + # Remove-Item -Force file_list.txt, register_list.txt, vault_data.txt + + # - name: Vault sync validation + # if: matrix.os == 'windows-latest' + # shell: python + # run: | + # import sys + + # # Define the values as environment variables + # NUM_OF_PUBLIC_FILES = int("$env:NUM_OF_PUBLIC_FILES") + # NUM_OF_PUBLIC_FILES_IN_VAULT = int("$env:NUM_OF_PUBLIC_FILES_IN_VAULT") + # NUM_OF_PRIVATE_FILES = int("$env:NUM_OF_PRIVATE_FILES") + # NUM_OF_PRIVATE_FILES_IN_VAULT = int("$env:NUM_OF_PRIVATE_FILES_IN_VAULT") + # NUM_OF_REGISTERS_FIRST = int("$env:NUM_OF_REGISTERS_first") + # NUM_OF_REGISTERS_IN_VAULT = int("$env:NUM_OF_REGISTERS_IN_VAULT") + + # # Assertions + # assert NUM_OF_PUBLIC_FILES == NUM_OF_PUBLIC_FILES_IN_VAULT, f"Error: Local public Files: {NUM_OF_PUBLIC_FILES} and vault public files: {NUM_OF_PUBLIC_FILES_IN_VAULT} are Not Equal" + # assert NUM_OF_PRIVATE_FILES == NUM_OF_PRIVATE_FILES_IN_VAULT, f"Error: Local private Files: {NUM_OF_PRIVATE_FILES} and vault private files: {NUM_OF_PRIVATE_FILES_IN_VAULT} are Not Equal" + # assert NUM_OF_REGISTERS_FIRST == NUM_OF_REGISTERS_IN_VAULT, f"Error: Local registers: {NUM_OF_REGISTERS_FIRST} and vault registers: {NUM_OF_REGISTERS_IN_VAULT} are Not Equal" + + # print("Vault synced successfully!") - name: load an existing vault from the network - if: matrix.os != 'windows-latest' run: ./target/release/autonomi --log-output-dest=data-dir vault load env: SN_LOG: "v" timeout-minutes: 2 - name: Time profiling for Different files + if: matrix.os != 'windows-latest' run: | # 1 MB python3 -c "with open('random_1MB.bin', 'wb') as f: f.write(bytearray([0xff] * 1 * 1024 * 1024))" From aa25b3a047c55de293e51f35aa7f9b5c6a4e9d9b Mon Sep 17 00:00:00 2001 From: Ermine Jose Date: Tue, 5 Nov 2024 22:06:20 +0530 Subject: [PATCH 4/5] fix: ensure workflow stops on error --- .github/workflows/merge.yml | 95 +++++++++++++++++++++---------------- 1 file changed, 53 insertions(+), 42 deletions(-) diff --git a/.github/workflows/merge.yml b/.github/workflows/merge.yml index f51d959db8..94f363b9a2 100644 --- a/.github/workflows/merge.yml +++ b/.github/workflows/merge.yml @@ -357,13 +357,18 @@ jobs: SN_LOG: "v" timeout-minutes: 2 - - name: Estimate cost to create a vault + - name: create local user data run: | # 1 MB python3 -c "with open('random.txt', 'wb') as f: f.write(bytearray([0xff] * 1 * 1024 * 1024))" ./target/release/autonomi --log-output-dest=data-dir file upload random.txt ./target/release/autonomi --log-output-dest=data-dir register create sample_new_register 1234 - ./target/release/autonomi --log-output-dest=data-dir vault cost + env: + SN_LOG: "v" + timeout-minutes: 2 + + - name: Estimate cost to create a vault + run: ./target/release/autonomi --log-output-dest=data-dir vault cost env: SN_LOG: "v" timeout-minutes: 2 @@ -377,6 +382,7 @@ jobs: - name: add more files - linux/macos if: matrix.os != 'windows-latest' run: | + set -e for i in {1..100}; do dd if=/dev/urandom of=random_file_$i.bin bs=1M count=1 status=none ./target/release/autonomi --log-output-dest=data-dir file upload random_file_$i.bin --public @@ -391,9 +397,12 @@ jobs: if: matrix.os == 'windows-latest' shell: pwsh run: | + $ErrorActionPreference = "Stop" for ($i = 1; $i -le 100; $i++) { + $fileName = "random_file_$i.bin" + $byteArray = [byte[]]@(0xFF) * (1MB) # Create a 1 MB array filled with 0xFF + [System.IO.File]::WriteAllBytes($fileName, $byteArray) - python3 -c "import sys; with open(sys.argv[1], 'wb') as f: f.write(bytearray([0xff] * 1 * 1024 * 1024))" "random_file_$i.bin" # Run autonomi commands ./target/release/autonomi --log-output-dest=data-dir file upload "random_file_$i.bin" --public ./target/release/autonomi --log-output-dest=data-dir file upload "random_file_$i.bin" @@ -413,6 +422,7 @@ jobs: if: matrix.os != 'windows-latest' shell: bash run: | + set -e NUM_OF_PUBLIC_FILES="" NUM_OF_PRIVATE_FILES="" NUM_OF_REGISTERS="" @@ -450,45 +460,45 @@ jobs: SN_LOG: "v" timeout-minutes: 15 - # - name: Set up variables - vault sync - windows - # if: matrix.os == 'windows-latest' - # shell: pwsh - # run: | - # # Initialize variables to empty - # $NUM_OF_PUBLIC_FILES = "" - # $NUM_OF_PRIVATE_FILES = "" - # $NUM_OF_REGISTERS = "" - # $NUM_OF_PUBLIC_FILES_IN_VAULT = "" - # $NUM_OF_PRIVATE_FILES_IN_VAULT = "" - # $NUM_OF_REGISTERS_IN_VAULT = "" - - # # Execute commands and save outputs to files - # ./target/release/autonomi --log-output-dest=data-dir file list > file_list.txt 2>&1 - # ./target/release/autonomi register list | Select-String "register" > register_list.txt - # ./target/release/autonomi --log-output-dest=data-dir vault load > vault_data.txt 2>&1 - - # # Parse the files and extract numbers - # $NUM_OF_PUBLIC_FILES = (Select-String "public" file_list.txt | ForEach-Object { $_ -match "\d+"; $matches[0] })[0] - # $NUM_OF_PRIVATE_FILES = (Select-String "private" file_list.txt | ForEach-Object { $_ -match "\d+"; $matches[0] })[0] - # $NUM_OF_REGISTERS = (Select-String "register" register_list.txt | ForEach-Object { $_ -match "\d+"; $matches[0] })[0] - - # # Get the first word only (PowerShell handles this without additional parsing) - # $NUM_OF_REGISTERS_first = $NUM_OF_REGISTERS -split '\s+' | Select-Object -First 1 - - # Write-Output "NUM_OF_REGISTERS is $NUM_OF_REGISTERS_first" - - # # Continue with vault data parsing - # $NUM_OF_PUBLIC_FILES_IN_VAULT = (Select-String "public" vault_data.txt | ForEach-Object { $_ -match "\d+"; $matches[0] })[0] - # $NUM_OF_PRIVATE_FILES_IN_VAULT = (Select-String "private" vault_data.txt | ForEach-Object { $_ -match "\d+"; $matches[0] })[0] - # $NUM_OF_REGISTERS_IN_VAULT = (Select-String "register" vault_data.txt | ForEach-Object { $_ -match "\d+"; $matches[0] })[0] - - # # Output summary - # Write-Output "Total Num of local public files is $NUM_OF_PUBLIC_FILES and in vault is $NUM_OF_PUBLIC_FILES_IN_VAULT" - # Write-Output "Total Num of local private files is $NUM_OF_PRIVATE_FILES and in vault is $NUM_OF_PRIVATE_FILES_IN_VAULT" - # Write-Output "Total Num of local registers is $NUM_OF_REGISTERS_first and in vault is $NUM_OF_REGISTERS_IN_VAULT" - - # # Clean up temporary files - # Remove-Item -Force file_list.txt, register_list.txt, vault_data.txt + - name: Set up variables - vault sync - windows + if: matrix.os == 'windows-latest' + shell: pwsh + run: | + # Initialize variables to empty + $NUM_OF_PUBLIC_FILES = "" + $NUM_OF_PRIVATE_FILES = "" + $NUM_OF_REGISTERS = "" + $NUM_OF_PUBLIC_FILES_IN_VAULT = "" + $NUM_OF_PRIVATE_FILES_IN_VAULT = "" + $NUM_OF_REGISTERS_IN_VAULT = "" + + # Execute commands and save outputs to files + ./target/release/autonomi --log-output-dest=data-dir file list > file_list.txt 2>&1 + ./target/release/autonomi register list | Select-String "register" > register_list.txt + ./target/release/autonomi --log-output-dest=data-dir vault load > vault_data.txt 2>&1 + + # Parse the files and extract numbers + $NUM_OF_PUBLIC_FILES = (Select-String "public" file_list.txt | ForEach-Object { $_ -match "\d+"; $matches[0] })[0] + $NUM_OF_PRIVATE_FILES = (Select-String "private" file_list.txt | ForEach-Object { $_ -match "\d+"; $matches[0] })[0] + $NUM_OF_REGISTERS = (Select-String "register" register_list.txt | ForEach-Object { $_ -match "\d+"; $matches[0] })[0] + + # Get the first word only (PowerShell handles this without additional parsing) + $NUM_OF_REGISTERS_first = $NUM_OF_REGISTERS -split '\s+' | Select-Object -First 1 + + Write-Output "NUM_OF_REGISTERS is $NUM_OF_REGISTERS_first" + + # Continue with vault data parsing + $NUM_OF_PUBLIC_FILES_IN_VAULT = (Select-String "public" vault_data.txt | ForEach-Object { $_ -match "\d+"; $matches[0] })[0] + $NUM_OF_PRIVATE_FILES_IN_VAULT = (Select-String "private" vault_data.txt | ForEach-Object { $_ -match "\d+"; $matches[0] })[0] + $NUM_OF_REGISTERS_IN_VAULT = (Select-String "register" vault_data.txt | ForEach-Object { $_ -match "\d+"; $matches[0] })[0] + + # Output summary + Write-Output "Total Num of local public files is $NUM_OF_PUBLIC_FILES and in vault is $NUM_OF_PUBLIC_FILES_IN_VAULT" + Write-Output "Total Num of local private files is $NUM_OF_PRIVATE_FILES and in vault is $NUM_OF_PRIVATE_FILES_IN_VAULT" + Write-Output "Total Num of local registers is $NUM_OF_REGISTERS_first and in vault is $NUM_OF_REGISTERS_IN_VAULT" + + # Clean up temporary files + Remove-Item -Force file_list.txt, register_list.txt, vault_data.txt # - name: Vault sync validation # if: matrix.os == 'windows-latest' @@ -520,6 +530,7 @@ jobs: - name: Time profiling for Different files if: matrix.os != 'windows-latest' run: | + set -e # 1 MB python3 -c "with open('random_1MB.bin', 'wb') as f: f.write(bytearray([0xff] * 1 * 1024 * 1024))" # 10 MB From 911d57e1776c9f497b3f7ad21f23b56aa3891f81 Mon Sep 17 00:00:00 2001 From: Ermine Jose Date: Wed, 6 Nov 2024 23:25:20 +0530 Subject: [PATCH 5/5] test: add python logic for comparison --- .github/workflows/merge.yml | 123 +++++++++++++++++++----------------- 1 file changed, 64 insertions(+), 59 deletions(-) diff --git a/.github/workflows/merge.yml b/.github/workflows/merge.yml index 94f363b9a2..964c28d01d 100644 --- a/.github/workflows/merge.yml +++ b/.github/workflows/merge.yml @@ -357,12 +357,20 @@ jobs: SN_LOG: "v" timeout-minutes: 2 - - name: create local user data - run: | - # 1 MB - python3 -c "with open('random.txt', 'wb') as f: f.write(bytearray([0xff] * 1 * 1024 * 1024))" - ./target/release/autonomi --log-output-dest=data-dir file upload random.txt - ./target/release/autonomi --log-output-dest=data-dir register create sample_new_register 1234 + - name: create local user file + run: echo random > random.txt + env: + SN_LOG: "v" + timeout-minutes: 2 + + - name: file upload + run: ./target/release/autonomi --log-output-dest=data-dir file upload random.txt + env: + SN_LOG: "v" + timeout-minutes: 2 + + - name: create a local register + run: ./target/release/autonomi --log-output-dest=data-dir register create sample_new_register 1234 env: SN_LOG: "v" timeout-minutes: 2 @@ -464,62 +472,59 @@ jobs: if: matrix.os == 'windows-latest' shell: pwsh run: | - # Initialize variables to empty - $NUM_OF_PUBLIC_FILES = "" - $NUM_OF_PRIVATE_FILES = "" - $NUM_OF_REGISTERS = "" - $NUM_OF_PUBLIC_FILES_IN_VAULT = "" - $NUM_OF_PRIVATE_FILES_IN_VAULT = "" - $NUM_OF_REGISTERS_IN_VAULT = "" - - # Execute commands and save outputs to files + $ErrorActionPreference = "Stop" ./target/release/autonomi --log-output-dest=data-dir file list > file_list.txt 2>&1 - ./target/release/autonomi register list | Select-String "register" > register_list.txt + ./target/release/autonomi register list > register_list.txt 2>&1 ./target/release/autonomi --log-output-dest=data-dir vault load > vault_data.txt 2>&1 + env: + SN_LOG: "v" + timeout-minutes: 15 - # Parse the files and extract numbers - $NUM_OF_PUBLIC_FILES = (Select-String "public" file_list.txt | ForEach-Object { $_ -match "\d+"; $matches[0] })[0] - $NUM_OF_PRIVATE_FILES = (Select-String "private" file_list.txt | ForEach-Object { $_ -match "\d+"; $matches[0] })[0] - $NUM_OF_REGISTERS = (Select-String "register" register_list.txt | ForEach-Object { $_ -match "\d+"; $matches[0] })[0] - - # Get the first word only (PowerShell handles this without additional parsing) - $NUM_OF_REGISTERS_first = $NUM_OF_REGISTERS -split '\s+' | Select-Object -First 1 - - Write-Output "NUM_OF_REGISTERS is $NUM_OF_REGISTERS_first" - - # Continue with vault data parsing - $NUM_OF_PUBLIC_FILES_IN_VAULT = (Select-String "public" vault_data.txt | ForEach-Object { $_ -match "\d+"; $matches[0] })[0] - $NUM_OF_PRIVATE_FILES_IN_VAULT = (Select-String "private" vault_data.txt | ForEach-Object { $_ -match "\d+"; $matches[0] })[0] - $NUM_OF_REGISTERS_IN_VAULT = (Select-String "register" vault_data.txt | ForEach-Object { $_ -match "\d+"; $matches[0] })[0] - - # Output summary - Write-Output "Total Num of local public files is $NUM_OF_PUBLIC_FILES and in vault is $NUM_OF_PUBLIC_FILES_IN_VAULT" - Write-Output "Total Num of local private files is $NUM_OF_PRIVATE_FILES and in vault is $NUM_OF_PRIVATE_FILES_IN_VAULT" - Write-Output "Total Num of local registers is $NUM_OF_REGISTERS_first and in vault is $NUM_OF_REGISTERS_IN_VAULT" - - # Clean up temporary files - Remove-Item -Force file_list.txt, register_list.txt, vault_data.txt - - # - name: Vault sync validation - # if: matrix.os == 'windows-latest' - # shell: python - # run: | - # import sys - - # # Define the values as environment variables - # NUM_OF_PUBLIC_FILES = int("$env:NUM_OF_PUBLIC_FILES") - # NUM_OF_PUBLIC_FILES_IN_VAULT = int("$env:NUM_OF_PUBLIC_FILES_IN_VAULT") - # NUM_OF_PRIVATE_FILES = int("$env:NUM_OF_PRIVATE_FILES") - # NUM_OF_PRIVATE_FILES_IN_VAULT = int("$env:NUM_OF_PRIVATE_FILES_IN_VAULT") - # NUM_OF_REGISTERS_FIRST = int("$env:NUM_OF_REGISTERS_first") - # NUM_OF_REGISTERS_IN_VAULT = int("$env:NUM_OF_REGISTERS_IN_VAULT") - - # # Assertions - # assert NUM_OF_PUBLIC_FILES == NUM_OF_PUBLIC_FILES_IN_VAULT, f"Error: Local public Files: {NUM_OF_PUBLIC_FILES} and vault public files: {NUM_OF_PUBLIC_FILES_IN_VAULT} are Not Equal" - # assert NUM_OF_PRIVATE_FILES == NUM_OF_PRIVATE_FILES_IN_VAULT, f"Error: Local private Files: {NUM_OF_PRIVATE_FILES} and vault private files: {NUM_OF_PRIVATE_FILES_IN_VAULT} are Not Equal" - # assert NUM_OF_REGISTERS_FIRST == NUM_OF_REGISTERS_IN_VAULT, f"Error: Local registers: {NUM_OF_REGISTERS_FIRST} and vault registers: {NUM_OF_REGISTERS_IN_VAULT} are Not Equal" - - # print("Vault synced successfully!") + - name: Vault sync validation + if: matrix.os == 'windows-latest' + shell: python + run: | + import re + def find_number_before_word(file_name, search_word): + """ + Reads a file and finds the number immediately preceding a specified word in a line. + + :param file_name: Name of the file to read. + :param search_word: Word to search for in the file. + :return: The number before the word as an integer, or None if not found. + """ + try: + with open(file_name, 'r') as file: + for line in file: + if search_word in line: + match = re.search(r'(\d+)\s+' + re.escape(search_word), line) + if match: + return int(match.group(1)) # Convert to integer + return None # Return None if no match is found + except FileNotFoundError: + print(f"Error: File '{file_name}' not found.") + return None + NUM_OF_PUBLIC_FILES = find_number_before_word("file_list.txt", "public") + print("NUM_OF_PUBLIC_FILES:", NUM_OF_PUBLIC_FILES) + NUM_OF_PRIVATE_FILES = find_number_before_word("file_list.txt", "private") + print("NUM_OF_PRIVATE_FILES:", NUM_OF_PRIVATE_FILES) + NUM_OF_REGISTERS_FILES = find_number_before_word("register_list.txt", "register") + print("NUM_OF_REGISTERS_FILES:", NUM_OF_REGISTERS_FILES) + NUM_OF_PUBLIC_FILES_IN_VAULT = find_number_before_word("vault_data.txt", "public") + print("NUM_OF_PUBLIC_FILES_IN_VAULT:", NUM_OF_PUBLIC_FILES_IN_VAULT) + NUM_OF_PRIVATE_FILES_IN_VAULT = find_number_before_word("vault_data.txt", "private") + print("NUM_OF_PRIVATE_FILES_IN_VAULT:", NUM_OF_PRIVATE_FILES_IN_VAULT) + NUM_OF_REGISTERS_IN_VAULT = find_number_before_word("vault_data.txt", "register") + print("NUM_OF_PRIVATE_FILES_IN_VAULT:", NUM_OF_PRIVATE_FILES_IN_VAULT) + + # Assertions + assert NUM_OF_PUBLIC_FILES == NUM_OF_PUBLIC_FILES_IN_VAULT, f"Error: Local public Files: {NUM_OF_PUBLIC_FILES} and vault public files: {NUM_OF_PUBLIC_FILES_IN_VAULT} are Not Equal" + assert NUM_OF_PRIVATE_FILES == NUM_OF_PRIVATE_FILES_IN_VAULT, f"Error: Local private Files: {NUM_OF_PRIVATE_FILES} and vault private files: {NUM_OF_PRIVATE_FILES_IN_VAULT} are Not Equal" + assert NUM_OF_REGISTERS_FILES == NUM_OF_REGISTERS_IN_VAULT, f"Error: Local registers: {NUM_OF_REGISTERS_FILES} and vault registers: {NUM_OF_REGISTERS_IN_VAULT} are Not Equal" + print("Vault synced successfully!") + env: + SN_LOG: "v" + timeout-minutes: 2 - name: load an existing vault from the network run: ./target/release/autonomi --log-output-dest=data-dir vault load