Skip to content

Commit

Permalink
Merge pull request #2250 from RolandSherwin/ci_bench
Browse files Browse the repository at this point in the history
fix(bench): use the correct client path
  • Loading branch information
RolandSherwin authored Oct 16, 2024
2 parents edc02c5 + 8fb0c1f commit 189aaa7
Show file tree
Hide file tree
Showing 17 changed files with 52 additions and 31 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/benchmark-prs.yml
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ on: pull_request
env:
CARGO_INCREMENTAL: "0"
RUST_BACKTRACE: 1
CLIENT_DATA_PATH: /home/runner/.local/share/safe/client
CLIENT_DATA_PATH: /home/runner/.local/share/safe/autonomi
NODE_DATA_PATH: /home/runner/.local/share/safe/node

jobs:
Expand Down
8 changes: 4 additions & 4 deletions .github/workflows/generate-benchmark-charts.yml
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ jobs:
# Criterion outputs the actual bench results to stderr "2>&1 tee output.txt" takes stderr,
# passes to tee which displays it in the terminal and writes to output.txt
run: |
cargo criterion --features=local --message-format=json 2>&1 -p autonomi | tee -a output.txt
cargo criterion --features=local --message-format=json 2>&1 -p autonomi-cli | tee -a output.txt
cat output.txt | rg benchmark-complete | jq -s 'map({
name: (.id | split("/"))[-1],
unit: "MiB/s",
Expand Down Expand Up @@ -158,17 +158,17 @@ jobs:
shell: bash
run: |
peak_mem_usage=$(
rg '"memory_used_mb":[^,]*' $CLIENT_DATA_PATH/logs --glob safe.* -o --no-line-number --no-filename |
rg '"memory_used_mb":[^,]*' $CLIENT_DATA_PATH/logs/*/*.log -o --no-line-number --no-filename |
awk -F':' '/"memory_used_mb":/{print $2}' |
sort -n |
tail -n 1
)
total_mem=$(
rg '"memory_used_mb":[^,]*' $CLIENT_DATA_PATH/logs --glob safe.* -o --no-line-number --no-filename |
rg '"memory_used_mb":[^,]*' $CLIENT_DATA_PATH/logs/*/*.log -o --no-line-number --no-filename |
awk -F':' '/"memory_used_mb":/ {sum += $2} END {printf "%.0f\n", sum}'
)
num_of_times=$(
rg "\"memory_used_mb\"" $CLIENT_DATA_PATH/logs --glob safe.* -c --stats |
rg "\"memory_used_mb\"" $CLIENT_DATA_PATH/logs/*/*.log -c --stats |
rg "(\d+) matches" |
rg "\d+" -o
)
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/memcheck.yml
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ on:

env:
SAFE_DATA_PATH: /home/runner/.local/share/safe
CLIENT_DATA_PATH: /home/runner/.local/share/safe/client
CLIENT_DATA_PATH: /home/runner/.local/share/safe/autonomi
NODE_DATA_PATH: /home/runner/.local/share/safe/node
BOOTSTRAP_NODE_DATA_PATH: /home/runner/.local/share/safe/bootstrap_node
RESTART_TEST_NODE_DATA_PATH: /home/runner/.local/share/safe/restart_node
Expand Down
12 changes: 6 additions & 6 deletions .github/workflows/merge.yml
Original file line number Diff line number Diff line change
Expand Up @@ -313,7 +313,7 @@ jobs:

- name: Delete current register signing key
shell: bash
run: rm -rf ${{ matrix.safe_path }}/client
run: rm -rf ${{ matrix.safe_path }}/autonomi

- name: Generate new register signing key
run: ./target/release/autonomi --log-output-dest=data-dir register generate-key
Expand Down Expand Up @@ -932,7 +932,7 @@ jobs:
# ls -l /home/runner/.local/share/safe
# rm -rf /home/runner/.local/share/safe/test_faucet
# rm -rf /home/runner/.local/share/safe/test_genesis
# rm -rf /home/runner/.local/share/safe/client
# rm -rf /home/runner/.local/share/safe/autonomi
# env:
# SN_LOG: "all"
# timeout-minutes: 5
Expand Down Expand Up @@ -973,7 +973,7 @@ jobs:
# ls -l /home/runner/.local/share/safe
# rm -rf /home/runner/.local/share/safe/test_faucet
# rm -rf /home/runner/.local/share/safe/test_genesis
# rm -rf /home/runner/.local/share/safe/client
# rm -rf /home/runner/.local/share/safe/autonomi
# ~/safe --log-output-dest=data-dir wallet create --no-password
# if GENESIS_PK=a9925296499299fdbf4412509d342a92e015f5b996e9acd1d2ab7f2326e3ad05934326efdc345345a95e973ac1bb6637 GENESIS_SK=40f6bbc870355c68138ac70b450b6425af02b49874df3f141b7018378ceaac66 nohup ~/faucet --log-output-dest=data-dir send 100000000 $(~/safe --log-output-dest=data-dir wallet address | tail -n 1); then
# echo "Faucet with different genesis key not rejected!"
Expand All @@ -995,7 +995,7 @@ jobs:
# ls -l /home/runner/.local/share/safe
# rm -rf /home/runner/.local/share/safe/test_faucet
# rm -rf /home/runner/.local/share/safe/test_genesis
# rm -rf /home/runner/.local/share/safe/client
# rm -rf /home/runner/.local/share/safe/autonomi
# target/release/faucet server &
# sleep 60
# env:
Expand Down Expand Up @@ -1161,15 +1161,15 @@ jobs:
# ls $CLIENT_DATA_PATH/wallet/confirmed_spends -l
# ls $CLIENT_DATA_PATH/logs -l
# env:
# CLIENT_DATA_PATH: /home/runner/.local/share/safe/client
# CLIENT_DATA_PATH: /home/runner/.local/share/safe/autonomi
# timeout-minutes: 1

# replication_bench_with_heavy_upload:
# if: "!startsWith(github.event.head_commit.message, 'chore(release):')"
# name: Replication bench with heavy upload
# runs-on: ubuntu-latest
# env:
# CLIENT_DATA_PATH: /home/runner/.local/share/safe/client
# CLIENT_DATA_PATH: /home/runner/.local/share/safe/autonomi
# steps:
# - uses: actions/checkout@v4

Expand Down
8 changes: 4 additions & 4 deletions .github/workflows/nightly_wan.yml
Original file line number Diff line number Diff line change
Expand Up @@ -129,7 +129,7 @@ jobs:
path: |
~/.local/share/safe/node/*/logs/*.log*
~/.local/share/safe/*/*/*.log*
~/.local/share/safe/client/logs/*/*.log*
~/.local/share/safe/autonomi/logs/*/*.log*
- name: destroy network
if: always()
Expand Down Expand Up @@ -235,7 +235,7 @@ jobs:
# path: |
# ~/.local/share/safe/node/*/logs/*.log*
# ~/.local/share/safe/*/*/*.log*
# ~/.local/share/safe/client/logs/*/*.log*
# ~/.local/share/safe/autonomi/logs/*/*.log*

# - name: destroy network
# uses: maidsafe/sn-testnet-control-action/destroy-network@main
Expand Down Expand Up @@ -349,7 +349,7 @@ jobs:
# path: |
# ~/.local/share/safe/node/*/logs/*.log*
# ~/.local/share/safe/*/*/*.log*
# ~/.local/share/safe/client/logs/*/*.log*
# ~/.local/share/safe/autonomi/logs/*/*.log*
#
# - name: Stop the WAN network
# if: always()
Expand Down Expand Up @@ -555,7 +555,7 @@ jobs:
# path: |
# ~/.local/share/safe/node/*/logs/*.log*
# ~/.local/share/safe/*/*/*.log*
# ~/.local/share/safe/client/logs/*/*.log*
# ~/.local/share/safe/autonomi/logs/*/*.log*
#
# - name: Stop the WAN network
# if: always()
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/nightly_wan_churn.yml
Original file line number Diff line number Diff line change
Expand Up @@ -127,7 +127,7 @@ jobs:
path: |
~/.local/share/safe/node/*/logs/*.log*
~/.local/share/safe/*/*/*.log*
~/.local/share/safe/client/logs/*/*.log*
~/.local/share/safe/autonomi/logs/*/*.log*
- name: Stop the WAN network
if: always()
Expand Down
4 changes: 2 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ Libp2p.<br>

### For Users

- [CLI](https://github.com/maidsafe/safe_network/blob/main/autonomi_cli/README.md) The Command Line
- [CLI](https://github.com/maidsafe/safe_network/blob/main/autonomi-cli/README.md) The Command Line
Interface, allowing users to interact with the network from their terminal.
- [Node](https://github.com/maidsafe//safe_network/blob/main/sn_node/README.md) The backbone of the
safe network. Nodes can be run on commodity hardware and provide storage space and validation of
Expand Down Expand Up @@ -65,7 +65,7 @@ cargo build --release --features=network-contacts --bin safenode

- [Autonomi API](https://github.com/maidsafe/safe_network/blob/main/autonomi/README.md) The client APIs
allowing use of the Autonomi Network to users and developers.
- [Autonomi CLI](https://github.com/maidsafe/safe_network/blob/main/autonomi_cli/README.md) The Command Line
- [Autonomi CLI](https://github.com/maidsafe/safe_network/blob/main/autonomi-cli/README.md) The Command Line
Interface, allowing users to interact with the network from their terminal.
- [Node](https://github.com/maidsafe/safe_network/blob/main/sn_node/README.md) The backbone of the
autonomi network. Nodes can be run on commodity hardware and run the Network.
Expand Down
2 changes: 1 addition & 1 deletion autonomi-cli/README.md
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
# A CLI for the Autonomi Network

```
Usage: autonomi_cli [OPTIONS] <COMMAND>
Usage: autonomi [OPTIONS] <COMMAND>
Commands:
file Operations related to file handling
Expand Down
2 changes: 1 addition & 1 deletion autonomi-cli/benches/files.rs
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,7 @@ fn get_cli_path() -> PathBuf {
path.push("target");
}
path.push("release");
path.push("autonomi_cli");
path.push("autonomi");
path
}

Expand Down
2 changes: 1 addition & 1 deletion autonomi-cli/src/access/data_dir.rs
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ pub fn get_client_data_dir_path() -> Result<PathBuf> {
let mut home_dirs = dirs_next::data_dir()
.ok_or_else(|| eyre!("Failed to obtain data dir, your OS might not be supported."))?;
home_dirs.push("safe");
home_dirs.push("client");
home_dirs.push("autonomi");
std::fs::create_dir_all(home_dirs.as_path()).wrap_err("Failed to create data dir")?;
Ok(home_dirs)
}
2 changes: 2 additions & 0 deletions autonomi-cli/src/actions/connect.rs
Original file line number Diff line number Diff line change
Expand Up @@ -24,10 +24,12 @@ pub async fn connect_to_network(peers: Vec<Multiaddr>) -> Result<Client> {

match Client::connect(&peers).await {
Ok(client) => {
info!("Connected to the Network");
progress_bar.finish_with_message("Connected to the Network");
Ok(client)
}
Err(e) => {
error!("Failed to connect to the network: {e}");
progress_bar.finish_with_message("Failed to connect to the network");
bail!("Failed to connect to the network: {e}")
}
Expand Down
2 changes: 2 additions & 0 deletions autonomi-cli/src/actions/download.rs
Original file line number Diff line number Diff line change
Expand Up @@ -41,12 +41,14 @@ pub async fn download(addr: &str, dest_path: &str, client: &mut Client) -> Resul
progress_bar.finish_and_clear();

if all_errs.is_empty() {
info!("Successfully downloaded data at: {addr}");
println!("Successfully downloaded data at: {addr}");
Ok(())
} else {
let err_no = all_errs.len();
eprintln!("{err_no} errors while downloading data at: {addr}");
eprintln!("{all_errs:#?}");
error!("Errors while downloading data at {addr}: {all_errs:#?}");
Err(eyre!("Errors while downloading data"))
}
}
5 changes: 5 additions & 0 deletions autonomi-cli/src/commands/file.rs
Original file line number Diff line number Diff line change
Expand Up @@ -17,13 +17,15 @@ pub async fn cost(file: &str, peers: Vec<Multiaddr>) -> Result<()> {
let client = crate::actions::connect_to_network(peers).await?;

println!("Getting upload cost...");
info!("Calculating cost for file: {file}");
let cost = client
.file_cost(&PathBuf::from(file))
.await
.wrap_err("Failed to calculate cost for file")?;

println!("Estimate cost to upload file: {file}");
println!("Total cost: {cost}");
info!("Total cost: {cost} for file: {file}");
Ok(())
}
pub async fn upload(file: &str, peers: Vec<Multiaddr>) -> Result<()> {
Expand All @@ -33,6 +35,7 @@ pub async fn upload(file: &str, peers: Vec<Multiaddr>) -> Result<()> {
let (upload_summary_thread, upload_completed_tx) = collect_upload_summary(event_receiver);

println!("Uploading data to network...");
info!("Uploading file: {file}");

let xor_name = client
.dir_upload(PathBuf::from(file), &wallet)
Expand All @@ -42,6 +45,7 @@ pub async fn upload(file: &str, peers: Vec<Multiaddr>) -> Result<()> {

println!("Successfully uploaded: {file}");
println!("At address: {addr}");
info!("Successfully uploaded: {file} at address: {addr}");
if let Ok(()) = upload_completed_tx.send(()) {
let summary = upload_summary_thread.await?;
if summary.record_count == 0 {
Expand All @@ -50,6 +54,7 @@ pub async fn upload(file: &str, peers: Vec<Multiaddr>) -> Result<()> {
println!("Number of chunks uploaded: {}", summary.record_count);
println!("Total cost: {} AttoTokens", summary.tokens_spent);
}
info!("Summary for upload of file {file} at {addr:?}: {summary:?}");
}

Ok(())
Expand Down
16 changes: 15 additions & 1 deletion autonomi-cli/src/commands/register.rs
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ pub fn generate_key(overwrite: bool) -> Result<()> {
// check if the key already exists
let key_path = crate::keys::get_register_signing_key_path()?;
if key_path.exists() && !overwrite {
error!("Register key already exists at: {key_path:?}");
return Err(eyre!("Register key already exists at: {}", key_path.display()))
.with_suggestion(|| "if you want to overwrite the existing key, run the command with the --overwrite flag")
.with_warning(|| "overwriting the existing key might result in loss of access to any existing registers created using that key");
Expand All @@ -30,6 +31,7 @@ pub fn generate_key(overwrite: bool) -> Result<()> {
let key = RegisterSecretKey::random();
let path = crate::keys::create_register_signing_key_file(key)
.wrap_err("Failed to create new register key")?;
info!("Created new register key at: {path:?}");
println!("✅ Created new register key at: {}", path.display());
Ok(())
}
Expand All @@ -43,6 +45,7 @@ pub async fn cost(name: &str, peers: Vec<Multiaddr>) -> Result<()> {
.register_cost(name.to_string(), register_key)
.await
.wrap_err("Failed to get cost for register")?;
info!("Estimated cost to create a register with name {name}: {cost}");
println!("✅ The estimated cost to create a register with name {name} is: {cost}");
Ok(())
}
Expand All @@ -56,8 +59,10 @@ pub async fn create(name: &str, value: &str, public: bool, peers: Vec<Multiaddr>
let (upload_summary_thread, upload_completed_tx) = collect_upload_summary(event_receiver);

println!("Creating register with name: {name}");
info!("Creating register with name: {name}");
let register = if public {
println!("With public write access");
info!("With public write access");
let permissions = RegisterPermissions::new_anyone_can_write();
client
.register_create_with_permissions(
Expand All @@ -71,6 +76,7 @@ pub async fn create(name: &str, value: &str, public: bool, peers: Vec<Multiaddr>
.wrap_err("Failed to create register")?
} else {
println!("With private write access");
info!("With private write access");
client
.register_create(
value.as_bytes().to_vec().into(),
Expand All @@ -87,6 +93,7 @@ pub async fn create(name: &str, value: &str, public: bool, peers: Vec<Multiaddr>
println!("✅ Register created at address: {address}");
println!("With name: {name}");
println!("And initial value: [{value}]");
info!("✅ Register created at address: {address} with name: {name}");

if let Ok(()) = upload_completed_tx.send(()) {
let summary = upload_summary_thread.await?;
Expand All @@ -95,6 +102,7 @@ pub async fn create(name: &str, value: &str, public: bool, peers: Vec<Multiaddr>
} else {
println!("Total cost: {} AttoTokens", summary.tokens_spent);
}
info!("Summary of register creation: {summary:?}");
}

Ok(())
Expand All @@ -116,20 +124,24 @@ pub async fn edit(address: String, name: bool, value: &str, peers: Vec<Multiaddr
};

println!("Getting register at address: {address}");
info!("Getting register at address: {address}");
let register = client
.register_get(address)
.await
.wrap_err(format!("Failed to get register at address: {address}"))?;
println!("Found register at address: {address}");

println!("Found register at address: {address}");
println!("Updating register with new value: {value}");
info!("Updating register at address: {address} with new value: {value}");

client
.register_update(register, value.as_bytes().to_vec().into(), register_key)
.await
.wrap_err(format!("Failed to update register at address: {address}"))?;

println!("✅ Successfully updated register");
println!("With value: [{value}]");
info!("Successfully updated register at address: {address}");

Ok(())
}
Expand All @@ -150,13 +162,15 @@ pub async fn get(address: String, name: bool, peers: Vec<Multiaddr>) -> Result<(
};

println!("Getting register at address: {address}");
info!("Getting register at address: {address}");
let register = client
.register_get(address)
.await
.wrap_err(format!("Failed to get register at address: {address}"))?;
let values = register.values();

println!("✅ Register found at address: {address}");
info!("Register found at address: {address}");
match values.as_slice() {
[one] => println!("With value: [{:?}]", String::from_utf8_lossy(one)),
_ => {
Expand Down
6 changes: 3 additions & 3 deletions autonomi-cli/src/opt.rs
Original file line number Diff line number Diff line change
Expand Up @@ -27,9 +27,9 @@ pub(crate) struct Opt {
/// `data-dir` is the default value.
///
/// The data directory location is platform specific:
/// - Linux: $HOME/.local/share/safe/client/logs
/// - macOS: $HOME/Library/Application Support/safe/client/logs
/// - Windows: C:\Users\<username>\AppData\Roaming\safe\client\logs
/// - Linux: $HOME/.local/share/safe/autonomi/logs
/// - macOS: $HOME/Library/Application Support/safe/autonomi/logs
/// - Windows: C:\Users\<username>\AppData\Roaming\safe\autonomi\logs
#[allow(rustdoc::invalid_html_tags)]
#[clap(long, value_parser = LogOutputDest::parse_from_str, verbatim_doc_comment, default_value = "data-dir")]
pub log_output_dest: LogOutputDest,
Expand Down
2 changes: 2 additions & 0 deletions autonomi/src/client/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -197,11 +197,13 @@ async fn handle_event_receiver(
}

/// Events that can be broadcasted by the client.
#[derive(Debug, Clone)]
pub enum ClientEvent {
UploadComplete(UploadSummary),
}

/// Summary of an upload operation.
#[derive(Debug, Clone)]
pub struct UploadSummary {
pub record_count: usize,
pub tokens_spent: Amount,
Expand Down
Loading

0 comments on commit 189aaa7

Please sign in to comment.