Skip to content

Commit

Permalink
Merge pull request #473 from Cerebellum-Network/chore/offload-api-def…
Browse files Browse the repository at this point in the history 
…s-from-pallet-mod

Offload aggregator's API JSON definitions from ddc-verification pallet mod
  • Loading branch information
@khssnv
khssnv authored Nov 12, 2024
2 parents 266eefc + 247f460 commit f2ea45b
Show file tree
Hide file tree
Showing 3 changed files with 396 additions and 361 deletions.
230 changes: 224 additions & 6 deletions pallets/ddc-verification/src/aggregator_client.rs
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
#![allow(dead_code)]

use ddc_primitives::{BucketId, DdcEra};
use ddc_primitives::{AggregatorInfo, BucketId, DdcEra};
use prost::Message;
use sp_io::offchain::timestamp;
use sp_runtime::offchain::{http, Duration};
Expand All @@ -23,7 +23,7 @@ impl<'a> AggregatorClient<'a> {
era_id: DdcEra,
limit: Option<u32>,
prev_token: Option<BucketId>,
) -> Result<Vec<pallet::BucketAggregateResponse>, http::Error> {
) -> Result<Vec<json::BucketAggregateResponse>, http::Error> {
let mut url = format!("{}/activity/buckets?eraId={}", self.base_url, era_id);
if let Some(limit) = limit {
url = format!("{}&limit={}", url, limit);
Expand All @@ -43,7 +43,7 @@ impl<'a> AggregatorClient<'a> {
era_id: DdcEra,
limit: Option<u32>,
prev_token: Option<String>, // node_id hex string
) -> Result<Vec<pallet::NodeAggregateResponse>, http::Error> {
) -> Result<Vec<json::NodeAggregateResponse>, http::Error> {
let mut url = format!("{}/activity/nodes?eraId={}", self.base_url, era_id);
if let Some(limit) = limit {
url = format!("{}&limit={}", url, limit);
Expand Down Expand Up @@ -102,7 +102,7 @@ impl<'a> AggregatorClient<'a> {
Ok(proto_response)
}

pub fn eras(&self) -> Result<Vec<pallet::AggregationEraResponse>, http::Error> {
pub fn eras(&self) -> Result<Vec<json::AggregationEraResponse>, http::Error> {
let url = format!("{}/activity/eras", self.base_url);
let response = self.get(&url, Accept::Any)?;
let body = response.body().collect::<Vec<u8>>();
Expand All @@ -118,7 +118,7 @@ impl<'a> AggregatorClient<'a> {
node_id: &str,
merkle_tree_node_id: u32,
levels: u16,
) -> Result<pallet::MerkleTreeNodeResponse, http::Error> {
) -> Result<json::MerkleTreeNodeResponse, http::Error> {
let url = format!(
"{}/activity/buckets/{}/traverse?eraId={}&nodeId={}&merkleTreeNodeId={}&levels={}",
self.base_url, bucket_id, era_id, node_id, merkle_tree_node_id, levels,
Expand All @@ -137,7 +137,7 @@ impl<'a> AggregatorClient<'a> {
node_id: &str,
merkle_tree_node_id: u32,
levels: u16,
) -> Result<pallet::MerkleTreeNodeResponse, http::Error> {
) -> Result<json::MerkleTreeNodeResponse, http::Error> {
let url = format!(
"{}/activity/nodes/{}/traverse?eraId={}&merkleTreeNodeId={}&levels={}",
self.base_url, node_id, era_id, merkle_tree_node_id, levels,
Expand Down Expand Up @@ -213,3 +213,221 @@ enum Accept {
Any,
Protobuf,
}

pub(crate) mod json {
use super::*;

/// Node aggregate response from aggregator.
#[derive(
Debug, Serialize, Deserialize, Clone, Hash, Ord, PartialOrd, PartialEq, Eq, Encode, Decode,
)]
pub struct NodeAggregateResponse {
/// Node id.
pub node_id: String,
/// Total amount of stored bytes.
pub stored_bytes: i64,
/// Total amount of transferred bytes.
pub transferred_bytes: u64,
/// Total number of puts.
pub number_of_puts: u64,
/// Total number of gets.
pub number_of_gets: u64,
}

/// DDC aggregation era
#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq, Encode, Decode)]
pub struct AggregationEraResponse {
pub id: DdcEra,
pub status: String,
pub start: i64,
pub end: i64,
pub processing_time: i64,
pub nodes_total: u32,
pub nodes_processed: u32,
pub records_processed: u32,
pub records_applied: u32,
pub records_discarded: u32,
pub attempt: u32,
}

/// Bucket aggregate response from aggregator.
#[derive(
Debug, Serialize, Deserialize, Clone, Hash, Ord, PartialOrd, PartialEq, Eq, Encode, Decode,
)]
pub struct BucketAggregateResponse {
/// Bucket id
pub bucket_id: BucketId,
/// Total amount of stored bytes.
pub stored_bytes: i64,
/// Total amount of transferred bytes.
pub transferred_bytes: u64,
/// Total number of puts.
pub number_of_puts: u64,
/// Total number of gets.
pub number_of_gets: u64,
/// Bucket sub aggregates.
pub sub_aggregates: Vec<BucketSubAggregateResponse>,
}

/// Sub aggregates of a bucket.
#[derive(
Debug, Serialize, Deserialize, Clone, Hash, Ord, PartialOrd, PartialEq, Eq, Encode, Decode,
)]
#[allow(non_snake_case)]
pub struct BucketSubAggregateResponse {
/// Node id.
pub NodeID: String,
/// Total amount of stored bytes.
pub stored_bytes: i64,
/// Total amount of transferred bytes.
pub transferred_bytes: u64,
/// Total number of puts.
pub number_of_puts: u64,
/// Total number of gets.
pub number_of_gets: u64,
}

/// Bucket activity per a DDC node.
#[derive(
Debug, Serialize, Deserialize, Clone, Ord, PartialOrd, PartialEq, Eq, Encode, Decode,
)]
pub struct BucketSubAggregate {
/// Bucket id
pub bucket_id: BucketId,
/// Node id.
pub node_id: String,
/// Total amount of stored bytes.
pub stored_bytes: i64,
/// Total amount of transferred bytes.
pub transferred_bytes: u64,
/// Total number of puts.
pub number_of_puts: u64,
/// Total number of gets.
pub number_of_gets: u64,
/// Aggregator data.
pub aggregator: AggregatorInfo,
}

#[derive(
Debug, Serialize, Deserialize, Clone, Ord, PartialOrd, PartialEq, Eq, Encode, Decode,
)]
pub struct NodeAggregate {
/// Node id.
pub node_id: String,
/// Total amount of stored bytes.
pub stored_bytes: i64,
/// Total amount of transferred bytes.
pub transferred_bytes: u64,
/// Total number of puts.
pub number_of_puts: u64,
/// Total number of gets.
pub number_of_gets: u64,
/// Node data.
pub aggregator: AggregatorInfo,
}

/// Challenge Response
#[derive(
Debug, Serialize, Deserialize, Clone, Hash, Ord, PartialOrd, PartialEq, Eq, Encode, Decode,
)]
pub struct ChallengeAggregateResponse {
/// proofs
pub proofs: Vec<Proof>, //todo! add optional fields
}

#[derive(
Debug, Serialize, Deserialize, Clone, Hash, Ord, PartialOrd, PartialEq, Eq, Encode, Decode,
)]
pub struct Proof {
pub merkle_tree_node_id: u32,
pub usage: Usage,
pub path: Vec<String>, //todo! add base64 deserialization
pub leafs: Vec<Leaf>,
}

#[derive(
Debug, Serialize, Deserialize, Clone, Hash, Ord, PartialOrd, PartialEq, Eq, Encode, Decode,
)]
pub struct Usage {
/// Total amount of stored bytes.
pub stored_bytes: i64,
/// Total amount of transferred bytes.
pub transferred_bytes: u64,
/// Total number of puts.
pub number_of_puts: u64,
/// Total number of gets.
pub number_of_gets: u64,
}

#[derive(
Debug, Serialize, Deserialize, Clone, Hash, Ord, PartialOrd, PartialEq, Eq, Encode, Decode,
)]
pub struct Leaf {
pub record: Record,
pub transferred_bytes: u64,
pub stored_bytes: i64,
// todo! add links if there is no record
}

#[derive(
Debug, Serialize, Deserialize, Clone, Hash, Ord, PartialOrd, PartialEq, Eq, Encode, Decode,
)]
#[allow(non_snake_case)]
pub struct Record {
pub id: String,
pub upstream: Upstream,
pub downstream: Vec<Downstream>,
pub timestamp: String,
pub signature: Signature,
}

#[derive(
Debug, Serialize, Deserialize, Clone, Hash, Ord, PartialOrd, PartialEq, Eq, Encode, Decode,
)]
pub struct Upstream {
pub request: Request,
}

#[derive(
Debug, Serialize, Deserialize, Clone, Hash, Ord, PartialOrd, PartialEq, Eq, Encode, Decode,
)]
pub struct Downstream {
pub request: Request,
}
#[derive(
Debug, Serialize, Deserialize, Clone, Hash, Ord, PartialOrd, PartialEq, Eq, Encode, Decode,
)]
#[allow(non_snake_case)]
pub struct Request {
pub requestId: String,
pub requestType: String,
pub contentType: String,
pub bucketId: String,
pub pieceCid: String,
pub offset: String,
pub size: String,
pub timestamp: String,
pub signature: Signature,
}

#[derive(
Debug, Serialize, Deserialize, Clone, Hash, Ord, PartialOrd, PartialEq, Eq, Encode, Decode,
)]
pub struct Signature {
pub algorithm: String,
pub signer: String,
pub value: String,
}

#[derive(
Debug, Serialize, Deserialize, Clone, Hash, Ord, PartialOrd, PartialEq, Eq, Encode, Decode,
)]
pub struct MerkleTreeNodeResponse {
pub merkle_tree_node_id: u32,
pub hash: String,
pub stored_bytes: i64,
pub transferred_bytes: u64,
pub number_of_puts: u64,
pub number_of_gets: u64,
}
}
Loading

0 comments on commit f2ea45b

Please sign in to comment.