From b8fee8fc6dbc26e020b6424321584ae396552339 Mon Sep 17 00:00:00 2001 From: Preston Evans Date: Tue, 2 May 2023 11:54:02 -0500 Subject: [PATCH] Make compatible with risc0. Introduce the CompactHeader type, which allows us to circumvent https://github.com/informalsystems/tendermint-rs/issues/1309. Update to Tendermint 0.32, which allows removal of unnecessary data copies. --- Cargo.toml | 14 ++- src/celestia.rs | 248 ++++++++++++++++++++++++++++++++++++++------ src/da_service.rs | 5 +- src/share_commit.rs | 9 +- src/types.rs | 12 +-- 5 files changed, 231 insertions(+), 57 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index a929ea3..408f976 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,7 +7,8 @@ edition = "2021" [dependencies] sovereign-sdk = { git = "https://github.com/Sovereign-Labs/sovereign.git", rev = "5e43c3ee9b5785abdca33b21c86fd38dbd9285e0" } -tendermint = "0.27" +tendermint = "0.32" +tendermint-proto = "0.32" prost = "0.11" prost-types = "0.11" @@ -26,8 +27,6 @@ anyhow = "1.0.62" jsonrpsee = { version = "0.16.2", features = ["http-client"], optional = true } tracing = "0.1.37" - -#nmt-rs = { path = "../nmt-rs", features = ["serde", "borsh"] } nmt-rs = { git = "https://github.com/Sovereign-Labs/nmt-rs.git", rev = "aec2dcdc279b381162537f5b20ce43d1d46dc42f", features = ["serde", "borsh"] } [dev-dependencies] @@ -37,8 +36,13 @@ postcard = { version = "1", features = ["use-std"]} prost-build = { version = "0.11" } -#[patch.'https://github.com/Sovereign-Labs/sovereign.git'] -#sovereign-sdk = { path = "../sovereign/sdk" } +[patch.crates-io] +# Patch tendermint until the "0.32" release lands. We need 0.32 to avoid a lot of unnecessary data copying during header verification +tendermint = { git = "https://github.com/informalsystems/tendermint-rs.git", rev = "e014de927abed7c5fcbf8186780a61b5c9c1e775" } +tendermint-proto = { git = "https://github.com/informalsystems/tendermint-rs.git", rev = "e014de927abed7c5fcbf8186780a61b5c9c1e775" } + +[patch.'https://github.com/Sovereign-Labs/sovereign.git'] +sovereign-sdk = { path = "../sovereign/sdk" } [features] default = ["native"] diff --git a/src/celestia.rs b/src/celestia.rs index d643ae0..2993690 100644 --- a/src/celestia.rs +++ b/src/celestia.rs @@ -1,4 +1,4 @@ -use std::ops::Range; +use std::{cell::RefCell, ops::Range}; use borsh::{BorshDeserialize, BorshSerialize}; use nmt_rs::NamespacedHash; @@ -8,8 +8,12 @@ use sovereign_sdk::core::traits::{ AddressTrait as Address, BlockHeaderTrait as BlockHeader, CanonicalHash, }; pub use tendermint::block::Header as TendermintHeader; +use tendermint::{crypto::default::Sha256, merkle::simple_hash_from_byte_vectors, Hash}; +use tendermint_proto::Protobuf; use tracing::debug; +pub use tendermint_proto::v0_34 as celestia_tm_version; + const NAMESPACED_HASH_LEN: usize = 48; use crate::{ @@ -26,6 +30,158 @@ pub struct MarshalledDataAvailabilityHeader { pub column_roots: Vec, } +#[derive(PartialEq, Eq, Debug, Clone, Serialize, Deserialize, BorshDeserialize, BorshSerialize)] +pub struct PartialBlockId { + pub hash: ProtobufHash, + pub part_set_header: Vec, +} + +/// A partially serialized tendermint header. Only fields which are actually inspected by +/// Jupiter are included in their raw form. Other fields are pre-encoded as protobufs. +/// +/// This type was first introduced as a way to circumvent a bug in tendermint-rs which prevents +/// a tendermint::block::Header from being deserialized in most formats except JSON. However +/// it also provides a significant efficiency benefit over the standard tendermint type, which +/// performs a complete protobuf serialization every time `.hash()` is called. +#[derive(PartialEq, Eq, Debug, Clone, Serialize, Deserialize)] +pub struct CompactHeader { + /// Header version + pub version: Vec, + + /// Chain ID + pub chain_id: Vec, + + /// Current block height + pub height: Vec, + + /// Current timestamp + pub time: Vec, + + /// Previous block info + pub last_block_id: Vec, + + /// Commit from validators from the last block + pub last_commit_hash: Vec, + + /// Merkle root of transaction hashes + pub data_hash: Option, + + /// Validators for the current block + pub validators_hash: Vec, + + /// Validators for the next block + pub next_validators_hash: Vec, + + /// Consensus params for the current block + pub consensus_hash: Vec, + + /// State after txs from the previous block + pub app_hash: Vec, + + /// Root hash of all results from the txs from the previous block + pub last_results_hash: Vec, + + /// Hash of evidence included in the block + pub evidence_hash: Vec, + + /// Original proposer of the block + pub proposer_address: Vec, +} + +trait EncodeTm34 { + fn encode_to_tm34_protobuf(&self) -> Result, BoxError>; +} + +impl From for CompactHeader { + fn from(value: TendermintHeader) -> Self { + let data_hash = if let Some(h) = value.data_hash { + match h { + Hash::Sha256(value) => Some(ProtobufHash(value)), + Hash::None => None, + } + } else { + None + }; + Self { + version: Protobuf::::encode_vec( + &value.version, + ) + .unwrap(), + chain_id: value.chain_id.encode_vec().unwrap(), + height: value.height.encode_vec().unwrap(), + time: value.time.encode_vec().unwrap(), + last_block_id: Protobuf::::encode_vec( + &value.last_block_id.unwrap_or_default(), + ) + .unwrap(), + last_commit_hash: value + .last_commit_hash + .unwrap_or_default() + .encode_vec() + .unwrap(), + data_hash, + validators_hash: value.validators_hash.encode_vec().unwrap(), + next_validators_hash: value.next_validators_hash.encode_vec().unwrap(), + consensus_hash: value.consensus_hash.encode_vec().unwrap(), + app_hash: value.app_hash.encode_vec().unwrap(), + last_results_hash: value + .last_results_hash + .unwrap_or_default() + .encode_vec() + .unwrap(), + evidence_hash: value + .evidence_hash + .unwrap_or_default() + .encode_vec() + .unwrap(), + proposer_address: value.proposer_address.encode_vec().unwrap(), + } + } +} + +#[derive(Clone, PartialEq, Eq, Debug, Serialize, Deserialize, BorshDeserialize, BorshSerialize)] +pub struct ProtobufHash(pub [u8; 32]); + +pub fn protobuf_encode(hash: &Option) -> Vec { + match hash { + Some(ProtobufHash(value)) => prost::Message::encode_to_vec(&value.to_vec()), + None => prost::Message::encode_to_vec(&vec![]), + } +} + +impl CompactHeader { + /// Hash this header + // TODO: this function can be made even more efficient. Rather than computing the block hash, + // we could provide the hash as a non-deterministic input and simply verify the correctness of the + // fields that we care about. + pub fn hash(&self) -> Hash { + // Note that if there is an encoding problem this will + // panic (as the golang code would): + // https://github.com/tendermint/tendermint/blob/134fe2896275bb926b49743c1e25493f6b24cc31/types/block.go#L393 + // https://github.com/tendermint/tendermint/blob/134fe2896275bb926b49743c1e25493f6b24cc31/types/encoding_helper.go#L9:6 + + let encoded_data_hash = protobuf_encode(&self.data_hash); + let fields_bytes = vec![ + &self.version, + &self.chain_id, + &self.height, + &self.time, + &self.last_block_id, + &self.last_commit_hash, + &encoded_data_hash, + &self.validators_hash, + &self.next_validators_hash, + &self.consensus_hash, + &self.app_hash, + &self.last_results_hash, + &self.evidence_hash, + &self.proposer_address, + ]; + + Hash::Sha256(simple_hash_from_byte_vectors::(&fields_bytes)) + } +} + #[derive( PartialEq, Debug, Clone, Deserialize, serde::Serialize, BorshDeserialize, BorshSerialize, )] @@ -60,6 +216,8 @@ impl TryFrom for DataAvailabilityHeader { } } +/// The response from the celestia `/header` endpoint. Must be converted to a +/// [`CelestiaHeader`] before use. #[derive(Deserialize, Serialize, PartialEq, Debug, Clone)] pub struct CelestiaHeaderResponse { pub header: tendermint::block::Header, @@ -75,10 +233,20 @@ pub struct NamespacedSharesResponse { #[derive(Debug, PartialEq, Clone, Deserialize, serde::Serialize)] pub struct CelestiaHeader { pub dah: DataAvailabilityHeader, - pub header: tendermint::block::Header, + pub header: CompactHeader, + #[serde(skip)] + cached_prev_hash: RefCell>, } impl CelestiaHeader { + pub fn new(dah: DataAvailabilityHeader, header: CompactHeader) -> Self { + Self { + dah, + header, + cached_prev_hash: RefCell::new(None), + } + } + pub fn square_size(&self) -> usize { self.dah.row_roots.len() } @@ -101,13 +269,22 @@ pub struct BlobWithSender { impl BlockHeader for CelestiaHeader { type Hash = TmHash; - fn prev_hash(&self) -> &Self::Hash { - self.header - .last_block_id - .as_ref() + fn prev_hash(&self) -> Self::Hash { + // Try to return the cached value + if let Some(hash) = self.cached_prev_hash.borrow().as_ref() { + return hash.clone(); + } + // If we reach this point, we know that the cach is empty - so there can't be any outstanding references to its value. + // That means its safe to borrow the cache mutably and populate it. + let mut cached_hash = self.cached_prev_hash.borrow_mut(); + let hash = + >::decode( + self.header.last_block_id.as_ref(), + ) .expect("must not call prev_hash on block with no predecessor") - .hash - .as_ref() + .hash; + *cached_hash = Some(TmHash(hash.clone())); + TmHash(hash) } } @@ -218,29 +395,32 @@ fn next_pfb(mut data: &mut BlobRefIterator) -> Result<(MsgPayForBlobs, TxPositio )) } -// #[cfg(test)] -// mod tests { -// use crate::CelestiaHeaderResponse; - -// const HEADER_RESPONSE_JSON: &[u8] = include_bytes!("./header_response.json"); -// #[test] -// fn test_tm_header_serde() { -// // TODO: Find breakage here. Then, re-enable this test. -// let original_header: CelestiaHeaderResponse = -// serde_json::from_slice(HEADER_RESPONSE_JSON).unwrap(); - -// let header: tendermint::block::Header = original_header.header; - -// let serialized_header: Vec = serde_json::to_vec_pretty(&header).unwrap(); -// println!( -// "header: {}", -// String::from_utf8(serialized_header.clone()).unwrap() -// ); -// // let deserialized_header: tendermint::block::Header = -// // serde_json::from_slice(&serialized_header).unwrap(); - -// let serialized_header = postcard::to_stdvec(&header).unwrap(); -// let deserialized_header: tendermint::block::Header = -// postcard::from_bytes(&serialized_header).unwrap(); -// } -// } +#[cfg(test)] +mod tests { + use crate::{CelestiaHeaderResponse, CompactHeader}; + + const HEADER_RESPONSE_JSON: &[u8] = include_bytes!("./header_response.json"); + + #[test] + fn test_compact_header_serde() { + let original_header: CelestiaHeaderResponse = + serde_json::from_slice(HEADER_RESPONSE_JSON).unwrap(); + + let header: CompactHeader = original_header.header.into(); + + let serialized_header = postcard::to_stdvec(&header).unwrap(); + let deserialized_header: CompactHeader = postcard::from_bytes(&serialized_header).unwrap(); + assert_eq!(deserialized_header, header) + } + + #[test] + fn test_compact_header_hash() { + let original_header: CelestiaHeaderResponse = + serde_json::from_slice(HEADER_RESPONSE_JSON).unwrap(); + + let tm_header = original_header.header.clone(); + let compact_header: CompactHeader = original_header.header.into(); + + assert_eq!(tm_header.hash(), compact_header.hash()); + } +} diff --git a/src/da_service.rs b/src/da_service.rs index 0b80ac2..bf06120 100644 --- a/src/da_service.rs +++ b/src/da_service.rs @@ -140,10 +140,7 @@ impl DaService for CelestiaService { } let filtered_block = FilteredCelestiaBlock { - header: CelestiaHeader { - header: unmarshalled_header.header, - dah, - }, + header: CelestiaHeader::new(dah, unmarshalled_header.header.into()), rollup_data: rollup_shares, relevant_pfbs: pfd_map, rollup_rows, diff --git a/src/share_commit.rs b/src/share_commit.rs index ef3bc28..0cc49e6 100644 --- a/src/share_commit.rs +++ b/src/share_commit.rs @@ -1,6 +1,6 @@ use crate::shares::{self, Share}; -use tendermint::merkle::simple_hash_from_byte_vectors; +use tendermint::{crypto::default::Sha256, merkle::simple_hash_from_byte_vectors}; // /// Calculates the size of the smallest square that could be used to commit // /// to this message, following Celestia's "non-interactive default rules" @@ -54,12 +54,7 @@ pub fn recreate_commitment( } subtree_roots.push(tree.root()); } - let h = simple_hash_from_byte_vectors( - subtree_roots - .into_iter() - .map(|x| x.as_ref().to_vec()) - .collect(), - ); + let h = simple_hash_from_byte_vectors::(&subtree_roots); Ok(h) } diff --git a/src/types.rs b/src/types.rs index f96c4ec..f9877d7 100644 --- a/src/types.rs +++ b/src/types.rs @@ -8,7 +8,7 @@ use sovereign_sdk::{ services::da::SlotData, Bytes, }; -use tendermint::merkle; +use tendermint::{crypto::default::Sha256, merkle}; use crate::{ pfb::MsgPayForBlobs, @@ -144,16 +144,14 @@ impl CelestiaHeader { pub fn validate_dah(&self) -> Result<(), ValidationError> { let rows_iter = self.dah.row_roots.iter(); let cols_iter = self.dah.column_roots.iter(); - let byte_vecs = rows_iter - .chain(cols_iter) - .map(|hash| hash.0.to_vec()) - .collect(); - let root = merkle::simple_hash_from_byte_vectors(byte_vecs); + let byte_vecs: Vec<&NamespacedHash> = rows_iter.chain(cols_iter).collect(); + let root = merkle::simple_hash_from_byte_vectors::(&byte_vecs); let data_hash = self .header .data_hash + .as_ref() .ok_or(ValidationError::MissingDataHash)?; - if &root != >::as_ref(&data_hash) { + if &root != &data_hash.0 { return Err(ValidationError::InvalidDataRoot); } Ok(())