Merge pull request #156 from nervosnetwork/quake/load-extension

chore: support ckb2023 load_extension syscall

Cargo.toml

@@ -9,24 +9,24 @@ homepage = "https://github.com/nervosnetwork/ckb-light-client"
 repository = "https://github.com/nervosnetwork/ckb-light-client"
 
 [dependencies]
-ckb-app-config    = "0.111.0"
-ckb-async-runtime = "0.111.0"
-ckb-stop-handler  = "0.111.0"
-ckb-constant      = "0.111.0"
-ckb-types         = "0.111.0"
-ckb-network       = "0.111.0"
-ckb-jsonrpc-types = "0.111.0"
-ckb-error         = "0.111.0"
-ckb-script        = "0.111.0"
-ckb-chain-spec    = "0.111.0"
-ckb-traits        = "0.111.0"
-ckb-resource      = "0.111.0"
-ckb-verification  = "0.111.0"
-ckb-systemtime    = "0.111.0"
-ckb-hash          = "0.111.0"
+ckb-app-config    = "0.113.0"
+ckb-async-runtime = "0.113.0"
+ckb-stop-handler  = "0.113.0"
+ckb-constant      = "0.113.0"
+ckb-types         = "0.113.0"
+ckb-network       = "0.113.0"
+ckb-jsonrpc-types = "0.113.0"
+ckb-error         = "0.113.0"
+ckb-script        = "0.113.0"
+ckb-chain-spec    = "0.113.0"
+ckb-traits        = "0.113.0"
+ckb-resource      = "0.113.0"
+ckb-verification  = "0.113.0"
+ckb-systemtime    = "0.113.0"
+ckb-hash          = "0.113.0"
 ckb-merkle-mountain-range = "0.5.1"
 golomb-coded-set = "0.2.0"
-rocksdb = { package = "ckb-rocksdb", version ="=0.20.0", features = ["snappy"], default-features = false }
+rocksdb = { package = "ckb-rocksdb", version ="=0.21.1", features = ["snappy"], default-features = false }
 numext-fixed-uint = { version = "0.1", features = ["support_rand", "support_heapsize", "support_serde"] }
 anyhow = "1.0.56"
 thiserror = "1.0.30"
@@ -46,12 +46,11 @@ jsonrpc-http-server = "18.0"
 jsonrpc-server-utils = "18.0"
 
 [dev-dependencies]
-ckb-launcher    = "0.111.0"
-ckb-shared      = "0.111.0"
-ckb-chain       = "0.111.0"
-ckb-tx-pool     = "0.111.0"
-ckb-store       = "0.111.0"
-ckb-systemtime  = { version = "0.111.0", features = ["enable_faketime"] }
+ckb-shared      = "0.113.0"
+ckb-chain       = "0.113.0"
+ckb-tx-pool     = "0.113.0"
+ckb-store       = "0.113.0"
+ckb-systemtime  = { version = "0.113.0", features = ["enable_faketime"] }
 tempfile = "3.0"
 rand = "0.6"
 serde_json = "1.0"

src/protocols/light_client/components/mod.rs

@@ -6,7 +6,7 @@ mod send_transactions_proof;
 #[cfg(test)]
 mod tests;
 
-pub(crate) use send_blocks_proof::SendBlocksProofProcess;
+pub(crate) use send_blocks_proof::{verify_extra_hash, SendBlocksProofProcess};
 pub(crate) use send_last_state::SendLastStateProcess;
 pub(crate) use send_last_state_proof::{verify_mmr_proof, SendLastStateProofProcess};
 pub(crate) use send_transactions_proof::SendTransactionsProofProcess;

src/protocols/light_client/components/send_blocks_proof.rs

@@ -1,8 +1,15 @@
 use ckb_network::{CKBProtocolContext, PeerIndex, SupportProtocols};
-use ckb_types::{packed, prelude::*, utilities::merkle_mountain_range::VerifiableHeader};
+use ckb_types::{
+    core::{ExtraHashView, HeaderView},
+    packed,
+    prelude::*,
+    utilities::merkle_mountain_range::VerifiableHeader,
+};
 use log::{debug, error};
 use rand::seq::SliceRandom;
 
+use crate::storage::HeaderWithExtension;
+
 use super::{
     super::{LightClientProtocol, Status, StatusCode},
     verify_mmr_proof,
@@ -109,6 +116,29 @@ impl<'a> SendBlocksProofProcess<'a> {
             // Check PoW for blocks
             return_if_failed!(self.protocol.check_pow_for_headers(headers.iter()));
 
+            // Check extra hash for blocks
+            let is_v1 = self.message.count_extra_fields() >= 2;
+            let extensions = if is_v1 {
+                let message_v1 =
+                    packed::SendBlocksProofV1Reader::new_unchecked(self.message.as_slice());
+                let uncle_hashes: Vec<_> = message_v1
+                    .blocks_uncles_hash()
+                    .iter()
+                    .map(|uncle_hashes| uncle_hashes.to_entity())
+                    .collect();
+
+                let extensions: Vec<_> = message_v1
+                    .blocks_extension()
+                    .iter()
+                    .map(|extension| extension.to_entity().to_opt())
+                    .collect();
+
+                return_if_failed!(verify_extra_hash(&headers, &uncle_hashes, &extensions));
+                extensions
+            } else {
+                vec![None; headers.len()]
+            };
+
             // Verify the proof
             return_if_failed!(verify_mmr_proof(
                 self.protocol.mmr_activated_epoch(),
@@ -184,9 +214,14 @@ impl<'a> SendBlocksProofProcess<'a> {
                 }
             }
 
-            for header in headers {
-                if self.protocol.peers().add_header(&header.hash()) {
-                    self.protocol.storage().add_fetched_header(&header.data());
+            for (header, extension) in headers.into_iter().zip(extensions.into_iter()) {
+                if self.protocol.peers().remove_fetching_header(&header.hash()) {
+                    self.protocol
+                        .storage()
+                        .add_fetched_header(&HeaderWithExtension {
+                            header: header.data(),
+                            extension,
+                        });
                 }
             }
         }
@@ -196,3 +231,31 @@ impl<'a> SendBlocksProofProcess<'a> {
         Status::ok()
     }
 }
+
+pub(crate) fn verify_extra_hash(
+    headers: &[HeaderView],
+    uncle_hashes: &[packed::Byte32],
+    extensions: &[Option<packed::Bytes>],
+) -> Result<(), Status> {
+    if headers.len() != uncle_hashes.len() || headers.len() != extensions.len() {
+        return Err(StatusCode::InvalidProof.into());
+    }
+
+    for ((header, uncle_hash), extension) in headers
+        .iter()
+        .zip(uncle_hashes.iter())
+        .zip(extensions.iter())
+    {
+        let expected_extension_hash = extension
+            .as_ref()
+            .map(|extension| extension.calc_raw_data_hash());
+        let extra_hash_view = ExtraHashView::new(uncle_hash.clone(), expected_extension_hash);
+        let expected_extra_hash = extra_hash_view.extra_hash();
+        let actual_extra_hash = header.extra_hash();
+        if expected_extra_hash != actual_extra_hash {
+            return Err(StatusCode::InvalidProof.into());
+        }
+    }
+
+    Ok(())
+}

src/protocols/light_client/components/send_transactions_proof.rs

@@ -6,6 +6,8 @@ use ckb_types::{
 };
 use log::{debug, error};
 
+use crate::{protocols::light_client::components::verify_extra_hash, storage::HeaderWithExtension};
+
 use super::{
     super::{LightClientProtocol, Status, StatusCode},
     verify_mmr_proof,
@@ -116,6 +118,29 @@ impl<'a> SendTransactionsProofProcess<'a> {
             // Check PoW for blocks
             return_if_failed!(self.protocol.check_pow_for_headers(headers.iter()));
 
+            // Check extra hash for blocks
+            let is_v1 = self.message.count_extra_fields() >= 2;
+            let extensions = if is_v1 {
+                let message_v1 =
+                    packed::SendTransactionsProofV1Reader::new_unchecked(self.message.as_slice());
+                let uncle_hashes: Vec<_> = message_v1
+                    .blocks_uncles_hash()
+                    .iter()
+                    .map(|uncle_hashes| uncle_hashes.to_entity())
+                    .collect();
+
+                let extensions: Vec<_> = message_v1
+                    .blocks_extension()
+                    .iter()
+                    .map(|extension| extension.to_entity().to_opt())
+                    .collect();
+
+                return_if_failed!(verify_extra_hash(&headers, &uncle_hashes, &extensions));
+                extensions
+            } else {
+                vec![None; headers.len()]
+            };
+
             // Verify the proof
             return_if_failed!(verify_mmr_proof(
                 self.protocol.mmr_activated_epoch(),
@@ -155,15 +180,21 @@ impl<'a> SendTransactionsProofProcess<'a> {
             }
             debug!("verify SendBlocksProof ok");
 
-            for filtered_block in filtered_blocks {
+            for (filtered_block, extension) in filtered_blocks.into_iter().zip(extensions.iter()) {
                 let header = filtered_block.header().into_view();
                 for tx in filtered_block.transactions() {
                     if self
                         .protocol
                         .peers()
-                        .add_transaction(&tx.calc_tx_hash(), &header.hash())
+                        .remove_fetching_transaction(&tx.calc_tx_hash(), &header.hash())
                     {
-                        self.protocol.storage().add_fetched_tx(&tx, &header.data());
+                        self.protocol.storage().add_fetched_tx(
+                            &tx,
+                            &HeaderWithExtension {
+                                header: header.data(),
+                                extension: extension.as_ref().cloned(),
+                            },
+                        );
                     }
                 }
             }

src/protocols/light_client/mod.rs

@@ -111,7 +111,7 @@ impl CKBProtocolHandler for LightClientProtocol {
         peer_index: PeerIndex,
         data: Bytes,
     ) {
-        let msg = match packed::LightClientMessageReader::from_slice(&data) {
+        let msg = match packed::LightClientMessageReader::from_compatible_slice(&data) {
             Ok(msg) => msg.to_enum(),
             _ => {
                 warn!(
@@ -718,7 +718,7 @@ impl LightClientProtocol {
 
         let now = unix_time_as_millis();
         let last_hash = tip_header.calc_header_hash();
-        for block_hashes_all in self
+        for block_hashes in self
             .peers
             .get_headers_to_fetch()
             .chunks(GET_BLOCKS_PROOF_LIMIT)
@@ -730,21 +730,9 @@ impl LightClientProtocol {
                     .unwrap_or(false)
             }) {
                 debug!("send block proof request to peer: {}", peer_index);
-                let mut block_hashes = Vec::with_capacity(block_hashes_all.len());
-                for block_hash in block_hashes_all {
-                    if block_hash == &last_hash {
-                        debug!("remove tip hash from block proof request {:#x}", last_hash);
-                        if self.peers().add_header(&last_hash) {
-                            debug!("fetching tip header, immediately add tip header to storage");
-                            self.storage().add_fetched_header(&tip_header);
-                        }
-                    } else {
-                        block_hashes.push(block_hash.clone());
-                    }
-                }
                 if !block_hashes.is_empty() {
                     let content = packed::GetBlocksProof::new_builder()
-                        .block_hashes(block_hashes.clone().pack())
+                        .block_hashes(block_hashes.to_vec().pack())
                         .last_hash(last_hash.clone())
                         .build();
                     let message = packed::LightClientMessage::new_builder()
@@ -763,7 +751,7 @@ impl LightClientProtocol {
                             format!("nc.send_message LightClientMessage, error: {:?}", err);
                         error!("{}", error_message);
                     }
-                    self.peers.fetching_idle_headers(&block_hashes, now);
+                    self.peers.fetching_idle_headers(block_hashes, now);
                 }
             } else {
                 debug!("all valid peers are busy for fetching blocks proof (headers)");

src/protocols/light_client/peers.rs

@@ -1386,13 +1386,17 @@ impl Peers {
         })
     }
 
-    pub(crate) fn add_header(&self, block_hash: &Byte32) -> bool {
+    pub(crate) fn remove_fetching_header(&self, block_hash: &Byte32) -> bool {
         self.fetching_headers.remove(block_hash).is_some()
     }
 
-    pub(crate) fn add_transaction(&self, tx_hash: &Byte32, block_hash: &Byte32) -> bool {
+    pub(crate) fn remove_fetching_transaction(
+        &self,
+        tx_hash: &Byte32,
+        block_hash: &Byte32,
+    ) -> bool {
         if self.fetching_txs.remove(tx_hash).is_some() {
-            self.add_header(block_hash);
+            self.remove_fetching_header(block_hash);
             true
         } else {
             false

src/protocols/relayer.rs

@@ -1,4 +1,3 @@
-use ckb_network::SupportProtocols;
 use ckb_network::{
     async_trait, bytes::Bytes, extract_peer_id, CKBProtocolContext, CKBProtocolHandler, PeerId,
     PeerIndex,
@@ -118,12 +117,7 @@ impl CKBProtocolHandler for RelayProtocol {
                     .tx_hashes(tx_hashes.pack())
                     .build();
                 let message = packed::RelayMessage::new_builder().set(content).build();
-
-                if let Err(err) = nc.send_message(
-                    SupportProtocols::RelayV2.protocol_id(),
-                    peer,
-                    message.as_bytes(),
-                ) {
+                if let Err(err) = nc.send_message_to(peer, message.as_bytes()) {
                     warn!(
                         "RelayProtocol failed to send RelayTransactionHashes message to peer={} since {:?}",
                         peer, err
@@ -216,9 +210,7 @@ impl CKBProtocolHandler for RelayProtocol {
                 {
                     let p2p_control = nc.p2p_control().expect("p2p_control should be exist");
                     for peer in self.connected_peers.get_peers_index() {
-                        if let Err(err) =
-                            p2p_control.open_protocol(peer, SupportProtocols::RelayV2.protocol_id())
-                        {
+                        if let Err(err) = p2p_control.open_protocol(peer, nc.protocol_id()) {
                             warn!(
                                 "RelayProtocol failed to open protocol to peer={} since {:?}",
                                 peer, err
@@ -241,11 +233,7 @@ impl CKBProtocolHandler for RelayProtocol {
                                 let message =
                                     packed::RelayMessage::new_builder().set(content).build();
 
-                                if let Err(err) = nc.send_message(
-                                    SupportProtocols::RelayV2.protocol_id(),
-                                    peer,
-                                    message.as_bytes(),
-                                ) {
+                                if let Err(err) = nc.send_message_to(peer, message.as_bytes()) {
                                     warn!(
                                         "RelayProtocol failed to send RelayTransactionHashes message to peer={} since {:?}",
                                         peer, err
@@ -263,7 +251,7 @@ impl CKBProtocolHandler for RelayProtocol {
                                 let _ = nc
                                     .p2p_control()
                                     .expect("p2p_control should be exist")
-                                    .close_protocol(peer, SupportProtocols::RelayV2.protocol_id());
+                                    .close_protocol(peer, nc.protocol_id());
                             }
                         }
                     }

src/service.rs

@@ -1267,13 +1267,8 @@ impl ChainRpc for ChainRpcImpl {
     }
 
     fn fetch_header(&self, block_hash: H256) -> Result<FetchStatus<HeaderView>> {
-        if let Some(value) = self.get_header(block_hash.clone())? {
-            if self.swc.storage().get_header(&block_hash.pack()).is_none() {
-                self.swc
-                    .storage()
-                    .add_fetched_header(&value.inner.clone().into());
-            }
-            return Ok(FetchStatus::Fetched { data: value });
+        if let Some(value) = self.swc.storage().get_header(&block_hash.pack()) {
+            return Ok(FetchStatus::Fetched { data: value.into() });
         }
         let now = unix_time_as_millis();
         if let Some((added_ts, first_sent, missing)) = self.swc.get_header_fetch_info(&block_hash) {