Add proper support for rounds where echo broadcasts are only sent to …

…a subset of nodes

manul/Cargo.toml

@@ -28,12 +28,14 @@ serde_json = { version = "1", default-features = false, features = ["alloc"], op
 
 [dev-dependencies]
 impls = "1"
+rand_core = { version = "0.6.4", default-features = false, features = ["getrandom"] }
 rand = { version = "0.8", default-features = false }
 serde_asn1_der = "0.8"
 criterion = "0.5"
 serde-persistent-deserializer = "0.3"
 postcard = { version = "1", default-features = false, features = ["alloc"] }
 serde_json = { version = "1", default-features = false, features = ["alloc"] }
+tracing-subscriber = { version = "0.3", features = ["env-filter"] }
 
 [features]
 testing = ["rand", "postcard", "serde_json", "serde-persistent-deserializer"]

manul/src/lib.rs

@@ -21,3 +21,6 @@ pub(crate) mod utils;
 
 #[cfg(any(test, feature = "testing"))]
 pub mod testing;
+
+#[cfg(test)]
+mod tests;

manul/src/protocol.rs

@@ -23,7 +23,8 @@ pub use errors::{
 };
 pub use message::{DirectMessage, EchoBroadcast, NormalBroadcast, ProtocolMessagePart};
 pub use round::{
-    AnotherRound, Artifact, FinalizeOutcome, FirstRound, PartyId, Payload, Protocol, ProtocolError, Round, RoundId,
+    AnotherRound, Artifact, EchoRoundParticipation, FinalizeOutcome, FirstRound, PartyId, Payload, Protocol,
+    ProtocolError, Round, RoundId,
 };
 pub use serialization::{Deserializer, Serializer};
 

manul/src/protocol/object_safe.rs

@@ -10,7 +10,7 @@ use rand_core::{CryptoRng, CryptoRngCore, RngCore};
 use super::{
     errors::{FinalizeError, LocalError, ReceiveError},
     message::{DirectMessage, EchoBroadcast, NormalBroadcast},
-    round::{Artifact, FinalizeOutcome, PartyId, Payload, Protocol, Round, RoundId},
+    round::{Artifact, EchoRoundParticipation, FinalizeOutcome, PartyId, Payload, Protocol, Round, RoundId},
     serialization::{Deserializer, Serializer},
 };
 
@@ -48,6 +48,10 @@ pub(crate) trait ObjectSafeRound<Id: PartyId>: 'static + Debug + Send + Sync {
 
     fn message_destinations(&self) -> &BTreeSet<Id>;
 
+    fn expecting_messages_from(&self) -> &BTreeSet<Id>;
+
+    fn echo_round_participation(&self) -> EchoRoundParticipation<Id>;
+
     fn make_direct_message_with_artifact(
         &self,
         rng: &mut dyn CryptoRngCore,
@@ -84,8 +88,6 @@ pub(crate) trait ObjectSafeRound<Id: PartyId>: 'static + Debug + Send + Sync {
         artifacts: BTreeMap<Id, Artifact>,
     ) -> Result<FinalizeOutcome<Id, Self::Protocol>, FinalizeError<Self::Protocol>>;
 
-    fn expecting_messages_from(&self) -> &BTreeSet<Id>;
-
     /// Returns the type ID of the implementing type.
     fn get_type_id(&self) -> core::any::TypeId;
 }
@@ -130,6 +132,14 @@ where
         self.round.message_destinations()
     }
 
+    fn expecting_messages_from(&self) -> &BTreeSet<Id> {
+        self.round.expecting_messages_from()
+    }
+
+    fn echo_round_participation(&self) -> EchoRoundParticipation<Id> {
+        self.round.echo_round_participation()
+    }
+
     fn make_direct_message_with_artifact(
         &self,
         rng: &mut dyn CryptoRngCore,
@@ -189,10 +199,6 @@ where
         self.round.finalize(&mut boxed_rng, payloads, artifacts)
     }
 
-    fn expecting_messages_from(&self) -> &BTreeSet<Id> {
-        self.round.expecting_messages_from()
-    }
-
     fn get_type_id(&self) -> core::any::TypeId {
         core::any::TypeId::of::<Self>()
     }

manul/src/protocol/round.rs

@@ -322,6 +322,25 @@ pub trait PartyId: 'static + Debug + Clone + Ord + Send + Sync {}
 
 impl<T> PartyId for T where T: 'static + Debug + Clone + Ord + Send + Sync {}
 
+/// The specific way the node participates in the echo round (if any).
+#[derive(Debug, Clone)]
+pub enum EchoRoundParticipation<Id> {
+    /// The default behavior: sends broadcasts and receives echoed messages, or does neither.
+    ///
+    /// That is, this node will be a part of the echo round if [`Round::make_echo_broadcast`] generates a message.
+    Default,
+
+    /// This node sends broadcasts that will be echoed, but does not receive any.
+    Send,
+
+    /// This node receives broadcasts that it needs to echo, but does not send any itself.
+    Receive {
+        /// The other participants of the echo round
+        /// (that is, the nodes to which echoed messages will be sent).
+        echo_targets: BTreeSet<Id>,
+    },
+}
+
 /**
 A type representing a single round of a protocol.
 
@@ -354,6 +373,21 @@ pub trait Round<Id: PartyId>: 'static + Debug + Send + Sync {
     ///   for each element of the returned set.
     fn message_destinations(&self) -> &BTreeSet<Id>;
 
+    /// Returns the set of node IDs from which this round expects messages.
+    ///
+    /// The execution layer will not call [`finalize`](`Self::finalize`) until all these nodes have responded
+    /// (and the corresponding [`receive_message`](`Self::receive_message`) finished successfully).
+    fn expecting_messages_from(&self) -> &BTreeSet<Id>;
+
+    /// Returns the specific way the node participates in the echo round following this round.
+    ///
+    /// Returns [`EchoRoundParticipation::Default`] by default; this works fine when every node
+    /// sends messages to every other one, or do not send or receive any echo broadcasts.
+    /// Otherwise, review the options in [`EchoRoundParticipation`] and pick the appropriate one.
+    fn echo_round_participation(&self) -> EchoRoundParticipation<Id> {
+        EchoRoundParticipation::Default
+    }
+
     /// Returns the direct message to the given destination and (maybe) an accompanying artifact.
     ///
     /// Return [`DirectMessage::none`] if this round does not send direct messages.
@@ -445,10 +479,4 @@ pub trait Round<Id: PartyId>: 'static + Debug + Send + Sync {
         payloads: BTreeMap<Id, Payload>,
         artifacts: BTreeMap<Id, Artifact>,
     ) -> Result<FinalizeOutcome<Id, Self::Protocol>, FinalizeError<Self::Protocol>>;
-
-    /// Returns the set of node IDs from which this round expects messages.
-    ///
-    /// The execution layer will not call [`finalize`](`Self::finalize`) until all these nodes have responded
-    /// (and the corresponding [`receive_message`](`Self::receive_message`) finished successfully).
-    fn expecting_messages_from(&self) -> &BTreeSet<Id>;
 }

manul/src/session/echo.rs

@@ -13,7 +13,7 @@ use tracing::debug;
 
 use super::{
     message::{MessageVerificationError, SignedMessagePart},
-    session::SessionParameters,
+    session::{EchoRoundInfo, SessionParameters},
     LocalError,
 };
 use crate::{
@@ -78,8 +78,7 @@ pub(crate) struct EchoRoundMessage<SP: SessionParameters> {
 pub struct EchoRound<P, SP: SessionParameters> {
     verifier: SP::Verifier,
     echo_broadcasts: BTreeMap<SP::Verifier, SignedMessagePart<EchoBroadcast>>,
-    destinations: BTreeSet<SP::Verifier>,
-    expected_echos: BTreeSet<SP::Verifier>,
+    echo_round_info: EchoRoundInfo<SP::Verifier>,
     main_round: Box<dyn ObjectSafeRound<SP::Verifier, Protocol = P>>,
     payloads: BTreeMap<SP::Verifier, Payload>,
     artifacts: BTreeMap<SP::Verifier, Artifact>,
@@ -94,25 +93,19 @@ where
         verifier: SP::Verifier,
         my_echo_broadcast: SignedMessagePart<EchoBroadcast>,
         echo_broadcasts: BTreeMap<SP::Verifier, SignedMessagePart<EchoBroadcast>>,
+        echo_round_info: EchoRoundInfo<SP::Verifier>,
         main_round: Box<dyn ObjectSafeRound<SP::Verifier, Protocol = P>>,
         payloads: BTreeMap<SP::Verifier, Payload>,
         artifacts: BTreeMap<SP::Verifier, Artifact>,
     ) -> Self {
-        let destinations = echo_broadcasts.keys().cloned().collect::<BTreeSet<_>>();
-
-        // Add our own echo message because we expect it to be sent back from other nodes.
-        let mut expected_echos = destinations.clone();
-        expected_echos.insert(verifier.clone());
-
         let mut echo_broadcasts = echo_broadcasts;
         echo_broadcasts.insert(verifier.clone(), my_echo_broadcast);
 
-        debug!("{:?}: initialized echo round with {:?}", verifier, destinations);
+        debug!("{:?}: initialized echo round with {:?}", verifier, echo_round_info);
         Self {
             verifier,
             echo_broadcasts,
-            destinations,
-            expected_echos,
+            echo_round_info,
             main_round,
             payloads,
             artifacts,
@@ -155,7 +148,7 @@ where
     }
 
     fn message_destinations(&self) -> &BTreeSet<SP::Verifier> {
-        &self.destinations
+        &self.echo_round_info.message_destinations
     }
 
     fn make_normal_broadcast(
@@ -181,7 +174,7 @@ where
     }
 
     fn expecting_messages_from(&self) -> &BTreeSet<SP::Verifier> {
-        &self.destinations
+        &self.echo_round_info.expecting_messages_from
     }
 
     fn receive_message(
@@ -200,16 +193,14 @@ where
 
         let message = normal_broadcast.deserialize::<EchoRoundMessage<SP>>(deserializer)?;
 
-        // Check that the received message contains entries from `destinations` sans `from`
+        // Check that the received message contains entries from `expected_echos`.
         // It is an unprovable fault.
 
-        let mut expected_keys = self.expected_echos.clone();
-        if !expected_keys.remove(from) {
-            return Err(ReceiveError::local(format!(
-                "The message sender {from:?} is missing from the expected senders {:?}",
-                self.destinations
-            )));
-        }
+        let mut expected_keys = self.echo_round_info.expected_echos.clone();
+
+        // We don't expect the node to send its echo the second time.
+        expected_keys.remove(from);
+
         let message_keys = message.echo_broadcasts.keys().cloned().collect::<BTreeSet<_>>();
 
         let missing_keys = expected_keys.difference(&message_keys).collect::<Vec<_>>();

manul/src/session/session.rs

@@ -23,9 +23,9 @@ use super::{
     LocalError, RemoteError,
 };
 use crate::protocol::{
-    Artifact, Deserializer, DirectMessage, EchoBroadcast, FinalizeError, FinalizeOutcome, FirstRound, NormalBroadcast,
-    ObjectSafeRound, ObjectSafeRoundWrapper, PartyId, Payload, Protocol, ProtocolMessagePart, ReceiveError,
-    ReceiveErrorType, Round, RoundId, Serializer,
+    Artifact, Deserializer, DirectMessage, EchoBroadcast, EchoRoundParticipation, FinalizeError, FinalizeOutcome,
+    FirstRound, NormalBroadcast, ObjectSafeRound, ObjectSafeRoundWrapper, PartyId, Payload, Protocol,
+    ProtocolMessagePart, ReceiveError, ReceiveErrorType, Round, RoundId, Serializer,
 };
 
 /// A set of types needed to execute a session.
@@ -97,6 +97,13 @@ impl AsRef<[u8]> for SessionId {
     }
 }
 
+#[derive(Debug)]
+pub(crate) struct EchoRoundInfo<Verifier> {
+    pub(crate) message_destinations: BTreeSet<Verifier>,
+    pub(crate) expecting_messages_from: BTreeSet<Verifier>,
+    pub(crate) expected_echos: BTreeSet<Verifier>,
+}
+
 /// An object encapsulating the currently active round, transport protocol,
 /// and the database of messages and errors from the previous rounds.
 #[derive(Debug)]
@@ -108,6 +115,7 @@ pub struct Session<P: Protocol, SP: SessionParameters> {
     deserializer: Deserializer,
     round: Box<dyn ObjectSafeRound<SP::Verifier, Protocol = P>>,
     message_destinations: BTreeSet<SP::Verifier>,
+    echo_round_info: Option<EchoRoundInfo<SP::Verifier>>,
     echo_broadcast: SignedMessagePart<EchoBroadcast>,
     normal_broadcast: SignedMessagePart<NormalBroadcast>,
     possible_next_rounds: BTreeSet<RoundId>,
@@ -182,10 +190,36 @@ where
 
         let message_destinations = round.message_destinations().clone();
 
-        let possible_next_rounds = if echo_broadcast.payload().is_none() {
-            round.possible_next_rounds()
-        } else {
+        let echo_round_participation = round.echo_round_participation();
+
+        let round_sends_echo_broadcast = !echo_broadcast.payload().is_none();
+        let echo_round_info = match echo_round_participation {
+            EchoRoundParticipation::Default => {
+                if round_sends_echo_broadcast {
+                    // Add our own echo message to the expected list because we expect it to be sent back from other nodes.
+                    let mut expected_echos = round.expecting_messages_from().clone();
+                    expected_echos.insert(verifier.clone());
+                    Some(EchoRoundInfo {
+                        message_destinations: message_destinations.clone(),
+                        expecting_messages_from: message_destinations.clone(),
+                        expected_echos,
+                    })
+                } else {
+                    None
+                }
+            }
+            EchoRoundParticipation::Send => None,
+            EchoRoundParticipation::Receive { echo_targets } => Some(EchoRoundInfo {
+                message_destinations: echo_targets.clone(),
+                expecting_messages_from: echo_targets,
+                expected_echos: round.expecting_messages_from().clone(),
+            }),
+        };
+
+        let possible_next_rounds = if echo_round_info.is_some() {
             BTreeSet::from([round.id().echo()])
+        } else {
+            round.possible_next_rounds()
         };
 
         Ok(Self {
@@ -199,6 +233,7 @@ where
             normal_broadcast,
             possible_next_rounds,
             message_destinations,
+            echo_round_info,
             transcript,
         })
     }
@@ -459,13 +494,12 @@ where
             accum.still_have_not_sent_messages,
         )?;
 
-        let echo_round_needed = !self.echo_broadcast.payload().is_none();
-
-        if echo_round_needed {
+        if let Some(echo_round_info) = self.echo_round_info {
             let round = Box::new(ObjectSafeRoundWrapper::new(EchoRound::<P, SP>::new(
                 verifier,
                 self.echo_broadcast,
                 transcript.echo_broadcasts(round_id)?,
+                echo_round_info,
                 self.round,
                 accum.payloads,
                 accum.artifacts,

manul/src/tests.rs

@@ -0,0 +1 @@
+mod partial_echo;