Support for larger confirm_req & confirm_ack messages

nano/core_test/message.cpp

@@ -90,6 +90,59 @@ TEST (message, publish_serialization)
 	ASSERT_EQ (nano::message_type::publish, header.type);
 }
 
+TEST (message, confirm_header_flags)
+{
+	nano::message_header header_v2{ nano::dev::network_params.network, nano::message_type::confirm_req };
+	header_v2.confirm_set_v2 (true);
+
+	const uint8_t value = 0b0110'1001;
+
+	header_v2.count_v2_set (value); // Max count value
+
+	ASSERT_TRUE (header_v2.confirm_is_v2 ());
+	ASSERT_EQ (header_v2.count_v2_get (), value);
+
+	std::vector<uint8_t> bytes;
+	{
+		nano::vectorstream stream (bytes);
+		header_v2.serialize (stream);
+	}
+	nano::bufferstream stream (bytes.data (), bytes.size ());
+
+	bool error = false;
+	nano::message_header header (error, stream);
+	ASSERT_FALSE (error);
+	ASSERT_EQ (nano::message_type::confirm_req, header.type);
+
+	ASSERT_TRUE (header.confirm_is_v2 ());
+	ASSERT_EQ (header.count_v2_get (), value);
+}
+
+TEST (message, confirm_header_flags_max)
+{
+	nano::message_header header_v2{ nano::dev::network_params.network, nano::message_type::confirm_req };
+	header_v2.confirm_set_v2 (true);
+	header_v2.count_v2_set (255); // Max count value
+
+	ASSERT_TRUE (header_v2.confirm_is_v2 ());
+	ASSERT_EQ (header_v2.count_v2_get (), 255);
+
+	std::vector<uint8_t> bytes;
+	{
+		nano::vectorstream stream (bytes);
+		header_v2.serialize (stream);
+	}
+	nano::bufferstream stream (bytes.data (), bytes.size ());
+
+	bool error = false;
+	nano::message_header header (error, stream);
+	ASSERT_FALSE (error);
+	ASSERT_EQ (nano::message_type::confirm_req, header.type);
+
+	ASSERT_TRUE (header.confirm_is_v2 ());
+	ASSERT_EQ (header.count_v2_get (), 255);
+}
+
 TEST (message, confirm_ack_hash_serialization)
 {
 	std::vector<nano::block_hash> hashes;

nano/node/messages.cpp

@@ -189,26 +189,71 @@ nano::block_type nano::message_header::block_type () const
 void nano::message_header::block_type_set (nano::block_type type_a)
 {
 	extensions &= ~block_type_mask;
-	extensions |= std::bitset<16> (static_cast<unsigned long long> (type_a) << 8);
+	extensions |= (extensions_bitset_t{ static_cast<unsigned long long> (type_a) } << 8);
 }
 
 uint8_t nano::message_header::count_get () const
 {
+	debug_assert (type == nano::message_type::confirm_ack || type == nano::message_type::confirm_req);
+	debug_assert (!flag_test (confirm_v2_flag)); // Only valid for v1
+
 	return static_cast<uint8_t> (((extensions & count_mask) >> 12).to_ullong ());
 }
 
 void nano::message_header::count_set (uint8_t count_a)
 {
-	debug_assert (count_a < 16);
+	debug_assert (type == nano::message_type::confirm_ack || type == nano::message_type::confirm_req);
+	debug_assert (!flag_test (confirm_v2_flag)); // Only valid for v1
+	debug_assert (count_a < 16); // Max 4 bits
+
 	extensions &= ~count_mask;
-	extensions |= std::bitset<16> (static_cast<unsigned long long> (count_a) << 12);
+	extensions |= ((extensions_bitset_t{ count_a } << 12) & count_mask);
+}
+
+/*
+ * We need those shenanigans because we need to keep compatibility with previous protocol versions (<= V25.1)
+ */
+
+uint8_t nano::message_header::count_v2_get () const
+{
+	debug_assert (type == nano::message_type::confirm_ack || type == nano::message_type::confirm_req);
+	debug_assert (flag_test (confirm_v2_flag)); // Only valid for v2
+
+	// Extract 2 parts of 4 bits
+	auto left = (extensions & count_v2_mask_left) >> 12;
+	auto right = (extensions & count_v2_mask_right) >> 4;
+
+	return static_cast<uint8_t> (((left << 4) | right).to_ullong ());
+}
+
+void nano::message_header::count_v2_set (uint8_t count)
+{
+	debug_assert (type == nano::message_type::confirm_ack || type == nano::message_type::confirm_req);
+	debug_assert (flag_test (confirm_v2_flag)); // Only valid for v2
+	debug_assert (count < 256); // Max 8 bits
+
+	extensions &= ~(count_v2_mask_left | count_v2_mask_right);
+
+	// Split count into 2 parts of 4 bits
+	extensions_bitset_t trim_mask{ 0xf };
+	auto left = (extensions_bitset_t{ count } >> 4) & trim_mask;
+	auto right = (extensions_bitset_t{ count }) & trim_mask;
+
+	extensions |= (left << 12) | (right << 4);
 }
 
-void nano::message_header::flag_set (uint8_t flag_a, bool enable)
+bool nano::message_header::flag_test (uint8_t flag) const
 {
-	// Flags from 8 are block_type & count
-	debug_assert (flag_a < 8);
-	extensions.set (flag_a, enable);
+	// Extension bits at index >= 8 are block type & count
+	debug_assert (flag < 8);
+	return extensions.test (flag);
+}
+
+void nano::message_header::flag_set (uint8_t flag, bool enable)
+{
+	// Extension bits at index >= 8 are block type & count
+	debug_assert (flag < 8);
+	extensions.set (flag, enable);
 }
 
 bool nano::message_header::bulk_pull_is_count_present () const
@@ -250,6 +295,18 @@ bool nano::message_header::frontier_req_is_only_confirmed_present () const
 	return result;
 }
 
+bool nano::message_header::confirm_is_v2 () const
+{
+	debug_assert (type == nano::message_type::confirm_ack || type == nano::message_type::confirm_req);
+	return flag_test (confirm_v2_flag);
+}
+
+void nano::message_header::confirm_set_v2 (bool value)
+{
+	debug_assert (type == nano::message_type::confirm_ack || type == nano::message_type::confirm_req);
+	flag_set (confirm_v2_flag, value);
+}
+
 std::size_t nano::message_header::payload_length_bytes () const
 {
 	switch (type)
@@ -282,7 +339,7 @@ std::size_t nano::message_header::payload_length_bytes () const
 		}
 		case nano::message_type::confirm_ack:
 		{
-			return nano::confirm_ack::size (count_get ());
+			return nano::confirm_ack::size (*this);
 		}
 		case nano::message_type::confirm_req:
 		{
@@ -559,7 +616,7 @@ bool nano::confirm_req::deserialize (nano::stream & stream_a)
 	bool result = false;
 	try
 	{
-		uint8_t const count = header.count_get ();
+		uint8_t const count = hash_count (header);
 		for (auto i (0); i != count && !result; ++i)
 		{
 			nano::block_hash block_hash (0);
@@ -605,9 +662,21 @@ std::string nano::confirm_req::roots_string () const
 	return result;
 }
 
+uint8_t nano::confirm_req::hash_count (const nano::message_header & header)
+{
+	if (header.confirm_is_v2 ())
+	{
+		return header.count_v2_get ();
+	}
+	else
+	{
+		return header.count_get ();
+	}
+}
+
 std::size_t nano::confirm_req::size (nano::message_header const & header)
 {
-	auto const count = header.count_get ();
+	auto const count = hash_count (header);
 	return count * (sizeof (decltype (roots_hashes)::value_type::first) + sizeof (decltype (roots_hashes)::value_type::second));
 }
 
@@ -641,8 +710,9 @@ nano::confirm_ack::confirm_ack (nano::network_constants const & constants, std::
 	message (constants, nano::message_type::confirm_ack),
 	vote (vote_a)
 {
-	debug_assert (vote_a->hashes.size () < 16);
+	debug_assert (vote->hashes.size () < 16);
 
+	header.block_type_set (nano::block_type::not_a_block);
 	header.count_set (static_cast<uint8_t> (vote_a->hashes.size ()));
 }
 
@@ -663,10 +733,22 @@ void nano::confirm_ack::visit (nano::message_visitor & visitor_a) const
 	visitor_a.confirm_ack (*this);
 }
 
-std::size_t nano::confirm_ack::size (std::size_t count)
+uint8_t nano::confirm_ack::hash_count (const nano::message_header & header)
 {
-	std::size_t result = sizeof (nano::account) + sizeof (nano::signature) + sizeof (uint64_t) + count * sizeof (nano::block_hash);
-	return result;
+	if (header.confirm_is_v2 ())
+	{
+		return header.count_v2_get ();
+	}
+	else
+	{
+		return header.count_get ();
+	}
+}
+
+std::size_t nano::confirm_ack::size (const nano::message_header & header)
+{
+	auto const count = hash_count (header);
+	return nano::vote::size (count);
 }
 
 std::string nano::confirm_ack::to_string () const

nano/node/messages.hpp

@@ -60,40 +60,59 @@ class message_visitor;
 class message_header final
 {
 public:
+	using extensions_bitset_t = std::bitset<16>;
+
 	message_header (nano::network_constants const &, nano::message_type);
 	message_header (bool &, nano::stream &);
+
 	void serialize (nano::stream &) const;
 	bool deserialize (nano::stream &);
-	nano::block_type block_type () const;
-	void block_type_set (nano::block_type);
-	uint8_t count_get () const;
-	void count_set (uint8_t);
+
+	std::string to_string () const;
+
+public: // Payload
 	nano::networks network;
 	uint8_t version_max;
 	uint8_t version_using;
 	uint8_t version_min;
-	std::string to_string () const;
+	nano::message_type type;
+	extensions_bitset_t extensions;
 
 public:
-	nano::message_type type;
-	std::bitset<16> extensions;
 	static std::size_t constexpr size = sizeof (nano::networks) + sizeof (version_max) + sizeof (version_using) + sizeof (version_min) + sizeof (type) + sizeof (/* extensions */ uint16_t);
 
-	void flag_set (uint8_t, bool enable = true);
+	bool flag_test (uint8_t flag) const;
+	void flag_set (uint8_t flag, bool enable = true);
+
+	nano::block_type block_type () const;
+	void block_type_set (nano::block_type);
+
+	uint8_t count_get () const;
+	void count_set (uint8_t);
+	uint8_t count_v2_get () const;
+	void count_v2_set (uint8_t);
+
 	static uint8_t constexpr bulk_pull_count_present_flag = 0;
 	static uint8_t constexpr bulk_pull_ascending_flag = 1;
 	bool bulk_pull_is_count_present () const;
 	bool bulk_pull_ascending () const;
+
 	static uint8_t constexpr frontier_req_only_confirmed = 1;
 	bool frontier_req_is_only_confirmed_present () const;
 
+	static uint8_t constexpr confirm_v2_flag = 0;
+	bool confirm_is_v2 () const;
+	void confirm_set_v2 (bool);
+
 	/** Size of the payload in bytes. For some messages, the payload size is based on header flags. */
 	std::size_t payload_length_bytes () const;
 	bool is_valid_message_type () const;
 
-	static std::bitset<16> constexpr block_type_mask{ 0x0f00 };
-	static std::bitset<16> constexpr count_mask{ 0xf000 };
-	static std::bitset<16> constexpr telemetry_size_mask{ 0x3ff };
+	static extensions_bitset_t constexpr block_type_mask{ 0x0f00 };
+	static extensions_bitset_t constexpr count_mask{ 0xf000 };
+	static extensions_bitset_t constexpr count_v2_mask_left{ 0xf000 };
+	static extensions_bitset_t constexpr count_v2_mask_right{ 0x00f0 };
+	static extensions_bitset_t constexpr telemetry_size_mask{ 0x3ff };
 };
 
 class message
@@ -148,6 +167,7 @@ class confirm_req final : public message
 	confirm_req (bool & error, nano::stream &, nano::message_header const &);
 	confirm_req (nano::network_constants const & constants, std::vector<std::pair<nano::block_hash, nano::root>> const &);
 	confirm_req (nano::network_constants const & constants, nano::block_hash const &, nano::root const &);
+
 	void serialize (nano::stream &) const override;
 	bool deserialize (nano::stream &);
 	void visit (nano::message_visitor &) const override;
@@ -157,20 +177,30 @@ class confirm_req final : public message
 
 	static std::size_t size (nano::message_header const &);
 
+private:
+	static uint8_t hash_count (nano::message_header const &);
+
 public: // Payload
 	std::vector<std::pair<nano::block_hash, nano::root>> roots_hashes;
 };
 
 class confirm_ack final : public message
 {
 public:
-	confirm_ack (bool &, nano::stream &, nano::message_header const &, nano::vote_uniquer * = nullptr);
+	confirm_ack (bool & error, nano::stream &, nano::message_header const &, nano::vote_uniquer * = nullptr);
 	confirm_ack (nano::network_constants const & constants, std::shared_ptr<nano::vote> const &);
+
 	void serialize (nano::stream &) const override;
 	void visit (nano::message_visitor &) const override;
 	bool operator== (nano::confirm_ack const &) const;
-	static std::size_t size (std::size_t count);
 	std::string to_string () const;
+
+	static std::size_t size (nano::message_header const &);
+
+private:
+	static uint8_t hash_count (nano::message_header const &);
+
+public: // Payload
 	std::shared_ptr<nano::vote> vote;
 };
 

nano/node/network.hpp

@@ -154,6 +154,7 @@ class network final
 	std::atomic<bool> stopped{ false };
 	static unsigned const broadcast_interval_ms = 10;
 	static std::size_t const buffer_size = 512;
+
 	static std::size_t const confirm_req_hashes_max = 7;
 	static std::size_t const confirm_ack_hashes_max = 12;
 };

nano/secure/vote.cpp

@@ -1,3 +1,4 @@
+#include <nano/lib/utility.hpp>
 #include <nano/secure/common.hpp>
 #include <nano/secure/vote.hpp>
 
@@ -13,11 +14,15 @@ nano::vote::vote (nano::account const & account_a, nano::raw_key const & prv_a,
 	timestamp_m{ packed_timestamp (timestamp_a, duration) },
 	account{ account_a }
 {
+	debug_assert (hashes.size () <= max_hashes);
+
 	signature = nano::sign_message (prv_a, account_a, hash ());
 }
 
 void nano::vote::serialize (nano::stream & stream_a) const
 {
+	debug_assert (hashes.size () <= max_hashes);
+
 	write (stream_a, account);
 	write (stream_a, signature);
 	write (stream_a, boost::endian::native_to_little (timestamp_m));
@@ -36,7 +41,7 @@ bool nano::vote::deserialize (nano::stream & stream_a)
 		nano::read (stream_a, signature.bytes);
 		nano::read (stream_a, timestamp_m);
 
-		while (stream_a.in_avail () > 0)
+		while (stream_a.in_avail () > 0 && hashes.size () < max_hashes)
 		{
 			nano::block_hash block_hash;
 			nano::read (stream_a, block_hash);
@@ -50,6 +55,12 @@ bool nano::vote::deserialize (nano::stream & stream_a)
 	return error;
 }
 
+std::size_t nano::vote::size (uint8_t count)
+{
+	debug_assert (count <= max_hashes);
+	return partial_size + count * sizeof (nano::block_hash);
+}
+
 std::string const nano::vote::hash_prefix = "vote ";
 
 nano::block_hash nano::vote::hash () const