From 767f3b1825737802b4147f3506a1718ef3ecb022 Mon Sep 17 00:00:00 2001 From: Guy Nir Date: Mon, 19 Aug 2024 16:08:41 +0300 Subject: [PATCH 1/6] feat: add poll_pending_boolean --- config/papyrus/default_config.json | 5 ++++ crates/papyrus_sync/src/lib.rs | 25 ++++++++++++------- .../src/sources/central_sync_test.rs | 1 + 3 files changed, 22 insertions(+), 9 deletions(-) diff --git a/config/papyrus/default_config.json b/config/papyrus/default_config.json index 9a0f2ddcb0..1b8638ee9e 100644 --- a/config/papyrus/default_config.json +++ b/config/papyrus/default_config.json @@ -403,5 +403,10 @@ "description": "Whether to verify incoming blocks.", "privacy": "Public", "value": true + }, + "sync.poll_pending_data": { + "description": "Whether to poll for pending data.", + "privacy": "Public", + "value": true } } \ No newline at end of file diff --git a/crates/papyrus_sync/src/lib.rs b/crates/papyrus_sync/src/lib.rs index faf9ba8717..22645c3490 100644 --- a/crates/papyrus_sync/src/lib.rs +++ b/crates/papyrus_sync/src/lib.rs @@ -70,6 +70,7 @@ pub struct SyncConfig { pub blocks_max_stream_size: u32, pub state_updates_max_stream_size: u32, pub verify_blocks: bool, + pub poll_pending_data: bool, } impl SerializeConfig for SyncConfig { @@ -125,6 +126,8 @@ impl Default for SyncConfig { blocks_max_stream_size: 1000, state_updates_max_stream_size: 1000, verify_blocks: true, + poll_pending_data: true, + } } } @@ -306,6 +309,7 @@ impl< self.pending_data.clone(), self.pending_classes.clone(), self.config.block_propagation_sleep_duration, + self.config.poll_pending_data, PENDING_SLEEP_DURATION, self.config.blocks_max_stream_size, ) @@ -664,6 +668,7 @@ fn stream_new_blocks< pending_data: Arc>, pending_classes: Arc>, block_propagation_sleep_duration: Duration, + poll_pending_data: bool, pending_sleep_duration: Duration, max_stream_size: u32, ) -> impl Stream> { @@ -682,15 +687,17 @@ fn stream_new_blocks< // Only if the node have the last block and state (without casms), sync pending data. if reader.begin_ro_txn()?.get_state_marker()? == header_marker{ // Here is the only place we update the pending data. - debug!("Start polling for pending data."); - sync_pending_data( - reader.clone(), - central_source.clone(), - pending_source.clone(), - pending_data.clone(), - pending_classes.clone(), - pending_sleep_duration, - ).await?; + if poll_pending_data { + debug!("Start polling for pending data."); + sync_pending_data( + reader.clone(), + central_source.clone(), + pending_source.clone(), + pending_data.clone(), + pending_classes.clone(), + pending_sleep_duration, + ).await?; + } } else{ debug!("Blocks syncing reached the last known block, waiting for blockchain to advance."); diff --git a/crates/papyrus_sync/src/sources/central_sync_test.rs b/crates/papyrus_sync/src/sources/central_sync_test.rs index 6826c55725..40773e844b 100644 --- a/crates/papyrus_sync/src/sources/central_sync_test.rs +++ b/crates/papyrus_sync/src/sources/central_sync_test.rs @@ -94,6 +94,7 @@ fn get_test_sync_config(verify_blocks: bool) -> SyncConfig { blocks_max_stream_size: STREAM_SIZE, state_updates_max_stream_size: STREAM_SIZE, verify_blocks, + poll_pending_data: true, } } From 6a36afe10b7958b1157053dbdd09995ec81adc1b Mon Sep 17 00:00:00 2001 From: Guy Nir Date: Tue, 20 Aug 2024 10:13:32 +0300 Subject: [PATCH 2/6] fix: add missing ser_param --- crates/papyrus_sync/src/lib.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/crates/papyrus_sync/src/lib.rs b/crates/papyrus_sync/src/lib.rs index 22645c3490..6a5a76eee2 100644 --- a/crates/papyrus_sync/src/lib.rs +++ b/crates/papyrus_sync/src/lib.rs @@ -113,6 +113,12 @@ impl SerializeConfig for SyncConfig { "Whether to verify incoming blocks.", ParamPrivacyInput::Public, ), + ser_param( + "poll_pending_data", + &self.poll_pending_data, + "Whether to poll for pending data.", + ParamPrivacyInput::Public, + ), ]) } } From bc7cf2e87e7152aa5aab8c445779f51523450baf Mon Sep 17 00:00:00 2001 From: Guy Nir Date: Tue, 20 Aug 2024 10:59:04 +0300 Subject: [PATCH 3/6] fix: add missing snapshot --- ..._config_test__dump_default_config.snap.new | 476 ++++++++++++++++++ 1 file changed, 476 insertions(+) create mode 100644 crates/papyrus_node/src/config/snapshots/papyrus_node__config__config_test__dump_default_config.snap.new diff --git a/crates/papyrus_node/src/config/snapshots/papyrus_node__config__config_test__dump_default_config.snap.new b/crates/papyrus_node/src/config/snapshots/papyrus_node__config__config_test__dump_default_config.snap.new new file mode 100644 index 0000000000..641025d780 --- /dev/null +++ b/crates/papyrus_node/src/config/snapshots/papyrus_node__config__config_test__dump_default_config.snap.new @@ -0,0 +1,476 @@ +--- +source: crates/papyrus_node/src/config/config_test.rs +assertion_line: 101 +expression: dumped_default_config +--- +{ + "base_layer.node_url": { + "description": "A required param! Ethereum node URL. A schema to match to Infura node: https://mainnet.infura.io/v3/, but any other node can be used.", + "param_type": "String", + "privacy": "Private" + }, + "base_layer.starknet_contract_address": { + "description": "Starknet contract address in ethereum.", + "value": "0xc662c410C0ECf747543f5bA90660f6ABeBD9C8c4", + "privacy": "Public" + }, + "central.class_cache_size": { + "description": "Size of class cache, must be a positive integer.", + "value": { + "$serde_json::private::Number": "100" + }, + "privacy": "Public" + }, + "central.concurrent_requests": { + "description": "Maximum number of concurrent requests to Starknet feeder-gateway for getting a type of data (for example, blocks).", + "value": { + "$serde_json::private::Number": "10" + }, + "privacy": "Public" + }, + "central.http_headers": { + "description": "'k1:v1 k2:v2 ...' headers for SN-client.", + "value": "", + "privacy": "Private" + }, + "central.max_classes_to_download": { + "description": "Maximum number of classes to download at a given time.", + "value": { + "$serde_json::private::Number": "20" + }, + "privacy": "Public" + }, + "central.max_state_updates_to_download": { + "description": "Maximum number of state updates to download at a given time.", + "value": { + "$serde_json::private::Number": "20" + }, + "privacy": "Public" + }, + "central.max_state_updates_to_store_in_memory": { + "description": "Maximum number of state updates to store in memory at a given time.", + "value": { + "$serde_json::private::Number": "20" + }, + "privacy": "Public" + }, + "central.retry_config.max_retries": { + "description": "Maximum number of retries before the node stops retrying.", + "value": { + "$serde_json::private::Number": "10" + }, + "privacy": "Public" + }, + "central.retry_config.retry_base_millis": { + "description": "Base waiting time after a failed request. After that, the time increases exponentially.", + "value": { + "$serde_json::private::Number": "30" + }, + "privacy": "Public" + }, + "central.retry_config.retry_max_delay_millis": { + "description": "Max waiting time after a failed request.", + "value": { + "$serde_json::private::Number": "30000" + }, + "privacy": "Public" + }, + "central.url": { + "description": "Starknet feeder-gateway URL. It should match chain_id.", + "value": "https://alpha-mainnet.starknet.io/", + "privacy": "Public" + }, + "collect_profiling_metrics": { + "description": "If true, collect profiling metrics for the node.", + "value": false, + "privacy": "Public" + }, + "consensus.#is_none": { + "description": "Flag for an optional field.", + "value": true, + "privacy": "TemporaryValue" + }, + "consensus.consensus_delay": { + "description": "Delay (seconds) before starting consensus to give time for network peering.", + "value": { + "$serde_json::private::Number": "5" + }, + "privacy": "Public" + }, + "consensus.network_topic": { + "description": "The network topic of the consensus.", + "value": "consensus", + "privacy": "Public" + }, + "consensus.num_validators": { + "description": "The number of validators in the consensus.", + "value": { + "$serde_json::private::Number": "4" + }, + "privacy": "Public" + }, + "consensus.start_height": { + "description": "The height to start the consensus from.", + "value": { + "$serde_json::private::Number": "0" + }, + "privacy": "Public" + }, + "consensus.test.#is_none": { + "description": "Flag for an optional field.", + "value": true, + "privacy": "TemporaryValue" + }, + "consensus.test.cache_size": { + "description": "The cache size for the test simulation.", + "value": { + "$serde_json::private::Number": "1000" + }, + "privacy": "Public" + }, + "consensus.test.drop_probability": { + "description": "The probability of dropping a message.", + "value": { + "$serde_json::private::Number": "0.0" + }, + "privacy": "Public" + }, + "consensus.test.invalid_probability": { + "description": "The probability of sending an invalid message.", + "value": { + "$serde_json::private::Number": "0.0" + }, + "privacy": "Public" + }, + "consensus.test.random_seed": { + "description": "The random seed for the test simulation to ensure repeatable test results.", + "value": { + "$serde_json::private::Number": "0" + }, + "privacy": "Public" + }, + "consensus.test.sync_topic": { + "description": "The network topic for sync messages.", + "value": "consensus_test_sync", + "privacy": "Public" + }, + "consensus.validator_id": { + "description": "A required param! The validator id of the node.", + "param_type": "String", + "privacy": "Public" + }, + "monitoring_gateway.collect_metrics": { + "description": "If true, collect and return metrics in the monitoring gateway.", + "value": false, + "privacy": "Public" + }, + "monitoring_gateway.metric_labels": { + "description": "'label1:value1 label2:value2 ...' additional labels for metrics.", + "value": "", + "privacy": "Public" + }, + "monitoring_gateway.present_full_config_secret": { + "description": "A secret for presenting the full general config. If no value is provided, the system will generate one.", + "param_type": "String", + "privacy": "Private" + }, + "monitoring_gateway.server_address": { + "description": "node's monitoring server.", + "value": "0.0.0.0:8081", + "privacy": "Public" + }, + "monitoring_gateway.starknet_url": { + "description": "The URL of a centralized Starknet gateway.", + "value": "https://alpha-mainnet.starknet.io/", + "privacy": "Public" + }, + "network.#is_none": { + "description": "Flag for an optional field.", + "value": true, + "privacy": "TemporaryValue" + }, + "network.bootstrap_peer_multiaddr": { + "description": "The multiaddress of the peer node. It should include the peer's id. For more info: https://docs.libp2p.io/concepts/fundamentals/peers/", + "value": "", + "privacy": "Public" + }, + "network.bootstrap_peer_multiaddr.#is_none": { + "description": "Flag for an optional field.", + "value": true, + "privacy": "TemporaryValue" + }, + "network.chain_id": { + "description": "The chain to follow. For more details see https://docs.starknet.io/documentation/architecture_and_concepts/Blocks/transactions/#chain-id.", + "value": "SN_MAIN", + "privacy": "Public" + }, + "network.idle_connection_timeout": { + "description": "Amount of time in seconds that a connection with no active sessions will stay alive.", + "value": { + "$serde_json::private::Number": "120" + }, + "privacy": "Public" + }, + "network.quic_port": { + "description": "The port that the node listens on for incoming quic connections.", + "value": { + "$serde_json::private::Number": "10001" + }, + "privacy": "Public" + }, + "network.secret_key": { + "description": "The secret key used for building the peer id. If it's an empty string a random one will be used.", + "value": "", + "privacy": "Private" + }, + "network.session_timeout": { + "description": "Maximal time in seconds that each session can take before failing on timeout.", + "value": { + "$serde_json::private::Number": "120" + }, + "privacy": "Public" + }, + "network.tcp_port": { + "description": "The port that the node listens on for incoming tcp connections.", + "value": { + "$serde_json::private::Number": "10000" + }, + "privacy": "Public" + }, + "p2p_sync.#is_none": { + "description": "Flag for an optional field.", + "value": true, + "privacy": "TemporaryValue" + }, + "p2p_sync.buffer_size": { + "description": "Size of the buffer for read from the storage and for incoming responses.", + "value": { + "$serde_json::private::Number": "100000" + }, + "privacy": "Public" + }, + "p2p_sync.num_block_state_diffs_per_query": { + "description": "The maximum amount of block's state diffs to ask from peers in each iteration.", + "value": { + "$serde_json::private::Number": "100" + }, + "privacy": "Public" + }, + "p2p_sync.num_headers_per_query": { + "description": "The maximum amount of headers to ask from peers in each iteration.", + "value": { + "$serde_json::private::Number": "10000" + }, + "privacy": "Public" + }, + "p2p_sync.num_transactions_per_query": { + "description": "The maximum amount of blocks to ask their transactions from peers in each iteration.", + "value": { + "$serde_json::private::Number": "100" + }, + "privacy": "Public" + }, + "p2p_sync.stop_sync_at_block_number": { + "description": "Stops the sync at given block number and closes the node cleanly. Used to run profiling on the node.", + "value": { + "$serde_json::private::Number": "1000" + }, + "privacy": "Public" + }, + "p2p_sync.stop_sync_at_block_number.#is_none": { + "description": "Flag for an optional field.", + "value": true, + "privacy": "TemporaryValue" + }, + "p2p_sync.wait_period_for_new_data": { + "description": "Time in seconds to wait when a query returned with partial data before sending a new query", + "value": { + "$serde_json::private::Number": "5" + }, + "privacy": "Public" + }, + "rpc.chain_id": { + "description": "The chain to follow. For more details see https://docs.starknet.io/documentation/architecture_and_concepts/Blocks/transactions/#chain-id.", + "value": "SN_MAIN", + "privacy": "Public" + }, + "rpc.collect_metrics": { + "description": "If true, collect metrics for the rpc.", + "value": false, + "privacy": "Public" + }, + "rpc.execution_config.eth_fee_contract_address": { + "description": "The eth fee token address to receive fees", + "value": "0x49d36570d4e46f48e99674bd3fcc84644ddd6b96f7c741b1562b82f9e004dc7", + "privacy": "Public" + }, + "rpc.execution_config.initial_gas_cost": { + "description": "The initial gas cost for a transaction", + "value": { + "$serde_json::private::Number": "10000000000" + }, + "privacy": "Public" + }, + "rpc.execution_config.strk_fee_contract_address": { + "description": "The strk fee token address to receive fees", + "value": "0x4718f5a0fc34cc1af16a1cdee98ffb20c31f5cd61d6ab07201858f4287c938d", + "privacy": "Public" + }, + "rpc.max_events_chunk_size": { + "description": "Maximum chunk size supported by the node in get_events requests.", + "value": { + "$serde_json::private::Number": "1000" + }, + "privacy": "Public" + }, + "rpc.max_events_keys": { + "description": "Maximum number of keys supported by the node in get_events requests.", + "value": { + "$serde_json::private::Number": "100" + }, + "privacy": "Public" + }, + "rpc.server_address": { + "description": "IP:PORT of the node`s JSON-RPC server.", + "value": "0.0.0.0:8080", + "privacy": "Public" + }, + "rpc.starknet_gateway_retry_config.max_retries": { + "description": "For communicating with Starknet gateway, maximum number of retries before the node stops retrying.", + "value": { + "$serde_json::private::Number": "5" + }, + "privacy": "Public" + }, + "rpc.starknet_gateway_retry_config.retry_base_millis": { + "description": "For communicating with Starknet gateway, base waiting time after a failed request. After that, the time increases exponentially.", + "value": { + "$serde_json::private::Number": "50" + }, + "privacy": "Public" + }, + "rpc.starknet_gateway_retry_config.retry_max_delay_millis": { + "description": "For communicating with Starknet gateway, max waiting time after a failed request.", + "value": { + "$serde_json::private::Number": "1000" + }, + "privacy": "Public" + }, + "rpc.starknet_url": { + "description": "URL for communicating with Starknet in write_api methods.", + "value": "https://alpha-mainnet.starknet.io/", + "privacy": "Public" + }, + "storage.db_config.chain_id": { + "description": "The chain to follow. For more details see https://docs.starknet.io/documentation/architecture_and_concepts/Blocks/transactions/#chain-id.", + "value": "SN_MAIN", + "privacy": "Public" + }, + "storage.db_config.enforce_file_exists": { + "description": "Whether to enforce that the path exists. If true, `open_env` fails when the mdbx.dat file does not exist.", + "value": false, + "privacy": "Public" + }, + "storage.db_config.growth_step": { + "description": "The growth step in bytes, must be greater than zero to allow the database to grow.", + "value": { + "$serde_json::private::Number": "4294967296" + }, + "privacy": "Public" + }, + "storage.db_config.max_size": { + "description": "The maximum size of the node's storage in bytes.", + "value": { + "$serde_json::private::Number": "1099511627776" + }, + "privacy": "Public" + }, + "storage.db_config.min_size": { + "description": "The minimum size of the node's storage in bytes.", + "value": { + "$serde_json::private::Number": "1048576" + }, + "privacy": "Public" + }, + "storage.db_config.path_prefix": { + "description": "Prefix of the path of the node's storage directory, the storage file path will be /. The path is not created automatically.", + "value": "./data", + "privacy": "Public" + }, + "storage.mmap_file_config.growth_step": { + "description": "The growth step in bytes, must be greater than max_object_size.", + "value": { + "$serde_json::private::Number": "1073741824" + }, + "privacy": "Public" + }, + "storage.mmap_file_config.max_object_size": { + "description": "The maximum size of a single object in the file in bytes", + "value": { + "$serde_json::private::Number": "268435456" + }, + "privacy": "Public" + }, + "storage.mmap_file_config.max_size": { + "description": "The maximum size of a memory mapped file in bytes. Must be greater than growth_step.", + "value": { + "$serde_json::private::Number": "1099511627776" + }, + "privacy": "Public" + }, + "storage.scope": { + "description": "The categories of data saved in storage.", + "value": "FullArchive", + "privacy": "Public" + }, + "sync.#is_none": { + "description": "Flag for an optional field.", + "value": false, + "privacy": "TemporaryValue" + }, + "sync.base_layer_propagation_sleep_duration": { + "description": "Time in seconds to poll the base layer to get the latest proved block.", + "value": { + "$serde_json::private::Number": "10" + }, + "privacy": "Public" + }, + "sync.block_propagation_sleep_duration": { + "description": "Time in seconds before checking for a new block after the node is synchronized.", + "value": { + "$serde_json::private::Number": "2" + }, + "privacy": "Public" + }, + "sync.blocks_max_stream_size": { + "description": "Max amount of blocks to download in a stream.", + "value": { + "$serde_json::private::Number": "1000" + }, + "privacy": "Public" + }, + "sync.poll_pending_data": { + "description": "Whether to poll for pending data.", + "value": true, + "privacy": "Public" + }, + "sync.recoverable_error_sleep_duration": { + "description": "Waiting time in seconds before restarting synchronization after a recoverable error.", + "value": { + "$serde_json::private::Number": "3" + }, + "privacy": "Public" + }, + "sync.state_updates_max_stream_size": { + "description": "Max amount of state updates to download in a stream.", + "value": { + "$serde_json::private::Number": "1000" + }, + "privacy": "Public" + }, + "sync.verify_blocks": { + "description": "Whether to verify incoming blocks.", + "value": true, + "privacy": "Public" + } +} From 65c7912f9869c36eaf4508180dde2dcc6c25d106 Mon Sep 17 00:00:00 2001 From: Guy Nir Date: Tue, 20 Aug 2024 11:04:40 +0300 Subject: [PATCH 4/6] fix: formatting --- crates/papyrus_sync/src/lib.rs | 3 +-- crates/papyrus_sync/src/sources/central_sync_test.rs | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/crates/papyrus_sync/src/lib.rs b/crates/papyrus_sync/src/lib.rs index 6a5a76eee2..468cf86fa9 100644 --- a/crates/papyrus_sync/src/lib.rs +++ b/crates/papyrus_sync/src/lib.rs @@ -133,7 +133,6 @@ impl Default for SyncConfig { state_updates_max_stream_size: 1000, verify_blocks: true, poll_pending_data: true, - } } } @@ -315,7 +314,7 @@ impl< self.pending_data.clone(), self.pending_classes.clone(), self.config.block_propagation_sleep_duration, - self.config.poll_pending_data, + self.config.poll_pending_data, PENDING_SLEEP_DURATION, self.config.blocks_max_stream_size, ) diff --git a/crates/papyrus_sync/src/sources/central_sync_test.rs b/crates/papyrus_sync/src/sources/central_sync_test.rs index 40773e844b..6a5dcb77c7 100644 --- a/crates/papyrus_sync/src/sources/central_sync_test.rs +++ b/crates/papyrus_sync/src/sources/central_sync_test.rs @@ -94,7 +94,7 @@ fn get_test_sync_config(verify_blocks: bool) -> SyncConfig { blocks_max_stream_size: STREAM_SIZE, state_updates_max_stream_size: STREAM_SIZE, verify_blocks, - poll_pending_data: true, + poll_pending_data: true, } } From 4e62729b2a3423c9cc35b4bea8d01613495e102c Mon Sep 17 00:00:00 2001 From: Guy Nir Date: Tue, 20 Aug 2024 12:18:38 +0300 Subject: [PATCH 5/6] fix: update the snapshot --- config/papyrus/default_config.json | 10 +- ...fig__config_test__dump_default_config.snap | 5 + ..._config_test__dump_default_config.snap.new | 476 ------------------ 3 files changed, 10 insertions(+), 481 deletions(-) delete mode 100644 crates/papyrus_node/src/config/snapshots/papyrus_node__config__config_test__dump_default_config.snap.new diff --git a/config/papyrus/default_config.json b/config/papyrus/default_config.json index 1b8638ee9e..ac5119e7f1 100644 --- a/config/papyrus/default_config.json +++ b/config/papyrus/default_config.json @@ -389,6 +389,11 @@ "privacy": "Public", "value": 1000 }, + "sync.poll_pending_data": { + "description": "Whether to poll for pending data.", + "privacy": "Public", + "value": true + }, "sync.recoverable_error_sleep_duration": { "description": "Waiting time in seconds before restarting synchronization after a recoverable error.", "privacy": "Public", @@ -403,10 +408,5 @@ "description": "Whether to verify incoming blocks.", "privacy": "Public", "value": true - }, - "sync.poll_pending_data": { - "description": "Whether to poll for pending data.", - "privacy": "Public", - "value": true } } \ No newline at end of file diff --git a/crates/papyrus_node/src/config/snapshots/papyrus_node__config__config_test__dump_default_config.snap b/crates/papyrus_node/src/config/snapshots/papyrus_node__config__config_test__dump_default_config.snap index b4a60c52ca..bb78c73260 100644 --- a/crates/papyrus_node/src/config/snapshots/papyrus_node__config__config_test__dump_default_config.snap +++ b/crates/papyrus_node/src/config/snapshots/papyrus_node__config__config_test__dump_default_config.snap @@ -458,6 +458,11 @@ expression: dumped_default_config }, "privacy": "Public" }, + "sync.poll_pending_data": { + "description": "Whether to poll for pending data.", + "value": true, + "privacy": "Public" + }, "sync.recoverable_error_sleep_duration": { "description": "Waiting time in seconds before restarting synchronization after a recoverable error.", "value": { diff --git a/crates/papyrus_node/src/config/snapshots/papyrus_node__config__config_test__dump_default_config.snap.new b/crates/papyrus_node/src/config/snapshots/papyrus_node__config__config_test__dump_default_config.snap.new deleted file mode 100644 index 641025d780..0000000000 --- a/crates/papyrus_node/src/config/snapshots/papyrus_node__config__config_test__dump_default_config.snap.new +++ /dev/null @@ -1,476 +0,0 @@ ---- -source: crates/papyrus_node/src/config/config_test.rs -assertion_line: 101 -expression: dumped_default_config ---- -{ - "base_layer.node_url": { - "description": "A required param! Ethereum node URL. A schema to match to Infura node: https://mainnet.infura.io/v3/, but any other node can be used.", - "param_type": "String", - "privacy": "Private" - }, - "base_layer.starknet_contract_address": { - "description": "Starknet contract address in ethereum.", - "value": "0xc662c410C0ECf747543f5bA90660f6ABeBD9C8c4", - "privacy": "Public" - }, - "central.class_cache_size": { - "description": "Size of class cache, must be a positive integer.", - "value": { - "$serde_json::private::Number": "100" - }, - "privacy": "Public" - }, - "central.concurrent_requests": { - "description": "Maximum number of concurrent requests to Starknet feeder-gateway for getting a type of data (for example, blocks).", - "value": { - "$serde_json::private::Number": "10" - }, - "privacy": "Public" - }, - "central.http_headers": { - "description": "'k1:v1 k2:v2 ...' headers for SN-client.", - "value": "", - "privacy": "Private" - }, - "central.max_classes_to_download": { - "description": "Maximum number of classes to download at a given time.", - "value": { - "$serde_json::private::Number": "20" - }, - "privacy": "Public" - }, - "central.max_state_updates_to_download": { - "description": "Maximum number of state updates to download at a given time.", - "value": { - "$serde_json::private::Number": "20" - }, - "privacy": "Public" - }, - "central.max_state_updates_to_store_in_memory": { - "description": "Maximum number of state updates to store in memory at a given time.", - "value": { - "$serde_json::private::Number": "20" - }, - "privacy": "Public" - }, - "central.retry_config.max_retries": { - "description": "Maximum number of retries before the node stops retrying.", - "value": { - "$serde_json::private::Number": "10" - }, - "privacy": "Public" - }, - "central.retry_config.retry_base_millis": { - "description": "Base waiting time after a failed request. After that, the time increases exponentially.", - "value": { - "$serde_json::private::Number": "30" - }, - "privacy": "Public" - }, - "central.retry_config.retry_max_delay_millis": { - "description": "Max waiting time after a failed request.", - "value": { - "$serde_json::private::Number": "30000" - }, - "privacy": "Public" - }, - "central.url": { - "description": "Starknet feeder-gateway URL. It should match chain_id.", - "value": "https://alpha-mainnet.starknet.io/", - "privacy": "Public" - }, - "collect_profiling_metrics": { - "description": "If true, collect profiling metrics for the node.", - "value": false, - "privacy": "Public" - }, - "consensus.#is_none": { - "description": "Flag for an optional field.", - "value": true, - "privacy": "TemporaryValue" - }, - "consensus.consensus_delay": { - "description": "Delay (seconds) before starting consensus to give time for network peering.", - "value": { - "$serde_json::private::Number": "5" - }, - "privacy": "Public" - }, - "consensus.network_topic": { - "description": "The network topic of the consensus.", - "value": "consensus", - "privacy": "Public" - }, - "consensus.num_validators": { - "description": "The number of validators in the consensus.", - "value": { - "$serde_json::private::Number": "4" - }, - "privacy": "Public" - }, - "consensus.start_height": { - "description": "The height to start the consensus from.", - "value": { - "$serde_json::private::Number": "0" - }, - "privacy": "Public" - }, - "consensus.test.#is_none": { - "description": "Flag for an optional field.", - "value": true, - "privacy": "TemporaryValue" - }, - "consensus.test.cache_size": { - "description": "The cache size for the test simulation.", - "value": { - "$serde_json::private::Number": "1000" - }, - "privacy": "Public" - }, - "consensus.test.drop_probability": { - "description": "The probability of dropping a message.", - "value": { - "$serde_json::private::Number": "0.0" - }, - "privacy": "Public" - }, - "consensus.test.invalid_probability": { - "description": "The probability of sending an invalid message.", - "value": { - "$serde_json::private::Number": "0.0" - }, - "privacy": "Public" - }, - "consensus.test.random_seed": { - "description": "The random seed for the test simulation to ensure repeatable test results.", - "value": { - "$serde_json::private::Number": "0" - }, - "privacy": "Public" - }, - "consensus.test.sync_topic": { - "description": "The network topic for sync messages.", - "value": "consensus_test_sync", - "privacy": "Public" - }, - "consensus.validator_id": { - "description": "A required param! The validator id of the node.", - "param_type": "String", - "privacy": "Public" - }, - "monitoring_gateway.collect_metrics": { - "description": "If true, collect and return metrics in the monitoring gateway.", - "value": false, - "privacy": "Public" - }, - "monitoring_gateway.metric_labels": { - "description": "'label1:value1 label2:value2 ...' additional labels for metrics.", - "value": "", - "privacy": "Public" - }, - "monitoring_gateway.present_full_config_secret": { - "description": "A secret for presenting the full general config. If no value is provided, the system will generate one.", - "param_type": "String", - "privacy": "Private" - }, - "monitoring_gateway.server_address": { - "description": "node's monitoring server.", - "value": "0.0.0.0:8081", - "privacy": "Public" - }, - "monitoring_gateway.starknet_url": { - "description": "The URL of a centralized Starknet gateway.", - "value": "https://alpha-mainnet.starknet.io/", - "privacy": "Public" - }, - "network.#is_none": { - "description": "Flag for an optional field.", - "value": true, - "privacy": "TemporaryValue" - }, - "network.bootstrap_peer_multiaddr": { - "description": "The multiaddress of the peer node. It should include the peer's id. For more info: https://docs.libp2p.io/concepts/fundamentals/peers/", - "value": "", - "privacy": "Public" - }, - "network.bootstrap_peer_multiaddr.#is_none": { - "description": "Flag for an optional field.", - "value": true, - "privacy": "TemporaryValue" - }, - "network.chain_id": { - "description": "The chain to follow. For more details see https://docs.starknet.io/documentation/architecture_and_concepts/Blocks/transactions/#chain-id.", - "value": "SN_MAIN", - "privacy": "Public" - }, - "network.idle_connection_timeout": { - "description": "Amount of time in seconds that a connection with no active sessions will stay alive.", - "value": { - "$serde_json::private::Number": "120" - }, - "privacy": "Public" - }, - "network.quic_port": { - "description": "The port that the node listens on for incoming quic connections.", - "value": { - "$serde_json::private::Number": "10001" - }, - "privacy": "Public" - }, - "network.secret_key": { - "description": "The secret key used for building the peer id. If it's an empty string a random one will be used.", - "value": "", - "privacy": "Private" - }, - "network.session_timeout": { - "description": "Maximal time in seconds that each session can take before failing on timeout.", - "value": { - "$serde_json::private::Number": "120" - }, - "privacy": "Public" - }, - "network.tcp_port": { - "description": "The port that the node listens on for incoming tcp connections.", - "value": { - "$serde_json::private::Number": "10000" - }, - "privacy": "Public" - }, - "p2p_sync.#is_none": { - "description": "Flag for an optional field.", - "value": true, - "privacy": "TemporaryValue" - }, - "p2p_sync.buffer_size": { - "description": "Size of the buffer for read from the storage and for incoming responses.", - "value": { - "$serde_json::private::Number": "100000" - }, - "privacy": "Public" - }, - "p2p_sync.num_block_state_diffs_per_query": { - "description": "The maximum amount of block's state diffs to ask from peers in each iteration.", - "value": { - "$serde_json::private::Number": "100" - }, - "privacy": "Public" - }, - "p2p_sync.num_headers_per_query": { - "description": "The maximum amount of headers to ask from peers in each iteration.", - "value": { - "$serde_json::private::Number": "10000" - }, - "privacy": "Public" - }, - "p2p_sync.num_transactions_per_query": { - "description": "The maximum amount of blocks to ask their transactions from peers in each iteration.", - "value": { - "$serde_json::private::Number": "100" - }, - "privacy": "Public" - }, - "p2p_sync.stop_sync_at_block_number": { - "description": "Stops the sync at given block number and closes the node cleanly. Used to run profiling on the node.", - "value": { - "$serde_json::private::Number": "1000" - }, - "privacy": "Public" - }, - "p2p_sync.stop_sync_at_block_number.#is_none": { - "description": "Flag for an optional field.", - "value": true, - "privacy": "TemporaryValue" - }, - "p2p_sync.wait_period_for_new_data": { - "description": "Time in seconds to wait when a query returned with partial data before sending a new query", - "value": { - "$serde_json::private::Number": "5" - }, - "privacy": "Public" - }, - "rpc.chain_id": { - "description": "The chain to follow. For more details see https://docs.starknet.io/documentation/architecture_and_concepts/Blocks/transactions/#chain-id.", - "value": "SN_MAIN", - "privacy": "Public" - }, - "rpc.collect_metrics": { - "description": "If true, collect metrics for the rpc.", - "value": false, - "privacy": "Public" - }, - "rpc.execution_config.eth_fee_contract_address": { - "description": "The eth fee token address to receive fees", - "value": "0x49d36570d4e46f48e99674bd3fcc84644ddd6b96f7c741b1562b82f9e004dc7", - "privacy": "Public" - }, - "rpc.execution_config.initial_gas_cost": { - "description": "The initial gas cost for a transaction", - "value": { - "$serde_json::private::Number": "10000000000" - }, - "privacy": "Public" - }, - "rpc.execution_config.strk_fee_contract_address": { - "description": "The strk fee token address to receive fees", - "value": "0x4718f5a0fc34cc1af16a1cdee98ffb20c31f5cd61d6ab07201858f4287c938d", - "privacy": "Public" - }, - "rpc.max_events_chunk_size": { - "description": "Maximum chunk size supported by the node in get_events requests.", - "value": { - "$serde_json::private::Number": "1000" - }, - "privacy": "Public" - }, - "rpc.max_events_keys": { - "description": "Maximum number of keys supported by the node in get_events requests.", - "value": { - "$serde_json::private::Number": "100" - }, - "privacy": "Public" - }, - "rpc.server_address": { - "description": "IP:PORT of the node`s JSON-RPC server.", - "value": "0.0.0.0:8080", - "privacy": "Public" - }, - "rpc.starknet_gateway_retry_config.max_retries": { - "description": "For communicating with Starknet gateway, maximum number of retries before the node stops retrying.", - "value": { - "$serde_json::private::Number": "5" - }, - "privacy": "Public" - }, - "rpc.starknet_gateway_retry_config.retry_base_millis": { - "description": "For communicating with Starknet gateway, base waiting time after a failed request. After that, the time increases exponentially.", - "value": { - "$serde_json::private::Number": "50" - }, - "privacy": "Public" - }, - "rpc.starknet_gateway_retry_config.retry_max_delay_millis": { - "description": "For communicating with Starknet gateway, max waiting time after a failed request.", - "value": { - "$serde_json::private::Number": "1000" - }, - "privacy": "Public" - }, - "rpc.starknet_url": { - "description": "URL for communicating with Starknet in write_api methods.", - "value": "https://alpha-mainnet.starknet.io/", - "privacy": "Public" - }, - "storage.db_config.chain_id": { - "description": "The chain to follow. For more details see https://docs.starknet.io/documentation/architecture_and_concepts/Blocks/transactions/#chain-id.", - "value": "SN_MAIN", - "privacy": "Public" - }, - "storage.db_config.enforce_file_exists": { - "description": "Whether to enforce that the path exists. If true, `open_env` fails when the mdbx.dat file does not exist.", - "value": false, - "privacy": "Public" - }, - "storage.db_config.growth_step": { - "description": "The growth step in bytes, must be greater than zero to allow the database to grow.", - "value": { - "$serde_json::private::Number": "4294967296" - }, - "privacy": "Public" - }, - "storage.db_config.max_size": { - "description": "The maximum size of the node's storage in bytes.", - "value": { - "$serde_json::private::Number": "1099511627776" - }, - "privacy": "Public" - }, - "storage.db_config.min_size": { - "description": "The minimum size of the node's storage in bytes.", - "value": { - "$serde_json::private::Number": "1048576" - }, - "privacy": "Public" - }, - "storage.db_config.path_prefix": { - "description": "Prefix of the path of the node's storage directory, the storage file path will be /. The path is not created automatically.", - "value": "./data", - "privacy": "Public" - }, - "storage.mmap_file_config.growth_step": { - "description": "The growth step in bytes, must be greater than max_object_size.", - "value": { - "$serde_json::private::Number": "1073741824" - }, - "privacy": "Public" - }, - "storage.mmap_file_config.max_object_size": { - "description": "The maximum size of a single object in the file in bytes", - "value": { - "$serde_json::private::Number": "268435456" - }, - "privacy": "Public" - }, - "storage.mmap_file_config.max_size": { - "description": "The maximum size of a memory mapped file in bytes. Must be greater than growth_step.", - "value": { - "$serde_json::private::Number": "1099511627776" - }, - "privacy": "Public" - }, - "storage.scope": { - "description": "The categories of data saved in storage.", - "value": "FullArchive", - "privacy": "Public" - }, - "sync.#is_none": { - "description": "Flag for an optional field.", - "value": false, - "privacy": "TemporaryValue" - }, - "sync.base_layer_propagation_sleep_duration": { - "description": "Time in seconds to poll the base layer to get the latest proved block.", - "value": { - "$serde_json::private::Number": "10" - }, - "privacy": "Public" - }, - "sync.block_propagation_sleep_duration": { - "description": "Time in seconds before checking for a new block after the node is synchronized.", - "value": { - "$serde_json::private::Number": "2" - }, - "privacy": "Public" - }, - "sync.blocks_max_stream_size": { - "description": "Max amount of blocks to download in a stream.", - "value": { - "$serde_json::private::Number": "1000" - }, - "privacy": "Public" - }, - "sync.poll_pending_data": { - "description": "Whether to poll for pending data.", - "value": true, - "privacy": "Public" - }, - "sync.recoverable_error_sleep_duration": { - "description": "Waiting time in seconds before restarting synchronization after a recoverable error.", - "value": { - "$serde_json::private::Number": "3" - }, - "privacy": "Public" - }, - "sync.state_updates_max_stream_size": { - "description": "Max amount of state updates to download in a stream.", - "value": { - "$serde_json::private::Number": "1000" - }, - "privacy": "Public" - }, - "sync.verify_blocks": { - "description": "Whether to verify incoming blocks.", - "value": true, - "privacy": "Public" - } -} From 84319185ed0e27b648fb23a46060775110eb9558 Mon Sep 17 00:00:00 2001 From: Guy Nir Date: Wed, 21 Aug 2024 12:12:16 +0300 Subject: [PATCH 6/6] fix: reviewer comments --- config/papyrus/default_config.json | 6 ++-- ...fig__config_test__dump_default_config.snap | 6 ++-- crates/papyrus_sync/src/lib.rs | 36 +++++++++---------- .../src/sources/central_sync_test.rs | 2 +- 4 files changed, 24 insertions(+), 26 deletions(-) diff --git a/config/papyrus/default_config.json b/config/papyrus/default_config.json index ac5119e7f1..4647c6717b 100644 --- a/config/papyrus/default_config.json +++ b/config/papyrus/default_config.json @@ -389,10 +389,10 @@ "privacy": "Public", "value": 1000 }, - "sync.poll_pending_data": { - "description": "Whether to poll for pending data.", + "sync.collect_pending_data": { + "description": "Whether to collect data on pending blocks.", "privacy": "Public", - "value": true + "value": false }, "sync.recoverable_error_sleep_duration": { "description": "Waiting time in seconds before restarting synchronization after a recoverable error.", diff --git a/crates/papyrus_node/src/config/snapshots/papyrus_node__config__config_test__dump_default_config.snap b/crates/papyrus_node/src/config/snapshots/papyrus_node__config__config_test__dump_default_config.snap index bb78c73260..090e23610c 100644 --- a/crates/papyrus_node/src/config/snapshots/papyrus_node__config__config_test__dump_default_config.snap +++ b/crates/papyrus_node/src/config/snapshots/papyrus_node__config__config_test__dump_default_config.snap @@ -458,9 +458,9 @@ expression: dumped_default_config }, "privacy": "Public" }, - "sync.poll_pending_data": { - "description": "Whether to poll for pending data.", - "value": true, + "sync.collect_pending_data": { + "description": "Whether to collect data on pending blocks.", + "value": false, "privacy": "Public" }, "sync.recoverable_error_sleep_duration": { diff --git a/crates/papyrus_sync/src/lib.rs b/crates/papyrus_sync/src/lib.rs index 468cf86fa9..10a57782ee 100644 --- a/crates/papyrus_sync/src/lib.rs +++ b/crates/papyrus_sync/src/lib.rs @@ -70,7 +70,7 @@ pub struct SyncConfig { pub blocks_max_stream_size: u32, pub state_updates_max_stream_size: u32, pub verify_blocks: bool, - pub poll_pending_data: bool, + pub collect_pending_data: bool, } impl SerializeConfig for SyncConfig { @@ -114,9 +114,9 @@ impl SerializeConfig for SyncConfig { ParamPrivacyInput::Public, ), ser_param( - "poll_pending_data", - &self.poll_pending_data, - "Whether to poll for pending data.", + "collect_pending_data", + &self.collect_pending_data, + "Whether to collect data on pending blocks.", ParamPrivacyInput::Public, ), ]) @@ -132,7 +132,7 @@ impl Default for SyncConfig { blocks_max_stream_size: 1000, state_updates_max_stream_size: 1000, verify_blocks: true, - poll_pending_data: true, + collect_pending_data: false, } } } @@ -314,7 +314,7 @@ impl< self.pending_data.clone(), self.pending_classes.clone(), self.config.block_propagation_sleep_duration, - self.config.poll_pending_data, + self.config.collect_pending_data, PENDING_SLEEP_DURATION, self.config.blocks_max_stream_size, ) @@ -673,7 +673,7 @@ fn stream_new_blocks< pending_data: Arc>, pending_classes: Arc>, block_propagation_sleep_duration: Duration, - poll_pending_data: bool, + collect_pending_data: bool, pending_sleep_duration: Duration, max_stream_size: u32, ) -> impl Stream> { @@ -690,19 +690,17 @@ fn stream_new_blocks< ); if header_marker == central_block_marker { // Only if the node have the last block and state (without casms), sync pending data. - if reader.begin_ro_txn()?.get_state_marker()? == header_marker{ + if collect_pending_data && reader.begin_ro_txn()?.get_state_marker()? == header_marker{ // Here is the only place we update the pending data. - if poll_pending_data { - debug!("Start polling for pending data."); - sync_pending_data( - reader.clone(), - central_source.clone(), - pending_source.clone(), - pending_data.clone(), - pending_classes.clone(), - pending_sleep_duration, - ).await?; - } + debug!("Start polling for pending data."); + sync_pending_data( + reader.clone(), + central_source.clone(), + pending_source.clone(), + pending_data.clone(), + pending_classes.clone(), + pending_sleep_duration, + ).await?; } else{ debug!("Blocks syncing reached the last known block, waiting for blockchain to advance."); diff --git a/crates/papyrus_sync/src/sources/central_sync_test.rs b/crates/papyrus_sync/src/sources/central_sync_test.rs index 6a5dcb77c7..6a90c6a765 100644 --- a/crates/papyrus_sync/src/sources/central_sync_test.rs +++ b/crates/papyrus_sync/src/sources/central_sync_test.rs @@ -94,7 +94,7 @@ fn get_test_sync_config(verify_blocks: bool) -> SyncConfig { blocks_max_stream_size: STREAM_SIZE, state_updates_max_stream_size: STREAM_SIZE, verify_blocks, - poll_pending_data: true, + collect_pending_data: false, } }