Merge branch 'develop' into networking-fixes/daemon-latch-rebased

.github/workflows/build_deploy.yml

@@ -102,51 +102,6 @@ jobs:
           AWS_DEFAULT_REGION: us-east-2
 
 
-  linux_rpm_job:
-    needs: prepare_build
-    if: ${{ needs.prepare_build.outputs.tag_created == 'true' }}
-    runs-on: ubuntu-22.04
-    timeout-minutes: 90
-    strategy:
-      matrix:
-        network: ["BETA", "LIVE"] #No path to build TEST exists ci/build-rhel.sh
-    steps:
-      - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 #v3.1.0
-        with:
-          submodules: "recursive"
-          ref: "develop" #build-rhel.sh needs develop branch and then sets the tag
-          repository: ${{ github.repository }}
-      - name: Build local/nano-env:rhel
-        run: ci/actions/linux/install_deps.sh
-        env:
-          COMPILER: rhel
-          DOCKER_REGISTRY: local
-      - name: RockyLinux 8 Base
-        run: ci/build-docker-image.sh docker/ci/Dockerfile-rhel local/nano-env:rhel
-      - name: Build Artifact
-        run: |
-          mkdir -p ${GITHUB_WORKSPACE}/artifacts
-          docker run -v ${GITHUB_WORKSPACE}:/workspace -v ${GITHUB_WORKSPACE}/artifacts:/root/rpmbuild \
-          local/nano-env:rhel /bin/bash -c " \
-          NETWORK=${{ matrix.network }} \
-          TAG=${{ needs.prepare_build.outputs.ci_tag }} \
-          REPO_TO_BUILD=${{ github.repository }} \
-          RPM_RELEASE=1 \
-          ci/build-rhel.sh"
-
-      - name: Deploy Artifacts
-        run: ci/actions/deploy.sh
-        env:
-          LINUX_RPM: 1
-          NETWORK: ${{ matrix.network }}
-          # TAG: ${{ needs.prepare_build.outputs.ci_tag }} # (not used in the deploy script if LINUX_RPM==1 )
-          S3_BUCKET_NAME: ${{ vars.S3_BUCKET_NAME }}
-          S3_BUILD_DIRECTORY: ${{ vars.S3_BUILD_DIRECTORY }}
-          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-          AWS_DEFAULT_REGION: us-east-2
-
-
   linux_docker_job:
     needs: prepare_build
     if: ${{ needs.prepare_build.outputs.tag_created == 'true' }}

.github/workflows/unit_tests.yml

@@ -143,4 +143,5 @@ jobs:
         if: steps.build.outcome == 'success' && (success() || failure())
         run: ../ci/tests/run-system-tests.sh
         working-directory: build
-        shell: bash
+        shell: bash
+        continue-on-error: true # FIXME: Investigate why this fails on Windows

ci/tests/run-system-tests.sh

@@ -1,9 +1,51 @@
 #!/bin/bash
-set -euo pipefail
+set -uo pipefail
 
 source "$(dirname "$BASH_SOURCE")/common.sh"
 
-BUILD_DIR=${1-${PWD}}
+# Path to the nano-node repository can be provided as an argument
+# Otherwise parent directory of working directory is assumed
+NANO_REPO_DIR=${1:-../}
+NANO_SYSTEST_DIR=${NANO_REPO_DIR}/systest
 
-export NANO_NODE_EXE=${BUILD_DIR}/nano_node$(get_exec_extension)
-cd ../systest && ./RUNALL
+# Allow TEST_TIMEOUT to be set from an environment variable
+TEST_TIMEOUT=${TEST_TIMEOUT:-300s}
+
+echo "Running systests from: ${NANO_SYSTEST_DIR}"
+
+# This assumes that the executables are in the current working directory
+export NANO_NODE_EXE=./nano_node$(get_exec_extension)
+export NANO_RPC_EXE=./nano_rpc$(get_exec_extension)
+
+overall_status=0
+
+for script in ${NANO_SYSTEST_DIR}/*.sh; do
+    name=$(basename ${script})
+
+    echo "::group::Running: $name"
+
+    # Redirecting output to a file to prevent it from being mixed with the output of the action
+    # Using timeout command to enforce time limits
+    timeout $TEST_TIMEOUT ./$script > "${name}.log" 2>&1
+    status=$?
+    cat "${name}.log"
+
+    echo "::endgroup::"
+
+    if [ $status -eq 0 ]; then
+        echo "Passed: $name"
+    elif [ $status -eq 124 ]; then
+        echo "::error::Systest timed out: $name"
+        overall_status=1
+    else
+        echo "::error::Systest failed: $name ($status)"
+        overall_status=1
+    fi
+done
+
+if [ $overall_status -eq 0 ]; then
+    echo "::notice::All systests passed"
+else
+    echo "::error::Some systests failed"
+    exit 1
+fi

nano/core_test/active_transactions.cpp

@@ -253,7 +253,7 @@ TEST (inactive_votes_cache, basic)
 	node.vote_processor.vote (vote, std::make_shared<nano::transport::inproc::channel> (node, node));
 	ASSERT_TIMELY_EQ (5s, node.vote_cache.size (), 1);
 	node.process_active (send);
-	ASSERT_TIMELY (5s, node.ledger.block_confirmed (node.store.tx_begin_read (), send->hash ()));
+	ASSERT_TIMELY (5s, node.ledger.block_confirmed (node.ledger.tx_begin_read (), send->hash ()));
 	ASSERT_EQ (1, node.stats.count (nano::stat::type::election, nano::stat::detail::vote_cached));
 }
 
@@ -430,7 +430,7 @@ TEST (inactive_votes_cache, multiple_votes)
 
 	ASSERT_TIMELY_EQ (5s, node.vote_cache.find (send1->hash ()).size (), 2);
 	ASSERT_EQ (1, node.vote_cache.size ());
-	node.scheduler.priority.activate (nano::dev::genesis_key.pub, node.store.tx_begin_read ());
+	node.scheduler.priority.activate (nano::dev::genesis_key.pub, node.ledger.tx_begin_read ());
 	std::shared_ptr<nano::election> election;
 	ASSERT_TIMELY (5s, election = node.active.election (send1->qualified_root ()));
 	ASSERT_EQ (3, election->votes ().size ()); // 2 votes and 1 default not_an_acount
@@ -534,7 +534,7 @@ TEST (inactive_votes_cache, election_start)
 	// An election is started for send6 but does not
 	ASSERT_FALSE (node.block_confirmed_or_being_confirmed (send3->hash ()));
 	// send7 cannot be voted on but an election should be started from inactive votes
-	ASSERT_FALSE (node.ledger.dependents_confirmed (node.store.tx_begin_read (), *send4));
+	ASSERT_FALSE (node.ledger.dependents_confirmed (node.ledger.tx_begin_read (), *send4));
 	node.process_active (send4);
 	ASSERT_TIMELY_EQ (5s, 7, node.ledger.cemented_count ());
 }
@@ -1016,9 +1016,9 @@ TEST (active_transactions, confirmation_consistency)
 		auto block (system.wallet (0)->send_action (nano::dev::genesis_key.pub, nano::public_key (), node.config.receive_minimum.number ()));
 		ASSERT_NE (nullptr, block);
 		system.deadline_set (5s);
-		while (!node.ledger.block_confirmed (node.store.tx_begin_read (), block->hash ()))
+		while (!node.ledger.block_confirmed (node.ledger.tx_begin_read (), block->hash ()))
 		{
-			node.scheduler.priority.activate (nano::dev::genesis_key.pub, node.store.tx_begin_read ());
+			node.scheduler.priority.activate (nano::dev::genesis_key.pub, node.ledger.tx_begin_read ());
 			ASSERT_NO_ERROR (system.poll (5ms));
 		}
 		ASSERT_NO_ERROR (system.poll_until_true (1s, [&node, &block, i] {
@@ -1159,19 +1159,19 @@ TEST (active_transactions, activate_account_chain)
 	ASSERT_EQ (nano::block_status::progress, node.process (open));
 	ASSERT_EQ (nano::block_status::progress, node.process (receive));
 
-	node.scheduler.priority.activate (nano::dev::genesis_key.pub, node.store.tx_begin_read ());
+	node.scheduler.priority.activate (nano::dev::genesis_key.pub, node.ledger.tx_begin_read ());
 	ASSERT_TIMELY (5s, node.active.election (send->qualified_root ()));
 	auto election1 = node.active.election (send->qualified_root ());
 	ASSERT_EQ (1, node.active.size ());
 	ASSERT_EQ (1, election1->blocks ().count (send->hash ()));
-	node.scheduler.priority.activate (nano::dev::genesis_key.pub, node.store.tx_begin_read ());
+	node.scheduler.priority.activate (nano::dev::genesis_key.pub, node.ledger.tx_begin_read ());
 	auto election2 = node.active.election (send->qualified_root ());
 	ASSERT_EQ (election2, election1);
 	election1->force_confirm ();
 	ASSERT_TIMELY (3s, node.block_confirmed (send->hash ()));
 	// On cementing, the next election is started
 	ASSERT_TIMELY (3s, node.active.active (send2->qualified_root ()));
-	node.scheduler.priority.activate (nano::dev::genesis_key.pub, node.store.tx_begin_read ());
+	node.scheduler.priority.activate (nano::dev::genesis_key.pub, node.ledger.tx_begin_read ());
 	auto election3 = node.active.election (send2->qualified_root ());
 	ASSERT_NE (nullptr, election3);
 	ASSERT_EQ (1, election3->blocks ().count (send2->hash ()));
@@ -1180,19 +1180,19 @@ TEST (active_transactions, activate_account_chain)
 	// On cementing, the next election is started
 	ASSERT_TIMELY (3s, node.active.active (open->qualified_root ()));
 	ASSERT_TIMELY (3s, node.active.active (send3->qualified_root ()));
-	node.scheduler.priority.activate (nano::dev::genesis_key.pub, node.store.tx_begin_read ());
+	node.scheduler.priority.activate (nano::dev::genesis_key.pub, node.ledger.tx_begin_read ());
 	auto election4 = node.active.election (send3->qualified_root ());
 	ASSERT_NE (nullptr, election4);
 	ASSERT_EQ (1, election4->blocks ().count (send3->hash ()));
-	node.scheduler.priority.activate (key.pub, node.store.tx_begin_read ());
+	node.scheduler.priority.activate (key.pub, node.ledger.tx_begin_read ());
 	auto election5 = node.active.election (open->qualified_root ());
 	ASSERT_NE (nullptr, election5);
 	ASSERT_EQ (1, election5->blocks ().count (open->hash ()));
 	election5->force_confirm ();
 	ASSERT_TIMELY (3s, node.block_confirmed (open->hash ()));
 	// Until send3 is also confirmed, the receive block should not activate
 	std::this_thread::sleep_for (200ms);
-	node.scheduler.priority.activate (key.pub, node.store.tx_begin_read ());
+	node.scheduler.priority.activate (key.pub, node.ledger.tx_begin_read ());
 	election4->force_confirm ();
 	ASSERT_TIMELY (3s, node.block_confirmed (send3->hash ()));
 	ASSERT_TIMELY (3s, node.active.active (receive->qualified_root ()));
@@ -1248,9 +1248,9 @@ TEST (active_transactions, activate_inactive)
 	ASSERT_TIMELY (5s, node.block_confirmed (send2->hash ()));
 	ASSERT_TIMELY (5s, node.block_confirmed (send->hash ()));
 
-	ASSERT_EQ (1, node.stats.count (nano::stat::type::confirmation_observer, nano::stat::detail::inactive_conf_height, nano::stat::dir::out));
-	ASSERT_EQ (1, node.stats.count (nano::stat::type::confirmation_observer, nano::stat::detail::active_quorum, nano::stat::dir::out));
-	ASSERT_EQ (0, node.stats.count (nano::stat::type::confirmation_observer, nano::stat::detail::active_conf_height, nano::stat::dir::out));
+	ASSERT_TIMELY_EQ (5s, 1, node.stats.count (nano::stat::type::confirmation_observer, nano::stat::detail::inactive_conf_height, nano::stat::dir::out));
+	ASSERT_TIMELY_EQ (5s, 1, node.stats.count (nano::stat::type::confirmation_observer, nano::stat::detail::active_quorum, nano::stat::dir::out));
+	ASSERT_ALWAYS_EQ (50ms, 0, node.stats.count (nano::stat::type::confirmation_observer, nano::stat::detail::active_conf_height, nano::stat::dir::out));
 
 	// The first block was not active so no activation takes place
 	ASSERT_FALSE (node.active.active (open->qualified_root ()) || node.block_confirmed_or_being_confirmed (open->hash ()));
@@ -1333,7 +1333,7 @@ TEST (active_transactions, vacancy)
 		ASSERT_EQ (nano::block_status::progress, node.process (send));
 		ASSERT_EQ (1, node.active.vacancy ());
 		ASSERT_EQ (0, node.active.size ());
-		node.scheduler.priority.activate (nano::dev::genesis_key.pub, node.store.tx_begin_read ());
+		node.scheduler.priority.activate (nano::dev::genesis_key.pub, node.ledger.tx_begin_read ());
 		ASSERT_TIMELY (1s, updated);
 		updated = false;
 		ASSERT_EQ (0, node.active.vacancy ());
@@ -1523,7 +1523,7 @@ TEST (active_transactions, allow_limited_overflow)
 	// Insert the first part of the blocks into normal election scheduler
 	for (auto const & block : blocks1)
 	{
-		node.scheduler.priority.activate (block->account (), node.store.tx_begin_read ());
+		node.scheduler.priority.activate (block->account (), node.ledger.tx_begin_read ());
 	}
 
 	// Ensure number of active elections reaches AEC limit and there is no overfill
@@ -1585,7 +1585,7 @@ TEST (active_transactions, allow_limited_overflow_adapt)
 	// Insert the first part of the blocks into normal election scheduler
 	for (auto const & block : blocks1)
 	{
-		node.scheduler.priority.activate (block->account (), node.store.tx_begin_read ());
+		node.scheduler.priority.activate (block->account (), node.ledger.tx_begin_read ());
 	}
 
 	// Ensure number of active elections reaches AEC limit and there is no overfill

nano/core_test/backlog.cpp

@@ -21,7 +21,7 @@ TEST (backlog, population)
 	nano::test::system system{};
 	auto & node = *system.add_node ();
 
-	node.backlog.activate_callback.add ([&] (nano::store::transaction const & transaction, nano::account const & account, nano::account_info const & account_info, nano::confirmation_height_info const & conf_info) {
+	node.backlog.activate_callback.add ([&] (nano::secure::transaction const & transaction, nano::account const & account, nano::account_info const & account_info, nano::confirmation_height_info const & conf_info) {
 		nano::lock_guard<nano::mutex> lock{ mutex };
 
 		activated.insert (account);

nano/core_test/block_store.cpp

@@ -1000,7 +1000,7 @@ TEST (mdb_block_store, sideband_height)
 	nano::stats stats;
 	nano::ledger ledger (store, stats, nano::dev::constants);
 	nano::block_builder builder;
-	auto transaction (store.tx_begin_write ());
+	auto transaction = ledger.tx_begin_write ();
 	store.initialize (transaction, ledger.cache, nano::dev::constants);
 	nano::work_pool pool{ nano::dev::network_params.network, std::numeric_limits<unsigned>::max () };
 	auto send = builder

nano/core_test/bootstrap.cpp

@@ -651,7 +651,7 @@ TEST (bootstrap_processor, push_diamond_pruning)
 	ASSERT_EQ (nano::block_status::progress, node1->process (receive));
 
 	{
-		auto transaction (node1->store.tx_begin_write ());
+		auto transaction = node1->ledger.tx_begin_write ();
 		node1->ledger.confirm (transaction, open->hash ());
 		ASSERT_EQ (1, node1->ledger.pruning_action (transaction, send1->hash (), 2));
 		ASSERT_EQ (1, node1->ledger.pruning_action (transaction, open->hash (), 1));

nano/core_test/bootstrap_ascending.cpp

@@ -252,7 +252,7 @@ TEST (bootstrap_ascending, trace_base)
 	//	std::cerr << "--------------- Start ---------------\n";
 	ASSERT_EQ (nano::block_status::progress, node0.process (send1));
 	ASSERT_EQ (nano::block_status::progress, node0.process (receive1));
-	ASSERT_EQ (node1.ledger.receivable_end (), node1.ledger.receivable_upper_bound (node1.store.tx_begin_read (), key.pub, 0));
+	ASSERT_EQ (node1.ledger.receivable_end (), node1.ledger.receivable_upper_bound (node1.ledger.tx_begin_read (), key.pub, 0));
 	//	std::cerr << "node0: " << node0.network.endpoint () << std::endl;
 	//	std::cerr << "node1: " << node1.network.endpoint () << std::endl;
 	ASSERT_TIMELY (10s, node1.block (receive1->hash ()) != nullptr);

nano/core_test/confirming_set.cpp

@@ -94,7 +94,7 @@ TEST (confirmation_callback, observer_callbacks)
 				 .build ();
 
 	{
-		auto transaction = node->store.tx_begin_write ();
+		auto transaction = node->ledger.tx_begin_write ();
 		ASSERT_EQ (nano::block_status::progress, node->ledger.process (transaction, send));
 		ASSERT_EQ (nano::block_status::progress, node->ledger.process (transaction, send1));
 	}
@@ -133,7 +133,7 @@ TEST (confirmation_callback, confirmed_history)
 				.work (*system.work.generate (latest))
 				.build ();
 	{
-		auto transaction = node->store.tx_begin_write ();
+		auto transaction = node->ledger.tx_begin_write ();
 		ASSERT_EQ (nano::block_status::progress, node->ledger.process (transaction, send));
 	}
 
@@ -159,18 +159,18 @@ TEST (confirmation_callback, confirmed_history)
 		ASSERT_EQ (0, node->active.recently_cemented.list ().size ());
 		ASSERT_TRUE (node->active.empty ());
 
-		auto transaction = node->store.tx_begin_read ();
+		auto transaction = node->ledger.tx_begin_read ();
 		ASSERT_FALSE (node->ledger.block_confirmed (transaction, send->hash ()));
 
 		ASSERT_TIMELY (10s, node->store.write_queue.contains (nano::store::writer::confirmation_height));
 
 		// Confirm that no inactive callbacks have been called when the confirmation height processor has already iterated over it, waiting to write
-		ASSERT_EQ (0, node->stats.count (nano::stat::type::confirmation_observer, nano::stat::detail::inactive_conf_height, nano::stat::dir::out));
+		ASSERT_ALWAYS_EQ (50ms, 0, node->stats.count (nano::stat::type::confirmation_observer, nano::stat::detail::inactive_conf_height, nano::stat::dir::out));
 	}
 
 	ASSERT_TIMELY (10s, !node->store.write_queue.contains (nano::store::writer::confirmation_height));
 
-	auto transaction = node->store.tx_begin_read ();
+	auto transaction = node->ledger.tx_begin_read ();
 	ASSERT_TRUE (node->ledger.block_confirmed (transaction, send->hash ()));
 
 	ASSERT_TIMELY_EQ (10s, node->active.size (), 0);
@@ -181,9 +181,9 @@ TEST (confirmation_callback, confirmed_history)
 	ASSERT_TRUE (node->active.empty ());
 
 	// Confirm the callback is not called under this circumstance
-	ASSERT_EQ (1, node->stats.count (nano::stat::type::confirmation_observer, nano::stat::detail::active_quorum, nano::stat::dir::out));
-	ASSERT_EQ (1, node->stats.count (nano::stat::type::confirmation_observer, nano::stat::detail::inactive_conf_height, nano::stat::dir::out));
-	ASSERT_EQ (2, node->stats.count (nano::stat::type::confirmation_height, nano::stat::detail::blocks_confirmed, nano::stat::dir::in));
+	ASSERT_TIMELY_EQ (5s, 1, node->stats.count (nano::stat::type::confirmation_observer, nano::stat::detail::active_quorum, nano::stat::dir::out));
+	ASSERT_TIMELY_EQ (5s, 1, node->stats.count (nano::stat::type::confirmation_observer, nano::stat::detail::inactive_conf_height, nano::stat::dir::out));
+	ASSERT_TIMELY_EQ (5s, 2, node->stats.count (nano::stat::type::confirmation_height, nano::stat::detail::blocks_confirmed, nano::stat::dir::in));
 	ASSERT_EQ (3, node->ledger.cemented_count ());
 	ASSERT_EQ (0, node->active.election_winner_details_size ());
 }
@@ -226,7 +226,7 @@ TEST (confirmation_callback, dependent_election)
 				 .work (*system.work.generate (send1->hash ()))
 				 .build ();
 	{
-		auto transaction = node->store.tx_begin_write ();
+		auto transaction = node->ledger.tx_begin_write ();
 		ASSERT_EQ (nano::block_status::progress, node->ledger.process (transaction, send));
 		ASSERT_EQ (nano::block_status::progress, node->ledger.process (transaction, send1));
 		ASSERT_EQ (nano::block_status::progress, node->ledger.process (transaction, send2));
@@ -244,9 +244,9 @@ TEST (confirmation_callback, dependent_election)
 	// Once the item added to the confirming set no longer exists, callbacks have completed
 	ASSERT_TIMELY (5s, !node->confirming_set.exists (send2->hash ()));
 
-	ASSERT_EQ (1, node->stats.count (nano::stat::type::confirmation_observer, nano::stat::detail::active_quorum, nano::stat::dir::out));
-	ASSERT_EQ (1, node->stats.count (nano::stat::type::confirmation_observer, nano::stat::detail::active_conf_height, nano::stat::dir::out));
-	ASSERT_EQ (1, node->stats.count (nano::stat::type::confirmation_observer, nano::stat::detail::inactive_conf_height, nano::stat::dir::out));
+	ASSERT_TIMELY_EQ (5s, 1, node->stats.count (nano::stat::type::confirmation_observer, nano::stat::detail::active_quorum, nano::stat::dir::out));
+	ASSERT_TIMELY_EQ (5s, 1, node->stats.count (nano::stat::type::confirmation_observer, nano::stat::detail::active_conf_height, nano::stat::dir::out));
+	ASSERT_TIMELY_EQ (5s, 1, node->stats.count (nano::stat::type::confirmation_observer, nano::stat::detail::inactive_conf_height, nano::stat::dir::out));
 	ASSERT_EQ (4, node->ledger.cemented_count ());
 
 	ASSERT_EQ (0, node->active.election_winner_details_size ());

nano/core_test/conflicts.cpp

@@ -31,7 +31,7 @@ TEST (conflicts, start_stop)
 	node1.work_generate_blocking (*send1);
 	ASSERT_EQ (nano::block_status::progress, node1.process (send1));
 	ASSERT_EQ (0, node1.active.size ());
-	node1.scheduler.priority.activate (nano::dev::genesis_key.pub, node1.store.tx_begin_read ());
+	node1.scheduler.priority.activate (nano::dev::genesis_key.pub, node1.ledger.tx_begin_read ());
 	ASSERT_TIMELY (5s, node1.active.election (send1->qualified_root ()));
 	auto election1 = node1.active.election (send1->qualified_root ());
 	ASSERT_EQ (1, node1.active.size ());
@@ -64,7 +64,7 @@ TEST (conflicts, add_existing)
 	ASSERT_TIMELY (5s, node1.block (send1->hash ()));
 
 	// instruct the election scheduler to trigger an election for send1
-	node1.scheduler.priority.activate (nano::dev::genesis_key.pub, node1.store.tx_begin_read ());
+	node1.scheduler.priority.activate (nano::dev::genesis_key.pub, node1.ledger.tx_begin_read ());
 
 	// wait for election to be started before processing send2
 	ASSERT_TIMELY (5s, node1.active.active (*send1));