diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index 45aa3beb287..00000000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,502 +0,0 @@ ---- -version: 2.1 -orbs: - win: circleci/windows@5.0 - -executors: - besu_executor_med: # 2cpu, 4G ram - docker: - - image: cimg/openjdk:21.0 - resource_class: medium - working_directory: ~/project - environment: - architecture: "amd64" - GRADLE_OPTS: -Dorg.gradle.daemon=false -Dorg.gradle.parallel=true -Dorg.gradle.workers.max=2 - - besu_arm64_executor_med: # 2cpu, 8G ram - machine: #https://circleci.com/developer/machine/image/ubuntu-2204 - image: ubuntu-2204:2022.10.2 - resource_class: arm.medium - working_directory: ~/project - environment: - architecture: "arm64" - GRADLE_OPTS: -Dorg.gradle.daemon=false -Dorg.gradle.parallel=true -Dorg.gradle.workers.max=2 - - besu_executor_xl: # 8cpu, 16G ram - docker: - - image: cimg/openjdk:21.0 - resource_class: xlarge - working_directory: ~/project - environment: - GRADLE_OPTS: -Dorg.gradle.daemon=false -Dorg.gradle.parallel=true -Dorg.gradle.workers.max=4 - - xl_machine_executor: - machine: #https://circleci.com/developer/machine/image/ubuntu-2204 - image: ubuntu-2204:2022.10.2 - resource_class: xlarge - - trivy_executor: - docker: - - image: docker:stable-git - resource_class: small - working_directory: ~/project - -commands: - prepare: - description: "Prepare" - steps: - - checkout - - run: - name: Install Packages - LibSodium, nssdb - command: | - sudo apt-get update - sudo apt-get install -y libsodium23 libsodium-dev libjemalloc-dev apt-transport-https haveged libnss3-tools - sudo service haveged restart - java --version - - restore_gradle_cache - restore_gradle_cache: - description: "Restore Gradle cache" - steps: - - restore_cache: - name: Restore cached gradle dependencies - keys: - - deps-{{ checksum "gradle/versions.gradle" }}-{{ .Branch }}-{{ .Revision }} - - deps-{{ checksum "gradle/versions.gradle" }} - - deps- - - capture_test_results: - description: "Capture test results" - steps: - - run: - name: Jacoco - command: | - ./gradlew --no-daemon jacocoTestReport - - run: - name: Gather test results - when: always - command: | - FILES=`find . -name test-results` - for FILE in $FILES - do - MODULE=`echo "$FILE" | sed -e 's@./\(.*\)/build/test-results@\1@'` - TARGET="build/test-results/$MODULE" - mkdir -p "$TARGET" - cp -rf ${FILE}/*/* "$TARGET" - done - - store_test_results: - path: build/test-results - - store_artifacts: - path: besu/build/reports/jacoco - - capture_test_logs: - description: "Capture test logs" - steps: - - store_artifacts: - path: acceptance-tests/tests/build/acceptanceTestLogs - destination: acceptance-tests-logs - - store_artifacts: - path: acceptance-tests/tests/build/jvmErrorLogs - -jobs: - assemble: - executor: besu_executor_xl - steps: - - prepare - - run: - name: Assemble - command: | - ./gradlew --no-daemon clean compileJava compileTestJava assemble - - save_cache: - name: Caching gradle dependencies - key: deps-{{ checksum "gradle/versions.gradle" }}-{{ .Branch }}-{{ .Revision }} - paths: - - .gradle - - ~/.gradle - - persist_to_workspace: - root: ~/project - paths: - - ./ - - store_artifacts: - name: Distribution artifacts - path: build/distributions - destination: distributions - when: always - - testWindows: - executor: win/default - steps: - - attach_workspace: - at: ~/project - - run: - name: Unzip Windows build - no_output_timeout: 20m - command: | - cd build/distributions - unzip besu-*.zip -d besu-tmp - cd besu-tmp - mv besu-* ../besu - - run: - name: Test Besu Windows executable - no_output_timeout: 10m - command: | - build\distributions\besu\bin\besu.bat --help - build\distributions\besu\bin\besu.bat --version - - dockerScan: - executor: trivy_executor - steps: - - checkout - - restore_gradle_cache - - setup_remote_docker: - docker_layer_caching: true - - run: - name: Install trivy - command: | - apk add --update-cache --upgrade curl bash - curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sh -s -- -b /usr/local/bin - - run: - name: Scan with trivy - shell: /bin/sh - command: | - for FILE in $(ls docker) - do - if [[ $FILE == "test.sh" || $FILE == "tests" ]]; then - continue - fi - docker pull -q "hyperledger/besu:develop-$FILE" - trivy -q image --exit-code 1 --no-progress --severity HIGH,CRITICAL "hyperledger/besu:develop-$FILE" - done - - unitTests: - executor: besu_executor_xl - steps: - - prepare - - attach_workspace: - at: ~/project - - run: - name: Build - no_output_timeout: 20m - command: | - ./gradlew --no-daemon build - - capture_test_results - - integrationTests: - executor: xl_machine_executor # needs the machine executor since privacy test uses container tests (docker) - steps: - - prepare - - attach_workspace: - at: ~/project - - run: - name: IntegrationTests - command: | - ./gradlew --no-daemon integrationTest - - run: - name: CompileJmh - command: | - ./gradlew --no-daemon compileJmh - - capture_test_results - - referenceTests: - executor: besu_executor_xl - steps: - - prepare - - attach_workspace: - at: ~/project - - run: - name: ReferenceTests - no_output_timeout: 40m - command: | - git submodule update --init --recursive - ./gradlew --no-daemon referenceTest - - capture_test_results - - acceptanceTests: - parallelism: 4 - executor: xl_machine_executor # needs the machine executor since privacy test uses container tests (docker) - steps: - - prepare - - attach_workspace: - at: ~/project - - run: - name: AcceptanceTests (Mainnet) - no_output_timeout: 20m - command: | - CLASSNAMES=$(circleci tests glob "acceptance-tests/tests/src/test/java/**/*.java" \ - | sed 's@.*/src/test/java/@@' \ - | sed 's@/@.@g' \ - | sed 's/.\{5\}$//' \ - | circleci tests split --split-by=timings --timings-type=classname) - # Format the arguments to "./gradlew test" - GRADLE_ARGS=$(echo $CLASSNAMES | awk '{for (i=1; i<=NF; i++) print "--tests",$i}') - ./gradlew --no-daemon acceptanceTestMainnet $GRADLE_ARGS - - capture_test_results - - capture_test_logs - - acceptanceTestsCliqueBft: - executor: besu_executor_med - steps: - - prepare - - attach_workspace: - at: ~/project - - run: - name: AcceptanceTests (Non-Mainnet) - no_output_timeout: 30m - command: | - ./gradlew --no-daemon --max-workers=1 acceptanceTestCliqueBft - - capture_test_results - - capture_test_logs - - acceptanceTestsPrivacy: - parallelism: 4 - executor: xl_machine_executor # needs the machine executor since it uses container tests (docker) - steps: - - prepare - - attach_workspace: - at: ~/project - - run: - name: AcceptanceTests (Non-Mainnet) - no_output_timeout: 20m - command: | - CLASSNAMES=$(circleci tests glob "acceptance-tests/tests/src/test/java/**/*.java" \ - | sed 's@.*/src/test/java/@@' \ - | sed 's@/@.@g' \ - | sed 's/.\{5\}$//' \ - | circleci tests split --split-by=timings --timings-type=classname) - # Format the arguments to "./gradlew test" - GRADLE_ARGS=$(echo $CLASSNAMES | awk '{for (i=1; i<=NF; i++) print "--tests",$i}') - ./gradlew --no-daemon acceptanceTestPrivacy $GRADLE_ARGS - - capture_test_results - - capture_test_logs - - acceptanceTestsPermissioning: - executor: besu_executor_med - steps: - - prepare - - attach_workspace: - at: ~/project - - run: - name: AcceptanceTests (Non-Mainnet) - no_output_timeout: 20m - command: | - ./gradlew --no-daemon --max-workers=1 acceptanceTestPermissioning - - capture_test_results - - capture_test_logs - - buildDocker: - executor: besu_executor_med - steps: - - prepare - - attach_workspace: - at: ~/project - - setup_remote_docker - - run: - name: hadoLint_openjdk_17 - command: | - docker run --rm -i hadolint/hadolint < docker/openjdk-17/Dockerfile - - run: - name: hadoLint_openjdk_17_debug - command: | - docker run --rm -i hadolint/hadolint < docker/openjdk-17-debug/Dockerfile - - run: - name: hadoLint_openjdk_latest - command: | - docker run --rm -i hadolint/hadolint < docker/openjdk-latest/Dockerfile - - run: - name: hadoLint_graalvm - command: | - docker run --rm -i hadolint/hadolint < docker/graalvm/Dockerfile - - run: - name: build image - command: | - ./gradlew --no-daemon distDocker - - run: - name: test image - command: | - mkdir -p docker/reports - curl -L https://github.com/aelsabbahy/goss/releases/download/v0.3.9/goss-linux-amd64 -o ./docker/tests/goss-linux-amd64 - ./gradlew --no-daemon testDocker - - buildArm64Docker: - executor: besu_arm64_executor_med - steps: - - prepare - - attach_workspace: - at: ~/project - - run: - name: hadoLint_openjdk_17 - command: | - docker run --rm -i hadolint/hadolint < docker/openjdk-17/Dockerfile - - run: - name: hadoLint_openjdk_latest - command: | - docker run --rm -i hadolint/hadolint < docker/openjdk-latest/Dockerfile - - run: - name: hadoLint_graalvm - command: | - docker run --rm -i hadolint/hadolint < docker/graalvm/Dockerfile - - run: - name: Java_17 - command: | - sudo apt install -q --assume-yes openjdk-17-jre-headless openjdk-17-jdk-headless - sudo update-java-alternatives -a - - run: - name: build image - command: | - ./gradlew --no-daemon distDocker - - run: - name: test image - command: | - mkdir -p docker/reports - curl -L https://github.com/aelsabbahy/goss/releases/download/v0.3.9/goss-linux-arm -o ./docker/tests/goss-linux-arm64 - ./gradlew --no-daemon testDocker - - publish: - executor: besu_executor_med - steps: - - prepare - - attach_workspace: - at: ~/project - - run: - name: Publish - command: | - ./gradlew --no-daemon artifactoryPublish - - publishDocker: - executor: besu_executor_med - steps: - - prepare - - attach_workspace: - at: ~/project - - setup_remote_docker - - run: - name: Publish Docker - command: | - docker login --username "${DOCKER_USER_RW}" --password "${DOCKER_PASSWORD_RW}" - ./gradlew --no-daemon "-Pbranch=${CIRCLE_BRANCH}" dockerUpload - - publishArm64Docker: - executor: besu_arm64_executor_med - steps: - - prepare - - attach_workspace: - at: ~/project - - run: - name: Java_17 - command: | - sudo apt install -q --assume-yes openjdk-17-jre-headless openjdk-17-jdk-headless - sudo update-java-alternatives -a - - run: - name: Publish Docker - command: | - docker login --username "${DOCKER_USER_RW}" --password "${DOCKER_PASSWORD_RW}" - ./gradlew --no-daemon "-Pbranch=${CIRCLE_BRANCH}" dockerUpload - manifestDocker: - executor: besu_executor_med - steps: - - prepare - - setup_remote_docker - - run: - name: Create and publish docker manifest - command: | - docker login --username "${DOCKER_USER_RW}" --password "${DOCKER_PASSWORD_RW}" - ./gradlew --no-daemon "-Pbranch=${CIRCLE_BRANCH}" --parallel manifestDocker - -workflows: - version: 2 - default: - jobs: - - assemble - - unitTests: - requires: - - assemble - - testWindows: - requires: - - assemble - - referenceTests: - requires: - - assemble - - integrationTests: - requires: - - assemble - - acceptanceTests: - requires: - - assemble - - acceptanceTestsCliqueBft: - requires: - - assemble - - acceptanceTests - - acceptanceTestsPermissioning: - requires: - - assemble - - buildDocker: - requires: - - assemble - - buildArm64Docker: - requires: - - buildDocker - - assemble - - publish: - filters: - branches: - only: - - main - - /^release-.*/ - requires: - - assemble - - integrationTests - - unitTests - - acceptanceTests - - referenceTests - - buildDocker - - publishDocker: - filters: - branches: - only: - - main - - /^release-.*/ - requires: - - assemble - - integrationTests - - unitTests - - acceptanceTests - - referenceTests - - buildDocker - context: - - besu-dockerhub-rw - - publishArm64Docker: - filters: - branches: - only: - - main - - /^release-.*/ - requires: - - integrationTests - - unitTests - - acceptanceTests - - referenceTests - - buildArm64Docker - context: - - besu-dockerhub-rw - - manifestDocker: - filters: - branches: - only: - - main - - /^release-.*/ - requires: - - publishDocker - - publishArm64Docker - context: - - besu-dockerhub-rw - - nightly: - triggers: - - schedule: - cron: "0 19 * * *" - filters: - branches: - only: - - main - jobs: - - assemble - - dockerScan diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ba6b4dc5dd..4d29b8f8af8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,14 @@ # Changelog -## Next Release +## Next release + +### Breaking Changes + +### Additions and Improvements + +### Bug fixes + +## 24.7.0 ### Upcoming Breaking Changes - Receipt compaction will be enabled by default in a future version of Besu. After this change it will not be possible to downgrade to the previous Besu version. @@ -24,6 +32,8 @@ - Refactored how code, initcode, and max stack size are configured in forks. [#7245](https://github.com/hyperledger/besu/pull/7245) - Nodes in a permissioned chain maintain (and retry) connections to bootnodes [#7257](https://github.com/hyperledger/besu/pull/7257) - Promote experimental `besu storage x-trie-log` subcommand to production-ready [#7278](https://github.com/hyperledger/besu/pull/7278) +- Enhanced BFT round-change diagnostics [#7271](https://github.com/hyperledger/besu/pull/7271) +- `--Xsnapsync-bft-enabled` option enables experimental support for snap sync with IBFT/QBFT permissioned Bonsai-DB chains [#7140](https://github.com/hyperledger/besu/pull/7140) ### Bug fixes - Validation errors ignored in accounts-allowlist and empty list [#7138](https://github.com/hyperledger/besu/issues/7138) @@ -31,6 +41,8 @@ - Fix "Could not confirm best peer had pivot block" [#7109](https://github.com/hyperledger/besu/issues/7109) - Fix "Chain Download Halt" [#6884](https://github.com/hyperledger/besu/issues/6884) + + ## 24.6.0 ### Breaking Changes diff --git a/besu/src/main/java/org/hyperledger/besu/RunnerBuilder.java b/besu/src/main/java/org/hyperledger/besu/RunnerBuilder.java index e9f9ef6268c..7ed627cfc17 100644 --- a/besu/src/main/java/org/hyperledger/besu/RunnerBuilder.java +++ b/besu/src/main/java/org/hyperledger/besu/RunnerBuilder.java @@ -688,13 +688,14 @@ public Runner build() { .map(nodePerms -> PeerPermissions.combine(nodePerms, defaultPeerPermissions)) .orElse(defaultPeerPermissions); + final EthPeers ethPeers = besuController.getEthPeers(); + LOG.info("Detecting NAT service."); final boolean fallbackEnabled = natMethod == NatMethod.AUTO || natMethodFallbackEnabled; final NatService natService = new NatService(buildNatManager(natMethod), fallbackEnabled); final NetworkBuilder inactiveNetwork = caps -> new NoopP2PNetwork(); final NetworkBuilder activeNetwork = caps -> { - final EthPeers ethPeers = besuController.getEthPeers(); return DefaultP2PNetwork.builder() .vertx(vertx) .nodeKey(nodeKey) @@ -709,8 +710,8 @@ public Runner build() { .blockchain(context.getBlockchain()) .blockNumberForks(besuController.getGenesisConfigOptions().getForkBlockNumbers()) .timestampForks(besuController.getGenesisConfigOptions().getForkBlockTimestamps()) - .allConnectionsSupplier(ethPeers::getAllConnections) - .allActiveConnectionsSupplier(ethPeers::getAllActiveConnections) + .allConnectionsSupplier(ethPeers::streamAllConnections) + .allActiveConnectionsSupplier(ethPeers::streamAllActiveConnections) .maxPeers(ethPeers.getMaxPeers()) .build(); }; @@ -721,9 +722,10 @@ public Runner build() { .subProtocols(subProtocols) .network(p2pEnabled ? activeNetwork : inactiveNetwork) .metricsSystem(metricsSystem) + .ethPeersShouldConnect(ethPeers::shouldTryToConnect) .build(); - besuController.getEthPeers().setRlpxAgent(networkRunner.getRlpxAgent()); + ethPeers.setRlpxAgent(networkRunner.getRlpxAgent()); final P2PNetwork network = networkRunner.getNetwork(); // ForkId in Ethereum Node Record needs updating when we transition to a new protocol spec diff --git a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java index 7e1be15fb65..12752f2e9a1 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java @@ -1552,10 +1552,11 @@ private void validateOptions() { } private void validateConsensusSyncCompatibilityOptions() { - // snap and checkpoint can't be used with BFT but can for clique - if (genesisConfigOptionsSupplier.get().isIbftLegacy() - || genesisConfigOptionsSupplier.get().isIbft2() - || genesisConfigOptionsSupplier.get().isQbft()) { + // snap and checkpoint are experimental for BFT + if ((genesisConfigOptionsSupplier.get().isIbftLegacy() + || genesisConfigOptionsSupplier.get().isIbft2() + || genesisConfigOptionsSupplier.get().isQbft()) + && !unstableSynchronizerOptions.isSnapSyncBftEnabled()) { final String errorSuffix = "can't be used with BFT networks"; if (SyncMode.CHECKPOINT.equals(syncMode) || SyncMode.X_CHECKPOINT.equals(syncMode)) { throw new ParameterException( @@ -2181,7 +2182,7 @@ private SynchronizerConfiguration buildSyncConfig() { return unstableSynchronizerOptions .toDomainObject() .syncMode(syncMode) - .fastSyncMinimumPeerCount(syncMinPeerCount) + .syncMinimumPeerCount(syncMinPeerCount) .build(); } @@ -2194,14 +2195,14 @@ private TransactionPoolConfiguration buildTransactionPoolConfiguration() { .saveFile((dataPath.resolve(txPoolConf.getSaveFile().getPath()).toFile())); if (genesisConfigOptionsSupplier.get().isZeroBaseFee()) { - logger.info( + logger.warn( "Forcing price bump for transaction replacement to 0, since we are on a zero basefee network"); txPoolConfBuilder.priceBump(Percentage.ZERO); } if (miningParametersSupplier.get().getMinTransactionGasPrice().equals(Wei.ZERO) && !transactionPoolOptions.isPriceBumpSet(commandLine)) { - logger.info( + logger.warn( "Forcing price bump for transaction replacement to 0, since min-gas-price is set to 0"); txPoolConfBuilder.priceBump(Percentage.ZERO); } @@ -2810,6 +2811,7 @@ && getDataStorageConfiguration().getBonsaiLimitTrieLogsEnabled()) { } builder.setSnapServerEnabled(this.unstableSynchronizerOptions.isSnapsyncServerEnabled()); + builder.setSnapSyncBftEnabled(this.unstableSynchronizerOptions.isSnapSyncBftEnabled()); builder.setTxPoolImplementation(buildTransactionPoolConfiguration().getTxPoolImplementation()); builder.setWorldStateUpdateMode(unstableEvmOptions.toDomainObject().worldUpdaterMode()); diff --git a/besu/src/main/java/org/hyperledger/besu/cli/ConfigurationOverviewBuilder.java b/besu/src/main/java/org/hyperledger/besu/cli/ConfigurationOverviewBuilder.java index 2d74ae8d26f..b2f89a349e2 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/ConfigurationOverviewBuilder.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/ConfigurationOverviewBuilder.java @@ -56,6 +56,7 @@ public class ConfigurationOverviewBuilder { private long trieLogRetentionLimit = 0; private Integer trieLogsPruningWindowSize = null; private boolean isSnapServerEnabled = false; + private boolean isSnapSyncBftEnabled = false; private TransactionPoolConfiguration.Implementation txPoolImplementation; private EvmConfiguration.WorldUpdaterMode worldStateUpdateMode; private Map environment; @@ -233,6 +234,17 @@ public ConfigurationOverviewBuilder setSnapServerEnabled(final boolean snapServe return this; } + /** + * Sets snap sync BFT enabled/disabled + * + * @param snapSyncBftEnabled bool to indicate if snap sync for BFT is enabled + * @return the builder + */ + public ConfigurationOverviewBuilder setSnapSyncBftEnabled(final boolean snapSyncBftEnabled) { + isSnapSyncBftEnabled = snapSyncBftEnabled; + return this; + } + /** * Sets trie logs pruning window size * @@ -357,6 +369,10 @@ public String build() { lines.add("Experimental Snap Sync server enabled"); } + if (isSnapSyncBftEnabled) { + lines.add("Experimental Snap Sync for BFT enabled"); + } + if (isBonsaiLimitTrieLogsEnabled) { final StringBuilder trieLogPruningString = new StringBuilder(); trieLogPruningString diff --git a/besu/src/main/java/org/hyperledger/besu/cli/options/unstable/SynchronizerOptions.java b/besu/src/main/java/org/hyperledger/besu/cli/options/unstable/SynchronizerOptions.java index e52839cceb4..95bbe0a2b1f 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/options/unstable/SynchronizerOptions.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/options/unstable/SynchronizerOptions.java @@ -85,6 +85,8 @@ public class SynchronizerOptions implements CLIOptions