Skip to content

Commit

Permalink
Merge branch 'master' into standardise-json-rpc-error-codes
Browse files Browse the repository at this point in the history
  • Loading branch information
@tbenr
tbenr authored Oct 17, 2024
2 parents ad9ece0 + da119c6 commit 7eb382a
Show file tree
Hide file tree
Showing 4 changed files with 150 additions and 20 deletions.
95 changes: 93 additions & 2 deletions build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@ import tech.pegasys.teku.depcheck.DepCheckPlugin

import java.text.SimpleDateFormat

import groovy.json.JsonSlurper

import static tech.pegasys.teku.repackage.Repackage.repackage

buildscript {
Expand Down Expand Up @@ -318,7 +320,8 @@ allprojects {
}
}

def refTestVersion = 'v1.5.0-alpha.8'
def nightly = System.getenv("NIGHTLY") != null
def refTestVersion = nightly ? "nightly" : "v1.5.0-alpha.8"
def blsRefTestVersion = 'v0.1.2'
def slashingProtectionInterchangeRefTestVersion = 'v5.3.0'
def refTestBaseUrl = 'https://github.com/ethereum/consensus-spec-tests/releases/download'
Expand All @@ -329,7 +332,87 @@ def blsRefTestDownloadDir = "${buildDir}/blsRefTests/${blsRefTestVersion}"
def slashingProtectionInterchangeRefTestDownloadDir = "${buildDir}/slashingProtectionInterchangeRefTests/${slashingProtectionInterchangeRefTestVersion}"
def refTestExpandDir = "${project.rootDir}/eth-reference-tests/src/referenceTest/resources/consensus-spec-tests/"

task downloadEthRefTests(type: Download) {
def downloadFile(String url, String token, File outputFile) {
println "Download ${outputFile.getName()} (${url})"
def connection = new URL(url).openConnection()
connection.setRequestProperty("Authorization", "token ${token}")
connection.getInputStream().withCloseable { inputStream ->
outputFile.withOutputStream { outputStream ->
outputStream << inputStream
}
}
}

def downloadArtifacts(String repo, Long runId, String token, String downloadDir) {
def artifactsApiUrl = "https://api.github.com/repos/${repo}/actions/runs/${runId}/artifacts"
def connection = new URL(artifactsApiUrl).openConnection()
connection.setRequestProperty("Authorization", "token ${token}")
connection.setRequestProperty("Accept", "application/vnd.github.v3+json")

def response = new JsonSlurper().parse(connection.getInputStream())
if (response.artifacts && response.artifacts.size() > 0) {
response.artifacts.each { artifact ->
// We can skip the log file
if (artifact.name.contains("consensustestgen.log")) {
return
}

def fileOutput = new File(downloadDir, "${artifact.name}.zip")
downloadFile(artifact.archive_download_url, token, fileOutput)
ant.unzip(src: fileOutput, dest: downloadDir)
fileOutput.delete()
}
return true
}
return false
}

static def getLatestRunId(String repo, String workflow, String branch, String token) {
def apiUrl = "https://api.github.com/repos/${repo}/actions/workflows/${workflow}/runs?branch=${branch}&status=success&per_page=1"
def connection = new URL(apiUrl).openConnection()
connection.setRequestProperty("Authorization", "token ${token}")
connection.setRequestProperty("Accept", "application/vnd.github.v3+json")

// Query & parse the ID out of the response
def response = new JsonSlurper().parse(connection.getInputStream())
if (response.workflow_runs && response.workflow_runs.size() > 0) {
return response.workflow_runs[0].id
}
return null
}

task downloadEthRefTestsNightly {
doLast {
def repo = "ethereum/consensus-specs"
def workflowFileName = "generate_vectors.yml"
def branch = "dev"

// We need a GitHub API token to download the artifacts
def githubToken = System.getenv("GITHUB_TOKEN")
if (!githubToken) {
println "Error: GITHUB_TOKEN environment variable is not set"
return
}

// Get the latest workflow run ID
def runId = getLatestRunId(repo, workflowFileName, branch, githubToken)
if (!runId) {
println "Error: Failed to get latest run ID"
return
}

// Create the download directory
file(refTestDownloadDir).mkdirs()

// Download artifacts for the run
def success = downloadArtifacts(repo, runId, githubToken, refTestDownloadDir)
if (!success) {
println "Error: Failed to download artifacts"
}
}
}

task downloadEthRefTestsStable(type: Download) {
src([
"${refTestBaseUrl}/${refTestVersion}/general.tar.gz",
"${refTestBaseUrl}/${refTestVersion}/minimal.tar.gz",
Expand All @@ -339,6 +422,14 @@ task downloadEthRefTests(type: Download) {
overwrite false
}

task downloadEthRefTests {
if (nightly) {
dependsOn tasks.findByName("downloadEthRefTestsNightly")
} else {
dependsOn tasks.findByName("downloadEthRefTestsStable")
}
}

task downloadBlsRefTests(type: Download) {
src([
"${blsRefTestBaseUrl}/${blsRefTestVersion}/bls_tests_yaml.tar.gz"
Expand Down
13 changes: 12 additions & 1 deletion ...es/chainstorage/src/main/java/tech/pegasys/teku/services/chainstorage/StorageService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@

import com.google.common.annotations.VisibleForTesting;
import java.nio.file.Path;
import java.time.Duration;
import java.util.Optional;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
Expand Down Expand Up @@ -53,6 +54,7 @@
import tech.pegasys.teku.storage.server.pruner.StatePruner;

public class StorageService extends Service implements StorageServiceFacade {
public static final Duration STATE_PRUNING_INTERVAL = Duration.ofMinutes(1);
private final StorageConfiguration config;
private volatile ChainStorage chainStorage;
private final ServiceConfig serviceConfig;
Expand Down Expand Up @@ -134,12 +136,20 @@ protected SafeFuture<?> doStart() {
configureStatePruner(
config.getRetainedSlots(),
storagePrunerAsyncRunner,
config.getStatePruningInterval(),
pruningTimingsLabelledGauge,
pruningActiveLabelledGauge);
} else if (!config.getDataStorageMode().storesFinalizedStates()) {
final Duration statePruningInterval =
config
.getStatePruningInterval()
.equals(StorageConfiguration.DEFAULT_STATE_PRUNING_INTERVAL)
? STATE_PRUNING_INTERVAL
: config.getStatePruningInterval();
configureStatePruner(
StorageConfiguration.DEFAULT_STORAGE_RETAINED_SLOTS,
storagePrunerAsyncRunner,
statePruningInterval,
pruningTimingsLabelledGauge,
pruningActiveLabelledGauge);
}
Expand Down Expand Up @@ -221,6 +231,7 @@ protected SafeFuture<?> doStart() {
void configureStatePruner(
final long slotsToRetain,
final AsyncRunner storagePrunerAsyncRunner,
final Duration pruningInterval,
final SettableLabelledGauge pruningTimingsLabelledGauge,
final SettableLabelledGauge pruningActiveLabelledGauge) {
if (config.getDataStorageCreateDbVersion() == DatabaseVersion.LEVELDB_TREE) {
Expand All @@ -240,7 +251,7 @@ void configureStatePruner(
config.getSpec(),
database,
storagePrunerAsyncRunner,
config.getStatePruningInterval(),
pruningInterval,
slotsToRetain,
config.getStatePruningLimit(),
"state",
Expand Down
56 changes: 39 additions & 17 deletions ...hainstorage/src/test/java/tech/pegasys/teku/services/chainstorage/StorageServiceTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,11 +20,14 @@
import static org.mockito.Mockito.when;

import java.nio.file.Path;
import java.time.Duration;
import java.util.Optional;
import org.hyperledger.besu.plugin.services.MetricsSystem;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.EnumSource;
import tech.pegasys.teku.ethereum.execution.types.Eth1Address;
import tech.pegasys.teku.infrastructure.async.SafeFuture;
import tech.pegasys.teku.infrastructure.async.StubAsyncRunner;
Expand Down Expand Up @@ -59,6 +62,10 @@ void setUp(@TempDir final Path tempDir) {
.thenReturn(StorageConfiguration.DEFAULT_MAX_KNOWN_NODE_CACHE_SIZE);
when(storageConfiguration.getDataStorageFrequency())
.thenReturn(StorageConfiguration.DEFAULT_STORAGE_FREQUENCY);
when(storageConfiguration.getStatePruningLimit())
.thenReturn(StorageConfiguration.DEFAULT_STATE_PRUNING_LIMIT);
when(storageConfiguration.getStatePruningInterval())
.thenReturn(StorageConfiguration.DEFAULT_STATE_PRUNING_INTERVAL);
when(storageConfiguration.getEth1DepositContract()).thenReturn(eth1DepositContract);
when(storageConfiguration.isStoreNonCanonicalBlocksEnabled()).thenReturn(false);
when(storageConfiguration.getSpec()).thenReturn(spec);
Expand All @@ -80,39 +87,54 @@ void setUp(@TempDir final Path tempDir) {
void shouldNotSetupStatePrunerWhenArchiveMode() {
when(storageConfiguration.getDataStorageMode()).thenReturn(StateStorageMode.ARCHIVE);
final SafeFuture<?> future = storageService.doStart();
final Optional<StatePruner> statePruner = storageService.getStatePruner();
final Optional<StatePruner> maybeStatePruner = storageService.getStatePruner();
assertThat(future).isCompleted();
assertThat(statePruner).isEmpty();
assertThat(maybeStatePruner).isEmpty();
}

@Test
void shouldSetupStatePrunerWhenArchiveModeAndRetentionSlotsEnabled() {
when(storageConfiguration.getDataStorageMode()).thenReturn(StateStorageMode.ARCHIVE);
when(storageConfiguration.getRetainedSlots()).thenReturn(5L);
final SafeFuture<?> future = storageService.doStart();
final Optional<StatePruner> statePruner = storageService.getStatePruner();
final Optional<StatePruner> maybeStatePruner = storageService.getStatePruner();
assertThat(future).isCompleted();
assertThat(statePruner).isPresent();
assertThat(storageService.getStatePruner().get().isRunning()).isTrue();
assertThat(maybeStatePruner).isPresent();
final StatePruner statePruner = maybeStatePruner.get();
assertThat(statePruner.isRunning()).isTrue();
assertThat(statePruner.getPruneInterval())
.isEqualTo(StorageConfiguration.DEFAULT_STATE_PRUNING_INTERVAL);
}

@Test
void shouldSetupStatePrunerWhenPruneMode() {
when(storageConfiguration.getDataStorageMode()).thenReturn(StateStorageMode.PRUNE);
@ParameterizedTest
@EnumSource(
value = StateStorageMode.class,
names = {"PRUNE", "MINIMAL"})
void shouldSetupStatePrunerWhenPruneMode(final StateStorageMode stateStorageMode) {
when(storageConfiguration.getDataStorageMode()).thenReturn(stateStorageMode);
final SafeFuture<?> future = storageService.doStart();
final Optional<StatePruner> statePruner = storageService.getStatePruner();
final Optional<StatePruner> maybeStatePruner = storageService.getStatePruner();
assertThat(future).isCompleted();
assertThat(statePruner).isPresent();
assertThat(storageService.getStatePruner().get().isRunning()).isTrue();
assertThat(maybeStatePruner).isPresent();
final StatePruner statePruner = maybeStatePruner.get();
assertThat(statePruner.isRunning()).isTrue();
assertThat(statePruner.getPruneInterval()).isEqualTo(StorageService.STATE_PRUNING_INTERVAL);
}

@Test
void shouldSetupStatePrunerWhenMinimalMode() {
when(storageConfiguration.getDataStorageMode()).thenReturn(StateStorageMode.MINIMAL);
@ParameterizedTest
@EnumSource(
value = StateStorageMode.class,
names = {"PRUNE", "MINIMAL"})
void shouldSetupStatePrunerWithCustomInterval(final StateStorageMode stateStorageMode) {
when(storageConfiguration.getDataStorageMode()).thenReturn(stateStorageMode);
final Duration customPruningInterval = Duration.ofSeconds(8);
when(storageConfiguration.getStatePruningInterval()).thenReturn(customPruningInterval);
final SafeFuture<?> future = storageService.doStart();
final Optional<StatePruner> statePruner = storageService.getStatePruner();
final Optional<StatePruner> maybeStatePruner = storageService.getStatePruner();
assertThat(future).isCompleted();
assertThat(statePruner).isPresent();
assertThat(storageService.getStatePruner().get().isRunning()).isTrue();
assertThat(maybeStatePruner).isPresent();
final StatePruner statePruner = maybeStatePruner.get();
assertThat(statePruner.isRunning()).isTrue();
assertThat(statePruner.getPruneInterval()).isEqualTo(customPruningInterval);
}
}
6 changes: 6 additions & 0 deletions storage/src/main/java/tech/pegasys/teku/storage/server/pruner/StatePruner.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@

package tech.pegasys.teku.storage.server.pruner;

import com.google.common.annotations.VisibleForTesting;
import java.time.Duration;
import java.util.Optional;
import java.util.concurrent.RejectedExecutionException;
Expand Down Expand Up @@ -114,4 +115,9 @@ private void pruneStates() {
LOG.debug("Shutting down", ex);
}
}

@VisibleForTesting
public Duration getPruneInterval() {
return pruneInterval;
}
}

0 comments on commit 7eb382a

Please sign in to comment.