From 7b2871ceda2e45204951ed7cf7fac19402c363f0 Mon Sep 17 00:00:00 2001 From: gconnect Date: Tue, 1 Oct 2024 22:50:38 +0100 Subject: [PATCH] Create custom Ephemery reset class and test Signed-off-by: gconnect --- .../chainstorage/EphemeryDatabaseReset.java | 63 +++++- .../services/chainstorage/StorageService.java | 43 +--- .../EphemeryDatabaseResetTest.java | 204 +++++++++--------- 3 files changed, 171 insertions(+), 139 deletions(-) diff --git a/services/chainstorage/src/main/java/tech/pegasys/teku/services/chainstorage/EphemeryDatabaseReset.java b/services/chainstorage/src/main/java/tech/pegasys/teku/services/chainstorage/EphemeryDatabaseReset.java index a37c8691e50..ea8fe45cc0b 100644 --- a/services/chainstorage/src/main/java/tech/pegasys/teku/services/chainstorage/EphemeryDatabaseReset.java +++ b/services/chainstorage/src/main/java/tech/pegasys/teku/services/chainstorage/EphemeryDatabaseReset.java @@ -1,2 +1,63 @@ -package tech.pegasys.teku.services.chainstorage;public class EphemeryDatabaseReset { +/* + * Copyright Consensys Software Inc., 2024 + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package tech.pegasys.teku.services.chainstorage; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import tech.pegasys.teku.infrastructure.exceptions.InvalidConfigurationException; +import tech.pegasys.teku.service.serviceutils.ServiceConfig; +import tech.pegasys.teku.storage.server.Database; +import tech.pegasys.teku.storage.server.VersionedDatabaseFactory; + +public class EphemeryDatabaseReset { + + /** This method is called only on Ephemery network when reset is due. */ + Database resetDatabaseAndCreate( + final ServiceConfig serviceConfig, final VersionedDatabaseFactory dbFactory) { + try { + final Path beaconDataDir = serviceConfig.getDataDirLayout().getBeaconDataDirectory(); + final Path validatorDataDir = serviceConfig.getDataDirLayout().getValidatorDataDirectory(); + final Path slashProtectionDir; + if (validatorDataDir.endsWith("slashprotection")) { + slashProtectionDir = validatorDataDir; + } else { + slashProtectionDir = validatorDataDir.resolve("slashprotection"); + } + deleteDirectoryRecursively(beaconDataDir); + deleteDirectoryRecursively(slashProtectionDir); + return dbFactory.createDatabase(); + } catch (final Exception ex) { + throw new InvalidConfigurationException( + "The existing ephemery database was old, and was unable to reset it.", ex); + } + } + + void deleteDirectoryRecursively(final Path path) throws IOException { + if (Files.isDirectory(path)) { + try (var stream = Files.walk(path)) { + stream + .sorted((o1, o2) -> o2.compareTo(o1)) + .forEach( + p -> { + try { + Files.delete(p); + } catch (IOException e) { + throw new RuntimeException("Failed to delete file: " + p, e); + } + }); + } + } + } } diff --git a/services/chainstorage/src/main/java/tech/pegasys/teku/services/chainstorage/StorageService.java b/services/chainstorage/src/main/java/tech/pegasys/teku/services/chainstorage/StorageService.java index 5a6bc95a3f7..12631b89626 100644 --- a/services/chainstorage/src/main/java/tech/pegasys/teku/services/chainstorage/StorageService.java +++ b/services/chainstorage/src/main/java/tech/pegasys/teku/services/chainstorage/StorageService.java @@ -16,10 +16,6 @@ import static tech.pegasys.teku.infrastructure.async.AsyncRunnerFactory.DEFAULT_MAX_QUEUE_SIZE; import static tech.pegasys.teku.spec.config.Constants.STORAGE_QUERY_CHANNEL_PARALLELISM; -import com.google.common.annotations.VisibleForTesting; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; import java.util.Optional; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -63,6 +59,7 @@ public class StorageService extends Service implements StorageServiceFacade { private final boolean depositSnapshotStorageEnabled; private final boolean blobSidecarsStorageCountersEnabled; private static final Logger LOG = LogManager.getLogger(); + private final EphemeryDatabaseReset ephemeryDatabaseReset; public StorageService( final ServiceConfig serviceConfig, @@ -73,6 +70,7 @@ public StorageService( this.config = storageConfiguration; this.depositSnapshotStorageEnabled = depositSnapshotStorageEnabled; this.blobSidecarsStorageCountersEnabled = blobSidecarsStorageCountersEnabled; + this.ephemeryDatabaseReset = new EphemeryDatabaseReset(); } @Override @@ -94,7 +92,7 @@ protected SafeFuture doStart() { try { database = dbFactory.createDatabase(); } catch (EphemeryException e) { - database = resetDatabaseAndCreate(serviceConfig, dbFactory); + database = ephemeryDatabaseReset.resetDatabaseAndCreate(serviceConfig, dbFactory); } final SettableLabelledGauge pruningTimingsLabelledGauge = SettableLabelledGauge.create( @@ -229,39 +227,4 @@ protected SafeFuture doStop() { public ChainStorage getChainStorage() { return chainStorage; } - - /** This method is called only on Ephemery network when reset is due. */ - @VisibleForTesting - Database resetDatabaseAndCreate( - final ServiceConfig serviceConfig, final VersionedDatabaseFactory dbFactory) { - try { - final Path beaconDataDir = serviceConfig.getDataDirLayout().getBeaconDataDirectory(); - final Path slashProtectionDir = - serviceConfig.getDataDirLayout().getValidatorDataDirectory().resolve("slashprotection"); - deleteDirectoryRecursively(beaconDataDir); - deleteDirectoryRecursively(slashProtectionDir); - - return dbFactory.createDatabase(); - } catch (final Exception ex) { - throw new InvalidConfigurationException( - "The existing ephemery database was old, and was unable to reset it.", ex); - } - } - - private void deleteDirectoryRecursively(final Path path) throws IOException { - if (Files.isDirectory(path)) { - try (var stream = Files.walk(path)) { - stream - .sorted((o1, o2) -> o2.compareTo(o1)) - .forEach( - p -> { - try { - Files.delete(p); - } catch (IOException e) { - throw new RuntimeException("Failed to delete file: " + p, e); - } - }); - } - } - } } diff --git a/services/chainstorage/src/test/java/tech/pegasys/teku/services/chainstorage/EphemeryDatabaseResetTest.java b/services/chainstorage/src/test/java/tech/pegasys/teku/services/chainstorage/EphemeryDatabaseResetTest.java index 3d27325ce1c..198761f0622 100644 --- a/services/chainstorage/src/test/java/tech/pegasys/teku/services/chainstorage/EphemeryDatabaseResetTest.java +++ b/services/chainstorage/src/test/java/tech/pegasys/teku/services/chainstorage/EphemeryDatabaseResetTest.java @@ -1,5 +1,30 @@ +/* + * Copyright Consensys Software Inc., 2024 + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + package tech.pegasys.teku.services.chainstorage; +import static java.nio.file.Files.createTempDirectory; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.io.IOException; +import java.nio.file.Path; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mockito.InjectMocks; @@ -8,104 +33,87 @@ import tech.pegasys.teku.infrastructure.exceptions.InvalidConfigurationException; import tech.pegasys.teku.service.serviceutils.ServiceConfig; import tech.pegasys.teku.service.serviceutils.layout.DataDirLayout; -import tech.pegasys.teku.storage.server.VersionedDatabaseFactory; import tech.pegasys.teku.storage.server.Database; +import tech.pegasys.teku.storage.server.VersionedDatabaseFactory; -import java.io.IOException; -import java.nio.file.Path; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.mockito.Mockito.doThrow; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -public class StorageServiceTest { - - - private ServiceConfig serviceConfig; - - @Mock - private VersionedDatabaseFactory dbFactory; - - @Mock - private Path beaconDataDir; - - @Mock - private Path slashProtectionDir; - - @Mock - private FileUtils fileUtils; - - @Mock - private EphemeryDatabaseReset ephemeryDatabaseReset; - - @Mock - private Database database; - -// private StorageService storageService; - - @BeforeEach - void setUp() { - MockitoAnnotations.openMocks(this); - serviceConfig = mock(ServiceConfig.class); - DataDirLayout dataDirLayout = mock(DataDirLayout.class); - when(serviceConfig.getDataDirLayout()).thenReturn(dataDirLayout); - when(serviceConfig.getDataDirLayout().getBeaconDataDirectory()).thenReturn(beaconDataDir); - when(serviceConfig.getDataDirLayout().getValidatorDataDirectory()).thenReturn(slashProtectionDir); - } - - @Test - void shouldResetDirectoriesAndCreateDatabase() throws IOException { - // Mock database creation - when(dbFactory.createDatabase()).thenReturn(database); - - // Call the method - Database result = ephemeryDatabaseReset.resetDatabaseAndCreate(serviceConfig, dbFactory); - - // Verify that directories were deleted - verify(fileUtils).deleteDirectoryRecursively(beaconDataDir); - verify(fileUtils).deleteDirectoryRecursively(slashProtectionDir.resolve("slashprotection")); - - // Verify that the database was created - verify(dbFactory).createDatabase(); - - // Assert that the result is the mock database - assertEquals(database, result); - } - - @Test - void shouldThrowInvalidConfigurationExceptionWhenDirectoryDeletionFails() throws IOException { - // Simulate an exception during directory deletion - doThrow(new IOException("Failed to delete directory")).when(fileUtils).deleteDirectoryRecursively(beaconDataDir); - - // Expect InvalidConfigurationException to be thrown - assertThrows(InvalidConfigurationException.class, () -> { - ephemeryDatabaseReset.resetDatabaseAndCreate(serviceConfig, dbFactory); - }); - - // Verify that database creation was not attempted - verify(dbFactory, never()).createDatabase(); - } - - @Test - void shouldThrowInvalidConfigurationExceptionWhenDatabaseCreationFails() throws IOException { - // Simulate successful directory deletion but failure in database creation - when(dbFactory.createDatabase()).thenThrow(new RuntimeException("Database creation failed")); - - // Expect InvalidConfigurationException to be thrown - assertThrows(InvalidConfigurationException.class, () -> { - ephemeryDatabaseReset.resetDatabaseAndCreate(serviceConfig, dbFactory); - }); - - // Verify that directories were deleted - verify(fileUtils).deleteDirectoryRecursively(beaconDataDir); - verify(fileUtils).deleteDirectoryRecursively(slashProtectionDir.resolve("slashprotection")); - - // Verify that database creation was attempted - verify(dbFactory).createDatabase(); - } +class EphemeryDatabaseResetTest { + + @Mock private ServiceConfig serviceConfig; + + @Mock private VersionedDatabaseFactory dbFactory; + + @Mock private DataDirLayout dataDirLayout; + private Path beaconDataDir; + private Path validatorDataDir; + private Path resolvedSlashProtectionDir; + + @Mock private Database database; + + @InjectMocks private EphemeryDatabaseReset ephemeryDatabaseReset; + + @BeforeEach + void setUp() throws IOException { + MockitoAnnotations.openMocks(this); + ephemeryDatabaseReset = spy(new EphemeryDatabaseReset()); + beaconDataDir = createTempDirectory("beaconDataDir"); + validatorDataDir = createTempDirectory("validatorDataDir"); + resolvedSlashProtectionDir = validatorDataDir.resolve("slashprotection"); + when(serviceConfig.getDataDirLayout()).thenReturn(dataDirLayout); + when(dataDirLayout.getBeaconDataDirectory()).thenReturn(beaconDataDir); + when(dataDirLayout.getValidatorDataDirectory()).thenReturn(validatorDataDir); + when(dataDirLayout.getValidatorDataDirectory().resolve("slashprotection")) + .thenReturn(resolvedSlashProtectionDir); + } + + @Test + void shouldResetDirectoriesAndCreateDatabase() throws IOException { + + when(dbFactory.createDatabase()).thenReturn(database); + when(dataDirLayout.getBeaconDataDirectory()).thenReturn(beaconDataDir); + when(dataDirLayout.getValidatorDataDirectory().resolve("slashprotection")) + .thenReturn(resolvedSlashProtectionDir); + + Database result = ephemeryDatabaseReset.resetDatabaseAndCreate(serviceConfig, dbFactory); + verify(ephemeryDatabaseReset).deleteDirectoryRecursively(beaconDataDir); + verify(ephemeryDatabaseReset).deleteDirectoryRecursively(resolvedSlashProtectionDir); + verify(dbFactory).createDatabase(); + assertEquals(database, result); + } + + @Test + void shouldThrowInvalidConfigurationExceptionWhenDirectoryDeletionFails() throws IOException { + doThrow(new IOException("Failed to delete directory")) + .when(ephemeryDatabaseReset) + .deleteDirectoryRecursively(beaconDataDir); + final InvalidConfigurationException exception = + assertThrows( + InvalidConfigurationException.class, + () -> { + ephemeryDatabaseReset.resetDatabaseAndCreate(serviceConfig, dbFactory); + }); + assertEquals( + "The existing ephemery database was old, and was unable to reset it.", + exception.getMessage()); + verify(dbFactory, never()).createDatabase(); + verify(ephemeryDatabaseReset, never()).deleteDirectoryRecursively(resolvedSlashProtectionDir); + } + + @Test + void shouldThrowInvalidConfigurationExceptionWhenDatabaseCreationFails() throws IOException { + doNothing().when(ephemeryDatabaseReset).deleteDirectoryRecursively(beaconDataDir); + doNothing().when(ephemeryDatabaseReset).deleteDirectoryRecursively(resolvedSlashProtectionDir); + when(dbFactory.createDatabase()).thenThrow(new RuntimeException("Database creation failed")); + final InvalidConfigurationException exception = + assertThrows( + InvalidConfigurationException.class, + () -> { + ephemeryDatabaseReset.resetDatabaseAndCreate(serviceConfig, dbFactory); + }); + assertEquals( + "The existing ephemery database was old, and was unable to reset it.", + exception.getMessage()); + verify(ephemeryDatabaseReset).deleteDirectoryRecursively(beaconDataDir); + verify(ephemeryDatabaseReset).deleteDirectoryRecursively(resolvedSlashProtectionDir); + verify(dbFactory).createDatabase(); + } } -