diff --git a/storage/src/main/java/tech/pegasys/teku/storage/archive/fsarchive/BlobSidecarJsonWriter.java b/storage/src/main/java/tech/pegasys/teku/storage/archive/fsarchive/BlobSidecarJsonWriter.java index 6917726e38c..b466a31a041 100644 --- a/storage/src/main/java/tech/pegasys/teku/storage/archive/fsarchive/BlobSidecarJsonWriter.java +++ b/storage/src/main/java/tech/pegasys/teku/storage/archive/fsarchive/BlobSidecarJsonWriter.java @@ -18,8 +18,9 @@ import java.io.IOException; import java.io.OutputStream; import java.util.List; +import java.util.Objects; import tech.pegasys.teku.infrastructure.json.JsonUtil; -import tech.pegasys.teku.infrastructure.json.types.DeserializableTypeDefinition; +import tech.pegasys.teku.infrastructure.json.types.SerializableTypeDefinition; import tech.pegasys.teku.spec.Spec; import tech.pegasys.teku.spec.SpecMilestone; import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar; @@ -28,20 +29,22 @@ public class BlobSidecarJsonWriter { - private final SchemaDefinitionCache schemaCache; + private final SerializableTypeDefinition> blobSidecarType; public BlobSidecarJsonWriter(final Spec spec) { - this.schemaCache = new SchemaDefinitionCache(spec); + SchemaDefinitionCache schemaCache = new SchemaDefinitionCache(spec); + this.blobSidecarType = + listOf( + SchemaDefinitionsDeneb.required(schemaCache.getSchemaDefinition(SpecMilestone.DENEB)) + .getBlobSidecarSchema() + .getJsonTypeDefinition()); } public void writeSlotBlobSidecars(final OutputStream out, final List blobSidecar) throws IOException { + Objects.requireNonNull(out); + Objects.requireNonNull(blobSidecar); - final DeserializableTypeDefinition blobSidecarType = - SchemaDefinitionsDeneb.required(schemaCache.getSchemaDefinition(SpecMilestone.DENEB)) - .getBlobSidecarSchema() - .getJsonTypeDefinition(); - - JsonUtil.serializeToBytes(blobSidecar, listOf(blobSidecarType), out); + JsonUtil.serializeToBytes(blobSidecar, blobSidecarType, out); } } diff --git a/storage/src/main/java/tech/pegasys/teku/storage/archive/fsarchive/FileSystemArchive.java b/storage/src/main/java/tech/pegasys/teku/storage/archive/fsarchive/FileSystemArchive.java index bc2a8a6473d..f98fe13bb0c 100644 --- a/storage/src/main/java/tech/pegasys/teku/storage/archive/fsarchive/FileSystemArchive.java +++ b/storage/src/main/java/tech/pegasys/teku/storage/archive/fsarchive/FileSystemArchive.java @@ -20,6 +20,7 @@ import java.io.IOException; import java.io.OutputStreamWriter; import java.nio.charset.StandardCharsets; +import java.nio.file.Files; import java.nio.file.Path; import java.util.List; import org.apache.logging.log4j.LogManager; @@ -35,9 +36,8 @@ * PathResolver method to decide where to write the files. */ public class FileSystemArchive implements DataArchive { - private static final String INDEX_FILE = "index.dat"; - - private static final Logger LOG = LogManager.getLogger(); + static final String INDEX_FILE = "index.dat"; + static final Logger LOG = LogManager.getLogger(); private final Path baseDirectory; private final BlobSidecarJsonWriter jsonWriter; @@ -51,7 +51,7 @@ public FileSystemArchive(final Spec spec, final Path baseDirectory) { public DataArchiveWriter> getBlobSidecarWriter() throws IOException { try { - File indexFile = baseDirectory.resolve(INDEX_FILE).toFile(); + final File indexFile = baseDirectory.resolve(INDEX_FILE).toFile(); return new FileSystemBlobSidecarWriter(indexFile); } catch (IOException e) { LOG.warn("Unable to create BlobSidecar archive writer", e); @@ -72,7 +72,7 @@ public FileSystemBlobSidecarWriter(final File indexFile) throws IOException { @Override public boolean archive(final List blobSidecars) { - if (blobSidecars == null || blobSidecars.isEmpty()) { + if (blobSidecars == null) { return true; } @@ -83,8 +83,12 @@ public boolean archive(final List blobSidecars) { return false; } - if (!file.getParentFile().mkdirs()) { - LOG.warn("Failed to write BlobSidecar. Could not make directories to: {}", file.toString()); + try { + Files.createDirectories(file.toPath().getParent()); + } catch (IOException e) { + LOG.warn( + "Failed to write BlobSidecar. Could not make directories to: {}", + file.getParentFile().toString()); return false; } @@ -93,7 +97,7 @@ public boolean archive(final List blobSidecars) { indexWriter.write(formatIndexOutput(slotAndBlockRoot)); indexWriter.newLine(); return true; - } catch (IOException e) { + } catch (IOException | NullPointerException e) { LOG.warn("Failed to write BlobSidecar.", e); return false; } diff --git a/storage/src/test/java/tech/pegasys/teku/storage/archive/fsarchive/BlobSidecarJsonWriterTest.java b/storage/src/test/java/tech/pegasys/teku/storage/archive/fsarchive/BlobSidecarJsonWriterTest.java new file mode 100644 index 00000000000..d63f4a3fdd1 --- /dev/null +++ b/storage/src/test/java/tech/pegasys/teku/storage/archive/fsarchive/BlobSidecarJsonWriterTest.java @@ -0,0 +1,96 @@ +/* + * Copyright Consensys Software Inc., 2024 + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package tech.pegasys.teku.storage.archive.fsarchive; + +import static org.junit.jupiter.api.Assertions.assertThrows; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import tech.pegasys.teku.spec.Spec; +import tech.pegasys.teku.spec.TestSpecFactory; +import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar; +import tech.pegasys.teku.spec.util.DataStructureUtil; + +public class BlobSidecarJsonWriterTest { + private static final Spec SPEC = TestSpecFactory.createMinimalDeneb(); + + BlobSidecarJsonWriter blobSidecarJsonWriter; + private DataStructureUtil dataStructureUtil; + + @BeforeEach + public void test() { + this.blobSidecarJsonWriter = new BlobSidecarJsonWriter(SPEC); + this.dataStructureUtil = new DataStructureUtil(SPEC); + } + + @Test + void testWriteSlotBlobSidecarsWithEmptyList() throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + List blobSidecars = new ArrayList<>(); + blobSidecarJsonWriter.writeSlotBlobSidecars(out, blobSidecars); + String json = out.toString(); + assert (json.equals("[]")); + } + + @Test + void testWriteSlotBlobSidecarsWithSingleElement() throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + List blobSidecars = new ArrayList<>(); + final BlobSidecar blobSidecar = + dataStructureUtil.randomBlobSidecarForBlock( + dataStructureUtil.randomSignedBeaconBlock(1), 0); + blobSidecars.add(blobSidecar); + blobSidecarJsonWriter.writeSlotBlobSidecars(out, blobSidecars); + String json = out.toString(); + assert (json.contains("index")); + assert (json.contains("blob")); + assert (json.contains("kzg_commitment")); + assert (json.contains("kzg_proof")); + assert (json.contains("signed_block_header")); + assert (json.contains("parent_root")); + assert (json.contains("state_root")); + assert (json.contains("body_root")); + assert (json.contains("signature")); + } + + @Test + void testWriteSlotBlobSidecarsNulls() { + assertThrows( + NullPointerException.class, () -> blobSidecarJsonWriter.writeSlotBlobSidecars(null, null)); + } + + @Test + void testWriteSlotBlobSidecarsNullOut() { + assertThrows( + NullPointerException.class, + () -> { + List blobSidecars = new ArrayList<>(); + blobSidecarJsonWriter.writeSlotBlobSidecars(null, blobSidecars); + }); + } + + @Test + void testWriteSlotBlobSidecarsNullList() { + assertThrows( + NullPointerException.class, + () -> { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + blobSidecarJsonWriter.writeSlotBlobSidecars(out, null); + }); + } +} diff --git a/storage/src/test/java/tech/pegasys/teku/storage/archive/fsarchive/FileSystemArchiveTest.java b/storage/src/test/java/tech/pegasys/teku/storage/archive/fsarchive/FileSystemArchiveTest.java index 063dc34df23..ce1f67655ec 100644 --- a/storage/src/test/java/tech/pegasys/teku/storage/archive/fsarchive/FileSystemArchiveTest.java +++ b/storage/src/test/java/tech/pegasys/teku/storage/archive/fsarchive/FileSystemArchiveTest.java @@ -13,13 +13,19 @@ package tech.pegasys.teku.storage.archive.fsarchive; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; +import java.io.File; +import java.io.FileInputStream; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; import java.util.List; +import java.util.stream.Stream; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import tech.pegasys.teku.infrastructure.unsigned.UInt64; @@ -29,11 +35,11 @@ import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar; import tech.pegasys.teku.spec.datastructures.blocks.SignedBeaconBlock; import tech.pegasys.teku.spec.datastructures.type.SszKZGProof; +import tech.pegasys.teku.spec.datastructures.util.SlotAndBlockRootAndBlobIndex; import tech.pegasys.teku.spec.logic.common.helpers.Predicates; import tech.pegasys.teku.spec.logic.versions.deneb.helpers.MiscHelpersDeneb; import tech.pegasys.teku.spec.schemas.SchemaDefinitionsDeneb; import tech.pegasys.teku.spec.util.DataStructureUtil; -import tech.pegasys.teku.storage.archive.DataArchive; import tech.pegasys.teku.storage.archive.DataArchiveWriter; public class FileSystemArchiveTest { @@ -48,12 +54,29 @@ public class FileSystemArchiveTest { schemaDefinitionsDeneb); private final DataStructureUtil dataStructureUtil = new DataStructureUtil(SPEC); - static DataArchive dataArchive; + static Path testTempDir; + static FileSystemArchive dataArchive; @BeforeAll static void beforeEach() throws IOException { - Path temp = Files.createTempDirectory("blobs"); - dataArchive = new FileSystemArchive(SPEC, temp); + testTempDir = Files.createTempDirectory("blobs"); + dataArchive = new FileSystemArchive(SPEC, testTempDir); + } + + @AfterEach + public void tearDown() throws IOException { + // Delete the temporary directory after each test + if (Files.exists(testTempDir)) { + try (Stream walk = Files.walk(testTempDir)) { + walk.map(Path::toFile) + .forEach( + file -> { + if (!file.delete()) { + file.deleteOnExit(); + } + }); + } + } } BlobSidecar createBlobSidecar() { @@ -65,11 +88,50 @@ BlobSidecar createBlobSidecar() { return miscHelpersDeneb.constructBlobSidecar(signedBeaconBlock, UInt64.ZERO, blob, proof); } + @Test + void testResolve() { + SlotAndBlockRootAndBlobIndex slotAndBlockRootAndBlobIndex = + new SlotAndBlockRootAndBlobIndex( + UInt64.ONE, dataStructureUtil.randomBytes32(), UInt64.ZERO); + File file = + dataArchive.resolve(testTempDir, slotAndBlockRootAndBlobIndex.getSlotAndBlockRoot()); + + // Check if the file path is correct. Doesn't check the intermediate directories. + assertTrue(file.toString().startsWith(testTempDir.toString())); + assertTrue( + file.toString() + .endsWith(slotAndBlockRootAndBlobIndex.getBlockRoot().toUnprefixedHexString())); + } + @Test void testWriteBlobSidecar() throws IOException { + DataArchiveWriter> blobWriter = dataArchive.getBlobSidecarWriter(); + ArrayList list = new ArrayList<>(); + BlobSidecar blobSidecar = createBlobSidecar(); + list.add(blobSidecar); + assertTrue(blobWriter.archive(list)); + blobWriter.close(); + + // Check if the file was written + FileInputStream fis = + new FileInputStream(testTempDir.resolve(FileSystemArchive.INDEX_FILE).toFile()); + String content = new String(fis.readAllBytes()); + assertEquals( + blobSidecar.getSlot().toString() + + " " + + blobSidecar.getSlotAndBlockRoot().getBlockRoot().toUnprefixedHexString() + + "\n", + content); + } + + @Test + void testFileAlreadyExists() throws IOException { DataArchiveWriter> blobWriter = dataArchive.getBlobSidecarWriter(); ArrayList list = new ArrayList<>(); list.add(createBlobSidecar()); assertTrue(blobWriter.archive(list)); + // Try to write the same file again + assertFalse(blobWriter.archive(list)); + blobWriter.close(); } }