From 9f5181058eaa8449c61a95f40e8740d53bb9654b Mon Sep 17 00:00:00 2001 From: mikera Date: Wed, 28 Aug 2024 07:20:24 +0100 Subject: [PATCH] Make Etch persistence throw checked IOExceptions --- .../convex/benchmarks/BigBlockBenchmark.java | 3 +- .../java/convex/benchmarks/EtchBenchmark.java | 17 +++--- .../convex/benchmarks/SignatureBenchmark.java | 8 ++- .../main/java/convex/cli/etch/EtchWrite.java | 5 ++ .../src/main/java/convex/core/data/Cells.java | 13 +++-- .../src/main/java/convex/core/data/Ref.java | 13 +++-- .../main/java/convex/core/store/AStore.java | 6 ++- .../java/convex/core/store/MemoryStore.java | 2 +- .../src/main/java/convex/etch/EtchStore.java | 37 ++++++------- .../test/java/convex/comms/GenTestFormat.java | 6 ++- .../src/test/java/convex/core/StateTest.java | 6 ++- .../java/convex/core/cpos/BeliefTest.java | 4 +- .../test/java/convex/core/data/BlobsTest.java | 5 +- .../java/convex/core/data/EncodingTest.java | 7 +-- .../convex/core/data/GenTestAnyValue.java | 8 +-- .../java/convex/core/data/ObjectsTest.java | 12 ++++- .../java/convex/core/data/ParamTestRefs.java | 2 +- .../convex/core/data/ParamTestValues.java | 3 +- .../test/java/convex/core/data/RefTest.java | 13 ++--- .../java/convex/core/data/SignedDataTest.java | 6 ++- .../java/convex/core/lang/ParamTestEvals.java | 3 +- .../test/java/convex/store/EtchStoreTest.java | 10 ++-- .../java/convex/store/MemoryStoreTest.java | 7 +-- .../test/java/convex/store/StoresTest.java | 19 ++++--- .../src/main/java/convex/api/ConvexLocal.java | 7 ++- .../java/convex/peer/BeliefPropagator.java | 54 ++++++++++++------- .../src/main/java/convex/peer/Server.java | 22 ++++---- .../java/convex/peer/TransactionHandler.java | 8 +-- .../test/java/convex/peer/MessageTest.java | 3 +- 29 files changed, 194 insertions(+), 115 deletions(-) diff --git a/convex-benchmarks/src/main/java/convex/benchmarks/BigBlockBenchmark.java b/convex-benchmarks/src/main/java/convex/benchmarks/BigBlockBenchmark.java index bc042d6a3..47814ef3a 100644 --- a/convex-benchmarks/src/main/java/convex/benchmarks/BigBlockBenchmark.java +++ b/convex-benchmarks/src/main/java/convex/benchmarks/BigBlockBenchmark.java @@ -1,5 +1,6 @@ package convex.benchmarks; +import java.io.IOException; import java.util.ArrayList; import java.util.Random; @@ -53,7 +54,7 @@ public class BigBlockBenchmark { } @Benchmark - public void benchmark() throws BadSignatureException { + public void benchmark() throws BadSignatureException, IOException { BlockResult br=state.applyBlock(block); Cells.persist(br.getState()); } diff --git a/convex-benchmarks/src/main/java/convex/benchmarks/EtchBenchmark.java b/convex-benchmarks/src/main/java/convex/benchmarks/EtchBenchmark.java index ee6bc5abe..1e77d6c5a 100644 --- a/convex-benchmarks/src/main/java/convex/benchmarks/EtchBenchmark.java +++ b/convex-benchmarks/src/main/java/convex/benchmarks/EtchBenchmark.java @@ -41,21 +41,22 @@ public class EtchBenchmark { static { try { store =EtchStore.createTemp(); + for (int i=0; i v=Vectors.of(0L,(long)i); + Ref r=v.getRef(); + refs[i]=r; + r.getHash(); + store.storeTopRef(r, Ref.STORED, null); + } } catch (IOException e) { throw new Error(e); } - for (int i=0; i v=Vectors.of(0L,(long)i); - Ref r=v.getRef(); - refs[i]=r; - r.getHash(); - store.storeTopRef(r, Ref.STORED, null); - } + System.out.println("Refs stored for testing"); } @Benchmark - public void writeData() { + public void writeData() throws IOException { AVector v=Vectors.of(1L,nonce++); store.storeTopRef(v.getRef(), Ref.STORED, null); } diff --git a/convex-benchmarks/src/main/java/convex/benchmarks/SignatureBenchmark.java b/convex-benchmarks/src/main/java/convex/benchmarks/SignatureBenchmark.java index b86c975ad..a1e056b37 100644 --- a/convex-benchmarks/src/main/java/convex/benchmarks/SignatureBenchmark.java +++ b/convex-benchmarks/src/main/java/convex/benchmarks/SignatureBenchmark.java @@ -1,5 +1,7 @@ package convex.benchmarks; +import java.io.IOException; + import org.openjdk.jmh.annotations.Benchmark; import org.openjdk.jmh.runner.Runner; import org.openjdk.jmh.runner.options.Options; @@ -25,7 +27,11 @@ public class SignatureBenchmark { private static SignedData makeSigned() { SignedData signed= KEYPAIR.signData(Blobs.fromHex("cafebabe")); - Cells.persist(signed); + try { + Cells.persist(signed); + } catch (IOException e) { + throw new Error(e); + } return signed; } diff --git a/convex-cli/src/main/java/convex/cli/etch/EtchWrite.java b/convex-cli/src/main/java/convex/cli/etch/EtchWrite.java index ffcd39f2d..4831d0445 100644 --- a/convex-cli/src/main/java/convex/cli/etch/EtchWrite.java +++ b/convex-cli/src/main/java/convex/cli/etch/EtchWrite.java @@ -1,5 +1,8 @@ package convex.cli.etch; +import java.io.IOException; + +import convex.cli.CLIError; import convex.core.data.ACell; import convex.core.data.Hash; import convex.core.data.Ref; @@ -34,6 +37,8 @@ public void run() { Hash h=Ref.get(cell).getHash(); println(h.toString()); informSuccess("Data saved with hash: "+h); + } catch (IOException e) { + throw new CLIError("Unable to write to store",e); } finally { store.close(); } diff --git a/convex-core/src/main/java/convex/core/data/Cells.java b/convex-core/src/main/java/convex/core/data/Cells.java index a3f0ce18b..89890e725 100644 --- a/convex-core/src/main/java/convex/core/data/Cells.java +++ b/convex-core/src/main/java/convex/core/data/Cells.java @@ -1,5 +1,6 @@ package convex.core.data; +import java.io.IOException; import java.lang.reflect.Array; import java.util.function.Consumer; @@ -126,8 +127,9 @@ public static boolean isValue(ACell a) { * Persist a cell in the current store * @param a Cell to persist * @return Cell after persisting (may be the same Cell if no change in cell hierarchy) + * @throws IOException */ - public static T persist(T a) { + public static T persist(T a) throws IOException { return persist(a,Stores.current()); } @@ -136,8 +138,9 @@ public static T persist(T a) { * @param a Cell to persist * @param store Store instance to persist in * @return Cell after persisting (may be the same Cell if no change in cell hierarchy) + * @throws IOException */ - public static T persist(T a, AStore store) { + public static T persist(T a, AStore store) throws IOException { Ref ref=Ref.get(a); Ref sref=store.storeTopRef(ref, Ref.PERSISTED, null); return sref.getValue(); @@ -148,8 +151,9 @@ public static T persist(T a, AStore store) { * @param a Cell to persist * @param store Store instance to persist in * @return Cell after persisting (may be the same Cell if no change in cell hierarchy) + * @throws IOException */ - public static T store(T a, AStore store) { + public static T store(T a, AStore store) throws IOException { Ref ref=Ref.get(a); Ref sref=store.storeTopRef(ref, Ref.STORED, null); return sref.getValue(); @@ -160,8 +164,9 @@ public static T store(T a, AStore store) { * @param a Cell to announce * @param noveltyHandler Handler for novelty values * @return Cell after announcing (may be the same Cell if no change in cell hierarchy) + * @throws IOException */ - public static T announce(T a, Consumer> noveltyHandler) { + public static T announce(T a, Consumer> noveltyHandler) throws IOException { if (a==null) { return null; // null is never "novelty" }; diff --git a/convex-core/src/main/java/convex/core/data/Ref.java b/convex-core/src/main/java/convex/core/data/Ref.java index 3a7f122f2..4422dafab 100644 --- a/convex-core/src/main/java/convex/core/data/Ref.java +++ b/convex-core/src/main/java/convex/core/data/Ref.java @@ -1,5 +1,6 @@ package convex.core.data; +import java.io.IOException; import java.util.ArrayList; import java.util.HashSet; import java.util.function.Consumer; @@ -404,11 +405,12 @@ public boolean isMarked() { * * @param noveltyHandler Novelty handler to call (may be null) * @return the persisted Ref + * @throws IOException * @throws MissingDataException If the Ref's value does not exist or has been * garbage collected before being persisted */ @SuppressWarnings("unchecked") - public Ref persist(Consumer> noveltyHandler) { + public Ref persist(Consumer> noveltyHandler) throws IOException { int status = getStatus(); if (status >= PERSISTED) return (Ref) this; // already persisted in some form AStore store=Stores.current(); @@ -423,8 +425,9 @@ public Ref persist(Consumer> noveltyHandler) { * * @throws MissingDataException if the Ref cannot be fully persisted. * @return the persisted Ref + * @throws IOException */ - public Ref persist() { + public Ref persist() throws IOException { return persist(null); } @@ -531,8 +534,9 @@ public final boolean isEmbedded() { * Status will be updated to STORED or higher. * * @return Ref with status of STORED or above + * @throws IOException */ - public Ref persistShallow() { + public Ref persistShallow() throws IOException { return persistShallow(null); } @@ -544,9 +548,10 @@ public Ref persistShallow() { * * @param noveltyHandler Novelty handler to call (may be null) * @return Ref with status of STORED or above + * @throws IOException */ @SuppressWarnings("unchecked") - public Ref persistShallow(Consumer> noveltyHandler) { + public Ref persistShallow(Consumer> noveltyHandler) throws IOException { AStore store=Stores.current(); return (Ref) store.storeTopRef((Ref)this, Ref.STORED, noveltyHandler); } diff --git a/convex-core/src/main/java/convex/core/store/AStore.java b/convex-core/src/main/java/convex/core/store/AStore.java index 71c1b2154..d7c2d597b 100644 --- a/convex-core/src/main/java/convex/core/store/AStore.java +++ b/convex-core/src/main/java/convex/core/store/AStore.java @@ -37,8 +37,9 @@ public abstract class AStore implements Closeable { * @param status Status to store at * @param noveltyHandler Novelty Handler function for Novelty detected. May be null. * @return The persisted Ref, of status STORED at minimum + * @throws IOException */ - public abstract Ref storeRef(Ref ref, int status,Consumer> noveltyHandler); + public abstract Ref storeRef(Ref ref, int status,Consumer> noveltyHandler) throws IOException; /** * Stores a top level @Ref in long term storage as defined by this store implementation. @@ -55,8 +56,9 @@ public abstract class AStore implements Closeable { * @param status Status to store at * @param noveltyHandler Novelty Handler function for Novelty detected. May be null. * @return The persisted Ref, of status STORED at minimum + * @throws IOException */ - public abstract Ref storeTopRef(Ref ref, int status,Consumer> noveltyHandler); + public abstract Ref storeTopRef(Ref ref, int status,Consumer> noveltyHandler) throws IOException; /** diff --git a/convex-core/src/main/java/convex/core/store/MemoryStore.java b/convex-core/src/main/java/convex/core/store/MemoryStore.java index 29efd3d08..2f635c14a 100644 --- a/convex-core/src/main/java/convex/core/store/MemoryStore.java +++ b/convex-core/src/main/java/convex/core/store/MemoryStore.java @@ -99,7 +99,7 @@ public Ref persistRef(Ref ref, Consumer> nove // need to do recursive persistence cell = cell.updateRefs(r -> { - return r.persist(noveltyHandler); + return persistRef(r,noveltyHandler,requiredStatus,false); }); ref=ref.withValue((T)cell); diff --git a/convex-core/src/main/java/convex/etch/EtchStore.java b/convex-core/src/main/java/convex/etch/EtchStore.java index 946052139..79d5ad5fd 100644 --- a/convex-core/src/main/java/convex/etch/EtchStore.java +++ b/convex-core/src/main/java/convex/etch/EtchStore.java @@ -133,18 +133,18 @@ public Ref readStoreRef(Hash hash) throws IOException { } @Override - public Ref storeRef(Ref ref, int status, Consumer> noveltyHandler) { + public Ref storeRef(Ref ref, int status, Consumer> noveltyHandler) throws IOException { return storeRef(ref, status, noveltyHandler, false); } @Override - public Ref storeTopRef(Ref ref, int status, Consumer> noveltyHandler) { + public Ref storeTopRef(Ref ref, int status, Consumer> noveltyHandler) throws IOException { return storeRef(ref, status, noveltyHandler, true); } @SuppressWarnings("unchecked") public Ref storeRef(Ref ref, int requiredStatus, Consumer> noveltyHandler, - boolean topLevel) { + boolean topLevel) throws IOException { // Get the value. If we are persisting, should be there! ACell cell = ref.getValue(); @@ -179,7 +179,11 @@ public Ref storeRef(Ref ref, int requiredStatus, Consume if ((requiredStatus > Ref.STORED) && (cell.getRefCount() > 0)) { // TODO: probably slow to rebuild these all the time! IRefFunction func = r -> { - return storeRef((Ref) r, requiredStatus, noveltyHandler, false); + try { + return storeRef((Ref) r, requiredStatus, noveltyHandler, false); + } catch (IOException e) { + throw Utils.sneakyThrow(e); + } }; // need to do recursive persistence @@ -206,23 +210,20 @@ public Ref storeRef(Ref ref, int requiredStatus, Consume } Ref result; - try { - // ensure status is set when we write to store - ref = ref.withMinimumStatus(requiredStatus); - cell.attachRef(ref); // make sure we are using current ref within cell - result = etch.write(fHash, (Ref) ref); - - if (!embedded) { - // Ensure we have soft Refpointing to this store - result = result.toSoft(this); - } - cell.attachRef(result); - addToCache(result); // cache for subsequent writes - } catch (IOException e) { - throw Utils.sneakyThrow(e); + // ensure status is set when we write to store + ref = ref.withMinimumStatus(requiredStatus); + cell.attachRef(ref); // make sure we are using current ref within cell + result = etch.write(fHash, (Ref) ref); + + if (!embedded) { + // Ensure we have soft Refpointing to this store + result = result.toSoft(this); } + cell.attachRef(result); + addToCache(result); // cache for subsequent writes + // call novelty handler if newly persisted non-embedded if (noveltyHandler != null) { if (!embedded) diff --git a/convex-core/src/test/java/convex/comms/GenTestFormat.java b/convex-core/src/test/java/convex/comms/GenTestFormat.java index f07ecd189..fe6a8c45a 100644 --- a/convex-core/src/test/java/convex/comms/GenTestFormat.java +++ b/convex-core/src/test/java/convex/comms/GenTestFormat.java @@ -2,6 +2,8 @@ import static org.junit.Assert.assertEquals; +import java.io.IOException; + import org.junit.runner.RunWith; import com.pholser.junit.quickcheck.From; @@ -35,7 +37,7 @@ public void messageRoundTrip(String str) throws BadFormatException { } @Property - public void primitiveRoundTrip(@From(PrimitiveGen.class) ACell prim) throws BadFormatException { + public void primitiveRoundTrip(@From(PrimitiveGen.class) ACell prim) throws BadFormatException, IOException { Blob b = Format.encodedBlob(prim); if (!Format.isEmbedded(prim)) { // persist in case large @@ -49,7 +51,7 @@ public void primitiveRoundTrip(@From(PrimitiveGen.class) ACell prim) throws BadF } @Property - public void dataRoundTrip(@From(ValueGen.class) ACell value) throws BadFormatException { + public void dataRoundTrip(@From(ValueGen.class) ACell value) throws BadFormatException, IOException { Ref pref = Ref.get(Cells.persist(value)); // ensure persisted Blob b = Format.encodedBlob(value); ACell o = Format.read(b); diff --git a/convex-core/src/test/java/convex/core/StateTest.java b/convex-core/src/test/java/convex/core/StateTest.java index 6ff74697d..0f90f71cf 100644 --- a/convex-core/src/test/java/convex/core/StateTest.java +++ b/convex-core/src/test/java/convex/core/StateTest.java @@ -5,6 +5,8 @@ import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertTrue; +import java.io.IOException; + import org.junit.jupiter.api.Test; import convex.core.crypto.AKeyPair; @@ -53,7 +55,7 @@ public void testInitialState() throws InvalidDataException { } @Test - public void testRoundTrip() throws BadFormatException { + public void testRoundTrip() throws BadFormatException, IOException { State s = INIT_STATE; assertEquals(0,s.getRef().getStatus()); @@ -97,7 +99,7 @@ public void testMultiCellTrip() throws BadFormatException { } @SuppressWarnings("unused") - @Test public void testStateRefs() { + @Test public void testStateRefs() throws IOException { AKeyPair kp=AKeyPair.createSeeded(578587); State s=Init.createState(Lists.of(kp.getAccountKey())); diff --git a/convex-core/src/test/java/convex/core/cpos/BeliefTest.java b/convex-core/src/test/java/convex/core/cpos/BeliefTest.java index 0c75b71bd..fa946746b 100644 --- a/convex-core/src/test/java/convex/core/cpos/BeliefTest.java +++ b/convex-core/src/test/java/convex/core/cpos/BeliefTest.java @@ -1,5 +1,7 @@ package convex.core.cpos; +import java.io.IOException; + import org.junit.jupiter.api.Test; import convex.core.Belief; @@ -30,7 +32,7 @@ public class BeliefTest { } @SuppressWarnings("unchecked") - @Test public void testBasicBelief() throws BadFormatException { + @Test public void testBasicBelief() throws BadFormatException, IOException { Order o=Order.create(); for (int i=0; iFormat.read(b)); } - @Test public void testDeltaEncoding() throws BadFormatException { + @Test public void testDeltaEncoding() throws BadFormatException, IOException { Blob randBlob=Blob.createRandom(new Random(1234), 2*Format.MAX_EMBEDDED_LENGTH); AVector v=Vectors.of(1,randBlob,randBlob); @@ -326,7 +327,7 @@ public void testFailedMissingEncoding() throws BadFormatException { } - @Test public void testBeliefEncoding() throws BadFormatException, InvalidDataException { + @Test public void testBeliefEncoding() throws BadFormatException, InvalidDataException, IOException { AKeyPair kp=Samples.KEY_PAIR; Order order=Order.create(); order.append(kp.signData(Block.create(0, Vectors.empty()))); @@ -436,7 +437,7 @@ public static void checkCodingSize(ACell c) { } - @Test public void testDeltaBeliefEncoding() throws BadFormatException { + @Test public void testDeltaBeliefEncoding() throws BadFormatException, IOException { AKeyPair kp=AKeyPair.createSeeded(101); Order o=Order.create(); o=o.append(kp.signData(Block.create(0, Vectors.of(kp.signData(Invoke.create(Address.ZERO, 1, o)))))); diff --git a/convex-core/src/test/java/convex/core/data/GenTestAnyValue.java b/convex-core/src/test/java/convex/core/data/GenTestAnyValue.java index 209e15da1..35e8c2e95 100644 --- a/convex-core/src/test/java/convex/core/data/GenTestAnyValue.java +++ b/convex-core/src/test/java/convex/core/data/GenTestAnyValue.java @@ -5,6 +5,8 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import java.io.IOException; + import org.junit.runner.RunWith; import com.pholser.junit.quickcheck.From; @@ -35,7 +37,7 @@ public void printFormats(@From(ValueGen.class) ACell o) { } @Property - public void genericTests(@From(ValueGen.class) ACell o) throws InvalidDataException, BadFormatException { + public void genericTests(@From(ValueGen.class) ACell o) throws InvalidDataException, BadFormatException, IOException { ObjectsTest.doAnyValueTests(o); } @@ -71,7 +73,7 @@ public void testFuzzing(@From(ValueGen.class) ACell o) throws InvalidDataExcepti } @Property - public void validEmbedded(@From(ValueGen.class) ACell o) throws InvalidDataException, BadFormatException { + public void validEmbedded(@From(ValueGen.class) ACell o) throws InvalidDataException, BadFormatException, IOException { if (Format.isEmbedded(o)) { Cells.persist(o); // NOTE: may have child refs to persist @@ -97,7 +99,7 @@ public void validEmbedded(@From(ValueGen.class) ACell o) throws InvalidDataExcep } @Property (trials=20) - public void dataRoundTrip(@From(ValueGen.class) ACell o) throws BadFormatException { + public void dataRoundTrip(@From(ValueGen.class) ACell o) throws BadFormatException, IOException { Blob data=Format.encodedBlob(o); // introduce a small offset to ensure blobs working correctly diff --git a/convex-core/src/test/java/convex/core/data/ObjectsTest.java b/convex-core/src/test/java/convex/core/data/ObjectsTest.java index 95392283f..7ed62ff18 100644 --- a/convex-core/src/test/java/convex/core/data/ObjectsTest.java +++ b/convex-core/src/test/java/convex/core/data/ObjectsTest.java @@ -8,6 +8,9 @@ import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + +import java.io.IOException; import convex.core.Constants; import convex.core.data.Refs.RefTreeStats; @@ -30,11 +33,16 @@ public class ObjectsTest { * *any* ACell where RT.isCVM(..) returns true and validate(...) succeeds. * * @param a Value to test + * @throws IOException */ public static void doAnyValueTests(ACell a) { Hash h=Hash.get(a); - a=Cells.persist(a); + try { + a=Cells.persist(a); + } catch (IOException e) { + fail(e); + } Ref r = Ref.get(a); assertEquals(h,r.getHash()); assertSame(a, r.getValue()); // shouldn't get GC'd because we have a strong reference @@ -315,7 +323,7 @@ private static void doCellRefTests(ACell a) { } @SuppressWarnings("unused") - private static void doCellStorageTest(ACell a) throws InvalidDataException { + private static void doCellStorageTest(ACell a) throws InvalidDataException, IOException { AStore temp=Stores.current(); try { diff --git a/convex-core/src/test/java/convex/core/data/ParamTestRefs.java b/convex-core/src/test/java/convex/core/data/ParamTestRefs.java index d8ce5839e..27177731a 100644 --- a/convex-core/src/test/java/convex/core/data/ParamTestRefs.java +++ b/convex-core/src/test/java/convex/core/data/ParamTestRefs.java @@ -39,7 +39,7 @@ public static Collection dataExamples() { } @Test - public void testStoreUsage() { + public void testStoreUsage() throws IOException { AStore temp=Stores.current(); try { diff --git a/convex-core/src/test/java/convex/core/data/ParamTestValues.java b/convex-core/src/test/java/convex/core/data/ParamTestValues.java index bf457bc41..e0cbd329e 100644 --- a/convex-core/src/test/java/convex/core/data/ParamTestValues.java +++ b/convex-core/src/test/java/convex/core/data/ParamTestValues.java @@ -6,6 +6,7 @@ import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; +import java.io.IOException; import java.util.Arrays; import java.util.Collection; @@ -115,7 +116,7 @@ public void testType() { } @Test - public void testHexRoundTrip() throws InvalidDataException, ValidationException { + public void testHexRoundTrip() throws InvalidDataException, ValidationException, IOException { Cells.persist(data); String hex = Format.encodedBlob(data).toHexString(); Blob d2 = Blob.fromHex(hex); diff --git a/convex-core/src/test/java/convex/core/data/RefTest.java b/convex-core/src/test/java/convex/core/data/RefTest.java index 8c89c3bdd..530993b86 100644 --- a/convex-core/src/test/java/convex/core/data/RefTest.java +++ b/convex-core/src/test/java/convex/core/data/RefTest.java @@ -7,6 +7,7 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import java.io.IOException; import java.util.ArrayList; import java.util.Random; import java.util.Set; @@ -55,7 +56,7 @@ public void testRefSet() { } @Test - public void testShallowPersist() { + public void testShallowPersist() throws IOException { Blob bb = Blob.createRandom(new Random(), 100); // unique blob but embedded assertTrue(bb.isEmbedded()); @@ -145,7 +146,7 @@ public void testEmbedded() { } @Test - public void testPersistEmbeddedNull() throws InvalidDataException { + public void testPersistEmbeddedNull() throws InvalidDataException, IOException { Ref nr = Ref.get(null); assertSame(Ref.NULL_VALUE, nr); assertSame(nr, nr.persist()); @@ -154,7 +155,7 @@ public void testPersistEmbeddedNull() throws InvalidDataException { } @Test - public void testPersistEmbeddedLong() { + public void testPersistEmbeddedLong() throws IOException { ACell val=RT.cvm(10001L); Ref nr = Ref.get(val); Ref nrp = nr.persist(); @@ -169,7 +170,7 @@ public void testPersistEmbeddedLong() { } @Test - public void testPersistNestedBlob() { + public void testPersistNestedBlob() throws IOException { ABlob bigBlob=Blobs.createRandom(17*Blob.CHUNK_LENGTH); // 16 full chunks plus one extra (3 levels) RefTreeStats rs=Refs.getRefTreeStats(bigBlob.getRef()); assertEquals(19,rs.total); @@ -189,7 +190,7 @@ public void testPersistNestedBlob() { } @Test - public void testGoodData() { + public void testGoodData() throws IOException { AVector value = Vectors.of(Keywords.FOO, Symbols.FOO); // a good ref Ref orig = value.getRef(); @@ -207,7 +208,7 @@ public void testGoodData() { } @Test - public void testCompare() { + public void testCompare() throws IOException { assertEquals(0, Ref.get(RT.cvm(1L)).compareTo(Cells.persist(RT.cvm(1L)).getRef())); assertEquals(1, Ref.get(RT.cvm(1L)).compareTo( Ref.forHash(Hash.fromHex("0000000000000000000000000000000000000000000000000000000000000000")))); diff --git a/convex-core/src/test/java/convex/core/data/SignedDataTest.java b/convex-core/src/test/java/convex/core/data/SignedDataTest.java index 8f7d3eb1a..6d018ba7f 100644 --- a/convex-core/src/test/java/convex/core/data/SignedDataTest.java +++ b/convex-core/src/test/java/convex/core/data/SignedDataTest.java @@ -8,6 +8,8 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import java.io.IOException; + import org.junit.jupiter.api.Test; import convex.core.crypto.AKeyPair; @@ -21,7 +23,7 @@ public class SignedDataTest { @SuppressWarnings("unchecked") @Test - public void testBadSignature() { + public void testBadSignature() throws IOException { Ref dref = Ref.get(RT.cvm(13L)); SignedData sd = SignedData.create(Samples.BAD_ACCOUNTKEY, Samples.BAD_SIGNATURE, dref); @@ -82,7 +84,7 @@ public void testEmbeddedSignature() throws BadSignatureException { @SuppressWarnings("unchecked") @Test - public void testSignatureCache() { + public void testSignatureCache() throws IOException { CVMLong cl=RT.cvm(1585856457); AKeyPair kp = InitTest.HERO_KEYPAIR; SignedData sd = kp.signData(cl); diff --git a/convex-core/src/test/java/convex/core/lang/ParamTestEvals.java b/convex-core/src/test/java/convex/core/lang/ParamTestEvals.java index 5b388d2eb..773c0019b 100644 --- a/convex-core/src/test/java/convex/core/lang/ParamTestEvals.java +++ b/convex-core/src/test/java/convex/core/lang/ParamTestEvals.java @@ -3,6 +3,7 @@ import static convex.test.Assertions.assertCVMEquals; import static org.junit.Assert.assertEquals; +import java.io.IOException; import java.util.Arrays; import java.util.Collection; @@ -100,7 +101,7 @@ public Context eval(String source) { } @Test - public void testOpRoundTrip() throws BadFormatException { + public void testOpRoundTrip() throws BadFormatException, IOException { AOp op = compile(source); Blob b = Format.encodedBlob(op); Cells.persist(op); // persist to allow re-creation diff --git a/convex-core/src/test/java/convex/store/EtchStoreTest.java b/convex-core/src/test/java/convex/store/EtchStoreTest.java index 66161d646..949148689 100644 --- a/convex-core/src/test/java/convex/store/EtchStoreTest.java +++ b/convex-core/src/test/java/convex/store/EtchStoreTest.java @@ -63,7 +63,7 @@ public class EtchStoreTest { } @Test - public void testEmptyStore() { + public void testEmptyStore() throws IOException { AStore oldStore = Stores.current(); try { Stores.setCurrent(store); @@ -129,7 +129,7 @@ public void testEmptyStore() { } @Test - public void testPersistedStatus() throws BadFormatException { + public void testPersistedStatus() throws BadFormatException, IOException { AStore oldStore = Stores.current(); try { Stores.setCurrent(store); @@ -161,7 +161,7 @@ public void testPersistedStatus() throws BadFormatException { } @Test - public void testBeliefAnnounce() { + public void testBeliefAnnounce() throws IOException { AStore oldStore = Stores.current(); AtomicLong counter=new AtomicLong(0L); @@ -239,7 +239,7 @@ public void testBeliefAnnounce() { } @Test - public void testNoveltyHandler() { + public void testNoveltyHandler() throws IOException { AStore oldStore = Stores.current(); ArrayList> al = new ArrayList<>(); try { @@ -266,7 +266,7 @@ public void testNoveltyHandler() { } } - @Test public void testDecodeCache() throws BadFormatException { + @Test public void testDecodeCache() throws BadFormatException, IOException { AStore oldStore = Stores.current(); try { Stores.setCurrent(store); diff --git a/convex-core/src/test/java/convex/store/MemoryStoreTest.java b/convex-core/src/test/java/convex/store/MemoryStoreTest.java index bd35117a2..b488a5494 100644 --- a/convex-core/src/test/java/convex/store/MemoryStoreTest.java +++ b/convex-core/src/test/java/convex/store/MemoryStoreTest.java @@ -6,6 +6,7 @@ import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; +import java.io.IOException; import java.util.ArrayList; import java.util.Random; import java.util.function.Consumer; @@ -35,7 +36,7 @@ public class MemoryStoreTest { private static final Hash BAD_HASH = Samples.BAD_HASH; @Test - public void testEmptyStore() { + public void testEmptyStore() throws IOException { AStore oldStore = Stores.current(); MemoryStore ms = new MemoryStore(); try { @@ -64,7 +65,7 @@ public void testEmptyStore() { } @Test - public void testPersistedStatus() throws BadFormatException { + public void testPersistedStatus() throws BadFormatException, IOException { // generate Hash of unique secure random bytes to test - should not already be // in store Blob value = Blob.createRandom(new Random(), Format.MAX_EMBEDDED_LENGTH); @@ -86,7 +87,7 @@ public void testPersistedStatus() throws BadFormatException { } @Test - public void testNoveltyHandler() { + public void testNoveltyHandler() throws IOException { AStore oldStore = Stores.current(); MemoryStore ms = new MemoryStore(); ArrayList> al = new ArrayList<>(); diff --git a/convex-core/src/test/java/convex/store/StoresTest.java b/convex-core/src/test/java/convex/store/StoresTest.java index 9c1d92126..74855e3e6 100644 --- a/convex-core/src/test/java/convex/store/StoresTest.java +++ b/convex-core/src/test/java/convex/store/StoresTest.java @@ -26,6 +26,7 @@ import convex.core.store.AStore; import convex.core.store.MemoryStore; import convex.core.store.Stores; +import convex.core.util.Utils; import convex.etch.EtchStore; import convex.test.Samples; @@ -40,7 +41,7 @@ public class StoresTest { } } - @Test public void testInitState() throws InvalidDataException { + @Test public void testInitState() throws InvalidDataException, IOException { AStore temp=Stores.current(); try { Stores.setCurrent(testStore); @@ -77,12 +78,16 @@ public class StoresTest { assertTrue(Cells.isCompletelyEncoded(ev)); AVector v=Vectors.of(Vectors.of(nv,ev),nv,ev); - + Consumer crossTest=x->{ - doCrossStoreTest(x,e1,e2); - doCrossStoreTest(x,m1,e1); - doCrossStoreTest(x,e2,m2); - doCrossStoreTest(x,m1,m2); + try { + doCrossStoreTest(x,e1,e2); + doCrossStoreTest(x,m1,e1); + doCrossStoreTest(x,e2,m2); + doCrossStoreTest(x,m1,m2); + } catch (IOException e) { + throw Utils.sneakyThrow(e); + } }; AStore temp=Stores.current(); @@ -104,7 +109,7 @@ public class StoresTest { } } - private void doCrossStoreTest(ACell a, AStore s1, AStore s2) { + private void doCrossStoreTest(ACell a, AStore s1, AStore s2) throws IOException { Hash ha=Cells.getHash(a); ACell a1=Cells.persist(a, s1); diff --git a/convex-peer/src/main/java/convex/api/ConvexLocal.java b/convex-peer/src/main/java/convex/api/ConvexLocal.java index fc0ed6b10..7c0eb49aa 100644 --- a/convex-peer/src/main/java/convex/api/ConvexLocal.java +++ b/convex-peer/src/main/java/convex/api/ConvexLocal.java @@ -1,5 +1,6 @@ package convex.api; +import java.io.IOException; import java.net.InetSocketAddress; import java.util.concurrent.CompletableFuture; import java.util.function.Predicate; @@ -55,7 +56,11 @@ public CompletableFuture acquire(Hash hash, AStore store) { if (ref==null) { f.completeExceptionally(new MissingDataException(peerStore,hash)); } else { - ref=store.storeTopRef(ref, Ref.PERSISTED, null); + try { + ref=store.storeTopRef(ref, Ref.PERSISTED, null); + } catch (IOException e) { + f.completeExceptionally(e); + } f.complete((T) ref.getValue()); } }); diff --git a/convex-peer/src/main/java/convex/peer/BeliefPropagator.java b/convex-peer/src/main/java/convex/peer/BeliefPropagator.java index 9ba7e5798..1795b2b22 100644 --- a/convex-peer/src/main/java/convex/peer/BeliefPropagator.java +++ b/convex-peer/src/main/java/convex/peer/BeliefPropagator.java @@ -1,5 +1,6 @@ package convex.peer; +import java.io.IOException; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; @@ -126,6 +127,8 @@ protected void loop() throws InterruptedException { // Wait for some new Beliefs to accumulate up to a given time Belief incomingBelief = awaitBelief(); + + // Try belief update boolean updated= maybeUpdateBelief(incomingBelief); if (updated) { @@ -143,15 +146,38 @@ protected void loop() throws InterruptedException { } } + maybeBroadcast(updated); + + // Persist Belief in all cases, just without announcing + // This is mainly in case we get missing data / sync requests for the Belief + // This is super cheap if already persisted, so no problem + try { + belief=Cells.persist(belief); + } catch (IOException e) { + throw Utils.sneakyThrow(e); + } + + /* Update Belief again after persistence. We want to be using + * Latest persisted version as much as possible + */ + server.updateBelief(belief); + } + + + protected void maybeBroadcast(boolean updated) throws InterruptedException { long ts=Utils.getCurrentTimestamp(); if (updated||(ts>lastBroadcastTime+BELIEF_REBROADCAST_DELAY)) { lastBroadcastTime=ts; - Message msg; - if (ts>lastFullBroadcastTime+BELIEF_FULL_BROADCAST_DELAY) { - msg=createFullUpdateMessage(); - lastFullBroadcastTime=ts; - } else { - msg=createQuickUpdateMessage(); + Message msg=null; + try { + if (ts>lastFullBroadcastTime+BELIEF_FULL_BROADCAST_DELAY) { + msg=createFullUpdateMessage(); + lastFullBroadcastTime=ts; + } else { + msg=createQuickUpdateMessage(); + } + } catch (IOException e) { + log.warn("IO Error attempting to create broadcast message",e); } // Actually broadcast the message to outbound connected Peers @@ -161,16 +187,6 @@ protected void loop() throws InterruptedException { log.warn("null message in BeliefPropagator!"); } } - - // Persist Belief in all cases, just without announcing - // This is mainly in case we get missing data / sync requests for the Belief - // This is super cheap if already persisted, so no problem - belief=Cells.persist(belief); - - /* Update Belief again after persistence. We want to be using - * Latest persisted version as much as possible - */ - server.updateBelief(belief); } @Override public void start() { @@ -184,7 +200,7 @@ protected void loop() throws InterruptedException { * @return true if Peer Belief changed, false otherwise * @throws InterruptedException */ - protected boolean maybeUpdateBelief(Belief newBelief) throws InterruptedException { + protected boolean maybeUpdateBelief(Belief newBelief) { // we are in full consensus if there are no unconfirmed blocks after the consensus point //boolean inConsensus=peer.getConsensusPoint()==peer.getPeerOrder().getBlockCount(); @@ -366,7 +382,7 @@ private void doBroadcast(Message msg) throws InterruptedException { beliefBroadcastCount++; } - private Message createFullUpdateMessage() { + private Message createFullUpdateMessage() throws IOException { ArrayList novelty=new ArrayList<>(); // At this point we know something updated our belief, so we want to rebroadcast @@ -390,7 +406,7 @@ private Message createFullUpdateMessage() { return msg; } - private Message createQuickUpdateMessage() { + private Message createQuickUpdateMessage() throws IOException { ArrayList novelty=new ArrayList<>(); // At this point we know something updated our belief, so we want to rebroadcast diff --git a/convex-peer/src/main/java/convex/peer/Server.java b/convex-peer/src/main/java/convex/peer/Server.java index 12d33d6dd..e33ce1eb7 100644 --- a/convex-peer/src/main/java/convex/peer/Server.java +++ b/convex-peer/src/main/java/convex/peer/Server.java @@ -596,19 +596,21 @@ private void processData(Message m) { ACell payload; try { payload = m.getPayload(); - } catch (BadFormatException e) { + Counters.peerDataReceived++; + + // Note: partial messages are handled in Connection now + Ref r = Ref.get(payload); + if (r.isEmbedded()) { + log.warn("DATA with embedded value: "+payload); + return; + } + r = r.persistShallow(); + } catch (BadFormatException | IOException e) { + log.debug("Error processing data: "+e.getMessage()); m.closeConnection(); return; } - Counters.peerDataReceived++; - - // Note: partial messages are handled in Connection now - Ref r = Ref.get(payload); - if (r.isEmbedded()) { - log.warn("DATA with embedded value: "+payload); - return; - } - r = r.persistShallow(); + } /** diff --git a/convex-peer/src/main/java/convex/peer/TransactionHandler.java b/convex-peer/src/main/java/convex/peer/TransactionHandler.java index d2b8c92c7..6111fb768 100644 --- a/convex-peer/src/main/java/convex/peer/TransactionHandler.java +++ b/convex-peer/src/main/java/convex/peer/TransactionHandler.java @@ -7,6 +7,7 @@ import java.util.concurrent.TimeUnit; import java.util.function.BiConsumer; import java.util.function.Consumer; +import java.io.IOException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -107,7 +108,7 @@ private void registerInterest(Hash signedTransactionHash, Message m) { interests.put(signedTransactionHash, m); } - protected void processMessage(Message m) { + protected void processMessage(Message m) throws InterruptedException { try { this.receivedTransactionCount++; @@ -135,10 +136,9 @@ protected void processMessage(Message m) { this.clientTransactionCount++; registerInterest(sd.getHash(), m); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } catch (BadFormatException e) { + } catch (BadFormatException | IOException e) { log.warn("Unandled exception in transaction handler",e); + m.closeConnection(); } } diff --git a/convex-peer/src/test/java/convex/peer/MessageTest.java b/convex-peer/src/test/java/convex/peer/MessageTest.java index 43c439fb8..382ede819 100644 --- a/convex-peer/src/test/java/convex/peer/MessageTest.java +++ b/convex-peer/src/test/java/convex/peer/MessageTest.java @@ -4,6 +4,7 @@ import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertThrows; +import java.io.IOException; import java.util.Random; import org.junit.jupiter.api.Test; @@ -35,7 +36,7 @@ public void testBadCode() { } @Test - public void testDataMessages() throws BadFormatException { + public void testDataMessages() throws BadFormatException, IOException { Blob b=Blob.createRandom(new Random(1256785), 1000); Cells.persist(b);