From f89e8b78f2a308cbc72e9610f905d46c56e950e1 Mon Sep 17 00:00:00 2001 From: Constantine Date: Wed, 17 Jan 2024 19:31:12 -0800 Subject: [PATCH] B 5 (#181) * Moar cleanup Mostly lambda cleanup, but a few minor bugs as well. * split out verifiers from Parameters to allow de/serialization * missed * Configure DHT URL template, rather than hard-wire * Configure DHT URL template, rather than hard-wire * add encryption caps, expand keystore to alias storage :( moar cleanup * add Key Encapsulation Method to EncryptionAlgorithm moar cleanup * forgot about the algo diff. Weird asymmetry * error messages * add context interceptor and provide bootstrapping * further cleanup of notes/certs/validations Fallout from the addition of event sequence to signature. No longer need event coordinates, just the identifier. * better error handling/logging for KerlDHT. * bootstrap optimization for context majority * far better logging. add support for getKeyState(id, seq) * ugh. forgot this * implement getKS(id, seq) - wow. Gate with empty check on reconcile. * bootstrapping accommodations. use diadem's cardinality, rather than context bft normalized cardinality. Ring* handle no members * moar bootstrapping accommodations. "up to 4" --- .../com/salesforce/apollo/choam/CHOAM.java | 42 +- .../salesforce/apollo/choam/Committee.java | 23 +- .../apollo/choam/GenesisAssembly.java | 55 +-- .../apollo/choam/GenesisContext.java | 6 +- .../salesforce/apollo/choam/Parameters.java | 8 +- .../com/salesforce/apollo/choam/Producer.java | 102 ++--- .../salesforce/apollo/choam/ViewAssembly.java | 19 +- .../salesforce/apollo/choam/ViewContext.java | 50 +-- .../apollo/choam/comm/Concierge.java | 8 +- .../apollo/choam/comm/Terminal.java | 2 +- .../apollo/choam/comm/TerminalClient.java | 2 +- .../apollo/choam/comm/TerminalServer.java | 15 +- .../apollo/choam/comm/TxnSubmission.java | 6 +- .../apollo/choam/comm/TxnSubmitClient.java | 6 +- .../apollo/choam/comm/TxnSubmitServer.java | 5 +- .../salesforce/apollo/choam/fsm/Combine.java | 60 +-- .../salesforce/apollo/choam/fsm/Driven.java | 6 +- .../salesforce/apollo/choam/fsm/Genesis.java | 43 +- .../apollo/choam/fsm/Reconfiguration.java | 49 +- .../apollo/choam/support/BatchingQueue.java | 131 +++--- .../apollo/choam/support/Bootstrapper.java | 8 +- .../apollo/choam/support/CheckpointState.java | 2 +- .../apollo/choam/support/ChoamMetrics.java | 22 +- .../choam/support/ChoamMetricsImpl.java | 60 ++- .../apollo/choam/support/DigestType.java | 4 +- .../support/ExponentialBackoffPolicy.java | 118 ++--- .../apollo/choam/support/HashedBlock.java | 13 +- .../choam/support/HashedCertifiedBlock.java | 3 +- .../choam/support/InvalidTransaction.java | 1 - .../apollo/choam/support/OneShot.java | 8 +- .../choam/support/ServiceUnavailable.java | 1 - .../apollo/choam/support/Store.java | 2 +- .../choam/support/TransactionFailed.java | 1 - .../apollo/choam/support/TxDataSource.java | 19 +- .../apollo/cryptography/EdDSAOperations.java | 4 +- .../cryptography/EncryptionAlgorithm.java | 163 +++++++ .../cryptography/SignatureAlgorithm.java | 42 +- .../salesforce/apollo/cryptography/XTest.java | 76 ++++ .../salesforce/apollo/fireflies/Binding.java | 86 ++-- .../apollo/fireflies/Bootstrapper.java | 244 ---------- .../apollo/fireflies/NoteWrapper.java | 7 +- .../com/salesforce/apollo/fireflies/View.java | 141 ++---- .../apollo/fireflies/ViewManagement.java | 226 +++++----- .../fireflies/comm/entrance/Entrance.java | 22 +- .../comm/entrance/EntranceClient.java | 13 - .../comm/entrance/EntranceServer.java | 58 +-- .../comm/entrance/EntranceService.java | 12 +- .../apollo/fireflies/ChurnTest.java | 6 +- .../salesforce/apollo/fireflies/E2ETest.java | 18 +- .../salesforce/apollo/fireflies/MtlsTest.java | 11 +- .../apollo/fireflies/SwarmTest.java | 22 +- .../client/GorgoneionClientTest.java | 21 +- .../apollo/gorgoneion/Gorgoneion.java | 95 ++-- .../apollo/gorgoneion/Parameters.java | 19 +- .../apollo/gorgoneion/GorgoneionTest.java | 32 +- grpc/src/main/proto/fireflies.proto | 14 +- grpc/src/main/proto/leyden.proto | 2 - .../apollo/demesnes/FireFliesTrace.java | 22 +- .../apollo/archipelago/MtlsServer.java | 9 +- .../salesforce/apollo/membership/Context.java | 18 +- .../apollo/membership/ContextImpl.java | 27 +- .../messaging/rbc/ReliableBroadcaster.java | 21 +- .../apollo/ring/RingCommunications.java | 8 +- .../salesforce/apollo/ring/RingIterator.java | 10 +- .../membership/messaging/rbc/RbcTest.java | 5 +- .../com/salesforce/apollo/model/Domain.java | 3 - .../apollo/model/ProcessContainerDomain.java | 34 +- .../apollo/model/ProcessDomain.java | 25 +- .../apollo/model/demesnes/DemesneImpl.java | 2 +- .../apollo/model/ContainmentDomainTest.java | 7 +- .../salesforce/apollo/model/DomainTest.java | 9 +- .../apollo/model/FireFliesTest.java | 25 +- pom.xml | 10 + .../apollo/comm/grpc/MtlsClient.java | 17 + stereotomy/pom.xml | 2 +- .../stereotomy/ControlledIdentifier.java | 2 +- .../apollo/stereotomy/EventValidation.java | 55 +-- .../salesforce/apollo/stereotomy/KERL.java | 40 +- .../apollo/stereotomy/KeyCoordinates.java | 10 +- .../apollo/stereotomy/Stereotomy.java | 12 +- .../apollo/stereotomy/StereotomyImpl.java | 52 +-- .../apollo/stereotomy/StereotomyKeyStore.java | 13 +- .../stereotomy/StereotomyValidator.java | 14 +- .../apollo/stereotomy/Verifiers.java | 30 +- .../apollo/stereotomy/caching/CachingKEL.java | 13 +- .../apollo/stereotomy/db/UniKERL.java | 27 +- .../apollo/stereotomy/db/UniKERLDirect.java | 12 +- .../stereotomy/event/AttachmentEvent.java | 22 +- .../event/DelegatedInceptionEvent.java | 5 +- ...eyStateWithEndorsementsAndValidations.java | 35 +- .../identifier/SelfAddressingIdentifier.java | 10 +- .../apollo/stereotomy/jks/FileKeyStore.java | 34 +- .../apollo/stereotomy/jks/JksKeyStore.java | 103 +++-- .../apollo/stereotomy/mem/MemKeyStore.java | 27 +- .../processing/KeyEventVerifier.java | 4 +- .../processing/KeyStateProcessor.java | 15 +- .../stereotomy/processing/Validator.java | 72 +-- .../services/proto/ProtoEventObserver.java | 5 +- .../services/proto/ProtoKERLAdapter.java | 20 +- .../apollo/stereotomy/FileKeyStoreTest.java | 7 +- .../apollo/stereotomy/JksKeyStoreTest.java | 7 +- .../apollo/stereotomy/StereotomyTests.java | 6 +- .../java/com/salesforce/apollo/thoth/Ani.java | 23 +- .../apollo/thoth/CombinedIntervals.java | 24 +- .../apollo/thoth/DirectPublisher.java | 21 +- .../com/salesforce/apollo/thoth/KerlDHT.java | 422 +++++++++--------- .../salesforce/apollo/thoth/KerlSpace.java | 55 +-- .../com/salesforce/apollo/thoth/Maat.java | 4 +- .../apollo/thoth/grpc/dht/DhtClient.java | 8 +- .../apollo/thoth/grpc/dht/DhtServer.java | 37 +- .../apollo/thoth/AbstractDhtTest.java | 7 +- .../apollo/thoth/BootstrappingTest.java | 9 +- .../apollo/thoth/KerlSpaceTest.java | 8 +- 113 files changed, 1713 insertions(+), 2045 deletions(-) create mode 100644 cryptography/src/main/java/com/salesforce/apollo/cryptography/EncryptionAlgorithm.java create mode 100644 cryptography/src/test/java/com/salesforce/apollo/cryptography/XTest.java delete mode 100644 fireflies/src/main/java/com/salesforce/apollo/fireflies/Bootstrapper.java diff --git a/choam/src/main/java/com/salesforce/apollo/choam/CHOAM.java b/choam/src/main/java/com/salesforce/apollo/choam/CHOAM.java index d004baddf7..09fde0e528 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/CHOAM.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/CHOAM.java @@ -142,7 +142,7 @@ public static Checkpoint checkpoint(DigestAlgorithm algo, File state, int segmen length = state.length(); } int count = (int) (length / segmentSize); - if (length != 0 && count * segmentSize < length) { + if (length != 0 && (long) count * segmentSize < length) { count++; } var accumulator = new HexBloom.HexAccumulator(count, crowns, initial); @@ -190,15 +190,8 @@ public static Digest hashOf(Transaction transaction, DigestAlgorithm digestAlgor } public static String print(Join join, DigestAlgorithm da) { - StringBuilder builder = new StringBuilder(); - builder.append("J[view: ") - .append(Digest.from(join.getView())) - .append(" member: ") - .append(ViewContext.print(join.getMember(), da)) - .append("certifications: ") - .append(join.getEndorsementsList().stream().map(c -> ViewContext.print(c, da)).toList()) - .append("]"); - return builder.toString(); + return "J[view: " + Digest.from(join.getView()) + " member: " + ViewContext.print(join.getMember(), da) + + "certifications: " + join.getEndorsementsList().stream().map(c -> ViewContext.print(c, da)).toList() + "]"; } public static Reconfigure reconfigure(Digest nextViewId, Map joins, Context context, @@ -208,7 +201,7 @@ public static Reconfigure reconfigure(Digest nextViewId, Map joins // Canonical labeling of the view members for Ethereal var remapped = rosterMap(context, joins.keySet()); - remapped.keySet().stream().sorted().map(d -> remapped.get(d)).forEach(m -> builder.addJoins(joins.get(m))); + remapped.keySet().stream().sorted().map(remapped::get).forEach(m -> builder.addJoins(joins.get(m))); var reconfigure = builder.build(); return reconfigure; @@ -234,7 +227,7 @@ public static Map rosterMap(Context baseContext, Collect // Canonical labeling of the view members for Ethereal var ring0 = baseContext.ring(0); - return members.stream().collect(Collectors.toMap(m -> ring0.hash(m), m -> m)); + return members.stream().collect(Collectors.toMap(ring0::hash, m -> m)); } public static List toGenesisData(List initializationData) { @@ -444,7 +437,7 @@ private void combine() { } private void combine(List messages) { - messages.forEach(m -> combine(m)); + messages.forEach(this::combine); transitions.combine(); } @@ -600,10 +593,7 @@ private boolean isNext(HashedBlock next) { return true; } final Digest prev = next.getPrevious(); - if (h.hash.equals(prev)) { - return true; - } - return false; + return h.hash.equals(prev); } private ViewMember join(Digest nextView, Digest from) { @@ -785,7 +775,7 @@ private Digest signatureHash(ByteString any) { .stream() .map(cert -> JohnHancock.from(cert.getSignature())) .map(sig -> sig.toDigest(params.digestAlgorithm())) - .reduce(Digest.from(cb.getBlock().getHeader().getBodyHash()), (a, b) -> a.xor(b)); + .reduce(Digest.from(cb.getBlock().getHeader().getBodyHash()), Digest::xor); } /** @@ -875,7 +865,7 @@ private void synchronize(SynchronizedState state) { state.lastCheckpoint != null ? state.lastCheckpoint.hash : state.genesis.hash, pending.size(), params.member().getId()); try { - linear.execute(() -> transitions.regenerated()); + linear.execute(transitions::regenerated); } catch (RejectedExecutionException e) { // ignore } @@ -982,7 +972,6 @@ public void anchor() { if (anchor != null) { log.info("Synchronizing from anchor: {} on: {}", anchor.hash, params.member().getId()); transitions.bootstrap(anchor); - return; } } @@ -1161,10 +1150,6 @@ public SubmitResult submitTxn(Transaction transaction) { log.debug("No link for: {} for submitting txn on: {}", target.getId(), params.member().getId()); return SubmitResult.newBuilder().setResult(Result.UNAVAILABLE).build(); } - // if (log.isTraceEnabled()) { - // log.trace("Submitting received txn: {} to: {} in: {} on: {}", - // hashOf(transaction, params.digestAlgorithm()), target.getId(), viewId, params.member().getId()); - // } return link.submit(transaction); } catch (StatusRuntimeException e) { log.trace("Failed submitting txn: {} status:{} to: {} in: {} on: {}", @@ -1191,8 +1176,7 @@ public boolean validate(HashedCertifiedBlock hb) { /** a member of the current committee */ private class Associate extends Administration { - private final Producer producer; - private final ViewContext viewContext; + private final Producer producer; Associate(HashedCertifiedBlock viewChange, Map validators, nextView nextView) { super(validators, new Digest( @@ -1204,8 +1188,8 @@ private class Associate extends Administration { params.digestAlgorithm().digest(nextView.member.getSignature().toByteString()), viewId, params.member().getId()); Signer signer = new SignerImpl(nextView.consensusKeyPair.getPrivate(), ULong.MIN); - viewContext = new ViewContext(context, params, signer, validators, constructBlock()); - producer = new Producer(viewContext, head.get(), checkpoint.get(), comm, getLabel()); + producer = new Producer(new ViewContext(context, params, signer, validators, constructBlock()), head.get(), + checkpoint.get(), comm, getLabel()); producer.start(); } @@ -1221,8 +1205,6 @@ public void complete() { @Override public SubmitResult submit(Transaction request) { - // log.trace("Submit txn: {} to producer on: {}", hashOf(request.getTransaction(), params.digestAlgorithm()), - // params().member()); return producer.submit(request); } } diff --git a/choam/src/main/java/com/salesforce/apollo/choam/Committee.java b/choam/src/main/java/com/salesforce/apollo/choam/Committee.java index 7b6b67ebe9..acd13df049 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/Committee.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/Committee.java @@ -6,21 +6,8 @@ */ package com.salesforce.apollo.choam; -import static com.salesforce.apollo.cryptography.QualifiedBase64.publicKey; - -import java.util.HashSet; -import java.util.Map; -import java.util.Set; -import java.util.stream.Collectors; - -import org.slf4j.Logger; - -import com.salesforce.apollo.choam.proto.Certification; -import com.salesforce.apollo.choam.proto.Reconfigure; -import com.salesforce.apollo.choam.proto.SubmitResult; +import com.salesforce.apollo.choam.proto.*; import com.salesforce.apollo.choam.proto.SubmitResult.Result; -import com.salesforce.apollo.choam.proto.Transaction; -import com.salesforce.apollo.choam.proto.ViewMember; import com.salesforce.apollo.choam.support.HashedCertifiedBlock; import com.salesforce.apollo.cryptography.Digest; import com.salesforce.apollo.cryptography.DigestAlgorithm; @@ -30,6 +17,14 @@ import com.salesforce.apollo.membership.Context; import com.salesforce.apollo.membership.ContextImpl; import com.salesforce.apollo.membership.Member; +import org.slf4j.Logger; + +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import static com.salesforce.apollo.cryptography.QualifiedBase64.publicKey; /** * @author hal.hildebrand diff --git a/choam/src/main/java/com/salesforce/apollo/choam/GenesisAssembly.java b/choam/src/main/java/com/salesforce/apollo/choam/GenesisAssembly.java index 85b46c08c3..5dc427c978 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/GenesisAssembly.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/GenesisAssembly.java @@ -9,16 +9,16 @@ import com.chiralbehaviors.tron.Fsm; import com.google.protobuf.ByteString; import com.google.protobuf.InvalidProtocolBufferException; -import com.salesforce.apollo.choam.proto.*; -import com.salesforce.apollo.cryptography.proto.PubKey; import com.salesforce.apollo.archipelago.RouterImpl.CommonCommunications; import com.salesforce.apollo.choam.comm.Terminal; import com.salesforce.apollo.choam.fsm.Genesis; +import com.salesforce.apollo.choam.proto.*; import com.salesforce.apollo.choam.support.HashedBlock; import com.salesforce.apollo.choam.support.HashedCertifiedBlock; import com.salesforce.apollo.choam.support.HashedCertifiedBlock.NullBlock; import com.salesforce.apollo.choam.support.OneShot; import com.salesforce.apollo.cryptography.Digest; +import com.salesforce.apollo.cryptography.proto.PubKey; import com.salesforce.apollo.ethereal.Config; import com.salesforce.apollo.ethereal.Dag; import com.salesforce.apollo.ethereal.DataSource; @@ -31,10 +31,7 @@ import org.slf4j.LoggerFactory; import java.security.PublicKey; -import java.util.Comparator; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; @@ -70,7 +67,7 @@ public GenesisAssembly(ViewContext vc, CommonCommunications comms, ds = new OneShot(); nextAssembly = Committee.viewMembersOf(view.context().getId(), params().context()) .stream() - .collect(Collectors.toMap(m -> m.getId(), m -> m)); + .collect(Collectors.toMap(Member::getId, m -> m)); if (!Dag.validate(nextAssembly.size())) { log.error("Invalid cardinality: {} for: {} on: {}", nextAssembly.size(), view.context().getId(), params().member().getId()); @@ -80,9 +77,9 @@ public GenesisAssembly(ViewContext vc, CommonCommunications comms, // Create a new context for reconfiguration final Digest reconPrefixed = view.context().getId().prefix("Genesis Assembly"); - Context reContext = new ContextImpl(reconPrefixed, view.context().memberCount(), - view.context().getProbabilityByzantine(), - view.context().getBias()); + Context reContext = new ContextImpl<>(reconPrefixed, view.context().memberCount(), + view.context().getProbabilityByzantine(), + view.context().getBias()); reContext.activate(view.context().activeMembers()); final Fsm fsm = Fsm.construct(this, Transitions.class, BrickLayer.INITIAL, true); @@ -101,8 +98,7 @@ public GenesisAssembly(ViewContext vc, CommonCommunications comms, config.setEpochLength(7).setNumberOfEpochs(3); config.setLabel("Genesis Assembly" + view.context().getId() + " on: " + params().member().getId()); controller = new Ethereal(config.build(), params().producer().maxBatchByteSize(), dataSource(), - (preblock, last) -> transitions.process(preblock, last), - epoch -> transitions.nextEpoch(epoch), label); + transitions::process, transitions::nextEpoch, label); coordinator = new ChRbcGossip(reContext, params().member(), controller.processor(), params().communications(), params().metrics() == null ? null : params().metrics().getGensisMetrics()); log.debug("Genesis Assembly: {} recontext: {} next assembly: {} on: {}", view.context().getId(), @@ -133,7 +129,7 @@ public void certify(List preblock, boolean last) { } catch (InvalidProtocolBufferException e) { return null; } - }).filter(v -> v != null).filter(v -> !v.equals(Validate.getDefaultInstance())).forEach(v -> certify(v)); + }).filter(Objects::nonNull).filter(v -> !v.equals(Validate.getDefaultInstance())).forEach(this::certify); } @Override @@ -162,7 +158,7 @@ public void gather(List preblock, boolean last) { } catch (InvalidProtocolBufferException e) { return null; } - }).filter(j -> j != null).filter(j -> !j.equals(Join.getDefaultInstance())).forEach(j -> join(j)); + }).filter(Objects::nonNull).filter(j -> !j.equals(Join.getDefaultInstance())).forEach(this::join); } @Override @@ -173,7 +169,7 @@ public void nominate() { .stream() .filter(p -> !p.member.equals(params().member())) .map(p -> view.generateValidation(p.join.getMember())) - .forEach(v -> validations.addValidations(v)); + .forEach(validations::addValidations); ds.setValue(validations.build().toByteString()); } @@ -187,10 +183,10 @@ public void nominations(List preblock, boolean last) { return null; } }) - .filter(v -> v != null) + .filter(Objects::nonNull) .flatMap(vs -> vs.getValidationsList().stream()) .filter(v -> !v.equals(Validate.getDefaultInstance())) - .forEach(v -> validate(v)); + .forEach(this::validate); } @Override @@ -199,7 +195,7 @@ public void publish() { witnesses.entrySet() .stream() .sorted(Comparator.comparing(e -> e.getKey().getId())) - .map(e -> e.getValue()) + .map(Map.Entry::getValue) .forEach(v -> b.addCertifications(v.getWitness())); view.publish(new HashedCertifiedBlock(params().digestAlgorithm(), b.build())); log.debug("Genesis block: {} published for: {} on: {}", reconfiguration.hash, view.context().getId(), @@ -247,19 +243,16 @@ private void certify(Validate v) { } private DataSource dataSource() { - return new DataSource() { - @Override - public ByteString getData() { - if (!started.get()) { - return ByteString.EMPTY; - } - try { - blockingThread = Thread.currentThread(); - final var take = ds.get(); - return take; - } finally { - blockingThread = null; - } + return () -> { + if (!started.get()) { + return ByteString.EMPTY; + } + try { + blockingThread = Thread.currentThread(); + final var take = ds.get(); + return take; + } finally { + blockingThread = null; } }; } diff --git a/choam/src/main/java/com/salesforce/apollo/choam/GenesisContext.java b/choam/src/main/java/com/salesforce/apollo/choam/GenesisContext.java index 8d7f4c801a..fc8d5bca94 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/GenesisContext.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/GenesisContext.java @@ -6,15 +6,15 @@ */ package com.salesforce.apollo.choam; -import java.util.Collections; - -import com.salesforce.apollo.choam.proto.Validate; import com.salesforce.apollo.choam.CHOAM.BlockProducer; +import com.salesforce.apollo.choam.proto.Validate; import com.salesforce.apollo.cryptography.Signer; import com.salesforce.apollo.cryptography.Verifier; import com.salesforce.apollo.membership.Context; import com.salesforce.apollo.membership.Member; +import java.util.Collections; + /** * @author hal.hildebrand */ diff --git a/choam/src/main/java/com/salesforce/apollo/choam/Parameters.java b/choam/src/main/java/com/salesforce/apollo/choam/Parameters.java index 1cc21136f2..8e2c6f3f3d 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/Parameters.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/Parameters.java @@ -11,12 +11,11 @@ import com.netflix.concurrency.limits.limit.AIMDLimit; import com.netflix.concurrency.limits.limiter.LifoBlockingLimiter; import com.netflix.concurrency.limits.limiter.SimpleLimiter; +import com.salesforce.apollo.archipelago.Router; +import com.salesforce.apollo.choam.CHOAM.TransactionExecutor; import com.salesforce.apollo.choam.proto.FoundationSeal; import com.salesforce.apollo.choam.proto.Join; import com.salesforce.apollo.choam.proto.Transaction; -import com.salesforce.apollo.stereotomy.event.proto.KERL_; -import com.salesforce.apollo.archipelago.Router; -import com.salesforce.apollo.choam.CHOAM.TransactionExecutor; import com.salesforce.apollo.choam.support.CheckpointState; import com.salesforce.apollo.choam.support.ChoamMetrics; import com.salesforce.apollo.choam.support.ExponentialBackoffPolicy; @@ -29,6 +28,7 @@ import com.salesforce.apollo.membership.Member; import com.salesforce.apollo.membership.SigningMember; import com.salesforce.apollo.membership.messaging.rbc.ReliableBroadcaster; +import com.salesforce.apollo.stereotomy.event.proto.KERL_; import org.h2.mvstore.MVStore; import org.h2.mvstore.OffHeapStore; import org.joou.ULong; @@ -574,7 +574,7 @@ public Limiter build(String name, MetricRegistry metrics) { .backoffRatio(backoffRatio) .build()) .build(); - return LifoBlockingLimiter.newBuilder(limiter) + return LifoBlockingLimiter.newBuilder(limiter) .backlogSize(backlogSize) .backlogTimeout(backlogDuration) .build(); diff --git a/choam/src/main/java/com/salesforce/apollo/choam/Producer.java b/choam/src/main/java/com/salesforce/apollo/choam/Producer.java index 422894277a..b918b40c8a 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/Producer.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/Producer.java @@ -6,38 +6,16 @@ */ package com.salesforce.apollo.choam; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.*; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicReference; - -import com.google.protobuf.ByteString; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import com.chiralbehaviors.tron.Fsm; +import com.google.protobuf.ByteString; import com.google.protobuf.InvalidProtocolBufferException; -import com.salesforce.apollo.choam.proto.Assemble; -import com.salesforce.apollo.choam.proto.Block; -import com.salesforce.apollo.choam.proto.CertifiedBlock; -import com.salesforce.apollo.choam.proto.Executions; -import com.salesforce.apollo.choam.proto.Reassemble; -import com.salesforce.apollo.choam.proto.SubmitResult; -import com.salesforce.apollo.choam.proto.SubmitResult.Result; -import com.salesforce.apollo.choam.proto.Transaction; -import com.salesforce.apollo.choam.proto.UnitData; -import com.salesforce.apollo.choam.proto.Validate; import com.salesforce.apollo.archipelago.RouterImpl.CommonCommunications; import com.salesforce.apollo.choam.comm.Terminal; import com.salesforce.apollo.choam.fsm.Driven; import com.salesforce.apollo.choam.fsm.Driven.Earner; import com.salesforce.apollo.choam.fsm.Driven.Transitions; +import com.salesforce.apollo.choam.proto.*; +import com.salesforce.apollo.choam.proto.SubmitResult.Result; import com.salesforce.apollo.choam.support.HashedBlock; import com.salesforce.apollo.choam.support.HashedCertifiedBlock; import com.salesforce.apollo.choam.support.TxDataSource; @@ -47,6 +25,15 @@ import com.salesforce.apollo.ethereal.Ethereal; import com.salesforce.apollo.ethereal.memberships.ChRbcGossip; import com.salesforce.apollo.membership.Member; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.*; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; /** * An "Earner" @@ -55,23 +42,24 @@ */ public class Producer { - private static final Logger log = LoggerFactory.getLogger(Producer.class); - private final AtomicBoolean assembled = new AtomicBoolean(); - private final AtomicReference assembly = new AtomicReference<>(); - private final AtomicReference checkpoint = new AtomicReference<>(); - private final CommonCommunications comms; - private final Ethereal controller; - private final ChRbcGossip coordinator; - private final TxDataSource ds; - private final int lastEpoch; - private final Set nextAssembly = new HashSet<>(); - private final Map pending = new ConcurrentHashMap<>(); - private final BlockingQueue pendingReassembles = new LinkedBlockingQueue<>(); - private final AtomicReference previousBlock = new AtomicReference<>(); - private final AtomicBoolean started = new AtomicBoolean(false); - private final Transitions transitions; - private final ViewContext view; - private volatile Digest nextViewId; + private static final Logger log = LoggerFactory.getLogger(Producer.class); + private final AtomicBoolean assembled = new AtomicBoolean(); + private final AtomicReference assembly = new AtomicReference<>(); + private final AtomicReference checkpoint = new AtomicReference<>(); + private final CommonCommunications comms; + private final Ethereal controller; + private final ChRbcGossip coordinator; + private final TxDataSource ds; + private final int lastEpoch; + private final Set nextAssembly = new HashSet<>(); + private final Map pending = new ConcurrentHashMap<>(); + private final BlockingQueue pendingReassembles = new LinkedBlockingQueue<>(); + private final AtomicReference previousBlock = new AtomicReference<>(); + private final AtomicBoolean started = new AtomicBoolean(false); + private final Transitions transitions; + private final ViewContext view; + private volatile Digest nextViewId; + public Producer(ViewContext view, HashedBlock lastBlock, HashedBlock checkpoint, CommonCommunications comms, String label) { assert view != null; @@ -115,8 +103,7 @@ public Producer(ViewContext view, HashedBlock lastBlock, HashedBlock checkpoint, config.setLabel("Producer" + getViewId() + " on: " + params().member().getId()); var producerMetrics = params().metrics() == null ? null : params().metrics().getProducerMetrics(); controller = new Ethereal(config.build(), params().producer().maxBatchByteSize() + (8 * 1024), ds, - (preblock, last) -> transitions.create(preblock, last), epoch -> newEpoch(epoch), - label); + transitions::create, this::newEpoch, label); coordinator = new ChRbcGossip(view.context(), params().member(), controller.processor(), params().communications(), producerMetrics); log.debug("Roster for: {} is: {} on: {}", getViewId(), view.roster(), params().member().getId()); @@ -179,22 +166,22 @@ private void create(List preblock, boolean last) { return UnitData.parseFrom(e); } catch (InvalidProtocolBufferException ex) { log.error("Error parsing unit data on: {}", params().member().getId()); - return (UnitData) null; + return null; } - }).filter(e -> e != null).toList(); + }).filter(Objects::nonNull).toList(); aggregate.stream() .flatMap(e -> e.getValidationsList().stream()) - .map(witness -> validate(witness)) - .filter(p -> p != null) + .map(this::validate) + .filter(Objects::nonNull) .filter(p -> !p.published.get()) .filter(p -> p.witnesses.size() >= params().majority()) - .forEach(p -> publish(p)); + .forEach(this::publish); var reass = Reassemble.newBuilder(); - aggregate.stream().flatMap(e -> e.getReassembliesList().stream()).forEach(r -> { - reass.addAllMembers(r.getMembersList()).addAllValidations(r.getValidationsList()); - }); + aggregate.stream() + .flatMap(e -> e.getReassembliesList().stream()) + .forEach(r -> reass.addAllMembers(r.getMembersList()).addAllValidations(r.getValidationsList())); if (reass.getMembersCount() > 0 || reass.getValidationsCount() > 0) { final var ass = assembly.get(); if (ass != null) { @@ -215,11 +202,11 @@ private void create(List preblock, boolean last) { log.trace("transactions: {} comb hash: {} height: {} on: {}", txns.size(), txns.stream() .map(t -> CHOAM.hashOf(t, params().digestAlgorithm())) - .reduce((a, b) -> a.xor(b)) + .reduce(Digest::xor) .orElse(null), lb.height().add(1), params().member().getId()); var builder = Executions.newBuilder(); - txns.forEach(e -> builder.addExecutions(e)); + txns.forEach(builder::addExecutions); var next = new HashedBlock(params().digestAlgorithm(), view.produce(lb.height().add(1), lb.hash, builder.build(), checkpoint.get())); @@ -273,16 +260,13 @@ private void produceAssemble() { } private void publish(PendingBlock p) { - // assert previousBlock.get().hash.equals(Digest.from(p.block.block.getHeader().getPrevious())) : "Pending block: " - // + p.block.hash + " previous: " + Digest.from(p.block.block.getHeader().getPrevious()) + " is not: " - // + previousBlock.get().hash; log.debug("Published pending: {} height: {} on: {}", p.block.hash, p.block.height(), params().member().getId()); p.published.set(true); pending.remove(p.block.hash); final var cb = CertifiedBlock.newBuilder() .setBlock(p.block.block) .addAllCertifications( - p.witnesses.values().stream().map(v -> v.getWitness()).toList()) + p.witnesses.values().stream().map(Validate::getWitness).toList()) .build(); view.publish(new HashedCertifiedBlock(params().digestAlgorithm(), cb)); } @@ -385,7 +369,7 @@ public void produceAssemble() { @Override public void reconfigure() { log.debug("Starting view reconfiguration for: {} on: {}", nextViewId, params().member().getId()); - assembly.set(new ViewAssembly(nextViewId, view, r -> addReassemble(r), comms) { + assembly.set(new ViewAssembly(nextViewId, view, Producer.this::addReassemble, comms) { @Override public void complete() { super.complete(); diff --git a/choam/src/main/java/com/salesforce/apollo/choam/ViewAssembly.java b/choam/src/main/java/com/salesforce/apollo/choam/ViewAssembly.java index c7cf17ad99..c4bf544c8c 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/ViewAssembly.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/ViewAssembly.java @@ -64,7 +64,7 @@ public ViewAssembly(Digest nextViewId, ViewContext vc, Consumer publ this.publisher = publisher; nextAssembly = Committee.viewMembersOf(nextViewId, params().context()) .stream() - .collect(Collectors.toMap(m -> m.getId(), m -> m)); + .collect(Collectors.toMap(Member::getId, m -> m)); var slice = new ArrayList<>(nextAssembly.values()); committee = new SliceIterator("Committee for " + nextViewId, params().member(), slice, comms); @@ -126,14 +126,14 @@ Consumer> inbound() { return lre -> { lre.stream() .flatMap(re -> re.getMembersList().stream()) - .map(e -> join(e)) - .filter(r -> r != null) + .map(this::join) + .filter(Objects::nonNull) .reduce((a, b) -> Reassemble.newBuilder(a) .addAllMembers(b.getMembersList()) .addAllValidations(b.getValidationsList()) .build()) .ifPresent(publisher); - lre.stream().flatMap(re -> re.getValidationsList().stream()).forEach(e -> validate(e)); + lre.stream().flatMap(re -> re.getValidationsList().stream()).forEach(this::validate); }; } @@ -144,7 +144,7 @@ private void completeSlice(AtomicReference retryDelay, AtomicReference final var delay = retryDelay.get(); if (delay.compareTo(params().producer().maxGossipDelay()) < 0) { - retryDelay.accumulateAndGet(Duration.ofMillis(100), (a, b) -> a.plus(b)); + retryDelay.accumulateAndGet(Duration.ofMillis(100), Duration::plus); } log.trace("Proposal incomplete of: {} gathered: {} desired: {}, retrying: {} on: {}", nextViewId, @@ -232,7 +232,7 @@ private Reassemble join(ViewMember vm) { } var validations = unassigned.remove(mid); if (validations != null) { - validations.forEach(v -> validate(v)); + validations.forEach(this::validate); } if (proposals.size() == nextAssembly.size()) { transitions.gathered(); @@ -248,7 +248,7 @@ private Reassemble join(ViewMember vm) { private Join joinOf(Proposed candidate) { final List witnesses = candidate.validations.values() .stream() - .map(v -> v.getWitness()) + .map(Validate::getWitness) .sorted( Comparator.comparing(c -> new Digest(c.getId()))) .collect(Collectors.toList()); @@ -299,9 +299,6 @@ private void validate(Validate v) { } } - record AJoin(Member m, Join j) { - } - private record Proposed(ViewMember vm, Member member, Map validations) { } @@ -364,7 +361,7 @@ public void gather() { } log.trace("Requesting Join from: {} on: {}", term.getMember().getId(), params().member().getId()); return term.join(nextViewId); - }, (futureSailor, term, m) -> consider(futureSailor, term, m), () -> completeSlice(retryDelay, reiterate), + }, ViewAssembly.this::consider, () -> completeSlice(retryDelay, reiterate), Executors.newScheduledThreadPool(1, Thread.ofVirtual().factory()), params().gossipDuration())); reiterate.get().run(); diff --git a/choam/src/main/java/com/salesforce/apollo/choam/ViewContext.java b/choam/src/main/java/com/salesforce/apollo/choam/ViewContext.java index 51986e6dd3..8800c1d729 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/ViewContext.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/ViewContext.java @@ -6,45 +6,35 @@ */ package com.salesforce.apollo.choam; -import static com.salesforce.apollo.cryptography.QualifiedBase64.publicKey; - -import java.util.HashMap; -import java.util.Map; - -import org.joou.ULong; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.salesforce.apollo.choam.proto.Assemble; -import com.salesforce.apollo.choam.proto.Block; -import com.salesforce.apollo.choam.proto.Certification; -import com.salesforce.apollo.choam.proto.Executions; -import com.salesforce.apollo.choam.proto.Join; -import com.salesforce.apollo.choam.proto.Validate; -import com.salesforce.apollo.choam.proto.ViewMember; import com.salesforce.apollo.choam.CHOAM.BlockProducer; +import com.salesforce.apollo.choam.proto.*; import com.salesforce.apollo.choam.support.HashedBlock; import com.salesforce.apollo.choam.support.HashedCertifiedBlock; -import com.salesforce.apollo.cryptography.Digest; -import com.salesforce.apollo.cryptography.DigestAlgorithm; -import com.salesforce.apollo.cryptography.JohnHancock; -import com.salesforce.apollo.cryptography.Signer; -import com.salesforce.apollo.cryptography.Verifier; +import com.salesforce.apollo.cryptography.*; import com.salesforce.apollo.membership.Context; import com.salesforce.apollo.membership.Member; +import org.joou.ULong; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.HashMap; +import java.util.Map; + +import static com.salesforce.apollo.cryptography.QualifiedBase64.publicKey; /** * @author hal.hildebrand */ public class ViewContext { - private final static Logger log = LoggerFactory.getLogger(ViewContext.class); - private final BlockProducer blockProducer; - private final Context context; - private final Parameters params; - private final Map roster; - private final Signer signer; - private final Map validators; + private final static Logger log = LoggerFactory.getLogger(ViewContext.class); + private final BlockProducer blockProducer; + private final Context context; + private final Parameters params; + private final Map roster; + private final Signer signer; + private final Map validators; + public ViewContext(Context context, Parameters params, Signer signer, Map validators, BlockProducer blockProducer) { this.blockProducer = blockProducer; @@ -158,8 +148,8 @@ public Map roster() { public boolean validate(HashedBlock block, Validate validate) { Verifier v = verifierOf(validate); - return v == null ? false : v.verify(JohnHancock.from(validate.getWitness().getSignature()), - block.block.getHeader().toByteString()); + return v != null && v.verify(JohnHancock.from(validate.getWitness().getSignature()), + block.block.getHeader().toByteString()); } public boolean validate(ViewMember vm, Validate validate) { diff --git a/choam/src/main/java/com/salesforce/apollo/choam/comm/Concierge.java b/choam/src/main/java/com/salesforce/apollo/choam/comm/Concierge.java index b9d0f2c4fa..099e8ccbac 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/comm/Concierge.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/comm/Concierge.java @@ -6,13 +6,7 @@ */ package com.salesforce.apollo.choam.comm; -import com.salesforce.apollo.choam.proto.BlockReplication; -import com.salesforce.apollo.choam.proto.Blocks; -import com.salesforce.apollo.choam.proto.CheckpointReplication; -import com.salesforce.apollo.choam.proto.CheckpointSegments; -import com.salesforce.apollo.choam.proto.Initial; -import com.salesforce.apollo.choam.proto.Synchronize; -import com.salesforce.apollo.choam.proto.ViewMember; +import com.salesforce.apollo.choam.proto.*; import com.salesforce.apollo.cryptography.Digest; /** diff --git a/choam/src/main/java/com/salesforce/apollo/choam/comm/Terminal.java b/choam/src/main/java/com/salesforce/apollo/choam/comm/Terminal.java index e01ac89e24..4fa4187ee6 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/comm/Terminal.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/comm/Terminal.java @@ -6,8 +6,8 @@ */ package com.salesforce.apollo.choam.comm; -import com.salesforce.apollo.choam.proto.*; import com.salesforce.apollo.archipelago.Link; +import com.salesforce.apollo.choam.proto.*; import com.salesforce.apollo.cryptography.Digest; import com.salesforce.apollo.membership.Member; import com.salesforce.apollo.membership.SigningMember; diff --git a/choam/src/main/java/com/salesforce/apollo/choam/comm/TerminalClient.java b/choam/src/main/java/com/salesforce/apollo/choam/comm/TerminalClient.java index fb766960e3..b8486d50dd 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/comm/TerminalClient.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/comm/TerminalClient.java @@ -6,9 +6,9 @@ */ package com.salesforce.apollo.choam.comm; -import com.salesforce.apollo.choam.proto.*; import com.salesforce.apollo.archipelago.ManagedServerChannel; import com.salesforce.apollo.archipelago.ServerConnectionCache.CreateClientCommunications; +import com.salesforce.apollo.choam.proto.*; import com.salesforce.apollo.choam.support.ChoamMetrics; import com.salesforce.apollo.cryptography.Digest; import com.salesforce.apollo.membership.Member; diff --git a/choam/src/main/java/com/salesforce/apollo/choam/comm/TerminalServer.java b/choam/src/main/java/com/salesforce/apollo/choam/comm/TerminalServer.java index 0ea30e1694..b780b0d607 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/comm/TerminalServer.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/comm/TerminalServer.java @@ -6,20 +6,13 @@ */ package com.salesforce.apollo.choam.comm; -import com.salesforce.apollo.choam.proto.BlockReplication; -import com.salesforce.apollo.choam.proto.Blocks; -import com.salesforce.apollo.choam.proto.CheckpointReplication; -import com.salesforce.apollo.choam.proto.CheckpointSegments; -import com.salesforce.apollo.choam.proto.Initial; -import com.salesforce.apollo.choam.proto.Synchronize; -import com.salesforce.apollo.choam.proto.TerminalGrpc.TerminalImplBase; -import com.salesforce.apollo.choam.proto.ViewMember; -import com.salesforce.apollo.cryptography.proto.Digeste; import com.salesforce.apollo.archipelago.RoutableService; +import com.salesforce.apollo.choam.proto.*; +import com.salesforce.apollo.choam.proto.TerminalGrpc.TerminalImplBase; import com.salesforce.apollo.choam.support.ChoamMetrics; import com.salesforce.apollo.cryptography.Digest; +import com.salesforce.apollo.cryptography.proto.Digeste; import com.salesforce.apollo.protocols.ClientIdentity; - import io.grpc.stub.StreamObserver; /** @@ -29,7 +22,7 @@ public class TerminalServer extends TerminalImplBase { @SuppressWarnings("unused") private final ChoamMetrics metrics; private final RoutableService router; - private ClientIdentity identity; + private final ClientIdentity identity; public TerminalServer(ClientIdentity identity, ChoamMetrics metrics, RoutableService router) { this.metrics = metrics; diff --git a/choam/src/main/java/com/salesforce/apollo/choam/comm/TxnSubmission.java b/choam/src/main/java/com/salesforce/apollo/choam/comm/TxnSubmission.java index 64e07d9dc9..d65cf9a497 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/comm/TxnSubmission.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/comm/TxnSubmission.java @@ -6,14 +6,14 @@ */ package com.salesforce.apollo.choam.comm; -import java.io.IOException; - +import com.salesforce.apollo.archipelago.Link; import com.salesforce.apollo.choam.proto.SubmitResult; import com.salesforce.apollo.choam.proto.Transaction; -import com.salesforce.apollo.archipelago.Link; import com.salesforce.apollo.membership.Member; import com.salesforce.apollo.membership.SigningMember; +import java.io.IOException; + /** * @author hal.hildebrand */ diff --git a/choam/src/main/java/com/salesforce/apollo/choam/comm/TxnSubmitClient.java b/choam/src/main/java/com/salesforce/apollo/choam/comm/TxnSubmitClient.java index 71c2daa33c..81ae0786af 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/comm/TxnSubmitClient.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/comm/TxnSubmitClient.java @@ -6,12 +6,12 @@ */ package com.salesforce.apollo.choam.comm; +import com.salesforce.apollo.archipelago.ManagedServerChannel; +import com.salesforce.apollo.archipelago.ServerConnectionCache.CreateClientCommunications; import com.salesforce.apollo.choam.proto.SubmitResult; import com.salesforce.apollo.choam.proto.Transaction; import com.salesforce.apollo.choam.proto.TransactionSubmissionGrpc; import com.salesforce.apollo.choam.proto.TransactionSubmissionGrpc.TransactionSubmissionBlockingStub; -import com.salesforce.apollo.archipelago.ManagedServerChannel; -import com.salesforce.apollo.archipelago.ServerConnectionCache.CreateClientCommunications; import com.salesforce.apollo.choam.support.ChoamMetrics; import com.salesforce.apollo.membership.Member; @@ -20,7 +20,7 @@ */ public class TxnSubmitClient implements TxnSubmission { - private final ManagedServerChannel channel; + private final ManagedServerChannel channel; private final TransactionSubmissionBlockingStub client; public TxnSubmitClient(ManagedServerChannel channel, ChoamMetrics metrics) { diff --git a/choam/src/main/java/com/salesforce/apollo/choam/comm/TxnSubmitServer.java b/choam/src/main/java/com/salesforce/apollo/choam/comm/TxnSubmitServer.java index e25c5d5159..366d47919a 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/comm/TxnSubmitServer.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/comm/TxnSubmitServer.java @@ -6,14 +6,13 @@ */ package com.salesforce.apollo.choam.comm; +import com.salesforce.apollo.archipelago.RoutableService; import com.salesforce.apollo.choam.proto.SubmitResult; import com.salesforce.apollo.choam.proto.Transaction; import com.salesforce.apollo.choam.proto.TransactionSubmissionGrpc.TransactionSubmissionImplBase; -import com.salesforce.apollo.archipelago.RoutableService; import com.salesforce.apollo.choam.support.ChoamMetrics; import com.salesforce.apollo.cryptography.Digest; import com.salesforce.apollo.protocols.ClientIdentity; - import io.grpc.StatusRuntimeException; import io.grpc.stub.StreamObserver; @@ -24,7 +23,7 @@ public class TxnSubmitServer extends TransactionSubmissionImplBase { @SuppressWarnings("unused") private final ChoamMetrics metrics; private final RoutableService router; - private ClientIdentity identity; + private final ClientIdentity identity; public TxnSubmitServer(ClientIdentity identity, ChoamMetrics metrics, RoutableService router) { this.metrics = metrics; diff --git a/choam/src/main/java/com/salesforce/apollo/choam/fsm/Combine.java b/choam/src/main/java/com/salesforce/apollo/choam/fsm/Combine.java index 24ac98106a..0ad13ecc19 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/fsm/Combine.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/fsm/Combine.java @@ -13,12 +13,27 @@ /** * @author hal.hildebrand - * */ public interface Combine { + String AWAIT_REGEN = "AWAIT_REGEN"; + String AWAIT_SYNC = "AWAIT_SYNC"; + + void anchor(); + + void awaitRegeneration(); + + void awaitSynchronization(); + + void cancelTimer(String timer); + + void combine(); + + void recover(HashedCertifiedBlock anchor); + + void regenerate(); + enum Merchantile implements Transitions { AWAITING_REGENERATION { - @Exit public void cancelTimer() { context().cancelTimer(Combine.AWAIT_REGEN); @@ -40,8 +55,7 @@ public Transitions synchronizationFailed() { public void synchronizeContext() { context().awaitRegeneration(); } - }, - BOOTSTRAPPING { + }, BOOTSTRAPPING { @Override public Transitions combine() { return null; // Just queue up any blocks @@ -51,9 +65,7 @@ public Transitions combine() { public Transitions synchronizing() { return SYNCHRONIZING; } - }, - CHECKPOINTING { - + }, CHECKPOINTING { @Override public Transitions combine() { return null; // Just queue up any blocks @@ -64,15 +76,12 @@ public Transitions finishCheckpoint() { fsm().pop().combine(); return null; } - }, - INITIAL { + }, INITIAL { @Override public Transitions start() { return RECOVERING; } - }, - OPERATIONAL { - + }, OPERATIONAL { @Override public Transitions beginCheckpoint() { fsm().push(CHECKPOINTING); @@ -85,8 +94,7 @@ public Transitions combine() { return null; } - }, - PROTOCOL_FAILURE, RECOVERING { + }, PROTOCOL_FAILURE, RECOVERING { @Override public Transitions bootstrap(HashedCertifiedBlock anchor) { context().recover(anchor); @@ -118,9 +126,7 @@ public Transitions synchronizationFailed() { public void synchronizeContext() { context().awaitSynchronization(); } - }, - REGENERATING { - + }, REGENERATING { @Override public Transitions combine() { context().combine(); @@ -131,8 +137,7 @@ public Transitions combine() { public void regenerateView() { context().regenerate(); } - }, - SYNCHRONIZING { + }, SYNCHRONIZING { @Override public Transitions combine() { return null; // Just queue up any blocks @@ -186,21 +191,4 @@ default Transitions synchronizing() { throw fsm().invalidTransitionOn(); } } - - static final String AWAIT_REGEN = "AWAIT_REGEN"; - static final String AWAIT_SYNC = "AWAIT_SYNC"; - - void anchor(); - - void awaitRegeneration(); - - void awaitSynchronization(); - - void cancelTimer(String timer); - - void combine(); - - void recover(HashedCertifiedBlock anchor); - - void regenerate(); } diff --git a/choam/src/main/java/com/salesforce/apollo/choam/fsm/Driven.java b/choam/src/main/java/com/salesforce/apollo/choam/fsm/Driven.java index 6b3f20ec7f..3e0ec9ce6f 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/fsm/Driven.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/fsm/Driven.java @@ -20,8 +20,8 @@ * @author hal.hildebrand */ public interface Driven { - public static String PERIODIC_VALIDATIONS = "PERIODIC_VALIDATIONS"; - public static String SYNC = "SYNC"; + String PERIODIC_VALIDATIONS = "PERIODIC_VALIDATIONS"; + String SYNC = "SYNC"; void assembled(); @@ -161,7 +161,7 @@ public Transitions viewComplete() { /** Transition events for the Producer FSM */ interface Transitions extends FsmExecutor { - static Logger log = LoggerFactory.getLogger(Transitions.class); + Logger log = LoggerFactory.getLogger(Transitions.class); default Transitions assembled() { return null; diff --git a/choam/src/main/java/com/salesforce/apollo/choam/fsm/Genesis.java b/choam/src/main/java/com/salesforce/apollo/choam/fsm/Genesis.java index b47bcdb012..a103981878 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/fsm/Genesis.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/fsm/Genesis.java @@ -14,9 +14,22 @@ /** * @author hal.hildebrand - * */ public interface Genesis { + void certify(); + + void certify(List preblock, boolean last); + + void gather(); + + void gather(List preblock, boolean last); + + void nominate(); + + void nominations(List preblock, boolean last); + + void publish(); + enum BrickLayer implements Transitions { CERTIFICATION { @@ -30,10 +43,8 @@ public Transitions process(List preblock, boolean last) { context().certify(preblock, last); return last ? PUBLISH : null; } - }, - FAIL { - }, - INITIAL { + }, FAIL { + }, INITIAL { @Entry public void gather() { context().gather(); @@ -50,8 +61,7 @@ public Transitions process(List preblock, boolean last) { context().gather(preblock, last); return null; } - }, - NOMINATION { + }, NOMINATION { @Override public Transitions nextEpoch(Integer epoch) { return CERTIFICATION; @@ -67,13 +77,12 @@ public Transitions process(List preblock, boolean last) { context().nominations(preblock, last); return null; } - }, - PUBLISH { + }, PUBLISH { @Entry public void publish() { context().publish(); } - }; + } } @@ -87,18 +96,4 @@ default Transitions process(List preblock, boolean last) { throw fsm().invalidTransitionOn(); } } - - public void certify(); - - public void certify(List preblock, boolean last); - - public void gather(); - - public void gather(List preblock, boolean last); - - public void nominate(); - - public void nominations(List preblock, boolean last); - - public void publish(); } diff --git a/choam/src/main/java/com/salesforce/apollo/choam/fsm/Reconfiguration.java b/choam/src/main/java/com/salesforce/apollo/choam/fsm/Reconfiguration.java index 6eb11c04f8..fe34e90d4f 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/fsm/Reconfiguration.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/fsm/Reconfiguration.java @@ -11,18 +11,27 @@ /** * @author hal.hildebrand - * */ public interface Reconfiguration { + void certify(); + + void complete(); + + void elect(); + + void failed(); + + void gather(); + + void nominate(); + enum Reconfigure implements Transitions { AWAIT_ASSEMBLY { @Override public Transitions assembled() { return GATHER; } - }, - CERTIFICATION { - + }, CERTIFICATION { @Override public Transitions certified() { return RECONFIGURE; @@ -47,9 +56,7 @@ public Transitions gathered() { public Transitions validation() { return CERTIFICATION; } - }, - GATHER { - + }, GATHER { @Entry public void assembly() { context().gather(); @@ -64,9 +71,7 @@ public Transitions election() { public Transitions gathered() { return NOMINATION; } - }, - NOMINATION { - + }, NOMINATION { @Entry public void nominate() { context().nominate(); @@ -76,8 +81,7 @@ public void nominate() { public Transitions nominated() { return CERTIFICATION; } - }, - PROTOCOL_FAILURE { + }, PROTOCOL_FAILURE { @Override public Transitions assembled() { return null; @@ -122,8 +126,7 @@ public void terminate() { public Transitions validation() { return null; } - }, - RECONFIGURE { + }, RECONFIGURE { @Override public Transitions complete() { return RECONFIGURED; @@ -133,9 +136,7 @@ public Transitions complete() { public void elect() { context().elect(); } - }, - RECONFIGURED { - + }, RECONFIGURED { @Override public Transitions complete() { return null; @@ -145,7 +146,7 @@ public Transitions complete() { public void completion() { context().complete(); } - }; + } } interface Transitions extends FsmExecutor { @@ -181,16 +182,4 @@ default Transitions validation() { return null; } } - - void certify(); - - void complete(); - - void elect(); - - void failed(); - - void gather(); - - void nominate(); } diff --git a/choam/src/main/java/com/salesforce/apollo/choam/support/BatchingQueue.java b/choam/src/main/java/com/salesforce/apollo/choam/support/BatchingQueue.java index d24f0b488e..386a8fd91d 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/support/BatchingQueue.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/support/BatchingQueue.java @@ -1,5 +1,8 @@ package com.salesforce.apollo.choam.support; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import java.time.Duration; import java.util.ArrayList; import java.util.Iterator; @@ -9,73 +12,18 @@ import java.util.concurrent.locks.ReentrantLock; import java.util.function.Function; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - public class BatchingQueue { - public class Batch { - private int byteSize; - private final List events = new ArrayList<>(); - - public Iterator iterator() { - return events.iterator(); - } - - @Override - public String toString() { - return String.format("Batch [bytes=%s, size=%s]", byteSize, events.size()); - } - - List getEvents() { - return events; - } - - private boolean addEvent(T event) { - if (taken == limit) { - return false; - } - if (events.size() == batchSize) { - if (!reapCurrentBatch()) { - log.trace("rejecting event size: {} added: {} taken: {}", size, added, taken); - return false; - } - } - final var eventSize = sizer.apply(event); - if (byteSize + eventSize > maxByteSize) { - if (!reapCurrentBatch()) { - log.trace("rejecting event size: {} added: {} taken: {}", size, added, taken); - return false; - } - } - - final var add = events.add(event); - if (add) { - size = size + 1; - byteSize += eventSize; - log.trace("adding event: {} size: {} added: {} taken: {}", eventSize, size, added, taken); - } - return add; - - } - - private void clear() { - events.clear(); - byteSize = 0; - } - } - - private final static Logger log = LoggerFactory.getLogger(BatchingQueue.class); - - private int added; - private final int batchSize; - private final Batch currentBatch; - private final int limit; - private final ReentrantLock lock; - private final int maxByteSize; - private final LinkedBlockingQueue> oldBatches = new LinkedBlockingQueue<>(); - private int size; - private final Function sizer; - private int taken; + private final static Logger log = LoggerFactory.getLogger(BatchingQueue.class); + private final int batchSize; + private final Batch currentBatch; + private final int limit; + private final ReentrantLock lock; + private final int maxByteSize; + private final LinkedBlockingQueue> oldBatches = new LinkedBlockingQueue<>(); + private final Function sizer; + private int added; + private int size; + private int taken; public BatchingQueue(int limit, int batchSize, Function sizer, int maxByteSize) { this.limit = limit; @@ -192,4 +140,55 @@ private boolean reapCurrentBatch() { currentBatch.clear(); return added <= limit; } + + public class Batch { + private final List events = new ArrayList<>(); + private int byteSize; + + public Iterator iterator() { + return events.iterator(); + } + + @Override + public String toString() { + return String.format("Batch [bytes=%s, size=%s]", byteSize, events.size()); + } + + List getEvents() { + return events; + } + + private boolean addEvent(T event) { + if (taken == limit) { + return false; + } + if (events.size() == batchSize) { + if (!reapCurrentBatch()) { + log.trace("rejecting event size: {} added: {} taken: {}", size, added, taken); + return false; + } + } + final var eventSize = sizer.apply(event); + if (byteSize + eventSize > maxByteSize) { + if (!reapCurrentBatch()) { + log.trace("rejecting event size: {} added: {} taken: {}", size, added, taken); + return false; + } + } + + final var add = events.add(event); + if (add) { + size = size + 1; + byteSize += eventSize; + log.trace("adding event: {} size: {} added: {} taken: {}", eventSize, size, added, taken); + } + return add; + + } + + private void clear() { + events.clear(); + byteSize = 0; + } + } } diff --git a/choam/src/main/java/com/salesforce/apollo/choam/support/Bootstrapper.java b/choam/src/main/java/com/salesforce/apollo/choam/support/Bootstrapper.java index 67e260ed62..d43b06e320 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/support/Bootstrapper.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/support/Bootstrapper.java @@ -238,7 +238,7 @@ private void computeGenesis(Map votes) { Map valid = votes.entrySet() .stream() .filter(e -> e.getValue().hasGenesis()) // Has a genesis - .filter(e -> genesis == null ? true : genesis.hash.equals(e.getKey())) // If + .filter(e -> genesis == null || genesis.hash.equals(e.getKey())) // If // restoring // from // known @@ -306,10 +306,8 @@ private void computeGenesis(Map votes) { .filter(i -> genesis.hash.equals( new HashedCertifiedBlock(params.digestAlgorithm(), i.getGenesis()).hash)) .filter(i -> i.hasCheckpoint()) - .filter(i -> lastCheckpoint != null ? true : lastCheckpoint != null ? - HashedBlock.height(i.getCheckpoint()) - .compareTo(lastCheckpoint) > 0 - : true) + .filter(i -> lastCheckpoint != null || lastCheckpoint == null + || HashedBlock.height(i.getCheckpoint()).compareTo(lastCheckpoint) > 0) .max((a, b) -> Long.compare(a.getCheckpoint().getBlock().getHeader().getHeight(), b.getCheckpoint().getBlock().getHeader().getHeight())) .orElse(null); diff --git a/choam/src/main/java/com/salesforce/apollo/choam/support/CheckpointState.java b/choam/src/main/java/com/salesforce/apollo/choam/support/CheckpointState.java index 34b35e43a6..9c101c81a2 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/support/CheckpointState.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/support/CheckpointState.java @@ -7,9 +7,9 @@ package com.salesforce.apollo.choam.support; import com.google.protobuf.ByteString; +import com.salesforce.apollo.bloomFilters.BloomFilter; import com.salesforce.apollo.choam.proto.Checkpoint; import com.salesforce.apollo.choam.proto.Slice; -import com.salesforce.apollo.bloomFilters.BloomFilter; import com.salesforce.apollo.cryptography.Digest; import com.salesforce.apollo.cryptography.HexBloom; import com.salesforce.apollo.utils.Utils; diff --git a/choam/src/main/java/com/salesforce/apollo/choam/support/ChoamMetrics.java b/choam/src/main/java/com/salesforce/apollo/choam/support/ChoamMetrics.java index f02d5acf40..77b62f1772 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/support/ChoamMetrics.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/support/ChoamMetrics.java @@ -29,31 +29,31 @@ public interface ChoamMetrics extends EndpointMetrics { void publishedBatch(int batchSize, int byteSize, int validations, int reassemblies); + void transactionCancelled(); + void transactionComplete(Throwable t); Timer transactionLatency(); + void transactionSubmissionError(); + + void transactionSubmitRateLimited(); + + void transactionSubmitRetriesExhausted(); + void transactionSubmitRetry(); void transactionSubmittedBufferFull(); void transactionSubmittedFail(); - void transactionSubmittedSuccess(); - - void transactionTimeout(); - void transactionSubmittedInvalidCommittee(); - void transactionSubmittedUnavailable(); - - void transactionSubmissionError(); - void transactionSubmittedInvalidResult(); - void transactionSubmitRetriesExhausted(); + void transactionSubmittedSuccess(); - void transactionSubmitRateLimited(); + void transactionSubmittedUnavailable(); - void transactionCancelled(); + void transactionTimeout(); } diff --git a/choam/src/main/java/com/salesforce/apollo/choam/support/ChoamMetricsImpl.java b/choam/src/main/java/com/salesforce/apollo/choam/support/ChoamMetricsImpl.java index 00c1ca1608..6ff2e0e84b 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/support/ChoamMetricsImpl.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/support/ChoamMetricsImpl.java @@ -50,8 +50,8 @@ public class ChoamMetricsImpl extends EndpointMetricsImpl implements ChoamMetric private final Meter transactionSubmissionError; private final Meter transactionSubmittedInvalidResult; private final Meter transactionSubmitRetriesExhausted; - private final Meter transactionSubmitRateLimited; - private final Meter transactionCancelled; + private final Meter transactionSubmitRateLimited; + private final Meter transactionCancelled; public ChoamMetricsImpl(Digest context, MetricRegistry registry) { super(registry); @@ -85,10 +85,8 @@ public ChoamMetricsImpl(Digest context, MetricRegistry registry) { name(context.shortString(), "transaction.submit.invalid.result")); transactionSubmitRetriesExhausted = registry.meter( name(context.shortString(), "transaction.submit.retries.exhausted")); - transactionSubmitRateLimited = registry.meter( - name(context.shortString(), "transaction.submit.rate.limited")); - transactionCancelled = registry.meter( - name(context.shortString(), "transaction.submit.cancelled")); + transactionSubmitRateLimited = registry.meter(name(context.shortString(), "transaction.submit.rate.limited")); + transactionCancelled = registry.meter(name(context.shortString(), "transaction.submit.cancelled")); } @Override @@ -126,6 +124,11 @@ public void publishedBatch(int transactions, int byteSize, int validations, int publishedReassemblies.mark(reassemblies); } + @Override + public void transactionCancelled() { + transactionCancelled.mark(); + } + @Override public void transactionComplete(Throwable t) { if (t != null) { @@ -147,43 +150,38 @@ public Timer transactionLatency() { } @Override - public void transactionSubmitRetry() { - transactionSubmitRetry.mark(); - } - - @Override - public void transactionSubmittedBufferFull() { - transactionSubmittedBufferFull.mark(); + public void transactionSubmissionError() { + transactionSubmissionError.mark(); } @Override - public void transactionSubmittedFail() { - transactionSubmitFailed.mark(); + public void transactionSubmitRateLimited() { + transactionSubmitRateLimited.mark(); } @Override - public void transactionSubmittedSuccess() { - transactionSubmitSuccess.mark(); + public void transactionSubmitRetriesExhausted() { + transactionSubmitRetriesExhausted.mark(); } @Override - public void transactionTimeout() { - transactionTimeout.mark(); + public void transactionSubmitRetry() { + transactionSubmitRetry.mark(); } @Override - public void transactionSubmittedInvalidCommittee() { - transactionSubmittedInvalidCommittee.mark(); + public void transactionSubmittedBufferFull() { + transactionSubmittedBufferFull.mark(); } @Override - public void transactionSubmittedUnavailable() { - transactionSubmittedUnavailable.mark(); + public void transactionSubmittedFail() { + transactionSubmitFailed.mark(); } @Override - public void transactionSubmissionError() { - transactionSubmissionError.mark(); + public void transactionSubmittedInvalidCommittee() { + transactionSubmittedInvalidCommittee.mark(); } @Override @@ -192,17 +190,17 @@ public void transactionSubmittedInvalidResult() { } @Override - public void transactionSubmitRetriesExhausted() { - transactionSubmitRetriesExhausted.mark(); + public void transactionSubmittedSuccess() { + transactionSubmitSuccess.mark(); } @Override - public void transactionSubmitRateLimited() { - transactionSubmitRateLimited.mark(); + public void transactionSubmittedUnavailable() { + transactionSubmittedUnavailable.mark(); } @Override - public void transactionCancelled() { - transactionCancelled.mark(); + public void transactionTimeout() { + transactionTimeout.mark(); } } diff --git a/choam/src/main/java/com/salesforce/apollo/choam/support/DigestType.java b/choam/src/main/java/com/salesforce/apollo/choam/support/DigestType.java index 2dd8f3d551..3d73ed69c1 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/support/DigestType.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/support/DigestType.java @@ -20,7 +20,7 @@ public class DigestType extends BasicDataType { @Override public int compare(Digest a, Digest b) { - return ((Digest) a).compareTo(((Digest) b)); + return a.compareTo(b); } @Override @@ -30,7 +30,7 @@ public Digest[] createStorage(int size) { @Override public int getMemory(Digest obj) { - return ((Digest) obj).getAlgorithm().digestLength() + 1; + return obj.getAlgorithm().digestLength() + 1; } @Override diff --git a/choam/src/main/java/com/salesforce/apollo/choam/support/ExponentialBackoffPolicy.java b/choam/src/main/java/com/salesforce/apollo/choam/support/ExponentialBackoffPolicy.java index 874311b60e..2f959a38be 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/support/ExponentialBackoffPolicy.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/support/ExponentialBackoffPolicy.java @@ -1,68 +1,17 @@ package com.salesforce.apollo.choam.support; -import static com.google.common.base.Preconditions.checkArgument; +import com.salesforce.apollo.utils.Entropy; import java.time.Duration; -import com.salesforce.apollo.utils.Entropy; +import static com.google.common.base.Preconditions.checkArgument; public class ExponentialBackoffPolicy { - public static class Builder { - private Duration initialBackoff = Duration.ofMillis(10); - private double jitter = .2; - private Duration maxBackoff = Duration.ofMillis(500); - private double multiplier = 1.6; - - public ExponentialBackoffPolicy build() { - return new ExponentialBackoffPolicy(initialBackoff, jitter, maxBackoff, multiplier); - } - - public Duration getInitialBackoff() { - return initialBackoff; - } - - public double getJitter() { - return jitter; - } - - public Duration getMaxBackoff() { - return maxBackoff; - } - - public double getMultiplier() { - return multiplier; - } - - public Builder setInitialBackoff(Duration initialBackoff) { - this.initialBackoff = initialBackoff; - return this; - } - - public Builder setJitter(double jitter) { - this.jitter = jitter; - return this; - } - - public Builder setMaxBackoff(Duration maxBackoff) { - this.maxBackoff = maxBackoff; - return this; - } - - public Builder setMultiplier(double multiplier) { - this.multiplier = multiplier; - return this; - } - } - - public static Builder newBuilder() { - return new Builder(); - } - - private final Duration initialBackoff; - private final double jitter; - private final Duration maxBackoff; - private final double multiplier; + private final Duration initialBackoff; + private final double jitter; + private final Duration maxBackoff; + private final double multiplier; private volatile Duration nextBackoff; public ExponentialBackoffPolicy(Duration initialBackoff, double jitter, Duration maxBackoff, double multiplier) { @@ -74,6 +23,10 @@ public ExponentialBackoffPolicy(Duration initialBackoff, double jitter, Duration nextBackoff = initialBackoff; } + public static Builder newBuilder() { + return new Builder(); + } + public Duration getInitialBackoff() { return initialBackoff; } @@ -95,8 +48,8 @@ public Duration nextBackoff() { nextBackoff = Duration.ofNanos((long) Math.min(currentBackoffNanos * multiplier, maxBackoff.toNanos())); - return Duration.ofNanos(currentBackoffNanos - + uniformRandom(-jitter * currentBackoffNanos, jitter * currentBackoffNanos)); + return Duration.ofNanos( + currentBackoffNanos + uniformRandom(-jitter * currentBackoffNanos, jitter * currentBackoffNanos)); } private long uniformRandom(double low, double high) { @@ -104,4 +57,51 @@ private long uniformRandom(double low, double high) { double mag = high - low; return (long) (Entropy.nextBitsStreamDouble() * mag + low); } + + public static class Builder { + private Duration initialBackoff = Duration.ofMillis(10); + private double jitter = .2; + private Duration maxBackoff = Duration.ofMillis(500); + private double multiplier = 1.6; + + public ExponentialBackoffPolicy build() { + return new ExponentialBackoffPolicy(initialBackoff, jitter, maxBackoff, multiplier); + } + + public Duration getInitialBackoff() { + return initialBackoff; + } + + public Builder setInitialBackoff(Duration initialBackoff) { + this.initialBackoff = initialBackoff; + return this; + } + + public double getJitter() { + return jitter; + } + + public Builder setJitter(double jitter) { + this.jitter = jitter; + return this; + } + + public Duration getMaxBackoff() { + return maxBackoff; + } + + public Builder setMaxBackoff(Duration maxBackoff) { + this.maxBackoff = maxBackoff; + return this; + } + + public double getMultiplier() { + return multiplier; + } + + public Builder setMultiplier(double multiplier) { + this.multiplier = multiplier; + return this; + } + } } diff --git a/choam/src/main/java/com/salesforce/apollo/choam/support/HashedBlock.java b/choam/src/main/java/com/salesforce/apollo/choam/support/HashedBlock.java index 7835dada50..96eec22471 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/support/HashedBlock.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/support/HashedBlock.java @@ -6,19 +6,18 @@ */ package com.salesforce.apollo.choam.support; -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.List; - -import org.joou.ULong; -import org.joou.Unsigned; - import com.google.protobuf.Message; import com.salesforce.apollo.choam.proto.Block; import com.salesforce.apollo.choam.proto.CertifiedBlock; import com.salesforce.apollo.choam.proto.Header; import com.salesforce.apollo.cryptography.Digest; import com.salesforce.apollo.cryptography.DigestAlgorithm; +import org.joou.ULong; +import org.joou.Unsigned; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; public class HashedBlock implements Comparable { public final Block block; diff --git a/choam/src/main/java/com/salesforce/apollo/choam/support/HashedCertifiedBlock.java b/choam/src/main/java/com/salesforce/apollo/choam/support/HashedCertifiedBlock.java index cb4b4eaa80..e453acf289 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/support/HashedCertifiedBlock.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/support/HashedCertifiedBlock.java @@ -6,11 +6,10 @@ */ package com.salesforce.apollo.choam.support; -import org.joou.ULong; - import com.salesforce.apollo.choam.proto.CertifiedBlock; import com.salesforce.apollo.cryptography.Digest; import com.salesforce.apollo.cryptography.DigestAlgorithm; +import org.joou.ULong; /** * @author hal.hildebrand diff --git a/choam/src/main/java/com/salesforce/apollo/choam/support/InvalidTransaction.java b/choam/src/main/java/com/salesforce/apollo/choam/support/InvalidTransaction.java index 1de2f9ec92..39297bd0ee 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/support/InvalidTransaction.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/support/InvalidTransaction.java @@ -8,7 +8,6 @@ /** * @author hal.hildebrand - * */ public class InvalidTransaction extends Exception { diff --git a/choam/src/main/java/com/salesforce/apollo/choam/support/OneShot.java b/choam/src/main/java/com/salesforce/apollo/choam/support/OneShot.java index 3c30c5272c..7ec1866cff 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/support/OneShot.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/support/OneShot.java @@ -6,14 +6,14 @@ */ package com.salesforce.apollo.choam.support; +import com.google.protobuf.ByteString; + import java.util.concurrent.CountDownLatch; import java.util.function.Supplier; -import com.google.protobuf.ByteString; - public class OneShot implements Supplier { - private CountDownLatch latch = new CountDownLatch(1); - private volatile ByteString value; + private final CountDownLatch latch = new CountDownLatch(1); + private volatile ByteString value; @Override public ByteString get() { diff --git a/choam/src/main/java/com/salesforce/apollo/choam/support/ServiceUnavailable.java b/choam/src/main/java/com/salesforce/apollo/choam/support/ServiceUnavailable.java index 7d72ffe14b..92905a0ed0 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/support/ServiceUnavailable.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/support/ServiceUnavailable.java @@ -8,7 +8,6 @@ /** * @author hal.hildebrand - * */ public class ServiceUnavailable extends RuntimeException { diff --git a/choam/src/main/java/com/salesforce/apollo/choam/support/Store.java b/choam/src/main/java/com/salesforce/apollo/choam/support/Store.java index fa21e05f18..cf85349a5b 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/support/Store.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/support/Store.java @@ -7,8 +7,8 @@ package com.salesforce.apollo.choam.support; import com.google.protobuf.InvalidProtocolBufferException; -import com.salesforce.apollo.choam.proto.*; import com.salesforce.apollo.bloomFilters.BloomFilter; +import com.salesforce.apollo.choam.proto.*; import com.salesforce.apollo.cryptography.Digest; import com.salesforce.apollo.cryptography.DigestAlgorithm; import org.h2.mvstore.MVMap; diff --git a/choam/src/main/java/com/salesforce/apollo/choam/support/TransactionFailed.java b/choam/src/main/java/com/salesforce/apollo/choam/support/TransactionFailed.java index befcb5811f..fe2362ca87 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/support/TransactionFailed.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/support/TransactionFailed.java @@ -8,7 +8,6 @@ /** * @author hal.hildebrand - * */ public class TransactionFailed extends Exception { diff --git a/choam/src/main/java/com/salesforce/apollo/choam/support/TxDataSource.java b/choam/src/main/java/com/salesforce/apollo/choam/support/TxDataSource.java index 1e89de4b1c..2972d4120d 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/support/TxDataSource.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/support/TxDataSource.java @@ -6,16 +6,6 @@ */ package com.salesforce.apollo.choam.support; -import java.time.Duration; -import java.time.Instant; -import java.util.ArrayList; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.atomic.AtomicBoolean; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import com.google.protobuf.ByteString; import com.salesforce.apollo.choam.proto.Reassemble; import com.salesforce.apollo.choam.proto.Transaction; @@ -23,6 +13,15 @@ import com.salesforce.apollo.choam.proto.Validate; import com.salesforce.apollo.ethereal.DataSource; import com.salesforce.apollo.membership.Member; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.atomic.AtomicBoolean; /** * The data source for CHOAM. Provides back pressure to the caller when the capacity of the receiver is exceeded. This diff --git a/cryptography/src/main/java/com/salesforce/apollo/cryptography/EdDSAOperations.java b/cryptography/src/main/java/com/salesforce/apollo/cryptography/EdDSAOperations.java index 9131e34710..50b9be23b4 100644 --- a/cryptography/src/main/java/com/salesforce/apollo/cryptography/EdDSAOperations.java +++ b/cryptography/src/main/java/com/salesforce/apollo/cryptography/EdDSAOperations.java @@ -72,7 +72,7 @@ public EdDSAOperations(SignatureAlgorithm signatureAlgorithm) { } } - private static void reverse(byte[] arr) { + public static void reverse(byte[] arr) { var i = 0; var j = arr.length - 1; @@ -83,7 +83,7 @@ private static void reverse(byte[] arr) { } } - private static void swap(byte[] arr, int i, int j) { + public static void swap(byte[] arr, int i, int j) { var tmp = arr[i]; arr[i] = arr[j]; arr[j] = tmp; diff --git a/cryptography/src/main/java/com/salesforce/apollo/cryptography/EncryptionAlgorithm.java b/cryptography/src/main/java/com/salesforce/apollo/cryptography/EncryptionAlgorithm.java new file mode 100644 index 0000000000..603ddb494d --- /dev/null +++ b/cryptography/src/main/java/com/salesforce/apollo/cryptography/EncryptionAlgorithm.java @@ -0,0 +1,163 @@ +package com.salesforce.apollo.cryptography; + +import javax.crypto.DecapsulateException; +import javax.crypto.KEM; +import javax.crypto.SecretKey; +import java.math.BigInteger; +import java.security.*; +import java.security.interfaces.EdECPrivateKey; +import java.security.interfaces.EdECPublicKey; +import java.security.interfaces.XECPublicKey; +import java.security.spec.InvalidKeySpecException; +import java.security.spec.NamedParameterSpec; +import java.security.spec.XECPublicKeySpec; + +public enum EncryptionAlgorithm { + X_25519 { + @Override + public String algorithmName() { + return "X25519"; + } + + @Override + public String curveName() { + return "Curve25519"; + } + + @Override + public int getCode() { + return 1; + } + + @Override + public int publicKeyLength() { + return 32; + } + + }, X_448 { + @Override + public String algorithmName() { + return "X448"; + } + + @Override + public String curveName() { + return "Curve448"; + } + + @Override + public int getCode() { + return 2; + } + + @Override + public int publicKeyLength() { + return 57; + } + }; + + public static final EncryptionAlgorithm DEFAULT = X_25519; + public static final String DHKEM = "DHKEM"; + public static final String XDH = "XDH"; + + public static EncryptionAlgorithm lookup(int code) { + return switch (code) { + case 0 -> throw new IllegalArgumentException("Uninitialized enum value"); + case 1 -> X_25519; + case 2 -> X_448; + default -> throw new IllegalArgumentException("Unknown code: " + code); + }; + } + + public static EncryptionAlgorithm lookup(PrivateKey privateKey) { + return switch (privateKey.getAlgorithm()) { + case XDH -> lookupX(((EdECPrivateKey) privateKey).getParams()); + case "x25519" -> X_25519; + case "x448" -> X_448; + default -> throw new IllegalArgumentException("Unknown algorithm: " + privateKey.getAlgorithm()); + }; + } + + public static EncryptionAlgorithm lookup(PublicKey publicKey) { + return switch (publicKey.getAlgorithm()) { + case XDH -> lookupX(((EdECPublicKey) publicKey).getParams()); + case "X25519" -> X_25519; + case "X448" -> X_448; + default -> throw new IllegalArgumentException("Unknown algorithm: " + publicKey.getAlgorithm()); + }; + } + + private static EncryptionAlgorithm lookupX(NamedParameterSpec params) { + var curveName = params.getName(); + return switch (curveName.toLowerCase()) { + case "x25519" -> X_25519; + case "x448" -> X_448; + default -> throw new IllegalArgumentException("Unknown edwards curve: " + curveName); + }; + } + + abstract public String algorithmName(); + + abstract public String curveName(); + + final public SecretKey decapsulate(PrivateKey privateKey, byte[] encapsulated, String algorithm) { + try { + var kem = KEM.getInstance(DHKEM); + return kem.newDecapsulator(privateKey).decapsulate(encapsulated, 0, encapsulated.length, algorithm); + } catch (NoSuchAlgorithmException | InvalidKeyException | DecapsulateException e) { + throw new IllegalArgumentException("error decapsulating", e); + } + } + + final public KEM.Encapsulated encapsulated(PublicKey publicKey) { + try { + var kem = KEM.getInstance(DHKEM); + return kem.newEncapsulator(publicKey).encapsulate(); + } catch (NoSuchAlgorithmException | InvalidKeyException e) { + throw new IllegalArgumentException("error encapsulating", e); + } + } + + final public byte[] encode(PublicKey publicKey) { + return ((XECPublicKey) publicKey).getU().toByteArray(); + } + + final public KeyPair generateKeyPair() { + try { + KeyPairGenerator kpg = KeyPairGenerator.getInstance(XDH); + kpg.initialize(getParamSpec()); + return kpg.generateKeyPair(); + } catch (NoSuchAlgorithmException | InvalidAlgorithmParameterException e) { + throw new IllegalArgumentException("Cannot generate key pair", e); + } + } + + final public KeyPair generateKeyPair(SecureRandom secureRandom) { + try { + KeyPairGenerator kpg = KeyPairGenerator.getInstance(XDH); + kpg.initialize(getParamSpec(), secureRandom); + return kpg.generateKeyPair(); + } catch (NoSuchAlgorithmException | InvalidAlgorithmParameterException e) { + throw new IllegalArgumentException("Cannot generate key pair", e); + } + } + + abstract public int getCode(); + + final public PublicKey publicKey(byte[] bytes) { + try { + KeyFactory kf = KeyFactory.getInstance(XDH); + BigInteger u = new BigInteger(bytes); + XECPublicKeySpec pubSpec = new XECPublicKeySpec(getParamSpec(), u); + return kf.generatePublic(pubSpec); + } catch (NoSuchAlgorithmException | InvalidKeySpecException e) { + throw new IllegalArgumentException("Cannot create public key", e); + } + } + + abstract public int publicKeyLength(); + + private NamedParameterSpec getParamSpec() { + return new NamedParameterSpec(algorithmName()); + } +} diff --git a/cryptography/src/main/java/com/salesforce/apollo/cryptography/SignatureAlgorithm.java b/cryptography/src/main/java/com/salesforce/apollo/cryptography/SignatureAlgorithm.java index 2e535d71b9..5965ff6e02 100644 --- a/cryptography/src/main/java/com/salesforce/apollo/cryptography/SignatureAlgorithm.java +++ b/cryptography/src/main/java/com/salesforce/apollo/cryptography/SignatureAlgorithm.java @@ -56,16 +56,6 @@ public KeyPair generateKeyPair(SecureRandom secureRandom) { return ops.generateKeyPair(secureRandom); } - @Override - public PrivateKey privateKey(byte[] bytes) { - return ops.privateKey(bytes); - } - - @Override - public int privateKeyLength() { - return 32; - } - @Override public PublicKey publicKey(byte[] bytes) { return ops.publicKey(bytes); @@ -141,16 +131,6 @@ public KeyPair generateKeyPair(SecureRandom secureRandom) { return ops.generateKeyPair(secureRandom); } - @Override - public PrivateKey privateKey(byte[] bytes) { - return ops.privateKey(bytes); - } - - @Override - public int privateKeyLength() { - return 56; - } - @Override public PublicKey publicKey(byte[] bytes) { return ops.publicKey(bytes); @@ -222,16 +202,6 @@ public KeyPair generateKeyPair(SecureRandom secureRandom) { return null; } - @Override - public PrivateKey privateKey(byte[] bytes) { - return null; - } - - @Override - public int privateKeyLength() { - return 0; - } - @Override public PublicKey publicKey(byte[] bytes) { return null; @@ -330,14 +300,6 @@ private static SignatureAlgorithm lookupEd(NamedParameterSpec params) { abstract public KeyPair generateKeyPair(SecureRandom secureRandom); - public KeyPair keyPair(byte[] bytes, byte[] publicKey) { - return new KeyPair(publicKey(publicKey), privateKey(bytes)); - } - - abstract public PrivateKey privateKey(byte[] bytes); - - abstract public int privateKeyLength(); - abstract public PublicKey publicKey(byte[] bytes); abstract public int publicKeyLength(); @@ -374,12 +336,12 @@ final public boolean verify(PublicKey publicKey, JohnHancock signature, ByteStri return verify(publicKey, signature, BbBackedInputStream.aggregate(message)); } - abstract protected boolean verify(PublicKey publicKey, byte[] signature, InputStream message); - abstract JohnHancock sign(ULong sequenceNumber, PrivateKey[] privateKeys, InputStream message); final boolean verify(PublicKey publicKey, JohnHancock signature, InputStream message) { return new DefaultVerifier(new PublicKey[] { publicKey }).verify(SigningThreshold.unweighted(1), signature, message); } + + abstract protected boolean verify(PublicKey publicKey, byte[] signature, InputStream message); } diff --git a/cryptography/src/test/java/com/salesforce/apollo/cryptography/XTest.java b/cryptography/src/test/java/com/salesforce/apollo/cryptography/XTest.java new file mode 100644 index 0000000000..8bcf48c822 --- /dev/null +++ b/cryptography/src/test/java/com/salesforce/apollo/cryptography/XTest.java @@ -0,0 +1,76 @@ +package com.salesforce.apollo.cryptography; + +import org.junit.jupiter.api.Test; + +import javax.crypto.KeyAgreement; +import java.security.SecureRandom; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * @author hal.hildebrand + **/ +public class XTest { + @Test + public void testEncoding() throws Exception { + var entropy = SecureRandom.getInstance("SHA1PRNG"); + entropy.setSeed(new byte[] { 6, 6, 6 }); + + var algorithm = EncryptionAlgorithm.X_25519; + var pair = algorithm.generateKeyPair(entropy); + assertNotNull(pair); + var encodedPublic = algorithm.encode(pair.getPublic()); + assertNotNull(encodedPublic); + var decodedPublic = algorithm.publicKey(encodedPublic); + assertNotNull(decodedPublic); + assertEquals(pair.getPublic(), decodedPublic); + } + + @Test + public void testKEM() throws Exception { + var entropy = SecureRandom.getInstance("SHA1PRNG"); + entropy.setSeed(new byte[] { 6, 6, 6 }); + + var algorithm = EncryptionAlgorithm.X_25519; + var pair1 = algorithm.generateKeyPair(entropy); + assertNotNull(pair1); + var pair2 = algorithm.generateKeyPair(entropy); + assertNotNull(pair2); + + var encapsulated = algorithm.encapsulated(pair2.getPublic()); + assertNotNull(encapsulated); + + var secretKey = algorithm.decapsulate(pair2.getPrivate(), encapsulated.encapsulation(), "Generic"); + + assertNotNull(secretKey); + assertEquals(encapsulated.key(), secretKey); + } + + @Test + public void testKeyAgreement() throws Exception { + var entropy = SecureRandom.getInstance("SHA1PRNG"); + entropy.setSeed(new byte[] { 6, 6, 6 }); + + var algorithm = EncryptionAlgorithm.X_25519; + var pair1 = algorithm.generateKeyPair(entropy); + assertNotNull(pair1); + var pair2 = algorithm.generateKeyPair(entropy); + assertNotNull(pair2); + + KeyAgreement ka = KeyAgreement.getInstance("XDH"); + KeyAgreement ka2 = KeyAgreement.getInstance("XDH"); + + ka.init(pair1.getPrivate()); + ka2.init(pair2.getPrivate()); + + ka.doPhase(pair2.getPublic(), true); + ka2.doPhase(pair1.getPublic(), true); + + byte[] secret1 = ka.generateSecret(); + assertNotNull(secret1); + byte[] secret2 = ka2.generateSecret(); + assertNotNull(secret2); + + assertArrayEquals(secret1, secret2); + } +} diff --git a/fireflies/src/main/java/com/salesforce/apollo/fireflies/Binding.java b/fireflies/src/main/java/com/salesforce/apollo/fireflies/Binding.java index 161164c5a4..30b290fdfa 100644 --- a/fireflies/src/main/java/com/salesforce/apollo/fireflies/Binding.java +++ b/fireflies/src/main/java/com/salesforce/apollo/fireflies/Binding.java @@ -8,6 +8,7 @@ import com.codahale.metrics.Timer; import com.google.common.collect.HashMultiset; +import com.google.common.collect.Multiset; import com.google.protobuf.ByteString; import com.salesforce.apollo.archipelago.RouterImpl.CommonCommunications; import com.salesforce.apollo.cryptography.Digest; @@ -166,7 +167,7 @@ private boolean completeGateway(Participant member, CompletableFuture gat var trust = trusts.entrySet() .stream() .filter(e -> e.getCount() >= majority) - .map(e -> e.getElement()) + .map(Multiset.Entry::getElement) .findFirst() .orElse(null); if (trust != null) { @@ -216,7 +217,7 @@ private Join join(Digest v) { return; } if (!r.isInitialized()) { - log.error("Empty seeding response on: {}", node.getId(), t); + log.error("Empty seeding response on: {}", node.getId()); return; } var view = Digest.from(r.getView()); @@ -237,7 +238,7 @@ private Join join(Digest v) { private void join(Redirect redirect, Digest v, Duration duration) { var sample = redirect.getSampleList() .stream() - .map(sn -> new NoteWrapper(sn.getNote(), digestAlgo)) + .map(sn -> new NoteWrapper(sn, digestAlgo)) .map(nw -> view.new Participant(nw)) .collect(Collectors.toList()); log.info("Redirecting to: {} context: {} sample: {} on: {}", v, this.context.getId(), sample.size(), @@ -264,51 +265,47 @@ private void join(Redirect redirect, Digest v, Duration duration) { final var join = join(v); final var abandon = new AtomicInteger(); var scheduler = Executors.newScheduledThreadPool(1, Thread.ofVirtual().factory()); - regate.set(() -> { - redirecting.iterate((link, m) -> { - log.debug("Joining: {} contacting: {} on: {}", v, link.getMember().getId(), node.getId()); - try { - var g = link.join(join, params.seedingTimeout()); - if (g == null || g.equals(Gateway.getDefaultInstance())) { - log.info("Gateway view: {} empty from: {} on: {}", v, link.getMember().getId(), node.getId()); - abandon.incrementAndGet(); - return null; - } - return g; - } catch (StatusRuntimeException sre) { - gatewaySRE(v, link, sre, abandon); - return null; - } catch (Throwable t) { - log.info("Gateway view: {} error: {} from: {} on: {}", v, t.toString(), link.getMember().getId(), - node.getId()); + regate.set(() -> redirecting.iterate((link, m) -> { + log.debug("Joining: {} contacting: {} on: {}", v, link.getMember().getId(), node.getId()); + try { + var g = link.join(join, params.seedingTimeout()); + if (g == null || g.equals(Gateway.getDefaultInstance())) { + log.info("Gateway view: {} empty from: {} on: {}", v, link.getMember().getId(), node.getId()); abandon.incrementAndGet(); return null; } - }, (futureSailor, link, m) -> completeGateway((Participant) m, gateway, futureSailor, trusts, - initialSeedSet, v, majority), () -> { - if (gateway.isDone()) { - return; - } - if (abandon.get() >= majority) { - log.info("Abandoning Gateway view: {} reseeding on: {}", v, node.getId()); - seeding(); + return g; + } catch (StatusRuntimeException sre) { + gatewaySRE(v, link, sre, abandon); + return null; + } catch (Throwable t) { + log.info("Gateway view: {} error: {} from: {} on: {}", v, t, link.getMember().getId(), node.getId()); + abandon.incrementAndGet(); + return null; + } + }, (futureSailor, link, m) -> completeGateway((Participant) m, gateway, futureSailor, trusts, initialSeedSet, v, + majority), () -> { + if (gateway.isDone()) { + return; + } + if (abandon.get() >= majority) { + log.info("Abandoning Gateway view: {} reseeding on: {}", v, node.getId()); + seeding(); + } else { + abandon.set(0); + if (retries.get() < params.joinRetries()) { + log.info("Failed to join view: {} retry: {} out of: {} on: {}", v, retries.incrementAndGet(), + params.joinRetries(), node.getId()); + trusts.clear(); + initialSeedSet.clear(); + scheduler.schedule(() -> Thread.ofVirtual().start(Utils.wrapped(regate.get(), log)), + Entropy.nextBitsStreamLong(params.retryDelay().toNanos()), TimeUnit.NANOSECONDS); } else { - abandon.set(0); - if (retries.get() < params.joinRetries()) { - log.info("Failed to join view: {} retry: {} out of: {} on: {}", v, retries.incrementAndGet(), - params.joinRetries(), node.getId()); - trusts.clear(); - initialSeedSet.clear(); - scheduler.schedule(() -> Thread.ofVirtual().start(Utils.wrapped(regate.get(), log)), - Entropy.nextBitsStreamLong(params.retryDelay().toNanos()), - TimeUnit.NANOSECONDS); - } else { - log.error("Failed to join view: {} cannot obtain majority Gateway on: {}", view, node.getId()); - view.stop(); - } + log.error("Failed to join view: {} cannot obtain majority Gateway on: {}", view, node.getId()); + view.stop(); } - }, scheduler, params.retryDelay()); - }); + } + }, scheduler, params.retryDelay())); regate.get().run(); } @@ -324,8 +321,7 @@ private NoteWrapper seedFor(Seed seed) { .setNote(Note.newBuilder() .setHost(seed.endpoint().getHostName()) .setPort(seed.endpoint().getPort()) - .setCoordinates( - seed.establishment().getCoordinates().toEventCoords()) + .setIdentifier(seed.identifier().toIdent()) .setEpoch(-1) .setMask(ByteString.copyFrom( Node.createInitialMask(context).toByteArray()))) diff --git a/fireflies/src/main/java/com/salesforce/apollo/fireflies/Bootstrapper.java b/fireflies/src/main/java/com/salesforce/apollo/fireflies/Bootstrapper.java deleted file mode 100644 index 579cf8a687..0000000000 --- a/fireflies/src/main/java/com/salesforce/apollo/fireflies/Bootstrapper.java +++ /dev/null @@ -1,244 +0,0 @@ -package com.salesforce.apollo.fireflies; - -import com.github.benmanes.caffeine.cache.CacheLoader; -import com.github.benmanes.caffeine.cache.Caffeine; -import com.github.benmanes.caffeine.cache.LoadingCache; -import com.github.benmanes.caffeine.cache.RemovalCause; -import com.google.common.collect.HashMultiset; -import com.salesforce.apollo.archipelago.RouterImpl; -import com.salesforce.apollo.cryptography.Verifier; -import com.salesforce.apollo.fireflies.comm.entrance.Entrance; -import com.salesforce.apollo.fireflies.proto.Validation; -import com.salesforce.apollo.membership.Member; -import com.salesforce.apollo.membership.SigningMember; -import com.salesforce.apollo.ring.SliceIterator; -import com.salesforce.apollo.stereotomy.*; -import com.salesforce.apollo.stereotomy.event.EstablishmentEvent; -import com.salesforce.apollo.stereotomy.event.proto.IdentAndSeq; -import com.salesforce.apollo.stereotomy.event.proto.KeyState_; -import com.salesforce.apollo.stereotomy.event.protobuf.KeyStateImpl; -import com.salesforce.apollo.stereotomy.identifier.Identifier; -import org.checkerframework.checker.nullness.qual.Nullable; -import org.joou.ULong; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.time.Duration; -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.Executors; - -/** - * Verifiers that delegate to a majority of the sample for event validation and verification - *

- * - * @author hal.hildebrand - **/ -public class Bootstrapper implements Verifiers { - private final static Logger log = LoggerFactory.getLogger(Bootstrapper.class); - private final List successors; - private final SigningMember member; - private final int majority; - private final LoadingCache ksSeq; - private final RouterImpl.CommonCommunications communications; - private final Duration operationTimeout; - private final Duration operationsFrequency; - - public Bootstrapper(S member, Duration operationTimeout, - List successors, int majority, - Duration operationsFrequency, - RouterImpl.CommonCommunications communications) { - this.member = member; - this.successors = new ArrayList<>(successors); - this.majority = majority; - this.communications = communications; - this.operationTimeout = operationTimeout; - this.operationsFrequency = operationsFrequency; - ksSeq = Caffeine.newBuilder() - .maximumSize(100) - .expireAfterWrite(Duration.ofMinutes(1)) - .removalListener((IdentifierSequence seq, KeyState ks, RemovalCause cause) -> log.trace( - "KeyState {} was removed ({})", seq, cause)) - .build(new CacheLoader() { - - @Override - public @Nullable KeyState load(IdentifierSequence key) throws Exception { - return delegate(key); - } - }); - } - - public EventValidation getValidator() { - return new EventValidation() { - - @Override - public KeyState keyState(Identifier identifier, ULong seqNum) { - log.trace("Get key state: {}:{} on: {}", identifier, seqNum, member.getId()); - return ksSeq.get(new IdentifierSequence(identifier, seqNum)); - } - - @Override - public boolean validate(EstablishmentEvent event) { - log.trace("Validate event: {} on: {}", event, member.getId()); - return validate(event.getCoordinates()); - } - - @Override - public boolean validate(EventCoordinates coordinates) { - log.trace("Validating coordinates: {} on: {}", coordinates, member.getId()); - return Bootstrapper.this.validate(coordinates); - } - }; - } - - @Override - public Optional verifierFor(Identifier identifier) { - return Optional.of(new BootstrapVerifier(identifier)); - } - - @Override - public Optional verifierFor(EventCoordinates coordinates) { - return Optional.of(new BootstrapVerifier(coordinates.getIdentifier())); - } - - protected KeyState getKeyState(Identifier identifier, ULong sequenceNumber) { - return ksSeq.get(new IdentifierSequence(identifier, sequenceNumber)); - } - - private boolean complete(CompletableFuture ksFuture, Optional futureSailor, - HashMultiset keystates, Member m) { - if (futureSailor.isEmpty()) { - return true; - } - if (ksFuture.isDone()) { - return true; - } - final var ks = futureSailor.get(); - keystates.add(ks); - - var vs = keystates.entrySet() - .stream() - .filter(e -> e.getCount() >= majority) - .map(e -> e.getElement()) - .findFirst() - .orElse(null); - if (vs != null) { - var keyState = new KeyStateImpl(vs); - if (ksFuture.complete(keyState)) { - log.debug("Key state: {} received majority on: {}", keyState.getCoordinates(), member.getId()); - return false; - } - } - return true; - } - - private boolean completeValidation(CompletableFuture valid, Optional futureSailor, - HashMultiset validations, Member m) { - if (futureSailor.isEmpty()) { - return true; - } - if (valid.isDone()) { - return true; - } - final var v = futureSailor.get(); - validations.add(v); - - var validation = validations.entrySet() - .stream() - .filter(e -> e.getCount() >= majority) - .map(e -> e.getElement()) - .findFirst(); - if (!validation.isEmpty()) { - if (valid.complete(validation.get())) { - log.debug("Validation: {} received majority on: {}", validation.get().getResult(), member.getId()); - return false; - } - } - return true; - } - - private KeyState delegate(IdentifierSequence idSeq) { - log.info("Get key state: {} from slice on: {}", idSeq, member.getId()); - var iterator = new SliceIterator<>("Retrieve KeyState", member, successors, communications); - final var identifierSeq = idSeq.toIdSeq(); - var ks = new CompletableFuture(); - HashMultiset keystates = HashMultiset.create(); - iterator.iterate((link, m) -> { - log.debug("Requesting Key State from: {} on: {}", link.getMember().getId(), member.getId()); - return link.keyState(identifierSeq); - }, (futureSailor, link, m) -> complete(ks, futureSailor, keystates, m), () -> { - if (!ks.isDone()) { - log.warn("Failed to retrieve key state: {} from slice on: {}", idSeq, member.getId()); - ks.complete(null); - } - }, Executors.newScheduledThreadPool(1, Thread.ofVirtual().factory()), operationsFrequency); - try { - return ks.get(); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } catch (ExecutionException e) { - log.warn("Unable to retrieve key state: {} on: {}", idSeq, member.getId()); - } - return null; - } - - private boolean validate(EventCoordinates coordinates) { - log.info("Validate event: {} from slice on: {}", coordinates, member.getId()); - var succ = successors.stream().filter(m -> coordinates.getIdentifier().equals(m.getId())).findFirst(); - if (succ.isPresent()) { - return true; - } - var iterator = new SliceIterator<>("Retrieve KeyState", member, successors, communications); - var valid = new CompletableFuture(); - HashMultiset validations = HashMultiset.create(); - iterator.iterate((link, m) -> { - log.debug("Requesting Validation: {} from: {} on: {}", coordinates, link.getMember().getId(), - member.getId()); - return link.validate(coordinates.toEventCoords()); - }, (futureSailor, link, m) -> completeValidation(valid, futureSailor, validations, m), () -> { - if (!valid.isDone()) { - log.warn("Failed to validate: {} from slice on: {}", coordinates, member.getId()); - valid.complete(Validation.newBuilder().setResult(false).build()); - } - }, Executors.newScheduledThreadPool(1, Thread.ofVirtual().factory()), operationsFrequency); - try { - return valid.get().getResult(); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } catch (ExecutionException e) { - log.warn("Unable to validate: {} on: {}", coordinates, member.getId()); - } - return false; - } - - private record IdentifierSequence(Identifier identifier, ULong seqNum) { - public IdentAndSeq toIdSeq() { - return IdentAndSeq.newBuilder() - .setIdentifier(identifier.toIdent()) - .setSequenceNumber(seqNum.longValue()) - .build(); - } - - @Override - public String toString() { - return "{" + "identifier=" + identifier + ", seqNum=" + seqNum + '}'; - } - } - - private class BootstrapVerifier extends KeyStateVerifier { - - public BootstrapVerifier(Identifier identifier) { - super(identifier); - } - - @Override - protected KeyState getKeyState(ULong sequenceNumber) { - var key = new IdentifierSequence(identifier, sequenceNumber); - log.info("Get key state: {} on: {}", key, member.getId()); - return ksSeq.get(key); - } - } -} diff --git a/fireflies/src/main/java/com/salesforce/apollo/fireflies/NoteWrapper.java b/fireflies/src/main/java/com/salesforce/apollo/fireflies/NoteWrapper.java index 4482c0d693..b180e64da1 100644 --- a/fireflies/src/main/java/com/salesforce/apollo/fireflies/NoteWrapper.java +++ b/fireflies/src/main/java/com/salesforce/apollo/fireflies/NoteWrapper.java @@ -12,7 +12,6 @@ import com.salesforce.apollo.fireflies.proto.Note; import com.salesforce.apollo.fireflies.proto.Note.Builder; import com.salesforce.apollo.fireflies.proto.SignedNote; -import com.salesforce.apollo.stereotomy.EventCoordinates; import com.salesforce.apollo.stereotomy.identifier.Identifier; import com.salesforce.apollo.stereotomy.identifier.SelfAddressingIdentifier; @@ -41,10 +40,6 @@ public Digest currentView() { return currentView; } - public EventCoordinates getCoordinates() { - return EventCoordinates.from(note.getNote().getCoordinates()); - } - public long getEpoch() { return note.getNote().getEpoch(); } @@ -62,7 +57,7 @@ public Digest getId() { } public SelfAddressingIdentifier getIdentifier() { - return (SelfAddressingIdentifier) Identifier.from(note.getNote().getCoordinates().getIdentifier()); + return (SelfAddressingIdentifier) Identifier.from(note.getNote().getIdentifier()); } public BitSet getMask() { diff --git a/fireflies/src/main/java/com/salesforce/apollo/fireflies/View.java b/fireflies/src/main/java/com/salesforce/apollo/fireflies/View.java index 97dc711ee6..006cd320c5 100644 --- a/fireflies/src/main/java/com/salesforce/apollo/fireflies/View.java +++ b/fireflies/src/main/java/com/salesforce/apollo/fireflies/View.java @@ -31,15 +31,9 @@ import com.salesforce.apollo.membership.*; import com.salesforce.apollo.membership.stereotomy.ControlledIdentifierMember; import com.salesforce.apollo.ring.RingCommunications; -import com.salesforce.apollo.stereotomy.EventCoordinates; import com.salesforce.apollo.stereotomy.EventValidation; import com.salesforce.apollo.stereotomy.Verifiers; -import com.salesforce.apollo.stereotomy.event.KeyEvent; -import com.salesforce.apollo.stereotomy.event.proto.EventCoords; -import com.salesforce.apollo.stereotomy.event.proto.IdentAndSeq; -import com.salesforce.apollo.stereotomy.event.proto.KeyEvent_; import com.salesforce.apollo.stereotomy.event.proto.KeyState_; -import com.salesforce.apollo.stereotomy.identifier.Identifier; import com.salesforce.apollo.stereotomy.identifier.SelfAddressingIdentifier; import com.salesforce.apollo.utils.BbBackedInputStream; import com.salesforce.apollo.utils.Entropy; @@ -47,7 +41,6 @@ import io.grpc.Status; import io.grpc.StatusRuntimeException; import io.grpc.stub.StreamObserver; -import org.joou.ULong; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -213,8 +206,8 @@ public void start(CompletableFuture onJoin, Duration d, List seedpod Entropy.secureShuffle(seeds); viewManagement.start(onJoin, seeds.isEmpty()); - log.info("Starting: {} cardinality: {} tolerance: {} seeds: {} on: {}", context.getId(), context.cardinality(), - context.toleranceLevel(), seeds.size(), node.getId()); + log.info("Starting: {} cardinality: {} tolerance: {} seeds: {} on: {}", context.getId(), + viewManagement.cardinality(), context.toleranceLevel(), seeds.size(), node.getId()); viewManagement.clear(); roundTimers.reset(); context.clear(); @@ -235,9 +228,7 @@ public void start(CompletableFuture onJoin, Duration d, List seedpod */ public void start(Runnable onJoin, Duration d, List seedpods) { final var futureSailor = new CompletableFuture(); - futureSailor.whenComplete((v, t) -> { - onJoin.run(); - }); + futureSailor.whenComplete((v, t) -> onJoin.run()); start(futureSailor, d, seedpods); } @@ -251,16 +242,14 @@ public void stop() { roundTimers.reset(); comm.deregister(context.getId()); pendingRebuttals.clear(); - context.active().forEach(m -> { - context.offline(m); - }); + context.active().forEach(context::offline); final var current = futureGossip; futureGossip = null; if (current != null) { current.cancel(true); } observations.clear(); - timers.values().forEach(t -> t.cancel()); + timers.values().forEach(RoundScheduler.Timer::cancel); timers.clear(); viewManagement.clear(); } @@ -320,12 +309,14 @@ boolean addToView(NoteWrapper note) { if (accused) { checkInvalidations(member); } - if (!viewManagement.joined() && context.totalCount() == context.cardinality()) { - assert context.totalCount() == context.cardinality(); + if (!viewManagement.joined() && context.totalCount() == viewManagement.cardinality()) { + assert context.totalCount() == viewManagement.cardinality(); viewManagement.join(); } else { - assert context.totalCount() <= context.cardinality() : "total: " + context.totalCount() + " card: " - + context.cardinality(); + // This assertion needs to accommodate invalid diadem cardinality during view installation, as the diadem + // is from the previous view until all joining member have... joined. + assert context.totalCount() <= Math.max(viewManagement.cardinality(), context.cardinality()) : "total: " + + context.totalCount() + " card: " + viewManagement.cardinality(); } return true; }); @@ -359,9 +350,9 @@ void finalizeViewChange() { HashMultiset ballots = HashMultiset.create(); observations.values().forEach(vc -> { final var leaving = new ArrayList<>( - vc.getChange().getLeavesList().stream().map(d -> Digest.from(d)).collect(Collectors.toSet())); + vc.getChange().getLeavesList().stream().map(Digest::from).collect(Collectors.toSet())); final var joining = new ArrayList<>( - vc.getChange().getJoinsList().stream().map(d -> Digest.from(d)).collect(Collectors.toSet())); + vc.getChange().getJoinsList().stream().map(Digest::from).collect(Collectors.toSet())); leaving.sort(Ordering.natural()); joining.sort(Ordering.natural()); ballots.add(new Ballot(Digest.from(vc.getChange().getCurrent()), leaving, joining, digestAlgo)); @@ -372,14 +363,14 @@ void finalizeViewChange() { .orElse(null); if (max != null && max.getCount() >= superMajority) { log.info("Fast path consensus successful: {} required: {} cardinality: {} for: {} on: {}", max, - superMajority, context.cardinality(), currentView(), node.getId()); + superMajority, viewManagement.cardinality(), currentView(), node.getId()); viewManagement.install(max.getElement()); observations.clear(); } else { @SuppressWarnings("unchecked") final var reversed = Comparator.comparing(e -> ((Entry) e).getCount()).reversed(); log.info("Fast path consensus failed: {}, required: {} cardinality: {} ballots: {} for: {} on: {}", - observations.size(), superMajority, context.cardinality(), + observations.size(), superMajority, viewManagement.cardinality(), ballots.entrySet().stream().sorted(reversed).limit(1).toList(), currentView(), node.getId()); } @@ -414,7 +405,7 @@ void introduced() { return viewManagement.join(duration, timer); } - void notifyListeners(List joining, List leaving) { + void notifyListeners(List joining, List leaving) { final var current = currentView(); lifecycleListeners.forEach((id, listener) -> { try { @@ -509,7 +500,7 @@ void scheduleFinalizeViewChange(final int finalizeViewRounds) { // log.trace("View change finalization scheduled: {} rounds for: {} joining: {} leaving: {} on: {}", // finalizeViewRounds, currentView(), joins.size(), context.getOffline().size(), node.getId()); timers.put(FINALIZE_VIEW_CHANGE, - roundTimers.schedule(FINALIZE_VIEW_CHANGE, () -> finalizeViewChange(), finalizeViewRounds)); + roundTimers.schedule(FINALIZE_VIEW_CHANGE, this::finalizeViewChange, finalizeViewRounds)); } void scheduleViewChange() { @@ -520,8 +511,7 @@ void scheduleViewChange(final int viewChangeRounds) { // log.trace("Schedule view change: {} rounds for: {} on: {}", viewChangeRounds, currentView(), // node.getId()); timers.put(SCHEDULED_VIEW_CHANGE, - roundTimers.schedule(SCHEDULED_VIEW_CHANGE, () -> viewManagement.maybeViewChange(), - viewChangeRounds)); + roundTimers.schedule(SCHEDULED_VIEW_CHANGE, viewManagement::maybeViewChange, viewChangeRounds)); } T stable(Callable call) { @@ -570,6 +560,10 @@ void tick() { roundTimers.tick(); } + boolean validate(SelfAddressingIdentifier identifier) { + return validation.validate(identifier); + } + void viewChange(Runnable r) { // log.error("Enter view change on: {}", node.getId()); final var lock = viewChange.writeLock(); @@ -588,7 +582,6 @@ void viewChange(Runnable r) { * @param ring - the index of the gossip ring the gossip is originating from in this view * @param link - the outbound communications to the paired member * @param ring - * @throws Exception */ protected Gossip gossip(Fireflies link, int ring) { tick(); @@ -861,7 +854,7 @@ private boolean addJoin(SignedNote sn) { return false; } - if (!validation.validate(note.getCoordinates())) { + if (!validation.validate(note.getIdentifier())) { log.trace("Invalid join note from {} on: {}", note.getId(), node.getId()); return false; } @@ -900,8 +893,8 @@ private boolean addToCurrentView(NoteWrapper note) { */ private void amplify(Participant target) { context.rings() - .filter(ring -> !target.isDisabled(ring.getIndex()) && target.equals( - ring.successor(node, m -> context.isActive(m)))) + .filter( + ring -> !target.isDisabled(ring.getIndex()) && target.equals(ring.successor(node, context::isActive))) .forEach(ring -> { log.trace("amplifying: {} ring: {} on: {}", target.getId(), ring.getIndex(), node.getId()); accuse(target, ring.getIndex(), new IllegalStateException("Amplifying accusation")); @@ -982,14 +975,13 @@ private void gc(Participant member) { private BloomFilter getAccusationsBff(long seed, double p) { BloomFilter bff = new BloomFilter.DigestBloomFilter(seed, Math.max(params.minimumBiffCardinality(), context.cardinality() * 2), p); - context.allMembers().flatMap(m -> m.getAccusations()).filter(e -> e != null).forEach(m -> bff.add(m.getHash())); + context.allMembers() + .flatMap(Participant::getAccusations) + .filter(Objects::nonNull) + .forEach(m -> bff.add(m.getHash())); return bff; } - private KeyEvent getEvent(EventCoordinates coordinates) { - return null; - } - /** * @param seed * @param p @@ -1281,7 +1273,7 @@ private void recover(Participant member) { return; } if (context.activate(member)) { - log.debug("Recovering: {} cardinality: {} count: {} on: {}", member.getId(), context.cardinality(), + log.debug("Recovering: {} cardinality: {} count: {} on: {}", member.getId(), viewManagement.cardinality(), context.totalCount(), node.getId()); } } @@ -1470,14 +1462,15 @@ public interface ViewLifecycleListener { * * @param context - the context for which the view change has occurred * @param viewId - the Digest identity of the new view - * @param joins - the list of joining member's establishment event - * @param leaves - the list of leaving member's ids + * @param joins - the list of joining member's id + * @param leaves - the list of leaving member's id */ - void viewChange(Context context, Digest viewId, List joins, List leaves); + void viewChange(Context context, Digest viewId, List joins, + List leaves); } - public record Seed(KeyEvent establishment, InetSocketAddress endpoint) { + public record Seed(SelfAddressingIdentifier identifier, InetSocketAddress endpoint) { } public class Node extends Participant implements SigningMember { @@ -1490,7 +1483,7 @@ public Node(ControlledIdentifierMember wrapped, InetSocketAddress endpoint) { .setEpoch(0) .setHost(endpoint.getHostName()) .setPort(endpoint.getPort()) - .setCoordinates(wrapped.getEvent().getCoordinates().toEventCoords()) + .setIdentifier(wrapped.getIdentifier().getIdentifier().toIdent()) .setMask(ByteString.copyFrom(nextMask().toByteArray())) .build(); var signedNote = SignedNote.newBuilder() @@ -1533,18 +1526,6 @@ public SelfAddressingIdentifier getIdentifier() { return wrapped.getIdentifier().getIdentifier(); } - @Override - public Seed_ getSeed() { - return Seed_.newBuilder() - .setNote(note.getWrapped()) - .setEstablishment(wrapped.getEvent().toKeyEvent_()) - .build(); - } - - public JohnHancock sign(byte[] message) { - return wrapped.sign(message); - } - @Override public JohnHancock sign(InputStream message) { return wrapped.sign(message); @@ -1589,7 +1570,6 @@ assert isValidMask(mask, context) : "Invalid mask: " + mask + " majority: " + co for (int i = 0; i < context.getRingCount() && i < accusations.length; i++) { if (accusations[i] != null) { mask.set(i, false); - continue; } } // clear masks from previous note @@ -1642,7 +1622,7 @@ void nextNote(Digest view) { void nextNote(long newEpoch, Digest view) { final var current = note; var n = current.newBuilder() - .setCoordinates(note.getCoordinates().toEventCoords()) + .setIdentifier(note.getIdentifier().toIdent()) .setEpoch(newEpoch) .setMask(ByteString.copyFrom(nextMask().toByteArray())) .setCurrentView(view.toDigeste()) @@ -1667,7 +1647,7 @@ void reset() { .setCurrentView(currentView().toDigeste()) .setHost(current.getHost()) .setPort(current.getPort()) - .setCoordinates(current.getCoordinates().toEventCoords()) + .setIdentifier(current.getIdentifier().toIdent()) .setMask(ByteString.copyFrom(nextMask().toByteArray())) .build(); SignedNote signedNote = SignedNote.newBuilder() @@ -1735,7 +1715,7 @@ public int getAccusationCount() { } public Iterable getEncodedAccusations() { - return getAccusations().map(w -> w.getWrapped()).toList(); + return getAccusations().map(AccusationWrapper::getWrapped).toList(); } @Override @@ -1747,13 +1727,8 @@ public SelfAddressingIdentifier getIdentifier() { return note.getIdentifier(); } - public Seed_ getSeed() { - final var establishment = getEvent(getNote().getCoordinates()); - return Seed_.newBuilder() - .setNote(note.getWrapped()) - .setEstablishment( - establishment == null ? KeyEvent_.getDefaultInstance() : establishment.toKeyEvent_()) - .build(); + public SignedNote getSignedNote() { + return note.getWrapped(); } @Override @@ -1837,7 +1812,7 @@ AccusationWrapper getAccusation(int ring) { } Stream getAccusations() { - return Arrays.asList(validAccusations).stream().filter(a -> a != null); + return Arrays.stream(validAccusations).filter(Objects::nonNull); } long getEpoch() { @@ -1903,21 +1878,6 @@ public void join(Join join, Digest from, StreamObserver responseObserve viewManagement.join(join, from, responseObserver, timer); } - @Override - public KeyState_ keyState(IdentAndSeq request, Digest from) { - var identifier = Identifier.from(request.getIdentifier()); - var seq = ULong.valueOf(request.getSequenceNumber()); - - if (!viewManagement.joined()) { - log.info("Not yet joined!, ignoring key state request: {}:{} from: {} on: {}", identifier, seq, from, - node.getId()); - return KeyState_.getDefaultInstance(); - } - - var keyState = validation.keyState(identifier, seq); - return keyState == null ? KeyState_.getDefaultInstance() : keyState.toKeyState_(); - } - /** * The first message in the anti-entropy protocol. Process any digests from the inbound gossip digest. Respond * with the Gossip that represents the digests newer or not known in this view, as well as updates from this @@ -2011,8 +1971,7 @@ public void update(State request, Digest from) { validate(from, request); final var ring = request.getRing(); if (!context.validRing(ring)) { - log.debug("invalid ring: {} current: {} from: {} on: {}", ring, currentView(), ring, from, - node.getId()); + log.debug("invalid ring: {} current: {} from: {} on: {}", ring, currentView(), from, node.getId()); throw new StatusRuntimeException( Status.INVALID_ARGUMENT.withDescription("No successor of: " + from)); } @@ -2033,19 +1992,5 @@ public void update(State request, Digest from) { } }); } - - @Override - public Validation validateCoords(EventCoords request, Digest from) { - var coordinates = EventCoordinates.from(request); - if (!viewManagement.joined()) { - log.info("Not yet joined!, ignoring validation request: {} from: {} on: {}", from, coordinates, - node.getId()); - return Validation.newBuilder().setResult(false).build(); - } - log.info("Validating event: {} for: {} on: {}", request, from, node.getId()); - var validate = validation.validate(coordinates); - log.info("Returning validate: {}:{} to: {} on: {}", coordinates, validate, from, node.getId()); - return Validation.newBuilder().setResult(validate).build(); - } } } diff --git a/fireflies/src/main/java/com/salesforce/apollo/fireflies/ViewManagement.java b/fireflies/src/main/java/com/salesforce/apollo/fireflies/ViewManagement.java index 09bb5878f0..0033e63201 100644 --- a/fireflies/src/main/java/com/salesforce/apollo/fireflies/ViewManagement.java +++ b/fireflies/src/main/java/com/salesforce/apollo/fireflies/ViewManagement.java @@ -21,7 +21,7 @@ import com.salesforce.apollo.membership.Context; import com.salesforce.apollo.membership.ReservoirSampler; import com.salesforce.apollo.ring.SliceIterator; -import com.salesforce.apollo.stereotomy.EventCoordinates; +import com.salesforce.apollo.stereotomy.identifier.SelfAddressingIdentifier; import com.salesforce.apollo.utils.Entropy; import com.salesforce.apollo.utils.Utils; import io.grpc.Status; @@ -61,9 +61,9 @@ public class ViewManagement { private final View view; private final AtomicReference vote = new AtomicReference<>(); private final Lock joinLock = new ReentrantLock(); + private final AtomicReference currentView = new AtomicReference<>(); + private final AtomicReference diadem = new AtomicReference<>(); private boolean bootstrap; - private AtomicReference currentView = new AtomicReference<>(); - private AtomicReference diadem = new AtomicReference<>(); private CompletableFuture onJoined; ViewManagement(View view, Context context, Parameters params, FireflyMetrics metrics, Node node, @@ -78,10 +78,6 @@ public class ViewManagement { bootstrapView = currentView.get(); } - public boolean contains(Digest member) { - return diadem.get().contains(member); - } - boolean addJoin(Digest id, NoteWrapper note) { return joins.put(id, note) == null; } @@ -106,8 +102,9 @@ void bootstrap(NoteWrapper nw, final Duration dur) { } } - Digest bootstrapView() { - return bootstrapView; + int cardinality() { + var hex = diadem.get(); + return hex != null ? hex.getCardinality() : context.cardinality(); } void clear() { @@ -119,6 +116,10 @@ void clearVote() { vote.set(null); } + boolean contains(Digest member) { + return diadem.get().contains(member); + } + Digest currentView() { return currentView.get(); } @@ -131,7 +132,7 @@ Digest currentView() { BloomFilter getJoinsBff(long seed, double p) { BloomFilter bff = new BloomFilter.DigestBloomFilter(seed, Math.max(params.minimumBiffCardinality(), joins.size() * 2), p); - joins.keySet().forEach(d -> bff.add(d)); + joins.keySet().forEach(bff::add); return bff; } @@ -149,7 +150,7 @@ void install(Ballot ballot) { context.offlineCount(), node.getId()); attempt.set(0); - ballot.leaving.stream().filter(d -> !node.getId().equals(d)).forEach(p -> view.remove(p)); + ballot.leaving.stream().filter(d -> !node.getId().equals(d)).forEach(view::remove); final var seedSet = context.sample(params.maximumTxfr(), Entropy.bitsStream(), node.getId()) .stream() @@ -157,24 +158,24 @@ void install(Ballot ballot) { .collect(Collectors.toSet()); context.rebalance(context.totalCount() + ballot.joining.size()); - var joining = new ArrayList(); + var joining = new ArrayList(); var pending = ballot.joining() .stream() - .map(d -> joins.remove(d)) - .filter(sn -> sn != null) - .peek(nw -> joining.add(nw.getCoordinates())) - .peek(nw -> view.addToView(nw)) + .map(joins::remove) + .filter(java.util.Objects::nonNull) + .peek(nw -> joining.add(nw.getIdentifier())) + .peek(view::addToView) .peek(nw -> { if (metrics != null) { metrics.joins().mark(); } }) .map(nw -> pendingJoins.remove(nw.getId())) - .filter(p -> p != null) + .filter(java.util.Objects::nonNull) .toList(); setDiadem( - HexBloom.construct(context.memberCount(), context.allMembers().map(p -> p.getId()), view.bootstrapView(), + HexBloom.construct(context.memberCount(), context.allMembers().map(Participant::getId), view.bootstrapView(), params.crowns())); view.reset(); // complete all pending joins @@ -191,8 +192,8 @@ void install(Ballot ballot) { log.info( "Installed view: {} from: {} crown: {} for context: {} cardinality: {} count: {} pending: {} leaving: {} joining: {} on: {}", - currentView.get(), previousView, diadem.get(), context.getId(), context.cardinality(), - context.allMembers().count(), pending.size(), ballot.leaving.size(), ballot.joining.size(), node.getId()); + currentView.get(), previousView, diadem.get(), context.getId(), cardinality(), context.allMembers().count(), + pending.size(), ballot.leaving.size(), ballot.joining.size(), node.getId()); view.notifyListeners(joining, ballot.leaving); } @@ -203,25 +204,25 @@ void install(Ballot ballot) { void join() { joinLock.lock(); try { - assert context.totalCount() == context.cardinality(); + assert context.totalCount() == cardinality(); if (joined()) { return; } var current = currentView(); - log.info("Joining view: {} cardinality: {} count: {} on: {}", current, context.cardinality(), - context.totalCount(), node.getId()); - var calculated = HexBloom.construct(context.totalCount(), context.allMembers().map(p -> p.getId()), + log.info("Joining view: {} cardinality: {} count: {} on: {}", current, cardinality(), context.totalCount(), + node.getId()); + var calculated = HexBloom.construct(context.totalCount(), context.allMembers().map(Participant::getId), view.bootstrapView(), params.crowns()); if (!current.equals(calculated.compactWrapped())) { log.error("Crown: {} does not produce view: {} cardinality: {} count: {} on: {}", - calculated.compactWrapped(), currentView(), context.cardinality(), context.totalCount(), + calculated.compactWrapped(), currentView(), cardinality(), context.totalCount(), node.getId()); view.stop(); throw new IllegalStateException("Invalid crown"); } setDiadem(calculated); - view.notifyListeners(context.allMembers().map(p -> p.note.getCoordinates()).toList(), + view.notifyListeners(context.allMembers().map(p -> p.note.getIdentifier()).toList(), Collections.emptyList()); view.scheduleViewChange(); @@ -229,8 +230,8 @@ void join() { if (metrics != null) { metrics.viewChanges().mark(); } - log.info("Joined view: {} cardinality: {} count: {} on: {}", current, context.cardinality(), - context.totalCount(), node.getId()); + log.info("Joined view: {} cardinality: {} count: {} on: {}", current, cardinality(), context.totalCount(), + node.getId()); onJoined.complete(null); } finally { joinLock.unlock(); @@ -245,19 +246,27 @@ void join(Join join, Digest from, StreamObserver responseObserver, Time "Not joined, ignored join of view: %s from: %s on: %s".formatted(joinView, from, node.getId())))); return; } + var note = new NoteWrapper(join.getNote(), digestAlgo); + if (!from.equals(note.getId())) { + log.info("Ignored join of view: {} from: {} does not match: {} on: {}", joinView, from, note.getId(), + node.getId()); + responseObserver.onError( + new StatusRuntimeException(Status.INVALID_ARGUMENT.withDescription("Member does not match note"))); + return; + } + if (!view.validate(note.getIdentifier())) { + responseObserver.onError( + new StatusRuntimeException(Status.INVALID_ARGUMENT.withDescription("Invalid identifier"))); + log.info("Ignored join of view: {} from: {} invalid identifier on: {}", joinView, from, node.getId()); + return; + } view.stable(() -> { var thisView = currentView(); - var note = new NoteWrapper(join.getNote(), digestAlgo); - if (!from.equals(note.getId())) { - responseObserver.onError( - new StatusRuntimeException(Status.INVALID_ARGUMENT.withDescription("Member does not match note"))); - return; - } log.debug("Join requested from: {} view: {} context: {} cardinality: {} on: {}", from, thisView, - context.getId(), context.cardinality(), node.getId()); + context.getId(), cardinality(), node.getId()); if (contains(from)) { log.debug("Already a member: {} view: {} context: {} cardinality: {} on: {}", from, thisView, - context.getId(), context.cardinality(), node.getId()); + context.getId(), cardinality(), node.getId()); joined(context.sample(params.maximumTxfr(), Entropy.bitsStream(), node.getId()) .stream() .map(p -> p.note.getWrapped()) @@ -272,8 +281,7 @@ void join(Join join, Digest from, StreamObserver responseObserver, Time if (!View.isValidMask(note.getMask(), context)) { log.warn( "Invalid join mask: {} majority: {} from member: {} view: {} context: {} cardinality: {} on: {}", - note.getMask(), context.majority(), from, thisView, context.getId(), context.cardinality(), - node.getId()); + note.getMask(), context.majority(true), from, thisView, context.getId(), cardinality(), node.getId()); } if (pendingJoins.size() >= params.maxPending()) { responseObserver.onError( @@ -282,7 +290,7 @@ void join(Join join, Digest from, StreamObserver responseObserver, Time } pendingJoins.computeIfAbsent(from, d -> seeds -> { log.info("Gateway established for: {} view: {} context: {} cardinality: {} on: {}", from, - currentView(), context.getId(), context.cardinality(), node.getId()); + currentView(), context.getId(), cardinality(), node.getId()); joined(seeds, from, responseObserver, timer); }); joins.put(note.getId(), note); @@ -292,45 +300,43 @@ void join(Join join, Digest from, StreamObserver responseObserver, Time } BiConsumer join(Duration duration, Timer.Context timer) { - return (bound, t) -> { - view.viewChange(() -> { - final var hex = bound.view(); - if (t != null) { - log.error("Failed to join view on: {}", node.getId(), t); - view.stop(); - return; - } + return (bound, t) -> view.viewChange(() -> { + final var hex = bound.view(); + if (t != null) { + log.error("Failed to join view on: {}", node.getId(), t); + view.stop(); + return; + } - log.info("Rebalancing to cardinality: {} (join) for: {} context: {} on: {}", hex.getCardinality(), - hex.compact(), context.getId(), node.getId()); - context.rebalance(hex.getCardinality()); - context.activate(node); - diadem.set(hex); - currentView.set(hex.compact()); + log.info("Rebalancing to cardinality: {} (join) for: {} context: {} on: {}", hex.getCardinality(), + hex.compact(), context.getId(), node.getId()); + context.rebalance(hex.getCardinality()); + context.activate(node); + diadem.set(hex); + currentView.set(hex.compact()); - bound.successors().forEach(nw -> view.addToView(nw)); - bound.initialSeedSet().forEach(nw -> view.addToView(nw)); + bound.successors().forEach(view::addToView); + bound.initialSeedSet().forEach(view::addToView); - view.reset(); + view.reset(); - context.allMembers().forEach(p -> p.clearAccusations()); + context.allMembers().forEach(Participant::clearAccusations); - view.schedule(duration); + view.schedule(duration); - if (timer != null) { - timer.stop(); - } + if (timer != null) { + timer.stop(); + } - view.introduced(); - log.info("Currently joining view: {} seeds: {} cardinality: {} count: {} on: {}", currentView.get(), - bound.successors().size(), context.cardinality(), context.totalCount(), node.getId()); - if (context.totalCount() == context.cardinality()) { - join(); - } else { - populate(new ArrayList(context.activeMembers())); - } - }); - }; + view.introduced(); + log.info("Currently joining view: {} seeds: {} cardinality: {} count: {} on: {}", currentView.get(), + bound.successors().size(), cardinality(), context.totalCount(), node.getId()); + if (context.totalCount() == cardinality()) { + join(); + } else { + populate(new ArrayList<>(context.activeMembers())); + } + }); } void joinUpdatesFor(BloomFilter joinBff, Builder builder) { @@ -349,7 +355,7 @@ boolean joined() { * start a view change if there's any offline members or joining members */ void maybeViewChange() { - if (context.offlineCount() > 0 || joins.size() > 0) { + if (context.offlineCount() > 0 || !joins.isEmpty()) { initiateViewChange(); } else { view.scheduleViewChange(); @@ -360,31 +366,24 @@ void populate(List sample) { var populate = new SliceIterator("Populate: " + context.getId(), node, sample, view.comm); var repopulate = new AtomicReference(); var scheduler = Executors.newScheduledThreadPool(1, Thread.ofVirtual().factory()); - repopulate.set(() -> { - populate.iterate((link, m) -> { - log.debug("Populating: {} contacting: {} on: {}", context.getId(), link.getMember().getId(), - node.getId()); - view.tick(); - return view.gossip(link, 0); - }, (futureSailor, link, m) -> { - futureSailor.ifPresent(g -> { - if (g.hasRedirect()) { - final Participant member = (Participant) link.getMember(); - if (g.hasRedirect()) { - view.stable(() -> view.redirect(member, g, 0)); - } - } else { - view.stable(() -> view.processUpdates(g)); - } - }); - return !joined(); - }, () -> { - if (!joined()) { - scheduler.schedule(() -> Thread.ofVirtual().start(Utils.wrapped(repopulate.get(), log)), 500, - TimeUnit.MILLISECONDS); + repopulate.set(() -> populate.iterate((link, m) -> { + return view.gossip(link, 0); + }, (futureSailor, link, m) -> { + futureSailor.ifPresent(g -> { + if (g.hasRedirect()) { + final Participant member = (Participant) link.getMember(); + view.stable(() -> view.redirect(member, g, 0)); + } else { + view.stable(() -> view.processUpdates(g)); } - }, scheduler, Duration.ofMillis(500)); - }); + }); + return !joined(); + }, () -> { + if (!joined()) { + scheduler.schedule(() -> Thread.ofVirtual().start(Utils.wrapped(repopulate.get(), log)), 500, + TimeUnit.MILLISECONDS); + } + }, scheduler, Duration.ofMillis(500))); repopulate.get().run(); } @@ -395,7 +394,7 @@ JoinGossip.Builder processJoins(BloomFilter bff) { joins.entrySet() .stream() .filter(m -> !bff.contains(m.getKey())) - .map(m -> m.getValue()) + .map(Map.Entry::getValue) .collect(new ReservoirSampler<>(params.maximumTxfr(), Entropy.bitsStream())) .forEach(n -> builder.addUpdates(n.getWrapped())); return builder; @@ -433,7 +432,8 @@ Redirect seed(Registration registration, Digest from) { return Redirect.getDefaultInstance(); } if (!bootstrapView.equals(requestView)) { - log.warn("Invalid bootstrap view: {} expected: {} from: {} on: {}", requestView, from, node.getId()); + log.warn("Invalid bootstrap view: {} expected: {} from: {} on: {}", bootstrapView, requestView, from, + node.getId()); return Redirect.getDefaultInstance(); } var note = new NoteWrapper(registration.getNote(), digestAlgo); @@ -442,6 +442,10 @@ Redirect seed(Registration registration, Digest from) { node.getId()); return Redirect.getDefaultInstance(); } + if (!view.validate(note.getIdentifier())) { + log.warn("Invalid identifier: {} from: {} on: {}", note.getIdentifier(), from, node.getId()); + return Redirect.getDefaultInstance(); + } return view.stable(() -> { var newMember = view.new Participant(note.getId()); final var sample = context.sample(params.maximumTxfr(), Entropy.bitsStream(), (Digest) null); @@ -450,8 +454,9 @@ Redirect seed(Registration registration, Digest from) { context.getId(), sample.size(), node.getId()); return Redirect.newBuilder() .setView(currentView().toDigeste()) - .addAllSample(sample.stream().filter(p -> p != null).map(p -> p.getSeed()).toList()) - .setCardinality(context.cardinality()) + .addAllSample( + sample.stream().filter(java.util.Objects::nonNull).map(Participant::getSignedNote).toList()) + .setCardinality(cardinality()) .setBootstrap(bootstrap) .setRings(context.getRingCount()) .build(); @@ -484,8 +489,8 @@ private void initiateViewChange() { .setCurrent(currentView().toDigeste()) .setAttempt(attempt.getAndIncrement()) .addAllLeaves( - view.streamShunned().map(id -> id.toDigeste()).collect(Collectors.toSet())) - .addAllJoins(joins.keySet().stream().map(id -> id.toDigeste()).toList()); + view.streamShunned().map(Digest::toDigeste).collect(Collectors.toSet())) + .addAllJoins(joins.keySet().stream().map(Digest::toDigeste).toList()); ViewChange change = builder.build(); vote.set(change); var signature = node.sign(change.toByteString()); @@ -501,11 +506,11 @@ private void initiateViewChange() { private void joined(Collection seedSet, Digest from, StreamObserver responseObserver, Timer.Context timer) { - var unique = new HashSet(seedSet); - final var initialSeeds = new ArrayList(seedSet); + var unique = new HashSet<>(seedSet); + final var initialSeeds = new ArrayList<>(seedSet); final var successors = new HashSet(); - context.successors(from, m -> context.isActive(m)).forEach(p -> { + context.successors(from, context::isActive).forEach(p -> { var sn = p.getNote().getWrapped(); if (unique.add(sn)) { initialSeeds.add(sn); @@ -538,19 +543,18 @@ private void setDiadem(final HexBloom hex) { record Ballot(Digest view, List leaving, List joining, int hash) { Ballot(Digest view, List leaving, List joining, DigestAlgorithm algo) { - this(view, leaving, joining, view.xor(joining.stream().reduce((a, b) -> a.xor(b)).orElse(algo.getOrigin())) + this(view, leaving, joining, view.xor(joining.stream().reduce(Digest::xor).orElse(algo.getOrigin())) .xor(leaving.stream() - .reduce((a, b) -> a.xor(b)) + .reduce(Digest::xor) .orElse(algo.getOrigin()) - .xor(joining.stream() - .reduce((a, b) -> a.xor(b)) - .orElse(algo.getOrigin()))) + .xor( + joining.stream().reduce(Digest::xor).orElse(algo.getOrigin()))) .hashCode()); } @Override public boolean equals(Object obj) { - if (obj != null && obj instanceof Ballot b) { + if (obj instanceof Ballot b) { return Objects.equal(view, b.view) && Objects.equal(leaving, b.leaving) && Objects.equal(joining, b.joining); } diff --git a/fireflies/src/main/java/com/salesforce/apollo/fireflies/comm/entrance/Entrance.java b/fireflies/src/main/java/com/salesforce/apollo/fireflies/comm/entrance/Entrance.java index b2cbd3f4b9..98b63d0533 100644 --- a/fireflies/src/main/java/com/salesforce/apollo/fireflies/comm/entrance/Entrance.java +++ b/fireflies/src/main/java/com/salesforce/apollo/fireflies/comm/entrance/Entrance.java @@ -8,11 +8,11 @@ import com.salesforce.apollo.archipelago.Link; import com.salesforce.apollo.fireflies.View.Node; -import com.salesforce.apollo.fireflies.proto.*; +import com.salesforce.apollo.fireflies.proto.Gateway; +import com.salesforce.apollo.fireflies.proto.Join; +import com.salesforce.apollo.fireflies.proto.Redirect; +import com.salesforce.apollo.fireflies.proto.Registration; import com.salesforce.apollo.membership.Member; -import com.salesforce.apollo.stereotomy.event.proto.EventCoords; -import com.salesforce.apollo.stereotomy.event.proto.IdentAndSeq; -import com.salesforce.apollo.stereotomy.event.proto.KeyState_; import java.io.IOException; import java.time.Duration; @@ -39,28 +39,14 @@ public Gateway join(Join join, Duration timeout) { return null; } - @Override - public KeyState_ keyState(IdentAndSeq idAndSeq) { - return null; - } - @Override public Redirect seed(Registration registration) { return null; } - - @Override - public Validation validate(EventCoords coords) { - return service.validateCoords(coords, getMember().getId()); - } }; } Gateway join(Join join, Duration timeout); - KeyState_ keyState(IdentAndSeq idAndSeq); - Redirect seed(Registration registration); - - Validation validate(EventCoords coords); } diff --git a/fireflies/src/main/java/com/salesforce/apollo/fireflies/comm/entrance/EntranceClient.java b/fireflies/src/main/java/com/salesforce/apollo/fireflies/comm/entrance/EntranceClient.java index 53953a4bf6..8ae3c58bb3 100644 --- a/fireflies/src/main/java/com/salesforce/apollo/fireflies/comm/entrance/EntranceClient.java +++ b/fireflies/src/main/java/com/salesforce/apollo/fireflies/comm/entrance/EntranceClient.java @@ -11,9 +11,6 @@ import com.salesforce.apollo.fireflies.FireflyMetrics; import com.salesforce.apollo.fireflies.proto.*; import com.salesforce.apollo.membership.Member; -import com.salesforce.apollo.stereotomy.event.proto.EventCoords; -import com.salesforce.apollo.stereotomy.event.proto.IdentAndSeq; -import com.salesforce.apollo.stereotomy.event.proto.KeyState_; import java.time.Duration; import java.util.concurrent.TimeUnit; @@ -69,11 +66,6 @@ public Gateway join(Join join, Duration timeout) { return result; } - @Override - public KeyState_ keyState(IdentAndSeq idAndSeq) { - return client.keyState(idAndSeq); - } - @Override public Redirect seed(Registration registration) { if (metrics != null) { @@ -94,9 +86,4 @@ public Redirect seed(Registration registration) { return result; } - @Override - public Validation validate(EventCoords coords) { - return client.validate(coords); - } - } diff --git a/fireflies/src/main/java/com/salesforce/apollo/fireflies/comm/entrance/EntranceServer.java b/fireflies/src/main/java/com/salesforce/apollo/fireflies/comm/entrance/EntranceServer.java index 7032592507..bee570e970 100644 --- a/fireflies/src/main/java/com/salesforce/apollo/fireflies/comm/entrance/EntranceServer.java +++ b/fireflies/src/main/java/com/salesforce/apollo/fireflies/comm/entrance/EntranceServer.java @@ -12,11 +12,11 @@ import com.salesforce.apollo.fireflies.FireflyMetrics; import com.salesforce.apollo.fireflies.View.Service; import com.salesforce.apollo.fireflies.proto.EntranceGrpc.EntranceImplBase; -import com.salesforce.apollo.fireflies.proto.*; +import com.salesforce.apollo.fireflies.proto.Gateway; +import com.salesforce.apollo.fireflies.proto.Join; +import com.salesforce.apollo.fireflies.proto.Redirect; +import com.salesforce.apollo.fireflies.proto.Registration; import com.salesforce.apollo.protocols.ClientIdentity; -import com.salesforce.apollo.stereotomy.event.proto.EventCoords; -import com.salesforce.apollo.stereotomy.event.proto.IdentAndSeq; -import com.salesforce.apollo.stereotomy.event.proto.KeyState_; import io.grpc.stub.StreamObserver; /** @@ -56,36 +56,6 @@ public void join(Join request, StreamObserver responseObserver) { }); } - @Override - public void keyState(IdentAndSeq request, StreamObserver responseObserver) { - if (metrics != null) { - var serializedSize = request.getSerializedSize(); - metrics.inboundBandwidth().mark(serializedSize); - metrics.inboundSeed().update(serializedSize); - } - Digest from = identity.getFrom(); - if (from == null) { - responseObserver.onError(new IllegalStateException("Member has been removed")); - return; - } - router.evaluate(responseObserver, s -> { - KeyState_ r; - try { - r = s.keyState(request, from); - } catch (Throwable t) { - responseObserver.onError(t); - return; - } - responseObserver.onNext(r); - responseObserver.onCompleted(); - if (metrics != null) { - var serializedSize = r.getSerializedSize(); - metrics.outboundBandwidth().mark(serializedSize); - metrics.outboundRedirect().update(serializedSize); - } - }); - } - @Override public void seed(Registration request, StreamObserver responseObserver) { Context timer = metrics == null ? null : metrics.inboundSeedDuration().time(); @@ -117,24 +87,4 @@ public void seed(Registration request, StreamObserver responseObserver } }); } - - @Override - public void validate(EventCoords request, StreamObserver responseObserver) { - Digest from = identity.getFrom(); - if (from == null) { - responseObserver.onError(new IllegalStateException("Member has been removed")); - return; - } - router.evaluate(responseObserver, s -> { - Validation r; - try { - r = s.validateCoords(request, from); - } catch (Throwable t) { - responseObserver.onError(t); - return; - } - responseObserver.onNext(r); - responseObserver.onCompleted(); - }); - } } diff --git a/fireflies/src/main/java/com/salesforce/apollo/fireflies/comm/entrance/EntranceService.java b/fireflies/src/main/java/com/salesforce/apollo/fireflies/comm/entrance/EntranceService.java index 0c2f3dff14..a468b6b1ef 100644 --- a/fireflies/src/main/java/com/salesforce/apollo/fireflies/comm/entrance/EntranceService.java +++ b/fireflies/src/main/java/com/salesforce/apollo/fireflies/comm/entrance/EntranceService.java @@ -8,10 +8,10 @@ import com.codahale.metrics.Timer.Context; import com.salesforce.apollo.cryptography.Digest; -import com.salesforce.apollo.fireflies.proto.*; -import com.salesforce.apollo.stereotomy.event.proto.EventCoords; -import com.salesforce.apollo.stereotomy.event.proto.IdentAndSeq; -import com.salesforce.apollo.stereotomy.event.proto.KeyState_; +import com.salesforce.apollo.fireflies.proto.Gateway; +import com.salesforce.apollo.fireflies.proto.Join; +import com.salesforce.apollo.fireflies.proto.Redirect; +import com.salesforce.apollo.fireflies.proto.Registration; import io.grpc.stub.StreamObserver; /** @@ -21,9 +21,5 @@ public interface EntranceService { void join(Join request, Digest from, StreamObserver responseObserver, Context timer); - KeyState_ keyState(IdentAndSeq request, Digest from); - Redirect seed(Registration request, Digest from); - - Validation validateCoords(EventCoords request, Digest from); } diff --git a/fireflies/src/test/java/com/salesforce/apollo/fireflies/ChurnTest.java b/fireflies/src/test/java/com/salesforce/apollo/fireflies/ChurnTest.java index 0d1cc40769..d2e831c34b 100644 --- a/fireflies/src/test/java/com/salesforce/apollo/fireflies/ChurnTest.java +++ b/fireflies/src/test/java/com/salesforce/apollo/fireflies/ChurnTest.java @@ -49,8 +49,8 @@ public class ChurnTest { private static final double P_BYZ = 0.3; private static Map> identities; private static KERL.AppendKERL kerl; - private List communications = new ArrayList<>(); - private List gateways = new ArrayList<>(); + private final List communications = new ArrayList<>(); + private final List gateways = new ArrayList<>(); private Map members; private MetricRegistry node0Registry; private MetricRegistry registry; @@ -96,7 +96,7 @@ public void churn() throws Exception { var seeds = members.values() .stream() - .map(m -> new Seed(m.getEvent(), new InetSocketAddress(0))) + .map(m -> new Seed(m.getIdentifier().getIdentifier(), new InetSocketAddress(0))) .limit(25) .toList(); diff --git a/fireflies/src/test/java/com/salesforce/apollo/fireflies/E2ETest.java b/fireflies/src/test/java/com/salesforce/apollo/fireflies/E2ETest.java index c6845c417b..adea831afa 100644 --- a/fireflies/src/test/java/com/salesforce/apollo/fireflies/E2ETest.java +++ b/fireflies/src/test/java/com/salesforce/apollo/fireflies/E2ETest.java @@ -49,21 +49,21 @@ public class E2ETest { private static final int BIAS = 2; private static final int CARDINALITY; private static final double P_BYZ = 0.1; - private static Map> identities; - private static boolean largeTests = Boolean.getBoolean( + private static final boolean largeTests = Boolean.getBoolean( "large_tests"); + private static Map> identities; private static KERL.AppendKERL kerl; static { CARDINALITY = largeTests ? 30 : 12; } - private List communications = new ArrayList<>(); - private List gateways = new ArrayList<>(); - private Map members; - private MetricRegistry node0Registry; - private MetricRegistry registry; - private List views; + private final List communications = new ArrayList<>(); + private final List gateways = new ArrayList<>(); + private Map members; + private MetricRegistry node0Registry; + private MetricRegistry registry; + private List views; @BeforeAll public static void beforeClass() throws Exception { @@ -102,7 +102,7 @@ public void smokin() throws Exception { final var seeds = members.values() .stream() - .map(m -> new Seed(m.getEvent(), new InetSocketAddress(0))) + .map(m -> new Seed(m.getIdentifier().getIdentifier(), new InetSocketAddress(0))) .limit(largeTests ? 10 : 1) .toList(); final var bootstrapSeed = seeds.subList(0, 1); diff --git a/fireflies/src/test/java/com/salesforce/apollo/fireflies/MtlsTest.java b/fireflies/src/test/java/com/salesforce/apollo/fireflies/MtlsTest.java index 2d9a30989a..46b034018c 100644 --- a/fireflies/src/test/java/com/salesforce/apollo/fireflies/MtlsTest.java +++ b/fireflies/src/test/java/com/salesforce/apollo/fireflies/MtlsTest.java @@ -68,8 +68,8 @@ public class MtlsTest { CARDINALITY = LARGE_TESTS ? 20 : 10; } - private List communications = new ArrayList<>(); - private List views; + private final List communications = new ArrayList<>(); + private List views; @BeforeAll public static void beforeClass() throws Exception { @@ -111,7 +111,7 @@ public void smoke() throws Exception { var ctxBuilder = Context.newBuilder().setCardinality(CARDINALITY); var seeds = members.stream() - .map(m -> new Seed(m.getEvent(), endpoints.get(m.getId()))) + .map(m -> new Seed(m.getIdentifier().getIdentifier(), endpoints.get(m.getId()))) .limit(LARGE_TESTS ? 24 : 3) .toList(); @@ -211,10 +211,7 @@ public SslContext forServer(ClientAuth clientAuth, String alias, CertificateVali @Override public Digest getMemberId(X509Certificate key) { - return ((SelfAddressingIdentifier) Stereotomy.decode(key) - .get() - .coordinates() - .getIdentifier()).getDigest(); + return ((SelfAddressingIdentifier) Stereotomy.decode(key).get().identifier()).getDigest(); } }; } diff --git a/fireflies/src/test/java/com/salesforce/apollo/fireflies/SwarmTest.java b/fireflies/src/test/java/com/salesforce/apollo/fireflies/SwarmTest.java index 16ea84559b..09a3f949f0 100644 --- a/fireflies/src/test/java/com/salesforce/apollo/fireflies/SwarmTest.java +++ b/fireflies/src/test/java/com/salesforce/apollo/fireflies/SwarmTest.java @@ -50,21 +50,21 @@ public class SwarmTest { private static final int BIAS = 3; private static final int CARDINALITY; private static final double P_BYZ = 0.1; - private static Map> identities; - private static boolean largeTests = Boolean.getBoolean( + private static final boolean largeTests = Boolean.getBoolean( "large_tests"); + private static Map> identities; private static KERL.AppendKERL kerl; static { CARDINALITY = largeTests ? 100 : 50; } - private List communications = new ArrayList<>(); - private List gateways = new ArrayList<>(); - private Map members; - private MetricRegistry node0Registry; - private MetricRegistry registry; - private List views; + private final List communications = new ArrayList<>(); + private final List gateways = new ArrayList<>(); + private Map members; + private MetricRegistry node0Registry; + private MetricRegistry registry; + private List views; @BeforeAll public static void beforeClass() throws Exception { @@ -103,7 +103,7 @@ public void swarm() throws Exception { final var seeds = members.values() .stream() - .map(m -> new Seed(m.getEvent(), new InetSocketAddress(0))) + .map(m -> new Seed(m.getIdentifier().getIdentifier(), new InetSocketAddress(0))) .limit(largeTests ? 100 : 10) .toList(); final var bootstrapSeed = seeds.subList(0, 1); @@ -197,8 +197,8 @@ public void swarm() throws Exception { private void initialize() { var parameters = Parameters.newBuilder() - .setMaxPending(largeTests ? 10 : 10) - .setMaximumTxfr(largeTests ? 20 : 20) + .setMaxPending(10) + .setMaximumTxfr(20) .setJoinRetries(30) .setFpr(0.00000125) .setSeedingTimout(Duration.ofSeconds(10)) diff --git a/gorgoneion-client/src/test/java/com/salesforce/apollo/gorgoneion/client/GorgoneionClientTest.java b/gorgoneion-client/src/test/java/com/salesforce/apollo/gorgoneion/client/GorgoneionClientTest.java index 7a47198981..9c9fcf51e1 100644 --- a/gorgoneion-client/src/test/java/com/salesforce/apollo/gorgoneion/client/GorgoneionClientTest.java +++ b/gorgoneion-client/src/test/java/com/salesforce/apollo/gorgoneion/client/GorgoneionClientTest.java @@ -19,7 +19,6 @@ import com.salesforce.apollo.gorgoneion.comm.admissions.AdmissionsService; import com.salesforce.apollo.gorgoneion.proto.SignedNonce; import com.salesforce.apollo.membership.Context; -import com.salesforce.apollo.membership.Member; import com.salesforce.apollo.membership.stereotomy.ControlledIdentifierMember; import com.salesforce.apollo.stereotomy.StereotomyImpl; import com.salesforce.apollo.stereotomy.event.proto.Validations; @@ -36,7 +35,6 @@ import java.time.Duration; import java.util.UUID; import java.util.concurrent.CountDownLatch; -import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.function.Function; import java.util.stream.IntStream; @@ -61,7 +59,7 @@ public void clientSmoke() throws Exception { var stereotomy = new StereotomyImpl(new MemKeyStore(), kerl, entropy); final var prefix = UUID.randomUUID().toString(); var member = new ControlledIdentifierMember(stereotomy.newIdentifier()); - var context = Context.newBuilder().setCardinality(1).build(); + var context = Context.newBuilder().setCardinality(1).build(); context.activate(member); // Gorgoneion service comms @@ -72,8 +70,7 @@ public void clientSmoke() throws Exception { var observer = mock(ProtoEventObserver.class); final var parameters = Parameters.newBuilder().setKerl(kerl).build(); @SuppressWarnings("unused") - var gorgon = new Gorgoneion(parameters, member, context, observer, gorgonRouter, - Executors.newScheduledThreadPool(1, Thread.ofVirtual().factory()), null); + var gorgon = new Gorgoneion(true, t -> true, parameters, member, context, observer, gorgonRouter, null); // The registering client var client = new ControlledIdentifierMember(stereotomy.newIdentifier()); @@ -142,7 +139,7 @@ public Void answer(InvocationOnMock invocation) { } }).when(observer).publish(Mockito.any(), Mockito.anyList()); - var context = Context.newBuilder().setCardinality(members.size()).build(); + var context = Context.newBuilder().setCardinality(members.size()).build(); for (ControlledIdentifierMember member : members) { context.activate(member); } @@ -154,11 +151,9 @@ public Void answer(InvocationOnMock invocation) { router.start(); return router; }) - .map(r -> new Gorgoneion(parameters, (ControlledIdentifierMember) r.getFrom(), - context, observer, r, - Executors.newScheduledThreadPool(2, Thread.ofVirtual() - .factory()), - null)) + .map(r -> new Gorgoneion(r.getFrom().equals(members.getFirst()), t -> true, + parameters, (ControlledIdentifierMember) r.getFrom(), + context, observer, r, null)) .toList(); // The registering client @@ -175,7 +170,7 @@ public Void answer(InvocationOnMock invocation) { clientRouter.start(); // Admin client link - var admin = clientComminications.connect(members.get(0)); + var admin = clientComminications.connect(members.getFirst()); assertNotNull(admin); Function attester = sn -> { @@ -186,7 +181,7 @@ public Void answer(InvocationOnMock invocation) { var invitation = gorgoneionClient.apply(Duration.ofSeconds(2_000)); assertNotNull(invitation); assertNotEquals(Validations.getDefaultInstance(), invitation); - assertTrue(invitation.getValidationsCount() >= context.majority()); + assertTrue(invitation.getValidationsCount() >= context.majority(true)); assertTrue(countdown.await(1, TimeUnit.SECONDS)); } diff --git a/gorgoneion/src/main/java/com/salesforce/apollo/gorgoneion/Gorgoneion.java b/gorgoneion/src/main/java/com/salesforce/apollo/gorgoneion/Gorgoneion.java index 3d8245df60..028927e5eb 100644 --- a/gorgoneion/src/main/java/com/salesforce/apollo/gorgoneion/Gorgoneion.java +++ b/gorgoneion/src/main/java/com/salesforce/apollo/gorgoneion/Gorgoneion.java @@ -9,12 +9,6 @@ import com.codahale.metrics.Timer; import com.google.protobuf.Empty; import com.google.protobuf.Timestamp; -import com.salesforce.apollo.gorgoneion.proto.*; -import com.salesforce.apollo.stereotomy.event.proto.Ident; -import com.salesforce.apollo.stereotomy.event.proto.KERL_; -import com.salesforce.apollo.stereotomy.event.proto.Validation_; -import com.salesforce.apollo.stereotomy.event.proto.Validations; -import com.salesforce.apollo.cryptography.proto.Digeste; import com.salesforce.apollo.archipelago.Router; import com.salesforce.apollo.archipelago.RouterImpl.CommonCommunications; import com.salesforce.apollo.cryptography.Digest; @@ -22,6 +16,7 @@ import com.salesforce.apollo.cryptography.Signer; import com.salesforce.apollo.cryptography.Verifier; import com.salesforce.apollo.cryptography.Verifier.DefaultVerifier; +import com.salesforce.apollo.cryptography.proto.Digeste; import com.salesforce.apollo.gorgoneion.comm.GorgoneionMetrics; import com.salesforce.apollo.gorgoneion.comm.admissions.AdmissionsServer; import com.salesforce.apollo.gorgoneion.comm.admissions.AdmissionsService; @@ -29,6 +24,7 @@ import com.salesforce.apollo.gorgoneion.comm.endorsement.EndorsementClient; import com.salesforce.apollo.gorgoneion.comm.endorsement.EndorsementServer; import com.salesforce.apollo.gorgoneion.comm.endorsement.EndorsementService; +import com.salesforce.apollo.gorgoneion.proto.*; import com.salesforce.apollo.membership.Context; import com.salesforce.apollo.membership.Member; import com.salesforce.apollo.membership.stereotomy.ControlledIdentifierMember; @@ -36,6 +32,10 @@ import com.salesforce.apollo.stereotomy.EventCoordinates; import com.salesforce.apollo.stereotomy.event.EstablishmentEvent; import com.salesforce.apollo.stereotomy.event.InceptionEvent; +import com.salesforce.apollo.stereotomy.event.proto.Ident; +import com.salesforce.apollo.stereotomy.event.proto.KERL_; +import com.salesforce.apollo.stereotomy.event.proto.Validation_; +import com.salesforce.apollo.stereotomy.event.proto.Validations; import com.salesforce.apollo.stereotomy.event.protobuf.ProtobufEventFactory; import com.salesforce.apollo.stereotomy.identifier.Identifier; import com.salesforce.apollo.stereotomy.identifier.SelfAddressingIdentifier; @@ -51,10 +51,8 @@ import java.util.HashSet; import java.util.Optional; import java.util.Set; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.*; +import java.util.function.Predicate; import static com.salesforce.apollo.stereotomy.event.protobuf.ProtobufEventFactory.digestOf; @@ -62,30 +60,35 @@ * @author hal.hildebrand */ public class Gorgoneion { - public static final Logger log = LoggerFactory.getLogger( - Gorgoneion.class); + public static final Logger log = LoggerFactory.getLogger(Gorgoneion.class); + @SuppressWarnings("unused") - private final CommonCommunications admissionsComm; - private final Context context; - private final CommonCommunications endorsementComm; - private final ControlledIdentifierMember member; - private final ProtoEventObserver observer; - private final Parameters parameters; - private final ScheduledExecutorService scheduler; - - public Gorgoneion(Parameters parameters, ControlledIdentifierMember member, Context context, - ProtoEventObserver observer, Router router, ScheduledExecutorService scheduler, - GorgoneionMetrics metrics) { - this(parameters, member, context, observer, router, scheduler, metrics, router); + private final CommonCommunications admissionsComm; + private final Context context; + private final CommonCommunications endorsementComm; + private final ControlledIdentifierMember member; + private final ProtoEventObserver observer; + private final Parameters parameters; + private final ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1, + Thread.ofVirtual() + .factory()); + private final Predicate verifier; + private final boolean bootstrap; + + public Gorgoneion(boolean bootstrap, Predicate verifier, Parameters parameters, + ControlledIdentifierMember member, Context context, ProtoEventObserver observer, + Router router, GorgoneionMetrics metrics) { + this(bootstrap, verifier, parameters, member, context, observer, router, metrics, router); } - public Gorgoneion(Parameters parameters, ControlledIdentifierMember member, Context context, - ProtoEventObserver observer, Router admissionsRouter, ScheduledExecutorService scheduler, - GorgoneionMetrics metrics, Router endorsementRouter) { + public Gorgoneion(boolean bootstrap, Predicate verifier, Parameters parameters, + ControlledIdentifierMember member, Context context, ProtoEventObserver observer, + Router admissionsRouter, GorgoneionMetrics metrics, Router endorsementRouter) { + this.bootstrap = bootstrap; + this.verifier = verifier; this.member = member; this.context = context; this.parameters = parameters; - this.scheduler = scheduler; this.observer = observer; admissionsComm = admissionsRouter.create(member, context.getId(), new Admit(), ":admissions", @@ -145,7 +148,7 @@ private SignedNonce generateNonce(KERL_ application) { if (identifier == null) { throw new IllegalArgumentException("No identifier"); } - log.debug("Generating nonce for: {} contacting: {} on: {}", identifier, identifier, member.getId()); + log.info("Generating nonce for: {} contacting: {} on: {}", identifier, identifier, member.getId()); var now = parameters.clock().instant(); final var ident = identifier.toIdent(); var nonce = Nonce.newBuilder() @@ -158,7 +161,7 @@ private SignedNonce generateNonce(KERL_ application) { var successors = context.totalCount() == 1 ? Collections.singletonList(member) : Context.uniqueSuccessors(context, digestOf(ident, parameters.digestAlgorithm())); - final var majority = context.totalCount() == 1 ? 1 : context.majority(); + final var majority = context.majority(true); final var redirecting = new SliceIterator<>("Nonce Endorsement", member, successors, endorsementComm); Set endorsements = Collections.newSetFromMap(new ConcurrentHashMap<>()); var generated = new CompletableFuture(); @@ -168,8 +171,10 @@ private SignedNonce generateNonce(KERL_ application) { return link.endorse(nonce, parameters.registrationTimeout()); }, (futureSailor, link, m) -> completeEndorsement(futureSailor, m, endorsements), () -> { if (endorsements.size() < majority) { - generated.completeExceptionally(new StatusRuntimeException( - Status.ABORTED.withDescription("Cannot gather required nonce endorsements"))); + generated.completeExceptionally(new StatusRuntimeException(Status.ABORTED.withDescription( + "Cannot gather required nonce endorsements: %s required: %s on: %s".formatted(endorsements.size(), + majority, + member.getId())))); } else { generated.complete(SignedNonce.newBuilder() .addSignatures(MemberSignature.newBuilder() @@ -190,7 +195,7 @@ private SignedNonce generateNonce(KERL_ application) { Thread.currentThread().interrupt(); return null; } catch (ExecutionException e) { - throw new RuntimeException(e); + throw new RuntimeException(e.getCause()); } } @@ -216,11 +221,11 @@ private void notarize(Credentials credentials, Validations validations) { var successors = Context.uniqueSuccessors(context, digestOf(identifier.toIdent(), parameters.digestAlgorithm())); - final var majority = context.activeCount() == 1 ? 0 : context.majority(); + final var majority = context.majority(true); SliceIterator redirecting = new SliceIterator<>("Enrollment", member, successors, endorsementComm); var completed = new HashSet(); redirecting.iterate((link, m) -> { - log.debug("Enrolling: {} contacting: {} on: {}", identifier, link.getMember().getId(), member.getId()); + log.info("Enrolling: {} contacting: {} on: {}", identifier, link.getMember().getId(), member.getId()); link.enroll(notarization, parameters.registrationTimeout()); return Empty.getDefaultInstance(); }, (futureSailor, link, m) -> completeEnrollment(futureSailor, m, completed), () -> { @@ -243,7 +248,7 @@ private Validations register(Credentials request) { var successors = Context.uniqueSuccessors(context, digestOf(identifier.toIdent(), parameters.digestAlgorithm())); - final var majority = context.activeCount() == 1 ? 0 : context.majority(); + final var majority = context.majority(true); final var redirecting = new SliceIterator<>("Credential verification", member, successors, endorsementComm); var verifications = new HashSet(); redirecting.iterate((link, m) -> { @@ -292,7 +297,7 @@ private Validation_ validate(Credentials credentials) { } private Validation_ verificationOf(Credentials credentials) { - if (parameters.verifier().test(credentials.getAttestation())) { + if (verifier.test(credentials.getAttestation())) { return validate(credentials); } return null; @@ -458,12 +463,13 @@ private boolean validate(Notarization request, Identifier identifier, KERL_ kerl } } // If there is only one active member in our context, it's us. - final var majority = count >= (context.activeCount() == 1 ? 1 : context.majority()); - if (!majority) { + var majority = context.majority(true); + if (count < majority) { log.warn("Invalid notarization, no majority: {} required: {} for: {} from: {} on: {}", count, - context.majority(), identifier, from, member.getId()); + majority, identifier, from, member.getId()); + return false; } - return majority; + return true; } else { log.warn("Invalid notarization, invalid kerl for: {} from: {} on: {}", identifier, from, member.getId()); @@ -516,9 +522,10 @@ private boolean validateCredentials(Credentials credentials, Digest from) { count++; } - if (count < context.majority()) { - log.warn("Invalid credential nonce, no majority signature: {} required > {} from: {} on: {}", count, - context.majority(), from, member.getId()); + var majority = context.majority(true); + if (count < majority) { + log.warn("Invalid credential nonce, no majority signature: {} required >= {} from: {} on: {}", count, + majority, from, member.getId()); return false; } diff --git a/gorgoneion/src/main/java/com/salesforce/apollo/gorgoneion/Parameters.java b/gorgoneion/src/main/java/com/salesforce/apollo/gorgoneion/Parameters.java index 0419f4c507..1871c09ee1 100644 --- a/gorgoneion/src/main/java/com/salesforce/apollo/gorgoneion/Parameters.java +++ b/gorgoneion/src/main/java/com/salesforce/apollo/gorgoneion/Parameters.java @@ -6,8 +6,8 @@ */ package com.salesforce.apollo.gorgoneion; -import com.salesforce.apollo.gorgoneion.proto.SignedAttestation; import com.salesforce.apollo.cryptography.DigestAlgorithm; +import com.salesforce.apollo.gorgoneion.proto.SignedAttestation; import com.salesforce.apollo.stereotomy.KERL; import java.time.Clock; @@ -17,8 +17,8 @@ /** * @author hal.hildebrand */ -public record Parameters(Predicate verifier, Clock clock, Duration registrationTimeout, - Duration frequency, DigestAlgorithm digestAlgorithm, Duration maxDuration, KERL kerl) { +public record Parameters(Clock clock, Duration registrationTimeout, Duration frequency, DigestAlgorithm digestAlgorithm, + Duration maxDuration, KERL kerl) { public static Builder newBuilder() { return new Builder(); @@ -38,10 +38,8 @@ public static class Builder { private Duration maxDuration = Duration.ofSeconds(30); private Duration registrationTimeout = Duration.ofSeconds(30); - private Predicate verifier = defaultVerifier; - public Parameters build() { - return new Parameters(verifier, clock, registrationTimeout, frequency, digestAlgorithm, maxDuration, kerl); + return new Parameters(clock, registrationTimeout, frequency, digestAlgorithm, maxDuration, kerl); } public Clock getClock() { @@ -97,15 +95,6 @@ public Builder setRegistrationTimeout(Duration registrationTimeout) { this.registrationTimeout = registrationTimeout; return this; } - - public Predicate getVerifier() { - return verifier; - } - - public Builder setVerifier(Predicate verifier) { - this.verifier = verifier; - return this; - } } } diff --git a/gorgoneion/src/test/java/com/salesforce/apollo/gorgoneion/GorgoneionTest.java b/gorgoneion/src/test/java/com/salesforce/apollo/gorgoneion/GorgoneionTest.java index 774f39020a..7d95574151 100644 --- a/gorgoneion/src/test/java/com/salesforce/apollo/gorgoneion/GorgoneionTest.java +++ b/gorgoneion/src/test/java/com/salesforce/apollo/gorgoneion/GorgoneionTest.java @@ -8,20 +8,19 @@ import com.google.protobuf.Any; import com.google.protobuf.Timestamp; -import com.salesforce.apollo.gorgoneion.proto.Attestation; -import com.salesforce.apollo.gorgoneion.proto.Credentials; -import com.salesforce.apollo.gorgoneion.proto.SignedAttestation; -import com.salesforce.apollo.stereotomy.event.proto.KERL_; -import com.salesforce.apollo.stereotomy.event.proto.Validations; import com.salesforce.apollo.archipelago.LocalServer; import com.salesforce.apollo.archipelago.ServerConnectionCache; import com.salesforce.apollo.cryptography.DigestAlgorithm; import com.salesforce.apollo.gorgoneion.comm.admissions.AdmissionsServer; import com.salesforce.apollo.gorgoneion.comm.admissions.AdmissionsService; +import com.salesforce.apollo.gorgoneion.proto.Attestation; +import com.salesforce.apollo.gorgoneion.proto.Credentials; +import com.salesforce.apollo.gorgoneion.proto.SignedAttestation; import com.salesforce.apollo.membership.Context; -import com.salesforce.apollo.membership.Member; import com.salesforce.apollo.membership.stereotomy.ControlledIdentifierMember; import com.salesforce.apollo.stereotomy.StereotomyImpl; +import com.salesforce.apollo.stereotomy.event.proto.KERL_; +import com.salesforce.apollo.stereotomy.event.proto.Validations; import com.salesforce.apollo.stereotomy.mem.MemKERL; import com.salesforce.apollo.stereotomy.mem.MemKeyStore; import com.salesforce.apollo.stereotomy.services.proto.ProtoEventObserver; @@ -31,7 +30,6 @@ import java.time.Duration; import java.time.Instant; import java.util.UUID; -import java.util.concurrent.Executors; import static org.junit.jupiter.api.Assertions.*; import static org.mockito.Mockito.mock; @@ -49,7 +47,7 @@ public void smokin() throws Exception { var stereotomy = new StereotomyImpl(new MemKeyStore(), kerl, entropy); final var prefix = UUID.randomUUID().toString(); var member = new ControlledIdentifierMember(stereotomy.newIdentifier()); - var context = Context.newBuilder().setCardinality(1).build(); + var context = Context.newBuilder().setCardinality(1).build(); context.activate(member); // Gorgoneion service comms @@ -59,9 +57,8 @@ public void smokin() throws Exception { // The kerl observer to publish admitted client KERLs to var observer = mock(ProtoEventObserver.class); @SuppressWarnings("unused") - var gorgon = new Gorgoneion(Parameters.newBuilder().setKerl(kerl).build(), member, context, observer, - gorgonRouter, Executors.newScheduledThreadPool(1, Thread.ofVirtual().factory()), - null); + var gorgon = new Gorgoneion(true, t -> true, Parameters.newBuilder().setKerl(kerl).build(), member, context, + observer, gorgonRouter, null); // The registering client var client = new ControlledIdentifierMember(stereotomy.newIdentifier()); @@ -69,7 +66,7 @@ public void smokin() throws Exception { // Registering client comms var clientRouter = new LocalServer(prefix, client).router(ServerConnectionCache.newBuilder().setTarget(2)); AdmissionsService admissions = mock(AdmissionsService.class); - var clientComminications = clientRouter.create(client, context.getId(), admissions, ":admissions", + var clientCommunications = clientRouter.create(client, context.getId(), admissions, ":admissions", r -> new AdmissionsServer( clientRouter.getClientIdentityProvider(), r, null), AdmissionsClient.getCreate(), @@ -77,7 +74,7 @@ public void smokin() throws Exception { clientRouter.start(); // Admin client link - var admin = clientComminications.connect(member); + var admin = clientCommunications.connect(member); assertNotNull(admin); @@ -86,9 +83,8 @@ public void smokin() throws Exception { final KERL_ cKerl = client.kerl(); var fs = admin.apply(cKerl, Duration.ofSeconds(1)); assertNotNull(fs); - var signedNonce = fs; - assertNotNull(signedNonce.getNonce()); - assertEquals(client.getIdentifier().getIdentifier().toIdent(), signedNonce.getNonce().getMember()); + assertNotNull(fs.getNonce()); + assertEquals(client.getIdentifier().getIdentifier().toIdent(), fs.getNonce().getMember()); // Create attestation final var now = Instant.now(); @@ -98,7 +94,7 @@ public void smokin() throws Exception { .setTimestamp(Timestamp.newBuilder() .setSeconds(now.getEpochSecond()) .setNanos(now.getNano())) - .setNonce(client.sign(signedNonce.toByteString()).toSig()) + .setNonce(client.sign(fs.toByteString()).toSig()) .setKerl(client.kerl()) .setAttestation(attestationDocument) .build(); @@ -109,7 +105,7 @@ public void smokin() throws Exception { .setSignature(client.sign( attestation.toByteString()).toSig()) .build()) - .setNonce(signedNonce) + .setNonce(fs) .build(), Duration.ofSeconds(1)); gorgonRouter.close(Duration.ofSeconds(1)); clientRouter.close(Duration.ofSeconds(1)); diff --git a/grpc/src/main/proto/fireflies.proto b/grpc/src/main/proto/fireflies.proto index 23edf780d6..b094959fb6 100644 --- a/grpc/src/main/proto/fireflies.proto +++ b/grpc/src/main/proto/fireflies.proto @@ -46,7 +46,7 @@ message SignedAccusation { message Note { int64 epoch = 1; crypto.Digeste currentView = 2; - stereotomy.EventCoords coordinates = 3; + stereotomy.Ident identifier = 3; bytes mask = 4; string host = 5; int32 port = 6; @@ -117,11 +117,6 @@ message Update { service Entrance { rpc seed (Registration) returns (Redirect) {} rpc join (Join) returns (Gateway) {} - - // Bootstrap validation - rpc validate(stereotomy.EventCoords) returns (Validation) {} - // Bootstrap verification key state - rpc keyState(stereotomy.IdentAndSeq) returns (stereotomy.KeyState_) {} } message Validation { @@ -138,12 +133,7 @@ message Redirect { int32 cardinality = 2; int32 rings = 3; bool bootstrap = 4; - repeated Seed_ sample = 6; -} - -message Seed_ { - SignedNote note = 1; - stereotomy.KeyEvent_ establishment = 5; + repeated SignedNote sample = 6; } message Join { diff --git a/grpc/src/main/proto/leyden.proto b/grpc/src/main/proto/leyden.proto index 03c6b2d9f4..2420e4cf19 100644 --- a/grpc/src/main/proto/leyden.proto +++ b/grpc/src/main/proto/leyden.proto @@ -6,10 +6,8 @@ option java_outer_classname = "LeydenProto"; option objc_class_prefix = "Ley"; import "google/protobuf/empty.proto"; -import "google/protobuf/any.proto"; import "stereotomy.proto"; -import "stereotomy-services.proto"; import "crypto.proto"; package leyden; diff --git a/isolates/src/test/java/com/salesforce/apollo/demesnes/FireFliesTrace.java b/isolates/src/test/java/com/salesforce/apollo/demesnes/FireFliesTrace.java index 308d9deb5d..469cce981a 100644 --- a/isolates/src/test/java/com/salesforce/apollo/demesnes/FireFliesTrace.java +++ b/isolates/src/test/java/com/salesforce/apollo/demesnes/FireFliesTrace.java @@ -26,8 +26,8 @@ import com.salesforce.apollo.membership.stereotomy.ControlledIdentifierMember; import com.salesforce.apollo.model.ProcessContainerDomain; import com.salesforce.apollo.model.ProcessDomain; -import com.salesforce.apollo.stereotomy.EventCoordinates; import com.salesforce.apollo.stereotomy.StereotomyImpl; +import com.salesforce.apollo.stereotomy.identifier.SelfAddressingIdentifier; import com.salesforce.apollo.stereotomy.identifier.spec.IdentifierSpecification; import com.salesforce.apollo.stereotomy.mem.MemKERL; import com.salesforce.apollo.stereotomy.mem.MemKeyStore; @@ -199,7 +199,9 @@ public void before() throws Exception { var context = new ContextImpl<>(DigestAlgorithm.DEFAULT.getLast(), CARDINALITY, 0.2, 3); final var member = new ControlledIdentifierMember(id); var localRouter = new LocalServer(prefix, member).router(ServerConnectionCache.newBuilder().setTarget(30)); - var pdParams = new ProcessDomain.ProcessDomainParameters("jdbc:h2:mem:", Duration.ofMinutes(1), + var pdParams = new ProcessDomain.ProcessDomainParameters("jdbc:h2:mem:%s-state".formatted(digest), + Duration.ofMinutes(1), + "jdbc:h2:mem:%s-dht".formatted(digest), checkpointDirBase, Duration.ofMillis(10), 0.00125, Duration.ofMinutes(1), 3, 10, 0.1); var node = new ProcessContainerDomain(group, member, pdParams, params, RuntimeParameters.newBuilder() @@ -221,22 +223,20 @@ public void smokin() throws Exception { long then = System.currentTimeMillis(); final var countdown = new CountDownLatch(domains.size()); final var seeds = Collections.singletonList( - new Seed(domains.get(0).getMember().getEvent(), new InetSocketAddress(0))); + new Seed(domains.getFirst().getMember().getIdentifier().getIdentifier(), new InetSocketAddress(0))); domains.forEach(d -> { var listener = new View.ViewLifecycleListener() { @Override - public void viewChange(Context context, Digest viewId, List joins, - List leaves) { + public void viewChange(Context context, Digest viewId, + List joins, List leaves) { if (context.totalCount() == CARDINALITY) { - System.out.println( - String.format("Full view: %s members: %s on: %s", viewId, context.totalCount(), - d.getMember().getId())); + System.out.printf("Full view: %s members: %s on: %s%n", viewId, context.totalCount(), + d.getMember().getId()); countdown.countDown(); } else { - System.out.println( - String.format("Members joining: %s members: %s on: %s", viewId, context.totalCount(), - d.getMember().getId())); + System.out.printf("Members joining: %s members: %s on: %s%n", viewId, context.totalCount(), + d.getMember().getId()); } } }; diff --git a/memberships/src/main/java/com/salesforce/apollo/archipelago/MtlsServer.java b/memberships/src/main/java/com/salesforce/apollo/archipelago/MtlsServer.java index 7ae6f4661c..cc61fee23f 100644 --- a/memberships/src/main/java/com/salesforce/apollo/archipelago/MtlsServer.java +++ b/memberships/src/main/java/com/salesforce/apollo/archipelago/MtlsServer.java @@ -152,7 +152,8 @@ public RouterImpl router(ServerConnectionCache.Builder cacheBuilder, Supplier
  • { - double DEFAULT_EPSILON = 0.99999; - static final String RING_HASH_TEMPLATE = "%s-%s-%s"; + double DEFAULT_EPSILON = 0.99999; + String RING_HASH_TEMPLATE = "%s-%s-%s"; static Digest hashFor(Digest ctxId, int ring, Digest d) { return d.prefix(ctxId, ring); @@ -270,7 +270,7 @@ static List uniqueSuccessors(final Context context, Digest diges /** * Answer true if the member is a successor of the supplied digest on any ring * - * @param member + * @param m * @param digest * @return */ @@ -279,7 +279,17 @@ static List uniqueSuccessors(final Context context, Digest diges /** * Answer the majority cardinality of the context, based on the current ring count */ - int majority(); + default int majority() { + return majority(false); + } + + /** + * Answer the majority cardinality of the context, based on the current ring count + * + * @param bootstrapped - if true, calculate correct majority for bootstrapping cases where totalCount < true + * majority + */ + int majority(boolean bootstrapped); /** * Answer the total member count (offline + active) tracked by this context diff --git a/memberships/src/main/java/com/salesforce/apollo/membership/ContextImpl.java b/memberships/src/main/java/com/salesforce/apollo/membership/ContextImpl.java index acdfbc37f3..6c41e9bfa9 100644 --- a/memberships/src/main/java/com/salesforce/apollo/membership/ContextImpl.java +++ b/memberships/src/main/java/com/salesforce/apollo/membership/ContextImpl.java @@ -210,11 +210,9 @@ public boolean equals(Object obj) { return false; Context other = (Context) obj; if (id == null) { - if (other.getId() != null) - return false; - } else if (!id.equals(other.getId())) - return false; - return true; + return other.getId() == null; + } else + return id.equals(other.getId()); } @Override @@ -341,6 +339,21 @@ public int majority() { return getRingCount() - toleranceLevel(); } + @Override + public int majority(boolean bootstrapped) { + var majority = getRingCount() - toleranceLevel(); + if (bootstrapped) { + return switch (totalCount()) { + case 1, 2 -> 1; + case 3 -> 2; + case 4 -> 3; + default -> majority; + }; + } else { + return majority; + } + } + @Override public int memberCount() { return members.size(); @@ -701,9 +714,7 @@ public String toString() { private void rebalance(int ringCount, ContextImpl contextImpl) { final var newHashes = new Digest[ringCount]; - for (int i = 0; i < Math.min(ringCount, hashes.length); i++) { - newHashes[i] = hashes[i]; - } + System.arraycopy(hashes, 0, newHashes, 0, Math.min(ringCount, hashes.length)); for (int i = Math.min(ringCount, hashes.length); i < newHashes.length; i++) { newHashes[i] = contextImpl.hashFor(member.getId(), i); } diff --git a/memberships/src/main/java/com/salesforce/apollo/membership/messaging/rbc/ReliableBroadcaster.java b/memberships/src/main/java/com/salesforce/apollo/membership/messaging/rbc/ReliableBroadcaster.java index de5ae9bb4c..2d90136de2 100644 --- a/memberships/src/main/java/com/salesforce/apollo/membership/messaging/rbc/ReliableBroadcaster.java +++ b/memberships/src/main/java/com/salesforce/apollo/membership/messaging/rbc/ReliableBroadcaster.java @@ -241,12 +241,15 @@ private void handle(Optional result, } Reconcile gossip = result.get(); buffer.receive(gossip.getUpdatesList()); - destination.link() - .update(ReconcileContext.newBuilder() - .setRing(destination.ring()) - .addAllUpdates(buffer.reconcile(BloomFilter.from(gossip.getDigests()), - destination.member().getId())) - .build()); + var biff = gossip.getDigests(); + if (!Biff.getDefaultInstance().equals(biff)) { + destination.link() + .update(ReconcileContext.newBuilder() + .setRing(destination.ring()) + .addAllUpdates(buffer.reconcile(BloomFilter.from(biff), + destination.member().getId())) + .build()); + } } finally { if (timer != null) { timer.stop(); @@ -403,9 +406,9 @@ public class Service implements Router.ServiceRouting { public Reconcile gossip(MessageBff request, Digest from) { Member predecessor = context.ring(request.getRing()).predecessor(member); if (predecessor == null || !from.equals(predecessor.getId())) { - log.info("Invalid inbound messages gossip on {}:{} from: {} on ring: {} - not predecessor: {}", - context.getId(), member.getId(), from, request.getRing(), - predecessor == null ? "" : predecessor.getId()); + log.trace("Invalid inbound messages gossip on {}:{} from: {} on ring: {} - not predecessor: {}", + context.getId(), member.getId(), from, request.getRing(), + predecessor == null ? "" : predecessor.getId()); return Reconcile.getDefaultInstance(); } return Reconcile.newBuilder() diff --git a/memberships/src/main/java/com/salesforce/apollo/ring/RingCommunications.java b/memberships/src/main/java/com/salesforce/apollo/ring/RingCommunications.java index c3d2649151..2494760779 100644 --- a/memberships/src/main/java/com/salesforce/apollo/ring/RingCommunications.java +++ b/memberships/src/main/java/com/salesforce/apollo/ring/RingCommunications.java @@ -65,7 +65,7 @@ public RingCommunications(Direction direction, Context context, SigningMember public void execute(BiFunction round, SyncHandler handler) { final var next = next(member.getId()); - if (next.member == null) { + if (next == null || next.member == null) { log.debug("No member for ring: {} on: {}", next.ring, member.getId()); handler.handle(Optional.empty(), next); return; @@ -162,14 +162,18 @@ private void execute(BiFunction round, SyncHandler linkFor(Digest digest) { final var current = currentIndex; - var successor = traversalOrder.get(current); + iteration successor = null; try { + successor = traversalOrder.get(current); final Comm link = comm.connect(successor.m); if (link == null) { log.trace("No connection to {} on: {}", successor.m == null ? "" : successor.m.getId(), member.getId()); } return new Destination<>(successor.m, link, successor.ring); + } catch (IndexOutOfBoundsException e) { + log.trace("No members to traver on: {}", member.getId()); + return null; } catch (Throwable e) { log.trace("error opening connection to {}: {} on: {}", successor.m == null ? "" : successor.m.getId(), (e.getCause() != null ? e.getCause() : e).getMessage(), member.getId()); diff --git a/memberships/src/main/java/com/salesforce/apollo/ring/RingIterator.java b/memberships/src/main/java/com/salesforce/apollo/ring/RingIterator.java index 6bbeb99e78..1b823c0b67 100644 --- a/memberships/src/main/java/com/salesforce/apollo/ring/RingIterator.java +++ b/memberships/src/main/java/com/salesforce/apollo/ring/RingIterator.java @@ -109,7 +109,7 @@ private void internalIterate(Digest digest, Runnable onMajority, BiFunction< var next = next(digest); log.trace("Iteration: {} tally: {} for digest: {} on: {} ring: {} complete: false on: {}", iteration(), tally.get(), digest, context.getId(), next.ring(), member.getId()); - if (next.link() == null) { + if (next == null || next.link() == null) { log.trace("No successor found for digest: {} on: {} iteration: {} traversed: {} ring: {} on: {}", digest, context.getId(), iteration(), traversed, context.ring(currentIndex).stream().toList(), member.getId()); @@ -174,16 +174,16 @@ private void proceed(Digest key, final boolean allow, Runnable onMajority, Runna if (!finalIteration) { log.trace( "Determining: {} continuation of: {} for digest: {} tally: {} majority: {} final itr: {} allow: {} on: {}", - current, key, context.getId(), tally.get(), context.majority(), finalIteration, allow, member.getId()); + current, key, context.getId(), tally.get(), context.majority(true), finalIteration, allow, member.getId()); } if (finalIteration && allow) { log.trace("Completing iteration: {} of: {} for digest: {} tally: {} on: {}", iteration(), key, context.getId(), tally.get(), member.getId()); if (failedMajority != null && !majorityFailed) { - if (tally.get() < context.majority()) { + if (tally.get() < context.majority(true)) { majorityFailed = true; log.debug("Failed to obtain majority of: {} for digest: {} tally: {} required: {} on: {}", key, - context.getId(), tally.get(), context.majority(), member.getId()); + context.getId(), tally.get(), context.majority(true), member.getId()); failedMajority.run(); } } @@ -195,7 +195,7 @@ private void proceed(Digest key, final boolean allow, Runnable onMajority, Runna member.getId()); } else { if (onMajority != null && !majoritySucceed) { - if (tally.get() >= context.majority()) { + if (tally.get() >= context.majority(true)) { majoritySucceed = true; log.debug("Obtained: {} majority of: {} for digest: {} tally: {} on: {}", current, key, context.getId(), tally.get(), member.getId()); diff --git a/memberships/src/test/java/com/salesforce/apollo/membership/messaging/rbc/RbcTest.java b/memberships/src/test/java/com/salesforce/apollo/membership/messaging/rbc/RbcTest.java index 5efd0ecc99..c1ceece38f 100644 --- a/memberships/src/test/java/com/salesforce/apollo/membership/messaging/rbc/RbcTest.java +++ b/memberships/src/test/java/com/salesforce/apollo/membership/messaging/rbc/RbcTest.java @@ -10,7 +10,6 @@ import com.codahale.metrics.MetricRegistry; import com.google.protobuf.ByteString; import com.google.protobuf.InvalidProtocolBufferException; -import com.salesforce.apollo.test.proto.ByteMessage; import com.salesforce.apollo.archipelago.LocalServer; import com.salesforce.apollo.archipelago.Router; import com.salesforce.apollo.archipelago.ServerConnectionCache; @@ -27,6 +26,7 @@ import com.salesforce.apollo.stereotomy.StereotomyImpl; import com.salesforce.apollo.stereotomy.mem.MemKERL; import com.salesforce.apollo.stereotomy.mem.MemKeyStore; +import com.salesforce.apollo.test.proto.ByteMessage; import com.salesforce.apollo.utils.Entropy; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Test; @@ -112,12 +112,13 @@ public void broadcast() throws Exception { view.registerHandler(receiver); receivers.put(view.getMember(), receiver); } + System.out.println(); int rounds = LARGE_TESTS ? 100 : 5; for (int r = 0; r < rounds; r++) { CountDownLatch latch = new CountDownLatch(messengers.size()); round.set(latch); var rnd = r; - System.out.print("\nround: %s ".formatted(r)); + System.out.printf("\nround: %s ", r); messengers.stream().forEach(view -> { byte[] rand = new byte[32]; Entropy.nextSecureBytes(rand); diff --git a/model/src/main/java/com/salesforce/apollo/model/Domain.java b/model/src/main/java/com/salesforce/apollo/model/Domain.java index 87112adb73..7d876b7908 100644 --- a/model/src/main/java/com/salesforce/apollo/model/Domain.java +++ b/model/src/main/java/com/salesforce/apollo/model/Domain.java @@ -47,8 +47,6 @@ import java.sql.Connection; import java.sql.JDBCType; import java.util.*; -import java.util.concurrent.Executor; -import java.util.concurrent.Executors; import static com.salesforce.apollo.cryptography.QualifiedBase64.qb64; import static java.nio.file.Path.of; @@ -62,7 +60,6 @@ abstract public class Domain { private static final Logger log = LoggerFactory.getLogger(Domain.class); - protected final Executor executor = Executors.newVirtualThreadPerTaskExecutor(); protected final CHOAM choam; protected final ControlledIdentifierMember member; protected final Mutator mutator; diff --git a/model/src/main/java/com/salesforce/apollo/model/ProcessContainerDomain.java b/model/src/main/java/com/salesforce/apollo/model/ProcessContainerDomain.java index 8c7882dad2..6f04e70618 100644 --- a/model/src/main/java/com/salesforce/apollo/model/ProcessContainerDomain.java +++ b/model/src/main/java/com/salesforce/apollo/model/ProcessContainerDomain.java @@ -42,9 +42,7 @@ import java.nio.file.Path; import java.time.Duration; import java.util.*; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.RejectedExecutionException; -import java.util.concurrent.TimeUnit; +import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicBoolean; import static com.salesforce.apollo.comm.grpc.DomainSocketServerInterceptor.IMPL; @@ -55,22 +53,22 @@ **/ public class ProcessContainerDomain extends ProcessDomain { - private final static Logger log = LoggerFactory.getLogger( + private final static Logger log = LoggerFactory.getLogger( ProcessContainerDomain.class); - private final static Class channelType = IMPL.getChannelType(); - - private final DomainSocketAddress bridge; - private final EventLoopGroup clientEventLoopGroup = IMPL.getEventLoopGroup(); - private final Path communicationsDirectory; - private final EventLoopGroup contextEventLoopGroup = IMPL.getEventLoopGroup(); - private final Map hostedDomains = new ConcurrentHashMap<>(); - private final DomainSocketAddress outerContextEndpoint; - private final Server outerContextService; - private final Portal portal; - private final DomainSocketAddress portalEndpoint; - private final EventLoopGroup portalEventLoopGroup = IMPL.getEventLoopGroup(); - private final Map routes = new HashMap<>(); - private final IdentifierSpecification.Builder subDomainSpecification; + private final static Class channelType = IMPL.getChannelType(); + protected final Executor executor = Executors.newVirtualThreadPerTaskExecutor(); + private final DomainSocketAddress bridge; + private final EventLoopGroup clientEventLoopGroup = IMPL.getEventLoopGroup(); + private final Path communicationsDirectory; + private final EventLoopGroup contextEventLoopGroup = IMPL.getEventLoopGroup(); + private final Map hostedDomains = new ConcurrentHashMap<>(); + private final DomainSocketAddress outerContextEndpoint; + private final Server outerContextService; + private final Portal portal; + private final DomainSocketAddress portalEndpoint; + private final EventLoopGroup portalEventLoopGroup = IMPL.getEventLoopGroup(); + private final Map routes = new HashMap<>(); + private final IdentifierSpecification.Builder subDomainSpecification; public ProcessContainerDomain(Digest group, ControlledIdentifierMember member, ProcessDomainParameters parameters, Parameters.Builder builder, Parameters.RuntimeParameters.Builder runtime, diff --git a/model/src/main/java/com/salesforce/apollo/model/ProcessDomain.java b/model/src/main/java/com/salesforce/apollo/model/ProcessDomain.java index 1f574cbc9e..ea90e9102e 100644 --- a/model/src/main/java/com/salesforce/apollo/model/ProcessDomain.java +++ b/model/src/main/java/com/salesforce/apollo/model/ProcessDomain.java @@ -19,7 +19,6 @@ import com.salesforce.apollo.membership.stereotomy.ControlledIdentifierMember; import com.salesforce.apollo.stereotomy.EventValidation; import com.salesforce.apollo.stereotomy.Verifiers; -import com.salesforce.apollo.stereotomy.identifier.SelfAddressingIdentifier; import com.salesforce.apollo.stereotomy.services.grpc.StereotomyMetrics; import com.salesforce.apollo.thoth.KerlDHT; import org.h2.jdbcx.JdbcConnectionPool; @@ -42,11 +41,11 @@ * @author hal.hildebrand */ public class ProcessDomain extends Domain { + private final static Logger log = LoggerFactory.getLogger(ProcessDomain.class); - private final static Logger log = LoggerFactory.getLogger(ProcessDomain.class); - protected final KerlDHT dht; - protected final View foundation; - private final UUID listener; + protected final KerlDHT dht; + protected final View foundation; + private final UUID listener; public ProcessDomain(Digest group, ControlledIdentifierMember member, ProcessDomainParameters parameters, Builder builder, Parameters.RuntimeParameters.Builder runtime, InetSocketAddress endpoint, @@ -57,9 +56,8 @@ public ProcessDomain(Digest group, ControlledIdentifierMember member, ProcessDom .setpByz(parameters.dhtPbyz) .setId(group) .build(); - final var dhtUrl = String.format("jdbc:h2:mem:%s-%s;DB_CLOSE_DELAY=-1", member.getId(), UUID.randomUUID()); - JdbcConnectionPool connectionPool = JdbcConnectionPool.create(dhtUrl, "", ""); - connectionPool.setMaxConnections(10); + JdbcConnectionPool connectionPool = JdbcConnectionPool.create(parameters.dhtDbUrl, "", ""); + connectionPool.setMaxConnections(parameters.jdbcMaxConnections()); dht = new KerlDHT(parameters.dhtOpsFrequency, params.context(), member, connectionPool, params.digestAlgorithm(), params.communications(), parameters.dhtOperationsTimeout, parameters.dhtFpr, stereotomyMetrics); @@ -103,9 +101,7 @@ public void stop() { protected ViewLifecycleListener listener() { return (context, id, join, leaving) -> { for (var d : join) { - if (d.getIdentifier() instanceof SelfAddressingIdentifier sai) { - params.context().activate(context.getMember(sai.getDigest())); - } + params.context().activate(context.getMember(d.getDigest())); } for (var d : leaving) { params.context().remove(d); @@ -124,8 +120,9 @@ protected void stopServices() { dht.stop(); } - public record ProcessDomainParameters(String dbURL, Duration dhtOperationsTimeout, Path checkpointBaseDir, - Duration dhtOpsFrequency, double dhtFpr, Duration dhtEventValidTO, - int dhtBias, int jdbcMaxConnections, double dhtPbyz) { + public record ProcessDomainParameters(String dbURL, Duration dhtOperationsTimeout, String dhtDbUrl, + Path checkpointBaseDir, Duration dhtOpsFrequency, double dhtFpr, + Duration dhtEventValidTO, int dhtBias, int jdbcMaxConnections, + double dhtPbyz) { } } diff --git a/model/src/main/java/com/salesforce/apollo/model/demesnes/DemesneImpl.java b/model/src/main/java/com/salesforce/apollo/model/demesnes/DemesneImpl.java index 02c5cc1ed1..843e320311 100644 --- a/model/src/main/java/com/salesforce/apollo/model/demesnes/DemesneImpl.java +++ b/model/src/main/java/com/salesforce/apollo/model/demesnes/DemesneImpl.java @@ -94,7 +94,7 @@ public DemesneImpl(DemesneParameters parameters) throws GeneralSecurityException entropy.nextBytes(pwd); final var password = Hex.hexChars(pwd); final Supplier passwordProvider = () -> password; - final var keystore = KeyStore.getInstance("JKS"); + final var keystore = KeyStore.getInstance("JCEKS"); keystore.load(null, password); diff --git a/model/src/test/java/com/salesforce/apollo/model/ContainmentDomainTest.java b/model/src/test/java/com/salesforce/apollo/model/ContainmentDomainTest.java index a33746955c..d63364eabb 100644 --- a/model/src/test/java/com/salesforce/apollo/model/ContainmentDomainTest.java +++ b/model/src/test/java/com/salesforce/apollo/model/ContainmentDomainTest.java @@ -85,9 +85,10 @@ public void before() throws Exception { final var member = new ControlledIdentifierMember(id); var localRouter = new LocalServer(prefix, member).router(ServerConnectionCache.newBuilder().setTarget(30)); routers.add(localRouter); - var dbUrl = String.format("jdbc:h2:mem:%s-%s;DB_CLOSE_DELAY=-1", member.getId(), UUID.randomUUID()); - var pdParams = new ProcessDomain.ProcessDomainParameters(dbUrl, Duration.ofMinutes(1), checkpointDirBase, - Duration.ofMillis(10), 0.00125, + var dbUrl = String.format("jdbc:h2:mem:sql-%s-%s;DB_CLOSE_DELAY=-1", member.getId(), UUID.randomUUID()); + var pdParams = new ProcessDomain.ProcessDomainParameters(dbUrl, Duration.ofMinutes(1), + "jdbc:h2:mem:%s-state".formatted(d), + checkpointDirBase, Duration.ofMillis(10), 0.00125, Duration.ofMinutes(1), 3, 10, 0.1); var domain = new ProcessContainerDomain(group, member, pdParams, params, RuntimeParameters.newBuilder() .setFoundation( diff --git a/model/src/test/java/com/salesforce/apollo/model/DomainTest.java b/model/src/test/java/com/salesforce/apollo/model/DomainTest.java index 25a714c972..6319aac6f4 100644 --- a/model/src/test/java/com/salesforce/apollo/model/DomainTest.java +++ b/model/src/test/java/com/salesforce/apollo/model/DomainTest.java @@ -215,10 +215,11 @@ public void before() throws Exception { final var member = new ControlledIdentifierMember(id); var localRouter = new LocalServer(prefix, member).router(ServerConnectionCache.newBuilder().setTarget(30)); routers.add(localRouter); - var dbUrl = String.format("jdbc:h2:mem:%s-%s;DB_CLOSE_DELAY=-1", member.getId(), UUID.randomUUID()); - var pdParams = new ProcessDomain.ProcessDomainParameters(dbUrl, Duration.ofMinutes(1), checkpointDirBase, - Duration.ofMillis(10), 0.00125, - Duration.ofMinutes(1), 3, 10, 0.1); + var dbUrl = String.format("jdbc:h2:mem:sql-%s-%s;DB_CLOSE_DELAY=-1", member.getId(), UUID.randomUUID()); + var pdParams = new ProcessDomain.ProcessDomainParameters(dbUrl, Duration.ofMinutes(1), + "jdbc:h2:mem:%s-state;DB_CLOSE_DELAY=-1".formatted( + d), checkpointDirBase, Duration.ofMillis(10), + 0.00125, Duration.ofMinutes(1), 3, 10, 0.1); var domain = new ProcessDomain(group, member, pdParams, params, RuntimeParameters.newBuilder() .setFoundation(sealed) .setContext(context) diff --git a/model/src/test/java/com/salesforce/apollo/model/FireFliesTest.java b/model/src/test/java/com/salesforce/apollo/model/FireFliesTest.java index b7aab9b7cf..3bdf3b80ce 100644 --- a/model/src/test/java/com/salesforce/apollo/model/FireFliesTest.java +++ b/model/src/test/java/com/salesforce/apollo/model/FireFliesTest.java @@ -23,8 +23,8 @@ import com.salesforce.apollo.membership.Context; import com.salesforce.apollo.membership.ContextImpl; import com.salesforce.apollo.membership.stereotomy.ControlledIdentifierMember; -import com.salesforce.apollo.stereotomy.EventCoordinates; import com.salesforce.apollo.stereotomy.StereotomyImpl; +import com.salesforce.apollo.stereotomy.identifier.SelfAddressingIdentifier; import com.salesforce.apollo.stereotomy.mem.MemKERL; import com.salesforce.apollo.stereotomy.mem.MemKeyStore; import com.salesforce.apollo.utils.Entropy; @@ -86,9 +86,10 @@ public void before() throws Exception { var context = new ContextImpl<>(DigestAlgorithm.DEFAULT.getLast(), CARDINALITY, 0.2, 3); final var member = new ControlledIdentifierMember(id); var localRouter = new LocalServer(prefix, member).router(ServerConnectionCache.newBuilder().setTarget(30)); - var dbUrl = String.format("jdbc:h2:mem:%s-%s;DB_CLOSE_DELAY=-1", member.getId(), UUID.randomUUID()); - var pdParams = new ProcessDomain.ProcessDomainParameters(dbUrl, Duration.ofSeconds(5), checkpointDirBase, - Duration.ofMillis(10), 0.00125, + var dbUrl = String.format("jdbc:h2:mem:sql-%s-%s;DB_CLOSE_DELAY=-1", member.getId(), UUID.randomUUID()); + var pdParams = new ProcessDomain.ProcessDomainParameters(dbUrl, Duration.ofSeconds(5), + "jdbc:h2:mem:%s-state".formatted(digest), + checkpointDirBase, Duration.ofMillis(10), 0.00125, Duration.ofSeconds(5), 3, 10, 0.1); var node = new ProcessDomain(group, member, pdParams, params, RuntimeParameters.newBuilder() .setFoundation(sealed) @@ -108,22 +109,20 @@ public void smokin() throws Exception { long then = System.currentTimeMillis(); final var countdown = new CountDownLatch(domains.size()); final var seeds = Collections.singletonList( - new Seed(domains.getFirst().getMember().getEvent(), new InetSocketAddress(0))); + new Seed(domains.getFirst().getMember().getIdentifier().getIdentifier(), new InetSocketAddress(0))); domains.forEach(d -> { var listener = new View.ViewLifecycleListener() { @Override - public void viewChange(Context context, Digest viewId, List joins, - List leaves) { + public void viewChange(Context context, Digest viewId, + List joins, List leaves) { if (context.totalCount() == CARDINALITY) { - System.out.println( - String.format("Full view: %s members: %s on: %s", viewId, context.totalCount(), - d.getMember().getId())); + System.out.printf("Full view: %s members: %s on: %s%n", viewId, context.totalCount(), + d.getMember().getId()); countdown.countDown(); } else { - System.out.println( - String.format("Members joining: %s members: %s on: %s", viewId, context.totalCount(), - d.getMember().getId())); + System.out.printf("Members joining: %s members: %s on: %s%n", viewId, context.totalCount(), + d.getMember().getId()); } } }; diff --git a/pom.xml b/pom.xml index 93364fc578..0fe5dc06df 100644 --- a/pom.xml +++ b/pom.xml @@ -552,6 +552,11 @@ ${graal.vm.version} provided + + org.netbeans.api + org-netbeans-modules-keyring + RELEASE200 + @@ -772,6 +777,11 @@ maven-surefire-plugin 3.1.2 + + org.apache.maven.plugins + maven-clean-plugin + 3.3.2 + org.apache.maven.plugins maven-compiler-plugin diff --git a/protocols/src/test/java/com/salesforce/apollo/comm/grpc/MtlsClient.java b/protocols/src/test/java/com/salesforce/apollo/comm/grpc/MtlsClient.java index d3bc3f599d..c22ae6e6df 100644 --- a/protocols/src/test/java/com/salesforce/apollo/comm/grpc/MtlsClient.java +++ b/protocols/src/test/java/com/salesforce/apollo/comm/grpc/MtlsClient.java @@ -12,6 +12,7 @@ import com.netflix.concurrency.limits.grpc.client.GrpcClientRequestContext; import com.salesforce.apollo.cryptography.ssl.CertificateValidator; import io.grpc.ManagedChannel; +import io.grpc.NameResolver; import io.grpc.Status; import io.grpc.netty.NettyChannelBuilder; import io.netty.handler.ssl.ClientAuth; @@ -59,6 +60,22 @@ public MtlsClient(SocketAddress address, ClientAuth clientAuth, String alias, X5 } + public MtlsClient(String target, NameResolver.Factory resolverFactory, ClientAuth clientAuth, String alias, + X509Certificate certificate, PrivateKey privateKey, CertificateValidator validator) { + + Limiter limiter = new GrpcClientLimiterBuilder().blockOnLimit(false).build(); + channel = NettyChannelBuilder.forTarget(target) + .nameResolverFactory(resolverFactory) + .defaultLoadBalancingPolicy("round_robin") + .executor(exec) + .sslContext(forClient(clientAuth, alias, certificate, privateKey, validator)) + .intercept(new ConcurrencyLimitClientInterceptor(limiter, + () -> Status.RESOURCE_EXHAUSTED.withDescription( + "Client side concurrency limit exceeded"))) + .build(); + + } + public ManagedChannel getChannel() { return channel; } diff --git a/stereotomy/pom.xml b/stereotomy/pom.xml index ff874279b1..7f99a70405 100644 --- a/stereotomy/pom.xml +++ b/stereotomy/pom.xml @@ -83,8 +83,8 @@ build-helper-maven-plugin + org.apache.maven.plugins maven-clean-plugin - 2.5 clean-db diff --git a/stereotomy/src/main/java/com/salesforce/apollo/stereotomy/ControlledIdentifier.java b/stereotomy/src/main/java/com/salesforce/apollo/stereotomy/ControlledIdentifier.java index 3c41144be8..074fef3ab2 100644 --- a/stereotomy/src/main/java/com/salesforce/apollo/stereotomy/ControlledIdentifier.java +++ b/stereotomy/src/main/java/com/salesforce/apollo/stereotomy/ControlledIdentifier.java @@ -77,7 +77,7 @@ public interface ControlledIdentifier extends BoundIdentif * is signed by this self same generated basic identifier *

    * A new key pair is generated and this becomes the signing key of the certificate. This new public key is then - * signed by this identifier's current key state's key(s).. + * signed by this identifier's current key state's key(s). *

    * The values are encoded into the SubjectDN of the certificate as follows: *

      diff --git a/stereotomy/src/main/java/com/salesforce/apollo/stereotomy/EventValidation.java b/stereotomy/src/main/java/com/salesforce/apollo/stereotomy/EventValidation.java index d40187f771..c02ddf89e3 100644 --- a/stereotomy/src/main/java/com/salesforce/apollo/stereotomy/EventValidation.java +++ b/stereotomy/src/main/java/com/salesforce/apollo/stereotomy/EventValidation.java @@ -8,7 +8,6 @@ import com.salesforce.apollo.stereotomy.event.EstablishmentEvent; import com.salesforce.apollo.stereotomy.identifier.Identifier; -import org.joou.ULong; /** * The EventValidation provides validation predicates for EstablishmentEvents @@ -19,82 +18,34 @@ public interface EventValidation { EventValidation NONE = new EventValidation() { - @Override - public KeyState keyState(Identifier id, ULong sequenceNumber) { - return null; - } - @Override public boolean validate(EstablishmentEvent event) { return true; } @Override - public boolean validate(EventCoordinates coordinates) { + public boolean validate(Identifier identifier) { return true; } }; EventValidation NO_VALIDATION = new EventValidation() { - @Override - public KeyState keyState(Identifier id, ULong sequenceNumber) { - return null; - } - @Override public boolean validate(EstablishmentEvent event) { return false; } @Override - public boolean validate(EventCoordinates coordinates) { + public boolean validate(Identifier identifier) { return false; } }; - KeyState keyState(Identifier id, ULong sequenceNumber); - - /** - * Answer true if the event indicated by the coordinates is validated. This means that thresholds have been met from - * indicated witnesses and trusted validators. - */ - boolean validate(EventCoordinates coordinates); - /** * Answer true if the event is validated. This means that thresholds have been met from indicated witnesses and * trusted validators. */ boolean validate(EstablishmentEvent event); - class DelegatedEventValidation implements EventValidation { - private volatile EventValidation delegate; - - public DelegatedEventValidation(EventValidation delegate) { - this.delegate = delegate; - } - - @Override - public KeyState keyState(Identifier id, ULong sequenceNumber) { - return delegate().keyState(id, sequenceNumber); - } - - public void setDelegate(EventValidation delegate) { - this.delegate = delegate; - } - - @Override - public boolean validate(EventCoordinates coordinates) { - return delegate().validate(coordinates); - } - - @Override - public boolean validate(EstablishmentEvent event) { - return delegate().validate(event); - } - - private EventValidation delegate() { - final var current = delegate; - return current; - } - } + boolean validate(Identifier identifier); } diff --git a/stereotomy/src/main/java/com/salesforce/apollo/stereotomy/KERL.java b/stereotomy/src/main/java/com/salesforce/apollo/stereotomy/KERL.java index d54224e291..293b571981 100644 --- a/stereotomy/src/main/java/com/salesforce/apollo/stereotomy/KERL.java +++ b/stereotomy/src/main/java/com/salesforce/apollo/stereotomy/KERL.java @@ -6,18 +6,18 @@ */ package com.salesforce.apollo.stereotomy; -import com.google.protobuf.InvalidProtocolBufferException; import com.salesforce.apollo.cryptography.JohnHancock; import com.salesforce.apollo.stereotomy.event.AttachmentEvent; import com.salesforce.apollo.stereotomy.event.AttachmentEvent.Attachment; import com.salesforce.apollo.stereotomy.event.KeyEvent; import com.salesforce.apollo.stereotomy.event.KeyStateWithEndorsementsAndValidations; import com.salesforce.apollo.stereotomy.event.proto.KeyEventWithAttachments; -import com.salesforce.apollo.stereotomy.event.protobuf.ProtobufEventFactory; import com.salesforce.apollo.stereotomy.identifier.Identifier; -import java.util.*; -import java.util.concurrent.CompletableFuture; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; /** * The Key Event Receipt Log @@ -29,7 +29,7 @@ public interface KERL extends KEL { default KeyStateWithEndorsementsAndValidations getKeyStateWithEndorsementsAndValidations( EventCoordinates coordinates) { var ks = getKeyStateWithAttachments(coordinates); - if (ks != null) { + if (ks == null) { return null; } return KeyStateWithEndorsementsAndValidations.create(ks.state(), ks.attachments().endorsements(), @@ -62,7 +62,6 @@ private EventWithAttachments completeKerl(EventCoordinates c, List kerl(KeyEvent event) { - var fs = new CompletableFuture>(); var result = new ArrayList(); Attachment a = getAttachment(event.getCoordinates()); @@ -82,35 +81,6 @@ interface AppendKERL extends KERL, AppendKEL { record EventWithAttachments(KeyEvent event, Attachment attachments) { - static EventWithAttachments fromBase64(String encoded) { - var decoder = Base64.getUrlDecoder(); - String[] split = encoded.split("\\|"); - Attachment attachment = null; - if (split.length == 3) { - try { - attachment = Attachment.of( - com.salesforce.apollo.stereotomy.event.proto.Attachment.parseFrom(decoder.decode(split[2]))); - } catch (InvalidProtocolBufferException e) { - throw new IllegalArgumentException("Invalid attachment: " + encoded); - } - } else if (split.length != 2) { - throw new IllegalArgumentException("Invalid encoding: " + encoded); - } - return new EventWithAttachments(ProtobufEventFactory.toKeyEvent(decoder.decode(split[1]), split[0]), - attachment); - } - - public String toBase64() { - var encoder = Base64.getUrlEncoder().withoutPadding(); - var attachBytes = - attachments == null ? com.salesforce.apollo.stereotomy.event.proto.Attachment.getDefaultInstance() - .toByteArray() - : attachments.toAttachemente().toByteArray(); - var encoded = - event.getIlk() + "|" + encoder.encodeToString(event.getBytes()) + "|" + encoder.encodeToString(attachBytes); - return encoded; - } - public KeyEventWithAttachments toKeyEvente() { var builder = KeyEventWithAttachments.newBuilder(); event.setEventOf(builder); diff --git a/stereotomy/src/main/java/com/salesforce/apollo/stereotomy/KeyCoordinates.java b/stereotomy/src/main/java/com/salesforce/apollo/stereotomy/KeyCoordinates.java index b2e2a46077..24c35322be 100644 --- a/stereotomy/src/main/java/com/salesforce/apollo/stereotomy/KeyCoordinates.java +++ b/stereotomy/src/main/java/com/salesforce/apollo/stereotomy/KeyCoordinates.java @@ -6,12 +6,12 @@ */ package com.salesforce.apollo.stereotomy; -import static java.util.Objects.requireNonNull; +import com.salesforce.apollo.stereotomy.event.EstablishmentEvent; +import com.salesforce.apollo.stereotomy.event.proto.KeyCoords; import java.util.Objects; -import com.salesforce.apollo.stereotomy.event.proto.KeyCoords; -import com.salesforce.apollo.stereotomy.event.EstablishmentEvent; +import static java.util.Objects.requireNonNull; /** * The coordinates of a key in the KEL @@ -22,6 +22,7 @@ public class KeyCoordinates { private final EventCoordinates establishmentEvent; private final int keyIndex; + public KeyCoordinates(EventCoordinates establishmentEvent, int keyIndex) { if (keyIndex < 0) { throw new IllegalArgumentException("keyIndex must be >= 0"); @@ -46,10 +47,9 @@ public boolean equals(Object obj) { if (this == obj) { return true; } - if (!(obj instanceof KeyCoordinates)) { + if (!(obj instanceof KeyCoordinates other)) { return false; } - KeyCoordinates other = (KeyCoordinates) obj; return Objects.equals(establishmentEvent, other.establishmentEvent) && keyIndex == other.keyIndex; } diff --git a/stereotomy/src/main/java/com/salesforce/apollo/stereotomy/Stereotomy.java b/stereotomy/src/main/java/com/salesforce/apollo/stereotomy/Stereotomy.java index 16e96ac8c8..4c3fa2185c 100644 --- a/stereotomy/src/main/java/com/salesforce/apollo/stereotomy/Stereotomy.java +++ b/stereotomy/src/main/java/com/salesforce/apollo/stereotomy/Stereotomy.java @@ -7,14 +7,14 @@ package com.salesforce.apollo.stereotomy; import com.google.protobuf.InvalidProtocolBufferException; -import com.salesforce.apollo.cryptography.proto.Sig; -import com.salesforce.apollo.stereotomy.event.proto.EventCoords; import com.salesforce.apollo.cryptography.DigestAlgorithm; import com.salesforce.apollo.cryptography.JohnHancock; import com.salesforce.apollo.cryptography.Verifier; +import com.salesforce.apollo.cryptography.proto.Sig; import com.salesforce.apollo.stereotomy.event.AttachmentEvent; import com.salesforce.apollo.stereotomy.event.DelegatedInceptionEvent; import com.salesforce.apollo.stereotomy.event.Version; +import com.salesforce.apollo.stereotomy.event.proto.Ident; import com.salesforce.apollo.stereotomy.identifier.Identifier; import com.salesforce.apollo.stereotomy.identifier.SelfAddressingIdentifier; import com.salesforce.apollo.stereotomy.identifier.spec.IdentifierSpecification; @@ -76,9 +76,9 @@ static Optional decode(X509Certificate cert) { getLogger().warn("Invalid certificate, missing \"DC\" of dn= {}", dn); return Optional.empty(); } - EventCoordinates keyCoords; + Identifier identifier; try { - keyCoords = EventCoordinates.from(EventCoords.parseFrom(Base64.getUrlDecoder().decode(id.getBytes()))); + identifier = Identifier.from(Ident.parseFrom(Base64.getUrlDecoder().decode(id.getBytes()))); } catch (InvalidProtocolBufferException e) { getLogger().debug("Unable to deserialize key event coordinates", e); return Optional.empty(); @@ -90,7 +90,7 @@ static Optional decode(X509Certificate cert) { getLogger().debug("Unable to deserialize signature", e); return Optional.empty(); } - return Optional.of(new Decoded(keyCoords, JohnHancock.from(sig))); + return Optional.of(new Decoded(identifier, JohnHancock.from(sig))); } /** @@ -186,6 +186,6 @@ DelegatedInceptionEvent newDelegatedIdentifier(Identifier controller, */ ControlledIdentifier newIdentifier(IdentifierSpecification.Builder spec); - record Decoded(EventCoordinates coordinates, JohnHancock signature) { + record Decoded(Identifier identifier, JohnHancock signature) { } } diff --git a/stereotomy/src/main/java/com/salesforce/apollo/stereotomy/StereotomyImpl.java b/stereotomy/src/main/java/com/salesforce/apollo/stereotomy/StereotomyImpl.java index 470b4a5fe5..b4340b101d 100644 --- a/stereotomy/src/main/java/com/salesforce/apollo/stereotomy/StereotomyImpl.java +++ b/stereotomy/src/main/java/com/salesforce/apollo/stereotomy/StereotomyImpl.java @@ -42,7 +42,7 @@ import static com.salesforce.apollo.stereotomy.identifier.QualifiedBase64Identifier.qb64; /** - * Direct mode implementation of a Stereotomy controller. This controller keeps it's own KEL/KERL and does not cooperate + * Direct mode implementation of a Stereotomy controller. This controller keeps its own KEL/KERL and does not cooperate * with other controllers * * @author hal.hildebrand @@ -69,14 +69,14 @@ public StereotomyImpl(StereotomyKeyStore keyStore, KERL.AppendKERL kerl, SecureR @Override public BoundIdentifier bindingOf(EventCoordinates coordinates) { KeyState lookup = kerl.getKeyState(coordinates); - return new ControlledIdentifierImpl(lookup); + return new ControlledIdentifierImpl<>(lookup); } @Override public ControlledIdentifier commit(DelegatedInceptionEvent delegation, AttachmentEvent commitment) { - List ks = kerl.append(Arrays.asList(delegation), Arrays.asList(commitment)); - var cid = new ControlledIdentifierImpl(ks.get(0)); + List ks = kerl.append(Collections.singletonList(delegation), Collections.singletonList(commitment)); + var cid = new ControlledIdentifierImpl(ks.getFirst()); log.info("New delegated identifier: {} coordinates: {}", cid.getIdentifier(), cid.getCoordinates()); return cid; } @@ -84,7 +84,7 @@ public ControlledIdentifier commit(DelegatedInceptionE @Override public ControlledIdentifier controlOf(D identifier) { KeyState lookup = kerl.getKeyState(identifier); - return new ControlledIdentifierImpl(lookup); + return new ControlledIdentifierImpl<>(lookup); } @Override @@ -134,7 +134,7 @@ private Optional getKeyPair(KeyCoordinates keyCoords) { return keyStore.getKey(keyCoords); } - private Optional getKeyPair(KeyState state, int keyIndex, EstablishmentEvent lastEstablishmentEvent) { + private Optional getKeyPair(int keyIndex, EstablishmentEvent lastEstablishmentEvent) { if (lastEstablishmentEvent == null) { return Optional.empty(); } @@ -152,7 +152,7 @@ private Signer getSigner(KeyState state) { var signers = new PrivateKey[state.getKeys().size()]; EstablishmentEvent e = getLastEstablishingEvent(state); for (int i = 0; i < signers.length; i++) { - Optional keyPair = getKeyPair(state, i, e); + Optional keyPair = getKeyPair(i, e); if (keyPair.isEmpty()) { log.warn("Last establishment event not found in KEL: {} : {} missing: {}", identifier, state.getCoordinates(), state.getLastEstablishmentEvent()); @@ -168,9 +168,7 @@ private InceptionEvent inception(Identifier delegatingIde IdentifierSpecification.Builder specification = spec.clone(); var initialKeyPair = specification.getSignatureAlgorithm().generateKeyPair(entropy); - KeyPair nextKeyPair = null; - - nextKeyPair = specification.getSignatureAlgorithm().generateKeyPair(entropy); + KeyPair nextKeyPair = specification.getSignatureAlgorithm().generateKeyPair(entropy); specification.addKey(initialKeyPair.getPublic()) .setSigningThreshold(unweighted(1)) @@ -212,7 +210,6 @@ private KeyEvent interaction(KeyState state, InteractionSpecification.Builder sp return eventFactory.interaction(specification.build()); } - @SuppressWarnings("unchecked") private ControlledIdentifier newIdentifier( ControlledIdentifier delegator, IdentifierSpecification.Builder spec) { log.warn("New identifier, controller: {}", delegator.getIdentifier()); @@ -222,20 +219,17 @@ private ControlledIdentifier newIdentifier( // Seal we need to verify the inception, based on the delegated inception // location var seals = InteractionSpecification.newBuilder() - .addAllSeals(Arrays.asList(EventSeal.construct(event.getIdentifier(), - event.hash( - kerl.getDigestAlgorithm()), - event.getSequenceNumber() - .longValue()))); + .addAllSeals(List.of(EventSeal.construct(event.getIdentifier(), event.hash( + kerl.getDigestAlgorithm()), event.getSequenceNumber().longValue()))); // Interaction event with the seal - KeyState ks = kerl.append(event); + kerl.append(event); var interaction = interaction(delegator, seals); // Attachment of the interaction event, verifying the delegated inception var attachment = eventFactory.attachment(event, new AttachmentImpl( EventSeal.construct(interaction.getIdentifier(), interaction.hash(kerl.getDigestAlgorithm()), interaction.getSequenceNumber().longValue()))); - var s = kerl.append(Collections.singletonList(interaction), Collections.singletonList(attachment)); + kerl.append(Collections.singletonList(interaction), Collections.singletonList(attachment)); var delegatedState = kerl.append(event); if (delegatedState == null) { log.warn("Unable to append inception event for identifier: {}", event.getIdentifier()); @@ -243,7 +237,7 @@ private ControlledIdentifier newIdentifier( } // Finally, the new delegated identifier - ControlledIdentifier cid = new ControlledIdentifierImpl(delegatedState); + ControlledIdentifier cid = new ControlledIdentifierImpl<>(delegatedState); log.info("New {} delegator: {} identifier: {} coordinates: {}", spec.getWitnesses().isEmpty() ? "Private" : "Public", cid.getDelegatingIdentifier().get(), @@ -329,11 +323,6 @@ public Set configurationTraits() { return getState().configurationTraits(); } - @Override - public boolean equals(Object o) { - return getState().equals(o); - } - @Override public byte[] getBytes() { return getState().getBytes(); @@ -399,11 +388,6 @@ public List getWitnesses() { return getState().getWitnesses(); } - @Override - public int hashCode() { - return getState().hashCode(); - } - @Override public boolean isDelegated() { return getState().isDelegated(); @@ -451,7 +435,7 @@ public EstablishmentEvent getLastEstablishingEvent() { @Override public Optional getVerifier() { - return Optional.of(new KerlVerifier(getIdentifier(), kerl)); + return Optional.of(new KerlVerifier<>(getIdentifier(), kerl)); } @Override @@ -470,7 +454,7 @@ public KeyState_ toKeyState_() { @Override KeyState getState() { - KeyState current = state; + final var current = state; return current; } @@ -493,7 +477,7 @@ public ControlledIdentifierImpl(KeyState state) { @Override public BoundIdentifier bind() { - return new BoundControllableIdentifier(getState()); + return new BoundControllableIdentifier<>(getState()); } @Override @@ -562,8 +546,8 @@ public CertificateWithPrivateKey provision(Instant validFrom, Duration valid, Li var dn = new BcX500NameDnImpl(String.format("UID=%s, DC=%s", Base64.getUrlEncoder() .encodeToString( - (getState().getCoordinates() - .toEventCoords() + (getState().getIdentifier() + .toIdent() .toByteArray())), Base64.getUrlEncoder() .encodeToString(signature.toSig().toByteArray()))); diff --git a/stereotomy/src/main/java/com/salesforce/apollo/stereotomy/StereotomyKeyStore.java b/stereotomy/src/main/java/com/salesforce/apollo/stereotomy/StereotomyKeyStore.java index 372bfaf002..1e3f3bfe3f 100644 --- a/stereotomy/src/main/java/com/salesforce/apollo/stereotomy/StereotomyKeyStore.java +++ b/stereotomy/src/main/java/com/salesforce/apollo/stereotomy/StereotomyKeyStore.java @@ -10,22 +10,27 @@ import java.util.Optional; /** - * The interface for a keystore that tracks KeyPairs by keyCoordinates, and - * knows about current and next keypairs associated with those coordinates - * - * @author hal.hildebrand + * The interface for a keystore that tracks KeyPairs by keyCoordinates, and knows about current and next keypairs + * associated with those coordinates * + * @author hal.hildebrand */ public interface StereotomyKeyStore { + Optional getKey(String alias); + Optional getKey(KeyCoordinates keyCoordinates); Optional getNextKey(KeyCoordinates keyCoordinates); void removeKey(KeyCoordinates keyCoordinates); + void removeKey(String alias); + void removeNextKey(KeyCoordinates keyCoordinates); + void storeKey(String alias, KeyPair keyPair); + void storeKey(KeyCoordinates keyCoordinates, KeyPair keyPair); void storeNextKey(KeyCoordinates keyCoordinates, KeyPair keyPair); diff --git a/stereotomy/src/main/java/com/salesforce/apollo/stereotomy/StereotomyValidator.java b/stereotomy/src/main/java/com/salesforce/apollo/stereotomy/StereotomyValidator.java index 2157d6ef84..26bd0f6541 100644 --- a/stereotomy/src/main/java/com/salesforce/apollo/stereotomy/StereotomyValidator.java +++ b/stereotomy/src/main/java/com/salesforce/apollo/stereotomy/StereotomyValidator.java @@ -6,18 +6,17 @@ */ package com.salesforce.apollo.stereotomy; -import static com.salesforce.apollo.stereotomy.identifier.QualifiedBase64Identifier.qb64; +import com.salesforce.apollo.cryptography.ssl.CertificateValidator; +import com.salesforce.apollo.stereotomy.Stereotomy.Decoded; +import com.salesforce.apollo.stereotomy.identifier.BasicIdentifier; import java.security.cert.CertificateException; import java.security.cert.X509Certificate; -import com.salesforce.apollo.cryptography.ssl.CertificateValidator; -import com.salesforce.apollo.stereotomy.Stereotomy.Decoded; -import com.salesforce.apollo.stereotomy.identifier.BasicIdentifier; +import static com.salesforce.apollo.stereotomy.identifier.QualifiedBase64Identifier.qb64; /** * @author hal.hildebrand - * */ public class StereotomyValidator implements CertificateValidator { @@ -37,10 +36,9 @@ public void validate(final X509Certificate cert) throws CertificateException { } final var qb64Id = qb64(basicId); Decoded decoder = decoded.get(); - var verifier = verifiers.verifierFor(decoded.get().coordinates()); + var verifier = verifiers.verifierFor(decoded.get().identifier()); if (verifier.isEmpty()) { - throw new CertificateException(String.format("No verifier for coordinates: %s", - decoded.get().coordinates())); + throw new CertificateException(String.format("No verifier for identifier: %s", decoded.get().identifier())); } if (!verifier.get().verify(decoder.signature(), qb64Id)) { throw new CertificateException(String.format("Cannot verify cert public key signature for %s", basicId)); diff --git a/stereotomy/src/main/java/com/salesforce/apollo/stereotomy/Verifiers.java b/stereotomy/src/main/java/com/salesforce/apollo/stereotomy/Verifiers.java index 370302e3e2..30ba52e101 100644 --- a/stereotomy/src/main/java/com/salesforce/apollo/stereotomy/Verifiers.java +++ b/stereotomy/src/main/java/com/salesforce/apollo/stereotomy/Verifiers.java @@ -118,50 +118,42 @@ private Verifiers delegate() { class FixedVerifiers implements Verifiers { private final Map verifiersByCoordinates; - private final Map verifiersByIdentifer; + private final Map verifiersByIdentifier; public FixedVerifiers(Map verifiersByCoordinates, - Map verifiersByIdentifer) { + Map verifiersByIdentifier) { this.verifiersByCoordinates = verifiersByCoordinates; - this.verifiersByIdentifer = verifiersByIdentifer; + this.verifiersByIdentifier = verifiersByIdentifier; } private FixedVerifiers(Pair verifiers) { verifiersByCoordinates = verifiers.coords; - verifiersByIdentifer = verifiers.ids; + verifiersByIdentifier = verifiers.ids; } private static Pair fromEvents(Collection states) { Map coords = new HashMap<>(); Map ids = new HashMap<>(); - states.forEach(ks -> { - coords.put(ks.getCoordinates(), new DefaultVerifier(ks.getKeys())); - }); - states.forEach(ks -> { - ids.put(ks.getIdentifier(), new DefaultVerifier(ks.getKeys())); - }); + states.forEach(ks -> coords.put(ks.getCoordinates(), new DefaultVerifier(ks.getKeys()))); + states.forEach(ks -> ids.put(ks.getIdentifier(), new DefaultVerifier(ks.getKeys()))); return new Pair(coords, ids); } private static Pair fromEventState( Collection states) { - return fromEvents(states.stream().map(ks -> ProtobufEventFactory.toKeyEvent(ks)).toList()); + return fromEvents(states.stream().map(ProtobufEventFactory::toKeyEvent).toList()); } private static Pair fromKeyState(Collection states) { Map coords = new HashMap<>(); Map ids = new HashMap<>(); - states.forEach(ks -> { - coords.put(ks.getCoordinates(), new DefaultVerifier(ks.getKeys())); - }); - states.forEach(ks -> { - ids.put(ks.getIdentifier(), new DefaultVerifier(ks.getKeys())); - }); + states.forEach(ks -> coords.put(ks.getCoordinates(), new DefaultVerifier(ks.getKeys()))); + states.forEach(ks -> ids.put(ks.getIdentifier(), new DefaultVerifier(ks.getKeys()))); return new Pair(coords, ids); } private static Pair fromKeyState_(Collection states) { - return fromKeyState(states.stream().map(ks -> new KeyStateImpl(ks)).map(ks -> (KeyState) ks).toList()); + return fromKeyState(states.stream().map(KeyStateImpl::new).map(ks -> (KeyState) ks).toList()); } @Override @@ -171,7 +163,7 @@ public Optional verifierFor(EventCoordinates coordinates) { @Override public Optional verifierFor(Identifier identifier) { - return Optional.ofNullable(verifiersByIdentifer.get(identifier)); + return Optional.ofNullable(verifiersByIdentifier.get(identifier)); } record Pair(Map coords, Map ids) { diff --git a/stereotomy/src/main/java/com/salesforce/apollo/stereotomy/caching/CachingKEL.java b/stereotomy/src/main/java/com/salesforce/apollo/stereotomy/caching/CachingKEL.java index 6323cb982d..33eb64c417 100644 --- a/stereotomy/src/main/java/com/salesforce/apollo/stereotomy/caching/CachingKEL.java +++ b/stereotomy/src/main/java/com/salesforce/apollo/stereotomy/caching/CachingKEL.java @@ -53,18 +53,18 @@ public CachingKEL(Function, ?> kelSupplier) { public CachingKEL(Function, ?> kelSupplier, Caffeine builder, Caffeine eventBuilder) { - ksCoords = builder.build(new CacheLoader() { + ksCoords = builder.build(new CacheLoader<>() { @Override - public @Nullable KeyState load(EventCoordinates key) throws Exception { + public @Nullable KeyState load(EventCoordinates key) { return complete(kel -> kel.getKeyState(key)); } }); this.kelSupplier = kelSupplier; - this.keyCoords = eventBuilder.build(new CacheLoader() { + this.keyCoords = eventBuilder.build(new CacheLoader<>() { @Override - public @Nullable KeyEvent load(EventCoordinates key) throws Exception { + public @Nullable KeyEvent load(EventCoordinates key) { return complete(kel -> kel.getKeyEvent(key)); } }); @@ -104,9 +104,6 @@ public List append(KeyEvent... events) { } try { return complete(kel -> kel.append(events)); - } catch (ClassCastException e) { - log.error("Cannot complete append", e); - return null; } catch (Throwable e) { log.error("Cannot complete append", e); return null; @@ -148,7 +145,7 @@ public Attachment getAttachment(EventCoordinates coordinates) { @Override public DigestAlgorithm getDigestAlgorithm() { try { - return complete(kel -> kel.getDigestAlgorithm()); + return complete(KEL::getDigestAlgorithm); } catch (Throwable e) { log.error("Cannot complete append", e); return null; diff --git a/stereotomy/src/main/java/com/salesforce/apollo/stereotomy/db/UniKERL.java b/stereotomy/src/main/java/com/salesforce/apollo/stereotomy/db/UniKERL.java index 30a89e7422..3923cb8ce4 100644 --- a/stereotomy/src/main/java/com/salesforce/apollo/stereotomy/db/UniKERL.java +++ b/stereotomy/src/main/java/com/salesforce/apollo/stereotomy/db/UniKERL.java @@ -7,12 +7,10 @@ package com.salesforce.apollo.stereotomy.db; import com.google.protobuf.InvalidProtocolBufferException; -import com.salesforce.apollo.cryptography.proto.Sig; -import com.salesforce.apollo.stereotomy.event.proto.EventCoords; -import com.salesforce.apollo.stereotomy.event.proto.Sealed; import com.salesforce.apollo.cryptography.Digest; import com.salesforce.apollo.cryptography.DigestAlgorithm; import com.salesforce.apollo.cryptography.JohnHancock; +import com.salesforce.apollo.cryptography.proto.Sig; import com.salesforce.apollo.stereotomy.DigestKERL; import com.salesforce.apollo.stereotomy.EventCoordinates; import com.salesforce.apollo.stereotomy.KeyState; @@ -21,6 +19,8 @@ import com.salesforce.apollo.stereotomy.event.AttachmentEvent.AttachmentImpl; import com.salesforce.apollo.stereotomy.event.KeyEvent; import com.salesforce.apollo.stereotomy.event.Seal; +import com.salesforce.apollo.stereotomy.event.proto.EventCoords; +import com.salesforce.apollo.stereotomy.event.proto.Sealed; import com.salesforce.apollo.stereotomy.event.protobuf.AttachmentEventImpl; import com.salesforce.apollo.stereotomy.event.protobuf.KeyStateImpl; import com.salesforce.apollo.stereotomy.event.protobuf.ProtobufEventFactory; @@ -42,6 +42,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; import java.util.zip.GZIPInputStream; @@ -74,7 +75,7 @@ public UniKERL(Connection connection, DigestAlgorithm digestAlgorithm) { } public static void append(DSLContext dsl, AttachmentEvent attachment) { - if (attachment.attachments().seals().size() == 0 && attachment.attachments().endorsements().size() == 0) { + if (attachment.attachments().seals().isEmpty() && attachment.attachments().endorsements().isEmpty()) { return; } var coordinates = attachment.coordinates(); @@ -116,7 +117,7 @@ public static void append(DSLContext dsl, AttachmentEvent attachment) { .whenNotMatchedThenInsert() .set(ATTACHMENT.FOR, id.value1()) .set(ATTACHMENT.SEAL, bytes) - .execute(), (a, b) -> a + b); + .execute(), Integer::sum); } log.info("appended: {} seals out of: {} coords: {}", count.get(), attachment.attachments().seals().size(), coordinates); @@ -130,7 +131,7 @@ public static void append(DSLContext dsl, AttachmentEvent attachment) { .set(RECEIPT.FOR, id.value1()) .set(RECEIPT.WITNESS, entry.getKey()) .set(RECEIPT.SIGNATURE, entry.getValue().toSig().toByteArray()) - .execute(), (a, b) -> a + b); + .execute(), Integer::sum); } log.info("appended: {} endorsements out of: {} coords: {}", count.get(), attachment.attachments().endorsements().size(), coordinates); @@ -255,7 +256,7 @@ public static byte[] appendEvent(Connection connection, byte[] event, String ilk public static void appendValidations(DSLContext dsl, EventCoordinates coordinates, Map validations) { - if (validations.size() == 0) { + if (validations.isEmpty()) { return; } final var identBytes = coordinates.getIdentifier().toIdent().toByteArray(); @@ -300,7 +301,7 @@ public static void appendValidations(DSLContext dsl, EventCoordinates coordinate vRec.setFor(l); vRec.setValidator(coords.toEventCoords().toByteArray()); vRec.setSignature(signature.toSig().toByteArray()); - result.accumulateAndGet(vRec.merge(), (a, b) -> a + b); + result.accumulateAndGet(vRec.merge(), Integer::sum); }); log.info("Inserted validations: {} out of : {} for event: {}", result.get(), validations.size(), coordinates); } @@ -308,7 +309,7 @@ public static void appendValidations(DSLContext dsl, EventCoordinates coordinate public static byte[] compress(byte[] input) { ByteArrayOutputStream baos = new ByteArrayOutputStream(); try (GZIPOutputStream gzos = new GZIPOutputStream(baos); - ByteArrayInputStream bais = new ByteArrayInputStream(input);) { + ByteArrayInputStream bais = new ByteArrayInputStream(input)) { bais.transferTo(gzos); gzos.finish(); gzos.flush(); @@ -322,7 +323,7 @@ public static byte[] compress(byte[] input) { public static byte[] decompress(byte[] input) { ByteArrayOutputStream baos = new ByteArrayOutputStream(); try (ByteArrayInputStream bais = new ByteArrayInputStream(input); - GZIPInputStream gis = new GZIPInputStream(bais);) { + GZIPInputStream gis = new GZIPInputStream(bais)) { gis.transferTo(baos); baos.flush(); } catch (IOException e) { @@ -394,7 +395,7 @@ public Attachment getAttachment(EventCoordinates coordinates) { return null; } }) - .filter(s -> s != null) + .filter(Objects::nonNull) .toList(); record receipt(int witness, Sig signature) { @@ -412,7 +413,7 @@ record receipt(int witness, Sig signature) { return null; } }) - .filter(s -> s != null) + .filter(Objects::nonNull) .collect(Collectors.toMap(r -> r.witness, r -> JohnHancock.from(r.signature))); return new AttachmentImpl(seals, receipts); } @@ -563,7 +564,7 @@ record validation(EventCoords coordinates, Sig signature) { return null; } }) - .filter(s -> s != null) + .filter(Objects::nonNull) .collect(Collectors.toMap(v -> EventCoordinates.from(v.coordinates), v -> JohnHancock.from(v.signature))); log.trace("Resolve validations: {} result: {}", coordinates, validations); diff --git a/stereotomy/src/main/java/com/salesforce/apollo/stereotomy/db/UniKERLDirect.java b/stereotomy/src/main/java/com/salesforce/apollo/stereotomy/db/UniKERLDirect.java index b1a7a9fd0f..2b490119e1 100644 --- a/stereotomy/src/main/java/com/salesforce/apollo/stereotomy/db/UniKERLDirect.java +++ b/stereotomy/src/main/java/com/salesforce/apollo/stereotomy/db/UniKERLDirect.java @@ -31,17 +31,13 @@ public UniKERLDirect(Connection connection, DigestAlgorithm digestAlgorithm) { @Override public KeyState append(KeyEvent event) { KeyState newState = processor.process(event); - dsl.transaction(ctx -> { - append(DSL.using(ctx), event, newState, digestAlgorithm); - }); + dsl.transaction(ctx -> append(DSL.using(ctx), event, newState, digestAlgorithm)); return newState; } @Override public Void append(List events) { - dsl.transaction(ctx -> { - events.forEach(event -> append(DSL.using(ctx), event)); - }); + dsl.transaction(ctx -> events.forEach(event -> append(DSL.using(ctx), event))); return null; } @@ -62,9 +58,7 @@ public List append(List events, List attach @Override public Void appendValidations(EventCoordinates coordinates, Map validations) { - dsl.transaction(ctx -> { - appendValidations(DSL.using(ctx), coordinates, validations); - }); + dsl.transaction(ctx -> appendValidations(DSL.using(ctx), coordinates, validations)); return null; } } diff --git a/stereotomy/src/main/java/com/salesforce/apollo/stereotomy/event/AttachmentEvent.java b/stereotomy/src/main/java/com/salesforce/apollo/stereotomy/event/AttachmentEvent.java index 3f6c1f7cbe..da4df706cd 100644 --- a/stereotomy/src/main/java/com/salesforce/apollo/stereotomy/event/AttachmentEvent.java +++ b/stereotomy/src/main/java/com/salesforce/apollo/stereotomy/event/AttachmentEvent.java @@ -6,16 +6,12 @@ */ package com.salesforce.apollo.stereotomy.event; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.TreeMap; -import java.util.stream.Collectors; - import com.salesforce.apollo.cryptography.JohnHancock; import com.salesforce.apollo.stereotomy.EventCoordinates; +import java.util.*; +import java.util.stream.Collectors; + /** * @author hal.hildebrand */ @@ -50,12 +46,12 @@ public List seals() { }; static Attachment of(com.salesforce.apollo.stereotomy.event.proto.Attachment attachment) { - return new AttachmentImpl(attachment.getSealsList().stream().map(s -> Seal.from(s)).toList(), + return new AttachmentImpl(attachment.getSealsList().stream().map(Seal::from).toList(), attachment.getEndorsementsMap() .entrySet() .stream() - .collect( - Collectors.toMap(e -> e.getKey(), e -> JohnHancock.of(e.getValue())))); + .collect(Collectors.toMap(Map.Entry::getKey, + e -> JohnHancock.of(e.getValue())))); } Map endorsements(); @@ -64,16 +60,16 @@ static Attachment of(com.salesforce.apollo.stereotomy.event.proto.Attachment att default com.salesforce.apollo.stereotomy.event.proto.Attachment toAttachemente() { var builder = com.salesforce.apollo.stereotomy.event.proto.Attachment.newBuilder(); - builder.addAllSeals(seals().stream().map(s -> s.toSealed()).toList()) + builder.addAllSeals(seals().stream().map(Seal::toSealed).toList()) .putAllEndorsements(endorsements().entrySet() .stream() .collect( - Collectors.toMap(e -> e.getKey(), e -> e.getValue().toSig()))); + Collectors.toMap(Map.Entry::getKey, e -> e.getValue().toSig()))); return builder.build(); } } - static class AttachmentImpl implements Attachment { + class AttachmentImpl implements Attachment { private final Map endorsements; private final List seals; diff --git a/stereotomy/src/main/java/com/salesforce/apollo/stereotomy/event/DelegatedInceptionEvent.java b/stereotomy/src/main/java/com/salesforce/apollo/stereotomy/event/DelegatedInceptionEvent.java index cf01108bc4..c6a022c739 100644 --- a/stereotomy/src/main/java/com/salesforce/apollo/stereotomy/event/DelegatedInceptionEvent.java +++ b/stereotomy/src/main/java/com/salesforce/apollo/stereotomy/event/DelegatedInceptionEvent.java @@ -10,14 +10,13 @@ /** * @author hal.hildebrand - * */ public interface DelegatedInceptionEvent extends InceptionEvent, DelegatedEstablishmentEvent { + Identifier getDelegatingPrefix(); + @Override default String getIlk() { return DELEGATED_INCEPTION_TYPE; } - - public Identifier getDelegatingPrefix(); } diff --git a/stereotomy/src/main/java/com/salesforce/apollo/stereotomy/event/KeyStateWithEndorsementsAndValidations.java b/stereotomy/src/main/java/com/salesforce/apollo/stereotomy/event/KeyStateWithEndorsementsAndValidations.java index 029f75e560..14255204a8 100644 --- a/stereotomy/src/main/java/com/salesforce/apollo/stereotomy/event/KeyStateWithEndorsementsAndValidations.java +++ b/stereotomy/src/main/java/com/salesforce/apollo/stereotomy/event/KeyStateWithEndorsementsAndValidations.java @@ -6,17 +6,17 @@ */ package com.salesforce.apollo.stereotomy.event; -import java.util.Map; -import java.util.TreeMap; -import java.util.stream.Collectors; - -import com.salesforce.apollo.stereotomy.event.proto.KeyStateWithEndorsementsAndValidations_; -import com.salesforce.apollo.stereotomy.event.proto.Validation_; import com.salesforce.apollo.cryptography.JohnHancock; import com.salesforce.apollo.stereotomy.EventCoordinates; import com.salesforce.apollo.stereotomy.KeyState; +import com.salesforce.apollo.stereotomy.event.proto.KeyStateWithEndorsementsAndValidations_; +import com.salesforce.apollo.stereotomy.event.proto.Validation_; import com.salesforce.apollo.stereotomy.event.protobuf.KeyStateImpl; +import java.util.Map; +import java.util.TreeMap; +import java.util.stream.Collectors; + /** * @author hal.hildebrand */ @@ -34,14 +34,14 @@ public static KeyStateWithEndorsementsAndValidations from(KeyStateWithEndorsemen ks.getEndorsementsMap() .entrySet() .stream() - .collect(Collectors.toMap(e -> e.getKey(), e -> JohnHancock.from(e.getValue())))), ks.getValidationsList() - .stream() - .collect( - Collectors.toMap( - e -> EventCoordinates.from( - e.getValidator()), - e -> JohnHancock.from( - e.getSignature())))); + .collect(Collectors.toMap(Map.Entry::getKey, e -> JohnHancock.from(e.getValue())))), ks.getValidationsList() + .stream() + .collect( + Collectors.toMap( + e -> EventCoordinates.from( + e.getValidator()), + e -> JohnHancock.from( + e.getSignature())))); } public KeyStateWithEndorsementsAndValidations_ toKS() { @@ -61,10 +61,9 @@ public KeyStateWithEndorsementsAndValidations_ toKS() { .toList()) .putAllEndorsements(endorsements.entrySet() .stream() - .collect( - Collectors.toMap(e -> e.getKey(), - e -> e.getValue() - .toSig()))) + .collect(Collectors.toMap( + Map.Entry::getKey, + e -> e.getValue().toSig()))) .build(); } diff --git a/stereotomy/src/main/java/com/salesforce/apollo/stereotomy/identifier/SelfAddressingIdentifier.java b/stereotomy/src/main/java/com/salesforce/apollo/stereotomy/identifier/SelfAddressingIdentifier.java index b69689d90a..90486075e5 100644 --- a/stereotomy/src/main/java/com/salesforce/apollo/stereotomy/identifier/SelfAddressingIdentifier.java +++ b/stereotomy/src/main/java/com/salesforce/apollo/stereotomy/identifier/SelfAddressingIdentifier.java @@ -6,11 +6,11 @@ */ package com.salesforce.apollo.stereotomy.identifier; -import java.util.Objects; - -import com.salesforce.apollo.stereotomy.event.proto.Ident; import com.salesforce.apollo.cryptography.Digest; import com.salesforce.apollo.cryptography.DigestAlgorithm; +import com.salesforce.apollo.stereotomy.event.proto.Ident; + +import java.util.Objects; /** * @author hal.hildebrand @@ -20,6 +20,7 @@ public class SelfAddressingIdentifier implements Identifier, Comparable passwordProvider, File f } @Override - protected void store(String alias, KeyPair keyPair) { - super.store(alias, keyPair); + public void removeKey(String alias) { + super.removeKey(alias); + save(); + } + + @Override + public void removeKey(KeyCoordinates keyCoordinates) { + super.removeKey(keyCoordinates); + save(); + } + + @Override + public void removeNextKey(KeyCoordinates keyCoordinates) { + super.removeNextKey(keyCoordinates); + save(); + } + + @Override + public void storeKey(String alias, KeyPair keyPair) { + super.storeKey(alias, keyPair); + save(); + } + + private void save() { try (var fos = new FileOutputStream(file)) { keyStore.store(fos, passwordProvider.get()); } catch (IOException | KeyStoreException | NoSuchAlgorithmException | CertificateException e) { diff --git a/stereotomy/src/main/java/com/salesforce/apollo/stereotomy/jks/JksKeyStore.java b/stereotomy/src/main/java/com/salesforce/apollo/stereotomy/jks/JksKeyStore.java index de662e958f..2b23f92381 100644 --- a/stereotomy/src/main/java/com/salesforce/apollo/stereotomy/jks/JksKeyStore.java +++ b/stereotomy/src/main/java/com/salesforce/apollo/stereotomy/jks/JksKeyStore.java @@ -6,16 +6,16 @@ */ package com.salesforce.apollo.stereotomy.jks; -import static com.salesforce.apollo.cryptography.QualifiedBase64.qb64; -import static com.salesforce.apollo.stereotomy.identifier.QualifiedBase64Identifier.qb64; +import com.salesforce.apollo.cryptography.cert.BcX500NameDnImpl; +import com.salesforce.apollo.cryptography.cert.CertExtension; +import com.salesforce.apollo.cryptography.cert.Certificates; +import com.salesforce.apollo.stereotomy.KeyCoordinates; +import com.salesforce.apollo.stereotomy.StereotomyKeyStore; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.math.BigInteger; -import java.security.KeyPair; -import java.security.KeyStore; -import java.security.KeyStoreException; -import java.security.NoSuchAlgorithmException; -import java.security.PrivateKey; -import java.security.UnrecoverableKeyException; +import java.security.*; import java.security.cert.Certificate; import java.security.cert.X509Certificate; import java.time.Instant; @@ -26,26 +26,27 @@ import java.util.concurrent.locks.ReentrantLock; import java.util.function.Supplier; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.salesforce.apollo.cryptography.cert.BcX500NameDnImpl; -import com.salesforce.apollo.cryptography.cert.CertExtension; -import com.salesforce.apollo.cryptography.cert.Certificates; -import com.salesforce.apollo.stereotomy.KeyCoordinates; -import com.salesforce.apollo.stereotomy.StereotomyKeyStore; +import static com.salesforce.apollo.cryptography.QualifiedBase64.qb64; +import static com.salesforce.apollo.stereotomy.identifier.QualifiedBase64Identifier.qb64; /** * @author hal.hildebrand - * */ public class JksKeyStore implements StereotomyKeyStore { - private static Logger log = LoggerFactory.getLogger(JksKeyStore.class); + private static final Logger log = LoggerFactory.getLogger(JksKeyStore.class); + protected final KeyStore keyStore; + protected final Supplier passwordProvider; + private final Lock lock = new ReentrantLock(); + + public JksKeyStore(KeyStore keyStore, Supplier passwordProvider) { + this.keyStore = keyStore; + this.passwordProvider = passwordProvider; + } public static String coordinateOrdering(KeyCoordinates coords) { var eventCoords = coords.getEstablishmentEvent(); - return qb64(eventCoords.getIdentifier()) + ':' + eventCoords.getSequenceNumber() + ':' - + qb64(eventCoords.getDigest()) + ":" + Integer.toString(coords.getKeyIndex()); + return qb64(eventCoords.getIdentifier()) + ':' + eventCoords.getSequenceNumber() + ':' + qb64( + eventCoords.getDigest()) + ":" + coords.getKeyIndex(); } private static String current(KeyCoordinates keyCoordinates) { @@ -56,14 +57,9 @@ private static String next(KeyCoordinates keyCoordinates) { return String.format("%s:%s", coordinateOrdering(keyCoordinates), "1"); } - protected final KeyStore keyStore; - protected final Supplier passwordProvider; - - private final Lock lock = new ReentrantLock(); - - public JksKeyStore(KeyStore keyStore, Supplier passwordProvider) { - this.keyStore = keyStore; - this.passwordProvider = passwordProvider; + @Override + public Optional getKey(String alias) { + return get(alias, null); } @Override @@ -79,12 +75,21 @@ public Optional getNextKey(KeyCoordinates keyCoordinates) { @Override public void removeKey(KeyCoordinates keyCoordinates) { try { - keyStore.deleteEntry(next(keyCoordinates)); + keyStore.deleteEntry(current(keyCoordinates)); } catch (KeyStoreException e) { throw new IllegalStateException("Error deleting current: " + keyCoordinates, e); } } + @Override + public void removeKey(String alias) { + try { + keyStore.deleteEntry(alias); + } catch (KeyStoreException e) { + throw new IllegalStateException("Error deleting: " + alias, e); + } + } + @Override public void removeNextKey(KeyCoordinates keyCoordinates) { try { @@ -94,11 +99,26 @@ public void removeNextKey(KeyCoordinates keyCoordinates) { } } + public void storeKey(final String alias, KeyPair keyPair) { + BcX500NameDnImpl dn = new BcX500NameDnImpl("CN=noop"); + BigInteger sn = BigInteger.valueOf(Long.MAX_VALUE); + var notBefore = Instant.now(); + var notAfter = Instant.now().plusSeconds(2_000_000_000); + List extensions = Collections.emptyList(); + X509Certificate selfSignedCert = Certificates.selfSign(true, dn, sn, keyPair, notBefore, notAfter, extensions); + try { + keyStore.setKeyEntry(alias, keyPair.getPrivate(), passwordProvider.get(), + new Certificate[] { selfSignedCert }); + } catch (KeyStoreException e) { + throw new IllegalStateException(e); + } + } + @Override public void storeKey(KeyCoordinates keyCoordinates, KeyPair keyPair) { lock.lock(); try { - store(current(keyCoordinates), keyPair); + storeKey(current(keyCoordinates), keyPair); } finally { lock.unlock(); } @@ -108,41 +128,26 @@ public void storeKey(KeyCoordinates keyCoordinates, KeyPair keyPair) { public void storeNextKey(KeyCoordinates keyCoordinates, KeyPair keyPair) { lock.lock(); try { - store(next(keyCoordinates), keyPair); + storeKey(next(keyCoordinates), keyPair); } finally { lock.unlock(); } } - protected void store(final String alias, KeyPair keyPair) { - BcX500NameDnImpl dn = new BcX500NameDnImpl("CN=noop"); - BigInteger sn = BigInteger.valueOf(Long.MAX_VALUE); - var notBefore = Instant.now(); - var notAfter = Instant.now().plusSeconds(2_000_000_000); - List extensions = Collections.emptyList(); - X509Certificate selfSignedCert = Certificates.selfSign(true, dn, sn, keyPair, notBefore, notAfter, extensions); - try { - keyStore.setKeyEntry(alias, keyPair.getPrivate(), passwordProvider.get(), - new Certificate[] { selfSignedCert }); - } catch (KeyStoreException e) { - throw new IllegalStateException(e); - } - } - private Optional get(String alias, KeyCoordinates keyCoordinates) { try { if (!keyStore.containsAlias(alias)) { return Optional.empty(); } } catch (KeyStoreException e) { - log.error("Unable to query keystore for: {}", keyCoordinates, e); + log.error("Unable to query keystore for: {}", keyCoordinates != null ? keyCoordinates : alias, e); return Optional.empty(); } Certificate cert; try { cert = keyStore.getCertificate(alias); } catch (KeyStoreException e) { - log.error("Unable to retrieve certificate for: {}", keyCoordinates, e); + log.error("Unable to retrieve certificate for: {}", keyCoordinates != null ? keyCoordinates : alias, e); return Optional.empty(); } var publicKey = cert.getPublicKey(); @@ -150,7 +155,7 @@ private Optional get(String alias, KeyCoordinates keyCoordinates) { try { privateKey = (PrivateKey) keyStore.getKey(alias, passwordProvider.get()); } catch (UnrecoverableKeyException | KeyStoreException | NoSuchAlgorithmException e) { - log.error("Unable to retrieve certificate for: {}", keyCoordinates, e); + log.error("Unable to retrieve certificate for: {}", keyCoordinates != null ? keyCoordinates : alias, e); return Optional.empty(); } return Optional.of(new KeyPair(publicKey, privateKey)); diff --git a/stereotomy/src/main/java/com/salesforce/apollo/stereotomy/mem/MemKeyStore.java b/stereotomy/src/main/java/com/salesforce/apollo/stereotomy/mem/MemKeyStore.java index 7480c578f3..b6a703e054 100644 --- a/stereotomy/src/main/java/com/salesforce/apollo/stereotomy/mem/MemKeyStore.java +++ b/stereotomy/src/main/java/com/salesforce/apollo/stereotomy/mem/MemKeyStore.java @@ -6,22 +6,27 @@ */ package com.salesforce.apollo.stereotomy.mem; +import com.salesforce.apollo.stereotomy.KeyCoordinates; +import com.salesforce.apollo.stereotomy.StereotomyKeyStore; + import java.security.KeyPair; import java.util.Map; import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; -import com.salesforce.apollo.stereotomy.KeyCoordinates; -import com.salesforce.apollo.stereotomy.StereotomyKeyStore; - /** * @author hal.hildebrand - * */ public class MemKeyStore implements StereotomyKeyStore { - private final Map keys = new ConcurrentHashMap<>(); - private final Map nextKeys = new ConcurrentHashMap<>(); + private final Map keys = new ConcurrentHashMap<>(); + private final Map nextKeys = new ConcurrentHashMap<>(); + private final Map aliasedKeys = new ConcurrentHashMap<>(); + + @Override + public Optional getKey(String alias) { + return Optional.ofNullable(aliasedKeys.get(alias)); + } @Override public Optional getKey(KeyCoordinates keyCoordinates) { @@ -38,11 +43,21 @@ public void removeKey(KeyCoordinates keyCoordinates) { this.keys.remove(keyCoordinates); } + @Override + public void removeKey(String alias) { + aliasedKeys.remove(alias); + } + @Override public void removeNextKey(KeyCoordinates keyCoordinates) { this.nextKeys.remove(keyCoordinates); } + @Override + public void storeKey(String alias, KeyPair keyPair) { + aliasedKeys.put(alias, keyPair); + } + @Override public void storeKey(KeyCoordinates coordinates, KeyPair keyPair) { this.keys.put(coordinates, keyPair); diff --git a/stereotomy/src/main/java/com/salesforce/apollo/stereotomy/processing/KeyEventVerifier.java b/stereotomy/src/main/java/com/salesforce/apollo/stereotomy/processing/KeyEventVerifier.java index 64df9db1b9..608be09d77 100644 --- a/stereotomy/src/main/java/com/salesforce/apollo/stereotomy/processing/KeyEventVerifier.java +++ b/stereotomy/src/main/java/com/salesforce/apollo/stereotomy/processing/KeyEventVerifier.java @@ -23,7 +23,7 @@ * @author hal.hildebrand */ public interface KeyEventVerifier { - static final Logger log = LoggerFactory.getLogger(KeyEventVerifier.class); + Logger log = LoggerFactory.getLogger(KeyEventVerifier.class); default JohnHancock verifyAuthentication(KeyState state, KeyEvent event, JohnHancock signatures, KEL kel) { KeyEvent lookup = kel.getKeyEvent(state.getLastEstablishmentEvent()); @@ -32,7 +32,7 @@ default JohnHancock verifyAuthentication(KeyState state, KeyEvent event, JohnHan } var kee = (EstablishmentEvent) lookup; var filtered = new DefaultVerifier(kee.getKeys()).filtered(kee.getSigningThreshold(), signatures, - event.getBytes()); + event.getBytes()); if (!filtered.verified()) { throw new UnmetSigningThresholdException(event); } diff --git a/stereotomy/src/main/java/com/salesforce/apollo/stereotomy/processing/KeyStateProcessor.java b/stereotomy/src/main/java/com/salesforce/apollo/stereotomy/processing/KeyStateProcessor.java index 8860dc5350..89b850f6be 100644 --- a/stereotomy/src/main/java/com/salesforce/apollo/stereotomy/processing/KeyStateProcessor.java +++ b/stereotomy/src/main/java/com/salesforce/apollo/stereotomy/processing/KeyStateProcessor.java @@ -44,8 +44,7 @@ public KeyState apply(KeyState currentState, KeyEvent event) { var witnessThreshold = currentState.getWitnessThreshold(); var witnesses = currentState.getWitnesses(); - if (event instanceof RotationEvent) { - var re = (RotationEvent) event; + if (event instanceof RotationEvent re) { signingThreshold = re.getSigningThreshold(); keys = re.getKeys(); nextKeyConfigugurationDigest = re.getNextKeysDigest(); @@ -55,13 +54,11 @@ public KeyState apply(KeyState currentState, KeyEvent event) { witnesses.removeAll(re.getWitnessesRemovedList()); witnesses.addAll(re.getWitnessesAddedList()); } - KeyState state = KeyStateImpl.newKeyState(event.getIdentifier(), signingThreshold, keys, - nextKeyConfigugurationDigest.orElse(null), witnessThreshold, - witnesses, currentState.configurationTraits(), event, - lastEstablishmentEvent, - currentState.getDelegatingIdentifier().orElse(null), - events.getDigestAlgorithm().digest(event.getBytes())); - return state; + return KeyStateImpl.newKeyState(event.getIdentifier(), signingThreshold, keys, + nextKeyConfigugurationDigest.orElse(null), witnessThreshold, witnesses, + currentState.configurationTraits(), event, lastEstablishmentEvent, + currentState.getDelegatingIdentifier().orElse(null), + events.getDigestAlgorithm().digest(event.getBytes())); } } diff --git a/stereotomy/src/main/java/com/salesforce/apollo/stereotomy/processing/Validator.java b/stereotomy/src/main/java/com/salesforce/apollo/stereotomy/processing/Validator.java index dfcfbc35da..44ffdedf16 100644 --- a/stereotomy/src/main/java/com/salesforce/apollo/stereotomy/processing/Validator.java +++ b/stereotomy/src/main/java/com/salesforce/apollo/stereotomy/processing/Validator.java @@ -35,7 +35,7 @@ * @author hal.hildebrand */ public interface Validator { - static final Logger log = LoggerFactory.getLogger(Validator.class); + Logger log = LoggerFactory.getLogger(Validator.class); static boolean distinct(Collection items) { if (items instanceof Set) { @@ -58,12 +58,13 @@ default boolean validate(Identifier identifier, JohnHancock signature, InputStre log.debug("Identifier: {} not found in KeyState", identifier); return false; } - for (KeyEvent lee = kel.getKeyEvent(currentState.getLastEstablishmentEvent()); lee != null; lee = kel.getKeyEvent(lee.getPrevious())) { + for (KeyEvent lee = kel.getKeyEvent(currentState.getLastEstablishmentEvent()); lee != null; + lee = kel.getKeyEvent(lee.getPrevious())) { var lastEstablishment = (EstablishmentEvent) lee; lastEstablishment.getKeys(); if (new DefaultVerifier(lastEstablishment.getKeys()).verify(lastEstablishment.getSigningThreshold(), - signature, message)) { + signature, message)) { return true; } } @@ -76,51 +77,51 @@ default void validateKeyEventData(KeyState state, KeyEvent event, KEL kel) { validateKeyConfiguration(ee); validate(ee.getIdentifier().isTransferable() || ee.getNextKeysDigest().isEmpty(), - "non-transferable prefix must not have a next key configuration"); + "non-transferable prefix must not have a next key configuration"); if (event instanceof InceptionEvent icp) { validate(icp.getSequenceNumber().equals(ULong.valueOf(0)), - "inception events must have a sequence number of 0"); + "inception events must have a sequence number of 0"); validateIdentifier(icp); validateInceptionWitnesses(icp); } else if (event instanceof RotationEvent rot) { validate(!(state.isDelegated()) || rot instanceof DelegatedRotationEvent, - "delegated identifiers must use delegated rotation event type"); + "delegated identifiers must use delegated rotation event type"); validate(rot.getSequenceNumber().compareTo(ULong.valueOf(0)) > 0, - "non-inception event must have a sequence number greater than 0 (s: %s)", - rot.getSequenceNumber()); + "non-inception event must have a sequence number greater than 0 (s: %s)", + rot.getSequenceNumber()); validate(event.getIdentifier().isTransferable(), - "only transferable identifiers can have rotation events"); + "only transferable identifiers can have rotation events"); KeyEvent lookup; lookup = kel.getKeyEvent(state.getLastEstablishmentEvent()); if (lookup == null) { - throw new InvalidKeyEventException(String.format("previous establishment event does not exist")); + throw new InvalidKeyEventException("previous establishment event does not exist"); } EstablishmentEvent lastEstablishmentEvent = (EstablishmentEvent) lookup; validate(lastEstablishmentEvent.getNextKeysDigest().isPresent(), - "previous establishment event must have a next key configuration for rotation"); + "previous establishment event must have a next key configuration for rotation"); var nextKeyConfigurationDigest = lastEstablishmentEvent.getNextKeysDigest().get(); - validate(KeyConfigurationDigester.matches(rot.getSigningThreshold(), rot.getKeys(), - nextKeyConfigurationDigest), - "digest of signing threshold and keys must match digest in previous establishment event"); + validate( + KeyConfigurationDigester.matches(rot.getSigningThreshold(), rot.getKeys(), nextKeyConfigurationDigest), + "digest of signing threshold and keys must match digest in previous establishment event"); validateRotationWitnesses(rot, state); } if (event instanceof DelegatedInceptionEvent dee) { validate(dee.getDelegatingPrefix() != null, - "delegated establishment event must contain referenced delegating identifier"); + "delegated establishment event must contain referenced delegating identifier"); } } else if (event instanceof InteractionEvent ixn) { validate(ixn.getSequenceNumber().compareTo(ULong.valueOf(0)) > 0, - "non-inception event must have a sequence number greater than 0 (s: %s)", ixn.getSequenceNumber()); + "non-inception event must have a sequence number greater than 0 (s: %s)", ixn.getSequenceNumber()); validate(!state.configurationTraits().contains(ConfigurationTrait.ESTABLISHMENT_EVENTS_ONLY), - "interaction events only permitted when identifier is not configured for establishment events only"); + "interaction events only permitted when identifier is not configured for establishment events only"); } } @@ -134,22 +135,22 @@ private void validateIdentifier(InceptionEvent event) { if (event.getIdentifier() instanceof BasicIdentifier bi) { validate(event.getKeys().size() == 1, "basic identifiers can only have a single key"); - validate(bi.getPublicKey().equals(event.getKeys().get(0)), "basic identifier key must match event key"); + validate(bi.getPublicKey().equals(event.getKeys().getFirst()), "basic identifier key must match event key"); } else if (event.getIdentifier() instanceof SelfAddressingIdentifier sap) { var digest = sap.getDigest().getAlgorithm().digest(event.getInceptionStatement()); validate(sap.getDigest().equals(digest), - "self-addressing identifier digests must match digest of inception statement"); + "self-addressing identifier digests must match digest of inception statement"); } else if (event.getIdentifier() instanceof SelfSigningIdentifier ssp) { validate(event.getKeys().size() == 1, "self-signing identifiers can only have a single key"); - var ops = SignatureAlgorithm.lookup(event.getKeys().get(0)); + var ops = SignatureAlgorithm.lookup(event.getKeys().getFirst()); new DefaultVerifier(event.getKeys()).verify(event.getSigningThreshold(), ssp.getSignature(), - event.getInceptionStatement()); - validate(ops.verify(event.getKeys().get(0), ssp.getSignature(), event.getInceptionStatement()), - "self-signing prefix signature must verify against inception statement"); + event.getInceptionStatement()); + validate(ops.verify(event.getKeys().getFirst(), ssp.getSignature(), event.getInceptionStatement()), + "self-signing prefix signature must verify against inception statement"); } else { throw new IllegalArgumentException("Unknown prefix type: " + event.getIdentifier().getClass()); @@ -163,12 +164,12 @@ private void validateInceptionWitnesses(InceptionEvent icp) { validate(distinct(icp.getWitnesses()), "witness set must not have duplicates"); validate(icp.getWitnessThreshold() > 0, - "witness threshold must be greater than 0 if witnesses are provided (given: threshold: %s, witnesses: %s", - icp.getWitnessThreshold(), icp.getWitnesses().size()); + "witness threshold must be greater than 0 if witnesses are provided (given: threshold: %s, witnesses: %s", + icp.getWitnessThreshold(), icp.getWitnesses().size()); validate(icp.getWitnessThreshold() <= icp.getWitnesses().size(), - "witness threshold must be less than or equal to the number of witnesses (given: threshold: %s, witnesses: %s", - icp.getWitnessThreshold(), icp.getWitnesses().size()); + "witness threshold must be less than or equal to the number of witnesses (given: threshold: %s, witnesses: %s", + icp.getWitnessThreshold(), icp.getWitnesses().size()); } } @@ -177,12 +178,11 @@ private void validateKeyConfiguration(EstablishmentEvent ee) { if (ee.getSigningThreshold() instanceof SigningThreshold.Unweighted) { validate(ee.getKeys().size() >= ((SigningThreshold.Unweighted) ee.getSigningThreshold()).getThreshold(), - "unweighted signing threshold must be less than or equals to the number of keys"); - } else if (ee.getSigningThreshold() instanceof SigningThreshold.Weighted) { - var weightedThreshold = ((SigningThreshold.Weighted) ee.getSigningThreshold()); + "unweighted signing threshold must be less than or equals to the number of keys"); + } else if (ee.getSigningThreshold() instanceof SigningThreshold.Weighted weightedThreshold) { var countOfWeights = SigningThreshold.countWeights(weightedThreshold.getWeights()); validate(ee.getKeys().size() == countOfWeights, - "weighted signing threshold must specify a weight for each key"); + "weighted signing threshold must specify a weight for each key"); } } @@ -192,13 +192,13 @@ private void validateRotationWitnesses(RotationEvent rot, KeyState state) { validate(distinct(rot.getWitnessesRemovedList()), "added witnesses must not have duplicates"); validate(state.getWitnesses().containsAll(rot.getWitnessesRemovedList()), - "removed witnesses must be present witness list"); + "removed witnesses must be present witness list"); validate(disjoint(rot.getWitnessesAddedList(), rot.getWitnessesRemovedList()), - "added and removed witnesses must be mutually exclusive"); + "added and removed witnesses must be mutually exclusive"); validate(disjoint(rot.getWitnessesAddedList(), state.getWitnesses()), - "added witnesses must not already be present in witness list"); + "added witnesses must not already be present in witness list"); var newWitnesses = new ArrayList<>(state.getWitnesses()); newWitnesses.removeAll(rot.getWitnessesRemovedList()); @@ -210,8 +210,8 @@ private void validateRotationWitnesses(RotationEvent rot, KeyState state) { validate(rot.getWitnessThreshold() == 0, "witness threshold must be 0 if no witnesses are specified"); } else { validate(rot.getWitnessThreshold() <= newWitnesses.size(), - "witness threshold must be less than or equal to the number of witnesses " - + "(threshold: %s, witnesses: %s)", rot.getWitnessThreshold(), newWitnesses.size()); + "witness threshold must be less than or equal to the number of witnesses " + + "(threshold: %s, witnesses: %s)", rot.getWitnessThreshold(), newWitnesses.size()); } } } diff --git a/stereotomy/src/main/java/com/salesforce/apollo/stereotomy/services/proto/ProtoEventObserver.java b/stereotomy/src/main/java/com/salesforce/apollo/stereotomy/services/proto/ProtoEventObserver.java index 414e22b105..733b9cda59 100644 --- a/stereotomy/src/main/java/com/salesforce/apollo/stereotomy/services/proto/ProtoEventObserver.java +++ b/stereotomy/src/main/java/com/salesforce/apollo/stereotomy/services/proto/ProtoEventObserver.java @@ -6,14 +6,13 @@ */ package com.salesforce.apollo.stereotomy.services.proto; -import java.util.List; -import java.util.concurrent.CompletableFuture; - import com.salesforce.apollo.stereotomy.event.proto.AttachmentEvent; import com.salesforce.apollo.stereotomy.event.proto.KERL_; import com.salesforce.apollo.stereotomy.event.proto.KeyEvent_; import com.salesforce.apollo.stereotomy.event.proto.Validations; +import java.util.List; + /** * @author hal.hildebrand */ diff --git a/stereotomy/src/main/java/com/salesforce/apollo/stereotomy/services/proto/ProtoKERLAdapter.java b/stereotomy/src/main/java/com/salesforce/apollo/stereotomy/services/proto/ProtoKERLAdapter.java index 19411fa155..d5c8af1d09 100644 --- a/stereotomy/src/main/java/com/salesforce/apollo/stereotomy/services/proto/ProtoKERLAdapter.java +++ b/stereotomy/src/main/java/com/salesforce/apollo/stereotomy/services/proto/ProtoKERLAdapter.java @@ -44,7 +44,7 @@ public ProtoKERLAdapter(KERL.AppendKERL kerl) { public List append(KERL_ k) { List events = new ArrayList<>(); List attachments = new ArrayList<>(); - k.getEventsList().stream().map(e -> ProtobufEventFactory.from(e)).forEach(ewa -> { + k.getEventsList().stream().map(ProtobufEventFactory::from).forEach(ewa -> { events.add(ewa.event()); attachments.add( ProtobufEventFactory.INSTANCE.attachment((EstablishmentEvent) ewa.event(), ewa.attachments())); @@ -59,7 +59,7 @@ public List append(KERL_ k) { public List append(List keyEventList) { KeyEvent[] events = new KeyEvent[keyEventList.size()]; int i = 0; - for (KeyEvent event : keyEventList.stream().map(ke -> ProtobufEventFactory.from(ke)).toList()) { + for (KeyEvent event : keyEventList.stream().map(ProtobufEventFactory::from).toList()) { events[i++] = event; } List keyStates = kerl.append(events); @@ -72,17 +72,21 @@ public List append(List keyEventList) { @Override public List append(List eventsList, List attachmentsList) { - return kerl.append(eventsList.stream().map(ke -> ProtobufEventFactory.from(ke)).toList(), - attachmentsList.stream() - .map(ae -> new AttachmentEventImpl(ae)) - .map(e -> (com.salesforce.apollo.stereotomy.event.AttachmentEvent) e) - .toList()).stream().map(ks -> ks == null ? null : ks.toKeyState_()).toList(); + return kerl.append(eventsList.stream().map(ProtobufEventFactory::from).toList(), attachmentsList.stream() + .map( + AttachmentEventImpl::new) + .map( + e -> (com.salesforce.apollo.stereotomy.event.AttachmentEvent) e) + .toList()) + .stream() + .map(ks -> ks == null ? null : ks.toKeyState_()) + .toList(); } @Override public Empty appendAttachments(List attachments) { kerl.append(attachments.stream() - .map(e -> new AttachmentEventImpl(e)) + .map(AttachmentEventImpl::new) .map(e -> (com.salesforce.apollo.stereotomy.event.AttachmentEvent) e) .toList()); return Empty.getDefaultInstance(); diff --git a/stereotomy/src/test/java/com/salesforce/apollo/stereotomy/FileKeyStoreTest.java b/stereotomy/src/test/java/com/salesforce/apollo/stereotomy/FileKeyStoreTest.java index a8c5f0bcaa..05d6ae7bd2 100644 --- a/stereotomy/src/test/java/com/salesforce/apollo/stereotomy/FileKeyStoreTest.java +++ b/stereotomy/src/test/java/com/salesforce/apollo/stereotomy/FileKeyStoreTest.java @@ -6,6 +6,8 @@ */ package com.salesforce.apollo.stereotomy; +import com.salesforce.apollo.stereotomy.jks.FileKeyStore; + import java.io.File; import java.io.IOException; import java.security.KeyStore; @@ -14,11 +16,8 @@ import java.security.cert.CertificateException; import java.util.function.Supplier; -import com.salesforce.apollo.stereotomy.jks.FileKeyStore; - /** * @author hal.hildebrand - * */ public class FileKeyStoreTest extends StereotomyTests { @@ -28,7 +27,7 @@ protected FileKeyStore initializeKeyStore() { file.delete(); final Supplier passwordProvider = () -> new char[] { 'f', 'o', 'o' }; try { - final var ks = KeyStore.getInstance("JKS"); + final var ks = KeyStore.getInstance("jceks"); ks.load(null, passwordProvider.get()); return new FileKeyStore(ks, passwordProvider, file); } catch (KeyStoreException | NoSuchAlgorithmException | CertificateException | IOException e) { diff --git a/stereotomy/src/test/java/com/salesforce/apollo/stereotomy/JksKeyStoreTest.java b/stereotomy/src/test/java/com/salesforce/apollo/stereotomy/JksKeyStoreTest.java index 454e23f295..1f59a95218 100644 --- a/stereotomy/src/test/java/com/salesforce/apollo/stereotomy/JksKeyStoreTest.java +++ b/stereotomy/src/test/java/com/salesforce/apollo/stereotomy/JksKeyStoreTest.java @@ -6,6 +6,8 @@ */ package com.salesforce.apollo.stereotomy; +import com.salesforce.apollo.stereotomy.jks.JksKeyStore; + import java.io.IOException; import java.security.KeyStore; import java.security.KeyStoreException; @@ -13,11 +15,8 @@ import java.security.cert.CertificateException; import java.util.function.Supplier; -import com.salesforce.apollo.stereotomy.jks.JksKeyStore; - /** * @author hal.hildebrand - * */ public class JksKeyStoreTest extends StereotomyTests { @@ -25,7 +24,7 @@ public class JksKeyStoreTest extends StereotomyTests { protected JksKeyStore initializeKeyStore() { final Supplier passwordProvider = () -> new char[] { 'f', 'o', 'o' }; try { - final var ks = KeyStore.getInstance("JKS"); + final var ks = KeyStore.getInstance("jceks"); ks.load(null, passwordProvider.get()); return new JksKeyStore(ks, passwordProvider); } catch (KeyStoreException | NoSuchAlgorithmException | CertificateException | IOException e) { diff --git a/stereotomy/src/test/java/com/salesforce/apollo/stereotomy/StereotomyTests.java b/stereotomy/src/test/java/com/salesforce/apollo/stereotomy/StereotomyTests.java index 3e3691ffd9..862dc7efbf 100644 --- a/stereotomy/src/test/java/com/salesforce/apollo/stereotomy/StereotomyTests.java +++ b/stereotomy/src/test/java/com/salesforce/apollo/stereotomy/StereotomyTests.java @@ -106,7 +106,7 @@ public void newIdentifier() throws Exception { ControlledIdentifier identifier = controller.newIdentifier(); // identifier - assertTrue(identifier.getIdentifier() instanceof SelfAddressingIdentifier); + assertInstanceOf(SelfAddressingIdentifier.class, identifier.getIdentifier()); var sap = (SelfAddressingIdentifier) identifier.getIdentifier(); assertEquals(DigestAlgorithm.DEFAULT, sap.getDigest().getAlgorithm()); assertEquals("4cb6958622749694aedff3d48b8e402524562813bf2bdd11894a528edc965b4d", @@ -167,7 +167,7 @@ public void newIdentifierFromIdentifier() throws Exception { IdentifierSpecification.newBuilder()); // identifier - assertTrue(identifier.getIdentifier() instanceof SelfAddressingIdentifier); + assertInstanceOf(SelfAddressingIdentifier.class, identifier.getIdentifier()); var sap = (SelfAddressingIdentifier) identifier.getIdentifier(); assertEquals(DigestAlgorithm.DEFAULT, sap.getDigest().getAlgorithm()); assertEquals("092126af01f80ca28e7a99bbdce229c029be3bbfcb791e29ccb7a64e8019a36f", @@ -260,7 +260,7 @@ private void provision(ControlledIdentifier identifier, Stereotomy controller var decoded = Stereotomy.decode(cert); assertFalse(decoded.isEmpty()); - assertEquals(identifier.getIdentifier(), decoded.get().coordinates().getIdentifier()); + assertEquals(identifier.getIdentifier(), decoded.get().identifier()); final var qb64Id = qb64(basicId); assertTrue(identifier.getVerifier().get().verify(decoded.get().signature(), qb64Id)); diff --git a/thoth/src/main/java/com/salesforce/apollo/thoth/Ani.java b/thoth/src/main/java/com/salesforce/apollo/thoth/Ani.java index 271d32124a..935363797e 100644 --- a/thoth/src/main/java/com/salesforce/apollo/thoth/Ani.java +++ b/thoth/src/main/java/com/salesforce/apollo/thoth/Ani.java @@ -18,7 +18,6 @@ import com.salesforce.apollo.stereotomy.event.KeyEvent; import com.salesforce.apollo.stereotomy.identifier.Identifier; import com.salesforce.apollo.utils.BbBackedInputStream; -import org.joou.ULong; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -51,22 +50,22 @@ public CertificateValidator certificateValidator(Duration timeout) { public EventValidation eventValidation(Duration timeout) { return new EventValidation() { - @Override - public KeyState keyState(Identifier id, ULong sequenceNumber) { - return kerl.getKeyState(id, sequenceNumber); - } - @Override public boolean validate(EstablishmentEvent event) { - log.trace("Validate event: {} on: {}", event, member); - return Ani.this.validateKerl(event, timeout); + log.trace("Validate event: {} on: {}", event.getCoordinates(), member); + var result = Ani.this.validateKerl(event, timeout); + log.info("Validate event: {}: {} on: {}", event, result, member); + return result; } @Override - public boolean validate(EventCoordinates coordinates) { - log.trace("Validating coordinates: {} on: {}", coordinates, member); - KeyEvent ke = kerl.getKeyEvent(coordinates); - return Ani.this.validateKerl(ke, timeout); + public boolean validate(Identifier identifier) { + log.trace("Validating identifier: {} on: {}", identifier, member); + var ks = kerl.getKeyState(identifier); + var ke = kerl.getKeyEvent(ks.getLastEstablishmentEvent()); + var result = Ani.this.validateKerl(ke, timeout); + log.info("Validating identifier: {}:{} on: {}", identifier, result, member); + return result; } }; } diff --git a/thoth/src/main/java/com/salesforce/apollo/thoth/CombinedIntervals.java b/thoth/src/main/java/com/salesforce/apollo/thoth/CombinedIntervals.java index 4155457f74..dfee7ca86a 100644 --- a/thoth/src/main/java/com/salesforce/apollo/thoth/CombinedIntervals.java +++ b/thoth/src/main/java/com/salesforce/apollo/thoth/CombinedIntervals.java @@ -7,10 +7,13 @@ package com.salesforce.apollo.thoth; -import com.salesforce.apollo.thoth.proto.Interval; import com.salesforce.apollo.cryptography.Digest; +import com.salesforce.apollo.thoth.proto.Interval; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; +import java.util.List; import java.util.function.Predicate; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -29,17 +32,10 @@ public CombinedIntervals(List allIntervals) { if (allIntervals.isEmpty()) { return; } - Collections.sort(allIntervals, new Comparator() { - @Override - public int compare(KeyInterval o1, KeyInterval o2) { - int comparison = o1.getBegin().compareTo(o2.getBegin()); - - return comparison == 0 // if both intervals begin the same - ? o1.getEnd().compareTo(o2.getEnd()) // compare their ends - : comparison; - } - }); - KeyInterval current = allIntervals.get(0); + // if both intervals begin the same + // compare their ends + allIntervals.sort(Comparator.comparing(KeyInterval::getBegin).thenComparing(KeyInterval::getEnd)); + KeyInterval current = allIntervals.getFirst(); intervals.add(current); for (int i = 1; i < allIntervals.size(); i++) { KeyInterval next = allIntervals.get(i); @@ -62,7 +58,7 @@ public Stream intervals() { @Override public boolean test(Digest t) { - return intervals.stream().filter(i -> i.test(t)).findFirst().isPresent(); + return intervals.stream().anyMatch(i -> i.test(t)); } public List toIntervals() { diff --git a/thoth/src/main/java/com/salesforce/apollo/thoth/DirectPublisher.java b/thoth/src/main/java/com/salesforce/apollo/thoth/DirectPublisher.java index b99b5cc93b..d6090ac474 100644 --- a/thoth/src/main/java/com/salesforce/apollo/thoth/DirectPublisher.java +++ b/thoth/src/main/java/com/salesforce/apollo/thoth/DirectPublisher.java @@ -6,6 +6,7 @@ */ package com.salesforce.apollo.thoth; +import com.salesforce.apollo.cryptography.Digest; import com.salesforce.apollo.stereotomy.event.proto.AttachmentEvent; import com.salesforce.apollo.stereotomy.event.proto.KERL_; import com.salesforce.apollo.stereotomy.event.proto.KeyEvent_; @@ -24,31 +25,33 @@ public class DirectPublisher implements ProtoEventObserver { private final static Logger log = LoggerFactory.getLogger(DirectPublisher.class); private final ProtoKERLAdapter kerl; + private final Digest member; - public DirectPublisher(ProtoKERLAdapter kerl) { + public DirectPublisher(Digest member, ProtoKERLAdapter kerl) { + this.member = member; this.kerl = kerl; } @Override public void publish(KERL_ kerl_, List validations) { - log.info("publishing KERL[{}] and validations[{}]", kerl_.getEventsCount(), validations.size()); - validations.stream().forEach(v -> kerl.appendValidations(v)); - log.info("published KERL[{}] and validations[{}]", kerl_.getEventsCount(), validations.size()); + log.info("publishing KERL[{}] and validations[{}] on: {}", kerl_.getEventsCount(), validations.size(), member); + validations.forEach(kerl::appendValidations); + log.info("published KERL[{}] and validations[{}] on: {}", kerl_.getEventsCount(), validations.size(), member); kerl.append(kerl_); } @Override public void publishAttachments(List attachments) { - log.info("Publishing attachments[{}]", attachments.size()); + log.info("Publishing attachments[{}] on: {}", attachments.size(), member); kerl.appendAttachments(attachments); - log.info("Published attachments[{}]", attachments.size()); + log.info("Published attachments[{}] on: {}", attachments.size(), member); } @Override public void publishEvents(List events, List validations) { - log.info("Publishing events[{}], validations[{}]", events.size(), validations.size()); - validations.forEach(v -> kerl.appendValidations(v)); + log.info("Publishing events[{}], validations[{}] on: {}", events.size(), validations.size(), member); + validations.forEach(kerl::appendValidations); kerl.append(events); - log.info("Published events[{}], validations[{}]", events.size(), validations.size()); + log.info("Published events[{}], validations[{}] on: {}", events.size(), validations.size(), member); } } diff --git a/thoth/src/main/java/com/salesforce/apollo/thoth/KerlDHT.java b/thoth/src/main/java/com/salesforce/apollo/thoth/KerlDHT.java index 45521e8e42..466566385d 100644 --- a/thoth/src/main/java/com/salesforce/apollo/thoth/KerlDHT.java +++ b/thoth/src/main/java/com/salesforce/apollo/thoth/KerlDHT.java @@ -143,7 +143,7 @@ public KerlDHT(Duration operationsFrequency, Context context, this.connectionPool = connectionPool; kerlPool = new UniKERLDirectPooled(connectionPool, digestAlgorithm); this.reconcile = new RingCommunications<>(this.context, member, reconcileComms); - this.kerlSpace = new KerlSpace(connectionPool); + this.kerlSpace = new KerlSpace(connectionPool, member.getId()); initializeSchema(); kerl = new CachingKERL(f -> { @@ -220,6 +220,10 @@ public KeyState_ append(AttachmentEvent event) { Thread.currentThread().interrupt(); return null; } catch (ExecutionException e) { + if (e.getCause() instanceof CompletionException ce) { + log.info("error appending event: {} on: {}", ce.getMessage(), member.getId()); + return KeyState_.getDefaultInstance(); + } throw new IllegalStateException(e.getCause()); } } @@ -229,7 +233,7 @@ public List append(KERL_ kerl) { if (kerl.getEventsList().isEmpty()) { return completeIt(Collections.emptyList()); } - final var event = kerl.getEventsList().get(0); + final var event = kerl.getEventsList().getFirst(); Digest identifier = digestOf(event, digestAlgorithm()); if (identifier == null) { return completeIt(Collections.emptyList()); @@ -255,6 +259,10 @@ public List append(KERL_ kerl) { Thread.currentThread().interrupt(); return null; } catch (ExecutionException e) { + if (e.getCause() instanceof CompletionException ce) { + log.info("error appending KERL: {} on: {}", ce.getMessage(), member.getId()); + return Collections.emptyList(); + } throw new IllegalStateException(e.getCause()); } } @@ -282,11 +290,15 @@ public KeyState_ append(KeyEvent_ event) { gathered)); try { var ks = result.get(); - return ks.getKeyStatesCount() == 0 ? KeyState_.getDefaultInstance() : ks.getKeyStatesList().get(0); + return ks.getKeyStatesCount() == 0 ? KeyState_.getDefaultInstance() : ks.getKeyStatesList().getFirst(); } catch (InterruptedException e) { Thread.currentThread().interrupt(); return null; } catch (ExecutionException e) { + if (e.getCause() instanceof CompletionException ce) { + log.info("error appending Key Event: {} on: {}", ce.getMessage(), member.getId()); + return KeyState_.getDefaultInstance(); + } throw new IllegalStateException(e.getCause()); } } @@ -297,9 +309,7 @@ public List append(List events) { return completeIt(Collections.emptyList()); } List states = new ArrayList<>(); - events.stream().map(e -> append(e)).forEach(ks -> { - states.add(ks); - }); + events.stream().map(this::append).forEach(states::add); return states; } @@ -309,9 +319,7 @@ public List append(List events, List atta return completeIt(Collections.emptyList()); } List states = new ArrayList<>(); - events.stream().map(e -> append(e)).forEach(ks -> { - states.add(ks); - }); + events.stream().map(this::append).forEach(states::add); attachments.forEach(this::append); return states; @@ -322,7 +330,7 @@ public Empty appendAttachments(List events) { if (events.isEmpty()) { return completeIt(Empty.getDefaultInstance()); } - final var event = events.get(0); + final var event = events.getFirst(); Digest identifier = digestAlgorithm().digest(event.getCoordinates().getIdentifier().toByteString()); if (identifier == null) { return completeIt(Empty.getDefaultInstance()); @@ -342,7 +350,19 @@ public Empty appendAttachments(List events) { "append attachments"), t -> completeIt(result, gathered)); - return Empty.getDefaultInstance(); + + try { + return result.get(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return null; + } catch (ExecutionException e) { + if (e.getCause() instanceof CompletionException ce) { + log.info("error appending attachments: {} on: {}", ce.getMessage(), member.getId()); + return Empty.getDefaultInstance(); + } + throw new IllegalStateException(e.getCause()); + } } @Override @@ -375,6 +395,10 @@ public Empty appendValidations(Validations validations) { Thread.currentThread().interrupt(); return null; } catch (ExecutionException e) { + if (e.getCause() instanceof CompletionException ce) { + log.info("error appending validations: {} on: {}", ce.getMessage(), member.getId()); + return Empty.getDefaultInstance(); + } throw new IllegalStateException(e.getCause()); } } @@ -411,28 +435,25 @@ public Attachment getAttachment(EventCoords coordinates) { Supplier isTimedOut = () -> Instant.now().isAfter(timedOut); var result = new CompletableFuture(); HashMultiset gathered = HashMultiset.create(); - new RingIterator<>(operationsFrequency, context, member, scheduler, dhtComms).noDuplicates() - .iterate(identifier, null, - (link, r) -> link.getAttachment( - coordinates), - () -> failedMajority( - result, - maxCount(gathered)), - (tally, futureSailor, destination) -> read( - result, gathered, tally, - futureSailor, identifier, - isTimedOut, destination, - "get attachment", - Attachment.getDefaultInstance()), - t -> failedMajority( - result, - maxCount(gathered))); + var operation = "getAttachment(%s)".formatted(EventCoordinates.from(coordinates)); + var iterator = new RingIterator(operationsFrequency, context, member, scheduler, dhtComms); + iterator.noDuplicates() + .iterate(identifier, null, (link, r) -> link.getAttachment(coordinates), + () -> failedMajority(result, maxCount(gathered), operation), + (tally, futureSailor, destination) -> read(result, gathered, tally, futureSailor, identifier, + isTimedOut, destination, "get attachment", + Attachment.getDefaultInstance()), + t -> failedMajority(result, maxCount(gathered), operation)); try { return result.get(); } catch (InterruptedException e) { Thread.currentThread().interrupt(); return null; } catch (ExecutionException e) { + if (e.getCause() instanceof CompletionException ce) { + log.info("error {} : {} on: {}", operation, ce.getMessage(), member.getId()); + return null; + } throw new IllegalStateException(e.getCause()); } } @@ -450,35 +471,33 @@ public KERL_ getKERL(Ident identifier) { Supplier isTimedOut = () -> Instant.now().isAfter(timedOut); var result = new CompletableFuture(); HashMultiset gathered = HashMultiset.create(); - new RingIterator<>(operationsFrequency, context, member, scheduler, dhtComms).noDuplicates() - .iterate(digest, null, - (link, r) -> link.getKERL( - identifier), - () -> failedMajority( - result, - maxCount(gathered)), - (tally, futureSailor, destination) -> read( - result, gathered, tally, - futureSailor, digest, - isTimedOut, destination, - "get kerl", - KERL_.getDefaultInstance()), - t -> failedMajority( - result, - maxCount(gathered))); + var operation = "getKerl(%s)".formatted(Identifier.from(identifier)); + var iterator = new RingIterator(operationsFrequency, context, member, scheduler, dhtComms); + iterator.noDuplicates() + .iterate(digest, null, (link, r) -> link.getKERL(identifier), + () -> failedMajority(result, maxCount(gathered), operation), + (tally, futureSailor, destination) -> read(result, gathered, tally, futureSailor, digest, + isTimedOut, destination, operation, + KERL_.getDefaultInstance()), + t -> failedMajority(result, maxCount(gathered), operation)); try { return result.get(); } catch (InterruptedException e) { Thread.currentThread().interrupt(); return null; } catch (ExecutionException e) { + if (e.getCause() instanceof CompletionException ce) { + log.info("error {} : {} on: {}", operation, ce.getMessage(), member.getId()); + return KERL_.getDefaultInstance(); + } throw new IllegalStateException(e.getCause()); } } @Override public KeyEvent_ getKeyEvent(EventCoords coordinates) { - log.trace("Get key event: {} on: {}", EventCoordinates.from(coordinates), member.getId()); + var operation = "getKeyEvent(%s)".formatted(EventCoordinates.from(coordinates)); + log.trace("{} on: {}", operation, member.getId()); if (coordinates == null) { return completeIt(KeyEvent_.getDefaultInstance()); } @@ -490,35 +509,32 @@ public KeyEvent_ getKeyEvent(EventCoords coordinates) { Supplier isTimedOut = () -> Instant.now().isAfter(timedOut); var result = new CompletableFuture(); HashMultiset gathered = HashMultiset.create(); - new RingIterator<>(operationsFrequency, context, member, scheduler, dhtComms).noDuplicates() - .iterate(digest, null, - (link, r) -> link.getKeyEvent( - coordinates), - () -> failedMajority( - result, - maxCount(gathered)), - (tally, futureSailor, destination) -> read( - result, gathered, tally, - futureSailor, digest, - isTimedOut, destination, - "get key event", - KeyEvent_.getDefaultInstance()), - t -> failedMajority( - result, - maxCount(gathered))); + var iterator = new RingIterator(operationsFrequency, context, member, scheduler, dhtComms); + iterator.noDuplicates() + .iterate(digest, null, (link, r) -> link.getKeyEvent(coordinates), + () -> failedMajority(result, maxCount(gathered), operation), + (tally, futureSailor, destination) -> read(result, gathered, tally, futureSailor, digest, + isTimedOut, destination, operation, + KeyEvent_.getDefaultInstance()), + t -> failedMajority(result, maxCount(gathered), operation)); try { return result.get(); } catch (InterruptedException e) { Thread.currentThread().interrupt(); return null; } catch (ExecutionException e) { + if (e.getCause() instanceof CompletionException ce) { + log.info("error {} : {} on: {}", operation, ce.getMessage(), member.getId()); + return KeyEvent_.getDefaultInstance(); + } throw new IllegalStateException(e.getCause()); } } @Override public KeyState_ getKeyState(EventCoords coordinates) { - log.info("Get key state: {} on: {}", EventCoordinates.from(coordinates), member.getId()); + var operation = "getKeyState(%s)".formatted(EventCoordinates.from(coordinates)); + log.info("{} on: {}", operation, member.getId()); if (coordinates == null) { return completeIt(KeyState_.getDefaultInstance()); } @@ -530,38 +546,32 @@ public KeyState_ getKeyState(EventCoords coordinates) { Supplier isTimedOut = () -> Instant.now().isAfter(timedOut); var result = new CompletableFuture(); HashMultiset gathered = HashMultiset.create(); - new RingIterator<>(operationsFrequency, context, member, scheduler, dhtComms).noDuplicates() - .iterate(digest, null, - (link, r) -> link.getKeyState( - coordinates), - () -> failedMajority( - result, - maxCount(gathered)), - (tally, futureSailor, destination) -> read( - result, gathered, tally, - futureSailor, digest, - isTimedOut, destination, - "get key state for coordinates", - KeyState_.getDefaultInstance()), - t -> failedMajority( - result, - maxCount(gathered))); + var iterator = new RingIterator(operationsFrequency, context, member, scheduler, dhtComms); + iterator.noDuplicates() + .iterate(digest, null, (link, r) -> link.getKeyState(coordinates), + () -> failedMajority(result, maxCount(gathered), operation), + (tally, futureSailor, destination) -> read(result, gathered, tally, futureSailor, digest, + isTimedOut, destination, operation, + KeyState_.getDefaultInstance()), + t -> failedMajority(result, maxCount(gathered), operation)); try { return result.get(); } catch (InterruptedException e) { Thread.currentThread().interrupt(); return null; } catch (ExecutionException e) { + if (e.getCause() instanceof CompletionException ce) { + log.info("error {} : {} on: {}", operation, ce.getMessage(), member.getId()); + return KeyState_.getDefaultInstance(); + } throw new IllegalStateException(e.getCause()); } } @Override public KeyState_ getKeyState(Ident identifier, long sequenceNumber) { - if (log.isInfoEnabled()) { - log.info("Get key state for: {} seq#: {} on: {}", Identifier.from(identifier), - ULong.valueOf(sequenceNumber), member.getId()); - } + var operation = "getKeyState(%s, %s)".formatted(Identifier.from(identifier), ULong.valueOf(sequenceNumber)); + log.info("{} on: {}", operation, member.getId()); if (identifier == null) { return completeIt(KeyState_.getDefaultInstance()); } @@ -574,38 +584,35 @@ public KeyState_ getKeyState(Ident identifier, long sequenceNumber) { Supplier isTimedOut = () -> Instant.now().isAfter(timedOut); var result = new CompletableFuture(); HashMultiset gathered = HashMultiset.create(); - new RingIterator<>(operationsFrequency, context, member, scheduler, dhtComms).noDuplicates() - .iterate(digest, null, - (link, r) -> link.getKeyState( - identAndSeq), - () -> failedMajority( - result, - maxCount(gathered)), - (tally, futureSailor, destination) -> read( - result, gathered, tally, - futureSailor, digest, - isTimedOut, destination, - "get key state for coordinates", - KeyState_.getDefaultInstance()), - t -> failedMajority( - result, - maxCount(gathered))); + var iterator = new RingIterator(operationsFrequency, context, member, scheduler, dhtComms); + iterator.noDuplicates() + .iterate(digest, null, (link, r) -> link.getKeyState(identAndSeq), + () -> failedMajority(result, maxCount(gathered), operation), + (tally, futureSailor, destination) -> read(result, gathered, tally, futureSailor, digest, + isTimedOut, destination, operation, + KeyState_.getDefaultInstance()), + t -> failedMajority(result, maxCount(gathered), operation)); try { return result.get(); } catch (InterruptedException e) { Thread.currentThread().interrupt(); return null; } catch (ExecutionException e) { + if (e.getCause() instanceof CompletionException ce) { + log.info("error {} : {} on: {}", operation, ce.getMessage(), member.getId()); + return KeyState_.getDefaultInstance(); + } throw new IllegalStateException(e.getCause()); } } @Override public KeyState_ getKeyState(Ident identifier) { - log.info("Get key state: {} on: {}", Identifier.from(identifier), member.getId()); if (identifier == null) { return completeIt(KeyState_.getDefaultInstance()); } + var operation = "getKeyState(%s)".formatted(Identifier.from(identifier)); + log.info("{} on: {}", operation, member.getId()); Digest digest = digestAlgorithm().digest(identifier.toByteString()); if (digest == null) { return completeIt(KeyState_.getDefaultInstance()); @@ -614,34 +621,31 @@ public KeyState_ getKeyState(Ident identifier) { Supplier isTimedOut = () -> Instant.now().isAfter(timedOut); var result = new CompletableFuture(); HashMultiset gathered = HashMultiset.create(); - new RingIterator<>(operationsFrequency, context, member, scheduler, dhtComms).iterate(digest, null, - (link, r) -> link.getKeyState( - identifier), - () -> failedMajority( - result, - maxCount(gathered)), - (tally, futureSailor, destination) -> read( - result, gathered, tally, - futureSailor, digest, - isTimedOut, destination, - "get current key state", - KeyState_.getDefaultInstance()), - t -> failedMajority( - result, - maxCount(gathered))); + var iterator = new RingIterator(operationsFrequency, context, member, scheduler, dhtComms); + iterator.iterate(digest, null, (link, r) -> link.getKeyState(identifier), + () -> failedMajority(result, maxCount(gathered), operation), + (tally, futureSailor, destination) -> read(result, gathered, tally, futureSailor, digest, + isTimedOut, destination, operation, + KeyState_.getDefaultInstance()), + t -> failedMajority(result, maxCount(gathered), operation)); try { return result.get(); } catch (InterruptedException e) { Thread.currentThread().interrupt(); return null; } catch (ExecutionException e) { + if (e.getCause() instanceof CompletionException ce) { + log.info("error {} : {} on: {}", operation, ce.getMessage(), member.getId()); + return KeyState_.getDefaultInstance(); + } throw new IllegalStateException(e.getCause()); } } @Override public KeyStateWithAttachments_ getKeyStateWithAttachments(EventCoords coordinates) { - log.info("Get key state with attachements: {} on: {}", EventCoordinates.from(coordinates), member.getId()); + var operation = "getKeyStateWithAttachments(%s)".formatted(EventCoordinates.from(coordinates)); + log.info("{} on: {}", operation, member.getId()); if (coordinates == null) { return completeIt(KeyStateWithAttachments_.getDefaultInstance()); } @@ -653,35 +657,31 @@ public KeyStateWithAttachments_ getKeyStateWithAttachments(EventCoords coordinat Supplier isTimedOut = () -> Instant.now().isAfter(timedOut); var result = new CompletableFuture(); HashMultiset gathered = HashMultiset.create(); - new RingIterator<>(operationsFrequency, context, member, scheduler, dhtComms).iterate(digest, null, - (link, r) -> link.getKeyStateWithAttachments( - coordinates), - () -> failedMajority( - result, - maxCount(gathered)), - (tally, futureSailor, destination) -> read( - result, gathered, tally, - futureSailor, digest, - isTimedOut, destination, - "get key state with attachments", - KeyStateWithAttachments_.getDefaultInstance()), - t -> failedMajority( - result, - maxCount(gathered))); + var iterator = new RingIterator(operationsFrequency, context, member, scheduler, dhtComms); + iterator.iterate(digest, null, (link, r) -> link.getKeyStateWithAttachments(coordinates), + () -> failedMajority(result, maxCount(gathered), operation), + (tally, futureSailor, destination) -> read(result, gathered, tally, futureSailor, digest, + isTimedOut, destination, operation, + KeyStateWithAttachments_.getDefaultInstance()), + t -> failedMajority(result, maxCount(gathered), operation)); try { return result.get(); } catch (InterruptedException e) { Thread.currentThread().interrupt(); return null; } catch (ExecutionException e) { + if (e.getCause() instanceof CompletionException ce) { + log.info("error {} on: {}", operation, member.getId(), ce); + return null; + } throw new IllegalStateException(e.getCause()); } } @Override public KeyStateWithEndorsementsAndValidations_ getKeyStateWithEndorsementsAndValidations(EventCoords coordinates) { - log.info("Get key state with endorsements and validations: {} on: {}", EventCoordinates.from(coordinates), - member.getId()); + var operation = "getKeyStateWithEndorsementsAndValidations(%s)".formatted(EventCoordinates.from(coordinates)); + log.info("{} on: {}", operation, member.getId()); if (coordinates == null) { return completeIt(KeyStateWithEndorsementsAndValidations_.getDefaultInstance()); } @@ -693,34 +693,31 @@ public KeyStateWithEndorsementsAndValidations_ getKeyStateWithEndorsementsAndVal Supplier isTimedOut = () -> Instant.now().isAfter(timedOut); var result = new CompletableFuture(); HashMultiset gathered = HashMultiset.create(); - new RingIterator<>(operationsFrequency, context, member, scheduler, dhtComms).iterate(digest, null, - (link, r) -> link.getKeyStateWithEndorsementsAndValidations( - coordinates), - () -> failedMajority( - result, - maxCount(gathered)), - (tally, futureSailor, destination) -> read( - result, gathered, tally, - futureSailor, digest, - isTimedOut, destination, - "get key state with endorsements", - KeyStateWithEndorsementsAndValidations_.getDefaultInstance()), - t -> failedMajority( - result, - maxCount(gathered))); + var iterator = new RingIterator(operationsFrequency, context, member, scheduler, dhtComms); + iterator.iterate(digest, null, (link, r) -> link.getKeyStateWithEndorsementsAndValidations(coordinates), + () -> failedMajority(result, maxCount(gathered), operation), + (tally, futureSailor, destination) -> read(result, gathered, tally, futureSailor, digest, + isTimedOut, destination, operation, + KeyStateWithEndorsementsAndValidations_.getDefaultInstance()), + t -> failedMajority(result, maxCount(gathered), operation)); try { return result.get(); } catch (InterruptedException e) { Thread.currentThread().interrupt(); return null; } catch (ExecutionException e) { + if (e.getCause() instanceof CompletionException ce) { + log.info("error {} : {} on: {}", operation, ce.getMessage(), member.getId()); + return null; + } throw new IllegalStateException(e.getCause()); } } @Override public Validations getValidations(EventCoords coordinates) { - log.info("Get validations: {} on: {}", EventCoordinates.from(coordinates), member.getId()); + var operation = "getValidations(%s)".formatted(EventCoordinates.from(coordinates)); + log.info("{} on: {}", operation, member.getId()); if (coordinates == null) { return completeIt(Validations.getDefaultInstance()); } @@ -732,27 +729,23 @@ public Validations getValidations(EventCoords coordinates) { Supplier isTimedOut = () -> Instant.now().isAfter(timedOut); var result = new CompletableFuture(); HashMultiset gathered = HashMultiset.create(); - new RingIterator<>(operationsFrequency, context, member, scheduler, dhtComms).iterate(identifier, null, - (link, r) -> link.getValidations( - coordinates), - () -> failedMajority( - result, - maxCount(gathered)), - (tally, futureSailor, destination) -> read( - result, gathered, tally, - futureSailor, identifier, - isTimedOut, destination, - "get validations", - Validations.getDefaultInstance()), - t -> failedMajority( - result, - maxCount(gathered))); + var iterator = new RingIterator(operationsFrequency, context, member, scheduler, dhtComms); + iterator.iterate(identifier, null, (link, r) -> link.getValidations(coordinates), + () -> failedMajority(result, maxCount(gathered), operation), + (tally, futureSailor, destination) -> read(result, gathered, tally, futureSailor, identifier, + isTimedOut, destination, operation, + Validations.getDefaultInstance()), + t -> failedMajority(result, maxCount(gathered), operation)); try { return result.get(); } catch (InterruptedException e) { Thread.currentThread().interrupt(); return null; } catch (ExecutionException e) { + if (e.getCause() instanceof CompletionException ce) { + log.info("error {} : {} on: {}", operation, ce.getMessage(), member.getId()); + return null; + } throw new IllegalStateException(e.getCause()); } } @@ -766,7 +759,7 @@ public Optional verifierFor(EventCoordinates coordinates) { @Override public Optional verifierFor(Identifier identifier) { - return Optional.of(new KerlVerifier(identifier, asKERL())); + return Optional.of(new KerlVerifier<>(identifier, asKERL())); } }; } @@ -777,7 +770,7 @@ public Entry max(HashMultiset gathered) { public int maxCount(HashMultiset gathered) { final var max = gathered.entrySet().stream().max(Ordering.natural().onResultOf(Multiset.Entry::getCount)); - return max.isEmpty() ? 0 : max.get().getCount(); + return max.map(Entry::getCount).orElse(0); } public void start(Duration duration) { @@ -812,26 +805,26 @@ private void completeIt(CompletableFuture result, HashMultiset gathere .max(Ordering.natural().onResultOf(Multiset.Entry::getCount)) .orElse(null); if (max != null) { - if (max.getCount() >= context.majority()) { + if (max.getCount() >= context.majority(true)) { try { result.complete(max.getElement()); } catch (Throwable t) { - log.error("Unable to complete it", t); + log.error("Unable to complete it on {}", member.getId(), t); } return; } } result.completeExceptionally(new CompletionException( - "Unable to achieve majority, max: " + (max == null ? 0 : max.getCount()) + " required: " + context.majority() - + " on: " + member.getId())); + "Unable to achieve majority, max: " + (max == null ? 0 : max.getCount()) + " required: " + context.majority( + true) + " on: " + member.getId())); } - private boolean failedMajority(CompletableFuture result, int maxAgree) { - log.error("Unable to achieve majority read, max: {} required: {} on: {}", maxAgree, context.majority(), - member.getId()); + private boolean failedMajority(CompletableFuture result, int maxAgree, String operation) { + log.error("Unable to achieve majority read: {}, max: {} required: {} on: {}", operation, maxAgree, + context.majority(true), member.getId()); return result.completeExceptionally(new CompletionException( - "Unable to achieve majority read, max: " + maxAgree + " required: " + context.majority() + " on: " - + member.getId())); + "Unable to achieve majority read: " + operation + ", max: " + maxAgree + " required: " + context.majority(true) + + " on: " + member.getId())); } private void initializeSchema() { @@ -878,55 +871,42 @@ private boolean mutate(HashMultiset gathered, Optional futureSailor, D Supplier isTimedOut, AtomicInteger tally, RingCommunications.Destination destination, String action) { if (futureSailor.isEmpty()) { - return !isTimedOut.get(); - } - T content = futureSailor.get(); - if (content != null) { - log.trace("{}: {} from: {} on: {}", action, identifier, destination.member().getId(), member.getId()); - gathered.add(content); - var max = gathered.entrySet() - .stream() - .max(Ordering.natural().onResultOf(Multiset.Entry::getCount)) - .orElse(null); - if (max != null) { - tally.set(max.getCount()); - } - return !isTimedOut.get(); - } else { log.debug("Failed {}: {} from: {} on: {}", action, identifier, destination.member().getId(), member.getId()); return !isTimedOut.get(); } + T content = futureSailor.get(); + log.trace("{}: {} from: {} on: {}", action, identifier, destination.member().getId(), member.getId()); + gathered.add(content); + gathered.entrySet() + .stream() + .max(Ordering.natural().onResultOf(Entry::getCount)) + .ifPresent(max -> tally.set(max.getCount())); + return !isTimedOut.get(); } private boolean read(CompletableFuture result, HashMultiset gathered, AtomicInteger tally, Optional futureSailor, Digest identifier, Supplier isTimedOut, RingCommunications.Destination destination, String action, T empty) { if (futureSailor.isEmpty()) { + log.debug("Failed {}: {} from: {} on: {}", action, identifier, destination.member().getId(), + member.getId()); return !isTimedOut.get(); } T content = futureSailor.get(); - if (content != null) { - log.trace("{}: {} from: {} on: {}", action, identifier, destination.member().getId(), member.getId()); - gathered.add(content); - var max = max(gathered); - if (max != null) { - tally.set(max.getCount()); - // If there is only one active member in our context, it's us. - final var majority = tally.get() >= (context.activeCount() == 1 ? 1 : context.majority()); - if (majority) { - result.complete(max.getElement()); - log.debug("Majority: {} achieved: {}: {} on: {}", max.getCount(), action, identifier, - member.getId()); - return false; - } + log.trace("{}: {} from: {} on: {}", action, identifier, destination.member().getId(), member.getId()); + gathered.add(content); + var max = max(gathered); + if (max != null) { + tally.set(max.getCount()); + final var majority = tally.get() >= context.majority(true); + if (majority) { + result.complete(max.getElement()); + log.debug("Majority: {} achieved: {}: {} on: {}", max.getCount(), action, identifier, member.getId()); + return false; } - return !isTimedOut.get(); - } else { - log.debug("Failed {}: {} from: {} on: {}", action, identifier, destination.member().getId(), - member.getId()); - return !isTimedOut.get(); } + return !isTimedOut.get(); } private void reconcile(Optional result, @@ -935,7 +915,7 @@ private void reconcile(Optional result, if (!started.get()) { return; } - if (!result.isEmpty()) { + if (result.isPresent()) { try { Update update = result.get(); if (update.getEventsCount() > 0) { @@ -945,7 +925,7 @@ private void reconcile(Optional result, } } catch (NoSuchElementException e) { reconcileLog.debug("null interval reconciliation with {} : {} on: {}", destination.member().getId(), - member.getId(), e.getCause()); + e.getMessage(), member.getId()); } } if (started.get()) { @@ -959,8 +939,8 @@ private Update reconcile(ReconciliationService link, Integer ring) { return null; } CombinedIntervals keyIntervals = keyIntervals(); - reconcileLog.trace("Interval reconciliation on ring: {} with: {} on: {} intervals: {}", ring, - link.getMember().getId(), member.getId(), keyIntervals); + reconcileLog.trace("Interval reconciliation on ring: {} with: {} intervals: {} on: {} ", ring, + link.getMember().getId(), keyIntervals, member.getId()); return link.reconcile(Intervals.newBuilder() .setRing(ring) .addAllIntervals(keyIntervals.toIntervals()) @@ -973,7 +953,7 @@ private void reconcile(ScheduledExecutorService scheduler, Duration duration) { return; } Thread.ofVirtual() - .start(() -> reconcile.execute((link, ring) -> reconcile(link, ring), + .start(() -> reconcile.execute(this::reconcile, (futureSailor, destination) -> reconcile(futureSailor, destination, scheduler, duration))); @@ -1021,8 +1001,8 @@ public KeyState append(KeyEvent event) { @Override public List append(KeyEvent... events) { List lks = super.append(events); - if (lks.size() > 0) { - updateLocationHash(lks.get(0).getCoordinates().getIdentifier()); + if (!lks.isEmpty()) { + updateLocationHash(lks.getFirst().getCoordinates().getIdentifier()); } return lks; } @@ -1031,8 +1011,8 @@ public List append(KeyEvent... events) { public List append(List events, List attachments) { List lks = super.append(events, attachments); - if (lks.size() > 0) { - updateLocationHash(lks.get(0).getCoordinates().getIdentifier()); + if (!lks.isEmpty()) { + updateLocationHash(lks.getFirst().getCoordinates().getIdentifier()); } return lks; } @@ -1130,9 +1110,7 @@ public KERL_ getKERL(Ident identifier) { @Override public KeyEvent_ getKeyEvent(EventCoords coordinates) { log.trace("get key event for coordinates on: {}", member.getId()); - final Function func = k -> { - return k.getKeyEvent(coordinates); - }; + final Function func = k -> k.getKeyEvent(coordinates); return complete(func); } @@ -1144,7 +1122,11 @@ public KeyState_ getKeyState(EventCoords coordinates) { @Override public KeyState_ getKeyState(Ident identifier, long sequenceNumber) { - return null; + if (log.isTraceEnabled()) { + log.trace("get key state for {}:{} on: {}", Identifier.from(identifier), ULong.valueOf(sequenceNumber), + member.getId()); + } + return complete(k -> k.getKeyState(identifier, sequenceNumber)); } @Override diff --git a/thoth/src/main/java/com/salesforce/apollo/thoth/KerlSpace.java b/thoth/src/main/java/com/salesforce/apollo/thoth/KerlSpace.java index f45cb8e897..88d7b8cfef 100644 --- a/thoth/src/main/java/com/salesforce/apollo/thoth/KerlSpace.java +++ b/thoth/src/main/java/com/salesforce/apollo/thoth/KerlSpace.java @@ -38,6 +38,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -58,12 +59,14 @@ public class KerlSpace { private static final Logger log = LoggerFactory.getLogger(KerlSpace.class); private final JdbcConnectionPool connectionPool; + private final Digest member; - public KerlSpace(JdbcConnectionPool connectionPool) { + public KerlSpace(JdbcConnectionPool connectionPool, Digest member) { this.connectionPool = connectionPool; + this.member = member; } - public static void upsert(DSLContext dsl, EventCoords coordinates, Attachment attachment) { + public static void upsert(DSLContext dsl, EventCoords coordinates, Attachment attachment, Digest member) { final var identBytes = coordinates.getIdentifier().toByteArray(); var ident = dsl.newRecord(IDENTIFIER); @@ -100,7 +103,7 @@ public static void upsert(DSLContext dsl, EventCoords coordinates, Attachment at vRec.insert(); } - public static void upsert(DSLContext context, KeyEvent event, DigestAlgorithm digestAlgorithm) { + public static void upsert(DSLContext context, KeyEvent event, DigestAlgorithm digestAlgorithm, Digest member) { final EventCoordinates prevCoords = event.getPrevious(); final var identBytes = event.getIdentifier().toIdent().toByteArray(); @@ -148,11 +151,11 @@ public static void upsert(DSLContext context, KeyEvent event, DigestAlgorithm di } } - public static void upsert(DSLContext dsl, Validations validations) { + public static void upsert(DSLContext dsl, Validations validations, Digest member) { final var coordinates = validations.getCoordinates(); final var logCoords = EventCoordinates.from(coordinates); final var logIdentifier = Identifier.from(coordinates.getIdentifier()); - log.trace("Upserting validations for: {}", logCoords); + log.trace("Upserting validations for: {} on: {}", logCoords, member); final var identBytes = coordinates.getIdentifier().toByteArray(); try { @@ -177,9 +180,9 @@ public static void upsert(DSLContext dsl, Validations validations) { ULong.valueOf(coordinates.getSequenceNumber()).toBigInteger()) .returningResult(PENDING_COORDINATES.ID) .fetchOne(); - log.trace("Id: {} for: {}", id.value1(), logCoords); + log.trace("Id: {} for: {} on: {}", id.value1(), logCoords, member); } catch (DataAccessException e) { - log.trace("access exception for: {}", logCoords, e); + log.trace("access exception for: {} on: {}", logCoords, e, member); // Already exists id = dsl.select(PENDING_COORDINATES.ID) .from(PENDING_COORDINATES) @@ -193,7 +196,7 @@ public static void upsert(DSLContext dsl, Validations validations) { .fetchOne(); } if (id == null) { - log.trace("Null coordinates ID for: {}", coordinates); + log.trace("Null coordinates ID for: {} on: {}", coordinates, member); return; } var vRec = dsl.newRecord(PENDING_VALIDATIONS); @@ -215,11 +218,11 @@ public Biff populate(long seed, CombinedIntervals intervals, double fpr) { try (var connection = connectionPool.getConnection()) { var dsl = DSL.using(connection); eventDigestsIn(intervals, dsl).forEach(d -> { - log.trace("Adding reconcile digest: {}", d); + log.trace("Adding reconcile digest: {} on: {}", d, member); bff.add(d); }); } catch (SQLException e) { - log.error("Unable populate bloom filter, cannot acquire JDBC connection", e); + log.error("Unable populate bloom filter, cannot acquire JDBC connection on: {}", member, e); } return bff.toBff(); } @@ -241,16 +244,16 @@ public Update.Builder reconcile(Intervals intervals, DigestKERL kerl) { .stream() .map(KeyInterval::new) .flatMap(i -> eventDigestsIn(i, dsl)) - .peek(d -> log.trace("reconcile digest: {}", d)) + .peek(d -> log.trace("reconcile digest: {} on: {}", d, member)) .filter(d -> !biff.contains(d)) - .peek(d -> log.trace("filtered reconcile digest: {}", d)) + .peek(d -> log.trace("filtered reconcile digest: {} on: {}", d, member)) .map(d -> event(d, dsl, kerl)) - .filter(ke -> ke != null) + .filter(Objects::nonNull) .forEach(update::addEvents); } catch (SQLException e) { - log.error("Unable to provide estimated cardinality, cannot acquire JDBC connection", e); - throw new IllegalStateException("Unable to provide estimated cardinality, cannot acquire JDBC connection", - e); + log.error("Unable to provide estimated cardinality, cannot acquire JDBC connection on: {}", member, e); + throw new IllegalStateException( + "Unable to provide estimated cardinality, cannot acquire JDBC connection on:" + member, e); } return update; } @@ -263,7 +266,7 @@ public Update.Builder reconcile(Intervals intervals, DigestKERL kerl) { */ public void update(List events, KERL.AppendKERL kerl) { if (events.isEmpty()) { - log.trace("No events to update"); + log.trace("No events to update on: {}", member); return; } @@ -277,18 +280,18 @@ public void update(List events, KERL.Appe for (var evente_ : events) { final var event = ProtobufEventFactory.from(evente_.getEvent()); if (!evente_.getValidations().equals(Validations.getDefaultInstance())) { - upsert(context, evente_.getValidations()); + upsert(context, evente_.getValidations(), member); } if (evente_.hasAttachment()) { - upsert(context, event.getCoordinates().toEventCoords(), evente_.getAttachment()); + upsert(context, event.getCoordinates().toEventCoords(), evente_.getAttachment(), member); } - upsert(context, event, digestAlgorithm); + upsert(context, event, digestAlgorithm, member); } }); commitPending(dsl, kerl); } catch (SQLException e) { - log.error("Unable to update events, cannot acquire JDBC connection", e); - throw new IllegalStateException("Unable to update events, cannot acquire JDBC connection", e); + log.error("Unable to update events, cannot acquire JDBC connection on: {}", member, e); + throw new IllegalStateException("Unable to update events, cannot acquire JDBC connection on: " + member, e); } } @@ -298,13 +301,13 @@ private int cardinality() { var dsl = DSL.using(connection); return dsl.fetchCount(dsl.selectFrom(IDENTIFIER)); } catch (SQLException e) { - log.error("Unable to provide estimated cardinality, cannot acquire JDBC connection", e); + log.error("Unable to provide estimated cardinality, cannot acquire JDBC connection on: {}", member, e); return 0; } } private void commitPending(DSLContext context, KERL.AppendKERL kerl) { - log.trace("Commit pending"); + log.trace("Commit pending on: {}", member); context.select(PENDING_COORDINATES.ID, PENDING_EVENT.EVENT, PENDING_COORDINATES.ILK) .from(PENDING_EVENT) .join(PENDING_COORDINATES) @@ -330,7 +333,7 @@ private void commitPending(DSLContext context, KERL.AppendKERL kerl) { .setAttachment(attach) .build()))); } catch (InvalidProtocolBufferException e) { - log.error("Cannot deserialize attachment", e); + log.error("Cannot deserialize attachment on: {}", member, e); } }); context.select(PENDING_VALIDATIONS.VALIDATIONS) @@ -348,7 +351,7 @@ private void commitPending(DSLContext context, KERL.AppendKERL kerl) { v -> JohnHancock.from( v.getSignature())))); } catch (InvalidProtocolBufferException e) { - log.error("Cannot deserialize validation", e); + log.error("Cannot deserialize validation on: {}", member, e); } }); kerl.append(event); diff --git a/thoth/src/main/java/com/salesforce/apollo/thoth/Maat.java b/thoth/src/main/java/com/salesforce/apollo/thoth/Maat.java index ad514cf797..e1aebbc9dd 100644 --- a/thoth/src/main/java/com/salesforce/apollo/thoth/Maat.java +++ b/thoth/src/main/java/com/salesforce/apollo/thoth/Maat.java @@ -128,10 +128,10 @@ record validator(EstablishmentEvent validating, JohnHancock signature) { r.validating.getIdentifier()); } } - var validated = verified >= context.majority(); + var validated = verified >= context.majority(true); log.trace("Validated: {} valid: {} out of: {} required: {} for: {} ", validated, verified, mapped.size(), - ctx.majority(), event.getCoordinates()); + ctx.majority(true), event.getCoordinates()); return validated; } } diff --git a/thoth/src/main/java/com/salesforce/apollo/thoth/grpc/dht/DhtClient.java b/thoth/src/main/java/com/salesforce/apollo/thoth/grpc/dht/DhtClient.java index 44b491f964..842036d28c 100644 --- a/thoth/src/main/java/com/salesforce/apollo/thoth/grpc/dht/DhtClient.java +++ b/thoth/src/main/java/com/salesforce/apollo/thoth/grpc/dht/DhtClient.java @@ -8,14 +8,14 @@ import com.codahale.metrics.Timer.Context; import com.google.protobuf.Empty; -import com.salesforce.apollo.stereotomy.event.proto.*; -import com.salesforce.apollo.stereotomy.services.grpc.proto.*; -import com.salesforce.apollo.thoth.proto.KerlDhtGrpc; import com.salesforce.apollo.archipelago.ManagedServerChannel; import com.salesforce.apollo.archipelago.ServerConnectionCache.CreateClientCommunications; import com.salesforce.apollo.membership.Member; +import com.salesforce.apollo.stereotomy.event.proto.*; import com.salesforce.apollo.stereotomy.services.grpc.StereotomyMetrics; +import com.salesforce.apollo.stereotomy.services.grpc.proto.*; import com.salesforce.apollo.stereotomy.services.proto.ProtoKERLService; +import com.salesforce.apollo.thoth.proto.KerlDhtGrpc; import java.io.IOException; import java.util.List; @@ -100,7 +100,7 @@ public KeyState_ getKeyState(Ident identifier) { @Override public KeyState_ getKeyState(IdentAndSeq identAndSeq) { - return null; + return service.getKeyState(identAndSeq.getIdentifier(), identAndSeq.getSequenceNumber()); } @Override diff --git a/thoth/src/main/java/com/salesforce/apollo/thoth/grpc/dht/DhtServer.java b/thoth/src/main/java/com/salesforce/apollo/thoth/grpc/dht/DhtServer.java index d6b654abbc..9d86f11682 100644 --- a/thoth/src/main/java/com/salesforce/apollo/thoth/grpc/dht/DhtServer.java +++ b/thoth/src/main/java/com/salesforce/apollo/thoth/grpc/dht/DhtServer.java @@ -8,12 +8,12 @@ import com.codahale.metrics.Timer.Context; import com.google.protobuf.Empty; -import com.salesforce.apollo.stereotomy.event.proto.*; -import com.salesforce.apollo.stereotomy.services.grpc.proto.*; -import com.salesforce.apollo.thoth.proto.KerlDhtGrpc.KerlDhtImplBase; import com.salesforce.apollo.archipelago.RoutableService; +import com.salesforce.apollo.stereotomy.event.proto.*; import com.salesforce.apollo.stereotomy.services.grpc.StereotomyMetrics; +import com.salesforce.apollo.stereotomy.services.grpc.proto.*; import com.salesforce.apollo.stereotomy.services.proto.ProtoKERLService; +import com.salesforce.apollo.thoth.proto.KerlDhtGrpc.KerlDhtImplBase; import io.grpc.Status; import io.grpc.StatusRuntimeException; import io.grpc.stub.StreamObserver; @@ -308,6 +308,37 @@ public void getKeyStateCoords(EventCoords request, StreamObserver res }); } + @Override + public void getKeyStateSeqNum(IdentAndSeq request, StreamObserver responseObserver) { + Context timer = metrics != null ? metrics.getKeyStateCoordsService().time() : null; + if (metrics != null) { + final var serializedSize = request.getSerializedSize(); + metrics.inboundBandwidth().mark(serializedSize); + metrics.inboundGetKeyStateCoordsRequest().mark(serializedSize); + } + routing.evaluate(responseObserver, s -> { + var response = s.getKeyState(request.getIdentifier(), request.getSequenceNumber()); + if (response == null) { + if (timer != null) { + timer.stop(); + } + responseObserver.onNext(KeyState_.getDefaultInstance()); + responseObserver.onCompleted(); + } + if (timer != null) { + timer.stop(); + } + var state = response == null ? KeyState_.getDefaultInstance() : response; + responseObserver.onNext(state); + responseObserver.onCompleted(); + if (metrics != null) { + final var serializedSize = state.getSerializedSize(); + metrics.outboundBandwidth().mark(serializedSize); + metrics.outboundGetKeyStateCoordsResponse().mark(serializedSize); + } + }); + } + @Override public void getKeyStateWithAttachments(EventCoords request, StreamObserver responseObserver) { diff --git a/thoth/src/test/java/com/salesforce/apollo/thoth/AbstractDhtTest.java b/thoth/src/test/java/com/salesforce/apollo/thoth/AbstractDhtTest.java index 2fa5dcca66..5ab0bb59a1 100644 --- a/thoth/src/test/java/com/salesforce/apollo/thoth/AbstractDhtTest.java +++ b/thoth/src/test/java/com/salesforce/apollo/thoth/AbstractDhtTest.java @@ -117,15 +117,14 @@ public void before() throws Exception { .mapToObj(i -> stereotomy.newIdentifier()) .collect(Collectors.toMap(controlled -> new ControlledIdentifierMember(controlled), controlled -> controlled)); - context = Context.newBuilder().setpByz(PBYZ).setCardinality(getCardinality()).build(); + context = Context.newBuilder().setpByz(PBYZ).setCardinality(getCardinality()).build(); ConcurrentSkipListMap serverMembers = new ConcurrentSkipListMap<>(); identities.keySet().forEach(member -> instantiate(member, context, serverMembers)); System.out.println(); System.out.println(); - System.out.println( - String.format("Cardinality: %s, Prob Byz: %s, Rings: %s Majority: %s", getCardinality(), PBYZ, - context.getRingCount(), context.majority())); + System.out.printf("Cardinality: %s, Prob Byz: %s, Rings: %s Majority: %s%n", getCardinality(), PBYZ, + context.getRingCount(), context.majority(true)); System.out.println(); } diff --git a/thoth/src/test/java/com/salesforce/apollo/thoth/BootstrappingTest.java b/thoth/src/test/java/com/salesforce/apollo/thoth/BootstrappingTest.java index 906ee0f2c7..0e2b3a787b 100644 --- a/thoth/src/test/java/com/salesforce/apollo/thoth/BootstrappingTest.java +++ b/thoth/src/test/java/com/salesforce/apollo/thoth/BootstrappingTest.java @@ -34,7 +34,6 @@ import java.security.SecureRandom; import java.time.Clock; import java.time.Duration; -import java.util.concurrent.Executors; import java.util.function.BiFunction; import java.util.function.Function; @@ -66,9 +65,9 @@ public void smokin() throws Exception { gate.set(true); var gorgoneions = routers.values().stream().map(r -> { var k = dhts.get(r.getFrom()).asKERL(); - return new Gorgoneion(Parameters.newBuilder().setKerl(k).build(), (ControlledIdentifierMember) r.getFrom(), - context, new DirectPublisher(new ProtoKERLAdapter(k)), r, - Executors.newScheduledThreadPool(2, Thread.ofVirtual().factory()), null); + return new Gorgoneion(r.getFrom().equals(dhts.firstKey()), t -> true, + Parameters.newBuilder().setKerl(k).build(), (ControlledIdentifierMember) r.getFrom(), + context, new DirectPublisher(r.getFrom().getId(), new ProtoKERLAdapter(k)), r, null); }).toList(); final var dht = (KerlDHT) dhts.values().stream().findFirst().get(); @@ -112,7 +111,7 @@ context, new DirectPublisher(new ProtoKERLAdapter(k)), r, final var invitation = gorgoneionClient.apply(Duration.ofSeconds(120)); assertNotNull(invitation); assertNotEquals(Validations.getDefaultInstance(), invitation); - assertTrue(invitation.getValidationsCount() >= context.majority()); + assertTrue(invitation.getValidationsCount() >= context.majority(true)); // Verify client KERL published Utils.waitForCondition(30_000, 1000, () -> testKerl.getKeyEvent(client.getEvent().getCoordinates()) != null); var keyS = testKerl.getKeyEvent(client.getEvent().getCoordinates()); diff --git a/thoth/src/test/java/com/salesforce/apollo/thoth/KerlSpaceTest.java b/thoth/src/test/java/com/salesforce/apollo/thoth/KerlSpaceTest.java index d0697552e8..7c0cde364a 100644 --- a/thoth/src/test/java/com/salesforce/apollo/thoth/KerlSpaceTest.java +++ b/thoth/src/test/java/com/salesforce/apollo/thoth/KerlSpaceTest.java @@ -6,13 +6,13 @@ */ package com.salesforce.apollo.thoth; -import com.salesforce.apollo.thoth.proto.Interval; -import com.salesforce.apollo.thoth.proto.Intervals; import com.salesforce.apollo.bloomFilters.BloomFilter; import com.salesforce.apollo.cryptography.DigestAlgorithm; import com.salesforce.apollo.stereotomy.StereotomyImpl; import com.salesforce.apollo.stereotomy.db.UniKERLDirectPooled; import com.salesforce.apollo.stereotomy.mem.MemKeyStore; +import com.salesforce.apollo.thoth.proto.Interval; +import com.salesforce.apollo.thoth.proto.Intervals; import liquibase.Liquibase; import liquibase.database.core.H2Database; import liquibase.exception.LiquibaseException; @@ -67,11 +67,11 @@ public void smokin() throws Exception { JdbcConnectionPool connectionPoolB = JdbcConnectionPool.create("jdbc:h2:mem:B;DB_CLOSE_DELAY=-1", "", ""); connectionPoolB.setMaxConnections(10); - var spaceA = new KerlSpace(connectionPoolA); + var spaceA = new KerlSpace(connectionPoolA, DigestAlgorithm.DEFAULT.getOrigin()); var stereotomyA = new StereotomyImpl(new MemKeyStore(), new UniKERLDirectPooled(connectionPoolA, digestAlgorithm).create(), entropy); - var spaceB = new KerlSpace(connectionPoolB); + var spaceB = new KerlSpace(connectionPoolB, DigestAlgorithm.DEFAULT.getLast()); var stereotomyB = new StereotomyImpl(new MemKeyStore(), new UniKERLDirectPooled(connectionPoolB, digestAlgorithm).create(), entropy);