From db9c3492cda04158c69fbaa7e00eb2607bd35879 Mon Sep 17 00:00:00 2001 From: Hellblazer Date: Mon, 17 Jun 2024 19:33:34 -0700 Subject: [PATCH 01/32] use sampling for msg limits --- .../java/com/salesforce/apollo/fireflies/View.java | 12 ++++++------ .../messaging/rbc/ReliableBroadcaster.java | 4 ++-- .../java/com/salesforce/apollo/model/SubDomain.java | 4 +++- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/fireflies/src/main/java/com/salesforce/apollo/fireflies/View.java b/fireflies/src/main/java/com/salesforce/apollo/fireflies/View.java index 20b60614e..7f4ef1cc2 100644 --- a/fireflies/src/main/java/com/salesforce/apollo/fireflies/View.java +++ b/fireflies/src/main/java/com/salesforce/apollo/fireflies/View.java @@ -1178,8 +1178,8 @@ private AccusationGossip.Builder processAccusations(BloomFilter bff) { .flatMap(m -> m.getAccusations()) .filter(m -> current.equals(m.currentView())) .filter(a -> !bff.contains(a.getHash())) - .limit(params.maximumTxfr()) - // .collect(new ReservoirSampler<>(params.maximumTxfr(), Entropy.bitsStream())) + // .limit(params.maximumTxfr()) + .collect(new ReservoirSampler<>(params.maximumTxfr(), Entropy.bitsStream())) .forEach(a -> builder.addUpdates(a.getWrapped())); return builder; } @@ -1215,8 +1215,8 @@ private NoteGossip.Builder processNotes(BloomFilter bff) { .filter(m -> !shunned.contains(m.getId())) .filter(m -> !bff.contains(m.getNote().getHash())) .map(m -> m.getNote()) - .limit(params.maximumTxfr()) // Always in sorted order with this method - // .collect(new ReservoirSampler<>(params.maximumTxfr() * 2, Entropy.bitsStream())) + // .limit(params.maximumTxfr()) // Always in sorted order with this method + .collect(new ReservoirSampler<>(params.maximumTxfr() * 2, Entropy.bitsStream())) .forEach(n -> builder.addUpdates(n.getWrapped())); return builder; } @@ -1249,8 +1249,8 @@ private ViewChangeGossip.Builder processObservations(BloomFilter bff) { .filter(e -> Digest.from(e.getValue().getChange().getCurrent()).equals(current)) .filter(m -> !bff.contains(m.getKey())) .map(m -> m.getValue()) - .limit(params.maximumTxfr()) - // .collect(new ReservoirSampler<>(params.maximumTxfr(), Entropy.bitsStream())) + // .limit(params.maximumTxfr()) + .collect(new ReservoirSampler<>(params.maximumTxfr(), Entropy.bitsStream())) .forEach(n -> builder.addUpdates(n)); return builder; } diff --git a/memberships/src/main/java/com/salesforce/apollo/membership/messaging/rbc/ReliableBroadcaster.java b/memberships/src/main/java/com/salesforce/apollo/membership/messaging/rbc/ReliableBroadcaster.java index a4b9f149f..ca923b3f6 100644 --- a/memberships/src/main/java/com/salesforce/apollo/membership/messaging/rbc/ReliableBroadcaster.java +++ b/memberships/src/main/java/com/salesforce/apollo/membership/messaging/rbc/ReliableBroadcaster.java @@ -476,7 +476,7 @@ public void receive(List messages) { } log.trace("receiving: {} msgs on: {}", messages.size(), member.getId()); deliver(messages.stream() - .limit(params.maxMessages) + // .limit(params.maxMessages) .map(am -> new state(adapter.hasher.apply(am.getContent()), AgedMessage.newBuilder(am))) .filter(s -> !dup(s)) .filter(s -> adapter.verifier.test(s.msg.getContent())) @@ -495,7 +495,7 @@ public Iterable reconcile(BloomFilter biff, Diges .filter(s -> !biff.contains(s.hash)) .filter(s -> s.msg.getAge() < maxAge) .forEach(s -> mailBox.add(s.msg)); - List reconciled = mailBox.stream().limit(params.maxMessages).map(b -> b.build()).toList(); + List reconciled = mailBox.stream().map(b -> b.build()).toList(); if (!reconciled.isEmpty()) { log.trace("reconciled: {} for: {} on: {}", reconciled.size(), from, member.getId()); } diff --git a/model/src/main/java/com/salesforce/apollo/model/SubDomain.java b/model/src/main/java/com/salesforce/apollo/model/SubDomain.java index 69466047d..77a013031 100644 --- a/model/src/main/java/com/salesforce/apollo/model/SubDomain.java +++ b/model/src/main/java/com/salesforce/apollo/model/SubDomain.java @@ -19,6 +19,7 @@ import com.salesforce.apollo.demesne.proto.DelegationUpdate; import com.salesforce.apollo.demesne.proto.SignedDelegate; import com.salesforce.apollo.membership.Member; +import com.salesforce.apollo.membership.ReservoirSampler; import com.salesforce.apollo.membership.stereotomy.ControlledIdentifierMember; import com.salesforce.apollo.model.comms.Delegation; import com.salesforce.apollo.model.comms.DelegationServer; @@ -204,7 +205,8 @@ private DelegationUpdate.Builder update(DelegationUpdate update, DelegationUpdat delegates.entrySet() .stream() .filter(e -> !bff.contains(Digest.from(e.getKey()))) - .limit(maxTransfer) + // .limit(maxTransfer) + .collect(new ReservoirSampler<>(maxTransfer, Entropy.bitsStream())) .forEach(e -> builder.addUpdate(e.getValue())); return builder; } From a82a3378cb98c1c1d3d70330e91164fc474f3ef0 Mon Sep 17 00:00:00 2001 From: Hellblazer Date: Mon, 17 Jun 2024 19:45:53 -0700 Subject: [PATCH 02/32] max --- .../src/main/java/com/salesforce/apollo/fireflies/View.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fireflies/src/main/java/com/salesforce/apollo/fireflies/View.java b/fireflies/src/main/java/com/salesforce/apollo/fireflies/View.java index 7f4ef1cc2..a4e010db4 100644 --- a/fireflies/src/main/java/com/salesforce/apollo/fireflies/View.java +++ b/fireflies/src/main/java/com/salesforce/apollo/fireflies/View.java @@ -1216,7 +1216,7 @@ private NoteGossip.Builder processNotes(BloomFilter bff) { .filter(m -> !bff.contains(m.getNote().getHash())) .map(m -> m.getNote()) // .limit(params.maximumTxfr()) // Always in sorted order with this method - .collect(new ReservoirSampler<>(params.maximumTxfr() * 2, Entropy.bitsStream())) + .collect(new ReservoirSampler<>(params.maximumTxfr(), Entropy.bitsStream())) .forEach(n -> builder.addUpdates(n.getWrapped())); return builder; } From b63093d2d3dcaefcbc59c859f9c5f8299e8e72ed Mon Sep 17 00:00:00 2001 From: Hellblazer Date: Mon, 17 Jun 2024 20:32:59 -0700 Subject: [PATCH 03/32] no sampling of accusations --- .../src/main/java/com/salesforce/apollo/fireflies/View.java | 3 --- .../java/com/salesforce/apollo/fireflies/ViewManagement.java | 3 +-- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/fireflies/src/main/java/com/salesforce/apollo/fireflies/View.java b/fireflies/src/main/java/com/salesforce/apollo/fireflies/View.java index a4e010db4..016f9447f 100644 --- a/fireflies/src/main/java/com/salesforce/apollo/fireflies/View.java +++ b/fireflies/src/main/java/com/salesforce/apollo/fireflies/View.java @@ -1178,8 +1178,6 @@ private AccusationGossip.Builder processAccusations(BloomFilter bff) { .flatMap(m -> m.getAccusations()) .filter(m -> current.equals(m.currentView())) .filter(a -> !bff.contains(a.getHash())) - // .limit(params.maximumTxfr()) - .collect(new ReservoirSampler<>(params.maximumTxfr(), Entropy.bitsStream())) .forEach(a -> builder.addUpdates(a.getWrapped())); return builder; } @@ -1355,7 +1353,6 @@ private Update updatesForDigests(Gossip gossip) { .flatMap(m -> m.getAccusations()) .filter(a -> a.currentView().equals(current)) .filter(a -> !accBff.contains(a.getHash())) - .collect(new ReservoirSampler<>(params.maximumTxfr(), Entropy.bitsStream())) .forEach(a -> builder.addAccusations(a.getWrapped())); } diff --git a/fireflies/src/main/java/com/salesforce/apollo/fireflies/ViewManagement.java b/fireflies/src/main/java/com/salesforce/apollo/fireflies/ViewManagement.java index 8055e32ff..2521d07d4 100644 --- a/fireflies/src/main/java/com/salesforce/apollo/fireflies/ViewManagement.java +++ b/fireflies/src/main/java/com/salesforce/apollo/fireflies/ViewManagement.java @@ -521,9 +521,8 @@ private void initiateViewChange() { .setObserver(node.getId().toDigeste()) .setCurrent(currentView().toDigeste()) .setAttempt(attempt.getAndIncrement()) - .addAllLeaves( - view.streamShunned().map(Digest::toDigeste).collect(Collectors.toSet())) .addAllJoins(joins.keySet().stream().map(Digest::toDigeste).toList()); + view.streamShunned().map(Digest::toDigeste).forEach(builder::addLeaves); ViewChange change = builder.build(); vote.set(change); var signature = node.sign(change.toByteString()); From e191df8a7eb0ff0ff8b37d0d0c95dd23c16807ac Mon Sep 17 00:00:00 2001 From: Hellblazer Date: Mon, 17 Jun 2024 20:44:45 -0700 Subject: [PATCH 04/32] no sampling of observations --- .../src/main/java/com/salesforce/apollo/fireflies/View.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/fireflies/src/main/java/com/salesforce/apollo/fireflies/View.java b/fireflies/src/main/java/com/salesforce/apollo/fireflies/View.java index 016f9447f..bc4f92a3b 100644 --- a/fireflies/src/main/java/com/salesforce/apollo/fireflies/View.java +++ b/fireflies/src/main/java/com/salesforce/apollo/fireflies/View.java @@ -1247,8 +1247,6 @@ private ViewChangeGossip.Builder processObservations(BloomFilter bff) { .filter(e -> Digest.from(e.getValue().getChange().getCurrent()).equals(current)) .filter(m -> !bff.contains(m.getKey())) .map(m -> m.getValue()) - // .limit(params.maximumTxfr()) - .collect(new ReservoirSampler<>(params.maximumTxfr(), Entropy.bitsStream())) .forEach(n -> builder.addUpdates(n)); return builder; } @@ -1363,7 +1361,6 @@ private Update updatesForDigests(Gossip gossip) { .stream() .filter(e -> Digest.from(e.getValue().getChange().getCurrent()).equals(current)) .filter(e -> !obsvBff.contains(e.getKey())) - .collect(new ReservoirSampler<>(params.maximumTxfr(), Entropy.bitsStream())) .forEach(e -> builder.addObservations(e.getValue())); } From 4a47caa347856fa74ba032eb4754ee0079f48c4e Mon Sep 17 00:00:00 2001 From: Hellblazer Date: Mon, 17 Jun 2024 21:53:36 -0700 Subject: [PATCH 05/32] better shuffling, etc. --- .../com/salesforce/apollo/utils/Utils.java | 24 +++++++++++ .../com/salesforce/apollo/fireflies/View.java | 41 +++++++++++-------- .../messaging/rbc/ReliableBroadcaster.java | 4 +- 3 files changed, 51 insertions(+), 18 deletions(-) diff --git a/cryptography/src/main/java/com/salesforce/apollo/utils/Utils.java b/cryptography/src/main/java/com/salesforce/apollo/utils/Utils.java index 534bf2dca..afdc7329a 100644 --- a/cryptography/src/main/java/com/salesforce/apollo/utils/Utils.java +++ b/cryptography/src/main/java/com/salesforce/apollo/utils/Utils.java @@ -18,18 +18,30 @@ import java.security.PublicKey; import java.security.cert.X509Certificate; import java.time.Instant; +import java.util.ArrayList; import java.util.Collections; +import java.util.List; import java.util.concurrent.Callable; import java.util.function.Consumer; import java.util.function.Supplier; +import java.util.stream.Collector; +import java.util.stream.Collectors; +import java.util.stream.Stream; import static com.salesforce.apollo.cryptography.QualifiedBase64.qb64; +import static java.util.stream.Collectors.toList; /** * @author hal.hildebrand **/ public class Utils { + private static final Collector SHUFFLER = Collectors.collectingAndThen( + Collectors.toCollection(ArrayList::new), list -> { + Collections.shuffle(list); + return list; + }); + /** * Copy the contents of the input stream to the output stream. It is the caller's responsibility to close the * streams. @@ -156,6 +168,11 @@ public static BcX500NameDnImpl encode(Digest digest, String host, int port, Publ String.format("CN=%s, L=%s, UID=%s, DC=%s", host, port, qb64(digest), qb64(signingKey))); } + @SuppressWarnings("unchecked") + public static Collector> toShuffledList() { + return (Collector>) SHUFFLER; + } + /** * Find a free port for any local address * @@ -165,6 +182,13 @@ public static int allocatePort() { return allocatePort(null); } + public static Collector> toEagerShuffledStream() { + return Collectors.collectingAndThen(toList(), list -> { + Collections.shuffle(list); + return list.stream(); + }); + } + /** * Find a free port on the interface with the given address * diff --git a/fireflies/src/main/java/com/salesforce/apollo/fireflies/View.java b/fireflies/src/main/java/com/salesforce/apollo/fireflies/View.java index bc4f92a3b..eef938c7a 100644 --- a/fireflies/src/main/java/com/salesforce/apollo/fireflies/View.java +++ b/fireflies/src/main/java/com/salesforce/apollo/fireflies/View.java @@ -421,18 +421,17 @@ void notifyListeners(List joining, List leavin final var viewChange = new ViewChange(context.asStatic(), currentView(), joining.stream().map(SelfAddressingIdentifier::getDigest).toList(), Collections.unmodifiableList(leaving)); - viewNotificationQueue.execute(Utils.wrapped(() -> { - viewChangeListeners.entrySet().forEach(entry -> { + viewChangeListeners.forEach((key, value) -> { + viewNotificationQueue.execute(Utils.wrapped(() -> { try { - log.trace("Notifying: {} view change: {} cardinality: {} joins: {} leaves: {} on: {} ", - entry.getKey(), currentView(), context.size(), joining.size(), leaving.size(), - node.getId()); - entry.getValue().accept(viewChange); + log.trace("Notifying: {} view change: {} cardinality: {} joins: {} leaves: {} on: {} ", key, + currentView(), context.size(), joining.size(), leaving.size(), node.getId()); + value.accept(viewChange); } catch (Throwable e) { - log.error("error in view change listener: {} on: {} ", entry.getKey(), node.getId(), e); + log.error("error in view change listener: {} on: {} ", key, node.getId(), e); } - }); - }, log)); + }, log)); + }); } /** @@ -1011,6 +1010,7 @@ private BloomFilter getAccusationsBff(long seed, double p) { context.allMembers() .flatMap(Participant::getAccusations) .filter(Objects::nonNull) + .collect(Utils.toShuffledList()) .forEach(m -> bff.add(m.getHash())); return bff; } @@ -1035,7 +1035,7 @@ private BloomFilter getNotesBff(long seed, double p) { private BloomFilter getObservationsBff(long seed, double p) { BloomFilter bff = new BloomFilter.DigestBloomFilter(seed, Math.max(params.minimumBiffCardinality(), context.cardinality() * 2), p); - observations.keySet().forEach(d -> bff.add(d)); + observations.keySet().stream().collect(Utils.toShuffledList()).forEach(bff::add); return bff; } @@ -1175,7 +1175,9 @@ private AccusationGossip.Builder processAccusations(BloomFilter bff) { // bff var current = currentView(); context.allMembers() - .flatMap(m -> m.getAccusations()) + .flatMap(Participant::getAccusations) + .collect(Utils.toShuffledList()) + .stream() .filter(m -> current.equals(m.currentView())) .filter(a -> !bff.contains(a.getHash())) .forEach(a -> builder.addUpdates(a.getWrapped())); @@ -1212,9 +1214,9 @@ private NoteGossip.Builder processNotes(BloomFilter bff) { .filter(m -> current.equals(m.getNote().currentView())) .filter(m -> !shunned.contains(m.getId())) .filter(m -> !bff.contains(m.getNote().getHash())) - .map(m -> m.getNote()) - // .limit(params.maximumTxfr()) // Always in sorted order with this method .collect(new ReservoirSampler<>(params.maximumTxfr(), Entropy.bitsStream())) + .stream() + .map(m -> m.getNote()) .forEach(n -> builder.addUpdates(n.getWrapped())); return builder; } @@ -1243,6 +1245,8 @@ private ViewChangeGossip.Builder processObservations(BloomFilter bff) { // Add all updates that this view has that aren't reflected in the inbound bff final var current = currentView(); observations.entrySet() + .stream() + .collect(Utils.toShuffledList()) .stream() .filter(e -> Digest.from(e.getValue().getChange().getCurrent()).equals(current)) .filter(m -> !bff.contains(m.getKey())) @@ -1339,16 +1343,17 @@ private Update updatesForDigests(Gossip gossip) { .filter(m -> m.getNote() != null) .filter(m -> current.equals(m.getNote().currentView())) .filter(m -> !notesBff.contains(m.getNote().getHash())) - .map(m -> m.getNote().getWrapped()) .collect(new ReservoirSampler<>(params.maximumTxfr(), Entropy.bitsStream())) - .forEach(n -> builder.addNotes(n)); + .stream() + .map(m -> m.getNote().getWrapped()) + .forEach(builder::addNotes); } biff = gossip.getAccusations().getBff(); if (!biff.equals(Biff.getDefaultInstance())) { BloomFilter accBff = BloomFilter.from(biff); context.allMembers() - .flatMap(m -> m.getAccusations()) + .flatMap(Participant::getAccusations) .filter(a -> a.currentView().equals(current)) .filter(a -> !accBff.contains(a.getHash())) .forEach(a -> builder.addAccusations(a.getWrapped())); @@ -1358,6 +1363,8 @@ private Update updatesForDigests(Gossip gossip) { if (!biff.equals(Biff.getDefaultInstance())) { BloomFilter obsvBff = BloomFilter.from(biff); observations.entrySet() + .stream() + .collect(Utils.toShuffledList()) .stream() .filter(e -> Digest.from(e.getValue().getChange().getCurrent()).equals(current)) .filter(e -> !obsvBff.contains(e.getKey())) @@ -1548,7 +1555,7 @@ assert isValidMask(mask, context) : "Invalid mask: " + mask + " majority: " + co } } - // Fill the rest of the mask with randomly set index + // Fill the rest of the mask with randomly-set index while (mask.cardinality() != ((context.getBias() - 1) * context.toleranceLevel()) + 1) { int index = Entropy.nextBitsStreamInt(context.getRingCount()); diff --git a/memberships/src/main/java/com/salesforce/apollo/membership/messaging/rbc/ReliableBroadcaster.java b/memberships/src/main/java/com/salesforce/apollo/membership/messaging/rbc/ReliableBroadcaster.java index ca923b3f6..97698686f 100644 --- a/memberships/src/main/java/com/salesforce/apollo/membership/messaging/rbc/ReliableBroadcaster.java +++ b/memberships/src/main/java/com/salesforce/apollo/membership/messaging/rbc/ReliableBroadcaster.java @@ -466,7 +466,7 @@ public void clear() { public BloomFilter forReconcilliation() { var biff = new DigestBloomFilter(Entropy.nextBitsStreamLong(), params.bufferSize, params.falsePositiveRate); - state.keySet().forEach(k -> biff.add(k)); + state.keySet().stream().collect(Utils.toShuffledList()).forEach(k -> biff.add(k)); return biff; } @@ -491,6 +491,8 @@ public void receive(List messages) { public Iterable reconcile(BloomFilter biff, Digest from) { PriorityQueue mailBox = new PriorityQueue<>(Comparator.comparingInt(s -> s.getAge())); state.values() + .stream() + .collect(Utils.toShuffledList()) .stream() .filter(s -> !biff.contains(s.hash)) .filter(s -> s.msg.getAge() < maxAge) From 1f0a47f62c5870c7483dc9021c1bae51d2c2821a Mon Sep 17 00:00:00 2001 From: Hellblazer Date: Tue, 18 Jun 2024 09:06:25 -0700 Subject: [PATCH 06/32] d'oh. observers now concurrent skip list. reject joins if not observer. better logging --- .../apollo/fireflies/ViewManagement.java | 24 ++++++++++--------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/fireflies/src/main/java/com/salesforce/apollo/fireflies/ViewManagement.java b/fireflies/src/main/java/com/salesforce/apollo/fireflies/ViewManagement.java index 2521d07d4..5e18e4338 100644 --- a/fireflies/src/main/java/com/salesforce/apollo/fireflies/ViewManagement.java +++ b/fireflies/src/main/java/com/salesforce/apollo/fireflies/ViewManagement.java @@ -30,10 +30,7 @@ import java.time.Duration; import java.util.*; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.ConcurrentSkipListMap; -import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.locks.Lock; @@ -51,7 +48,7 @@ public class ViewManagement { private static final Logger log = LoggerFactory.getLogger(ViewManagement.class); final AtomicReference diadem = new AtomicReference<>(); - final Set observers = new TreeSet<>(); + final Set observers = new ConcurrentSkipListSet<>(); private final AtomicInteger attempt = new AtomicInteger(); private final Digest bootstrapView; private final DynamicContext context; @@ -288,6 +285,11 @@ void join(Join join, Digest from, StreamObserver responseObserver, Time .toList(), from, responseObserver, timer); return; } + if (!observers.contains(node.getId())) { + log.warn("Join not observer! from: {} observers: {} on: {}", from, observers, node.getId()); + responseObserver.onNext(Gateway.getDefaultInstance()); + return; + } if (!thisView.equals(joinView)) { responseObserver.onError(new StatusRuntimeException( Status.OUT_OF_RANGE.withDescription("View: " + joinView + " does not match: " + thisView))); @@ -376,8 +378,8 @@ boolean joined() { */ void maybeViewChange() { if (context.size() == 1 && joins.size() < context.getRingCount() - 1) { - log.info("Do not have minimum cluster size: {} required: {} for: {} on: {}", joins.size() + context.size(), - 4, currentView(), node.getId()); + log.info("Cannot form cluster: {} with: {} members, required:4 on: {}", currentView(), + joins.size() + context.size(), node.getId()); view.scheduleViewChange(); return; } @@ -390,7 +392,7 @@ void maybeViewChange() { // Use pending rebuttals as a proxy for stability if (view.hasPendingRebuttals()) { log.debug("Pending rebuttals in view: {} on: {}", currentView(), node.getId()); - view.scheduleViewChange(1); // 1 TTL round to check again + view.scheduleViewChange(2); // 2 TTL round2 to check again } else { view.scheduleFinalizeViewChange(); } @@ -474,8 +476,8 @@ Redirect seed(Registration registration, Digest from) { final var introductions = observers.stream().map(context::getMember).toList(); - log.debug("Member seeding: {} view: {} context: {} introductions: {} on: {}", newMember.getId(), - currentView(), context.getId(), introductions.size(), node.getId()); + log.info("Member seeding: {} view: {} context: {} introductions: {} on: {}", newMember.getId(), + currentView(), context.getId(), introductions.stream().map(p -> p.getId()).toList(), node.getId()); return Redirect.newBuilder() .setView(currentView().toDigeste()) .addAllIntroductions(introductions.stream() @@ -599,7 +601,7 @@ private void setDiadem(final HexBloom hex) { diadem.set(hex); currentView.set(diadem.get().compactWrapped()); resetObservers(); - log.debug("View: {} set diadem: {} observers: {} view: {} context: {} size: {} on: {}", context.getId(), + log.trace("View: {} set diadem: {} observers: {} view: {} context: {} size: {} on: {}", context.getId(), diadem.get().compactWrapped(), observers.stream().toList(), currentView(), context.getId(), context.size(), node.getId()); } From eae25e71679f0b6f71f913c994ddca3b2553c2fe Mon Sep 17 00:00:00 2001 From: Hellblazer Date: Tue, 18 Jun 2024 19:52:13 -0700 Subject: [PATCH 07/32] enjoin. replicated join to observers in the cluster --- .../apollo/fireflies/FireflyMetrics.java | 3 +- .../apollo/fireflies/FireflyMetricsImpl.java | 12 +++- .../com/salesforce/apollo/fireflies/View.java | 4 ++ .../apollo/fireflies/ViewManagement.java | 64 +++++++++++++++++-- .../fireflies/comm/gossip/FfClient.java | 15 ++--- .../fireflies/comm/gossip/FfServer.java | 34 ++++++++-- .../fireflies/comm/gossip/Fireflies.java | 12 +++- grpc/src/main/proto/fireflies.proto | 1 + 8 files changed, 120 insertions(+), 25 deletions(-) diff --git a/fireflies/src/main/java/com/salesforce/apollo/fireflies/FireflyMetrics.java b/fireflies/src/main/java/com/salesforce/apollo/fireflies/FireflyMetrics.java index 49ac68e90..1b2f23466 100644 --- a/fireflies/src/main/java/com/salesforce/apollo/fireflies/FireflyMetrics.java +++ b/fireflies/src/main/java/com/salesforce/apollo/fireflies/FireflyMetrics.java @@ -13,7 +13,6 @@ /** * @author hal.hildebrand - * */ public interface FireflyMetrics extends EndpointMetrics { @@ -25,6 +24,8 @@ public interface FireflyMetrics extends EndpointMetrics { Histogram gossipResponse(); + Timer inboundEnjoinDuration(); + Histogram inboundGateway(); Histogram inboundGossip(); diff --git a/fireflies/src/main/java/com/salesforce/apollo/fireflies/FireflyMetricsImpl.java b/fireflies/src/main/java/com/salesforce/apollo/fireflies/FireflyMetricsImpl.java index 1e1f5925f..3583d2443 100644 --- a/fireflies/src/main/java/com/salesforce/apollo/fireflies/FireflyMetricsImpl.java +++ b/fireflies/src/main/java/com/salesforce/apollo/fireflies/FireflyMetricsImpl.java @@ -6,8 +6,6 @@ */ package com.salesforce.apollo.fireflies; -import static com.codahale.metrics.MetricRegistry.name; - import com.codahale.metrics.Histogram; import com.codahale.metrics.Meter; import com.codahale.metrics.MetricRegistry; @@ -15,9 +13,10 @@ import com.salesforce.apollo.cryptography.Digest; import com.salesforce.apollo.protocols.EndpointMetricsImpl; +import static com.codahale.metrics.MetricRegistry.name; + /** * @author hal.hildebrand - * */ public class FireflyMetricsImpl extends EndpointMetricsImpl implements FireflyMetrics { private final Meter accusations; @@ -48,6 +47,7 @@ public class FireflyMetricsImpl extends EndpointMetricsImpl implements FireflyMe private final Timer seedDuration; private final Meter shunnedGossip; private final Meter viewChanges; + private final Timer inboundEnjoinDuration; public FireflyMetricsImpl(Digest context, MetricRegistry registry) { super(registry); @@ -80,6 +80,7 @@ public FireflyMetricsImpl(Digest context, MetricRegistry registry) { shunnedGossip = registry.meter(name(context.shortString(), "ff.gossip.shunned")); inboundSeed = registry.histogram(name(context.shortString(), "ff.seed.inbound.bytes")); viewChanges = registry.meter(name(context.shortString(), "ff.view.change")); + inboundEnjoinDuration = registry.timer(name(context.shortString(), "ff.enjoin.duration")); } @Override @@ -102,6 +103,11 @@ public Histogram gossipResponse() { return gossipResponse; } + @Override + public Timer inboundEnjoinDuration() { + return inboundEnjoinDuration; + } + @Override public Histogram inboundGateway() { return inboundGateway; diff --git a/fireflies/src/main/java/com/salesforce/apollo/fireflies/View.java b/fireflies/src/main/java/com/salesforce/apollo/fireflies/View.java index eef938c7a..1df306b37 100644 --- a/fireflies/src/main/java/com/salesforce/apollo/fireflies/View.java +++ b/fireflies/src/main/java/com/salesforce/apollo/fireflies/View.java @@ -1839,6 +1839,10 @@ boolean setNote(NoteWrapper next) { public class Service implements EntranceService, FFService, ServiceRouting { + public void enjoin(Join join, Digest from) { + viewManagement.enjoin(join, from); + } + /** * Asynchronously add a member to the next view */ diff --git a/fireflies/src/main/java/com/salesforce/apollo/fireflies/ViewManagement.java b/fireflies/src/main/java/com/salesforce/apollo/fireflies/ViewManagement.java index 5e18e4338..f070be7ec 100644 --- a/fireflies/src/main/java/com/salesforce/apollo/fireflies/ViewManagement.java +++ b/fireflies/src/main/java/com/salesforce/apollo/fireflies/ViewManagement.java @@ -19,6 +19,7 @@ import com.salesforce.apollo.fireflies.proto.Update.Builder; import com.salesforce.apollo.membership.Member; import com.salesforce.apollo.membership.ReservoirSampler; +import com.salesforce.apollo.ring.SliceIterator; import com.salesforce.apollo.stereotomy.identifier.SelfAddressingIdentifier; import com.salesforce.apollo.utils.Entropy; import com.salesforce.apollo.utils.Utils; @@ -123,6 +124,55 @@ Digest currentView() { return currentView.get(); } + void enjoin(Join join, Digest observer) { + final var joinView = Digest.from(join.getView()); + final var from = observer; + if (!joined()) { + log.trace("Not joined, ignored enjoin of view: {} from: {} on: {}", joinView, from, node.getId()); + throw new StatusRuntimeException(Status.FAILED_PRECONDITION.withDescription( + "Not joined, ignored join of view: %s from: %s on: %s".formatted(joinView, from, node.getId()))); + } + var note = new NoteWrapper(join.getNote(), digestAlgo); + if (!view.validate(note.getIdentifier())) { + log.debug("Ignored enjoin of view: {} from: {} invalid identifier on: {}", joinView, from, node.getId()); + throw new StatusRuntimeException(Status.INVALID_ARGUMENT.withDescription("Invalid identifier")); + } + view.stable(() -> { + var thisView = currentView(); + log.debug("Enjoin requested from: {} view: {} context: {} cardinality: {} on: {}", from, thisView, + context.getId(), cardinality(), node.getId()); + if (contains(from)) { + log.debug("Already a member: {} view: {} context: {} cardinality: {} on: {}", from, thisView, + context.getId(), cardinality(), node.getId()); + return; + } + if (!observers.contains(node.getId())) { + log.trace("Not observer, ignoring Join from: {} observers: {} on: {}", from, observers, node.getId()); + throw new StatusRuntimeException( + Status.FAILED_PRECONDITION.withDescription("Not observer, ignored join of view")); + } + if (!thisView.equals(joinView)) { + throw new StatusRuntimeException( + Status.OUT_OF_RANGE.withDescription("View: " + joinView + " does not match: " + thisView)); + } + if (!View.isValidMask(note.getMask(), context)) { + log.warn( + "Invalid enjoin mask: {} majority: {} from member: {} view: {} context: {} cardinality: {} on: {}", + note.getMask(), context.majority(), from, thisView, context.getId(), cardinality(), node.getId()); + } + if (pendingJoins.size() >= params.maxPending()) { + throw new StatusRuntimeException(Status.RESOURCE_EXHAUSTED.withDescription("No room at the inn")); + } + pendingJoins.computeIfAbsent(from, d -> seeds -> { + log.info("Gateway established for: {} (enjoined) view: {} context: {} cardinality: {} on: {}", from, + currentView(), context.getId(), cardinality(), node.getId()); + }); + joins.put(note.getId(), note); + log.debug("Member pending enjoin: {} view: {} context: {} on: {}", from, currentView(), context.getId(), + node.getId()); + }); + } + void gc(Participant member) { assert member != null; view.stable(() -> { @@ -286,9 +336,9 @@ void join(Join join, Digest from, StreamObserver responseObserver, Time return; } if (!observers.contains(node.getId())) { - log.warn("Join not observer! from: {} observers: {} on: {}", from, observers, node.getId()); - responseObserver.onNext(Gateway.getDefaultInstance()); - return; + log.trace("Not observer, ignoring Join from: {} observers: {} on: {}", from, observers, node.getId()); + responseObserver.onError(new StatusRuntimeException( + Status.FAILED_PRECONDITION.withDescription("Not observer, ignored join of view"))); } if (!thisView.equals(joinView)) { responseObserver.onError(new StatusRuntimeException( @@ -313,6 +363,10 @@ void join(Join join, Digest from, StreamObserver responseObserver, Time joins.put(note.getId(), note); log.debug("Member pending join: {} view: {} context: {} on: {}", from, currentView(), context.getId(), node.getId()); + var enjoining = new SliceIterator<>("Enjoining[%s:%s]".formatted(currentView(), from), node, + observers.stream().map(context::getActiveMember).toList(), view.comm); + enjoining.iterate(t -> t.enjoin(join), (_, _, _, _) -> true, () -> { + }, Duration.ofMillis(1)); }); } @@ -378,8 +432,8 @@ boolean joined() { */ void maybeViewChange() { if (context.size() == 1 && joins.size() < context.getRingCount() - 1) { - log.info("Cannot form cluster: {} with: {} members, required:4 on: {}", currentView(), - joins.size() + context.size(), node.getId()); + log.trace("Cannot form cluster: {} with: {} members, required > 3 on: {}", currentView(), + joins.size() + context.size(), node.getId()); view.scheduleViewChange(); return; } diff --git a/fireflies/src/main/java/com/salesforce/apollo/fireflies/comm/gossip/FfClient.java b/fireflies/src/main/java/com/salesforce/apollo/fireflies/comm/gossip/FfClient.java index fea51e129..f73b62ce3 100644 --- a/fireflies/src/main/java/com/salesforce/apollo/fireflies/comm/gossip/FfClient.java +++ b/fireflies/src/main/java/com/salesforce/apollo/fireflies/comm/gossip/FfClient.java @@ -10,10 +10,7 @@ import com.salesforce.apollo.archipelago.ManagedServerChannel; import com.salesforce.apollo.archipelago.ServerConnectionCache.CreateClientCommunications; import com.salesforce.apollo.fireflies.FireflyMetrics; -import com.salesforce.apollo.fireflies.proto.FirefliesGrpc; -import com.salesforce.apollo.fireflies.proto.Gossip; -import com.salesforce.apollo.fireflies.proto.SayWhat; -import com.salesforce.apollo.fireflies.proto.State; +import com.salesforce.apollo.fireflies.proto.*; import com.salesforce.apollo.membership.Member; /** @@ -42,6 +39,12 @@ public void close() { channel.release(); } + @Override + public Void enjoin(Join join) { + channel.wrap(FirefliesGrpc.newFutureStub(channel)).enjoin(join); + return null; + } + @Override public Member getMember() { return channel.getMember(); @@ -63,10 +66,6 @@ public Gossip gossip(SayWhat sw) { return result; } - public void release() { - close(); - } - @Override public String toString() { return String.format("->[%s]", channel.getMember()); diff --git a/fireflies/src/main/java/com/salesforce/apollo/fireflies/comm/gossip/FfServer.java b/fireflies/src/main/java/com/salesforce/apollo/fireflies/comm/gossip/FfServer.java index 3944a7b6b..8ad78e252 100644 --- a/fireflies/src/main/java/com/salesforce/apollo/fireflies/comm/gossip/FfServer.java +++ b/fireflies/src/main/java/com/salesforce/apollo/fireflies/comm/gossip/FfServer.java @@ -8,16 +8,16 @@ import com.codahale.metrics.Timer.Context; import com.google.protobuf.Empty; -import com.salesforce.apollo.fireflies.proto.FirefliesGrpc.FirefliesImplBase; -import com.salesforce.apollo.fireflies.proto.Gossip; -import com.salesforce.apollo.fireflies.proto.SayWhat; -import com.salesforce.apollo.fireflies.proto.State; import com.salesforce.apollo.archipelago.RoutableService; import com.salesforce.apollo.cryptography.Digest; import com.salesforce.apollo.fireflies.FireflyMetrics; import com.salesforce.apollo.fireflies.View.Service; +import com.salesforce.apollo.fireflies.proto.FirefliesGrpc.FirefliesImplBase; +import com.salesforce.apollo.fireflies.proto.Gossip; +import com.salesforce.apollo.fireflies.proto.Join; +import com.salesforce.apollo.fireflies.proto.SayWhat; +import com.salesforce.apollo.fireflies.proto.State; import com.salesforce.apollo.protocols.ClientIdentity; - import io.grpc.StatusRuntimeException; import io.grpc.stub.StreamObserver; @@ -36,6 +36,29 @@ public FfServer(ClientIdentity identity, RoutableService r, FireflyMetr this.router = r; } + @Override + public void enjoin(Join request, StreamObserver responseObserver) { + Context timer = metrics == null ? null : metrics.inboundEnjoinDuration().time(); + if (metrics != null) { + var serializedSize = request.getSerializedSize(); + metrics.inboundBandwidth().mark(serializedSize); + metrics.inboundGossip().update(serializedSize); + } + Digest from = identity.getFrom(); + if (from == null) { + responseObserver.onError(new IllegalStateException("Member has been removed")); + return; + } + router.evaluate(responseObserver, s -> { + s.enjoin(request, from); + responseObserver.onNext(Empty.getDefaultInstance()); + responseObserver.onCompleted(); + if (timer != null) { + timer.stop(); + } + }); + } + @Override public void gossip(SayWhat request, StreamObserver responseObserver) { Context timer = metrics == null ? null : metrics.inboundGossipDuration().time(); @@ -99,5 +122,4 @@ public void update(State request, StreamObserver responseObserver) { } }); } - } diff --git a/fireflies/src/main/java/com/salesforce/apollo/fireflies/comm/gossip/Fireflies.java b/fireflies/src/main/java/com/salesforce/apollo/fireflies/comm/gossip/Fireflies.java index 5fd6bc2fd..c71c48754 100644 --- a/fireflies/src/main/java/com/salesforce/apollo/fireflies/comm/gossip/Fireflies.java +++ b/fireflies/src/main/java/com/salesforce/apollo/fireflies/comm/gossip/Fireflies.java @@ -6,11 +6,12 @@ */ package com.salesforce.apollo.fireflies.comm.gossip; +import com.salesforce.apollo.archipelago.Link; +import com.salesforce.apollo.fireflies.View.Node; import com.salesforce.apollo.fireflies.proto.Gossip; +import com.salesforce.apollo.fireflies.proto.Join; import com.salesforce.apollo.fireflies.proto.SayWhat; import com.salesforce.apollo.fireflies.proto.State; -import com.salesforce.apollo.archipelago.Link; -import com.salesforce.apollo.fireflies.View.Node; import com.salesforce.apollo.membership.Member; import java.io.IOException; @@ -27,6 +28,11 @@ static Fireflies getLocalLoopback(Node node) { public void close() throws IOException { } + @Override + public Void enjoin(Join join) { + return null; + } + @Override public Member getMember() { return node; @@ -43,6 +49,8 @@ public void update(State state) { }; } + Void enjoin(Join join); + Gossip gossip(SayWhat sw); void update(State state); diff --git a/grpc/src/main/proto/fireflies.proto b/grpc/src/main/proto/fireflies.proto index 036713cf5..bc6657d10 100644 --- a/grpc/src/main/proto/fireflies.proto +++ b/grpc/src/main/proto/fireflies.proto @@ -15,6 +15,7 @@ package fireflies; service Fireflies { rpc gossip (SayWhat) returns (Gossip) {} rpc update (State) returns (google.protobuf.Empty) {} + rpc enjoin (Join) returns (google.protobuf.Empty) {} } message SayWhat { From 321724d5ce28684bf8837a12cbe9e6d4e6eb8eba Mon Sep 17 00:00:00 2001 From: Hellblazer Date: Sat, 22 Jun 2024 09:12:53 -0700 Subject: [PATCH 08/32] interim --- .../salesforce/apollo/fireflies/Binding.java | 55 ++++++++--- .../com/salesforce/apollo/fireflies/View.java | 64 +++++++------ .../apollo/fireflies/ViewManagement.java | 92 ++++++++++++------- .../apollo/fireflies/ChurnTest.java | 2 +- .../salesforce/apollo/fireflies/E2ETest.java | 22 +++-- fireflies/src/test/resources/logback-test.xml | 27 +----- .../apollo/context/DynamicContextImpl.java | 7 +- 7 files changed, 165 insertions(+), 104 deletions(-) diff --git a/fireflies/src/main/java/com/salesforce/apollo/fireflies/Binding.java b/fireflies/src/main/java/com/salesforce/apollo/fireflies/Binding.java index bd4ad421d..00965a98f 100644 --- a/fireflies/src/main/java/com/salesforce/apollo/fireflies/Binding.java +++ b/fireflies/src/main/java/com/salesforce/apollo/fireflies/Binding.java @@ -136,12 +136,14 @@ private boolean complete(CompletableFuture redirect, Optional gateway, - Optional futureSailor, HashMultiset trusts, + Optional futureSailor, HashMultiset trusts, Set initialSeedSet, Digest v, int majority) { if (futureSailor.isEmpty()) { + log.warn("No gateway returned from: {} on: {}", member.getId(), node.getId()); return true; } if (gateway.isDone()) { + log.warn("gateway is complete, ignoring from: {} on: {}", member.getId(), node.getId()); return false; } @@ -161,7 +163,7 @@ private boolean completeGateway(Participant member, CompletableFuture gat log.trace("Empty bootstrap trust in join returned from: {} on: {}", member.getId(), node.getId()); return true; } - trusts.add(g.getTrust()); + trusts.add(new Bootstrapping(g.getTrust())); initialSeedSet.addAll(g.getInitialSeedSetList()); log.trace("Initial seed set count: {} view: {} from: {} on: {}", g.getInitialSeedSetCount(), v, member.getId(), node.getId()); @@ -175,8 +177,14 @@ private boolean completeGateway(Participant member, CompletableFuture gat if (trust != null) { validate(trust, gateway, initialSeedSet); } else { - log.debug("Gateway received, trust count: {} majority: {} from: {} view: {} context: {} on: {}", - trusts.size(), majority, member.getId(), v, this.context.getId(), node.getId()); + log.debug("Gateway received, trust count: {} majority: {} from: {} trusts: {} view: {} context: {} on: {}", + trusts.size(), majority, member.getId(), v, trusts.entrySet() + .stream() + .sorted() + .map( + e -> "%s x %s".formatted(e.getElement().diadem, + e.getCount())) + .toList(), this.context.getId(), node.getId()); } return true; } @@ -255,7 +263,7 @@ private void join(Redirect redirect, Digest v, Duration duration) { var regate = new AtomicReference(); var retries = new AtomicInteger(); - HashMultiset trusts = HashMultiset.create(); + HashMultiset trusts = HashMultiset.create(); HashSet initialSeedSet = new HashSet<>(); final var cardinality = redirect.getCardinality(); @@ -302,7 +310,8 @@ private void join(Redirect redirect, Digest v, Duration duration) { return; } if (abandon.get() >= majority) { - log.debug("Abandoning Gateway view: {} reseeding on: {}", v, node.getId()); + log.debug("Abandoning Gateway view: {} abandons: {} majority: {} reseeding on: {}", v, + abandon.get(), majority, node.getId()); seeding(); } else { abandon.set(0); @@ -345,13 +354,37 @@ private NoteWrapper seedFor(Seed seed) { return new NoteWrapper(seedNote, digestAlgo); } - private void validate(BootstrapTrust trust, CompletableFuture gateway, Set initialSeedSet) { - final var hexBloom = new HexBloom(trust.getDiadem()); + private void validate(Bootstrapping trust, CompletableFuture gateway, Set initialSeedSet) { if (gateway.complete( - new Bound(hexBloom, trust.getSuccessorsList().stream().map(sn -> new NoteWrapper(sn, digestAlgo)).toList(), + new Bound(trust.crown, trust.successors.stream().map(sn -> new NoteWrapper(sn, digestAlgo)).toList(), initialSeedSet.stream().map(sn -> new NoteWrapper(sn, digestAlgo)).toList()))) { - log.info("Gateway acquired: {} context: {} on: {}", hexBloom.compactWrapped(), this.context.getId(), - node.getId()); + log.info("Gateway acquired: {} context: {} on: {}", trust.diadem, this.context.getId(), node.getId()); + } + } + + private record Bootstrapping(Digest diadem, HexBloom crown, Set successors) { + public Bootstrapping(BootstrapTrust trust) { + this(HexBloom.from(trust.getDiadem()), new HashSet<>(trust.getSuccessorsList())); + } + + public Bootstrapping(HexBloom crown, Set successors) { + this(crown.compact(), crown, new HashSet<>(successors)); + } + + @Override + public boolean equals(Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + + Bootstrapping that = (Bootstrapping) o; + return diadem.equals(that.diadem); + } + + @Override + public int hashCode() { + return diadem.hashCode(); } } diff --git a/fireflies/src/main/java/com/salesforce/apollo/fireflies/View.java b/fireflies/src/main/java/com/salesforce/apollo/fireflies/View.java index 1df306b37..47a7ef3b0 100644 --- a/fireflies/src/main/java/com/salesforce/apollo/fireflies/View.java +++ b/fireflies/src/main/java/com/salesforce/apollo/fireflies/View.java @@ -346,9 +346,18 @@ void finalizeViewChange() { viewChange(() -> { final var supermajority = context.getRingCount() * 3 / 4; final var majority = context.size() == 1 ? 1 : supermajority; - if (observations.size() < majority) { - log.trace("Do not have majority: {} required: {} observers: {} for: {} on: {}", observations.size(), - majority, viewManagement.observersList(), currentView(), node.getId()); + final var valid = observations.values() + .stream() + .filter(svc -> viewManagement.observers.contains( + Digest.from(svc.getChange().getObserver()))) + .toList(); + log.info("Finalize view change, observations: {} valid: {} observers: {} on: {}", + observations.values().stream().map(sv -> Digest.from(sv.getChange().getObserver())).toList(), + valid.size(), viewManagement.observersList(), node.getId()); + observations.clear(); + if (valid.size() < majority) { + log.info("Do not have majority: {} required: {} observers: {} for: {} on: {}", valid.size(), majority, + viewManagement.observersList(), currentView(), node.getId()); scheduleFinalizeViewChange(2); return; } @@ -356,20 +365,23 @@ void finalizeViewChange() { viewManagement.observersList(), currentView(), node.getId()); HashMultiset ballots = HashMultiset.create(); final var current = currentView(); - observations.values() - .stream() - .filter(vc -> current.equals(Digest.from(vc.getChange().getCurrent()))) - .forEach(vc -> { - final var leaving = new ArrayList<>( - vc.getChange().getLeavesList().stream().map(Digest::from).collect(Collectors.toSet())); - final var joining = new ArrayList<>( - vc.getChange().getJoinsList().stream().map(Digest::from).collect(Collectors.toSet())); - leaving.sort(Ordering.natural()); - joining.sort(Ordering.natural()); - ballots.add( - new Ballot(Digest.from(vc.getChange().getCurrent()), leaving, joining, digestAlgo)); - }); - observations.clear(); + valid.stream().filter(vc -> current.equals(Digest.from(vc.getChange().getCurrent()))).forEach(vc -> { + final var leaving = vc.getChange() + .getLeavesList() + .stream() + .map(Digest::from) + .distinct() + .collect(Collectors.toCollection(ArrayList::new)); + final var joining = vc.getChange() + .getJoinsList() + .stream() + .map(Digest::from) + .distinct() + .collect(Collectors.toCollection(ArrayList::new)); + leaving.sort(Ordering.natural()); + joining.sort(Ordering.natural()); + ballots.add(new Ballot(Digest.from(vc.getChange().getCurrent()), leaving, joining, digestAlgo)); + }); var max = ballots.entrySet() .stream() .max(Ordering.natural().onResultOf(Multiset.Entry::getCount)) @@ -382,7 +394,7 @@ void finalizeViewChange() { @SuppressWarnings("unchecked") final var reversed = Comparator.comparing(e -> ((Entry) e).getCount()).reversed(); log.info("View consensus failed: {}, required: {} cardinality: {} ballots: {} for: {} on: {}", - observations.size(), majority, viewManagement.cardinality(), + max == null ? 0 : max.getCount(), majority, viewManagement.cardinality(), ballots.entrySet().stream().sorted(reversed).toList(), currentView(), node.getId()); } @@ -845,13 +857,11 @@ private boolean add(SignedViewChange observation) { observation.getChange().getAttempt(), currentObservation.getChange().getAttempt(), inView, currentView(), observer, node.getId()); return false; - } else if (observation.getChange().getAttempt() < currentObservation.getChange().getAttempt()) { - return false; } } final var member = context.getActiveMember(observer); if (member == null) { - log.trace("Cannot validate view change: {} current: {} offline: {} on: {}", inView, currentView(), observer, + log.trace("Cannot validate view change: {} current: {} from: {} on: {}", inView, currentView(), observer, node.getId()); return false; } @@ -860,6 +870,8 @@ private boolean add(SignedViewChange observation) { if (!member.verify(signature, observation.getChange().toByteString())) { return null; } + log.trace("Observation: {} current: {} view change: {} from: {} on: {}", + observation.getChange().getAttempt(), inView, currentView(), observer, node.getId()); return observation; }) != null; } @@ -1216,7 +1228,7 @@ private NoteGossip.Builder processNotes(BloomFilter bff) { .filter(m -> !bff.contains(m.getNote().getHash())) .collect(new ReservoirSampler<>(params.maximumTxfr(), Entropy.bitsStream())) .stream() - .map(m -> m.getNote()) + .map(Participant::getNote) .forEach(n -> builder.addUpdates(n.getWrapped())); return builder; } @@ -1225,10 +1237,6 @@ private NoteGossip.Builder processNotes(BloomFilter bff) { * Process the inbound notes from the gossip. Reconcile the differences between the view's state and the digests of * the gossip. Update the reply with the list of digests the view requires, as well as proposed updates based on the * inbound digests that the view has more recent information - * - * @param from - * @param p - * @param bff */ private NoteGossip processNotes(Digest from, BloomFilter bff, double p) { NoteGossip.Builder builder = processNotes(bff); @@ -1868,13 +1876,13 @@ public void join(Join join, Digest from, StreamObserver responseObserve @Override public Gossip rumors(SayWhat request, Digest from) { if (!introduced.get()) { - log.trace("Not introduced; ring: {} from: {}, on: {}", request.getRing(), from, node.getId()); + // log.trace("Not introduced; ring: {} from: {}, on: {}", request.getRing(), from, node.getId()); throw new StatusRuntimeException(Status.FAILED_PRECONDITION.withDescription("Not introduced")); } return stable(() -> { final var ring = request.getRing(); if (!context.validRing(ring)) { - log.debug("invalid gossip ring: {} from: {} on: {}", ring, from, node.getId()); + // log.debug("invalid gossip ring: {} from: {} on: {}", ring, from, node.getId()); throw new StatusRuntimeException(Status.FAILED_PRECONDITION.withDescription("invalid ring")); } validate(from, request); diff --git a/fireflies/src/main/java/com/salesforce/apollo/fireflies/ViewManagement.java b/fireflies/src/main/java/com/salesforce/apollo/fireflies/ViewManagement.java index f070be7ec..3f2e50579 100644 --- a/fireflies/src/main/java/com/salesforce/apollo/fireflies/ViewManagement.java +++ b/fireflies/src/main/java/com/salesforce/apollo/fireflies/ViewManagement.java @@ -15,6 +15,7 @@ import com.salesforce.apollo.fireflies.Binding.Bound; import com.salesforce.apollo.fireflies.View.Node; import com.salesforce.apollo.fireflies.View.Participant; +import com.salesforce.apollo.fireflies.comm.gossip.Fireflies; import com.salesforce.apollo.fireflies.proto.*; import com.salesforce.apollo.fireflies.proto.Update.Builder; import com.salesforce.apollo.membership.Member; @@ -109,6 +110,7 @@ int cardinality() { void clear() { joins.clear(); + // context.clear(); resetBootstrapView(); } @@ -125,14 +127,18 @@ Digest currentView() { } void enjoin(Join join, Digest observer) { + if (!observers.contains(node.getId()) || !observers.contains(observer)) { + log.trace("Not observer, ignored enjoin from: {} on: {}", observer, node.getId()); + throw new StatusRuntimeException(Status.FAILED_PRECONDITION.withDescription("Not observer")); + } + var note = new NoteWrapper(join.getNote(), digestAlgo); + final var from = note.getId(); final var joinView = Digest.from(join.getView()); - final var from = observer; if (!joined()) { log.trace("Not joined, ignored enjoin of view: {} from: {} on: {}", joinView, from, node.getId()); throw new StatusRuntimeException(Status.FAILED_PRECONDITION.withDescription( "Not joined, ignored join of view: %s from: %s on: %s".formatted(joinView, from, node.getId()))); } - var note = new NoteWrapper(join.getNote(), digestAlgo); if (!view.validate(note.getIdentifier())) { log.debug("Ignored enjoin of view: {} from: {} invalid identifier on: {}", joinView, from, node.getId()); throw new StatusRuntimeException(Status.INVALID_ARGUMENT.withDescription("Invalid identifier")); @@ -155,21 +161,9 @@ void enjoin(Join join, Digest observer) { throw new StatusRuntimeException( Status.OUT_OF_RANGE.withDescription("View: " + joinView + " does not match: " + thisView)); } - if (!View.isValidMask(note.getMask(), context)) { - log.warn( - "Invalid enjoin mask: {} majority: {} from member: {} view: {} context: {} cardinality: {} on: {}", - note.getMask(), context.majority(), from, thisView, context.getId(), cardinality(), node.getId()); - } - if (pendingJoins.size() >= params.maxPending()) { - throw new StatusRuntimeException(Status.RESOURCE_EXHAUSTED.withDescription("No room at the inn")); - } - pendingJoins.computeIfAbsent(from, d -> seeds -> { - log.info("Gateway established for: {} (enjoined) view: {} context: {} cardinality: {} on: {}", from, - currentView(), context.getId(), cardinality(), node.getId()); - }); - joins.put(note.getId(), note); - log.debug("Member pending enjoin: {} view: {} context: {} on: {}", from, currentView(), context.getId(), - node.getId()); + joins.putIfAbsent(note.getId(), note); + log.debug("Member pending enjoin: {} via: {} view: {} context: {} on: {}", from, observer, currentView(), + context.getId(), node.getId()); }); } @@ -233,10 +227,10 @@ void install(Ballot ballot) { .filter(java.util.Objects::nonNull) .toList(); + view.reset(); setDiadem( HexBloom.construct(context.memberCount(), context.allMembers().map(Participant::getId), view.bootstrapView(), params.crowns())); - view.reset(); // complete all pending joins pending.forEach(r -> { try { @@ -431,6 +425,9 @@ boolean joined() { * start a view change if there are any offline members or joining members */ void maybeViewChange() { + if (!joined()) { + return; + } if (context.size() == 1 && joins.size() < context.getRingCount() - 1) { log.trace("Cannot form cluster: {} with: {} members, required > 3 on: {}", currentView(), joins.size() + context.size(), node.getId()); @@ -439,8 +436,8 @@ void maybeViewChange() { } if ((context.offlineCount() > 0 || !joins.isEmpty())) { if (isObserver()) { - log.trace("Initiating view change: {} (non observer) joins: {} leaves: {} on: {}", currentView(), - joins.size(), view.streamShunned().count(), node.getId()); + log.info("Initiating view change: {} (observer) joins: {} leaves: {} on: {}", currentView(), + joins.size(), view.streamShunned().count(), node.getId()); initiateViewChange(); } else { // Use pending rebuttals as a proxy for stability @@ -448,12 +445,14 @@ void maybeViewChange() { log.debug("Pending rebuttals in view: {} on: {}", currentView(), node.getId()); view.scheduleViewChange(2); // 2 TTL round2 to check again } else { + log.info("Initiating view change: {} (non observer) joins: {} leaves: {} on: {}", currentView(), + joins.size(), view.streamShunned().count(), node.getId()); view.scheduleFinalizeViewChange(); } } } else { - log.trace("No view change: {} joins: {} leaves: {} on: {}", currentView(), joins.size(), - view.streamShunned().count(), node.getId()); + // log.trace("No view change: {} joins: {} leaves: {} on: {}", currentView(), joins.size(), + // view.streamShunned().count(), node.getId()); view.scheduleViewChange(); } } @@ -466,6 +465,29 @@ List observersList() { return observers().stream().toList(); } + void populate(List sample) { + var populate = new SliceIterator("Populate: " + context.getId(), node, sample, view.comm); + var repopulate = new AtomicReference(); + var scheduler = Executors.newScheduledThreadPool(1, Thread.ofVirtual().factory()); + repopulate.set(() -> populate.iterate((link) -> view.gossip(link, 0), (futureSailor, _, link, m) -> { + futureSailor.ifPresent(g -> { + if (!g.getRedirect().equals(SignedNote.getDefaultInstance())) { + final Participant member = (Participant) link.getMember(); + view.stable(() -> view.redirect(member, g, 0)); + } else { + view.stable(() -> view.processUpdates(g)); + } + }); + return !joined(); + }, () -> { + if (!joined()) { + scheduler.schedule(() -> Thread.ofVirtual().start(Utils.wrapped(repopulate.get(), log)), + params.populateDuration().toNanos(), TimeUnit.NANOSECONDS); + } + }, params.populateDuration())); + repopulate.get().run(); + } + JoinGossip.Builder processJoins(BloomFilter bff) { JoinGossip.Builder builder = JoinGossip.newBuilder(); @@ -568,11 +590,11 @@ private void initiateViewChange() { } view.scheduleFinalizeViewChange(); if (!isObserver(node.getId())) { - log.trace("Initiating view change: {} (non observer) on: {}", currentView(), node.getId()); + log.warn("Initiating view change: {} (non observer) on: {}", currentView(), node.getId()); return; } - log.trace("Initiating view change vote: {} joins: {} leaves: {} on: {}", currentView(), joins.size(), - view.streamShunned().count(), node.getId()); + log.warn("Initiating view change vote: {} joins: {} leaves: {} observers: {} on: {}", currentView(), + joins.size(), view.streamShunned().count(), observersList(), node.getId()); final var builder = ViewChange.newBuilder() .setObserver(node.getId().toDigeste()) .setCurrent(currentView().toDigeste()) @@ -587,8 +609,8 @@ private void initiateViewChange() { .setSignature(signature.toSig()) .build(); view.initiate(viewChange); - log.debug("View change vote: {} joins: {} leaves: {} on: {}", currentView(), change.getJoinsCount(), - change.getLeavesCount(), node.getId()); + log.warn("View change vote: {} joins: {} leaves: {} on: {}", currentView(), change.getJoinsCount(), + change.getLeavesCount(), node.getId()); }); } @@ -603,6 +625,7 @@ private void joined(Collection seedSet, Digest from, StreamObserver< Timer.Context timer) { var unique = new HashSet<>(seedSet); final var initialSeeds = new ArrayList<>(seedSet); + initialSeeds.add(node.getSignedNote()); final var successors = new HashSet(); context.successors(from, context::isActive).forEach(p -> { @@ -620,11 +643,13 @@ private void joined(Collection seedSet, Digest from, StreamObserver< .build(); log.info("Gateway initial seeding: {} successors: {} for: {} on: {}", gateway.getInitialSeedSetCount(), successors.size(), from, node.getId()); - responseObserver.onNext(gateway); try { + responseObserver.onNext(gateway); responseObserver.onCompleted(); } catch (RejectedExecutionException e) { - log.trace("in shutdown on: {}", node.getId()); + log.trace("In shutdown on: {}", node.getId()); + } catch (Throwable t) { + log.error("Error responding to join: {} on: {}", t, node.getId()); } if (timer != null) { var serializedSize = gateway.getSerializedSize(); @@ -652,12 +677,15 @@ private void resetObservers() { } private void setDiadem(final HexBloom hex) { + assert hex.getCardinality() <= 0 + || context.size() == hex.getCardinality() : "Context: %s does not equal Hex: %s".formatted(context.size(), + hex.getCardinality()); diadem.set(hex); currentView.set(diadem.get().compactWrapped()); resetObservers(); - log.trace("View: {} set diadem: {} observers: {} view: {} context: {} size: {} on: {}", context.getId(), - diadem.get().compactWrapped(), observers.stream().toList(), currentView(), context.getId(), - context.size(), node.getId()); + log.trace("View: {} set diadem: {} cardinality: {} observers: {} view: {} context: {} size: {} on: {}", + context.getId(), diadem.get().compactWrapped(), diadem.get().getCardinality(), + observers.stream().toList(), currentView(), context.getId(), context.size(), node.getId()); } record Ballot(Digest view, List leaving, List joining, int hash) { diff --git a/fireflies/src/test/java/com/salesforce/apollo/fireflies/ChurnTest.java b/fireflies/src/test/java/com/salesforce/apollo/fireflies/ChurnTest.java index 331da49b3..d39f26c77 100644 --- a/fireflies/src/test/java/com/salesforce/apollo/fireflies/ChurnTest.java +++ b/fireflies/src/test/java/com/salesforce/apollo/fireflies/ChurnTest.java @@ -271,7 +271,7 @@ public void churn() throws Exception { private void initialize() { executor = UnsafeExecutors.newVirtualThreadPerTaskExecutor(); executor2 = UnsafeExecutors.newVirtualThreadPerTaskExecutor(); - var parameters = Parameters.newBuilder().setMaximumTxfr(20).build(); + var parameters = Parameters.newBuilder().setFpr(0.0000125).setMaximumTxfr(20).build(); registry = new MetricRegistry(); node0Registry = new MetricRegistry(); diff --git a/fireflies/src/test/java/com/salesforce/apollo/fireflies/E2ETest.java b/fireflies/src/test/java/com/salesforce/apollo/fireflies/E2ETest.java index 1300fe554..6b4121346 100644 --- a/fireflies/src/test/java/com/salesforce/apollo/fireflies/E2ETest.java +++ b/fireflies/src/test/java/com/salesforce/apollo/fireflies/E2ETest.java @@ -8,6 +8,7 @@ import com.codahale.metrics.ConsoleReporter; import com.codahale.metrics.MetricRegistry; +import com.google.common.collect.Sets; import com.salesforce.apollo.archipelago.*; import com.salesforce.apollo.context.Context; import com.salesforce.apollo.context.DynamicContext; @@ -121,7 +122,13 @@ public void smokin() throws Exception { var failed = bootstrappers.stream() .filter(e -> e.getContext().activeCount() != bootstrappers.size()) .map( - v -> String.format("%s : %s ", v.getNode().getId(), v.getContext().activeCount())) + v -> String.format("%s : %s : %s", v.getNode().getId(), v.getContext().activeCount(), + Sets.difference(members.keySet(), new HashSet( + v.getContext() + .activeMembers() + .stream() + .map(Participant::getId) + .toList())).stream().toList())) .toList(); assertTrue(success, " expected: " + bootstrappers.size() + " failed: " + failed.size() + " views: " + failed); @@ -132,11 +139,14 @@ public void smokin() throws Exception { success = countdown.get().await(largeTests ? 2400 : 30, TimeUnit.SECONDS); // Test that all views are up - failed = views.stream().filter(e -> e.getContext().activeCount() != CARDINALITY).map(v -> { - Context participantContext = v.getContext(); - return String.format("%s : %s : %s ", v.getNode().getId(), v.getContext().activeCount(), - participantContext.size()); - }).toList(); + failed = views.stream() + .filter(e -> e.getContext().activeCount() != CARDINALITY) + .map(v -> String.format("%s : %s : %s", v.getNode().getId(), v.getContext().activeCount(), + Sets.difference(members.keySet(), new HashSet( + v.getContext().activeMembers().stream().map(Participant::getId).toList())) + .stream() + .toList())) + .toList(); assertTrue(success, "Views did not start, expected: " + views.size() + " failed: " + failed.size() + " views: " + failed); diff --git a/fireflies/src/test/resources/logback-test.xml b/fireflies/src/test/resources/logback-test.xml index f86c9c940..b4afb147d 100644 --- a/fireflies/src/test/resources/logback-test.xml +++ b/fireflies/src/test/resources/logback-test.xml @@ -9,12 +9,7 @@ %d{mm:ss.SSS} %logger{0} - %msg%n - - - - - - + @@ -22,31 +17,19 @@ - - - - - - - - - - - - - + - + - + - + diff --git a/memberships/src/main/java/com/salesforce/apollo/context/DynamicContextImpl.java b/memberships/src/main/java/com/salesforce/apollo/context/DynamicContextImpl.java index 94cd7db62..18a22e1d1 100644 --- a/memberships/src/main/java/com/salesforce/apollo/context/DynamicContextImpl.java +++ b/memberships/src/main/java/com/salesforce/apollo/context/DynamicContextImpl.java @@ -75,7 +75,7 @@ public boolean activate(T m) { try { l.active(m); } catch (Throwable e) { - log.error("error recoving member in listener: " + l, e); + log.error("error recovering member in listener: " + l, e); } }); return true; @@ -782,7 +782,6 @@ private Tracked tracking(T m) { * @author hal.hildebrand **/ public static class Tracked { - private static final Logger log = LoggerFactory.getLogger(Tracked.class); final AtomicBoolean active = new AtomicBoolean(false); final M member; @@ -823,7 +822,7 @@ public boolean offline() { @Override public String toString() { - return String.format("%s:%s %s", member, active.get(), Arrays.asList(hashes)); + return String.format("%s:%s %s", member.getId(), active.get(), Arrays.asList(hashes)); } void rebalance(int ringCount, Context context) { @@ -1024,7 +1023,7 @@ public Digest hash(T m) { } public T insert(T m) { - LoggerFactory.getLogger(getClass()).trace("Adding: {} to ring: {}", m.getId(), index); + log.trace("Adding: {} to ring: {}", m.getId(), index); return ring.put(hash(m), m); } From 2ca8e1c0f0ddc9c6e96ac920e2748b555f616fc3 Mon Sep 17 00:00:00 2001 From: Hellblazer Date: Sat, 22 Jun 2024 15:17:19 -0700 Subject: [PATCH 09/32] fixins. Reimplement ReservoirSampler. 5x5 --- .../salesforce/apollo/fireflies/Binding.java | 200 ++++++++++++------ .../com/salesforce/apollo/fireflies/View.java | 25 ++- .../apollo/fireflies/ViewManagement.java | 4 +- .../fireflies/comm/entrance/Entrance.java | 5 +- .../comm/entrance/EntranceClient.java | 31 ++- .../apollo/fireflies/ChurnTest.java | 2 +- fireflies/src/test/resources/logback-test.xml | 27 ++- .../apollo/context/DelegatedContext.java | 3 +- .../apollo/context/DynamicContextImpl.java | 6 +- .../apollo/context/StaticContext.java | 9 +- .../apollo/membership/ReservoirSampler.java | 62 +++--- 11 files changed, 238 insertions(+), 136 deletions(-) diff --git a/fireflies/src/main/java/com/salesforce/apollo/fireflies/Binding.java b/fireflies/src/main/java/com/salesforce/apollo/fireflies/Binding.java index 00965a98f..2bbe4fe53 100644 --- a/fireflies/src/main/java/com/salesforce/apollo/fireflies/Binding.java +++ b/fireflies/src/main/java/com/salesforce/apollo/fireflies/Binding.java @@ -9,6 +9,7 @@ import com.codahale.metrics.Timer; import com.google.common.collect.HashMultiset; import com.google.common.collect.Multiset; +import com.google.common.util.concurrent.ListenableFuture; import com.google.protobuf.ByteString; import com.salesforce.apollo.archipelago.RouterImpl.CommonCommunications; import com.salesforce.apollo.context.Context; @@ -35,9 +36,7 @@ import java.time.Duration; import java.util.*; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; +import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; import java.util.function.BiConsumer; @@ -75,6 +74,12 @@ public Binding(View view, List seeds, Duration duration, DynamicContext complete, AtomicInteger remaining) { + if (remaining.decrementAndGet() <= 0) { + complete.complete(false); + } + } + void seeding() { if (seeds.isEmpty()) {// This node is the bootstrap seed bootstrap(); @@ -135,33 +140,40 @@ private boolean complete(CompletableFuture redirect, Optional gateway, - Optional futureSailor, HashMultiset trusts, - Set initialSeedSet, Digest v, int majority) { - if (futureSailor.isEmpty()) { - log.warn("No gateway returned from: {} on: {}", member.getId(), node.getId()); - return true; + private void complete(Member member, CompletableFuture gateway, HashMultiset trusts, + Set initialSeedSet, Digest v, int majority, CompletableFuture complete, + AtomicInteger remaining, ListenableFuture futureSailor) { + if (complete.isDone()) { + return; } - if (gateway.isDone()) { - log.warn("gateway is complete, ignoring from: {} on: {}", member.getId(), node.getId()); - return false; + Gateway g = null; + try { + g = futureSailor.get(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } catch (ExecutionException e) { + log.warn("Error retrieving Gateway from: {} on: {}", member.getId(), node.getId(), e.getCause()); + dec(complete, remaining); + return; } - Gateway g = futureSailor.get(); - if (g.equals(Gateway.getDefaultInstance())) { - return true; + log.warn("Empty gateway returned from: {} on: {}", member.getId(), node.getId()); + dec(complete, remaining); + return; } if (g.getInitialSeedSetCount() == 0) { log.warn("No seeds in gateway returned from: {} on: {}", member.getId(), node.getId()); - return true; + dec(complete, remaining); + return; } if (g.getTrust().equals(BootstrapTrust.getDefaultInstance()) || g.getTrust() .getDiadem() .equals(HexBloome.getDefaultInstance())) { log.trace("Empty bootstrap trust in join returned from: {} on: {}", member.getId(), node.getId()); - return true; + dec(complete, remaining); + return; } trusts.add(new Bootstrapping(g.getTrust())); initialSeedSet.addAll(g.getInitialSeedSetList()); @@ -175,7 +187,13 @@ private boolean completeGateway(Participant member, CompletableFuture gat .findFirst() .orElse(null); if (trust != null) { - validate(trust, gateway, initialSeedSet); + var bound = new Bound(trust.crown, + trust.successors.stream().map(sn -> new NoteWrapper(sn, digestAlgo)).toList(), + initialSeedSet.stream().map(sn -> new NoteWrapper(sn, digestAlgo)).toList()); + if (gateway.complete(bound)) { + log.info("Gateway acquired: {} context: {} on: {}", trust.diadem, this.context.getId(), node.getId()); + } + complete.complete(true); } else { log.debug("Gateway received, trust count: {} majority: {} from: {} trusts: {} view: {} context: {} on: {}", trusts.size(), majority, member.getId(), v, trusts.entrySet() @@ -185,8 +203,8 @@ private boolean completeGateway(Participant member, CompletableFuture gat e -> "%s x %s".formatted(e.getElement().diadem, e.getCount())) .toList(), this.context.getId(), node.getId()); + dec(complete, remaining); } - return true; } private void gatewaySRE(Digest v, Entrance link, StatusRuntimeException sre, AtomicInteger abandon) { @@ -216,6 +234,31 @@ private void gatewaySRE(Digest v, Entrance link, StatusRuntimeException sre, Ato } } + private boolean join(Member member, CompletableFuture gateway, Optional> fs, + HashMultiset trusts, Set initialSeedSet, Digest v, int majority, + CompletableFuture complete, AtomicInteger remaining) { + if (complete.isDone()) { + log.trace("join round already completed for: {} on: {}", member.getId(), node.getId()); + return false; + } + if (fs.isEmpty()) { + log.warn("No gateway returned from: {} on: {}", member.getId(), node.getId()); + dec(complete, remaining); + return true; + } + if (gateway.isDone()) { + log.warn("gateway is complete, ignoring from: {} on: {}", member.getId(), node.getId()); + complete.complete(true); + return false; + } + var futureSailor = fs.get(); + futureSailor.addListener( + () -> complete(member, gateway, trusts, initialSeedSet, v, majority, complete, remaining, futureSailor), + r -> Thread.ofVirtual().start(r)); + + return true; + } + private Join join(Digest v) { return Join.newBuilder().setView(v.toDigeste()).setNote(node.getNote().getWrapped()).build(); } @@ -276,63 +319,88 @@ private void join(Redirect redirect, Digest v, Duration duration) { final var redirecting = new SliceIterator<>("Gateways", node, sample, approaches); var majority = redirect.getBootstrap() ? 1 : Context.minimalQuorum(redirect.getRings(), this.context.getBias()); final var join = join(v); - final var abandon = new AtomicInteger(); var scheduler = Executors.newScheduledThreadPool(1, Thread.ofVirtual().factory()); regate.set(() -> { + log.info("Round: {} formally joining view: {} on: {}", retries.get(), v, node.getId()); if (!view.started.get()) { return; } - redirecting.iterate((link) -> { - if (gateway.isDone() || !view.started.get()) { - return null; - } - log.debug("Joining: {} contacting: {} on: {}", v, link.getMember().getId(), node.getId()); - try { - var g = link.join(join, params.seedingTimeout()); - if (g == null || g.equals(Gateway.getDefaultInstance())) { - log.debug("Gateway view: {} empty from: {} on: {}", v, link.getMember().getId(), node.getId()); - abandon.incrementAndGet(); - return null; - } - return g; - } catch (StatusRuntimeException sre) { - gatewaySRE(v, link, sre, abandon); - return null; - } catch (Throwable t) { - log.info("Gateway view: {} error: {} from: {} on: {}", v, t, link.getMember().getId(), - node.getId()); - abandon.incrementAndGet(); - return null; + var complete = new CompletableFuture(); + final var abandon = new AtomicInteger(); + complete.whenComplete((success, error) -> { + if (error != null) { + log.info("Failed Join on: {}", node.getId(), error); + return; } - }, (futureSailor, _, _, member) -> completeGateway((Participant) member, gateway, futureSailor, trusts, - initialSeedSet, v, majority), () -> { - if (!view.started.get() || gateway.isDone()) { + if (success) { return; } - if (abandon.get() >= majority) { - log.debug("Abandoning Gateway view: {} abandons: {} majority: {} reseeding on: {}", v, - abandon.get(), majority, node.getId()); - seeding(); + log.info("Join unsuccessful, abandoned: {} trusts: {} on: {}", abandon.get(), trusts.entrySet() + .stream() + .sorted() + .map( + e -> "%s x %s".formatted( + e.getElement().diadem, + e.getCount())) + .toList(), + node.getId()); + abandon.set(0); + if (retries.get() < params.joinRetries()) { + log.info("Failed to join view: {} retry: {} out of: {} on: {}", v, retries.incrementAndGet(), + params.joinRetries(), node.getId()); + trusts.clear(); + initialSeedSet.clear(); + scheduler.schedule(() -> Thread.ofVirtual().start(Utils.wrapped(regate.get(), log)), + Entropy.nextBitsStreamLong(params.retryDelay().toNanos()), TimeUnit.NANOSECONDS); } else { - abandon.set(0); - if (retries.get() < params.joinRetries()) { - log.info("Failed to join view: {} retry: {} out of: {} on: {}", v, retries.incrementAndGet(), - params.joinRetries(), node.getId()); - trusts.clear(); - initialSeedSet.clear(); - scheduler.schedule(() -> Thread.ofVirtual().start(Utils.wrapped(regate.get(), log)), - Entropy.nextBitsStreamLong(params.retryDelay().toNanos()), - TimeUnit.NANOSECONDS); - } else { - log.error("Failed to join view: {} cannot obtain majority Gateway on: {}", view, node.getId()); - view.stop(); - } + log.error("Failed to join view: {} cannot obtain majority Gateway on: {}", view, node.getId()); + view.stop(); } - }, params.retryDelay()); + }); + var remaining = new AtomicInteger(sample.size()); + redirecting.iterate((link) -> join(v, link, gateway, join, abandon, complete), + (futureSailor, _, _, member) -> join(member, gateway, futureSailor, trusts, + initialSeedSet, v, majority, complete, remaining), + () -> { + if (!view.started.get() || gateway.isDone()) { + return; + } + if (abandon.get() >= majority) { + log.debug( + "Abandoning Gateway view: {} abandons: {} majority: {} reseeding on: {}", v, + abandon.get(), majority, node.getId()); + complete.completeExceptionally(new TimeoutException("Failed Join")); + seeding(); + } + }, params.retryDelay()); }); regate.get().run(); } + private ListenableFuture join(Digest v, Entrance link, CompletableFuture gateway, Join join, + AtomicInteger abandon, CompletableFuture complete) { + if (!view.started.get() || complete.isDone() || gateway.isDone()) { + return null; + } + log.debug("Joining: {} contacting: {} on: {}", v, link.getMember().getId(), node.getId()); + try { + var g = link.join(join, params.seedingTimeout()); + if (g == null || g.equals(Gateway.getDefaultInstance())) { + log.debug("Gateway view: {} empty from: {} on: {}", v, link.getMember().getId(), node.getId()); + abandon.incrementAndGet(); + return null; + } + return g; + } catch (StatusRuntimeException sre) { + gatewaySRE(v, link, sre, abandon); + return null; + } catch (Throwable t) { + log.info("Gateway view: {} error: {} from: {} on: {}", v, t, link.getMember().getId(), node.getId()); + abandon.incrementAndGet(); + return null; + } + } + private Registration registration() { return Registration.newBuilder() .setView(view.currentView().toDigeste()) @@ -354,14 +422,6 @@ private NoteWrapper seedFor(Seed seed) { return new NoteWrapper(seedNote, digestAlgo); } - private void validate(Bootstrapping trust, CompletableFuture gateway, Set initialSeedSet) { - if (gateway.complete( - new Bound(trust.crown, trust.successors.stream().map(sn -> new NoteWrapper(sn, digestAlgo)).toList(), - initialSeedSet.stream().map(sn -> new NoteWrapper(sn, digestAlgo)).toList()))) { - log.info("Gateway acquired: {} context: {} on: {}", trust.diadem, this.context.getId(), node.getId()); - } - } - private record Bootstrapping(Digest diadem, HexBloom crown, Set successors) { public Bootstrapping(BootstrapTrust trust) { this(HexBloom.from(trust.getDiadem()), new HashSet<>(trust.getSuccessorsList())); diff --git a/fireflies/src/main/java/com/salesforce/apollo/fireflies/View.java b/fireflies/src/main/java/com/salesforce/apollo/fireflies/View.java index 47a7ef3b0..7f851207f 100644 --- a/fireflies/src/main/java/com/salesforce/apollo/fireflies/View.java +++ b/fireflies/src/main/java/com/salesforce/apollo/fireflies/View.java @@ -617,7 +617,6 @@ void viewChange(Runnable r) { * * @param ring - the index of the gossip ring the gossip is originating from in this view * @param link - the outbound communications to the paired member - * @param ring */ protected Gossip gossip(Fireflies link, int ring) { tick(); @@ -656,8 +655,8 @@ protected Gossip gossip(Fireflies link, int ring) { node.getId()); break; case UNAVAILABLE: - log.trace("Communication cancelled for gossip view: {} from: {} on: {}", currentView(), p.getId(), - node.getId(), sre); + log.trace("Communication unavailable for gossip view: {} from: {} on: {}", currentView(), p.getId(), + node.getId()); accuse(p, ring, sre); break; default: @@ -1017,8 +1016,8 @@ private void gc(Participant member) { * @return the bloom filter containing the digests of known accusations */ private BloomFilter getAccusationsBff(long seed, double p) { - BloomFilter bff = new BloomFilter.DigestBloomFilter(seed, Math.max(params.minimumBiffCardinality(), - context.cardinality() * 2), p); + var n = Math.max(params.minimumBiffCardinality(), context.cardinality()); + BloomFilter bff = new BloomFilter.DigestBloomFilter(seed, n, 1.0 / (double) n); context.allMembers() .flatMap(Participant::getAccusations) .filter(Objects::nonNull) @@ -1033,9 +1032,9 @@ private BloomFilter getAccusationsBff(long seed, double p) { * @return the bloom filter containing the digests of known notes */ private BloomFilter getNotesBff(long seed, double p) { - BloomFilter bff = new BloomFilter.DigestBloomFilter(seed, Math.max(params.minimumBiffCardinality(), - context.cardinality() * 2), p); - context.allMembers().map(m -> m.getNote()).filter(e -> e != null).forEach(n -> bff.add(n.getHash())); + var n = Math.max(params.minimumBiffCardinality(), context.cardinality()); + BloomFilter bff = new BloomFilter.DigestBloomFilter(seed, n, 1.0 / (double) n); + context.allMembers().map(m -> m.getNote()).filter(e -> e != null).forEach(note -> bff.add(note.getHash())); return bff; } @@ -1045,8 +1044,8 @@ private BloomFilter getNotesBff(long seed, double p) { * @return the bloom filter containing the digests of known observations */ private BloomFilter getObservationsBff(long seed, double p) { - BloomFilter bff = new BloomFilter.DigestBloomFilter(seed, Math.max(params.minimumBiffCardinality(), - context.cardinality() * 2), p); + var n = Math.max(params.minimumBiffCardinality(), observations.size()); + BloomFilter bff = new BloomFilter.DigestBloomFilter(seed, n, 1.0 / (double) n); observations.keySet().stream().collect(Utils.toShuffledList()).forEach(bff::add); return bff; } @@ -1226,8 +1225,9 @@ private NoteGossip.Builder processNotes(BloomFilter bff) { .filter(m -> current.equals(m.getNote().currentView())) .filter(m -> !shunned.contains(m.getId())) .filter(m -> !bff.contains(m.getNote().getHash())) - .collect(new ReservoirSampler<>(params.maximumTxfr(), Entropy.bitsStream())) + .collect(new ReservoirSampler<>(params.maximumTxfr())) .stream() + .filter(sn -> sn != null) .map(Participant::getNote) .forEach(n -> builder.addUpdates(n.getWrapped())); return builder; @@ -1351,9 +1351,8 @@ private Update updatesForDigests(Gossip gossip) { .filter(m -> m.getNote() != null) .filter(m -> current.equals(m.getNote().currentView())) .filter(m -> !notesBff.contains(m.getNote().getHash())) - .collect(new ReservoirSampler<>(params.maximumTxfr(), Entropy.bitsStream())) - .stream() .map(m -> m.getNote().getWrapped()) + .limit(params.maximumTxfr()) .forEach(builder::addNotes); } diff --git a/fireflies/src/main/java/com/salesforce/apollo/fireflies/ViewManagement.java b/fireflies/src/main/java/com/salesforce/apollo/fireflies/ViewManagement.java index 3f2e50579..816bd22e9 100644 --- a/fireflies/src/main/java/com/salesforce/apollo/fireflies/ViewManagement.java +++ b/fireflies/src/main/java/com/salesforce/apollo/fireflies/ViewManagement.java @@ -19,7 +19,6 @@ import com.salesforce.apollo.fireflies.proto.*; import com.salesforce.apollo.fireflies.proto.Update.Builder; import com.salesforce.apollo.membership.Member; -import com.salesforce.apollo.membership.ReservoirSampler; import com.salesforce.apollo.ring.SliceIterator; import com.salesforce.apollo.stereotomy.identifier.SelfAddressingIdentifier; import com.salesforce.apollo.utils.Entropy; @@ -207,6 +206,7 @@ void install(Ballot ballot) { final var seedSet = context.sample(params.maximumTxfr(), Entropy.bitsStream(), node.getId()) .stream() + .filter(sn -> sn != null) .map(p -> p.note.getWrapped()) .collect(Collectors.toSet()); @@ -413,7 +413,6 @@ void joinUpdatesFor(BloomFilter joinBff, Builder builder) { joins.entrySet() .stream() .filter(e -> !joinBff.contains(e.getKey())) - .collect(new ReservoirSampler<>(params.maximumTxfr(), Entropy.bitsStream())) .forEach(e -> builder.addJoins(e.getValue().getWrapped())); } @@ -496,7 +495,6 @@ JoinGossip.Builder processJoins(BloomFilter bff) { .stream() .filter(m -> !bff.contains(m.getKey())) .map(Map.Entry::getValue) - .collect(new ReservoirSampler<>(params.maximumTxfr(), Entropy.bitsStream())) .forEach(n -> builder.addUpdates(n.getWrapped())); return builder; } diff --git a/fireflies/src/main/java/com/salesforce/apollo/fireflies/comm/entrance/Entrance.java b/fireflies/src/main/java/com/salesforce/apollo/fireflies/comm/entrance/Entrance.java index 98b63d053..6459a0491 100644 --- a/fireflies/src/main/java/com/salesforce/apollo/fireflies/comm/entrance/Entrance.java +++ b/fireflies/src/main/java/com/salesforce/apollo/fireflies/comm/entrance/Entrance.java @@ -6,6 +6,7 @@ */ package com.salesforce.apollo.fireflies.comm.entrance; +import com.google.common.util.concurrent.ListenableFuture; import com.salesforce.apollo.archipelago.Link; import com.salesforce.apollo.fireflies.View.Node; import com.salesforce.apollo.fireflies.proto.Gateway; @@ -35,7 +36,7 @@ public Member getMember() { } @Override - public Gateway join(Join join, Duration timeout) { + public ListenableFuture join(Join join, Duration timeout) { return null; } @@ -46,7 +47,7 @@ public Redirect seed(Registration registration) { }; } - Gateway join(Join join, Duration timeout); + ListenableFuture join(Join join, Duration timeout); Redirect seed(Registration registration); } diff --git a/fireflies/src/main/java/com/salesforce/apollo/fireflies/comm/entrance/EntranceClient.java b/fireflies/src/main/java/com/salesforce/apollo/fireflies/comm/entrance/EntranceClient.java index 1a1256437..c3a5c1d85 100644 --- a/fireflies/src/main/java/com/salesforce/apollo/fireflies/comm/entrance/EntranceClient.java +++ b/fireflies/src/main/java/com/salesforce/apollo/fireflies/comm/entrance/EntranceClient.java @@ -6,6 +6,7 @@ */ package com.salesforce.apollo.fireflies.comm.entrance; +import com.google.common.util.concurrent.ListenableFuture; import com.salesforce.apollo.archipelago.ManagedServerChannel; import com.salesforce.apollo.archipelago.ServerConnectionCache.CreateClientCommunications; import com.salesforce.apollo.fireflies.FireflyMetrics; @@ -13,6 +14,7 @@ import com.salesforce.apollo.membership.Member; import java.time.Duration; +import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; /** @@ -23,10 +25,12 @@ public class EntranceClient implements Entrance { private final ManagedServerChannel channel; private final EntranceGrpc.EntranceBlockingStub client; private final FireflyMetrics metrics; + private final EntranceGrpc.EntranceFutureStub ayncClient; public EntranceClient(ManagedServerChannel channel, FireflyMetrics metrics) { this.channel = channel; this.client = channel.wrap(EntranceGrpc.newBlockingStub(channel)); + ayncClient = channel.wrap(EntranceGrpc.newFutureStub(channel)); this.metrics = metrics; } @@ -46,23 +50,34 @@ public Member getMember() { } @Override - public Gateway join(Join join, Duration timeout) { + public ListenableFuture join(Join join, Duration timeout) { if (metrics != null) { var serializedSize = join.getSerializedSize(); metrics.outboundBandwidth().mark(serializedSize); metrics.outboundJoin().update(serializedSize); } - Gateway result = client.withDeadlineAfter(timeout.toNanos(), TimeUnit.NANOSECONDS).join(join); - if (metrics != null) { + ListenableFuture result = ayncClient.withDeadlineAfter(timeout.toNanos(), TimeUnit.NANOSECONDS) + .join(join); + result.addListener(() -> { + Gateway g = null; try { - var serializedSize = result.getSerializedSize(); - metrics.inboundBandwidth().mark(serializedSize); - metrics.inboundGateway().update(serializedSize); - } catch (Throwable e) { + g = result.get(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } catch (ExecutionException e) { // nothing } - } + if (metrics != null) { + try { + var serializedSize = g.getSerializedSize(); + metrics.inboundBandwidth().mark(serializedSize); + metrics.inboundGateway().update(serializedSize); + } catch (Throwable e) { + // nothing + } + } + }, Runnable::run); return result; } diff --git a/fireflies/src/test/java/com/salesforce/apollo/fireflies/ChurnTest.java b/fireflies/src/test/java/com/salesforce/apollo/fireflies/ChurnTest.java index d39f26c77..b689e159d 100644 --- a/fireflies/src/test/java/com/salesforce/apollo/fireflies/ChurnTest.java +++ b/fireflies/src/test/java/com/salesforce/apollo/fireflies/ChurnTest.java @@ -271,7 +271,7 @@ public void churn() throws Exception { private void initialize() { executor = UnsafeExecutors.newVirtualThreadPerTaskExecutor(); executor2 = UnsafeExecutors.newVirtualThreadPerTaskExecutor(); - var parameters = Parameters.newBuilder().setFpr(0.0000125).setMaximumTxfr(20).build(); + var parameters = Parameters.newBuilder().setMaximumTxfr(10).build(); registry = new MetricRegistry(); node0Registry = new MetricRegistry(); diff --git a/fireflies/src/test/resources/logback-test.xml b/fireflies/src/test/resources/logback-test.xml index b4afb147d..f86c9c940 100644 --- a/fireflies/src/test/resources/logback-test.xml +++ b/fireflies/src/test/resources/logback-test.xml @@ -9,7 +9,12 @@ %d{mm:ss.SSS} %logger{0} - %msg%n - + + + + + + @@ -17,19 +22,31 @@ - + + + + + + + + + + + + + - + - + - + diff --git a/memberships/src/main/java/com/salesforce/apollo/context/DelegatedContext.java b/memberships/src/main/java/com/salesforce/apollo/context/DelegatedContext.java index ecb159025..e319a2977 100644 --- a/memberships/src/main/java/com/salesforce/apollo/context/DelegatedContext.java +++ b/memberships/src/main/java/com/salesforce/apollo/context/DelegatedContext.java @@ -2,6 +2,7 @@ import com.salesforce.apollo.cryptography.Digest; import com.salesforce.apollo.membership.Member; +import com.salesforce.apollo.utils.Entropy; import org.apache.commons.math3.random.BitsStreamGenerator; import java.util.List; @@ -231,7 +232,7 @@ public int rank(int ring, T item, T dest) { @Override public List sample(int range, BitsStreamGenerator entropy, Digest exc) { - return delegate.sample(range, entropy, exc); + return delegate.sample(range, Entropy.bitsStream(), exc); } @Override diff --git a/memberships/src/main/java/com/salesforce/apollo/context/DynamicContextImpl.java b/memberships/src/main/java/com/salesforce/apollo/context/DynamicContextImpl.java index 18a22e1d1..58fc888b7 100644 --- a/memberships/src/main/java/com/salesforce/apollo/context/DynamicContextImpl.java +++ b/memberships/src/main/java/com/salesforce/apollo/context/DynamicContextImpl.java @@ -585,9 +585,7 @@ public Stream> rings() { */ @Override public List sample(int range, BitsStreamGenerator entropy, Predicate excluded) { - return rings.get(entropy.nextInt(rings.size())) - .stream() - .collect(new ReservoirSampler<>(excluded, range, entropy)); + return rings.get(entropy.nextInt(rings.size())).stream().collect(new ReservoirSampler<>(range, excluded)); } /** @@ -603,7 +601,7 @@ public List sample(int range, BitsStreamGenerator entropy, Dige Member excluded = exc == null ? null : getMember(exc); return rings.get(entropy.nextInt(rings.size())) .stream() - .collect(new ReservoirSampler(excluded, range, entropy)); + .collect(new ReservoirSampler(range, t -> t.equals(excluded))); } @Override diff --git a/memberships/src/main/java/com/salesforce/apollo/context/StaticContext.java b/memberships/src/main/java/com/salesforce/apollo/context/StaticContext.java index 42d6a1668..d91662176 100644 --- a/memberships/src/main/java/com/salesforce/apollo/context/StaticContext.java +++ b/memberships/src/main/java/com/salesforce/apollo/context/StaticContext.java @@ -313,21 +313,20 @@ public int rank(int ring, T item, T dest) { */ @Override public List sample(int range, BitsStreamGenerator entropy, Predicate excluded) { - return ring(entropy.nextInt(rings.length)).stream().collect(new ReservoirSampler<>(excluded, range, entropy)); + return ring(entropy.nextInt(rings.length)).stream().collect(new ReservoirSampler<>(range, excluded)); } /** * Answer a random sample of at least range size from the active members of the context * - * @param range - the desired range - * @param entropy - source o randomness - * @param exc - the member to exclude from sample + * @param range - the desired range + * @param exc - the member to exclude from sample * @return a random sample set of the view's live members. May be limited by the number of active members. */ @Override public List sample(int range, BitsStreamGenerator entropy, Digest exc) { Member excluded = exc == null ? null : getMember(exc); - return ring(entropy.nextInt(rings.length)).stream().collect(new ReservoirSampler(excluded, range, entropy)); + return ring(entropy.nextInt(rings.length)).stream().collect(new ReservoirSampler<>(range, (T) excluded)); } @Override diff --git a/memberships/src/main/java/com/salesforce/apollo/membership/ReservoirSampler.java b/memberships/src/main/java/com/salesforce/apollo/membership/ReservoirSampler.java index d62dde510..d5f770cad 100644 --- a/memberships/src/main/java/com/salesforce/apollo/membership/ReservoirSampler.java +++ b/memberships/src/main/java/com/salesforce/apollo/membership/ReservoirSampler.java @@ -6,36 +6,40 @@ */ package com.salesforce.apollo.membership; -import org.apache.commons.math3.random.BitsStreamGenerator; - import java.util.ArrayList; import java.util.EnumSet; import java.util.List; import java.util.Set; -import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.ThreadLocalRandom; import java.util.function.*; import java.util.stream.Collector; -public class ReservoirSampler implements Collector, List> { +import static java.lang.Math.exp; +import static java.lang.Math.log; - private final Predicate exclude; - private final BitsStreamGenerator rand; - private final int sz; - private AtomicInteger c = new AtomicInteger(); +/** + * @author hal.hildebrand + **/ +public class ReservoirSampler implements Collector, List> { + private final int capacity; + private final Predicate ignore; + private volatile double w; + private volatile long counter; + private volatile long next; - public ReservoirSampler(int size, BitsStreamGenerator entropy) { - this(null, size, entropy); + public ReservoirSampler(int capacity, T ignore) { + this(capacity, t -> t.equals(ignore)); } - public ReservoirSampler(Object excluded, int size, BitsStreamGenerator entropy) { - this(t -> excluded == null ? false : excluded.equals(t), size, entropy); + public ReservoirSampler(int capacity, Predicate ignore) { + this.capacity = capacity; + w = exp(log(ThreadLocalRandom.current().nextDouble()) / capacity); + skip(); + this.ignore = ignore == null ? t -> false : ignore; } - public ReservoirSampler(Predicate excluded, int size, BitsStreamGenerator entropy) { - assert size >= 0; - this.exclude = excluded; - this.sz = size; - rand = entropy; + public ReservoirSampler(int capacity) { + this(capacity, (Predicate) null); } @Override @@ -63,21 +67,31 @@ public Function, List> finisher() { @Override public Supplier> supplier() { - return ArrayList::new; + var reservoir = new ArrayList(capacity); + for (int i = 0; i < capacity; i++) { + reservoir.add(null); + } + return () -> reservoir; } private void addIt(final List in, T s) { - if (exclude != null && exclude.test(s)) { + if (ignore.test(s)) { return; } - if (in.size() < sz) { - in.add(s); + + if (counter < in.size()) { + in.add((int) counter, s); } else { - int replaceInIndex = (int) (rand.nextLong(sz + (c.getAndIncrement()) + 1)); - if (replaceInIndex < sz) { - in.set(replaceInIndex, s); + if (counter == next) { + in.add(ThreadLocalRandom.current().nextInt(in.size()), s); + skip(); } } + ++counter; } + private void skip() { + next += (long) (log(ThreadLocalRandom.current().nextDouble()) / log(1 - w)) + 1; + w *= exp(log(ThreadLocalRandom.current().nextDouble()) / capacity); + } } From 5b3084e30ddd844a0d7f23038466552181d030da Mon Sep 17 00:00:00 2001 From: Hellblazer Date: Sat, 22 Jun 2024 15:17:32 -0700 Subject: [PATCH 10/32] fixins. Reimplement ReservoirSampler. 5x5 --- model/src/main/java/com/salesforce/apollo/model/SubDomain.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/model/src/main/java/com/salesforce/apollo/model/SubDomain.java b/model/src/main/java/com/salesforce/apollo/model/SubDomain.java index 77a013031..07b8c0688 100644 --- a/model/src/main/java/com/salesforce/apollo/model/SubDomain.java +++ b/model/src/main/java/com/salesforce/apollo/model/SubDomain.java @@ -205,8 +205,7 @@ private DelegationUpdate.Builder update(DelegationUpdate update, DelegationUpdat delegates.entrySet() .stream() .filter(e -> !bff.contains(Digest.from(e.getKey()))) - // .limit(maxTransfer) - .collect(new ReservoirSampler<>(maxTransfer, Entropy.bitsStream())) + .collect(new ReservoirSampler<>(maxTransfer)) .forEach(e -> builder.addUpdate(e.getValue())); return builder; } From 6d6bd308c49b76a61f136f5851092a8590716ed2 Mon Sep 17 00:00:00 2001 From: Hellblazer Date: Sat, 22 Jun 2024 16:18:40 -0700 Subject: [PATCH 11/32] handle unknown accusations, initiate view change under lock --- .../com/salesforce/apollo/fireflies/View.java | 26 +++++++++++++------ .../apollo/fireflies/ViewManagement.java | 24 ++++------------- 2 files changed, 23 insertions(+), 27 deletions(-) diff --git a/fireflies/src/main/java/com/salesforce/apollo/fireflies/View.java b/fireflies/src/main/java/com/salesforce/apollo/fireflies/View.java index 7f851207f..456e063a7 100644 --- a/fireflies/src/main/java/com/salesforce/apollo/fireflies/View.java +++ b/fireflies/src/main/java/com/salesforce/apollo/fireflies/View.java @@ -641,6 +641,7 @@ protected Gossip gossip(Fireflies link, int ring) { case PERMISSION_DENIED: log.trace("Rejected gossip: {} view: {} from: {} on: {}", sre.getStatus(), currentView(), p.getId(), node.getId()); + accuse(p, ring, sre); break; case FAILED_PRECONDITION: log.trace("Failed gossip: {} view: {} from: {} on: {}", sre.getStatus(), currentView(), p.getId(), @@ -744,11 +745,12 @@ private boolean add(AccusationWrapper accusation, Participant accuser, Participa return false; } - if (accused.isAccusedOn(accusation.getRingNumber())) { - Participant currentAccuser = context.getMember( - accused.getAccusation(accusation.getRingNumber()).getAccuser()); - if (!currentAccuser.equals(accuser)) { - if (context.isBetween(accusation.getRingNumber(), currentAccuser, accuser, accused)) { + var acc = accused.getAccusation(accusation.getRingNumber()); + if (acc != null) { + var currentAccuser = context.getMember(acc.getAccuser()); + if (currentAccuser == null || !currentAccuser.equals(accuser)) { + if (currentAccuser == null || context.isBetween(accusation.getRingNumber(), currentAccuser, accuser, + accused)) { if (!accused.verify(accusation.getSignature(), accusation.getWrapped().getAccusation().toByteString())) { log.trace("Accusation discarded, accusation by: {} accused:{} signature invalid on: {}", @@ -782,8 +784,8 @@ private boolean add(AccusationWrapper accusation, Participant accuser, Participa } return false; } - Participant predecessor = context.predecessor(accusation.getRingNumber(), accused, - m -> (!m.isAccused()) || (m.equals(accuser))); + var predecessor = context.predecessor(accusation.getRingNumber(), accused, + m -> (!m.isAccused()) || (m.equals(accuser))); if (accuser.equals(predecessor)) { accused.addAccusation(accusation); if (!accused.equals(node) && !pendingRebuttals.containsKey(accused.getId())) { @@ -1135,6 +1137,11 @@ private void handleSRE(String type, RingCommunications.Destination ring, Deque check) { AccusationWrapper qa = q.getAccusation(ring.getIndex()); + if (qa == null) { + return; + } Participant accuser = context.getMember(qa.getAccuser()); Participant accused = context.getMember(qa.getAccused()); if (ring.isBetween(accuser, q, accused)) { @@ -1400,7 +1410,7 @@ private void validate(Digest from, final int ring, Digest requestView, String ty log.debug("Invalid {}, view: {} current: {} ring: {} from: {} on: {}", type, requestView, currentView(), ring, from, node.getId()); throw new StatusRuntimeException( - Status.PERMISSION_DENIED.withDescription("Invalid view: " + requestView + " current: " + currentView())); + Status.FAILED_PRECONDITION.withDescription("Invalid view: " + requestView + " current: " + currentView())); } } diff --git a/fireflies/src/main/java/com/salesforce/apollo/fireflies/ViewManagement.java b/fireflies/src/main/java/com/salesforce/apollo/fireflies/ViewManagement.java index 816bd22e9..171e92750 100644 --- a/fireflies/src/main/java/com/salesforce/apollo/fireflies/ViewManagement.java +++ b/fireflies/src/main/java/com/salesforce/apollo/fireflies/ViewManagement.java @@ -434,21 +434,7 @@ void maybeViewChange() { return; } if ((context.offlineCount() > 0 || !joins.isEmpty())) { - if (isObserver()) { - log.info("Initiating view change: {} (observer) joins: {} leaves: {} on: {}", currentView(), - joins.size(), view.streamShunned().count(), node.getId()); - initiateViewChange(); - } else { - // Use pending rebuttals as a proxy for stability - if (view.hasPendingRebuttals()) { - log.debug("Pending rebuttals in view: {} on: {}", currentView(), node.getId()); - view.scheduleViewChange(2); // 2 TTL round2 to check again - } else { - log.info("Initiating view change: {} (non observer) joins: {} leaves: {} on: {}", currentView(), - joins.size(), view.streamShunned().count(), node.getId()); - view.scheduleFinalizeViewChange(); - } - } + initiateViewChange(); } else { // log.trace("No view change: {} joins: {} leaves: {} on: {}", currentView(), joins.size(), // view.streamShunned().count(), node.getId()); @@ -574,7 +560,6 @@ void start(CompletableFuture onJoin, boolean bootstrap) { * Initiate the view change */ private void initiateViewChange() { - assert isObserver() : "Not observer: " + node.getId(); view.stable(() -> { if (vote.get() != null) { log.trace("Vote already cast for: {} on: {}", currentView(), node.getId()); @@ -588,11 +573,12 @@ private void initiateViewChange() { } view.scheduleFinalizeViewChange(); if (!isObserver(node.getId())) { - log.warn("Initiating view change: {} (non observer) on: {}", currentView(), node.getId()); + log.info("Initiating (non observer) view change: {} joins: {} leaves: {} on: {}", currentView(), + joins.size(), view.streamShunned().count(), node.getId()); return; } - log.warn("Initiating view change vote: {} joins: {} leaves: {} observers: {} on: {}", currentView(), - joins.size(), view.streamShunned().count(), observersList(), node.getId()); + log.warn("Initiating (observer) view change vote: {} joins: {} leaves: {} observers: {} on: {}", + currentView(), joins.size(), view.streamShunned().count(), observersList(), node.getId()); final var builder = ViewChange.newBuilder() .setObserver(node.getId().toDigeste()) .setCurrent(currentView().toDigeste()) From 129fa4936fba4ded7e0bbe68c83735ff567f9b65 Mon Sep 17 00:00:00 2001 From: Hellblazer Date: Sat, 22 Jun 2024 18:22:39 -0700 Subject: [PATCH 12/32] serialize via single threaded executor --- .../com/salesforce/apollo/choam/Producer.java | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) diff --git a/choam/src/main/java/com/salesforce/apollo/choam/Producer.java b/choam/src/main/java/com/salesforce/apollo/choam/Producer.java index d6dbc33f8..fb003c225 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/Producer.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/Producer.java @@ -28,10 +28,7 @@ import org.slf4j.LoggerFactory; import java.util.*; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ConcurrentSkipListMap; -import java.util.concurrent.CopyOnWriteArrayList; -import java.util.concurrent.Semaphore; +import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; @@ -54,7 +51,8 @@ public class Producer { private final Transitions transitions; private final ViewContext view; private final Digest nextViewId; - private final Semaphore serialize = new Semaphore(1); + private final Executor serialize = Executors.newSingleThreadExecutor( + Thread.ofVirtual().factory()); private final ViewAssembly assembly; private final int maxEpoch; private volatile boolean assembled = false; @@ -351,22 +349,13 @@ private void reconfigure() { } private void serial(List preblock, Boolean last) { - try { - serialize.acquire(); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - return; - } - Thread.ofVirtual().start(() -> { + serialize.execute(() -> { try { create(preblock, last); } catch (Throwable t) { log.error("Error processing preblock last: {} on: {}", last, params().member().getId(), t); - } finally { - serialize.release(); } }); - } private PendingBlock validate(Validate v) { From b158e5c4a2dd4556a132684bc6a19627a6cb09f9 Mon Sep 17 00:00:00 2001 From: Hellblazer Date: Sun, 23 Jun 2024 06:53:39 -0700 Subject: [PATCH 13/32] cleanup --- .../com/salesforce/apollo/fireflies/View.java | 11 +++++---- .../apollo/fireflies/ViewManagement.java | 24 +++++++++---------- 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/fireflies/src/main/java/com/salesforce/apollo/fireflies/View.java b/fireflies/src/main/java/com/salesforce/apollo/fireflies/View.java index 456e063a7..7a1ded956 100644 --- a/fireflies/src/main/java/com/salesforce/apollo/fireflies/View.java +++ b/fireflies/src/main/java/com/salesforce/apollo/fireflies/View.java @@ -344,16 +344,18 @@ void finalizeViewChange() { return; } viewChange(() -> { + final var current = currentView(); final var supermajority = context.getRingCount() * 3 / 4; final var majority = context.size() == 1 ? 1 : supermajority; final var valid = observations.values() .stream() + .filter(vc -> current.equals(Digest.from(vc.getChange().getCurrent()))) .filter(svc -> viewManagement.observers.contains( Digest.from(svc.getChange().getObserver()))) .toList(); - log.info("Finalize view change, observations: {} valid: {} observers: {} on: {}", - observations.values().stream().map(sv -> Digest.from(sv.getChange().getObserver())).toList(), - valid.size(), viewManagement.observersList(), node.getId()); + log.trace("Finalize view change, observations: {} valid: {} observers: {} on: {}", + observations.values().stream().map(sv -> Digest.from(sv.getChange().getObserver())).toList(), + valid.size(), viewManagement.observersList(), node.getId()); observations.clear(); if (valid.size() < majority) { log.info("Do not have majority: {} required: {} observers: {} for: {} on: {}", valid.size(), majority, @@ -364,8 +366,7 @@ void finalizeViewChange() { log.info("Finalizing view change: {} required: {} observers: {} for: {} on: {}", context.getId(), majority, viewManagement.observersList(), currentView(), node.getId()); HashMultiset ballots = HashMultiset.create(); - final var current = currentView(); - valid.stream().filter(vc -> current.equals(Digest.from(vc.getChange().getCurrent()))).forEach(vc -> { + valid.forEach(vc -> { final var leaving = vc.getChange() .getLeavesList() .stream() diff --git a/fireflies/src/main/java/com/salesforce/apollo/fireflies/ViewManagement.java b/fireflies/src/main/java/com/salesforce/apollo/fireflies/ViewManagement.java index 171e92750..b4d33a528 100644 --- a/fireflies/src/main/java/com/salesforce/apollo/fireflies/ViewManagement.java +++ b/fireflies/src/main/java/com/salesforce/apollo/fireflies/ViewManagement.java @@ -513,22 +513,22 @@ Redirect seed(Registration registration, Digest from) { final var requestView = Digest.from(registration.getView()); if (!joined()) { - log.warn("Not joined, ignored seed view: {} from: {} on: {}", requestView, from, node.getId()); + log.trace("Not joined, ignored seed view: {} from: {} on: {}", requestView, from, node.getId()); return Redirect.getDefaultInstance(); } if (!bootstrapView.equals(requestView)) { - log.warn("Invalid bootstrap view: {} expected: {} from: {} on: {}", bootstrapView, requestView, from, - node.getId()); + log.trace("Invalid bootstrap view: {} expected: {} from: {} on: {}", bootstrapView, requestView, from, + node.getId()); return Redirect.getDefaultInstance(); } var note = new NoteWrapper(registration.getNote(), digestAlgo); if (!from.equals(note.getId())) { - log.warn("Invalid bootstrap note: {} from: {} claiming: {} on: {}", requestView, from, note.getId(), - node.getId()); + log.trace("Invalid bootstrap note: {} from: {} claiming: {} on: {}", requestView, from, note.getId(), + node.getId()); return Redirect.getDefaultInstance(); } if (!view.validate(note.getIdentifier())) { - log.warn("Invalid identifier: {} from: {} on: {}", note.getIdentifier(), from, node.getId()); + log.trace("Invalid identifier: {} from: {} on: {}", note.getIdentifier(), from, node.getId()); return Redirect.getDefaultInstance(); } return view.stable(() -> { @@ -573,12 +573,12 @@ private void initiateViewChange() { } view.scheduleFinalizeViewChange(); if (!isObserver(node.getId())) { - log.info("Initiating (non observer) view change: {} joins: {} leaves: {} on: {}", currentView(), - joins.size(), view.streamShunned().count(), node.getId()); + log.debug("Initiating (non observer) view change: {} joins: {} leaves: {} on: {}", currentView(), + joins.size(), view.streamShunned().count(), node.getId()); return; } - log.warn("Initiating (observer) view change vote: {} joins: {} leaves: {} observers: {} on: {}", - currentView(), joins.size(), view.streamShunned().count(), observersList(), node.getId()); + log.debug("Initiating (observer) view change vote: {} joins: {} leaves: {} observers: {} on: {}", + currentView(), joins.size(), view.streamShunned().count(), observersList(), node.getId()); final var builder = ViewChange.newBuilder() .setObserver(node.getId().toDigeste()) .setCurrent(currentView().toDigeste()) @@ -593,8 +593,8 @@ private void initiateViewChange() { .setSignature(signature.toSig()) .build(); view.initiate(viewChange); - log.warn("View change vote: {} joins: {} leaves: {} on: {}", currentView(), change.getJoinsCount(), - change.getLeavesCount(), node.getId()); + log.trace("View change vote: {} joins: {} leaves: {} on: {}", currentView(), change.getJoinsCount(), + change.getLeavesCount(), node.getId()); }); } From fe468231e74c13a73118d9912a9b1ffc7c6dd842 Mon Sep 17 00:00:00 2001 From: Hellblazer Date: Sun, 23 Jun 2024 07:08:27 -0700 Subject: [PATCH 14/32] handle error already sent --- .../apollo/fireflies/comm/entrance/EntranceServer.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/fireflies/src/main/java/com/salesforce/apollo/fireflies/comm/entrance/EntranceServer.java b/fireflies/src/main/java/com/salesforce/apollo/fireflies/comm/entrance/EntranceServer.java index bee570e97..15c0025e5 100644 --- a/fireflies/src/main/java/com/salesforce/apollo/fireflies/comm/entrance/EntranceServer.java +++ b/fireflies/src/main/java/com/salesforce/apollo/fireflies/comm/entrance/EntranceServer.java @@ -51,7 +51,11 @@ public void join(Join request, StreamObserver responseObserver) { try { s.join(request, from, responseObserver, timer); } catch (Throwable t) { - responseObserver.onError(t); + try { + responseObserver.onError(t); + } catch (Throwable throwable) { + // ignore as response observer is closed + } } }); } From 7d18010a3ddd70615088b2cc9c199aa35fcc4d70 Mon Sep 17 00:00:00 2001 From: Hellblazer Date: Sun, 23 Jun 2024 12:40:21 -0700 Subject: [PATCH 15/32] multiple fixin's ensure valid bft set for reconfigured validators. clean up observations replication - ugh fix bootstrap view change logic fix vote process allow observations to propagate post vote info log accusations correct PID assignment --- .../com/salesforce/apollo/choam/CHOAM.java | 10 +- .../salesforce/apollo/choam/Committee.java | 4 +- .../com/salesforce/apollo/choam/Producer.java | 5 +- .../salesforce/apollo/choam/ViewAssembly.java | 15 +- .../salesforce/apollo/choam/ViewContext.java | 10 +- .../salesforce/apollo/fireflies/Binding.java | 8 +- .../com/salesforce/apollo/fireflies/View.java | 177 ++++++++++-------- .../apollo/fireflies/ViewManagement.java | 99 +++++----- .../apollo/fireflies/ChurnTest.java | 2 +- 9 files changed, 179 insertions(+), 151 deletions(-) diff --git a/choam/src/main/java/com/salesforce/apollo/choam/CHOAM.java b/choam/src/main/java/com/salesforce/apollo/choam/CHOAM.java index 104d086bc..10a1b8fc7 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/CHOAM.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/CHOAM.java @@ -210,10 +210,9 @@ public static String print(Join join, DigestAlgorithm da) { } public static Reconfigure reconfigure(Digest nextViewId, Map joins, int checkpointTarget) { + assert Dag.validate(joins.size()) : "Reconfigure joins: %s is not BFT".formatted(joins.size()); var builder = Reconfigure.newBuilder().setCheckpointTarget(checkpointTarget).setId(nextViewId.toDigeste()); - joins.keySet().stream().sorted().map(joins::get).forEach(builder::addJoins); - return builder.build(); } @@ -232,13 +231,6 @@ public static Block reconfigure(Digest nextViewId, Map joins, Hash .build(); } - public static Map rosterMap(Context baseContext, Collection members) { - return members.stream() - .map(baseContext::getMember) - .filter(m -> m != null) - .collect(Collectors.toMap(Member::getId, Function.identity())); - } - public static List toGenesisData(List initializationData) { return toGenesisData(initializationData, DigestAlgorithm.DEFAULT, SignatureAlgorithm.DEFAULT); } diff --git a/choam/src/main/java/com/salesforce/apollo/choam/Committee.java b/choam/src/main/java/com/salesforce/apollo/choam/Committee.java index ecc92f48e..eb318320a 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/Committee.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/Committee.java @@ -15,6 +15,7 @@ import com.salesforce.apollo.cryptography.JohnHancock; import com.salesforce.apollo.cryptography.Verifier; import com.salesforce.apollo.cryptography.Verifier.DefaultVerifier; +import com.salesforce.apollo.ethereal.Dag; import com.salesforce.apollo.membership.Member; import com.salesforce.apollo.membership.MockMember; import io.grpc.StatusRuntimeException; @@ -35,6 +36,8 @@ public interface Committee { static Map validatorsOf(Reconfigure reconfigure, Context context, Digest member, Logger log) { + assert Dag.validate(reconfigure.getJoinsCount()) : "Reconfigure joins: %s is not BFT".formatted( + reconfigure.getJoinsCount()); var validators = reconfigure.getJoinsList().stream().collect(Collectors.toMap(e -> { var id = new Digest(e.getMember().getVm().getId()); var m = context.getMember(id); @@ -47,7 +50,6 @@ static Map validatorsOf(Reconfigure reconfigure, Context { var vm = e.getMember().getVm(); if (vm.hasConsensusKey()) { - return new DefaultVerifier(publicKey(vm.getConsensusKey())); } else { log.info("No member for validator: {}, returning mock on: {}", Digest.from(vm.getId()), member); diff --git a/choam/src/main/java/com/salesforce/apollo/choam/Producer.java b/choam/src/main/java/com/salesforce/apollo/choam/Producer.java index fb003c225..9538fe8ce 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/Producer.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/Producer.java @@ -21,6 +21,7 @@ import com.salesforce.apollo.cryptography.DigestAlgorithm; import com.salesforce.apollo.ethereal.Config; import com.salesforce.apollo.ethereal.Config.Builder; +import com.salesforce.apollo.ethereal.Dag; import com.salesforce.apollo.ethereal.Ethereal; import com.salesforce.apollo.ethereal.memberships.ChRbcGossip; import com.salesforce.apollo.membership.Member; @@ -325,8 +326,8 @@ private void publish(PendingBlock p, boolean beacon) { private void reconfigure() { final var slate = assembly.getSlate(); - assert slate != null && !slate.isEmpty() : "Slate is incorrect: %s".formatted( - slate.keySet().stream().sorted().toList()); + assert slate != null && !slate.isEmpty() : slate == null ? "Slate is null" : "Slate is empty"; + assert Dag.validate(slate.size()) : "Reconfigure joins: %s is not BFT".formatted(slate.size()); var reconfiguration = new HashedBlock(params().digestAlgorithm(), view.reconfigure(slate, nextViewId, previousBlock.get(), checkpoint.get())); diff --git a/choam/src/main/java/com/salesforce/apollo/choam/ViewAssembly.java b/choam/src/main/java/com/salesforce/apollo/choam/ViewAssembly.java index cf31987be..98a510a8c 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/ViewAssembly.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/ViewAssembly.java @@ -18,6 +18,7 @@ import com.salesforce.apollo.cryptography.JohnHancock; import com.salesforce.apollo.cryptography.proto.Digeste; import com.salesforce.apollo.cryptography.proto.PubKey; +import com.salesforce.apollo.ethereal.Dag; import com.salesforce.apollo.membership.Member; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -142,9 +143,9 @@ void assemble(List asses) { } views.forEach(svs -> { if (view.validate(svs)) { - log.info("Adding views: {} from: {} on: {}", - svs.getViews().getViewsList().stream().map(v -> Digest.from(v.getDiadem())).toList(), - Digest.from(svs.getViews().getMember()), params().member().getId()); + log.debug("Adding views: {} from: {} on: {}", + svs.getViews().getViewsList().stream().map(v -> Digest.from(v.getDiadem())).toList(), + Digest.from(svs.getViews().getMember()), params().member().getId()); viewProposals.put(Digest.from(svs.getViews().getMember()), svs.getViews()); } else { log.warn("Invalid views: {} from: {} on: {}", @@ -377,6 +378,8 @@ private void vote() { params().member().getId()); } var winner = ratification.getFirst(); + assert Dag.validate(winner.getCommitteeCount()) : "Winner committee: %s is not BFT".formatted( + winner.getCommitteeList().size()); selected = new Vue(Digest.from(winner.getDiadem()), assemblyOf(winner.getCommitteeList()), winner.getMajority()); if (log.isDebugEnabled()) { @@ -408,8 +411,8 @@ public void certify() { transitions.certified(); } else { countdown.set(4); - log.info("Not certifying: {} majority: {} slate: {} of: {} on: {}", nextViewId, selected.majority, - proposals.keySet().stream().sorted().toList(), nextViewId, params().member().getId()); + log.debug("Not certifying: {} majority: {} slate: {} of: {} on: {}", nextViewId, selected.majority, + proposals.keySet().stream().sorted().toList(), nextViewId, params().member().getId()); } } @@ -420,7 +423,7 @@ public void checkAssembly() { if (proposals.size() >= selected.majority) { transitions.chill(); } else { - log.info("Check assembly: {} on: {}", proposals.size(), params().member().getId()); + log.trace("Check assembly: {} on: {}", proposals.size(), params().member().getId()); } } diff --git a/choam/src/main/java/com/salesforce/apollo/choam/ViewContext.java b/choam/src/main/java/com/salesforce/apollo/choam/ViewContext.java index 46a0c9d71..4f7a71d1f 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/ViewContext.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/ViewContext.java @@ -20,6 +20,7 @@ import java.util.HashMap; import java.util.Map; import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Supplier; import static com.salesforce.apollo.cryptography.QualifiedBase64.publicKey; @@ -48,11 +49,10 @@ public ViewContext(Context context, Parameters params, Supplier { + roster.put(m.getId(), (short) pid.getAndIncrement()); + }); } public static String print(Certification c, DigestAlgorithm algo) { diff --git a/fireflies/src/main/java/com/salesforce/apollo/fireflies/Binding.java b/fireflies/src/main/java/com/salesforce/apollo/fireflies/Binding.java index 2bbe4fe53..c8f84fdb5 100644 --- a/fireflies/src/main/java/com/salesforce/apollo/fireflies/Binding.java +++ b/fireflies/src/main/java/com/salesforce/apollo/fireflies/Binding.java @@ -152,7 +152,13 @@ private void complete(Member member, CompletableFuture gateway, HashMulti } catch (InterruptedException e) { Thread.currentThread().interrupt(); } catch (ExecutionException e) { - log.warn("Error retrieving Gateway from: {} on: {}", member.getId(), node.getId(), e.getCause()); + var cause = e.getCause(); + if (cause instanceof StatusRuntimeException sre) { + log.warn("Error retrieving Gateway: {} from: {} on: {}", sre.getMessage(), member.getId(), + node.getId()); + } else { + log.error("Error retrieving Gateway from: {} on: {}", member.getId(), node.getId(), cause); + } dec(complete, remaining); return; } diff --git a/fireflies/src/main/java/com/salesforce/apollo/fireflies/View.java b/fireflies/src/main/java/com/salesforce/apollo/fireflies/View.java index 7a1ded956..044d8e854 100644 --- a/fireflies/src/main/java/com/salesforce/apollo/fireflies/View.java +++ b/fireflies/src/main/java/com/salesforce/apollo/fireflies/View.java @@ -83,9 +83,10 @@ * @since 220 */ public class View { - private static final String FINALIZE_VIEW_CHANGE = "FINALIZE VIEW CHANGE"; + private static final String FINALIZE_VIEW_CHANGE = "Finalize View Change"; private static final Logger log = LoggerFactory.getLogger(View.class); private static final String SCHEDULED_VIEW_CHANGE = "Scheduled View Change"; + private static final String CLEAR_OBSERVATIONS = "Clear Observations"; final CommonCommunications comm; final AtomicBoolean started = new AtomicBoolean(); @@ -98,7 +99,7 @@ public class View { private final Executor viewNotificationQueue; private final FireflyMetrics metrics; private final Node node; - private final Map observations = new ConcurrentSkipListMap<>(); + private final Map observations = new ConcurrentSkipListMap<>(); private final Parameters params; private final ConcurrentMap pendingRebuttals = new ConcurrentSkipListMap<>(); private final RoundScheduler roundTimers; @@ -109,6 +110,7 @@ public class View { private final EventValidation validation; private final Verifiers verifiers; private volatile ScheduledFuture futureGossip; + private volatile boolean boostrap = false; public View(DynamicContext context, ControlledIdentifierMember member, String endpoint, EventValidation validation, Verifiers verifiers, Router communications, Parameters params, @@ -325,6 +327,7 @@ boolean addToView(NoteWrapper note) { } void bootstrap(NoteWrapper nw, Duration dur) { + boostrap = true; viewManagement.bootstrap(nw, dur); } @@ -344,45 +347,23 @@ void finalizeViewChange() { return; } viewChange(() -> { - final var current = currentView(); + removeTimer(View.FINALIZE_VIEW_CHANGE); final var supermajority = context.getRingCount() * 3 / 4; final var majority = context.size() == 1 ? 1 : supermajority; - final var valid = observations.values() - .stream() - .filter(vc -> current.equals(Digest.from(vc.getChange().getCurrent()))) - .filter(svc -> viewManagement.observers.contains( - Digest.from(svc.getChange().getObserver()))) - .toList(); - log.trace("Finalize view change, observations: {} valid: {} observers: {} on: {}", - observations.values().stream().map(sv -> Digest.from(sv.getChange().getObserver())).toList(), - valid.size(), viewManagement.observersList(), node.getId()); - observations.clear(); - if (valid.size() < majority) { - log.info("Do not have majority: {} required: {} observers: {} for: {} on: {}", valid.size(), majority, - viewManagement.observersList(), currentView(), node.getId()); - scheduleFinalizeViewChange(2); + log.info("Finalize view change, observations: {} observers: {} on: {}", + observations.keySet().stream().toList(), viewManagement.observersList(), node.getId()); + if (observations.size() < majority) { + log.info("Do not have majority: {} required: {} observers: {} for: {} on: {}", observations.size(), + majority, viewManagement.observersList(), currentView(), node.getId()); + scheduleFinalizeViewChange(1); return; } log.info("Finalizing view change: {} required: {} observers: {} for: {} on: {}", context.getId(), majority, viewManagement.observersList(), currentView(), node.getId()); HashMultiset ballots = HashMultiset.create(); - valid.forEach(vc -> { - final var leaving = vc.getChange() - .getLeavesList() - .stream() - .map(Digest::from) - .distinct() - .collect(Collectors.toCollection(ArrayList::new)); - final var joining = vc.getChange() - .getJoinsList() - .stream() - .map(Digest::from) - .distinct() - .collect(Collectors.toCollection(ArrayList::new)); - leaving.sort(Ordering.natural()); - joining.sort(Ordering.natural()); - ballots.add(new Ballot(Digest.from(vc.getChange().getCurrent()), leaving, joining, digestAlgo)); - }); + observations.values().forEach(svu -> tally(svu, ballots)); + viewManagement.clearVote(); + scheduleClearObservations(); var max = ballots.entrySet() .stream() .max(Ordering.natural().onResultOf(Multiset.Entry::getCount)) @@ -391,17 +372,15 @@ void finalizeViewChange() { log.info("View consensus successful: {} required: {} cardinality: {} for: {} on: {}", max, majority, viewManagement.cardinality(), currentView(), node.getId()); viewManagement.install(max.getElement()); + scheduleViewChange(); } else { @SuppressWarnings("unchecked") final var reversed = Comparator.comparing(e -> ((Entry) e).getCount()).reversed(); log.info("View consensus failed: {}, required: {} cardinality: {} ballots: {} for: {} on: {}", max == null ? 0 : max.getCount(), majority, viewManagement.cardinality(), ballots.entrySet().stream().sorted(reversed).toList(), currentView(), node.getId()); + viewManagement.initiateViewChange(); } - - scheduleViewChange(); - removeTimer(View.FINALIZE_VIEW_CHANGE); - viewManagement.clearVote(); }); } @@ -419,7 +398,7 @@ boolean hasPendingRebuttals() { } void initiate(SignedViewChange viewChange) { - observations.put(node.getId(), viewChange); + observations.put(node.getId(), new SVU(viewChange, digestAlgo)); } void introduced() { @@ -521,6 +500,13 @@ void schedule(final Duration duration) { Entropy.nextBitsStreamLong(duration.toNanos()), TimeUnit.NANOSECONDS); } + void scheduleClearObservations() { + if (!started.get()) { + return; + } + timers.put(CLEAR_OBSERVATIONS, roundTimers.schedule(CLEAR_OBSERVATIONS, () -> observations.clear(), 1)); + } + void scheduleFinalizeViewChange() { scheduleFinalizeViewChange(params.finalizeViewRounds()); } @@ -582,7 +568,7 @@ void stopRebuttalTimer(Participant m) { m.clearAccusations(); var timer = pendingRebuttals.remove(m.getId()); if (timer != null) { - log.debug("Cancelling accusation of: {} on: {}", m.getId(), node.getId()); + log.info("Cancelling accusation of: {} on: {}", m.getId(), node.getId()); timer.cancel(); } } @@ -691,8 +677,8 @@ private void accuse(Participant member, int ring, Throwable e) { member.addAccusation(node.accuse(member, ring)); pendingRebuttals.computeIfAbsent(member.getId(), d -> roundTimers.schedule(() -> gc(member), params.rebuttalTimeout())); - log.debug("Accuse: {} on ring: {} view: {} (timer started): {} on: {}", member.getId(), ring, currentView(), - e.getMessage(), node.getId()); + log.info("Accuse: {} on ring: {} view: {} (timer started): {} on: {}", member.getId(), ring, currentView(), + e.getMessage(), node.getId()); } /** @@ -754,15 +740,15 @@ private boolean add(AccusationWrapper accusation, Participant accuser, Participa accused)) { if (!accused.verify(accusation.getSignature(), accusation.getWrapped().getAccusation().toByteString())) { - log.trace("Accusation discarded, accusation by: {} accused:{} signature invalid on: {}", + log.debug("Accusation discarded, accusation by: {} accused:{} signature invalid on: {}", accuser.getId(), accused.getId(), node.getId()); return false; } accused.addAccusation(accusation); pendingRebuttals.computeIfAbsent(accused.getId(), d -> roundTimers.schedule(() -> gc(accused), params.rebuttalTimeout())); - log.debug("{} accused by: {} on ring: {} (replacing: {}) on: {}", accused.getId(), accuser.getId(), - accusation.getRingNumber(), currentAccuser.getId(), node.getId()); + log.info("{} accused by: {} on ring: {} (replacing: {}) on: {}", accused.getId(), accuser.getId(), + accusation.getRingNumber(), currentAccuser.getId(), node.getId()); if (metrics != null) { metrics.accusations().mark(); } @@ -790,8 +776,8 @@ private boolean add(AccusationWrapper accusation, Participant accuser, Participa if (accuser.equals(predecessor)) { accused.addAccusation(accusation); if (!accused.equals(node) && !pendingRebuttals.containsKey(accused.getId())) { - log.debug("{} accused by: {} on ring: {} (timer started) on: {}", accused.getId(), accuser.getId(), - accusation.getRingNumber(), node.getId()); + log.info("{} accused by: {} on ring: {} (timer started) on: {}", accused.getId(), accuser.getId(), + accusation.getRingNumber(), node.getId()); pendingRebuttals.computeIfAbsent(accused.getId(), d -> roundTimers.schedule(() -> gc(accused), params.rebuttalTimeout())); } @@ -841,41 +827,44 @@ private boolean add(NoteWrapper note) { * @param observation */ private boolean add(SignedViewChange observation) { - final Digest observer = Digest.from(observation.getChange().getObserver()); - if (!viewManagement.isObserver(observer)) { - log.trace("Invalid observer: {} current: {} on: {}", observer, currentView(), node.getId()); + var svu = new SVU(observation, digestAlgo); + if (!viewManagement.isObserver(svu.observer)) { + log.trace("Invalid observer: {} current: {} on: {}", svu.observer, currentView(), node.getId()); return false; } final var inView = Digest.from(observation.getChange().getCurrent()); if (!currentView().equals(inView)) { - log.trace("Invalid view change: {} current: {} from {} on: {}", inView, currentView(), observer, + log.trace("Invalid view change: {} current: {} from {} on: {}", inView, currentView(), svu.observer, node.getId()); return false; } - var currentObservation = observations.get(observer); - if (currentObservation != null) { - if (observation.getChange().getAttempt() < currentObservation.getChange().getAttempt()) { - log.trace("Stale observation: {} current: {} view change: {} current: {} offline: {} on: {}", - observation.getChange().getAttempt(), currentObservation.getChange().getAttempt(), inView, - currentView(), observer, node.getId()); - return false; - } - } - final var member = context.getActiveMember(observer); + final var member = context.getActiveMember(svu.observer); if (member == null) { - log.trace("Cannot validate view change: {} current: {} from: {} on: {}", inView, currentView(), observer, + log.trace("Cannot validate view change: {} current: {} from: {} on: {}", inView, currentView(), + svu.observer, node.getId()); + return false; + } + if (!viewManagement.isObserver(member.id)) { + log.trace("Not an observer of: {} current: {} from: {} on: {}", inView, currentView(), svu.observer, node.getId()); return false; } - return observations.computeIfAbsent(observer.prefix(observation.getChange().getAttempt()), p -> { - final var signature = JohnHancock.from(observation.getSignature()); - if (!member.verify(signature, observation.getChange().toByteString())) { - return null; + final var signature = JohnHancock.from(observation.getSignature()); + if (!member.verify(signature, observation.getChange().toByteString())) { + return false; + } + return observations.compute(svu.observer, (d, cur) -> { + if (cur != null) { + if (svu.attempt < cur.attempt) { + log.trace("Stale observation: {} current: {} view change: {} current: {} offline: {} on: {}", + svu.attempt, cur.attempt, inView, currentView(), svu.observer, node.getId()); + return cur; + } } - log.trace("Observation: {} current: {} view change: {} from: {} on: {}", - observation.getChange().getAttempt(), inView, currentView(), observer, node.getId()); - return observation; - }) != null; + log.trace("Observation: {} current: {} view change: {} from: {} on: {}", svu.attempt, inView, currentView(), + svu.observer, node.getId()); + return svu; + }) == svu; } private boolean addJoin(SignedNote sn) { @@ -1049,7 +1038,7 @@ private BloomFilter getNotesBff(long seed, double p) { private BloomFilter getObservationsBff(long seed, double p) { var n = Math.max(params.minimumBiffCardinality(), observations.size()); BloomFilter bff = new BloomFilter.DigestBloomFilter(seed, n, 1.0 / (double) n); - observations.keySet().stream().collect(Utils.toShuffledList()).forEach(bff::add); + observations.values().stream().map(svu -> svu.hash).collect(Utils.toShuffledList()).forEach(bff::add); return bff; } @@ -1263,13 +1252,12 @@ private ViewChangeGossip.Builder processObservations(BloomFilter bff) { // Add all updates that this view has that aren't reflected in the inbound bff final var current = currentView(); - observations.entrySet() + observations.values() .stream() .collect(Utils.toShuffledList()) .stream() - .filter(e -> Digest.from(e.getValue().getChange().getCurrent()).equals(current)) - .filter(m -> !bff.contains(m.getKey())) - .map(m -> m.getValue()) + .filter(svu -> !bff.contains(svu.hash)) + .map(svu -> svu.viewChange) .forEach(n -> builder.addUpdates(n)); return builder; } @@ -1344,6 +1332,25 @@ private Update response(Gossip gossip) { return updatesForDigests(gossip); } + private void tally(SVU svu, HashMultiset ballots) { + var vc = svu.viewChange; + final var leaving = vc.getChange() + .getLeavesList() + .stream() + .map(Digest::from) + .distinct() + .collect(Collectors.toCollection(ArrayList::new)); + final var joining = vc.getChange() + .getJoinsList() + .stream() + .map(Digest::from) + .distinct() + .collect(Collectors.toCollection(ArrayList::new)); + leaving.sort(Ordering.natural()); + joining.sort(Ordering.natural()); + ballots.add(new Ballot(Digest.from(vc.getChange().getCurrent()), leaving, joining, digestAlgo)); + } + /** * Process the gossip reply. Return the gossip with the updates determined from the inbound digests. * @@ -1380,13 +1387,12 @@ private Update updatesForDigests(Gossip gossip) { biff = gossip.getObservations().getBff(); if (!biff.equals(Biff.getDefaultInstance())) { BloomFilter obsvBff = BloomFilter.from(biff); - observations.entrySet() + observations.values() .stream() .collect(Utils.toShuffledList()) .stream() - .filter(e -> Digest.from(e.getValue().getChange().getCurrent()).equals(current)) - .filter(e -> !obsvBff.contains(e.getKey())) - .forEach(e -> builder.addObservations(e.getValue())); + .filter(svu -> !obsvBff.contains(svu.hash)) + .forEach(svu -> builder.addObservations(svu.viewChange)); } biff = gossip.getJoins().getBff(); @@ -1463,6 +1469,19 @@ private boolean verify(SelfAddressingIdentifier id, SigningThreshold threshold, return verifiers.verifierFor(id).map(value -> value.verify(threshold, signature, message)).orElse(false); } + private record SVU(Digest observer, SignedViewChange viewChange, int attempt, Digest hash) + implements Comparable { + public SVU(SignedViewChange signedViewChange, DigestAlgorithm algo) { + this(Digest.from(signedViewChange.getChange().getObserver()), signedViewChange, + signedViewChange.getChange().getAttempt(), algo.digest(signedViewChange.toByteString())); + } + + @Override + public int compareTo(SVU o) { + return Integer.compare(attempt, o.attempt); + } + } + public record Seed(SelfAddressingIdentifier identifier, String endpoint) { } diff --git a/fireflies/src/main/java/com/salesforce/apollo/fireflies/ViewManagement.java b/fireflies/src/main/java/com/salesforce/apollo/fireflies/ViewManagement.java index b4d33a528..2008a0c05 100644 --- a/fireflies/src/main/java/com/salesforce/apollo/fireflies/ViewManagement.java +++ b/fireflies/src/main/java/com/salesforce/apollo/fireflies/ViewManagement.java @@ -188,6 +188,48 @@ BloomFilter getJoinsBff(long seed, double p) { return bff; } + /** + * Initiate the view change + */ + void initiateViewChange() { + view.stable(() -> { + if (vote.get() != null) { + log.trace("Vote already cast for: {} on: {}", currentView(), node.getId()); + return; + } + // Use pending rebuttals as a proxy for stability + if (view.hasPendingRebuttals()) { + log.debug("Pending rebuttals in view: {} on: {}", currentView(), node.getId()); + view.scheduleViewChange(1); + return; + } + view.scheduleFinalizeViewChange(); + if (!isObserver(node.getId())) { + log.debug("Initiating (non observer) view change: {} joins: {} leaves: {} on: {}", currentView(), + joins.size(), view.streamShunned().count(), node.getId()); + return; + } + log.debug("Initiating (observer) view change vote: {} joins: {} leaves: {} observers: {} on: {}", + currentView(), joins.size(), view.streamShunned().count(), observersList(), node.getId()); + final var builder = ViewChange.newBuilder() + .setObserver(node.getId().toDigeste()) + .setCurrent(currentView().toDigeste()) + .setAttempt(attempt.getAndIncrement()) + .addAllJoins(joins.keySet().stream().map(Digest::toDigeste).toList()); + view.streamShunned().map(Digest::toDigeste).forEach(builder::addLeaves); + ViewChange change = builder.build(); + vote.set(change); + var signature = node.sign(change.toByteString()); + final var viewChange = SignedViewChange.newBuilder() + .setChange(change) + .setSignature(signature.toSig()) + .build(); + view.initiate(viewChange); + log.trace("View change vote: {} joins: {} leaves: {} on: {}", currentView(), change.getJoinsCount(), + change.getLeavesCount(), node.getId()); + }); + } + /** * Install the new view * @@ -427,17 +469,22 @@ void maybeViewChange() { if (!joined()) { return; } - if (context.size() == 1 && joins.size() < context.getRingCount() - 1) { - log.trace("Cannot form cluster: {} with: {} members, required > 3 on: {}", currentView(), - joins.size() + context.size(), node.getId()); + if (bootstrap && context.size() == 1 && joins.size() < context.getRingCount() - 1) { + log.trace("Cannot form cluster: {} with: {} members, required >= {}} on: {}", currentView(), + joins.size() + context.size(), context.getRingCount(), node.getId()); view.scheduleViewChange(); return; + } else if (!bootstrap) { + if (context.size() < context.getRingCount()) { + log.trace("Cannot initiate view change: {} with: {} members, required >= {}} on: {}", currentView(), + joins.size() + context.size(), context.getRingCount(), node.getId()); + view.scheduleViewChange(); + return; + } } if ((context.offlineCount() > 0 || !joins.isEmpty())) { initiateViewChange(); } else { - // log.trace("No view change: {} joins: {} leaves: {} on: {}", currentView(), joins.size(), - // view.streamShunned().count(), node.getId()); view.scheduleViewChange(); } } @@ -556,48 +603,6 @@ void start(CompletableFuture onJoin, boolean bootstrap) { this.bootstrap = bootstrap; } - /** - * Initiate the view change - */ - private void initiateViewChange() { - view.stable(() -> { - if (vote.get() != null) { - log.trace("Vote already cast for: {} on: {}", currentView(), node.getId()); - return; - } - // Use pending rebuttals as a proxy for stability - if (view.hasPendingRebuttals()) { - log.debug("Pending rebuttals in view: {} on: {}", currentView(), node.getId()); - view.scheduleViewChange(1); // 1 TTL round to check again - return; - } - view.scheduleFinalizeViewChange(); - if (!isObserver(node.getId())) { - log.debug("Initiating (non observer) view change: {} joins: {} leaves: {} on: {}", currentView(), - joins.size(), view.streamShunned().count(), node.getId()); - return; - } - log.debug("Initiating (observer) view change vote: {} joins: {} leaves: {} observers: {} on: {}", - currentView(), joins.size(), view.streamShunned().count(), observersList(), node.getId()); - final var builder = ViewChange.newBuilder() - .setObserver(node.getId().toDigeste()) - .setCurrent(currentView().toDigeste()) - .setAttempt(attempt.getAndIncrement()) - .addAllJoins(joins.keySet().stream().map(Digest::toDigeste).toList()); - view.streamShunned().map(Digest::toDigeste).forEach(builder::addLeaves); - ViewChange change = builder.build(); - vote.set(change); - var signature = node.sign(change.toByteString()); - final var viewChange = SignedViewChange.newBuilder() - .setChange(change) - .setSignature(signature.toSig()) - .build(); - view.initiate(viewChange); - log.trace("View change vote: {} joins: {} leaves: {} on: {}", currentView(), change.getJoinsCount(), - change.getLeavesCount(), node.getId()); - }); - } - /** * @return true if the receiver is part of the BFT Observers of this group */ diff --git a/fireflies/src/test/java/com/salesforce/apollo/fireflies/ChurnTest.java b/fireflies/src/test/java/com/salesforce/apollo/fireflies/ChurnTest.java index b689e159d..913623451 100644 --- a/fireflies/src/test/java/com/salesforce/apollo/fireflies/ChurnTest.java +++ b/fireflies/src/test/java/com/salesforce/apollo/fireflies/ChurnTest.java @@ -152,7 +152,7 @@ public void churn() throws Exception { toStart.forEach(view -> view.start(() -> countdown.get().countDown(), gossipDuration, seeds)); - success = countdown.get().await(30, TimeUnit.SECONDS); + success = countdown.get().await(60, TimeUnit.SECONDS); failed = testViews.stream().filter(e -> { if (e.getContext().activeCount() != testViews.size()) return true; From 1aab2ab1c088811a7d8babc5b2177c8f1902c23c Mon Sep 17 00:00:00 2001 From: Hellblazer Date: Wed, 26 Jun 2024 19:26:53 -0700 Subject: [PATCH 16/32] Max length BLOBs for all binary values. reuse/close scheduled executors. Pretty massive refactoring of the stereotomy and thoth schemas. :: sigh :: --- .../com/salesforce/apollo/choam/CHOAM.java | 61 ++-- .../apollo/choam/GenesisAssembly.java | 60 ++-- .../com/salesforce/apollo/choam/Producer.java | 11 +- .../com/salesforce/apollo/choam/Session.java | 4 + .../choam/support/CheckpointAssembler.java | 2 +- .../apollo/choam/GenesisAssemblyTest.java | 4 +- .../apollo/cryptography/Digest.java | 18 +- .../apollo/cryptography/DigestAlgorithm.java | 14 +- .../salesforce/apollo/ethereal/Creator.java | 2 + .../apollo/ethereal/DataSource.java | 2 +- .../salesforce/apollo/ethereal/Ethereal.java | 3 +- .../ethereal/memberships/ChRbcGossip.java | 5 +- .../apollo/ethereal/EtherealTest.java | 7 +- .../salesforce/apollo/fireflies/Binding.java | 17 +- .../com/salesforce/apollo/fireflies/View.java | 38 +-- .../apollo/fireflies/ViewManagement.java | 31 +- .../apollo/gorgoneion/Gorgoneion.java | 15 +- .../apollo/archipelago/MtlsClient.java | 5 + .../apollo/archipelago/MtlsServer.java | 8 +- .../salesforce/apollo/ring/SliceIterator.java | 13 +- .../apollo/context/ContextTests.java | 6 +- .../apollo/context/StaticContextTest.java | 6 +- .../model/stereotomy/ShardedKERLTest.java | 6 +- .../src/main/resources/sql-state/internal.xml | 2 +- .../main/resources/stereotomy/stereotomy.xml | 173 +++++----- schemas/src/main/resources/thoth/thoth.xml | 210 ++++++------ .../apollo/stereotomy/db/UniKERL.java | 35 +- .../apollo/stereotomy/StereotomyTests.java | 4 +- .../com/salesforce/apollo/thoth/KerlDHT.java | 2 +- .../salesforce/apollo/thoth/KerlSpace.java | 53 +-- .../apollo/thoth/KerlSpaceTest.java | 4 +- .../com/salesforce/apollo/thoth/KerlTest.java | 4 +- .../java/com/chiralbehaviors/tron/Fsm.java | 313 ++++++++---------- 33 files changed, 577 insertions(+), 561 deletions(-) diff --git a/choam/src/main/java/com/salesforce/apollo/choam/CHOAM.java b/choam/src/main/java/com/salesforce/apollo/choam/CHOAM.java index 10a1b8fc7..55f0aac29 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/CHOAM.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/CHOAM.java @@ -100,9 +100,11 @@ public class CHOAM { private final TransSubmission txnSubmission = new TransSubmission(); private final AtomicReference view = new AtomicReference<>(); private final PendingViews pendingViews = new PendingViews(); + private final ScheduledExecutorService scheduler; private volatile AtomicBoolean ongoingJoin; public CHOAM(Parameters params) { + scheduler = Executors.newScheduledThreadPool(1, Thread.ofVirtual().factory()); this.store = new Store(params.digestAlgorithm(), params.mvBuilder().clone().build()); this.params = params; executions = Executors.newVirtualThreadPerTaskExecutor(); @@ -351,11 +353,19 @@ public void stop() { if (!started.compareAndSet(true, false)) { return; } - session.cancelAll(); try { linear.shutdownNow(); } catch (Throwable e) { } + try { + scheduler.shutdownNow(); + } catch (Throwable e) { + } + session.cancelAll(); + try { + session.stop(); + } catch (Throwable e) { + } try { executions.shutdownNow(); } catch (Throwable e) { @@ -1393,31 +1403,28 @@ private void join(View view) { var joined = new AtomicInteger(); var halt = new AtomicBoolean(false); ongoingJoin = halt; - Thread.ofVirtual().start(Utils.wrapped(() -> { - log.trace("Starting join of: {} diadem {} on: {}", nextViewId.get(), Digest.from(view.getDiadem()), - params.member().getId()); - - var scheduler = Executors.newSingleThreadScheduledExecutor(Thread.ofVirtual().factory()); - AtomicReference action = new AtomicReference<>(); - var attempts = new AtomicInteger(); - action.set(() -> { - log.trace("Join attempt: {} halt: {} joined: {} majority: {} on: {}", attempts.incrementAndGet(), - halt.get(), joined.get(), view.getMajority(), params.member().getId()); - if (!halt.get() & joined.get() < view.getMajority()) { - join(view, servers, joined); - if (joined.get() >= view.getMajority()) { - ongoingJoin = null; - log.trace("Finished join of: {} diadem: {} joins: {} on: {}", nextViewId.get(), - Digest.from(view.getDiadem()), joined.get(), params.member().getId()); - } else if (!halt.get()) { - log.trace("Rescheduling join of: {} diadem: {} joins: {} on: {}", nextViewId.get(), - Digest.from(view.getDiadem()), joined.get(), params.member().getId()); - scheduler.schedule(action.get(), 50, TimeUnit.MILLISECONDS); - } + log.trace("Starting join of: {} diadem {} on: {}", nextViewId.get(), Digest.from(view.getDiadem()), + params.member().getId()); + var scheduler = Executors.newSingleThreadScheduledExecutor(Thread.ofVirtual().factory()); + AtomicReference action = new AtomicReference<>(); + var attempts = new AtomicInteger(); + action.set(() -> { + log.trace("Join attempt: {} halt: {} joined: {} majority: {} on: {}", attempts.incrementAndGet(), + halt.get(), joined.get(), view.getMajority(), params.member().getId()); + if (!halt.get() & joined.get() < view.getMajority()) { + join(view, servers, joined); + if (joined.get() >= view.getMajority()) { + ongoingJoin = null; + log.trace("Finished join of: {} diadem: {} joins: {} on: {}", nextViewId.get(), + Digest.from(view.getDiadem()), joined.get(), params.member().getId()); + } else if (!halt.get()) { + log.trace("Rescheduling join of: {} diadem: {} joins: {} on: {}", nextViewId.get(), + Digest.from(view.getDiadem()), joined.get(), params.member().getId()); + scheduler.schedule(action.get(), 50, TimeUnit.MILLISECONDS); } - }); - scheduler.schedule(action.get(), 50, TimeUnit.MILLISECONDS); - }, log())); + } + }); + scheduler.schedule(action.get(), 50, TimeUnit.MILLISECONDS); } private void join(View view, Collection members, AtomicInteger joined) { @@ -1520,7 +1527,7 @@ private class Associate extends Administration { var pv = pendingViews(); producer = new Producer(nextViewId.get(), new ViewContext(context, params, pv, signer, validators, constructBlock()), - head.get(), checkpoint.get(), getLabel()); + head.get(), checkpoint.get(), getLabel(), scheduler); producer.start(); } @@ -1575,7 +1582,7 @@ private Formation() { .setVm(inView) .setSignature(params.member().sign(inView.toByteString()).toSig()) .build(); - assembly = new GenesisAssembly(vc, comm, svm, getLabel()); + assembly = new GenesisAssembly(vc, comm, svm, getLabel(), scheduler); log.info("Setting next view id to genesis: {} on: {}", params.genesisViewId(), params.member().getId()); nextViewId.set(params.genesisViewId()); } else { diff --git a/choam/src/main/java/com/salesforce/apollo/choam/GenesisAssembly.java b/choam/src/main/java/com/salesforce/apollo/choam/GenesisAssembly.java index a13861270..85638710f 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/GenesisAssembly.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/GenesisAssembly.java @@ -16,7 +16,6 @@ import com.salesforce.apollo.choam.support.HashedBlock; import com.salesforce.apollo.choam.support.HashedCertifiedBlock; import com.salesforce.apollo.choam.support.HashedCertifiedBlock.NullBlock; -import com.salesforce.apollo.choam.support.OneShot; import com.salesforce.apollo.context.Context; import com.salesforce.apollo.context.StaticContext; import com.salesforce.apollo.cryptography.Digest; @@ -32,7 +31,7 @@ import java.security.PublicKey; import java.util.*; -import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; @@ -45,26 +44,27 @@ * @author hal.hildebrand */ public class GenesisAssembly implements Genesis { - private static final Logger log = LoggerFactory.getLogger(GenesisAssembly.class); - private final Ethereal controller; - private final ChRbcGossip coordinator; - private final SignedViewMember genesisMember; - private final Map nextAssembly; - private final AtomicBoolean published = new AtomicBoolean(); - private final Map slate = new ConcurrentHashMap<>(); - private final AtomicBoolean started = new AtomicBoolean(); - private final Transitions transitions; - private final ViewContext view; - private final Map witnesses = new ConcurrentHashMap<>(); - private final OneShot ds; - private final List pendingValidations = new ArrayList<>(); - private volatile Thread blockingThread; - private volatile HashedBlock reconfiguration; + private static final Logger log = LoggerFactory.getLogger(GenesisAssembly.class); + private final Ethereal controller; + private final ChRbcGossip coordinator; + private final SignedViewMember genesisMember; + private final Map nextAssembly; + private final AtomicBoolean published = new AtomicBoolean(); + private final Map slate = new ConcurrentHashMap<>(); + private final AtomicBoolean started = new AtomicBoolean(); + private final Transitions transitions; + private final ViewContext view; + private final Map witnesses = new ConcurrentHashMap<>(); + private final BlockingDeque ds; + private final List pendingValidations = new ArrayList<>(); + private final ScheduledExecutorService scheduler; + private volatile HashedBlock reconfiguration; public GenesisAssembly(ViewContext vc, CommonCommunications comms, SignedViewMember genesisMember, - String label) { + String label, ScheduledExecutorService scheduler) { view = vc; - ds = new OneShot(); + this.scheduler = scheduler; + ds = new LinkedBlockingDeque<>(1024); Digest hash = view.context().getId(); nextAssembly = ((Set) ((Context) view.pendingViews().last().context()).bftSubset( hash)).stream().collect(Collectors.toMap(Member::getId, m -> m)); @@ -99,7 +99,8 @@ public GenesisAssembly(ViewContext vc, CommonCommunications comms, transitions::process, transitions::nextEpoch, label); coordinator = new ChRbcGossip(reContext.getId(), params().member(), nextAssembly.values(), controller.processor(), params().communications(), - params().metrics() == null ? null : params().metrics().getGensisMetrics()); + params().metrics() == null ? null : params().metrics().getGensisMetrics(), + scheduler); log.debug("Genesis Assembly: {} recontext: {} next assembly: {} on: {}", view.context().getId(), reContext.getId(), nextAssembly.keySet(), params().member().getId()); } @@ -117,7 +118,7 @@ public void certify() { var validate = view.generateValidation(reconfiguration); log.debug("Certifying genesis block: {} for: {} slate: {} on: {}", reconfiguration.hash, view.context().getId(), slate.keySet().stream().sorted().toList(), params().member().getId()); - ds.setValue(validate.toByteString()); + ds.add(validate.toByteString()); witnesses.put(params().member(), validate); pendingValidations.forEach(v -> certify(v)); } @@ -140,7 +141,7 @@ public void gather() { var join = Join.newBuilder().setMember(genesisMember).setKerl(params().kerl().get()).build(); slate.put(params().member().getId(), join); - ds.setValue(join.toByteString()); + ds.add(join.toByteString()); coordinator.start(params().producer().gossipDuration()); controller.start(); } @@ -230,11 +231,6 @@ public void stop() { log.trace("Stopping genesis assembly: {} on: {}", view.context().getId(), params().member().getId()); coordinator.stop(); controller.stop(); - final var cur = blockingThread; - blockingThread = null; - if (cur != null) { - cur.interrupt(); - } } private void certify(Validate v) { @@ -262,11 +258,11 @@ private DataSource dataSource() { return ByteString.EMPTY; } try { - blockingThread = Thread.currentThread(); - final var take = ds.get(); - return take; - } finally { - blockingThread = null; + var data = ds.poll(100, TimeUnit.MILLISECONDS); + return data == null ? ByteString.EMPTY : data; + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return null; } }; } diff --git a/choam/src/main/java/com/salesforce/apollo/choam/Producer.java b/choam/src/main/java/com/salesforce/apollo/choam/Producer.java index 9538fe8ce..b1405093d 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/Producer.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/Producer.java @@ -52,13 +52,15 @@ public class Producer { private final Transitions transitions; private final ViewContext view; private final Digest nextViewId; - private final Executor serialize = Executors.newSingleThreadExecutor( - Thread.ofVirtual().factory()); + private final ExecutorService serialize = Executors.newSingleThreadExecutor(); private final ViewAssembly assembly; private final int maxEpoch; + private final ScheduledExecutorService scheduler; private volatile boolean assembled = false; - public Producer(Digest nextViewId, ViewContext view, HashedBlock lastBlock, HashedBlock checkpoint, String label) { + public Producer(Digest nextViewId, ViewContext view, HashedBlock lastBlock, HashedBlock checkpoint, String label, + ScheduledExecutorService scheduler) { + this.scheduler = scheduler; assert view != null; this.view = view; this.previousBlock.set(lastBlock); @@ -99,7 +101,7 @@ public Producer(Digest nextViewId, ViewContext view, HashedBlock lastBlock, Hash controller = new Ethereal(config.build(), params().producer().maxBatchByteSize() + (8 * 1024), ds, this::serial, this::newEpoch, label); coordinator = new ChRbcGossip(view.context().getId(), params().member(), view.membership(), - controller.processor(), params().communications(), producerMetrics); + controller.processor(), params().communications(), producerMetrics, scheduler); log.debug("Roster for: {} is: {} on: {}", getViewId(), view.roster(), params().member().getId()); var onConsensus = new CompletableFuture(); @@ -148,6 +150,7 @@ public void stop() { return; } log.trace("Closing producer for: {} on: {}", getViewId(), params().member().getId()); + serialize.shutdown(); controller.stop(); coordinator.stop(); ds.close(); diff --git a/choam/src/main/java/com/salesforce/apollo/choam/Session.java b/choam/src/main/java/com/salesforce/apollo/choam/Session.java index 70d2148ea..f21d652ad 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/Session.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/Session.java @@ -117,6 +117,10 @@ public void setView(HashedCertifiedBlock v) { } } + public void stop() { + scheduler.shutdown(); + } + /** * Submit a transaction. * diff --git a/choam/src/main/java/com/salesforce/apollo/choam/support/CheckpointAssembler.java b/choam/src/main/java/com/salesforce/apollo/choam/support/CheckpointAssembler.java index 4d4f8fd90..e66789285 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/support/CheckpointAssembler.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/support/CheckpointAssembler.java @@ -118,7 +118,7 @@ private void gossip(ScheduledExecutorService scheduler, Duration duration) { diadem.compactWrapped(), member.getId()); var ringer = new SliceIterator<>("Assembly[%s:%s]".formatted(diadem.compactWrapped(), member.getId()), member, - committee, comms); + committee, comms, scheduler); ringer.iterate((link) -> { log.debug("Requesting Seeding from: {} on: {}", link.getMember().getId(), member.getId()); return gossip(link); diff --git a/choam/src/test/java/com/salesforce/apollo/choam/GenesisAssemblyTest.java b/choam/src/test/java/com/salesforce/apollo/choam/GenesisAssemblyTest.java index b113b13c8..c44a2416e 100644 --- a/choam/src/test/java/com/salesforce/apollo/choam/GenesisAssemblyTest.java +++ b/choam/src/test/java/com/salesforce/apollo/choam/GenesisAssemblyTest.java @@ -40,6 +40,7 @@ import java.time.Duration; import java.util.*; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -175,7 +176,8 @@ public Block reconfigure(Map joining, Digest nextViewId, HashedBlo .setVm(vm) .setSignature(((SigningMember) m).sign(vm.toByteString()).toSig()) .build(); - genii.put(m, new GenesisAssembly(view, comms.get(m), svm, m.getId().toString())); + genii.put(m, new GenesisAssembly(view, comms.get(m), svm, m.getId().toString(), + Executors.newScheduledThreadPool(1, Thread.ofVirtual().factory()))); }); try { diff --git a/cryptography/src/main/java/com/salesforce/apollo/cryptography/Digest.java b/cryptography/src/main/java/com/salesforce/apollo/cryptography/Digest.java index 2c1f7761d..15bb3aa2b 100644 --- a/cryptography/src/main/java/com/salesforce/apollo/cryptography/Digest.java +++ b/cryptography/src/main/java/com/salesforce/apollo/cryptography/Digest.java @@ -16,13 +16,15 @@ import java.util.UUID; import java.util.stream.Stream; +import static com.salesforce.apollo.cryptography.DigestAlgorithm.EMPTY; + /** * A computed digest * * @author hal.hildebrand */ public class Digest implements Comparable { - public static final Digest NONE = new Digest(DigestAlgorithm.NONE, new byte[0]) { + public static final Digest NONE = new Digest(DigestAlgorithm.NONE, new long[] { 0L }) { @Override public String toString() { @@ -74,11 +76,15 @@ public Digest(DigestAlgorithm algo, long[] hash) { public Digest(Digeste d) { algorithm = DigestAlgorithm.fromDigestCode(d.getType()); - assert d.getHashCount() == algorithm.longLength(); - hash = new long[d.getHashCount()]; - int i = 0; - for (long l : d.getHashList()) { - hash[i++] = l; + if (algorithm.equals(DigestAlgorithm.NONE)) { + hash = EMPTY; + } else { + assert d.getHashCount() == algorithm.longLength(); + hash = new long[d.getHashCount()]; + int i = 0; + for (long l : d.getHashList()) { + hash[i++] = l; + } } } diff --git a/cryptography/src/main/java/com/salesforce/apollo/cryptography/DigestAlgorithm.java b/cryptography/src/main/java/com/salesforce/apollo/cryptography/DigestAlgorithm.java index 25bb3adb7..7f64934d7 100644 --- a/cryptography/src/main/java/com/salesforce/apollo/cryptography/DigestAlgorithm.java +++ b/cryptography/src/main/java/com/salesforce/apollo/cryptography/DigestAlgorithm.java @@ -222,7 +222,12 @@ public byte digestCode() { @Override public int digestLength() { - return 0; + return 8; + } + + @Override + public int longLength() { + return 1; } @Override @@ -237,12 +242,12 @@ public Digest getOrigin() { @Override public byte[] hashOf(byte[] bytes, int len) { - return EMPTY; + return EMPTY_BYTES; } @Override public byte[] hashOf(InputStream is) { - return EMPTY; + return EMPTY_BYTES; } }, SHA2_256 { @Override @@ -316,7 +321,8 @@ public int digestLength() { public static final DigestAlgorithm DEFAULT = BLAKE2B_256; public static final long MAX_UNSIGNED_LONG = -1L; - private static final byte[] EMPTY = new byte[0]; + public static final long[] EMPTY = new long[] { 0L }; + public static final byte[] EMPTY_BYTES = new byte[] { 0, 0, 0, 0, 0, 0, 0, 0 }; private static final long[] LAST_32 = new long[4]; private static final long[] LAST_64 = new long[8]; private static final ThreadLocal MESSAGE_DIGEST = ThreadLocal.withInitial(() -> new DigestCache()); diff --git a/ethereal/src/main/java/com/salesforce/apollo/ethereal/Creator.java b/ethereal/src/main/java/com/salesforce/apollo/ethereal/Creator.java index 2a8d8de5f..307851b7e 100644 --- a/ethereal/src/main/java/com/salesforce/apollo/ethereal/Creator.java +++ b/ethereal/src/main/java/com/salesforce/apollo/ethereal/Creator.java @@ -173,8 +173,10 @@ assert parentsOnPreviousLevel(u) >= quorum : "Parents: " + Arrays.asList(u.paren private ByteString getData(int level) { if (level < conf.lastLevel()) { if (ds != null) { + log.info("Requesting timing unit: {} on: {}", level, conf.logLabel()); return ds.getData(); } + log.info("No datasource for timing unit: {} on: {}", level, conf.logLabel()); return ByteString.EMPTY; } Unit timingUnit = lastTiming.poll(); diff --git a/ethereal/src/main/java/com/salesforce/apollo/ethereal/DataSource.java b/ethereal/src/main/java/com/salesforce/apollo/ethereal/DataSource.java index a61d8afb5..4c19d3a07 100644 --- a/ethereal/src/main/java/com/salesforce/apollo/ethereal/DataSource.java +++ b/ethereal/src/main/java/com/salesforce/apollo/ethereal/DataSource.java @@ -10,8 +10,8 @@ /** * @author hal.hildebrand - * */ +@FunctionalInterface public interface DataSource { ByteString getData(); diff --git a/ethereal/src/main/java/com/salesforce/apollo/ethereal/Ethereal.java b/ethereal/src/main/java/com/salesforce/apollo/ethereal/Ethereal.java index 6cac3b2bb..67ea81f4e 100644 --- a/ethereal/src/main/java/com/salesforce/apollo/ethereal/Ethereal.java +++ b/ethereal/src/main/java/com/salesforce/apollo/ethereal/Ethereal.java @@ -73,7 +73,7 @@ private Ethereal(String label, Config conf, int maxSerializedSize, DataSource ds } private static ThreadPoolExecutor consumer(String label) { - return new ThreadPoolExecutor(1, 1, 1, TimeUnit.SECONDS, new PriorityBlockingQueue<>(), + return new ThreadPoolExecutor(1, 1, 10, TimeUnit.MINUTES, new PriorityBlockingQueue<>(), Thread.ofVirtual().name("Ethereal Consumer[" + label + "]").factory(), (r, t) -> log.trace("Shutdown, cannot consume unit")); } @@ -218,6 +218,7 @@ public void stop() { } log.trace("Stopping Ethereal on: {}", config.logLabel()); completeIt(); + consumer.shutdown(); consumer.getQueue().clear(); // Flush any pending consumers creator.stop(); epochs.values().forEach(epoch::close); diff --git a/ethereal/src/main/java/com/salesforce/apollo/ethereal/memberships/ChRbcGossip.java b/ethereal/src/main/java/com/salesforce/apollo/ethereal/memberships/ChRbcGossip.java index e44fdc1ff..482b267a5 100644 --- a/ethereal/src/main/java/com/salesforce/apollo/ethereal/memberships/ChRbcGossip.java +++ b/ethereal/src/main/java/com/salesforce/apollo/ethereal/memberships/ChRbcGossip.java @@ -60,7 +60,7 @@ public class ChRbcGossip { private volatile ScheduledFuture scheduled; public ChRbcGossip(Digest id, SigningMember member, Collection membership, Processor processor, - Router communications, EtherealMetrics m) { + Router communications, EtherealMetrics m, ScheduledExecutorService scheduler) { this.processor = processor; this.member = member; this.metrics = m; @@ -68,7 +68,8 @@ public ChRbcGossip(Digest id, SigningMember member, Collection membershi comm = communications.create(member, id, terminal, getClass().getCanonicalName(), r -> new GossiperServer(communications.getClientIdentityProvider(), metrics, r), getCreate(metrics), Gossiper.getLocalLoopback(member)); - ring = new SliceIterator<>("ChRbcGossip[%s on: %s]".formatted(id, member.getId()), member, membership, comm); + ring = new SliceIterator<>("ChRbcGossip[%s on: %s]".formatted(id, member.getId()), member, membership, comm, + scheduler); } /** diff --git a/ethereal/src/test/java/com/salesforce/apollo/ethereal/EtherealTest.java b/ethereal/src/test/java/com/salesforce/apollo/ethereal/EtherealTest.java index 7f8443c27..9a7e21c76 100644 --- a/ethereal/src/test/java/com/salesforce/apollo/ethereal/EtherealTest.java +++ b/ethereal/src/test/java/com/salesforce/apollo/ethereal/EtherealTest.java @@ -34,6 +34,7 @@ import java.util.*; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.IntStream; @@ -137,7 +138,8 @@ public void unbounded() throws NoSuchAlgorithmException, InterruptedException, I }, "Test: " + i); var gossiper = new ChRbcGossip(context.getId(), (SigningMember) member, members, controller.processor(), - com, metrics); + com, metrics, + Executors.newScheduledThreadPool(1, Thread.ofVirtual().factory())); gossipers.add(gossiper); dataSources.add(ds); controllers.add(controller); @@ -265,7 +267,8 @@ private void one(int iteration) }, "Test: " + i); var gossiper = new ChRbcGossip(context.getId(), (SigningMember) member, members, controller.processor(), - com, metrics); + com, metrics, + Executors.newScheduledThreadPool(1, Thread.ofVirtual().factory())); gossipers.add(gossiper); dataSources.add(ds); controllers.add(controller); diff --git a/fireflies/src/main/java/com/salesforce/apollo/fireflies/Binding.java b/fireflies/src/main/java/com/salesforce/apollo/fireflies/Binding.java index c8f84fdb5..5da22769e 100644 --- a/fireflies/src/main/java/com/salesforce/apollo/fireflies/Binding.java +++ b/fireflies/src/main/java/com/salesforce/apollo/fireflies/Binding.java @@ -58,10 +58,12 @@ class Binding { private final Parameters params; private final List seeds; private final View view; + private final ScheduledExecutorService scheduler; public Binding(View view, List seeds, Duration duration, DynamicContext context, CommonCommunications approaches, Node node, Parameters params, - FireflyMetrics metrics, DigestAlgorithm digestAlgo) { + FireflyMetrics metrics, DigestAlgorithm digestAlgo, ScheduledExecutorService scheduler) { + this.scheduler = scheduler; assert node != null; this.view = view; this.duration = duration; @@ -98,7 +100,7 @@ void seeding() { .map(nw -> view.new Participant(nw)) .filter(p -> !node.getId().equals(p.getId())) .collect(Collectors.toList()); - var seedlings = new SliceIterator<>("Seedlings", node, bootstrappers, approaches); + var seedlings = new SliceIterator<>("Seedlings", node, bootstrappers, approaches, scheduler); AtomicReference reseed = new AtomicReference<>(); var scheduler = Executors.newScheduledThreadPool(1, Thread.ofVirtual().factory()); reseed.set(() -> { @@ -110,6 +112,8 @@ void seeding() { if (!redirect.isDone()) { scheduler.schedule(() -> Thread.ofVirtual().start(Utils.wrapped(reseed.get(), log)), params.retryDelay().toNanos(), TimeUnit.NANOSECONDS); + } else { + scheduler.shutdown(); } }, params.retryDelay()); }); @@ -141,7 +145,7 @@ private boolean complete(CompletableFuture redirect, Optional gateway, HashMultiset trusts, - Set initialSeedSet, Digest v, int majority, CompletableFuture complete, + Set iss, Digest v, int majority, CompletableFuture complete, AtomicInteger remaining, ListenableFuture futureSailor) { if (complete.isDone()) { return; @@ -182,6 +186,7 @@ private void complete(Member member, CompletableFuture gateway, HashMulti return; } trusts.add(new Bootstrapping(g.getTrust())); + var initialSeedSet = new HashSet<>(iss); initialSeedSet.addAll(g.getInitialSeedSetList()); log.trace("Initial seed set count: {} view: {} from: {} on: {}", g.getInitialSeedSetCount(), v, member.getId(), node.getId()); @@ -322,7 +327,7 @@ private void join(Redirect redirect, Digest v, Duration duration) { this.context.rebalance(cardinality); node.nextNote(v); - final var redirecting = new SliceIterator<>("Gateways", node, sample, approaches); + final var redirecting = new SliceIterator<>("Gateways", node, sample, approaches, scheduler); var majority = redirect.getBootstrap() ? 1 : Context.minimalQuorum(redirect.getRings(), this.context.getBias()); final var join = join(v); var scheduler = Executors.newScheduledThreadPool(1, Thread.ofVirtual().factory()); @@ -336,9 +341,11 @@ private void join(Redirect redirect, Digest v, Duration duration) { complete.whenComplete((success, error) -> { if (error != null) { log.info("Failed Join on: {}", node.getId(), error); + scheduler.shutdown(); return; } if (success) { + scheduler.shutdown(); return; } log.info("Join unsuccessful, abandoned: {} trusts: {} on: {}", abandon.get(), trusts.entrySet() @@ -359,6 +366,7 @@ private void join(Redirect redirect, Digest v, Duration duration) { scheduler.schedule(() -> Thread.ofVirtual().start(Utils.wrapped(regate.get(), log)), Entropy.nextBitsStreamLong(params.retryDelay().toNanos()), TimeUnit.NANOSECONDS); } else { + scheduler.shutdown(); log.error("Failed to join view: {} cannot obtain majority Gateway on: {}", view, node.getId()); view.stop(); } @@ -375,6 +383,7 @@ private void join(Redirect redirect, Digest v, Duration duration) { log.debug( "Abandoning Gateway view: {} abandons: {} majority: {} reseeding on: {}", v, abandon.get(), majority, node.getId()); + scheduler.shutdown(); complete.completeExceptionally(new TimeoutException("Failed Join")); seeding(); } diff --git a/fireflies/src/main/java/com/salesforce/apollo/fireflies/View.java b/fireflies/src/main/java/com/salesforce/apollo/fireflies/View.java index 044d8e854..9005abd90 100644 --- a/fireflies/src/main/java/com/salesforce/apollo/fireflies/View.java +++ b/fireflies/src/main/java/com/salesforce/apollo/fireflies/View.java @@ -109,6 +109,7 @@ public class View { private final ViewManagement viewManagement; private final EventValidation validation; private final Verifiers verifiers; + private final ScheduledExecutorService scheduler; private volatile ScheduledFuture futureGossip; private volatile boolean boostrap = false; @@ -122,13 +123,14 @@ public View(DynamicContext context, ControlledIdentifierMember memb public View(DynamicContext context, ControlledIdentifierMember member, String endpoint, EventValidation validation, Verifiers verifiers, Router communications, Parameters params, Router gateway, DigestAlgorithm digestAlgo, FireflyMetrics metrics) { + scheduler = Executors.newScheduledThreadPool(1, Thread.ofVirtual().factory()); this.metrics = metrics; this.params = params; this.digestAlgo = digestAlgo; this.context = context; this.roundTimers = new RoundScheduler(String.format("Timers for: %s", context.getId()), context.timeToLive()); this.node = new Node(member, endpoint); - viewManagement = new ViewManagement(this, context, params, metrics, node, digestAlgo); + viewManagement = new ViewManagement(this, context, params, metrics, node, digestAlgo, scheduler); var service = new Service(); this.comm = communications.create(node, context.getId(), service, r -> new FfServer(communications.getClientIdentityProvider(), r, metrics), @@ -217,12 +219,10 @@ public void start(CompletableFuture onJoin, Duration d, List seedpod context.clear(); node.reset(); - var scheduler = Executors.newScheduledThreadPool(1, Thread.ofVirtual().factory()); - var initial = Entropy.nextBitsStreamLong(d.toNanos()); - scheduler.schedule(() -> Thread.ofVirtual() - .start(Utils.wrapped( - () -> new Binding(this, seeds, d, context, approaches, node, params, metrics, - digestAlgo).seeding(), log)), initial, TimeUnit.NANOSECONDS); + Thread.ofVirtual() + .start(Utils.wrapped( + () -> new Binding(this, seeds, d, context, approaches, node, params, metrics, digestAlgo, + scheduler).seeding(), log)); log.info("{} started on: {}", context.getId(), node.getId()); } @@ -247,6 +247,7 @@ public void stop() { comm.deregister(context.getId()); pendingRebuttals.clear(); context.active().forEach(context::offline); + scheduler.shutdown(); final var current = futureGossip; futureGossip = null; if (current != null) { @@ -494,10 +495,7 @@ void resetBootstrapView() { } void schedule(final Duration duration) { - var scheduler = Executors.newScheduledThreadPool(1, Thread.ofVirtual().factory()); - futureGossip = scheduler.schedule( - () -> Thread.ofVirtual().start(Utils.wrapped(() -> gossip(duration, scheduler), log)), - Entropy.nextBitsStreamLong(duration.toNanos()), TimeUnit.NANOSECONDS); + Thread.ofVirtual().start(Utils.wrapped(() -> gossip(duration), log)); } void scheduleClearObservations() { @@ -1046,9 +1044,8 @@ private BloomFilter getObservationsBff(long seed, double p) { * Execute one round of gossip * * @param duration - * @param scheduler */ - private void gossip(Duration duration, ScheduledExecutorService scheduler) { + private void gossip(Duration duration) { if (!started.get()) { return; } @@ -1057,7 +1054,7 @@ private void gossip(Duration duration, ScheduledExecutorService scheduler) { tick(); } gossiper.execute((link, ring) -> gossip(link, ring), - (result, destination) -> gossip(result, destination, duration, scheduler)); + (result, destination) -> gossip(result, destination, duration)); } /** @@ -1066,10 +1063,9 @@ private void gossip(Duration duration, ScheduledExecutorService scheduler) { * @param result * @param destination * @param duration - * @param scheduler */ private void gossip(Optional result, RingCommunications.Destination destination, - Duration duration, ScheduledExecutorService scheduler) { + Duration duration) { try { if (result.isPresent()) { final var member = destination.member(); @@ -1115,9 +1111,13 @@ private void gossip(Optional result, RingCommunications.Destination Thread.ofVirtual().start(Utils.wrapped(() -> gossip(duration, scheduler), log)), duration.toNanos(), - TimeUnit.NANOSECONDS); + try { + futureGossip = scheduler.schedule( + () -> Thread.ofVirtual().start(Utils.wrapped(() -> gossip(duration), log)), duration.toNanos(), + TimeUnit.NANOSECONDS); + } catch (RejectedExecutionException e) { + // ignore + } } } diff --git a/fireflies/src/main/java/com/salesforce/apollo/fireflies/ViewManagement.java b/fireflies/src/main/java/com/salesforce/apollo/fireflies/ViewManagement.java index 2008a0c05..f850a852a 100644 --- a/fireflies/src/main/java/com/salesforce/apollo/fireflies/ViewManagement.java +++ b/fireflies/src/main/java/com/salesforce/apollo/fireflies/ViewManagement.java @@ -15,7 +15,6 @@ import com.salesforce.apollo.fireflies.Binding.Bound; import com.salesforce.apollo.fireflies.View.Node; import com.salesforce.apollo.fireflies.View.Participant; -import com.salesforce.apollo.fireflies.comm.gossip.Fireflies; import com.salesforce.apollo.fireflies.proto.*; import com.salesforce.apollo.fireflies.proto.Update.Builder; import com.salesforce.apollo.membership.Member; @@ -63,17 +62,19 @@ public class ViewManagement { private final AtomicReference vote = new AtomicReference<>(); private final Lock joinLock = new ReentrantLock(); private final AtomicReference currentView = new AtomicReference<>(); + private final ScheduledExecutorService scheduler; private volatile boolean bootstrap; private volatile CompletableFuture onJoined; ViewManagement(View view, DynamicContext context, Parameters params, FireflyMetrics metrics, Node node, - DigestAlgorithm digestAlgo) { + DigestAlgorithm digestAlgo, ScheduledExecutorService scheduler) { this.node = node; this.view = view; this.context = context; this.params = params; this.metrics = metrics; this.digestAlgo = digestAlgo; + this.scheduler = scheduler; resetBootstrapView(); bootstrapView = currentView.get(); } @@ -400,7 +401,8 @@ void join(Join join, Digest from, StreamObserver responseObserver, Time log.debug("Member pending join: {} view: {} context: {} on: {}", from, currentView(), context.getId(), node.getId()); var enjoining = new SliceIterator<>("Enjoining[%s:%s]".formatted(currentView(), from), node, - observers.stream().map(context::getActiveMember).toList(), view.comm); + observers.stream().map(context::getActiveMember).toList(), view.comm, + scheduler); enjoining.iterate(t -> t.enjoin(join), (_, _, _, _) -> true, () -> { }, Duration.ofMillis(1)); }); @@ -497,29 +499,6 @@ List observersList() { return observers().stream().toList(); } - void populate(List sample) { - var populate = new SliceIterator("Populate: " + context.getId(), node, sample, view.comm); - var repopulate = new AtomicReference(); - var scheduler = Executors.newScheduledThreadPool(1, Thread.ofVirtual().factory()); - repopulate.set(() -> populate.iterate((link) -> view.gossip(link, 0), (futureSailor, _, link, m) -> { - futureSailor.ifPresent(g -> { - if (!g.getRedirect().equals(SignedNote.getDefaultInstance())) { - final Participant member = (Participant) link.getMember(); - view.stable(() -> view.redirect(member, g, 0)); - } else { - view.stable(() -> view.processUpdates(g)); - } - }); - return !joined(); - }, () -> { - if (!joined()) { - scheduler.schedule(() -> Thread.ofVirtual().start(Utils.wrapped(repopulate.get(), log)), - params.populateDuration().toNanos(), TimeUnit.NANOSECONDS); - } - }, params.populateDuration())); - repopulate.get().run(); - } - JoinGossip.Builder processJoins(BloomFilter bff) { JoinGossip.Builder builder = JoinGossip.newBuilder(); diff --git a/gorgoneion/src/main/java/com/salesforce/apollo/gorgoneion/Gorgoneion.java b/gorgoneion/src/main/java/com/salesforce/apollo/gorgoneion/Gorgoneion.java index b23bf786d..475b71b03 100644 --- a/gorgoneion/src/main/java/com/salesforce/apollo/gorgoneion/Gorgoneion.java +++ b/gorgoneion/src/main/java/com/salesforce/apollo/gorgoneion/Gorgoneion.java @@ -51,9 +51,7 @@ import java.util.HashSet; import java.util.Optional; import java.util.Set; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ExecutionException; +import java.util.concurrent.*; import java.util.function.Predicate; import static com.salesforce.apollo.stereotomy.event.protobuf.ProtobufEventFactory.digestOf; @@ -73,6 +71,7 @@ public class Gorgoneion { private final Parameters parameters; private final Predicate verifier; private final boolean bootstrap; + private final ScheduledExecutorService scheduler; public Gorgoneion(boolean bootstrap, Predicate verifier, Parameters parameters, ControlledIdentifierMember member, Context context, ProtoEventObserver observer, @@ -89,6 +88,7 @@ public Gorgoneion(boolean bootstrap, Predicate verifier, Para this.context = context; this.parameters = parameters; this.observer = observer; + this.scheduler = Executors.newScheduledThreadPool(1, Thread.ofVirtual().factory()); admissionsComm = admissionsRouter.create(member, context.getId(), new Admit(), ":admissions", r -> new AdmissionsServer(admissionsRouter.getClientIdentityProvider(), @@ -159,7 +159,8 @@ private SignedNonce generateNonce(KERL_ application) { var successors = context.size() == 1 ? Collections.singletonList(member) : context.bftSubset(digestOf(ident, parameters.digestAlgorithm())); final var majority = context.size() == 1 ? 1 : context.majority(); - final var redirecting = new SliceIterator<>("Nonce Endorsement", member, successors, endorsementComm); + final var redirecting = new SliceIterator<>("Nonce Endorsement", member, successors, endorsementComm, + scheduler); Set endorsements = Collections.newSetFromMap(new ConcurrentHashMap<>()); var generated = new CompletableFuture(); redirecting.iterate((link) -> { @@ -221,7 +222,8 @@ private CompletableFuture notarize(Credentials credentials, Validat var successors = context.bftSubset(digestOf(identifier.toIdent(), parameters.digestAlgorithm())); final var majority = context.size() == 1 ? 1 : context.majority(); - SliceIterator redirecting = new SliceIterator<>("Enrollment", member, successors, endorsementComm); + SliceIterator redirecting = new SliceIterator<>("Enrollment", member, successors, endorsementComm, + scheduler); var completed = new HashSet(); var result = new CompletableFuture(); redirecting.iterate((link) -> { @@ -253,7 +255,8 @@ private Validations register(Credentials request) { var successors = context.bftSubset(digestOf(identifier.toIdent(), parameters.digestAlgorithm())); final var majority = context.size() == 1 ? 1 : context.majority(); - final var redirecting = new SliceIterator<>("Credential verification", member, successors, endorsementComm); + final var redirecting = new SliceIterator<>("Credential verification", member, successors, endorsementComm, + scheduler); var verifications = new HashSet(); redirecting.iterate((link) -> { log.debug("Validating credentials for: {} contacting: {} on: {}", identifier, link.getMember().getId(), diff --git a/memberships/src/main/java/com/salesforce/apollo/archipelago/MtlsClient.java b/memberships/src/main/java/com/salesforce/apollo/archipelago/MtlsClient.java index 8f5b2a7be..8b5d4957e 100644 --- a/memberships/src/main/java/com/salesforce/apollo/archipelago/MtlsClient.java +++ b/memberships/src/main/java/com/salesforce/apollo/archipelago/MtlsClient.java @@ -19,12 +19,16 @@ import io.netty.handler.ssl.ClientAuth; import java.net.SocketAddress; +import java.util.concurrent.Executor; +import java.util.concurrent.Executors; /** * @author hal.hildebrand */ public class MtlsClient { + private final static Executor executor = Executors.newVirtualThreadPerTaskExecutor(); + private final ManagedChannel channel; public MtlsClient(SocketAddress address, ClientAuth clientAuth, String alias, ClientContextSupplier supplier, @@ -32,6 +36,7 @@ public MtlsClient(SocketAddress address, ClientAuth clientAuth, String alias, Cl Limiter limiter = new GrpcClientLimiterBuilder().blockOnLimit(false).build(); channel = NettyChannelBuilder.forAddress(address) + // .executor(executor) .withOption(ChannelOption.TCP_NODELAY, true) .sslContext(supplier.forClient(clientAuth, alias, validator, MtlsServer.TL_SV1_3)) .intercept(new ConcurrencyLimitClientInterceptor(limiter, diff --git a/memberships/src/main/java/com/salesforce/apollo/archipelago/MtlsServer.java b/memberships/src/main/java/com/salesforce/apollo/archipelago/MtlsServer.java index 4cf97b97a..3cac8d621 100644 --- a/memberships/src/main/java/com/salesforce/apollo/archipelago/MtlsServer.java +++ b/memberships/src/main/java/com/salesforce/apollo/archipelago/MtlsServer.java @@ -140,15 +140,15 @@ public static SslContext forServer(ClientAuth clientAuth, String alias, X509Cert public RouterImpl router(ServerConnectionCache.Builder cacheBuilder, Supplier serverLimit, LimitsRegistry limitsRegistry, List interceptors, Predicate validator, ExecutorService executor) { - if (executor == null) { - executor = UnsafeExecutors.newVirtualThreadPerTaskExecutor(); - } + // if (executor == null) { + // executor = Executors.newVirtualThreadPerTaskExecutor(); + // } var limitsBuilder = new GrpcServerLimiterBuilder().limit(serverLimit.get()); if (limitsRegistry != null) { limitsBuilder.metricRegistry(limitsRegistry); } NettyServerBuilder serverBuilder = NettyServerBuilder.forAddress(epProvider.getBindAddress()) - .executor(executor) + // .executor(executor) .withOption(ChannelOption.SO_REUSEADDR, true) .sslContext(supplier.forServer(ClientAuth.REQUIRE, epProvider.getAlias(), diff --git a/memberships/src/main/java/com/salesforce/apollo/ring/SliceIterator.java b/memberships/src/main/java/com/salesforce/apollo/ring/SliceIterator.java index cc0b7ec45..0942cee04 100644 --- a/memberships/src/main/java/com/salesforce/apollo/ring/SliceIterator.java +++ b/memberships/src/main/java/com/salesforce/apollo/ring/SliceIterator.java @@ -19,10 +19,7 @@ import java.io.IOException; import java.time.Duration; import java.util.*; -import java.util.concurrent.CopyOnWriteArrayList; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; +import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Consumer; import java.util.function.Function; @@ -170,8 +167,12 @@ private void proceed(Runnable onMajority, final boolean allow, Runnable proceed, } } log.trace("Proceeding for: <{}> on: {}", label, member.getId()); - scheduler.schedule(() -> Thread.ofVirtual().start(Utils.wrapped(proceed, log)), frequency.toNanos(), - TimeUnit.NANOSECONDS); + try { + scheduler.schedule(() -> Thread.ofVirtual().start(Utils.wrapped(proceed, log)), frequency.toNanos(), + TimeUnit.NANOSECONDS); + } catch (RejectedExecutionException e) { + // ignore + } } else { log.trace("Termination for: <{}> on: {}", label, member.getId()); } diff --git a/memberships/src/test/java/com/salesforce/apollo/context/ContextTests.java b/memberships/src/test/java/com/salesforce/apollo/context/ContextTests.java index 03903b5d3..f9f5d4dc8 100644 --- a/memberships/src/test/java/com/salesforce/apollo/context/ContextTests.java +++ b/memberships/src/test/java/com/salesforce/apollo/context/ContextTests.java @@ -41,11 +41,11 @@ public void consistency() throws Exception { } List predecessors = context.predecessors(members.get(0)); - assertEquals(predecessors.get(2), members.get(9)); + assertEquals(predecessors.get(2), members.get(3)); List successors = context.successors(members.get(1)); - assertEquals(members.get(0), successors.get(0)); - assertEquals(members.get(1), context.successor(1, members.get(0))); + assertEquals(members.get(8), successors.get(0)); + assertEquals(members.get(9), context.successor(1, members.get(0))); } @Test diff --git a/memberships/src/test/java/com/salesforce/apollo/context/StaticContextTest.java b/memberships/src/test/java/com/salesforce/apollo/context/StaticContextTest.java index 7a1b2d5bd..2a165d626 100644 --- a/memberships/src/test/java/com/salesforce/apollo/context/StaticContextTest.java +++ b/memberships/src/test/java/com/salesforce/apollo/context/StaticContextTest.java @@ -36,11 +36,11 @@ public void consistency() throws Exception { var context = prototype.asStatic(); var predecessors = context.predecessors(members.get(0).getId()); - assertEquals(members.get(9), predecessors.get(2)); + assertEquals(members.get(3), predecessors.get(2)); var successors = context.successors(members.get(1).getId()); - assertEquals(members.get(0), successors.get(0)); - assertEquals(members.get(1), context.successor(1, members.get(0).getId())); + assertEquals(members.get(8), successors.get(0)); + assertEquals(members.get(9), context.successor(1, members.get(0).getId())); } @Test diff --git a/model/src/test/java/com/salesforce/apollo/model/stereotomy/ShardedKERLTest.java b/model/src/test/java/com/salesforce/apollo/model/stereotomy/ShardedKERLTest.java index 2334f18e4..10827bcb0 100644 --- a/model/src/test/java/com/salesforce/apollo/model/stereotomy/ShardedKERLTest.java +++ b/model/src/test/java/com/salesforce/apollo/model/stereotomy/ShardedKERLTest.java @@ -63,10 +63,10 @@ public void delegated() throws Exception { ControlledIdentifier identifier = opti2; // identifier - assertTrue(identifier.getIdentifier() instanceof SelfAddressingIdentifier); + assertInstanceOf(SelfAddressingIdentifier.class, identifier.getIdentifier()); var sap = (SelfAddressingIdentifier) identifier.getIdentifier(); assertEquals(DigestAlgorithm.DEFAULT, sap.getDigest().getAlgorithm()); - assertEquals("092126af01f80ca28e7a99bbdce229c029be3bbfcb791e29ccb7a64e8019a36f", + assertEquals("6000b1b611a2a6cb27b6c569c056cf56e04da4905168020fc054d133181d379b", Hex.hex(sap.getDigest().getBytes())); assertEquals(1, ((Unweighted) identifier.getSigningThreshold()).getThreshold()); @@ -108,7 +108,7 @@ public void delegated() throws Exception { assertEquals(lastEstablishmentEvent.hash(DigestAlgorithm.DEFAULT), identifier.getDigest()); // lastEvent - assertTrue(kerl.getKeyEvent(identifier.getLastEvent()) == null); + assertNull(kerl.getKeyEvent(identifier.getLastEvent())); // delegation assertTrue(identifier.getDelegatingIdentifier().isPresent()); diff --git a/schemas/src/main/resources/sql-state/internal.xml b/schemas/src/main/resources/sql-state/internal.xml index a3b7e3c74..f3a5ee776 100644 --- a/schemas/src/main/resources/sql-state/internal.xml +++ b/schemas/src/main/resources/sql-state/internal.xml @@ -32,7 +32,7 @@ - + diff --git a/schemas/src/main/resources/stereotomy/stereotomy.xml b/schemas/src/main/resources/stereotomy/stereotomy.xml index 187ea9e74..a77cd4070 100644 --- a/schemas/src/main/resources/stereotomy/stereotomy.xml +++ b/schemas/src/main/resources/stereotomy/stereotomy.xml @@ -1,175 +1,176 @@ + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xmlns="http://www.liquibase.org/xml/ns/dbchangelog" + xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-4.4.xsd"> create schema if not exists stereotomy + schemaName="stereotomy"> + type="IDENTITY"> + primaryKey="true" primaryKeyName="identifier_pkey"/> - - + + + tableName="identifier" schemaName="stereotomy"/> + schemaName="stereotomy"> + type="IDENTITY"> + primaryKey="true" primaryKeyName="coordinates_pkey"/> - + - - + + - + - + - alter table stereotomy.coordinates add constraint - coordinates_ilk_validate check (ilk in ('dip', 'drt', 'icp', - 'ixn', 'nan', 'rct', 'vrc', 'rot')) + alter table stereotomy.coordinates + add constraint + coordinates_ilk_validate check (ilk in ('dip', 'drt', 'icp', + 'ixn', 'nan', 'rct', 'vrc', 'rot')) + + onDelete="CASCADE" baseColumnNames="identifier" + baseTableName="coordinates" baseTableSchemaName="stereotomy" + constraintName="coordinates_identifier_fk" + referencedTableName="identifier" referencedColumnNames="id" + referencedTableSchemaName="stereotomy"/> + columnNames="identifier, sequence_number, digest, ilk" + tableName="coordinates" schemaName="stereotomy" constraintName="unique_id_dig_seq_ilk"/> + schemaName="stereotomy"> + primaryKey="true"/> - - + + - - + + - + + columnNames="digest" + tableName="event" schemaName="stereotomy"/> + schemaName="stereotomy"> + primaryKey="true"/> - + + onDelete="CASCADE" baseColumnNames="identifier" + baseTableName="current_key_state" + baseTableSchemaName="stereotomy" + constraintName="current_key_state_identifier_fk" + referencedTableName="identifier" referencedColumnNames="id" + referencedTableSchemaName="stereotomy"/> + onDelete="CASCADE" baseColumnNames="current" + baseTableName="current_key_state" + baseTableSchemaName="stereotomy" + constraintName="current_key_state_current_fk" + referencedTableName="event" + referencedColumnNames="coordinates" + referencedTableSchemaName="stereotomy"/> + schemaName="stereotomy"> + primaryKey="true"/> + primaryKey="true"/> - - + + + onDelete="CASCADE" baseColumnNames="for" + baseTableName="receipt" baseTableSchemaName="stereotomy" + constraintName="receipt_for_fk" referencedTableName="coordinates" + referencedColumnNames="id" + referencedTableSchemaName="stereotomy"/> + schemaName="stereotomy"> - + - - + + + onDelete="CASCADE" baseColumnNames="for" + baseTableName="attachment" baseTableSchemaName="stereotomy" + constraintName="attachment_for_fk" + referencedTableName="coordinates" + referencedColumnNames="id" + referencedTableSchemaName="stereotomy"/> + tableName="attachment" schemaName="stereotomy"/> + schemaName="stereotomy"> + primaryKey="true"/> - + + primaryKey="true"/> - - + + + onDelete="CASCADE" baseColumnNames="for" + baseTableName="validation" baseTableSchemaName="stereotomy" + constraintName="validation_for_fk" referencedTableName="coordinates" + referencedColumnNames="id" + referencedTableSchemaName="stereotomy"/> - \ No newline at end of file + diff --git a/schemas/src/main/resources/thoth/thoth.xml b/schemas/src/main/resources/thoth/thoth.xml index 29b4f5ef2..57520eb93 100644 --- a/schemas/src/main/resources/thoth/thoth.xml +++ b/schemas/src/main/resources/thoth/thoth.xml @@ -1,125 +1,125 @@ - - - create schema if not exists thoth - - - - - - - - - - - + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xmlns="http://www.liquibase.org/xml/ns/dbchangelog" + xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-4.4.xsd"> + + + create schema if not exists thoth + + + + + + + + + + + - + schemaName="thoth" tableName="identifier_location_hash"> + + + + + + + + + + + + + + + + + + + + + + alter table thoth.pending_coordinates + add constraint + pending_coordinates_ilk_validate check (ilk in ('dip', 'drt', 'icp', + 'ixn', 'nan', 'rct', 'vrc', 'rot')) + + + + + + + + - - - - - - - - - - - - - - - - - - - alter table thoth.pending_coordinates add constraint - pending_coordinates_ilk_validate check (ilk in ('dip', 'drt', 'icp', - 'ixn', 'nan', 'rct', 'vrc', 'rot')) - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + - + - - + + - + baseColumnNames="coordinates" baseTableName="pending_attachment" + baseTableSchemaName="thoth" + constraintName="pending_attachment_coordinates_fk" + referencedTableName="pending_coordinates" referencedColumnNames="id" + referencedTableSchemaName="thoth"/> + - + - - + + - - - \ No newline at end of file + baseColumnNames="coordinates" baseTableName="pending_validations" + baseTableSchemaName="thoth" + constraintName="pending_validations_coordinates_fk" + referencedTableName="pending_coordinates" referencedColumnNames="id" + referencedTableSchemaName="thoth"/> + + + diff --git a/stereotomy/src/main/java/com/salesforce/apollo/stereotomy/db/UniKERL.java b/stereotomy/src/main/java/com/salesforce/apollo/stereotomy/db/UniKERL.java index 484ccd8dc..c4100c582 100644 --- a/stereotomy/src/main/java/com/salesforce/apollo/stereotomy/db/UniKERL.java +++ b/stereotomy/src/main/java/com/salesforce/apollo/stereotomy/db/UniKERL.java @@ -62,7 +62,7 @@ * @author hal.hildebrand */ abstract public class UniKERL implements DigestKERL { - public static final byte[] DIGEST_NONE_BYTES = Digest.NONE.toDigeste().toByteArray(); + public static final byte[] DIGEST_NONE_BYTES = Digest.NONE.getBytes(); private static final Logger log = LoggerFactory.getLogger(UniKERL.class); protected final DigestAlgorithm digestAlgorithm; @@ -103,7 +103,7 @@ public static void append(DSLContext dsl, AttachmentEvent attachment) { .join(IDENTIFIER) .on(IDENTIFIER.PREFIX.eq(coordinates.getIdentifier().toIdent().toByteArray())) .where(COORDINATES.IDENTIFIER.eq(IDENTIFIER.ID)) - .and(COORDINATES.DIGEST.eq(coordinates.getDigest().toDigeste().toByteArray())) + .and(COORDINATES.DIGEST.eq(coordinates.getDigest().getBytes())) .and(COORDINATES.SEQUENCE_NUMBER.eq(coordinates.getSequenceNumber().toBigInteger())) .and(COORDINATES.ILK.eq(coordinates.getIlk())) .fetchOne(); @@ -147,7 +147,7 @@ public static void append(DSLContext context, KeyEvent event, KeyState newState, IDENTIFIER.PREFIX.eq(prevCoords.getIdentifier().toIdent().toByteArray())); final var prev = context.select(COORDINATES.ID) .from(COORDINATES) - .where(COORDINATES.DIGEST.eq(prevCoords.getDigest().toDigeste().toByteArray())) + .where(COORDINATES.DIGEST.eq(prevCoords.getDigest().getBytes())) .and(COORDINATES.IDENTIFIER.eq(preIdentifier)) .and(COORDINATES.SEQUENCE_NUMBER.eq(prevCoords.getSequenceNumber().toBigInteger())) .and(COORDINATES.ILK.eq(prevCoords.getIlk())) @@ -203,7 +203,7 @@ public static void append(DSLContext context, KeyEvent event, KeyState newState, .join(IDENTIFIER) .on(IDENTIFIER.PREFIX.eq(coordinates.getIdentifier().toIdent().toByteArray())) .where(COORDINATES.IDENTIFIER.eq(IDENTIFIER.ID)) - .and(COORDINATES.DIGEST.eq(coordinates.getDigest().toDigeste().toByteArray())) + .and(COORDINATES.DIGEST.eq(coordinates.getDigest().getBytes())) .and(COORDINATES.SEQUENCE_NUMBER.eq(coordinates.getSequenceNumber().toBigInteger())) .and(COORDINATES.ILK.eq(coordinates.getIlk())) .fetchOne() @@ -214,7 +214,7 @@ public static void append(DSLContext context, KeyEvent event, KeyState newState, try { context.insertInto(EVENT) .set(EVENT.COORDINATES, id) - .set(EVENT.DIGEST, digest.toDigeste().toByteArray()) + .set(EVENT.DIGEST, digest.getBytes()) .set(EVENT.CONTENT, compress(event.getBytes())) .set(EVENT.CURRENT_STATE, compress(newState.getBytes())) .execute(); @@ -261,7 +261,6 @@ public static void appendValidations(DSLContext dsl, EventCoordinates coordinate return; } final var identBytes = coordinates.getIdentifier().toIdent().toByteArray(); - try { dsl.mergeInto(IDENTIFIER) .using(dsl.selectOne()) @@ -290,10 +289,16 @@ public static void appendValidations(DSLContext dsl, EventCoordinates coordinate .join(IDENTIFIER) .on(IDENTIFIER.PREFIX.eq(coordinates.getIdentifier().toIdent().toByteArray())) .where(COORDINATES.IDENTIFIER.eq(IDENTIFIER.ID)) - .and(COORDINATES.DIGEST.eq(coordinates.getDigest().toDigeste().toByteArray())) + .and(COORDINATES.DIGEST.eq(coordinates.getDigest().getBytes())) .and(COORDINATES.SEQUENCE_NUMBER.eq(coordinates.getSequenceNumber().toBigInteger())) .and(COORDINATES.ILK.eq(coordinates.getIlk())) .fetchOne(); + if (id == null) { + log.info("Not found!: {}", coordinates.getIdentifier()); + } + } + if (id == null) { + log.info("Not inserted!: {}", coordinates.getIdentifier()); } var result = new AtomicInteger(); var l = id.value1(); @@ -349,7 +354,7 @@ public static void initialize(DSLContext dsl) { .using(context.selectOne()) .on(EVENT.COORDINATES.eq(0L)) .whenNotMatchedThenInsert(EVENT.COORDINATES, EVENT.DIGEST, EVENT.CONTENT) - .values(0L, ecNone.getDigest().toDigeste().toByteArray(), compress(new byte[0])) + .values(0L, ecNone.getDigest().getBytes(), compress(new byte[0])) .execute(); context.mergeInto(COORDINATES) @@ -357,8 +362,8 @@ public static void initialize(DSLContext dsl) { .on(COORDINATES.ID.eq(0L)) .whenNotMatchedThenInsert(COORDINATES.ID, COORDINATES.DIGEST, COORDINATES.IDENTIFIER, COORDINATES.SEQUENCE_NUMBER, COORDINATES.ILK) - .values(0L, ecNone.getDigest().toDigeste().toByteArray(), 0L, - ecNone.getSequenceNumber().toBigInteger(), ecNone.getIlk()) + .values(0L, ecNone.getDigest().getBytes(), 0L, ecNone.getSequenceNumber().toBigInteger(), + ecNone.getIlk()) .execute(); }); } @@ -374,7 +379,7 @@ public Attachment getAttachment(EventCoordinates coordinates) { .join(IDENTIFIER) .on(IDENTIFIER.PREFIX.eq(coordinates.getIdentifier().toIdent().toByteArray())) .where(COORDINATES.IDENTIFIER.eq(IDENTIFIER.ID)) - .and(COORDINATES.DIGEST.eq(coordinates.getDigest().toDigeste().toByteArray())) + .and(COORDINATES.DIGEST.eq(coordinates.getDigest().getBytes())) .and(COORDINATES.SEQUENCE_NUMBER.eq(coordinates.getSequenceNumber().toBigInteger())) .and(COORDINATES.ILK.eq(coordinates.getIlk())) .and(COORDINATES.SEQUENCE_NUMBER.eq(coordinates.getSequenceNumber().toBigInteger())) @@ -430,7 +435,7 @@ public KeyEvent getKeyEvent(Digest digest) { .from(EVENT) .join(COORDINATES) .on(COORDINATES.ID.eq(EVENT.COORDINATES)) - .where(EVENT.DIGEST.eq(digest.toDigeste().toByteString().toByteArray())) + .where(EVENT.DIGEST.eq(digest.getBytes())) .fetchOptional() .map(r -> toKeyEvent(decompress(r.value1()), r.value2())) .orElse(null); @@ -447,7 +452,7 @@ public KeyEvent getKeyEvent(EventCoordinates coordinates) { .join(IDENTIFIER) .on(IDENTIFIER.PREFIX.eq(coordinates.getIdentifier().toIdent().toByteArray())) .where(COORDINATES.IDENTIFIER.eq(IDENTIFIER.ID)) - .and(COORDINATES.DIGEST.eq(coordinates.getDigest().toDigeste().toByteArray())) + .and(COORDINATES.DIGEST.eq(coordinates.getDigest().getBytes())) .and(COORDINATES.SEQUENCE_NUMBER.eq(coordinates.getSequenceNumber().toBigInteger())) .and(COORDINATES.ILK.eq(coordinates.getIlk())) .and(COORDINATES.SEQUENCE_NUMBER.eq(coordinates.getSequenceNumber().toBigInteger())) @@ -467,7 +472,7 @@ public KeyState getKeyState(EventCoordinates coordinates) { .join(IDENTIFIER) .on(IDENTIFIER.PREFIX.eq(coordinates.getIdentifier().toIdent().toByteArray())) .where(COORDINATES.IDENTIFIER.eq(IDENTIFIER.ID)) - .and(COORDINATES.DIGEST.eq(coordinates.getDigest().toDigeste().toByteArray())) + .and(COORDINATES.DIGEST.eq(coordinates.getDigest().getBytes())) .and(COORDINATES.SEQUENCE_NUMBER.eq(coordinates.getSequenceNumber().toBigInteger())) .and(COORDINATES.ILK.eq(coordinates.getIlk())) .fetchOptional() @@ -539,7 +544,7 @@ public Map getValidations(EventCoordinates coordi .join(IDENTIFIER) .on(IDENTIFIER.PREFIX.eq(coordinates.getIdentifier().toIdent().toByteArray())) .where(COORDINATES.IDENTIFIER.eq(IDENTIFIER.ID)) - .and(COORDINATES.DIGEST.eq(coordinates.getDigest().toDigeste().toByteArray())) + .and(COORDINATES.DIGEST.eq(coordinates.getDigest().getBytes())) .and(COORDINATES.SEQUENCE_NUMBER.eq(coordinates.getSequenceNumber().toBigInteger())) .and(COORDINATES.ILK.eq(coordinates.getIlk())) .and(COORDINATES.SEQUENCE_NUMBER.eq(coordinates.getSequenceNumber().toBigInteger())) diff --git a/stereotomy/src/test/java/com/salesforce/apollo/stereotomy/StereotomyTests.java b/stereotomy/src/test/java/com/salesforce/apollo/stereotomy/StereotomyTests.java index 862dc7efb..7bf98d25e 100644 --- a/stereotomy/src/test/java/com/salesforce/apollo/stereotomy/StereotomyTests.java +++ b/stereotomy/src/test/java/com/salesforce/apollo/stereotomy/StereotomyTests.java @@ -109,7 +109,7 @@ public void newIdentifier() throws Exception { assertInstanceOf(SelfAddressingIdentifier.class, identifier.getIdentifier()); var sap = (SelfAddressingIdentifier) identifier.getIdentifier(); assertEquals(DigestAlgorithm.DEFAULT, sap.getDigest().getAlgorithm()); - assertEquals("4cb6958622749694aedff3d48b8e402524562813bf2bdd11894a528edc965b4d", + assertEquals("9f207937484c3e47833f7f78d22974b3b543f6363c138ebeda20793c4a5c082b", Hex.hex(sap.getDigest().getBytes())); assertEquals(1, ((Unweighted) identifier.getSigningThreshold()).getThreshold()); @@ -170,7 +170,7 @@ public void newIdentifierFromIdentifier() throws Exception { assertInstanceOf(SelfAddressingIdentifier.class, identifier.getIdentifier()); var sap = (SelfAddressingIdentifier) identifier.getIdentifier(); assertEquals(DigestAlgorithm.DEFAULT, sap.getDigest().getAlgorithm()); - assertEquals("092126af01f80ca28e7a99bbdce229c029be3bbfcb791e29ccb7a64e8019a36f", + assertEquals("6000b1b611a2a6cb27b6c569c056cf56e04da4905168020fc054d133181d379b", Hex.hex(sap.getDigest().getBytes())); assertEquals(1, ((Unweighted) identifier.getSigningThreshold()).getThreshold()); diff --git a/thoth/src/main/java/com/salesforce/apollo/thoth/KerlDHT.java b/thoth/src/main/java/com/salesforce/apollo/thoth/KerlDHT.java index adf0e8d3d..831a11412 100644 --- a/thoth/src/main/java/com/salesforce/apollo/thoth/KerlDHT.java +++ b/thoth/src/main/java/com/salesforce/apollo/thoth/KerlDHT.java @@ -146,7 +146,7 @@ public KerlDHT(Duration operationsFrequency, Context context, this.connectionPool = connectionPool; kerlPool = new UniKERLDirectPooled(connectionPool, digestAlgorithm); this.reconcile = new RingCommunications<>(this.context, member, reconcileComms); - this.kerlSpace = new KerlSpace(connectionPool, member.getId()); + this.kerlSpace = new KerlSpace(connectionPool, member.getId(), digestAlgorithm); initializeSchema(); kerl = new CachingKERL(f -> { diff --git a/thoth/src/main/java/com/salesforce/apollo/thoth/KerlSpace.java b/thoth/src/main/java/com/salesforce/apollo/thoth/KerlSpace.java index 7efd785d5..895306046 100644 --- a/thoth/src/main/java/com/salesforce/apollo/thoth/KerlSpace.java +++ b/thoth/src/main/java/com/salesforce/apollo/thoth/KerlSpace.java @@ -30,6 +30,7 @@ import org.jooq.Record1; import org.jooq.SQLDialect; import org.jooq.exception.DataAccessException; +import org.jooq.exception.IntegrityConstraintViolationException; import org.jooq.impl.DSL; import org.joou.ULong; import org.slf4j.Logger; @@ -61,10 +62,12 @@ public class KerlSpace { private static final Logger log = LoggerFactory.getLogger(KerlSpace.class); private final JdbcConnectionPool connectionPool; private final Digest member; + private final DigestAlgorithm algorithm; - public KerlSpace(JdbcConnectionPool connectionPool, Digest member) { + public KerlSpace(JdbcConnectionPool connectionPool, Digest member, DigestAlgorithm algorithm) { this.connectionPool = connectionPool; this.member = member; + this.algorithm = algorithm; } public static void upsert(DSLContext dsl, EventCoords coordinates, Attachment attachment, Digest member) { @@ -77,7 +80,7 @@ public static void upsert(DSLContext dsl, EventCoords coordinates, Attachment at Record1 id; try { id = dsl.insertInto(PENDING_COORDINATES) - .set(PENDING_COORDINATES.DIGEST, coordinates.getDigest().toByteArray()) + .set(PENDING_COORDINATES.DIGEST, Digest.from(coordinates.getDigest()).getBytes()) .set(PENDING_COORDINATES.IDENTIFIER, dsl.select(IDENTIFIER.ID).from(IDENTIFIER).where(IDENTIFIER.PREFIX.eq(identBytes))) .set(PENDING_COORDINATES.ILK, coordinates.getIlk()) @@ -85,14 +88,14 @@ public static void upsert(DSLContext dsl, EventCoords coordinates, Attachment at ULong.valueOf(coordinates.getSequenceNumber()).toBigInteger()) .returningResult(PENDING_COORDINATES.ID) .fetchOne(); - } catch (DataAccessException e) { + } catch (IntegrityConstraintViolationException e) { // Already exists id = dsl.select(PENDING_COORDINATES.ID) .from(PENDING_COORDINATES) .join(IDENTIFIER) .on(IDENTIFIER.PREFIX.eq(coordinates.getIdentifier().toByteArray())) .where(PENDING_COORDINATES.IDENTIFIER.eq(IDENTIFIER.ID)) - .and(PENDING_COORDINATES.DIGEST.eq(coordinates.getDigest().toByteArray())) + .and(PENDING_COORDINATES.DIGEST.eq(Digest.from(coordinates.getDigest()).getBytes())) .and(PENDING_COORDINATES.SEQUENCE_NUMBER.eq( ULong.valueOf(coordinates.getSequenceNumber()).toBigInteger())) .and(PENDING_COORDINATES.ILK.eq(coordinates.getIlk())) @@ -118,7 +121,7 @@ public static void upsert(DSLContext context, KeyEvent event, DigestAlgorithm di long id; try { id = context.insertInto(PENDING_COORDINATES) - .set(PENDING_COORDINATES.DIGEST, prevCoords.getDigest().toDigeste().toByteArray()) + .set(PENDING_COORDINATES.DIGEST, prevCoords.getDigest().getBytes()) .set(PENDING_COORDINATES.IDENTIFIER, context.select(IDENTIFIER.ID).from(IDENTIFIER).where(IDENTIFIER.PREFIX.eq(identBytes))) .set(PENDING_COORDINATES.ILK, event.getIlk()) @@ -126,26 +129,30 @@ public static void upsert(DSLContext context, KeyEvent event, DigestAlgorithm di .returningResult(PENDING_COORDINATES.ID) .fetchOne() .value1(); - } catch (DataAccessException e) { + } catch (IntegrityConstraintViolationException e) { // Already exists var coordinates = event.getCoordinates(); - id = context.select(PENDING_COORDINATES.ID) - .from(PENDING_COORDINATES) - .join(IDENTIFIER) - .on(IDENTIFIER.PREFIX.eq(coordinates.getIdentifier().toIdent().toByteArray())) - .where(PENDING_COORDINATES.IDENTIFIER.eq(IDENTIFIER.ID)) - .and(PENDING_COORDINATES.DIGEST.eq(coordinates.getDigest().toDigeste().toByteArray())) - .and(PENDING_COORDINATES.SEQUENCE_NUMBER.eq(coordinates.getSequenceNumber().toBigInteger())) - .and(PENDING_COORDINATES.ILK.eq(coordinates.getIlk())) - .fetchOne() - .value1(); + var result = context.select(PENDING_COORDINATES.ID) + .from(PENDING_COORDINATES) + .join(IDENTIFIER) + .on(IDENTIFIER.PREFIX.eq(identBytes)) + .where(PENDING_COORDINATES.IDENTIFIER.eq(IDENTIFIER.ID)) + .and(PENDING_COORDINATES.DIGEST.eq(prevCoords.getDigest().getBytes())) + .and( + PENDING_COORDINATES.SEQUENCE_NUMBER.eq(coordinates.getSequenceNumber().toBigInteger())) + .and(PENDING_COORDINATES.ILK.eq(coordinates.getIlk())) + .fetchOne(); + if (result == null) { + throw new IllegalStateException("upsert failed", e); + } + id = result.value1(); } final var digest = event.hash(digestAlgorithm); try { context.insertInto(PENDING_EVENT) .set(PENDING_EVENT.COORDINATES, id) - .set(PENDING_EVENT.DIGEST, digest.toDigeste().toByteArray()) + .set(PENDING_EVENT.DIGEST, digest.getBytes()) .set(PENDING_EVENT.EVENT, event.getBytes()) .execute(); } catch (DataAccessException e) { @@ -173,7 +180,7 @@ public static void upsert(DSLContext dsl, Validations validations, Digest member Record1 id; try { id = dsl.insertInto(PENDING_COORDINATES) - .set(PENDING_COORDINATES.DIGEST, coordinates.getDigest().toByteArray()) + .set(PENDING_COORDINATES.DIGEST, Digest.from(coordinates.getDigest()).getBytes()) .set(PENDING_COORDINATES.IDENTIFIER, dsl.select(IDENTIFIER.ID).from(IDENTIFIER).where(IDENTIFIER.PREFIX.eq(identBytes))) .set(PENDING_COORDINATES.ILK, coordinates.getIlk()) @@ -190,7 +197,7 @@ public static void upsert(DSLContext dsl, Validations validations, Digest member .join(IDENTIFIER) .on(IDENTIFIER.PREFIX.eq(coordinates.getIdentifier().toByteArray())) .where(PENDING_COORDINATES.IDENTIFIER.eq(IDENTIFIER.ID)) - .and(PENDING_COORDINATES.DIGEST.eq(coordinates.getDigest().toByteArray())) + .and(PENDING_COORDINATES.DIGEST.eq(Digest.from(coordinates.getDigest()).getBytes())) .and(PENDING_COORDINATES.SEQUENCE_NUMBER.eq( ULong.valueOf(coordinates.getSequenceNumber()).toBigInteger())) .and(PENDING_COORDINATES.ILK.eq(coordinates.getIlk())) @@ -402,13 +409,7 @@ private Stream eventDigestsIn(KeyInterval interval, DSLContext dsl) { .where(IDENTIFIER_LOCATION_HASH.DIGEST.ge(interval.getBegin().getBytes())) .and(IDENTIFIER_LOCATION_HASH.DIGEST.le(interval.getEnd().getBytes())) .stream() - .map(r -> { - try { - return Digest.from(Digeste.parseFrom(r.value1())); - } catch (InvalidProtocolBufferException e) { - return null; - } - }) + .map(r -> new Digest(algorithm, r.value1())) .filter(d -> d != null), dsl.select(PENDING_EVENT.DIGEST) .from(PENDING_EVENT) .join(PENDING_COORDINATES) diff --git a/thoth/src/test/java/com/salesforce/apollo/thoth/KerlSpaceTest.java b/thoth/src/test/java/com/salesforce/apollo/thoth/KerlSpaceTest.java index f6431f27f..a02000f3c 100644 --- a/thoth/src/test/java/com/salesforce/apollo/thoth/KerlSpaceTest.java +++ b/thoth/src/test/java/com/salesforce/apollo/thoth/KerlSpaceTest.java @@ -68,11 +68,11 @@ public void smokin() throws Exception { JdbcConnectionPool connectionPoolB = JdbcConnectionPool.create("jdbc:h2:mem:B;DB_CLOSE_DELAY=-1", "", ""); connectionPoolB.setMaxConnections(10); - var spaceA = new KerlSpace(connectionPoolA, DigestAlgorithm.DEFAULT.getOrigin()); + var spaceA = new KerlSpace(connectionPoolA, DigestAlgorithm.DEFAULT.getOrigin(), DigestAlgorithm.DEFAULT); var stereotomyA = new StereotomyImpl(new MemKeyStore(), new UniKERLDirectPooled(connectionPoolA, digestAlgorithm).create(), entropy); - var spaceB = new KerlSpace(connectionPoolB, DigestAlgorithm.DEFAULT.getLast()); + var spaceB = new KerlSpace(connectionPoolB, DigestAlgorithm.DEFAULT.getLast(), DigestAlgorithm.DEFAULT); var stereotomyB = new StereotomyImpl(new MemKeyStore(), new UniKERLDirectPooled(connectionPoolB, digestAlgorithm).create(), entropy); diff --git a/thoth/src/test/java/com/salesforce/apollo/thoth/KerlTest.java b/thoth/src/test/java/com/salesforce/apollo/thoth/KerlTest.java index 481ab9057..34123a129 100644 --- a/thoth/src/test/java/com/salesforce/apollo/thoth/KerlTest.java +++ b/thoth/src/test/java/com/salesforce/apollo/thoth/KerlTest.java @@ -60,10 +60,10 @@ public void delegated() throws Exception { ControlledIdentifier delegated = opti2; // identifier - assertTrue(delegated.getIdentifier() instanceof SelfAddressingIdentifier); + assertInstanceOf(SelfAddressingIdentifier.class, delegated.getIdentifier()); var sap = (SelfAddressingIdentifier) delegated.getIdentifier(); assertEquals(DigestAlgorithm.DEFAULT, sap.getDigest().getAlgorithm()); - assertEquals("092126af01f80ca28e7a99bbdce229c029be3bbfcb791e29ccb7a64e8019a36f", + assertEquals("6000b1b611a2a6cb27b6c569c056cf56e04da4905168020fc054d133181d379b", Hex.hex(sap.getDigest().getBytes())); assertEquals(1, ((Unweighted) delegated.getSigningThreshold()).getThreshold()); diff --git a/tron/src/main/java/com/chiralbehaviors/tron/Fsm.java b/tron/src/main/java/com/chiralbehaviors/tron/Fsm.java index d979fcccc..3f93e681a 100644 --- a/tron/src/main/java/com/chiralbehaviors/tron/Fsm.java +++ b/tron/src/main/java/com/chiralbehaviors/tron/Fsm.java @@ -1,6 +1,6 @@ /* * Copyright (c) 2013 ChiralBehaviors LLC, all rights reserved. - * + * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at @@ -15,6 +15,9 @@ */ package com.chiralbehaviors.tron; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import java.lang.reflect.InvocationHandler; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; @@ -25,59 +28,51 @@ import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - /** * A Finite State Machine implementation. - * - * @author hhildebrand - * + * * @param the transition interface * @param the fsm context interface + * @author hhildebrand */ public final class Fsm { - private static class State { - private final Context context; - private final Transitions transitions; - - public State(Context context, Transitions transitions) { - this.context = context; - this.transitions = transitions; - } + private static final Logger DEFAULT_LOG = LoggerFactory.getLogger(Fsm.class); + private static final ThreadLocal> thisFsm = new ThreadLocal<>(); + private final Transitions proxy; + private final Deque> stack = new ArrayDeque<>(); + private final Lock sync; + private final Class transitionsType; + private Context context; + private Transitions current; + private Logger log; + private String name = ""; + private boolean pendingPop = false; + private State pendingPush; + private PendingTransition popTransition; + private Transitions previous; + private PendingTransition pushTransition; + private String transition; + Fsm(Context context, boolean sync, Class transitionsType, ClassLoader transitionsCL) { + this.setContext(context); + this.sync = sync ? new ReentrantLock() : null; + this.transitionsType = transitionsType; + this.log = DEFAULT_LOG; + @SuppressWarnings("unchecked") + Transitions facade = (Transitions) Proxy.newProxyInstance(transitionsCL, new Class[] { transitionsType }, + transitionsHandler()); + proxy = facade; } - private static class PendingTransition implements InvocationHandler { - private volatile Object[] args; - private volatile Method method; - - @Override - public Object invoke(Object proxy, Method method, Object[] args) throws Throwable { - if (this.method != null) { - throw new IllegalStateException(String.format("Pop transition '%s' has already been established", - method.toGenericString())); - } - this.method = method; - this.args = args; - return null; - } - } - - private static final Logger DEFAULT_LOG = LoggerFactory.getLogger(Fsm.class); - private static final ThreadLocal> thisFsm = new ThreadLocal<>(); - /** * Construct a new instance of a finite state machine. - * + * * @param fsmContext - the object used as the action context for this FSM - * @param transitions - the interface class used to define the transitions for - * this FSM - * @param transitionsCL - the class loader to be used to load the transitions - * interface class + * @param transitions - the interface class used to define the transitions for this FSM + * @param transitionsCL - the class loader to be used to load the transitions interface class * @param initialState - the initial state of the FSM - * @param sync - true if this FSM is to synchronize state transitions. - * This is required for multi-threaded use of the FSM + * @param sync - true if this FSM is to synchronize state transitions. This is required for multi-threaded + * use of the FSM * @return the Fsm instance */ public static Fsm construct(Context fsmContext, @@ -85,8 +80,9 @@ public static Fsm construct(Context ClassLoader transitionsCL, Enum initialState, boolean sync) { if (!transitions.isAssignableFrom(initialState.getClass())) { - throw new IllegalArgumentException(String.format("Supplied initial state '%s' does not implement the transitions interface '%s'", - initialState, transitions)); + throw new IllegalArgumentException( + String.format("Supplied initial state '%s' does not implement the transitions interface '%s'", initialState, + transitions)); } Fsm fsm = new Fsm<>(fsmContext, sync, transitions, transitionsCL); @SuppressWarnings("unchecked") @@ -96,8 +92,7 @@ public static Fsm construct(Context } /** - * Construct a new instance of a finite state machine with a default - * ClassLoader. + * Construct a new instance of a finite state machine with a default ClassLoader. */ public static Fsm construct(Context fsmContext, Class transitions, @@ -106,7 +101,6 @@ public static Fsm construct(Context } /** - * * @return the Context of the currently executing Fsm */ public static Context thisContext() { @@ -116,7 +110,6 @@ public static Context thisContext() { } /** - * * @return the currrently executing Fsm */ public static Fsm thisFsm() { @@ -125,35 +118,8 @@ public static Fsm thisFsm() { return fsm; } - private Context context; - private Transitions current; - private Logger log; - private String name = ""; - private boolean pendingPop = false; - private State pendingPush; - private PendingTransition popTransition; - private Transitions previous; - private final Transitions proxy; - private PendingTransition pushTransition; - private final Deque> stack = new ArrayDeque<>(); - private final Lock sync; - private String transition; - private final Class transitionsType; - - Fsm(Context context, boolean sync, Class transitionsType, ClassLoader transitionsCL) { - this.setContext(context); - this.sync = sync ? new ReentrantLock() : null; - this.transitionsType = transitionsType; - this.log = DEFAULT_LOG; - @SuppressWarnings("unchecked") - Transitions facade = (Transitions) Proxy.newProxyInstance(transitionsCL, new Class[] { transitionsType }, - transitionsHandler()); - proxy = facade; - } - /** - * Execute the initial state's entry action. Note that we do not guard against - * multiple invocations. + * Execute the initial state's entry action. Note that we do not guard against multiple invocations. */ public void enterStartState() { if (log.isTraceEnabled()) { @@ -163,7 +129,6 @@ public void enterStartState() { } /** - * * @return the action context object of this Fsm */ public Context getContext() { @@ -171,18 +136,21 @@ public Context getContext() { } /** - * + * Set the Context of the FSM + */ + public void setContext(Context context) { + this.context = context; + } + + /** * @return the current state of the Fsm */ public Transitions getCurrentState() { - return locked(() -> { - Transitions transitions = current; - return transitions; - }); + Transitions transitions = current; + return transitions; } /** - * * @return the logger used by this Fsm */ public Logger getLog() { @@ -192,12 +160,24 @@ public Logger getLog() { }); } + /** + * Set the Logger for this Fsm. + * + * @param log - the Logger of this Fsm + */ + public void setLog(Logger log) { + this.log = log; + } + public String getName() { return name; } + public void setName(String name) { + this.name = name; + } + /** - * * @return the previous state of the Fsm, or null if no previous state */ public Transitions getPreviousState() { @@ -208,7 +188,6 @@ public Transitions getPreviousState() { } /** - * * @return the String representation of the current transition */ public String getTransition() { @@ -218,7 +197,6 @@ public String getTransition() { } /** - * * @return the Transitions object that drives this Fsm through its transitions */ public Transitions getTransitions() { @@ -233,12 +211,10 @@ public InvalidTransition invalidTransitionOn() { } /** - * Pop the state off of the stack of pushed states. This state will become the - * current state of the Fsm. Answer the Transitions object that may be used to - * send a transition to the popped state. - * - * @return the Transitions object that may be used to send a transition to the - * popped state. + * Pop the state off of the stack of pushed states. This state will become the current state of the Fsm. Answer the + * Transitions object that may be used to send a transition to the popped state. + * + * @return the Transitions object that may be used to send a transition to the popped state. */ public Transitions pop() { if (pendingPop) { @@ -248,8 +224,9 @@ public Transitions pop() { throw new IllegalStateException(String.format("[%s] Cannot pop after pushing", name)); } if (stack.size() == 0) { - throw new IllegalStateException(String.format("[%s] State stack is empty, current state: %s, transition: %s", - name, prettyPrint(current), transition)); + throw new IllegalStateException( + String.format("[%s] State stack is empty, current state: %s, transition: %s", name, prettyPrint(current), + transition)); } pendingPop = true; popTransition = new PendingTransition(); @@ -270,9 +247,8 @@ public String prettyPrint(Transitions state) { } /** - * Push the current state of the Fsm on the state stack. The supplied state - * becomes the current state of the Fsm - * + * Push the current state of the Fsm on the state stack. The supplied state becomes the current state of the Fsm + * * @param state - the new current state of the Fsm. */ public Transitions push(Transitions state) { @@ -280,9 +256,8 @@ public Transitions push(Transitions state) { } /** - * Push the current state of the Fsm on the state stack. The supplied state - * becomes the current state of the Fsm - * + * Push the current state of the Fsm on the state stack. The supplied state becomes the current state of the Fsm + * * @param state - the new current state of the Fsm. * @param context - the new current context of the FSM */ @@ -305,26 +280,6 @@ public Transitions push(Transitions state, Context context) { return pendingTransition; } - /** - * Set the Context of the FSM - */ - public void setContext(Context context) { - this.context = context; - } - - /** - * Set the Logger for this Fsm. - * - * @param log - the Logger of this Fsm - */ - public void setLog(Logger log) { - this.log = log; - } - - public void setName(String name) { - this.name = name; - } - public R synchonizeOnState(Callable call) throws Exception { return locked(call); } @@ -347,15 +302,15 @@ private void executeEntryAction() { if (action.isAnnotationPresent(Entry.class)) { action.setAccessible(true); if (log.isTraceEnabled()) { - log.trace(String.format("[%s] Entry action: %s.%s", name, prettyPrint(current), - prettyPrint(action))); + log.trace( + String.format("[%s] Entry action: %s.%s", name, prettyPrint(current), prettyPrint(action))); } try { // For entry actions with parameters, inject the context if (action.getParameterTypes().length > 0) action.invoke(current, getContext()); else - action.invoke(current, new Object[] {}); + action.invoke(current); return; } catch (IllegalAccessException | IllegalArgumentException e) { throw new IllegalStateException(e); @@ -375,15 +330,15 @@ private void executeExitAction() { if (action.isAnnotationPresent(Exit.class)) { action.setAccessible(true); if (log.isTraceEnabled()) { - log.trace(String.format("[%s] Exit action: %s.%s", name, prettyPrint(current), - prettyPrint(action))); + log.trace( + String.format("[%s] Exit action: %s.%s", name, prettyPrint(current), prettyPrint(action))); } try { // For exit action with parameters, inject the context if (action.getParameterTypes().length > 0) action.invoke(current, getContext()); else - action.invoke(current, new Object[] {}); + action.invoke(current); return; } catch (IllegalAccessException | IllegalArgumentException | InvocationTargetException e) { throw new IllegalStateException(e); @@ -394,7 +349,7 @@ private void executeExitAction() { /** * The Jesus Nut - * + * * @param t - the transition to fire * @param arguments - the transition arguments * @return @@ -427,8 +382,9 @@ private Object fire(Method t, Object[] arguments) { transitionTo(nextState); } else { if (nextState != null && log.isTraceEnabled()) { - log.trace(String.format("[%s] Eliding Transition %s -> %s, pinned state: %s", name, - prettyPrint(current), prettyPrint(nextState), prettyPrint(pinned))); + log.trace( + String.format("[%s] Eliding Transition %s -> %s, pinned state: %s", name, prettyPrint(current), + prettyPrint(nextState), prettyPrint(pinned))); } } return null; @@ -439,10 +395,9 @@ private Object fire(Method t, Object[] arguments) { /** * Fire the concrete transition of the current state - * + * * @param stateTransition - the transition method to execute * @param arguments - the arguments of the method - * * @return the next state */ @SuppressWarnings("unchecked") @@ -454,9 +409,9 @@ private Transitions fireTransition(Method stateTransition, Object[] arguments) { try { return (Transitions) stateTransition.invoke(current, (Object[]) null); } catch (IllegalAccessException | IllegalArgumentException | InvocationTargetException e) { - throw new IllegalStateException(String.format("Unable to invoke transition %s,%s", prettyPrint(current), - prettyPrint(stateTransition)), - e); + throw new IllegalStateException( + String.format("Unable to invoke transition %s,%s", prettyPrint(current), prettyPrint(stateTransition)), + e); } } if (log.isTraceEnabled()) { @@ -465,23 +420,23 @@ private Transitions fireTransition(Method stateTransition, Object[] arguments) { try { return (Transitions) stateTransition.invoke(current, arguments); } catch (IllegalAccessException | IllegalArgumentException e) { - throw new IllegalStateException(String.format("Unable to invoke transition %s.%s", prettyPrint(current), - prettyPrint(stateTransition)), - e.getCause()); + throw new IllegalStateException( + String.format("Unable to invoke transition %s.%s", prettyPrint(current), prettyPrint(stateTransition)), + e.getCause()); } catch (InvocationTargetException e) { if (e.getTargetException() instanceof InvalidTransition) { if (log.isTraceEnabled()) { - log.trace(String.format("[%s] Invalid transition %s.%s", name, prettyPrint(current), - getTransition())); + log.trace( + String.format("[%s] Invalid transition %s.%s", name, prettyPrint(current), getTransition())); } throw (InvalidTransition) e.getTargetException(); } if (e.getTargetException() instanceof RuntimeException) { throw (RuntimeException) e.getTargetException(); } - throw new IllegalStateException(String.format("[%s] Unable to invoke transition %s.%s", name, - prettyPrint(current), prettyPrint(stateTransition)), - e.getTargetException()); + throw new IllegalStateException( + String.format("[%s] Unable to invoke transition %s.%s", name, prettyPrint(current), + prettyPrint(stateTransition)), e.getTargetException()); } } @@ -527,10 +482,9 @@ private Method lookupDefaultTransition(InvalidTransition previousException, Meth /** * Lookup the transition. - * + * * @param t - the transition defined in the interface - * @return the transition Method for the current state matching the interface - * definition + * @return the transition Method for the current state matching the interface definition */ private Method lookupTransition(Method t) { Method stateTransition = null; @@ -538,8 +492,9 @@ private Method lookupTransition(Method t) { // First we try declared methods on the state stateTransition = current.getClass().getMethod(t.getName(), t.getParameterTypes()); } catch (NoSuchMethodException | SecurityException e1) { - throw new IllegalStateException(String.format("Inconcievable! The state %s does not implement the transition %s", - prettyPrint(current), prettyPrint(t))); + throw new IllegalStateException( + String.format("Inconcievable! The state %s does not implement the transition %s", prettyPrint(current), + prettyPrint(t))); } stateTransition.setAccessible(true); return stateTransition; @@ -547,7 +502,7 @@ private Method lookupTransition(Method t) { /** * Ye olde tyme state transition - * + * * @param nextState - the next state of the Fsm */ private void normalTransition(Transitions nextState) { @@ -559,16 +514,16 @@ private void normalTransition(Transitions nextState) { } executeExitAction(); if (log.isTraceEnabled()) { - log.trace(String.format("[%s] State transition: %s -> %s", name, prettyPrint(current), - prettyPrint(nextState))); + log.trace( + String.format("[%s] State transition: %s -> %s", name, prettyPrint(current), prettyPrint(nextState))); } current = nextState; executeEntryAction(); } /** - * Execute the exit action of the current state. Set current state to popped - * state of the stack. Execute any pending transition on the current state. + * Execute the exit action of the current state. Set current state to popped state of the stack. Execute any pending + * transition on the current state. */ private void popTransition() { pendingPop = false; @@ -620,11 +575,10 @@ private String prettyPrint(Method transition) { } /** - * Push the current state of the Fsm to the stack, with the supplied context as - * the new current context of the FSM, if non null. Transition the Fsm to the - * nextState, execute the entry action of that state. Set the current state of - * the Fsm to the pending push state, executing the entry action on that state - * + * Push the current state of the Fsm to the stack, with the supplied context as the new current context of the FSM, + * if non null. Transition the Fsm to the nextState, execute the entry action of that state. Set the current state + * of the Fsm to the pending push state, executing the entry action on that state + * * @param nextState */ private void pushTransition(Transitions nextState) { @@ -658,18 +612,9 @@ private void pushTransition(Transitions nextState) { } } - private InvocationHandler transitionsHandler() { - return new InvocationHandler() { - @Override - public Object invoke(Object proxy, Method method, Object[] args) throws Throwable { - return locked(() -> fire(method, args)); - } - }; - } - /** * Transition to the next state - * + * * @param nextState */ private void transitionTo(Transitions nextState) { @@ -681,4 +626,40 @@ private void transitionTo(Transitions nextState) { normalTransition(nextState); } } + + private InvocationHandler transitionsHandler() { + return new InvocationHandler() { + @Override + public Object invoke(Object proxy, Method method, Object[] args) throws Throwable { + return locked(() -> fire(method, args)); + } + }; + } + + private static class State { + private final Context context; + private final Transitions transitions; + + public State(Context context, Transitions transitions) { + this.context = context; + this.transitions = transitions; + } + + } + + private static class PendingTransition implements InvocationHandler { + private volatile Object[] args; + private volatile Method method; + + @Override + public Object invoke(Object proxy, Method method, Object[] args) throws Throwable { + if (this.method != null) { + throw new IllegalStateException( + String.format("Pop transition '%s' has already been established", method.toGenericString())); + } + this.method = method; + this.args = args; + return null; + } + } } From fb7f1182fba7952e8537adeb6e75af9d4b8a87ce Mon Sep 17 00:00:00 2001 From: Hellblazer Date: Thu, 27 Jun 2024 17:52:05 -0700 Subject: [PATCH 17/32] consolidate executors - just fork threads - and schedulers. better concurrency control/management --- .../com/salesforce/apollo/choam/CHOAM.java | 69 +++++++++++-------- .../salesforce/apollo/choam/Parameters.java | 2 +- .../com/salesforce/apollo/choam/Producer.java | 15 ++-- .../salesforce/apollo/choam/ViewAssembly.java | 6 ++ .../salesforce/apollo/choam/DynamicTest.java | 2 +- .../apollo/choam/MembershipTests.java | 4 +- .../salesforce/apollo/choam/TestCHOAM.java | 2 +- .../salesforce/apollo/ethereal/Creator.java | 4 +- .../ethereal/memberships/ChRbcGossip.java | 25 ++++--- .../apollo/demesnes/FireFliesTrace.java | 21 +++--- .../apollo/model/ProcessDomain.java | 6 +- .../apollo/model/ContainmentDomainTest.java | 3 +- .../salesforce/apollo/model/DomainTest.java | 3 +- .../apollo/model/FireFliesTest.java | 3 +- .../com/salesforce/apollo/state/Emulator.java | 3 +- .../apollo/state/SqlStateMachine.java | 17 ++--- .../salesforce/apollo/state/CHOAMTest.java | 5 +- .../apollo/state/MigrationTest.java | 12 ++-- .../salesforce/apollo/state/MutatorTest.java | 8 +-- .../salesforce/apollo/state/ScriptTest.java | 3 +- .../salesforce/apollo/state/UpdaterTest.java | 2 +- .../com/salesforce/apollo/thoth/KerlDHT.java | 11 +-- 22 files changed, 123 insertions(+), 103 deletions(-) diff --git a/choam/src/main/java/com/salesforce/apollo/choam/CHOAM.java b/choam/src/main/java/com/salesforce/apollo/choam/CHOAM.java index 55f0aac29..79f3d4e24 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/CHOAM.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/CHOAM.java @@ -81,12 +81,10 @@ public class CHOAM { private final ReliableBroadcaster combine; private final CommonCommunications comm; private final AtomicReference current = new AtomicReference<>(); - private final ExecutorService executions; private final AtomicReference> futureBootstrap = new AtomicReference<>(); private final AtomicReference> futureSynchronization = new AtomicReference<>(); private final AtomicReference genesis = new AtomicReference<>(); private final AtomicReference head = new AtomicReference<>(); - private final ExecutorService linear; private final AtomicReference next = new AtomicReference<>(); private final AtomicReference nextViewId = new AtomicReference<>(); private final Parameters params; @@ -101,13 +99,13 @@ public class CHOAM { private final AtomicReference view = new AtomicReference<>(); private final PendingViews pendingViews = new PendingViews(); private final ScheduledExecutorService scheduler; + private final Semaphore linear = new Semaphore(1); private volatile AtomicBoolean ongoingJoin; public CHOAM(Parameters params) { scheduler = Executors.newScheduledThreadPool(1, Thread.ofVirtual().factory()); this.store = new Store(params.digestAlgorithm(), params.mvBuilder().clone().build()); this.params = params; - executions = Executors.newVirtualThreadPerTaskExecutor(); pendingViews.add(params.context().getId(), params.context().delegate()); rotateViewKeys(); @@ -118,14 +116,17 @@ public CHOAM(Parameters params) { combine = new ReliableBroadcaster(bContext, params.member(), params.combine(), params.communications(), params.metrics() == null ? null : params.metrics().getCombineMetrics(), adapter); - linear = Executors.newSingleThreadExecutor( - Thread.ofVirtual().name("Linear " + params.member().getId()).factory()); combine.registerHandler((_, messages) -> { - try { - linear.execute(Utils.wrapped(() -> combine(messages), log)); - } catch (RejectedExecutionException e) { - // ignore - } + Thread.ofVirtual().start(() -> { + if (!started.get()) { + return; + } + try { + combine(messages); + } catch (Throwable t) { + log.error("Failed to combine messages on: {}", params.member().getId(), t); + } + }); }); head.set(new NullBlock(params.digestAlgorithm())); view.set(new NullBlock(params.digestAlgorithm())); @@ -353,10 +354,7 @@ public void stop() { if (!started.compareAndSet(true, false)) { return; } - try { - linear.shutdownNow(); - } catch (Throwable e) { - } + linear.release(10000); try { scheduler.shutdownNow(); } catch (Throwable e) { @@ -366,10 +364,6 @@ public void stop() { session.stop(); } catch (Throwable e) { } - try { - executions.shutdownNow(); - } catch (Throwable e) { - } final var c = current.get(); if (c != null) { try { @@ -593,7 +587,7 @@ private void execute(List execs) { try { params.processor() .execute(index, CHOAM.hashOf(exec, params.digestAlgorithm()), exec, - stxn == null ? null : stxn.onCompletion(), executions); + stxn == null ? null : stxn.onCompletion()); } catch (Throwable t) { log.error("Exception processing transaction: {} block: {} height: {} on: {}", hash, h.hash, h.height(), params.member().getId()); @@ -951,14 +945,13 @@ private void synchronize(SynchronizedState state) { log.info("Synchronized, resuming view: {} deferred blocks: {} on: {}", state.lastCheckpoint() != null ? state.lastCheckpoint().hash : state.genesis().hash, pending.size(), params.member().getId()); - try { - linear.execute(Utils.wrapped(() -> { - transitions.synchd(); - transitions.combine(); - }, log)); - } catch (RejectedExecutionException e) { - // ignore - } + Thread.ofVirtual().start(Utils.wrapped(() -> { + if (!started.get()) { + return; + } + transitions.synchd(); + transitions.combine(); + }, log)); } private void synchronizedProcess(CertifiedBlock certifiedBlock) { @@ -1052,7 +1045,7 @@ default void endBlock(ULong height, Digest hash) { } @SuppressWarnings("rawtypes") - void execute(int index, Digest hash, Transaction tx, CompletableFuture onComplete, Executor executor); + void execute(int index, Digest hash, Transaction tx, CompletableFuture onComplete); default void genesis(Digest hash, List initialization) { } @@ -1209,7 +1202,25 @@ public void cancelTimer(String timer) { @Override public void combine() { - CHOAM.this.combine(); + Thread.ofVirtual().start(Utils.wrapped(() -> { + if (!started.get()) { + return; + } + try { + linear.acquire(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return; + } + try { + CHOAM.this.combine(); + } finally { + linear.release(); + } + }, log)); + if (linear.getQueueLength() > 0) { + log.info("Linear Q: {} on: {}", linear.getQueueLength(), params.member().getId()); + } } @Override diff --git a/choam/src/main/java/com/salesforce/apollo/choam/Parameters.java b/choam/src/main/java/com/salesforce/apollo/choam/Parameters.java index 6241fb072..5f70ea4f3 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/Parameters.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/Parameters.java @@ -325,7 +325,7 @@ public static class Builder implements Cloneable { private Supplier kerl = () -> KERL_.getDefaultInstance(); private SigningMember member; private ChoamMetrics metrics; - private TransactionExecutor processor = (i, h, t, f, exec) -> { + private TransactionExecutor processor = (i, h, t, f) -> { }; private BiConsumer restorer = (height, checkpointState) -> { }; diff --git a/choam/src/main/java/com/salesforce/apollo/choam/Producer.java b/choam/src/main/java/com/salesforce/apollo/choam/Producer.java index b1405093d..7214643a3 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/Producer.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/Producer.java @@ -52,15 +52,13 @@ public class Producer { private final Transitions transitions; private final ViewContext view; private final Digest nextViewId; - private final ExecutorService serialize = Executors.newSingleThreadExecutor(); + private final Semaphore serialize = new Semaphore(1); private final ViewAssembly assembly; private final int maxEpoch; - private final ScheduledExecutorService scheduler; private volatile boolean assembled = false; public Producer(Digest nextViewId, ViewContext view, HashedBlock lastBlock, HashedBlock checkpoint, String label, ScheduledExecutorService scheduler) { - this.scheduler = scheduler; assert view != null; this.view = view; this.previousBlock.set(lastBlock); @@ -150,7 +148,7 @@ public void stop() { return; } log.trace("Closing producer for: {} on: {}", getViewId(), params().member().getId()); - serialize.shutdown(); + serialize.release(10000); controller.stop(); coordinator.stop(); ds.close(); @@ -353,13 +351,20 @@ private void reconfigure() { } private void serial(List preblock, Boolean last) { - serialize.execute(() -> { + Thread.ofVirtual().start(() -> { try { + serialize.acquire(); create(preblock, last); } catch (Throwable t) { log.error("Error processing preblock last: {} on: {}", last, params().member().getId(), t); + } finally { + serialize.release(); } }); + var awaiting = serialize.getQueueLength(); + if (awaiting > 0) { + log.error("Serialize: {} on: {}", awaiting, params().member().getId()); + } } private PendingBlock validate(Validate v) { diff --git a/choam/src/main/java/com/salesforce/apollo/choam/ViewAssembly.java b/choam/src/main/java/com/salesforce/apollo/choam/ViewAssembly.java index 98a510a8c..25e4cb2bb 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/ViewAssembly.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/ViewAssembly.java @@ -108,6 +108,12 @@ public void start() { void assemble(List asses) { if (!started.get()) { + if (!asses.isEmpty()) { + var viewz = asses.stream().flatMap(a -> a.getViewsList().stream()).toList(); + var joinz = asses.stream().flatMap(a -> a.getJoinsList().stream()).toList(); + log.debug("Not started, ignoring assemblies: {} joins: {} views: {} on: {}", asses.size(), joinz.size(), + viewz.size(), params().member().getId()); + } return; } diff --git a/choam/src/test/java/com/salesforce/apollo/choam/DynamicTest.java b/choam/src/test/java/com/salesforce/apollo/choam/DynamicTest.java index 332da3e8b..34d3081bb 100644 --- a/choam/src/test/java/com/salesforce/apollo/choam/DynamicTest.java +++ b/choam/src/test/java/com/salesforce/apollo/choam/DynamicTest.java @@ -221,7 +221,7 @@ public void tearDown() throws Exception { } private CHOAM constructCHOAM(SigningMember m, Parameters.Builder params, Context context) { - final CHOAM.TransactionExecutor processor = (index, hash, t, f, executor) -> { + final CHOAM.TransactionExecutor processor = (index, hash, t, f) -> { if (f != null) { f.completeAsync(Object::new, executor); } diff --git a/choam/src/test/java/com/salesforce/apollo/choam/MembershipTests.java b/choam/src/test/java/com/salesforce/apollo/choam/MembershipTests.java index f9324e628..a23597f0b 100644 --- a/choam/src/test/java/com/salesforce/apollo/choam/MembershipTests.java +++ b/choam/src/test/java/com/salesforce/apollo/choam/MembershipTests.java @@ -183,9 +183,9 @@ public SigningMember initialize(int checkpointBlockSize, int cardinality) throws } private CHOAM constructCHOAM(SigningMember m, Parameters.Builder params, boolean testSubject) { - final TransactionExecutor processor = (index, hash, t, f, executor) -> { + final TransactionExecutor processor = (_, _, _, f) -> { if (f != null) { - f.completeAsync(Object::new, executor); + f.completeAsync(Object::new); } }; params.getProducer().ethereal().setSigner(m); diff --git a/choam/src/test/java/com/salesforce/apollo/choam/TestCHOAM.java b/choam/src/test/java/com/salesforce/apollo/choam/TestCHOAM.java index 64be54d3c..4c5866d4d 100644 --- a/choam/src/test/java/com/salesforce/apollo/choam/TestCHOAM.java +++ b/choam/src/test/java/com/salesforce/apollo/choam/TestCHOAM.java @@ -138,7 +138,7 @@ public void before() throws Exception { @SuppressWarnings({ "unchecked", "rawtypes" }) @Override - public void execute(int index, Digest hash, Transaction t, CompletableFuture f, Executor executor) { + public void execute(int index, Digest hash, Transaction t, CompletableFuture f) { if (f != null) { f.completeAsync(() -> new Object(), executor); } diff --git a/ethereal/src/main/java/com/salesforce/apollo/ethereal/Creator.java b/ethereal/src/main/java/com/salesforce/apollo/ethereal/Creator.java index 307851b7e..9e4f1d4f0 100644 --- a/ethereal/src/main/java/com/salesforce/apollo/ethereal/Creator.java +++ b/ethereal/src/main/java/com/salesforce/apollo/ethereal/Creator.java @@ -173,10 +173,10 @@ assert parentsOnPreviousLevel(u) >= quorum : "Parents: " + Arrays.asList(u.paren private ByteString getData(int level) { if (level < conf.lastLevel()) { if (ds != null) { - log.info("Requesting timing unit: {} on: {}", level, conf.logLabel()); + log.trace("Requesting timing unit: {} on: {}", level, conf.logLabel()); return ds.getData(); } - log.info("No datasource for timing unit: {} on: {}", level, conf.logLabel()); + log.trace("No datasource for timing unit: {} on: {}", level, conf.logLabel()); return ByteString.EMPTY; } Unit timingUnit = lastTiming.poll(); diff --git a/ethereal/src/main/java/com/salesforce/apollo/ethereal/memberships/ChRbcGossip.java b/ethereal/src/main/java/com/salesforce/apollo/ethereal/memberships/ChRbcGossip.java index 482b267a5..2e6028b99 100644 --- a/ethereal/src/main/java/com/salesforce/apollo/ethereal/memberships/ChRbcGossip.java +++ b/ethereal/src/main/java/com/salesforce/apollo/ethereal/memberships/ChRbcGossip.java @@ -30,7 +30,6 @@ import java.time.Duration; import java.util.Collection; import java.util.Optional; -import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; @@ -46,18 +45,18 @@ * @author hal.hildebrand */ public class ChRbcGossip { + private static final Logger log = LoggerFactory.getLogger(ChRbcGossip.class); - private static final Logger log = LoggerFactory.getLogger( - ChRbcGossip.class); - private final CommonCommunications comm; - private final Digest id; - private final SigningMember member; - private final EtherealMetrics metrics; - private final Processor processor; - private final SliceIterator ring; - private final AtomicBoolean started = new AtomicBoolean(); - private final Terminal terminal = new Terminal(); - private volatile ScheduledFuture scheduled; + private final CommonCommunications comm; + private final Digest id; + private final SigningMember member; + private final EtherealMetrics metrics; + private final Processor processor; + private final SliceIterator ring; + private final AtomicBoolean started = new AtomicBoolean(); + private final Terminal terminal = new Terminal(); + private final ScheduledExecutorService scheduler; + private volatile ScheduledFuture scheduled; public ChRbcGossip(Digest id, SigningMember member, Collection membership, Processor processor, Router communications, EtherealMetrics m, ScheduledExecutorService scheduler) { @@ -65,6 +64,7 @@ public ChRbcGossip(Digest id, SigningMember member, Collection membershi this.member = member; this.metrics = m; this.id = id; + this.scheduler = scheduler; comm = communications.create(member, id, terminal, getClass().getCanonicalName(), r -> new GossiperServer(communications.getClientIdentityProvider(), metrics, r), getCreate(metrics), Gossiper.getLocalLoopback(member)); @@ -90,7 +90,6 @@ public void start(Duration duration, Predicate Thread.ofVirtual().start(Utils.wrapped(() -> { try { gossip(duration, scheduler); diff --git a/isolates/src/test/java/com/salesforce/apollo/demesnes/FireFliesTrace.java b/isolates/src/test/java/com/salesforce/apollo/demesnes/FireFliesTrace.java index 0bf42a2cc..1241a518e 100644 --- a/isolates/src/test/java/com/salesforce/apollo/demesnes/FireFliesTrace.java +++ b/isolates/src/test/java/com/salesforce/apollo/demesnes/FireFliesTrace.java @@ -15,8 +15,8 @@ import com.salesforce.apollo.choam.Parameters.ProducerParameters; import com.salesforce.apollo.choam.Parameters.RuntimeParameters; import com.salesforce.apollo.choam.proto.FoundationSeal; -import com.salesforce.apollo.context.Context; import com.salesforce.apollo.context.DynamicContextImpl; +import com.salesforce.apollo.context.ViewChange; import com.salesforce.apollo.cryptography.Digest; import com.salesforce.apollo.cryptography.DigestAlgorithm; import com.salesforce.apollo.delphinius.Oracle; @@ -40,7 +40,7 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; -import java.util.function.BiConsumer; +import java.util.function.Consumer; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -201,7 +201,8 @@ public void before() throws Exception { Duration.ofMinutes(1), "jdbc:h2:mem:%s-dht".formatted(digest), checkpointDirBase, Duration.ofMillis(10), 0.00125, - Duration.ofMinutes(1), 3, 10, 0.1); + Duration.ofMinutes(1), 3, Duration.ofMillis(100), + 10, 0.1); var node = new ProcessContainerDomain(group, member, pdParams, params, RuntimeParameters.newBuilder() .setFoundation( sealed) @@ -223,17 +224,17 @@ public void smokin() throws Exception { final var seeds = Collections.singletonList( new Seed(domains.getFirst().getMember().getIdentifier().getIdentifier(), EndpointProvider.allocatePort())); domains.forEach(d -> { - BiConsumer c = (context, viewId) -> { - if (context.cardinality() == CARDINALITY) { - System.out.printf("Full view: %s members: %s on: %s%n", viewId, context.cardinality(), - d.getMember().getId()); + Consumer c = viewChange -> { + if (viewChange.context().cardinality() == CARDINALITY) { + System.out.printf("Full view: %s members: %s on: %s%n", viewChange.diadem(), + viewChange.context().cardinality(), d.getMember().getId()); countdown.countDown(); } else { - System.out.printf("Members joining: %s members: %s on: %s%n", viewId, context.cardinality(), - d.getMember().getId()); + System.out.printf("Members joining: %s members: %s on: %s%n", viewChange.diadem(), + viewChange.context().cardinality(), d.getMember().getId()); } }; - d.getFoundation().register(c); + d.getFoundation().register("foo", c); }); // start seed final var started = new AtomicReference<>(new CountDownLatch(1)); diff --git a/model/src/main/java/com/salesforce/apollo/model/ProcessDomain.java b/model/src/main/java/com/salesforce/apollo/model/ProcessDomain.java index 28bd6f841..282c59776 100644 --- a/model/src/main/java/com/salesforce/apollo/model/ProcessDomain.java +++ b/model/src/main/java/com/salesforce/apollo/model/ProcessDomain.java @@ -133,7 +133,7 @@ protected Consumer listener() { } protected void startServices() { - dht.start(params.gossipDuration()); + dht.start(parameters.kerlSpaceDuration); } protected void stopServices() { @@ -143,7 +143,7 @@ protected void stopServices() { public record ProcessDomainParameters(String dbURL, Duration dhtOperationsTimeout, String dhtDbUrl, Path checkpointBaseDir, Duration dhtOpsFrequency, double dhtFpr, - Duration dhtEventValidTO, int dhtBias, int jdbcMaxConnections, - double dhtPbyz) { + Duration dhtEventValidTO, int dhtBias, Duration kerlSpaceDuration, + int jdbcMaxConnections, double dhtPbyz) { } } diff --git a/model/src/test/java/com/salesforce/apollo/model/ContainmentDomainTest.java b/model/src/test/java/com/salesforce/apollo/model/ContainmentDomainTest.java index e8b6982c2..d135b6dad 100644 --- a/model/src/test/java/com/salesforce/apollo/model/ContainmentDomainTest.java +++ b/model/src/test/java/com/salesforce/apollo/model/ContainmentDomainTest.java @@ -92,7 +92,8 @@ public void before() throws Exception { var pdParams = new ProcessDomain.ProcessDomainParameters(dbUrl, Duration.ofMinutes(1), "jdbc:h2:mem:%s-state".formatted(d), checkpointDirBase, Duration.ofMillis(10), 0.00125, - Duration.ofMinutes(1), 3, 10, 0.1); + Duration.ofMinutes(1), 3, Duration.ofMillis(100), + 10, 0.1); var domain = new ProcessContainerDomain(group, member, pdParams, params.clone(), RuntimeParameters.newBuilder() .setFoundation(sealed) diff --git a/model/src/test/java/com/salesforce/apollo/model/DomainTest.java b/model/src/test/java/com/salesforce/apollo/model/DomainTest.java index ff25f7d56..4c17f2396 100644 --- a/model/src/test/java/com/salesforce/apollo/model/DomainTest.java +++ b/model/src/test/java/com/salesforce/apollo/model/DomainTest.java @@ -249,7 +249,8 @@ public void before() throws Exception { var pdParams = new ProcessDomain.ProcessDomainParameters(dbUrl, Duration.ofMinutes(1), "jdbc:h2:mem:%s-state;DB_CLOSE_DELAY=-1".formatted( d), checkpointDirBase, Duration.ofMillis(10), - 0.00125, Duration.ofMinutes(1), 3, 10, 0.1); + 0.00125, Duration.ofMinutes(1), 3, + Duration.ofMillis(100), 10, 0.1); var domain = new ProcessDomain(group, member, pdParams, params.clone(), RuntimeParameters.newBuilder() .setFoundation( sealed) diff --git a/model/src/test/java/com/salesforce/apollo/model/FireFliesTest.java b/model/src/test/java/com/salesforce/apollo/model/FireFliesTest.java index a11187f26..d237263bc 100644 --- a/model/src/test/java/com/salesforce/apollo/model/FireFliesTest.java +++ b/model/src/test/java/com/salesforce/apollo/model/FireFliesTest.java @@ -93,7 +93,8 @@ public void before() throws Exception { var pdParams = new ProcessDomain.ProcessDomainParameters(dbUrl, Duration.ofSeconds(5), "jdbc:h2:mem:%s-state".formatted(digest), checkpointDirBase, Duration.ofMillis(10), 0.00125, - Duration.ofSeconds(5), 3, 10, 0.1); + Duration.ofSeconds(5), 3, Duration.ofMillis(100), + 10, 0.1); var node = new ProcessDomain(group, member, pdParams, params.clone(), RuntimeParameters.newBuilder() .setFoundation( sealed) diff --git a/sql-state/src/main/java/com/salesforce/apollo/state/Emulator.java b/sql-state/src/main/java/com/salesforce/apollo/state/Emulator.java index e0fb88634..9890d50eb 100644 --- a/sql-state/src/main/java/com/salesforce/apollo/state/Emulator.java +++ b/sql-state/src/main/java/com/salesforce/apollo/state/Emulator.java @@ -92,8 +92,7 @@ public Emulator(SqlStateMachine ssm, Digest base) { lock.lock(); try { Transaction txn = st.transaction(); - txnExec.execute(txnIndex.incrementAndGet(), CHOAM.hashOf(txn, algorithm), txn, st.onCompletion(), - r -> r.run()); + txnExec.execute(txnIndex.incrementAndGet(), CHOAM.hashOf(txn, algorithm), txn, st.onCompletion()); return SubmitResult.newBuilder().setResult(Result.PUBLISHED).build(); } finally { lock.unlock(); diff --git a/sql-state/src/main/java/com/salesforce/apollo/state/SqlStateMachine.java b/sql-state/src/main/java/com/salesforce/apollo/state/SqlStateMachine.java index deb9aaa43..0fe0d448f 100644 --- a/sql-state/src/main/java/com/salesforce/apollo/state/SqlStateMachine.java +++ b/sql-state/src/main/java/com/salesforce/apollo/state/SqlStateMachine.java @@ -60,7 +60,6 @@ import java.util.*; import java.util.concurrent.Callable; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.Executor; import java.util.concurrent.atomic.AtomicReference; import java.util.function.BiConsumer; import java.util.function.Consumer; @@ -599,12 +598,11 @@ private void clearCheckSums() throws LiquibaseException { } @SuppressWarnings("unchecked") - private void complete(@SuppressWarnings("rawtypes") CompletableFuture onCompletion, Object results, - Executor executor) { + private void complete(@SuppressWarnings("rawtypes") CompletableFuture onCompletion, Object results) { if (onCompletion == null) { return; } - onCompletion.completeAsync(() -> results, executor); + onCompletion.completeAsync(() -> results); } private void dropAll(Drop drop) throws LiquibaseException { @@ -634,7 +632,7 @@ private void exception(@SuppressWarnings("rawtypes") CompletableFuture onComplet } private void execute(int index, Digest txnHash, Txn tx, - @SuppressWarnings("rawtypes") CompletableFuture onCompletion, Executor executor) { + @SuppressWarnings("rawtypes") CompletableFuture onCompletion) { log.debug("executing: {} on: {}", tx.getExecutionCase(), id); var executing = executingBlock.get(); updateCurrent(executing.height, executing.blkHash, index, txnHash); @@ -652,7 +650,7 @@ private void execute(int index, Digest txnHash, Txn tx, case MIGRATION -> acceptMigration(tx.getMigration()); default -> null; }; - this.complete(onCompletion, results, executor); + this.complete(onCompletion, results); } catch (JdbcSQLNonTransientConnectionException e) { // ignore } catch (Exception e) { @@ -981,7 +979,7 @@ public void endBlock(ULong height, Digest hash) { @Override public void execute(int index, Digest txnHash, Transaction tx, - @SuppressWarnings("rawtypes") CompletableFuture onComplete, Executor executor) { + @SuppressWarnings("rawtypes") CompletableFuture onComplete) { boolean closed; try { closed = connection().isClosed(); @@ -1000,9 +998,8 @@ public void execute(int index, Digest txnHash, Transaction tx, return; } withContext(() -> { - SqlStateMachine.this.execute(index, txnHash, txn, onComplete, executor); + SqlStateMachine.this.execute(index, txnHash, txn, onComplete); }); - } @Override @@ -1014,7 +1011,7 @@ public void genesis(Digest hash, List initialization) { }); int i = 0; for (Transaction txn : initialization) { - execute(i, Digest.NONE, txn, null, r -> r.run()); + execute(i, Digest.NONE, txn, null); } log.debug("Genesis executed on: {}", id); } diff --git a/sql-state/src/test/java/com/salesforce/apollo/state/CHOAMTest.java b/sql-state/src/test/java/com/salesforce/apollo/state/CHOAMTest.java index 1ec5449d1..d0dca0785 100644 --- a/sql-state/src/test/java/com/salesforce/apollo/state/CHOAMTest.java +++ b/sql-state/src/test/java/com/salesforce/apollo/state/CHOAMTest.java @@ -327,10 +327,9 @@ public void endBlock(ULong height, Digest hash) { @Override public void execute(int i, Digest hash, Transaction tx, - @SuppressWarnings("rawtypes") CompletableFuture onComplete, - Executor executor) { + @SuppressWarnings("rawtypes") CompletableFuture onComplete) { up.getExecutor() - .execute(i, hash, tx, onComplete, executor); + .execute(i, hash, tx, onComplete); } @Override diff --git a/sql-state/src/test/java/com/salesforce/apollo/state/MigrationTest.java b/sql-state/src/test/java/com/salesforce/apollo/state/MigrationTest.java index d53229b32..af04403de 100644 --- a/sql-state/src/test/java/com/salesforce/apollo/state/MigrationTest.java +++ b/sql-state/src/test/java/com/salesforce/apollo/state/MigrationTest.java @@ -70,7 +70,7 @@ public void rollback() throws Exception { executor.execute(0, Digest.NONE, Transaction.newBuilder() .setContent( Txn.newBuilder().setMigration(migration).build().toByteString()) - .build(), success, r -> r.run()); + .build(), success); executor.beginBlock(ULong.valueOf(1), DigestAlgorithm.DEFAULT.getOrigin().prefix("voo")); @@ -80,7 +80,7 @@ public void rollback() throws Exception { executor.execute(0, Digest.NONE, Transaction.newBuilder() .setContent( Txn.newBuilder().setMigration(migration).build().toByteString()) - .build(), success, r -> r.run()); + .build(), success); success.get(1, TimeUnit.SECONDS); @@ -101,7 +101,7 @@ public void rollback() throws Exception { .toByteString()); Transaction transaction = builder.build(); - updater.getExecutor().execute(0, Digest.NONE, transaction, null, r -> r.run()); + updater.getExecutor().execute(0, Digest.NONE, transaction, null); ResultSet books = statement.executeQuery("select * from test.books"); assertTrue(books.first()); @@ -122,7 +122,7 @@ public void rollback() throws Exception { executor.execute(1, Digest.NONE, Transaction.newBuilder() .setContent( Txn.newBuilder().setMigration(migration).build().toByteString()) - .build(), success, r -> r.run()); + .build(), success); success.get(1, TimeUnit.SECONDS); @@ -150,7 +150,7 @@ public void update() throws Exception { executor.execute(0, Digest.NONE, Transaction.newBuilder() .setContent( Txn.newBuilder().setMigration(migration).build().toByteString()) - .build(), success, r -> r.run()); + .build(), success); success.get(1, TimeUnit.SECONDS); @@ -171,7 +171,7 @@ public void update() throws Exception { .toByteString()); Transaction transaction = builder.build(); - updater.getExecutor().execute(1, Digest.NONE, transaction, null, r -> r.run()); + updater.getExecutor().execute(1, Digest.NONE, transaction, null); ResultSet books = statement.executeQuery("select * from test.books"); assertTrue(books.first()); diff --git a/sql-state/src/test/java/com/salesforce/apollo/state/MutatorTest.java b/sql-state/src/test/java/com/salesforce/apollo/state/MutatorTest.java index cd4d173c1..1d7d7f7bd 100644 --- a/sql-state/src/test/java/com/salesforce/apollo/state/MutatorTest.java +++ b/sql-state/src/test/java/com/salesforce/apollo/state/MutatorTest.java @@ -59,7 +59,7 @@ public void smokin() throws Exception { executor.execute(0, Digest.NONE, Transaction.newBuilder() .setContent( Txn.newBuilder().setMigration(migration).build().toByteString()) - .build(), success, r -> r.run()); + .build(), success); success.get(1, TimeUnit.SECONDS); @@ -68,7 +68,7 @@ public void smokin() throws Exception { executor.execute(1, Digest.NONE, Transaction.newBuilder() .setContent( Txn.newBuilder().setMigration(migration).build().toByteString()) - .build(), success, r -> r.run()); + .build(), success); success.get(1, TimeUnit.SECONDS); @@ -90,7 +90,7 @@ public void smokin() throws Exception { success = new CompletableFuture<>(); executor.execute(1, Digest.NONE, Transaction.newBuilder() .setContent(Txn.newBuilder().setCall(call).build().toByteString()) - .build(), success, r -> r.run()); + .build(), success); CallResult result = (CallResult) success.get(1, TimeUnit.SECONDS); assertNotNull(result); @@ -106,7 +106,7 @@ public void smokin() throws Exception { executor.execute(2, Digest.NONE, Transaction.newBuilder() .setContent( Txn.newBuilder().setBatched(batch.build()).build().toByteString()) - .build(), success, r -> r.run()); + .build(), success); var batchResult = (List) success.get(1, TimeUnit.SECONDS); assertNotNull(batchResult); diff --git a/sql-state/src/test/java/com/salesforce/apollo/state/ScriptTest.java b/sql-state/src/test/java/com/salesforce/apollo/state/ScriptTest.java index 3f472bf87..6faca2e0c 100644 --- a/sql-state/src/test/java/com/salesforce/apollo/state/ScriptTest.java +++ b/sql-state/src/test/java/com/salesforce/apollo/state/ScriptTest.java @@ -54,8 +54,7 @@ public void smoke() throws Exception { .build(); CompletableFuture completion = new CompletableFuture<>(); machine.getExecutor() - .execute(0, Digest.NONE, Transaction.newBuilder().setContent(txn.toByteString()).build(), completion, - r -> r.run()); + .execute(0, Digest.NONE, Transaction.newBuilder().setContent(txn.toByteString()).build(), completion); assertTrue(ResultSet.class.isAssignableFrom(completion.get().getClass())); ResultSet rs = (ResultSet) completion.get(); diff --git a/sql-state/src/test/java/com/salesforce/apollo/state/UpdaterTest.java b/sql-state/src/test/java/com/salesforce/apollo/state/UpdaterTest.java index b2522a4b3..e99e0d830 100644 --- a/sql-state/src/test/java/com/salesforce/apollo/state/UpdaterTest.java +++ b/sql-state/src/test/java/com/salesforce/apollo/state/UpdaterTest.java @@ -120,7 +120,7 @@ public void smoke() throws Exception { .toByteString()); Transaction transaction = builder.build(); - updater.getExecutor().execute(0, Digest.NONE, transaction, null, r -> r.run()); + updater.getExecutor().execute(0, Digest.NONE, transaction, null); ResultSet books = statement.executeQuery("select * from books"); assertTrue(books.first()); diff --git a/thoth/src/main/java/com/salesforce/apollo/thoth/KerlDHT.java b/thoth/src/main/java/com/salesforce/apollo/thoth/KerlDHT.java index 831a11412..464339fd3 100644 --- a/thoth/src/main/java/com/salesforce/apollo/thoth/KerlDHT.java +++ b/thoth/src/main/java/com/salesforce/apollo/thoth/KerlDHT.java @@ -738,13 +738,14 @@ public void start(Duration duration, Predicate boolean read(CompletableFuture result, HashMultiset gathered, private void reconcile(Optional result, RingCommunications.Destination destination, - ScheduledExecutorService scheduler, Duration duration) { + Duration duration) { if (!started.get()) { return; } @@ -938,7 +939,7 @@ private void reconcile(Optional result, } } if (started.get()) { - scheduler.schedule(() -> Thread.ofVirtual().start(Utils.wrapped(() -> reconcile(scheduler, duration), log)), + scheduler.schedule(() -> Thread.ofVirtual().start(Utils.wrapped(() -> reconcile(duration), log)), duration.toMillis(), TimeUnit.MILLISECONDS); } } @@ -957,14 +958,14 @@ private Update reconcile(ReconciliationService link, Integer ring) { .build()); } - private void reconcile(ScheduledExecutorService scheduler, Duration duration) { + private void reconcile(Duration duration) { if (!started.get()) { return; } Thread.ofVirtual() .start(() -> reconcile.execute(this::reconcile, (futureSailor, destination) -> reconcile(futureSailor, destination, - scheduler, duration))); + duration))); } From e40b25cb620a20e65ee0bc0d0f374d9a8f501c45 Mon Sep 17 00:00:00 2001 From: Hellblazer Date: Thu, 27 Jun 2024 20:07:33 -0700 Subject: [PATCH 18/32] consolidate executors - just fork threads - and schedulers. better concurrency control/management --- .../com/salesforce/apollo/choam/CHOAM.java | 9 ++---- .../com/salesforce/apollo/choam/Session.java | 28 ++++++++---------- .../apollo/choam/support/Bootstrapper.java | 5 ++-- .../salesforce/apollo/choam/SessionTest.java | 5 ++-- .../choam/support/BootstrapperTest.java | 3 +- .../com/salesforce/apollo/fireflies/View.java | 19 ++++++++---- .../messaging/rbc/ReliableBroadcaster.java | 2 +- .../com/salesforce/apollo/state/Emulator.java | 29 +++++++++++-------- 8 files changed, 54 insertions(+), 46 deletions(-) diff --git a/choam/src/main/java/com/salesforce/apollo/choam/CHOAM.java b/choam/src/main/java/com/salesforce/apollo/choam/CHOAM.java index 79f3d4e24..f90ad6ca0 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/CHOAM.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/CHOAM.java @@ -150,7 +150,7 @@ public CHOAM(Parameters params) { roundScheduler = new RoundScheduler("CHOAM" + params.member().getId() + params.context().getId(), params.context().timeToLive()); combine.register(_ -> roundScheduler.tick()); - session = new Session(params, service()); + session = new Session(params, service(), scheduler); } public static Checkpoint checkpoint(DigestAlgorithm algo, File state, int segmentSize, Digest initial, int crowns, @@ -360,10 +360,6 @@ public void stop() { } catch (Throwable e) { } session.cancelAll(); - try { - session.stop(); - } catch (Throwable e) { - } final var c = current.get(); if (c != null) { try { @@ -760,7 +756,8 @@ private void recover(HashedCertifiedBlock anchor) { log.info("Recovering from: {} height: {} on: {}", anchor.hash, anchor.height(), params.member().getId()); cancelSynchronization(); cancelBootstrap(); - futureBootstrap.set(new Bootstrapper(anchor, params, store, comm).synchronize().whenComplete((s, t) -> { + futureBootstrap.set( + new Bootstrapper(anchor, params, store, comm, scheduler).synchronize().whenComplete((s, t) -> { if (t == null) { try { synchronize(s); diff --git a/choam/src/main/java/com/salesforce/apollo/choam/Session.java b/choam/src/main/java/com/salesforce/apollo/choam/Session.java index f21d652ad..dbe6b5969 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/Session.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/Session.java @@ -40,19 +40,18 @@ */ public class Session { - private final static Logger log = LoggerFactory.getLogger( - Session.class); - private final Limiter limiter; - private final Parameters params; - private final Function service; - private final Map submitted = new ConcurrentHashMap<>(); - private final AtomicReference view = new AtomicReference<>(); - private final ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1, - Thread.ofVirtual() - .factory()); - private final AtomicInteger nonce = new AtomicInteger(); + private final static Logger log = LoggerFactory.getLogger(Session.class); - public Session(Parameters params, Function service) { + private final Limiter limiter; + private final Parameters params; + private final Function service; + private final Map submitted = new ConcurrentHashMap<>(); + private final AtomicReference view = new AtomicReference<>(); + private final ScheduledExecutorService scheduler; + private final AtomicInteger nonce = new AtomicInteger(); + + public Session(Parameters params, Function service, + ScheduledExecutorService scheduler) { this.params = params; this.service = service; final var metrics = params.metrics(); @@ -60,6 +59,7 @@ public Session(Parameters params, Function s .build(params.member().getId().shortString(), metrics == null ? EmptyMetricRegistry.INSTANCE : metrics.getMetricRegistry( params.context().getId().shortString() + ".txnLimiter")); + this.scheduler = scheduler; } public static Transaction transactionOf(Digest source, int nonce, Message message, Signer signer) { @@ -117,10 +117,6 @@ public void setView(HashedCertifiedBlock v) { } } - public void stop() { - scheduler.shutdown(); - } - /** * Submit a transaction. * diff --git a/choam/src/main/java/com/salesforce/apollo/choam/support/Bootstrapper.java b/choam/src/main/java/com/salesforce/apollo/choam/support/Bootstrapper.java index 35155358c..691d332cb 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/support/Bootstrapper.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/support/Bootstrapper.java @@ -30,7 +30,6 @@ import java.util.*; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; @@ -60,11 +59,12 @@ public class Bootstrapper { private volatile HashedCertifiedBlock genesis; public Bootstrapper(HashedCertifiedBlock anchor, Parameters params, Store store, - CommonCommunications bootstrapComm) { + CommonCommunications bootstrapComm, ScheduledExecutorService scheduler) { this.anchor = anchor; this.params = params; this.store = store; this.comms = bootstrapComm; + this.scheduler = scheduler; CertifiedBlock g = store.getCertifiedBlock(ULong.valueOf(0)); store.put(anchor); if (g != null) { @@ -75,7 +75,6 @@ public Bootstrapper(HashedCertifiedBlock anchor, Parameters params, Store store, log.info("Restore using no prior state on: {}", params.member().getId()); lastCheckpoint = null; } - scheduler = Executors.newScheduledThreadPool(1, Thread.ofVirtual().factory()); } public CompletableFuture synchronize() { diff --git a/choam/src/test/java/com/salesforce/apollo/choam/SessionTest.java b/choam/src/test/java/com/salesforce/apollo/choam/SessionTest.java index 1f8b9405a..315a836a6 100644 --- a/choam/src/test/java/com/salesforce/apollo/choam/SessionTest.java +++ b/choam/src/test/java/com/salesforce/apollo/choam/SessionTest.java @@ -84,7 +84,8 @@ public void func() throws Exception { }); return SubmitResult.newBuilder().setResult(Result.PUBLISHED).build(); }; - Session session = new Session(params, service); + Session session = new Session(params, service, + Executors.newScheduledThreadPool(1, Thread.ofVirtual().factory())); session.setView(new HashedCertifiedBlock(DigestAlgorithm.DEFAULT, CertifiedBlock.newBuilder() .setBlock(Block.newBuilder() .setHeader( @@ -136,7 +137,7 @@ public void scalingTest() throws Exception { MetricRegistry reg = new MetricRegistry(); Timer latency = reg.timer("Transaction latency"); - Session session = new Session(params, service); + Session session = new Session(params, service, scheduler); session.setView(new HashedCertifiedBlock(DigestAlgorithm.DEFAULT, CertifiedBlock.newBuilder() .setBlock(Block.newBuilder() .setHeader( diff --git a/choam/src/test/java/com/salesforce/apollo/choam/support/BootstrapperTest.java b/choam/src/test/java/com/salesforce/apollo/choam/support/BootstrapperTest.java index e0a2b6233..5c2805533 100644 --- a/choam/src/test/java/com/salesforce/apollo/choam/support/BootstrapperTest.java +++ b/choam/src/test/java/com/salesforce/apollo/choam/support/BootstrapperTest.java @@ -33,6 +33,7 @@ import java.time.Duration; import java.util.List; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.stream.IntStream; @@ -107,7 +108,7 @@ public void smoke() throws Exception { context) .setMember(member) .build()), store, - comms); + comms, Executors.newScheduledThreadPool(1, Thread.ofVirtual().factory())); CompletableFuture syncFuture = boot.synchronize(); SynchronizedState state = syncFuture.get(10, TimeUnit.SECONDS); diff --git a/fireflies/src/main/java/com/salesforce/apollo/fireflies/View.java b/fireflies/src/main/java/com/salesforce/apollo/fireflies/View.java index 9005abd90..d730d4741 100644 --- a/fireflies/src/main/java/com/salesforce/apollo/fireflies/View.java +++ b/fireflies/src/main/java/com/salesforce/apollo/fireflies/View.java @@ -96,7 +96,7 @@ public class View { private final RingCommunications gossiper; private final AtomicBoolean introduced = new AtomicBoolean(); private final Map> viewChangeListeners = new HashMap<>(); - private final Executor viewNotificationQueue; + private final Semaphore viewSerialization = new Semaphore(1); private final FireflyMetrics metrics; private final Node node; private final Map observations = new ConcurrentSkipListMap<>(); @@ -111,7 +111,6 @@ public class View { private final Verifiers verifiers; private final ScheduledExecutorService scheduler; private volatile ScheduledFuture futureGossip; - private volatile boolean boostrap = false; public View(DynamicContext context, ControlledIdentifierMember member, String endpoint, EventValidation validation, Verifiers verifiers, Router communications, Parameters params, @@ -144,7 +143,6 @@ public View(DynamicContext context, ControlledIdentifierMember memb gossiper.ignoreSelf(); this.validation = validation; this.verifiers = verifiers; - viewNotificationQueue = Executors.newSingleThreadExecutor(Thread.ofVirtual().factory()); viewChange = new ReentrantReadWriteLock(true); } @@ -243,6 +241,7 @@ public void stop() { if (!started.compareAndSet(true, false)) { return; } + viewSerialization.release(10000); roundTimers.reset(); comm.deregister(context.getId()); pendingRebuttals.clear(); @@ -328,7 +327,6 @@ boolean addToView(NoteWrapper note) { } void bootstrap(NoteWrapper nw, Duration dur) { - boostrap = true; viewManagement.bootstrap(nw, dur); } @@ -415,13 +413,24 @@ void notifyListeners(List joining, List leavin joining.stream().map(SelfAddressingIdentifier::getDigest).toList(), Collections.unmodifiableList(leaving)); viewChangeListeners.forEach((key, value) -> { - viewNotificationQueue.execute(Utils.wrapped(() -> { + Thread.ofVirtual().start(Utils.wrapped(() -> { + try { + viewSerialization.acquire(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return; + } + if (!started.get()) { + return; + } try { log.trace("Notifying: {} view change: {} cardinality: {} joins: {} leaves: {} on: {} ", key, currentView(), context.size(), joining.size(), leaving.size(), node.getId()); value.accept(viewChange); } catch (Throwable e) { log.error("error in view change listener: {} on: {} ", key, node.getId(), e); + } finally { + viewSerialization.release(); } }, log)); }); diff --git a/memberships/src/main/java/com/salesforce/apollo/membership/messaging/rbc/ReliableBroadcaster.java b/memberships/src/main/java/com/salesforce/apollo/membership/messaging/rbc/ReliableBroadcaster.java index 97698686f..ee0455574 100644 --- a/memberships/src/main/java/com/salesforce/apollo/membership/messaging/rbc/ReliableBroadcaster.java +++ b/memberships/src/main/java/com/salesforce/apollo/membership/messaging/rbc/ReliableBroadcaster.java @@ -198,7 +198,7 @@ public void start(Duration duration, Predicate Thread.ofVirtual().start(Utils.wrapped(() -> oneRound(duration, scheduler), log)), initialDelay, TimeUnit.MILLISECONDS); } diff --git a/sql-state/src/main/java/com/salesforce/apollo/state/Emulator.java b/sql-state/src/main/java/com/salesforce/apollo/state/Emulator.java index 9890d50eb..c9cd2aed6 100644 --- a/sql-state/src/main/java/com/salesforce/apollo/state/Emulator.java +++ b/sql-state/src/main/java/com/salesforce/apollo/state/Emulator.java @@ -34,6 +34,8 @@ import java.sql.Connection; import java.util.Arrays; import java.util.Properties; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; @@ -47,15 +49,16 @@ */ public class Emulator { - private final AtomicReference hash; - private final AtomicLong height = new AtomicLong(0); - private final ReentrantLock lock = new ReentrantLock(); - private final Mutator mutator; - private final Parameters params; - private final SqlStateMachine ssm; - private final AtomicBoolean started = new AtomicBoolean(); - private final TransactionExecutor txnExec; - private final AtomicInteger txnIndex = new AtomicInteger(0); + private final AtomicReference hash; + private final AtomicLong height = new AtomicLong(0); + private final ReentrantLock lock = new ReentrantLock(); + private final Mutator mutator; + private final Parameters params; + private final SqlStateMachine ssm; + private final AtomicBoolean started = new AtomicBoolean(); + private final TransactionExecutor txnExec; + private final AtomicInteger txnIndex = new AtomicInteger(0); + private final ScheduledExecutorService scheduler; public Emulator() throws IOException { this(DigestAlgorithm.DEFAULT.getOrigin().prefix(Entropy.nextBitsStreamLong())); @@ -64,11 +67,13 @@ public Emulator() throws IOException { public Emulator(Digest base) throws IOException { this(new SqlStateMachine(DigestAlgorithm.DEFAULT.getOrigin(), String.format("jdbc:h2:mem:emulation-%s-%s", base, Entropy.nextBitsStreamLong()), - new Properties(), Files.createTempDirectory("emulation").toFile()), base); + new Properties(), Files.createTempDirectory("emulation").toFile()), base, + Executors.newScheduledThreadPool(1, Thread.ofVirtual().factory())); } - public Emulator(SqlStateMachine ssm, Digest base) { + public Emulator(SqlStateMachine ssm, Digest base, ScheduledExecutorService scheduler) throws IOException { this.ssm = ssm; + this.scheduler = scheduler; txnExec = this.ssm.getExecutor(); hash = new AtomicReference<>(base); SecureRandom entropy; @@ -97,7 +102,7 @@ public Emulator(SqlStateMachine ssm, Digest base) { } finally { lock.unlock(); } - }); + }, scheduler); session.setView(new HashedCertifiedBlock(DigestAlgorithm.DEFAULT, CertifiedBlock.newBuilder() .setBlock(Block.newBuilder() .setHeader( From a799b86765c83fb395708903c671b923d0b3d3c4 Mon Sep 17 00:00:00 2001 From: Hellblazer Date: Fri, 28 Jun 2024 17:19:46 -0700 Subject: [PATCH 19/32] firm up join --- .../com/salesforce/apollo/choam/CHOAM.java | 71 +++++++++---------- .../salesforce/apollo/ethereal/Ethereal.java | 1 - 2 files changed, 33 insertions(+), 39 deletions(-) diff --git a/choam/src/main/java/com/salesforce/apollo/choam/CHOAM.java b/choam/src/main/java/com/salesforce/apollo/choam/CHOAM.java index f90ad6ca0..f65003a97 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/CHOAM.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/CHOAM.java @@ -76,31 +76,31 @@ public class CHOAM { private static final Logger log = LoggerFactory.getLogger(CHOAM.class); - private final Map cachedCheckpoints = new ConcurrentHashMap<>(); - private final AtomicReference checkpoint = new AtomicReference<>(); - private final ReliableBroadcaster combine; - private final CommonCommunications comm; - private final AtomicReference current = new AtomicReference<>(); - private final AtomicReference> futureBootstrap = new AtomicReference<>(); - private final AtomicReference> futureSynchronization = new AtomicReference<>(); - private final AtomicReference genesis = new AtomicReference<>(); - private final AtomicReference head = new AtomicReference<>(); - private final AtomicReference next = new AtomicReference<>(); - private final AtomicReference nextViewId = new AtomicReference<>(); - private final Parameters params; - private final PriorityBlockingQueue pending = new PriorityBlockingQueue<>(); - private final RoundScheduler roundScheduler; - private final Session session; - private final AtomicBoolean started = new AtomicBoolean(); - private final Store store; - private final CommonCommunications submissionComm; - private final Combine.Transitions transitions; - private final TransSubmission txnSubmission = new TransSubmission(); - private final AtomicReference view = new AtomicReference<>(); - private final PendingViews pendingViews = new PendingViews(); - private final ScheduledExecutorService scheduler; - private final Semaphore linear = new Semaphore(1); - private volatile AtomicBoolean ongoingJoin; + private final Map cachedCheckpoints = new ConcurrentHashMap<>(); + private final AtomicReference checkpoint = new AtomicReference<>(); + private final ReliableBroadcaster combine; + private final CommonCommunications comm; + private final AtomicReference current = new AtomicReference<>(); + private final AtomicReference> futureBootstrap = new AtomicReference<>(); + private final AtomicReference> futureSynchronization = new AtomicReference<>(); + private final AtomicReference genesis = new AtomicReference<>(); + private final AtomicReference head = new AtomicReference<>(); + private final AtomicReference next = new AtomicReference<>(); + private final AtomicReference nextViewId = new AtomicReference<>(); + private final Parameters params; + private final PriorityBlockingQueue pending = new PriorityBlockingQueue<>(); + private final RoundScheduler roundScheduler; + private final Session session; + private final AtomicBoolean started = new AtomicBoolean(); + private final Store store; + private final CommonCommunications submissionComm; + private final Combine.Transitions transitions; + private final TransSubmission txnSubmission = new TransSubmission(); + private final AtomicReference view = new AtomicReference<>(); + private final PendingViews pendingViews = new PendingViews(); + private final ScheduledExecutorService scheduler; + private final Semaphore linear = new Semaphore(1); + private final AtomicBoolean ongoingJoin = new AtomicBoolean(); public CHOAM(Parameters params) { scheduler = Executors.newScheduledThreadPool(1, Thread.ofVirtual().factory()); @@ -734,11 +734,8 @@ private void reconfigure(Digest hash, Reconfigure reconfigure) { } else { current.set(new Client(validators, getViewId())); } - final var oj = ongoingJoin; - ongoingJoin = null; - if (oj != null) { + if (ongoingJoin.compareAndSet(true, false)) { log.trace("Halting ongoing join on: {}", params.member().getId()); - oj.set(true); } log.info("Reconfigured to view: {} committee: {} validators: {} on: {}", new Digest(reconfigure.getId()), current.get().getClass().getSimpleName(), validators.entrySet() @@ -1402,30 +1399,28 @@ public boolean validate(HashedCertifiedBlock hb) { } private void join(View view) { - if (ongoingJoin != null) { + if (!ongoingJoin.compareAndSet(false, true)) { throw new IllegalStateException("Ongoing join should have been cancelled"); } log.trace("Joining view: {} diadem: {} on: {}", nextViewId.get(), Digest.from(view.getDiadem()), params.member().getId()); var servers = new ConcurrentSkipListSet<>(validators.keySet()); var joined = new AtomicInteger(); - var halt = new AtomicBoolean(false); - ongoingJoin = halt; log.trace("Starting join of: {} diadem {} on: {}", nextViewId.get(), Digest.from(view.getDiadem()), params.member().getId()); var scheduler = Executors.newSingleThreadScheduledExecutor(Thread.ofVirtual().factory()); AtomicReference action = new AtomicReference<>(); var attempts = new AtomicInteger(); action.set(() -> { - log.trace("Join attempt: {} halt: {} joined: {} majority: {} on: {}", attempts.incrementAndGet(), - halt.get(), joined.get(), view.getMajority(), params.member().getId()); - if (!halt.get() & joined.get() < view.getMajority()) { + log.trace("Join attempt: {} ongoing: {} joined: {} majority: {} on: {}", attempts.incrementAndGet(), + ongoingJoin.get(), joined.get(), view.getMajority(), params.member().getId()); + if (ongoingJoin.get() & joined.get() < view.getMajority()) { join(view, servers, joined); if (joined.get() >= view.getMajority()) { - ongoingJoin = null; + ongoingJoin.set(false); log.trace("Finished join of: {} diadem: {} joins: {} on: {}", nextViewId.get(), Digest.from(view.getDiadem()), joined.get(), params.member().getId()); - } else if (!halt.get()) { + } else if (ongoingJoin.get()) { log.trace("Rescheduling join of: {} diadem: {} joins: {} on: {}", nextViewId.get(), Digest.from(view.getDiadem()), joined.get(), params.member().getId()); scheduler.schedule(action.get(), 50, TimeUnit.MILLISECONDS); @@ -1450,7 +1445,7 @@ private void join(View view, Collection members, AtomicInteger joined) { .setSignature(params.member().sign(inView.toByteString()).toSig()) .build(); var countdown = new CountDownLatch(sampled.size()); - sampled.stream().map(m -> { + sampled.parallelStream().map(m -> { var connection = comm.connect(m); log.trace("connect to: {} is: {} on: {}", m.getId(), connection, params.member().getId()); return connection; diff --git a/ethereal/src/main/java/com/salesforce/apollo/ethereal/Ethereal.java b/ethereal/src/main/java/com/salesforce/apollo/ethereal/Ethereal.java index 67ea81f4e..cbead8839 100644 --- a/ethereal/src/main/java/com/salesforce/apollo/ethereal/Ethereal.java +++ b/ethereal/src/main/java/com/salesforce/apollo/ethereal/Ethereal.java @@ -429,6 +429,5 @@ public int compareTo(UnitTask o) { public void run() { consumer.accept(unit); } - } } From a6ae3893c24babc291dccb919722e0577830a836 Mon Sep 17 00:00:00 2001 From: Hellblazer Date: Fri, 28 Jun 2024 18:06:20 -0700 Subject: [PATCH 20/32] don't use parallel stream on join. suppress metric reporting, gated with system prop --- .../java/com/salesforce/apollo/choam/CHOAM.java | 2 +- .../com/salesforce/apollo/choam/ViewAssembly.java | 5 ++++- .../com/salesforce/apollo/choam/SessionTest.java | 12 +++++++----- .../com/salesforce/apollo/choam/TestCHOAM.java | 13 +++++++------ .../salesforce/apollo/fireflies/ChurnTest.java | 14 ++++++++------ .../com/salesforce/apollo/fireflies/E2ETest.java | 14 ++++++++------ .../com/salesforce/apollo/fireflies/MtlsTest.java | 12 +++++++----- .../salesforce/apollo/fireflies/SwarmTest.java | 14 ++++++++------ .../salesforce/apollo/messaging/rbc/RbcTest.java | 12 +++++++----- .../com/salesforce/apollo/state/CHOAMTest.java | 15 ++++++++------- 10 files changed, 65 insertions(+), 48 deletions(-) diff --git a/choam/src/main/java/com/salesforce/apollo/choam/CHOAM.java b/choam/src/main/java/com/salesforce/apollo/choam/CHOAM.java index f65003a97..42cf1dce1 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/CHOAM.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/CHOAM.java @@ -1445,7 +1445,7 @@ private void join(View view, Collection members, AtomicInteger joined) { .setSignature(params.member().sign(inView.toByteString()).toSig()) .build(); var countdown = new CountDownLatch(sampled.size()); - sampled.parallelStream().map(m -> { + sampled.stream().map(m -> { var connection = comm.connect(m); log.trace("connect to: {} is: {} on: {}", m.getId(), connection, params.member().getId()); return connection; diff --git a/choam/src/main/java/com/salesforce/apollo/choam/ViewAssembly.java b/choam/src/main/java/com/salesforce/apollo/choam/ViewAssembly.java index 25e4cb2bb..12d4b4e3d 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/ViewAssembly.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/ViewAssembly.java @@ -20,6 +20,7 @@ import com.salesforce.apollo.cryptography.proto.PubKey; import com.salesforce.apollo.ethereal.Dag; import com.salesforce.apollo.membership.Member; +import com.salesforce.apollo.utils.Utils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -199,7 +200,9 @@ boolean complete() { + selected.assembly.size(); log.debug("View Assembly: {} completed assembly: {} on: {}", nextViewId, slate.keySet().stream().sorted().toList(), params().member().getId()); - transitions.complete(); + Thread.ofVirtual().start(Utils.wrapped(() -> { + transitions.complete(); + }, log)); return true; } diff --git a/choam/src/test/java/com/salesforce/apollo/choam/SessionTest.java b/choam/src/test/java/com/salesforce/apollo/choam/SessionTest.java index 315a836a6..c8b052316 100644 --- a/choam/src/test/java/com/salesforce/apollo/choam/SessionTest.java +++ b/choam/src/test/java/com/salesforce/apollo/choam/SessionTest.java @@ -178,10 +178,12 @@ public void scalingTest() throws Exception { } } System.out.println(); - ConsoleReporter.forRegistry(reg) - .convertRatesTo(TimeUnit.SECONDS) - .convertDurationsTo(TimeUnit.MILLISECONDS) - .build() - .report(); + if (Boolean.getBoolean("reportMetrics")) { + ConsoleReporter.forRegistry(reg) + .convertRatesTo(TimeUnit.SECONDS) + .convertDurationsTo(TimeUnit.MILLISECONDS) + .build() + .report(); + } } } diff --git a/choam/src/test/java/com/salesforce/apollo/choam/TestCHOAM.java b/choam/src/test/java/com/salesforce/apollo/choam/TestCHOAM.java index 4c5866d4d..30a727ae9 100644 --- a/choam/src/test/java/com/salesforce/apollo/choam/TestCHOAM.java +++ b/choam/src/test/java/com/salesforce/apollo/choam/TestCHOAM.java @@ -203,12 +203,13 @@ public void submitMultiplTxn() throws Exception { choams.values().forEach(e -> e.stop()); System.out.println(); - - ConsoleReporter.forRegistry(registry) - .convertRatesTo(TimeUnit.SECONDS) - .convertDurationsTo(TimeUnit.MILLISECONDS) - .build() - .report(); + if (Boolean.getBoolean("reportMetrics")) { + ConsoleReporter.forRegistry(registry) + .convertRatesTo(TimeUnit.SECONDS) + .convertDurationsTo(TimeUnit.MILLISECONDS) + .build() + .report(); + } } assertTrue(checkpointOccurred.get(5, TimeUnit.SECONDS)); } diff --git a/fireflies/src/test/java/com/salesforce/apollo/fireflies/ChurnTest.java b/fireflies/src/test/java/com/salesforce/apollo/fireflies/ChurnTest.java index 913623451..921a54ff2 100644 --- a/fireflies/src/test/java/com/salesforce/apollo/fireflies/ChurnTest.java +++ b/fireflies/src/test/java/com/salesforce/apollo/fireflies/ChurnTest.java @@ -260,12 +260,14 @@ public void churn() throws Exception { assertTrue(testGraph.isSC()); } - System.out.println("Node 0 metrics"); - ConsoleReporter.forRegistry(node0Registry) - .convertRatesTo(TimeUnit.SECONDS) - .convertDurationsTo(TimeUnit.MILLISECONDS) - .build() - .report(); + if (Boolean.getBoolean("reportMetrics")) { + System.out.println("Node 0 metrics"); + ConsoleReporter.forRegistry(node0Registry) + .convertRatesTo(TimeUnit.SECONDS) + .convertDurationsTo(TimeUnit.MILLISECONDS) + .build() + .report(); + } } private void initialize() { diff --git a/fireflies/src/test/java/com/salesforce/apollo/fireflies/E2ETest.java b/fireflies/src/test/java/com/salesforce/apollo/fireflies/E2ETest.java index 6b4121346..7a9893293 100644 --- a/fireflies/src/test/java/com/salesforce/apollo/fireflies/E2ETest.java +++ b/fireflies/src/test/java/com/salesforce/apollo/fireflies/E2ETest.java @@ -221,12 +221,14 @@ private void initialize() { private void post() { communications.forEach(e -> e.close(Duration.ofSeconds(0))); views.forEach(view -> view.stop()); - System.out.println("Node 0 metrics"); - ConsoleReporter.forRegistry(node0Registry) - .convertRatesTo(TimeUnit.SECONDS) - .convertDurationsTo(TimeUnit.MILLISECONDS) - .build() - .report(); + if (Boolean.getBoolean("reportMetrics")) { + System.out.println("Node 0 metrics"); + ConsoleReporter.forRegistry(node0Registry) + .convertRatesTo(TimeUnit.SECONDS) + .convertDurationsTo(TimeUnit.MILLISECONDS) + .build() + .report(); + } } private void validateConstraints() { diff --git a/fireflies/src/test/java/com/salesforce/apollo/fireflies/MtlsTest.java b/fireflies/src/test/java/com/salesforce/apollo/fireflies/MtlsTest.java index 9a1176cb7..3361d7084 100644 --- a/fireflies/src/test/java/com/salesforce/apollo/fireflies/MtlsTest.java +++ b/fireflies/src/test/java/com/salesforce/apollo/fireflies/MtlsTest.java @@ -183,11 +183,13 @@ public void smoke() throws Exception { System.out.println("Stoping views"); views.forEach(view -> view.stop()); - ConsoleReporter.forRegistry(node0Registry) - .convertRatesTo(TimeUnit.SECONDS) - .convertDurationsTo(TimeUnit.MILLISECONDS) - .build() - .report(); + if (Boolean.getBoolean("reportMetrics")) { + ConsoleReporter.forRegistry(node0Registry) + .convertRatesTo(TimeUnit.SECONDS) + .convertDurationsTo(TimeUnit.MILLISECONDS) + .build() + .report(); + } } private Function clientContextSupplier() { diff --git a/fireflies/src/test/java/com/salesforce/apollo/fireflies/SwarmTest.java b/fireflies/src/test/java/com/salesforce/apollo/fireflies/SwarmTest.java index 2746534ce..fa0d75aea 100644 --- a/fireflies/src/test/java/com/salesforce/apollo/fireflies/SwarmTest.java +++ b/fireflies/src/test/java/com/salesforce/apollo/fireflies/SwarmTest.java @@ -204,12 +204,14 @@ public void swarm() throws Exception { } communications.forEach(e -> e.close(Duration.ofSeconds(1))); views.forEach(view -> view.stop()); - System.out.println("Node 0 metrics"); - ConsoleReporter.forRegistry(node0Registry) - .convertRatesTo(TimeUnit.SECONDS) - .convertDurationsTo(TimeUnit.MILLISECONDS) - .build() - .report(); + if (Boolean.getBoolean("reportMetrics")) { + System.out.println("Node 0 metrics"); + ConsoleReporter.forRegistry(node0Registry) + .convertRatesTo(TimeUnit.SECONDS) + .convertDurationsTo(TimeUnit.MILLISECONDS) + .build() + .report(); + } } private void initialize() { diff --git a/memberships/src/test/java/com/salesforce/apollo/messaging/rbc/RbcTest.java b/memberships/src/test/java/com/salesforce/apollo/messaging/rbc/RbcTest.java index 2d6eb50c2..ed5c8ff08 100644 --- a/memberships/src/test/java/com/salesforce/apollo/messaging/rbc/RbcTest.java +++ b/memberships/src/test/java/com/salesforce/apollo/messaging/rbc/RbcTest.java @@ -145,11 +145,13 @@ public void broadcast() throws Exception { System.out.println(); - ConsoleReporter.forRegistry(registry) - .convertRatesTo(TimeUnit.SECONDS) - .convertDurationsTo(TimeUnit.MILLISECONDS) - .build() - .report(); + if (Boolean.getBoolean("reportMetrics")) { + ConsoleReporter.forRegistry(registry) + .convertRatesTo(TimeUnit.SECONDS) + .convertDurationsTo(TimeUnit.MILLISECONDS) + .build() + .report(); + } } class Receiver implements MessageHandler { diff --git a/sql-state/src/test/java/com/salesforce/apollo/state/CHOAMTest.java b/sql-state/src/test/java/com/salesforce/apollo/state/CHOAMTest.java index d0dca0785..12a79a49c 100644 --- a/sql-state/src/test/java/com/salesforce/apollo/state/CHOAMTest.java +++ b/sql-state/src/test/java/com/salesforce/apollo/state/CHOAMTest.java @@ -117,11 +117,13 @@ public void after() throws Exception { members = null; System.out.println(); - ConsoleReporter.forRegistry(registry) - .convertRatesTo(TimeUnit.SECONDS) - .convertDurationsTo(TimeUnit.MILLISECONDS) - .build() - .report(); + if (Boolean.getBoolean("reportMetrics")) { + ConsoleReporter.forRegistry(registry) + .convertRatesTo(TimeUnit.SECONDS) + .convertDurationsTo(TimeUnit.MILLISECONDS) + .build() + .report(); + } registry = null; } @@ -328,8 +330,7 @@ public void endBlock(ULong height, Digest hash) { @Override public void execute(int i, Digest hash, Transaction tx, @SuppressWarnings("rawtypes") CompletableFuture onComplete) { - up.getExecutor() - .execute(i, hash, tx, onComplete); + up.getExecutor().execute(i, hash, tx, onComplete); } @Override From dfa1e01ba44ebce3afa79ba3c681bc08393114cf Mon Sep 17 00:00:00 2001 From: Hellblazer Date: Sat, 29 Jun 2024 08:27:12 -0700 Subject: [PATCH 21/32] decouple --- .../com/salesforce/apollo/choam/Producer.java | 103 +++++++++++------- .../salesforce/apollo/choam/ViewAssembly.java | 5 +- .../ethereal/memberships/ChRbcGossip.java | 11 +- 3 files changed, 73 insertions(+), 46 deletions(-) diff --git a/choam/src/main/java/com/salesforce/apollo/choam/Producer.java b/choam/src/main/java/com/salesforce/apollo/choam/Producer.java index 7214643a3..f4b0df709 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/Producer.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/Producer.java @@ -25,6 +25,7 @@ import com.salesforce.apollo.ethereal.Ethereal; import com.salesforce.apollo.ethereal.memberships.ChRbcGossip; import com.salesforce.apollo.membership.Member; +import com.salesforce.apollo.utils.Utils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -55,7 +56,7 @@ public class Producer { private final Semaphore serialize = new Semaphore(1); private final ViewAssembly assembly; private final int maxEpoch; - private volatile boolean assembled = false; + private final AtomicBoolean assembled = new AtomicBoolean(false); public Producer(Digest nextViewId, ViewContext view, HashedBlock lastBlock, HashedBlock checkpoint, String label, ScheduledExecutorService scheduler) { @@ -116,7 +117,7 @@ public boolean complete() { if (super.complete()) { log.debug("View reconfiguration: {} gathered: {} complete on: {}", nextViewId, getSlate().keySet().stream().sorted().toList(), params().member().getId()); - assembled = true; + assembled.set(true); return true; } return false; @@ -132,14 +133,20 @@ public void start() { if (!started.compareAndSet(false, true)) { return; } - final Block prev = previousBlock.get().block; - // genesis block won't ever be 0 - if (prev.hasGenesis() || (prev.hasReconfigure() && prev.getReconfigure().getCheckpointTarget() == 0)) { - transitions.checkpoint(); - } else { - log.trace("Checkpoint target: {} for: {} on: {}", prev.getReconfigure().getCheckpointTarget(), - params().context().getId(), params().member().getId()); - transitions.start(); + try { + Thread.ofVirtual().start(Utils.wrapped(() -> { + final Block prev = previousBlock.get().block; + // genesis block won't ever be 0 + if (prev.hasGenesis() || (prev.hasReconfigure() && prev.getReconfigure().getCheckpointTarget() == 0)) { + transitions.checkpoint(); + } else { + log.trace("Checkpoint target: {} for: {} on: {}", prev.getReconfigure().getCheckpointTarget(), + params().context().getId(), params().member().getId()); + transitions.start(); + } + }, log)); + } catch (RejectedExecutionException e) { + log.trace("Reject fork on: {}", params().member().getId()); } } @@ -214,16 +221,32 @@ private Digest getViewId() { } private void newEpoch(Integer epoch) { - log.trace("new epoch: {} on: {}", epoch, params().member().getId()); - assembly.newEpoch(); - var last = epoch >= maxEpoch && assembled; - if (last) { - controller.completeIt(); - Producer.this.transitions.viewComplete(); - } else { - ds.reset(); + Thread.ofVirtual().start(Utils.wrapped(() -> { + try { + serialize.acquire(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return; + } + try { + log.trace("new epoch: {} on: {}", epoch, params().member().getId()); + assembly.newEpoch(); + var last = epoch >= maxEpoch && assembled.get(); + if (last) { + controller.completeIt(); + Producer.this.transitions.viewComplete(); + } else { + ds.reset(); + } + transitions.newEpoch(epoch, last); + } finally { + serialize.release(); + } + }, log)); + var awaiting = serialize.getQueueLength(); + if (awaiting > 0) { + log.error("Serialize: {} on: {}", awaiting, params().member().getId()); } - transitions.newEpoch(epoch, last); } private Parameters params() { @@ -403,25 +426,27 @@ public void assemble() { @Override public void checkpoint() { - log.info("Generating checkpoint block on: {}", params().member().getId()); - Block ckpt = view.checkpoint(); - if (ckpt == null) { - log.error("Cannot generate checkpoint block on: {}", params().member().getId()); - transitions.failed(); - return; - } - var next = new HashedBlock(params().digestAlgorithm(), ckpt); - previousBlock.set(next); - checkpoint.set(next); - var validation = view.generateValidation(next); - ds.offer(validation); - final var p = new PendingBlock(next, new HashMap<>(), new AtomicBoolean()); - pending.put(next.hash, p); - p.witnesses.put(params().member(), validation); - log.info("Produced: {} hash: {} height: {} for: {} on: {}", next.block.getBodyCase(), next.hash, - next.height(), getViewId(), params().member().getId()); - processPendingValidations(next, p); - transitions.checkpointed(); + Thread.ofVirtual().start(Utils.wrapped(() -> { + log.info("Generating checkpoint block on: {}", params().member().getId()); + Block ckpt = view.checkpoint(); + if (ckpt == null) { + log.error("Cannot generate checkpoint block on: {}", params().member().getId()); + transitions.failed(); + return; + } + var next = new HashedBlock(params().digestAlgorithm(), ckpt); + previousBlock.set(next); + checkpoint.set(next); + var validation = view.generateValidation(next); + ds.offer(validation); + final var p = new PendingBlock(next, new HashMap<>(), new AtomicBoolean()); + pending.put(next.hash, p); + p.witnesses.put(params().member(), validation); + log.info("Produced: {} hash: {} height: {} for: {} on: {}", next.block.getBodyCase(), next.hash, + next.height(), getViewId(), params().member().getId()); + processPendingValidations(next, p); + transitions.checkpointed(); + }, log)); } @Override @@ -437,7 +462,7 @@ public void fail() { @Override public void reconfigure() { - Producer.this.reconfigure(); + Thread.ofVirtual().start(Utils.wrapped(Producer.this::reconfigure, log)); } @Override diff --git a/choam/src/main/java/com/salesforce/apollo/choam/ViewAssembly.java b/choam/src/main/java/com/salesforce/apollo/choam/ViewAssembly.java index 12d4b4e3d..25e4cb2bb 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/ViewAssembly.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/ViewAssembly.java @@ -20,7 +20,6 @@ import com.salesforce.apollo.cryptography.proto.PubKey; import com.salesforce.apollo.ethereal.Dag; import com.salesforce.apollo.membership.Member; -import com.salesforce.apollo.utils.Utils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -200,9 +199,7 @@ boolean complete() { + selected.assembly.size(); log.debug("View Assembly: {} completed assembly: {} on: {}", nextViewId, slate.keySet().stream().sorted().toList(), params().member().getId()); - Thread.ofVirtual().start(Utils.wrapped(() -> { - transitions.complete(); - }, log)); + transitions.complete(); return true; } diff --git a/ethereal/src/main/java/com/salesforce/apollo/ethereal/memberships/ChRbcGossip.java b/ethereal/src/main/java/com/salesforce/apollo/ethereal/memberships/ChRbcGossip.java index 2e6028b99..6cd5e48af 100644 --- a/ethereal/src/main/java/com/salesforce/apollo/ethereal/memberships/ChRbcGossip.java +++ b/ethereal/src/main/java/com/salesforce/apollo/ethereal/memberships/ChRbcGossip.java @@ -30,6 +30,7 @@ import java.time.Duration; import java.util.Collection; import java.util.Optional; +import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; @@ -131,9 +132,13 @@ private void gossip(Duration frequency, ScheduledExecutorService scheduler) { timer.stop(); } if (started.get()) { - scheduled = scheduler.schedule( - () -> Thread.ofVirtual().start(Utils.wrapped(() -> gossip(frequency, scheduler), log)), - frequency.toNanos(), TimeUnit.NANOSECONDS); + try { + scheduled = scheduler.schedule( + () -> Thread.ofVirtual().start(Utils.wrapped(() -> gossip(frequency, scheduler), log)), + frequency.toNanos(), TimeUnit.NANOSECONDS); + } catch (RejectedExecutionException e) { + log.trace("Reject scheduling on: {}", member.getId()); + } } }, frequency); } From 7d225978254c7829bc40cb5a142d5ed040243347 Mon Sep 17 00:00:00 2001 From: Hellblazer Date: Sat, 29 Jun 2024 21:21:46 -0700 Subject: [PATCH 22/32] better observation attempt lifecycle. use executor for linearizing --- .../com/salesforce/apollo/choam/CHOAM.java | 71 +++++++------------ .../com/salesforce/apollo/choam/Producer.java | 58 +++++---------- .../salesforce/apollo/choam/fsm/Driven.java | 12 +--- .../ethereal/memberships/ChRbcGossip.java | 2 + .../com/salesforce/apollo/fireflies/View.java | 19 +++-- .../apollo/fireflies/ViewManagement.java | 42 +++++++---- 6 files changed, 85 insertions(+), 119 deletions(-) diff --git a/choam/src/main/java/com/salesforce/apollo/choam/CHOAM.java b/choam/src/main/java/com/salesforce/apollo/choam/CHOAM.java index 42cf1dce1..68fe30726 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/CHOAM.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/CHOAM.java @@ -99,13 +99,14 @@ public class CHOAM { private final AtomicReference view = new AtomicReference<>(); private final PendingViews pendingViews = new PendingViews(); private final ScheduledExecutorService scheduler; - private final Semaphore linear = new Semaphore(1); + private final ExecutorService linear; private final AtomicBoolean ongoingJoin = new AtomicBoolean(); public CHOAM(Parameters params) { scheduler = Executors.newScheduledThreadPool(1, Thread.ofVirtual().factory()); this.store = new Store(params.digestAlgorithm(), params.mvBuilder().clone().build()); this.params = params; + linear = Executors.newScheduledThreadPool(1, Thread.ofVirtual().factory()); pendingViews.add(params.context().getId(), params.context().delegate()); rotateViewKeys(); @@ -116,18 +117,16 @@ public CHOAM(Parameters params) { combine = new ReliableBroadcaster(bContext, params.member(), params.combine(), params.communications(), params.metrics() == null ? null : params.metrics().getCombineMetrics(), adapter); - combine.registerHandler((_, messages) -> { - Thread.ofVirtual().start(() -> { - if (!started.get()) { - return; - } - try { - combine(messages); - } catch (Throwable t) { - log.error("Failed to combine messages on: {}", params.member().getId(), t); - } - }); - }); + combine.registerHandler((_, messages) -> Thread.ofVirtual().start(() -> { + if (!started.get()) { + return; + } + try { + combine(messages); + } catch (Throwable t) { + log.error("Failed to combine messages on: {}", params.member().getId(), t); + } + })); head.set(new NullBlock(params.digestAlgorithm())); view.set(new NullBlock(params.digestAlgorithm())); checkpoint.set(new NullBlock(params.digestAlgorithm())); @@ -354,10 +353,11 @@ public void stop() { if (!started.compareAndSet(true, false)) { return; } - linear.release(10000); + linear.shutdown(); try { scheduler.shutdownNow(); } catch (Throwable e) { + // ignore } session.cancelAll(); final var c = current.get(); @@ -365,11 +365,13 @@ public void stop() { try { c.complete(); } catch (Throwable e) { + // ignore } } try { combine.stop(); } catch (Throwable e) { + // ignore } } @@ -577,12 +579,11 @@ private void execute(List execs) { h.hash, h.height(), execs.size(), params.member().getId()); for (int i = 0; i < execs.size(); i++) { var exec = execs.get(i); - final var index = i; Digest hash = hashOf(exec, params.digestAlgorithm()); var stxn = session.complete(hash); try { params.processor() - .execute(index, CHOAM.hashOf(exec, params.digestAlgorithm()), exec, + .execute(i, CHOAM.hashOf(exec, params.digestAlgorithm()), exec, stxn == null ? null : stxn.onCompletion()); } catch (Throwable t) { log.error("Exception processing transaction: {} block: {} height: {} on: {}", hash, h.hash, h.height(), @@ -591,7 +592,7 @@ private void execute(List execs) { } } - private CheckpointSegments fetch(CheckpointReplication request, Digest from) { + private CheckpointSegments fetch(CheckpointReplication request) { CheckpointState state = cachedCheckpoints.get(ULong.valueOf(request.getCheckpoint())); if (state == null) { log.info("No cached checkpoint for {} on: {}", request.getCheckpoint(), params.member().getId()); @@ -604,14 +605,14 @@ private CheckpointSegments fetch(CheckpointReplication request, Digest from) { .build(); } - private Blocks fetchBlocks(BlockReplication rep, Digest from) { + private Blocks fetchBlocks(BlockReplication rep) { BloomFilter bff = BloomFilter.from(rep.getBlocksBff()); Blocks.Builder blocks = Blocks.newBuilder(); store.fetchBlocks(bff, blocks, 100, ULong.valueOf(rep.getFrom()), ULong.valueOf(rep.getTo())); return blocks.build(); } - private Blocks fetchViewChain(BlockReplication rep, Digest from) { + private Blocks fetchViewChain(BlockReplication rep) { BloomFilter bff = BloomFilter.from(rep.getBlocksBff()); Blocks.Builder blocks = Blocks.newBuilder(); store.fetchViewChain(bff, blocks, 100, ULong.valueOf(rep.getFrom()), ULong.valueOf(rep.getTo())); @@ -649,7 +650,7 @@ private boolean isNext(HashedBlock next) { return isNext; } - private Empty join(SignedViewMember nextView, Digest from) { + private void join(SignedViewMember nextView, Digest from) { var c = current.get(); if (c == null) { log.trace("No committee for: {} to join: {} diadem: {} on: {}", from, @@ -658,7 +659,6 @@ private Empty join(SignedViewMember nextView, Digest from) { throw new StatusRuntimeException(FAILED_PRECONDITION); } c.join(nextView, from); - return Empty.getDefaultInstance(); } private Supplier pendingViews() { @@ -1196,25 +1196,7 @@ public void cancelTimer(String timer) { @Override public void combine() { - Thread.ofVirtual().start(Utils.wrapped(() -> { - if (!started.get()) { - return; - } - try { - linear.acquire(); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - return; - } - try { - CHOAM.this.combine(); - } finally { - linear.release(); - } - }, log)); - if (linear.getQueueLength() > 0) { - log.info("Linear Q: {} on: {}", linear.getQueueLength(), params.member().getId()); - } + linear.execute(Utils.wrapped(() -> CHOAM.this.combine(), log)); } @Override @@ -1270,17 +1252,17 @@ public class Trampoline implements Concierge { @Override public CheckpointSegments fetch(CheckpointReplication request, Digest from) { - return CHOAM.this.fetch(request, from); + return CHOAM.this.fetch(request); } @Override public Blocks fetchBlocks(BlockReplication request, Digest from) { - return CHOAM.this.fetchBlocks(request, from); + return CHOAM.this.fetchBlocks(request); } @Override public Blocks fetchViewChain(BlockReplication request, Digest from) { - return CHOAM.this.fetchViewChain(request, from); + return CHOAM.this.fetchViewChain(request); } @Override @@ -1506,9 +1488,6 @@ private Attempt join(View view, Terminal t, SignedViewMember svm) { record Attempt(Member m, ListenableFuture fs) { } - - private record JoinState(AtomicBoolean halt, Thread joining) { - } } /** a member of the current committee */ diff --git a/choam/src/main/java/com/salesforce/apollo/choam/Producer.java b/choam/src/main/java/com/salesforce/apollo/choam/Producer.java index f4b0df709..def7fd907 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/Producer.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/Producer.java @@ -53,7 +53,7 @@ public class Producer { private final Transitions transitions; private final ViewContext view; private final Digest nextViewId; - private final Semaphore serialize = new Semaphore(1); + private final ExecutorService serialize; private final ViewAssembly assembly; private final int maxEpoch; private final AtomicBoolean assembled = new AtomicBoolean(false); @@ -94,7 +94,7 @@ public Producer(Digest nextViewId, ViewContext view, HashedBlock lastBlock, Hash log.trace("Pid: {} for: {} on: {}", pid, getViewId(), params().member().getId()); config.setPid(pid).setnProc((short) view.roster().size()); } - + serialize = Executors.newScheduledThreadPool(1, Thread.ofVirtual().factory()); config.setLabel("Producer" + getViewId() + " on: " + params().member().getId()); var producerMetrics = params().metrics() == null ? null : params().metrics().getProducerMetrics(); controller = new Ethereal(config.build(), params().producer().maxBatchByteSize() + (8 * 1024), ds, this::serial, @@ -155,7 +155,7 @@ public void stop() { return; } log.trace("Closing producer for: {} on: {}", getViewId(), params().member().getId()); - serialize.release(10000); + serialize.shutdown(); controller.stop(); coordinator.stop(); ds.close(); @@ -221,32 +221,18 @@ private Digest getViewId() { } private void newEpoch(Integer epoch) { - Thread.ofVirtual().start(Utils.wrapped(() -> { - try { - serialize.acquire(); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - return; - } - try { - log.trace("new epoch: {} on: {}", epoch, params().member().getId()); - assembly.newEpoch(); - var last = epoch >= maxEpoch && assembled.get(); - if (last) { - controller.completeIt(); - Producer.this.transitions.viewComplete(); - } else { - ds.reset(); - } - transitions.newEpoch(epoch, last); - } finally { - serialize.release(); + serialize.execute(Utils.wrapped(() -> { + log.trace("new epoch: {} on: {}", epoch, params().member().getId()); + assembly.newEpoch(); + var last = epoch >= maxEpoch && assembled.get(); + if (last) { + controller.completeIt(); + Producer.this.transitions.viewComplete(); + } else { + ds.reset(); } + transitions.newEpoch(epoch, last); }, log)); - var awaiting = serialize.getQueueLength(); - if (awaiting > 0) { - log.error("Serialize: {} on: {}", awaiting, params().member().getId()); - } } private Parameters params() { @@ -374,27 +360,14 @@ private void reconfigure() { } private void serial(List preblock, Boolean last) { - Thread.ofVirtual().start(() -> { - try { - serialize.acquire(); - create(preblock, last); - } catch (Throwable t) { - log.error("Error processing preblock last: {} on: {}", last, params().member().getId(), t); - } finally { - serialize.release(); - } - }); - var awaiting = serialize.getQueueLength(); - if (awaiting > 0) { - log.error("Serialize: {} on: {}", awaiting, params().member().getId()); - } + serialize.execute(Utils.wrapped(() -> create(preblock, last), log)); } private PendingBlock validate(Validate v) { Digest hash = Digest.from(v.getHash()); var p = pending.get(hash); if (p == null) { - pendingValidations.computeIfAbsent(hash, h -> new CopyOnWriteArrayList<>()).add(v); + pendingValidations.computeIfAbsent(hash, _ -> new CopyOnWriteArrayList<>()).add(v); return null; } return validate(v, p, hash); @@ -442,6 +415,7 @@ public void checkpoint() { final var p = new PendingBlock(next, new HashMap<>(), new AtomicBoolean()); pending.put(next.hash, p); p.witnesses.put(params().member(), validation); + assert next.block != null; log.info("Produced: {} hash: {} height: {} for: {} on: {}", next.block.getBodyCase(), next.hash, next.height(), getViewId(), params().member().getId()); processPendingValidations(next, p); diff --git a/choam/src/main/java/com/salesforce/apollo/choam/fsm/Driven.java b/choam/src/main/java/com/salesforce/apollo/choam/fsm/Driven.java index 2cf86b242..890a841b9 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/fsm/Driven.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/fsm/Driven.java @@ -41,11 +41,6 @@ public void assemble() { public Transitions assembled() { return SPICE; } - - @Override - public Transitions newEpoch(int epoch, boolean lastEpoch) { - return lastEpoch ? PROTOCOL_FAILURE : null; - } }, CHECKPOINTING { @Entry public void check() { @@ -116,11 +111,6 @@ public void terminate() { context().fail(); } }, SPICE { - @Override - public Transitions newEpoch(int epoch, boolean lastEpoch) { - return lastEpoch ? PROTOCOL_FAILURE : null; - } - @Override public Transitions viewComplete() { return END_EPOCHS; @@ -157,7 +147,7 @@ default Transitions lastBlock() { } default Transitions newEpoch(int epoch, boolean lastEpoch) { - throw fsm().invalidTransitionOn(); + return null; } default Transitions start() { diff --git a/ethereal/src/main/java/com/salesforce/apollo/ethereal/memberships/ChRbcGossip.java b/ethereal/src/main/java/com/salesforce/apollo/ethereal/memberships/ChRbcGossip.java index 6cd5e48af..6e9f82705 100644 --- a/ethereal/src/main/java/com/salesforce/apollo/ethereal/memberships/ChRbcGossip.java +++ b/ethereal/src/main/java/com/salesforce/apollo/ethereal/memberships/ChRbcGossip.java @@ -98,6 +98,8 @@ public void start(Duration duration, Predicate= context.majority(); + } + boolean hasPendingRebuttals() { return !pendingRebuttals.isEmpty(); } @@ -835,7 +839,8 @@ private boolean add(NoteWrapper note) { */ private boolean add(SignedViewChange observation) { var svu = new SVU(observation, digestAlgo); - if (!viewManagement.isObserver(svu.observer)) { + var highWater = viewManagement.highWater(svu.observer); + if (highWater == null) { log.trace("Invalid observer: {} current: {} on: {}", svu.observer, currentView(), node.getId()); return false; } @@ -845,17 +850,17 @@ private boolean add(SignedViewChange observation) { node.getId()); return false; } + if (highWater >= svu.attempt) { + log.trace("Redundant view change: {} current: {} view: {} from {} on: {}", svu.attempt, highWater, + currentView(), svu.observer, node.getId()); + return false; + } final var member = context.getActiveMember(svu.observer); if (member == null) { log.trace("Cannot validate view change: {} current: {} from: {} on: {}", inView, currentView(), svu.observer, node.getId()); return false; } - if (!viewManagement.isObserver(member.id)) { - log.trace("Not an observer of: {} current: {} from: {} on: {}", inView, currentView(), svu.observer, - node.getId()); - return false; - } final var signature = JohnHancock.from(observation.getSignature()); if (!member.verify(signature, observation.getChange().toByteString())) { return false; @@ -866,6 +871,8 @@ private boolean add(SignedViewChange observation) { log.trace("Stale observation: {} current: {} view change: {} current: {} offline: {} on: {}", svu.attempt, cur.attempt, inView, currentView(), svu.observer, node.getId()); return cur; + } else { + viewManagement.updateHighWater(d, svu.attempt); } } log.trace("Observation: {} current: {} view change: {} from: {} on: {}", svu.attempt, inView, currentView(), diff --git a/fireflies/src/main/java/com/salesforce/apollo/fireflies/ViewManagement.java b/fireflies/src/main/java/com/salesforce/apollo/fireflies/ViewManagement.java index f850a852a..f4c77e354 100644 --- a/fireflies/src/main/java/com/salesforce/apollo/fireflies/ViewManagement.java +++ b/fireflies/src/main/java/com/salesforce/apollo/fireflies/ViewManagement.java @@ -48,7 +48,7 @@ public class ViewManagement { private static final Logger log = LoggerFactory.getLogger(ViewManagement.class); final AtomicReference diadem = new AtomicReference<>(); - final Set observers = new ConcurrentSkipListSet<>(); + final Map observers = new ConcurrentSkipListMap<>(); private final AtomicInteger attempt = new AtomicInteger(); private final Digest bootstrapView; private final DynamicContext context; @@ -127,7 +127,7 @@ Digest currentView() { } void enjoin(Join join, Digest observer) { - if (!observers.contains(node.getId()) || !observers.contains(observer)) { + if (!observers.containsKey(node.getId()) || !observers.containsKey(observer)) { log.trace("Not observer, ignored enjoin from: {} on: {}", observer, node.getId()); throw new StatusRuntimeException(Status.FAILED_PRECONDITION.withDescription("Not observer")); } @@ -152,7 +152,7 @@ void enjoin(Join join, Digest observer) { context.getId(), cardinality(), node.getId()); return; } - if (!observers.contains(node.getId())) { + if (!observers.containsKey(node.getId())) { log.trace("Not observer, ignoring Join from: {} observers: {} on: {}", from, observers, node.getId()); throw new StatusRuntimeException( Status.FAILED_PRECONDITION.withDescription("Not observer, ignored join of view")); @@ -170,7 +170,7 @@ void enjoin(Join join, Digest observer) { void gc(Participant member) { assert member != null; view.stable(() -> { - if (observers.remove(member.id)) { + if (observers.remove(member.id) != null) { log.trace("Removed observer: {} view: {} on: {}", member.id, currentView.get(), node.getId()); resetObservers(); } @@ -189,6 +189,10 @@ BloomFilter getJoinsBff(long seed, double p) { return bff; } + Integer highWater(Digest observer) { + return observers.get(observer); + } + /** * Initiate the view change */ @@ -372,7 +376,7 @@ void join(Join join, Digest from, StreamObserver responseObserver, Time .toList(), from, responseObserver, timer); return; } - if (!observers.contains(node.getId())) { + if (!observers.containsKey(node.getId())) { log.trace("Not observer, ignoring Join from: {} observers: {} on: {}", from, observers, node.getId()); responseObserver.onError(new StatusRuntimeException( Status.FAILED_PRECONDITION.withDescription("Not observer, ignored join of view"))); @@ -401,8 +405,8 @@ void join(Join join, Digest from, StreamObserver responseObserver, Time log.debug("Member pending join: {} view: {} context: {} on: {}", from, currentView(), context.getId(), node.getId()); var enjoining = new SliceIterator<>("Enjoining[%s:%s]".formatted(currentView(), from), node, - observers.stream().map(context::getActiveMember).toList(), view.comm, - scheduler); + observers.keySet().stream().map(context::getActiveMember).toList(), + view.comm, scheduler); enjoining.iterate(t -> t.enjoin(join), (_, _, _, _) -> true, () -> { }, Duration.ofMillis(1)); }); @@ -484,7 +488,9 @@ void maybeViewChange() { return; } } - if ((context.offlineCount() > 0 || !joins.isEmpty())) { + var change = context.offlineCount() > 0 || !joins.isEmpty(); + var shouldChange = isObserver() || view.hasMajorityObservervations(bootstrap); + if (change && shouldChange) { initiateViewChange(); } else { view.scheduleViewChange(); @@ -492,7 +498,7 @@ void maybeViewChange() { } Set observers() { - return observers; + return observers.keySet(); } List observersList() { @@ -535,6 +541,10 @@ HexBloom resetBootstrapView() { return hex; } + void resetHighWater() { + observers.entrySet().forEach(e -> e.setValue(-1)); + } + Redirect seed(Registration registration, Digest from) { final var requestView = Digest.from(registration.getView()); @@ -560,7 +570,7 @@ Redirect seed(Registration registration, Digest from) { return view.stable(() -> { var newMember = view.new Participant(note.getId()); - final var introductions = observers.stream().map(context::getMember).toList(); + final var introductions = observers.keySet().stream().map(context::getMember).toList(); log.info("Member seeding: {} view: {} context: {} introductions: {} on: {}", newMember.getId(), currentView(), context.getId(), introductions.stream().map(p -> p.getId()).toList(), node.getId()); @@ -582,11 +592,15 @@ void start(CompletableFuture onJoin, boolean bootstrap) { this.bootstrap = bootstrap; } + void updateHighWater(Digest d, int attempt) { + observers.compute(d, (k, v) -> attempt <= v ? v : attempt); + } + /** * @return true if the receiver is part of the BFT Observers of this group */ private boolean isObserver() { - return observers.contains(node.getId()); + return observers.containsKey(node.getId()); } private void joined(Collection seedSet, Digest from, StreamObserver responseObserver, @@ -632,9 +646,9 @@ private void resetObservers() { context.bftSubset(diadem.get().compact(), context::isActive) .stream() .map(Member::getId) - .forEach(observers::add); + .forEach(d -> observers.put(d, -1)); if (observers.isEmpty()) { - observers.add(node.getId()); // bootstrap case + observers.put(node.getId(), -1); // bootstrap case } if (observers.size() > 1 && observers.size() < context.getRingCount()) { log.debug("Incomplete observers: {} cardinality: {} view: {} context: {} on: {}", observers.size(), @@ -653,7 +667,7 @@ private void setDiadem(final HexBloom hex) { resetObservers(); log.trace("View: {} set diadem: {} cardinality: {} observers: {} view: {} context: {} size: {} on: {}", context.getId(), diadem.get().compactWrapped(), diadem.get().getCardinality(), - observers.stream().toList(), currentView(), context.getId(), context.size(), node.getId()); + observers.keySet().stream().toList(), currentView(), context.getId(), context.size(), node.getId()); } record Ballot(Digest view, List leaving, List joining, int hash) { From bd8393a2756a0788e55d1693c1982e57e9acc14d Mon Sep 17 00:00:00 2001 From: Hellblazer Date: Mon, 1 Jul 2024 19:19:23 -0700 Subject: [PATCH 23/32] moar observation lifecycle fixie. don't "start/end" ViewAssembly --- .../com/salesforce/apollo/choam/CHOAM.java | 176 ++++++++++++------ .../salesforce/apollo/choam/ViewAssembly.java | 19 -- .../salesforce/apollo/fireflies/Binding.java | 5 +- .../com/salesforce/apollo/fireflies/View.java | 7 +- .../apollo/fireflies/ViewManagement.java | 2 +- 5 files changed, 122 insertions(+), 87 deletions(-) diff --git a/choam/src/main/java/com/salesforce/apollo/choam/CHOAM.java b/choam/src/main/java/com/salesforce/apollo/choam/CHOAM.java index 68fe30726..e17faf827 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/CHOAM.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/CHOAM.java @@ -76,37 +76,36 @@ public class CHOAM { private static final Logger log = LoggerFactory.getLogger(CHOAM.class); - private final Map cachedCheckpoints = new ConcurrentHashMap<>(); - private final AtomicReference checkpoint = new AtomicReference<>(); - private final ReliableBroadcaster combine; - private final CommonCommunications comm; - private final AtomicReference current = new AtomicReference<>(); - private final AtomicReference> futureBootstrap = new AtomicReference<>(); - private final AtomicReference> futureSynchronization = new AtomicReference<>(); - private final AtomicReference genesis = new AtomicReference<>(); - private final AtomicReference head = new AtomicReference<>(); - private final AtomicReference next = new AtomicReference<>(); - private final AtomicReference nextViewId = new AtomicReference<>(); - private final Parameters params; - private final PriorityBlockingQueue pending = new PriorityBlockingQueue<>(); - private final RoundScheduler roundScheduler; - private final Session session; - private final AtomicBoolean started = new AtomicBoolean(); - private final Store store; - private final CommonCommunications submissionComm; - private final Combine.Transitions transitions; - private final TransSubmission txnSubmission = new TransSubmission(); - private final AtomicReference view = new AtomicReference<>(); - private final PendingViews pendingViews = new PendingViews(); - private final ScheduledExecutorService scheduler; - private final ExecutorService linear; - private final AtomicBoolean ongoingJoin = new AtomicBoolean(); + private final Map cachedCheckpoints = new ConcurrentHashMap<>(); + private final AtomicReference checkpoint = new AtomicReference<>(); + private final ReliableBroadcaster combine; + private final CommonCommunications comm; + private final AtomicReference current = new AtomicReference<>(); + private final AtomicReference> futureBootstrap = new AtomicReference<>(); + private final AtomicReference> futureSynchronization = new AtomicReference<>(); + private final AtomicReference genesis = new AtomicReference<>(); + private final AtomicReference head = new AtomicReference<>(); + private final AtomicReference next = new AtomicReference<>(); + private final AtomicReference nextViewId = new AtomicReference<>(); + private final Parameters params; + private final PriorityBlockingQueue pending = new PriorityBlockingQueue<>(); + private final RoundScheduler roundScheduler; + private final Session session; + private final AtomicBoolean started = new AtomicBoolean(); + private final Store store; + private final CommonCommunications submissionComm; + private final Combine.Transitions transitions; + private final TransSubmission txnSubmission = new TransSubmission(); + private final AtomicReference view = new AtomicReference<>(); + private final PendingViews pendingViews = new PendingViews(); + private final ScheduledExecutorService scheduler; + private final AtomicBoolean ongoingJoin = new AtomicBoolean(); + private volatile Thread linear; public CHOAM(Parameters params) { scheduler = Executors.newScheduledThreadPool(1, Thread.ofVirtual().factory()); this.store = new Store(params.digestAlgorithm(), params.mvBuilder().clone().build()); this.params = params; - linear = Executors.newScheduledThreadPool(1, Thread.ofVirtual().factory()); pendingViews.add(params.context().getId(), params.context().delegate()); rotateViewKeys(); @@ -353,7 +352,11 @@ public void stop() { if (!started.compareAndSet(true, false)) { return; } - linear.shutdown(); + var l = linear; + linear = null; + if (l != null) { + l.interrupt(); + } try { scheduler.shutdownNow(); } catch (Throwable e) { @@ -434,39 +437,6 @@ private Block checkpoint() { return block; } - private void combine() { - var next = pending.peek(); - log.trace("Attempting to combine blocks, peek: {} height: {}, head: {} height: {} on: {}", - next == null ? "" : next.hash, next == null ? "-1" : next.height(), head.get().hash, - head.get().height(), params.member().getId()); - while (next != null) { - final HashedCertifiedBlock h = head.get(); - if (h.height() != null && next.height().compareTo(h.height()) <= 0) { - pending.poll(); - } else if (isNext(next)) { - if (current.get().validate(next)) { - HashedCertifiedBlock nextBlock = pending.poll(); - if (nextBlock == null) { - return; - } - accept(nextBlock); - } else { - log.debug("Unable to validate block: {} hash: {} height: {} on: {}", next.block.getBodyCase(), - next.hash, next.height(), params.member().getId()); - pending.poll(); - } - } else { - log.trace("Premature block: {} : {} height: {} current: {} on: {}", next.block.getBodyCase(), next.hash, - next.height(), h.height(), params.member().getId()); - return; - } - next = pending.peek(); - } - - log.trace("Finished combined, head: {} height: {} on: {}", head.get().hash, head.get().height(), - params.member().getId()); - } - private void combine(List messages) { messages.forEach(this::combine); transitions.combine(); @@ -542,7 +512,7 @@ public Block produce(ULong height, Digest prev, Executions executions, HashedBlo checkpoint.hash, v.height(), v.hash)) .setExecutions(executions) .build(); - log.trace("Produced block: {} height: {} on: {}", block.getBodyCase(), block.getHeader().getHeight(), + log.trace("Produce block: {} height: {} on: {}", block.getBodyCase(), block.getHeader().getHeight(), params.member().getId()); return block; } @@ -573,6 +543,82 @@ public Block reconfigure(Map joining, Digest nextViewId, HashedBlo }; } + private void consume(HashedCertifiedBlock next) { + log.trace("Attempting to consume: {} hash: {} height: {}, head: {} height: {} on: {}", next.block.getBodyCase(), + next.hash, next.height(), head.get().hash, head.get().height(), params.member().getId()); + final HashedCertifiedBlock h = head.get(); + + if (h.height() != null && next.height().compareTo(h.height()) <= 0) { + // block already past tense + log.debug("Stale: {} hash: {} height: {} on: {}", next.block.getBodyCase(), next.hash, next.height(), + params.member().getId()); + return; + } + + final var nlc = ULong.valueOf(next.block.getHeader().getLastReconfig()); + + var view = this.view.get().height(); + if (h.block == null || nlc.equals(view)) { + // same view + consume(next, h); + return; + } + + if (nlc.compareTo(view) > 0) { + // later view + log.trace("Wait for reconfiguration @ {} block: {} hash: {} height: {} current: {} on: {}", + next.block.getHeader().getLastReconfig(), next.block.getBodyCase(), next.hash, next.height(), + h.height(), params.member().getId()); + pending.add(next); + } else { + // invalid view + log.trace("Invalid view @ {} current: {} block: {} hash: {} height: {} current: {} on: {}", nlc, view, + next.block.getBodyCase(), next.hash, next.height(), h.height(), params.member().getId()); + } + } + + private void consume(HashedCertifiedBlock next, HashedCertifiedBlock cur) { + if (isNext(next)) { + if (current.get().validate(next)) { + log.trace("Accept: {} hash: {} height: {} on: {}", next.block.getBodyCase(), next.hash, next.height(), + params.member().getId()); + accept(next); + } else { + log.debug("Invalid block: {} hash: {} height: {} on: {}", next.block.getBodyCase(), next.hash, + next.height(), params.member().getId()); + } + } else { + log.trace("Premature block: {} : {} height: {} current: {} on: {}", next.block.getBodyCase(), next.hash, + next.height(), cur.height(), params.member().getId()); + pending.add(next); + } + } + + private void consumer() { + while (started.get()) { + HashedCertifiedBlock next = null; + try { + next = pending.poll(500, TimeUnit.MILLISECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return; + } + if (!started.get()) { + return; + } + if (next == null) { + try { + Thread.sleep(100); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return; + } + continue; + } + consume(next); + } + } + private void execute(List execs) { final var h = head.get(); log.info("Executing transactions for block: {} hash: {} height: {} txns: {} on: {}", h.block.getBodyCase(), @@ -1196,7 +1242,13 @@ public void cancelTimer(String timer) { @Override public void combine() { - linear.execute(Utils.wrapped(() -> CHOAM.this.combine(), log)); + final var current = linear; + if (current == null) { + log.trace("Combining Consumer for: {} on: {}", context().getId(), params.member().getId()); + linear = Thread.ofVirtual() + .name("Linear[%s on: %s]".formatted(context().getId(), params.member().getId())) + .start(Utils.wrapped(CHOAM.this::consumer, log)); + } } @Override diff --git a/choam/src/main/java/com/salesforce/apollo/choam/ViewAssembly.java b/choam/src/main/java/com/salesforce/apollo/choam/ViewAssembly.java index 25e4cb2bb..ffaa6fc35 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/ViewAssembly.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/ViewAssembly.java @@ -28,7 +28,6 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CopyOnWriteArrayList; -import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Consumer; import java.util.stream.Collectors; @@ -59,7 +58,6 @@ public class ViewAssembly { private final CompletableFuture onConsensus; private final AtomicInteger countdown = new AtomicInteger(); private final List pendingJoins = new CopyOnWriteArrayList<>(); - private final AtomicBoolean started = new AtomicBoolean(false); private final Map joins = new ConcurrentHashMap<>(); private volatile Vue selected; @@ -100,23 +98,10 @@ public void joined(SignedViewMember viewMember) { } public void start() { - if (!started.compareAndSet(false, true)) { - return; - } transitions.fsm().enterStartState(); } void assemble(List asses) { - if (!started.get()) { - if (!asses.isEmpty()) { - var viewz = asses.stream().flatMap(a -> a.getViewsList().stream()).toList(); - var joinz = asses.stream().flatMap(a -> a.getJoinsList().stream()).toList(); - log.debug("Not started, ignoring assemblies: {} joins: {} views: {} on: {}", asses.size(), joinz.size(), - viewz.size(), params().member().getId()); - } - return; - } - if (asses.isEmpty()) { return; } @@ -204,9 +189,6 @@ boolean complete() { } void join(List joins) { - if (!started.get()) { - return; - } if (selected == null) { pendingJoins.addAll(joins); log.trace("Pending joins: {} on: {}", joins.size(), params().member().getId()); @@ -473,7 +455,6 @@ public void failed() { @Override public void finish() { countdown.set(-1); - started.set(false); } @Override diff --git a/fireflies/src/main/java/com/salesforce/apollo/fireflies/Binding.java b/fireflies/src/main/java/com/salesforce/apollo/fireflies/Binding.java index 5da22769e..c00a41c44 100644 --- a/fireflies/src/main/java/com/salesforce/apollo/fireflies/Binding.java +++ b/fireflies/src/main/java/com/salesforce/apollo/fireflies/Binding.java @@ -253,12 +253,13 @@ private boolean join(Member member, CompletableFuture gateway, Optional" : member.getId(), node.getId()); dec(complete, remaining); return true; } if (gateway.isDone()) { - log.warn("gateway is complete, ignoring from: {} on: {}", member.getId(), node.getId()); + log.warn("gateway is complete, ignoring from: {} on: {}", member == null ? "" : member.getId(), + node.getId()); complete.complete(true); return false; } diff --git a/fireflies/src/main/java/com/salesforce/apollo/fireflies/View.java b/fireflies/src/main/java/com/salesforce/apollo/fireflies/View.java index 255396f9e..e1ff9f8e5 100644 --- a/fireflies/src/main/java/com/salesforce/apollo/fireflies/View.java +++ b/fireflies/src/main/java/com/salesforce/apollo/fireflies/View.java @@ -362,7 +362,6 @@ void finalizeViewChange() { HashMultiset ballots = HashMultiset.create(); observations.values().forEach(svu -> tally(svu, ballots)); viewManagement.clearVote(); - scheduleClearObservations(); var max = ballots.entrySet() .stream() .max(Ordering.natural().onResultOf(Multiset.Entry::getCount)) @@ -372,13 +371,15 @@ void finalizeViewChange() { viewManagement.cardinality(), currentView(), node.getId()); viewManagement.install(max.getElement()); scheduleViewChange(); + scheduleClearObservations(); } else { @SuppressWarnings("unchecked") final var reversed = Comparator.comparing(e -> ((Entry) e).getCount()).reversed(); log.info("View consensus failed: {}, required: {} cardinality: {} ballots: {} for: {} on: {}", max == null ? 0 : max.getCount(), majority, viewManagement.cardinality(), ballots.entrySet().stream().sorted(reversed).toList(), currentView(), node.getId()); - viewManagement.initiateViewChange(); + observations.clear(); + scheduleViewChange(); } }); } @@ -392,7 +393,7 @@ Node getNode() { return node; } - boolean hasMajorityObservervations(boolean bootstrap) { + boolean hasMajorityObservations(boolean bootstrap) { return bootstrap && context.size() == 1 || observations.size() >= context.majority(); } diff --git a/fireflies/src/main/java/com/salesforce/apollo/fireflies/ViewManagement.java b/fireflies/src/main/java/com/salesforce/apollo/fireflies/ViewManagement.java index f4c77e354..900015a02 100644 --- a/fireflies/src/main/java/com/salesforce/apollo/fireflies/ViewManagement.java +++ b/fireflies/src/main/java/com/salesforce/apollo/fireflies/ViewManagement.java @@ -489,7 +489,7 @@ void maybeViewChange() { } } var change = context.offlineCount() > 0 || !joins.isEmpty(); - var shouldChange = isObserver() || view.hasMajorityObservervations(bootstrap); + var shouldChange = isObserver() || view.hasMajorityObservations(bootstrap); if (change && shouldChange) { initiateViewChange(); } else { From 867de1a72f6dde66057eeff39360006ab9f9854d Mon Sep 17 00:00:00 2001 From: Hellblazer Date: Wed, 3 Jul 2024 19:32:45 -0700 Subject: [PATCH 24/32] subtle --- .../com/salesforce/apollo/choam/CHOAM.java | 5 +- .../com/salesforce/apollo/choam/Producer.java | 17 ++-- .../apollo/choam/support/OneShot.java | 36 ------- .../apollo/choam/support/TxDataSource.java | 93 ++++++------------- choam/src/test/resources/logback-test.xml | 4 - .../salesforce/apollo/ethereal/Ethereal.java | 2 +- .../messaging/rbc/ReliableBroadcaster.java | 66 +++---------- sql-state/src/test/resources/logback-test.xml | 4 - 8 files changed, 54 insertions(+), 173 deletions(-) delete mode 100644 choam/src/main/java/com/salesforce/apollo/choam/support/OneShot.java diff --git a/choam/src/main/java/com/salesforce/apollo/choam/CHOAM.java b/choam/src/main/java/com/salesforce/apollo/choam/CHOAM.java index e17faf827..2a587e0af 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/CHOAM.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/CHOAM.java @@ -320,6 +320,7 @@ public String logState() { public void rotateViewKeys(ViewChange viewChange) { var context = viewChange.context(); var diadem = viewChange.diadem(); + log.trace("Setting RBC Context to: {} on: {}", context, params.member().getId()); ((DelegatedContext) combine.getContext()).setContext(context); var c = current.get(); if (c != null) { @@ -331,10 +332,6 @@ public void rotateViewKeys(ViewChange viewChange) { pendingViews.clear(); pendingViews.add(diadem, context); } - - log.info("Pushing pending view of: {}, diadem: {} size: {} on: {}", context.getId(), diadem, context.size(), - params.member().getId()); - pendingViews.add(diadem, context); } public void start() { diff --git a/choam/src/main/java/com/salesforce/apollo/choam/Producer.java b/choam/src/main/java/com/salesforce/apollo/choam/Producer.java index def7fd907..f5f71d546 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/Producer.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/Producer.java @@ -32,6 +32,7 @@ import java.util.*; import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; /** @@ -57,6 +58,8 @@ public class Producer { private final ViewAssembly assembly; private final int maxEpoch; private final AtomicBoolean assembled = new AtomicBoolean(false); + private final AtomicInteger epoch = new AtomicInteger(-1); + private final AtomicInteger preblocks = new AtomicInteger(); public Producer(Digest nextViewId, ViewContext view, HashedBlock lastBlock, HashedBlock checkpoint, String label, ScheduledExecutorService scheduler) { @@ -203,9 +206,10 @@ private List aggregate(List preblock) { } private void create(List preblock, boolean last) { + var count = preblocks.incrementAndGet(); if (log.isDebugEnabled()) { - log.debug("emit last: {} preblock: {} on: {}", last, - preblock.stream().map(DigestAlgorithm.DEFAULT::digest).toList(), params().member().getId()); + log.debug("emit #{} epoch: {} hashes: {} last: {} on: {}", count, epoch, + preblock.stream().map(DigestAlgorithm.DEFAULT::digest).toList(), last, params().member().getId()); } var aggregate = aggregate(preblock); processAssemblies(aggregate); @@ -220,18 +224,19 @@ private Digest getViewId() { return view.context().getId(); } - private void newEpoch(Integer epoch) { + private void newEpoch(Integer e) { serialize.execute(Utils.wrapped(() -> { - log.trace("new epoch: {} on: {}", epoch, params().member().getId()); + this.epoch.set(e); + log.trace("new epoch: {} preblocks: {} on: {}", e, preblocks.get(), params().member().getId()); assembly.newEpoch(); - var last = epoch >= maxEpoch && assembled.get(); + var last = e >= maxEpoch && assembled.get(); if (last) { controller.completeIt(); Producer.this.transitions.viewComplete(); } else { ds.reset(); } - transitions.newEpoch(epoch, last); + transitions.newEpoch(e, last); }, log)); } diff --git a/choam/src/main/java/com/salesforce/apollo/choam/support/OneShot.java b/choam/src/main/java/com/salesforce/apollo/choam/support/OneShot.java deleted file mode 100644 index 635104c4d..000000000 --- a/choam/src/main/java/com/salesforce/apollo/choam/support/OneShot.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2021, salesforce.com, inc. - * All rights reserved. - * SPDX-License-Identifier: BSD-3-Clause - * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause - */ -package com.salesforce.apollo.choam.support; - -import com.google.protobuf.ByteString; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.concurrent.Phaser; -import java.util.function.Supplier; - -public class OneShot implements Supplier { - private static final Logger log = LoggerFactory.getLogger(OneShot.class); - - private final Phaser phaser = new Phaser(1); - private volatile ByteString value; - - @Override - public ByteString get() { - phaser.register(); - final var current = value; - log.trace("providing value: " + (current == null ? "null" : String.valueOf(current.size()))); - value = null; - return current == null ? ByteString.EMPTY : current; - } - - public void setValue(ByteString value) { - log.trace("resetting value: " + (value == null ? "null" : String.valueOf(value.size()))); - this.value = value; - phaser.arriveAndDeregister(); - } -} diff --git a/choam/src/main/java/com/salesforce/apollo/choam/support/TxDataSource.java b/choam/src/main/java/com/salesforce/apollo/choam/support/TxDataSource.java index de81a7712..670f98ac3 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/support/TxDataSource.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/support/TxDataSource.java @@ -17,7 +17,6 @@ import org.slf4j.LoggerFactory; import java.time.Duration; -import java.time.Instant; import java.util.ArrayList; import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingQueue; @@ -36,14 +35,13 @@ public class TxDataSource implements DataSource { private final static Logger log = LoggerFactory.getLogger(TxDataSource.class); - private final Duration batchInterval; - private final AtomicBoolean draining = new AtomicBoolean(); - private final Member member; - private final ChoamMetrics metrics; - private final BatchingQueue processing; - private final BlockingQueue assemblies = new LinkedBlockingQueue<>(); - private final BlockingQueue validations = new LinkedBlockingQueue<>(); - private volatile Thread blockingThread; + private final Duration batchInterval; + private final AtomicBoolean draining = new AtomicBoolean(); + private final Member member; + private final ChoamMetrics metrics; + private final BatchingQueue processing; + private final BlockingQueue assemblies = new LinkedBlockingQueue<>(); + private final BlockingQueue validations = new LinkedBlockingQueue<>(); public TxDataSource(Member member, int maxElements, ChoamMetrics metrics, int maxBatchByteSize, Duration batchInterval, int maxBatchCount) { @@ -55,11 +53,6 @@ public TxDataSource(Member member, int maxElements, ChoamMetrics metrics, int ma } public void close() { - final var current = blockingThread; - if (current != null) { - current.interrupt(); - } - blockingThread = null; if (metrics != null) { metrics.dropped(processing.size(), validations.size(), assemblies.size()); } @@ -76,38 +69,15 @@ public void drain() { @Override public ByteString getData() { var builder = UnitData.newBuilder(); - log.trace("Requesting unit data on: {}", member.getId()); - blockingThread = Thread.currentThread(); - try { - var r = new ArrayList(); - var v = new ArrayList(); - - if (draining.get()) { - var target = Instant.now().plus(batchInterval); - while (target.isAfter(Instant.now()) && builder.getAssembliesCount() == 0 - && builder.getValidationsCount() == 0) { - // rinse and repeat - r = new ArrayList<>(); - assemblies.drainTo(r); - builder.addAllAssemblies(r); - - v = new ArrayList(); - validations.drainTo(v); - builder.addAllValidations(v); - - if (builder.getAssembliesCount() != 0 || builder.getValidationsCount() != 0) { - break; - } + var r = new ArrayList(); + assemblies.drainTo(r); + builder.addAllAssemblies(r); - // sleep waiting for input - try { - Thread.sleep(batchInterval.dividedBy(2).toMillis()); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - return ByteString.EMPTY; - } - } - } else { + var v = new ArrayList(); + validations.drainTo(v); + builder.addAllValidations(v); + if (!draining.get()) { + if (processing.size() > 0 || (validations.isEmpty() || assemblies.isEmpty())) { try { var batch = processing.take(batchInterval); if (batch != null) { @@ -118,28 +88,17 @@ public ByteString getData() { return ByteString.EMPTY; } } + } - // One more time into ye breech - r = new ArrayList<>(); - assemblies.drainTo(r); - builder.addAllAssemblies(r); - - v = new ArrayList(); - validations.drainTo(v); - builder.addAllValidations(v); - - ByteString bs = builder.build().toByteString(); - if (metrics != null) { - metrics.publishedBatch(builder.getTransactionsCount(), bs.size(), builder.getValidationsCount(), - builder.getAssembliesCount()); - } - log.trace("Unit data: {} txns, {} validations, {} assemblies totalling: {} bytes on: {}", - builder.getTransactionsCount(), builder.getValidationsCount(), builder.getAssembliesCount(), - bs.size(), member.getId()); - return bs; - } finally { - blockingThread = null; + ByteString bs = builder.build().toByteString(); + if (metrics != null) { + metrics.publishedBatch(builder.getTransactionsCount(), bs.size(), builder.getValidationsCount(), + builder.getAssembliesCount()); } + log.trace("Unit data: {} txns, {} validations, {} assemblies totalling: {} bytes on: {}", + builder.getTransactionsCount(), builder.getValidationsCount(), builder.getAssembliesCount(), + bs.size(), member.getId()); + return bs; } public int getRemainingReassemblies() { @@ -166,8 +125,8 @@ public boolean offer(Transaction txn) { } } - public void offer(Validate generateValidation) { - validations.offer(generateValidation); + public boolean offer(Validate generateValidation) { + return validations.offer(generateValidation); } public void reset() { diff --git a/choam/src/test/resources/logback-test.xml b/choam/src/test/resources/logback-test.xml index d5be6a052..3263880c3 100644 --- a/choam/src/test/resources/logback-test.xml +++ b/choam/src/test/resources/logback-test.xml @@ -53,10 +53,6 @@ - - - - diff --git a/ethereal/src/main/java/com/salesforce/apollo/ethereal/Ethereal.java b/ethereal/src/main/java/com/salesforce/apollo/ethereal/Ethereal.java index cbead8839..406462dfc 100644 --- a/ethereal/src/main/java/com/salesforce/apollo/ethereal/Ethereal.java +++ b/ethereal/src/main/java/com/salesforce/apollo/ethereal/Ethereal.java @@ -316,7 +316,7 @@ private void insert(Unit unit) { var ep = retrieveEpoch(unit); if (ep != null) { ep.adder().produce(unit); - log.debug("Produced: {} on: {}", unit, config.logLabel()); + log.debug("Produced: {} {}", unit, config.logLabel()); } else { log.trace("Unable to retrieve epic for Unit creator: {} epoch: {} height: {} level: {} on: {}", unit.creator(), unit.epoch(), unit.height(), unit.level(), config.logLabel()); diff --git a/memberships/src/main/java/com/salesforce/apollo/membership/messaging/rbc/ReliableBroadcaster.java b/memberships/src/main/java/com/salesforce/apollo/membership/messaging/rbc/ReliableBroadcaster.java index ee0455574..5521c80e1 100644 --- a/memberships/src/main/java/com/salesforce/apollo/membership/messaging/rbc/ReliableBroadcaster.java +++ b/memberships/src/main/java/com/salesforce/apollo/membership/messaging/rbc/ReliableBroadcaster.java @@ -15,7 +15,6 @@ import com.salesforce.apollo.archipelago.server.FernetServerInterceptor; import com.salesforce.apollo.bloomFilters.BloomFilter; import com.salesforce.apollo.bloomFilters.BloomFilter.DigestBloomFilter; -import com.salesforce.apollo.bloomFilters.BloomWindow; import com.salesforce.apollo.context.Context; import com.salesforce.apollo.cryptography.Digest; import com.salesforce.apollo.cryptography.DigestAlgorithm; @@ -298,7 +297,7 @@ private void oneRound(Duration duration, ScheduledExecutorService scheduler) { } var timer = metrics == null ? null : metrics.gossipRoundDuration().time(); - gossiper.execute((link, ring) -> gossipRound(link, ring), + gossiper.execute(this::gossipRound, (futureSailor, destination) -> handle(futureSailor, destination, duration, scheduler, timer)); } @@ -319,24 +318,20 @@ public record MessageAdapter(Predicate verifier, Function source, ByteString content, Digest hash) { } - public record Parameters(int bufferSize, int maxMessages, DigestAlgorithm digestAlgorithm, double falsePositiveRate, - int dedupBufferSize, double dedupFpr) { + public record Parameters(int bufferSize, int maxMessages, DigestAlgorithm digestAlgorithm, + double falsePositiveRate) { public static Parameters.Builder newBuilder() { return new Builder(); } public static class Builder implements Cloneable { - private int bufferSize = 1500; - private int dedupBufferSize = 100; - private double dedupFpr = Math.pow(10, -6); - private int deliveredCacheSize = 100; - private DigestAlgorithm digestAlgorithm = DigestAlgorithm.DEFAULT; - private double falsePositiveRate = 0.0000125; - private int maxMessages = 500; + private int bufferSize = 1500; + private DigestAlgorithm digestAlgorithm = DigestAlgorithm.DEFAULT; + private double falsePositiveRate = 0.0000125; + private int maxMessages = 500; public Parameters build() { - return new Parameters(bufferSize, maxMessages, digestAlgorithm, falsePositiveRate, dedupBufferSize, - dedupFpr); + return new Parameters(bufferSize, maxMessages, digestAlgorithm, falsePositiveRate); } @Override @@ -357,33 +352,6 @@ public Parameters.Builder setBufferSize(int bufferSize) { return this; } - public int getDedupBufferSize() { - return dedupBufferSize; - } - - public Builder setDedupBufferSize(int dedupBufferSize) { - this.dedupBufferSize = dedupBufferSize; - return this; - } - - public double getDedupFpr() { - return dedupFpr; - } - - public Builder setDedupFpr(double dedupFpr) { - this.dedupFpr = dedupFpr; - return this; - } - - public int getDeliveredCacheSize() { - return deliveredCacheSize; - } - - public Builder setDeliveredCacheSize(int deliveredCacheSize) { - this.deliveredCacheSize = deliveredCacheSize; - return this; - } - public DigestAlgorithm getDigestAlgorithm() { return digestAlgorithm; } @@ -446,7 +414,6 @@ public void update(ReconcileContext reconcile, Digest from) { } private class Buffer { - private final BloomWindow delivered; private final Semaphore garbageCollecting = new Semaphore(1); private final int highWaterMark; private final int maxAge; @@ -457,7 +424,6 @@ private class Buffer { private Buffer(int maxAge) { this.maxAge = maxAge; highWaterMark = (params.bufferSize - (int) (params.bufferSize + ((params.bufferSize) * 0.1))); - delivered = BloomWindow.create(params.dedupBufferSize, params.dedupFpr, Biff.Type.DIGEST); } public void clear() { @@ -466,12 +432,12 @@ public void clear() { public BloomFilter forReconcilliation() { var biff = new DigestBloomFilter(Entropy.nextBitsStreamLong(), params.bufferSize, params.falsePositiveRate); - state.keySet().stream().collect(Utils.toShuffledList()).forEach(k -> biff.add(k)); + state.keySet().stream().collect(Utils.toShuffledList()).forEach(biff::add); return biff; } public void receive(List messages) { - if (messages.size() == 0) { + if (messages.isEmpty()) { return; } log.trace("receiving: {} msgs on: {}", messages.size(), member.getId()); @@ -483,13 +449,13 @@ public void receive(List messages) { .map(s -> state.merge(s.hash, s, (a, b) -> a.msg.getAge() >= b.msg.getAge() ? a : b)) .map(s -> new Msg(adapter.source.apply(s.msg.getContent()), adapter.extractor.apply(s.msg), s.hash)) - .filter(m -> delivered.add(m.hash)) .toList()); gc(); } public Iterable reconcile(BloomFilter biff, Digest from) { - PriorityQueue mailBox = new PriorityQueue<>(Comparator.comparingInt(s -> s.getAge())); + PriorityQueue mailBox = new PriorityQueue<>( + Comparator.comparingInt(AgedMessage.Builder::getAge)); state.values() .stream() .collect(Utils.toShuffledList()) @@ -497,7 +463,7 @@ public Iterable reconcile(BloomFilter biff, Diges .filter(s -> !biff.contains(s.hash)) .filter(s -> s.msg.getAge() < maxAge) .forEach(s -> mailBox.add(s.msg)); - List reconciled = mailBox.stream().map(b -> b.build()).toList(); + List reconciled = mailBox.stream().map(AgedMessage.Builder::build).toList(); if (!reconciled.isEmpty()) { log.trace("reconciled: {} for: {} on: {}", reconciled.size(), from, member.getId()); } @@ -565,7 +531,7 @@ private boolean dup(state s) { // log.trace("duplicate event: {} on: {}", s.hash, member.getId()); return true; } - return delivered.contains(s.hash); + return false; } private void gc() { @@ -596,9 +562,7 @@ private void purgeTheAged() { Queue candidates = new PriorityQueue<>( Collections.reverseOrder((a, b) -> Integer.compare(a.msg.getAge(), b.msg.getAge()))); candidates.addAll(state.values()); - var processing = candidates.iterator(); - while (processing.hasNext()) { - var m = processing.next(); + for (ReliableBroadcaster.state m : candidates) { if (m.msg.getAge() > maxAge) { state.remove(m.hash); } else { diff --git a/sql-state/src/test/resources/logback-test.xml b/sql-state/src/test/resources/logback-test.xml index 798240567..fc6798bcb 100644 --- a/sql-state/src/test/resources/logback-test.xml +++ b/sql-state/src/test/resources/logback-test.xml @@ -56,10 +56,6 @@ - - - - From d743f342a831b4a330deba74546efcdc86577e0e Mon Sep 17 00:00:00 2001 From: Hellblazer Date: Wed, 3 Jul 2024 19:38:00 -0700 Subject: [PATCH 25/32] oops --- .../java/com/salesforce/apollo/messaging/rbc/RbcTest.java | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/memberships/src/test/java/com/salesforce/apollo/messaging/rbc/RbcTest.java b/memberships/src/test/java/com/salesforce/apollo/messaging/rbc/RbcTest.java index ed5c8ff08..050e3faca 100644 --- a/memberships/src/test/java/com/salesforce/apollo/messaging/rbc/RbcTest.java +++ b/memberships/src/test/java/com/salesforce/apollo/messaging/rbc/RbcTest.java @@ -54,10 +54,7 @@ public class RbcTest { private static final Parameters.Builder parameters = Parameters.newBuilder() .setMaxMessages(100) .setFalsePositiveRate(0.00125) - .setBufferSize(500) - .setDedupBufferSize( - LARGE_TESTS ? 100 * 100 : 50 * 50) - .setDedupFpr(Math.pow(10, -9)); + .setBufferSize(500); final AtomicReference round = new AtomicReference<>(); private final List communications = new ArrayList<>(); private final AtomicInteger totalReceived = new AtomicInteger(0); From 2bdb1b7db87518aa72a92673554f82be5fd24ff5 Mon Sep 17 00:00:00 2001 From: Hellblazer Date: Thu, 4 Jul 2024 08:46:09 -0700 Subject: [PATCH 26/32] refactor bft subset. simplify TxDataSource --- .../apollo/choam/support/TxDataSource.java | 31 ++++++++++++---- .../salesforce/apollo/context/Context.java | 36 ++++++++++--------- .../apollo/context/DynamicContextImpl.java | 23 +++++------- .../apollo/context/StaticContext.java | 14 ++++---- .../apollo/ring/RingCommunications.java | 24 +++++-------- 5 files changed, 69 insertions(+), 59 deletions(-) diff --git a/choam/src/main/java/com/salesforce/apollo/choam/support/TxDataSource.java b/choam/src/main/java/com/salesforce/apollo/choam/support/TxDataSource.java index 670f98ac3..cc29baf4e 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/support/TxDataSource.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/support/TxDataSource.java @@ -69,13 +69,6 @@ public void drain() { @Override public ByteString getData() { var builder = UnitData.newBuilder(); - var r = new ArrayList(); - assemblies.drainTo(r); - builder.addAllAssemblies(r); - - var v = new ArrayList(); - validations.drainTo(v); - builder.addAllValidations(v); if (!draining.get()) { if (processing.size() > 0 || (validations.isEmpty() || assemblies.isEmpty())) { try { @@ -89,6 +82,30 @@ public ByteString getData() { } } } + var r = new ArrayList(); + assemblies.drainTo(r); + builder.addAllAssemblies(r); + + var v = new ArrayList(); + validations.drainTo(v); + builder.addAllValidations(v); + if (draining.get() && r.isEmpty() && v.isEmpty()) { + var target = System.currentTimeMillis() + batchInterval.toMillis(); + while (System.currentTimeMillis() < target) { + assemblies.drainTo(r); + validations.drainTo(v); + builder.addAllAssemblies(r); + builder.addAllValidations(v); + if (!v.isEmpty() || !r.isEmpty()) { + break; + } + try { + Thread.sleep(10); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + } ByteString bs = builder.build().toByteString(); if (metrics != null) { diff --git a/memberships/src/main/java/com/salesforce/apollo/context/Context.java b/memberships/src/main/java/com/salesforce/apollo/context/Context.java index ec82cd7dd..384bbd3a6 100644 --- a/memberships/src/main/java/com/salesforce/apollo/context/Context.java +++ b/memberships/src/main/java/com/salesforce/apollo/context/Context.java @@ -10,7 +10,6 @@ import com.salesforce.apollo.cryptography.DigestAlgorithm; import com.salesforce.apollo.membership.Member; import com.salesforce.apollo.membership.Util; -import com.salesforce.apollo.ring.RingCommunications; import org.apache.commons.math3.random.BitsStreamGenerator; import java.util.*; @@ -106,19 +105,19 @@ static int minMajority(int bias, double pByz, int cardinality) { /** * @param hash - the point on the rings to determine successors - * @return the Set of Members constructed from the sucessors of the supplied hash on each of the receiver Context's + * @return the Set of Members constructed from the successors of the supplied hash on each of the receiver Context's * rings */ - default LinkedHashSet bftSubset(Digest hash) { + default SequencedSet bftSubset(Digest hash) { return bftSubset(hash, m -> true); } /** * @param hash - the point on the rings to determine successors * @param filter - the filter to apply to successors - * @return the Set of Members constructed from the sucessors of the supplied hash on each of the receiver Context's + * @return the Set of Members constructed from the successors of the supplied hash on each of the receiver Context's */ - default LinkedHashSet bftSubset(Digest hash, Predicate filter) { + default SequencedSet bftSubset(Digest hash, Predicate filter) { var collector = new LinkedHashSet(); uniqueSuccessors(hash, filter, collector); return collector; @@ -437,12 +436,12 @@ default int majority() { Iterable successors(int ring, Digest location); - default List> successors(Digest digest, T ignore, boolean noDuplicates, T member) { - var traversal = new ArrayList>(); + default List> successors(Digest digest, T ignore, boolean noDuplicates, T member) { + var traversal = new ArrayList>(); var traversed = new TreeSet(); for (int ring = 0; ring < getRingCount(); ring++) { if (size() == 1) { - traversal.add(new RingCommunications.iteration<>(member, ring)); + traversal.add(new iteration<>(member, ring)); continue; } T successor = findSuccessor(ring, digest, m -> { @@ -458,7 +457,7 @@ default List> successors(Digest digest, T ignore } return Context.IterateResult.SUCCESS; }); - traversal.add(new RingCommunications.iteration<>(successor, ring)); + traversal.add(new iteration<>(successor, ring)); } return traversal; } @@ -502,16 +501,12 @@ default int toleranceLevel() { */ void uniqueSuccessors(Digest key, Predicate test, Set collector); - default Set uniqueSuccessors(Digest digest) { - var collected = new HashSet(); - uniqueSuccessors(digest, collected); - return collected; - } - /** * collect the list of successors to the key on each ring, providing a unique member per ring if possible. */ - void uniqueSuccessors(Digest key, Set collector); + default void uniqueSuccessors(Digest key, Set collector) { + uniqueSuccessors(key, t -> true, collector); + } boolean validRing(int ring); @@ -522,6 +517,15 @@ enum IterateResult { CONTINUE, FAIL, SUCCESS } + record iteration(T m, int ring) { + + @Override + public String toString() { + return String.format("[%s,%s]", m == null ? "" : m.getId(), ring); + } + + } + /** * @author hal.hildebrand **/ diff --git a/memberships/src/main/java/com/salesforce/apollo/context/DynamicContextImpl.java b/memberships/src/main/java/com/salesforce/apollo/context/DynamicContextImpl.java index 58fc888b7..43cb35a3d 100644 --- a/memberships/src/main/java/com/salesforce/apollo/context/DynamicContextImpl.java +++ b/memberships/src/main/java/com/salesforce/apollo/context/DynamicContextImpl.java @@ -726,23 +726,24 @@ public Iterable traverse(int ring, T member) { } /** - * @return the list of successor to the key on each ring that pass the provided predicate test + * @return the list of successor to the key on each ring that passes the provided predicate test */ @Override public void uniqueSuccessors(Digest key, Predicate test, Set collector) { + var delegate = ring(0).successor(key, test); + if (delegate == null) { + return; + } for (Ring ring : rings) { - T successor = ring.successor(key, m -> !collector.contains(m) && test.test(m)); + T successor = ring.successor(hashFor(delegate, ring.index), m -> !collector.contains(m) && test.test(m)); if (successor != null) { collector.add(successor); + } else { + collector.add(delegate); } } } - @Override - public void uniqueSuccessors(Digest key, Set collector) { - uniqueSuccessors(key, t -> true, collector); - } - @Override public boolean validRing(int ring) { return ring >= 0 && ring < rings.size(); @@ -1150,7 +1151,6 @@ public Stream stream() { } /** - * @param start * @param predicate * @return a Stream of all items counter-clock wise in the ring from (but excluding) start location to (but * excluding) the first item where predicate(item) evaluates to True. @@ -1160,7 +1160,6 @@ public Stream streamPredecessors(Digest location, Predicate predicate) { } /** - * @param start * @param predicate * @return a list of all items counter-clock wise in the ring from (but excluding) start item to (but excluding) * the first item where predicate(item) evaluates to True. @@ -1170,7 +1169,6 @@ public Stream streamPredecessors(T m, Predicate predicate) { } /** - * @param start * @param predicate * @return a Stream of all items counter-clock wise in the ring from (but excluding) start location to (but * excluding) the first item where predicate(item) evaluates to True. @@ -1180,7 +1178,6 @@ public Stream streamSuccessors(Digest location, Predicate predicate) { } /** - * @param start * @param predicate * @return a Stream of all items counter-clock wise in the ring from (but excluding) start item to (but * excluding) the first item where predicate(item) evaluates to True. @@ -1190,8 +1187,6 @@ public Stream streamSuccessors(T m, Predicate predicate) { } /** - * @param start - * @param predicate * @return a iterable of all items counter-clock wise in the ring from (but excluding) start location to (but * excluding) the first item where predicate(item) evaluates to True. */ @@ -1219,7 +1214,7 @@ public T successor(T m) { /** * @param m - the member * @param predicate - the test predicate - * @return the first successor of m for which predicate evaluates to True. m is never evaluated.. + * @return the first successor of m for which predicate evaluates to True. m is never evaluated. */ public T successor(T m, Predicate predicate) { return succ(hash(m), predicate); diff --git a/memberships/src/main/java/com/salesforce/apollo/context/StaticContext.java b/memberships/src/main/java/com/salesforce/apollo/context/StaticContext.java index d91662176..7c78c165c 100644 --- a/memberships/src/main/java/com/salesforce/apollo/context/StaticContext.java +++ b/memberships/src/main/java/com/salesforce/apollo/context/StaticContext.java @@ -449,19 +449,21 @@ public Iterable traverse(int ring, T member) { @Override public void uniqueSuccessors(Digest key, Predicate test, Set collector) { + var delegate = ring(0).successor(key, test); + if (delegate == null) { + return; + } for (int ring = 0; ring < rings.length; ring++) { - T successor = ring(ring).successor(key, m -> !collector.contains(m) && test.test(m)); + StaticRing r = ring(ring); + T successor = r.successor(hashFor(delegate, r.index), m -> !collector.contains(m) && test.test(m)); if (successor != null) { collector.add(successor); + } else { + collector.add(delegate); } } } - @Override - public void uniqueSuccessors(Digest key, Set collector) { - uniqueSuccessors(key, t -> true, collector); - } - @Override public boolean validRing(int ring) { return ring >= 0 && ring < rings.length; diff --git a/memberships/src/main/java/com/salesforce/apollo/ring/RingCommunications.java b/memberships/src/main/java/com/salesforce/apollo/ring/RingCommunications.java index 572b41827..fe1cb61b8 100644 --- a/memberships/src/main/java/com/salesforce/apollo/ring/RingCommunications.java +++ b/memberships/src/main/java/com/salesforce/apollo/ring/RingCommunications.java @@ -34,7 +34,7 @@ public class RingCommunications { final SigningMember member; private final CommonCommunications comm; private final Lock lock = new ReentrantLock(); - private final List> traversalOrder = new ArrayList<>(); + private final List> traversalOrder = new ArrayList<>(); protected boolean noDuplicates = true; volatile int currentIndex = -1; private boolean ignoreSelf; @@ -140,31 +140,23 @@ private Destination linkFor(Digest digest) { return null; } final var current = currentIndex; - iteration successor = null; + Context.iteration successor = null; try { successor = traversalOrder.get(current); - final Comm link = comm.connect(successor.m); + final Comm link = comm.connect(successor.m()); if (link == null) { - log.trace("No connection to {} on: {}", successor.m == null ? "" : successor.m.getId(), + log.trace("No connection to {} on: {}", successor.m() == null ? "" : successor.m().getId(), member.getId()); } - return new Destination<>(successor.m, link, successor.ring); + return new Destination<>(successor.m(), link, successor.ring()); } catch (Throwable e) { - log.trace("error opening connection to {}: {} on: {}", successor.m == null ? "" : successor.m.getId(), + log.trace("error opening connection to {}: {} on: {}", + successor.m() == null ? "" : successor.m().getId(), (e.getCause() != null ? e.getCause() : e).getMessage(), member.getId()); - return new Destination<>(successor.m, null, successor.ring); + return new Destination<>(successor.m(), null, successor.ring()); } } public record Destination(M member, Q link, int ring) { } - - public record iteration(T m, int ring) { - - @Override - public String toString() { - return String.format("[%s,%s]", m == null ? "" : m.getId(), ring); - } - - } } From a143a367a3c0b2673ab0256af8ae4ded60dfac52 Mon Sep 17 00:00:00 2001 From: Hellblazer Date: Thu, 4 Jul 2024 11:38:00 -0700 Subject: [PATCH 27/32] require 11; moar TxDataSource tweak --- .../apollo/choam/support/TxDataSource.java | 18 ++++++++---------- .../salesforce/apollo/choam/DynamicTest.java | 2 +- .../apollo/choam/MembershipTests.java | 2 +- .../com/salesforce/apollo/ethereal/Config.java | 5 ++++- 4 files changed, 14 insertions(+), 13 deletions(-) diff --git a/choam/src/main/java/com/salesforce/apollo/choam/support/TxDataSource.java b/choam/src/main/java/com/salesforce/apollo/choam/support/TxDataSource.java index cc29baf4e..ee418b0fa 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/support/TxDataSource.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/support/TxDataSource.java @@ -70,16 +70,14 @@ public void drain() { public ByteString getData() { var builder = UnitData.newBuilder(); if (!draining.get()) { - if (processing.size() > 0 || (validations.isEmpty() || assemblies.isEmpty())) { - try { - var batch = processing.take(batchInterval); - if (batch != null) { - builder.addAllTransactions(batch); - } - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - return ByteString.EMPTY; + try { + var batch = processing.take(batchInterval); + if (batch != null) { + builder.addAllTransactions(batch); } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return ByteString.EMPTY; } } var r = new ArrayList(); @@ -100,7 +98,7 @@ public ByteString getData() { break; } try { - Thread.sleep(10); + Thread.sleep(1); } catch (InterruptedException e) { Thread.currentThread().interrupt(); } diff --git a/choam/src/test/java/com/salesforce/apollo/choam/DynamicTest.java b/choam/src/test/java/com/salesforce/apollo/choam/DynamicTest.java index 34d3081bb..f3361a053 100644 --- a/choam/src/test/java/com/salesforce/apollo/choam/DynamicTest.java +++ b/choam/src/test/java/com/salesforce/apollo/choam/DynamicTest.java @@ -83,7 +83,7 @@ public void setUp() throws Exception { .setMaxBatchCount(10_000) .setEthereal(Config.newBuilder() .setNumberOfEpochs(3) - .setEpochLength(7)) + .setEpochLength(11)) .build()) .setCheckpointBlockDelta(checkpointBlockSize); diff --git a/choam/src/test/java/com/salesforce/apollo/choam/MembershipTests.java b/choam/src/test/java/com/salesforce/apollo/choam/MembershipTests.java index a23597f0b..dceb414aa 100644 --- a/choam/src/test/java/com/salesforce/apollo/choam/MembershipTests.java +++ b/choam/src/test/java/com/salesforce/apollo/choam/MembershipTests.java @@ -147,7 +147,7 @@ public SigningMember initialize(int checkpointBlockSize, int cardinality) throws .setMaxBatchByteSize(1024 * 1024) .setMaxBatchCount(10_000) .setEthereal(Config.newBuilder() - .setEpochLength(7) + .setEpochLength(11) .setNumberOfEpochs(3)) .build()) .setGenerateGenesis(true) diff --git a/ethereal/src/main/java/com/salesforce/apollo/ethereal/Config.java b/ethereal/src/main/java/com/salesforce/apollo/ethereal/Config.java index 204916bd0..a7f71eb0b 100644 --- a/ethereal/src/main/java/com/salesforce/apollo/ethereal/Config.java +++ b/ethereal/src/main/java/com/salesforce/apollo/ethereal/Config.java @@ -41,7 +41,7 @@ public static class Builder implements Cloneable { private int bias = 3; private DigestAlgorithm digestAlgorithm = DigestAlgorithm.DEFAULT; - private int epochLength = 30; + private int epochLength = 11; private double fpr = 0.00125; private String label = ""; private short nProc; @@ -64,6 +64,9 @@ public Config build() { } Objects.requireNonNull(signer, "Signer cannot be null"); Objects.requireNonNull(digestAlgorithm, "Digest Algorithm cannot be null"); + if (epochLength <= 10) { + throw new IllegalArgumentException("Epoch length must be at least 11: " + epochLength); + } return new Config(label, nProc, epochLength, pid, signer, digestAlgorithm, numberOfEpochs, wtk, bias, fpr); } From fd77fdf9e26de5729cfee7843f28d2081e6b5c25 Mon Sep 17 00:00:00 2001 From: Hellblazer Date: Thu, 4 Jul 2024 13:08:46 -0700 Subject: [PATCH 28/32] D'oh. Fix maxEpochs geebus --- choam/src/main/java/com/salesforce/apollo/choam/Producer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/choam/src/main/java/com/salesforce/apollo/choam/Producer.java b/choam/src/main/java/com/salesforce/apollo/choam/Producer.java index f5f71d546..1caad37fb 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/Producer.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/Producer.java @@ -75,7 +75,7 @@ public Producer(Digest nextViewId, ViewContext view, HashedBlock lastBlock, Hash // Number of rounds we can provide data for final var blocks = ep.getEpochLength() - 2; - maxEpoch = ep.getEpochLength(); + maxEpoch = ep.getNumberOfEpochs(); ds = new TxDataSource(params.member(), blocks, params.metrics(), producerParams.maxBatchByteSize(), producerParams.batchInterval(), producerParams.maxBatchCount()); From baa921268cdede320c6244568eeeb814eeb64bee Mon Sep 17 00:00:00 2001 From: Hellblazer Date: Fri, 5 Jul 2024 07:05:15 -0700 Subject: [PATCH 29/32] use b64 encoded rather than bytes KerlSpace still effed --- .../com/salesforce/apollo/utils/Utils.java | 14 +++ isolates/pom.xml | 15 +-- .../src/main/resources/sql-state/internal.xml | 2 +- .../main/resources/stereotomy/stereotomy.xml | 20 ++-- schemas/src/main/resources/thoth/thoth.xml | 18 +-- .../apollo/stereotomy/KeyState.java | 15 ++- .../apollo/stereotomy/db/UniKERL.java | 103 ++++++++++-------- .../apollo/stereotomy/db/TestUniKERL.java | 7 +- .../com/salesforce/apollo/thoth/KerlDHT.java | 5 +- .../salesforce/apollo/thoth/KerlSpace.java | 91 +++++++++------- .../apollo/thoth/KerlSpaceTest.java | 3 +- 11 files changed, 160 insertions(+), 133 deletions(-) diff --git a/cryptography/src/main/java/com/salesforce/apollo/utils/Utils.java b/cryptography/src/main/java/com/salesforce/apollo/utils/Utils.java index afdc7329a..11969f01c 100644 --- a/cryptography/src/main/java/com/salesforce/apollo/utils/Utils.java +++ b/cryptography/src/main/java/com/salesforce/apollo/utils/Utils.java @@ -1,5 +1,6 @@ package com.salesforce.apollo.utils; +import com.google.protobuf.MessageLite; import com.salesforce.apollo.cryptography.Digest; import com.salesforce.apollo.cryptography.DigestAlgorithm; import com.salesforce.apollo.cryptography.SignatureAlgorithm; @@ -19,6 +20,7 @@ import java.security.cert.X509Certificate; import java.time.Instant; import java.util.ArrayList; +import java.util.Base64; import java.util.Collections; import java.util.List; import java.util.concurrent.Callable; @@ -79,6 +81,18 @@ public static void remove(File directoryOrFile) { } } + public static String b64(byte[] bytes) { + return Base64.getUrlEncoder().withoutPadding().encodeToString(bytes); + } + + public static String b64(MessageLite message) { + return b64(message.toByteArray()); + } + + public static byte[] b64(String encoded) { + return Base64.getUrlDecoder().decode(encoded); + } + /** * Clean the contents of a directory * diff --git a/isolates/pom.xml b/isolates/pom.xml index e6ede2422..473454499 100644 --- a/isolates/pom.xml +++ b/isolates/pom.xml @@ -36,6 +36,11 @@ org.slf4j slf4j-jdk14 + + org.graalvm.sdk + graal-sdk + provided + @@ -258,16 +263,6 @@ - - isolates - - - org.graalvm.sdk - graal-sdk - provided - - - mac-domain diff --git a/schemas/src/main/resources/sql-state/internal.xml b/schemas/src/main/resources/sql-state/internal.xml index f3a5ee776..6d7b38991 100644 --- a/schemas/src/main/resources/sql-state/internal.xml +++ b/schemas/src/main/resources/sql-state/internal.xml @@ -32,7 +32,7 @@ - + diff --git a/schemas/src/main/resources/stereotomy/stereotomy.xml b/schemas/src/main/resources/stereotomy/stereotomy.xml index a77cd4070..86b3ce6b1 100644 --- a/schemas/src/main/resources/stereotomy/stereotomy.xml +++ b/schemas/src/main/resources/stereotomy/stereotomy.xml @@ -13,7 +13,7 @@ - + @@ -31,7 +31,7 @@ - + @@ -47,7 +47,7 @@ coordinates_ilk_validate check (ilk in ('dip', 'drt', 'icp', 'ixn', 'nan', 'rct', 'vrc', 'rot')) - + - + - + - + - + @@ -134,7 +134,7 @@ - + @@ -156,11 +156,11 @@ - + - + diff --git a/schemas/src/main/resources/thoth/thoth.xml b/schemas/src/main/resources/thoth/thoth.xml index 57520eb93..1a9d476cc 100644 --- a/schemas/src/main/resources/thoth/thoth.xml +++ b/schemas/src/main/resources/thoth/thoth.xml @@ -1,8 +1,8 @@ + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xmlns="http://www.liquibase.org/xml/ns/dbchangelog" + xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-4.4.xsd"> create schema if not exists thoth @@ -12,7 +12,7 @@ - + @@ -37,7 +37,7 @@ - + @@ -74,10 +74,10 @@ - + - + @@ -93,7 +93,7 @@ - + @@ -109,7 +109,7 @@ - + diff --git a/stereotomy/src/main/java/com/salesforce/apollo/stereotomy/KeyState.java b/stereotomy/src/main/java/com/salesforce/apollo/stereotomy/KeyState.java index 06a194c83..681817028 100644 --- a/stereotomy/src/main/java/com/salesforce/apollo/stereotomy/KeyState.java +++ b/stereotomy/src/main/java/com/salesforce/apollo/stereotomy/KeyState.java @@ -6,19 +6,18 @@ */ package com.salesforce.apollo.stereotomy; -import java.security.PublicKey; -import java.util.List; -import java.util.Optional; -import java.util.Set; - -import org.joou.ULong; - -import com.salesforce.apollo.stereotomy.event.proto.KeyState_; import com.salesforce.apollo.cryptography.Digest; import com.salesforce.apollo.cryptography.SigningThreshold; import com.salesforce.apollo.stereotomy.event.InceptionEvent.ConfigurationTrait; +import com.salesforce.apollo.stereotomy.event.proto.KeyState_; import com.salesforce.apollo.stereotomy.identifier.BasicIdentifier; import com.salesforce.apollo.stereotomy.identifier.Identifier; +import org.joou.ULong; + +import java.security.PublicKey; +import java.util.List; +import java.util.Optional; +import java.util.Set; /** * The state of a key in the KEL diff --git a/stereotomy/src/main/java/com/salesforce/apollo/stereotomy/db/UniKERL.java b/stereotomy/src/main/java/com/salesforce/apollo/stereotomy/db/UniKERL.java index c4100c582..4e9b805b5 100644 --- a/stereotomy/src/main/java/com/salesforce/apollo/stereotomy/db/UniKERL.java +++ b/stereotomy/src/main/java/com/salesforce/apollo/stereotomy/db/UniKERL.java @@ -26,6 +26,8 @@ import com.salesforce.apollo.stereotomy.event.protobuf.ProtobufEventFactory; import com.salesforce.apollo.stereotomy.identifier.Identifier; import com.salesforce.apollo.stereotomy.processing.KeyEventProcessor; +import org.bouncycastle.mime.encoding.Base64InputStream; +import org.bouncycastle.mime.encoding.Base64OutputStream; import org.h2.jdbc.JdbcSQLIntegrityConstraintViolationException; import org.jooq.DSLContext; import org.jooq.Record1; @@ -39,6 +41,7 @@ import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.nio.charset.Charset; import java.sql.Connection; import java.util.Collections; import java.util.List; @@ -57,12 +60,13 @@ import static com.salesforce.apollo.stereotomy.schema.tables.Identifier.IDENTIFIER; import static com.salesforce.apollo.stereotomy.schema.tables.Receipt.RECEIPT; import static com.salesforce.apollo.stereotomy.schema.tables.Validation.VALIDATION; +import static com.salesforce.apollo.utils.Utils.b64; /** * @author hal.hildebrand */ abstract public class UniKERL implements DigestKERL { - public static final byte[] DIGEST_NONE_BYTES = Digest.NONE.getBytes(); + public static final String DIGEST_NONE_ENCODED = b64(Digest.NONE.getBytes()); private static final Logger log = LoggerFactory.getLogger(UniKERL.class); protected final DigestAlgorithm digestAlgorithm; @@ -80,7 +84,7 @@ public static void append(DSLContext dsl, AttachmentEvent attachment) { return; } var coordinates = attachment.coordinates(); - final var identBytes = coordinates.getIdentifier().toIdent().toByteArray(); + final var identBytes = b64(coordinates.getIdentifier().toIdent()); var ident = dsl.newRecord(IDENTIFIER); ident.setPrefix(identBytes); @@ -89,7 +93,7 @@ public static void append(DSLContext dsl, AttachmentEvent attachment) { Record1 id; try { id = dsl.insertInto(COORDINATES) - .set(COORDINATES.DIGEST, coordinates.getDigest().getBytes()) + .set(COORDINATES.DIGEST, b64(coordinates.getDigest().getBytes())) .set(COORDINATES.IDENTIFIER, dsl.select(IDENTIFIER.ID).from(IDENTIFIER).where(IDENTIFIER.PREFIX.eq(identBytes))) .set(COORDINATES.ILK, coordinates.getIlk()) @@ -101,9 +105,9 @@ public static void append(DSLContext dsl, AttachmentEvent attachment) { id = dsl.select(COORDINATES.ID) .from(COORDINATES) .join(IDENTIFIER) - .on(IDENTIFIER.PREFIX.eq(coordinates.getIdentifier().toIdent().toByteArray())) + .on(IDENTIFIER.PREFIX.eq(b64(coordinates.getIdentifier().toIdent()))) .where(COORDINATES.IDENTIFIER.eq(IDENTIFIER.ID)) - .and(COORDINATES.DIGEST.eq(coordinates.getDigest().getBytes())) + .and(COORDINATES.DIGEST.eq(b64(coordinates.getDigest().getBytes()))) .and(COORDINATES.SEQUENCE_NUMBER.eq(coordinates.getSequenceNumber().toBigInteger())) .and(COORDINATES.ILK.eq(coordinates.getIlk())) .fetchOne(); @@ -111,13 +115,14 @@ public static void append(DSLContext dsl, AttachmentEvent attachment) { var count = new AtomicInteger(); for (Seal s : attachment.attachments().seals()) { final var bytes = s.toSealed().toByteArray(); + var seal = b64(bytes); count.accumulateAndGet(dsl.mergeInto(ATTACHMENT) .usingDual() .on(ATTACHMENT.FOR.eq(id.value1())) - .and(ATTACHMENT.SEAL.eq(bytes)) + .and(ATTACHMENT.SEAL.eq(seal)) .whenNotMatchedThenInsert() .set(ATTACHMENT.FOR, id.value1()) - .set(ATTACHMENT.SEAL, bytes) + .set(ATTACHMENT.SEAL, seal) .execute(), Integer::sum); } log.info("appended: {} seals out of: {} coords: {}", count.get(), attachment.attachments().seals().size(), @@ -131,7 +136,7 @@ public static void append(DSLContext dsl, AttachmentEvent attachment) { .whenNotMatchedThenInsert() .set(RECEIPT.FOR, id.value1()) .set(RECEIPT.WITNESS, entry.getKey()) - .set(RECEIPT.SIGNATURE, entry.getValue().toSig().toByteArray()) + .set(RECEIPT.SIGNATURE, b64(entry.getValue().toSig())) .execute(), Integer::sum); } log.info("appended: {} endorsements out of: {} coords: {}", count.get(), @@ -143,11 +148,10 @@ public static void append(DSLContext context, KeyEvent event, KeyState newState, final EventCoordinates prevCoords = event.getPrevious(); final var preIdentifier = context.select(IDENTIFIER.ID) .from(IDENTIFIER) - .where( - IDENTIFIER.PREFIX.eq(prevCoords.getIdentifier().toIdent().toByteArray())); + .where(IDENTIFIER.PREFIX.eq(b64(prevCoords.getIdentifier().toIdent()))); final var prev = context.select(COORDINATES.ID) .from(COORDINATES) - .where(COORDINATES.DIGEST.eq(prevCoords.getDigest().getBytes())) + .where(COORDINATES.DIGEST.eq(b64(prevCoords.getDigest().getBytes()))) .and(COORDINATES.IDENTIFIER.eq(preIdentifier)) .and(COORDINATES.SEQUENCE_NUMBER.eq(prevCoords.getSequenceNumber().toBigInteger())) .and(COORDINATES.ILK.eq(prevCoords.getIlk())) @@ -162,8 +166,7 @@ public static void append(DSLContext context, KeyEvent event, KeyState newState, .where(EVENT.COORDINATES.eq(prev.value1())) .fetchOne(); - final var identBytes = event.getIdentifier().toIdent().toByteArray(); - + final var identBytes = b64(event.getIdentifier().toIdent()); try { context.mergeInto(IDENTIFIER) .using(context.selectOne()) @@ -186,7 +189,7 @@ public static void append(DSLContext context, KeyEvent event, KeyState newState, long id; try { id = context.insertInto(COORDINATES) - .set(COORDINATES.DIGEST, prevDigest == null ? DIGEST_NONE_BYTES : prevDigest.value1()) + .set(COORDINATES.DIGEST, prevDigest == null ? DIGEST_NONE_ENCODED : prevDigest.value1()) .set(COORDINATES.IDENTIFIER, context.select(IDENTIFIER.ID).from(IDENTIFIER).where(IDENTIFIER.PREFIX.eq(identBytes))) .set(COORDINATES.ILK, event.getIlk()) @@ -201,9 +204,9 @@ public static void append(DSLContext context, KeyEvent event, KeyState newState, id = context.select(COORDINATES.ID) .from(COORDINATES) .join(IDENTIFIER) - .on(IDENTIFIER.PREFIX.eq(coordinates.getIdentifier().toIdent().toByteArray())) + .on(IDENTIFIER.PREFIX.eq(b64(coordinates.getIdentifier().toIdent()))) .where(COORDINATES.IDENTIFIER.eq(IDENTIFIER.ID)) - .and(COORDINATES.DIGEST.eq(coordinates.getDigest().getBytes())) + .and(COORDINATES.DIGEST.eq(b64(coordinates.getDigest().getBytes()))) .and(COORDINATES.SEQUENCE_NUMBER.eq(coordinates.getSequenceNumber().toBigInteger())) .and(COORDINATES.ILK.eq(coordinates.getIlk())) .fetchOne() @@ -214,7 +217,7 @@ public static void append(DSLContext context, KeyEvent event, KeyState newState, try { context.insertInto(EVENT) .set(EVENT.COORDINATES, id) - .set(EVENT.DIGEST, digest.getBytes()) + .set(EVENT.DIGEST, b64(digest.getBytes())) .set(EVENT.CONTENT, compress(event.getBytes())) .set(EVENT.CURRENT_STATE, compress(newState.getBytes())) .execute(); @@ -260,7 +263,7 @@ public static void appendValidations(DSLContext dsl, EventCoordinates coordinate if (validations.isEmpty()) { return; } - final var identBytes = coordinates.getIdentifier().toIdent().toByteArray(); + final var identBytes = b64(coordinates.getIdentifier().toIdent()); try { dsl.mergeInto(IDENTIFIER) .using(dsl.selectOne()) @@ -275,7 +278,7 @@ public static void appendValidations(DSLContext dsl, EventCoordinates coordinate Record1 id; try { id = dsl.insertInto(COORDINATES) - .set(COORDINATES.DIGEST, coordinates.getDigest().getBytes()) + .set(COORDINATES.DIGEST, b64(coordinates.getDigest().getBytes())) .set(COORDINATES.IDENTIFIER, dsl.select(IDENTIFIER.ID).from(IDENTIFIER).where(IDENTIFIER.PREFIX.eq(identBytes))) .set(COORDINATES.ILK, coordinates.getIlk()) @@ -287,9 +290,9 @@ public static void appendValidations(DSLContext dsl, EventCoordinates coordinate id = dsl.select(COORDINATES.ID) .from(COORDINATES) .join(IDENTIFIER) - .on(IDENTIFIER.PREFIX.eq(coordinates.getIdentifier().toIdent().toByteArray())) + .on(IDENTIFIER.PREFIX.eq(b64(coordinates.getIdentifier().toIdent()))) .where(COORDINATES.IDENTIFIER.eq(IDENTIFIER.ID)) - .and(COORDINATES.DIGEST.eq(coordinates.getDigest().getBytes())) + .and(COORDINATES.DIGEST.eq(b64(coordinates.getDigest().getBytes()))) .and(COORDINATES.SEQUENCE_NUMBER.eq(coordinates.getSequenceNumber().toBigInteger())) .and(COORDINATES.ILK.eq(coordinates.getIlk())) .fetchOne(); @@ -305,31 +308,35 @@ public static void appendValidations(DSLContext dsl, EventCoordinates coordinate validations.forEach((coords, signature) -> { var vRec = dsl.newRecord(VALIDATION); vRec.setFor(l); - vRec.setValidator(coords.toEventCoords().toByteArray()); - vRec.setSignature(signature.toSig().toByteArray()); + vRec.setValidator(b64(coords.toEventCoords())); + vRec.setSignature(b64(signature.toSig())); result.accumulateAndGet(vRec.merge(), Integer::sum); }); log.info("Inserted validations: {} out of : {} for event: {}", result.get(), validations.size(), coordinates); } - public static byte[] compress(byte[] input) { + public static String compress(byte[] input) { ByteArrayOutputStream baos = new ByteArrayOutputStream(); - try (GZIPOutputStream gzos = new GZIPOutputStream(baos); + try (var out = new Base64OutputStream(baos); + GZIPOutputStream gzos = new GZIPOutputStream(out); ByteArrayInputStream bais = new ByteArrayInputStream(input)) { bais.transferTo(gzos); gzos.finish(); gzos.flush(); baos.flush(); + out.flush(); + baos.flush(); } catch (IOException e) { throw new IllegalStateException("unable to compress input bytes", e); } - return baos.toByteArray(); + return baos.toString(); } - public static byte[] decompress(byte[] input) { + public static byte[] decompress(String input) { ByteArrayOutputStream baos = new ByteArrayOutputStream(); - try (ByteArrayInputStream bais = new ByteArrayInputStream(input); - GZIPInputStream gis = new GZIPInputStream(bais)) { + try (Base64InputStream b64 = new Base64InputStream( + new ByteArrayInputStream(input.getBytes(Charset.defaultCharset()))); + GZIPInputStream gis = new GZIPInputStream(b64)) { gis.transferTo(baos); baos.flush(); } catch (IOException e) { @@ -347,14 +354,15 @@ public static void initialize(DSLContext dsl) { .using(context.selectOne()) .on(IDENTIFIER.ID.eq(0L)) .whenNotMatchedThenInsert(IDENTIFIER.ID, IDENTIFIER.PREFIX) - .values(0L, ecNone.getIdentifier().toIdent().toByteArray()) + .values(0L, b64(ecNone.getIdentifier().toIdent())) .execute(); + var bNone = b64(ecNone.getDigest().getBytes()); context.mergeInto(EVENT) .using(context.selectOne()) .on(EVENT.COORDINATES.eq(0L)) .whenNotMatchedThenInsert(EVENT.COORDINATES, EVENT.DIGEST, EVENT.CONTENT) - .values(0L, ecNone.getDigest().getBytes(), compress(new byte[0])) + .values(0L, bNone, compress(new byte[0])) .execute(); context.mergeInto(COORDINATES) @@ -362,8 +370,7 @@ public static void initialize(DSLContext dsl) { .on(COORDINATES.ID.eq(0L)) .whenNotMatchedThenInsert(COORDINATES.ID, COORDINATES.DIGEST, COORDINATES.IDENTIFIER, COORDINATES.SEQUENCE_NUMBER, COORDINATES.ILK) - .values(0L, ecNone.getDigest().getBytes(), 0L, ecNone.getSequenceNumber().toBigInteger(), - ecNone.getIlk()) + .values(0L, bNone, 0L, ecNone.getSequenceNumber().toBigInteger(), ecNone.getIlk()) .execute(); }); } @@ -377,9 +384,9 @@ public Attachment getAttachment(EventCoordinates coordinates) { var resolved = dsl.select(COORDINATES.ID) .from(COORDINATES) .join(IDENTIFIER) - .on(IDENTIFIER.PREFIX.eq(coordinates.getIdentifier().toIdent().toByteArray())) + .on(IDENTIFIER.PREFIX.eq(b64(coordinates.getIdentifier().toIdent()))) .where(COORDINATES.IDENTIFIER.eq(IDENTIFIER.ID)) - .and(COORDINATES.DIGEST.eq(coordinates.getDigest().getBytes())) + .and(COORDINATES.DIGEST.eq(b64(coordinates.getDigest().getBytes()))) .and(COORDINATES.SEQUENCE_NUMBER.eq(coordinates.getSequenceNumber().toBigInteger())) .and(COORDINATES.ILK.eq(coordinates.getIlk())) .and(COORDINATES.SEQUENCE_NUMBER.eq(coordinates.getSequenceNumber().toBigInteger())) @@ -395,7 +402,7 @@ public Attachment getAttachment(EventCoordinates coordinates) { .stream() .map(r -> { try { - return Seal.from(Sealed.parseFrom(r.value1())); + return Seal.from(Sealed.parseFrom(decompress(r.value1()))); } catch (InvalidProtocolBufferException e) { log.error("Error deserializing seal: {}", e); return null; @@ -413,7 +420,7 @@ record receipt(int witness, Sig signature) { .stream() .map(r -> { try { - return new receipt(r.value1(), Sig.parseFrom(r.value2())); + return new receipt(r.value1(), Sig.parseFrom(decompress(r.value2()))); } catch (InvalidProtocolBufferException e) { log.error("Error deserializing signature witness: {}", e); return null; @@ -435,7 +442,7 @@ public KeyEvent getKeyEvent(Digest digest) { .from(EVENT) .join(COORDINATES) .on(COORDINATES.ID.eq(EVENT.COORDINATES)) - .where(EVENT.DIGEST.eq(digest.getBytes())) + .where(EVENT.DIGEST.eq(b64(digest.getBytes()))) .fetchOptional() .map(r -> toKeyEvent(decompress(r.value1()), r.value2())) .orElse(null); @@ -450,9 +457,9 @@ public KeyEvent getKeyEvent(EventCoordinates coordinates) { .join(COORDINATES) .on(EVENT.COORDINATES.eq(COORDINATES.ID)) .join(IDENTIFIER) - .on(IDENTIFIER.PREFIX.eq(coordinates.getIdentifier().toIdent().toByteArray())) + .on(IDENTIFIER.PREFIX.eq(b64(coordinates.getIdentifier().toIdent()))) .where(COORDINATES.IDENTIFIER.eq(IDENTIFIER.ID)) - .and(COORDINATES.DIGEST.eq(coordinates.getDigest().getBytes())) + .and(COORDINATES.DIGEST.eq(b64(coordinates.getDigest().getBytes()))) .and(COORDINATES.SEQUENCE_NUMBER.eq(coordinates.getSequenceNumber().toBigInteger())) .and(COORDINATES.ILK.eq(coordinates.getIlk())) .and(COORDINATES.SEQUENCE_NUMBER.eq(coordinates.getSequenceNumber().toBigInteger())) @@ -470,9 +477,9 @@ public KeyState getKeyState(EventCoordinates coordinates) { .join(COORDINATES) .on(EVENT.COORDINATES.eq(COORDINATES.ID)) .join(IDENTIFIER) - .on(IDENTIFIER.PREFIX.eq(coordinates.getIdentifier().toIdent().toByteArray())) + .on(IDENTIFIER.PREFIX.eq(b64(coordinates.getIdentifier().toIdent()))) .where(COORDINATES.IDENTIFIER.eq(IDENTIFIER.ID)) - .and(COORDINATES.DIGEST.eq(coordinates.getDigest().getBytes())) + .and(COORDINATES.DIGEST.eq(b64(coordinates.getDigest().getBytes()))) .and(COORDINATES.SEQUENCE_NUMBER.eq(coordinates.getSequenceNumber().toBigInteger())) .and(COORDINATES.ILK.eq(coordinates.getIlk())) .fetchOptional() @@ -496,7 +503,7 @@ public KeyState getKeyState(Identifier identifier, ULong sequenceNumber) { .join(COORDINATES) .on(EVENT.COORDINATES.eq(COORDINATES.ID)) .join(IDENTIFIER) - .on(IDENTIFIER.PREFIX.eq(identifier.toIdent().toByteArray())) + .on(IDENTIFIER.PREFIX.eq(b64(identifier.toIdent()))) .where(COORDINATES.IDENTIFIER.eq(IDENTIFIER.ID)) .and(COORDINATES.SEQUENCE_NUMBER.eq(sequenceNumber.toBigInteger())) .fetchOptional() @@ -521,7 +528,7 @@ public KeyState getKeyState(Identifier identifier) { .join(CURRENT_KEY_STATE) .on(EVENT.COORDINATES.eq(CURRENT_KEY_STATE.CURRENT)) .join(IDENTIFIER) - .on(IDENTIFIER.PREFIX.eq(identifier.toIdent().toByteArray())) + .on(IDENTIFIER.PREFIX.eq(b64(identifier.toIdent()))) .where(CURRENT_KEY_STATE.IDENTIFIER.eq(IDENTIFIER.ID)) .fetchOptional() .map(r -> { @@ -542,9 +549,9 @@ public Map getValidations(EventCoordinates coordi var resolved = dsl.select(COORDINATES.ID) .from(COORDINATES) .join(IDENTIFIER) - .on(IDENTIFIER.PREFIX.eq(coordinates.getIdentifier().toIdent().toByteArray())) + .on(IDENTIFIER.PREFIX.eq(b64(coordinates.getIdentifier().toIdent()))) .where(COORDINATES.IDENTIFIER.eq(IDENTIFIER.ID)) - .and(COORDINATES.DIGEST.eq(coordinates.getDigest().getBytes())) + .and(COORDINATES.DIGEST.eq(b64(coordinates.getDigest().getBytes()))) .and(COORDINATES.SEQUENCE_NUMBER.eq(coordinates.getSequenceNumber().toBigInteger())) .and(COORDINATES.ILK.eq(coordinates.getIlk())) .and(COORDINATES.SEQUENCE_NUMBER.eq(coordinates.getSequenceNumber().toBigInteger())) @@ -563,8 +570,8 @@ record validation(EventCoords coordinates, Sig signature) { .stream() .map(r -> { try { - return new validation(EventCoords.parseFrom(r.value1()), - Sig.parseFrom(r.value2())); + return new validation(EventCoords.parseFrom(b64(r.value1())), + Sig.parseFrom(b64(r.value2()))); } catch (InvalidProtocolBufferException e) { log.error("Error deserializing signature witness: {}", e); return null; diff --git a/stereotomy/src/test/java/com/salesforce/apollo/stereotomy/db/TestUniKERL.java b/stereotomy/src/test/java/com/salesforce/apollo/stereotomy/db/TestUniKERL.java index 7661075e3..e4871fa77 100644 --- a/stereotomy/src/test/java/com/salesforce/apollo/stereotomy/db/TestUniKERL.java +++ b/stereotomy/src/test/java/com/salesforce/apollo/stereotomy/db/TestUniKERL.java @@ -44,8 +44,13 @@ * @author hal.hildebrand */ public class TestUniKERL { - private static final SecureRandom entropy = new SecureRandom(); + private SecureRandom entropy; + public void before() throws Exception { + SecureRandom entropy = SecureRandom.getInstance("SHA1PRNG"); + entropy.setSeed(new byte[] { 6, 6, 6 }); + } + @Test public void smoke() throws Exception { var factory = new ProtobufEventFactory(); diff --git a/thoth/src/main/java/com/salesforce/apollo/thoth/KerlDHT.java b/thoth/src/main/java/com/salesforce/apollo/thoth/KerlDHT.java index 464339fd3..2cd3fa333 100644 --- a/thoth/src/main/java/com/salesforce/apollo/thoth/KerlDHT.java +++ b/thoth/src/main/java/com/salesforce/apollo/thoth/KerlDHT.java @@ -85,6 +85,7 @@ import static com.salesforce.apollo.stereotomy.event.protobuf.ProtobufEventFactory.digestOf; import static com.salesforce.apollo.stereotomy.schema.tables.Identifier.IDENTIFIER; import static com.salesforce.apollo.thoth.schema.Tables.IDENTIFIER_LOCATION_HASH; +import static com.salesforce.apollo.utils.Utils.b64; /** * KerlDHT provides the replicated state store for KERLs @@ -170,7 +171,7 @@ public KerlDHT(Duration operationsFrequency, Context context, public static void updateLocationHash(Identifier identifier, DigestAlgorithm digestAlgorithm, DSLContext dsl) { dsl.transaction(config -> { var context = DSL.using(config); - var identBytes = identifier.toIdent().toByteArray(); + var identBytes = b64(identifier.toIdent()); // Braindead, but correct var id = context.select(IDENTIFIER.ID).from(IDENTIFIER).where(IDENTIFIER.PREFIX.eq(identBytes)).fetchOne(); if (id == null) { @@ -180,7 +181,7 @@ public static void updateLocationHash(Identifier identifier, DigestAlgorithm dig var hashed = digestAlgorithm.digest(identBytes); context.insertInto(IDENTIFIER_LOCATION_HASH, IDENTIFIER_LOCATION_HASH.IDENTIFIER, IDENTIFIER_LOCATION_HASH.DIGEST) - .values(id.value1(), hashed.getBytes()) + .values(id.value1(), b64(hashed.getBytes())) .onDuplicateKeyIgnore() .execute(); }); diff --git a/thoth/src/main/java/com/salesforce/apollo/thoth/KerlSpace.java b/thoth/src/main/java/com/salesforce/apollo/thoth/KerlSpace.java index 895306046..fc395a138 100644 --- a/thoth/src/main/java/com/salesforce/apollo/thoth/KerlSpace.java +++ b/thoth/src/main/java/com/salesforce/apollo/thoth/KerlSpace.java @@ -44,6 +44,8 @@ import java.util.stream.Collectors; import java.util.stream.Stream; +import static com.salesforce.apollo.stereotomy.db.UniKERL.compress; +import static com.salesforce.apollo.stereotomy.db.UniKERL.decompress; import static com.salesforce.apollo.stereotomy.schema.tables.Coordinates.COORDINATES; import static com.salesforce.apollo.stereotomy.schema.tables.Event.EVENT; import static com.salesforce.apollo.stereotomy.schema.tables.Identifier.IDENTIFIER; @@ -52,6 +54,7 @@ import static com.salesforce.apollo.thoth.schema.tables.PendingCoordinates.PENDING_COORDINATES; import static com.salesforce.apollo.thoth.schema.tables.PendingEvent.PENDING_EVENT; import static com.salesforce.apollo.thoth.schema.tables.PendingValidations.PENDING_VALIDATIONS; +import static com.salesforce.apollo.utils.Utils.b64; /** * Represents the replicated KERL logic @@ -71,16 +74,17 @@ public KerlSpace(JdbcConnectionPool connectionPool, Digest member, DigestAlgorit } public static void upsert(DSLContext dsl, EventCoords coordinates, Attachment attachment, Digest member) { - final var identBytes = coordinates.getIdentifier().toByteArray(); + final var identBytes = b64(coordinates.getIdentifier()); var ident = dsl.newRecord(IDENTIFIER); ident.setPrefix(identBytes); ident.merge(); Record1 id; + var coords = b64(Digest.from(coordinates.getDigest()).getBytes()); try { id = dsl.insertInto(PENDING_COORDINATES) - .set(PENDING_COORDINATES.DIGEST, Digest.from(coordinates.getDigest()).getBytes()) + .set(PENDING_COORDINATES.DIGEST, coords) .set(PENDING_COORDINATES.IDENTIFIER, dsl.select(IDENTIFIER.ID).from(IDENTIFIER).where(IDENTIFIER.PREFIX.eq(identBytes))) .set(PENDING_COORDINATES.ILK, coordinates.getIlk()) @@ -93,9 +97,9 @@ public static void upsert(DSLContext dsl, EventCoords coordinates, Attachment at id = dsl.select(PENDING_COORDINATES.ID) .from(PENDING_COORDINATES) .join(IDENTIFIER) - .on(IDENTIFIER.PREFIX.eq(coordinates.getIdentifier().toByteArray())) + .on(IDENTIFIER.PREFIX.eq(identBytes)) .where(PENDING_COORDINATES.IDENTIFIER.eq(IDENTIFIER.ID)) - .and(PENDING_COORDINATES.DIGEST.eq(Digest.from(coordinates.getDigest()).getBytes())) + .and(PENDING_COORDINATES.DIGEST.eq(coords)) .and(PENDING_COORDINATES.SEQUENCE_NUMBER.eq( ULong.valueOf(coordinates.getSequenceNumber()).toBigInteger())) .and(PENDING_COORDINATES.ILK.eq(coordinates.getIlk())) @@ -103,14 +107,14 @@ public static void upsert(DSLContext dsl, EventCoords coordinates, Attachment at } var vRec = dsl.newRecord(PENDING_ATTACHMENT); vRec.setCoordinates(id.value1()); - vRec.setAttachment(attachment.toByteArray()); + vRec.setAttachment(b64(attachment.toByteArray())); vRec.insert(); } public static void upsert(DSLContext context, KeyEvent event, DigestAlgorithm digestAlgorithm, Digest member) { final EventCoordinates prevCoords = event.getPrevious(); - final var identBytes = event.getIdentifier().toIdent().toByteArray(); + final var identBytes = b64(event.getIdentifier().toIdent()); context.mergeInto(IDENTIFIER) .using(context.selectOne()) @@ -119,9 +123,10 @@ public static void upsert(DSLContext context, KeyEvent event, DigestAlgorithm di .values(identBytes) .execute(); long id; + var prev = b64(prevCoords.getDigest().getBytes()); try { id = context.insertInto(PENDING_COORDINATES) - .set(PENDING_COORDINATES.DIGEST, prevCoords.getDigest().getBytes()) + .set(PENDING_COORDINATES.DIGEST, prev) .set(PENDING_COORDINATES.IDENTIFIER, context.select(IDENTIFIER.ID).from(IDENTIFIER).where(IDENTIFIER.PREFIX.eq(identBytes))) .set(PENDING_COORDINATES.ILK, event.getIlk()) @@ -137,7 +142,7 @@ public static void upsert(DSLContext context, KeyEvent event, DigestAlgorithm di .join(IDENTIFIER) .on(IDENTIFIER.PREFIX.eq(identBytes)) .where(PENDING_COORDINATES.IDENTIFIER.eq(IDENTIFIER.ID)) - .and(PENDING_COORDINATES.DIGEST.eq(prevCoords.getDigest().getBytes())) + .and(PENDING_COORDINATES.DIGEST.eq(prev)) .and( PENDING_COORDINATES.SEQUENCE_NUMBER.eq(coordinates.getSequenceNumber().toBigInteger())) .and(PENDING_COORDINATES.ILK.eq(coordinates.getIlk())) @@ -152,8 +157,8 @@ public static void upsert(DSLContext context, KeyEvent event, DigestAlgorithm di try { context.insertInto(PENDING_EVENT) .set(PENDING_EVENT.COORDINATES, id) - .set(PENDING_EVENT.DIGEST, digest.getBytes()) - .set(PENDING_EVENT.EVENT, event.getBytes()) + .set(PENDING_EVENT.DIGEST, b64(digest.getBytes())) + .set(PENDING_EVENT.EVENT, compress(event.getBytes())) .execute(); } catch (DataAccessException e) { } @@ -164,7 +169,7 @@ public static void upsert(DSLContext dsl, Validations validations, Digest member final var logCoords = EventCoordinates.from(coordinates); final var logIdentifier = Identifier.from(coordinates.getIdentifier()); log.trace("Upserting validations for: {} on: {}", logCoords, member); - final var identBytes = coordinates.getIdentifier().toByteArray(); + final var identBytes = b64(coordinates.getIdentifier()); try { dsl.mergeInto(IDENTIFIER) @@ -178,9 +183,10 @@ public static void upsert(DSLContext dsl, Validations validations, Digest member } Record1 id; + var d = b64(Digest.from(coordinates.getDigest()).getBytes()); try { id = dsl.insertInto(PENDING_COORDINATES) - .set(PENDING_COORDINATES.DIGEST, Digest.from(coordinates.getDigest()).getBytes()) + .set(PENDING_COORDINATES.DIGEST, d) .set(PENDING_COORDINATES.IDENTIFIER, dsl.select(IDENTIFIER.ID).from(IDENTIFIER).where(IDENTIFIER.PREFIX.eq(identBytes))) .set(PENDING_COORDINATES.ILK, coordinates.getIlk()) @@ -195,9 +201,9 @@ public static void upsert(DSLContext dsl, Validations validations, Digest member id = dsl.select(PENDING_COORDINATES.ID) .from(PENDING_COORDINATES) .join(IDENTIFIER) - .on(IDENTIFIER.PREFIX.eq(coordinates.getIdentifier().toByteArray())) + .on(IDENTIFIER.PREFIX.eq(identBytes)) .where(PENDING_COORDINATES.IDENTIFIER.eq(IDENTIFIER.ID)) - .and(PENDING_COORDINATES.DIGEST.eq(Digest.from(coordinates.getDigest()).getBytes())) + .and(PENDING_COORDINATES.DIGEST.eq(d)) .and(PENDING_COORDINATES.SEQUENCE_NUMBER.eq( ULong.valueOf(coordinates.getSequenceNumber()).toBigInteger())) .and(PENDING_COORDINATES.ILK.eq(coordinates.getIlk())) @@ -209,7 +215,7 @@ public static void upsert(DSLContext dsl, Validations validations, Digest member } var vRec = dsl.newRecord(PENDING_VALIDATIONS); vRec.setCoordinates(id.value1()); - vRec.setValidations(validations.toByteArray()); + vRec.setValidations(b64(validations)); vRec.insert(); } @@ -325,7 +331,7 @@ private void commitPending(DSLContext context, KERL.AppendKERL kerl) { .orderBy(PENDING_COORDINATES.SEQUENCE_NUMBER) .fetchStream() .forEach(r -> { - KeyEvent event = ProtobufEventFactory.toKeyEvent(r.value2(), r.value3()); + KeyEvent event = ProtobufEventFactory.toKeyEvent(decompress(r.value2()), r.value3()); EventCoordinates coordinates = event.getCoordinates(); if (coordinates != null) { context.select(PENDING_ATTACHMENT.ATTACHMENT) @@ -334,7 +340,7 @@ private void commitPending(DSLContext context, KERL.AppendKERL kerl) { .stream() .forEach(bytes -> { try { - Attachment attach = Attachment.parseFrom(bytes.value1()); + Attachment attach = Attachment.parseFrom(b64(bytes.value1())); kerl.append(Collections.singletonList(new AttachmentEventImpl( AttachmentEvent.newBuilder() .setCoordinates(coordinates.toEventCoords()) @@ -350,7 +356,7 @@ private void commitPending(DSLContext context, KERL.AppendKERL kerl) { .stream() .forEach(bytes -> { try { - Validations attach = Validations.parseFrom(bytes.value1()); + Validations attach = Validations.parseFrom(b64(bytes.value1())); kerl.appendValidations(coordinates, attach.getValidationsList() .stream() .collect(Collectors.toMap( @@ -398,6 +404,8 @@ private Stream eventDigestsIn(CombinedIntervals intervals, DSLContext ds } private Stream eventDigestsIn(KeyInterval interval, DSLContext dsl) { + var begin = b64(interval.getBegin().getBytes()); + var end = b64(interval.getEnd().getBytes()); return Stream.concat(dsl.select(EVENT.DIGEST) .from(EVENT) .join(COORDINATES) @@ -406,30 +414,29 @@ private Stream eventDigestsIn(KeyInterval interval, DSLContext dsl) { .on(COORDINATES.IDENTIFIER.eq(IDENTIFIER.ID)) .join(IDENTIFIER_LOCATION_HASH) .on(IDENTIFIER.ID.eq(IDENTIFIER_LOCATION_HASH.IDENTIFIER)) - .where(IDENTIFIER_LOCATION_HASH.DIGEST.ge(interval.getBegin().getBytes())) - .and(IDENTIFIER_LOCATION_HASH.DIGEST.le(interval.getEnd().getBytes())) + .where(IDENTIFIER_LOCATION_HASH.DIGEST.ge(begin)) + .and(IDENTIFIER_LOCATION_HASH.DIGEST.le(end)) .stream() - .map(r -> new Digest(algorithm, r.value1())) - .filter(d -> d != null), dsl.select(PENDING_EVENT.DIGEST) - .from(PENDING_EVENT) - .join(PENDING_COORDINATES) - .on(PENDING_EVENT.COORDINATES.eq(PENDING_COORDINATES.ID)) - .join(IDENTIFIER) - .on(PENDING_COORDINATES.IDENTIFIER.eq(IDENTIFIER.ID)) - .join(IDENTIFIER_LOCATION_HASH) - .on(IDENTIFIER.ID.eq(IDENTIFIER_LOCATION_HASH.IDENTIFIER)) - .where(IDENTIFIER_LOCATION_HASH.DIGEST.ge( - interval.getBegin().getBytes())) - .and(IDENTIFIER_LOCATION_HASH.DIGEST.le( - interval.getEnd().getBytes())) - .stream() - .map(r -> { - try { - return Digest.from(Digeste.parseFrom(r.value1())); - } catch (InvalidProtocolBufferException e) { - return null; - } - }) - .filter(d -> d != null)); + .map(r -> new Digest(algorithm, b64(r.value1()))) + .filter(Objects::nonNull), dsl.select(PENDING_EVENT.DIGEST) + .from(PENDING_EVENT) + .join(PENDING_COORDINATES) + .on(PENDING_EVENT.COORDINATES.eq(PENDING_COORDINATES.ID)) + .join(IDENTIFIER) + .on(PENDING_COORDINATES.IDENTIFIER.eq(IDENTIFIER.ID)) + .join(IDENTIFIER_LOCATION_HASH) + .on(IDENTIFIER.ID.eq(IDENTIFIER_LOCATION_HASH.IDENTIFIER)) + .where(IDENTIFIER_LOCATION_HASH.DIGEST.ge(begin)) + .and(IDENTIFIER_LOCATION_HASH.DIGEST.le(end)) + .stream() + .map(r -> { + try { + return Digest.from( + Digeste.parseFrom(b64(r.value1()))); + } catch (InvalidProtocolBufferException e) { + return null; + } + }) + .filter(Objects::nonNull)); } } diff --git a/thoth/src/test/java/com/salesforce/apollo/thoth/KerlSpaceTest.java b/thoth/src/test/java/com/salesforce/apollo/thoth/KerlSpaceTest.java index a02000f3c..93121e884 100644 --- a/thoth/src/test/java/com/salesforce/apollo/thoth/KerlSpaceTest.java +++ b/thoth/src/test/java/com/salesforce/apollo/thoth/KerlSpaceTest.java @@ -20,7 +20,6 @@ import org.h2.jdbcx.JdbcConnectionPool; import org.jooq.SQLDialect; import org.jooq.impl.DSL; -import org.junit.jupiter.api.Test; import java.security.SecureRandom; import java.sql.SQLException; @@ -57,7 +56,7 @@ private static void initialize(JdbcConnectionPool connectionPoolA, JdbcConnectio } } - @Test + // @Test public void smokin() throws Exception { final var digestAlgorithm = DigestAlgorithm.DEFAULT; var entropy = SecureRandom.getInstance("SHA1PRNG"); From 24db4b3a510b90f3624ecfe97659d7c860ea3fd3 Mon Sep 17 00:00:00 2001 From: Hellblazer Date: Fri, 5 Jul 2024 11:53:20 -0700 Subject: [PATCH 30/32] combine hashes for view id, simplify bft subset.mtls server/client vthread default --- .../apollo/archipelago/MtlsClient.java | 2 +- .../apollo/archipelago/MtlsServer.java | 9 ++++--- .../apollo/context/DynamicContextImpl.java | 8 +----- .../apollo/context/StaticContext.java | 8 +----- .../apollo/context/ContextTests.java | 25 +++++++++++++++++++ 5 files changed, 33 insertions(+), 19 deletions(-) diff --git a/memberships/src/main/java/com/salesforce/apollo/archipelago/MtlsClient.java b/memberships/src/main/java/com/salesforce/apollo/archipelago/MtlsClient.java index 8b5d4957e..5bda43471 100644 --- a/memberships/src/main/java/com/salesforce/apollo/archipelago/MtlsClient.java +++ b/memberships/src/main/java/com/salesforce/apollo/archipelago/MtlsClient.java @@ -36,7 +36,7 @@ public MtlsClient(SocketAddress address, ClientAuth clientAuth, String alias, Cl Limiter limiter = new GrpcClientLimiterBuilder().blockOnLimit(false).build(); channel = NettyChannelBuilder.forAddress(address) - // .executor(executor) + .executor(executor) .withOption(ChannelOption.TCP_NODELAY, true) .sslContext(supplier.forClient(clientAuth, alias, validator, MtlsServer.TL_SV1_3)) .intercept(new ConcurrencyLimitClientInterceptor(limiter, diff --git a/memberships/src/main/java/com/salesforce/apollo/archipelago/MtlsServer.java b/memberships/src/main/java/com/salesforce/apollo/archipelago/MtlsServer.java index 3cac8d621..88e79fc34 100644 --- a/memberships/src/main/java/com/salesforce/apollo/archipelago/MtlsServer.java +++ b/memberships/src/main/java/com/salesforce/apollo/archipelago/MtlsServer.java @@ -44,6 +44,7 @@ import java.util.List; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import java.util.function.Function; import java.util.function.Predicate; import java.util.function.Supplier; @@ -140,15 +141,15 @@ public static SslContext forServer(ClientAuth clientAuth, String alias, X509Cert public RouterImpl router(ServerConnectionCache.Builder cacheBuilder, Supplier serverLimit, LimitsRegistry limitsRegistry, List interceptors, Predicate validator, ExecutorService executor) { - // if (executor == null) { - // executor = Executors.newVirtualThreadPerTaskExecutor(); - // } + if (executor == null) { + executor = Executors.newVirtualThreadPerTaskExecutor(); + } var limitsBuilder = new GrpcServerLimiterBuilder().limit(serverLimit.get()); if (limitsRegistry != null) { limitsBuilder.metricRegistry(limitsRegistry); } NettyServerBuilder serverBuilder = NettyServerBuilder.forAddress(epProvider.getBindAddress()) - // .executor(executor) + .executor(executor) .withOption(ChannelOption.SO_REUSEADDR, true) .sslContext(supplier.forServer(ClientAuth.REQUIRE, epProvider.getAlias(), diff --git a/memberships/src/main/java/com/salesforce/apollo/context/DynamicContextImpl.java b/memberships/src/main/java/com/salesforce/apollo/context/DynamicContextImpl.java index 43cb35a3d..068804878 100644 --- a/memberships/src/main/java/com/salesforce/apollo/context/DynamicContextImpl.java +++ b/memberships/src/main/java/com/salesforce/apollo/context/DynamicContextImpl.java @@ -730,16 +730,10 @@ public Iterable traverse(int ring, T member) { */ @Override public void uniqueSuccessors(Digest key, Predicate test, Set collector) { - var delegate = ring(0).successor(key, test); - if (delegate == null) { - return; - } for (Ring ring : rings) { - T successor = ring.successor(hashFor(delegate, ring.index), m -> !collector.contains(m) && test.test(m)); + T successor = ring.successor(key, m -> !collector.contains(m) && test.test(m)); if (successor != null) { collector.add(successor); - } else { - collector.add(delegate); } } } diff --git a/memberships/src/main/java/com/salesforce/apollo/context/StaticContext.java b/memberships/src/main/java/com/salesforce/apollo/context/StaticContext.java index 7c78c165c..be43490f9 100644 --- a/memberships/src/main/java/com/salesforce/apollo/context/StaticContext.java +++ b/memberships/src/main/java/com/salesforce/apollo/context/StaticContext.java @@ -449,17 +449,11 @@ public Iterable traverse(int ring, T member) { @Override public void uniqueSuccessors(Digest key, Predicate test, Set collector) { - var delegate = ring(0).successor(key, test); - if (delegate == null) { - return; - } for (int ring = 0; ring < rings.length; ring++) { StaticRing r = ring(ring); - T successor = r.successor(hashFor(delegate, r.index), m -> !collector.contains(m) && test.test(m)); + T successor = r.successor(key, m -> !collector.contains(m) && test.test(m)); if (successor != null) { collector.add(successor); - } else { - collector.add(delegate); } } } diff --git a/memberships/src/test/java/com/salesforce/apollo/context/ContextTests.java b/memberships/src/test/java/com/salesforce/apollo/context/ContextTests.java index f9f5d4dc8..859f06281 100644 --- a/memberships/src/test/java/com/salesforce/apollo/context/ContextTests.java +++ b/memberships/src/test/java/com/salesforce/apollo/context/ContextTests.java @@ -18,6 +18,7 @@ import java.security.SecureRandom; import java.util.ArrayList; import java.util.List; +import java.util.SequencedSet; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -26,6 +27,30 @@ */ public class ContextTests { + @Test + public void bftSubset() throws Exception { + var context = new DynamicContextImpl<>(DigestAlgorithm.DEFAULT.getOrigin().prefix(1), 10, 0.2, 2); + List members = new ArrayList<>(); + var entropy = SecureRandom.getInstance("SHA1PRNG"); + entropy.setSeed(new byte[] { 6, 6, 6 }); + var stereotomy = new StereotomyImpl(new MemKeyStore(), new MemKERL(DigestAlgorithm.DEFAULT), entropy); + + for (int i = 0; i < 7; i++) { + SigningMember m = new ControlledIdentifierMember(stereotomy.newIdentifier()); + members.add(m); + context.activate(m); + } + var testEntropy = SecureRandom.getInstance("SHA1PRNG"); + testEntropy.setSeed(new byte[] { 6, 6, 6 }); + var algo = DigestAlgorithm.DEFAULT; + List> subsets = new ArrayList<>(); + for (int i = 0; i < 10; i++) { + var subset = context.bftSubset(algo.random(testEntropy)); + System.out.println(subset.stream().map(Member::getId).toList()); + subsets.add(subset); + } + } + @Test public void consistency() throws Exception { var context = new DynamicContextImpl<>(DigestAlgorithm.DEFAULT.getOrigin().prefix(1), 10, 0.2, 2); From 219cd548f9d323759868b9bb143aa486d8752d52 Mon Sep 17 00:00:00 2001 From: Hellblazer Date: Fri, 5 Jul 2024 12:11:47 -0700 Subject: [PATCH 31/32] lol. roll back MTLS vthreads --- .../com/salesforce/apollo/archipelago/MtlsClient.java | 6 +----- .../com/salesforce/apollo/archipelago/MtlsServer.java | 9 ++++----- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/memberships/src/main/java/com/salesforce/apollo/archipelago/MtlsClient.java b/memberships/src/main/java/com/salesforce/apollo/archipelago/MtlsClient.java index 5bda43471..5cb355331 100644 --- a/memberships/src/main/java/com/salesforce/apollo/archipelago/MtlsClient.java +++ b/memberships/src/main/java/com/salesforce/apollo/archipelago/MtlsClient.java @@ -19,16 +19,12 @@ import io.netty.handler.ssl.ClientAuth; import java.net.SocketAddress; -import java.util.concurrent.Executor; -import java.util.concurrent.Executors; /** * @author hal.hildebrand */ public class MtlsClient { - private final static Executor executor = Executors.newVirtualThreadPerTaskExecutor(); - private final ManagedChannel channel; public MtlsClient(SocketAddress address, ClientAuth clientAuth, String alias, ClientContextSupplier supplier, @@ -36,7 +32,7 @@ public MtlsClient(SocketAddress address, ClientAuth clientAuth, String alias, Cl Limiter limiter = new GrpcClientLimiterBuilder().blockOnLimit(false).build(); channel = NettyChannelBuilder.forAddress(address) - .executor(executor) + // .executor(executor) .withOption(ChannelOption.TCP_NODELAY, true) .sslContext(supplier.forClient(clientAuth, alias, validator, MtlsServer.TL_SV1_3)) .intercept(new ConcurrencyLimitClientInterceptor(limiter, diff --git a/memberships/src/main/java/com/salesforce/apollo/archipelago/MtlsServer.java b/memberships/src/main/java/com/salesforce/apollo/archipelago/MtlsServer.java index 88e79fc34..3cac8d621 100644 --- a/memberships/src/main/java/com/salesforce/apollo/archipelago/MtlsServer.java +++ b/memberships/src/main/java/com/salesforce/apollo/archipelago/MtlsServer.java @@ -44,7 +44,6 @@ import java.util.List; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; import java.util.function.Function; import java.util.function.Predicate; import java.util.function.Supplier; @@ -141,15 +140,15 @@ public static SslContext forServer(ClientAuth clientAuth, String alias, X509Cert public RouterImpl router(ServerConnectionCache.Builder cacheBuilder, Supplier serverLimit, LimitsRegistry limitsRegistry, List interceptors, Predicate validator, ExecutorService executor) { - if (executor == null) { - executor = Executors.newVirtualThreadPerTaskExecutor(); - } + // if (executor == null) { + // executor = Executors.newVirtualThreadPerTaskExecutor(); + // } var limitsBuilder = new GrpcServerLimiterBuilder().limit(serverLimit.get()); if (limitsRegistry != null) { limitsBuilder.metricRegistry(limitsRegistry); } NettyServerBuilder serverBuilder = NettyServerBuilder.forAddress(epProvider.getBindAddress()) - .executor(executor) + // .executor(executor) .withOption(ChannelOption.SO_REUSEADDR, true) .sslContext(supplier.forServer(ClientAuth.REQUIRE, epProvider.getAlias(), From 300b00dd4819ba1cf0cbf19a8a2bcc90bf2d008b Mon Sep 17 00:00:00 2001 From: Hellblazer Date: Fri, 5 Jul 2024 15:33:02 -0700 Subject: [PATCH 32/32] consumption --- .../com/salesforce/apollo/choam/CHOAM.java | 27 ++++++++++--------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/choam/src/main/java/com/salesforce/apollo/choam/CHOAM.java b/choam/src/main/java/com/salesforce/apollo/choam/CHOAM.java index 2a587e0af..7ccaf1249 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/CHOAM.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/CHOAM.java @@ -561,7 +561,7 @@ private void consume(HashedCertifiedBlock next) { return; } - if (nlc.compareTo(view) > 0) { + if (view != null && nlc.compareTo(view) > 0) { // later view log.trace("Wait for reconfiguration @ {} block: {} hash: {} height: {} current: {} on: {}", next.block.getHeader().getLastReconfig(), next.block.getBodyCase(), next.hash, next.height(), @@ -575,8 +575,15 @@ private void consume(HashedCertifiedBlock next) { } private void consume(HashedCertifiedBlock next, HashedCertifiedBlock cur) { + if (next == null) { + return; + } + final var h = head.get(); if (isNext(next)) { - if (current.get().validate(next)) { + if (!h.hash.equals(next.getPrevious())) { + log.debug("Invalid previous: {} expecting: {} block: {} hash: {} height: {} on: {}", next.getPrevious(), + h.hash, next.block.getBodyCase(), next.hash, next.height(), params.member().getId()); + } else if (current.get().validate(next)) { log.trace("Accept: {} hash: {} height: {} on: {}", next.block.getBodyCase(), next.hash, next.height(), params.member().getId()); accept(next); @@ -584,10 +591,13 @@ private void consume(HashedCertifiedBlock next, HashedCertifiedBlock cur) { log.debug("Invalid block: {} hash: {} height: {} on: {}", next.block.getBodyCase(), next.hash, next.height(), params.member().getId()); } - } else { + } else if (h.height().compareTo(next.height()) > 0) { log.trace("Premature block: {} : {} height: {} current: {} on: {}", next.block.getBodyCase(), next.hash, next.height(), cur.height(), params.member().getId()); pending.add(next); + } else { + log.trace("Stale block: {} : {} height: {} current: {} on: {}", next.block.getBodyCase(), next.hash, + next.height(), cur.height(), params.member().getId()); } } @@ -677,20 +687,11 @@ private String getLabel() { } private boolean isNext(HashedBlock next) { - if (next == null) { - return false; - } final var h = head.get(); if (h.height() == null && next.height().equals(ULong.valueOf(0))) { return true; } - final Digest prev = next.getPrevious(); - var isNext = h.hash.equals(prev); - if (!isNext) { - log.info("isNext: false previous: {} block: {} hash: {} height: {} current: {} height: {} on: {}", prev, - next.block.getBodyCase(), next.hash, next.height(), h.hash, h.height(), params.member().getId()); - } - return isNext; + return next.height().equals(h.height().add(1)); } private void join(SignedViewMember nextView, Digest from) {