From d1149d182bfb2417194b51ee66e0ee0d4e450887 Mon Sep 17 00:00:00 2001 From: Hellblazer Date: Sat, 13 Apr 2024 08:55:34 -0700 Subject: [PATCH 01/10] interim --- .../com/salesforce/apollo/choam/CHOAM.java | 26 ++++++++++--------- .../com/salesforce/apollo/choam/Producer.java | 3 +++ .../com/salesforce/apollo/choam/Session.java | 9 ++++--- 3 files changed, 23 insertions(+), 15 deletions(-) diff --git a/choam/src/main/java/com/salesforce/apollo/choam/CHOAM.java b/choam/src/main/java/com/salesforce/apollo/choam/CHOAM.java index d475aa91b..2b1783083 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/CHOAM.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/CHOAM.java @@ -23,9 +23,11 @@ import com.salesforce.apollo.choam.support.HashedCertifiedBlock.NullBlock; import com.salesforce.apollo.context.Context; import com.salesforce.apollo.context.DelegatedContext; +import com.salesforce.apollo.context.StaticContext; import com.salesforce.apollo.cryptography.*; import com.salesforce.apollo.cryptography.Signer.SignerImpl; import com.salesforce.apollo.cryptography.proto.PubKey; +import com.salesforce.apollo.ethereal.Dag; import com.salesforce.apollo.membership.GroupIterator; import com.salesforce.apollo.membership.Member; import com.salesforce.apollo.membership.RoundScheduler; @@ -44,7 +46,10 @@ import java.io.FileInputStream; import java.io.IOException; import java.security.KeyPair; -import java.util.*; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; @@ -394,12 +399,6 @@ private boolean checkJoin(Digest nextView, Digest from) { source.getId(), params.member().getId()); return false; } - final Set members = Committee.viewMembersOf(nextView, pendingView().get()); - if (!members.contains(params.member())) { - log.debug("Not a member of view: {} invalid join request from: {} members: {} on: {}", nextView, - source.getId(), members.stream().map(m -> m.getId()).toList(), params.member().getId()); - return false; - } return true; } @@ -736,12 +735,12 @@ private void reconfigure(Reconfigure reconfigure) { view.set(h); session.setView(h); if (validators.containsKey(params.member())) { - try { + if (Dag.validate(validators.size())) { current.set(new Associate(h, validators, currentView)); - } catch (IllegalArgumentException e) { + } else { + log.warn("Reconfiguration to associate failed: {} in view: {} on:{}", validators.size(), + new Digest(reconfigure.getId()), params.member().getId()); current.set(new Client(validators, getViewId())); - log.debug("unable to create consensus: {} defaulting to committee: {} on: {}", e.getMessage(), - current.get().getClass().getSimpleName(), params.member().getId()); } } else { current.set(new Client(validators, getViewId())); @@ -1262,6 +1261,8 @@ public SubmitResult submitTxn(Transaction transaction) { log.debug("No link for: {} for submitting txn on: {}", target.getId(), params.member().getId()); return SubmitResult.newBuilder().setResult(Result.UNAVAILABLE).build(); } + log.trace("Submitting txn: {} to: {} in view: {} on: {}", hashOf(transaction, params.digestAlgorithm()), + link.getMember().getId(), viewId, params.member().getId()); return link.submit(transaction); } catch (StatusRuntimeException e) { log.trace("Failed submitting txn: {} status:{} to: {} in: {} on: {}", @@ -1294,7 +1295,8 @@ private class Associate extends Administration { super(validators, new Digest( viewChange.block.hasGenesis() ? viewChange.block.getGenesis().getInitialView().getId() : viewChange.block.getReconfigure().getId())); - var context = Committee.viewFor(viewId, params.context()); + var context = new StaticContext<>(viewId, params.context().getProbabilityByzantine(), 3, + validators.keySet(), params.context().getEpsilon(), validators.size()); log.trace("Using consensus key: {} sig: {} for view: {} on: {}", params.digestAlgorithm().digest(nextView.consensusKeyPair.getPublic().getEncoded()), params.digestAlgorithm().digest(nextView.member.getSignature().toByteString()), viewId, diff --git a/choam/src/main/java/com/salesforce/apollo/choam/Producer.java b/choam/src/main/java/com/salesforce/apollo/choam/Producer.java index d8af594e1..0ff834cb1 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/Producer.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/Producer.java @@ -150,6 +150,9 @@ public void stop() { } public SubmitResult submit(Transaction transaction) { + if (!started.get()) { + return SubmitResult.newBuilder().setResult(Result.NO_COMMITTEE).build(); + } if (ds.offer(transaction)) { return SubmitResult.newBuilder().setResult(Result.PUBLISHED).build(); } else { diff --git a/choam/src/main/java/com/salesforce/apollo/choam/Session.java b/choam/src/main/java/com/salesforce/apollo/choam/Session.java index d7185aea9..99aa48076 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/Session.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/Session.java @@ -89,9 +89,11 @@ public static boolean verify(Transaction transaction, Verifier verifier) { public static CompletableFuture retryNesting(Supplier> supplier, int maxRetries) { CompletableFuture cf = supplier.get(); for (int i = 0; i < maxRetries; i++) { - cf = cf.thenApply(CompletableFuture::completedFuture) - .exceptionally(__ -> supplier.get()) - .thenCompose(java.util.function.Function.identity()); + final var attempt = i; + cf = cf.thenApply(CompletableFuture::completedFuture).exceptionally(__ -> { + log.warn("resubmitting, next attempt: {}", attempt); + return supplier.get(); + }).thenCompose(java.util.function.Function.identity()); } return cf; } @@ -310,6 +312,7 @@ private Submission submit(SubmittedTransaction stx) { return new Submission(SubmitResult.newBuilder().setResult(SubmitResult.Result.RATE_LIMITED).build(), listener); } + log.debug("Submitting txn: {} on: {}", stx.hash(), params.member().getId()); return new Submission(service.apply(stx), listener); } From e9362f7a35b4f86d98e03ccd56717b0f8e82a2e4 Mon Sep 17 00:00:00 2001 From: Hellblazer Date: Sat, 13 Apr 2024 08:56:43 -0700 Subject: [PATCH 02/10] delete --- VAT.txt | 59 --------------------------------------------------------- 1 file changed, 59 deletions(-) delete mode 100644 VAT.txt diff --git a/VAT.txt b/VAT.txt deleted file mode 100644 index f19bbec6e..000000000 --- a/VAT.txt +++ /dev/null @@ -1,59 +0,0 @@ -Internal Error occurred. -org.junit.platform.commons.JUnitException: TestEngine with ID 'junit-jupiter' failed to discover tests - at org.junit.platform.launcher.core.EngineDiscoveryOrchestrator.discoverEngineRoot(EngineDiscoveryOrchestrator.java:160) - at org.junit.platform.launcher.core.EngineDiscoveryOrchestrator.discoverSafely(EngineDiscoveryOrchestrator.java:132) - at org.junit.platform.launcher.core.EngineDiscoveryOrchestrator.discover(EngineDiscoveryOrchestrator.java:107) - at org.junit.platform.launcher.core.EngineDiscoveryOrchestrator.discover(EngineDiscoveryOrchestrator.java:78) - at org.junit.platform.launcher.core.DefaultLauncher.discover(DefaultLauncher.java:99) - at org.junit.platform.launcher.core.DefaultLauncher.execute(DefaultLauncher.java:85) - at org.junit.platform.launcher.core.DelegatingLauncher.execute(DelegatingLauncher.java:47) - at org.junit.platform.launcher.core.SessionPerRequestLauncher.execute(SessionPerRequestLauncher.java:63) - at com.intellij.junit5.JUnit5IdeaTestRunner.startRunnerWithArgs(JUnit5IdeaTestRunner.java:57) - at com.intellij.rt.junit.IdeaTestRunner$Repeater$1.execute(IdeaTestRunner.java:38) - at com.intellij.rt.execution.junit.TestsRepeater.repeat(TestsRepeater.java:11) - at com.intellij.rt.junit.IdeaTestRunner$Repeater.startRunnerWithArgs(IdeaTestRunner.java:35) - at com.intellij.rt.junit.JUnitStarter.prepareStreamsAndStart(JUnitStarter.java:232) - at com.intellij.rt.junit.JUnitStarter.main(JUnitStarter.java:55) -Caused by: org.junit.platform.commons.JUnitException: ClassSelector [className = 'com.salesforce.apollo.choam.ViewAssemblyTest', classLoader = null] resolution failed - at org.junit.platform.launcher.listeners.discovery.AbortOnFailureLauncherDiscoveryListener.selectorProcessed(AbortOnFailureLauncherDiscoveryListener.java:39) - at org.junit.platform.engine.support.discovery.EngineDiscoveryRequestResolution.resolveCompletely(EngineDiscoveryRequestResolution.java:103) - at org.junit.platform.engine.support.discovery.EngineDiscoveryRequestResolution.run(EngineDiscoveryRequestResolution.java:83) - at org.junit.platform.engine.support.discovery.EngineDiscoveryRequestResolver.resolve(EngineDiscoveryRequestResolver.java:113) - at org.junit.jupiter.engine.discovery.DiscoverySelectorResolver.resolveSelectors(DiscoverySelectorResolver.java:46) - at org.junit.jupiter.engine.JupiterTestEngine.discover(JupiterTestEngine.java:69) - at org.junit.platform.launcher.core.EngineDiscoveryOrchestrator.discoverEngineRoot(EngineDiscoveryOrchestrator.java:152) - ... 13 more -Caused by: org.junit.platform.commons.PreconditionViolationException: Could not load class with name: com.salesforce.apollo.choam.ViewAssemblyTest - at org.junit.platform.engine.discovery.ClassSelector.lambda$getJavaClass$0(ClassSelector.java:95) - at org.junit.platform.commons.function.Try$Failure.getOrThrow(Try.java:335) - at org.junit.platform.engine.discovery.ClassSelector.getJavaClass(ClassSelector.java:94) - at org.junit.jupiter.engine.discovery.ClassSelectorResolver.resolve(ClassSelectorResolver.java:66) - at org.junit.platform.engine.support.discovery.EngineDiscoveryRequestResolution.lambda$resolve$2(EngineDiscoveryRequestResolution.java:135) - at java.base/java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:212) - at java.base/java.util.ArrayList$ArrayListSpliterator.tryAdvance(ArrayList.java:1686) - at java.base/java.util.stream.ReferencePipeline.forEachWithCancel(ReferencePipeline.java:144) - at java.base/java.util.stream.AbstractPipeline.copyIntoWithCancel(AbstractPipeline.java:574) - at java.base/java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:560) - at java.base/java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:546) - at java.base/java.util.stream.FindOps$FindOp.evaluateSequential(FindOps.java:150) - at java.base/java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:265) - at java.base/java.util.stream.ReferencePipeline.findFirst(ReferencePipeline.java:662) - at org.junit.platform.engine.support.discovery.EngineDiscoveryRequestResolution.resolve(EngineDiscoveryRequestResolution.java:189) - at org.junit.platform.engine.support.discovery.EngineDiscoveryRequestResolution.resolve(EngineDiscoveryRequestResolution.java:126) - at org.junit.platform.engine.support.discovery.EngineDiscoveryRequestResolution.resolveCompletely(EngineDiscoveryRequestResolution.java:92) - ... 18 more -Caused by: java.lang.ClassNotFoundException: com.salesforce.apollo.choam.ViewAssemblyTest - at java.base/jdk.internal.loader.BuiltinClassLoader.loadClass(BuiltinClassLoader.java:641) - at java.base/jdk.internal.loader.ClassLoaders$AppClassLoader.loadClass(ClassLoaders.java:188) - at java.base/java.lang.ClassLoader.loadClass(ClassLoader.java:525) - at java.base/java.lang.Class.forName0(Native Method) - at java.base/java.lang.Class.forName(Class.java:529) - at java.base/java.lang.Class.forName(Class.java:508) - at org.junit.platform.commons.util.ReflectionUtils.lambda$tryToLoadClass$9(ReflectionUtils.java:832) - at org.junit.platform.commons.function.Try.lambda$call$0(Try.java:57) - at org.junit.platform.commons.function.Try.of(Try.java:93) - at org.junit.platform.commons.function.Try.call(Try.java:57) - at org.junit.platform.commons.util.ReflectionUtils.tryToLoadClass(ReflectionUtils.java:795) - at org.junit.platform.commons.util.ReflectionUtils.tryToLoadClass(ReflectionUtils.java:751) - at org.junit.platform.engine.discovery.ClassSelector.getJavaClass(ClassSelector.java:92) - ... 32 more From 17da1224242b27d2c10eca7c0b531c3375891cd8 Mon Sep 17 00:00:00 2001 From: Hellblazer Date: Sat, 13 Apr 2024 16:02:08 -0700 Subject: [PATCH 03/10] revert dat and clean --- choam/src/main/java/com/salesforce/apollo/choam/CHOAM.java | 4 +++- choam/src/main/java/com/salesforce/apollo/choam/Session.java | 2 +- .../src/main/java/com/salesforce/apollo/ethereal/Adder.java | 2 +- .../main/java/com/salesforce/apollo/ethereal/Ethereal.java | 2 ++ .../salesforce/apollo/ethereal/memberships/ChRbcGossip.java | 2 +- .../src/main/java/com/salesforce/apollo/fireflies/View.java | 2 +- grpc/src/main/proto/choam.proto | 1 + 7 files changed, 10 insertions(+), 5 deletions(-) diff --git a/choam/src/main/java/com/salesforce/apollo/choam/CHOAM.java b/choam/src/main/java/com/salesforce/apollo/choam/CHOAM.java index 2b1783083..a751be158 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/CHOAM.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/CHOAM.java @@ -1237,7 +1237,9 @@ public Logger log() { @Override public void nextView(Context pendingView) { var previous = CHOAM.this.pendingView.getAndSet(pendingView); - log.info("Pending view: {} size: {} on: {}", nextViewId.get(), pendingView.size(), params.member().getId()); + log.info("Pending context for view: {} size: {} on: {}", + nextViewId.get() == null ? "" : nextViewId.get(), pendingView.size(), + params.member().getId()); } @Override diff --git a/choam/src/main/java/com/salesforce/apollo/choam/Session.java b/choam/src/main/java/com/salesforce/apollo/choam/Session.java index 99aa48076..31c09603e 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/Session.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/Session.java @@ -91,7 +91,7 @@ public static CompletableFuture retryNesting(Supplier { - log.warn("resubmitting, next attempt: {}", attempt); + log.trace("resubmitting after attempt: {}", attempt + 1); return supplier.get(); }).thenCompose(java.util.function.Function.identity()); } diff --git a/ethereal/src/main/java/com/salesforce/apollo/ethereal/Adder.java b/ethereal/src/main/java/com/salesforce/apollo/ethereal/Adder.java index be093a8ce..0d040d191 100644 --- a/ethereal/src/main/java/com/salesforce/apollo/ethereal/Adder.java +++ b/ethereal/src/main/java/com/salesforce/apollo/ethereal/Adder.java @@ -294,7 +294,7 @@ void commit(Digest digest, short member) { return; } - // Check for existing proposal + // Check for an existing proposal if (wpu == null) { log.trace("Committed, but no proposal: {} count: {} on: {}", digest, committed.size(), conf.logLabel()); return; diff --git a/ethereal/src/main/java/com/salesforce/apollo/ethereal/Ethereal.java b/ethereal/src/main/java/com/salesforce/apollo/ethereal/Ethereal.java index fc89abd77..f11310a96 100644 --- a/ethereal/src/main/java/com/salesforce/apollo/ethereal/Ethereal.java +++ b/ethereal/src/main/java/com/salesforce/apollo/ethereal/Ethereal.java @@ -68,6 +68,8 @@ private Ethereal(String label, Config conf, int maxSerializedSize, DataSource ds log.trace("Sending: {} on: {}", u, config.logLabel()); insert(u); }, epoch -> new epochProofImpl(config, epoch, new sharesDB(config, new ConcurrentHashMap<>()))); + + log.trace("Configured {} processes {}", config.nProc(), config.logLabel()); } private static ThreadPoolExecutor consumer(String label) { diff --git a/ethereal/src/main/java/com/salesforce/apollo/ethereal/memberships/ChRbcGossip.java b/ethereal/src/main/java/com/salesforce/apollo/ethereal/memberships/ChRbcGossip.java index ff8c048a4..69b7a0685 100644 --- a/ethereal/src/main/java/com/salesforce/apollo/ethereal/memberships/ChRbcGossip.java +++ b/ethereal/src/main/java/com/salesforce/apollo/ethereal/memberships/ChRbcGossip.java @@ -145,7 +145,7 @@ private Update gossipRound(Gossiper link, int ring) { */ private void handle(Optional result, RingCommunications.Destination destination, Duration duration, ScheduledExecutorService scheduler, Timer.Context timer) { - if (!started.get() || destination.link() == null) { + if (!started.get() || destination == null || destination.link() == null) { if (timer != null) { timer.stop(); } diff --git a/fireflies/src/main/java/com/salesforce/apollo/fireflies/View.java b/fireflies/src/main/java/com/salesforce/apollo/fireflies/View.java index a4153e0a9..7c9b702c8 100644 --- a/fireflies/src/main/java/com/salesforce/apollo/fireflies/View.java +++ b/fireflies/src/main/java/com/salesforce/apollo/fireflies/View.java @@ -1508,7 +1508,7 @@ public static BitSet createInitialMask(DynamicContext context) { int nbits = context.getRingCount(); BitSet mask = new BitSet(nbits); List random = new ArrayList<>(); - for (int i = 0; i < ((context.getBias() - 1) * context.toleranceLevel()) + 1; i++) { + for (int i = 0; i < context.majority(); i++) { random.add(true); } for (int i = 0; i < context.toleranceLevel(); i++) { diff --git a/grpc/src/main/proto/choam.proto b/grpc/src/main/proto/choam.proto index 4f7e5af9c..1cc17b0d0 100644 --- a/grpc/src/main/proto/choam.proto +++ b/grpc/src/main/proto/choam.proto @@ -154,6 +154,7 @@ message Validate { message Reassemble { repeated SignedViewMember members = 1; repeated Validate validations = 2; + repeated crypto.Digeste slate = 3; } message Validations { From 77d99b5b876df6c3ec18bc00ab92f933f763657f Mon Sep 17 00:00:00 2001 From: Hellblazer Date: Sat, 13 Apr 2024 20:30:52 -0700 Subject: [PATCH 04/10] Simplify genesis and view reconfiguration remove unnecessary signatures n' such. --- .../com/salesforce/apollo/choam/CHOAM.java | 35 ++-- .../apollo/choam/GenesisAssembly.java | 111 ++--------- .../com/salesforce/apollo/choam/Producer.java | 66 ++++--- .../salesforce/apollo/choam/ViewAssembly.java | 179 ++++-------------- .../salesforce/apollo/choam/ViewContext.java | 6 +- .../salesforce/apollo/choam/fsm/Genesis.java | 20 +- .../apollo/choam/support/TxDataSource.java | 4 +- .../apollo/choam/GenesisAssemblyTest.java | 4 +- grpc/src/main/proto/choam.proto | 10 +- 9 files changed, 134 insertions(+), 301 deletions(-) diff --git a/choam/src/main/java/com/salesforce/apollo/choam/CHOAM.java b/choam/src/main/java/com/salesforce/apollo/choam/CHOAM.java index a751be158..436bedeb3 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/CHOAM.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/CHOAM.java @@ -184,7 +184,7 @@ public static Checkpoint checkpoint(DigestAlgorithm algo, File state, int segmen return cp; } - public static Block genesis(Digest id, Map joins, HashedBlock head, Context context, + public static Block genesis(Digest id, Map joins, HashedBlock head, Context context, HashedBlock lastViewChange, Parameters params, HashedBlock lastCheckpoint, Iterable initialization) { var reconfigure = reconfigure(id, joins, context, params.checkpointBlockDelta()); @@ -202,26 +202,23 @@ public static Digest hashOf(Transaction transaction, DigestAlgorithm digestAlgor public static String print(Join join, DigestAlgorithm da) { return "J[view: " + Digest.from(join.getMember().getVm().getView()) + " member: " + ViewContext.print( - join.getMember(), da) + "certifications: " + join.getEndorsementsList() - .stream() - .map(c -> ViewContext.print(c, da)) - .toList() + "]"; + join.getMember(), da) + "]"; } - public static Reconfigure reconfigure(Digest nextViewId, Map joins, Context context, + public static Reconfigure reconfigure(Digest nextViewId, Map joins, Context context, int checkpointTarget) { var builder = Reconfigure.newBuilder().setCheckpointTarget(checkpointTarget).setId(nextViewId.toDigeste()); // Canonical labeling of the view members for Ethereal var remapped = rosterMap(context, joins.keySet()); - remapped.keySet().stream().sorted().map(remapped::get).forEach(m -> builder.addJoins(joins.get(m))); + remapped.keySet().stream().sorted().map(remapped::get).forEach(m -> builder.addJoins(joins.get(m.getId()))); var reconfigure = builder.build(); return reconfigure; } - public static Block reconfigure(Digest nextViewId, Map joins, HashedBlock head, + public static Block reconfigure(Digest nextViewId, Map joins, HashedBlock head, Context context, HashedBlock lastViewChange, Parameters params, HashedBlock lastCheckpoint) { final Block lvc = lastViewChange.block; @@ -237,10 +234,8 @@ public static Block reconfigure(Digest nextViewId, Map joins, Hash .build(); } - public static Map rosterMap(Context baseContext, Collection members) { - - // Canonical labeling of the view members for Ethereal - return members.stream().collect(Collectors.toMap(Member::getId, m -> m)); + public static Map rosterMap(Context baseContext, Collection members) { + return members.stream().collect(Collectors.toMap(m -> m, m -> baseContext.getMember(m))); } public static List toGenesisData(List initializationData) { @@ -497,11 +492,17 @@ public Block checkpoint() { } @Override - public Block genesis(Map joining, Digest nextViewId, HashedBlock previous) { + public Block genesis(Map joining, Digest nextViewId, HashedBlock previous) { final HashedCertifiedBlock cp = checkpoint.get(); final HashedCertifiedBlock v = view.get(); var g = CHOAM.genesis(nextViewId, joining, previous, params.context(), v, params, cp, - params.genesisData().apply(joining)); + params.genesisData() + .apply(joining.keySet() + .stream() + .map(m -> params.context().getMember(m)) + .filter(m -> m != null) + .collect( + Collectors.toMap(m -> m, m -> joining.get(m.getId()))))); log.info("Create genesis: {} on: {}", nextViewId, params.member().getId()); return g; } @@ -548,7 +549,7 @@ public void publish(Digest hash, CertifiedBlock cb) { } @Override - public Block reconfigure(Map joining, Digest nextViewId, HashedBlock previous, + public Block reconfigure(Map joining, Digest nextViewId, HashedBlock previous, HashedBlock checkpoint) { final HashedCertifiedBlock v = view.get(); var block = CHOAM.reconfigure(nextViewId, joining, previous, pendingView().get(), v, params, @@ -1008,7 +1009,7 @@ private void synchronizedProcess(CertifiedBlock certifiedBlock) { public interface BlockProducer { Block checkpoint(); - Block genesis(Map joining, Digest nextViewId, HashedBlock previous); + Block genesis(Map joining, Digest nextViewId, HashedBlock previous); void onFailure(); @@ -1018,7 +1019,7 @@ public interface BlockProducer { void publish(Digest hash, CertifiedBlock cb); - Block reconfigure(Map joining, Digest nextViewId, HashedBlock previous, HashedBlock checkpoint); + Block reconfigure(Map joining, Digest nextViewId, HashedBlock previous, HashedBlock checkpoint); } @FunctionalInterface diff --git a/choam/src/main/java/com/salesforce/apollo/choam/GenesisAssembly.java b/choam/src/main/java/com/salesforce/apollo/choam/GenesisAssembly.java index 9b004deb7..9ff7eaae3 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/GenesisAssembly.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/GenesisAssembly.java @@ -44,19 +44,19 @@ * @author hal.hildebrand */ public class GenesisAssembly implements Genesis { - private static final Logger log = LoggerFactory.getLogger(GenesisAssembly.class); + private static final Logger log = LoggerFactory.getLogger(GenesisAssembly.class); private final Ethereal controller; private final ChRbcGossip coordinator; private final SignedViewMember genesisMember; private final Map nextAssembly; - private final Map proposals = new ConcurrentHashMap<>(); - private final AtomicBoolean published = new AtomicBoolean(); - private final Map slate = new ConcurrentHashMap<>(); - private final AtomicBoolean started = new AtomicBoolean(); + private final AtomicBoolean published = new AtomicBoolean(); + private final Map slate = new ConcurrentHashMap<>(); + private final AtomicBoolean started = new AtomicBoolean(); private final Transitions transitions; private final ViewContext view; - private final Map witnesses = new ConcurrentHashMap<>(); + private final Map witnesses = new ConcurrentHashMap<>(); private final OneShot ds; + private final List pendingValidations = new ArrayList<>(); private volatile Thread blockingThread; private volatile HashedBlock reconfiguration; @@ -104,13 +104,9 @@ public GenesisAssembly(ViewContext vc, CommonCommunications comms, @Override public void certify() { - proposals.values() - .stream() - .filter(p -> p.certifications.size() == nextAssembly.size()) - .forEach(p -> slate.put(p.member(), joinOf(p))); if (slate.size() != nextAssembly.size()) { log.info("Not certifying genesis for: {} slate incomplete: {} on: {}", view.context().getId(), - slate.keySet().stream().map(m -> m.getId()).toList(), params().member().getId()); + slate.keySet().stream().sorted().toList(), params().member().getId()); return; } assert slate.size() == nextAssembly.size() : "Expected: %s members, slate: %s".formatted(nextAssembly.size(), @@ -120,8 +116,10 @@ public void certify() { params().digestAlgorithm()))); var validate = view.generateValidation(reconfiguration); log.debug("Certifying genesis block: {} for: {} slate: {} on: {}", reconfiguration.hash, view.context().getId(), - slate.keySet().stream().map(m -> m.getId()).toList(), params().member().getId()); + slate.keySet().stream().sorted().toList(), params().member().getId()); ds.setValue(validate.toByteString()); + witnesses.put(params().member(), validate); + pendingValidations.forEach(v -> certify(v)); } @Override @@ -139,15 +137,8 @@ public void certify(List preblock, boolean last) { @Override public void gather() { log.info("Gathering next assembly on: {}", params().member().getId()); - var certification = view.generateValidation(genesisMember).getWitness(); - var join = Join.newBuilder() - .setMember(genesisMember) - .addEndorsements(certification) - .setKerl(params().kerl().get()) - .build(); - var proposed = new Proposed(join, params().member()); - proposed.certifications.put(params().member(), certification); - proposals.put(params().member().getId(), proposed); + var join = Join.newBuilder().setMember(genesisMember).setKerl(params().kerl().get()).build(); + slate.put(params().member().getId(), join); ds.setValue(join.toByteString()); coordinator.start(params().producer().gossipDuration()); @@ -172,19 +163,6 @@ public void gather(List preblock, boolean last) { .forEach(this::join); } - @Override - public void nominate() { - var validations = Validations.newBuilder(); - proposals.values() - .stream() - .filter(p -> !p.member.equals(params().member())) - .map(p -> view.generateValidation(p.join.getMember())) - .forEach(validations::addValidations); - ds.setValue(validations.build().toByteString()); - log.info("Nominations of: {} validations: {} on: {}", params().context().getId(), - validations.getValidationsCount(), params().member().getId()); - } - @Override public void nominations(List preblock, boolean last) { preblock.stream() @@ -198,8 +176,7 @@ public void nominations(List preblock, boolean last) { }) .filter(Objects::nonNull) .flatMap(vs -> vs.getValidationsList().stream()) - .filter(v -> !v.equals(Validate.getDefaultInstance())) - .forEach(this::validate); + .filter(v -> !v.equals(Validate.getDefaultInstance())); } @Override @@ -258,6 +235,9 @@ public void stop() { } private void certify(Validate v) { + if (reconfiguration == null) { + pendingValidations.add(v); + } log.trace("Validating reconfiguration block: {} height: {} on: {}", reconfiguration.hash, reconfiguration.height(), params().member().getId()); if (!view.validate(reconfiguration, v)) { @@ -326,64 +306,15 @@ private void join(Join join) { ViewContext.print(svm, params().digestAlgorithm()), params().member().getId()); return; } - if (log.isTraceEnabled()) { - log.trace("Valid view member: {} on: {}", ViewContext.print(svm, params().digestAlgorithm()), - params().member().getId()); - } - var proposed = proposals.computeIfAbsent(mid, k -> new Proposed(join, m)); - if (join.getEndorsementsList().size() == 1) { - proposed.certifications.computeIfAbsent(m, k -> join.getEndorsements(0)); + if (slate.putIfAbsent(m.getId(), join) == null) { + if (log.isTraceEnabled()) { + log.trace("Add view member: {} to slate on: {}", ViewContext.print(svm, params().digestAlgorithm()), + params().member().getId()); + } } } - private Join joinOf(Proposed candidate) { - final List witnesses = candidate.certifications.values() - .stream() - .sorted( - Comparator.comparing(c -> new Digest(c.getId()))) - .collect(Collectors.toList()); - return Join.newBuilder(candidate.join).clearEndorsements().addAllEndorsements(witnesses).build(); - } - private Parameters params() { return view.params(); } - - private void validate(Validate v) { - final var cid = Digest.from(v.getWitness().getId()); - var certifier = view.context().getMember(cid); - if (certifier == null) { - log.warn("Unknown certifier: {} on: {}", cid, params().member().getId()); - return; // do not have the join yet - } - final var vid = Digest.from(v.getHash()); - final var member = nextAssembly.get(vid); - if (member == null) { - return; - } - var proposed = proposals.get(vid); - if (proposed == null) { - log.warn("Invalid certification, unknown view join: {} on: {}", vid, params().member().getId()); - return; // do not have the join yet - } - if (!view.validate(proposed.join.getMember(), v)) { - log.warn("Invalid certification for view join: {} from: {} on: {}", vid, - Digest.from(v.getWitness().getId()), params().member().getId()); - return; - } - var prev = proposed.certifications.put(certifier, v.getWitness()); - if (prev == null) { - log.debug("New validation of view member: {} using certifier: {} witnesses: {} on: {}", member.getId(), - certifier.getId(), proposed.certifications.values().size(), params().member().getId()); - } else { - log.debug("Redundant validation of view member: {} hash: {} using certifier: {} on: {}", member.getId(), - vid, certifier.getId(), params().member().getId()); - } - } - - private record Proposed(Join join, Member member, Map certifications) { - public Proposed(Join join, Member member) { - this(join, member, new HashMap<>()); - } - } } diff --git a/choam/src/main/java/com/salesforce/apollo/choam/Producer.java b/choam/src/main/java/com/salesforce/apollo/choam/Producer.java index 0ff834cb1..dd6954b33 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/Producer.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/Producer.java @@ -21,6 +21,7 @@ import com.salesforce.apollo.choam.support.TxDataSource; import com.salesforce.apollo.cryptography.Digest; import com.salesforce.apollo.cryptography.DigestAlgorithm; +import com.salesforce.apollo.cryptography.JohnHancock; import com.salesforce.apollo.ethereal.Config; import com.salesforce.apollo.ethereal.Config.Builder; import com.salesforce.apollo.ethereal.Ethereal; @@ -30,10 +31,8 @@ import org.slf4j.LoggerFactory; import java.util.*; -import java.util.concurrent.BlockingQueue; import java.util.concurrent.ConcurrentSkipListMap; import java.util.concurrent.CopyOnWriteArrayList; -import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; @@ -53,9 +52,9 @@ public class Producer { private final ChRbcGossip coordinator; private final TxDataSource ds; private final int lastEpoch; - private final Set nextAssembly = new HashSet<>(); + private final Map nextAssembly = new HashMap<>(); private final Map pending = new ConcurrentSkipListMap<>(); - private final BlockingQueue pendingReassembles = new LinkedBlockingQueue<>(); + private final List pendingJoins = new CopyOnWriteArrayList<>(); private final Map> pendingValidations = new ConcurrentSkipListMap<>(); private final AtomicReference previousBlock = new AtomicReference<>(); private final AtomicBoolean reconfigured = new AtomicBoolean(); @@ -161,9 +160,11 @@ public SubmitResult submit(Transaction transaction) { } private void addReassemble(Reassemble r) { - log.trace("Adding reassembly members: {} validations: {} on: {}", r.getMembersCount(), r.getValidationsCount(), - params().member().getId()); - ds.offer(r); + if (ds.offer(r)) { + log.trace("Adding joins: {} on: {}", r.getMembersList(), params().member().getId()); + } else { + log.trace("Cannot add joins: {} on: {}", r.getMembersCount(), params().member().getId()); + } } private void create(List preblock, boolean last) { @@ -188,19 +189,14 @@ private void create(List preblock, boolean last) { .filter(p -> p.witnesses.size() >= params().majority()) .forEach(this::publish); - var reass = Reassemble.newBuilder(); - aggregate.stream() - .flatMap(e -> e.getReassembliesList().stream()) - .forEach(r -> reass.addAllMembers(r.getMembersList()).addAllValidations(r.getValidationsList())); + var joins = aggregate.stream().flatMap(e -> e.getJoinsList().stream()).filter(j -> validate(j)).toList(); final var ass = assembly.get(); if (ass != null) { - log.trace("Consuming reassemblies: {} members: {} validations: {} on: {}", aggregate.size(), - reass.getMembersCount(), reass.getValidationsCount(), params().member().getId()); - ass.inbound().accept(Collections.singletonList(reass.build())); + log.trace("Consuming joins: {} on: {}", aggregate.size(), joins.size(), params().member().getId()); + ass.inbound().accept(joins); } else { - log.trace("Pending reassemblies: {} members: {} validations: {} on: {}", aggregate.size(), - reass.getMembersCount(), reass.getValidationsCount(), params().member().getId()); - pendingReassembles.add(reass.build()); + log.trace("Pending joins: {} on: {}", aggregate.size(), joins.size(), params().member().getId()); + pendingJoins.addAll(joins); } HashedBlock lb = previousBlock.get(); @@ -261,7 +257,9 @@ private void processPendingValidations(HashedBlock block, PendingBlock p) { private void produceAssemble() { final var vlb = previousBlock.get(); nextViewId = vlb.hash; - nextAssembly.addAll(Committee.viewMembersOf(nextViewId, view.pendingView())); + for (var m : Committee.viewMembersOf(nextViewId, view.pendingView())) { + nextAssembly.put(m.getId(), m); + } log.debug("Assembling: {} on: {}", nextViewId, params().member().getId()); final var assemble = new HashedBlock(params().digestAlgorithm(), view.produce(vlb.height().add(1), vlb.hash, Assemble.newBuilder() @@ -293,6 +291,28 @@ private void publish(PendingBlock p) { view.publish(new HashedCertifiedBlock(params().digestAlgorithm(), cb)); } + private boolean validate(SignedJoin join) { + var mid = Digest.from(join.getMember()); + var m = nextAssembly.get(mid); + if (m == null) { + log.trace("Cannot validate join view: {} of: {} signed by: {} on: {}", + Digest.from(join.getJoin().getVm().getView()), Digest.from(join.getJoin().getVm().getId()), mid, + params().member().getId()); + return false; + } + var validated = m.verify(JohnHancock.from(join.getSignature()), join.getJoin().toByteString()); + if (!validated) { + log.trace("Cannot validate view join: {} of: {} signed by: {} on: {}", + Digest.from(join.getJoin().getVm().getView()), Digest.from(join.getJoin().getVm().getId()), mid, + params().member().getId()); + } else { + log.trace("Validated view join: {} of: {} signed by: {} on: {}", + Digest.from(join.getJoin().getVm().getView()), Digest.from(join.getJoin().getVm().getId()), mid, + params().member().getId()); + } + return validated; + } + private PendingBlock validate(Validate v) { Digest hash = Digest.from(v.getHash()); var p = pending.get(hash); @@ -336,8 +356,8 @@ public void assembled() { ds.offer(validation); // controller.completeIt(); log.info("Produced: {} hash: {} height: {} slate: {} on: {}", reconfiguration.block.getBodyCase(), - reconfiguration.hash, reconfiguration.height(), - slate.keySet().stream().map(m -> m.getId()).sorted().toList(), params().member().getId()); + reconfiguration.hash, reconfiguration.height(), slate.keySet().stream().sorted().toList(), + params().member().getId()); processPendingValidations(reconfiguration, p); } @@ -420,9 +440,9 @@ public void complete() { }); assembly.get().start(); assembly.get().assembled(); - List reasses = new ArrayList<>(); - pendingReassembles.drainTo(reasses); - assembly.get().inbound().accept(reasses); + var joins = new ArrayList<>(pendingJoins); + pendingJoins.clear(); + assembly.get().inbound().accept(joins); } @Override diff --git a/choam/src/main/java/com/salesforce/apollo/choam/ViewAssembly.java b/choam/src/main/java/com/salesforce/apollo/choam/ViewAssembly.java index 105d0d98d..1cfc03a25 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/ViewAssembly.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/ViewAssembly.java @@ -12,7 +12,10 @@ import com.salesforce.apollo.choam.fsm.Reconfiguration; import com.salesforce.apollo.choam.fsm.Reconfiguration.Reconfigure; import com.salesforce.apollo.choam.fsm.Reconfiguration.Transitions; -import com.salesforce.apollo.choam.proto.*; +import com.salesforce.apollo.choam.proto.Join; +import com.salesforce.apollo.choam.proto.Reassemble; +import com.salesforce.apollo.choam.proto.SignedJoin; +import com.salesforce.apollo.choam.proto.SignedViewMember; import com.salesforce.apollo.context.Context; import com.salesforce.apollo.cryptography.Digest; import com.salesforce.apollo.cryptography.JohnHancock; @@ -26,7 +29,10 @@ import java.security.PublicKey; import java.time.Duration; import java.util.*; -import java.util.concurrent.*; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentSkipListMap; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; @@ -48,10 +54,9 @@ public class ViewAssembly { protected final Transitions transitions; private final AtomicBoolean cancelSlice = new AtomicBoolean(); private final Digest nextViewId; - private final Map proposals = new ConcurrentHashMap<>(); + private final Map proposals = new ConcurrentHashMap<>(); private final Consumer publisher; - private final Map slate = new ConcurrentSkipListMap<>(); - private final Map> unassigned = new ConcurrentHashMap<>(); + private final Map slate = new ConcurrentSkipListMap<>(); private final ViewContext view; private final CommonCommunications comms; private final Set polled = Collections.newSetFromMap( @@ -77,7 +82,7 @@ public ViewAssembly(Digest nextViewId, ViewContext vc, Consumer publ nextAssembly.keySet(), params().member().getId()); } - public Map getSlate() { + public Map getSlate() { return slate; } @@ -95,34 +100,21 @@ void assembled() { void complete() { cancelSlice.set(true); - proposals.values() + proposals.entrySet() .stream() - .filter(p -> p.validations.size() >= params().majority()) - .forEach(p -> slate.put(p.member(), joinOf(p))); + .forEach(e -> slate.put(e.getKey(), Join.newBuilder().setMember(e.getValue()).build())); Context pendingContext = view.pendingView(); - if (slate.size() >= pendingContext.majority()) { - log.debug("View Assembly: {} completed with: {} members on: {}", nextViewId, slate.size(), - params().member().getId()); - } else { - log.debug("Failed view assembly completion, election required: {} slate: {} of: {} on: {}", - pendingContext.majority(), proposals.values() - .stream() - .map(p -> String.format("%s:%s", p.member.getId(), - p.validations.size())) - .sorted() - .toList(), nextViewId, params().member().getId()); - transitions.complete(); - } + log.debug("View Assembly: {} completed with: {} members on: {}", nextViewId, slate.size(), + params().member().getId()); } void finalElection() { transitions.complete(); } - Consumer> inbound() { + Consumer> inbound() { return lre -> { - lre.stream().flatMap(re -> re.getMembersList().stream()).forEach(vm -> join(vm, false)); - lre.stream().flatMap(re -> re.getValidationsList().stream()).forEach(this::validate); + lre.forEach(vm -> join(vm.getJoin(), false)); }; } @@ -152,7 +144,6 @@ private boolean consider(Optional futureSailor, Terminal term, } SignedViewMember signedViewMember; signedViewMember = futureSailor.get(); - log.debug("Join reply from: {} on: {}", term.getMember().getId(), params().member().getId()); if (signedViewMember.equals(SignedViewMember.getDefaultInstance())) { log.debug("Empty join response from: {} on: {}", term.getMember().getId(), params().member().getId()); return !gathered(); @@ -163,6 +154,7 @@ private boolean consider(Optional futureSailor, Terminal term, params().member().getId()); return !gathered(); } + log.debug("Join reply from: {} on: {}", term.getMember().getId(), params().member().getId()); join(signedViewMember, true); return !gathered(); } @@ -176,14 +168,6 @@ private boolean gathered() { return false; } - private Reassemble getMemberProposal() { - return Reassemble.newBuilder() - .addAllMembers(proposals.values().stream().map(p -> p.vm).toList()) - .addAllValidations( - proposals.values().stream().flatMap(p -> p.validations.values().stream()).toList()) - .build(); - } - private void join(SignedViewMember svm, boolean direct) { final var mid = Digest.from(svm.getVm().getId()); final var m = nextAssembly.get(mid); @@ -233,117 +217,38 @@ private void join(SignedViewMember svm, boolean direct) { } return; } - AtomicBoolean newJoin = new AtomicBoolean(); - - var proposed = proposals.computeIfAbsent(mid, k -> { - newJoin.set(true); - return new Proposed(svm, m, new ConcurrentSkipListMap<>()); - }); - - var builder = Reassemble.newBuilder(); - proposed.validations.computeIfAbsent(params().member(), k -> { - var validate = view.generateValidation(svm); - builder.addValidations(validate); - return validate; - }); - - if (newJoin.get()) { + if (proposals.putIfAbsent(mid, svm) == null) { if (log.isTraceEnabled()) { log.trace("Adding view member: {} on: {}", ViewContext.print(svm, params().digestAlgorithm()), params().member().getId()); } if (direct) { - builder.addMembers(svm); - } - var validations = unassigned.remove(mid); - if (validations != null) { - validations.forEach(this::validate); + publisher.accept(Reassemble.newBuilder().addMembers(svm).build()); } } polled.add(mid); - var reass = builder.build(); - if (reass.isInitialized()) { - publisher.accept(reass); - } - } - - private Join joinOf(Proposed candidate) { - final List witnesses = candidate.validations.values() - .stream() - .map(Validate::getWitness) - .sorted( - Comparator.comparing(c -> new Digest(c.getId()))) - .toList(); - return Join.newBuilder().setMember(candidate.vm).addAllEndorsements(witnesses).build(); } private Parameters params() { return view.params(); } - private void validate(Validate v) { - final var cid = Digest.from(v.getWitness().getId()); - var certifier = view.context().getMember(cid); - if (certifier == null) { - log.warn("Unknown certifier: {} on: {}", cid, params().member().getId()); - return; - } - final var digest = Digest.from(v.getHash()); - final var member = nextAssembly.get(digest); - if (member == null) { - log.warn("Unknown next view member: {} on: {}", digest, params().member().getId()); - return; - } - var proposed = proposals.get(digest); - if (proposed == null) { - log.warn("Unassigned certification, unknown view join: {} on: {}", digest, params().member().getId()); - unassigned.computeIfAbsent(digest, d -> new CopyOnWriteArrayList<>()).add(v); - return; - } - if (!view.validate(proposed.vm, v)) { - log.warn("Invalid certification for view join: {} from: {} on: {}", digest, - Digest.from(v.getWitness().getId()), params().member().getId()); - return; - } - var newCertifier = new AtomicBoolean(); - proposed.validations.computeIfAbsent(certifier, k -> { - log.debug("Validation of view member: {}:{} using certifier: {} on: {}", member.getId(), digest, - certifier.getId(), params().member().getId()); - newCertifier.set(true); - return v; - }); - if (newCertifier.get()) { - transitions.validation(); - } - } - - private record Proposed(SignedViewMember vm, Member member, Map validations) { + private record Proposed(SignedViewMember vm, Member member) { } private class Recon implements Reconfiguration { @Override public void certify() { - var certified = proposals.entrySet() - .stream() - .filter(p -> p.getValue().validations.size() >= params().majority()) - .map(Map.Entry::getKey) - .sorted() - .toList(); - Context memberContext = view.pendingView(); - var required = memberContext.majority(); - if (certified.size() >= required) { + if (proposals.size() == nextAssembly.size()) { cancelSlice.set(true); - log.debug("Certifying: {} required: {} of: {} slate: {} on: {}", certified.size(), required, - nextViewId, certified, params().member().getId()); + log.debug("Certifying: {} required: {} of: {} slate: {} on: {}", proposals.size(), nextAssembly.size(), + nextViewId, proposals.keySet().stream().sorted().toList(), params().member().getId()); transitions.certified(); } else { - log.debug("Not certifying: {} required: {} slate: {} of: {} on: {}", certified.size(), required, - proposals.entrySet() - .stream() - .map(e -> String.format("%s:%s", e.getKey(), e.getValue().validations.size())) - .sorted() - .toList(), nextViewId, params().member().getId()); + log.debug("Not certifying: {} required: {} slate: {} of: {} on: {}", proposals.size(), + nextAssembly.size(), proposals.entrySet().stream().sorted().toList(), nextViewId, + params().member().getId()); } } @@ -354,33 +259,17 @@ public void complete() { @Override public void elect() { - proposals.values() - .stream() - .filter(p -> p.validations.size() >= params().majority()) - .sorted(Comparator.comparing(p -> p.member.getId())) - .forEach(p -> slate.put(p.member(), joinOf(p))); - if (slate.size() >= view.pendingView().majority()) { + proposals.entrySet().stream().forEach(e -> slate.put(e.getKey(), joinOf(e.getValue()))); + if (slate.size() == view.pendingView().getRingCount()) { cancelSlice.set(true); log.debug("Electing: {} of: {} slate: {} proposals: {} on: {}", slate.size(), nextViewId, - slate.keySet().stream().map(Member::getId).sorted().toList(), proposals.values() - .stream() - .map( - p -> String.format( - "%s:%s", - p.member.getId(), - p.validations.size())) - .sorted() - .toList(), + slate.keySet().stream().sorted().toList(), proposals.keySet().stream().sorted().toList(), params().member().getId()); transitions.complete(); } else { Context memberContext = view.pendingView(); log.error("Failed election, required: {} slate: {} of: {} on: {}", memberContext.majority(), - proposals.values() - .stream() - .map(p -> String.format("%s:%s", p.member.getId(), p.validations.size())) - .sorted() - .toList(), nextViewId, params().member().getId()); + proposals.keySet().stream().sorted().toList(), nextViewId, params().member().getId()); } } @@ -415,8 +304,12 @@ public void gather() { @Override public void nominate() { - publisher.accept(getMemberProposal()); + // publisher.accept(getMemberProposal()); transitions.nominated(); } + + private Join joinOf(SignedViewMember vm) { + return Join.newBuilder().setMember(vm).build(); + } } } diff --git a/choam/src/main/java/com/salesforce/apollo/choam/ViewContext.java b/choam/src/main/java/com/salesforce/apollo/choam/ViewContext.java index c06aedb9a..1a64d79ae 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/ViewContext.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/ViewContext.java @@ -47,7 +47,7 @@ public ViewContext(Context context, Parameters params, Supplier m.getId()).toList()); short pid = 0; for (Digest d : remapped.keySet().stream().sorted().toList()) { roster.put(remapped.get(d).getId(), pid++); @@ -121,7 +121,7 @@ public Validate generateValidation(SignedViewMember svm) { return validation; } - public Block genesis(Map slate, Digest nextViewId, HashedBlock previous) { + public Block genesis(Map slate, Digest nextViewId, HashedBlock previous) { return blockProducer.genesis(slate, nextViewId, previous); } @@ -156,7 +156,7 @@ public void publish(HashedCertifiedBlock block) { blockProducer.publish(block.hash, block.certifiedBlock); } - public Block reconfigure(Map aggregate, Digest nextViewId, HashedBlock lastBlock, + public Block reconfigure(Map aggregate, Digest nextViewId, HashedBlock lastBlock, HashedBlock checkpoint) { return blockProducer.reconfigure(aggregate, nextViewId, lastBlock, checkpoint); } diff --git a/choam/src/main/java/com/salesforce/apollo/choam/fsm/Genesis.java b/choam/src/main/java/com/salesforce/apollo/choam/fsm/Genesis.java index c8323c15c..0cdf3ac47 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/fsm/Genesis.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/fsm/Genesis.java @@ -24,8 +24,6 @@ public interface Genesis { void gather(List preblock, boolean last); - void nominate(); - void nominations(List preblock, boolean last); void publish(); @@ -52,7 +50,7 @@ public void gather() { @Override public Transitions nextEpoch(Integer epoch) { - return epoch.equals(0) ? null : NOMINATION; + return epoch.equals(0) ? null : CERTIFICATION; } @@ -61,22 +59,6 @@ public Transitions process(List preblock, boolean last) { context().gather(preblock, last); return null; } - }, NOMINATION { - @Override - public Transitions nextEpoch(Integer epoch) { - return CERTIFICATION; - } - - @Entry - public void nominate() { - context().nominate(); - } - - @Override - public Transitions process(List preblock, boolean last) { - context().nominations(preblock, last); - return null; - } }, PUBLISH { @Entry public void publish() { diff --git a/choam/src/main/java/com/salesforce/apollo/choam/support/TxDataSource.java b/choam/src/main/java/com/salesforce/apollo/choam/support/TxDataSource.java index 775a01a5a..5acb7c4cb 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/support/TxDataSource.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/support/TxDataSource.java @@ -156,8 +156,8 @@ public int getRemainingValidations() { return validations.size(); } - public void offer(Reassemble reassembly) { - reassemblies.offer(reassembly); + public boolean offer(Reassemble reassembly) { + return reassemblies.offer(reassembly); } public boolean offer(Transaction txn) { diff --git a/choam/src/test/java/com/salesforce/apollo/choam/GenesisAssemblyTest.java b/choam/src/test/java/com/salesforce/apollo/choam/GenesisAssemblyTest.java index 974363d66..83d441916 100644 --- a/choam/src/test/java/com/salesforce/apollo/choam/GenesisAssemblyTest.java +++ b/choam/src/test/java/com/salesforce/apollo/choam/GenesisAssemblyTest.java @@ -137,7 +137,7 @@ public Block checkpoint() { } @Override - public Block genesis(Map joining, Digest nextViewId, HashedBlock previous) { + public Block genesis(Map joining, Digest nextViewId, HashedBlock previous) { return CHOAM.genesis(viewId, joining, previous, committee, previous, built, previous, Collections.emptyList()); } @@ -163,7 +163,7 @@ public void publish(Digest hash, CertifiedBlock cb) { } @Override - public Block reconfigure(Map joining, Digest nextViewId, HashedBlock previous, + public Block reconfigure(Map joining, Digest nextViewId, HashedBlock previous, HashedBlock checkpoint) { return null; } diff --git a/grpc/src/main/proto/choam.proto b/grpc/src/main/proto/choam.proto index 1cc17b0d0..5c5c7416c 100644 --- a/grpc/src/main/proto/choam.proto +++ b/grpc/src/main/proto/choam.proto @@ -104,12 +104,18 @@ message UnitData { repeated Validate validations = 1; repeated Transaction transactions = 2; repeated Reassemble reassemblies = 3; + repeated SignedJoin joins = 4; } message Join { SignedViewMember member = 1; - repeated Certification endorsements = 2; - stereotomy.KERL_ kerl = 3; + stereotomy.KERL_ kerl = 2; +} + +message SignedJoin { + crypto.Digeste member = 1; + SignedViewMember join = 2; + crypto.Sig signature = 3; } message ViewMember { From 45399c038c6d2d97cb619b17a18c73aefddd633c Mon Sep 17 00:00:00 2001 From: Hellblazer Date: Sun, 14 Apr 2024 17:40:17 -0700 Subject: [PATCH 05/10] Further simplification of View Reassembly. Remove ASSEMBLE block type. Further clean up of view assembly --- choam/pom.xml | 4 + .../com/salesforce/apollo/choam/CHOAM.java | 59 ++++---- .../salesforce/apollo/choam/Committee.java | 3 - .../com/salesforce/apollo/choam/Producer.java | 130 +++++------------- .../salesforce/apollo/choam/ViewAssembly.java | 51 +++---- .../salesforce/apollo/choam/ViewContext.java | 56 +++++++- .../salesforce/apollo/choam/fsm/Driven.java | 28 +--- .../apollo/choam/support/ChoamMetrics.java | 2 +- .../choam/support/ChoamMetricsImpl.java | 8 +- .../apollo/choam/support/TxDataSource.java | 44 +++--- .../apollo/choam/GenesisAssemblyTest.java | 5 - grpc/src/main/proto/choam.proto | 14 +- .../salesforce/apollo/model/DomainTest.java | 2 +- .../apollo/model/FireFliesTest.java | 2 +- model/src/test/resources/logback-test.xml | 2 +- pom.xml | 5 + 16 files changed, 175 insertions(+), 240 deletions(-) diff --git a/choam/pom.xml b/choam/pom.xml index d4c07c80e..1a91f65b4 100644 --- a/choam/pom.xml +++ b/choam/pom.xml @@ -30,6 +30,10 @@ org.jooq joou + + com.googlecode.concurrentlinkedhashmap + concurrentlinkedhashmap-lru + diff --git a/choam/src/main/java/com/salesforce/apollo/choam/CHOAM.java b/choam/src/main/java/com/salesforce/apollo/choam/CHOAM.java index 436bedeb3..d05256ed5 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/CHOAM.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/CHOAM.java @@ -11,6 +11,7 @@ import com.google.protobuf.ByteString; import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.Message; +import com.googlecode.concurrentlinkedhashmap.ConcurrentLinkedHashMap; import com.salesforce.apollo.archipelago.RouterImpl.CommonCommunications; import com.salesforce.apollo.bloomFilters.BloomFilter; import com.salesforce.apollo.choam.comm.*; @@ -95,11 +96,15 @@ public class CHOAM { private final TransSubmission txnSubmission = new TransSubmission(); private final AtomicReference view = new AtomicReference<>(); private final AtomicReference> pendingView = new AtomicReference<>(); + private final ConcurrentLinkedHashMap> pendingViews; public CHOAM(Parameters params) { this.store = new Store(params.digestAlgorithm(), params.mvBuilder().clone().build()); this.params = params; executions = Executors.newVirtualThreadPerTaskExecutor(); + pendingViews = new ConcurrentLinkedHashMap.Builder>().maximumWeightedCapacity(100) + .initialCapacity(10) + .build(); nextView(); var bContext = new DelegatedContext<>(params.context()); @@ -321,10 +326,15 @@ public void nextView(Context context, Digest diadem) { if (c != null) { c.nextView(context); } else { - log.info("Acquiring new view, diadem: {} size: {} on: {}", diadem, context.size(), params.member().getId()); + log.info("Acquiring new view of: {}, diadem: {} size: {} on: {}", context.getId(), diadem, context.size(), + params.member().getId()); params.context().setContext(context); pendingView.set(null); } + + log.info("Pushing pending view of: {}, diadem: {} size: {} on: {}", context.getId(), diadem, context.size(), + params.member().getId()); + pendingViews.putIfAbsent(diadem, context); } public void start() { @@ -512,20 +522,6 @@ public void onFailure() { transitions.fail(); } - @Override - public Block produce(ULong height, Digest prev, Assemble assemble, HashedBlock checkpoint) { - final HashedCertifiedBlock v = view.get(); - var block = Block.newBuilder() - .setHeader( - buildHeader(params.digestAlgorithm(), assemble, prev, height, checkpoint.height(), - checkpoint.hash, v.height(), v.hash)) - .setAssemble(assemble) - .build(); - log.trace("Produced block: {} height: {} on: {}", block.getBodyCase(), block.getHeader().getHeight(), - params.member().getId()); - return block; - } - @Override public Block produce(ULong height, Digest prev, Executions executions, HashedBlock checkpoint) { final HashedCertifiedBlock v = view.get(); @@ -674,22 +670,19 @@ private Supplier> pendingView() { }; } + private Supplier>> pendingViews() { + return () -> pendingViews; + } + private void process() { final var c = current.get(); final HashedCertifiedBlock h = head.get(); log.info("Begin block: {} hash: {} height: {} committee: {} on: {}", h.block.getBodyCase(), h.hash, h.height(), c.getClass().getSimpleName(), params.member().getId()); switch (h.block.getBodyCase()) { - case ASSEMBLE: { - params.processor().beginBlock(h.height(), h.hash); - nextViewId.set(Digest.from(h.block.getAssemble().getNextView())); - log.info("Assembled next view id: {} on: {}", nextViewId.get(), params.member().getId()); - c.assembled(); - break; - } case RECONFIGURE: { params.processor().beginBlock(h.height(), h.hash); - reconfigure(h.block.getReconfigure()); + reconfigure(h.hash, h.block.getReconfigure()); break; } case GENESIS: { @@ -697,7 +690,7 @@ private void process() { cancelBootstrap(); transitions.regenerated(); genesisInitialization(h, h.block.getGenesis().getInitializeList()); - reconfigure(h.block.getGenesis().getInitialView()); + reconfigure(h.hash, h.block.getGenesis().getInitialView()); break; } case EXECUTIONS: { @@ -719,9 +712,9 @@ private void process() { params.member().getId()); } - private void reconfigure(Reconfigure reconfigure) { - log.info("Clearing next view id on: {}", params.member().getId()); - nextViewId.set(null); + private void reconfigure(Digest hash, Reconfigure reconfigure) { + log.info("Setting next view id: {} on: {}", hash, params.member().getId()); + nextViewId.set(hash); var pv = pendingView.getAndSet(null); if (pv != null) { // always advance view. @@ -741,7 +734,7 @@ private void reconfigure(Reconfigure reconfigure) { } else { log.warn("Reconfiguration to associate failed: {} in view: {} on:{}", validators.size(), new Digest(reconfigure.getId()), params.member().getId()); - current.set(new Client(validators, getViewId())); + transitions.fail(); } } else { current.set(new Client(validators, getViewId())); @@ -1013,8 +1006,6 @@ public interface BlockProducer { void onFailure(); - Block produce(ULong height, Digest prev, Assemble assemble, HashedBlock checkpoint); - Block produce(ULong height, Digest prev, Executions executions, HashedBlock checkpoint); void publish(Digest hash, CertifiedBlock cb); @@ -1306,16 +1297,12 @@ private class Associate extends Administration { params.member().getId()); Signer signer = new SignerImpl(nextView.consensusKeyPair.getPrivate(), ULong.MIN); Supplier> pv = pendingView(); - producer = new Producer(new ViewContext(context, params, pv, signer, validators, constructBlock()), + producer = new Producer(nextViewId.get(), + new ViewContext(context, params, pv, signer, validators, constructBlock()), head.get(), checkpoint.get(), comm, getLabel()); producer.start(); } - @Override - public void assembled() { - producer.assembled(); - } - @Override public void complete() { producer.stop(); diff --git a/choam/src/main/java/com/salesforce/apollo/choam/Committee.java b/choam/src/main/java/com/salesforce/apollo/choam/Committee.java index c7ad7fcf4..a9c86256c 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/Committee.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/Committee.java @@ -64,9 +64,6 @@ static Set viewMembersOf(Digest hash, Context baseContex void accept(HashedCertifiedBlock next); - default void assembled() { - } - void complete(); boolean isMember(); diff --git a/choam/src/main/java/com/salesforce/apollo/choam/Producer.java b/choam/src/main/java/com/salesforce/apollo/choam/Producer.java index dd6954b33..31219e253 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/Producer.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/Producer.java @@ -21,7 +21,6 @@ import com.salesforce.apollo.choam.support.TxDataSource; import com.salesforce.apollo.cryptography.Digest; import com.salesforce.apollo.cryptography.DigestAlgorithm; -import com.salesforce.apollo.cryptography.JohnHancock; import com.salesforce.apollo.ethereal.Config; import com.salesforce.apollo.ethereal.Config.Builder; import com.salesforce.apollo.ethereal.Ethereal; @@ -45,7 +44,6 @@ public class Producer { private static final Logger log = LoggerFactory.getLogger(Producer.class); private final AtomicBoolean assembled = new AtomicBoolean(); - private final AtomicReference assembly = new AtomicReference<>(); private final AtomicReference checkpoint = new AtomicReference<>(); private final CommonCommunications comms; private final Ethereal controller; @@ -61,15 +59,17 @@ public class Producer { private final AtomicBoolean started = new AtomicBoolean(false); private final Transitions transitions; private final ViewContext view; - private volatile Digest nextViewId; + private final Digest nextViewId; + private ViewAssembly assembly; - public Producer(ViewContext view, HashedBlock lastBlock, HashedBlock checkpoint, + public Producer(Digest nextViewId, ViewContext view, HashedBlock lastBlock, HashedBlock checkpoint, CommonCommunications comms, String label) { assert view != null; this.view = view; this.previousBlock.set(lastBlock); this.comms = comms; this.checkpoint.set(checkpoint); + this.nextViewId = nextViewId; final Parameters params = view.params(); final var producerParams = params.producer(); @@ -112,22 +112,14 @@ public Producer(ViewContext view, HashedBlock lastBlock, HashedBlock checkpoint, log.debug("Roster for: {} is: {} on: {}", getViewId(), view.roster(), params().member().getId()); } - public void assembled() { - transitions.assembled(); - } - - public Digest getNextViewId() { - final Digest current = nextViewId; - return current; - } - public void start() { if (!started.compareAndSet(false, true)) { return; } + reconfigure(); final Block prev = previousBlock.get().block; - if (prev.hasReconfigure() && prev.getReconfigure().getCheckpointTarget() == 0) { // genesis block won't ever be - // 0 + // genesis block won't ever be 0 + if (prev.hasReconfigure() && prev.getReconfigure().getCheckpointTarget() == 0) { transitions.checkpoint(); } else { transitions.start(); @@ -141,7 +133,7 @@ public void stop() { log.trace("Closing producer for: {} on: {}", getViewId(), params().member().getId()); controller.stop(); coordinator.stop(); - final var c = assembly.get(); + final var c = assembly; if (c != null) { c.stop(); } @@ -159,11 +151,11 @@ public SubmitResult submit(Transaction transaction) { } } - private void addReassemble(Reassemble r) { - if (ds.offer(r)) { - log.trace("Adding joins: {} on: {}", r.getMembersList(), params().member().getId()); + private void addJoin(SignedJoin signedJoin) { + if (ds.offer(signedJoin)) { + log.trace("Adding on: {}", params().member().getId()); } else { - log.trace("Cannot add joins: {} on: {}", r.getMembersCount(), params().member().getId()); + log.trace("Cannot add join on: {}", params().member().getId()); } } @@ -189,13 +181,13 @@ private void create(List preblock, boolean last) { .filter(p -> p.witnesses.size() >= params().majority()) .forEach(this::publish); - var joins = aggregate.stream().flatMap(e -> e.getJoinsList().stream()).filter(j -> validate(j)).toList(); - final var ass = assembly.get(); + var joins = aggregate.stream().flatMap(e -> e.getJoinsList().stream()).filter(view::validate).toList(); + final var ass = assembly; if (ass != null) { - log.trace("Consuming joins: {} on: {}", aggregate.size(), joins.size(), params().member().getId()); + log.trace("Consuming {} units, {} joins on: {}", aggregate.size(), joins.size(), params().member().getId()); ass.inbound().accept(joins); } else { - log.trace("Pending joins: {} on: {}", aggregate.size(), joins.size(), params().member().getId()); + log.trace("Pending {} units, {} joins on: {}", aggregate.size(), joins.size(), params().member().getId()); pendingJoins.addAll(joins); } @@ -254,30 +246,6 @@ private void processPendingValidations(HashedBlock block, PendingBlock p) { } } - private void produceAssemble() { - final var vlb = previousBlock.get(); - nextViewId = vlb.hash; - for (var m : Committee.viewMembersOf(nextViewId, view.pendingView())) { - nextAssembly.put(m.getId(), m); - } - log.debug("Assembling: {} on: {}", nextViewId, params().member().getId()); - final var assemble = new HashedBlock(params().digestAlgorithm(), view.produce(vlb.height().add(1), vlb.hash, - Assemble.newBuilder() - .setNextView( - vlb.hash.toDigeste()) - .build(), - checkpoint.get())); - previousBlock.set(assemble); - final var validation = view.generateValidation(assemble); - final var p = new PendingBlock(assemble, new HashMap<>(), new AtomicBoolean()); - pending.put(assemble.hash, p); - p.witnesses.put(params().member(), validation); - ds.offer(validation); - log.debug("Produced block: {} hash: {} height: {} from: {} on: {}", assemble.block.getBodyCase(), assemble.hash, - assemble.height(), getViewId(), params().member().getId()); - processPendingValidations(assemble, p); - } - private void publish(PendingBlock p) { assert p.witnesses.size() >= params().majority() : "Publishing non majority block"; log.debug("Published pending: {} hash: {} height: {} witnesses: {} on: {}", p.block.block.getBodyCase(), @@ -291,26 +259,23 @@ private void publish(PendingBlock p) { view.publish(new HashedCertifiedBlock(params().digestAlgorithm(), cb)); } - private boolean validate(SignedJoin join) { - var mid = Digest.from(join.getMember()); - var m = nextAssembly.get(mid); - if (m == null) { - log.trace("Cannot validate join view: {} of: {} signed by: {} on: {}", - Digest.from(join.getJoin().getVm().getView()), Digest.from(join.getJoin().getVm().getId()), mid, - params().member().getId()); - return false; - } - var validated = m.verify(JohnHancock.from(join.getSignature()), join.getJoin().toByteString()); - if (!validated) { - log.trace("Cannot validate view join: {} of: {} signed by: {} on: {}", - Digest.from(join.getJoin().getVm().getView()), Digest.from(join.getJoin().getVm().getId()), mid, - params().member().getId()); - } else { - log.trace("Validated view join: {} of: {} signed by: {} on: {}", - Digest.from(join.getJoin().getVm().getView()), Digest.from(join.getJoin().getVm().getId()), mid, - params().member().getId()); - } - return validated; + private void reconfigure() { + log.debug("Starting view reconfiguration: {} on: {}", nextViewId, params().member().getId()); + assembly = new ViewAssembly(nextViewId, view, Producer.this::addJoin, comms) { + @Override + public void complete() { + super.complete(); + log.debug("View reconfiguration: {} gathered: {} complete on: {}", nextViewId, getSlate().size(), + params().member().getId()); + assembled.set(true); + Producer.this.transitions.viewComplete(); + } + }; + assembly.start(); + assembly.assembled(); + var joins = new ArrayList<>(pendingJoins); + pendingJoins.clear(); + assembly.inbound().accept(joins); } private PendingBlock validate(Validate v) { @@ -345,7 +310,7 @@ public void assembled() { log.debug("assembly already complete on: {}", params().member().getId()); return; } - final var slate = assembly.get().getSlate(); + final var slate = assembly.getSlate(); var reconfiguration = new HashedBlock(params().digestAlgorithm(), view.reconfigure(slate, nextViewId, previousBlock.get(), checkpoint.get())); @@ -368,7 +333,7 @@ public void checkAssembly() { if (dropped != 0) { log.warn("Dropped txns: {} on: {}", dropped, params().member().getId()); } - final var viewAssembly = assembly.get(); + final var viewAssembly = assembly; if (viewAssembly == null) { log.error("Assemble block never processed on: {}", params().member().getId()); transitions.failed(); @@ -420,31 +385,6 @@ public void fail() { view.onFailure(); } - @Override - public void produceAssemble() { - Producer.this.produceAssemble(); - } - - @Override - public void reconfigure() { - log.debug("Starting view reconfiguration: {} on: {}", nextViewId, params().member().getId()); - assembly.set(new ViewAssembly(nextViewId, view, Producer.this::addReassemble, comms) { - @Override - public void complete() { - super.complete(); - log.debug("View reconfiguration: {} gathered: {} complete on: {}", nextViewId, getSlate().size(), - params().member().getId()); - assembled.set(true); - Producer.this.transitions.viewComplete(); - } - }); - assembly.get().start(); - assembly.get().assembled(); - var joins = new ArrayList<>(pendingJoins); - pendingJoins.clear(); - assembly.get().inbound().accept(joins); - } - @Override public void startProduction() { log.debug("Starting production for: {} on: {}", getViewId(), params().member().getId()); diff --git a/choam/src/main/java/com/salesforce/apollo/choam/ViewAssembly.java b/choam/src/main/java/com/salesforce/apollo/choam/ViewAssembly.java index 1cfc03a25..847aac3c7 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/ViewAssembly.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/ViewAssembly.java @@ -13,7 +13,6 @@ import com.salesforce.apollo.choam.fsm.Reconfiguration.Reconfigure; import com.salesforce.apollo.choam.fsm.Reconfiguration.Transitions; import com.salesforce.apollo.choam.proto.Join; -import com.salesforce.apollo.choam.proto.Reassemble; import com.salesforce.apollo.choam.proto.SignedJoin; import com.salesforce.apollo.choam.proto.SignedViewMember; import com.salesforce.apollo.context.Context; @@ -55,7 +54,7 @@ public class ViewAssembly { private final AtomicBoolean cancelSlice = new AtomicBoolean(); private final Digest nextViewId; private final Map proposals = new ConcurrentHashMap<>(); - private final Consumer publisher; + private final Consumer publisher; private final Map slate = new ConcurrentSkipListMap<>(); private final ViewContext view; private final CommonCommunications comms; @@ -63,7 +62,7 @@ public class ViewAssembly { new ConcurrentSkipListMap<>()); private volatile Map nextAssembly; - public ViewAssembly(Digest nextViewId, ViewContext vc, Consumer publisher, + public ViewAssembly(Digest nextViewId, ViewContext vc, Consumer publisher, CommonCommunications comms) { view = vc; this.nextViewId = nextViewId; @@ -118,23 +117,17 @@ Consumer> inbound() { }; } - private void completeSlice(AtomicReference retryDelay, AtomicReference reiterate) { + private void completeSlice(Duration retryDelay, AtomicReference reiterate) { if (gathered()) { return; } - final var delay = retryDelay.get(); - if (delay.compareTo(params().producer().maxGossipDelay()) < 0) { - retryDelay.accumulateAndGet(Duration.ofMillis(100), Duration::plus); - } - - log.trace("Proposal incomplete of: {} polled: {}, total: {} majority: {}, retrying: {} on: {}", nextViewId, - polled.stream().sorted().toList(), nextAssembly.size(), params().majority(), delay, - params().member().getId()); + log.trace("Proposal incomplete of: {} polled: {}, total: {} retrying: {} on: {}", nextViewId, + polled.stream().sorted().toList(), nextAssembly.size(), retryDelay, params().member().getId()); if (!cancelSlice.get()) { Executors.newScheduledThreadPool(1, Thread.ofVirtual().factory()) - .schedule(() -> Thread.ofVirtual().start(Utils.wrapped(reiterate.get(), log)), delay.toMillis(), - TimeUnit.MILLISECONDS); + .schedule(() -> Thread.ofVirtual().start(Utils.wrapped(reiterate.get(), log)), + retryDelay.toNanos(), TimeUnit.NANOSECONDS); } } @@ -217,16 +210,25 @@ private void join(SignedViewMember svm, boolean direct) { } return; } + polled.add(mid); if (proposals.putIfAbsent(mid, svm) == null) { - if (log.isTraceEnabled()) { - log.trace("Adding view member: {} on: {}", ViewContext.print(svm, params().digestAlgorithm()), - params().member().getId()); - } if (direct) { - publisher.accept(Reassemble.newBuilder().addMembers(svm).build()); + var sig = params().member().sign(svm.toByteString()).toSig(); + publisher.accept(SignedJoin.newBuilder() + .setJoin(svm) + .setMember(params().member().getId().toDigeste()) + .setSignature(sig) + .build()); + if (log.isTraceEnabled()) { + log.trace("Publishing view member: {} sig: {} on: {}", + ViewContext.print(svm, params().digestAlgorithm()), + params().digestAlgorithm().digest(sig.toByteString()), params().member().getId()); + } + } else if (log.isTraceEnabled()) { + log.trace("Adding discovered view member: {} on: {}", + ViewContext.print(svm, params().digestAlgorithm()), params().member().getId()); } } - polled.add(mid); } private Parameters params() { @@ -262,13 +264,12 @@ public void elect() { proposals.entrySet().stream().forEach(e -> slate.put(e.getKey(), joinOf(e.getValue()))); if (slate.size() == view.pendingView().getRingCount()) { cancelSlice.set(true); - log.debug("Electing: {} of: {} slate: {} proposals: {} on: {}", slate.size(), nextViewId, - slate.keySet().stream().sorted().toList(), proposals.keySet().stream().sorted().toList(), - params().member().getId()); + log.debug("Electing: {} of: {} slate: {} on: {}", slate.size(), nextViewId, + slate.keySet().stream().sorted().toList(), params().member().getId()); transitions.complete(); } else { Context memberContext = view.pendingView(); - log.error("Failed election, required: {} slate: {} of: {} on: {}", memberContext.majority(), + log.error("Failed election, required: {} slate: {} of: {} on: {}", view.pendingView().getRingCount(), proposals.keySet().stream().sorted().toList(), nextViewId, params().member().getId()); } } @@ -284,7 +285,7 @@ public void failed() { public void gather() { log.trace("Gathering assembly for: {} on: {}", nextViewId, params().member().getId()); AtomicReference reiterate = new AtomicReference<>(); - AtomicReference retryDelay = new AtomicReference<>(Duration.ofMillis(10)); + var retryDelay = Duration.ofMillis(10); reiterate.set(() -> { nextAssembly = Committee.viewMembersOf(nextViewId, view.pendingView()) .stream() diff --git a/choam/src/main/java/com/salesforce/apollo/choam/ViewContext.java b/choam/src/main/java/com/salesforce/apollo/choam/ViewContext.java index 1a64d79ae..d7aa35adc 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/ViewContext.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/ViewContext.java @@ -144,10 +144,6 @@ public Context pendingView() { return pendingView.get(); } - public Block produce(ULong l, Digest hash, Assemble assemble, HashedBlock checkpoint) { - return blockProducer.produce(l, hash, assemble, checkpoint); - } - public Block produce(ULong l, Digest hash, Executions executions, HashedBlock checkpoint) { return blockProducer.produce(l, hash, executions, checkpoint); } @@ -191,12 +187,35 @@ public boolean validate(SignedViewMember svm, Validate validate) { return valid; } + public boolean validate(SignedJoin join) { + if (true) { + return true; + } + Verifier v = verifierOf(join); + if (v == null) { + log.debug("no verifier: {} for join: {} on: {}", Digest.from(join.getMember()), + Digest.from(join.getJoin().getVm().getId()), params.member().getId()); + return false; + } + var validated = v.verify(JohnHancock.from(join.getSignature()), join.getJoin().toByteString()); + if (!validated) { + log.trace("Cannot validate view join: [{}] sig: {} signed by: {} on: {}", + print(join.getJoin(), params.digestAlgorithm()), + params.digestAlgorithm().digest(join.getSignature().toByteString()), + Digest.from(join.getMember()), params().member().getId()); + } else { + log.trace("Validated view join: [{}] signed by: {} on: {}", print(join.getJoin(), params.digestAlgorithm()), + Digest.from(join.getMember()), params().member().getId()); + } + return validated; + } + protected Verifier verifierOf(Validate validate) { final var mid = Digest.from(validate.getWitness().getId()); var m = context.getMember(mid); if (m == null) { if (log.isDebugEnabled()) { - log.debug("Unable to validate key by non existent validator: [{}] on: {}", + log.debug("Unable to get verifier by non existent member: [{}] on: {}", print(validate, params.digestAlgorithm()), params.member().getId()); } return null; @@ -204,11 +223,36 @@ protected Verifier verifierOf(Validate validate) { Verifier v = validators.get(m); if (v == null) { if (log.isDebugEnabled()) { - log.debug("Unable to validate key by non existent validator: [{}] on: {}", + log.debug("Unable to get verifier by non existent validator: [{}] on: {}", print(validate, params.digestAlgorithm()), params.member().getId()); } return null; } return v; } + + protected Verifier verifierOf(SignedJoin sj) { + final var mid = Digest.from(sj.getMember()); + var m = context.getMember(mid); + if (m == null) { + if (log.isDebugEnabled()) { + log.debug("Unable to get verifier by non existent member: [{}] on: {}", + String.format("id: %s sig: %s", Digest.from(sj.getMember()), + params.digestAlgorithm().digest(sj.getSignature().toByteString())), + params.member().getId()); + } + return null; + } + Verifier v = validators.get(m); + if (v == null) { + if (log.isDebugEnabled()) { + log.debug("Unable to validate key by non existent validator: [{}] on: {}", + String.format("id: %s sig: %s", Digest.from(sj.getMember()), + params.digestAlgorithm().digest(sj.getSignature().toByteString())), + params.member().getId()); + } + return null; + } + return v; + } } diff --git a/choam/src/main/java/com/salesforce/apollo/choam/fsm/Driven.java b/choam/src/main/java/com/salesforce/apollo/choam/fsm/Driven.java index d0f8c43e8..88ae2bada 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/fsm/Driven.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/fsm/Driven.java @@ -33,20 +33,10 @@ public interface Driven { void fail(); - void produceAssemble(); - - void reconfigure(); - void startProduction(); enum Earner implements Driven.Transitions { AWAIT_VIEW { - @Override - public Transitions assembled() { - context().assembled(); - return null; - } - @Entry public void checkAssembly() { context().checkAssembly(); @@ -125,21 +115,9 @@ public void terminate() { context().fail(); } }, SPICE { - @Override - public Transitions assembled() { - context().reconfigure(); - return null; - } - @Override public Transitions newEpoch(int epoch, int lastEpoch) { - if (lastEpoch == epoch) { - return AWAIT_VIEW; - } - if (epoch == 0) { - context().produceAssemble(); - } - return null; + return (lastEpoch == epoch) ? AWAIT_VIEW : null; } @Entry @@ -158,10 +136,6 @@ public Transitions viewComplete() { interface Transitions extends FsmExecutor { Logger log = LoggerFactory.getLogger(Transitions.class); - default Transitions assembled() { - return null; - } - default Transitions checkpoint() { throw fsm().invalidTransitionOn(); } diff --git a/choam/src/main/java/com/salesforce/apollo/choam/support/ChoamMetrics.java b/choam/src/main/java/com/salesforce/apollo/choam/support/ChoamMetrics.java index 77b62f177..f1d6129f9 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/support/ChoamMetrics.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/support/ChoamMetrics.java @@ -27,7 +27,7 @@ public interface ChoamMetrics extends EndpointMetrics { EtherealMetrics getProducerMetrics(); - void publishedBatch(int batchSize, int byteSize, int validations, int reassemblies); + void publishedBatch(int batchSize, int byteSize, int validations, int joins); void transactionCancelled(); diff --git a/choam/src/main/java/com/salesforce/apollo/choam/support/ChoamMetricsImpl.java b/choam/src/main/java/com/salesforce/apollo/choam/support/ChoamMetricsImpl.java index 6ff2e0e84..71b732242 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/support/ChoamMetricsImpl.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/support/ChoamMetricsImpl.java @@ -35,7 +35,7 @@ public class ChoamMetricsImpl extends EndpointMetricsImpl implements ChoamMetric private final EtherealMetrics genesisMetrics; private final EtherealMetrics producerMetrics; private final Histogram publishedBytes; - private final Meter publishedReassemblies; + private final Meter publishedJoins; private final Meter publishedTransactions; private final Meter publishedValidations; private final MetricRegistry registry; @@ -67,7 +67,7 @@ public ChoamMetricsImpl(Digest context, MetricRegistry registry) { cancelledTransactions = registry.meter(name(context.shortString(), "transactions.cancelled")); publishedTransactions = registry.meter(name(context.shortString(), "transactions.published")); publishedBytes = registry.histogram(name(context.shortString(), "unit.bytes")); - publishedReassemblies = registry.meter(name(context.shortString(), "reassemblies.published")); + publishedJoins = registry.meter(name(context.shortString(), "joins.published")); publishedValidations = registry.meter(name(context.shortString(), "validations.published")); transactionLatency = registry.timer(name(context.shortString(), "transaction.latency")); transactionSubmitRetry = registry.meter(name(context.shortString(), "transaction.submit.retry")); @@ -117,11 +117,11 @@ public EtherealMetrics getProducerMetrics() { } @Override - public void publishedBatch(int transactions, int byteSize, int validations, int reassemblies) { + public void publishedBatch(int transactions, int byteSize, int validations, int joins) { publishedTransactions.mark(transactions); publishedBytes.update(byteSize); publishedValidations.mark(validations); - publishedReassemblies.mark(reassemblies); + publishedJoins.mark(joins); } @Override diff --git a/choam/src/main/java/com/salesforce/apollo/choam/support/TxDataSource.java b/choam/src/main/java/com/salesforce/apollo/choam/support/TxDataSource.java index 5acb7c4cb..4a34fc326 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/support/TxDataSource.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/support/TxDataSource.java @@ -7,7 +7,7 @@ package com.salesforce.apollo.choam.support; import com.google.protobuf.ByteString; -import com.salesforce.apollo.choam.proto.Reassemble; +import com.salesforce.apollo.choam.proto.SignedJoin; import com.salesforce.apollo.choam.proto.Transaction; import com.salesforce.apollo.choam.proto.UnitData; import com.salesforce.apollo.choam.proto.Validate; @@ -37,13 +37,13 @@ public class TxDataSource implements DataSource { private final static Logger log = LoggerFactory.getLogger(TxDataSource.class); private final Duration batchInterval; - private final AtomicBoolean draining = new AtomicBoolean(); + private final AtomicBoolean draining = new AtomicBoolean(); private final ExponentialBackoffPolicy drainPolicy; private final Member member; private final ChoamMetrics metrics; private final BatchingQueue processing; - private final BlockingQueue reassemblies = new LinkedBlockingQueue<>(); - private final BlockingQueue validations = new LinkedBlockingQueue<>(); + private final BlockingQueue joins = new LinkedBlockingQueue<>(); + private final BlockingQueue validations = new LinkedBlockingQueue<>(); private volatile Thread blockingThread; public TxDataSource(Member member, int maxElements, ChoamMetrics metrics, int maxBatchByteSize, @@ -63,10 +63,10 @@ public void close() { } blockingThread = null; if (metrics != null) { - metrics.dropped(processing.size(), validations.size(), reassemblies.size()); + metrics.dropped(processing.size(), validations.size(), joins.size()); } log.trace("Closing with remaining txns: {}({}:{}) validations: {} reassemblies: {} on: {}", processing.size(), - processing.added(), processing.taken(), validations.size(), reassemblies.size(), member.getId()); + processing.added(), processing.taken(), validations.size(), joins.size(), member.getId()); } public void drain() { @@ -81,23 +81,23 @@ public ByteString getData() { log.trace("Requesting unit data on: {}", member.getId()); blockingThread = Thread.currentThread(); try { - var r = new ArrayList(); + var r = new ArrayList(); var v = new ArrayList(); if (draining.get()) { var target = Instant.now().plus(drainPolicy.nextBackoff()); - while (target.isAfter(Instant.now()) && builder.getReassembliesCount() == 0 + while (target.isAfter(Instant.now()) && builder.getJoinsCount() == 0 && builder.getValidationsCount() == 0) { // rinse and repeat - r = new ArrayList(); - reassemblies.drainTo(r); - builder.addAllReassemblies(r); + r = new ArrayList<>(); + joins.drainTo(r); + builder.addAllJoins(r); v = new ArrayList(); validations.drainTo(v); builder.addAllValidations(v); - if (builder.getReassembliesCount() != 0 || builder.getValidationsCount() != 0) { + if (builder.getJoinsCount() != 0 || builder.getValidationsCount() != 0) { break; } @@ -122,9 +122,9 @@ public ByteString getData() { } // One more time into ye breech - r = new ArrayList(); - reassemblies.drainTo(r); - builder.addAllReassemblies(r); + r = new ArrayList<>(); + joins.drainTo(r); + builder.addAllJoins(r); v = new ArrayList(); validations.drainTo(v); @@ -133,11 +133,11 @@ public ByteString getData() { ByteString bs = builder.build().toByteString(); if (metrics != null) { metrics.publishedBatch(builder.getTransactionsCount(), bs.size(), builder.getValidationsCount(), - builder.getReassembliesCount()); + builder.getJoinsCount()); } - log.trace("Unit data: {} txns, {} validations, {} reassemblies totalling: {} bytes on: {}", - builder.getTransactionsCount(), builder.getValidationsCount(), builder.getReassembliesCount(), - bs.size(), member.getId()); + log.trace("Unit data: {} txns, {} validations, {} joins totalling: {} bytes on: {}", + builder.getTransactionsCount(), builder.getValidationsCount(), builder.getJoinsCount(), bs.size(), + member.getId()); return bs; } finally { blockingThread = null; @@ -145,7 +145,7 @@ public ByteString getData() { } public int getRemainingReassemblies() { - return reassemblies.size(); + return joins.size(); } public int getRemainingTransactions() { @@ -156,8 +156,8 @@ public int getRemainingValidations() { return validations.size(); } - public boolean offer(Reassemble reassembly) { - return reassemblies.offer(reassembly); + public boolean offer(SignedJoin signedJoin) { + return joins.offer(signedJoin); } public boolean offer(Transaction txn) { diff --git a/choam/src/test/java/com/salesforce/apollo/choam/GenesisAssemblyTest.java b/choam/src/test/java/com/salesforce/apollo/choam/GenesisAssemblyTest.java index 83d441916..136b7e37a 100644 --- a/choam/src/test/java/com/salesforce/apollo/choam/GenesisAssemblyTest.java +++ b/choam/src/test/java/com/salesforce/apollo/choam/GenesisAssemblyTest.java @@ -147,11 +147,6 @@ public void onFailure() { // do nothing } - @Override - public Block produce(ULong height, Digest prev, Assemble assemble, HashedBlock checkpoint) { - return null; - } - @Override public Block produce(ULong height, Digest prev, Executions executions, HashedBlock checkpoint) { return null; diff --git a/grpc/src/main/proto/choam.proto b/grpc/src/main/proto/choam.proto index 5c5c7416c..21d653cb8 100644 --- a/grpc/src/main/proto/choam.proto +++ b/grpc/src/main/proto/choam.proto @@ -47,7 +47,6 @@ message Block { Reconfigure reconfigure = 3; Checkpoint checkpoint = 4; Executions executions = 5; - Assemble assemble = 6; } } @@ -84,10 +83,6 @@ message Executions { repeated Transaction executions = 1; } -message Assemble { - crypto.Digeste nextView = 1; -} - message FoundationSeal { stereotomy.KeyEvent_ foundation = 1; crypto.Sig signature = 2; @@ -103,8 +98,7 @@ message Transaction { message UnitData { repeated Validate validations = 1; repeated Transaction transactions = 2; - repeated Reassemble reassemblies = 3; - repeated SignedJoin joins = 4; + repeated SignedJoin joins = 3; } message Join { @@ -157,12 +151,6 @@ message Validate { Certification witness = 2; } -message Reassemble { - repeated SignedViewMember members = 1; - repeated Validate validations = 2; - repeated crypto.Digeste slate = 3; -} - message Validations { repeated Validate validations = 1; } diff --git a/model/src/test/java/com/salesforce/apollo/model/DomainTest.java b/model/src/test/java/com/salesforce/apollo/model/DomainTest.java index 4c72ce176..45a6e9e47 100644 --- a/model/src/test/java/com/salesforce/apollo/model/DomainTest.java +++ b/model/src/test/java/com/salesforce/apollo/model/DomainTest.java @@ -258,7 +258,7 @@ private Builder params() { .setMaxBatchCount(3000) .build()) .setCheckpointBlockDelta(200); - params.getProducer().ethereal().setNumberOfEpochs(4); + params.getProducer().ethereal().setEpochLength(7).setNumberOfEpochs(3); return params; } } diff --git a/model/src/test/java/com/salesforce/apollo/model/FireFliesTest.java b/model/src/test/java/com/salesforce/apollo/model/FireFliesTest.java index 9140f7793..509577801 100644 --- a/model/src/test/java/com/salesforce/apollo/model/FireFliesTest.java +++ b/model/src/test/java/com/salesforce/apollo/model/FireFliesTest.java @@ -181,7 +181,7 @@ private Builder params() { .build()) .setCheckpointBlockDelta(200); - params.getProducer().ethereal().setNumberOfEpochs(5); + params.getProducer().ethereal().setEpochLength(7).setNumberOfEpochs(3); return params; } } diff --git a/model/src/test/resources/logback-test.xml b/model/src/test/resources/logback-test.xml index 6c5f75706..87a97a0aa 100644 --- a/model/src/test/resources/logback-test.xml +++ b/model/src/test/resources/logback-test.xml @@ -62,7 +62,7 @@ - + diff --git a/pom.xml b/pom.xml index 977d4005d..c1a3a9a7e 100644 --- a/pom.xml +++ b/pom.xml @@ -558,6 +558,11 @@ org-netbeans-modules-keyring RELEASE200 + + com.googlecode.concurrentlinkedhashmap + concurrentlinkedhashmap-lru + 1.4.2 + From 39fabd16415a09495bcd477865af6ad4f39aaed0 Mon Sep 17 00:00:00 2001 From: Hellblazer Date: Sun, 14 Apr 2024 17:55:49 -0700 Subject: [PATCH 06/10] correct signing key --- .../salesforce/apollo/choam/ViewAssembly.java | 7 ++-- .../salesforce/apollo/choam/ViewContext.java | 41 ++++++++++++------- 2 files changed, 31 insertions(+), 17 deletions(-) diff --git a/choam/src/main/java/com/salesforce/apollo/choam/ViewAssembly.java b/choam/src/main/java/com/salesforce/apollo/choam/ViewAssembly.java index 847aac3c7..ec33dabea 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/ViewAssembly.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/ViewAssembly.java @@ -213,16 +213,17 @@ private void join(SignedViewMember svm, boolean direct) { polled.add(mid); if (proposals.putIfAbsent(mid, svm) == null) { if (direct) { - var sig = params().member().sign(svm.toByteString()).toSig(); + var signature = view.sign(svm); publisher.accept(SignedJoin.newBuilder() .setJoin(svm) .setMember(params().member().getId().toDigeste()) - .setSignature(sig) + .setSignature(signature.toSig()) .build()); if (log.isTraceEnabled()) { log.trace("Publishing view member: {} sig: {} on: {}", ViewContext.print(svm, params().digestAlgorithm()), - params().digestAlgorithm().digest(sig.toByteString()), params().member().getId()); + params().digestAlgorithm().digest(signature.toSig().toByteString()), + params().member().getId()); } } else if (log.isTraceEnabled()) { log.trace("Adding discovered view member: {} on: {}", diff --git a/choam/src/main/java/com/salesforce/apollo/choam/ViewContext.java b/choam/src/main/java/com/salesforce/apollo/choam/ViewContext.java index d7aa35adc..59a55e0e7 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/ViewContext.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/ViewContext.java @@ -82,8 +82,10 @@ public Context context() { } public Validate generateValidation(HashedBlock block) { - log.trace("Signing: {} block: {} height: {} on: {}", block.block.getBodyCase(), block.hash, block.height(), - params.member().getId()); + if (log.isTraceEnabled()) { + log.trace("Signing: {} block: {} height: {} on: {}", block.block.getBodyCase(), block.hash, block.height(), + params.member().getId()); + } JohnHancock signature = signer.sign(block.block.getHeader().toByteString()); if (signature == null) { log.error("Unable to sign: {} block: {} height: {} on: {}", block.block.getBodyCase(), block.hash, @@ -161,11 +163,21 @@ public Map roster() { return roster; } + public JohnHancock sign(SignedViewMember svm) { + if (log.isTraceEnabled()) { + log.trace("Signing: {} on: {}", print(svm, params.digestAlgorithm()), params.member().getId()); + } + return signer.sign(svm.toByteString()); + } + public boolean validate(HashedBlock block, Validate validate) { Verifier v = verifierOf(validate); if (v == null) { - log.debug("no validation witness: {} for: {} block: {} on: {}", Digest.from(validate.getWitness().getId()), - block.block.getBodyCase(), block.hash, params.member().getId()); + if (log.isDebugEnabled()) { + log.debug("no validation witness: {} for: {} block: {} on: {}", + Digest.from(validate.getWitness().getId()), block.block.getBodyCase(), block.hash, + params.member().getId()); + } return false; } return v.verify(JohnHancock.from(validate.getWitness().getSignature()), block.block.getHeader().toByteString()); @@ -188,22 +200,23 @@ public boolean validate(SignedViewMember svm, Validate validate) { } public boolean validate(SignedJoin join) { - if (true) { - return true; - } Verifier v = verifierOf(join); if (v == null) { - log.debug("no verifier: {} for join: {} on: {}", Digest.from(join.getMember()), - Digest.from(join.getJoin().getVm().getId()), params.member().getId()); + if (log.isDebugEnabled()) { + log.debug("no verifier: {} for join: {} on: {}", Digest.from(join.getMember()), + Digest.from(join.getJoin().getVm().getId()), params.member().getId()); + } return false; } var validated = v.verify(JohnHancock.from(join.getSignature()), join.getJoin().toByteString()); if (!validated) { - log.trace("Cannot validate view join: [{}] sig: {} signed by: {} on: {}", - print(join.getJoin(), params.digestAlgorithm()), - params.digestAlgorithm().digest(join.getSignature().toByteString()), - Digest.from(join.getMember()), params().member().getId()); - } else { + if (log.isTraceEnabled()) { + log.trace("Cannot validate view join: [{}] sig: {} signed by: {} on: {}", + print(join.getJoin(), params.digestAlgorithm()), + params.digestAlgorithm().digest(join.getSignature().toByteString()), + Digest.from(join.getMember()), params().member().getId()); + } + } else if (log.isTraceEnabled()) { log.trace("Validated view join: [{}] signed by: {} on: {}", print(join.getJoin(), params.digestAlgorithm()), Digest.from(join.getMember()), params().member().getId()); } From 6e1e5be2fa684fe9989720eeb0dda8524a480aec Mon Sep 17 00:00:00 2001 From: Hellblazer Date: Sun, 14 Apr 2024 18:54:11 -0700 Subject: [PATCH 07/10] correct signing key. active at genesis, not (missing) assemble --- choam/src/main/java/com/salesforce/apollo/choam/CHOAM.java | 2 +- choam/src/main/java/com/salesforce/apollo/choam/Producer.java | 1 + .../java/com/salesforce/apollo/model/ContainmentDomainTest.java | 2 +- model/src/test/resources/logback-test.xml | 2 +- 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/choam/src/main/java/com/salesforce/apollo/choam/CHOAM.java b/choam/src/main/java/com/salesforce/apollo/choam/CHOAM.java index d05256ed5..217c179e0 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/CHOAM.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/CHOAM.java @@ -262,7 +262,7 @@ public boolean active() { final var c = current.get(); HashedCertifiedBlock h = head.get(); return (c != null && h != null && transitions.fsm().getCurrentState() == Mercantile.OPERATIONAL) - && c instanceof Administration && h.height().compareTo(ULong.valueOf(1)) >= 0; + && c instanceof Administration && h.height().compareTo(ULong.valueOf(0)) >= 0; } public DelegatedContext context() { diff --git a/choam/src/main/java/com/salesforce/apollo/choam/Producer.java b/choam/src/main/java/com/salesforce/apollo/choam/Producer.java index 31219e253..08c8f597e 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/Producer.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/Producer.java @@ -343,6 +343,7 @@ public void checkAssembly() { log.debug("Final view assembly election on: {}", params().member().getId()); if (assembled.get()) { assembled(); + controller.completeIt(); } } diff --git a/model/src/test/java/com/salesforce/apollo/model/ContainmentDomainTest.java b/model/src/test/java/com/salesforce/apollo/model/ContainmentDomainTest.java index a12f17a6e..5dd3cbf23 100644 --- a/model/src/test/java/com/salesforce/apollo/model/ContainmentDomainTest.java +++ b/model/src/test/java/com/salesforce/apollo/model/ContainmentDomainTest.java @@ -131,7 +131,7 @@ private Builder params() { .setMaxBatchCount(3000) .build()) .setCheckpointBlockDelta(200); - params.getProducer().ethereal().setNumberOfEpochs(4); + params.getProducer().ethereal().setEpochLength(4).setNumberOfEpochs(4); return params; } } diff --git a/model/src/test/resources/logback-test.xml b/model/src/test/resources/logback-test.xml index 87a97a0aa..e8a22c6a4 100644 --- a/model/src/test/resources/logback-test.xml +++ b/model/src/test/resources/logback-test.xml @@ -22,7 +22,7 @@ - + From 5f9f93a7ea1ee227fbf63800722fdc508d2c56fd Mon Sep 17 00:00:00 2001 From: Hellblazer Date: Mon, 15 Apr 2024 09:47:38 -0700 Subject: [PATCH 08/10] prepare for view synchronization; moar refactoring around view change. --- .../com/salesforce/apollo/choam/Producer.java | 16 +++--- .../salesforce/apollo/choam/ViewAssembly.java | 30 +++++++---- .../salesforce/apollo/choam/ViewContext.java | 51 +++++++++++-------- .../apollo/choam/fsm/Reconfiguration.java | 16 ++++++ .../apollo/choam/support/TxDataSource.java | 38 +++++++------- grpc/src/main/proto/choam.proto | 23 ++++++++- 6 files changed, 115 insertions(+), 59 deletions(-) diff --git a/choam/src/main/java/com/salesforce/apollo/choam/Producer.java b/choam/src/main/java/com/salesforce/apollo/choam/Producer.java index 08c8f597e..961fa217f 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/Producer.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/Producer.java @@ -52,7 +52,7 @@ public class Producer { private final int lastEpoch; private final Map nextAssembly = new HashMap<>(); private final Map pending = new ConcurrentSkipListMap<>(); - private final List pendingJoins = new CopyOnWriteArrayList<>(); + private final List pendingAssemblies = new CopyOnWriteArrayList<>(); private final Map> pendingValidations = new ConcurrentSkipListMap<>(); private final AtomicReference previousBlock = new AtomicReference<>(); private final AtomicBoolean reconfigured = new AtomicBoolean(); @@ -151,8 +151,8 @@ public SubmitResult submit(Transaction transaction) { } } - private void addJoin(SignedJoin signedJoin) { - if (ds.offer(signedJoin)) { + private void addAssembly(Assemblies assemblies) { + if (ds.offer(assemblies)) { log.trace("Adding on: {}", params().member().getId()); } else { log.trace("Cannot add join on: {}", params().member().getId()); @@ -181,14 +181,14 @@ private void create(List preblock, boolean last) { .filter(p -> p.witnesses.size() >= params().majority()) .forEach(this::publish); - var joins = aggregate.stream().flatMap(e -> e.getJoinsList().stream()).filter(view::validate).toList(); + var joins = aggregate.stream().flatMap(e -> e.getAssembliesList().stream()).toList(); final var ass = assembly; if (ass != null) { log.trace("Consuming {} units, {} joins on: {}", aggregate.size(), joins.size(), params().member().getId()); ass.inbound().accept(joins); } else { log.trace("Pending {} units, {} joins on: {}", aggregate.size(), joins.size(), params().member().getId()); - pendingJoins.addAll(joins); + pendingAssemblies.addAll(joins); } HashedBlock lb = previousBlock.get(); @@ -261,7 +261,7 @@ private void publish(PendingBlock p) { private void reconfigure() { log.debug("Starting view reconfiguration: {} on: {}", nextViewId, params().member().getId()); - assembly = new ViewAssembly(nextViewId, view, Producer.this::addJoin, comms) { + assembly = new ViewAssembly(nextViewId, view, Producer.this::addAssembly, comms) { @Override public void complete() { super.complete(); @@ -273,8 +273,8 @@ public void complete() { }; assembly.start(); assembly.assembled(); - var joins = new ArrayList<>(pendingJoins); - pendingJoins.clear(); + var joins = new ArrayList<>(pendingAssemblies); + pendingAssemblies.clear(); assembly.inbound().accept(joins); } diff --git a/choam/src/main/java/com/salesforce/apollo/choam/ViewAssembly.java b/choam/src/main/java/com/salesforce/apollo/choam/ViewAssembly.java index ec33dabea..2db8e9d6f 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/ViewAssembly.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/ViewAssembly.java @@ -12,6 +12,7 @@ import com.salesforce.apollo.choam.fsm.Reconfiguration; import com.salesforce.apollo.choam.fsm.Reconfiguration.Reconfigure; import com.salesforce.apollo.choam.fsm.Reconfiguration.Transitions; +import com.salesforce.apollo.choam.proto.Assemblies; import com.salesforce.apollo.choam.proto.Join; import com.salesforce.apollo.choam.proto.SignedJoin; import com.salesforce.apollo.choam.proto.SignedViewMember; @@ -54,7 +55,7 @@ public class ViewAssembly { private final AtomicBoolean cancelSlice = new AtomicBoolean(); private final Digest nextViewId; private final Map proposals = new ConcurrentHashMap<>(); - private final Consumer publisher; + private final Consumer publisher; private final Map slate = new ConcurrentSkipListMap<>(); private final ViewContext view; private final CommonCommunications comms; @@ -62,7 +63,7 @@ public class ViewAssembly { new ConcurrentSkipListMap<>()); private volatile Map nextAssembly; - public ViewAssembly(Digest nextViewId, ViewContext vc, Consumer publisher, + public ViewAssembly(Digest nextViewId, ViewContext vc, Consumer publisher, CommonCommunications comms) { view = vc; this.nextViewId = nextViewId; @@ -111,12 +112,17 @@ void finalElection() { transitions.complete(); } - Consumer> inbound() { + Consumer> inbound() { return lre -> { - lre.forEach(vm -> join(vm.getJoin(), false)); + lre.forEach(ass -> assemble(ass)); }; } + private void assemble(Assemblies ass) { + ass.getJoinsList().stream().filter(sj -> view.validate(sj)).forEach(sj -> join(sj.getJoin(), false)); + ass.getViewsList().stream().filter(sv -> view.validate(sv)); + } + private void completeSlice(Duration retryDelay, AtomicReference reiterate) { if (gathered()) { return; @@ -214,10 +220,12 @@ private void join(SignedViewMember svm, boolean direct) { if (proposals.putIfAbsent(mid, svm) == null) { if (direct) { var signature = view.sign(svm); - publisher.accept(SignedJoin.newBuilder() - .setJoin(svm) - .setMember(params().member().getId().toDigeste()) - .setSignature(signature.toSig()) + publisher.accept(Assemblies.newBuilder() + .addJoins(SignedJoin.newBuilder() + .setJoin(svm) + .setMember(params().member().getId().toDigeste()) + .setSignature(signature.toSig()) + .build()) .build()); if (log.isTraceEnabled()) { log.trace("Publishing view member: {} sig: {} on: {}", @@ -306,10 +314,14 @@ public void gather() { @Override public void nominate() { - // publisher.accept(getMemberProposal()); transitions.nominated(); } + @Override + public void viewAgreement() { + + } + private Join joinOf(SignedViewMember vm) { return Join.newBuilder().setMember(vm).build(); } diff --git a/choam/src/main/java/com/salesforce/apollo/choam/ViewContext.java b/choam/src/main/java/com/salesforce/apollo/choam/ViewContext.java index 59a55e0e7..2af557f0a 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/ViewContext.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/ViewContext.java @@ -223,35 +223,44 @@ public boolean validate(SignedJoin join) { return validated; } - protected Verifier verifierOf(Validate validate) { - final var mid = Digest.from(validate.getWitness().getId()); - var m = context.getMember(mid); - if (m == null) { + public boolean validate(SignedViews sv) { + Verifier v = verifierOf(sv); + if (v == null) { if (log.isDebugEnabled()) { - log.debug("Unable to get verifier by non existent member: [{}] on: {}", - print(validate, params.digestAlgorithm()), params.member().getId()); + log.debug("no verifier: {} for signed view on: {}", Digest.from(sv.getViews().getMember()), + params.member().getId()); } - return null; + return false; } - Verifier v = validators.get(m); - if (v == null) { - if (log.isDebugEnabled()) { - log.debug("Unable to get verifier by non existent validator: [{}] on: {}", - print(validate, params.digestAlgorithm()), params.member().getId()); + var validated = v.verify(JohnHancock.from(sv.getSignature()), sv.getViews().toByteString()); + if (!validated) { + if (log.isTraceEnabled()) { + log.trace("Cannot validate views signed by: {} on: {}", Digest.from(sv.getViews().getMember()), + params().member().getId()); } - return null; + } else if (log.isTraceEnabled()) { + log.trace("Validated views signed by: {} on: {}", Digest.from(sv.getViews().getMember()), + params().member().getId()); } - return v; + return validated; + } + + protected Verifier verifierOf(Validate validate) { + return getVerifier(context.getMember(Digest.from(validate.getWitness().getId()))); } protected Verifier verifierOf(SignedJoin sj) { - final var mid = Digest.from(sj.getMember()); - var m = context.getMember(mid); + return getVerifier(context.getMember(Digest.from(sj.getMember()))); + } + + protected Verifier verifierOf(SignedViews sv) { + return getVerifier(context.getMember(Digest.from(sv.getViews().getMember()))); + } + + private Verifier getVerifier(Member m) { if (m == null) { if (log.isDebugEnabled()) { - log.debug("Unable to get verifier by non existent member: [{}] on: {}", - String.format("id: %s sig: %s", Digest.from(sj.getMember()), - params.digestAlgorithm().digest(sj.getSignature().toByteString())), + log.debug("Unable to get verifier by non existent member: {} on: {}", m.getId(), params.member().getId()); } return null; @@ -259,9 +268,7 @@ protected Verifier verifierOf(SignedJoin sj) { Verifier v = validators.get(m); if (v == null) { if (log.isDebugEnabled()) { - log.debug("Unable to validate key by non existent validator: [{}] on: {}", - String.format("id: %s sig: %s", Digest.from(sj.getMember()), - params.digestAlgorithm().digest(sj.getSignature().toByteString())), + log.debug("Unable to validate key by non existent validator: {} on: {}", m.getId(), params.member().getId()); } return null; diff --git a/choam/src/main/java/com/salesforce/apollo/choam/fsm/Reconfiguration.java b/choam/src/main/java/com/salesforce/apollo/choam/fsm/Reconfiguration.java index 66b4e51e4..df6705942 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/fsm/Reconfiguration.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/fsm/Reconfiguration.java @@ -25,6 +25,8 @@ public interface Reconfiguration { void nominate(); + void viewAgreement(); + enum Reconfigure implements Transitions { AWAIT_ASSEMBLY { @Override @@ -131,6 +133,16 @@ public Transitions complete() { public void completion() { context().complete(); } + }, VIEW_AGREEMENT { + @Entry + public void viewConsensus() { + context().viewAgreement(); + } + + @Override + public Transitions viewDetermined() { + return GATHER; + } } } @@ -162,5 +174,9 @@ default Transitions nominated() { default Transitions validation() { return null; } + + default Transitions viewDetermined() { + throw fsm().invalidTransitionOn(); + } } } diff --git a/choam/src/main/java/com/salesforce/apollo/choam/support/TxDataSource.java b/choam/src/main/java/com/salesforce/apollo/choam/support/TxDataSource.java index 4a34fc326..350908e90 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/support/TxDataSource.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/support/TxDataSource.java @@ -7,7 +7,7 @@ package com.salesforce.apollo.choam.support; import com.google.protobuf.ByteString; -import com.salesforce.apollo.choam.proto.SignedJoin; +import com.salesforce.apollo.choam.proto.Assemblies; import com.salesforce.apollo.choam.proto.Transaction; import com.salesforce.apollo.choam.proto.UnitData; import com.salesforce.apollo.choam.proto.Validate; @@ -42,7 +42,7 @@ public class TxDataSource implements DataSource { private final Member member; private final ChoamMetrics metrics; private final BatchingQueue processing; - private final BlockingQueue joins = new LinkedBlockingQueue<>(); + private final BlockingQueue assemblies = new LinkedBlockingQueue<>(); private final BlockingQueue validations = new LinkedBlockingQueue<>(); private volatile Thread blockingThread; @@ -63,10 +63,10 @@ public void close() { } blockingThread = null; if (metrics != null) { - metrics.dropped(processing.size(), validations.size(), joins.size()); + metrics.dropped(processing.size(), validations.size(), assemblies.size()); } - log.trace("Closing with remaining txns: {}({}:{}) validations: {} reassemblies: {} on: {}", processing.size(), - processing.added(), processing.taken(), validations.size(), joins.size(), member.getId()); + log.trace("Closing with remaining txns: {}({}:{}) validations: {} assemblies: {} on: {}", processing.size(), + processing.added(), processing.taken(), validations.size(), assemblies.size(), member.getId()); } public void drain() { @@ -81,23 +81,23 @@ public ByteString getData() { log.trace("Requesting unit data on: {}", member.getId()); blockingThread = Thread.currentThread(); try { - var r = new ArrayList(); + var r = new ArrayList(); var v = new ArrayList(); if (draining.get()) { var target = Instant.now().plus(drainPolicy.nextBackoff()); - while (target.isAfter(Instant.now()) && builder.getJoinsCount() == 0 + while (target.isAfter(Instant.now()) && builder.getAssembliesCount() == 0 && builder.getValidationsCount() == 0) { // rinse and repeat r = new ArrayList<>(); - joins.drainTo(r); - builder.addAllJoins(r); + assemblies.drainTo(r); + builder.addAllAssemblies(r); v = new ArrayList(); validations.drainTo(v); builder.addAllValidations(v); - if (builder.getJoinsCount() != 0 || builder.getValidationsCount() != 0) { + if (builder.getAssembliesCount() != 0 || builder.getValidationsCount() != 0) { break; } @@ -123,8 +123,8 @@ public ByteString getData() { // One more time into ye breech r = new ArrayList<>(); - joins.drainTo(r); - builder.addAllJoins(r); + assemblies.drainTo(r); + builder.addAllAssemblies(r); v = new ArrayList(); validations.drainTo(v); @@ -133,11 +133,11 @@ public ByteString getData() { ByteString bs = builder.build().toByteString(); if (metrics != null) { metrics.publishedBatch(builder.getTransactionsCount(), bs.size(), builder.getValidationsCount(), - builder.getJoinsCount()); + builder.getAssembliesCount()); } - log.trace("Unit data: {} txns, {} validations, {} joins totalling: {} bytes on: {}", - builder.getTransactionsCount(), builder.getValidationsCount(), builder.getJoinsCount(), bs.size(), - member.getId()); + log.trace("Unit data: {} txns, {} validations, {} assemblies totalling: {} bytes on: {}", + builder.getTransactionsCount(), builder.getValidationsCount(), builder.getAssembliesCount(), + bs.size(), member.getId()); return bs; } finally { blockingThread = null; @@ -145,7 +145,7 @@ public ByteString getData() { } public int getRemainingReassemblies() { - return joins.size(); + return assemblies.size(); } public int getRemainingTransactions() { @@ -156,8 +156,8 @@ public int getRemainingValidations() { return validations.size(); } - public boolean offer(SignedJoin signedJoin) { - return joins.offer(signedJoin); + public boolean offer(Assemblies assemblies) { + return this.assemblies.offer(assemblies); } public boolean offer(Transaction txn) { diff --git a/grpc/src/main/proto/choam.proto b/grpc/src/main/proto/choam.proto index 21d653cb8..0d301c1e1 100644 --- a/grpc/src/main/proto/choam.proto +++ b/grpc/src/main/proto/choam.proto @@ -98,7 +98,12 @@ message Transaction { message UnitData { repeated Validate validations = 1; repeated Transaction transactions = 2; - repeated SignedJoin joins = 3; + repeated Assemblies assemblies = 3; +} + +message Assemblies { + repeated SignedJoin joins = 1; + repeated SignedViews views = 2; } message Join { @@ -112,6 +117,22 @@ message SignedJoin { crypto.Sig signature = 3; } +message SignedViews { + Views views = 1; + crypto.Sig signature = 2; +} + +message Views { + crypto.Digeste member = 1; + crypto.Digeste vid = 2; + repeated View views = 3; +} + +message View { + crypto.Digeste diadem = 1; + repeated crypto.Digeste committee = 2; +} + message ViewMember { crypto.Digeste id = 1; crypto.Digeste view = 2; From a58f92ec50a78bfd3eda0ad37e5aa9da70305e0d Mon Sep 17 00:00:00 2001 From: Hellblazer Date: Wed, 17 Apr 2024 17:45:39 -0700 Subject: [PATCH 09/10] refactor to ordered list of pending views current context determined by the advance of the pending views the current Context of the CHOAM (params.context) is defined as the head view in the pending views of the CHOAM, thus teh pending views are never empty. reassembly currently uses the last pending view of the CHOAM --- choam/pom.xml | 4 - .../com/salesforce/apollo/choam/CHOAM.java | 186 ++++++++++++------ .../salesforce/apollo/choam/Committee.java | 2 +- .../apollo/choam/GenesisAssembly.java | 2 +- .../apollo/choam/GenesisContext.java | 2 +- .../com/salesforce/apollo/choam/Producer.java | 91 +++++---- .../salesforce/apollo/choam/ViewAssembly.java | 79 ++++---- .../salesforce/apollo/choam/ViewContext.java | 30 +-- .../apollo/choam/GenesisAssemblyTest.java | 7 +- choam/src/test/resources/logback-test.xml | 6 +- grpc/src/main/proto/choam.proto | 6 + pom.xml | 10 - 12 files changed, 256 insertions(+), 169 deletions(-) diff --git a/choam/pom.xml b/choam/pom.xml index 1a91f65b4..d4c07c80e 100644 --- a/choam/pom.xml +++ b/choam/pom.xml @@ -30,10 +30,6 @@ org.jooq joou - - com.googlecode.concurrentlinkedhashmap - concurrentlinkedhashmap-lru - diff --git a/choam/src/main/java/com/salesforce/apollo/choam/CHOAM.java b/choam/src/main/java/com/salesforce/apollo/choam/CHOAM.java index 217c179e0..ce4d98875 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/CHOAM.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/CHOAM.java @@ -11,7 +11,6 @@ import com.google.protobuf.ByteString; import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.Message; -import com.googlecode.concurrentlinkedhashmap.ConcurrentLinkedHashMap; import com.salesforce.apollo.archipelago.RouterImpl.CommonCommunications; import com.salesforce.apollo.bloomFilters.BloomFilter; import com.salesforce.apollo.choam.comm.*; @@ -47,14 +46,13 @@ import java.io.FileInputStream; import java.io.IOException; import java.security.KeyPair; -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.function.Supplier; import java.util.stream.Collectors; @@ -95,16 +93,13 @@ public class CHOAM { private final Combine.Transitions transitions; private final TransSubmission txnSubmission = new TransSubmission(); private final AtomicReference view = new AtomicReference<>(); - private final AtomicReference> pendingView = new AtomicReference<>(); - private final ConcurrentLinkedHashMap> pendingViews; + private final PendingViews pendingViews = new PendingViews(); public CHOAM(Parameters params) { this.store = new Store(params.digestAlgorithm(), params.mvBuilder().clone().build()); this.params = params; executions = Executors.newVirtualThreadPerTaskExecutor(); - pendingViews = new ConcurrentLinkedHashMap.Builder>().maximumWeightedCapacity(100) - .initialCapacity(10) - .build(); + pendingViews.add(params.context().getId(), params.context().delegate()); nextView(); var bContext = new DelegatedContext<>(params.context()); @@ -189,10 +184,9 @@ public static Checkpoint checkpoint(DigestAlgorithm algo, File state, int segmen return cp; } - public static Block genesis(Digest id, Map joins, HashedBlock head, Context context, - HashedBlock lastViewChange, Parameters params, HashedBlock lastCheckpoint, - Iterable initialization) { - var reconfigure = reconfigure(id, joins, context, params.checkpointBlockDelta()); + public static Block genesis(Digest id, Map joins, HashedBlock head, HashedBlock lastViewChange, + Parameters params, HashedBlock lastCheckpoint, Iterable initialization) { + var reconfigure = reconfigure(id, joins, params.checkpointBlockDelta()); return Block.newBuilder() .setHeader(buildHeader(params.digestAlgorithm(), reconfigure, head.hash, ULong.valueOf(0), lastCheckpoint.height(), lastCheckpoint.hash, lastViewChange.height(), @@ -210,27 +204,21 @@ public static String print(Join join, DigestAlgorithm da) { join.getMember(), da) + "]"; } - public static Reconfigure reconfigure(Digest nextViewId, Map joins, Context context, - int checkpointTarget) { + public static Reconfigure reconfigure(Digest nextViewId, Map joins, int checkpointTarget) { var builder = Reconfigure.newBuilder().setCheckpointTarget(checkpointTarget).setId(nextViewId.toDigeste()); - // Canonical labeling of the view members for Ethereal - var remapped = rosterMap(context, joins.keySet()); + joins.keySet().stream().sorted().map(joins::get).forEach(builder::addJoins); - remapped.keySet().stream().sorted().map(remapped::get).forEach(m -> builder.addJoins(joins.get(m.getId()))); - - var reconfigure = builder.build(); - return reconfigure; + return builder.build(); } public static Block reconfigure(Digest nextViewId, Map joins, HashedBlock head, - Context context, HashedBlock lastViewChange, Parameters params, - HashedBlock lastCheckpoint) { + HashedBlock lastViewChange, Parameters params, HashedBlock lastCheckpoint) { final Block lvc = lastViewChange.block; int lastTarget = lvc.hasGenesis() ? lvc.getGenesis().getInitialView().getCheckpointTarget() : lvc.getReconfigure().getCheckpointTarget(); int checkpointTarget = lastTarget == 0 ? params.checkpointBlockDelta() : lastTarget - 1; - var reconfigure = reconfigure(nextViewId, joins, context, checkpointTarget); + var reconfigure = reconfigure(nextViewId, joins, checkpointTarget); return Block.newBuilder() .setHeader(buildHeader(params.digestAlgorithm(), reconfigure, head.hash, head.height().add(1), lastCheckpoint.height(), lastCheckpoint.hash, lastViewChange.height(), @@ -324,17 +312,18 @@ public void nextView(Context context, Digest diadem) { ((DelegatedContext) combine.getContext()).setContext(context); var c = current.get(); if (c != null) { - c.nextView(context); + c.nextView(diadem, context); } else { log.info("Acquiring new view of: {}, diadem: {} size: {} on: {}", context.getId(), diadem, context.size(), params.member().getId()); params.context().setContext(context); - pendingView.set(null); + pendingViews.clear(); + pendingViews.add(diadem, context); } log.info("Pushing pending view of: {}, diadem: {} size: {} on: {}", context.getId(), diadem, context.size(), params.member().getId()); - pendingViews.putIfAbsent(diadem, context); + pendingViews.add(diadem, context); } public void start() { @@ -505,14 +494,20 @@ public Block checkpoint() { public Block genesis(Map joining, Digest nextViewId, HashedBlock previous) { final HashedCertifiedBlock cp = checkpoint.get(); final HashedCertifiedBlock v = view.get(); - var g = CHOAM.genesis(nextViewId, joining, previous, params.context(), v, params, cp, - params.genesisData() - .apply(joining.keySet() - .stream() - .map(m -> params.context().getMember(m)) - .filter(m -> m != null) - .collect( - Collectors.toMap(m -> m, m -> joining.get(m.getId()))))); + var g = CHOAM.genesis(nextViewId, joining, previous, v, params, cp, params.genesisData() + .apply(joining.keySet() + .stream() + .map( + m -> params.context() + .getMember( + m)) + .filter( + m -> m != null) + .collect( + Collectors.toMap( + m -> m, + m -> joining.get( + m.getId()))))); log.info("Create genesis: {} on: {}", nextViewId, params.member().getId()); return g; } @@ -548,8 +543,7 @@ public void publish(Digest hash, CertifiedBlock cb) { public Block reconfigure(Map joining, Digest nextViewId, HashedBlock previous, HashedBlock checkpoint) { final HashedCertifiedBlock v = view.get(); - var block = CHOAM.reconfigure(nextViewId, joining, previous, pendingView().get(), v, params, - checkpoint); + var block = CHOAM.reconfigure(nextViewId, joining, previous, v, params, checkpoint); log.trace("Produced block: {} height: {} on: {}", block.getBodyCase(), block.getHeader().getHeight(), params.member().getId()); return block; @@ -663,17 +657,12 @@ private void nextView() { .build(), keyPair)); } - private Supplier> pendingView() { + private Supplier pendingViews() { return () -> { - var v = pendingView.get(); - return v == null ? params.context() : v; + return pendingViews; }; } - private Supplier>> pendingViews() { - return () -> pendingViews; - } - private void process() { final var c = current.get(); final HashedCertifiedBlock h = head.get(); @@ -715,10 +704,10 @@ private void process() { private void reconfigure(Digest hash, Reconfigure reconfigure) { log.info("Setting next view id: {} on: {}", hash, params.member().getId()); nextViewId.set(hash); - var pv = pendingView.getAndSet(null); + var pv = pendingViews.advance(); if (pv != null) { // always advance view. - params.context().setContext(pv); + params.context().setContext(pv.context); } final Committee c = current.get(); c.complete(); @@ -1028,6 +1017,88 @@ default void genesis(Digest hash, List initialization) { } } + public static class PendingViews { + private final ReadWriteLock lock = new ReentrantReadWriteLock(); + private final LinkedHashMap views = new LinkedHashMap<>(); + + public void add(Digest diadem, Context context) { + final var l = lock.writeLock(); + try { + l.lock(); + views.putIfAbsent(diadem, new PendingView(diadem, context)); + } finally { + l.unlock(); + } + } + + public PendingView advance() { + final var l = lock.writeLock(); + try { + l.lock(); + var last = views.lastEntry(); + if (last == null) { + return null; + } + views.clear(); + views.put(last.getKey(), last.getValue()); + return last.getValue(); + } finally { + l.unlock(); + } + } + + public void clear() { + final var l = lock.writeLock(); + try { + l.lock(); + views.clear(); + } finally { + l.unlock(); + } + } + + public PendingView get(Digest diadem) { + return views.get(diadem); + } + + public PendingView last() { + final var l = lock.readLock(); + try { + l.lock(); + var last = views.lastEntry(); + return last == null ? null : last.getValue(); + } finally { + l.unlock(); + } + } + + public PendingView pop() { + final var l = lock.writeLock(); + try { + l.lock(); + var v = views.pollFirstEntry(); + return v == null ? null : v.getValue(); + } finally { + l.unlock(); + } + } + } + + public record PendingView(Digest diadem, Context context) { + /** + * Answer the view created by finding the successors of the supplied hash on this Context + * + * @param hash - the "cut" across the rings of the context, determining the successors and thus the committee + * members of the view + * @return the View determined by this Context and the supplied hash value + */ + public View getView(Digest hash) { + var builder = View.newBuilder().setDiadem(diadem.toDigeste()); + Committee.viewMembersOf(hash, context).forEach(d -> builder.addCommittee(d.getId().toDigeste())); + return builder.build(); + } + } + record nextView(ViewMember member, KeyPair consensusKeyPair) { } @@ -1036,7 +1107,7 @@ public class Combiner implements Combine { @Override public void anchor() { HashedCertifiedBlock anchor = pending.poll(); - var pending = pendingView().get(); + var pending = pendingViews.last().context; if (anchor != null && pending.totalCount() >= pending.majority()) { log.info("Synchronizing from anchor: {} cardinality: {} on: {}", anchor.hash, pending.totalCount(), params.member().getId()); @@ -1227,8 +1298,8 @@ public Logger log() { } @Override - public void nextView(Context pendingView) { - var previous = CHOAM.this.pendingView.getAndSet(pendingView); + public void nextView(Digest diadem, Context pendingView) { + pendingViews.add(diadem, pendingView); log.info("Pending context for view: {} size: {} on: {}", nextViewId.get() == null ? "" : nextViewId.get(), pendingView.size(), params.member().getId()); @@ -1296,7 +1367,7 @@ private class Associate extends Administration { params.digestAlgorithm().digest(nextView.member.getSignature().toByteString()), viewId, params.member().getId()); Signer signer = new SignerImpl(nextView.consensusKeyPair.getPrivate(), ULong.MIN); - Supplier> pv = pendingView(); + var pv = pendingViews(); producer = new Producer(nextViewId.get(), new ViewContext(context, params, pv, signer, validators, constructBlock()), head.get(), checkpoint.get(), comm, getLabel()); @@ -1335,8 +1406,8 @@ private Formation() { params.digestAlgorithm().digest(c.consensusKeyPair.getPublic().getEncoded()), params.digestAlgorithm().digest(c.member.getSignature().toByteString()), params.member().getId()); - Signer signer = new SignerImpl(c.consensusKeyPair.getPrivate(), ULong.MIN); - Supplier> supp = pendingView(); + var signer = new SignerImpl(c.consensusKeyPair.getPrivate(), ULong.MIN); + var supp = pendingViews(); ViewContext vc = new GenesisContext(formation, supp, params, signer, constructBlock()); var inView = ViewMember.newBuilder(c.member).setView(params.genesisViewId().toDigeste()).build(); var svm = SignedViewMember.newBuilder() @@ -1380,7 +1451,6 @@ public SignedViewMember join(Digest nextView, Digest from) { return SignedViewMember.getDefaultInstance(); } final var c = next.get(); - var cd = pendingView().get(); var inView = ViewMember.newBuilder(c.member).setView(nextView.toDigeste()).build(); if (log.isDebugEnabled()) { @@ -1399,11 +1469,11 @@ public Logger log() { } @Override - public void nextView(Context pendingView) { + public void nextView(Digest diadem, Context pendingView) { log.info("Cancelling formation, acquiring new view, size: {} on: {}", pendingView.size(), params.member().getId()); params.context().setContext(pendingView); - CHOAM.this.pendingView.set(null); + pendingViews.add(diadem, pendingView); transitions.nextView(); } @@ -1465,10 +1535,10 @@ public Logger log() { } @Override - public void nextView(Context pendingView) { + public void nextView(Digest diadem, Context pendingView) { log.info("Acquiring new view, size: {} on: {}", pendingView.size(), params.member().getId()); params.context().setContext(pendingView); - CHOAM.this.pendingView.set(null); + pendingViews.add(diadem, pendingView); } @Override diff --git a/choam/src/main/java/com/salesforce/apollo/choam/Committee.java b/choam/src/main/java/com/salesforce/apollo/choam/Committee.java index a9c86256c..21734f6ad 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/Committee.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/Committee.java @@ -72,7 +72,7 @@ static Set viewMembersOf(Digest hash, Context baseContex Logger log(); - void nextView(Context pendingView); + void nextView(Digest diadem, Context pendingView); Parameters params(); diff --git a/choam/src/main/java/com/salesforce/apollo/choam/GenesisAssembly.java b/choam/src/main/java/com/salesforce/apollo/choam/GenesisAssembly.java index 9ff7eaae3..865a954b5 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/GenesisAssembly.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/GenesisAssembly.java @@ -64,7 +64,7 @@ public GenesisAssembly(ViewContext vc, CommonCommunications comms, String label) { view = vc; ds = new OneShot(); - nextAssembly = Committee.viewMembersOf(view.context().getId(), view.pendingView()) + nextAssembly = Committee.viewMembersOf(view.context().getId(), view.pendingViews().last().context()) .stream() .collect(Collectors.toMap(Member::getId, m -> m)); if (!Dag.validate(nextAssembly.size())) { diff --git a/choam/src/main/java/com/salesforce/apollo/choam/GenesisContext.java b/choam/src/main/java/com/salesforce/apollo/choam/GenesisContext.java index 3b00b0ad9..c2af3bcce 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/GenesisContext.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/GenesisContext.java @@ -21,7 +21,7 @@ */ public class GenesisContext extends ViewContext { - public GenesisContext(Context context, Supplier> pendingView, Parameters params, + public GenesisContext(Context context, Supplier pendingView, Parameters params, Signer signer, BlockProducer blockProducer) { super(context, params, pendingView, signer, Collections.emptyMap(), blockProducer); } diff --git a/choam/src/main/java/com/salesforce/apollo/choam/Producer.java b/choam/src/main/java/com/salesforce/apollo/choam/Producer.java index 961fa217f..5366d85c3 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/Producer.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/Producer.java @@ -159,11 +159,7 @@ private void addAssembly(Assemblies assemblies) { } } - private void create(List preblock, boolean last) { - if (log.isDebugEnabled()) { - log.debug("emit last: {} preblock: {} on: {}", last, - preblock.stream().map(DigestAlgorithm.DEFAULT::digest).toList(), params().member().getId()); - } + private List aggregate(List preblock) { var aggregate = preblock.stream().map(e -> { try { return UnitData.parseFrom(e); @@ -180,26 +176,70 @@ private void create(List preblock, boolean last) { .filter(p -> !p.published.get()) .filter(p -> p.witnesses.size() >= params().majority()) .forEach(this::publish); + return aggregate; + } + + private void create(List preblock, boolean last) { + if (log.isDebugEnabled()) { + log.debug("emit last: {} preblock: {} on: {}", last, + preblock.stream().map(DigestAlgorithm.DEFAULT::digest).toList(), params().member().getId()); + } + var aggregate = aggregate(preblock); + processAssemblies(aggregate); + processTransactions(last, aggregate); + if (last) { + started.set(true); + transitions.lastBlock(); + } + } + private Digest getViewId() { + return view.context().getId(); + } + + private void newEpoch(Integer epoch) { + log.trace("new epoch: {} on: {}", epoch, params().member().getId()); + transitions.newEpoch(epoch, lastEpoch); + } + + private Parameters params() { + return view.params(); + } + + private void processAssemblies(List aggregate) { var joins = aggregate.stream().flatMap(e -> e.getAssembliesList().stream()).toList(); final var ass = assembly; if (ass != null) { - log.trace("Consuming {} units, {} joins on: {}", aggregate.size(), joins.size(), params().member().getId()); + log.trace("Consuming {} units, {} assemblies on: {}", aggregate.size(), joins.size(), + params().member().getId()); ass.inbound().accept(joins); } else { - log.trace("Pending {} units, {} joins on: {}", aggregate.size(), joins.size(), params().member().getId()); + log.trace("Pending {} units, {} assemblies on: {}", aggregate.size(), joins.size(), + params().member().getId()); pendingAssemblies.addAll(joins); } + } + + private void processPendingValidations(HashedBlock block, PendingBlock p) { + var pending = pendingValidations.remove(block.hash); + if (pending != null) { + pending.forEach(v -> validate(v, p, block.hash)); + if (p.witnesses.size() >= params().majority()) { + publish(p); + } + } + } + private void processTransactions(boolean last, List aggregate) { HashedBlock lb = previousBlock.get(); final var txns = aggregate.stream().flatMap(e -> e.getTransactionsList().stream()).toList(); if (!txns.isEmpty()) { - log.trace("transactions: {} comb hash: {} height: {} on: {}", txns.size(), txns.stream() - .map(t -> CHOAM.hashOf(t, - params().digestAlgorithm())) - .reduce(Digest::xor) - .orElse(null), + log.trace("transactions: {} combined hash: {} height: {} on: {}", txns.size(), txns.stream() + .map(t -> CHOAM.hashOf(t, + params().digestAlgorithm())) + .reduce(Digest::xor) + .orElse(null), lb.height().add(1), params().member().getId()); var builder = Executions.newBuilder(); txns.forEach(builder::addExecutions); @@ -217,33 +257,6 @@ private void create(List preblock, boolean last) { next.hash, next.height(), lb.hash, last, params().member().getId()); processPendingValidations(next, p); } - if (last) { - started.set(true); - transitions.lastBlock(); - } - } - - private Digest getViewId() { - return view.context().getId(); - } - - private void newEpoch(Integer epoch) { - log.trace("new epoch: {} on: {}", epoch, params().member().getId()); - transitions.newEpoch(epoch, lastEpoch); - } - - private Parameters params() { - return view.params(); - } - - private void processPendingValidations(HashedBlock block, PendingBlock p) { - var pending = pendingValidations.remove(block.hash); - if (pending != null) { - pending.forEach(v -> validate(v, p, block.hash)); - if (p.witnesses.size() >= params().majority()) { - publish(p); - } - } } private void publish(PendingBlock p) { diff --git a/choam/src/main/java/com/salesforce/apollo/choam/ViewAssembly.java b/choam/src/main/java/com/salesforce/apollo/choam/ViewAssembly.java index 2db8e9d6f..b4a3b7b20 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/ViewAssembly.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/ViewAssembly.java @@ -7,15 +7,13 @@ package com.salesforce.apollo.choam; import com.chiralbehaviors.tron.Fsm; +import com.google.common.collect.HashMultiset; import com.salesforce.apollo.archipelago.RouterImpl.CommonCommunications; import com.salesforce.apollo.choam.comm.Terminal; import com.salesforce.apollo.choam.fsm.Reconfiguration; import com.salesforce.apollo.choam.fsm.Reconfiguration.Reconfigure; import com.salesforce.apollo.choam.fsm.Reconfiguration.Transitions; -import com.salesforce.apollo.choam.proto.Assemblies; -import com.salesforce.apollo.choam.proto.Join; -import com.salesforce.apollo.choam.proto.SignedJoin; -import com.salesforce.apollo.choam.proto.SignedViewMember; +import com.salesforce.apollo.choam.proto.*; import com.salesforce.apollo.context.Context; import com.salesforce.apollo.cryptography.Digest; import com.salesforce.apollo.cryptography.JohnHancock; @@ -42,9 +40,11 @@ import static com.salesforce.apollo.cryptography.QualifiedBase64.signature; /** - * View reconfiguration. Attempts to create a new view reconfiguration. View reconfiguration needs at least 2f+1 - * certified members from the next view. The protocol finishes with a list of at least 2f+1 Joins with at least 2f+1 - * certifications from the current view, or fails + * View reconfiguration. Attempts to create a new view reconfiguration. The protocol comes to an agreement on the + * underlying Context membership view diadem and thus the membership of the next assembly. Members propose their + * current views and reach 2/3 agreement on the diadem. The next assembly members are contacted to retrieve their view + * keys and signed commitments on such. Protocol terminates after this next slate of members stabilizes or the last + * epoch occurs. * * @author hal.hildebrand */ @@ -56,12 +56,12 @@ public class ViewAssembly { private final Digest nextViewId; private final Map proposals = new ConcurrentHashMap<>(); private final Consumer publisher; - private final Map slate = new ConcurrentSkipListMap<>(); + private final Map slate = new HashMap<>(); private final ViewContext view; private final CommonCommunications comms; private final Set polled = Collections.newSetFromMap( new ConcurrentSkipListMap<>()); - private volatile Map nextAssembly; + private volatile View selected; public ViewAssembly(Digest nextViewId, ViewContext vc, Consumer publisher, CommonCommunications comms) { @@ -74,12 +74,8 @@ public ViewAssembly(Digest nextViewId, ViewContext vc, Consumer publ Reconfigure.AWAIT_ASSEMBLY, true); this.transitions = fsm.getTransitions(); fsm.setName("View Assembly%s on: %s".formatted(nextViewId, params().member().getId())); - - nextAssembly = Committee.viewMembersOf(nextViewId, view.pendingView()) - .stream() - .collect(Collectors.toMap(Member::getId, m -> m)); - log.debug("View reconfiguration from: {} to: {}, next assembly: {} on: {}", view.context().getId(), nextViewId, - nextAssembly.keySet(), params().member().getId()); + log.debug("View reconfiguration from: {} to: {} on: {}", view.context().getId(), nextViewId, + params().member().getId()); } public Map getSlate() { @@ -103,7 +99,6 @@ void complete() { proposals.entrySet() .stream() .forEach(e -> slate.put(e.getKey(), Join.newBuilder().setMember(e.getValue()).build())); - Context pendingContext = view.pendingView(); log.debug("View Assembly: {} completed with: {} members on: {}", nextViewId, slate.size(), params().member().getId()); } @@ -114,13 +109,21 @@ void finalElection() { Consumer> inbound() { return lre -> { - lre.forEach(ass -> assemble(ass)); + lre.forEach(this::assemble); }; } private void assemble(Assemblies ass) { - ass.getJoinsList().stream().filter(sj -> view.validate(sj)).forEach(sj -> join(sj.getJoin(), false)); - ass.getViewsList().stream().filter(sv -> view.validate(sv)); + ass.getJoinsList().stream().filter(view::validate).forEach(sj -> join(sj.getJoin(), false)); + var candidates = HashMultiset.create(); + var majority = params().context().toleranceLevel() + 1; + for (SignedViews svs : ass.getViewsList()) { + if (view.validate(svs)) { + candidates.addAll(svs.getViews().getViewsList()); + } + } + candidates.entrySet().stream().filter(e -> e.getCount() >= majority).forEach(e -> { + }); } private void completeSlice(Duration retryDelay, AtomicReference reiterate) { @@ -129,7 +132,7 @@ private void completeSlice(Duration retryDelay, AtomicReference reiter } log.trace("Proposal incomplete of: {} polled: {}, total: {} retrying: {} on: {}", nextViewId, - polled.stream().sorted().toList(), nextAssembly.size(), retryDelay, params().member().getId()); + polled.stream().sorted().toList(), selected.assembly.size(), retryDelay, params().member().getId()); if (!cancelSlice.get()) { Executors.newScheduledThreadPool(1, Thread.ofVirtual().factory()) .schedule(() -> Thread.ofVirtual().start(Utils.wrapped(reiterate.get(), log)), @@ -159,7 +162,7 @@ private boolean consider(Optional futureSailor, Terminal term, } private boolean gathered() { - if (polled.size() == nextAssembly.size()) { + if (polled.size() == selected.assembly.size()) { log.trace("Polled all +"); cancelSlice.set(true); return true; @@ -169,7 +172,7 @@ private boolean gathered() { private void join(SignedViewMember svm, boolean direct) { final var mid = Digest.from(svm.getVm().getId()); - final var m = nextAssembly.get(mid); + final var m = selected.assembly.get(mid); if (m == null) { if (log.isTraceEnabled()) { log.trace("Invalid view member: {} on: {}", ViewContext.print(svm, params().digestAlgorithm()), @@ -244,21 +247,28 @@ private Parameters params() { return view.params(); } - private record Proposed(SignedViewMember vm, Member member) { + private record View(Digest diadem, Map assembly, Context context) { + public View(CHOAM.PendingView view, Digest viewId) { + this(view.diadem(), Committee.viewMembersOf(viewId, view.context()) + .stream() + .collect(Collectors.toMap(m -> m.getId(), m -> m)), view.context()); + + } } private class Recon implements Reconfiguration { @Override public void certify() { - if (proposals.size() == nextAssembly.size()) { + if (proposals.size() == selected.assembly.size()) { cancelSlice.set(true); - log.debug("Certifying: {} required: {} of: {} slate: {} on: {}", proposals.size(), nextAssembly.size(), - nextViewId, proposals.keySet().stream().sorted().toList(), params().member().getId()); + log.debug("Certifying: {} required: {} of: {} slate: {} on: {}", proposals.size(), + selected.assembly.size(), nextViewId, proposals.keySet().stream().sorted().toList(), + params().member().getId()); transitions.certified(); } else { log.debug("Not certifying: {} required: {} slate: {} of: {} on: {}", proposals.size(), - nextAssembly.size(), proposals.entrySet().stream().sorted().toList(), nextViewId, + selected.assembly.size(), proposals.entrySet().stream().sorted().toList(), nextViewId, params().member().getId()); } } @@ -270,15 +280,14 @@ public void complete() { @Override public void elect() { - proposals.entrySet().stream().forEach(e -> slate.put(e.getKey(), joinOf(e.getValue()))); - if (slate.size() == view.pendingView().getRingCount()) { + proposals.entrySet().forEach(e -> slate.put(e.getKey(), joinOf(e.getValue()))); + if (slate.size() == view.context().getRingCount()) { cancelSlice.set(true); log.debug("Electing: {} of: {} slate: {} on: {}", slate.size(), nextViewId, slate.keySet().stream().sorted().toList(), params().member().getId()); transitions.complete(); } else { - Context memberContext = view.pendingView(); - log.error("Failed election, required: {} slate: {} of: {} on: {}", view.pendingView().getRingCount(), + log.error("Failed election, required: {} slate: {} of: {} on: {}", view.context().getRingCount(), proposals.keySet().stream().sorted().toList(), nextViewId, params().member().getId()); } } @@ -292,14 +301,12 @@ public void failed() { @Override public void gather() { + selected = new View(view.pendingViews().last(), nextViewId); log.trace("Gathering assembly for: {} on: {}", nextViewId, params().member().getId()); AtomicReference reiterate = new AtomicReference<>(); var retryDelay = Duration.ofMillis(10); reiterate.set(() -> { - nextAssembly = Committee.viewMembersOf(nextViewId, view.pendingView()) - .stream() - .collect(Collectors.toMap(Member::getId, m -> m)); - var slice = new ArrayList<>(nextAssembly.values()); + var slice = new ArrayList<>(selected.assembly.values()); var committee = new SliceIterator<>("Committee for " + nextViewId, params().member(), slice, comms); committee.iterate((term, m) -> { if (polled.contains(m.getId())) { @@ -319,7 +326,7 @@ public void nominate() { @Override public void viewAgreement() { - + view.pendingViews(); } private Join joinOf(SignedViewMember vm) { diff --git a/choam/src/main/java/com/salesforce/apollo/choam/ViewContext.java b/choam/src/main/java/com/salesforce/apollo/choam/ViewContext.java index 2af557f0a..7220660be 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/ViewContext.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/ViewContext.java @@ -28,26 +28,26 @@ */ public class ViewContext { - private final static Logger log = LoggerFactory.getLogger(ViewContext.class); - private final BlockProducer blockProducer; - private final Context context; - private final Parameters params; - private final Map roster; - private final Signer signer; - private final Map validators; - private final Supplier> pendingView; - - public ViewContext(Context context, Parameters params, Supplier> pendingView, Signer signer, - Map validators, BlockProducer blockProducer) { + private final static Logger log = LoggerFactory.getLogger(ViewContext.class); + private final BlockProducer blockProducer; + private final Context context; + private final Parameters params; + private final Map roster; + private final Signer signer; + private final Map validators; + private final Supplier pendingViews; + + public ViewContext(Context context, Parameters params, Supplier pendingViews, + Signer signer, Map validators, BlockProducer blockProducer) { this.blockProducer = blockProducer; this.context = context; this.roster = new HashMap<>(); this.params = params; this.signer = signer; this.validators = validators; - this.pendingView = pendingView; + this.pendingViews = pendingViews; - var remapped = CHOAM.rosterMap(params.context(), context.allMembers().map(m -> m.getId()).toList()); + var remapped = CHOAM.rosterMap(params.context(), context.allMembers().map(Member::getId).toList()); short pid = 0; for (Digest d : remapped.keySet().stream().sorted().toList()) { roster.put(remapped.get(d).getId(), pid++); @@ -142,8 +142,8 @@ public Parameters params() { return params; } - public Context pendingView() { - return pendingView.get(); + public CHOAM.PendingViews pendingViews() { + return pendingViews.get(); } public Block produce(ULong l, Digest hash, Executions executions, HashedBlock checkpoint) { diff --git a/choam/src/test/java/com/salesforce/apollo/choam/GenesisAssemblyTest.java b/choam/src/test/java/com/salesforce/apollo/choam/GenesisAssemblyTest.java index 136b7e37a..679ab9e5e 100644 --- a/choam/src/test/java/com/salesforce/apollo/choam/GenesisAssemblyTest.java +++ b/choam/src/test/java/com/salesforce/apollo/choam/GenesisAssemblyTest.java @@ -138,8 +138,7 @@ public Block checkpoint() { @Override public Block genesis(Map joining, Digest nextViewId, HashedBlock previous) { - return CHOAM.genesis(viewId, joining, previous, committee, previous, built, previous, - Collections.emptyList()); + return CHOAM.genesis(viewId, joining, previous, previous, built, previous, Collections.emptyList()); } @Override @@ -163,7 +162,9 @@ public Block reconfigure(Map joining, Digest nextViewId, HashedBlo return null; } }; - var view = new GenesisContext(committee, () -> base, built, sm, reconfigure); + var pending = new CHOAM.PendingViews(); + pending.add(base.getId(), base); + var view = new GenesisContext(committee, () -> pending, built, sm, reconfigure); KeyPair keyPair = params.getViewSigAlgorithm().generateKeyPair(); final PubKey consensus = bs(keyPair.getPublic()); diff --git a/choam/src/test/resources/logback-test.xml b/choam/src/test/resources/logback-test.xml index b10bd4445..c1cacf519 100644 --- a/choam/src/test/resources/logback-test.xml +++ b/choam/src/test/resources/logback-test.xml @@ -24,6 +24,10 @@ + + + + @@ -36,7 +40,7 @@ - + diff --git a/grpc/src/main/proto/choam.proto b/grpc/src/main/proto/choam.proto index 0d301c1e1..31c5af11d 100644 --- a/grpc/src/main/proto/choam.proto +++ b/grpc/src/main/proto/choam.proto @@ -133,6 +133,12 @@ message View { repeated crypto.Digeste committee = 2; } +message SignedView { + crypto.Digeste member = 1; + View view = 2; + crypto.Sig signature = 3; +} + message ViewMember { crypto.Digeste id = 1; crypto.Digeste view = 2; diff --git a/pom.xml b/pom.xml index c1a3a9a7e..f09238cf9 100644 --- a/pom.xml +++ b/pom.xml @@ -553,16 +553,6 @@ ${graal.vm.version} provided - - org.netbeans.api - org-netbeans-modules-keyring - RELEASE200 - - - com.googlecode.concurrentlinkedhashmap - concurrentlinkedhashmap-lru - 1.4.2 - From 3e406a5f1b7a7f68489797a3f75e28cb07f63d6c Mon Sep 17 00:00:00 2001 From: Hellblazer Date: Thu, 18 Apr 2024 09:54:21 -0700 Subject: [PATCH 10/10] View agreement on reassembly. F*ck yea --- .../com/salesforce/apollo/choam/CHOAM.java | 6 + .../salesforce/apollo/choam/ViewAssembly.java | 122 +++++++++++++++--- .../salesforce/apollo/choam/ViewContext.java | 7 + .../apollo/choam/fsm/Reconfiguration.java | 7 +- .../apollo/cryptography/Digest.java | 3 + 5 files changed, 126 insertions(+), 19 deletions(-) diff --git a/choam/src/main/java/com/salesforce/apollo/choam/CHOAM.java b/choam/src/main/java/com/salesforce/apollo/choam/CHOAM.java index ce4d98875..2f8889e90 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/CHOAM.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/CHOAM.java @@ -1061,6 +1061,12 @@ public PendingView get(Digest diadem) { return views.get(diadem); } + public Views.Builder getViews(Digest hash) { + var builder = Views.newBuilder(); + views.values().stream().map(pv -> pv.getView(hash)).forEach(v -> builder.addViews(v)); + return builder; + } + public PendingView last() { final var l = lock.readLock(); try { diff --git a/choam/src/main/java/com/salesforce/apollo/choam/ViewAssembly.java b/choam/src/main/java/com/salesforce/apollo/choam/ViewAssembly.java index b4a3b7b20..351e74ee7 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/ViewAssembly.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/ViewAssembly.java @@ -8,15 +8,16 @@ import com.chiralbehaviors.tron.Fsm; import com.google.common.collect.HashMultiset; +import com.google.common.collect.Multiset; import com.salesforce.apollo.archipelago.RouterImpl.CommonCommunications; import com.salesforce.apollo.choam.comm.Terminal; import com.salesforce.apollo.choam.fsm.Reconfiguration; import com.salesforce.apollo.choam.fsm.Reconfiguration.Reconfigure; import com.salesforce.apollo.choam.fsm.Reconfiguration.Transitions; import com.salesforce.apollo.choam.proto.*; -import com.salesforce.apollo.context.Context; import com.salesforce.apollo.cryptography.Digest; import com.salesforce.apollo.cryptography.JohnHancock; +import com.salesforce.apollo.cryptography.proto.Digeste; import com.salesforce.apollo.cryptography.proto.PubKey; import com.salesforce.apollo.membership.Member; import com.salesforce.apollo.ring.SliceIterator; @@ -50,16 +51,17 @@ */ public class ViewAssembly { - private final static Logger log = LoggerFactory.getLogger(ViewAssembly.class); + private final static Logger log = LoggerFactory.getLogger(ViewAssembly.class); protected final Transitions transitions; - private final AtomicBoolean cancelSlice = new AtomicBoolean(); + private final AtomicBoolean cancelSlice = new AtomicBoolean(); private final Digest nextViewId; - private final Map proposals = new ConcurrentHashMap<>(); + private final Map viewProposals = new ConcurrentHashMap<>(); + private final Map proposals = new ConcurrentHashMap<>(); private final Consumer publisher; - private final Map slate = new HashMap<>(); + private final Map slate = new HashMap<>(); private final ViewContext view; private final CommonCommunications comms; - private final Set polled = Collections.newSetFromMap( + private final Set polled = Collections.newSetFromMap( new ConcurrentSkipListMap<>()); private volatile View selected; @@ -108,22 +110,62 @@ void finalElection() { } Consumer> inbound() { - return lre -> { - lre.forEach(this::assemble); - }; + return lre -> lre.forEach(this::assemble); } private void assemble(Assemblies ass) { + log.info("Assembling {} joins and {} views on: {}", ass.getJoinsCount(), ass.getViewsCount(), + params().member().getId()); ass.getJoinsList().stream().filter(view::validate).forEach(sj -> join(sj.getJoin(), false)); - var candidates = HashMultiset.create(); - var majority = params().context().toleranceLevel() + 1; + if (selected != null) { + log.info("View already selected: {} on: {}", selected.diadem, params().member().getId()); + return; + } for (SignedViews svs : ass.getViewsList()) { if (view.validate(svs)) { - candidates.addAll(svs.getViews().getViewsList()); + viewProposals.put(Digest.from(svs.getViews().getMember()), svs.getViews()); } } - candidates.entrySet().stream().filter(e -> e.getCount() >= majority).forEach(e -> { - }); + vote(); + } + + private Map assemblyOf(List committee) { + var last = view.pendingViews().last(); + return committee.stream() + .map(d -> last.context().getMember(Digest.from(d))) + .collect(Collectors.toMap(m -> m.getId(), m -> m)); + } + + private void castVote() { + var views = view.pendingViews() + .getViews(nextViewId) + .setMember(params().member().getId().toDigeste()) + .setVid(nextViewId.toDigeste()) + .build(); + log.info("Voting for: {} on: {}", nextViewId, params().member().getId()); + publisher.accept(Assemblies.newBuilder() + .addViews( + SignedViews.newBuilder().setViews(views).setSignature(view.sign(views).toSig())) + .build()); + } + + private void castVote(Views vs, List majorities, + Multiset consensus) { + var ordered = vs.getViewsList().stream().map(v -> Digest.from(v.getDiadem())).toList(); + var lastIndex = -1; + com.salesforce.apollo.choam.proto.View last = null; + for (var v : majorities) { + var i = ordered.indexOf(Digest.from(v.getDiadem())); + if (i != -1) { + if (i > lastIndex) { + last = v; + lastIndex = i; + } + } + } + if (last != null) { + consensus.add(last); + } } private void completeSlice(Duration retryDelay, AtomicReference reiterate) { @@ -247,11 +289,53 @@ private Parameters params() { return view.params(); } - private record View(Digest diadem, Map assembly, Context context) { + private void vote() { + Multiset candidates = HashMultiset.create(); + viewProposals.values().forEach(v -> candidates.addAll(v.getViewsList())); + var majority = params().majority(); + var majorities = candidates.entrySet() + .stream() + .filter(e -> e.getCount() >= majority) + .map(e -> e.getElement()) + .toList(); + if (majorities.isEmpty()) { + log.info("No majority views on: {}", params().member().getId()); + return; + } + if (log.isTraceEnabled()) { + log.trace("Majority views: {} on: {}", majorities.stream().map(v -> Digest.from(v.getDiadem())), + params().member().getId()); + } + Multiset consensus = HashMultiset.create(); + viewProposals.values().forEach(vs -> { + castVote(vs, majorities, consensus); + }); + var ratification = consensus.entrySet() + .stream() + .filter(e -> e.getCount() >= majority) + .map(e -> e.getElement()) + .collect(Collectors.toList()); + if (consensus.isEmpty()) { + log.info("No consensus views on: {}", params().member().getId()); + return; + } else if (log.isTraceEnabled()) { + log.trace("Consensus views: {} on: {}", ratification.stream().map(v -> Digest.from(v.getDiadem())), + params().member().getId()); + } + ratification.sort(Comparator.comparing(v -> Digest.from(v.getDiadem()))); + var winner = ratification.getFirst(); + selected = new View(Digest.from(winner.getDiadem()), assemblyOf(winner.getCommitteeList())); + if (log.isDebugEnabled()) { + log.debug("Selected view: {} on: {}", selected, params().member().getId()); + } + transitions.viewDetermined(); + } + + private record View(Digest diadem, Map assembly) { public View(CHOAM.PendingView view, Digest viewId) { this(view.diadem(), Committee.viewMembersOf(viewId, view.context()) .stream() - .collect(Collectors.toMap(m -> m.getId(), m -> m)), view.context()); + .collect(Collectors.toMap(m -> m.getId(), m -> m))); } } @@ -301,7 +385,9 @@ public void failed() { @Override public void gather() { - selected = new View(view.pendingViews().last(), nextViewId); + if (selected == null) { + selected = new View(view.pendingViews().last(), nextViewId); + } log.trace("Gathering assembly for: {} on: {}", nextViewId, params().member().getId()); AtomicReference reiterate = new AtomicReference<>(); var retryDelay = Duration.ofMillis(10); @@ -326,7 +412,7 @@ public void nominate() { @Override public void viewAgreement() { - view.pendingViews(); + ViewAssembly.this.castVote(); } private Join joinOf(SignedViewMember vm) { diff --git a/choam/src/main/java/com/salesforce/apollo/choam/ViewContext.java b/choam/src/main/java/com/salesforce/apollo/choam/ViewContext.java index 7220660be..b00dd32ec 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/ViewContext.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/ViewContext.java @@ -170,6 +170,13 @@ public JohnHancock sign(SignedViewMember svm) { return signer.sign(svm.toByteString()); } + public JohnHancock sign(Views views) { + if (log.isTraceEnabled()) { + log.trace("Signing views on: {}", params.member().getId()); + } + return signer.sign(views.toByteString()); + } + public boolean validate(HashedBlock block, Validate validate) { Verifier v = verifierOf(validate); if (v == null) { diff --git a/choam/src/main/java/com/salesforce/apollo/choam/fsm/Reconfiguration.java b/choam/src/main/java/com/salesforce/apollo/choam/fsm/Reconfiguration.java index df6705942..d69d515a3 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/fsm/Reconfiguration.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/fsm/Reconfiguration.java @@ -31,7 +31,7 @@ enum Reconfigure implements Transitions { AWAIT_ASSEMBLY { @Override public Transitions assembled() { - return GATHER; + return VIEW_AGREEMENT; } }, CERTIFICATION { @Override @@ -63,6 +63,11 @@ public void assembly() { public Transitions gathered() { return NOMINATION; } + + @Override + public Transitions viewDetermined() { + return null; + } }, NOMINATION { @Entry public void nominate() { diff --git a/cryptography/src/main/java/com/salesforce/apollo/cryptography/Digest.java b/cryptography/src/main/java/com/salesforce/apollo/cryptography/Digest.java index 1264b814e..de61d2eb4 100644 --- a/cryptography/src/main/java/com/salesforce/apollo/cryptography/Digest.java +++ b/cryptography/src/main/java/com/salesforce/apollo/cryptography/Digest.java @@ -123,6 +123,9 @@ public int compareTo(Digest id) { if (id == this) { return 0; } + if (hash.length != id.hash.length) { + throw new IllegalArgumentException("hash length incorrect for algorithm"); + } for (int i = 0; i < hash.length; i++) { int compare = Long.compareUnsigned(hash[i], id.hash[i]); if (compare != 0) {