From 695baaadd6695cb80fe5bc8a8ac8a40f4183dbd4 Mon Sep 17 00:00:00 2001 From: Hellblazer Date: Sun, 28 Jan 2024 13:27:05 -0800 Subject: [PATCH] Use the compact digest of the view diadem - do'h fix up view listener --- .../com/salesforce/apollo/choam/CHOAM.java | 44 +++++++++---------- .../com/salesforce/apollo/choam/Producer.java | 8 ++-- .../salesforce/apollo/choam/ViewAssembly.java | 7 ++- .../salesforce/apollo/choam/ViewContext.java | 4 +- .../apollo/choam/GenesisAssemblyTest.java | 10 ++--- .../apollo/choam/ViewAssemblyTest.java | 3 +- .../com/salesforce/apollo/fireflies/View.java | 21 +++++---- grpc/src/main/proto/choam.proto | 6 +-- .../apollo/demesnes/FireFliesTrace.java | 15 ++++--- .../apollo/model/ProcessDomain.java | 3 +- .../apollo/model/FireFliesTest.java | 11 +++-- 11 files changed, 62 insertions(+), 70 deletions(-) diff --git a/choam/src/main/java/com/salesforce/apollo/choam/CHOAM.java b/choam/src/main/java/com/salesforce/apollo/choam/CHOAM.java index 9d1583603..03c431f71 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/CHOAM.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/CHOAM.java @@ -87,12 +87,12 @@ public class CHOAM { private final Combine.Transitions transitions; private final TransSubmission txnSubmission = new TransSubmission(); private final AtomicReference view = new AtomicReference<>(); - private final AtomicReference diadem = new AtomicReference<>(); + private final AtomicReference diadem = new AtomicReference<>(); public CHOAM(Parameters params) { this.store = new Store(params.digestAlgorithm(), params.mvBuilder().clone().build()); this.params = params; - diadem.set(new HexBloom(params.digestAlgorithm().getLast())); + diadem.set(params.digestAlgorithm().getLast()); executions = Executors.newVirtualThreadPerTaskExecutor(); nextView(); @@ -175,7 +175,7 @@ public static Checkpoint checkpoint(DigestAlgorithm algo, File state, int segmen return cp; } - public static Block genesis(Digest id, HexBloom diadem, Map joins, HashedBlock head, + public static Block genesis(Digest id, Digest diadem, Map joins, HashedBlock head, Context context, HashedBlock lastViewChange, Parameters params, HashedBlock lastCheckpoint, Iterable initialization) { var reconfigure = reconfigure(id, diadem, joins, context, params, params.checkpointBlockDelta()); @@ -196,12 +196,12 @@ public static String print(Join join, DigestAlgorithm da) { + "certifications: " + join.getEndorsementsList().stream().map(c -> ViewContext.print(c, da)).toList() + "]"; } - public static Reconfigure reconfigure(Digest nextViewId, HexBloom diadem, Map joins, + public static Reconfigure reconfigure(Digest nextViewId, Digest diadem, Map joins, Context context, Parameters params, int checkpointTarget) { var builder = Reconfigure.newBuilder() .setCheckpointTarget(checkpointTarget) .setId(nextViewId.toDigeste()) - .setView(diadem.toHexBloome()); + .setView(diadem.toDigeste()); // Canonical labeling of the view members for Ethereal var remapped = rosterMap(context, joins.keySet()); @@ -212,7 +212,7 @@ public static Reconfigure reconfigure(Digest nextViewId, HexBloom diadem, Map joins, HashedBlock head, + public static Block reconfigure(Digest nextViewId, Digest diadem, Map joins, HashedBlock head, Context context, HashedBlock lastViewChange, Parameters params, HashedBlock lastCheckpoint) { final Block lvc = lastViewChange.block; @@ -299,8 +299,8 @@ public String logState() { params.member().getId()); } - public void setDiadem(HexBloom diadem) { - log.info("Set diadem: {} on: {}", diadem.compact(), params.member().getId()); + public void setDiadem(Digest diadem) { + log.info("Set diadem: {} on: {}", diadem, params.member().getId()); this.diadem.set(diadem); } @@ -308,7 +308,7 @@ public void start() { if (!started.compareAndSet(false, true)) { return; } - log.info("CHOAM startup: {} diadem: {}, majority: {} on: {}", params.context().getId(), diadem.get().compact(), + log.info("CHOAM startup: {} diadem: {}, majority: {} on: {}", params.context().getId(), diadem.get(), params.majority(), params.member().getId()); combine.start(params.producer().gossipDuration()); transitions.fsm().enterStartState(); @@ -475,7 +475,7 @@ public Block checkpoint() { } @Override - public HexBloom diadem() { + public Digest diadem() { return diadem.get(); } @@ -484,7 +484,7 @@ public Block genesis(Map joining, Digest nextViewId, HashedBlock p final HashedCertifiedBlock cp = checkpoint.get(); final HashedCertifiedBlock v = view.get(); var current = diadem.get(); - log.info("Create genesis: {}", diadem.get().compact()); + log.info("Create genesis: {}", diadem.get()); return CHOAM.genesis(nextViewId, current, joining, previous, params.context(), v, params, cp, params.genesisData().apply(joining)); } @@ -634,8 +634,7 @@ private void nextView() { var current = diadem.get(); log.trace("Generated next view consensus key: {} sig: {} diadem: {} on: {}", params.digestAlgorithm().digest(pubKey.getEncoded()), - params.digestAlgorithm().digest(signed.toSig().toByteString()), current.compact(), - params.member().getId()); + params.digestAlgorithm().digest(signed.toSig().toByteString()), current, params.member().getId()); next.set(new nextView(ViewMember.newBuilder() .setId(params.member().getId().toDigeste()) .setConsensusKey(pubKey) @@ -652,14 +651,13 @@ private void process() { case ASSEMBLE: { params.processor().beginBlock(h.height(), h.hash); nextViewId.set(Digest.from(h.block.getAssemble().getNextView())); - var ass = HexBloom.from(h.block.getAssemble().getDiadem()); + var ass = Digest.from(h.block.getAssemble().getDiadem()); var current = diadem.getAndSet(ass); - if (!current.equivalent(ass)) { - log.info("Next view id: {} diadem: {} does match current: {} on: {}", nextViewId.get(), ass.compact(), - current.compact(), params.member().getId()); + if (!current.equals(ass)) { + log.info("Next view id: {} diadem: {} does match current: {} on: {}", nextViewId.get(), ass, current, + params.member().getId()); } - log.info("Next view id: {} diadem: {} on: {}", nextViewId.get(), diadem.get().compact(), - params.member().getId()); + log.info("Next view id: {} diadem: {} on: {}", nextViewId.get(), diadem.get(), params.member().getId()); c.assembled(); break; } @@ -963,7 +961,7 @@ private void synchronizedProcess(CertifiedBlock certifiedBlock) { public interface BlockProducer { Block checkpoint(); - HexBloom diadem(); + Digest diadem(); Block genesis(Map joining, Digest nextViewId, HashedBlock previous); @@ -1159,7 +1157,7 @@ public ViewMember join(Digest nextView, Digest from) { log.debug("Joining view: {} from: {} view member: {} on: {}", nextView, from, ViewContext.print(c.member, params.digestAlgorithm()), params.member().getId()); } - return ViewMember.newBuilder(c.member).setDiadem(diadem.get().toIdentityHexBloome()).build(); + return ViewMember.newBuilder(c.member).setDiadem(diadem.get().toDigeste()).build(); } @Override @@ -1307,13 +1305,11 @@ public ViewMember join(Digest nextView, Digest from) { } final var c = next.get(); var cd = diadem.get(); - assert cd.equivalent(HexBloom.from(cd.toIdentityHexBloome())) : "Deser: {} not equal to: {}".formatted( - HexBloom.from(cd.toIdentityHexBloome()), cd); if (log.isDebugEnabled()) { log.debug("Joining view: {} diadem: {} from: {} view member: {} on: {}", nextView, cd, from, ViewContext.print(c.member, params.digestAlgorithm()), params.member().getId()); } - return ViewMember.newBuilder(c.member).setDiadem(cd.toIdentityHexBloome()).build(); + return ViewMember.newBuilder(c.member).setDiadem(cd.toDigeste()).build(); } @Override diff --git a/choam/src/main/java/com/salesforce/apollo/choam/Producer.java b/choam/src/main/java/com/salesforce/apollo/choam/Producer.java index 19198d000..c6f121cc8 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/Producer.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/Producer.java @@ -244,13 +244,13 @@ private void produceAssemble() { nextViewId = vlb.hash; nextAssembly.addAll(Committee.viewMembersOf(nextViewId, params().context())); var diadem = view.diadem(); - log.debug("Assembling: {} diadem: {} on: {}", nextViewId, diadem.compact(), params().member().getId()); + log.debug("Assembling: {} diadem: {} on: {}", nextViewId, diadem, params().member().getId()); final var assemble = new HashedBlock(params().digestAlgorithm(), view.produce(vlb.height().add(1), vlb.hash, Assemble.newBuilder() .setNextView( vlb.hash.toDigeste()) .setDiadem( - diadem.toIdentityHexBloome()) + diadem.toDigeste()) .build(), checkpoint.get())); previousBlock.set(assemble); @@ -373,14 +373,14 @@ public void produceAssemble() { @Override public void reconfigure() { - log.debug("Starting view reconfiguration: {} diadem: {} on: {}", nextViewId, view.diadem().compact(), + log.debug("Starting view reconfiguration: {} diadem: {} on: {}", nextViewId, view.diadem(), params().member().getId()); assembly.set(new ViewAssembly(nextViewId, view, Producer.this::addReassemble, comms) { @Override public void complete() { super.complete(); log.debug("View reconfiguration: {} diadem: {} gathered: {} complete on: {}", nextViewId, - view.diadem().compact(), getSlate().size(), params().member().getId()); + view.diadem(), getSlate().size(), params().member().getId()); assembled.set(true); Producer.this.transitions.viewComplete(); } diff --git a/choam/src/main/java/com/salesforce/apollo/choam/ViewAssembly.java b/choam/src/main/java/com/salesforce/apollo/choam/ViewAssembly.java index 8e2f2e76b..1b6040bdc 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/ViewAssembly.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/ViewAssembly.java @@ -14,7 +14,6 @@ import com.salesforce.apollo.choam.fsm.Reconfiguration.Transitions; import com.salesforce.apollo.choam.proto.*; import com.salesforce.apollo.cryptography.Digest; -import com.salesforce.apollo.cryptography.HexBloom; import com.salesforce.apollo.cryptography.proto.PubKey; import com.salesforce.apollo.membership.Member; import com.salesforce.apollo.ring.SliceIterator; @@ -199,10 +198,10 @@ private Reassemble join(ViewMember vm) { } return null; } - final var hex = HexBloom.from(vm.getDiadem()); + final var hex = Digest.from(vm.getDiadem()); var diadem = view.diadem(); - if (!diadem.equivalent(hex)) { - log.warn("Invalid diadem: {} not equivalent to: {} vm: {} on: {}", hex.compact(), diadem.compact(), + if (!diadem.equals(hex)) { + log.warn("Invalid diadem: {} not equivalent to: {} vm: {} on: {}", hex, diadem, ViewContext.print(vm, params().digestAlgorithm()), params().member().getId()); return null; } diff --git a/choam/src/main/java/com/salesforce/apollo/choam/ViewContext.java b/choam/src/main/java/com/salesforce/apollo/choam/ViewContext.java index bedbeb553..7022345d3 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/ViewContext.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/ViewContext.java @@ -74,8 +74,8 @@ public Context context() { return context; } - public HexBloom diadem() { - return blockProducer == null ? new HexBloom() : blockProducer.diadem(); + public Digest diadem() { + return blockProducer == null ? DigestAlgorithm.DEFAULT.getLast() : blockProducer.diadem(); } public Validate generateValidation(HashedBlock block) { diff --git a/choam/src/test/java/com/salesforce/apollo/choam/GenesisAssemblyTest.java b/choam/src/test/java/com/salesforce/apollo/choam/GenesisAssemblyTest.java index 65c1c990e..304f6341e 100644 --- a/choam/src/test/java/com/salesforce/apollo/choam/GenesisAssemblyTest.java +++ b/choam/src/test/java/com/salesforce/apollo/choam/GenesisAssemblyTest.java @@ -20,7 +20,6 @@ import com.salesforce.apollo.choam.support.HashedBlock; import com.salesforce.apollo.cryptography.Digest; import com.salesforce.apollo.cryptography.DigestAlgorithm; -import com.salesforce.apollo.cryptography.HexBloom; import com.salesforce.apollo.cryptography.Signer; import com.salesforce.apollo.cryptography.proto.PubKey; import com.salesforce.apollo.membership.Context; @@ -138,14 +137,14 @@ public Block checkpoint() { } @Override - public HexBloom diadem() { - return new HexBloom(); + public Digest diadem() { + return DigestAlgorithm.DEFAULT.getLast(); } @Override public Block genesis(Map joining, Digest nextViewId, HashedBlock previous) { - return CHOAM.genesis(viewId, new HexBloom(), joining, previous, committee, previous, built, - previous, Collections.emptyList()); + return CHOAM.genesis(viewId, DigestAlgorithm.DEFAULT.getLast(), joining, previous, committee, + previous, built, previous, Collections.emptyList()); } @Override @@ -169,7 +168,6 @@ public Block reconfigure(Map joining, Digest nextViewId, HashedBlo return null; } }; - HexBloom diadem = new HexBloom(); var view = new GenesisContext(committee, built, sm, reconfigure); KeyPair keyPair = params.getViewSigAlgorithm().generateKeyPair(); diff --git a/choam/src/test/java/com/salesforce/apollo/choam/ViewAssemblyTest.java b/choam/src/test/java/com/salesforce/apollo/choam/ViewAssemblyTest.java index 55db2e157..a4926251f 100644 --- a/choam/src/test/java/com/salesforce/apollo/choam/ViewAssemblyTest.java +++ b/choam/src/test/java/com/salesforce/apollo/choam/ViewAssemblyTest.java @@ -130,7 +130,7 @@ private void buildAssemblies() { public ViewMember answer(InvocationOnMock invocation) throws Throwable { return ViewMember.newBuilder() .setId(m.getId().toDigeste()) - .setDiadem(new HexBloom().toIdentityHexBloome()) + .setDiadem(DigestAlgorithm.DEFAULT.getLast().toDigeste()) .setConsensusKey(consensus) .setSignature(((Signer) m).sign(consensus.toByteString()).toSig()) .build(); @@ -161,7 +161,6 @@ public ViewMember answer(InvocationOnMock invocation) throws Throwable { e -> new Verifier.DefaultVerifier( e.getValue().getPublic()))); Map views = new HashMap<>(); - HexBloom diadem = new HexBloom(DigestAlgorithm.DEFAULT.getLast()); context.active().forEach(m -> { SigningMember sm = (SigningMember) m; Router router = communications.get(m); diff --git a/fireflies/src/main/java/com/salesforce/apollo/fireflies/View.java b/fireflies/src/main/java/com/salesforce/apollo/fireflies/View.java index a61eff86b..0b1ecf6ab 100644 --- a/fireflies/src/main/java/com/salesforce/apollo/fireflies/View.java +++ b/fireflies/src/main/java/com/salesforce/apollo/fireflies/View.java @@ -409,13 +409,12 @@ void introduced() { void notifyListeners(List joining, List leaving) { final var current = currentView(); viewNotificationQueue.execute(Utils.wrapped(() -> { - var diadem = viewManagement.diadem.get(); lifecycleListeners.forEach(listener -> { try { - log.trace("Notifying: {} view change: {} diadem: {} cardinality: {} joins: {} leaves: {} on: {} ", - listener, currentView(), diadem, context.totalCount(), joining.size(), leaving.size(), - node.getId()); - listener.viewChange(i -> context.getMember(i.getDigest()), diadem, current, joining, leaving); + log.trace("Notifying: {} view change: {} cardinality: {} joins: {} leaves: {} on: {} ", listener, + currentView(), context.totalCount(), joining.size(), leaving.size(), node.getId()); + listener.viewChange(i -> context.getMember(i.getDigest()), current, viewManagement.cardinality(), + joining, leaving); } catch (Throwable e) { log.error("error in view change listener: {} on: {} ", listener, node.getId(), e); } @@ -1476,13 +1475,13 @@ public interface ViewLifecycleListener { /** * Notification of a view change event * - * @param members - the source of Members for supplied identifiers - * @param diadem - the wrapped diadem of the view - * @param viewId - the Digest identity of the new view - * @param joins - the list of joining member's id - * @param leaves - the list of leaving member's id + * @param members - the source of Members for supplied identifiers + * @param viewId - the compact Digest identifying the new view + * @param cardinality - the cardinality of the new view + * @param joins - the list of joining member's id + * @param leaves - the list of leaving member's id */ - void viewChange(Function members, HexBloom diadem, Digest viewId, + void viewChange(Function members, Digest viewId, int cardinality, List joins, List leaves); } diff --git a/grpc/src/main/proto/choam.proto b/grpc/src/main/proto/choam.proto index 07fa87acd..c811abae1 100644 --- a/grpc/src/main/proto/choam.proto +++ b/grpc/src/main/proto/choam.proto @@ -69,7 +69,7 @@ message Genesis { message Reconfigure { crypto.Digeste id = 1; - crypto.HexBloome view = 2; + crypto.Digeste view = 2; int32 checkpointTarget = 3; repeated Join joins = 4; } @@ -87,7 +87,7 @@ message Executions { message Assemble { crypto.Digeste nextView = 1; - crypto.HexBloome diadem = 2; + crypto.Digeste diadem = 2; } message FoundationSeal { @@ -117,7 +117,7 @@ message Join { message ViewMember { crypto.Digeste id = 1; - crypto.HexBloome diadem = 2; + crypto.Digeste diadem = 2; crypto.PubKey consensusKey = 3; crypto.Sig signature = 4; } diff --git a/isolates/src/test/java/com/salesforce/apollo/demesnes/FireFliesTrace.java b/isolates/src/test/java/com/salesforce/apollo/demesnes/FireFliesTrace.java index ddfc30ddd..a0b09094a 100644 --- a/isolates/src/test/java/com/salesforce/apollo/demesnes/FireFliesTrace.java +++ b/isolates/src/test/java/com/salesforce/apollo/demesnes/FireFliesTrace.java @@ -16,7 +16,6 @@ import com.salesforce.apollo.choam.proto.FoundationSeal; import com.salesforce.apollo.cryptography.Digest; import com.salesforce.apollo.cryptography.DigestAlgorithm; -import com.salesforce.apollo.cryptography.HexBloom; import com.salesforce.apollo.delphinius.Oracle; import com.salesforce.apollo.delphinius.Oracle.Assertion; import com.salesforce.apollo.fireflies.View; @@ -228,14 +227,14 @@ public void smokin() throws Exception { var listener = new View.ViewLifecycleListener() { @Override - public void viewChange(Function context, HexBloom diadem, - Digest viewId, List joins, List leaves) { - if (diadem.getCardinality() == CARDINALITY) { - System.out.printf("Full view: %s members: %s on: %s%n", viewId, diadem.getCardinality(), + public void viewChange(Function context, Digest viewId, + int cardinality, List joins, List leaves) { + if (cardinality == CARDINALITY) { + System.out.printf("Full view: %s members: %s on: %s%n", viewId, cardinality, d.getMember().getId()); countdown.countDown(); } else { - System.out.printf("Members joining: %s members: %s on: %s%n", viewId, diadem.getCardinality(), + System.out.printf("Members joining: %s members: %s on: %s%n", viewId, cardinality, d.getMember().getId()); } } @@ -245,7 +244,9 @@ public void viewChange(Function cont // start seed final var started = new AtomicReference<>(new CountDownLatch(1)); - domains.get(0).getFoundation().start(() -> started.get().countDown(), gossipDuration, Collections.emptyList()); + domains.getFirst() + .getFoundation() + .start(() -> started.get().countDown(), gossipDuration, Collections.emptyList()); if (!started.get().await(10, TimeUnit.SECONDS)) { throw new IllegalStateException("Cannot start up kernel"); } diff --git a/model/src/main/java/com/salesforce/apollo/model/ProcessDomain.java b/model/src/main/java/com/salesforce/apollo/model/ProcessDomain.java index 3d2472a57..92c0a9988 100644 --- a/model/src/main/java/com/salesforce/apollo/model/ProcessDomain.java +++ b/model/src/main/java/com/salesforce/apollo/model/ProcessDomain.java @@ -116,7 +116,8 @@ public void stop() { } protected ViewLifecycleListener listener() { - return (context, diadem, id, join, leaving) -> { + return (context, id, cardinality, join, leaving) -> { + choam.setDiadem(id); for (var d : join) { params.context().activate(context.apply(d)); } diff --git a/model/src/test/java/com/salesforce/apollo/model/FireFliesTest.java b/model/src/test/java/com/salesforce/apollo/model/FireFliesTest.java index 4cb690d07..c4ea05526 100644 --- a/model/src/test/java/com/salesforce/apollo/model/FireFliesTest.java +++ b/model/src/test/java/com/salesforce/apollo/model/FireFliesTest.java @@ -16,7 +16,6 @@ import com.salesforce.apollo.choam.proto.FoundationSeal; import com.salesforce.apollo.cryptography.Digest; import com.salesforce.apollo.cryptography.DigestAlgorithm; -import com.salesforce.apollo.cryptography.HexBloom; import com.salesforce.apollo.delphinius.Oracle; import com.salesforce.apollo.fireflies.View; import com.salesforce.apollo.fireflies.View.Seed; @@ -114,14 +113,14 @@ public void smokin() throws Exception { var listener = new View.ViewLifecycleListener() { @Override - public void viewChange(Function context, HexBloom diadem, - Digest viewId, List joins, List leaves) { - if (diadem.getCardinality() == CARDINALITY) { - System.out.printf("Full view: %s members: %s on: %s%n", viewId, diadem.getCardinality(), + public void viewChange(Function context, Digest viewId, + int cardinality, List joins, List leaves) { + if (cardinality == CARDINALITY) { + System.out.printf("Full view: %s members: %s on: %s%n", viewId, cardinality, d.getMember().getId()); countdown.countDown(); } else { - System.out.printf("Members joining: %s members: %s on: %s%n", viewId, diadem.getCardinality(), + System.out.printf("Members joining: %s members: %s on: %s%n", viewId, cardinality, d.getMember().getId()); } }