diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index 542009684..634a83f87 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -4,7 +4,6 @@ on: [ push ] jobs: build: - runs-on: ubuntu-latest steps: diff --git a/.gitignore b/.gitignore index e63ce2e46..171458f3e 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ **/.classpath **/.DS_Store **/bin/ +**/.run/ /bin/ *.csv **/dependency-reduced-pom.xml diff --git a/choam/src/main/java/com/salesforce/apollo/choam/CHOAM.java b/choam/src/main/java/com/salesforce/apollo/choam/CHOAM.java index d0bf95adf..e014625c7 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/CHOAM.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/CHOAM.java @@ -880,9 +880,6 @@ private SubmitResult submit(Transaction request, Digest from) { } private Initial sync(Synchronize request, Digest from) { - if (from == null) { - return Initial.getDefaultInstance(); - } final HashedCertifiedBlock g = genesis.get(); if (g != null) { Initial.Builder initial = Initial.newBuilder(); @@ -897,13 +894,10 @@ private Initial sync(Synchronize request, Digest from) { } final ULong lastReconfig = ULong.valueOf(cp.block.getHeader().getLastReconfig()); HashedCertifiedBlock lastView = null; - if (lastReconfig.equals(ULong.valueOf(0))) { - lastView = cp; - } else { - var stored = store.getCertifiedBlock(lastReconfig); - if (stored != null) { - lastView = new HashedCertifiedBlock(params.digestAlgorithm(), stored); - } + + var stored = store.getCertifiedBlock(lastReconfig); + if (stored != null) { + lastView = new HashedCertifiedBlock(params.digestAlgorithm(), stored); } if (lastView == null) { lastView = g; @@ -1120,7 +1114,8 @@ public record PendingView(Digest diadem, Context context) { */ public View getView(Digest hash) { var builder = View.newBuilder().setDiadem(diadem.toDigeste()).setMajority(context.majority()); - Committee.viewMembersOf(hash, context).forEach(d -> builder.addCommittee(d.getId().toDigeste())); + ((Context) context).bftSubset(hash).forEach( + d -> builder.addCommittee(d.getId().toDigeste())); return builder.build(); } } @@ -1134,8 +1129,8 @@ public class Combiner implements Combine { public void anchor() { HashedCertifiedBlock anchor = pending.poll(); var pending = pendingViews.last().context; - if (anchor != null && pending.totalCount() >= pending.majority()) { - log.info("Synchronizing from anchor: {} cardinality: {} on: {}", anchor.hash, pending.totalCount(), + if (anchor != null && pending.size() >= pending.majority()) { + log.info("Synchronizing from anchor: {} cardinality: {} on: {}", anchor.hash, pending.size(), params.member().getId()); transitions.bootstrap(anchor); } @@ -1181,9 +1176,10 @@ public void awaitSynchronization() { synchronizationFailed(); } catch (IllegalStateException e) { final var c = current.get(); + Context memberContext = context(); log.debug( "Synchronization quorum formation failed: {}, members: {} desired: {} required: {}, no anchor to recover from: {} on: {}", - e.getMessage(), context().totalCount(), context().getRingCount(), params.majority(), + e.getMessage(), memberContext.size(), context().getRingCount(), params.majority(), c == null ? "" : c.getClass().getSimpleName(), params.member().getId()); awaitSynchronization(); } @@ -1227,7 +1223,8 @@ public void rotateViewKeys() { private void synchronizationFailed() { cancelSynchronization(); - var activeCount = context().totalCount(); + Context memberContext = context(); + var activeCount = memberContext.size(); var majority = params.majority(); if (params.generateGenesis() && activeCount >= majority) { if (current.get() == null && current.compareAndSet(null, new Formation())) { diff --git a/choam/src/main/java/com/salesforce/apollo/choam/Committee.java b/choam/src/main/java/com/salesforce/apollo/choam/Committee.java index bbb2713ec..ecc92f48e 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/Committee.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/Committee.java @@ -20,7 +20,6 @@ import io.grpc.StatusRuntimeException; import org.slf4j.Logger; -import java.util.HashSet; import java.util.Map; import java.util.Objects; import java.util.Set; @@ -63,23 +62,12 @@ static Map validatorsOf(Reconfigure reconfigure, Context viewFor(Digest hash, Context baseContext) { - Set successors = viewMembersOf(hash, baseContext); + Set successors = (Set) baseContext.bftSubset(hash); var newView = new StaticContext<>(hash, baseContext.getProbabilityByzantine(), 3, successors, baseContext.getEpsilon(), successors.size()); return newView; } - static Set viewMembersOf(Digest hash, Context baseContext) { - Set successors = new HashSet<>(); - baseContext.successors(hash, m -> { - if (successors.size() == baseContext.getRingCount()) { - return false; - } - return successors.add(m); - }); - return successors; - } - void accept(HashedCertifiedBlock next); default void assemble(Assemble assemble) { diff --git a/choam/src/main/java/com/salesforce/apollo/choam/GenesisAssembly.java b/choam/src/main/java/com/salesforce/apollo/choam/GenesisAssembly.java index cebbb1d38..f8cdbee1b 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/GenesisAssembly.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/GenesisAssembly.java @@ -17,6 +17,7 @@ import com.salesforce.apollo.choam.support.HashedCertifiedBlock; import com.salesforce.apollo.choam.support.HashedCertifiedBlock.NullBlock; import com.salesforce.apollo.choam.support.OneShot; +import com.salesforce.apollo.context.Context; import com.salesforce.apollo.context.StaticContext; import com.salesforce.apollo.cryptography.Digest; import com.salesforce.apollo.cryptography.proto.PubKey; @@ -64,9 +65,9 @@ public GenesisAssembly(ViewContext vc, CommonCommunications comms, String label) { view = vc; ds = new OneShot(); - nextAssembly = Committee.viewMembersOf(view.context().getId(), view.pendingViews().last().context()) - .stream() - .collect(Collectors.toMap(Member::getId, m -> m)); + Digest hash = view.context().getId(); + nextAssembly = ((Set) ((Context) view.pendingViews().last().context()).bftSubset( + hash)).stream().collect(Collectors.toMap(Member::getId, m -> m)); if (!Dag.validate(nextAssembly.size())) { throw new IllegalStateException("Invalid BFT cardinality: " + nextAssembly.size()); } @@ -96,7 +97,8 @@ public GenesisAssembly(ViewContext vc, CommonCommunications comms, config.setLabel("Genesis Assembly" + view.context().getId() + " on: " + params().member().getId()); controller = new Ethereal(config.build(), params().producer().maxBatchByteSize(), dataSource(), transitions::process, transitions::nextEpoch, label); - coordinator = new ChRbcGossip(reContext, params().member(), controller.processor(), params().communications(), + coordinator = new ChRbcGossip(reContext.getId(), params().member(), nextAssembly.values(), + controller.processor(), params().communications(), params().metrics() == null ? null : params().metrics().getGensisMetrics()); log.debug("Genesis Assembly: {} recontext: {} next assembly: {} on: {}", view.context().getId(), reContext.getId(), nextAssembly.keySet(), params().member().getId()); diff --git a/choam/src/main/java/com/salesforce/apollo/choam/Producer.java b/choam/src/main/java/com/salesforce/apollo/choam/Producer.java index 28519e234..8db4483ca 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/Producer.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/Producer.java @@ -79,8 +79,8 @@ public Producer(Digest nextViewId, ViewContext view, HashedBlock lastBlock, Hash producerParams.batchInterval(), producerParams.maxBatchCount(), params().drainPolicy().build()); - log.info("Producer max elements: {} reconfiguration epoch: {} on: {}", blocks, maxEpoch, - params.member().getId()); + log.debug("Producer max elements: {} reconfiguration epoch: {} on: {}", blocks, maxEpoch, + params.member().getId()); var fsm = Fsm.construct(new DriveIn(), Transitions.class, Earner.INITIAL, true); fsm.setName("Producer%s on: %s".formatted(getViewId(), params.member().getId())); @@ -99,10 +99,10 @@ public Producer(Digest nextViewId, ViewContext view, HashedBlock lastBlock, Hash config.setLabel("Producer" + getViewId() + " on: " + params().member().getId()); var producerMetrics = params().metrics() == null ? null : params().metrics().getProducerMetrics(); - controller = new Ethereal(config.build(), params().producer().maxBatchByteSize() + (8 * 1024), ds, - (preblock, last) -> serial(preblock, last), this::newEpoch, label); - coordinator = new ChRbcGossip(view.context(), params().member(), controller.processor(), - params().communications(), producerMetrics); + controller = new Ethereal(config.build(), params().producer().maxBatchByteSize() + (8 * 1024), ds, this::serial, + this::newEpoch, label); + coordinator = new ChRbcGossip(view.context().getId(), params().member(), view.membership(), + controller.processor(), params().communications(), producerMetrics); log.debug("Roster for: {} is: {} on: {}", getViewId(), view.roster(), params().member().getId()); var onConsensus = new CompletableFuture(); @@ -343,9 +343,9 @@ private void reconfigure() { pending.put(reconfiguration.hash, p); p.witnesses.put(params().member(), validation); ds.offer(validation); - log.info("Produced: {} hash: {} height: {} slate: {} on: {}", reconfiguration.block.getBodyCase(), - reconfiguration.hash, reconfiguration.height(), slate.keySet().stream().sorted().toList(), - params().member().getId()); + log.trace("Produced: {} hash: {} height: {} slate: {} on: {}", reconfiguration.block.getBodyCase(), + reconfiguration.hash, reconfiguration.height(), slate.keySet().stream().sorted().toList(), + params().member().getId()); processPendingValidations(reconfiguration, p); log.trace("Draining on: {}", params().member().getId()); diff --git a/choam/src/main/java/com/salesforce/apollo/choam/Session.java b/choam/src/main/java/com/salesforce/apollo/choam/Session.java index b680d0737..70d2148ea 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/Session.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/Session.java @@ -90,8 +90,8 @@ public static CompletableFuture retryNesting(Supplier cf = supplier.get(); for (int i = 0; i < maxRetries; i++) { final var attempt = i; - cf = cf.thenApply(CompletableFuture::completedFuture).exceptionally(__ -> { - log.trace("resubmitting after attempt: {}", attempt + 1); + cf = cf.thenApply(CompletableFuture::completedFuture).exceptionally(e -> { + log.info("resubmitting after attempt: {} exception: {}", attempt + 1, e.toString()); return supplier.get(); }).thenCompose(java.util.function.Function.identity()); } diff --git a/choam/src/main/java/com/salesforce/apollo/choam/ViewAssembly.java b/choam/src/main/java/com/salesforce/apollo/choam/ViewAssembly.java index 33d536491..baf5250ee 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/ViewAssembly.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/ViewAssembly.java @@ -107,7 +107,7 @@ void assemble(List asses) { .toList(); var views = asses.stream().flatMap(a -> a.getViewsList().stream()).filter(SignedViews::hasViews).toList(); - log.info("Assembling joins: {} views: {} on: {}", joins.size(), views.size(), params().member().getId()); + log.debug("Assembling joins: {} views: {} on: {}", joins.size(), views.size(), params().member().getId()); joins.forEach(sj -> join(sj.getJoin(), false)); if (selected != null) { @@ -124,7 +124,7 @@ void assemble(List asses) { Digest.from(svs.getViews().getMember()), params().member().getId()); viewProposals.put(Digest.from(svs.getViews().getMember()), svs.getViews()); } else { - log.info("Invalid views: {} from: {} on: {}", + log.warn("Invalid views: {} from: {} on: {}", svs.getViews().getViewsList().stream().map(v -> Digest.from(v.getDiadem())).toList(), Digest.from(svs.getViews().getMember()), params().member().getId()); } @@ -140,14 +140,14 @@ void assemble(List asses) { boolean complete() { if (selected == null) { - log.info("Cannot complete view assembly: {} as selected is null on: {}", nextViewId, - params().member().getId()); + log.error("Cannot complete view assembly: {} as selected is null on: {}", nextViewId, + params().member().getId()); transitions.failed(); return false; } if (proposals.size() < selected.majority) { - log.info("Cannot complete view assembly: {} proposed: {} required: {} on: {}", nextViewId, - proposals.keySet().stream().sorted().toList(), selected.majority, params().member().getId()); + log.error("Cannot complete view assembly: {} proposed: {} required: {} on: {}", nextViewId, + proposals.keySet().stream().sorted().toList(), selected.majority, params().member().getId()); transitions.failed(); return false; } @@ -290,9 +290,9 @@ private void propose() { .setMember(params().member().getId().toDigeste()) .setVid(nextViewId.toDigeste()) .build(); - log.info("Proposing for: {} views: {} on: {}", nextViewId, - views.getViewsList().stream().map(v -> Digest.from(v.getDiadem())).toList(), - params().member().getId()); + log.debug("Proposing for: {} views: {} on: {}", nextViewId, + views.getViewsList().stream().map(v -> Digest.from(v.getDiadem())).toList(), + params().member().getId()); publisher.accept(Assemblies.newBuilder() .addViews( SignedViews.newBuilder().setViews(views).setSignature(view.sign(views).toSig())) diff --git a/choam/src/main/java/com/salesforce/apollo/choam/ViewContext.java b/choam/src/main/java/com/salesforce/apollo/choam/ViewContext.java index a9678206d..c204030db 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/ViewContext.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/ViewContext.java @@ -19,6 +19,7 @@ import java.util.HashMap; import java.util.Map; +import java.util.Set; import java.util.function.Supplier; import static com.salesforce.apollo.cryptography.QualifiedBase64.publicKey; @@ -135,6 +136,10 @@ public Signer getSigner() { return signer; } + public Set membership() { + return validators.keySet(); + } + /** * The process has failed */ @@ -244,7 +249,7 @@ public boolean validate(SignedViews sv) { Verifier v = verifierOf(sv); if (v == null) { if (log.isDebugEnabled()) { - log.debug("no verifier: {} for signed view on: {}", Digest.from(sv.getViews().getMember()), + log.debug("No verifier: {} for signed view on: {}", Digest.from(sv.getViews().getMember()), params.member().getId()); } return false; diff --git a/choam/src/main/java/com/salesforce/apollo/choam/support/Bootstrapper.java b/choam/src/main/java/com/salesforce/apollo/choam/support/Bootstrapper.java index 60ccc78d3..35155358c 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/support/Bootstrapper.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/support/Bootstrapper.java @@ -28,14 +28,12 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.Comparator; -import java.util.HashMap; -import java.util.Map; -import java.util.Optional; +import java.util.*; import java.util.concurrent.CompletableFuture; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; @@ -43,22 +41,23 @@ * @author hal.hildebrand */ public class Bootstrapper { - private static final Logger log = LoggerFactory.getLogger( - Bootstrapper.class); - private final HashedCertifiedBlock anchor; - private final CompletableFuture anchorSynchronized = new CompletableFuture<>(); - private final CommonCommunications comms; - private final ULong lastCheckpoint; - private final Parameters params; - private final Store store; - private final CompletableFuture sync = new CompletableFuture<>(); - private final CompletableFuture viewChainSynchronized = new CompletableFuture<>(); - private final ScheduledExecutorService scheduler; - private volatile HashedCertifiedBlock checkpoint; - private volatile CompletableFuture checkpointAssembled; - private volatile CheckpointState checkpointState; - private volatile HashedCertifiedBlock checkpointView; - private volatile HashedCertifiedBlock genesis; + private static final Logger log = LoggerFactory.getLogger(Bootstrapper.class); + + private final HashedCertifiedBlock anchor; + private final CompletableFuture anchorSynchronized = new CompletableFuture<>(); + private final CommonCommunications comms; + private final ULong lastCheckpoint; + private final Parameters params; + private final Store store; + private final CompletableFuture sync = new CompletableFuture<>(); + private final CompletableFuture viewChainSynchronized = new CompletableFuture<>(); + private final ScheduledExecutorService scheduler; + private final AtomicInteger sampleIndex = new AtomicInteger(); + private volatile HashedCertifiedBlock checkpoint; + private volatile CompletableFuture checkpointAssembled; + private volatile CheckpointState checkpointState; + private volatile HashedCertifiedBlock checkpointView; + private volatile HashedCertifiedBlock genesis; public Bootstrapper(HashedCertifiedBlock anchor, Parameters params, Store store, CommonCommunications bootstrapComm) { @@ -85,12 +84,19 @@ public CompletableFuture synchronize() { } private void anchor(AtomicReference start, ULong end) { + if (end.equals(ULong.valueOf(0)) && start.get().equals(ULong.valueOf(1))) { + validateAnchor(); + return; + } final var randomCut = params.digestAlgorithm().random(); log.trace("Anchoring from: {} to: {} cut: {} on: {}", start.get(), end, randomCut, params.member().getId()); - new RingIterator<>(params.gossipDuration(), params.context(), params.member(), comms, true, scheduler).iterate( - randomCut, (link, ring) -> anchor(link, start, end), - (tally, futureSailor, destination) -> completeAnchor(futureSailor, start, end, destination), - t -> scheduleAnchorCompletion(start, end)); + var iterator = new RingIterator(params.gossipDuration(), params.context(), params.member(), + comms, true, scheduler); + iterator.ignoreSelf(); + iterator.noDuplicates(); + iterator.iterate(randomCut, (link, ring) -> anchor(link, start, end), + (tally, futureSailor, destination) -> completeAnchor(futureSailor, start, end, destination), + t -> scheduleAnchorCompletion(start, end)); } private Blocks anchor(Terminal link, AtomicReference start, ULong end) { @@ -119,11 +125,20 @@ private void checkpointCompletion(int threshold, Initial mostRecent) { assert !checkpointView.height() .equals(Unsigned.ulong(0)) : "Should not attempt when bootstrapping from genesis"; var crown = HexBloom.from(checkpoint.block.getCheckpoint().getCrown()); + assert !crown.crowns().isEmpty() : "Crowns should not be empty"; log.info("Assembling from checkpoint: {}:{} crown: {} last cp: {} on: {}", checkpoint.height(), checkpoint.hash, crown.compactWrapped(), Digest.from(checkpoint.block.getHeader().getLastCheckpointHash()), params.member().getId()); - - CheckpointAssembler assembler = new CheckpointAssembler(params.gossipDuration(), checkpoint.height(), + var committee = checkpointView.certifiedBlock.getBlock() + .getReconfigure() + .getJoinsList() + .stream() + .map(j -> j.getMember().getVm().getId()) + .map(Digest::from) + .map(d -> params.context().getMember(d)) + .filter(Objects::nonNull) + .toList(); + CheckpointAssembler assembler = new CheckpointAssembler(committee, params.gossipDuration(), checkpoint.height(), checkpoint.block.getCheckpoint(), params.member(), store, comms, params.context(), threshold, params.digestAlgorithm()); @@ -179,10 +194,13 @@ private boolean completeAnchor(Optional futureSailor, AtomicReference
    start, ULong end) { - new RingIterator<>(params.gossipDuration(), params.context(), params.member(), scheduler, comms).iterate( - params.digestAlgorithm().random(), (link, ring) -> completeViewChain(link, start, end), - (tally, result, destination) -> completeViewChain(result, start, end, destination), - t -> scheduleViewChainCompletion(start, end)); + var iterator = new RingIterator(params.gossipDuration(), params.context(), params.member(), + scheduler, comms); + iterator.noDuplicates(); + iterator.ignoreSelf(); + iterator.iterate(params.digestAlgorithm().random(), (link, ring) -> completeViewChain(link, start, end), + (tally, result, destination) -> completeViewChain(result, start, end, destination), + t -> scheduleViewChainCompletion(start, end)); } private boolean completeViewChain(Optional futureSailor, AtomicReference start, ULong end, @@ -244,33 +262,15 @@ private void computeGenesis(Map votes) { // If restoring from known genesis... .filter(e -> genesis == null || genesis.hash.equals(e.getKey())) .filter(e -> { - if (e.getValue().hasGenesis()) { - if (lastCheckpoint != null - && lastCheckpoint.compareTo(ULong.valueOf(0)) > 0) { - log.trace( - "Rejecting genesis from: {} last checkpoint: {} > 0 on: {}", - e.getKey(), lastCheckpoint, params.member().getId()); - return false; - } - log.trace("Accepting genesis from: {} on: {}", e.getKey(), - params.member().getId()); - return true; - } - if (!e.getValue().hasCheckpoint()) { - log.trace( - "Rejecting: {} has no checkpoint. last checkpoint from: {} > 0 on: {}", - e.getKey(), lastCheckpoint, params.member().getId()); + if (lastCheckpoint != null + && lastCheckpoint.compareTo(ULong.valueOf(0)) > 0) { + log.trace("Rejecting genesis from: {} last checkpoint: {} > 0 on: {}", + e.getKey(), lastCheckpoint, params.member().getId()); return false; } - - ULong checkpointViewHeight = HashedBlock.height( - e.getValue().getCheckpointView().getBlock()); - ULong recordedCheckpointViewHeight = ULong.valueOf( - e.getValue().getCheckpoint().getBlock().getHeader().getLastReconfig()); - // checkpoint's view should match - log.trace("Accepting checkpoint from: {} on: {}", e.getKey(), + log.trace("Accepting genesis from: {} on: {}", e.getKey(), params.member().getId()); - return checkpointViewHeight.equals(recordedCheckpointViewHeight); + return true; }) .peek(e -> tally.add(new HashedCertifiedBlock(params.digestAlgorithm(), e.getValue().getGenesis()))) @@ -343,15 +343,15 @@ private void computeGenesis(Map votes) { checkpointView == null ? genesis.hash : checkpoint.hash, params.member().getId()); sync.complete(new SynchronizedState(genesis, checkpointView, checkpoint, checkpointState)); } else { - log.error("Failed synchronizing to {} from: {} last view: {} on: {}", genesis.hash, + log.error("Failed synchronizing to: {} from: {} last view: {} on: {}", genesis.hash, checkpoint == null ? genesis.hash : checkpoint.hash, - checkpointView == null ? genesis.hash : checkpoint.hash, t); + checkpointView == null ? genesis.hash : checkpoint.hash, params.member().getId(), t); sync.completeExceptionally(t); } }).exceptionally(t -> { - log.error("Failed synchronizing to {} from: {} last view: {} on: {}", genesis.hash, + log.error("Failed synchronizing to: {} from: {} last view: {} on: {}", genesis.hash, checkpoint == null ? genesis.hash : checkpoint.hash, - checkpointView == null ? genesis.hash : checkpoint.hash, t); + checkpointView == null ? genesis.hash : checkpoint.hash, params.member().getId(), t); sync.completeExceptionally(t); return null; }); @@ -366,11 +366,22 @@ private void sample() { } HashMap votes = new HashMap<>(); Synchronize s = Synchronize.newBuilder().setHeight(anchor.height().longValue()).build(); - final var randomCut = params.digestAlgorithm().random(); - new RingIterator<>(params.gossipDuration(), params.context(), params.member(), comms, true, scheduler).iterate( - randomCut, (link, ring) -> synchronize(s, link), - (tally, futureSailor, destination) -> synchronize(futureSailor, votes, destination), - t -> computeGenesis(votes)); + var si = sampleIndex.getAndIncrement(); + var member = params.context().getMember(si, Entropy.nextBitsStreamInt(params.context().getRingCount())); + if (member == null) { + log.warn("No members: {} to sample on: {}", params.context().size(), params.member().getId()); + computeGenesis(votes); + return; + } + var randomCut = member.getId(); + log.info("Random cut: {} on: {}", randomCut, params.member().getId()); + var iterator = new RingIterator<>(params.gossipDuration(), params.context(), params.member(), comms, true, + scheduler); + iterator.allowDuplicates(); + iterator.ignoreSelf(); + iterator.iterate(randomCut, (link, _) -> synchronize(s, link), + (_, futureSailor, destination) -> synchronize(futureSailor, votes, destination), + t -> computeGenesis(votes)); } private void scheduleAnchorCompletion(AtomicReference start, ULong anchorTo) { diff --git a/choam/src/main/java/com/salesforce/apollo/choam/support/CheckpointAssembler.java b/choam/src/main/java/com/salesforce/apollo/choam/support/CheckpointAssembler.java index 809d764fc..e382e8f5b 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/support/CheckpointAssembler.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/support/CheckpointAssembler.java @@ -19,7 +19,7 @@ import com.salesforce.apollo.cryptography.HexBloom; import com.salesforce.apollo.membership.Member; import com.salesforce.apollo.membership.SigningMember; -import com.salesforce.apollo.ring.RingIterator; +import com.salesforce.apollo.ring.SliceIterator; import com.salesforce.apollo.utils.Entropy; import com.salesforce.apollo.utils.Utils; import org.h2.mvstore.MVMap; @@ -28,6 +28,8 @@ import org.slf4j.LoggerFactory; import java.time.Duration; +import java.util.ArrayList; +import java.util.List; import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ScheduledExecutorService; @@ -51,10 +53,12 @@ public class CheckpointAssembler { private final SigningMember member; private final MVMap state; private final HexBloom diadem; + private final List committee; - public CheckpointAssembler(Duration frequency, ULong height, Checkpoint checkpoint, SigningMember member, - Store store, CommonCommunications comms, Context context, - double falsePositiveRate, DigestAlgorithm digestAlgorithm) { + public CheckpointAssembler(List committee, Duration frequency, ULong height, Checkpoint checkpoint, + SigningMember member, Store store, CommonCommunications comms, + Context context, double falsePositiveRate, DigestAlgorithm digestAlgorithm) { + this.committee = new ArrayList<>(committee); this.height = height; this.member = member; this.checkpoint = checkpoint; @@ -112,11 +116,19 @@ private void gossip(ScheduledExecutorService scheduler, Duration duration) { } log.info("Assembly of checkpoint: {} segments: {} crown: {} on: {}", height, checkpoint.getCount(), diadem.compactWrapped(), member.getId()); - var ringer = new RingIterator<>(frequency, context, member, comms, true, scheduler); - ringer.iterate(digestAlgorithm.random(), (link, ring) -> gossip(link), - (tally, result, destination) -> gossip(result), t -> scheduler.schedule( - () -> Thread.ofVirtual().start(Utils.wrapped(() -> gossip(scheduler, duration), log)), duration.toMillis(), - TimeUnit.MILLISECONDS)); + + var ringer = new SliceIterator<>("Assembly[%s:%s]".formatted(diadem.compactWrapped(), member.getId()), member, + committee, comms); + ringer.iterate((link, m) -> { + log.debug("Requesting Seeding from: {} on: {}", link.getMember().getId(), member.getId()); + return gossip(link); + }, (result, link, m) -> gossip(result), () -> { + if (!assembled.isDone()) { + scheduler.schedule( + () -> Thread.ofVirtual().start(Utils.wrapped(() -> gossip(scheduler, duration), log)), + duration.toMillis(), TimeUnit.MILLISECONDS); + } + }, duration); } diff --git a/choam/src/test/java/com/salesforce/apollo/choam/TestCHOAM.java b/choam/src/test/java/com/salesforce/apollo/choam/TestCHOAM.java index 4efedc695..8f0aff039 100644 --- a/choam/src/test/java/com/salesforce/apollo/choam/TestCHOAM.java +++ b/choam/src/test/java/com/salesforce/apollo/choam/TestCHOAM.java @@ -191,7 +191,7 @@ public void submitMultiplTxn() throws Exception { transactioneers.stream().forEach(e -> e.start()); try { - final var complete = countdown.await(LARGE_TESTS ? 3200 : 120, TimeUnit.SECONDS); + final var complete = countdown.await(LARGE_TESTS ? 3200 : 240, TimeUnit.SECONDS); assertTrue(complete, "All clients did not complete: " + transactioneers.stream() .map(t -> t.getCompleted()) .filter(i -> i < max) diff --git a/choam/src/test/java/com/salesforce/apollo/choam/support/CheckpointAssemblerTest.java b/choam/src/test/java/com/salesforce/apollo/choam/support/CheckpointAssemblerTest.java index 6a0e2199e..8a1f95dcf 100644 --- a/choam/src/test/java/com/salesforce/apollo/choam/support/CheckpointAssemblerTest.java +++ b/choam/src/test/java/com/salesforce/apollo/choam/support/CheckpointAssemblerTest.java @@ -125,12 +125,18 @@ public CheckpointSegments answer(InvocationOnMock invocation) throws Throwable { return CheckpointSegments.newBuilder().addAllSegments(fetched).build(); } }); + when(client.getMember()).then(new Answer<>() { + @Override + public Member answer(InvocationOnMock invocation) { + return members.get(1); + } + }); @SuppressWarnings("unchecked") CommonCommunications comm = mock(CommonCommunications.class); when(comm.connect(any())).thenReturn(client); Store store2 = new Store(DigestAlgorithm.DEFAULT, new MVStore.Builder().open()); - CheckpointAssembler boot = new CheckpointAssembler(Duration.ofMillis(10), ULong.valueOf(0), checkpoint, + CheckpointAssembler boot = new CheckpointAssembler(members, Duration.ofMillis(10), ULong.valueOf(0), checkpoint, bootstrapping, store2, comm, context, 0.00125, DigestAlgorithm.DEFAULT); ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1, Thread.ofVirtual().factory()); diff --git a/choam/src/test/resources/logback-test.xml b/choam/src/test/resources/logback-test.xml index 830a6923d..d5be6a052 100644 --- a/choam/src/test/resources/logback-test.xml +++ b/choam/src/test/resources/logback-test.xml @@ -25,6 +25,10 @@ + + + + @@ -33,7 +37,7 @@ - + diff --git a/cryptography/src/main/java/com/salesforce/apollo/bloomFilters/Hash.java b/cryptography/src/main/java/com/salesforce/apollo/bloomFilters/Hash.java index 010200f17..f0b431a19 100644 --- a/cryptography/src/main/java/com/salesforce/apollo/bloomFilters/Hash.java +++ b/cryptography/src/main/java/com/salesforce/apollo/bloomFilters/Hash.java @@ -6,20 +6,142 @@ */ package com.salesforce.apollo.bloomFilters; -import java.nio.ByteBuffer; -import java.util.stream.IntStream; - +import com.salesforce.apollo.cryptography.Digest; import org.joou.ULong; -import com.salesforce.apollo.cryptography.Digest; +import java.nio.ByteBuffer; +import java.util.stream.IntStream; /** * @author hal.hildebrand - * */ public abstract class Hash { -// public static int HITS = 0; -// public static int MISSES = 0; + // public static int HITS = 0; + // public static int MISSES = 0; + + public static final long MERSENNE_31 = (long) (Math.pow(2, 32) - 1); // 2147483647 + public final int k; + public final int m; + public final long seed; + protected final Hasher hasher; + + public Hash(long seed, int n, double p) { + m = optimalM(n, p); + k = optimalK(n, m); + assert m > 0 && k <= 30; + this.seed = seed; + hasher = newHasher(); + } + + public Hash(long seed, int k, int m) { + assert m > 0 && k <= 30; + this.seed = seed; + this.k = k; + this.m = m; + hasher = newHasher(); + } + + /** + * @param k - the number of hashes + * @param m - the number of entries, bits or counters that K hashes to + * @param n - the number of elements in the set + * @return the false positive probability for the specified number of hashes K, population M entries and N elements + */ + public static double fpp(int k, int m, int n) { + double Kd = k; + double Md = m; + double Nd = n; + return Math.pow(1 - Math.exp(-Kd / (Md / Nd)), Kd); + } + + /** + * @param m - the number of entries (bits) + * @param k - the number of hashes + * @param fpp - the false positive probability + * @return the number of elements that a bloom filter can hold with M entries (bits) K hashes and the specified + * false positive rate + */ + public static int n(int m, int k, double fpp) { + double Kd = k; + double Md = m; + return (int) Math.ceil(Md / (-Kd / Math.log(1 - Math.exp(Math.log(fpp) / Kd)))); + } + + /** + * Computes the optimal k (number of hashes per element inserted in Bloom filter), given the expected insertions and + * total number of bits in the Bloom filter. + * + * See http://en.wikipedia.org/wiki/File:Bloom_filter_fp_probability.svg for the formula. + * + * @param n expected insertions (must be positive) + * @param m total number of bits in Bloom filter (must be positive) + */ + public static int optimalK(long n, long m) { + // (m / n) * log(2), but avoid truncation due to division! + return Math.max(1, (int) Math.round(((double) m) / ((double) n) * Math.log(2))); + } + + /** + * Computes m (total bits of Bloom filter) which is expected to achieve, for the specified expected insertions, the + * required false positive probability. + * + * See http://en.wikipedia.org/wiki/Bloom_filter#Probability_of_false_positives for the formula. + * + * @param n expected insertions (must be positive) + * @param p false positive rate (must be 0 < p < 1) + */ + public static int optimalM(long n, double p) { + if (p == 0) { + p = Double.MIN_VALUE; + } + return Math.max(8, (int) (-n * Math.log(p) / (Math.log(2) * Math.log(2)))); + } + + @Override + public Hash clone() { + Hasher clone = hasher.clone(); + return new Hash(seed, k, m) { + + @Override + protected Hasher newHasher() { + return clone; + } + }; + } + + public boolean equivalent(Hash other) { + return getK() == other.getK() && getM() == other.getM() && getSeed() == other.getSeed(); + } + + public double fpp(int n) { + return fpp(k, m, n); + } + + public int getK() { + return k; + } + + public int getM() { + return m; + } + + public long getSeed() { + return seed; + } + + public int[] hashes(M key) { + return hasher.hashes(k, key, m, seed); + } + + public long identityHash(M key) { + return hasher.identityHash(key, seed); + } + + public IntStream locations(M key) { + return hasher.locations(k, key, m, seed); + } + + abstract protected Hasher newHasher(); public static class BytesHasher extends Hasher { @@ -69,27 +191,25 @@ abstract public static class Hasher { private static final long C2 = 0x4cf5ad432745937fL; private static final long CHUNK_SIZE = 16; private static final int MAX_HASHING_ATTEMPTS = 500; - - static int toInt(byte value) { - return value & 0xFF; - } - - private static void throwMax(int k, int m, int[] hashes) { - throw new IllegalStateException("Cannot find: " + k + " unique hashes for m: " + m + " after: " - + MAX_HASHING_ATTEMPTS + " hashing attempts. found: " + IntStream.of(hashes).mapToObj(e -> e).toList()); - } - long h1; long h2; int length; - public Hasher() { } - public Hasher(M key, long seed) { process(key, seed); } + static int toInt(byte value) { + return value & 0xFF; + } + + private static void throwMax(int k, int m, int[] hashes) { + throw new IllegalStateException( + "Cannot find: " + k + " unique hashes for m: " + m + " after: " + MAX_HASHING_ATTEMPTS + + " hashing attempts. found: " + IntStream.of(hashes).mapToObj(e -> e).toList()); + } + /** * Generate K unique hash locations for M elements, using the seed. */ @@ -114,11 +234,11 @@ public int[] hashes(int k, M key, int m, long seed) { } if (!found) { hashes[i++] = hash; -// HITS++; + // HITS++; } else { h2 += Primes.PRIMES[prime]; prime = ++prime % Primes.PRIMES.length; -// MISSES++; + // MISSES++; } combinedHash += h2; } @@ -149,9 +269,6 @@ public void processAdditional(M value) { makeHash(); } - @Override - protected abstract Hasher clone(); - IntStream locations(int k, M key, int m, long seed) { return IntStream.of(hashes(k, key, m, seed)); } @@ -203,6 +320,9 @@ void process(String key) { abstract void processIt(M key); + @Override + protected abstract Hasher clone(); + private void bmix64(long k1, long k2) { h1 ^= mixK1(k1); @@ -392,133 +512,4 @@ protected void processIt(ULong key) { } } - - public static final long MERSENNE_31 = (long) (Math.pow(2, 32) - 1); // 2147483647 - - /** - * @param k - the number of hashes - * @param m - the number of entries, bits or counters that K hashes to - * @param n - the number of elements in the set - * - * @return the false positive probability for the specified number of hashes K, - * population M entries and N elements - */ - public static double fpp(int k, int m, int n) { - double Kd = k; - double Md = m; - double Nd = n; - return Math.pow(1 - Math.exp(-Kd / (Md / Nd)), Kd); - } - - /** - * @param m - the number of entries (bits) - * @param k - the number of hashes - * @param fpp - the false positive probability - * @return the number of elements that a bloom filter can hold with M entries - * (bits) K hashes and the specified false positive rate - */ - public static int n(int m, int k, double fpp) { - double Kd = k; - double Md = m; - return (int) Math.ceil(Md / (-Kd / Math.log(1 - Math.exp(Math.log(fpp) / Kd)))); - } - - /** - * Computes the optimal k (number of hashes per element inserted in Bloom - * filter), given the expected insertions and total number of bits in the Bloom - * filter. - * - * See http://en.wikipedia.org/wiki/File:Bloom_filter_fp_probability.svg for the - * formula. - * - * @param n expected insertions (must be positive) - * @param m total number of bits in Bloom filter (must be positive) - */ - public static int optimalK(long n, long m) { - // (m / n) * log(2), but avoid truncation due to division! - return Math.max(1, (int) Math.round(((double) m) / ((double) n) * Math.log(2))); - } - - /** - * Computes m (total bits of Bloom filter) which is expected to achieve, for the - * specified expected insertions, the required false positive probability. - * - * See http://en.wikipedia.org/wiki/Bloom_filter#Probability_of_false_positives - * for the formula. - * - * @param n expected insertions (must be positive) - * @param p false positive rate (must be 0 < p < 1) - */ - public static int optimalM(long n, double p) { - if (p == 0) { - p = Double.MIN_VALUE; - } - return Math.max(8, (int) (-n * Math.log(p) / (Math.log(2) * Math.log(2)))); - } - - public final int k; - public final int m; - public final long seed; - - protected final Hasher hasher; - - public Hash(long seed, int n, double p) { - m = optimalM(n, p); - k = optimalK(n, m); - this.seed = seed; - hasher = newHasher(); - } - - public Hash(long seed, int k, int m) { - this.seed = seed; - this.k = k; - this.m = m; - hasher = newHasher(); - } - - @Override - public Hash clone() { - Hasher clone = hasher.clone(); - return new Hash(seed, k, m) { - - @Override - protected Hasher newHasher() { - return clone; - } - }; - } - - public boolean equivalent(Hash other) { - return getK() == other.getK() && getM() == other.getM() && getSeed() == other.getSeed(); - } - - public double fpp(int n) { - return fpp(k, m, n); - } - - public int getK() { - return k; - } - - public int getM() { - return m; - } - - public long getSeed() { - return seed; - } - - public int[] hashes(M key) { - return hasher.hashes(k, key, m, seed); - } - - public long identityHash(M key) { - return hasher.identityHash(key, seed); - } - - public IntStream locations(M key) { - return hasher.locations(k, key, m, seed); - } - - abstract protected Hasher newHasher(); } diff --git a/cryptography/src/main/java/com/salesforce/apollo/cryptography/HexBloom.java b/cryptography/src/main/java/com/salesforce/apollo/cryptography/HexBloom.java index c2480c1fe..f6f29f885 100644 --- a/cryptography/src/main/java/com/salesforce/apollo/cryptography/HexBloom.java +++ b/cryptography/src/main/java/com/salesforce/apollo/cryptography/HexBloom.java @@ -27,7 +27,6 @@ */ public class HexBloom { - public static final double DEFAULT_FPR = 0.0001; public static final long DEFAULT_SEED = Primes.PRIMES[666]; private static final Function IDENTITY = d -> d; private static final int MINIMUM_BFF_CARD = 100; @@ -41,29 +40,30 @@ public HexBloom(Digest initial, int count) { var hashes = hashes(count); crowns = new Digest[count]; cardinality = 0; - membership = new BloomFilter.DigestBloomFilter(0x666, MINIMUM_BFF_CARD, DEFAULT_FPR); + var cardinality = Math.max(MINIMUM_BFF_CARD, count); + membership = new BloomFilter.DigestBloomFilter(0x666, MINIMUM_BFF_CARD, 1.0 / (double) cardinality); for (int i = 0; i < crowns.length; i++) { crowns[i] = hashes.get(i).apply(initial); } } public HexBloom(Digest initial, List> hashes) { - assert hashes.size() > 0; + assert !hashes.isEmpty(); crowns = new Digest[hashes.size()]; cardinality = 0; - membership = new BloomFilter.DigestBloomFilter(0x666, MINIMUM_BFF_CARD, DEFAULT_FPR); + membership = new BloomFilter.DigestBloomFilter(0x666, MINIMUM_BFF_CARD, 1.0 / (double) MINIMUM_BFF_CARD); for (int i = 0; i < crowns.length; i++) { crowns[i] = hashes.get(i).apply(initial); } } public HexBloom(HexBloome hb) { - this(hb.getCardinality(), hb.getCrownsList().stream().map(d -> Digest.from(d)).toList(), + this(hb.getCardinality(), hb.getCrownsList().stream().map(Digest::from).toList(), BloomFilter.from(hb.getMembership())); } public HexBloom(int cardinality, List crowns, BloomFilter membership) { - assert crowns.size() > 0; + assert !crowns.isEmpty(); this.crowns = new Digest[crowns.size()]; for (int i = 0; i < crowns.size(); i++) { this.crowns[i] = crowns.get(i); @@ -90,7 +90,7 @@ public HexBloom(Digest initial) { * @return the HexBloom built according to spec */ public static HexBloom construct(int count, Stream digests, Digest initialCrown, int crowns) { - return construct(count, digests, Collections.emptyList(), hashes(crowns), initialCrown, DEFAULT_FPR); + return construct(count, digests, Collections.emptyList(), hashes(crowns), initialCrown); } /** @@ -105,7 +105,7 @@ public static HexBloom construct(int count, Stream digests, Digest initi */ public static HexBloom construct(int currentCount, Stream currentMembership, List added, Digest initialCrown, int count) { - return construct(currentCount, currentMembership, added, hashes(count), initialCrown, DEFAULT_FPR); + return construct(currentCount, currentMembership, added, hashes(count), initialCrown); } /** @@ -120,7 +120,7 @@ public static HexBloom construct(int currentCount, Stream currentMembers */ public static HexBloom construct(int currentCount, Stream currentMembership, List added, List crowns, List removed) { - return construct(currentCount, currentMembership, added, crowns, removed, hashes(crowns.size()), DEFAULT_FPR); + return construct(currentCount, currentMembership, added, crowns, removed, hashes(crowns.size())); } /** @@ -132,33 +132,32 @@ public static HexBloom construct(int currentCount, Stream currentMembers * @param crowns - the current crown state corresponding to the currentMembership * @param removed - digests removed that are present in the currentMembership list * @param hashes - the list of functions for computing the hash of a digest for a given crown - * @param fpr - desired false positive rate for membership bloomfilter * @return the HexBloom representing the new state */ public static HexBloom construct(int currentCount, Stream currentMembership, List added, - List crowns, List removed, List> hashes, - double fpr) { - assert crowns.size() > 0; + List crowns, List removed, List> hashes) { + assert !crowns.isEmpty(); if (hashes.size() != crowns.size()) { throw new IllegalArgumentException( "Size of supplied hash functions: " + hashes.size() + " must equal the # of crowns: " + crowns.size()); } var cardinality = currentCount + added.size() - removed.size(); - var membership = new BloomFilter.DigestBloomFilter(DEFAULT_SEED, Math.max(MINIMUM_BFF_CARD, cardinality), fpr); - var crwns = crowns.stream().map(d -> new AtomicReference<>(d)).toList(); + var n = Math.max(MINIMUM_BFF_CARD, cardinality); + var membership = new BloomFilter.DigestBloomFilter(DEFAULT_SEED, n, 1.0 / (double) n); + var crwns = crowns.stream().map(AtomicReference::new).toList(); added.forEach(d -> { for (int i = 0; i < crwns.size(); i++) { - crwns.get(i).accumulateAndGet(hashes.get(i).apply(d), (a, b) -> a.xor(b)); + crwns.get(i).accumulateAndGet(hashes.get(i).apply(d), Digest::xor); } membership.add(d); }); removed.forEach(d -> { for (int i = 0; i < crwns.size(); i++) { - crwns.get(i).accumulateAndGet(hashes.get(i).apply(d), (a, b) -> a.xor(b)); + crwns.get(i).accumulateAndGet(hashes.get(i).apply(d), Digest::xor); } }); - currentMembership.forEach(d -> membership.add(d)); - return new HexBloom(cardinality, crwns.stream().map(ad -> ad.get()).toList(), membership); + currentMembership.forEach(membership::add); + return new HexBloom(cardinality, crwns.stream().map(AtomicReference::get).toList(), membership); } /** @@ -169,81 +168,32 @@ public static HexBloom construct(int currentCount, Stream currentMembers * @param added - the added member digests * @param hashes - the supplied crown hash functions * @param initialCrown - the initial value of the crowns - * @param fpr - the false positive rate for the membership bloom filter * @return the HexBloom built according to spec */ public static HexBloom construct(int currentCount, Stream currentMembership, List added, - List> hashes, Digest initialCrown, double fpr) { - assert hashes.size() > 0; + List> hashes, Digest initialCrown) { + assert !hashes.isEmpty(); var cardinality = currentCount + added.size(); - var membership = new BloomFilter.DigestBloomFilter(DEFAULT_SEED, Math.max(MINIMUM_BFF_CARD, cardinality), fpr); + var n = Math.max(MINIMUM_BFF_CARD, cardinality); + var membership = new BloomFilter.DigestBloomFilter(DEFAULT_SEED, n, 1.0 / (double) n); - var crwns = IntStream.range(0, hashes.size()) - .mapToObj(i -> hashes.get(i).apply(initialCrown)) - .map(d -> new AtomicReference<>(d)) - .toList(); + var crwns = hashes.stream().map(hash -> hash.apply(initialCrown)).map(AtomicReference::new).toList(); currentMembership.forEach(d -> { for (int i = 0; i < crwns.size(); i++) { - crwns.get(i).accumulateAndGet(hashes.get(i).apply(d), (a, b) -> a.xor(b)); + crwns.get(i).accumulateAndGet(hashes.get(i).apply(d), Digest::xor); } membership.add(d); }); added.forEach(d -> { for (int i = 0; i < crwns.size(); i++) { - crwns.get(i).accumulateAndGet(hashes.get(i).apply(d), (a, b) -> a.xor(b)); + crwns.get(i).accumulateAndGet(hashes.get(i).apply(d), Digest::xor); } membership.add(d); }); return new HexBloom(cardinality, crwns.stream().map(ad -> ad.get()).toList(), membership); } - /** - * Construct a HexBloom with a membership bloomfilter using the default false positive rate and the default hash - * transforms for each crown - * - * @param currentMembership - list of digests that correspond to the supplied crowns - * @param added - digests added that are not present in the currentMembership list - * @param crowns - the current crown state corresponding to the currentMembership - * @param removed - digests removed that are present in the currentMembership list - * @return the HexBloom representing the new state - */ - public static HexBloom construct(List currentMembership, List added, List crowns, - List removed) { - return construct(currentMembership, added, crowns, removed, hashes(crowns.size()), DEFAULT_FPR); - } - - /** - * Construct a HexBloom with a membership bloomfilter using the default false positive rate - * - * @param currentMembership - list of digests that correspond to the supplied crowns - * @param added - digests added that are not present in the currentMembership list - * @param crowns - the current crown state corresponding to the currentMembership - * @param removed - digests removed that are present in the currentMembership list - * @param hashes - the list of functions for computing the hash of a digest for a given crown - * @return the HexBloom representing the new state - */ - public static HexBloom construct(List currentMembership, List added, List crowns, - List removed, List> hashes) { - return construct(currentMembership, added, crowns, removed, hashes, DEFAULT_FPR); - } - - /** - * Construct a HexBloom. - * - * @param currentMembership - list of digests that correspond to the supplied crowns - * @param added - digests added that are not present in the currentMembership list - * @param crowns - the current crown state corresponding to the currentMembership - * @param removed - digests removed that are present in the currentMembership list - * @param hashes - the list of functions for computing the hash of a digest for a given crown - * @param fpr - desired false positive rate for membership bloomfilter - * @return the HexBloom representing the new state - */ - public static HexBloom construct(List currentMembership, List added, List crowns, - List removed, List> hashes, double fpr) { - return construct(currentMembership.size(), currentMembership.stream(), added, crowns, removed, hashes, fpr); - } - public static HexBloom from(HexBloome hb) { assert !HexBloome.getDefaultInstance().equals(hb); return new HexBloom(hb); @@ -252,7 +202,6 @@ public static HexBloom from(HexBloome hb) { /** * Answer the default hash for a crown positiion * - * @param index * @return the hash transform */ public static Function hash(int index) { @@ -261,19 +210,13 @@ public static Function hash(int index) { /** * Answer the default hash transforms for the number of crowns - * - * @param crowns - * @return */ public static List> hashes(int crowns) { - return IntStream.range(0, crowns).mapToObj(i -> hash(i)).toList(); + return IntStream.range(0, crowns).mapToObj(HexBloom::hash).toList(); } /** * Answer the default wrapping hash for a crown positiion - * - * @param index - * @return the wrapping hash transform */ public static Function hashWrap(int index) { return d -> d.prefix(index); @@ -281,12 +224,9 @@ public static Function hashWrap(int index) { /** * Answer the default wrapping hash transforms for the number of crowns - * - * @param crowns - * @return */ public static List> hashWraps(int crowns) { - return IntStream.range(0, crowns).mapToObj(i -> hashWrap(i)).toList(); + return IntStream.range(0, crowns).mapToObj(HexBloom::hashWrap).toList(); } public HexBloom add(Digest d, List> hashes) { @@ -312,7 +252,7 @@ public Digest compact() { if (crowns.length == 1) { return crowns[0]; } - return Arrays.asList(crowns).stream().reduce(crowns[0].getAlgorithm().getOrigin(), (a, b) -> a.xor(b)); + return Arrays.stream(crowns).reduce(crowns[0].getAlgorithm().getOrigin(), Digest::xor); } /** @@ -334,7 +274,7 @@ public Digest compactWrapped(List> hashes) { .mapToObj(i -> hashes.get(i).apply(crowns[i])) .toList() .stream() - .reduce(crowns[0].getAlgorithm().getOrigin(), (a, b) -> a.xor(b)); + .reduce(crowns[0].getAlgorithm().getOrigin(), Digest::xor); } public boolean contains(Digest digest) { @@ -370,9 +310,6 @@ public HexBloome toHexBloome() { /** * Answer the serialized receiver, with crowns hashed using the supplied hashes functions - * - * @param hashes - * @return */ public HexBloome toHexBloome(List> hashes) { if (hashes.size() != crowns.length) { @@ -388,8 +325,6 @@ public HexBloome toHexBloome(List> hashes) { /** * Answer the serialized receiver, using no transformation on the crowns - * - * @return */ public HexBloome toIdentityHexBloome() { return toHexBloome(IntStream.range(0, crowns.length).mapToObj(i -> IDENTITY).toList()); @@ -401,8 +336,8 @@ public String toString() { } /** - * Validate that the supplied members matches the receiver's crowns. All members must be included in the membership - * bloomfilter, all calculated crowns must match and the cardinality must match. + * Validate that the supplied members match the receiver's crowns. All members must be included in the membership + * bloomfilter, all calculated crowns must match, and the cardinality must match. * * @param members - list of member digests * @return true if validated @@ -412,8 +347,8 @@ public boolean validate(List members) { } /** - * Validate that the supplied members matches the receiver's crowns. All members must be included in the membership - * bloomfilter, all calculated crowns must match and the cardinality must match. + * Validate that the supplied members match the receiver's crowns. All members must be included in the membership + * bloomfilter, all calculated crowns must match, and the cardinality must match. * * @param members - lsit of member digests * @param hashes - hash functions for computing crowns @@ -424,8 +359,8 @@ public boolean validate(List members, List> has } /** - * Validate that the supplied members matches the receiver's crowns. All members must be included in the membership - * bloomfilter, all calculated crowns must match and the cardinality must match. + * Validate that the supplied members match the receiver's crowns. All members must be included in the membership + * bloomfilter, all calculated crowns must match, and the cardinality must match. * * @param members - stream of member digests * @return true if validated @@ -435,8 +370,8 @@ public boolean validate(Stream members) { } /** - * Validate that the supplied members matches the receiver's crowns. All members must be included in the membership - * bloomfilter, all calculated crowns must match and the cardinality must match. + * Validate that the supplied members match the receiver's crowns. All members must be included in the membership + * bloomfilter, all calculated crowns must match, and the cardinality must match. * * @param members - stream of member digests * @param hashes - hash functions for computing crowns @@ -448,12 +383,12 @@ public boolean validate(Stream members, List> h "Size of supplied hash functions: " + hashes.size() + " must equal the # of crowns: " + crowns.length); } var count = new AtomicInteger(); - var calculated = IntStream.range(0, crowns.length) - .mapToObj(i -> new AtomicReference(crowns[i].getAlgorithm().getOrigin())) - .toList(); + var calculated = Arrays.stream(crowns) + .map(crown -> new AtomicReference(crown.getAlgorithm().getOrigin())) + .toList(); members.forEach(d -> { for (int i = 0; i < crowns.length; i++) { - calculated.get(i).accumulateAndGet(hashes.get(i).apply(d), (a, b) -> a.xor(b)); + calculated.get(i).accumulateAndGet(hashes.get(i).apply(d), Digest::xor); } count.incrementAndGet(); }); @@ -526,30 +461,20 @@ public static class Accumulator { protected final List> hashes; protected int currentCount = 0; - public Accumulator(int cardinality, int crowns, Digest initial, double fpr) { - this(cardinality, hashes(crowns), initial, fpr); - } - - public Accumulator(int cardinality, List> crownHashes, Digest initial, double fpr) { + public Accumulator(int cardinality, List> crownHashes, Digest initial) { if (cardinality < 0) { throw new IllegalArgumentException(("Cardinality must be >= 0")); } if (crownHashes == null || crownHashes.isEmpty()) { throw new IllegalArgumentException("Crown hashes must not be null or empty"); } - if (fpr <= 0) { - throw new IllegalArgumentException("False positive rate must be > 0"); - } this.cardinality = cardinality; this.hashes = crownHashes; - accumulators = IntStream.range(0, hashes.size()) - .mapToObj(i -> hashes.get(i).apply(initial)) - .map(d -> new AtomicReference<>(d)) - .toList(); + accumulators = hashes.stream().map(hash -> hash.apply(initial)).map(AtomicReference::new).toList(); } public Accumulator(int cardinality, int crowns, Digest initial) { - this(cardinality, crowns, initial, DEFAULT_FPR); + this(cardinality, hashes(crowns), initial); } public void add(Digest digest) { @@ -558,7 +483,7 @@ public void add(Digest digest) { } currentCount++; for (int i = 0; i < accumulators.size(); i++) { - accumulators.get(i).accumulateAndGet(hashes.get(i).apply(digest), (a, b) -> a.xor(b)); + accumulators.get(i).accumulateAndGet(hashes.get(i).apply(digest), Digest::xor); } } @@ -571,12 +496,12 @@ public Digest compactWrapped(List> hashes) { "Size of supplied hash functions: " + hashes.size() + " must equal the # of crowns: " + accumulators.size()); } - var algorithm = accumulators.get(0).get().getAlgorithm(); + var algorithm = accumulators.getFirst().get().getAlgorithm(); return IntStream.range(0, accumulators.size()) .mapToObj(i -> hashes.get(i).apply(accumulators.get(i).get())) .toList() .stream() - .reduce(algorithm.getOrigin(), (a, b) -> a.xor(b)); + .reduce(algorithm.getOrigin(), Digest::xor); } /** @@ -587,7 +512,7 @@ public Digest compactWrapped() { } public List crowns() { - return accumulators.stream().map(ar -> ar.get()).toList(); + return accumulators.stream().map(AtomicReference::get).toList(); } public List wrappedCrowns() { @@ -604,17 +529,14 @@ public List wrappedCrowns(List> wrapingHash) { public static class HexAccumulator extends Accumulator { private final BloomFilter membership; - public HexAccumulator(int cardinality, int crowns, Digest initial, double fpr) { - this(cardinality, hashes(crowns), initial, fpr); - } - - public HexAccumulator(int cardinality, List> crownHashes, Digest initial, double fpr) { - super(cardinality, crownHashes, initial, fpr); - membership = new BloomFilter.DigestBloomFilter(DEFAULT_SEED, Math.max(MINIMUM_BFF_CARD, cardinality), fpr); + public HexAccumulator(int cardinality, int crowns, Digest initial) { + this(cardinality, hashes(crowns), initial); } - public HexAccumulator(int cardinality, int crowns, Digest initial) { - this(cardinality, crowns, initial, DEFAULT_FPR); + public HexAccumulator(int cardinality, List> crownHashes, Digest initial) { + super(cardinality, crownHashes, initial); + var n = Math.max(MINIMUM_BFF_CARD, cardinality); + membership = new BloomFilter.DigestBloomFilter(DEFAULT_SEED, n, 1.0 / (double) n); } @Override @@ -625,7 +547,7 @@ public void add(Digest digest) { public HexBloom build() { assert currentCount == cardinality : "Did not add all members, missing: " + (cardinality - currentCount); - return new HexBloom(cardinality, accumulators.stream().map(ar -> ar.get()).toList(), membership); + return new HexBloom(cardinality, accumulators.stream().map(AtomicReference::get).toList(), membership); } } } diff --git a/ethereal/src/main/java/com/salesforce/apollo/ethereal/Config.java b/ethereal/src/main/java/com/salesforce/apollo/ethereal/Config.java index 2651e6872..204916bd0 100644 --- a/ethereal/src/main/java/com/salesforce/apollo/ethereal/Config.java +++ b/ethereal/src/main/java/com/salesforce/apollo/ethereal/Config.java @@ -42,7 +42,7 @@ public static class Builder implements Cloneable { private int bias = 3; private DigestAlgorithm digestAlgorithm = DigestAlgorithm.DEFAULT; private int epochLength = 30; - private double fpr = 0.000125; + private double fpr = 0.00125; private String label = ""; private short nProc; private int numberOfEpochs = 3; // < 0 for unbounded diff --git a/ethereal/src/main/java/com/salesforce/apollo/ethereal/Ethereal.java b/ethereal/src/main/java/com/salesforce/apollo/ethereal/Ethereal.java index f11310a96..6cac3b2bb 100644 --- a/ethereal/src/main/java/com/salesforce/apollo/ethereal/Ethereal.java +++ b/ethereal/src/main/java/com/salesforce/apollo/ethereal/Ethereal.java @@ -141,8 +141,8 @@ public String dump() { public Processor processor() { return new Processor() { @Override - public Gossip gossip(Digest context, int ring) { - final var builder = Gossip.newBuilder().setRing(ring); + public Gossip gossip(Digest context) { + final var builder = Gossip.newBuilder(); final var current = currentEpoch.get(); epochs.entrySet() .stream() diff --git a/ethereal/src/main/java/com/salesforce/apollo/ethereal/Processor.java b/ethereal/src/main/java/com/salesforce/apollo/ethereal/Processor.java index e663c30aa..8601d01dd 100644 --- a/ethereal/src/main/java/com/salesforce/apollo/ethereal/Processor.java +++ b/ethereal/src/main/java/com/salesforce/apollo/ethereal/Processor.java @@ -7,9 +7,9 @@ package com.salesforce.apollo.ethereal; +import com.salesforce.apollo.cryptography.Digest; import com.salesforce.apollo.ethereal.proto.Gossip; import com.salesforce.apollo.ethereal.proto.Update; -import com.salesforce.apollo.cryptography.Digest; /** * @author hal.hildebrand @@ -20,10 +20,9 @@ public interface Processor { * First phase request. Answer the gossip for the current state of the receiver * * @param context - the digest id of the context for routing - * @param ring - the ring we're gossiping on * @return the Gossip */ - Gossip gossip(Digest context, int ring); + Gossip gossip(Digest context); /** * First phase reply. Answer the Update from the receiver's state, based on the suppled Have diff --git a/ethereal/src/main/java/com/salesforce/apollo/ethereal/memberships/ChRbcGossip.java b/ethereal/src/main/java/com/salesforce/apollo/ethereal/memberships/ChRbcGossip.java index 69b7a0685..829b7e2de 100644 --- a/ethereal/src/main/java/com/salesforce/apollo/ethereal/memberships/ChRbcGossip.java +++ b/ethereal/src/main/java/com/salesforce/apollo/ethereal/memberships/ChRbcGossip.java @@ -6,11 +6,9 @@ */ package com.salesforce.apollo.ethereal.memberships; -import com.codahale.metrics.Timer; import com.salesforce.apollo.archipelago.Router; import com.salesforce.apollo.archipelago.RouterImpl.CommonCommunications; import com.salesforce.apollo.archipelago.server.FernetServerInterceptor; -import com.salesforce.apollo.context.Context; import com.salesforce.apollo.cryptography.Digest; import com.salesforce.apollo.ethereal.Processor; import com.salesforce.apollo.ethereal.memberships.comm.EtherealMetrics; @@ -22,7 +20,7 @@ import com.salesforce.apollo.ethereal.proto.Update; import com.salesforce.apollo.membership.Member; import com.salesforce.apollo.membership.SigningMember; -import com.salesforce.apollo.ring.RingCommunications; +import com.salesforce.apollo.ring.SliceIterator; import com.salesforce.apollo.utils.Entropy; import com.salesforce.apollo.utils.Utils; import io.grpc.StatusRuntimeException; @@ -30,6 +28,9 @@ import org.slf4j.LoggerFactory; import java.time.Duration; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; import java.util.Optional; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; @@ -51,29 +52,28 @@ public class ChRbcGossip { private static final Logger log = LoggerFactory.getLogger( ChRbcGossip.class); private final CommonCommunications comm; - private final Context context; + private final Digest id; private final SigningMember member; private final EtherealMetrics metrics; private final Processor processor; - private final RingCommunications ring; + private final SliceIterator ring; private final AtomicBoolean started = new AtomicBoolean(); private final Terminal terminal = new Terminal(); + private final List membership; private volatile ScheduledFuture scheduled; - public ChRbcGossip(Context context, SigningMember member, Processor processor, Router communications, - EtherealMetrics m) { + public ChRbcGossip(Digest id, SigningMember member, Collection membership, Processor processor, + Router communications, EtherealMetrics m) { this.processor = processor; - this.context = context; this.member = member; this.metrics = m; - comm = communications.create(member, context.getId(), terminal, getClass().getCanonicalName(), + this.id = id; + this.membership = new ArrayList<>(membership); + comm = communications.create(member, id, terminal, getClass().getCanonicalName(), r -> new GossiperServer(communications.getClientIdentityProvider(), metrics, r), getCreate(metrics), Gossiper.getLocalLoopback(member)); - ring = new RingCommunications<>(context, member, this.comm); - } - - public Context getContext() { - return context; + ring = new SliceIterator("ChRbcGossip[%s on: %s]".formatted(id, member.getId()), member, membership, + comm); } /** @@ -91,16 +91,20 @@ public void start(Duration duration, Predicate Thread.ofVirtual().start(Utils.wrapped(() -> { - try { - oneRound(duration, scheduler); - } catch (Throwable e) { - log.error("Error in gossip on: {}", member.getId(), e); - } - }, log)), initialDelay.toMillis(), TimeUnit.MILLISECONDS); + log.trace("Starting GossipService[{}] on: {}", id, member.getId()); + comm.register(id, terminal, validator); + try { + var scheduler = Executors.newScheduledThreadPool(1, Thread.ofVirtual().factory()); + scheduler.schedule(() -> Thread.ofVirtual().start(Utils.wrapped(() -> { + try { + gossip(duration, scheduler); + } catch (Throwable e) { + log.error("Error in gossip on: {}", member.getId(), e); + } + }, log)), initialDelay.toMillis(), TimeUnit.MILLISECONDS); + } catch (Throwable e) { + log.error("Error in gossip on: {}", member.getId(), e); + } } /** @@ -110,8 +114,8 @@ public void stop() { if (!started.compareAndSet(true, false)) { return; } - log.trace("Stopping GossipService [{}] for {}", context.getId(), member.getId()); - comm.deregister(context.getId()); + log.trace("Stopping GossipService [{}] for {}", id, member.getId()); + comm.deregister(id); final var current = scheduled; scheduled = null; if (current != null) { @@ -119,23 +123,42 @@ public void stop() { } } + private void gossip(Duration frequency, ScheduledExecutorService scheduler) { + if (!started.get()) { + return; + } + var timer = metrics == null ? null : metrics.gossipRoundDuration().time(); + ring.iterate((link, _) -> gossipRound(link), (result, link, _) -> { + handle(result, link); + return true; + }, () -> { + if (timer != null) { + timer.stop(); + } + if (started.get()) { + scheduled = scheduler.schedule( + () -> Thread.ofVirtual().start(Utils.wrapped(() -> gossip(frequency, scheduler), log)), + frequency.toNanos(), TimeUnit.NANOSECONDS); + } + }, frequency); + } + /** * Perform the first phase of the gossip. Send our partner the Have state of the receiver */ - private Update gossipRound(Gossiper link, int ring) { + private Update gossipRound(Gossiper link) { if (!started.get()) { return null; } - log.trace("gossiping[{}] with {} ring: {} on {}", context.getId(), link.getMember(), ring, member); + log.trace("gossiping[{}] with {} on {}", id, link.getMember(), member); try { - return link.gossip(processor.gossip(context.getId(), ring)); + return link.gossip(processor.gossip(id)); } catch (StatusRuntimeException e) { - log.debug("gossiping[{}] failed: {} with: {} with {} ring: {} on: {}", context.getId(), e.getMessage(), - member.getId(), ring, link.getMember().getId(), member.getId()); + log.debug("gossiping[{}] failed: {} with: {} with {} on: {}", id, e.getMessage(), member.getId(), + link.getMember().getId(), member.getId()); return null; } catch (Throwable e) { - log.warn("gossiping[{}] failed: {} from {} with {} ring: {} on: {}", context.getId(), member.getId(), ring, - link.getMember().getId(), ring, member.getId(), e); + log.warn("gossiping:{} with: {} failed on: {}", id, link.getMember().getId(), member.getId(), e); return null; } } @@ -143,61 +166,33 @@ private Update gossipRound(Gossiper link, int ring) { /** * The second phase of the gossip. Handle the update from our gossip partner */ - private void handle(Optional result, RingCommunications.Destination destination, - Duration duration, ScheduledExecutorService scheduler, Timer.Context timer) { - if (!started.get() || destination == null || destination.link() == null) { - if (timer != null) { - timer.stop(); - } + private void handle(Optional result, Gossiper link) { + if (!started.get()) { return; } - try { - if (result.isEmpty()) { - if (timer != null) { - timer.stop(); - } - return; - } - Update update = result.get(); - if (update.equals(Update.getDefaultInstance())) { - return; - } - try { - var u = processor.update(update); - if (!Update.getDefaultInstance().equals(u)) { - log.trace("Gossip update with: {} on: {}", destination.member().getId(), member.getId()); - destination.link() - .update(ContextUpdate.newBuilder().setRing(destination.ring()).setUpdate(u).build()); - } - } catch (StatusRuntimeException e) { - log.debug("gossiping[{}] failed: {} with: {} with {} ring: {} on: {}", context.getId(), e.getMessage(), - member.getId(), ring, destination.member().getId(), member.getId()); - } catch (Throwable e) { - log.warn("gossiping[{}] failed: {} with: {} with {} ring: {} on: {}", context.getId(), e.getMessage(), - member.getId(), ring, destination.member().getId(), member.getId(), e); - } - } finally { - if (timer != null) { - timer.stop(); - } - if (started.get()) { - scheduled = scheduler.schedule( - () -> Thread.ofVirtual().start(Utils.wrapped(() -> oneRound(duration, scheduler), log)), - duration.toMillis(), TimeUnit.MILLISECONDS); - } + if (link == null) { + return; } - } - - /** - * Perform one round of gossip - */ - private void oneRound(Duration duration, ScheduledExecutorService scheduler) { - if (!started.get()) { + if (result.isEmpty()) { return; } - var timer = metrics == null ? null : metrics.gossipRoundDuration().time(); - ring.execute(this::gossipRound, - (result, destination) -> handle(result, destination, duration, scheduler, timer)); + Update update = result.get(); + if (update.equals(Update.getDefaultInstance())) { + return; + } + try { + var u = processor.update(update); + if (!Update.getDefaultInstance().equals(u)) { + log.trace("Gossip update with: {} on: {}", link.getMember().getId(), member.getId()); + link.update(ContextUpdate.newBuilder().setUpdate(u).build()); + } + } catch (StatusRuntimeException e) { + log.debug("gossiping[{}] failed: {} with: {} with {} on: {}", id, e.getMessage(), member.getId(), + link.getMember().getId(), member.getId()); + } catch (Throwable e) { + log.warn("gossiping[{}] failed: {} with: {} with {} on: {}", id, e.getMessage(), member.getId(), + link.getMember().getId(), member.getId(), e); + } } /** @@ -206,13 +201,6 @@ private void oneRound(Duration duration, ScheduledExecutorService scheduler) { private class Terminal implements GossiperService, Router.ServiceRouting { @Override public Update gossip(Gossip request, Digest from) { - Member predecessor = context.predecessor(request.getRing(), member); - if (predecessor == null || !from.equals(predecessor.getId())) { - log.debug("Invalid inbound gossip context: {} from: {} on ring: {} - not predecessor: {} on: {}", - context.getId(), from, request.getRing(), - predecessor == null ? "" : predecessor.getId(), member.getId()); - return Update.getDefaultInstance(); - } final var update = processor.gossip(request); log.trace("GossipService received from: {} missing: {} on: {}", from, update.getMissingCount(), member.getId()); @@ -221,13 +209,6 @@ public Update gossip(Gossip request, Digest from) { @Override public void update(ContextUpdate request, Digest from) { - Member predecessor = context.predecessor(request.getRing(), member); - if (predecessor == null || !from.equals(predecessor.getId())) { - log.debug("Invalid inbound update context:{} from: {} on ring: {} - not predecessor: {} on: {}", - context.getId(), from, request.getRing(), - predecessor == null ? "" : predecessor.getId(), member.getId()); - return; - } log.trace("gossip update with {} on: {}", from, member.getId()); processor.updateFrom(request.getUpdate()); } diff --git a/ethereal/src/test/java/com/salesforce/apollo/ethereal/EtherealTest.java b/ethereal/src/test/java/com/salesforce/apollo/ethereal/EtherealTest.java index 250e02f11..c6f38106f 100644 --- a/ethereal/src/test/java/com/salesforce/apollo/ethereal/EtherealTest.java +++ b/ethereal/src/test/java/com/salesforce/apollo/ethereal/EtherealTest.java @@ -13,7 +13,7 @@ import com.salesforce.apollo.archipelago.LocalServer; import com.salesforce.apollo.archipelago.Router; import com.salesforce.apollo.archipelago.ServerConnectionCache; -import com.salesforce.apollo.context.StaticContext; +import com.salesforce.apollo.context.DynamicContext; import com.salesforce.apollo.cryptography.DigestAlgorithm; import com.salesforce.apollo.cryptography.Signer; import com.salesforce.apollo.ethereal.memberships.ChRbcGossip; @@ -97,8 +97,12 @@ public void unbounded() throws NoSuchAlgorithmException, InterruptedException, I .map(ControlledIdentifierMember::new) .map(e -> (Member) e) .toList(); - - StaticContext context = new StaticContext<>(DigestAlgorithm.DEFAULT.getOrigin(), 0.1, members, 3); + DynamicContext context = DynamicContext.newBuilder() + .setBias(3) + .setpByz(0.1) + .setId(DigestAlgorithm.DEFAULT.getOrigin()) + .build(); + context.activate(members); var metrics = new EtherealMetricsImpl(context.getId(), "test", registry); var builder = Config.newBuilder().setnProc((short) NPROC).setNumberOfEpochs(-1).setEpochLength(EPOCH_LENGTH); @@ -132,7 +136,8 @@ public void unbounded() throws NoSuchAlgorithmException, InterruptedException, I } }, "Test: " + i); - var gossiper = new ChRbcGossip(context, (SigningMember) member, controller.processor(), com, metrics); + var gossiper = new ChRbcGossip(context.getId(), (SigningMember) member, members, controller.processor(), + com, metrics); gossipers.add(gossiper); dataSources.add(ds); controllers.add(controller); @@ -219,7 +224,12 @@ private void one(int iteration) .map(e -> (Member) e) .toList(); - StaticContext context = new StaticContext<>(DigestAlgorithm.DEFAULT.getOrigin(), 0.1, members, 3); + DynamicContext context = DynamicContext.newBuilder() + .setBias(3) + .setpByz(0.1) + .setId(DigestAlgorithm.DEFAULT.getOrigin()) + .build(); + context.activate(members); var metrics = new EtherealMetricsImpl(context.getId(), "test", registry); var builder = Config.newBuilder() .setnProc((short) NPROC) @@ -254,7 +264,8 @@ private void one(int iteration) } }, "Test: " + i); - var gossiper = new ChRbcGossip(context, (SigningMember) member, controller.processor(), com, metrics); + var gossiper = new ChRbcGossip(context.getId(), (SigningMember) member, members, controller.processor(), + com, metrics); gossipers.add(gossiper); dataSources.add(ds); controllers.add(controller); diff --git a/fireflies/src/main/java/com/salesforce/apollo/fireflies/Binding.java b/fireflies/src/main/java/com/salesforce/apollo/fireflies/Binding.java index 4e107d8a9..adaf5837c 100644 --- a/fireflies/src/main/java/com/salesforce/apollo/fireflies/Binding.java +++ b/fireflies/src/main/java/com/salesforce/apollo/fireflies/Binding.java @@ -183,13 +183,13 @@ private boolean completeGateway(Participant member, CompletableFuture gat private void gatewaySRE(Digest v, Entrance link, StatusRuntimeException sre, AtomicInteger abandon) { switch (sre.getStatus().getCode()) { case OUT_OF_RANGE -> { - log.info("Gateway view: {} invalid: {} from: {} on: {}", v, sre.getMessage(), link.getMember().getId(), - node.getId()); + log.debug("Gateway view: {} invalid: {} from: {} on: {}", v, sre.getMessage(), link.getMember().getId(), + node.getId()); abandon.incrementAndGet(); } case FAILED_PRECONDITION -> { - log.info("Gateway view: {} unavailable: {} from: {} on: {}", v, sre.getMessage(), link.getMember().getId(), - node.getId()); + log.trace("Gateway view: {} unavailable: {} from: {} on: {}", v, sre.getMessage(), link.getMember().getId(), + node.getId()); abandon.incrementAndGet(); } case PERMISSION_DENIED -> { @@ -198,8 +198,8 @@ private void gatewaySRE(Digest v, Entrance link, StatusRuntimeException sre, Ato abandon.incrementAndGet(); } case RESOURCE_EXHAUSTED -> { - log.info("Gateway view: {} full: {} from: {} on: {}", v, sre.getMessage(), link.getMember().getId(), - node.getId()); + log.debug("Gateway view: {} full: {} from: {} on: {}", v, sre.getMessage(), link.getMember().getId(), + node.getId()); abandon.incrementAndGet(); } default -> log.info("Join view: {} error: {} from: {} on: {}", v, sre.getMessage(), link.getMember().getId(), @@ -224,13 +224,13 @@ private Join join(Digest v) { Thread.ofVirtual().start(Utils.wrapped(() -> { var view = Digest.from(r.getView()); - log.info("Rebalancing to cardinality: {} (validate) for: {} context: {} on: {}", r.getCardinality(), - view, context.getId(), node.getId()); + log.debug("Rebalancing to cardinality: {} (validate) for: {} context: {} on: {}", r.getCardinality(), + view, context.getId(), node.getId()); this.context.rebalance(r.getCardinality()); node.nextNote(view); - log.debug("Completing redirect to view: {} context: {} sample: {} on: {}", view, this.context.getId(), - r.getSampleCount(), node.getId()); + log.debug("Completing redirect to view: {} context: {} introductions: {} on: {}", view, + this.context.getId(), r.getIntroductionsCount(), node.getId()); if (timer != null) { timer.close(); } @@ -240,7 +240,7 @@ private Join join(Digest v) { } private void join(Redirect redirect, Digest v, Duration duration) { - var sample = redirect.getSampleList() + var sample = redirect.getIntroductionsList() .stream() .map(sn -> new NoteWrapper(sn, digestAlgo)) .map(nw -> view.new Participant(nw)) @@ -259,8 +259,8 @@ private void join(Redirect redirect, Digest v, Duration duration) { final var cardinality = redirect.getCardinality(); - log.info("Rebalancing to cardinality: {} (join) for: {} context: {} on: {}", cardinality, v, context.getId(), - node.getId()); + log.debug("Rebalancing to cardinality: {} (join) for: {} context: {} on: {}", cardinality, v, context.getId(), + node.getId()); this.context.rebalance(cardinality); node.nextNote(v); @@ -281,7 +281,7 @@ private void join(Redirect redirect, Digest v, Duration duration) { try { var g = link.join(join, params.seedingTimeout()); if (g == null || g.equals(Gateway.getDefaultInstance())) { - log.info("Gateway view: {} empty from: {} on: {}", v, link.getMember().getId(), node.getId()); + log.debug("Gateway view: {} empty from: {} on: {}", v, link.getMember().getId(), node.getId()); abandon.incrementAndGet(); return null; } @@ -301,7 +301,7 @@ private void join(Redirect redirect, Digest v, Duration duration) { return; } if (abandon.get() >= majority) { - log.info("Abandoning Gateway view: {} reseeding on: {}", v, node.getId()); + log.debug("Abandoning Gateway view: {} reseeding on: {}", v, node.getId()); seeding(); } else { abandon.set(0); diff --git a/fireflies/src/main/java/com/salesforce/apollo/fireflies/Parameters.java b/fireflies/src/main/java/com/salesforce/apollo/fireflies/Parameters.java index ec5aa1df1..ebe6d6e10 100644 --- a/fireflies/src/main/java/com/salesforce/apollo/fireflies/Parameters.java +++ b/fireflies/src/main/java/com/salesforce/apollo/fireflies/Parameters.java @@ -31,7 +31,7 @@ public static class Builder { /** * False positive rate for bloom filter state replication (high fpr is good) */ - private double fpr = 0.0125; + private double fpr = 0.00125; /** * Number of retries when joining until giving up */ @@ -59,7 +59,7 @@ public static class Builder { /** * Timeout for contacting seed gateways during seeding and join operations */ - private Duration seedingTimout = Duration.ofSeconds(5); + private Duration seedingTimout = Duration.ofSeconds(15); /** * Max number of times to attempt validation when joining a view */ diff --git a/fireflies/src/main/java/com/salesforce/apollo/fireflies/View.java b/fireflies/src/main/java/com/salesforce/apollo/fireflies/View.java index af5f58d1b..b8f0c2f83 100644 --- a/fireflies/src/main/java/com/salesforce/apollo/fireflies/View.java +++ b/fireflies/src/main/java/com/salesforce/apollo/fireflies/View.java @@ -138,6 +138,7 @@ public View(DynamicContext context, ControlledIdentifierMember memb EntranceClient.getCreate(metrics), Entrance.getLocalLoopback(node, service)); gossiper = new RingCommunications<>(context, node, comm); gossiper.allowDuplicates(); + gossiper.ignoreSelf(); this.validation = validation; this.verifiers = verifiers; } @@ -309,14 +310,14 @@ boolean addToView(NoteWrapper note) { if (accused) { checkInvalidations(member); } - if (!viewManagement.joined() && context.totalCount() == viewManagement.cardinality()) { - assert context.totalCount() == viewManagement.cardinality(); + if (!viewManagement.joined() && context.size() == viewManagement.cardinality()) { + assert context.size() == viewManagement.cardinality(); viewManagement.join(); } else { // This assertion needs to accommodate invalid diadem cardinality during view installation, as the diadem // is from the previous view until all joining member have... joined. - assert context.totalCount() <= Math.max(viewManagement.cardinality(), context.cardinality()) : "total: " - + context.totalCount() + " card: " + viewManagement.cardinality(); + assert context.size() <= Math.max(viewManagement.cardinality(), context.cardinality()) : "total: " + + context.size() + " card: " + viewManagement.cardinality(); } return true; }); @@ -342,41 +343,47 @@ void finalizeViewChange() { return; } viewChange(() -> { - final var majority = context.size() == 1 ? 1 : context.majority(); - if (observations.size() < majority) { - log.trace("Do not have majority: {} required: {} observers: {} for: {} on: {}", observations.size(), - majority, viewManagement.observers.stream().toList(), currentView(), node.getId()); + final var superMajority = + context.size() == 1 ? 1 : context.getRingCount() - ((context.getRingCount() - 1) / 4); + if (observations.size() < superMajority) { + log.trace("Do not have superMajority: {} required: {} observers: {} for: {} on: {}", + observations.size(), superMajority, viewManagement.observersList(), currentView(), + node.getId()); scheduleFinalizeViewChange(2); return; } - log.info("Finalizing view change: {} required: {} observers: {} for: {} on: {}", context.getId(), majority, - viewManagement.observers.stream().toList(), currentView(), node.getId()); + log.info("Finalizing view change: {} required: {} observers: {} for: {} on: {}", context.getId(), + superMajority, viewManagement.observersList(), currentView(), node.getId()); HashMultiset ballots = HashMultiset.create(); - observations.values().forEach(vc -> { - final var leaving = new ArrayList<>( - vc.getChange().getLeavesList().stream().map(Digest::from).collect(Collectors.toSet())); - final var joining = new ArrayList<>( - vc.getChange().getJoinsList().stream().map(Digest::from).collect(Collectors.toSet())); - leaving.sort(Ordering.natural()); - joining.sort(Ordering.natural()); - ballots.add(new Ballot(Digest.from(vc.getChange().getCurrent()), leaving, joining, digestAlgo)); - }); + final var current = currentView(); + observations.values() + .stream() + .filter(vc -> current.equals(Digest.from(vc.getChange().getCurrent()))) + .forEach(vc -> { + final var leaving = new ArrayList<>( + vc.getChange().getLeavesList().stream().map(Digest::from).collect(Collectors.toSet())); + final var joining = new ArrayList<>( + vc.getChange().getJoinsList().stream().map(Digest::from).collect(Collectors.toSet())); + leaving.sort(Ordering.natural()); + joining.sort(Ordering.natural()); + ballots.add( + new Ballot(Digest.from(vc.getChange().getCurrent()), leaving, joining, digestAlgo)); + }); + observations.clear(); var max = ballots.entrySet() .stream() .max(Ordering.natural().onResultOf(Multiset.Entry::getCount)) .orElse(null); - if (max != null && max.getCount() >= majority) { - log.info("View consensus successful: {} required: {} cardinality: {} for: {} on: {}", max, majority, - viewManagement.cardinality(), currentView(), node.getId()); + if (max != null && max.getCount() >= superMajority) { + log.info("View consensus successful: {} required: {} cardinality: {} for: {} on: {}", max, + superMajority, viewManagement.cardinality(), currentView(), node.getId()); viewManagement.install(max.getElement()); - observations.clear(); } else { @SuppressWarnings("unchecked") final var reversed = Comparator.comparing(e -> ((Entry) e).getCount()).reversed(); log.info("View consensus failed: {}, required: {} cardinality: {} ballots: {} for: {} on: {}", - observations.size(), majority, viewManagement.cardinality(), - ballots.entrySet().stream().sorted(reversed).limit(1).toList(), currentView(), node.getId()); - observations.clear(); + observations.size(), superMajority, viewManagement.cardinality(), + ballots.entrySet().stream().sorted(reversed).toList(), currentView(), node.getId()); } scheduleViewChange(); @@ -417,7 +424,7 @@ void notifyListeners(List joining, List leavin viewChangeListeners.forEach(listener -> { try { log.trace("Notifying: {} view change: {} cardinality: {} joins: {} leaves: {} on: {} ", listener, - currentView(), context.totalCount(), joining.size(), leaving.size(), node.getId()); + currentView(), context.size(), joining.size(), leaving.size(), node.getId()); listener.accept(sc, current); } catch (Throwable e) { log.error("error in view change listener: {} on: {} ", listener, node.getId(), e); @@ -616,7 +623,6 @@ protected Gossip gossip(Fireflies link, int ring) { .setRing(ring) .setGossip(commonDigests()) .build()); - log.trace("gossiping with: {} on: {}", link.getMember().getId(), node.getId()); try { return link.gossip(gossip); } catch (Throwable e) { @@ -673,7 +679,7 @@ private void accuse(Participant member, int ring, Throwable e) { pendingRebuttals.computeIfAbsent(member.getId(), d -> roundTimers.schedule(() -> gc(member), params.rebuttalTimeout())); log.debug("Accuse {} on ring {} view: {} (timer started): {} on: {}", member.getId(), ring, currentView(), - e.toString(), node.getId()); + e.getMessage(), node.getId()); } /** @@ -804,6 +810,8 @@ private boolean add(NoteWrapper note) { } if (!isValidMask(note.getMask(), context)) { + log.debug("Invalid mask of: {} cardinality: {} on: {}", note.getId(), note.getMask().cardinality(), + node.getId()); if (metrics != null) { metrics.filteredNotes().mark(); } @@ -820,7 +828,7 @@ private boolean add(NoteWrapper note) { */ private boolean add(SignedViewChange observation) { final Digest observer = Digest.from(observation.getChange().getObserver()); - if (!viewManagement.observers.contains(observer)) { + if (!viewManagement.isObserver(observer)) { log.trace("Invalid observer: {} current: {} on: {}", observer, currentView(), node.getId()); return false; } @@ -985,9 +993,10 @@ private void gc(Participant member) { if (context.isActive(member)) { amplify(member); } - log.debug("Garbage collecting: {} on: {}", member.getId(), node.getId()); + log.debug("Garbage collecting: {} view: {} on: {}", member.getId(), viewManagement.currentView(), node.getId()); context.offline(member); shunned.add(member.getId()); + viewManagement.gc(member); } /** @@ -1280,8 +1289,8 @@ private void processUpdates(List notes, List accus var jCount = joins.stream().filter(j -> addJoin(j)).count(); if (notes.size() + accusations.size() + observe.size() + joins.size() != 0) { log.trace("Updating, members: {} notes: {}:{} accusations: {}:{} observations: {}:{} joins: {}:{} on: {}", - context.totalCount(), nCount, notes.size(), aCount, accusations.size(), oCount, observe.size(), - jCount, joins.size(), node.getId()); + context.size(), nCount, notes.size(), aCount, accusations.size(), oCount, observe.size(), jCount, + joins.size(), node.getId()); } } @@ -1297,48 +1306,10 @@ private void recover(Participant member) { } if (context.activate(member)) { log.trace("Recovering: {} cardinality: {} count: {} on: {}", member.getId(), viewManagement.cardinality(), - context.totalCount(), node.getId()); + context.size(), node.getId()); } } - /** - * Redirect the member to the successor from this view's perspective - * - * @param member - * @param ring - * @param successor - * @param digests - * @return the Gossip containing the successor's Identity and Note from this view - */ - private Gossip redirectTo(Participant member, int ring, Participant successor, Digests digests) { - assert member != null; - assert successor != null; - if (successor.getNote() == null) { - log.debug("Cannot redirect: {} to: {} on ring: {} as note is null on: {}", member.getId(), - successor.getId(), ring, node.getId()); - return Gossip.getDefaultInstance(); - } - - var identity = successor.getNote(); - if (identity == null) { - log.debug("Cannot redirect: {} to: {} on ring: {} as note is null on: {}", member.getId(), - successor.getId(), ring, node.getId()); - return Gossip.getDefaultInstance(); - } - var gossip = Gossip.newBuilder() - .setRedirect(successor.getNote().getWrapped()) - .setNotes(processNotes(BloomFilter.from(digests.getNoteBff()))) - .setAccusations(processAccusations(BloomFilter.from(digests.getAccusationBff()))) - .setObservations(processObservations(BloomFilter.from(digests.getObservationBff()))) - .setJoins(viewManagement.processJoins(BloomFilter.from(digests.getJoinBiff()))) - .build(); - log.trace("Redirect: {} to: {} on ring: {} notes: {} acc: {} obv: {} joins: {} on: {}", member.getId(), - successor.getId(), ring, gossip.getNotes().getUpdatesCount(), - gossip.getAccusations().getUpdatesCount(), gossip.getObservations().getUpdatesCount(), - gossip.getJoins().getUpdatesCount(), node.getId()); - return gossip; - } - /** * Process the gossip response, providing the updates requested by the the other member and processing the updates * provided by the other member @@ -1405,18 +1376,18 @@ private Update updatesForDigests(Gossip gossip) { return builder.build(); } - private void validate(Digest from, final int ring, Digest requestView) { + private void validate(Digest from, final int ring, Digest requestView, String type) { if (shunned.contains(from)) { - log.trace("Member is shunned: {} on: {}", from, node.getId()); - throw new StatusRuntimeException(Status.UNKNOWN.withDescription("Member is shunned: " + from)); + log.trace("Member is shunned: {} cannot {} on: {}", type, from, node.getId()); + throw new StatusRuntimeException(Status.UNKNOWN.withDescription("Member is shunned")); } if (!started.get()) { - log.trace("Currently offline, send unknown to: {} on: {}", from, node.getId()); - throw new StatusRuntimeException(Status.UNKNOWN.withDescription("Member: " + node.getId() + " is offline")); + log.trace("Currently offline, cannot {}, send unknown to: {} on: {}", type, from, node.getId()); + throw new StatusRuntimeException(Status.UNKNOWN.withDescription("Considered offline")); } if (!requestView.equals(currentView())) { - log.debug("Invalid view: {} current: {} ring: {} from: {} on: {}", requestView, currentView(), ring, from, - node.getId()); + log.debug("Invalid {}, view: {} current: {} ring: {} from: {} on: {}", type, requestView, currentView(), + ring, from, node.getId()); throw new StatusRuntimeException( Status.PERMISSION_DENIED.withDescription("Invalid view: " + requestView + " current: " + currentView())); } @@ -1424,9 +1395,10 @@ private void validate(Digest from, final int ring, Digest requestView) { private void validate(Digest from, NoteWrapper note, Digest requestView, final int ring) { if (!from.equals(note.getId())) { - throw new StatusRuntimeException(Status.UNAUTHENTICATED.withDescription("Member does not match: " + from)); + log.debug("Invalid {} note: {} from: {} ring: {} on: {}", "gossip", note.getId(), from, ring, node.getId()); + throw new StatusRuntimeException(Status.UNAUTHENTICATED.withDescription("Note member does not match")); } - validate(from, ring, requestView); + validate(from, ring, requestView, "gossip"); } private void validate(Digest from, SayWhat request) { @@ -1447,7 +1419,7 @@ private void validate(Digest from, SayWhat request) { private void validate(Digest from, State request) { var valid = false; try { - validate(from, request.getRing(), Digest.from(request.getView())); + validate(from, request.getRing(), Digest.from(request.getView()), "update"); valid = true; } finally { if (!valid && metrics != null) { @@ -1895,18 +1867,16 @@ public void join(Join join, Digest from, StreamObserver responseObserve @Override public Gossip rumors(SayWhat request, Digest from) { if (!introduced.get()) { - log.trace("Not introduced!, ring: {} on: {}", request.getRing(), node.getId()); - throw new StatusRuntimeException(Status.FAILED_PRECONDITION.withDescription( - "Not introduced!, ring: %s from: %s on: %s".formatted(request.getRing(), from, node.getId()))); + log.trace("Not introduced; ring: {} from: {}, on: {}", request.getRing(), from, node.getId()); + throw new StatusRuntimeException(Status.FAILED_PRECONDITION.withDescription("Not introduced")); } return stable(() -> { - validate(from, request); final var ring = request.getRing(); if (!context.validRing(ring)) { - log.debug("invalid ring: {} from: {} on: {}", ring, from, node.getId()); - throw new StatusRuntimeException(Status.FAILED_PRECONDITION.withDescription( - "invalid ring: %s from: %s on: %s".formatted(ring, from, node.getId()))); + log.debug("invalid gossip ring: {} from: {} on: {}", ring, from, node.getId()); + throw new StatusRuntimeException(Status.FAILED_PRECONDITION.withDescription("invalid ring")); } + validate(from, request); Participant member = context.getActiveMember(from); if (member == null) { @@ -1914,34 +1884,30 @@ public Gossip rumors(SayWhat request, Digest from) { member = context.getActiveMember(from); if (member == null) { log.debug("Not active member: {} on: {}", from, node.getId()); - throw new StatusRuntimeException(Status.PERMISSION_DENIED.withDescription( - "Not active member: %s on: %s".formatted(from, node.getId()))); + throw new StatusRuntimeException(Status.PERMISSION_DENIED.withDescription("Not active member")); } } Participant successor = context.successor(ring, member, m -> context.isActive(m.getId())); if (successor == null) { log.debug("No active successor on ring: {} from: {} on: {}", ring, from, node.getId()); - throw new StatusRuntimeException(Status.FAILED_PRECONDITION.withDescription( - "No active successor on ring: %s from: %s on: %s".formatted(ring, from, node.getId()))); + throw new StatusRuntimeException(Status.FAILED_PRECONDITION.withDescription("No active successor")); } Gossip g; + var builder = Gossip.newBuilder(); final var digests = request.getGossip(); if (!successor.equals(node)) { - g = redirectTo(member, ring, successor, digests); - log.debug("Redirected: {} on: {}", member.getId(), node.getId()); - } else { - g = Gossip.newBuilder() - .setNotes(processNotes(from, BloomFilter.from(digests.getNoteBff()), params.fpr())) - .setAccusations( - processAccusations(BloomFilter.from(digests.getAccusationBff()), params.fpr())) - .setObservations( - processObservations(BloomFilter.from(digests.getObservationBff()), params.fpr())) - .setJoins( - viewManagement.processJoins(BloomFilter.from(digests.getJoinBiff()), params.fpr())) - .build(); + builder.setRedirect(successor.getNote().getWrapped()); + log.debug("Redirected: {} to: {} on: {}", member.getId(), successor.id, node.getId()); } + g = builder.setNotes(processNotes(from, BloomFilter.from(digests.getNoteBff()), params.fpr())) + .setAccusations( + processAccusations(BloomFilter.from(digests.getAccusationBff()), params.fpr())) + .setObservations( + processObservations(BloomFilter.from(digests.getObservationBff()), params.fpr())) + .setJoins(viewManagement.processJoins(BloomFilter.from(digests.getJoinBiff()), params.fpr())) + .build(); if (g.getNotes().getUpdatesCount() + g.getAccusations().getUpdatesCount() + g.getObservations() .getUpdatesCount() + g.getJoins().getUpdatesCount() != 0) { @@ -1978,15 +1944,13 @@ public void update(State request, Digest from) { final var ring = request.getRing(); if (!context.validRing(ring)) { log.debug("invalid ring: {} current: {} from: {} on: {}", ring, currentView(), from, node.getId()); - throw new StatusRuntimeException( - Status.INVALID_ARGUMENT.withDescription("No successor of: " + from)); + throw new StatusRuntimeException(Status.INVALID_ARGUMENT.withDescription("Invalid ring")); } Participant member = context.getActiveMember(from); Participant successor = context.successor(ring, member, m -> context.isActive(m.getId())); if (successor == null) { log.debug("No successor, invalid update from: {} on ring: {} on: {}", from, ring, node.getId()); - throw new StatusRuntimeException( - Status.FAILED_PRECONDITION.withDescription("No successor of: " + from)); + throw new StatusRuntimeException(Status.FAILED_PRECONDITION.withDescription("No successor")); } if (!successor.equals(node)) { return; diff --git a/fireflies/src/main/java/com/salesforce/apollo/fireflies/ViewManagement.java b/fireflies/src/main/java/com/salesforce/apollo/fireflies/ViewManagement.java index 5a62ea8d0..3104ab20d 100644 --- a/fireflies/src/main/java/com/salesforce/apollo/fireflies/ViewManagement.java +++ b/fireflies/src/main/java/com/salesforce/apollo/fireflies/ViewManagement.java @@ -7,7 +7,6 @@ package com.salesforce.apollo.fireflies; import com.codahale.metrics.Timer; -import com.google.common.base.Objects; import com.salesforce.apollo.bloomFilters.BloomFilter; import com.salesforce.apollo.context.DynamicContext; import com.salesforce.apollo.cryptography.Digest; @@ -126,6 +125,16 @@ Digest currentView() { return currentView.get(); } + void gc(Participant member) { + assert member != null; + view.stable(() -> { + if (observers.remove(member.id)) { + log.trace("Removed observer: {} view: {} on: {}", member.id, currentView.get(), node.getId()); + resetObservers(); + } + }); + } + /** * @param seed * @param p @@ -159,7 +168,7 @@ void install(Ballot ballot) { .map(p -> p.note.getWrapped()) .collect(Collectors.toSet()); - context.rebalance(context.totalCount() + ballot.joining.size()); + context.rebalance(context.size() + ballot.joining.size()); var joining = new ArrayList(); var pending = ballot.joining() .stream() @@ -200,26 +209,29 @@ void install(Ballot ballot) { view.notifyListeners(joining, ballot.leaving); } + boolean isObserver(Digest observer) { + return observers().contains(observer); + } + /** - * Formally join the view. Calculate the HEX-BLOOM crown and view, fail and stop if does not match currentView + * Formally join the view. Calculate the HEX-BLOOM crown and view, fail and stop if it does not match currentView */ void join() { joinLock.lock(); try { - assert context.totalCount() == cardinality(); + assert context.size() == cardinality(); if (joined()) { return; } var current = currentView(); - log.info("Joining view: {} cardinality: {} count: {} on: {}", current, cardinality(), context.totalCount(), + log.info("Joining view: {} cardinality: {} count: {} on: {}", current, cardinality(), context.size(), node.getId()); - var calculated = HexBloom.construct(context.totalCount(), context.allMembers().map(Participant::getId), + var calculated = HexBloom.construct(context.size(), context.allMembers().map(Participant::getId), view.bootstrapView(), params.crowns()); if (!current.equals(calculated.compactWrapped())) { log.error("Crown: {} does not produce view: {} cardinality: {} count: {} on: {}", - calculated.compactWrapped(), currentView(), cardinality(), context.totalCount(), - node.getId()); + calculated.compactWrapped(), currentView(), cardinality(), context.size(), node.getId()); view.stop(); throw new IllegalStateException("Invalid crown"); } @@ -232,7 +244,7 @@ void join() { if (metrics != null) { metrics.viewChanges().mark(); } - log.info("Joined view: {} cardinality: {} count: {} on: {}", current, cardinality(), context.totalCount(), + log.info("Joined view: {} cardinality: {} count: {} on: {}", current, cardinality(), context.size(), node.getId()); onJoined.complete(null); } finally { @@ -250,8 +262,8 @@ void join(Join join, Digest from, StreamObserver responseObserver, Time } var note = new NoteWrapper(join.getNote(), digestAlgo); if (!from.equals(note.getId())) { - log.info("Ignored join of view: {} from: {} does not match: {} on: {}", joinView, from, note.getId(), - node.getId()); + log.debug("Ignored join of view: {} from: {} does not match: {} on: {}", joinView, from, note.getId(), + node.getId()); responseObserver.onError( new StatusRuntimeException(Status.INVALID_ARGUMENT.withDescription("Member does not match note"))); return; @@ -259,7 +271,7 @@ void join(Join join, Digest from, StreamObserver responseObserver, Time if (!view.validate(note.getIdentifier())) { responseObserver.onError( new StatusRuntimeException(Status.INVALID_ARGUMENT.withDescription("Invalid identifier"))); - log.info("Ignored join of view: {} from: {} invalid identifier on: {}", joinView, from, node.getId()); + log.debug("Ignored join of view: {} from: {} invalid identifier on: {}", joinView, from, node.getId()); return; } view.stable(() -> { @@ -312,8 +324,8 @@ void join(Join join, Digest from, StreamObserver responseObserver, Time view.viewChange(() -> { final var hex = bound.view(); - log.info("Rebalancing to cardinality: {} (join) for: {} context: {} on: {}", hex.getCardinality(), - hex.compactWrapped(), context.getId(), node.getId()); + log.debug("Rebalancing to cardinality: {} (join) for: {} context: {} on: {}", hex.getCardinality(), + hex.compactWrapped(), context.getId(), node.getId()); context.rebalance(hex.getCardinality()); context.activate(node); diadem.set(hex); @@ -333,12 +345,13 @@ void join(Join join, Digest from, StreamObserver responseObserver, Time } view.introduced(); - log.info("Currently joining view: {} seeds: {} cardinality: {} count: {} on: {}", currentView.get(), - bound.successors().size(), cardinality(), context.totalCount(), node.getId()); - if (context.totalCount() == cardinality()) { + log.debug("Currently joining view: {} seeds: {} cardinality: {} count: {} on: {}", + currentView.get(), bound.successors().size(), cardinality(), context.size(), + node.getId()); + if (context.size() == cardinality()) { join(); } else { - populate(new ArrayList<>(context.activeMembers())); + // populate(new ArrayList<>(context.activeMembers())); } }); }, log)); @@ -363,6 +376,8 @@ boolean joined() { void maybeViewChange() { if ((context.offlineCount() > 0 || !joins.isEmpty())) { if (isObserver()) { + log.trace("Initiating view change: {} (non observer) joins: {} leaves: {} on: {}", currentView(), + joins.size(), view.streamShunned().count(), node.getId()); initiateViewChange(); } else { // Use pending rebuttals as a proxy for stability @@ -374,10 +389,20 @@ void maybeViewChange() { } } } else { + log.trace("No view change: {} joins: {} leaves: {} on: {}", currentView(), joins.size(), + view.streamShunned().count(), node.getId()); view.scheduleViewChange(); } } + Set observers() { + return observers; + } + + List observersList() { + return observers().stream().toList(); + } + void populate(List sample) { var populate = new SliceIterator("Populate: " + context.getId(), node, sample, view.comm); var repopulate = new AtomicReference(); @@ -464,14 +489,17 @@ Redirect seed(Registration registration, Digest from) { } return view.stable(() -> { var newMember = view.new Participant(note.getId()); - final var sample = context.sample(params.maximumTxfr(), Entropy.bitsStream(), (Digest) null); - log.info("Member seeding: {} view: {} context: {} sample: {} on: {}", newMember.getId(), currentView(), - context.getId(), sample.size(), node.getId()); + final var introductions = observers.stream().map(context::getMember).toList(); + + log.debug("Member seeding: {} view: {} context: {} introductions: {} on: {}", newMember.getId(), + currentView(), context.getId(), introductions.size(), node.getId()); return Redirect.newBuilder() .setView(currentView().toDigeste()) - .addAllSample( - sample.stream().filter(java.util.Objects::nonNull).map(Participant::getSignedNote).toList()) + .addAllIntroductions(introductions.stream() + .filter(java.util.Objects::nonNull) + .map(Participant::getSignedNote) + .toList()) .setCardinality(cardinality()) .setBootstrap(bootstrap) .setRings(context.getRingCount()) @@ -501,6 +529,10 @@ private void initiateViewChange() { return; } view.scheduleFinalizeViewChange(); + if (!isObserver(node.getId())) { + log.trace("Initiating view change: {} (non observer) on: {}", currentView(), node.getId()); + return; + } log.trace("Initiating view change vote: {} joins: {} leaves: {} on: {}", currentView(), joins.size(), view.streamShunned().count(), node.getId()); final var builder = ViewChange.newBuilder() @@ -518,8 +550,8 @@ private void initiateViewChange() { .setSignature(signature.toSig()) .build(); view.initiate(viewChange); - log.info("View change vote: {} joins: {} leaves: {} on: {}", currentView(), change.getJoinsCount(), - change.getLeavesCount(), node.getId()); + log.debug("View change vote: {} joins: {} leaves: {} on: {}", currentView(), change.getJoinsCount(), + change.getLeavesCount(), node.getId()); }); } @@ -565,32 +597,44 @@ private void joined(Collection seedSet, Digest from, StreamObserver< } } + private void resetObservers() { + observers.clear(); + context.bftSubset(diadem.get().compact(), context::isActive) + .stream() + .map(Member::getId) + .forEach(observers::add); + if (observers.isEmpty()) { + observers.add(node.getId()); // bootstrap case + } + if (observers.size() > 1 && observers.size() < context.getRingCount()) { + log.debug("Incomplete observers: {} cardinality: {} view: {} context: {} on: {}", observers.size(), + context.cardinality(), currentView(), context.getId(), node.getId()); + assert observers.size() > 1 && observers.size() < context.getRingCount(); + } + log.trace("Reset observers: {} cardinality: {} view: {} context: {} on: {}", observers.size(), + context.cardinality(), currentView(), context.getId(), node.getId()); + } + private void setDiadem(final HexBloom hex) { diadem.set(hex); currentView.set(diadem.get().compactWrapped()); - observers.clear(); - context.bftSubset(hex.compact()).stream().map(Member::getId).forEach(observers::add); - log.info("View: {} set diadem: {} observers: {} on: {}", context.getId(), diadem.get().compactWrapped(), - observers.stream().toList(), node.getId()); + resetObservers(); + log.debug("View: {} set diadem: {} observers: {} view: {} context: {} size: {} on: {}", context.getId(), + diadem.get().compactWrapped(), observers.stream().toList(), currentView(), context.getId(), + context.size(), node.getId()); } record Ballot(Digest view, List leaving, List joining, int hash) { Ballot(Digest view, List leaving, List joining, DigestAlgorithm algo) { - this(view, leaving, joining, view.xor(joining.stream().reduce(Digest::xor).orElse(algo.getOrigin())) - .xor(leaving.stream() - .reduce(Digest::xor) - .orElse(algo.getOrigin()) - .xor( - joining.stream().reduce(Digest::xor).orElse(algo.getOrigin()))) - .hashCode()); + this(view, leaving, joining, Objects.hash(view, joining, leaving)); } @Override public boolean equals(Object obj) { if (obj instanceof Ballot b) { - return Objects.equal(view, b.view) && Objects.equal(leaving, b.leaving) && Objects.equal(joining, - b.joining); + return Objects.equals(view, b.view) && Objects.equals(leaving, b.leaving) && Objects.equals(joining, + b.joining); } return false; } @@ -602,7 +646,7 @@ public int hashCode() { @Override public String toString() { - return String.format("{h: %s, j: %s, l: %s}", hash, joining.size(), leaving.size()); + return String.format("{v: %s, h: %s, j: %s, l: %s}", view, hash, joining.size(), leaving.size()); } } } diff --git a/fireflies/src/test/java/com/salesforce/apollo/fireflies/ChurnTest.java b/fireflies/src/test/java/com/salesforce/apollo/fireflies/ChurnTest.java index 2c9af24ef..d209c439b 100644 --- a/fireflies/src/test/java/com/salesforce/apollo/fireflies/ChurnTest.java +++ b/fireflies/src/test/java/com/salesforce/apollo/fireflies/ChurnTest.java @@ -9,6 +9,7 @@ import com.codahale.metrics.ConsoleReporter; import com.codahale.metrics.MetricRegistry; import com.salesforce.apollo.archipelago.*; +import com.salesforce.apollo.context.Context; import com.salesforce.apollo.context.DynamicContext; import com.salesforce.apollo.cryptography.Digest; import com.salesforce.apollo.cryptography.DigestAlgorithm; @@ -42,7 +43,7 @@ public class ChurnTest { private static final int CARDINALITY = 100; - private static final double P_BYZ = 0.3; + private static final double P_BYZ = 0.2; private static Map> identities; private static KERL.AppendKERL kerl; private final List communications = new ArrayList<>(); @@ -143,45 +144,52 @@ public void churn() throws Exception { toStart.forEach(view -> view.start(() -> countdown.get().countDown(), gossipDuration, seeds)); success = countdown.get().await(30, TimeUnit.SECONDS); - failed = testViews.stream() - .filter(e -> e.getContext().activeCount() != testViews.size() - || e.getContext().totalCount() != testViews.size()) - .sorted(Comparator.comparing(v -> v.getContext().activeCount())) - .map(v -> String.format("%s : %s : %s ", v.getNode().getId(), v.getContext().totalCount(), - v.getContext().activeCount())) - .toList(); + failed = testViews.stream().filter(e -> { + if (e.getContext().activeCount() != testViews.size()) + return true; + Context participantContext = e.getContext(); + return participantContext.size() != testViews.size(); + }).sorted(Comparator.comparing(v -> v.getContext().activeCount())).map(v -> { + Context participantContext = v.getContext(); + return String.format("%s : %s : %s ", v.getNode().getId(), participantContext.size(), + v.getContext().activeCount()); + }).toList(); assertTrue(success, " expected: " + testViews.size() + " failed: " + failed.size() + " views: " + failed); success = Utils.waitForCondition(30_000, 1_000, () -> { return testViews.stream() .map(v -> v.getContext()) - .filter( - ctx -> ctx.totalCount() != testViews.size() || ctx.activeCount() != testViews.size()) + .filter(ctx -> ctx.size() != testViews.size() || ctx.activeCount() != testViews.size()) .count() == 0; }); - failed = testViews.stream() - .filter(e -> e.getContext().activeCount() != testViews.size() - || e.getContext().totalCount() != testViews.size()) - .sorted(Comparator.comparing(v -> v.getContext().activeCount())) - .map(v -> String.format("%s : %s : %s ", v.getNode().getId(), v.getContext().totalCount(), - v.getContext().activeCount())) - .toList(); + failed = testViews.stream().filter(e -> { + if (e.getContext().activeCount() != testViews.size()) + return true; + Context participantContext = e.getContext(); + return participantContext.size() != testViews.size(); + }).sorted(Comparator.comparing(v -> v.getContext().activeCount())).map(v -> { + Context participantContext = v.getContext(); + return String.format("%s : %s : %s ", v.getNode().getId(), participantContext.size(), + v.getContext().activeCount()); + }).toList(); assertTrue(success, " expected: " + testViews.size() + " failed: " + failed.size() + " views: " + failed); success = Utils.waitForCondition(30_000, 1_000, () -> { return testViews.stream() .map(v -> v.getContext()) - .filter( - ctx -> ctx.totalCount() != testViews.size() || ctx.activeCount() != testViews.size()) + .filter(ctx -> ctx.size() != testViews.size() || ctx.activeCount() != testViews.size()) .count() == 0; }); - failed = testViews.stream() - .filter(e -> e.getContext().activeCount() != testViews.size() - || e.getContext().totalCount() != testViews.size()) - .sorted(Comparator.comparing(v -> v.getContext().activeCount())) - .map(v -> String.format("%s : %s : %s ", v.getNode().getId(), v.getContext().totalCount(), - v.getContext().activeCount())) - .toList(); + failed = testViews.stream().filter(e -> { + if (e.getContext().activeCount() != testViews.size()) + return true; + Context participantContext = e.getContext(); + return participantContext.size() != testViews.size(); + }).sorted(Comparator.comparing(v -> v.getContext().activeCount())).map(v -> { + Context participantContext = v.getContext(); + return String.format("%s : %s : %s ", v.getNode().getId(), participantContext.size(), + v.getContext().activeCount()); + }).toList(); assertTrue(success, " expected: " + testViews.size() + " failed: " + failed.size() + " views: " + failed); System.out.println( @@ -213,10 +221,13 @@ public void churn() throws Exception { // System.out.println("** Removed: " + removed); then = System.currentTimeMillis(); success = Utils.waitForCondition(60_000, 1_000, () -> { - return expected.stream().filter(view -> view.getContext().totalCount() > expected.size()).count() == 0; + return expected.stream().filter(view -> { + Context participantContext = view.getContext(); + return participantContext.size() > expected.size(); + }).count() == 0; }); failed = expected.stream() - .filter(e -> e.getContext().activeCount() != testViews.size()) + .filter(e -> e.getContext().activeCount() != expected.size()) .sorted(Comparator.comparing(v -> v.getContext().activeCount())) .map(v -> String.format("%s : %s ", v.getNode().getId(), v.getContext().activeCount())) .toList(); diff --git a/fireflies/src/test/java/com/salesforce/apollo/fireflies/E2ETest.java b/fireflies/src/test/java/com/salesforce/apollo/fireflies/E2ETest.java index 1b72a86a9..a6e58f479 100644 --- a/fireflies/src/test/java/com/salesforce/apollo/fireflies/E2ETest.java +++ b/fireflies/src/test/java/com/salesforce/apollo/fireflies/E2ETest.java @@ -9,6 +9,7 @@ import com.codahale.metrics.ConsoleReporter; import com.codahale.metrics.MetricRegistry; import com.salesforce.apollo.archipelago.*; +import com.salesforce.apollo.context.Context; import com.salesforce.apollo.context.DynamicContext; import com.salesforce.apollo.cryptography.Digest; import com.salesforce.apollo.cryptography.DigestAlgorithm; @@ -131,11 +132,11 @@ public void smokin() throws Exception { success = countdown.get().await(largeTests ? 2400 : 30, TimeUnit.SECONDS); // Test that all views are up - failed = views.stream() - .filter(e -> e.getContext().activeCount() != CARDINALITY) - .map(v -> String.format("%s : %s : %s ", v.getNode().getId(), v.getContext().activeCount(), - v.getContext().totalCount())) - .toList(); + failed = views.stream().filter(e -> e.getContext().activeCount() != CARDINALITY).map(v -> { + Context participantContext = v.getContext(); + return String.format("%s : %s : %s ", v.getNode().getId(), v.getContext().activeCount(), + participantContext.size()); + }).toList(); assertTrue(success, "Views did not start, expected: " + views.size() + " failed: " + failed.size() + " views: " + failed); @@ -144,11 +145,11 @@ public void smokin() throws Exception { }); // Test that all views are up - failed = views.stream() - .filter(e -> e.getContext().activeCount() != CARDINALITY) - .map(v -> String.format("%s : %s : %s ", v.getNode().getId(), v.getContext().activeCount(), - v.getContext().totalCount())) - .toList(); + failed = views.stream().filter(e -> e.getContext().activeCount() != CARDINALITY).map(v -> { + Context participantContext = v.getContext(); + return String.format("%s : %s : %s ", v.getNode().getId(), v.getContext().activeCount(), + participantContext.size()); + }).toList(); assertTrue(success || failed.isEmpty(), "Views did not stabilize, expected: " + views.size() + " failed: " + failed.size() + " views: " + failed); diff --git a/fireflies/src/test/java/com/salesforce/apollo/fireflies/SwarmTest.java b/fireflies/src/test/java/com/salesforce/apollo/fireflies/SwarmTest.java index 5b1167d8d..0e4f93dc6 100644 --- a/fireflies/src/test/java/com/salesforce/apollo/fireflies/SwarmTest.java +++ b/fireflies/src/test/java/com/salesforce/apollo/fireflies/SwarmTest.java @@ -9,6 +9,7 @@ import com.codahale.metrics.ConsoleReporter; import com.codahale.metrics.MetricRegistry; import com.salesforce.apollo.archipelago.*; +import com.salesforce.apollo.context.Context; import com.salesforce.apollo.context.DynamicContext; import com.salesforce.apollo.cryptography.Digest; import com.salesforce.apollo.cryptography.DigestAlgorithm; @@ -132,11 +133,11 @@ public void swarm() throws Exception { success = countdown.get().await(largeTests ? 2400 : 120, TimeUnit.SECONDS); // Test that all views are up - failed = views.stream() - .filter(e -> e.getContext().activeCount() != CARDINALITY) - .map(v -> String.format("%s : %s : %s : %s ", v.getNode().getId(), v.getContext().cardinality(), - v.getContext().activeCount(), v.getContext().totalCount())) - .toList(); + failed = views.stream().filter(e -> e.getContext().activeCount() != CARDINALITY).map(v -> { + Context participantContext = v.getContext(); + return String.format("%s : %s : %s : %s ", v.getNode().getId(), v.getContext().cardinality(), + v.getContext().activeCount(), participantContext.size()); + }).toList(); assertTrue(success, "Views did not start, expected: " + views.size() + " failed: " + failed.size() + " views: " + failed); @@ -145,11 +146,11 @@ public void swarm() throws Exception { }); // Test that all views are up - failed = views.stream() - .filter(e -> e.getContext().activeCount() != CARDINALITY) - .map(v -> String.format("%s : %s : %s ", v.getNode().getId(), v.getContext().activeCount(), - v.getContext().totalCount())) - .toList(); + failed = views.stream().filter(e -> e.getContext().activeCount() != CARDINALITY).map(v -> { + Context participantContext = v.getContext(); + return String.format("%s : %s : %s ", v.getNode().getId(), v.getContext().activeCount(), + participantContext.size()); + }).toList(); assertTrue(success, "Views did not stabilize, expected: " + views.size() + " failed: " + failed.size() + " views: " + failed); @@ -207,7 +208,6 @@ private void initialize() { .setMaxPending(50) .setMaximumTxfr(20) .setJoinRetries(30) - .setFpr(0.00000125) .setSeedingTimout(Duration.ofSeconds(10)) .setRetryDelay(Duration.ofMillis(largeTests ? 1000 : 200)) .build(); diff --git a/fireflies/src/test/resources/logback-test.xml b/fireflies/src/test/resources/logback-test.xml index 603d038bc..f86c9c940 100644 --- a/fireflies/src/test/resources/logback-test.xml +++ b/fireflies/src/test/resources/logback-test.xml @@ -14,7 +14,7 @@ - + diff --git a/gorgoneion/src/main/java/com/salesforce/apollo/gorgoneion/Gorgoneion.java b/gorgoneion/src/main/java/com/salesforce/apollo/gorgoneion/Gorgoneion.java index 3f13824cf..937c584b5 100644 --- a/gorgoneion/src/main/java/com/salesforce/apollo/gorgoneion/Gorgoneion.java +++ b/gorgoneion/src/main/java/com/salesforce/apollo/gorgoneion/Gorgoneion.java @@ -157,10 +157,9 @@ private SignedNonce generateNonce(KERL_ application) { .setTimestamp(Timestamp.newBuilder().setSeconds(now.getEpochSecond()).setNanos(now.getNano())) .build(); - var successors = context.totalCount() == 1 ? Collections.singletonList(member) - : Context.uniqueSuccessors(context, digestOf(ident, - parameters.digestAlgorithm())); - final var majority = context.totalCount() == 1 ? 1 : context.majority(); + var successors = context.size() == 1 ? Collections.singletonList(member) + : context.bftSubset(digestOf(ident, parameters.digestAlgorithm())); + final var majority = context.size() == 1 ? 1 : context.majority(); final var redirecting = new SliceIterator<>("Nonce Endorsement", member, successors, endorsementComm); Set endorsements = Collections.newSetFromMap(new ConcurrentHashMap<>()); var generated = new CompletableFuture(); @@ -221,9 +220,8 @@ private void notarize(Credentials credentials, Validations validations) { .setValidations(validations) .build(); - var successors = Context.uniqueSuccessors(context, - digestOf(identifier.toIdent(), parameters.digestAlgorithm())); - final var majority = context.totalCount() == 1 ? 1 : context.majority(); + var successors = context.bftSubset(digestOf(identifier.toIdent(), parameters.digestAlgorithm())); + final var majority = context.size() == 1 ? 1 : context.majority(); SliceIterator redirecting = new SliceIterator<>("Enrollment", member, successors, endorsementComm); var completed = new HashSet(); redirecting.iterate((link, m) -> { @@ -248,9 +246,8 @@ private Validations register(Credentials request) { var validated = new CompletableFuture(); - var successors = Context.uniqueSuccessors(context, - digestOf(identifier.toIdent(), parameters.digestAlgorithm())); - final var majority = context.totalCount() == 1 ? 1 : context.majority(); + var successors = context.bftSubset(digestOf(identifier.toIdent(), parameters.digestAlgorithm())); + final var majority = context.size() == 1 ? 1 : context.majority(); final var redirecting = new SliceIterator<>("Credential verification", member, successors, endorsementComm); var verifications = new HashSet(); redirecting.iterate((link, m) -> { @@ -472,7 +469,7 @@ private boolean validate(Notarization request, Identifier identifier, KERL_ kerl } } // If there is only one active member in our context, it's us. - var majority = context.totalCount() == 1 ? 1 : context.majority(); + var majority = context.size() == 1 ? 1 : context.majority(); if (count < majority) { log.warn("Invalid notarization, no majority: {} required: {} for: {} from: {} on: {}", count, majority, identifier, from, member.getId()); @@ -531,7 +528,7 @@ private boolean validateCredentials(Credentials credentials, Digest from) { count++; } - var majority = context.totalCount() == 1 ? 1 : context.majority(); + var majority = context.size() == 1 ? 1 : context.majority(); if (count < majority) { log.warn("Invalid credential nonce, no majority signature: {} required >= {} from: {} on: {}", count, majority, from, member.getId()); diff --git a/grpc/src/main/proto/ethereal.proto b/grpc/src/main/proto/ethereal.proto index da9f52823..91620e271 100644 --- a/grpc/src/main/proto/ethereal.proto +++ b/grpc/src/main/proto/ethereal.proto @@ -15,9 +15,8 @@ service Gossiper { } message Gossip { - int32 ring = 1; - crypto.Biff have = 2; - repeated Have haves = 3; + crypto.Biff have = 1; + repeated Have haves = 2; } message Have { diff --git a/grpc/src/main/proto/fireflies.proto b/grpc/src/main/proto/fireflies.proto index 44398361a..036713cf5 100644 --- a/grpc/src/main/proto/fireflies.proto +++ b/grpc/src/main/proto/fireflies.proto @@ -132,7 +132,7 @@ message Redirect { int32 cardinality = 2; int32 rings = 3; bool bootstrap = 4; - repeated SignedNote sample = 6; + repeated SignedNote introductions = 6; } message Join { diff --git a/isolate-ftesting/src/test/java/com/salesforce/apollo/domain/DemesneIsolateTest.java b/isolate-ftesting/src/test/java/com/salesforce/apollo/domain/DemesneIsolateTest.java index c43ceed88..f166dd4e9 100644 --- a/isolate-ftesting/src/test/java/com/salesforce/apollo/domain/DemesneIsolateTest.java +++ b/isolate-ftesting/src/test/java/com/salesforce/apollo/domain/DemesneIsolateTest.java @@ -121,7 +121,7 @@ public void register(SubContext context) { .setParent(parentAddress) .setCommDirectory(commDirectory.toString()) .setMaxTransfer(100) - .setFalsePositiveRate(.125) + .setFalsePositiveRate(.00125) .build(); var demesne = new JniBridge(parameters); Builder specification = IdentifierSpecification.newBuilder(); diff --git a/isolates/src/test/java/com/salesforce/apollo/demesnes/DemesneSmoke.java b/isolates/src/test/java/com/salesforce/apollo/demesnes/DemesneSmoke.java index 4c13cd545..965402473 100644 --- a/isolates/src/test/java/com/salesforce/apollo/demesnes/DemesneSmoke.java +++ b/isolates/src/test/java/com/salesforce/apollo/demesnes/DemesneSmoke.java @@ -171,7 +171,6 @@ public void register(SubContext context) { .setParent(parentAddress) .setCommDirectory(commDirectory.toString()) .setMaxTransfer(100) - .setFalsePositiveRate(.125) .build(); final var demesne = new DemesneImpl(parameters); Builder specification = IdentifierSpecification.newBuilder(); diff --git a/leyden/src/main/java/com/salesforce/apollo/leyden/LeydenJar.java b/leyden/src/main/java/com/salesforce/apollo/leyden/LeydenJar.java index 2ad10c645..a945046d6 100644 --- a/leyden/src/main/java/com/salesforce/apollo/leyden/LeydenJar.java +++ b/leyden/src/main/java/com/salesforce/apollo/leyden/LeydenJar.java @@ -105,8 +105,7 @@ public void bind(Binding bound) { Supplier isTimedOut = () -> Instant.now().isAfter(timedOut); var result = new CompletableFuture(); var gathered = HashMultiset.create(); - var iterate = new RingIterator(operationsFrequency, context, member, scheduler, - binderComms); + var iterate = new RingIterator<>(operationsFrequency, context, member, scheduler, binderComms); iterate.iterate(hash, null, (link, r) -> { link.bind(bound); return ""; @@ -133,8 +132,7 @@ public Bound get(Key keyAndToken) { Supplier isTimedOut = () -> Instant.now().isAfter(timedOut); var result = new CompletableFuture(); var gathered = HashMultiset.create(); - var iterate = new RingIterator(operationsFrequency, context, member, scheduler, - binderComms); + var iterate = new RingIterator<>(operationsFrequency, context, member, scheduler, binderComms); iterate.iterate(hash, null, (link, r) -> { var bound = link.get(keyAndToken); log.debug("Get {}: bound: <{}:{}> from: {} on: {}", hash, bound.getKey().toStringUtf8(), @@ -188,8 +186,7 @@ public void unbind(Key keyAndToken) { Supplier isTimedOut = () -> Instant.now().isAfter(timedOut); var result = new CompletableFuture(); var gathered = HashMultiset.create(); - var iterate = new RingIterator(operationsFrequency, context, member, scheduler, - binderComms); + var iterate = new RingIterator<>(operationsFrequency, context, member, scheduler, binderComms); iterate.iterate(hash, null, (link, r) -> { link.unbind(keyAndToken); return ""; @@ -221,7 +218,7 @@ private void add(Digest hash, Bound bound, Digest digest) { private Stream bindingsIn(KeyInterval i) { Iterator it = new Iterator() { private final Iterator iterate = bottled.keyIterator(i.getBegin()); - private Digest next; + private Digest next; { if (iterate.hasNext()) { diff --git a/memberships/src/main/java/com/salesforce/apollo/archipelago/LocalServer.java b/memberships/src/main/java/com/salesforce/apollo/archipelago/LocalServer.java index 1f28baf49..74403835d 100644 --- a/memberships/src/main/java/com/salesforce/apollo/archipelago/LocalServer.java +++ b/memberships/src/main/java/com/salesforce/apollo/archipelago/LocalServer.java @@ -79,9 +79,6 @@ public RouterImpl router(ServerConnectionCache.Builder cacheBuilder, Supplier
  • serverBuilder = InProcessServerBuilder.forName(name) .executor(Executors.newVirtualThreadPerTaskExecutor()) - .scheduledExecutorService( - Executors.newScheduledThreadPool(100, Thread.ofVirtual() - .factory())) .intercept(ConcurrencyLimitServerInterceptor.newBuilder( limitsBuilder.build()) .statusSupplier( @@ -105,7 +102,6 @@ private ManagedChannel connectTo(Member to) { final var name = String.format(NAME_TEMPLATE, prefix, qb64(to.getId())); final InProcessChannelBuilder builder = InProcessChannelBuilder.forName(name) .executor(executor) - .offloadExecutor(executor) .usePlaintext() .intercept(clientInterceptor); disableTrash(builder); diff --git a/memberships/src/main/java/com/salesforce/apollo/archipelago/MtlsClient.java b/memberships/src/main/java/com/salesforce/apollo/archipelago/MtlsClient.java index 2ee89f441..58db9ec8e 100644 --- a/memberships/src/main/java/com/salesforce/apollo/archipelago/MtlsClient.java +++ b/memberships/src/main/java/com/salesforce/apollo/archipelago/MtlsClient.java @@ -34,7 +34,6 @@ public MtlsClient(SocketAddress address, ClientAuth clientAuth, String alias, Cl Limiter limiter = new GrpcClientLimiterBuilder().blockOnLimit(false).build(); channel = NettyChannelBuilder.forAddress(address) .executor(executor) - .offloadExecutor(executor) .withOption(ChannelOption.TCP_NODELAY, true) .sslContext(supplier.forClient(clientAuth, alias, validator, MtlsServer.TL_SV1_3)) .intercept(new ConcurrencyLimitClientInterceptor(limiter, diff --git a/memberships/src/main/java/com/salesforce/apollo/context/Context.java b/memberships/src/main/java/com/salesforce/apollo/context/Context.java index ed43d7fa0..b1038e4db 100644 --- a/memberships/src/main/java/com/salesforce/apollo/context/Context.java +++ b/memberships/src/main/java/com/salesforce/apollo/context/Context.java @@ -10,12 +10,10 @@ import com.salesforce.apollo.cryptography.DigestAlgorithm; import com.salesforce.apollo.membership.Member; import com.salesforce.apollo.membership.Util; +import com.salesforce.apollo.ring.RingCommunications; import org.apache.commons.math3.random.BitsStreamGenerator; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Set; +import java.util.*; import java.util.function.Function; import java.util.function.Predicate; import java.util.stream.Stream; @@ -34,18 +32,6 @@ public interface Context { double DEFAULT_EPSILON = 0.99999; - static List uniqueSuccessors(Context context, Digest digest) { - Set post = new HashSet<>(); - context.successors(digest, m -> { - if (post.size() == context.getRingCount()) { - return false; - } - return post.add(m); - }); - var successors = new ArrayList<>(post); - return successors; - } - static Digest hashFor(Digest ctxId, int ring, Digest d) { return d.prefix(ctxId, ring); } @@ -127,15 +113,19 @@ static int minMajority(int bias, double pByz, int cardinality) { * @return the Set of Members constructed from the sucessors of the supplied hash on each of the receiver Context's * rings */ - default Set bftSubset(Digest hash) { - Set successors = new HashSet<>(); - successors(hash, m -> { - if (successors.size() == getRingCount()) { - return false; - } - return successors.add(m); - }); - return successors; + default LinkedHashSet bftSubset(Digest hash) { + return bftSubset(hash, m -> true); + } + + /** + * @param hash - the point on the rings to determine successors + * @param filter - the filter to apply to successors + * @return the Set of Members constructed from the sucessors of the supplied hash on each of the receiver Context's + */ + default LinkedHashSet bftSubset(Digest hash, Predicate filter) { + var collector = new LinkedHashSet(); + uniqueSuccessors(hash, filter, collector); + return collector; } /** @@ -208,6 +198,13 @@ default int diameter() { */ T getMember(Digest memberID); + /** + * @param i + * @param ring + * @return the i'th Member in Ring 0 of the receiver + */ + T getMember(int i, int ring); + /** * Answer the probability {0, 1} that any given member is byzantine */ @@ -466,6 +463,32 @@ default int majority() { Iterable successors(int ring, Digest location); + default List> successors(Digest digest, T ignore, boolean noDuplicates, T member) { + var traversal = new ArrayList>(); + var traversed = new TreeSet(); + for (int ring = 0; ring < getRingCount(); ring++) { + if (size() == 1) { + traversal.add(new RingCommunications.iteration<>(member, ring)); + continue; + } + T successor = findSuccessor(ring, digest, m -> { + if (ignore != null && ignore.equals(m)) { + return Context.IterateResult.CONTINUE; + } + if (noDuplicates) { + if (traversed.add(m)) { + return Context.IterateResult.SUCCESS; + } else { + return Context.IterateResult.CONTINUE; + } + } + return Context.IterateResult.SUCCESS; + }); + traversal.add(new RingCommunications.iteration<>(successor, ring)); + } + return traversal; + } + /** * @param ring * @param predicate @@ -498,17 +521,29 @@ default int toleranceLevel() { return (getRingCount() - 1) / getBias(); } - /** - * @return the total number of members - */ - int totalCount(); - /** * @param member * @return the iteratator to traverse the ring starting at the member */ Iterable traverse(int ring, T member); + /** + * collect the list of successors to the key on each ring that pass the provided predicate test, and providing a + * unique member per ring if possible. + */ + void uniqueSuccessors(Digest key, Predicate test, Set collector); + + default Set uniqueSuccessors(Digest digest) { + var collected = new HashSet(); + uniqueSuccessors(digest, collected); + return collected; + } + + /** + * collect the list of successors to the key on each ring, providing a unique member per ring if possible. + */ + void uniqueSuccessors(Digest key, Set collector); + boolean validRing(int ring); /** diff --git a/memberships/src/main/java/com/salesforce/apollo/context/DelegatedContext.java b/memberships/src/main/java/com/salesforce/apollo/context/DelegatedContext.java index d0265f05c..ecb159025 100644 --- a/memberships/src/main/java/com/salesforce/apollo/context/DelegatedContext.java +++ b/memberships/src/main/java/com/salesforce/apollo/context/DelegatedContext.java @@ -5,6 +5,7 @@ import org.apache.commons.math3.random.BitsStreamGenerator; import java.util.List; +import java.util.Set; import java.util.function.Function; import java.util.function.Predicate; import java.util.stream.Stream; @@ -94,6 +95,11 @@ public T getMember(Digest memberID) { return delegate.getMember(memberID); } + @Override + public T getMember(int i, int ring) { + return delegate.getMember(i, ring); + } + @Override public double getProbabilityByzantine() { return delegate.getProbabilityByzantine(); @@ -333,13 +339,18 @@ public int toleranceLevel() { } @Override - public int totalCount() { - return delegate.totalCount(); + public Iterable traverse(int ring, T member) { + return delegate.traverse(ring, member); + } + + @Override + public void uniqueSuccessors(Digest key, Predicate test, Set collector) { + delegate.uniqueSuccessors(key, test, collector); } @Override - public Iterable traverse(int ring, T member) { - return delegate.traverse(ring, member); + public void uniqueSuccessors(Digest key, Set collector) { + delegate.uniqueSuccessors(key, collector); } @Override diff --git a/memberships/src/main/java/com/salesforce/apollo/context/DynamicContextImpl.java b/memberships/src/main/java/com/salesforce/apollo/context/DynamicContextImpl.java index dce34e277..94cd7db62 100644 --- a/memberships/src/main/java/com/salesforce/apollo/context/DynamicContextImpl.java +++ b/memberships/src/main/java/com/salesforce/apollo/context/DynamicContextImpl.java @@ -258,6 +258,13 @@ public T getMember(Digest memberID) { return tracked == null ? null : tracked.member(); } + @Override + public T getMember(int i, int r) { + i = i % size(); + var ring = ring(i); + return (T) ring.get(i); + } + @Override public Collection getOffline() { return members.values().stream().filter(e -> !e.isActive()).map(Tracked::member).toList(); @@ -716,13 +723,26 @@ public String toString() { } @Override - public int totalCount() { - return members.size(); + public Iterable traverse(int ring, T member) { + return ring(ring).traverse(member); } + /** + * @return the list of successor to the key on each ring that pass the provided predicate test + */ @Override - public Iterable traverse(int ring, T member) { - return ring(ring).traverse(member); + public void uniqueSuccessors(Digest key, Predicate test, Set collector) { + for (Ring ring : rings) { + T successor = ring.successor(key, m -> !collector.contains(m) && test.test(m)); + if (successor != null) { + collector.add(successor); + } + } + } + + @Override + public void uniqueSuccessors(Digest key, Set collector) { + uniqueSuccessors(key, t -> true, collector); } @Override @@ -1213,7 +1233,7 @@ public Iterable successors(Digest location) { } /** - * @param start + * @param location * @param predicate * @return an Iterable of all items counter-clock wise in the ring from (but excluding) start location to (but * excluding) the first item where predicate(item) evaluates to True. @@ -1223,7 +1243,7 @@ public Iterable successors(Digest location, Predicate predicate) { } /** - * @param start + * @param m * @param predicate * @return an Iterable of all items counter-clock wise in the ring from (but excluding) start item to (but * excluding) the first item where predicate(item) evaluates to True. diff --git a/memberships/src/main/java/com/salesforce/apollo/context/Search.java b/memberships/src/main/java/com/salesforce/apollo/context/Search.java new file mode 100644 index 000000000..11c3ec672 --- /dev/null +++ b/memberships/src/main/java/com/salesforce/apollo/context/Search.java @@ -0,0 +1,122 @@ +package com.salesforce.apollo.context; + +import com.salesforce.apollo.utils.Pair; + +import java.util.Comparator; +import java.util.List; + +/** + * Search used to search in an ordered collection of Vector of type T comparisons are done using K which can be + * extracted from T + * + * @param vector type + * @param the key used for sorting to extract from T and to compare + * @author lindenb + */ +public abstract class Search { + private final Comparator comparator; + + /** + * Search + * + * @param comparator used to compare two keys of type K + */ + public Search(Comparator comparator) { + this.comparator = comparator; + } + + /** C+ equals range */ + public Pair equal_range(List dataVector, K select) { + return equal_range(dataVector, 0, dataVector.size(), select); + } + + /** + * C+ equals range + * + * @param bounds array of two integers [begin; end] + * @param select the value to search + */ + public Pair equal_range(List dataVector, Pair bounds, K select) { + return equal_range(dataVector, bounds.a(), bounds.b(), select); + } + + /** C+ equals range */ + public Pair equal_range(List dataVector, int first, int last, K subject) { + int left = lower_bound(dataVector, first, last, subject); + int right = upper_bound(dataVector, left, last, subject); + return new Pair(left, right); + } + + /** @return the internal comparator */ + public Comparator getComparator() { + return this.comparator; + } + + /** method used to extract the key(K) from an object (T) */ + public abstract K getKey(T value); + + /** @return True if the vector is sorted */ + public boolean isSorted(List dataVector) { + return isSorted(dataVector, 0, dataVector.size()); + } + + /** @return True if the vector is sorted between begin and end */ + public boolean isSorted(List dataVector, int begin, int end) { + while (begin + 1 < end) { + if (this.comparator.compare(getKey(dataVector.get(begin)), getKey(dataVector.get(begin + 1))) > 0) { + return false; + } + ++begin; + } + return true; + } + + /** C+ lower_bound */ + public int lower_bound(List dataVector, K select) { + return lower_bound(dataVector, 0, dataVector.size(), select); + } + + /** C+ lower_bound */ + public int lower_bound(List dataVector, int first, int last, K select) { + int len = last - first; + while (len > 0) { + int half = len / 2; + int middle = first + half; + T x = dataVector.get(middle); + if (this.comparator.compare(getKey(x), select) < 0) { + first = middle + 1; + len -= half + 1; + } else { + len = half; + } + } + return first; + } + + @Override + public String toString() { + return "Algorithm(" + this.comparator + ")"; + } + + /** C+ upper_bound */ + public int upper_bound(List dataVector, K select) { + return upper_bound(dataVector, 0, dataVector.size(), select); + } + + /** C+ upper_bound */ + public int upper_bound(List dataVector, int first, int last, K select) { + int len = last - first; + while (len > 0) { + int half = len / 2; + int middle = first + half; + T x = dataVector.get(middle); + if (this.comparator.compare(select, getKey(x)) < 0) { + len = half; + } else { + first = middle + 1; + len -= half + 1; + } + } + return first; + } +} diff --git a/memberships/src/main/java/com/salesforce/apollo/context/StaticContext.java b/memberships/src/main/java/com/salesforce/apollo/context/StaticContext.java index b88083b54..42d6a1668 100644 --- a/memberships/src/main/java/com/salesforce/apollo/context/StaticContext.java +++ b/memberships/src/main/java/com/salesforce/apollo/context/StaticContext.java @@ -140,6 +140,13 @@ public T getMember(Digest memberID) { return index >= 0 && (members[index].member.getId().equals(memberID)) ? members[index].member : null; } + @Override + public T getMember(int i, int r) { + i = i % size(); + var ring = new StaticRing(r); + return ring.get(i); + } + @Override public double getProbabilityByzantine() { return pByz; @@ -437,13 +444,23 @@ public Iterable successors(int ring, T m, Predicate predicate) { } @Override - public int totalCount() { - return members.length; + public Iterable traverse(int ring, T member) { + return ring(ring).traverse(member); } @Override - public Iterable traverse(int ring, T member) { - return ring(ring).traverse(member); + public void uniqueSuccessors(Digest key, Predicate test, Set collector) { + for (int ring = 0; ring < rings.length; ring++) { + T successor = ring(ring).successor(key, m -> !collector.contains(m) && test.test(m)); + if (successor != null) { + collector.add(successor); + } + } + } + + @Override + public void uniqueSuccessors(Digest key, Set collector) { + uniqueSuccessors(key, t -> true, collector); } @Override @@ -461,7 +478,7 @@ private Digest[] hashesFor(T m) { } private void initialize(Collection members) { - record ringMapping(Tracked m, short i) { + record ringMapping(Tracked m, int i) { } { var i = 0; @@ -472,7 +489,7 @@ record ringMapping(Tracked m, short i) { Arrays.sort(this.members, Comparator.comparing(t -> t.member)); for (int j = 0; j < rings.length; j++) { var mapped = new TreeMap>(); - for (short i = 0; i < this.members.length; i++) { + for (var i = 0; i < this.members.length; i++) { var m = this.members[i]; mapped.put(Context.hashFor(id, j, m.member.getId()), new ringMapping<>(m, i)); } @@ -580,6 +597,10 @@ public T findSuccessor(T m, Function predicate) { return succ(hashFor(m), predicate); } + public T get(int i) { + return ring().get(i, members).member; + } + public Digest hashFor(Digest d) { return Context.hashFor(id, index, d); } @@ -692,8 +713,8 @@ public Stream stream() { @Override public Iterator iterator() { return new Iterator() { - private final Ring ring = ring(); - private int current = 0; + private final Ring ring = ring(); + private int current = 0; @Override public boolean hasNext() { diff --git a/memberships/src/main/java/com/salesforce/apollo/context/StaticSearch.java b/memberships/src/main/java/com/salesforce/apollo/context/StaticSearch.java new file mode 100644 index 000000000..de23a1568 --- /dev/null +++ b/memberships/src/main/java/com/salesforce/apollo/context/StaticSearch.java @@ -0,0 +1,120 @@ +package com.salesforce.apollo.context; + +import com.salesforce.apollo.utils.Pair; + +import java.util.Comparator; + +/** + * StaticSearch is used to search in an Array of type T comparisons are done using K which can be extracted from T + * + * @param vector type + * @param the key used for sorting to extract from T and to compare + * @author lindenb + */ +public abstract class StaticSearch { + private final Comparator comparator; + + /** + * Search + * + * @param comparator used to compare two keys of type K + */ + public StaticSearch(Comparator comparator) { + this.comparator = comparator; + } + + /** C+ equals range */ + public Pair equal_range(T[] dataVector, K select) { + return equal_range(dataVector, 0, dataVector.length, select); + } + + /** + * C+ equals range + * + * @param bounds array of two integers [begin; end] + * @param select the value to search + */ + public Pair equal_range(T[] dataVector, Pair bounds, K select) { + return equal_range(dataVector, bounds.a(), bounds.b(), select); + } + + /** C+ equals range */ + public Pair equal_range(T[] dataVector, int first, int last, K subject) { + int left = lower_bound(dataVector, first, last, subject); + int right = upper_bound(dataVector, left, last, subject); + return new Pair(left, right); + } + + /** @return the internal comparator */ + public Comparator getComparator() { + return this.comparator; + } + + /** method used to extract the key(K) from an object (T) */ + public abstract K getKey(T value); + + /** @return True if the vector is sorted */ + public boolean isSorted(T[] dataVector) { + return isSorted(dataVector, 0, dataVector.length); + } + + /** @return True if the vector is sorted between begin and end */ + public boolean isSorted(T[] dataVector, int begin, int end) { + while (begin + 1 < end) { + if (this.comparator.compare(getKey(dataVector[begin]), getKey(dataVector[begin + 1])) > 0) { + return false; + } + ++begin; + } + return true; + } + + /** C+ lower_bound */ + public int lower_bound(T[] dataVector, K select) { + return lower_bound(dataVector, 0, dataVector.length, select); + } + + /** C+ lower_bound */ + public int lower_bound(T[] dataVector, int first, int last, K select) { + int len = last - first; + while (len > 0) { + int half = len / 2; + int middle = first + half; + T x = dataVector[middle]; + if (this.comparator.compare(getKey(x), select) < 0) { + first = middle + 1; + len -= half + 1; + } else { + len = half; + } + } + return first; + } + + @Override + public String toString() { + return "Algorithm(" + this.comparator + ")"; + } + + /** C+ upper_bound */ + public int upper_bound(T[] dataVector, K select) { + return upper_bound(dataVector, 0, dataVector.length, select); + } + + /** C+ upper_bound */ + public int upper_bound(T[] dataVector, int first, int last, K select) { + int len = last - first; + while (len > 0) { + int half = len / 2; + int middle = first + half; + T x = dataVector[middle]; + if (this.comparator.compare(select, getKey(x)) < 0) { + len = half; + } else { + first = middle + 1; + len -= half + 1; + } + } + return first; + } +} diff --git a/memberships/src/main/java/com/salesforce/apollo/membership/messaging/rbc/ReliableBroadcaster.java b/memberships/src/main/java/com/salesforce/apollo/membership/messaging/rbc/ReliableBroadcaster.java index 86ef23faa..8de6acb5d 100644 --- a/memberships/src/main/java/com/salesforce/apollo/membership/messaging/rbc/ReliableBroadcaster.java +++ b/memberships/src/main/java/com/salesforce/apollo/membership/messaging/rbc/ReliableBroadcaster.java @@ -330,7 +330,7 @@ public static class Builder implements Cloneable { private double dedupFpr = Math.pow(10, -9); private int deliveredCacheSize = 100; private DigestAlgorithm digestAlgorithm = DigestAlgorithm.DEFAULT; - private double falsePositiveRate = 0.000125; + private double falsePositiveRate = 0.00125; private int maxMessages = 500; public Parameters build() { diff --git a/memberships/src/main/java/com/salesforce/apollo/ring/RingCommunications.java b/memberships/src/main/java/com/salesforce/apollo/ring/RingCommunications.java index 91286415a..572b41827 100644 --- a/memberships/src/main/java/com/salesforce/apollo/ring/RingCommunications.java +++ b/memberships/src/main/java/com/salesforce/apollo/ring/RingCommunications.java @@ -20,11 +20,9 @@ import java.util.ArrayList; import java.util.List; import java.util.Optional; -import java.util.TreeSet; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; import java.util.function.BiFunction; -import java.util.function.Function; /** * @author hal.hildebrand @@ -35,7 +33,6 @@ public class RingCommunications { final Context context; final SigningMember member; private final CommonCommunications comm; - private final Direction direction; private final Lock lock = new ReentrantLock(); private final List> traversalOrder = new ArrayList<>(); protected boolean noDuplicates = true; @@ -48,13 +45,6 @@ public RingCommunications(Context context, SigningMember member, CommonCommun public RingCommunications(Context context, SigningMember member, CommonCommunications comm, boolean ignoreSelf) { - this(Direction.SUCCESSOR, context, member, comm, ignoreSelf); - } - - public RingCommunications(Direction direction, Context context, SigningMember member, - CommonCommunications comm, boolean ignoreSelf) { - assert direction != null && context != null && member != null && comm != null; - this.direction = direction; this.context = context; this.member = member; this.comm = comm; @@ -105,43 +95,15 @@ public String toString() { return "RingCommunications [" + context.getId() + ":" + member.getId() + ":" + currentIndex + "]"; } - @SuppressWarnings("unchecked") - List> calculateTraversal(Digest digest) { - var traversal = new ArrayList>(); - var traversed = new TreeSet(); - for (int ring = 0; ring < context.getRingCount(); ring++) { - if (context.size() == 1) { - traversal.add(new iteration<>((T) member, ring)); - continue; - } - T successor = direction.retrieve(context, ring, digest, m -> { - if (ignoreSelf && m.equals(member)) { - return Context.IterateResult.CONTINUE; - } - if (noDuplicates) { - if (traversed.add(m)) { - return Context.IterateResult.SUCCESS; - } else { - return Context.IterateResult.CONTINUE; - } - } - return Context.IterateResult.SUCCESS; - }); - if (successor != null) { - traversal.add(new iteration<>(successor == null ? (T) member : successor, ring)); - } - } - return traversal; - } - final RingCommunications.Destination next(Digest digest) { lock.lock(); try { final var current = currentIndex; final var count = traversalOrder.size(); if (count == 0 || current == count - 1) { + var successors = context.successors(digest, ignoreSelf ? (T) member : null, noDuplicates, (T) member); traversalOrder.clear(); - traversalOrder.addAll(calculateTraversal(digest)); + traversalOrder.addAll(successors); Entropy.secureShuffle(traversalOrder); log.trace("New traversal order: {}:{} on: {}", context.getRingCount(), traversalOrder, member.getId()); } @@ -194,44 +156,10 @@ private Destination linkFor(Digest digest) { } } - public enum Direction { - PREDECESSOR { - @Override - public T retrieve(Context context, int ring, Digest hash, - Function test) { - return context.findPredecessor(ring, hash, test); - } - - @Override - public T retrieve(Context context, int ring, T member, - Function test) { - return context.findPredecessor(ring, member, test); - } - }, SUCCESSOR { - @Override - public T retrieve(Context context, int ring, Digest hash, - Function test) { - return context.findSuccessor(ring, hash, test); - } - - @Override - public T retrieve(Context context, int ring, T member, - Function test) { - return context.findSuccessor(ring, member, test); - } - }; - - public abstract T retrieve(Context context, int ring, Digest hash, - Function test); - - public abstract T retrieve(Context context, int ring, T member, - Function test); - } - public record Destination(M member, Q link, int ring) { } - private record iteration(T m, int ring) { + public record iteration(T m, int ring) { @Override public String toString() { diff --git a/memberships/src/main/java/com/salesforce/apollo/ring/RingIterator.java b/memberships/src/main/java/com/salesforce/apollo/ring/RingIterator.java index 338564441..c582c09e3 100644 --- a/memberships/src/main/java/com/salesforce/apollo/ring/RingIterator.java +++ b/memberships/src/main/java/com/salesforce/apollo/ring/RingIterator.java @@ -51,18 +51,6 @@ public RingIterator(Duration frequency, Context context, SigningMember member this(frequency, context, member, comm, false, scheduler); } - public RingIterator(Duration frequency, Direction direction, Context context, SigningMember member, - CommonCommunications comm, boolean ignoreSelf, ScheduledExecutorService scheduler) { - super(direction, context, member, comm, ignoreSelf); - this.scheduler = scheduler; - this.frequency = frequency; - } - - public RingIterator(Duration frequency, Direction direction, Context context, SigningMember member, - ScheduledExecutorService scheduler, CommonCommunications comm) { - this(frequency, direction, context, member, comm, false, scheduler); - } - @Override public RingIterator allowDuplicates() { super.allowDuplicates(); diff --git a/memberships/src/main/java/com/salesforce/apollo/ring/SliceIterator.java b/memberships/src/main/java/com/salesforce/apollo/ring/SliceIterator.java index a64b3b5a2..4b255669c 100644 --- a/memberships/src/main/java/com/salesforce/apollo/ring/SliceIterator.java +++ b/memberships/src/main/java/com/salesforce/apollo/ring/SliceIterator.java @@ -18,9 +18,11 @@ import java.io.IOException; import java.time.Duration; +import java.util.Collection; import java.util.Iterator; import java.util.List; import java.util.Optional; +import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; @@ -31,25 +33,30 @@ * @author hal.hildebrand */ public class SliceIterator { - private static final Logger log = LoggerFactory.getLogger(SliceIterator.class); - private final CommonCommunications comm; - private final String label; - private final SigningMember member; - private final List slice; - private final ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1, - Thread.ofVirtual() - .factory()); - private Member current; - private Iterator currentIteration; + private static final Logger log = LoggerFactory.getLogger(SliceIterator.class); - public SliceIterator(String label, SigningMember member, List slice, + private final CommonCommunications comm; + private final String label; + private final SigningMember member; + private final List slice; + private final ScheduledExecutorService scheduler; + private Member current; + private Iterator currentIteration; + + public SliceIterator(String label, SigningMember member, Collection slice, CommonCommunications comm) { - assert member != null && slice != null && comm != null; + this(label, member, slice, comm, Executors.newScheduledThreadPool(1, Thread.ofVirtual().factory())); + } + + public SliceIterator(String label, SigningMember member, Collection s, + CommonCommunications comm, ScheduledExecutorService scheduler) { + assert member != null && s != null && comm != null; this.label = label; this.member = member; - this.slice = slice; + this.slice = new CopyOnWriteArrayList<>(s); this.comm = comm; - Entropy.secureShuffle(slice); + this.scheduler = scheduler; + Entropy.secureShuffle(this.slice); this.currentIteration = slice.iterator(); log.debug("Slice for: <{}> is: {} on: {}", label, slice.stream().map(m -> m.getId()).toList(), member.getId()); } @@ -71,9 +78,10 @@ private void internalIterate(BiFunction round, SlicePredica Consumer allowed = allow -> proceed(allow, proceed, onComplete, frequency); try (Comm link = next()) { - if (link == null) { + if (link == null || link.getMember() == null) { log.trace("No link for iteration of: <{}> on: {}", label, member.getId()); - allowed.accept(handler.handle(Optional.empty(), link, slice.get(slice.size() - 1))); + allowed.accept( + handler.handle(Optional.empty(), link, slice.isEmpty() ? null : slice.get(slice.size() - 1))); return; } log.trace("Iteration of: <{}> to: {} on: {}", label, link.getMember().getId(), member.getId()); @@ -108,6 +116,9 @@ private Comm next() { Entropy.secureShuffle(slice); currentIteration = slice.iterator(); } + if (!currentIteration.hasNext()) { + return null; + } current = currentIteration.next(); return linkFor(current); } diff --git a/memberships/src/test/java/com/salesforce/apollo/context/ContextTests.java b/memberships/src/test/java/com/salesforce/apollo/context/ContextTests.java index 4193bd873..03903b5d3 100644 --- a/memberships/src/test/java/com/salesforce/apollo/context/ContextTests.java +++ b/memberships/src/test/java/com/salesforce/apollo/context/ContextTests.java @@ -47,4 +47,26 @@ public void consistency() throws Exception { assertEquals(members.get(0), successors.get(0)); assertEquals(members.get(1), context.successor(1, members.get(0))); } + + @Test + public void successors() throws Exception { + var context = new DynamicContextImpl<>(DigestAlgorithm.DEFAULT.getOrigin().prefix(1), 10, 0.2, 2); + List members = new ArrayList<>(); + var entropy = SecureRandom.getInstance("SHA1PRNG"); + entropy.setSeed(new byte[] { 6, 6, 6 }); + var stereotomy = new StereotomyImpl(new MemKeyStore(), new MemKERL(DigestAlgorithm.DEFAULT), entropy); + + for (int i = 0; i < 50; i++) { + SigningMember m = new ControlledIdentifierMember(stereotomy.newIdentifier()); + members.add(m); + context.activate(m); + } + var successors = context.bftSubset(members.get(10).getId()); + assertEquals(context.getRingCount(), successors.size()); + + successors = context.bftSubset(members.get(10).getId(), m -> { + return context.isActive(m); + }); + assertEquals(context.getRingCount(), successors.size()); + } } diff --git a/memberships/src/test/java/com/salesforce/apollo/ring/RingCommunicationsTest.java b/memberships/src/test/java/com/salesforce/apollo/ring/RingCommunicationsTest.java index 8998468d0..3c2f6b81a 100644 --- a/memberships/src/test/java/com/salesforce/apollo/ring/RingCommunicationsTest.java +++ b/memberships/src/test/java/com/salesforce/apollo/ring/RingCommunicationsTest.java @@ -28,8 +28,8 @@ public class RingCommunicationsTest { @Test public void smokin() throws Exception { - var serverMember1 = new SigningMemberImpl(Utils.getMember(0), ULong.MIN); - var serverMember2 = new SigningMemberImpl(Utils.getMember(1), ULong.MIN); + var serverMember1 = new SigningMemberImpl(Utils.getMember(1), ULong.MIN); + var serverMember2 = new SigningMemberImpl(Utils.getMember(2), ULong.MIN); var pinged1 = new AtomicBoolean(); var pinged2 = new AtomicBoolean(); @@ -77,24 +77,20 @@ public Any ping(Any request) { .setFactory(to -> InProcessChannelBuilder.forName(name).build()); var router = new RouterImpl(serverMember1, serverBuilder, cacheBuilder, null); try { - RouterImpl.CommonCommunications commsA = router.create(serverMember1, - context.getId(), - new ServiceImpl(local1, "A"), - "A", ServerImpl::new, - TestItClient::new, local1); + var commsA = router.create(serverMember1, context.getId(), new ServiceImpl(local1, "A"), "A", + ServerImpl::new, TestItClient::new, local1); - RouterImpl.CommonCommunications commsB = router.create(serverMember2, - context.getId(), - new ServiceImpl(local2, "B"), - "B", ServerImpl::new, - TestItClient::new, local2); + router.create(serverMember2, context.getId(), new ServiceImpl(local2, "B"), "B", ServerImpl::new, + TestItClient::new, local2); router.start(); var sync = new RingCommunications(context, serverMember1, commsA); - sync.allowDuplicates(); + sync.noDuplicates(); var countdown = new CountDownLatch(1); - sync.execute((link, round) -> link.ping(Any.getDefaultInstance()), - (result, destination) -> countdown.countDown()); + for (var i = 0; i < context.getRingCount(); i++) { + sync.execute((link, round) -> link.ping(Any.getDefaultInstance()), + (result, destination) -> countdown.countDown()); + } assertTrue(countdown.await(1, TimeUnit.SECONDS), "Completed: " + countdown.getCount()); assertFalse(pinged1.get()); assertTrue(pinged2.get()); diff --git a/memberships/src/test/java/com/salesforce/apollo/ring/RingIteratorTest.java b/memberships/src/test/java/com/salesforce/apollo/ring/RingIteratorTest.java index 15857b392..bf9d9bf67 100644 --- a/memberships/src/test/java/com/salesforce/apollo/ring/RingIteratorTest.java +++ b/memberships/src/test/java/com/salesforce/apollo/ring/RingIteratorTest.java @@ -93,8 +93,8 @@ public Any ping(Any request) { router.start(); var frequency = Duration.ofMillis(1); var scheduler = Executors.newScheduledThreadPool(1, Thread.ofVirtual().factory()); - var sync = new RingIterator(frequency, context, serverMember1, scheduler, commsA); - sync.allowDuplicates(); + var sync = new RingIterator<>(frequency, context, serverMember1, scheduler, commsA); + sync.noDuplicates(); var countdown = new CountDownLatch(3); sync.iterate(context.getId(), (link, round) -> link.ping(Any.getDefaultInstance()), (round, result, link) -> { diff --git a/model/src/main/java/com/salesforce/apollo/model/ProcessDomain.java b/model/src/main/java/com/salesforce/apollo/model/ProcessDomain.java index 7bcc251b8..51645eded 100644 --- a/model/src/main/java/com/salesforce/apollo/model/ProcessDomain.java +++ b/model/src/main/java/com/salesforce/apollo/model/ProcessDomain.java @@ -122,8 +122,8 @@ protected BiConsumer listener() { return (context, diadem) -> { choam.rotateViewKeys(context, diadem); - log.info("View change: {} for: {} cardinality: {} on: {}", diadem, params.context().getId(), - context.totalCount(), params.member().getId()); + log.info("View change: {} for: {} cardinality: {} on: {}", diadem, params.context().getId(), context.size(), + params.member().getId()); }; } diff --git a/model/src/test/java/com/salesforce/apollo/model/demesnes/DemesneTest.java b/model/src/test/java/com/salesforce/apollo/model/demesnes/DemesneTest.java index bd31b9010..877e7661b 100644 --- a/model/src/test/java/com/salesforce/apollo/model/demesnes/DemesneTest.java +++ b/model/src/test/java/com/salesforce/apollo/model/demesnes/DemesneTest.java @@ -250,7 +250,7 @@ public void register(SubContext context) { .setParent(parentAddress) .setCommDirectory(commDirectory.toString()) .setMaxTransfer(100) - .setFalsePositiveRate(.125) + .setFalsePositiveRate(.00125) .build(); final var demesne = new DemesneImpl(parameters); Builder specification = IdentifierSpecification.newBuilder(); diff --git a/pom.xml b/pom.xml index 4aadc61cb..c4879bd2b 100644 --- a/pom.xml +++ b/pom.xml @@ -780,10 +780,11 @@ org.apache.maven.plugins maven-surefire-plugin - 3.1.2 + 3.2.5 ${forks} true + -Xmx10G -Xms4G -Djdk.tracePinnedThreads=full diff --git a/thoth/src/main/java/com/salesforce/apollo/thoth/KerlDHT.java b/thoth/src/main/java/com/salesforce/apollo/thoth/KerlDHT.java index 291995bde..ff748b5c3 100644 --- a/thoth/src/main/java/com/salesforce/apollo/thoth/KerlDHT.java +++ b/thoth/src/main/java/com/salesforce/apollo/thoth/KerlDHT.java @@ -59,6 +59,7 @@ import org.h2.jdbcx.JdbcConnectionPool; import org.jooq.DSLContext; import org.jooq.SQLDialect; +import org.jooq.exception.DataAccessException; import org.jooq.impl.DSL; import org.joou.ULong; import org.slf4j.Logger; @@ -198,7 +199,7 @@ public KeyState_ append(AttachmentEvent event) { Supplier isTimedOut = () -> Instant.now().isAfter(timedOut); var result = new CompletableFuture(); HashMultiset gathered = HashMultiset.create(); - var iterator = new RingIterator(operationsFrequency, context, member, scheduler, dhtComms); + var iterator = new RingIterator<>(operationsFrequency, context, member, scheduler, dhtComms); iterator.dontIgnoreSelf(); iterator.iterate(identifier, null, (link, r) -> link.append(Collections.emptyList(), Collections.singletonList(event)), null, @@ -234,7 +235,7 @@ public List append(KERL_ kerl) { Supplier isTimedOut = () -> Instant.now().isAfter(timedOut); var result = new CompletableFuture(); HashMultiset gathered = HashMultiset.create(); - var iterator = new RingIterator(operationsFrequency, context, member, scheduler, dhtComms); + var iterator = new RingIterator<>(operationsFrequency, context, member, scheduler, dhtComms); iterator.dontIgnoreSelf(); iterator.iterate(identifier, null, (link, r) -> link.append(kerl), null, (tally, futureSailor, destination) -> mutate(gathered, futureSailor, identifier, isTimedOut, @@ -247,7 +248,7 @@ public List append(KERL_ kerl) { return null; } catch (ExecutionException e) { if (e.getCause() instanceof CompletionException ce) { - log.info("error appending KERL: {} on: {}", ce.getMessage(), member.getId()); + log.warn("error appending KERL: {} on: {}", ce.getMessage(), member.getId()); return Collections.emptyList(); } throw new IllegalStateException(e.getCause()); @@ -263,7 +264,7 @@ public KeyState_ append(KeyEvent_ event) { Supplier isTimedOut = () -> Instant.now().isAfter(timedOut); var result = new CompletableFuture(); HashMultiset gathered = HashMultiset.create(); - var iterator = new RingIterator(operationsFrequency, context, member, scheduler, dhtComms); + var iterator = new RingIterator<>(operationsFrequency, context, member, scheduler, dhtComms); iterator.dontIgnoreSelf(); iterator.iterate(identifier, null, (link, r) -> link.append(Collections.singletonList(event)), null, (tally, futureSailor, destination) -> mutate(gathered, futureSailor, identifier, isTimedOut, @@ -277,7 +278,7 @@ public KeyState_ append(KeyEvent_ event) { return null; } catch (ExecutionException e) { if (e.getCause() instanceof CompletionException ce) { - log.info("error appending Key Event: {} on: {}", ce.getMessage(), member.getId()); + log.warn("error appending Key Event: {} on: {}", ce.getMessage(), member.getId()); return KeyState_.getDefaultInstance(); } throw new IllegalStateException(e.getCause()); @@ -320,7 +321,7 @@ public Empty appendAttachments(List events) { Supplier isTimedOut = () -> Instant.now().isAfter(timedOut); var result = new CompletableFuture(); HashMultiset gathered = HashMultiset.create(); - var iterator = new RingIterator(operationsFrequency, context, member, scheduler, dhtComms); + var iterator = new RingIterator<>(operationsFrequency, context, member, scheduler, dhtComms); iterator.dontIgnoreSelf(); iterator.iterate(identifier, null, (link, r) -> link.appendAttachments(events), null, (tally, futureSailor, destination) -> mutate(gathered, futureSailor, identifier, isTimedOut, @@ -334,7 +335,7 @@ public Empty appendAttachments(List events) { return null; } catch (ExecutionException e) { if (e.getCause() instanceof CompletionException ce) { - log.info("error appending attachments: {} on: {}", ce.getMessage(), member.getId()); + log.warn("error appending attachments: {} on: {}", ce.getMessage(), member.getId()); return Empty.getDefaultInstance(); } throw new IllegalStateException(e.getCause()); @@ -354,7 +355,7 @@ public Empty appendValidations(Validations validations) { Supplier isTimedOut = () -> Instant.now().isAfter(timedOut); var result = new CompletableFuture(); HashMultiset gathered = HashMultiset.create(); - var iterator = new RingIterator(operationsFrequency, context, member, scheduler, dhtComms); + var iterator = new RingIterator<>(operationsFrequency, context, member, scheduler, dhtComms); iterator.dontIgnoreSelf(); iterator.iterate(identifier, null, (link, r) -> link.appendValidations(validations), null, (tally, futureSailor, destination) -> mutate(gathered, futureSailor, identifier, isTimedOut, @@ -367,7 +368,7 @@ public Empty appendValidations(Validations validations) { return null; } catch (ExecutionException e) { if (e.getCause() instanceof CompletionException ce) { - log.info("error appending validations: {} on: {}", ce.getMessage(), member.getId()); + log.warn("error appending validations: {} on: {}", ce.getMessage(), member.getId()); return Empty.getDefaultInstance(); } throw new IllegalStateException(e.getCause()); @@ -407,7 +408,7 @@ public Attachment getAttachment(EventCoords coordinates) { var result = new CompletableFuture(); HashMultiset gathered = HashMultiset.create(); var operation = "getAttachment(%s)".formatted(EventCoordinates.from(coordinates)); - var iterator = new RingIterator(operationsFrequency, context, member, scheduler, dhtComms); + var iterator = new RingIterator<>(operationsFrequency, context, member, scheduler, dhtComms); iterator.iterate(identifier, null, (link, r) -> link.getAttachment(coordinates), () -> failedMajority(result, maxCount(gathered), operation), (tally, futureSailor, destination) -> read(result, gathered, tally, futureSailor, identifier, @@ -421,7 +422,7 @@ public Attachment getAttachment(EventCoords coordinates) { return null; } catch (ExecutionException e) { if (e.getCause() instanceof CompletionException ce) { - log.info("error {} : {} on: {}", operation, ce.getMessage(), member.getId()); + log.warn("error {} : {} on: {}", operation, ce.getMessage(), member.getId()); return null; } throw new IllegalStateException(e.getCause()); @@ -442,7 +443,7 @@ public KERL_ getKERL(Ident identifier) { var result = new CompletableFuture(); HashMultiset gathered = HashMultiset.create(); var operation = "getKerl(%s)".formatted(Identifier.from(identifier)); - var iterator = new RingIterator(operationsFrequency, context, member, scheduler, dhtComms); + var iterator = new RingIterator<>(operationsFrequency, context, member, scheduler, dhtComms); iterator.iterate(digest, null, (link, r) -> link.getKERL(identifier), () -> failedMajority(result, maxCount(gathered), operation), (tally, futureSailor, destination) -> read(result, gathered, tally, futureSailor, digest, @@ -456,7 +457,7 @@ public KERL_ getKERL(Ident identifier) { return null; } catch (ExecutionException e) { if (e.getCause() instanceof CompletionException ce) { - log.info("error {} : {} on: {}", operation, ce.getMessage(), member.getId()); + log.warn("error {} : {} on: {}", operation, ce.getMessage(), member.getId()); return KERL_.getDefaultInstance(); } throw new IllegalStateException(e.getCause()); @@ -481,7 +482,7 @@ public KeyEvent_ getKeyEvent(EventCoords coordinates) { Supplier isTimedOut = () -> Instant.now().isAfter(timedOut); var result = new CompletableFuture(); HashMultiset gathered = HashMultiset.create(); - var iterator = new RingIterator(operationsFrequency, context, member, scheduler, dhtComms); + var iterator = new RingIterator<>(operationsFrequency, context, member, scheduler, dhtComms); iterator.iterate(digest, null, (link, r) -> link.getKeyEvent(coordinates), () -> failedMajority(result, maxCount(gathered), operation), (tally, futureSailor, destination) -> read(result, gathered, tally, futureSailor, digest, @@ -517,7 +518,7 @@ public KeyState_ getKeyState(EventCoords coordinates) { Supplier isTimedOut = () -> Instant.now().isAfter(timedOut); var result = new CompletableFuture(); HashMultiset gathered = HashMultiset.create(); - var iterator = new RingIterator(operationsFrequency, context, member, scheduler, dhtComms); + var iterator = new RingIterator<>(operationsFrequency, context, member, scheduler, dhtComms); iterator.iterate(digest, null, (link, r) -> link.getKeyState(coordinates), () -> failedMajority(result, maxCount(gathered), operation), (tally, futureSailor, destination) -> read(result, gathered, tally, futureSailor, digest, @@ -531,7 +532,7 @@ public KeyState_ getKeyState(EventCoords coordinates) { return null; } catch (ExecutionException e) { if (e.getCause() instanceof CompletionException ce) { - log.info("error {} : {} on: {}", operation, ce.getMessage(), member.getId()); + log.warn("error {} : {} on: {}", operation, ce.getMessage(), member.getId()); return KeyState_.getDefaultInstance(); } throw new IllegalStateException(e.getCause()); @@ -541,7 +542,7 @@ public KeyState_ getKeyState(EventCoords coordinates) { @Override public KeyState_ getKeyState(Ident identifier, long sequenceNumber) { var operation = "getKeyState(%s, %s)".formatted(Identifier.from(identifier), ULong.valueOf(sequenceNumber)); - log.info("{} on: {}", operation, member.getId()); + log.warn("{} on: {}", operation, member.getId()); if (identifier == null) { return KeyState_.getDefaultInstance(); } @@ -554,7 +555,7 @@ public KeyState_ getKeyState(Ident identifier, long sequenceNumber) { Supplier isTimedOut = () -> Instant.now().isAfter(timedOut); var result = new CompletableFuture(); HashMultiset gathered = HashMultiset.create(); - var iterator = new RingIterator(operationsFrequency, context, member, scheduler, dhtComms); + var iterator = new RingIterator<>(operationsFrequency, context, member, scheduler, dhtComms); iterator.iterate(digest, null, (link, r) -> link.getKeyState(identAndSeq), () -> failedMajority(result, maxCount(gathered), operation), (tally, futureSailor, destination) -> read(result, gathered, tally, futureSailor, digest, @@ -568,7 +569,7 @@ public KeyState_ getKeyState(Ident identifier, long sequenceNumber) { return null; } catch (ExecutionException e) { if (e.getCause() instanceof CompletionException ce) { - log.info("error {} : {} on: {}", operation, ce.getMessage(), member.getId()); + log.warn("error {} : {} on: {}", operation, ce.getMessage(), member.getId()); return KeyState_.getDefaultInstance(); } throw new IllegalStateException(e.getCause()); @@ -590,7 +591,7 @@ public KeyState_ getKeyState(Ident identifier) { Supplier isTimedOut = () -> Instant.now().isAfter(timedOut); var result = new CompletableFuture(); HashMultiset gathered = HashMultiset.create(); - var iterator = new RingIterator(operationsFrequency, context, member, scheduler, dhtComms); + var iterator = new RingIterator<>(operationsFrequency, context, member, scheduler, dhtComms); iterator.iterate(digest, null, (link, r) -> link.getKeyState(identifier), () -> failedMajority(result, maxCount(gathered), operation), (tally, futureSailor, destination) -> read(result, gathered, tally, futureSailor, digest, @@ -604,7 +605,7 @@ public KeyState_ getKeyState(Ident identifier) { return null; } catch (ExecutionException e) { if (e.getCause() instanceof CompletionException ce) { - log.info("error {} : {} on: {}", operation, ce.getMessage(), member.getId()); + log.warn("error {} : {} on: {}", operation, ce.getMessage(), member.getId()); return KeyState_.getDefaultInstance(); } throw new IllegalStateException(e.getCause()); @@ -626,7 +627,7 @@ public KeyStateWithAttachments_ getKeyStateWithAttachments(EventCoords coordinat Supplier isTimedOut = () -> Instant.now().isAfter(timedOut); var result = new CompletableFuture(); HashMultiset gathered = HashMultiset.create(); - var iterator = new RingIterator(operationsFrequency, context, member, scheduler, dhtComms); + var iterator = new RingIterator<>(operationsFrequency, context, member, scheduler, dhtComms); iterator.iterate(digest, null, (link, r) -> link.getKeyStateWithAttachments(coordinates), () -> failedMajority(result, maxCount(gathered), operation), (tally, futureSailor, destination) -> read(result, gathered, tally, futureSailor, digest, @@ -640,7 +641,7 @@ public KeyStateWithAttachments_ getKeyStateWithAttachments(EventCoords coordinat return null; } catch (ExecutionException e) { if (e.getCause() instanceof CompletionException ce) { - log.info("error {} on: {}", operation, member.getId(), ce); + log.warn("error {} on: {}", operation, member.getId(), ce); return null; } throw new IllegalStateException(e.getCause()); @@ -662,7 +663,7 @@ public KeyStateWithEndorsementsAndValidations_ getKeyStateWithEndorsementsAndVal Supplier isTimedOut = () -> Instant.now().isAfter(timedOut); var result = new CompletableFuture(); HashMultiset gathered = HashMultiset.create(); - var iterator = new RingIterator(operationsFrequency, context, member, scheduler, dhtComms); + var iterator = new RingIterator<>(operationsFrequency, context, member, scheduler, dhtComms); iterator.iterate(digest, null, (link, r) -> link.getKeyStateWithEndorsementsAndValidations(coordinates), () -> failedMajority(result, maxCount(gathered), operation), (tally, futureSailor, destination) -> read(result, gathered, tally, futureSailor, digest, @@ -676,7 +677,7 @@ public KeyStateWithEndorsementsAndValidations_ getKeyStateWithEndorsementsAndVal return null; } catch (ExecutionException e) { if (e.getCause() instanceof CompletionException ce) { - log.info("error {} : {} on: {}", operation, ce.getMessage(), member.getId()); + log.warn("error {} : {} on: {}", operation, ce.getMessage(), member.getId()); return null; } throw new IllegalStateException(e.getCause()); @@ -698,7 +699,7 @@ public Validations getValidations(EventCoords coordinates) { Supplier isTimedOut = () -> Instant.now().isAfter(timedOut); var result = new CompletableFuture(); HashMultiset gathered = HashMultiset.create(); - var iterator = new RingIterator(operationsFrequency, context, member, scheduler, dhtComms); + var iterator = new RingIterator<>(operationsFrequency, context, member, scheduler, dhtComms); iterator.iterate(identifier, null, (link, r) -> link.getValidations(coordinates), () -> failedMajority(result, maxCount(gathered), operation), (tally, futureSailor, destination) -> read(result, gathered, tally, futureSailor, identifier, @@ -712,7 +713,7 @@ public Validations getValidations(EventCoords coordinates) { return null; } catch (ExecutionException e) { if (e.getCause() instanceof CompletionException ce) { - log.info("error {} : {} on: {}", operation, ce.getMessage(), member.getId()); + log.warn("error {} : {} on: {}", operation, ce.getMessage(), member.getId()); return null; } throw new IllegalStateException(e.getCause()); @@ -946,6 +947,8 @@ private void updateLocationHash(Identifier identifier) { log.error("Cannot update location hash for: {} on: {}", identifier, member.getId()); throw new IllegalStateException( "Cannot update location hash S for: %s on: %s".formatted(identifier, member.getId())); + } catch (DataAccessException e) { + log.trace("Duplicate location hash for: {} on: {}", identifier, member.getId()); } } @@ -1046,31 +1049,31 @@ private class Service implements ProtoKERLService { @Override public List append(KERL_ kerl_) { - log.info("appending kerl on: {}", member.getId()); + log.debug("appending kerl on: {}", member.getId()); return complete(k -> k.append(kerl_)); } @Override public List append(List events) { - log.info("appending events on: {}", member.getId()); + log.debug("appending events on: {}", member.getId()); return complete(k -> k.append(events)); } @Override public List append(List events, List attachments) { - log.info("appending events and attachments on: {}", member.getId()); + log.debug("appending events and attachments on: {}", member.getId()); return complete(k -> k.append(events, attachments)); } @Override public Empty appendAttachments(List attachments) { - log.info("append attachments on: {}", member.getId()); + log.debug("append attachments on: {}", member.getId()); return complete(k -> k.appendAttachments(attachments)); } @Override public Empty appendValidations(Validations validations) { - log.info("append validations on: {}", member.getId()); + log.debug("append validations on: {}", member.getId()); return complete(k -> k.appendValidations(validations)); } diff --git a/thoth/src/main/java/com/salesforce/apollo/thoth/Maat.java b/thoth/src/main/java/com/salesforce/apollo/thoth/Maat.java index 7b25f6ad9..4cb27ad8a 100644 --- a/thoth/src/main/java/com/salesforce/apollo/thoth/Maat.java +++ b/thoth/src/main/java/com/salesforce/apollo/thoth/Maat.java @@ -82,10 +82,10 @@ public boolean validate(EstablishmentEvent event) { return false; } final Context ctx = context; - var successors = Context.uniqueSuccessors(ctx, digestOf(event.getIdentifier().toIdent(), digest.getAlgorithm())) - .stream() - .map(m -> m.getId()) - .collect(Collectors.toSet()); + var successors = ctx.bftSubset(digestOf(event.getIdentifier().toIdent(), digest.getAlgorithm())) + .stream() + .map(m -> m.getId()) + .collect(Collectors.toSet()); record validator(EstablishmentEvent validating, JohnHancock signature) { }