diff --git a/fireflies/src/main/java/com/salesforce/apollo/fireflies/Binding.java b/fireflies/src/main/java/com/salesforce/apollo/fireflies/Binding.java index bd4ad421d..00965a98f 100644 --- a/fireflies/src/main/java/com/salesforce/apollo/fireflies/Binding.java +++ b/fireflies/src/main/java/com/salesforce/apollo/fireflies/Binding.java @@ -136,12 +136,14 @@ private boolean complete(CompletableFuture redirect, Optional gateway, - Optional futureSailor, HashMultiset trusts, + Optional futureSailor, HashMultiset trusts, Set initialSeedSet, Digest v, int majority) { if (futureSailor.isEmpty()) { + log.warn("No gateway returned from: {} on: {}", member.getId(), node.getId()); return true; } if (gateway.isDone()) { + log.warn("gateway is complete, ignoring from: {} on: {}", member.getId(), node.getId()); return false; } @@ -161,7 +163,7 @@ private boolean completeGateway(Participant member, CompletableFuture gat log.trace("Empty bootstrap trust in join returned from: {} on: {}", member.getId(), node.getId()); return true; } - trusts.add(g.getTrust()); + trusts.add(new Bootstrapping(g.getTrust())); initialSeedSet.addAll(g.getInitialSeedSetList()); log.trace("Initial seed set count: {} view: {} from: {} on: {}", g.getInitialSeedSetCount(), v, member.getId(), node.getId()); @@ -175,8 +177,14 @@ private boolean completeGateway(Participant member, CompletableFuture gat if (trust != null) { validate(trust, gateway, initialSeedSet); } else { - log.debug("Gateway received, trust count: {} majority: {} from: {} view: {} context: {} on: {}", - trusts.size(), majority, member.getId(), v, this.context.getId(), node.getId()); + log.debug("Gateway received, trust count: {} majority: {} from: {} trusts: {} view: {} context: {} on: {}", + trusts.size(), majority, member.getId(), v, trusts.entrySet() + .stream() + .sorted() + .map( + e -> "%s x %s".formatted(e.getElement().diadem, + e.getCount())) + .toList(), this.context.getId(), node.getId()); } return true; } @@ -255,7 +263,7 @@ private void join(Redirect redirect, Digest v, Duration duration) { var regate = new AtomicReference(); var retries = new AtomicInteger(); - HashMultiset trusts = HashMultiset.create(); + HashMultiset trusts = HashMultiset.create(); HashSet initialSeedSet = new HashSet<>(); final var cardinality = redirect.getCardinality(); @@ -302,7 +310,8 @@ private void join(Redirect redirect, Digest v, Duration duration) { return; } if (abandon.get() >= majority) { - log.debug("Abandoning Gateway view: {} reseeding on: {}", v, node.getId()); + log.debug("Abandoning Gateway view: {} abandons: {} majority: {} reseeding on: {}", v, + abandon.get(), majority, node.getId()); seeding(); } else { abandon.set(0); @@ -345,13 +354,37 @@ private NoteWrapper seedFor(Seed seed) { return new NoteWrapper(seedNote, digestAlgo); } - private void validate(BootstrapTrust trust, CompletableFuture gateway, Set initialSeedSet) { - final var hexBloom = new HexBloom(trust.getDiadem()); + private void validate(Bootstrapping trust, CompletableFuture gateway, Set initialSeedSet) { if (gateway.complete( - new Bound(hexBloom, trust.getSuccessorsList().stream().map(sn -> new NoteWrapper(sn, digestAlgo)).toList(), + new Bound(trust.crown, trust.successors.stream().map(sn -> new NoteWrapper(sn, digestAlgo)).toList(), initialSeedSet.stream().map(sn -> new NoteWrapper(sn, digestAlgo)).toList()))) { - log.info("Gateway acquired: {} context: {} on: {}", hexBloom.compactWrapped(), this.context.getId(), - node.getId()); + log.info("Gateway acquired: {} context: {} on: {}", trust.diadem, this.context.getId(), node.getId()); + } + } + + private record Bootstrapping(Digest diadem, HexBloom crown, Set successors) { + public Bootstrapping(BootstrapTrust trust) { + this(HexBloom.from(trust.getDiadem()), new HashSet<>(trust.getSuccessorsList())); + } + + public Bootstrapping(HexBloom crown, Set successors) { + this(crown.compact(), crown, new HashSet<>(successors)); + } + + @Override + public boolean equals(Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + + Bootstrapping that = (Bootstrapping) o; + return diadem.equals(that.diadem); + } + + @Override + public int hashCode() { + return diadem.hashCode(); } } diff --git a/fireflies/src/main/java/com/salesforce/apollo/fireflies/View.java b/fireflies/src/main/java/com/salesforce/apollo/fireflies/View.java index 1df306b37..47a7ef3b0 100644 --- a/fireflies/src/main/java/com/salesforce/apollo/fireflies/View.java +++ b/fireflies/src/main/java/com/salesforce/apollo/fireflies/View.java @@ -346,9 +346,18 @@ void finalizeViewChange() { viewChange(() -> { final var supermajority = context.getRingCount() * 3 / 4; final var majority = context.size() == 1 ? 1 : supermajority; - if (observations.size() < majority) { - log.trace("Do not have majority: {} required: {} observers: {} for: {} on: {}", observations.size(), - majority, viewManagement.observersList(), currentView(), node.getId()); + final var valid = observations.values() + .stream() + .filter(svc -> viewManagement.observers.contains( + Digest.from(svc.getChange().getObserver()))) + .toList(); + log.info("Finalize view change, observations: {} valid: {} observers: {} on: {}", + observations.values().stream().map(sv -> Digest.from(sv.getChange().getObserver())).toList(), + valid.size(), viewManagement.observersList(), node.getId()); + observations.clear(); + if (valid.size() < majority) { + log.info("Do not have majority: {} required: {} observers: {} for: {} on: {}", valid.size(), majority, + viewManagement.observersList(), currentView(), node.getId()); scheduleFinalizeViewChange(2); return; } @@ -356,20 +365,23 @@ void finalizeViewChange() { viewManagement.observersList(), currentView(), node.getId()); HashMultiset ballots = HashMultiset.create(); final var current = currentView(); - observations.values() - .stream() - .filter(vc -> current.equals(Digest.from(vc.getChange().getCurrent()))) - .forEach(vc -> { - final var leaving = new ArrayList<>( - vc.getChange().getLeavesList().stream().map(Digest::from).collect(Collectors.toSet())); - final var joining = new ArrayList<>( - vc.getChange().getJoinsList().stream().map(Digest::from).collect(Collectors.toSet())); - leaving.sort(Ordering.natural()); - joining.sort(Ordering.natural()); - ballots.add( - new Ballot(Digest.from(vc.getChange().getCurrent()), leaving, joining, digestAlgo)); - }); - observations.clear(); + valid.stream().filter(vc -> current.equals(Digest.from(vc.getChange().getCurrent()))).forEach(vc -> { + final var leaving = vc.getChange() + .getLeavesList() + .stream() + .map(Digest::from) + .distinct() + .collect(Collectors.toCollection(ArrayList::new)); + final var joining = vc.getChange() + .getJoinsList() + .stream() + .map(Digest::from) + .distinct() + .collect(Collectors.toCollection(ArrayList::new)); + leaving.sort(Ordering.natural()); + joining.sort(Ordering.natural()); + ballots.add(new Ballot(Digest.from(vc.getChange().getCurrent()), leaving, joining, digestAlgo)); + }); var max = ballots.entrySet() .stream() .max(Ordering.natural().onResultOf(Multiset.Entry::getCount)) @@ -382,7 +394,7 @@ void finalizeViewChange() { @SuppressWarnings("unchecked") final var reversed = Comparator.comparing(e -> ((Entry) e).getCount()).reversed(); log.info("View consensus failed: {}, required: {} cardinality: {} ballots: {} for: {} on: {}", - observations.size(), majority, viewManagement.cardinality(), + max == null ? 0 : max.getCount(), majority, viewManagement.cardinality(), ballots.entrySet().stream().sorted(reversed).toList(), currentView(), node.getId()); } @@ -845,13 +857,11 @@ private boolean add(SignedViewChange observation) { observation.getChange().getAttempt(), currentObservation.getChange().getAttempt(), inView, currentView(), observer, node.getId()); return false; - } else if (observation.getChange().getAttempt() < currentObservation.getChange().getAttempt()) { - return false; } } final var member = context.getActiveMember(observer); if (member == null) { - log.trace("Cannot validate view change: {} current: {} offline: {} on: {}", inView, currentView(), observer, + log.trace("Cannot validate view change: {} current: {} from: {} on: {}", inView, currentView(), observer, node.getId()); return false; } @@ -860,6 +870,8 @@ private boolean add(SignedViewChange observation) { if (!member.verify(signature, observation.getChange().toByteString())) { return null; } + log.trace("Observation: {} current: {} view change: {} from: {} on: {}", + observation.getChange().getAttempt(), inView, currentView(), observer, node.getId()); return observation; }) != null; } @@ -1216,7 +1228,7 @@ private NoteGossip.Builder processNotes(BloomFilter bff) { .filter(m -> !bff.contains(m.getNote().getHash())) .collect(new ReservoirSampler<>(params.maximumTxfr(), Entropy.bitsStream())) .stream() - .map(m -> m.getNote()) + .map(Participant::getNote) .forEach(n -> builder.addUpdates(n.getWrapped())); return builder; } @@ -1225,10 +1237,6 @@ private NoteGossip.Builder processNotes(BloomFilter bff) { * Process the inbound notes from the gossip. Reconcile the differences between the view's state and the digests of * the gossip. Update the reply with the list of digests the view requires, as well as proposed updates based on the * inbound digests that the view has more recent information - * - * @param from - * @param p - * @param bff */ private NoteGossip processNotes(Digest from, BloomFilter bff, double p) { NoteGossip.Builder builder = processNotes(bff); @@ -1868,13 +1876,13 @@ public void join(Join join, Digest from, StreamObserver responseObserve @Override public Gossip rumors(SayWhat request, Digest from) { if (!introduced.get()) { - log.trace("Not introduced; ring: {} from: {}, on: {}", request.getRing(), from, node.getId()); + // log.trace("Not introduced; ring: {} from: {}, on: {}", request.getRing(), from, node.getId()); throw new StatusRuntimeException(Status.FAILED_PRECONDITION.withDescription("Not introduced")); } return stable(() -> { final var ring = request.getRing(); if (!context.validRing(ring)) { - log.debug("invalid gossip ring: {} from: {} on: {}", ring, from, node.getId()); + // log.debug("invalid gossip ring: {} from: {} on: {}", ring, from, node.getId()); throw new StatusRuntimeException(Status.FAILED_PRECONDITION.withDescription("invalid ring")); } validate(from, request); diff --git a/fireflies/src/main/java/com/salesforce/apollo/fireflies/ViewManagement.java b/fireflies/src/main/java/com/salesforce/apollo/fireflies/ViewManagement.java index f070be7ec..3f2e50579 100644 --- a/fireflies/src/main/java/com/salesforce/apollo/fireflies/ViewManagement.java +++ b/fireflies/src/main/java/com/salesforce/apollo/fireflies/ViewManagement.java @@ -15,6 +15,7 @@ import com.salesforce.apollo.fireflies.Binding.Bound; import com.salesforce.apollo.fireflies.View.Node; import com.salesforce.apollo.fireflies.View.Participant; +import com.salesforce.apollo.fireflies.comm.gossip.Fireflies; import com.salesforce.apollo.fireflies.proto.*; import com.salesforce.apollo.fireflies.proto.Update.Builder; import com.salesforce.apollo.membership.Member; @@ -109,6 +110,7 @@ int cardinality() { void clear() { joins.clear(); + // context.clear(); resetBootstrapView(); } @@ -125,14 +127,18 @@ Digest currentView() { } void enjoin(Join join, Digest observer) { + if (!observers.contains(node.getId()) || !observers.contains(observer)) { + log.trace("Not observer, ignored enjoin from: {} on: {}", observer, node.getId()); + throw new StatusRuntimeException(Status.FAILED_PRECONDITION.withDescription("Not observer")); + } + var note = new NoteWrapper(join.getNote(), digestAlgo); + final var from = note.getId(); final var joinView = Digest.from(join.getView()); - final var from = observer; if (!joined()) { log.trace("Not joined, ignored enjoin of view: {} from: {} on: {}", joinView, from, node.getId()); throw new StatusRuntimeException(Status.FAILED_PRECONDITION.withDescription( "Not joined, ignored join of view: %s from: %s on: %s".formatted(joinView, from, node.getId()))); } - var note = new NoteWrapper(join.getNote(), digestAlgo); if (!view.validate(note.getIdentifier())) { log.debug("Ignored enjoin of view: {} from: {} invalid identifier on: {}", joinView, from, node.getId()); throw new StatusRuntimeException(Status.INVALID_ARGUMENT.withDescription("Invalid identifier")); @@ -155,21 +161,9 @@ void enjoin(Join join, Digest observer) { throw new StatusRuntimeException( Status.OUT_OF_RANGE.withDescription("View: " + joinView + " does not match: " + thisView)); } - if (!View.isValidMask(note.getMask(), context)) { - log.warn( - "Invalid enjoin mask: {} majority: {} from member: {} view: {} context: {} cardinality: {} on: {}", - note.getMask(), context.majority(), from, thisView, context.getId(), cardinality(), node.getId()); - } - if (pendingJoins.size() >= params.maxPending()) { - throw new StatusRuntimeException(Status.RESOURCE_EXHAUSTED.withDescription("No room at the inn")); - } - pendingJoins.computeIfAbsent(from, d -> seeds -> { - log.info("Gateway established for: {} (enjoined) view: {} context: {} cardinality: {} on: {}", from, - currentView(), context.getId(), cardinality(), node.getId()); - }); - joins.put(note.getId(), note); - log.debug("Member pending enjoin: {} view: {} context: {} on: {}", from, currentView(), context.getId(), - node.getId()); + joins.putIfAbsent(note.getId(), note); + log.debug("Member pending enjoin: {} via: {} view: {} context: {} on: {}", from, observer, currentView(), + context.getId(), node.getId()); }); } @@ -233,10 +227,10 @@ void install(Ballot ballot) { .filter(java.util.Objects::nonNull) .toList(); + view.reset(); setDiadem( HexBloom.construct(context.memberCount(), context.allMembers().map(Participant::getId), view.bootstrapView(), params.crowns())); - view.reset(); // complete all pending joins pending.forEach(r -> { try { @@ -431,6 +425,9 @@ boolean joined() { * start a view change if there are any offline members or joining members */ void maybeViewChange() { + if (!joined()) { + return; + } if (context.size() == 1 && joins.size() < context.getRingCount() - 1) { log.trace("Cannot form cluster: {} with: {} members, required > 3 on: {}", currentView(), joins.size() + context.size(), node.getId()); @@ -439,8 +436,8 @@ void maybeViewChange() { } if ((context.offlineCount() > 0 || !joins.isEmpty())) { if (isObserver()) { - log.trace("Initiating view change: {} (non observer) joins: {} leaves: {} on: {}", currentView(), - joins.size(), view.streamShunned().count(), node.getId()); + log.info("Initiating view change: {} (observer) joins: {} leaves: {} on: {}", currentView(), + joins.size(), view.streamShunned().count(), node.getId()); initiateViewChange(); } else { // Use pending rebuttals as a proxy for stability @@ -448,12 +445,14 @@ void maybeViewChange() { log.debug("Pending rebuttals in view: {} on: {}", currentView(), node.getId()); view.scheduleViewChange(2); // 2 TTL round2 to check again } else { + log.info("Initiating view change: {} (non observer) joins: {} leaves: {} on: {}", currentView(), + joins.size(), view.streamShunned().count(), node.getId()); view.scheduleFinalizeViewChange(); } } } else { - log.trace("No view change: {} joins: {} leaves: {} on: {}", currentView(), joins.size(), - view.streamShunned().count(), node.getId()); + // log.trace("No view change: {} joins: {} leaves: {} on: {}", currentView(), joins.size(), + // view.streamShunned().count(), node.getId()); view.scheduleViewChange(); } } @@ -466,6 +465,29 @@ List observersList() { return observers().stream().toList(); } + void populate(List sample) { + var populate = new SliceIterator("Populate: " + context.getId(), node, sample, view.comm); + var repopulate = new AtomicReference(); + var scheduler = Executors.newScheduledThreadPool(1, Thread.ofVirtual().factory()); + repopulate.set(() -> populate.iterate((link) -> view.gossip(link, 0), (futureSailor, _, link, m) -> { + futureSailor.ifPresent(g -> { + if (!g.getRedirect().equals(SignedNote.getDefaultInstance())) { + final Participant member = (Participant) link.getMember(); + view.stable(() -> view.redirect(member, g, 0)); + } else { + view.stable(() -> view.processUpdates(g)); + } + }); + return !joined(); + }, () -> { + if (!joined()) { + scheduler.schedule(() -> Thread.ofVirtual().start(Utils.wrapped(repopulate.get(), log)), + params.populateDuration().toNanos(), TimeUnit.NANOSECONDS); + } + }, params.populateDuration())); + repopulate.get().run(); + } + JoinGossip.Builder processJoins(BloomFilter bff) { JoinGossip.Builder builder = JoinGossip.newBuilder(); @@ -568,11 +590,11 @@ private void initiateViewChange() { } view.scheduleFinalizeViewChange(); if (!isObserver(node.getId())) { - log.trace("Initiating view change: {} (non observer) on: {}", currentView(), node.getId()); + log.warn("Initiating view change: {} (non observer) on: {}", currentView(), node.getId()); return; } - log.trace("Initiating view change vote: {} joins: {} leaves: {} on: {}", currentView(), joins.size(), - view.streamShunned().count(), node.getId()); + log.warn("Initiating view change vote: {} joins: {} leaves: {} observers: {} on: {}", currentView(), + joins.size(), view.streamShunned().count(), observersList(), node.getId()); final var builder = ViewChange.newBuilder() .setObserver(node.getId().toDigeste()) .setCurrent(currentView().toDigeste()) @@ -587,8 +609,8 @@ private void initiateViewChange() { .setSignature(signature.toSig()) .build(); view.initiate(viewChange); - log.debug("View change vote: {} joins: {} leaves: {} on: {}", currentView(), change.getJoinsCount(), - change.getLeavesCount(), node.getId()); + log.warn("View change vote: {} joins: {} leaves: {} on: {}", currentView(), change.getJoinsCount(), + change.getLeavesCount(), node.getId()); }); } @@ -603,6 +625,7 @@ private void joined(Collection seedSet, Digest from, StreamObserver< Timer.Context timer) { var unique = new HashSet<>(seedSet); final var initialSeeds = new ArrayList<>(seedSet); + initialSeeds.add(node.getSignedNote()); final var successors = new HashSet(); context.successors(from, context::isActive).forEach(p -> { @@ -620,11 +643,13 @@ private void joined(Collection seedSet, Digest from, StreamObserver< .build(); log.info("Gateway initial seeding: {} successors: {} for: {} on: {}", gateway.getInitialSeedSetCount(), successors.size(), from, node.getId()); - responseObserver.onNext(gateway); try { + responseObserver.onNext(gateway); responseObserver.onCompleted(); } catch (RejectedExecutionException e) { - log.trace("in shutdown on: {}", node.getId()); + log.trace("In shutdown on: {}", node.getId()); + } catch (Throwable t) { + log.error("Error responding to join: {} on: {}", t, node.getId()); } if (timer != null) { var serializedSize = gateway.getSerializedSize(); @@ -652,12 +677,15 @@ private void resetObservers() { } private void setDiadem(final HexBloom hex) { + assert hex.getCardinality() <= 0 + || context.size() == hex.getCardinality() : "Context: %s does not equal Hex: %s".formatted(context.size(), + hex.getCardinality()); diadem.set(hex); currentView.set(diadem.get().compactWrapped()); resetObservers(); - log.trace("View: {} set diadem: {} observers: {} view: {} context: {} size: {} on: {}", context.getId(), - diadem.get().compactWrapped(), observers.stream().toList(), currentView(), context.getId(), - context.size(), node.getId()); + log.trace("View: {} set diadem: {} cardinality: {} observers: {} view: {} context: {} size: {} on: {}", + context.getId(), diadem.get().compactWrapped(), diadem.get().getCardinality(), + observers.stream().toList(), currentView(), context.getId(), context.size(), node.getId()); } record Ballot(Digest view, List leaving, List joining, int hash) { diff --git a/fireflies/src/test/java/com/salesforce/apollo/fireflies/ChurnTest.java b/fireflies/src/test/java/com/salesforce/apollo/fireflies/ChurnTest.java index 331da49b3..d39f26c77 100644 --- a/fireflies/src/test/java/com/salesforce/apollo/fireflies/ChurnTest.java +++ b/fireflies/src/test/java/com/salesforce/apollo/fireflies/ChurnTest.java @@ -271,7 +271,7 @@ public void churn() throws Exception { private void initialize() { executor = UnsafeExecutors.newVirtualThreadPerTaskExecutor(); executor2 = UnsafeExecutors.newVirtualThreadPerTaskExecutor(); - var parameters = Parameters.newBuilder().setMaximumTxfr(20).build(); + var parameters = Parameters.newBuilder().setFpr(0.0000125).setMaximumTxfr(20).build(); registry = new MetricRegistry(); node0Registry = new MetricRegistry(); diff --git a/fireflies/src/test/java/com/salesforce/apollo/fireflies/E2ETest.java b/fireflies/src/test/java/com/salesforce/apollo/fireflies/E2ETest.java index 1300fe554..6b4121346 100644 --- a/fireflies/src/test/java/com/salesforce/apollo/fireflies/E2ETest.java +++ b/fireflies/src/test/java/com/salesforce/apollo/fireflies/E2ETest.java @@ -8,6 +8,7 @@ import com.codahale.metrics.ConsoleReporter; import com.codahale.metrics.MetricRegistry; +import com.google.common.collect.Sets; import com.salesforce.apollo.archipelago.*; import com.salesforce.apollo.context.Context; import com.salesforce.apollo.context.DynamicContext; @@ -121,7 +122,13 @@ public void smokin() throws Exception { var failed = bootstrappers.stream() .filter(e -> e.getContext().activeCount() != bootstrappers.size()) .map( - v -> String.format("%s : %s ", v.getNode().getId(), v.getContext().activeCount())) + v -> String.format("%s : %s : %s", v.getNode().getId(), v.getContext().activeCount(), + Sets.difference(members.keySet(), new HashSet( + v.getContext() + .activeMembers() + .stream() + .map(Participant::getId) + .toList())).stream().toList())) .toList(); assertTrue(success, " expected: " + bootstrappers.size() + " failed: " + failed.size() + " views: " + failed); @@ -132,11 +139,14 @@ public void smokin() throws Exception { success = countdown.get().await(largeTests ? 2400 : 30, TimeUnit.SECONDS); // Test that all views are up - failed = views.stream().filter(e -> e.getContext().activeCount() != CARDINALITY).map(v -> { - Context participantContext = v.getContext(); - return String.format("%s : %s : %s ", v.getNode().getId(), v.getContext().activeCount(), - participantContext.size()); - }).toList(); + failed = views.stream() + .filter(e -> e.getContext().activeCount() != CARDINALITY) + .map(v -> String.format("%s : %s : %s", v.getNode().getId(), v.getContext().activeCount(), + Sets.difference(members.keySet(), new HashSet( + v.getContext().activeMembers().stream().map(Participant::getId).toList())) + .stream() + .toList())) + .toList(); assertTrue(success, "Views did not start, expected: " + views.size() + " failed: " + failed.size() + " views: " + failed); diff --git a/fireflies/src/test/resources/logback-test.xml b/fireflies/src/test/resources/logback-test.xml index f86c9c940..b4afb147d 100644 --- a/fireflies/src/test/resources/logback-test.xml +++ b/fireflies/src/test/resources/logback-test.xml @@ -9,12 +9,7 @@ %d{mm:ss.SSS} %logger{0} - %msg%n - - - - - - + @@ -22,31 +17,19 @@ - - - - - - - - - - - - - + - + - + - + diff --git a/memberships/src/main/java/com/salesforce/apollo/context/DynamicContextImpl.java b/memberships/src/main/java/com/salesforce/apollo/context/DynamicContextImpl.java index 94cd7db62..18a22e1d1 100644 --- a/memberships/src/main/java/com/salesforce/apollo/context/DynamicContextImpl.java +++ b/memberships/src/main/java/com/salesforce/apollo/context/DynamicContextImpl.java @@ -75,7 +75,7 @@ public boolean activate(T m) { try { l.active(m); } catch (Throwable e) { - log.error("error recoving member in listener: " + l, e); + log.error("error recovering member in listener: " + l, e); } }); return true; @@ -782,7 +782,6 @@ private Tracked tracking(T m) { * @author hal.hildebrand **/ public static class Tracked { - private static final Logger log = LoggerFactory.getLogger(Tracked.class); final AtomicBoolean active = new AtomicBoolean(false); final M member; @@ -823,7 +822,7 @@ public boolean offline() { @Override public String toString() { - return String.format("%s:%s %s", member, active.get(), Arrays.asList(hashes)); + return String.format("%s:%s %s", member.getId(), active.get(), Arrays.asList(hashes)); } void rebalance(int ringCount, Context context) { @@ -1024,7 +1023,7 @@ public Digest hash(T m) { } public T insert(T m) { - LoggerFactory.getLogger(getClass()).trace("Adding: {} to ring: {}", m.getId(), index); + log.trace("Adding: {} to ring: {}", m.getId(), index); return ring.put(hash(m), m); }