From ce9f397c5083aa3f8d568f469cdc5c2fd582530b Mon Sep 17 00:00:00 2001 From: Hellblazer Date: Mon, 20 May 2024 19:52:21 -0700 Subject: [PATCH] misc clean up --- .../apollo/bloomFilters/BloomFilter.java | 145 ++++++++++++------ .../apollo/cryptography/HexBloom.java | 20 +++ .../salesforce/apollo/fireflies/Binding.java | 2 +- .../com/salesforce/apollo/fireflies/View.java | 10 +- 4 files changed, 118 insertions(+), 59 deletions(-) diff --git a/cryptography/src/main/java/com/salesforce/apollo/bloomFilters/BloomFilter.java b/cryptography/src/main/java/com/salesforce/apollo/bloomFilters/BloomFilter.java index d452b0774..d2f898823 100644 --- a/cryptography/src/main/java/com/salesforce/apollo/bloomFilters/BloomFilter.java +++ b/cryptography/src/main/java/com/salesforce/apollo/bloomFilters/BloomFilter.java @@ -15,13 +15,13 @@ import static com.salesforce.apollo.cryptography.proto.Biff.Type.*; /** - * Simplified Bloom filter for multiple types, with setable seeds and other parameters. + * Simplified Bloom filter for multiple types, with settable seeds and other parameters. * * @author hal.hildebrand */ abstract public class BloomFilter { - private final BitSet bits; - private final Hash h; + final BitSet bits; + final Hash h; private BloomFilter(Hash h) { this(h, new BitSet(h.getM())); @@ -34,42 +34,28 @@ private BloomFilter(Hash h, BitSet bits) { @SuppressWarnings("unchecked") public static BloomFilter create(long seed, int n, double p, Biff.Type type) { - switch (type) { - case DIGEST: - return (BloomFilter) new DigestBloomFilter(seed, n, p); - case INT: - return (BloomFilter) new IntBloomFilter(seed, n, p); - case LONG: - return (BloomFilter) new LongBloomFilter(seed, n, p); - case BYTES: - return (BloomFilter) new BytesBloomFilter(seed, n, p); - case STRING: - return (BloomFilter) new StringBloomFilter(seed, n, p); - case ULONG: - return (BloomFilter) new ULongBloomFilter(seed, n, p); - default: - throw new IllegalArgumentException("Invalid type: " + type); - } + return switch (type) { + case DIGEST -> (BloomFilter) new DigestBloomFilter(seed, n, p); + case INT -> (BloomFilter) new IntBloomFilter(seed, n, p); + case LONG -> (BloomFilter) new LongBloomFilter(seed, n, p); + case BYTES -> (BloomFilter) new BytesBloomFilter(seed, n, p); + case STRING -> (BloomFilter) new StringBloomFilter(seed, n, p); + case ULONG -> (BloomFilter) new ULongBloomFilter(seed, n, p); + default -> throw new IllegalArgumentException("Invalid type: " + type); + }; } @SuppressWarnings("unchecked") public static BloomFilter create(long seed, int m, int k, long[] bits, Biff.Type type) { - switch (type) { - case DIGEST: - return (BloomFilter) new DigestBloomFilter(seed, m, k, bits); - case INT: - return (BloomFilter) new IntBloomFilter(seed, m, k, bits); - case LONG: - return (BloomFilter) new LongBloomFilter(seed, m, k, bits); - case BYTES: - return (BloomFilter) new BytesBloomFilter(seed, m, k, bits); - case STRING: - return (BloomFilter) new StringBloomFilter(seed, m, k, bits); - case ULONG: - return (BloomFilter) new ULongBloomFilter(seed, m, k, bits); - default: - throw new IllegalArgumentException("Invalid type: " + type); - } + return switch (type) { + case DIGEST -> (BloomFilter) new DigestBloomFilter(seed, m, k, bits); + case INT -> (BloomFilter) new IntBloomFilter(seed, m, k, bits); + case LONG -> (BloomFilter) new LongBloomFilter(seed, m, k, bits); + case BYTES -> (BloomFilter) new BytesBloomFilter(seed, m, k, bits); + case STRING -> (BloomFilter) new StringBloomFilter(seed, m, k, bits); + case ULONG -> (BloomFilter) new ULongBloomFilter(seed, m, k, bits); + default -> throw new IllegalArgumentException("Invalid type: " + type); + }; } public static BloomFilter from(Biff bff) { @@ -106,6 +92,8 @@ public void clear() { bits.clear(); } + public abstract BloomFilter clone(); + public boolean contains(T element) { for (int hash : h.hashes(element)) { if (!bits.get(hash)) { @@ -116,8 +104,7 @@ public boolean contains(T element) { } public boolean equivalent(BloomFilter other) { - var equiv = h.equivalent(other.h) && bits.equals(other.bits); - return equiv; + return h.equivalent(other.h) && bits.equals(other.bits); } public double fpp(int n) { @@ -126,9 +113,9 @@ public double fpp(int n) { /** * Estimates the current population of the Bloom filter (see: - * http://en.wikipedia.org/wiki/Bloom_filter#Approximating_the_number_of_items_in_a_Bloom_filter + * ... * - * @return the estimated amount of elements in the filter + * @return the estimated number of elements in the filter */ public double getEstimatedPopulation() { return population(bits, h.getK(), h.getM()); @@ -148,7 +135,7 @@ public Biff toBff() { public static class BytesBloomFilter extends BloomFilter { public BytesBloomFilter(long seed, int n, double p) { - super(new Hash(seed, n, p) { + super(new Hash<>(seed, n, p) { @Override protected Hasher newHasher() { return new BytesHasher(); @@ -157,7 +144,7 @@ protected Hasher newHasher() { } public BytesBloomFilter(long seed, int m, int k, long[] bytes) { - super(new Hash(seed, k, m) { + super(new Hash<>(seed, k, m) { @Override protected Hasher newHasher() { return new BytesHasher(); @@ -165,6 +152,15 @@ protected Hasher newHasher() { }, BitSet.valueOf(bytes)); } + public BytesBloomFilter(Hash hash, BitSet bitSet) { + super(hash, bitSet); + } + + @Override + public BloomFilter clone() { + return new BytesBloomFilter(h.clone(), (BitSet) bits.clone()); + } + @Override protected Biff.Type getType() { return BYTES; @@ -173,8 +169,12 @@ protected Biff.Type getType() { public static class DigestBloomFilter extends BloomFilter { + public DigestBloomFilter(Hash hash, BitSet bitSet) { + super(hash, bitSet); + } + public DigestBloomFilter(long seed, int n, double p) { - super(new Hash(seed, n, p) { + super(new Hash<>(seed, n, p) { @Override protected Hasher newHasher() { return new DigestHasher(); @@ -183,7 +183,7 @@ protected Hasher newHasher() { } public DigestBloomFilter(long seed, int m, int k, long[] bytes) { - super(new Hash(seed, k, m) { + super(new Hash<>(seed, k, m) { @Override protected Hasher newHasher() { return new DigestHasher(); @@ -191,6 +191,11 @@ protected Hasher newHasher() { }, BitSet.valueOf(bytes)); } + @Override + public BloomFilter clone() { + return new DigestBloomFilter(h.clone(), (BitSet) bits.clone()); + } + @Override protected Biff.Type getType() { return DIGEST; @@ -200,8 +205,12 @@ protected Biff.Type getType() { public static class IntBloomFilter extends BloomFilter { + public IntBloomFilter(Hash hash, BitSet bitSet) { + super(hash, bitSet); + } + public IntBloomFilter(long seed, int n, double p) { - super(new Hash(seed, n, p) { + super(new Hash<>(seed, n, p) { @Override protected Hasher newHasher() { return new IntHasher(); @@ -210,7 +219,7 @@ protected Hasher newHasher() { } public IntBloomFilter(long seed, int m, int k, long[] bits) { - super(new Hash(seed, k, m) { + super(new Hash<>(seed, k, m) { @Override protected Hasher newHasher() { return new IntHasher(); @@ -218,6 +227,11 @@ protected Hasher newHasher() { }, BitSet.valueOf(bits)); } + @Override + public BloomFilter clone() { + return new IntBloomFilter(h.clone(), (BitSet) bits.clone()); + } + @Override protected Biff.Type getType() { return INT; @@ -226,8 +240,13 @@ protected Biff.Type getType() { } public static class LongBloomFilter extends BloomFilter { + + public LongBloomFilter(Hash hash, BitSet bitSet) { + super(hash, bitSet); + } + public LongBloomFilter(long seed, int n, double p) { - super(new Hash(seed, n, p) { + super(new Hash<>(seed, n, p) { @Override protected Hasher newHasher() { return new LongHasher(); @@ -236,7 +255,7 @@ protected Hasher newHasher() { } public LongBloomFilter(long seed, int m, int k, long[] bits) { - super(new Hash(seed, k, m) { + super(new Hash<>(seed, k, m) { @Override protected Hasher newHasher() { return new LongHasher(); @@ -244,6 +263,11 @@ protected Hasher newHasher() { }, BitSet.valueOf(bits)); } + @Override + public BloomFilter clone() { + return new LongBloomFilter(h.clone(), (BitSet) bits.clone()); + } + @Override protected Biff.Type getType() { return LONG; @@ -253,8 +277,12 @@ protected Biff.Type getType() { public static class StringBloomFilter extends BloomFilter { + public StringBloomFilter(Hash hash, BitSet bitSet) { + super(hash, bitSet); + } + public StringBloomFilter(long seed, int n, double p) { - super(new Hash(seed, n, p) { + super(new Hash<>(seed, n, p) { @Override protected Hasher newHasher() { return new StringHasher(); @@ -263,7 +291,7 @@ protected Hasher newHasher() { } public StringBloomFilter(long seed, int m, int k, long[] bytes) { - super(new Hash(seed, k, m) { + super(new Hash<>(seed, k, m) { @Override protected Hasher newHasher() { return new StringHasher(); @@ -271,6 +299,11 @@ protected Hasher newHasher() { }, BitSet.valueOf(bytes)); } + @Override + public BloomFilter clone() { + return new StringBloomFilter(h.clone(), (BitSet) bits.clone()); + } + @Override protected Biff.Type getType() { return STRING; @@ -278,8 +311,13 @@ protected Biff.Type getType() { } public static class ULongBloomFilter extends BloomFilter { + + public ULongBloomFilter(Hash hash, BitSet bitSet) { + super(hash, bitSet); + } + public ULongBloomFilter(long seed, int n, double p) { - super(new Hash(seed, n, p) { + super(new Hash<>(seed, n, p) { @Override protected Hasher newHasher() { return new ULongHasher(); @@ -288,7 +326,7 @@ protected Hasher newHasher() { } public ULongBloomFilter(long seed, int m, int k, long[] bits) { - super(new Hash(seed, k, m) { + super(new Hash<>(seed, k, m) { @Override protected Hasher newHasher() { return new ULongHasher(); @@ -296,6 +334,11 @@ protected Hasher newHasher() { }, BitSet.valueOf(bits)); } + @Override + public BloomFilter clone() { + return new ULongBloomFilter(h.clone(), (BitSet) bits.clone()); + } + @Override protected Biff.Type getType() { return ULONG; diff --git a/cryptography/src/main/java/com/salesforce/apollo/cryptography/HexBloom.java b/cryptography/src/main/java/com/salesforce/apollo/cryptography/HexBloom.java index 90714cf0d..c2480c1fe 100644 --- a/cryptography/src/main/java/com/salesforce/apollo/cryptography/HexBloom.java +++ b/cryptography/src/main/java/com/salesforce/apollo/cryptography/HexBloom.java @@ -26,6 +26,7 @@ * @author hal.hildebrand */ public class HexBloom { + public static final double DEFAULT_FPR = 0.0001; public static final long DEFAULT_SEED = Primes.PRIMES[666]; private static final Function IDENTITY = d -> d; @@ -288,6 +289,25 @@ public static List> hashWraps(int crowns) { return IntStream.range(0, crowns).mapToObj(i -> hashWrap(i)).toList(); } + public HexBloom add(Digest d, List> hashes) { + return addAll(Collections.singletonList(d), hashes); + } + + public HexBloom addAll(List added, List> hashes) { + var nextCard = cardinality + added.size(); + var nextMembership = membership.clone(); + var crwns = Arrays.stream(crowns).map(AtomicReference::new).toList(); + + added.forEach(d -> { + for (int i = 0; i < crwns.size(); i++) { + crwns.get(i).accumulateAndGet(hashes.get(i).apply(d), Digest::xor); + } + nextMembership.add(d); + }); + + return new HexBloom(nextCard, crwns.stream().map(AtomicReference::get).toList(), nextMembership); + } + public Digest compact() { if (crowns.length == 1) { return crowns[0]; diff --git a/fireflies/src/main/java/com/salesforce/apollo/fireflies/Binding.java b/fireflies/src/main/java/com/salesforce/apollo/fireflies/Binding.java index 91597bb46..4e107d8a9 100644 --- a/fireflies/src/main/java/com/salesforce/apollo/fireflies/Binding.java +++ b/fireflies/src/main/java/com/salesforce/apollo/fireflies/Binding.java @@ -274,7 +274,7 @@ private void join(Redirect redirect, Digest v, Duration duration) { return; } redirecting.iterate((link, m) -> { - if (!view.started.get()) { + if (gateway.isDone() || !view.started.get()) { return null; } log.debug("Joining: {} contacting: {} on: {}", v, link.getMember().getId(), node.getId()); diff --git a/fireflies/src/main/java/com/salesforce/apollo/fireflies/View.java b/fireflies/src/main/java/com/salesforce/apollo/fireflies/View.java index 7b2b499b2..65346def8 100644 --- a/fireflies/src/main/java/com/salesforce/apollo/fireflies/View.java +++ b/fireflies/src/main/java/com/salesforce/apollo/fireflies/View.java @@ -619,11 +619,6 @@ protected Gossip gossip(Fireflies link, int ring) { return link.gossip(gossip); } catch (Throwable e) { final var p = (Participant) link.getMember(); - if (!viewManagement.joined()) { - log.debug("Exception: {} bootstrap gossiping with:S {} view: {} on: {}", e.getMessage(), p.getId(), - currentView(), node.getId()); - return null; - } if (e instanceof StatusRuntimeException sre) { switch (sre.getStatus().getCode()) { case PERMISSION_DENIED: @@ -653,7 +648,8 @@ protected Gossip gossip(Fireflies link, int ring) { } } else { - log.debug("Exception gossiping with: {} view: {} on: {}", p.getId(), currentView(), node.getId(), e); + log.debug("Exception gossiping joined: {} with: {} view: {} on: {}", viewManagement.joined(), p.getId(), + currentView(), node.getId(), e); accuse(p, ring, e); } return null; @@ -1298,7 +1294,7 @@ private void recover(Participant member) { return; } if (context.activate(member)) { - log.debug("Recovering: {} cardinality: {} count: {} on: {}", member.getId(), viewManagement.cardinality(), + log.trace("Recovering: {} cardinality: {} count: {} on: {}", member.getId(), viewManagement.cardinality(), context.totalCount(), node.getId()); } }