Properly handle committee members who did not supply their view signi…

…ng key in the CHOAM view reconfiguration.

Some clean up on builders, adding missing clone()

choam/src/main/java/com/salesforce/apollo/choam/Committee.java

@@ -16,6 +16,7 @@
 import com.salesforce.apollo.cryptography.Verifier;
 import com.salesforce.apollo.cryptography.Verifier.DefaultVerifier;
 import com.salesforce.apollo.membership.Member;
+import com.salesforce.apollo.membership.MockMember;
 import org.slf4j.Logger;
 
 import java.util.HashSet;
@@ -31,12 +32,14 @@
 public interface Committee {
 
     static Map<Member, Verifier> validatorsOf(Reconfigure reconfigure, Context<Member> context) {
-        var validators = reconfigure.getJoinsList()
-                                    .stream()
-                                    .collect(
-                                    Collectors.toMap(e -> context.getMember(new Digest(e.getMember().getVm().getId())),
-                                                     e -> (Verifier) new DefaultVerifier(
-                                                     publicKey(e.getMember().getVm().getConsensusKey()))));
+        var validators = reconfigure.getJoinsList().stream().collect(Collectors.toMap(e -> {
+            var id = new Digest(e.getMember().getVm().getId());
+            var m = context.getMember(id);
+            return m == null ? new MockMember(id) : m;
+        }, e -> {
+            var vm = e.getMember().getVm();
+            return vm.hasConsensusKey() ? new DefaultVerifier(publicKey(vm.getConsensusKey())) : Verifier.NO_VERIFIER;
+        }));
         assert !validators.isEmpty() : "No validators in this reconfiguration of: " + context.getId();
         return validators;
     }

choam/src/main/java/com/salesforce/apollo/choam/Parameters.java

@@ -570,7 +570,7 @@ public Builder setMaxGossipDelay(Duration maxGossipDelay) {
         }
     }
 
-    public static class LimiterBuilder {
+    public static class LimiterBuilder implements Cloneable {
         private Duration backlogDuration = Duration.ofSeconds(1);
         private int      backlogSize     = 1_000;
         private double   backoffRatio    = 0.5;
@@ -597,6 +597,14 @@ public Limiter<Void> build(String name, MetricRegistry metrics) {
                                       .build();
         }
 
+        public LimiterBuilder clone() {
+            try {
+                return (LimiterBuilder) super.clone();
+            } catch (CloneNotSupportedException e) {
+                throw new IllegalStateException(e);
+            }
+        }
+
         public int getBacklogSize() {
             return backlogSize;
         }
@@ -706,11 +714,20 @@ public Parameters build(RuntimeParameters runtime) {
 
         @Override
         public Builder clone() {
+            Builder clone;
             try {
-                return (Builder) super.clone();
+                clone = (Builder) super.clone();
             } catch (CloneNotSupportedException e) {
                 throw new IllegalStateException("well, that was unexpected");
             }
+            clone.setMvBuilder(mvBuilder.clone());
+            clone.setProducer(
+            new ProducerParameters(producer.ethereal.clone(), producer.gossipDuration, producer.maxBatchByteSize(),
+                                   producer.batchInterval, producer.maxBatchCount(), producer.maxGossipDelay));
+            clone.setTxnLimiterBuilder(txnLimiterBuilder.clone());
+            clone.setSubmitPolicy(submitPolicy.clone());
+            clone.setDrainPolicy(drainPolicy.clone());
+            return clone;
         }
 
         public BootstrapParameters getBootstrap() {

choam/src/main/java/com/salesforce/apollo/choam/ViewAssembly.java

@@ -9,6 +9,7 @@
 import com.chiralbehaviors.tron.Fsm;
 import com.google.common.collect.HashMultiset;
 import com.google.common.collect.Multiset;
+import com.google.common.collect.Sets;
 import com.salesforce.apollo.archipelago.RouterImpl.CommonCommunications;
 import com.salesforce.apollo.choam.comm.Terminal;
 import com.salesforce.apollo.choam.fsm.Reconfiguration;
@@ -98,9 +99,16 @@ void assembled() {
 
     void complete() {
         cancelSlice.set(true);
-        proposals.entrySet()
-                 .stream()
-                 .forEach(e -> slate.put(e.getKey(), Join.newBuilder().setMember(e.getValue()).build()));
+        proposals.entrySet().forEach(e -> slate.put(e.getKey(), Join.newBuilder().setMember(e.getValue()).build()));
+        // Fill out the slate with the unreachable members of the next assembly
+        Sets.difference(selected.assembly.keySet(), proposals.keySet())
+            .forEach(m -> slate.put(m, Join.newBuilder()
+                                           .setMember(SignedViewMember.newBuilder()
+                                                                      .setVm(ViewMember.newBuilder()
+                                                                                       .setId(m.toDigeste())
+                                                                                       .setView(
+                                                                                       nextViewId.toDigeste())))
+                                           .build()));
         log.debug("View Assembly: {} completed with: {} members on: {}", nextViewId, slate.size(),
                   params().member().getId());
     }

choam/src/main/java/com/salesforce/apollo/choam/support/ExponentialBackoffPolicy.java

@@ -58,7 +58,7 @@ private long uniformRandom(double low, double high) {
         return (long) (Entropy.nextBitsStreamDouble() * mag + low);
     }
 
-    public static class Builder {
+    public static class Builder implements Cloneable {
         private Duration initialBackoff = Duration.ofMillis(10);
         private double   jitter         = .2;
         private Duration maxBackoff     = Duration.ofMillis(500);
@@ -68,6 +68,14 @@ public ExponentialBackoffPolicy build() {
             return new ExponentialBackoffPolicy(initialBackoff, jitter, maxBackoff, multiplier);
         }
 
+        public Builder clone() {
+            try {
+                return (Builder) super.clone();
+            } catch (CloneNotSupportedException e) {
+                throw new IllegalStateException(e);
+            }
+        }
+
         public Duration getInitialBackoff() {
             return initialBackoff;
         }

choam/src/test/java/com/salesforce/apollo/choam/DynamicTest.java

@@ -0,0 +1,122 @@
+package com.salesforce.apollo.choam;
+
+import com.salesforce.apollo.archipelago.LocalServer;
+import com.salesforce.apollo.archipelago.Router;
+import com.salesforce.apollo.archipelago.ServerConnectionCache;
+import com.salesforce.apollo.context.Context;
+import com.salesforce.apollo.context.DynamicContext;
+import com.salesforce.apollo.cryptography.DigestAlgorithm;
+import com.salesforce.apollo.membership.Member;
+import com.salesforce.apollo.membership.SigningMember;
+import com.salesforce.apollo.membership.stereotomy.ControlledIdentifierMember;
+import com.salesforce.apollo.stereotomy.StereotomyImpl;
+import com.salesforce.apollo.stereotomy.mem.MemKERL;
+import com.salesforce.apollo.stereotomy.mem.MemKeyStore;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.security.SecureRandom;
+import java.time.Duration;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+/**
+ * @author hal.hildebrand
+ **/
+public class DynamicTest {
+    private static final int cardinality         = 10;
+    private static final int checkpointBlockSize = 10;
+
+    private List<Member>                        members;
+    private SecureRandom                        entropy;
+    private Map<Member, Router>                 routers;
+    private Map<Member, CHOAM>                  choams;
+    private Map<Member, DynamicContext<Member>> contexts;
+
+    @BeforeEach
+    public void setUp() throws Exception {
+        choams = new HashMap<>();
+        contexts = new HashMap<>();
+        var contextBuilder = DynamicContext.newBuilder().setBias(3);
+        entropy = SecureRandom.getInstance("SHA1PRNG");
+        entropy.setSeed(new byte[] { 6, 6, 6 });
+        var stereotomy = new StereotomyImpl(new MemKeyStore(), new MemKERL(DigestAlgorithm.DEFAULT), entropy);
+
+        members = IntStream.range(0, cardinality)
+                           .mapToObj(i -> stereotomy.newIdentifier())
+                           .map(ControlledIdentifierMember::new)
+                           .map(e -> (Member) e)
+                           .toList();
+        members = IntStream.range(0, cardinality)
+                           .mapToObj(i -> stereotomy.newIdentifier())
+                           .map(ControlledIdentifierMember::new)
+                           .map(e -> (Member) e)
+                           .toList();
+
+        final var prefix = UUID.randomUUID().toString();
+        routers = members.stream()
+                         .collect(Collectors.toMap(m -> m, m -> new LocalServer(prefix, m).router(
+                         ServerConnectionCache.newBuilder().setTarget(cardinality * 2))));
+
+        var template = Parameters.newBuilder()
+                                 .setGenerateGenesis(true)
+                                 .setBootstrap(Parameters.BootstrapParameters.newBuilder()
+                                                                             .setGossipDuration(Duration.ofMillis(20))
+                                                                             .build())
+                                 .setGenesisViewId(DigestAlgorithm.DEFAULT.getOrigin())
+                                 .setGossipDuration(Duration.ofMillis(10))
+                                 .setProducer(Parameters.ProducerParameters.newBuilder()
+                                                                           .setGossipDuration(Duration.ofMillis(20))
+                                                                           .setBatchInterval(Duration.ofMillis(10))
+                                                                           .setMaxBatchByteSize(1024 * 1024)
+                                                                           .setMaxBatchCount(10_000)
+                                                                           .build())
+                                 .setGenerateGenesis(true)
+                                 .setCheckpointBlockDelta(checkpointBlockSize);
+        members.forEach(m -> {
+            var context = contextBuilder.build();
+            contexts.put(m, (DynamicContext<Member>) context);
+            choams.put(m, constructCHOAM((SigningMember) m, template.clone(), context));
+        });
+    }
+
+    @Test
+    public void smokin() throws Exception {
+
+    }
+
+    @AfterEach
+    public void tearDown() throws Exception {
+        if (choams != null) {
+            choams.values().forEach(CHOAM::stop);
+            choams = null;
+        }
+        if (routers != null) {
+            routers.values().forEach(e -> e.close(Duration.ofSeconds(1)));
+            routers = null;
+        }
+        members = null;
+    }
+
+    private CHOAM constructCHOAM(SigningMember m, Parameters.Builder params, Context<Member> context) {
+        final CHOAM.TransactionExecutor processor = (index, hash, t, f, executor) -> {
+            if (f != null) {
+                f.completeAsync(Object::new, executor);
+            }
+        };
+
+        params.getProducer().ethereal().setSigner(m);
+        var choam = new CHOAM(params.build(Parameters.RuntimeParameters.newBuilder()
+                                                                       .setMember(m)
+                                                                       .setCommunications(routers.get(m))
+                                                                       .setProcessor(processor)
+                                                                       .setContext(context)
+                                                                       .build()));
+        return choam;
+    }
+}

cryptography/src/main/java/com/salesforce/apollo/cryptography/Verifier.java

@@ -22,6 +22,8 @@
  * @author hal.hildebrand
  */
 public interface Verifier {
+    Verifier NO_VERIFIER = new NoVerifier();
+
     default Filtered filtered(SigningThreshold threshold, JohnHancock signature, byte[]... message) {
         return filtered(threshold, signature, BbBackedInputStream.aggregate(message));
     }
@@ -145,6 +147,25 @@ public boolean verify(SigningThreshold threshold, JohnHancock signature, InputSt
         }
     }
 
+    class NoVerifier implements Verifier {
+
+        @Override
+        public Filtered filtered(SigningThreshold threshold, JohnHancock signature, InputStream message) {
+            return new Filtered(false, signature.signatureCount(), signature);
+        }
+
+        @Override
+        public boolean verify(JohnHancock signature, InputStream message) {
+            return false;
+        }
+
+        @Override
+        public boolean verify(SigningThreshold threshold, JohnHancock signature, InputStream message) {
+            return false;
+        }
+
+    }
+
     class MockVerifier implements Verifier {
 
         @Override

memberships/src/test/java/com/salesforce/apollo/messaging/rbc/RbcTest.java

@@ -55,7 +55,7 @@ public class RbcTest {
     private static final boolean                         LARGE_TESTS    = Boolean.getBoolean("large_tests");
     private static final Parameters.Builder              parameters     = Parameters.newBuilder()
                                                                                     .setMaxMessages(100)
-                                                                                    .setFalsePositiveRate(0.0125)
+                                                                                    .setFalsePositiveRate(0.00125)
                                                                                     .setBufferSize(500)
                                                                                     .setDedupBufferSize(
                                                                                     LARGE_TESTS ? 100 * 100 : 50 * 50)