From 5a926c55f663c7a92f6fd38270beb3bc30b94502 Mon Sep 17 00:00:00 2001 From: Hellblazer Date: Mon, 3 Jun 2024 17:02:13 -0700 Subject: [PATCH 01/22] Saner resource management. Don't set exec on clients. --- .../salesforce/apollo/choam/DynamicTest.java | 2 +- .../apollo/choam/GenesisAssemblyTest.java | 2 +- .../apollo/choam/MembershipTests.java | 2 +- .../salesforce/apollo/choam/TestCHOAM.java | 4 +- .../apollo/ethereal/EtherealTest.java | 4 +- .../apollo/fireflies/ChurnTest.java | 10 +-- .../salesforce/apollo/fireflies/E2ETest.java | 6 +- .../client/GorgoneionClientTest.java | 8 +- .../apollo/gorgoneion/GorgoneionTest.java | 4 +- .../apollo/demesnes/FireFliesTrace.java | 2 +- .../apollo/leyden/LeydenJarTest.java | 2 +- .../apollo/archipelago/Enclave.java | 15 ++-- .../apollo/archipelago/LocalServer.java | 15 ++-- .../apollo/archipelago/MtlsClient.java | 4 +- .../apollo/archipelago/MtlsServer.java | 13 +-- .../apollo/archipelago/RouterImpl.java | 42 +++++---- .../apollo/archipelago/RouterSupplier.java | 23 ++++- .../archipelago/ServerConnectionCache.java | 42 ++++++--- .../apollo/archipelago/UnsafeExecutors.java | 88 +++++++++++++++++++ .../messaging/rbc/ReliableBroadcaster.java | 11 +-- .../apollo/archipelago/EnclaveTest.java | 6 +- .../apollo/archipelago/FernetTest.java | 4 +- .../apollo/archipelago/LocalServerTest.java | 4 +- .../apollo/archipelago/RouterTest.java | 2 +- .../apollo/messaging/rbc/RbcTest.java | 4 +- .../apollo/ring/RingCommunicationsTest.java | 2 +- .../apollo/ring/RingIteratorTest.java | 2 +- .../apollo/ring/SliceIteratorTest.java | 2 +- .../apollo/model/ContainmentDomainTest.java | 2 +- .../salesforce/apollo/model/DomainTest.java | 2 +- .../apollo/model/FireFliesTest.java | 2 +- .../apollo/model/demesnes/DemesneTest.java | 6 +- .../comm/grpc/ForwardingManagedChannel.java | 68 ++++++++++++++ .../apollo/state/AbstractLifecycleTest.java | 4 +- .../salesforce/apollo/state/CHOAMTest.java | 4 +- .../stereotomy/services/grpc/TestBinder.java | 4 +- .../services/grpc/TestEventObserver.java | 4 +- .../services/grpc/TestEventValidation.java | 4 +- .../services/grpc/TestKerlService.java | 4 +- .../services/grpc/TestResolver.java | 4 +- .../apollo/thoth/AbstractDhtTest.java | 2 +- .../apollo/thoth/BootstrappingTest.java | 2 +- .../apollo/thoth/DhtRebalanceTest.java | 2 +- .../apollo/thoth/PublisherTest.java | 4 +- 44 files changed, 321 insertions(+), 122 deletions(-) create mode 100644 protocols/src/main/java/com/salesforce/apollo/comm/grpc/ForwardingManagedChannel.java diff --git a/choam/src/test/java/com/salesforce/apollo/choam/DynamicTest.java b/choam/src/test/java/com/salesforce/apollo/choam/DynamicTest.java index d3d583f35..fe29a7e45 100644 --- a/choam/src/test/java/com/salesforce/apollo/choam/DynamicTest.java +++ b/choam/src/test/java/com/salesforce/apollo/choam/DynamicTest.java @@ -212,7 +212,7 @@ public void tearDown() throws Exception { choams = null; } if (routers != null) { - routers.values().forEach(e -> e.close(Duration.ofSeconds(1))); + routers.values().forEach(e -> e.close(Duration.ofSeconds(0))); routers = null; } members = null; diff --git a/choam/src/test/java/com/salesforce/apollo/choam/GenesisAssemblyTest.java b/choam/src/test/java/com/salesforce/apollo/choam/GenesisAssemblyTest.java index 079dbe402..b113b13c8 100644 --- a/choam/src/test/java/com/salesforce/apollo/choam/GenesisAssemblyTest.java +++ b/choam/src/test/java/com/salesforce/apollo/choam/GenesisAssemblyTest.java @@ -183,7 +183,7 @@ public Block reconfigure(Map joining, Digest nextViewId, HashedBlo genii.values().forEach(GenesisAssembly::start); complete.await(15, TimeUnit.SECONDS); } finally { - communications.values().forEach(r -> r.close(Duration.ofSeconds(1))); + communications.values().forEach(r -> r.close(Duration.ofSeconds(0))); genii.values().forEach(GenesisAssembly::stop); } } diff --git a/choam/src/test/java/com/salesforce/apollo/choam/MembershipTests.java b/choam/src/test/java/com/salesforce/apollo/choam/MembershipTests.java index f56f69436..effd854f5 100644 --- a/choam/src/test/java/com/salesforce/apollo/choam/MembershipTests.java +++ b/choam/src/test/java/com/salesforce/apollo/choam/MembershipTests.java @@ -207,7 +207,7 @@ private void shutdown() { choams = null; } if (routers != null) { - routers.values().forEach(e -> e.close(Duration.ofSeconds(1))); + routers.values().forEach(e -> e.close(Duration.ofSeconds(0))); routers = null; } } diff --git a/choam/src/test/java/com/salesforce/apollo/choam/TestCHOAM.java b/choam/src/test/java/com/salesforce/apollo/choam/TestCHOAM.java index 8f0aff039..391706bf2 100644 --- a/choam/src/test/java/com/salesforce/apollo/choam/TestCHOAM.java +++ b/choam/src/test/java/com/salesforce/apollo/choam/TestCHOAM.java @@ -75,7 +75,7 @@ public class TestCHOAM { @AfterEach public void after() throws Exception { if (routers != null) { - routers.values().forEach(e -> e.close(Duration.ofSeconds(1))); + routers.values().forEach(e -> e.close(Duration.ofSeconds(0))); routers = null; } if (choams != null) { @@ -197,7 +197,7 @@ public void submitMultiplTxn() throws Exception { .filter(i -> i < max) .count()); } finally { - routers.values().forEach(e -> e.close(Duration.ofSeconds(1))); + routers.values().forEach(e -> e.close(Duration.ofSeconds(0))); choams.values().forEach(e -> e.stop()); System.out.println(); diff --git a/ethereal/src/test/java/com/salesforce/apollo/ethereal/EtherealTest.java b/ethereal/src/test/java/com/salesforce/apollo/ethereal/EtherealTest.java index c6f38106f..7f8443c27 100644 --- a/ethereal/src/test/java/com/salesforce/apollo/ethereal/EtherealTest.java +++ b/ethereal/src/test/java/com/salesforce/apollo/ethereal/EtherealTest.java @@ -160,7 +160,7 @@ public void unbounded() throws NoSuchAlgorithmException, InterruptedException, I } finally { controllers.forEach(Ethereal::stop); gossipers.forEach(ChRbcGossip::stop); - comms.forEach(e -> e.close(Duration.ofSeconds(1))); + comms.forEach(e -> e.close(Duration.ofSeconds(0))); } final var expected = expectedEpochs * (EPOCH_LENGTH - 1); @@ -287,7 +287,7 @@ private void one(int iteration) controllers.forEach(c -> System.out.println(c.dump())); controllers.forEach(Ethereal::stop); gossipers.forEach(ChRbcGossip::stop); - comms.forEach(e -> e.close(Duration.ofSeconds(1))); + comms.forEach(e -> e.close(Duration.ofSeconds(0))); } final var expected = NUM_EPOCHS * (EPOCH_LENGTH - 1); diff --git a/fireflies/src/test/java/com/salesforce/apollo/fireflies/ChurnTest.java b/fireflies/src/test/java/com/salesforce/apollo/fireflies/ChurnTest.java index d209c439b..9825e45f6 100644 --- a/fireflies/src/test/java/com/salesforce/apollo/fireflies/ChurnTest.java +++ b/fireflies/src/test/java/com/salesforce/apollo/fireflies/ChurnTest.java @@ -74,10 +74,10 @@ public void after() { views.clear(); } - communications.forEach(e -> e.close(Duration.ofSeconds(1))); + communications.forEach(e -> e.close(Duration.ofSeconds(0))); communications.clear(); - gateways.forEach(e -> e.close(Duration.ofSeconds(1))); + gateways.forEach(e -> e.close(Duration.ofSeconds(0))); gateways.clear(); } @@ -210,8 +210,8 @@ public void churn() throws Exception { for (int j = c.size() - 1; j >= c.size() - delta; j--) { final var view = c.get(j); view.stop(); - r.get(j).close(Duration.ofSeconds(1)); - g.get(j).close(Duration.ofSeconds(1)); + r.get(j).close(Duration.ofSeconds(0)); + g.get(j).close(Duration.ofSeconds(0)); removed.add(view.getNode().getId()); } c = c.subList(0, c.size() - delta); @@ -239,7 +239,7 @@ public void churn() throws Exception { } views.forEach(e -> e.stop()); - communications.forEach(e -> e.close(Duration.ofSeconds(1))); + communications.forEach(e -> e.close(Duration.ofSeconds(0))); System.out.println(); diff --git a/fireflies/src/test/java/com/salesforce/apollo/fireflies/E2ETest.java b/fireflies/src/test/java/com/salesforce/apollo/fireflies/E2ETest.java index a6e58f479..1300fe554 100644 --- a/fireflies/src/test/java/com/salesforce/apollo/fireflies/E2ETest.java +++ b/fireflies/src/test/java/com/salesforce/apollo/fireflies/E2ETest.java @@ -83,10 +83,10 @@ public void after() { views.clear(); } - communications.forEach(e -> e.close(Duration.ofSeconds(1))); + communications.forEach(e -> e.close(Duration.ofSeconds(0))); communications.clear(); - gateways.forEach(e -> e.close(Duration.ofSeconds(1))); + gateways.forEach(e -> e.close(Duration.ofSeconds(0))); gateways.clear(); } @@ -209,7 +209,7 @@ private void initialize() { } private void post() { - communications.forEach(e -> e.close(Duration.ofSeconds(1))); + communications.forEach(e -> e.close(Duration.ofSeconds(0))); views.forEach(view -> view.stop()); System.out.println("Node 0 metrics"); ConsoleReporter.forRegistry(node0Registry) diff --git a/gorgoneion-client/src/test/java/com/salesforce/apollo/gorgoneion/client/GorgoneionClientTest.java b/gorgoneion-client/src/test/java/com/salesforce/apollo/gorgoneion/client/GorgoneionClientTest.java index c84e85eac..5b3f962c4 100644 --- a/gorgoneion-client/src/test/java/com/salesforce/apollo/gorgoneion/client/GorgoneionClientTest.java +++ b/gorgoneion-client/src/test/java/com/salesforce/apollo/gorgoneion/client/GorgoneionClientTest.java @@ -97,8 +97,8 @@ public void clientSmoke() throws Exception { var invitation = gorgoneionClient.apply(Duration.ofSeconds(60)); - gorgonRouter.close(Duration.ofSeconds(1)); - clientRouter.close(Duration.ofSeconds(1)); + gorgonRouter.close(Duration.ofSeconds(0)); + clientRouter.close(Duration.ofSeconds(0)); assertNotNull(invitation); assertNotEquals(Validations.getDefaultInstance(), invitation); @@ -113,10 +113,10 @@ public void clientSmoke() throws Exception { @AfterEach public void closeRouters() { if (gorgonRouter != null) { - gorgonRouter.close(Duration.ofSeconds(3)); + gorgonRouter.close(Duration.ofSeconds(0)); } if (clientRouter != null) { - clientRouter.close(Duration.ofSeconds(3)); + clientRouter.close(Duration.ofSeconds(0)); } } diff --git a/gorgoneion/src/test/java/com/salesforce/apollo/gorgoneion/GorgoneionTest.java b/gorgoneion/src/test/java/com/salesforce/apollo/gorgoneion/GorgoneionTest.java index 57386081a..b31f69259 100644 --- a/gorgoneion/src/test/java/com/salesforce/apollo/gorgoneion/GorgoneionTest.java +++ b/gorgoneion/src/test/java/com/salesforce/apollo/gorgoneion/GorgoneionTest.java @@ -109,8 +109,8 @@ public void smokin() throws Exception { .build()) .setNonce(fs) .build(), Duration.ofSeconds(1)); - gorgonRouter.close(Duration.ofSeconds(1)); - clientRouter.close(Duration.ofSeconds(1)); + gorgonRouter.close(Duration.ofSeconds(0)); + clientRouter.close(Duration.ofSeconds(0)); assertNotNull(invitation); assertNotEquals(Validations.getDefaultInstance(), invitation); assertEquals(1, invitation.getValidationsCount()); diff --git a/isolates/src/test/java/com/salesforce/apollo/demesnes/FireFliesTrace.java b/isolates/src/test/java/com/salesforce/apollo/demesnes/FireFliesTrace.java index 8e2a95111..0bf42a2cc 100644 --- a/isolates/src/test/java/com/salesforce/apollo/demesnes/FireFliesTrace.java +++ b/isolates/src/test/java/com/salesforce/apollo/demesnes/FireFliesTrace.java @@ -169,7 +169,7 @@ public static void smoke(Oracle oracle) throws Exception { public void after() { domains.forEach(n -> n.stop()); domains.clear(); - routers.values().forEach(r -> r.close(Duration.ofSeconds(1))); + routers.values().forEach(r -> r.close(Duration.ofSeconds(0))); routers.clear(); } diff --git a/leyden/src/test/java/com/salesforce/apollo/leyden/LeydenJarTest.java b/leyden/src/test/java/com/salesforce/apollo/leyden/LeydenJarTest.java index 1e5ffce2e..1944a54cc 100644 --- a/leyden/src/test/java/com/salesforce/apollo/leyden/LeydenJarTest.java +++ b/leyden/src/test/java/com/salesforce/apollo/leyden/LeydenJarTest.java @@ -46,7 +46,7 @@ public class LeydenJarTest { @AfterEach public void after() { - routers.values().forEach(r -> r.close(Duration.ofSeconds(2))); + routers.values().forEach(r -> r.close(Duration.ofSeconds(0))); routers.clear(); dhts.values().forEach(t -> t.stop()); dhts.clear(); diff --git a/memberships/src/main/java/com/salesforce/apollo/archipelago/Enclave.java b/memberships/src/main/java/com/salesforce/apollo/archipelago/Enclave.java index 5ae364d6c..08a830894 100644 --- a/memberships/src/main/java/com/salesforce/apollo/archipelago/Enclave.java +++ b/memberships/src/main/java/com/salesforce/apollo/archipelago/Enclave.java @@ -27,7 +27,7 @@ import org.slf4j.LoggerFactory; import java.util.List; -import java.util.concurrent.Executor; +import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.function.Consumer; import java.util.function.Predicate; @@ -46,7 +46,6 @@ public class Enclave implements RouterSupplier { private final static Class channelType = IMPL.getChannelType(); private static final Logger log = LoggerFactory.getLogger(Enclave.class); - private final Executor executor = Executors.newVirtualThreadPerTaskExecutor(); private final DomainSocketAddress bridge; private final Consumer contextRegistration; private final DomainSocketAddress endpoint; @@ -63,10 +62,6 @@ public Enclave(Member from, DomainSocketAddress endpoint, DomainSocketAddress br this.fromString = qb64(from.getId()); } - public void close() { - eventLoopGroup.shutdownGracefully(); - } - /** * @return the DomainSocketAddress for this Enclave */ @@ -77,7 +72,10 @@ public DomainSocketAddress getEndpoint() { @Override public RouterImpl router(ServerConnectionCache.Builder cacheBuilder, Supplier serverLimit, LimitsRegistry limitsRegistry, List interceptors, - Predicate validator) { + Predicate validator, ExecutorService executor) { + if (executor == null) { + executor = Executors.newVirtualThreadPerTaskExecutor(); + } var limitsBuilder = new GrpcServerLimiterBuilder().limit(serverLimit.get()); if (limitsRegistry != null) { limitsBuilder.metricRegistry(limitsRegistry); @@ -111,7 +109,7 @@ public Digest getAgent() { public Digest getFrom() { return Constants.SERVER_CLIENT_ID_KEY.get(); } - }, contextRegistration, validator); + }, contextRegistration, validator, executor); } private ManagedChannel connectTo(Member to) { @@ -132,7 +130,6 @@ public void start(Listener responseListener, Metadata headers) { }; final var builder = NettyChannelBuilder.forAddress(bridge) .withOption(ChannelOption.TCP_NODELAY, true) - .executor(executor) .eventLoopGroup(eventLoopGroup) .channelType(channelType) .usePlaintext() diff --git a/memberships/src/main/java/com/salesforce/apollo/archipelago/LocalServer.java b/memberships/src/main/java/com/salesforce/apollo/archipelago/LocalServer.java index fcee2a5af..781217edb 100644 --- a/memberships/src/main/java/com/salesforce/apollo/archipelago/LocalServer.java +++ b/memberships/src/main/java/com/salesforce/apollo/archipelago/LocalServer.java @@ -25,7 +25,8 @@ import java.lang.reflect.Method; import java.util.List; -import java.util.concurrent.Executor; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import java.util.function.Predicate; import java.util.function.Supplier; @@ -39,7 +40,6 @@ public class LocalServer implements RouterSupplier { private static final Logger log = LoggerFactory.getLogger(LocalServer.class); private static final String NAME_TEMPLATE = "%s-%s"; - private final Executor executor = UnsafeExecutors.newVirtualThreadPerTaskExecutor(); private final ClientInterceptor clientInterceptor; private final Member from; private final String prefix; @@ -70,15 +70,17 @@ public Member getFrom() { @Override public RouterImpl router(ServerConnectionCache.Builder cacheBuilder, Supplier serverLimit, LimitsRegistry limitsRegistry, List interceptors, - Predicate validator) { + Predicate validator, ExecutorService executor) { + if (executor == null) { + executor = Executors.newVirtualThreadPerTaskExecutor(); + } String name = String.format(NAME_TEMPLATE, prefix, qb64(from.getId())); var limitsBuilder = new GrpcServerLimiterBuilder().limit(serverLimit.get()); if (limitsRegistry != null) { limitsBuilder.metricRegistry(limitsRegistry); } ServerBuilder serverBuilder = InProcessServerBuilder.forName(name) - .executor( - UnsafeExecutors.newVirtualThreadPerTaskExecutor()) + .executor(executor) .intercept(ConcurrencyLimitServerInterceptor.newBuilder( limitsBuilder.build()) .statusSupplier( @@ -95,13 +97,12 @@ public Digest getFrom() { return Constants.SERVER_CLIENT_ID_KEY.get(); } }, d -> { - }, validator); + }, validator, executor); } private ManagedChannel connectTo(Member to) { final var name = String.format(NAME_TEMPLATE, prefix, qb64(to.getId())); final InProcessChannelBuilder builder = InProcessChannelBuilder.forName(name) - .executor(executor) .usePlaintext() .intercept(clientInterceptor); disableTrash(builder); diff --git a/memberships/src/main/java/com/salesforce/apollo/archipelago/MtlsClient.java b/memberships/src/main/java/com/salesforce/apollo/archipelago/MtlsClient.java index 58db9ec8e..8f5b2a7be 100644 --- a/memberships/src/main/java/com/salesforce/apollo/archipelago/MtlsClient.java +++ b/memberships/src/main/java/com/salesforce/apollo/archipelago/MtlsClient.java @@ -19,7 +19,6 @@ import io.netty.handler.ssl.ClientAuth; import java.net.SocketAddress; -import java.util.concurrent.Executor; /** * @author hal.hildebrand @@ -29,11 +28,10 @@ public class MtlsClient { private final ManagedChannel channel; public MtlsClient(SocketAddress address, ClientAuth clientAuth, String alias, ClientContextSupplier supplier, - CertificateValidator validator, Executor executor) { + CertificateValidator validator) { Limiter limiter = new GrpcClientLimiterBuilder().blockOnLimit(false).build(); channel = NettyChannelBuilder.forAddress(address) - .executor(executor) .withOption(ChannelOption.TCP_NODELAY, true) .sslContext(supplier.forClient(clientAuth, alias, validator, MtlsServer.TL_SV1_3)) .intercept(new ConcurrencyLimitClientInterceptor(limiter, diff --git a/memberships/src/main/java/com/salesforce/apollo/archipelago/MtlsServer.java b/memberships/src/main/java/com/salesforce/apollo/archipelago/MtlsServer.java index 0a800fa92..540d41119 100644 --- a/memberships/src/main/java/com/salesforce/apollo/archipelago/MtlsServer.java +++ b/memberships/src/main/java/com/salesforce/apollo/archipelago/MtlsServer.java @@ -43,7 +43,7 @@ import java.security.cert.X509Certificate; import java.util.List; import java.util.concurrent.ExecutionException; -import java.util.concurrent.Executor; +import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.function.Function; import java.util.function.Predicate; @@ -63,7 +63,6 @@ public class MtlsServer implements RouterSupplier { private final Member from; private final Context.Key sslSessionContext = Context.key("SSLSession"); private final ServerContextSupplier supplier; - private final Executor executor; public MtlsServer(Member from, EndpointProvider epProvider, Function contextSupplier, ServerContextSupplier supplier) { @@ -71,7 +70,6 @@ public MtlsServer(Member from, EndpointProvider epProvider, Function() { @Override public Digest load(X509Certificate key) throws Exception { @@ -142,7 +140,10 @@ public static SslContext forServer(ClientAuth clientAuth, String alias, X509Cert @Override public RouterImpl router(ServerConnectionCache.Builder cacheBuilder, Supplier serverLimit, LimitsRegistry limitsRegistry, List interceptors, - Predicate validator) { + Predicate validator, ExecutorService executor) { + if (executor == null) { + executor = Executors.newVirtualThreadPerTaskExecutor(); + } var limitsBuilder = new GrpcServerLimiterBuilder().limit(serverLimit.get()); if (limitsRegistry != null) { limitsBuilder.metricRegistry(limitsRegistry); @@ -174,14 +175,14 @@ public Digest getFrom() { } }; return new RouterImpl(from, serverBuilder, cacheBuilder.setFactory(t -> connectTo(t)), identity, c -> { - }, validator); + }, validator, executor); } private ManagedChannel connectTo(Member to) { var address = epProvider.addressFor(to); log.debug("Connecting to: {} address: {} on: {}", to.getId(), address, from.getId()); return new MtlsClient(address, epProvider.getClientAuth(), epProvider.getAlias(), contextSupplier.apply(from), - epProvider.getValidator(), executor).getChannel(); + epProvider.getValidator()).getChannel(); } private X509Certificate getCert() { diff --git a/memberships/src/main/java/com/salesforce/apollo/archipelago/RouterImpl.java b/memberships/src/main/java/com/salesforce/apollo/archipelago/RouterImpl.java index e7506c4e9..57705f081 100644 --- a/memberships/src/main/java/com/salesforce/apollo/archipelago/RouterImpl.java +++ b/memberships/src/main/java/com/salesforce/apollo/archipelago/RouterImpl.java @@ -23,6 +23,8 @@ import java.time.Duration; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Consumer; @@ -39,38 +41,47 @@ */ public class RouterImpl implements Router { - private final static Logger log = LoggerFactory.getLogger( - RouterImpl.class); - private final ServerConnectionCache cache; - private final ClientIdentity clientIdentityProvider; - private final Consumer contextRegistration; - private final Member from; - private final MutableHandlerRegistry registry = new MutableHandlerRegistry(); - private final Server server; - private final Map> services = new ConcurrentHashMap<>(); - private final AtomicBoolean started = new AtomicBoolean(); - private final Predicate validator; + private final static Logger log = LoggerFactory.getLogger(RouterImpl.class); + + private final ServerConnectionCache cache; + private final ClientIdentity clientIdentityProvider; + private final Consumer contextRegistration; + private final Member from; + private final MutableHandlerRegistry registry = new MutableHandlerRegistry(); + private final Server server; + private final Map> services = new ConcurrentHashMap<>(); + private final AtomicBoolean started = new AtomicBoolean(); + private final Predicate validator; + private final ExecutorService executor; public RouterImpl(Member from, ServerBuilder serverBuilder, ServerConnectionCache.Builder cacheBuilder, ClientIdentity clientIdentityProvider) { this(from, serverBuilder, cacheBuilder, clientIdentityProvider, d -> { - }); + }, Executors.newVirtualThreadPerTaskExecutor()); } public RouterImpl(Member from, ServerBuilder serverBuilder, ServerConnectionCache.Builder cacheBuilder, - ClientIdentity clientIdentityProvider, Consumer contextRegistration) { - this(from, serverBuilder, cacheBuilder, clientIdentityProvider, contextRegistration, null); + ClientIdentity clientIdentityProvider, ExecutorService executor) { + this(from, serverBuilder, cacheBuilder, clientIdentityProvider, d -> { + }, executor); + } + + public RouterImpl(Member from, ServerBuilder serverBuilder, ServerConnectionCache.Builder cacheBuilder, + ClientIdentity clientIdentityProvider, Consumer contextRegistration, + ExecutorService executor) { + this(from, serverBuilder, cacheBuilder, clientIdentityProvider, contextRegistration, null, executor); } public RouterImpl(Member from, ServerBuilder serverBuilder, ServerConnectionCache.Builder cacheBuilder, ClientIdentity clientIdentityProvider, Consumer contextRegistration, - Predicate validator) { + Predicate validator, ExecutorService executor) { this.server = serverBuilder.fallbackHandlerRegistry(registry).intercept(serverInterceptor()).build(); this.cache = cacheBuilder.clone().setMember(from.getId()).build(); this.clientIdentityProvider = clientIdentityProvider; this.contextRegistration = contextRegistration; this.from = from; this.validator = validator; + this.executor = executor; } public static ClientInterceptor clientInterceptor(Digest ctx) { @@ -124,6 +135,7 @@ public void close(Duration await) { } catch (InterruptedException e) { Thread.currentThread().interrupt(); } + executor.shutdown(); } @Override diff --git a/memberships/src/main/java/com/salesforce/apollo/archipelago/RouterSupplier.java b/memberships/src/main/java/com/salesforce/apollo/archipelago/RouterSupplier.java index 5a8d71df2..a88882f09 100644 --- a/memberships/src/main/java/com/salesforce/apollo/archipelago/RouterSupplier.java +++ b/memberships/src/main/java/com/salesforce/apollo/archipelago/RouterSupplier.java @@ -13,6 +13,7 @@ import java.util.Collections; import java.util.List; +import java.util.concurrent.*; import java.util.function.Predicate; import java.util.function.Supplier; @@ -20,6 +21,18 @@ * @author hal.hildebrand */ public interface RouterSupplier { + static ExecutorService newCachedThreadPool(int corePoolSize, ThreadFactory threadFactory) { + return newCachedThreadPool(corePoolSize, threadFactory, true); + } + + static ExecutorService newCachedThreadPool(int corePoolSize, ThreadFactory threadFactory, boolean preStart) { + var threadPoolExecutor = new ThreadPoolExecutor(corePoolSize, Integer.MAX_VALUE, 60L, TimeUnit.SECONDS, + new SynchronousQueue(), threadFactory); + if (preStart) { + threadPoolExecutor.prestartAllCoreThreads(); + } + return threadPoolExecutor; + } default Router router() { return router(ServerConnectionCache.newBuilder(), RouterImpl::defaultServerLimit, null); @@ -39,8 +52,14 @@ default Router router(ServerConnectionCache.Builder cacheBuilder, Supplier serverLimit, + LimitsRegistry limitsRegistry, List interceptors, + Predicate validator) { + return router(cacheBuilder, serverLimit, limitsRegistry, interceptors, validator, null); + + } + Router router(ServerConnectionCache.Builder cacheBuilder, Supplier serverLimit, LimitsRegistry limitsRegistry, List interceptors, - Predicate validator); - + Predicate validator, ExecutorService executor); } diff --git a/memberships/src/main/java/com/salesforce/apollo/archipelago/ServerConnectionCache.java b/memberships/src/main/java/com/salesforce/apollo/archipelago/ServerConnectionCache.java index fca126e8d..d568bd361 100644 --- a/memberships/src/main/java/com/salesforce/apollo/archipelago/ServerConnectionCache.java +++ b/memberships/src/main/java/com/salesforce/apollo/archipelago/ServerConnectionCache.java @@ -20,6 +20,7 @@ import java.time.Duration; import java.time.Instant; import java.util.*; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.locks.ReentrantLock; import java.util.function.Supplier; @@ -44,18 +45,19 @@ */ public class ServerConnectionCache { - private final static Logger log = LoggerFactory.getLogger( - ServerConnectionCache.class); - private final Map cache = new HashMap<>(); - private final Clock clock; - private final ServerConnectionFactory factory; - private final ReentrantLock lock = new ReentrantLock(true); - private final ServerConnectionCacheMetrics metrics; - private final Duration minIdle; - private final PriorityQueue queue = new PriorityQueue<>(); - private final int target; - private final Digest member; - private final CallCredentials credentials; + private final static Logger log = LoggerFactory.getLogger(ServerConnectionCache.class); + + private final Map cache = new HashMap<>(); + private final Clock clock; + private final ServerConnectionFactory factory; + private final ReentrantLock lock = new ReentrantLock(true); + private final ServerConnectionCacheMetrics metrics; + private final Duration minIdle; + private final PriorityQueue queue = new PriorityQueue<>(); + private final int target; + private final Digest member; + private final CallCredentials credentials; + private final AtomicBoolean open = new AtomicBoolean(true); public ServerConnectionCache(Digest member, CallCredentials credentials, ServerConnectionFactory factory, int target, Duration minIdle, Clock clock, ServerConnectionCacheMetrics metrics) { @@ -74,6 +76,9 @@ public static Builder newBuilder() { } public ManagedServerChannel borrow(Digest context, Member to) { + if (!open.get()) { + throw new IllegalStateException("not open on: " + member); + } return lock(() -> { if (cache.size() >= target) { log.debug("Cache target open connections exceeded: {}, opening to: {} on: {}", target, to.getId(), @@ -110,15 +115,21 @@ public ManagedServerChannel borrow(Digest context, Member to) { } public T borrow(Digest context, Member to, CreateClientCommunications createFunction) { + if (!open.get()) { + throw new IllegalStateException("not open on: " + member); + } return createFunction.create(borrow(context, to)); } public void close() { + if (!open.compareAndSet(true, false)) { + return; + } lock(() -> { log.info("Closing connection cache on: {}", member); for (ReleasableManagedChannel conn : new ArrayList<>(cache.values())) { try { - conn.channel.shutdownNow(); + conn.channel.shutdown(); if (metrics != null) { metrics.channelOpenDuration().update(Duration.between(conn.created, Instant.now(clock))); metrics.openConnections().dec(); @@ -134,6 +145,9 @@ public void close() { } public void release(ReleasableManagedChannel connection) { + if (!open.get()) { + return; + } lock(() -> { if (connection.decrementBorrow()) { log.debug("Releasing connection to: {} on: {}", connection.member.getId(), member); @@ -150,7 +164,7 @@ public void release(ReleasableManagedChannel connection) { private boolean close(ReleasableManagedChannel connection) { if (connection.isCloseable()) { try { - connection.channel.shutdownNow(); + connection.channel.shutdown(); } catch (Throwable t) { log.debug("Error closing connection to: {} on: {}", connection.member.getId(), connection.member); } diff --git a/memberships/src/main/java/com/salesforce/apollo/archipelago/UnsafeExecutors.java b/memberships/src/main/java/com/salesforce/apollo/archipelago/UnsafeExecutors.java index 1b0e35cb6..eec5c0926 100644 --- a/memberships/src/main/java/com/salesforce/apollo/archipelago/UnsafeExecutors.java +++ b/memberships/src/main/java/com/salesforce/apollo/archipelago/UnsafeExecutors.java @@ -62,6 +62,94 @@ private static void setExecutor(Object builder, Object executor) { } } + public static ThreadPoolExecutor newCachedThreadPool(int corePoolSize) { + return newCachedThreadPool(corePoolSize, true); + } + + public static ThreadPoolExecutor newCachedThreadPool(int corePoolSize, boolean prestart) { + var executorService = newCachedThreadPool(corePoolSize, new ForkJoinPool()); + if (prestart) { + executorService.prestartAllCoreThreads(); + } + return executorService; + } + + public static ThreadPoolExecutor newCachedThreadPool(int corePoolSize, ExecutorService executor) { + ThreadFactory factory = r -> { + var builder = Thread.ofVirtual(); + setExecutor(builder, executor); + return builder.unstarted(r); + }; + return new ThreadPoolExecutor(corePoolSize, Integer.MAX_VALUE, 60L, TimeUnit.SECONDS, + new SynchronousQueue(), factory) { + @Override + public boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedException { + return executor.awaitTermination(timeout, unit); + } + + @Override + public boolean isShutdown() { + return executor.isShutdown(); + } + + @Override + public boolean isTerminated() { + return executor.isTerminated() && super.isTerminated(); + } + + @Override + public void shutdown() { + executor.shutdown(); + super.shutdown(); + } + + @Override + public List shutdownNow() { + var returned = executor.shutdownNow(); + super.shutdownNow(); + return returned; + } + }; + } + + public static ExecutorService newFixedThreadPool(int nThreads, ExecutorService executor) { + ThreadFactory factory = r -> { + var builder = Thread.ofVirtual(); + setExecutor(builder, executor); + return builder.unstarted(r); + }; + return new ThreadPoolExecutor(nThreads, nThreads, 0L, TimeUnit.MILLISECONDS, + new LinkedBlockingQueue(), factory) { + @Override + public boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedException { + return executor.awaitTermination(timeout, unit); + } + + @Override + public boolean isShutdown() { + return executor.isShutdown(); + } + + @Override + public boolean isTerminated() { + return executor.isTerminated() && super.isTerminated(); + } + + @Override + public void shutdown() { + executor.shutdown(); + super.shutdown(); + } + + @Override + public List shutdownNow() { + var returned = executor.shutdownNow(); + super.shutdownNow(); + return returned; + } + }; + } + private static class BTB { private int characteristics; private long counter; diff --git a/memberships/src/main/java/com/salesforce/apollo/membership/messaging/rbc/ReliableBroadcaster.java b/memberships/src/main/java/com/salesforce/apollo/membership/messaging/rbc/ReliableBroadcaster.java index 8de6acb5d..290f5299a 100644 --- a/memberships/src/main/java/com/salesforce/apollo/membership/messaging/rbc/ReliableBroadcaster.java +++ b/memberships/src/main/java/com/salesforce/apollo/membership/messaging/rbc/ReliableBroadcaster.java @@ -77,6 +77,7 @@ public ReliableBroadcaster(Context context, SigningMember member, Parame r -> new RbcServer(communications.getClientIdentityProvider(), metrics, r), getCreate(metrics), ReliableBroadcast.getLocalLoopback(member)); gossiper = new RingCommunications<>(context, member, this.comm); + gossiper.ignoreSelf(); this.adapter = adapter; } @@ -230,18 +231,18 @@ private Reconcile gossipRound(ReliableBroadcast link, int ring) { if (!started.get()) { return null; } - log.trace("rbc gossiping[{}] with: {} ring: {} on: {}", buffer.round(), member.getId(), + log.trace("rbc gossiping[{}:{}] with: {} ring: {} on: {}", context.getId(), buffer.round(), link.getMember().getId(), ring, member.getId()); try { return link.gossip( MessageBff.newBuilder().setRing(ring).setDigests(buffer.forReconcilliation().toBff()).build()); } catch (StatusRuntimeException sre) { - log.trace("rbc gossiping[{}] failed: {} with: {} ring: {} on: {}", buffer.round(), sre.getStatus(), - link.getMember().getId(), ring, member.getId()); + log.trace("rbc gossiping[{}:{}] failed: {} with: {} ring: {} on: {}", context.getId(), buffer.round(), + sre.getStatus(), link.getMember().getId(), ring, member.getId()); return null; } catch (Throwable e) { - log.trace("rbc gossiping[{}] failed with: {} ring: {} on: {}", buffer.round(), link.getMember().getId(), - ring, member.getId(), e); + log.trace("rbc gossiping[{}:{}] failed with: {} ring: {} on: {}", context.getId(), buffer.round(), + link.getMember().getId(), ring, member.getId(), e); return null; } } diff --git a/memberships/src/test/java/com/salesforce/apollo/archipelago/EnclaveTest.java b/memberships/src/test/java/com/salesforce/apollo/archipelago/EnclaveTest.java index 55e8c78f2..85ce08693 100644 --- a/memberships/src/test/java/com/salesforce/apollo/archipelago/EnclaveTest.java +++ b/memberships/src/test/java/com/salesforce/apollo/archipelago/EnclaveTest.java @@ -142,9 +142,9 @@ public void smokin() throws Exception { msg = resultB.unpack(ByteMessage.class); assertEquals("Hello Server B", msg.getContents().toStringUtf8()); - portal.close(Duration.ofSeconds(1)); - router1.close(Duration.ofSeconds(1)); - router2.close(Duration.ofSeconds(1)); + portal.close(Duration.ofSeconds(0)); + router1.close(Duration.ofSeconds(0)); + router2.close(Duration.ofSeconds(0)); } private ManagedChannel handler(DomainSocketAddress address) { diff --git a/memberships/src/test/java/com/salesforce/apollo/archipelago/FernetTest.java b/memberships/src/test/java/com/salesforce/apollo/archipelago/FernetTest.java index 615a4cf43..612a34e13 100644 --- a/memberships/src/test/java/com/salesforce/apollo/archipelago/FernetTest.java +++ b/memberships/src/test/java/com/salesforce/apollo/archipelago/FernetTest.java @@ -115,8 +115,8 @@ public void smokin() throws Exception { assertNotNull(resultB); assertEquals("Hello Server A", resultB.unpack(ByteMessage.class).getContents().toStringUtf8()); - routerA.close(Duration.ofSeconds(1)); - routerB.close(Duration.ofSeconds(1)); + routerA.close(Duration.ofSeconds(0)); + routerB.close(Duration.ofSeconds(0)); } public interface TestIt { diff --git a/memberships/src/test/java/com/salesforce/apollo/archipelago/LocalServerTest.java b/memberships/src/test/java/com/salesforce/apollo/archipelago/LocalServerTest.java index 3099ac332..3c7cf715d 100644 --- a/memberships/src/test/java/com/salesforce/apollo/archipelago/LocalServerTest.java +++ b/memberships/src/test/java/com/salesforce/apollo/archipelago/LocalServerTest.java @@ -84,8 +84,8 @@ public void smokin() throws Exception { assertNotNull(resultB); assertEquals("Hello Server A", resultB.unpack(ByteMessage.class).getContents().toStringUtf8()); - routerA.close(Duration.ofSeconds(1)); - routerB.close(Duration.ofSeconds(1)); + routerA.close(Duration.ofSeconds(0)); + routerB.close(Duration.ofSeconds(0)); } public interface TestIt { diff --git a/memberships/src/test/java/com/salesforce/apollo/archipelago/RouterTest.java b/memberships/src/test/java/com/salesforce/apollo/archipelago/RouterTest.java index 919c8aaaa..884142ded 100644 --- a/memberships/src/test/java/com/salesforce/apollo/archipelago/RouterTest.java +++ b/memberships/src/test/java/com/salesforce/apollo/archipelago/RouterTest.java @@ -83,7 +83,7 @@ public Any ping(Any request) { msg = resultB.unpack(ByteMessage.class); assertEquals("Hello Server B", msg.getContents().toStringUtf8()); - router.close(Duration.ofSeconds(1)); + router.close(Duration.ofSeconds(0)); } public interface TestIt { diff --git a/memberships/src/test/java/com/salesforce/apollo/messaging/rbc/RbcTest.java b/memberships/src/test/java/com/salesforce/apollo/messaging/rbc/RbcTest.java index c7cf320ce..b9fc84dfb 100644 --- a/memberships/src/test/java/com/salesforce/apollo/messaging/rbc/RbcTest.java +++ b/memberships/src/test/java/com/salesforce/apollo/messaging/rbc/RbcTest.java @@ -70,7 +70,7 @@ public void after() { if (messengers != null) { messengers.forEach(e -> e.stop()); } - communications.forEach(e -> e.close(Duration.ofMillis(1))); + communications.forEach(e -> e.close(Duration.ofMillis(0))); } @Test @@ -140,7 +140,7 @@ public void broadcast() throws Exception { receiver.reset(); } } - communications.forEach(e -> e.close(Duration.ofMillis(1))); + communications.forEach(e -> e.close(Duration.ofMillis(0))); System.out.println(); diff --git a/memberships/src/test/java/com/salesforce/apollo/ring/RingCommunicationsTest.java b/memberships/src/test/java/com/salesforce/apollo/ring/RingCommunicationsTest.java index 3c2f6b81a..fc0ac9be5 100644 --- a/memberships/src/test/java/com/salesforce/apollo/ring/RingCommunicationsTest.java +++ b/memberships/src/test/java/com/salesforce/apollo/ring/RingCommunicationsTest.java @@ -95,7 +95,7 @@ public Any ping(Any request) { assertFalse(pinged1.get()); assertTrue(pinged2.get()); } finally { - router.close(Duration.ofSeconds(5)); + router.close(Duration.ofSeconds(0)); } } } diff --git a/memberships/src/test/java/com/salesforce/apollo/ring/RingIteratorTest.java b/memberships/src/test/java/com/salesforce/apollo/ring/RingIteratorTest.java index bf9d9bf67..00db46be2 100644 --- a/memberships/src/test/java/com/salesforce/apollo/ring/RingIteratorTest.java +++ b/memberships/src/test/java/com/salesforce/apollo/ring/RingIteratorTest.java @@ -105,7 +105,7 @@ public Any ping(Any request) { assertFalse(pinged1.get()); assertTrue(pinged2.get()); } finally { - router.close(Duration.ofSeconds(2)); + router.close(Duration.ofSeconds(0)); } } } diff --git a/memberships/src/test/java/com/salesforce/apollo/ring/SliceIteratorTest.java b/memberships/src/test/java/com/salesforce/apollo/ring/SliceIteratorTest.java index 500fd165f..b0abd0790 100644 --- a/memberships/src/test/java/com/salesforce/apollo/ring/SliceIteratorTest.java +++ b/memberships/src/test/java/com/salesforce/apollo/ring/SliceIteratorTest.java @@ -103,7 +103,7 @@ public Any ping(Any request) { assertTrue(pinged1.get()); assertTrue(pinged2.get()); } finally { - router.close(Duration.ofSeconds(2)); + router.close(Duration.ofSeconds(0)); } } } diff --git a/model/src/test/java/com/salesforce/apollo/model/ContainmentDomainTest.java b/model/src/test/java/com/salesforce/apollo/model/ContainmentDomainTest.java index c316dd9d7..6494cbecb 100644 --- a/model/src/test/java/com/salesforce/apollo/model/ContainmentDomainTest.java +++ b/model/src/test/java/com/salesforce/apollo/model/ContainmentDomainTest.java @@ -55,7 +55,7 @@ public class ContainmentDomainTest { public void after() { domains.forEach(Domain::stop); domains.clear(); - routers.forEach(r -> r.close(Duration.ofSeconds(100))); + routers.forEach(r -> r.close(Duration.ofSeconds(0))); routers.clear(); } diff --git a/model/src/test/java/com/salesforce/apollo/model/DomainTest.java b/model/src/test/java/com/salesforce/apollo/model/DomainTest.java index 9b915ed9a..dd5f0503f 100644 --- a/model/src/test/java/com/salesforce/apollo/model/DomainTest.java +++ b/model/src/test/java/com/salesforce/apollo/model/DomainTest.java @@ -215,7 +215,7 @@ public static void smoke(Oracle oracle) throws Exception { public void after() { domains.forEach(Domain::stop); domains.clear(); - routers.forEach(r -> r.close(Duration.ofSeconds(1))); + routers.forEach(r -> r.close(Duration.ofSeconds(0))); routers.clear(); } diff --git a/model/src/test/java/com/salesforce/apollo/model/FireFliesTest.java b/model/src/test/java/com/salesforce/apollo/model/FireFliesTest.java index cb9b466d1..beb936b33 100644 --- a/model/src/test/java/com/salesforce/apollo/model/FireFliesTest.java +++ b/model/src/test/java/com/salesforce/apollo/model/FireFliesTest.java @@ -60,7 +60,7 @@ public class FireFliesTest { public void after() { domains.forEach(n -> n.stop()); domains.clear(); - routers.values().forEach(r -> r.close(Duration.ofSeconds(1))); + routers.values().forEach(r -> r.close(Duration.ofSeconds(0))); routers.clear(); } diff --git a/model/src/test/java/com/salesforce/apollo/model/demesnes/DemesneTest.java b/model/src/test/java/com/salesforce/apollo/model/demesnes/DemesneTest.java index 877e7661b..202d4c4a6 100644 --- a/model/src/test/java/com/salesforce/apollo/model/demesnes/DemesneTest.java +++ b/model/src/test/java/com/salesforce/apollo/model/demesnes/DemesneTest.java @@ -186,9 +186,9 @@ public void portal() throws Exception { msg = resultB.unpack(ByteMessage.class); assertEquals("Hello Server B", msg.getContents().toStringUtf8()); - portal.close(Duration.ofSeconds(1)); - router1.close(Duration.ofSeconds(1)); - router2.close(Duration.ofSeconds(1)); + portal.close(Duration.ofSeconds(0)); + router1.close(Duration.ofSeconds(0)); + router2.close(Duration.ofSeconds(0)); } @Test diff --git a/protocols/src/main/java/com/salesforce/apollo/comm/grpc/ForwardingManagedChannel.java b/protocols/src/main/java/com/salesforce/apollo/comm/grpc/ForwardingManagedChannel.java new file mode 100644 index 000000000..550eb705a --- /dev/null +++ b/protocols/src/main/java/com/salesforce/apollo/comm/grpc/ForwardingManagedChannel.java @@ -0,0 +1,68 @@ +// +// Source code recreated from a .class file by IntelliJ IDEA +// (powered by FernFlower decompiler) +// + +package com.salesforce.apollo.comm.grpc; + +import com.google.common.base.MoreObjects; +import io.grpc.*; + +import java.util.concurrent.TimeUnit; + +public class ForwardingManagedChannel extends ManagedChannel { + private final ManagedChannel delegate; + + ForwardingManagedChannel(ManagedChannel delegate) { + this.delegate = delegate; + } + + public String authority() { + return this.delegate.authority(); + } + + public boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedException { + return this.delegate.awaitTermination(timeout, unit); + } + + public void enterIdle() { + this.delegate.enterIdle(); + } + + public ConnectivityState getState(boolean requestConnection) { + return this.delegate.getState(requestConnection); + } + + public boolean isShutdown() { + return this.delegate.isShutdown(); + } + + public boolean isTerminated() { + return this.delegate.isTerminated(); + } + + public ClientCall newCall( + MethodDescriptor methodDescriptor, CallOptions callOptions) { + return this.delegate.newCall(methodDescriptor, callOptions); + } + + public void notifyWhenStateChanged(ConnectivityState source, Runnable callback) { + this.delegate.notifyWhenStateChanged(source, callback); + } + + public void resetConnectBackoff() { + this.delegate.resetConnectBackoff(); + } + + public ManagedChannel shutdown() { + return this.delegate.shutdown(); + } + + public ManagedChannel shutdownNow() { + return this.delegate.shutdownNow(); + } + + public String toString() { + return MoreObjects.toStringHelper(this).add("delegate", this.delegate).toString(); + } +} diff --git a/sql-state/src/test/java/com/salesforce/apollo/state/AbstractLifecycleTest.java b/sql-state/src/test/java/com/salesforce/apollo/state/AbstractLifecycleTest.java index 3c8802818..028f4b67b 100644 --- a/sql-state/src/test/java/com/salesforce/apollo/state/AbstractLifecycleTest.java +++ b/sql-state/src/test/java/com/salesforce/apollo/state/AbstractLifecycleTest.java @@ -111,7 +111,7 @@ private static Txn initialInsert() { @AfterEach public void after() throws Exception { if (routers != null) { - routers.values().forEach(e -> e.close(Duration.ofSeconds(1))); + routers.values().forEach(e -> e.close(Duration.ofSeconds(0))); routers = null; } if (choams != null) { @@ -209,7 +209,7 @@ protected void post() throws Exception { .toList()); choams.values().forEach(e -> e.stop()); - routers.values().forEach(e -> e.close(Duration.ofSeconds(1))); + routers.values().forEach(e -> e.close(Duration.ofSeconds(0))); final ULong target = updaters.values() .stream() .map(ssm -> ssm.getCurrentBlock()) diff --git a/sql-state/src/test/java/com/salesforce/apollo/state/CHOAMTest.java b/sql-state/src/test/java/com/salesforce/apollo/state/CHOAMTest.java index 433ef9acb..e8cb71761 100644 --- a/sql-state/src/test/java/com/salesforce/apollo/state/CHOAMTest.java +++ b/sql-state/src/test/java/com/salesforce/apollo/state/CHOAMTest.java @@ -96,7 +96,7 @@ private static Txn initialInsert() { @AfterEach public void after() throws Exception { if (routers != null) { - routers.values().forEach(e -> e.close(Duration.ofSeconds(1))); + routers.values().forEach(e -> e.close(Duration.ofSeconds(0))); routers = null; } if (choams != null) { @@ -230,7 +230,7 @@ public void submitMultiplTxn() throws Exception { .toList()); } finally { choams.values().forEach(e -> e.stop()); - routers.values().forEach(e -> e.close(Duration.ofSeconds(1))); + routers.values().forEach(e -> e.close(Duration.ofSeconds(0))); System.out.println("Final block height: " + members.stream() .map(m -> updaters.get(m)) diff --git a/stereotomy-services/src/test/java/com/salesforce/apollo/stereotomy/services/grpc/TestBinder.java b/stereotomy-services/src/test/java/com/salesforce/apollo/stereotomy/services/grpc/TestBinder.java index 3a0926d20..18233c2bf 100644 --- a/stereotomy-services/src/test/java/com/salesforce/apollo/stereotomy/services/grpc/TestBinder.java +++ b/stereotomy-services/src/test/java/com/salesforce/apollo/stereotomy/services/grpc/TestBinder.java @@ -42,11 +42,11 @@ public class TestBinder { @AfterEach public void after() { if (serverRouter != null) { - serverRouter.close(Duration.ofMillis(1)); + serverRouter.close(Duration.ofMillis(0)); serverRouter = null; } if (clientRouter != null) { - clientRouter.close(Duration.ofMillis(1)); + clientRouter.close(Duration.ofMillis(0)); clientRouter = null; } } diff --git a/stereotomy-services/src/test/java/com/salesforce/apollo/stereotomy/services/grpc/TestEventObserver.java b/stereotomy-services/src/test/java/com/salesforce/apollo/stereotomy/services/grpc/TestEventObserver.java index 9ba52e7b9..eab2a64a1 100644 --- a/stereotomy-services/src/test/java/com/salesforce/apollo/stereotomy/services/grpc/TestEventObserver.java +++ b/stereotomy-services/src/test/java/com/salesforce/apollo/stereotomy/services/grpc/TestEventObserver.java @@ -44,11 +44,11 @@ public class TestEventObserver { @AfterEach public void after() { if (serverRouter != null) { - serverRouter.close(Duration.ofSeconds(1)); + serverRouter.close(Duration.ofSeconds(0)); serverRouter = null; } if (clientRouter != null) { - clientRouter.close(Duration.ofSeconds(1)); + clientRouter.close(Duration.ofSeconds(0)); clientRouter = null; } } diff --git a/stereotomy-services/src/test/java/com/salesforce/apollo/stereotomy/services/grpc/TestEventValidation.java b/stereotomy-services/src/test/java/com/salesforce/apollo/stereotomy/services/grpc/TestEventValidation.java index 0ac4b5ac1..3b69cf798 100644 --- a/stereotomy-services/src/test/java/com/salesforce/apollo/stereotomy/services/grpc/TestEventValidation.java +++ b/stereotomy-services/src/test/java/com/salesforce/apollo/stereotomy/services/grpc/TestEventValidation.java @@ -39,11 +39,11 @@ public class TestEventValidation { @AfterEach public void after() { if (serverRouter != null) { - serverRouter.close(Duration.ofSeconds(1)); + serverRouter.close(Duration.ofSeconds(0)); serverRouter = null; } if (clientRouter != null) { - clientRouter.close(Duration.ofSeconds(1)); + clientRouter.close(Duration.ofSeconds(0)); clientRouter = null; } } diff --git a/stereotomy-services/src/test/java/com/salesforce/apollo/stereotomy/services/grpc/TestKerlService.java b/stereotomy-services/src/test/java/com/salesforce/apollo/stereotomy/services/grpc/TestKerlService.java index 1c4721d54..e7d81f368 100644 --- a/stereotomy-services/src/test/java/com/salesforce/apollo/stereotomy/services/grpc/TestKerlService.java +++ b/stereotomy-services/src/test/java/com/salesforce/apollo/stereotomy/services/grpc/TestKerlService.java @@ -51,11 +51,11 @@ public class TestKerlService { @AfterEach public void after() { if (serverRouter != null) { - serverRouter.close(Duration.ofSeconds(1)); + serverRouter.close(Duration.ofSeconds(0)); serverRouter = null; } if (clientRouter != null) { - clientRouter.close(Duration.ofSeconds(1)); + clientRouter.close(Duration.ofSeconds(0)); clientRouter = null; } } diff --git a/stereotomy-services/src/test/java/com/salesforce/apollo/stereotomy/services/grpc/TestResolver.java b/stereotomy-services/src/test/java/com/salesforce/apollo/stereotomy/services/grpc/TestResolver.java index 19da10932..705e23e77 100644 --- a/stereotomy-services/src/test/java/com/salesforce/apollo/stereotomy/services/grpc/TestResolver.java +++ b/stereotomy-services/src/test/java/com/salesforce/apollo/stereotomy/services/grpc/TestResolver.java @@ -40,11 +40,11 @@ public class TestResolver { @AfterEach public void after() { if (serverRouter != null) { - serverRouter.close(Duration.ofSeconds(1)); + serverRouter.close(Duration.ofSeconds(0)); serverRouter = null; } if (clientRouter != null) { - clientRouter.close(Duration.ofSeconds(1)); + clientRouter.close(Duration.ofSeconds(0)); clientRouter = null; } } diff --git a/thoth/src/test/java/com/salesforce/apollo/thoth/AbstractDhtTest.java b/thoth/src/test/java/com/salesforce/apollo/thoth/AbstractDhtTest.java index fd22fde20..a761af4d6 100644 --- a/thoth/src/test/java/com/salesforce/apollo/thoth/AbstractDhtTest.java +++ b/thoth/src/test/java/com/salesforce/apollo/thoth/AbstractDhtTest.java @@ -100,7 +100,7 @@ public static RotationEvent rotation(KeyPair prevNext, final Digest prevDigest, @AfterEach public void after() { - routers.values().forEach(r -> r.close(Duration.ofSeconds(2))); + routers.values().forEach(r -> r.close(Duration.ofSeconds(0))); routers.clear(); dhts.values().forEach(t -> t.stop()); dhts.clear(); diff --git a/thoth/src/test/java/com/salesforce/apollo/thoth/BootstrappingTest.java b/thoth/src/test/java/com/salesforce/apollo/thoth/BootstrappingTest.java index 208189545..e1021f3be 100644 --- a/thoth/src/test/java/com/salesforce/apollo/thoth/BootstrappingTest.java +++ b/thoth/src/test/java/com/salesforce/apollo/thoth/BootstrappingTest.java @@ -50,7 +50,7 @@ public class BootstrappingTest extends AbstractDhtTest { @AfterEach public void closeClient() throws Exception { if (clientRouter != null) { - clientRouter.close(Duration.ofSeconds(3)); + clientRouter.close(Duration.ofSeconds(0)); } } diff --git a/thoth/src/test/java/com/salesforce/apollo/thoth/DhtRebalanceTest.java b/thoth/src/test/java/com/salesforce/apollo/thoth/DhtRebalanceTest.java index 71fb3662a..62fa3dbaf 100644 --- a/thoth/src/test/java/com/salesforce/apollo/thoth/DhtRebalanceTest.java +++ b/thoth/src/test/java/com/salesforce/apollo/thoth/DhtRebalanceTest.java @@ -54,7 +54,7 @@ public class DhtRebalanceTest { @AfterEach public void afterIt() throws Exception { - routers.values().forEach(r -> r.close(Duration.ofSeconds(1))); + routers.values().forEach(r -> r.close(Duration.ofSeconds(0))); routers.clear(); dhts.clear(); contexts.clear(); diff --git a/thoth/src/test/java/com/salesforce/apollo/thoth/PublisherTest.java b/thoth/src/test/java/com/salesforce/apollo/thoth/PublisherTest.java index 485fe03ac..f74821219 100644 --- a/thoth/src/test/java/com/salesforce/apollo/thoth/PublisherTest.java +++ b/thoth/src/test/java/com/salesforce/apollo/thoth/PublisherTest.java @@ -79,8 +79,8 @@ public void smokin() throws Exception { client.publish(KERL_.getDefaultInstance(), Collections.emptyList()); client.publishEvents(Collections.emptyList(), Collections.emptyList()); } finally { - clientRouter.close(Duration.ofSeconds(1)); - serverRouter.close(Duration.ofSeconds(1)); + clientRouter.close(Duration.ofSeconds(0)); + serverRouter.close(Duration.ofSeconds(0)); } } From 4d4c2e031b6732e64f094bb6b9280e393561adb7 Mon Sep 17 00:00:00 2001 From: Hellblazer Date: Mon, 3 Jun 2024 19:06:48 -0700 Subject: [PATCH 02/22] don't share forks. Use UE for model domain testing. --- .../com/salesforce/apollo/archipelago/RouterSupplier.java | 4 ++++ .../salesforce/apollo/archipelago/UnsafeExecutors.java | 2 +- .../salesforce/apollo/model/ContainmentDomainTest.java | 8 +++----- .../test/java/com/salesforce/apollo/model/DomainTest.java | 8 +++----- .../java/com/salesforce/apollo/model/FireFliesTest.java | 8 +++----- pom.xml | 2 +- 6 files changed, 15 insertions(+), 17 deletions(-) diff --git a/memberships/src/main/java/com/salesforce/apollo/archipelago/RouterSupplier.java b/memberships/src/main/java/com/salesforce/apollo/archipelago/RouterSupplier.java index a88882f09..8277eb526 100644 --- a/memberships/src/main/java/com/salesforce/apollo/archipelago/RouterSupplier.java +++ b/memberships/src/main/java/com/salesforce/apollo/archipelago/RouterSupplier.java @@ -42,6 +42,10 @@ default Router router(ServerConnectionCache.Builder cacheBuilder) { return router(cacheBuilder, RouterImpl::defaultServerLimit, null); } + default Router router(ServerConnectionCache.Builder cacheBuilder, ExecutorService executor) { + return router(cacheBuilder, RouterImpl::defaultServerLimit, null, Collections.emptyList(), null, executor); + } + default Router router(ServerConnectionCache.Builder cacheBuilder, Supplier serverLimit, LimitsRegistry limitsRegistry) { return router(cacheBuilder, serverLimit, limitsRegistry, Collections.emptyList()); diff --git a/memberships/src/main/java/com/salesforce/apollo/archipelago/UnsafeExecutors.java b/memberships/src/main/java/com/salesforce/apollo/archipelago/UnsafeExecutors.java index eec5c0926..936f42498 100644 --- a/memberships/src/main/java/com/salesforce/apollo/archipelago/UnsafeExecutors.java +++ b/memberships/src/main/java/com/salesforce/apollo/archipelago/UnsafeExecutors.java @@ -39,7 +39,7 @@ public class UnsafeExecutors { } public static ExecutorService newVirtualThreadPerTaskExecutor() { - return virtualThreadExecutor(new ForkJoinPool()); + return virtualThreadExecutor(Executors.newWorkStealingPool()); } public static B configureBuilderExecutor(B builder, Executor executor) { diff --git a/model/src/test/java/com/salesforce/apollo/model/ContainmentDomainTest.java b/model/src/test/java/com/salesforce/apollo/model/ContainmentDomainTest.java index 6494cbecb..7b042156b 100644 --- a/model/src/test/java/com/salesforce/apollo/model/ContainmentDomainTest.java +++ b/model/src/test/java/com/salesforce/apollo/model/ContainmentDomainTest.java @@ -6,10 +6,7 @@ */ package com.salesforce.apollo.model; -import com.salesforce.apollo.archipelago.EndpointProvider; -import com.salesforce.apollo.archipelago.LocalServer; -import com.salesforce.apollo.archipelago.Router; -import com.salesforce.apollo.archipelago.ServerConnectionCache; +import com.salesforce.apollo.archipelago.*; import com.salesforce.apollo.choam.Parameters; import com.salesforce.apollo.choam.Parameters.Builder; import com.salesforce.apollo.choam.Parameters.RuntimeParameters; @@ -84,7 +81,8 @@ public void before() throws Exception { final var group = DigestAlgorithm.DEFAULT.getOrigin(); identities.forEach((d, id) -> { final var member = new ControlledIdentifierMember(id); - var localRouter = new LocalServer(prefix, member).router(ServerConnectionCache.newBuilder().setTarget(30)); + var localRouter = new LocalServer(prefix, member).router(ServerConnectionCache.newBuilder().setTarget(30), + UnsafeExecutors.newVirtualThreadPerTaskExecutor()); routers.add(localRouter); var dbUrl = String.format("jdbc:h2:mem:sql-%s-%s;DB_CLOSE_DELAY=-1", member.getId(), UUID.randomUUID()); var pdParams = new ProcessDomain.ProcessDomainParameters(dbUrl, Duration.ofMinutes(1), diff --git a/model/src/test/java/com/salesforce/apollo/model/DomainTest.java b/model/src/test/java/com/salesforce/apollo/model/DomainTest.java index dd5f0503f..ce361beae 100644 --- a/model/src/test/java/com/salesforce/apollo/model/DomainTest.java +++ b/model/src/test/java/com/salesforce/apollo/model/DomainTest.java @@ -6,10 +6,7 @@ */ package com.salesforce.apollo.model; -import com.salesforce.apollo.archipelago.EndpointProvider; -import com.salesforce.apollo.archipelago.LocalServer; -import com.salesforce.apollo.archipelago.Router; -import com.salesforce.apollo.archipelago.ServerConnectionCache; +import com.salesforce.apollo.archipelago.*; import com.salesforce.apollo.choam.Parameters; import com.salesforce.apollo.choam.Parameters.Builder; import com.salesforce.apollo.choam.Parameters.RuntimeParameters; @@ -240,7 +237,8 @@ public void before() throws Exception { final var group = DigestAlgorithm.DEFAULT.getOrigin(); identities.forEach((d, id) -> { final var member = new ControlledIdentifierMember(id); - var localRouter = new LocalServer(prefix, member).router(ServerConnectionCache.newBuilder().setTarget(30)); + var localRouter = new LocalServer(prefix, member).router(ServerConnectionCache.newBuilder().setTarget(30), + UnsafeExecutors.newVirtualThreadPerTaskExecutor()); routers.add(localRouter); var dbUrl = String.format("jdbc:h2:mem:sql-%s-%s;DB_CLOSE_DELAY=-1", member.getId(), UUID.randomUUID()); var pdParams = new ProcessDomain.ProcessDomainParameters(dbUrl, Duration.ofMinutes(1), diff --git a/model/src/test/java/com/salesforce/apollo/model/FireFliesTest.java b/model/src/test/java/com/salesforce/apollo/model/FireFliesTest.java index beb936b33..4161111a9 100644 --- a/model/src/test/java/com/salesforce/apollo/model/FireFliesTest.java +++ b/model/src/test/java/com/salesforce/apollo/model/FireFliesTest.java @@ -6,10 +6,7 @@ */ package com.salesforce.apollo.model; -import com.salesforce.apollo.archipelago.EndpointProvider; -import com.salesforce.apollo.archipelago.LocalServer; -import com.salesforce.apollo.archipelago.Router; -import com.salesforce.apollo.archipelago.ServerConnectionCache; +import com.salesforce.apollo.archipelago.*; import com.salesforce.apollo.choam.Parameters; import com.salesforce.apollo.choam.Parameters.Builder; import com.salesforce.apollo.choam.Parameters.RuntimeParameters; @@ -84,7 +81,8 @@ public void before() throws Exception { identities.forEach((digest, id) -> { var context = new DynamicContextImpl<>(DigestAlgorithm.DEFAULT.getLast(), CARDINALITY, 0.2, 3); final var member = new ControlledIdentifierMember(id); - var localRouter = new LocalServer(prefix, member).router(ServerConnectionCache.newBuilder().setTarget(30)); + var localRouter = new LocalServer(prefix, member).router(ServerConnectionCache.newBuilder().setTarget(30), + UnsafeExecutors.newVirtualThreadPerTaskExecutor()); var dbUrl = String.format("jdbc:h2:mem:sql-%s-%s;DB_CLOSE_DELAY=-1", member.getId(), UUID.randomUUID()); var pdParams = new ProcessDomain.ProcessDomainParameters(dbUrl, Duration.ofSeconds(5), "jdbc:h2:mem:%s-state".formatted(digest), diff --git a/pom.xml b/pom.xml index c4879bd2b..557face93 100644 --- a/pom.xml +++ b/pom.xml @@ -783,7 +783,7 @@ 3.2.5 ${forks} - true + false -Xmx10G -Xms4G -Djdk.tracePinnedThreads=full From 3708893e56b19e7ae45856f93ecdfe84c5652be9 Mon Sep 17 00:00:00 2001 From: Hellblazer Date: Mon, 3 Jun 2024 19:10:48 -0700 Subject: [PATCH 03/22] remove --- .../comm/grpc/ForwardingManagedChannel.java | 68 ------------------- 1 file changed, 68 deletions(-) delete mode 100644 protocols/src/main/java/com/salesforce/apollo/comm/grpc/ForwardingManagedChannel.java diff --git a/protocols/src/main/java/com/salesforce/apollo/comm/grpc/ForwardingManagedChannel.java b/protocols/src/main/java/com/salesforce/apollo/comm/grpc/ForwardingManagedChannel.java deleted file mode 100644 index 550eb705a..000000000 --- a/protocols/src/main/java/com/salesforce/apollo/comm/grpc/ForwardingManagedChannel.java +++ /dev/null @@ -1,68 +0,0 @@ -// -// Source code recreated from a .class file by IntelliJ IDEA -// (powered by FernFlower decompiler) -// - -package com.salesforce.apollo.comm.grpc; - -import com.google.common.base.MoreObjects; -import io.grpc.*; - -import java.util.concurrent.TimeUnit; - -public class ForwardingManagedChannel extends ManagedChannel { - private final ManagedChannel delegate; - - ForwardingManagedChannel(ManagedChannel delegate) { - this.delegate = delegate; - } - - public String authority() { - return this.delegate.authority(); - } - - public boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedException { - return this.delegate.awaitTermination(timeout, unit); - } - - public void enterIdle() { - this.delegate.enterIdle(); - } - - public ConnectivityState getState(boolean requestConnection) { - return this.delegate.getState(requestConnection); - } - - public boolean isShutdown() { - return this.delegate.isShutdown(); - } - - public boolean isTerminated() { - return this.delegate.isTerminated(); - } - - public ClientCall newCall( - MethodDescriptor methodDescriptor, CallOptions callOptions) { - return this.delegate.newCall(methodDescriptor, callOptions); - } - - public void notifyWhenStateChanged(ConnectivityState source, Runnable callback) { - this.delegate.notifyWhenStateChanged(source, callback); - } - - public void resetConnectBackoff() { - this.delegate.resetConnectBackoff(); - } - - public ManagedChannel shutdown() { - return this.delegate.shutdown(); - } - - public ManagedChannel shutdownNow() { - return this.delegate.shutdownNow(); - } - - public String toString() { - return MoreObjects.toStringHelper(this).add("delegate", this.delegate).toString(); - } -} From 1abedb6f4144dfb44dd3753a162ca03cbd9b6939 Mon Sep 17 00:00:00 2001 From: Hellblazer Date: Tue, 4 Jun 2024 09:51:41 -0700 Subject: [PATCH 04/22] logging, fpr adjus --- .../java/com/salesforce/apollo/choam/CHOAM.java | 10 ++++++++-- .../com/salesforce/apollo/choam/ViewContext.java | 14 +++++--------- .../com/salesforce/apollo/choam/TestCHOAM.java | 11 +++++------ .../com/salesforce/apollo/ethereal/Adder.java | 12 ++++++------ .../com/salesforce/apollo/archipelago/Portal.java | 15 ++++++++------- .../messaging/rbc/ReliableBroadcaster.java | 6 ++++-- 6 files changed, 36 insertions(+), 32 deletions(-) diff --git a/choam/src/main/java/com/salesforce/apollo/choam/CHOAM.java b/choam/src/main/java/com/salesforce/apollo/choam/CHOAM.java index f0970605d..9cd8de9af 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/CHOAM.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/CHOAM.java @@ -360,8 +360,14 @@ public void stop() { return; } session.cancelAll(); - linear.shutdown(); - executions.shutdown(); + try { + linear.shutdown(); + } catch (Throwable e) { + } + try { + executions.shutdown(); + } catch (Throwable e) { + } final var c = current.get(); if (c != null) { c.complete(); diff --git a/choam/src/main/java/com/salesforce/apollo/choam/ViewContext.java b/choam/src/main/java/com/salesforce/apollo/choam/ViewContext.java index c204030db..6b84839e7 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/ViewContext.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/ViewContext.java @@ -248,19 +248,15 @@ public boolean validate(SignedJoin join) { public boolean validate(SignedViews sv) { Verifier v = verifierOf(sv); if (v == null) { - if (log.isDebugEnabled()) { - log.debug("No verifier: {} for signed view on: {}", Digest.from(sv.getViews().getMember()), - params.member().getId()); - } + log.debug("No verifier: {} for signed view on: {}", Digest.from(sv.getViews().getMember()), + params.member().getId()); return false; } var validated = v.verify(JohnHancock.from(sv.getSignature()), sv.getViews().toByteString()); if (!validated) { - if (log.isTraceEnabled()) { - log.trace("Cannot validate views signed by: {} on: {}", Digest.from(sv.getViews().getMember()), - params().member().getId()); - } - } else if (log.isTraceEnabled()) { + log.trace("Cannot validate views signed by: {} on: {}", Digest.from(sv.getViews().getMember()), + params().member().getId()); + } else { log.trace("Validated views signed by: {} on: {}", Digest.from(sv.getViews().getMember()), params().member().getId()); } diff --git a/choam/src/test/java/com/salesforce/apollo/choam/TestCHOAM.java b/choam/src/test/java/com/salesforce/apollo/choam/TestCHOAM.java index 391706bf2..0769d13de 100644 --- a/choam/src/test/java/com/salesforce/apollo/choam/TestCHOAM.java +++ b/choam/src/test/java/com/salesforce/apollo/choam/TestCHOAM.java @@ -6,7 +6,6 @@ */ package com.salesforce.apollo.choam; -import com.codahale.metrics.ConsoleReporter; import com.codahale.metrics.MetricRegistry; import com.salesforce.apollo.archipelago.LocalServer; import com.salesforce.apollo.archipelago.Router; @@ -202,11 +201,11 @@ public void submitMultiplTxn() throws Exception { System.out.println(); - ConsoleReporter.forRegistry(registry) - .convertRatesTo(TimeUnit.SECONDS) - .convertDurationsTo(TimeUnit.MILLISECONDS) - .build() - .report(); + // ConsoleReporter.forRegistry(registry) + // .convertRatesTo(TimeUnit.SECONDS) + // .convertDurationsTo(TimeUnit.MILLISECONDS) + // .build() + // .report(); } assertTrue(checkpointOccurred.get(5, TimeUnit.SECONDS)); } diff --git a/ethereal/src/main/java/com/salesforce/apollo/ethereal/Adder.java b/ethereal/src/main/java/com/salesforce/apollo/ethereal/Adder.java index 0d040d191..b920398de 100644 --- a/ethereal/src/main/java/com/salesforce/apollo/ethereal/Adder.java +++ b/ethereal/src/main/java/com/salesforce/apollo/ethereal/Adder.java @@ -600,8 +600,8 @@ private boolean decodeParents(Waiting wp) { * Answer the bloom filter with the commits the receiver has */ private Biff haveCommits() { - var bff = new DigestBloomFilter(Entropy.nextBitsStreamLong(), conf.epochLength() * 2 * conf.nProc() * 2, - conf.fpr()); + var n = conf.epochLength() * conf.nProc() * 4; + var bff = new DigestBloomFilter(Entropy.nextBitsStreamLong(), n, 1.0 / ((double) n * 2.0)); signedCommits.keySet().forEach(d -> bff.add(d)); return bff.toBff(); } @@ -610,8 +610,8 @@ private Biff haveCommits() { * Answer the bloom filter with the prevotes the receiver has */ private Biff havePreVotes() { - var bff = new DigestBloomFilter(Entropy.nextBitsStreamLong(), conf.epochLength() * 2 * conf.nProc() * 2, - conf.fpr()); + var n = conf.epochLength() * conf.nProc() * 4; + var bff = new DigestBloomFilter(Entropy.nextBitsStreamLong(), n, 1.0 / ((double) n * 2)); signedPrevotes.keySet().forEach(d -> bff.add(d)); return bff.toBff(); } @@ -620,8 +620,8 @@ private Biff havePreVotes() { * Answer the bloom filter with the units the receiver has */ private Biff haveUnits() { - var bff = new DigestBloomFilter(Entropy.nextBitsStreamLong(), conf.epochLength() * 2 * conf.nProc() * 2, - conf.fpr()); + var n = conf.epochLength() * conf.nProc() * 4; + var bff = new DigestBloomFilter(Entropy.nextBitsStreamLong(), n, 1.0 / ((double) n * 2)); waiting.keySet().forEach(d -> bff.add(d)); dag.have(bff); return bff.toBff(); diff --git a/memberships/src/main/java/com/salesforce/apollo/archipelago/Portal.java b/memberships/src/main/java/com/salesforce/apollo/archipelago/Portal.java index f0a04955a..4c17fb968 100644 --- a/memberships/src/main/java/com/salesforce/apollo/archipelago/Portal.java +++ b/memberships/src/main/java/com/salesforce/apollo/archipelago/Portal.java @@ -21,7 +21,7 @@ import java.io.IOException; import java.time.Duration; -import java.util.concurrent.Executor; +import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.function.Function; @@ -38,12 +38,12 @@ public class Portal { private final static Class channelType = IMPL.getChannelType(); - private final Executor executor = Executors.newVirtualThreadPerTaskExecutor(); - private final String agent; - private final EventLoopGroup eventLoopGroup = IMPL.getEventLoopGroup(); - private final Demultiplexer inbound; - private final Duration keepAlive; - private final Demultiplexer outbound; + private final ExecutorService executor = Executors.newVirtualThreadPerTaskExecutor(); + private final String agent; + private final EventLoopGroup eventLoopGroup = IMPL.getEventLoopGroup(); + private final Demultiplexer inbound; + private final Duration keepAlive; + private final Demultiplexer outbound; public Portal(Digest agent, ServerBuilder inbound, Function outbound, DomainSocketAddress bridge, Duration keepAlive, Function router) { @@ -63,6 +63,7 @@ public Portal(Digest agent, ServerBuilder inbound, Function forReconcilliation() { - var biff = new DigestBloomFilter(Entropy.nextBitsStreamLong(), params.bufferSize, params.falsePositiveRate); + var biff = new DigestBloomFilter(Entropy.nextBitsStreamLong(), params.bufferSize, + 1.0 / ((double) params.bufferSize * 2.0)); state.keySet().forEach(k -> biff.add(k)); return biff; } From 332780960a559f6e087f4b2a56a16f8f889e30aa Mon Sep 17 00:00:00 2001 From: Hellblazer Date: Tue, 4 Jun 2024 15:06:08 -0700 Subject: [PATCH 05/22] Clean and Smooth Join by requiring bft majorities before publish, etc. Clean up pending/etc logic as well. --- .../com/salesforce/apollo/choam/CHOAM.java | 74 +++--- .../com/salesforce/apollo/choam/Producer.java | 22 +- .../salesforce/apollo/choam/ViewAssembly.java | 225 ++++++++++-------- .../salesforce/apollo/choam/ViewContext.java | 3 +- .../salesforce/apollo/choam/fsm/Driven.java | 10 - .../salesforce/apollo/choam/TestCHOAM.java | 11 +- .../apollo/cryptography/Digest.java | 2 +- .../salesforce/apollo/context/Context.java | 33 +-- .../salesforce/apollo/ring/SliceIterator.java | 14 +- 9 files changed, 181 insertions(+), 213 deletions(-) diff --git a/choam/src/main/java/com/salesforce/apollo/choam/CHOAM.java b/choam/src/main/java/com/salesforce/apollo/choam/CHOAM.java index 9cd8de9af..dd2c43281 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/CHOAM.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/CHOAM.java @@ -1393,21 +1393,15 @@ private void join(View view) { } log.info("Joining view: {} diadem: {} on: {}", nextViewId.get(), Digest.from(view.getDiadem()), params.member().getId()); - var servers = new GroupIterator(validators.keySet()); - var joined = new HashSet(); + var servers = new ConcurrentSkipListSet<>(validators.keySet()); var delay = Duration.ofMillis(Entropy.nextSecureInt(5)); + var joined = new AtomicInteger(); Thread.ofPlatform().start(() -> { log.error("Starting join of: {} diadem {} on: {}", nextViewId.get(), Digest.from(view.getDiadem()), params.member().getId()); - while (!joining.isDone() && joined.size() < view.getMajority()) { - try { - Thread.sleep(delay.toMillis()); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - return; - } + while (!joining.isDone() && joined.get() < view.getMajority()) { join(view, servers, joined); } log.info("Finishing join of: {} diadem: {} on: {}", nextViewId.get(), Digest.from(view.getDiadem()), @@ -1416,32 +1410,10 @@ private void join(View view) { }); } - private void join(View view, GroupIterator servers, HashSet joined) { - Member target = servers.next(); - if (joined.contains(target)) { - log.trace("Already joined with: {} view: {} diadem: {} on: {}", target.getId(), nextViewId.get(), - Digest.from(view.getDiadem()), params.member().getId()); - return; - } - try (var link = comm.connect(target)) { - join(view, link, target, joined); - } catch (StatusRuntimeException e) { - log.trace("Failed join attempt with: {} view: {} diadem: {} status:{} on: {}", target.getId(), - nextViewId, Digest.from(view.getDiadem()), e.getStatus(), params.member().getId()); - } catch (Throwable e) { - log.trace("Failed join attempt with: {} view: {} diadem: {} on: {}", target.getId(), nextViewId, - Digest.from(view.getDiadem()), params.member().getId(), e); - } - } + private void join(View view, Collection servers, AtomicInteger joined) { - private void join(View view, Terminal link, Member target, HashSet joined) { - if (link == null) { - log.debug("No link for: {} for joining: {} on: {}", target.getId(), Digest.from(view.getDiadem()), - params.member().getId()); - return; - } - log.trace("Joining view: {} diadem: {} on: {}", viewId, Digest.from(view.getDiadem()), - params.member().getId()); + log.trace("Joining view: {} diadem: {} servers: {} on: {}", viewId, Digest.from(view.getDiadem()), + servers.stream().map(Member::getId).toList(), params.member().getId()); final var c = next.get(); var inView = ViewMember.newBuilder(c.member) .setDiadem(view.getDiadem()) @@ -1451,17 +1423,31 @@ private void join(View view, Terminal link, Member target, HashSet joine .setVm(inView) .setSignature(params.member().sign(inView.toByteString()).toSig()) .build(); + var countdown = new CountDownLatch(servers.size()); + + servers.stream().map(comm::connect).filter(Objects::nonNull).forEach(t -> { + Thread.ofVirtual().start(() -> { + try { + t.join(svm); + servers.remove(t.getMember()); + joined.incrementAndGet(); + countdown.countDown(); + log.trace("Joined with: {} view: {} diadem: {} on: {}", t.getMember().getId(), viewId, + Digest.from(view.getDiadem()), params.member().getId()); + } catch (StatusRuntimeException sre) { + log.trace("Failed join attempt: {} with: {} view: {} diadem: {} on: {}", sre.getStatus(), + t.getMember().getId(), nextViewId, Digest.from(view.getDiadem()), + params.member().getId(), sre); + } catch (Throwable throwable) { + log.trace("Failed join attempt with: {} view: {} diadem: {} on: {}", t.getMember().getId(), + nextViewId, Digest.from(view.getDiadem()), params.member().getId(), throwable); + } + }); + }); try { - link.join(svm); - joined.add(target); - log.trace("Joined with: {} view: {} diadem: {} on: {}", target.getId(), viewId, - Digest.from(view.getDiadem()), params.member().getId()); - } catch (StatusRuntimeException sre) { - log.trace("Failed join attempt: {} with: {} view: {} diadem: {} on: {}", sre.getStatus(), - target.getId(), nextViewId, Digest.from(view.getDiadem()), params.member().getId(), sre); - } catch (Throwable t) { - log.trace("Failed join attempt with: {} view: {} diadem: {} on: {}", target.getId(), nextViewId, - Digest.from(view.getDiadem()), params.member().getId(), t); + countdown.await(2, TimeUnit.SECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); } } } diff --git a/choam/src/main/java/com/salesforce/apollo/choam/Producer.java b/choam/src/main/java/com/salesforce/apollo/choam/Producer.java index 8db4483ca..2f1623a08 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/Producer.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/Producer.java @@ -128,7 +128,7 @@ public boolean complete() { } public void join(SignedViewMember viewMember) { - assembly.join(viewMember, true); + assembly.joined(viewMember); } public void start() { @@ -357,21 +357,22 @@ private void reconfigure() { } private void serial(List preblock, Boolean last) { + try { + serialize.acquire(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return; + } Thread.ofVirtual().start(() -> { try { - serialize.acquire(); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - return; - } - try { - transitions.create(preblock, last); + create(preblock, last); } catch (Throwable t) { log.error("Error processing preblock last: {} on: {}", last, params().member().getId(), t); } finally { serialize.release(); } }); + } private PendingBlock validate(Validate v) { @@ -436,11 +437,6 @@ public void complete() { stop(); } - @Override - public void create(List preblock, boolean last) { - Producer.this.create(preblock, last); - } - @Override public void fail() { stop(); diff --git a/choam/src/main/java/com/salesforce/apollo/choam/ViewAssembly.java b/choam/src/main/java/com/salesforce/apollo/choam/ViewAssembly.java index 96d5cbd37..d922991fb 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/ViewAssembly.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/ViewAssembly.java @@ -59,6 +59,7 @@ public class ViewAssembly { private final AtomicInteger countdown = new AtomicInteger(); private final List pendingJoins = new CopyOnWriteArrayList<>(); private final AtomicBoolean started = new AtomicBoolean(false); + private final Map joins = new ConcurrentHashMap<>(); private volatile Vue selected; public ViewAssembly(Digest nextViewId, ViewContext vc, Consumer publisher, @@ -80,6 +81,26 @@ public Map getSlate() { return slate; } + public void joined(SignedViewMember viewMember) { + final var mid = Digest.from(viewMember.getVm().getId()); + if (!validate(mid, viewMember)) { + return; + } + joins.put(mid, SignedJoin.newBuilder() + .setMember(params().member().getId().toDigeste()) + .setJoin(viewMember) + .setSignature(view.sign(viewMember).toSig()) + .build()); + if (selected != null && joins.size() >= selected.majority) { + publishJoins(); + } else if (selected == null) { + log.trace("Awaiting view selection to publish joins: {} on: {}", joins.size(), params().member().getId()); + } else { + log.trace("Awaiting required majority: {} of: {} to publish joins: {} on: {}", selected.diadem, + selected.majority, joins.size(), params().member().getId()); + } + } + public void start() { if (!started.compareAndSet(false, true)) { return; @@ -95,23 +116,31 @@ void assemble(List asses) { if (asses.isEmpty()) { return; } + var viewz = asses.stream().flatMap(a -> a.getViewsList().stream()).toList(); + var joinz = asses.stream().flatMap(a -> a.getJoinsList().stream()).toList(); + log.debug("Assemblies: {} joins: {} views: {} on: {}", asses.size(), joinz.size(), viewz.size(), + params().member().getId()); - var joins = asses.stream() - .flatMap(a -> a.getJoinsList().stream()) + var joins = joinz.stream() + .filter(SignedJoin::hasJoin) .filter(view -> !proposals.containsKey(Digest.from(view.getJoin().getVm().getId()))) .filter(signedJoin -> !SignedJoin.getDefaultInstance().equals(signedJoin)) .filter(view::validate) .toList(); - var views = asses.stream().flatMap(a -> a.getViewsList().stream()).filter(SignedViews::hasViews).toList(); + if (!joins.isEmpty()) { + log.debug("Assembling joins: {} on: {}", joins.size(), params().member().getId()); + join(joins.stream().map(SignedJoin::getJoin).toList()); + } - log.debug("Assembling joins: {} views: {} on: {}", joins.size(), views.size(), params().member().getId()); + var views = viewz.stream().filter(SignedViews::hasViews).toList(); + if (views.isEmpty()) { + return; + } + log.debug("Assembling views: {} on: {}", views.size(), params().member().getId()); - joins.forEach(sj -> join(sj.getJoin(), false)); if (selected != null) { - if (!views.isEmpty()) { - log.trace("Already selected: {}, ignoring views: {} on: {}", selected.diadem, views.size(), - params().member().getId()); - } + log.trace("Already selected: {}, ignoring views: {} on: {}", selected.diadem, views.size(), + params().member().getId()); return; } views.forEach(svs -> { @@ -170,87 +199,25 @@ boolean complete() { return true; } - void join(SignedViewMember svm, boolean direct) { + void join(List joins) { if (!started.get()) { return; } - - final var mid = Digest.from(svm.getVm().getId()); - if (proposals.containsKey(mid)) { - log.trace("Redundant join from: {} on: {}", print(svm, params().digestAlgorithm()), - params().member().getId()); - return; - } if (selected == null) { - pendingJoins.add(svm); - log.trace("Pending join from: {} on: {}", print(svm, params().digestAlgorithm()), - params().member().getId()); + pendingJoins.addAll(joins); + log.trace("Pending joins: {} on: {}", joins.size(), params().member().getId()); return; } - final var m = selected.assembly.get(mid); - if (m == null) { - if (log.isTraceEnabled()) { - log.trace("Invalid view member: {} on: {}", print(svm, params().digestAlgorithm()), + for (var svm : joins) { + final var mid = Digest.from(svm.getVm().getId()); + if (proposals.containsKey(mid)) { + log.trace("Redundant join from: {} on: {}", print(svm, params().digestAlgorithm()), params().member().getId()); + continue; } - return; - } - var viewId = Digest.from(svm.getVm().getView()); - if (!nextViewId.equals(viewId)) { - if (log.isTraceEnabled()) { - log.trace("Invalid view id for member: {} on: {}", print(svm, params().digestAlgorithm()), - params().member().getId()); + if (validate(mid, svm)) { + proposals.put(mid, svm); } - return; - } - if (log.isDebugEnabled()) { - log.debug("Join of: {} on: {}", print(svm, params().digestAlgorithm()), params().member().getId()); - } - - if (!m.verify(JohnHancock.from(svm.getSignature()), svm.getVm().toByteString())) { - if (log.isTraceEnabled()) { - log.trace("Invalid signature for view member: {} on: {}", print(svm, params().digestAlgorithm()), - params().member().getId()); - } - return; - } - - PubKey encoded = svm.getVm().getConsensusKey(); - - if (!m.verify(signature(svm.getVm().getSignature()), encoded.toByteString())) { - if (log.isTraceEnabled()) { - log.trace("Could not verify consensus key from view member: {} on: {}", - print(svm, params().digestAlgorithm()), params().member().getId()); - } - return; - } - - PublicKey consensusKey = publicKey(encoded); - if (consensusKey == null) { - if (log.isTraceEnabled()) { - log.trace("Could not deserialize consensus key from view member: {} on: {}", - print(svm, params().digestAlgorithm()), params().member().getId()); - } - return; - } - - if (direct) { - var signature = view.sign(svm); - publisher.accept(Assemblies.newBuilder() - .addJoins(SignedJoin.newBuilder() - .setJoin(svm) - .setMember(params().member().getId().toDigeste()) - .setSignature(signature.toSig()) - .build()) - .build()); - if (log.isTraceEnabled()) { - log.trace("Publishing view member: {} sig: {} on: {}", print(svm, params().digestAlgorithm()), - params().digestAlgorithm().digest(signature.toSig().toByteString()), - params().member().getId()); - } - } else if (proposals.putIfAbsent(mid, svm) == null) { - log.trace("Adding discovered view member: {} on: {}", print(svm, params().digestAlgorithm()), - params().member().getId()); } checkAssembly(); } @@ -271,6 +238,9 @@ private Map assemblyOf(List committee) { } private void checkAssembly() { + if (selected == null) { + return; + } if (proposals.size() == selected.majority) { transitions.certified(); } else if (proposals.size() >= selected.majority) { @@ -282,6 +252,24 @@ private Parameters params() { return view.params(); } + private void propose(Views vs, List majorities, Multiset consensus) { + var ordered = vs.getViewsList().stream().map(v -> Digest.from(v.getDiadem())).toList(); + var lastIndex = -1; + View last = null; + for (var v : majorities) { + var i = ordered.indexOf(Digest.from(v.getDiadem())); + if (i != -1) { + if (i > lastIndex) { + last = v; + lastIndex = i; + } + } + } + if (last != null) { + consensus.add(last); + } + } + private void propose() { var views = view.pendingViews() .getViews(nextViewId) @@ -297,22 +285,62 @@ private void propose() { .build()); } - private void propose(Views vs, List majorities, Multiset consensus) { - var ordered = vs.getViewsList().stream().map(v -> Digest.from(v.getDiadem())).toList(); - var lastIndex = -1; - View last = null; - for (var v : majorities) { - var i = ordered.indexOf(Digest.from(v.getDiadem())); - if (i != -1) { - if (i > lastIndex) { - last = v; - lastIndex = i; - } + private void publishJoins() { + log.trace("Publish joins: {} on: {}", joins.size(), params().member().getId()); + var b = Assemblies.newBuilder(); + joins.values().forEach(b::addJoins); + publisher.accept(b.build()); + } + + private boolean validate(Digest mid, SignedViewMember svm) { + final var m = selected.assembly.get(mid); + if (m == null) { + if (log.isTraceEnabled()) { + log.trace("Invalid view member: {} on: {}", print(svm, params().digestAlgorithm()), + params().member().getId()); } + return false; } - if (last != null) { - consensus.add(last); + var viewId = Digest.from(svm.getVm().getView()); + if (!nextViewId.equals(viewId)) { + if (log.isTraceEnabled()) { + log.trace("Invalid view id for member: {} on: {}", print(svm, params().digestAlgorithm()), + params().member().getId()); + } + return false; + } + if (log.isDebugEnabled()) { + log.debug("Join of: {} on: {}", print(svm, params().digestAlgorithm()), params().member().getId()); } + + if (!m.verify(JohnHancock.from(svm.getSignature()), svm.getVm().toByteString())) { + if (log.isTraceEnabled()) { + log.trace("Invalid signature for view member: {} on: {}", print(svm, params().digestAlgorithm()), + params().member().getId()); + } + return false; + } + + PubKey encoded = svm.getVm().getConsensusKey(); + + if (!m.verify(signature(svm.getVm().getSignature()), encoded.toByteString())) { + if (log.isTraceEnabled()) { + log.trace("Could not verify consensus key from view member: {} on: {}", + print(svm, params().digestAlgorithm()), params().member().getId()); + } + return false; + } + + PublicKey consensusKey = publicKey(encoded); + if (consensusKey == null) { + if (log.isTraceEnabled()) { + log.trace("Could not deserialize consensus key from view member: {} on: {}", + print(svm, params().digestAlgorithm()), params().member().getId()); + } + return false; + } + log.trace("Validated svm: {} on: {}", print(svm, params().digestAlgorithm()), params().member().getId()); + return true; } private void vote() { @@ -359,7 +387,10 @@ private void vote() { } onConsensus.complete(selected); transitions.viewAcquired(); - pendingJoins.forEach(svm -> join(svm, false)); + if (joins.size() >= selected.majority) { + publishJoins(); + } + join(pendingJoins); pendingJoins.clear(); } @@ -412,9 +443,5 @@ public void finish() { public void publishViews() { propose(); } - - private Join joinOf(SignedViewMember vm) { - return Join.newBuilder().setMember(vm).build(); - } } } diff --git a/choam/src/main/java/com/salesforce/apollo/choam/ViewContext.java b/choam/src/main/java/com/salesforce/apollo/choam/ViewContext.java index 6b84839e7..46a0c9d71 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/ViewContext.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/ViewContext.java @@ -278,8 +278,7 @@ protected Verifier verifierOf(SignedViews sv) { private Verifier getVerifier(Member m) { if (m == null) { if (log.isDebugEnabled()) { - log.debug("Unable to get verifier by non existent member: {} on: {}", m.getId(), - params.member().getId()); + log.debug("Unable to get verifier by non existent member on: {}", params.member().getId()); } return null; } diff --git a/choam/src/main/java/com/salesforce/apollo/choam/fsm/Driven.java b/choam/src/main/java/com/salesforce/apollo/choam/fsm/Driven.java index 9b290ed26..2cf86b242 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/fsm/Driven.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/fsm/Driven.java @@ -8,12 +8,9 @@ import com.chiralbehaviors.tron.Entry; import com.chiralbehaviors.tron.FsmExecutor; -import com.google.protobuf.ByteString; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.List; - /** * Leaf action interface for the Producer FSM * @@ -27,8 +24,6 @@ public interface Driven { void complete(); - void create(List preblock, boolean last); - void fail(); void reconfigure(); @@ -149,11 +144,6 @@ default Transitions checkpointed() { throw fsm().invalidTransitionOn(); } - default Transitions create(List preblock, boolean last) { - context().create(preblock, last); - return null; - } - default Transitions establish() { throw fsm().invalidTransitionOn(); } diff --git a/choam/src/test/java/com/salesforce/apollo/choam/TestCHOAM.java b/choam/src/test/java/com/salesforce/apollo/choam/TestCHOAM.java index 0769d13de..391706bf2 100644 --- a/choam/src/test/java/com/salesforce/apollo/choam/TestCHOAM.java +++ b/choam/src/test/java/com/salesforce/apollo/choam/TestCHOAM.java @@ -6,6 +6,7 @@ */ package com.salesforce.apollo.choam; +import com.codahale.metrics.ConsoleReporter; import com.codahale.metrics.MetricRegistry; import com.salesforce.apollo.archipelago.LocalServer; import com.salesforce.apollo.archipelago.Router; @@ -201,11 +202,11 @@ public void submitMultiplTxn() throws Exception { System.out.println(); - // ConsoleReporter.forRegistry(registry) - // .convertRatesTo(TimeUnit.SECONDS) - // .convertDurationsTo(TimeUnit.MILLISECONDS) - // .build() - // .report(); + ConsoleReporter.forRegistry(registry) + .convertRatesTo(TimeUnit.SECONDS) + .convertDurationsTo(TimeUnit.MILLISECONDS) + .build() + .report(); } assertTrue(checkpointOccurred.get(5, TimeUnit.SECONDS)); } diff --git a/cryptography/src/main/java/com/salesforce/apollo/cryptography/Digest.java b/cryptography/src/main/java/com/salesforce/apollo/cryptography/Digest.java index de61d2eb4..2c1f7761d 100644 --- a/cryptography/src/main/java/com/salesforce/apollo/cryptography/Digest.java +++ b/cryptography/src/main/java/com/salesforce/apollo/cryptography/Digest.java @@ -124,7 +124,7 @@ public int compareTo(Digest id) { return 0; } if (hash.length != id.hash.length) { - throw new IllegalArgumentException("hash length incorrect for algorithm"); + return -1; } for (int i = 0; i < hash.length; i++) { int compare = Long.compareUnsigned(hash[i], id.hash[i]); diff --git a/memberships/src/main/java/com/salesforce/apollo/context/Context.java b/memberships/src/main/java/com/salesforce/apollo/context/Context.java index b1038e4db..ec82cd7dd 100644 --- a/memberships/src/main/java/com/salesforce/apollo/context/Context.java +++ b/memberships/src/main/java/com/salesforce/apollo/context/Context.java @@ -95,15 +95,11 @@ static int minMajority(int bias, double pByz, int cardinality) { Stream allMembers(); /** - * @param start - * @param stop * @return Return all counter-clockwise items between (but not including) start and stop */ Iterable betweenPredecessors(int ring, T start, T stop); /** - * @param start - * @param stop * @return all clockwise items between (but not including) start item and stop item. */ Iterable betweenSuccessor(int ring, T start, T stop); @@ -199,8 +195,6 @@ default int diameter() { T getMember(Digest memberID); /** - * @param i - * @param ring * @return the i'th Member in Ring 0 of the receiver */ T getMember(int i, int ring); @@ -253,10 +247,6 @@ default Digest hashFor(Digest d, int ring) { /** * Answer true if the member is a successor of the supplied digest on any ring - * - * @param m - * @param digest - * @return */ boolean isSuccessorOf(T m, Digest digest); @@ -317,8 +307,6 @@ default int majority() { Iterable predecessors(int ring, Digest location); /** - * @param location - * @param predicate * @return an Iterable of all items counter-clock wise in the ring from (but excluding) start location to (but * excluding) the first item where predicate(item) evaluates to True. */ @@ -327,8 +315,6 @@ default int majority() { Iterable predecessors(int ring, T start); /** - * @param start - * @param predicate * @return an Iterable of all items counter-clock wise in the ring from (but excluding) start location to (but * excluding) the first item where predicate(item) evaluates to True. */ @@ -376,46 +362,34 @@ default int majority() { /** * Stream the members of the ring in hashed order - * - * @param ring - * @return */ Stream stream(int ring); /** - * @param ring - * @param predicate * @return a Stream of all items counter-clock wise in the ring from (but excluding) start location to (but * excluding) the first item where predicate(item) evaluates to True. */ Stream streamPredecessors(int ring, Digest location, Predicate predicate); /** - * @param ring - * @param predicate * @return a list of all items counter-clock wise in the ring from (but excluding) start item to (but excluding) the * first item where predicate(item) evaluates to True. */ Stream streamPredecessors(int ring, T m, Predicate predicate); /** - * @param ring - * @param predicate * @return a Stream of all items counter-clock wise in the ring from (but excluding) start location to (but * excluding) the first item where predicate(item) evaluates to True. */ Stream streamSuccessors(int ring, Digest location, Predicate predicate); /** - * @param ring - * @param predicate * @return a Stream of all items counter-clock wise in the ring from (but excluding) start item to (but excluding) * the first item where predicate(item) evaluates to True. */ Stream streamSuccessors(int ring, T m, Predicate predicate); /** - * @param ring * @return a iterable of all items counter-clock wise in the ring from (but excluding) start location to (but * excluding) the first item */ @@ -490,16 +464,12 @@ default List> successors(Digest digest, T ignore } /** - * @param ring - * @param predicate * @return an Iterable of all items counter-clock wise in the ring from (but excluding) start location to (but * excluding) the first item where predicate(item) evaluates to True. */ Iterable successors(int ring, Digest location, Predicate predicate); /** - * @param ring - * @param predicate * @return an Iterable of all items counter-clock wise in the ring from (but excluding) start item to (but * excluding) the first item where predicate(item) evaluates to True. */ @@ -522,8 +492,7 @@ default int toleranceLevel() { } /** - * @param member - * @return the iteratator to traverse the ring starting at the member + * @return the iterator to traverse the ring starting at the member */ Iterable traverse(int ring, T member); diff --git a/memberships/src/main/java/com/salesforce/apollo/ring/SliceIterator.java b/memberships/src/main/java/com/salesforce/apollo/ring/SliceIterator.java index 137db0a44..ba283f25f 100644 --- a/memberships/src/main/java/com/salesforce/apollo/ring/SliceIterator.java +++ b/memberships/src/main/java/com/salesforce/apollo/ring/SliceIterator.java @@ -32,13 +32,13 @@ public class SliceIterator { private static final Logger log = LoggerFactory.getLogger(SliceIterator.class); - private final CommonCommunications comm; - private final String label; - private final SigningMember member; - private final List slice; - private final ScheduledExecutorService scheduler; - private Member current; - private Iterator currentIteration; + private final CommonCommunications comm; + private final String label; + private final SigningMember member; + private final List slice; + private final ScheduledExecutorService scheduler; + private volatile Member current; + private volatile Iterator currentIteration; public SliceIterator(String label, SigningMember member, Collection slice, CommonCommunications comm) { From 310321bb9640460c5f80211990a338f9ce94875e Mon Sep 17 00:00:00 2001 From: Hellblazer Date: Tue, 4 Jun 2024 16:19:29 -0700 Subject: [PATCH 06/22] will it blend? --- choam/pom.xml | 11 +++++++++++ .../java/com/salesforce/apollo/choam/DynamicTest.java | 4 +++- .../com/salesforce/apollo/choam/MembershipTests.java | 4 +++- .../java/com/salesforce/apollo/choam/TestCHOAM.java | 8 +++----- .../com/salesforce/apollo/fireflies/ChurnTest.java | 5 +++-- .../com/salesforce/apollo/fireflies/SwarmTest.java | 5 +++-- model/pom.xml | 7 +++++++ .../apollo/model/ContainmentDomainTest.java | 3 ++- .../java/com/salesforce/apollo/model/DomainTest.java | 3 ++- .../com/salesforce/apollo/model/FireFliesTest.java | 3 ++- pom.xml | 2 +- .../apollo/state/AbstractLifecycleTest.java | 5 ++++- .../java/com/salesforce/apollo/state/CHOAMTest.java | 5 ++++- 13 files changed, 48 insertions(+), 17 deletions(-) diff --git a/choam/pom.xml b/choam/pom.xml index d4c07c80e..6ee71890c 100644 --- a/choam/pom.xml +++ b/choam/pom.xml @@ -53,4 +53,15 @@ test + + + + org.apache.maven.plugins + maven-surefire-plugin + + false + + + + diff --git a/choam/src/test/java/com/salesforce/apollo/choam/DynamicTest.java b/choam/src/test/java/com/salesforce/apollo/choam/DynamicTest.java index fe29a7e45..ebd508af6 100644 --- a/choam/src/test/java/com/salesforce/apollo/choam/DynamicTest.java +++ b/choam/src/test/java/com/salesforce/apollo/choam/DynamicTest.java @@ -3,6 +3,7 @@ import com.salesforce.apollo.archipelago.LocalServer; import com.salesforce.apollo.archipelago.Router; import com.salesforce.apollo.archipelago.ServerConnectionCache; +import com.salesforce.apollo.archipelago.UnsafeExecutors; import com.salesforce.apollo.choam.support.ExponentialBackoffPolicy; import com.salesforce.apollo.context.Context; import com.salesforce.apollo.context.DynamicContext; @@ -63,9 +64,10 @@ public void setUp() throws Exception { .toList(); final var prefix = UUID.randomUUID().toString(); + var executor = UnsafeExecutors.newVirtualThreadPerTaskExecutor(); routers = members.stream() .collect(Collectors.toMap(m -> m, m -> new LocalServer(prefix, m).router( - ServerConnectionCache.newBuilder().setTarget(cardinality * 2)))); + ServerConnectionCache.newBuilder().setTarget(cardinality * 2), executor))); var template = Parameters.newBuilder() .setGenerateGenesis(true) diff --git a/choam/src/test/java/com/salesforce/apollo/choam/MembershipTests.java b/choam/src/test/java/com/salesforce/apollo/choam/MembershipTests.java index effd854f5..febe8f415 100644 --- a/choam/src/test/java/com/salesforce/apollo/choam/MembershipTests.java +++ b/choam/src/test/java/com/salesforce/apollo/choam/MembershipTests.java @@ -9,6 +9,7 @@ import com.salesforce.apollo.archipelago.LocalServer; import com.salesforce.apollo.archipelago.Router; import com.salesforce.apollo.archipelago.ServerConnectionCache; +import com.salesforce.apollo.archipelago.UnsafeExecutors; import com.salesforce.apollo.choam.CHOAM.TransactionExecutor; import com.salesforce.apollo.choam.Parameters.BootstrapParameters; import com.salesforce.apollo.choam.Parameters.ProducerParameters; @@ -170,9 +171,10 @@ public SigningMember initialize(int checkpointBlockSize, int cardinality) throws SigningMember testSubject = new ControlledIdentifierMember(stereotomy.newIdentifier()); final var prefix = UUID.randomUUID().toString(); + var executor = UnsafeExecutors.newVirtualThreadPerTaskExecutor(); routers = members.stream() .collect(Collectors.toMap(Member::getId, m -> new LocalServer(prefix, m).router( - ServerConnectionCache.newBuilder().setTarget(cardinality)))); + ServerConnectionCache.newBuilder().setTarget(cardinality), executor))); routers.put(testSubject.getId(), new LocalServer(prefix, testSubject).router( ServerConnectionCache.newBuilder().setTarget(cardinality))); choams = new HashMap<>(); diff --git a/choam/src/test/java/com/salesforce/apollo/choam/TestCHOAM.java b/choam/src/test/java/com/salesforce/apollo/choam/TestCHOAM.java index 391706bf2..66d941748 100644 --- a/choam/src/test/java/com/salesforce/apollo/choam/TestCHOAM.java +++ b/choam/src/test/java/com/salesforce/apollo/choam/TestCHOAM.java @@ -8,10 +8,7 @@ import com.codahale.metrics.ConsoleReporter; import com.codahale.metrics.MetricRegistry; -import com.salesforce.apollo.archipelago.LocalServer; -import com.salesforce.apollo.archipelago.Router; -import com.salesforce.apollo.archipelago.ServerConnectionCache; -import com.salesforce.apollo.archipelago.ServerConnectionCacheMetricsImpl; +import com.salesforce.apollo.archipelago.*; import com.salesforce.apollo.choam.CHOAM.TransactionExecutor; import com.salesforce.apollo.choam.Parameters.ProducerParameters; import com.salesforce.apollo.choam.Parameters.RuntimeParameters; @@ -124,11 +121,12 @@ public void before() throws Exception { .toList(); var context = new StaticContext<>(origin, 0.2, members, 3); final var prefix = UUID.randomUUID().toString(); + var executor = UnsafeExecutors.newVirtualThreadPerTaskExecutor(); routers = members.stream() .collect(Collectors.toMap(m -> m.getId(), m -> new LocalServer(prefix, m).router( ServerConnectionCache.newBuilder() .setMetrics(new ServerConnectionCacheMetricsImpl(registry)) - .setTarget(CARDINALITY)))); + .setTarget(CARDINALITY), executor))); choams = members.stream().collect(Collectors.toMap(m -> m.getId(), m -> { var recording = new AtomicInteger(); blocks.put(m.getId(), recording); diff --git a/fireflies/src/test/java/com/salesforce/apollo/fireflies/ChurnTest.java b/fireflies/src/test/java/com/salesforce/apollo/fireflies/ChurnTest.java index 9825e45f6..dcfc99b4f 100644 --- a/fireflies/src/test/java/com/salesforce/apollo/fireflies/ChurnTest.java +++ b/fireflies/src/test/java/com/salesforce/apollo/fireflies/ChurnTest.java @@ -273,6 +273,7 @@ private void initialize() { AtomicBoolean frist = new AtomicBoolean(true); final var prefix = UUID.randomUUID().toString(); final var gatewayPrefix = UUID.randomUUID().toString(); + var executor = UnsafeExecutors.newVirtualThreadPerTaskExecutor(); views = members.values().stream().map(node -> { DynamicContext context = ctxBuilder.build(); FireflyMetricsImpl metrics = new FireflyMetricsImpl(context.getId(), @@ -288,8 +289,8 @@ private void initialize() { .setMetrics( new ServerConnectionCacheMetricsImpl( frist.getAndSet(false) - ? node0Registry - : registry))); + ? node0Registry : registry)), + executor); comms.start(); communications.add(comms); diff --git a/fireflies/src/test/java/com/salesforce/apollo/fireflies/SwarmTest.java b/fireflies/src/test/java/com/salesforce/apollo/fireflies/SwarmTest.java index 0e4f93dc6..b18c32a3e 100644 --- a/fireflies/src/test/java/com/salesforce/apollo/fireflies/SwarmTest.java +++ b/fireflies/src/test/java/com/salesforce/apollo/fireflies/SwarmTest.java @@ -226,6 +226,7 @@ private void initialize() { AtomicBoolean frist = new AtomicBoolean(true); final var prefix = UUID.randomUUID().toString(); final var gatewayPrefix = UUID.randomUUID().toString(); + var executor = UnsafeExecutors.newVirtualThreadPerTaskExecutor(); views = members.values().stream().map(node -> { DynamicContext context = ctxBuilder.build(); FireflyMetricsImpl metrics = new FireflyMetricsImpl(context.getId(), @@ -241,8 +242,8 @@ private void initialize() { .setMetrics( new ServerConnectionCacheMetricsImpl( frist.getAndSet(false) - ? node0Registry - : registry))); + ? node0Registry : registry)), + executor); comms.start(); communications.add(comms); diff --git a/model/pom.xml b/model/pom.xml index 3b9acd54a..2db95ab73 100644 --- a/model/pom.xml +++ b/model/pom.xml @@ -76,6 +76,13 @@ + + org.apache.maven.plugins + maven-surefire-plugin + + false + + org.apache.maven.plugins maven-antrun-plugin diff --git a/model/src/test/java/com/salesforce/apollo/model/ContainmentDomainTest.java b/model/src/test/java/com/salesforce/apollo/model/ContainmentDomainTest.java index 7b042156b..8df39f7d0 100644 --- a/model/src/test/java/com/salesforce/apollo/model/ContainmentDomainTest.java +++ b/model/src/test/java/com/salesforce/apollo/model/ContainmentDomainTest.java @@ -79,10 +79,11 @@ public void before() throws Exception { var sealed = FoundationSeal.newBuilder().build(); final var group = DigestAlgorithm.DEFAULT.getOrigin(); + var executor = UnsafeExecutors.newVirtualThreadPerTaskExecutor(); identities.forEach((d, id) -> { final var member = new ControlledIdentifierMember(id); var localRouter = new LocalServer(prefix, member).router(ServerConnectionCache.newBuilder().setTarget(30), - UnsafeExecutors.newVirtualThreadPerTaskExecutor()); + executor); routers.add(localRouter); var dbUrl = String.format("jdbc:h2:mem:sql-%s-%s;DB_CLOSE_DELAY=-1", member.getId(), UUID.randomUUID()); var pdParams = new ProcessDomain.ProcessDomainParameters(dbUrl, Duration.ofMinutes(1), diff --git a/model/src/test/java/com/salesforce/apollo/model/DomainTest.java b/model/src/test/java/com/salesforce/apollo/model/DomainTest.java index ce361beae..3cb29dd42 100644 --- a/model/src/test/java/com/salesforce/apollo/model/DomainTest.java +++ b/model/src/test/java/com/salesforce/apollo/model/DomainTest.java @@ -235,10 +235,11 @@ public void before() throws Exception { var sealed = FoundationSeal.newBuilder().build(); final var group = DigestAlgorithm.DEFAULT.getOrigin(); + var executor = UnsafeExecutors.newVirtualThreadPerTaskExecutor(); identities.forEach((d, id) -> { final var member = new ControlledIdentifierMember(id); var localRouter = new LocalServer(prefix, member).router(ServerConnectionCache.newBuilder().setTarget(30), - UnsafeExecutors.newVirtualThreadPerTaskExecutor()); + executor); routers.add(localRouter); var dbUrl = String.format("jdbc:h2:mem:sql-%s-%s;DB_CLOSE_DELAY=-1", member.getId(), UUID.randomUUID()); var pdParams = new ProcessDomain.ProcessDomainParameters(dbUrl, Duration.ofMinutes(1), diff --git a/model/src/test/java/com/salesforce/apollo/model/FireFliesTest.java b/model/src/test/java/com/salesforce/apollo/model/FireFliesTest.java index 4161111a9..c6a48aa41 100644 --- a/model/src/test/java/com/salesforce/apollo/model/FireFliesTest.java +++ b/model/src/test/java/com/salesforce/apollo/model/FireFliesTest.java @@ -78,11 +78,12 @@ public void before() throws Exception { Digest group = DigestAlgorithm.DEFAULT.getOrigin(); var sealed = FoundationSeal.newBuilder().build(); + var executor = UnsafeExecutors.newVirtualThreadPerTaskExecutor(); identities.forEach((digest, id) -> { var context = new DynamicContextImpl<>(DigestAlgorithm.DEFAULT.getLast(), CARDINALITY, 0.2, 3); final var member = new ControlledIdentifierMember(id); var localRouter = new LocalServer(prefix, member).router(ServerConnectionCache.newBuilder().setTarget(30), - UnsafeExecutors.newVirtualThreadPerTaskExecutor()); + executor); var dbUrl = String.format("jdbc:h2:mem:sql-%s-%s;DB_CLOSE_DELAY=-1", member.getId(), UUID.randomUUID()); var pdParams = new ProcessDomain.ProcessDomainParameters(dbUrl, Duration.ofSeconds(5), "jdbc:h2:mem:%s-state".formatted(digest), diff --git a/pom.xml b/pom.xml index 557face93..c4879bd2b 100644 --- a/pom.xml +++ b/pom.xml @@ -783,7 +783,7 @@ 3.2.5 ${forks} - false + true -Xmx10G -Xms4G -Djdk.tracePinnedThreads=full diff --git a/sql-state/src/test/java/com/salesforce/apollo/state/AbstractLifecycleTest.java b/sql-state/src/test/java/com/salesforce/apollo/state/AbstractLifecycleTest.java index 028f4b67b..cc2d78819 100644 --- a/sql-state/src/test/java/com/salesforce/apollo/state/AbstractLifecycleTest.java +++ b/sql-state/src/test/java/com/salesforce/apollo/state/AbstractLifecycleTest.java @@ -9,6 +9,7 @@ import com.salesforce.apollo.archipelago.LocalServer; import com.salesforce.apollo.archipelago.Router; import com.salesforce.apollo.archipelago.ServerConnectionCache; +import com.salesforce.apollo.archipelago.UnsafeExecutors; import com.salesforce.apollo.choam.CHOAM; import com.salesforce.apollo.choam.Parameters; import com.salesforce.apollo.choam.Parameters.BootstrapParameters; @@ -157,8 +158,10 @@ public void before() throws Exception { members.stream().filter(s -> s != testSubject).forEach(s -> context.activate(s)); final var prefix = UUID.randomUUID().toString(); + var executor = UnsafeExecutors.newVirtualThreadPerTaskExecutor(); routers = members.stream().collect(Collectors.toMap(m -> m.getId(), m -> { - var localRouter = new LocalServer(prefix, m).router(ServerConnectionCache.newBuilder().setTarget(30)); + var localRouter = new LocalServer(prefix, m).router(ServerConnectionCache.newBuilder().setTarget(30), + executor); return localRouter; })); routers.put(testSubject.getId(), diff --git a/sql-state/src/test/java/com/salesforce/apollo/state/CHOAMTest.java b/sql-state/src/test/java/com/salesforce/apollo/state/CHOAMTest.java index e8cb71761..e3a526a70 100644 --- a/sql-state/src/test/java/com/salesforce/apollo/state/CHOAMTest.java +++ b/sql-state/src/test/java/com/salesforce/apollo/state/CHOAMTest.java @@ -11,6 +11,7 @@ import com.salesforce.apollo.archipelago.LocalServer; import com.salesforce.apollo.archipelago.Router; import com.salesforce.apollo.archipelago.ServerConnectionCache; +import com.salesforce.apollo.archipelago.UnsafeExecutors; import com.salesforce.apollo.choam.CHOAM; import com.salesforce.apollo.choam.CHOAM.TransactionExecutor; import com.salesforce.apollo.choam.Parameters; @@ -154,8 +155,10 @@ public void before() throws Exception { }).map(cpk -> new ControlledIdentifierMember(cpk)).map(e -> (SigningMember) e).toList(); members.forEach(m -> context.activate(m)); final var prefix = UUID.randomUUID().toString(); + var executor = UnsafeExecutors.newVirtualThreadPerTaskExecutor(); routers = members.stream().collect(Collectors.toMap(m -> m.getId(), m -> { - var localRouter = new LocalServer(prefix, m).router(ServerConnectionCache.newBuilder().setTarget(30)); + var localRouter = new LocalServer(prefix, m).router(ServerConnectionCache.newBuilder().setTarget(30), + executor); return localRouter; })); choams = members.stream() From 1d540a6485c33044f20a4f4500f08f342eccf1fb Mon Sep 17 00:00:00 2001 From: Hellblazer Date: Wed, 5 Jun 2024 08:08:25 -0700 Subject: [PATCH 07/22] cleaner committee join, revert ReliableBroadcaster fpr changes The bloom window needs a very low FPR --- .../com/salesforce/apollo/choam/CHOAM.java | 47 +++++++++---------- .../com/salesforce/apollo/choam/Producer.java | 6 +-- choam/src/test/resources/logback-test.xml | 2 +- .../messaging/rbc/ReliableBroadcaster.java | 6 +-- pom.xml | 4 +- 5 files changed, 30 insertions(+), 35 deletions(-) diff --git a/choam/src/main/java/com/salesforce/apollo/choam/CHOAM.java b/choam/src/main/java/com/salesforce/apollo/choam/CHOAM.java index dd2c43281..449305faf 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/CHOAM.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/CHOAM.java @@ -35,7 +35,6 @@ import com.salesforce.apollo.membership.messaging.rbc.ReliableBroadcaster.MessageAdapter; import com.salesforce.apollo.membership.messaging.rbc.ReliableBroadcaster.Msg; import com.salesforce.apollo.messaging.proto.AgedMessageOrBuilder; -import com.salesforce.apollo.utils.Entropy; import com.salesforce.apollo.utils.Utils; import io.grpc.StatusRuntimeException; import org.h2.mvstore.MVMap; @@ -47,7 +46,6 @@ import java.io.FileInputStream; import java.io.IOException; import java.security.KeyPair; -import java.time.Duration; import java.util.*; import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicBoolean; @@ -99,7 +97,6 @@ public class CHOAM { private final TransSubmission txnSubmission = new TransSubmission(); private final AtomicReference view = new AtomicReference<>(); private final PendingViews pendingViews = new PendingViews(); - private final AtomicReference> join = new AtomicReference<>(); public CHOAM(Parameters params) { this.store = new Store(params.digestAlgorithm(), params.mvBuilder().clone().build()); @@ -711,10 +708,6 @@ private void process() { private void reconfigure(Digest hash, Reconfigure reconfigure) { log.info("Setting next view id: {} on: {}", hash, params.member().getId()); - var j = join.getAndSet(null); - if (j != null) { - j.cancel(true); - } nextViewId.set(hash); var pv = pendingViews.advance(); if (pv != null) { @@ -1282,10 +1275,10 @@ public Initial sync(Synchronize request, Digest from) { /** abstract class to maintain the common state */ private abstract class Administration implements Committee { - protected final Digest viewId; - - private final GroupIterator servers; - private final Map validators; + protected final Digest viewId; + private final GroupIterator servers; + private final Map validators; + private volatile JoinState ongoingJoin; public Administration(Map validators, Digest viewId) { this.validators = validators; @@ -1295,6 +1288,12 @@ public Administration(Map validators, Digest viewId) { @Override public void accept(HashedCertifiedBlock hb) { + final var oj = ongoingJoin; + ongoingJoin = null; + if (oj != null) { + oj.halt.set(true); + oj.joining.interrupt(); + } process(); } @@ -1384,30 +1383,25 @@ public boolean validate(HashedCertifiedBlock hb) { } private void join(View view) { - var joining = new CompletableFuture(); - if (!join.compareAndSet(null, joining)) { - log.info("Ongoing join of: {} should have been cancelled on: {}", Digest.from(view.getDiadem()), - params.member().getId()); - transitions.fail(); - return; + if (ongoingJoin != null) { + throw new IllegalStateException("Ongoing join should have been cancelled"); } log.info("Joining view: {} diadem: {} on: {}", nextViewId.get(), Digest.from(view.getDiadem()), params.member().getId()); var servers = new ConcurrentSkipListSet<>(validators.keySet()); - - var delay = Duration.ofMillis(Entropy.nextSecureInt(5)); var joined = new AtomicInteger(); + var halt = new AtomicBoolean(false); - Thread.ofPlatform().start(() -> { + ongoingJoin = new JoinState(halt, Thread.ofVirtual().start(Utils.wrapped(() -> { log.error("Starting join of: {} diadem {} on: {}", nextViewId.get(), Digest.from(view.getDiadem()), params.member().getId()); - while (!joining.isDone() && joined.get() < view.getMajority()) { + while (!halt.get() & joined.get() < view.getMajority()) { join(view, servers, joined); } log.info("Finishing join of: {} diadem: {} on: {}", nextViewId.get(), Digest.from(view.getDiadem()), params.member().getId()); - joining.complete(null); - }); + ongoingJoin = null; + }, log()))); } private void join(View view, Collection servers, AtomicInteger joined) { @@ -1426,7 +1420,7 @@ private void join(View view, Collection servers, AtomicInteger joined) { var countdown = new CountDownLatch(servers.size()); servers.stream().map(comm::connect).filter(Objects::nonNull).forEach(t -> { - Thread.ofVirtual().start(() -> { + Thread.ofVirtual().start(Utils.wrapped(() -> { try { t.join(svm); servers.remove(t.getMember()); @@ -1442,7 +1436,7 @@ private void join(View view, Collection servers, AtomicInteger joined) { log.trace("Failed join attempt with: {} view: {} diadem: {} on: {}", t.getMember().getId(), nextViewId, Digest.from(view.getDiadem()), params.member().getId(), throwable); } - }); + }, log())); }); try { countdown.await(2, TimeUnit.SECONDS); @@ -1450,6 +1444,9 @@ private void join(View view, Collection servers, AtomicInteger joined) { Thread.currentThread().interrupt(); } } + + private record JoinState(AtomicBoolean halt, Thread joining) { + } } /** a member of the current committee */ diff --git a/choam/src/main/java/com/salesforce/apollo/choam/Producer.java b/choam/src/main/java/com/salesforce/apollo/choam/Producer.java index 2f1623a08..19bc5580f 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/Producer.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/Producer.java @@ -234,7 +234,7 @@ private Parameters params() { private void processAssemblies(List aggregate) { var aggs = aggregate.stream().flatMap(e -> e.getAssembliesList().stream()).toList(); - log.trace("Consuming {} assemblies from {} units on: {}", aggs.size(), aggregate.size(), + log.trace("Consuming: {} assemblies from: {} units on: {}", aggs.size(), aggregate.size(), params().member().getId()); assembly.assemble(aggs); } @@ -313,14 +313,14 @@ private void publish(PendingBlock p) { } private void publish(PendingBlock p, boolean beacon) { - assert p.witnesses.size() >= params().majority() : "Publishing non majority block"; + assert p.witnesses.size() >= params().majority() : "Attempt to publish non majority block"; var publish = p.published.compareAndSet(false, true); if (!publish && !beacon) { log.trace("Already published: {} hash: {} height: {} witnesses: {} on: {}", p.block.block.getBodyCase(), p.block.hash, p.block.height(), p.witnesses.values().size(), params().member().getId()); return; } - log.trace("Publishing {}pending: {} hash: {} height: {} witnesses: {} on: {}", beacon ? "(beacon) " : "", + log.trace("Publishing {}: {} hash: {} height: {} witnesses: {} on: {}", beacon ? "(beacon) " : "(pending)", p.block.block.getBodyCase(), p.block.hash, p.block.height(), p.witnesses.values().size(), params().member().getId()); final var cb = CertifiedBlock.newBuilder() diff --git a/choam/src/test/resources/logback-test.xml b/choam/src/test/resources/logback-test.xml index d5be6a052..9f1825d70 100644 --- a/choam/src/test/resources/logback-test.xml +++ b/choam/src/test/resources/logback-test.xml @@ -33,7 +33,7 @@ - + diff --git a/memberships/src/main/java/com/salesforce/apollo/membership/messaging/rbc/ReliableBroadcaster.java b/memberships/src/main/java/com/salesforce/apollo/membership/messaging/rbc/ReliableBroadcaster.java index ba02e114e..290f5299a 100644 --- a/memberships/src/main/java/com/salesforce/apollo/membership/messaging/rbc/ReliableBroadcaster.java +++ b/memberships/src/main/java/com/salesforce/apollo/membership/messaging/rbc/ReliableBroadcaster.java @@ -457,8 +457,7 @@ private class Buffer { private Buffer(int maxAge) { this.maxAge = maxAge; highWaterMark = (params.bufferSize - (int) (params.bufferSize + ((params.bufferSize) * 0.1))); - delivered = BloomWindow.create(params.dedupBufferSize, 1.0 / ((double) params.dedupBufferSize * 2.0), - Biff.Type.DIGEST); + delivered = BloomWindow.create(params.dedupBufferSize, params.dedupFpr, Biff.Type.DIGEST); } public void clear() { @@ -466,8 +465,7 @@ public void clear() { } public BloomFilter forReconcilliation() { - var biff = new DigestBloomFilter(Entropy.nextBitsStreamLong(), params.bufferSize, - 1.0 / ((double) params.bufferSize * 2.0)); + var biff = new DigestBloomFilter(Entropy.nextBitsStreamLong(), params.bufferSize, params.falsePositiveRate); state.keySet().forEach(k -> biff.add(k)); return biff; } diff --git a/pom.xml b/pom.xml index c4879bd2b..9e386ff13 100644 --- a/pom.xml +++ b/pom.xml @@ -783,8 +783,8 @@ 3.2.5 ${forks} - true - -Xmx10G -Xms4G + false + -Xmx10G -Xms100M -Djdk.tracePinnedThreads=full From 6a19047a5756d30d80675ef8fbde0d0461402754 Mon Sep 17 00:00:00 2001 From: Hellblazer Date: Thu, 6 Jun 2024 07:56:19 -0700 Subject: [PATCH 08/22] Believe the view Reconfiguration is now fixed. Significant work on join protocol. Added new Chillin' state to ensure we get full membership through slow joiners. --- choam/pom.xml | 11 - .../com/salesforce/apollo/choam/CHOAM.java | 204 +++++++++++------- .../salesforce/apollo/choam/ViewAssembly.java | 41 ++-- .../apollo/choam/comm/Terminal.java | 11 +- .../apollo/choam/comm/TerminalClient.java | 7 +- .../apollo/choam/fsm/Reconfiguration.java | 42 +++- .../salesforce/apollo/choam/DynamicTest.java | 4 +- .../apollo/choam/MembershipTests.java | 4 +- .../salesforce/apollo/choam/TestCHOAM.java | 8 +- choam/src/test/resources/logback-test.xml | 2 +- .../apollo/fireflies/ChurnTest.java | 5 +- .../apollo/fireflies/SwarmTest.java | 5 +- .../messaging/rbc/ReliableBroadcaster.java | 4 +- model/pom.xml | 7 - .../apollo/model/ContainmentDomainTest.java | 9 +- .../salesforce/apollo/model/DomainTest.java | 9 +- .../apollo/model/FireFliesTest.java | 9 +- pom.xml | 2 +- .../apollo/state/AbstractLifecycleTest.java | 5 +- .../salesforce/apollo/state/CHOAMTest.java | 5 +- 20 files changed, 239 insertions(+), 155 deletions(-) diff --git a/choam/pom.xml b/choam/pom.xml index 6ee71890c..d4c07c80e 100644 --- a/choam/pom.xml +++ b/choam/pom.xml @@ -53,15 +53,4 @@ test - - - - org.apache.maven.plugins - maven-surefire-plugin - - false - - - - diff --git a/choam/src/main/java/com/salesforce/apollo/choam/CHOAM.java b/choam/src/main/java/com/salesforce/apollo/choam/CHOAM.java index 449305faf..c50ed31b1 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/CHOAM.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/CHOAM.java @@ -7,6 +7,7 @@ package com.salesforce.apollo.choam; import com.chiralbehaviors.tron.Fsm; +import com.google.common.util.concurrent.ListenableFuture; import com.google.protobuf.ByteString; import com.google.protobuf.Empty; import com.google.protobuf.InvalidProtocolBufferException; @@ -37,6 +38,7 @@ import com.salesforce.apollo.messaging.proto.AgedMessageOrBuilder; import com.salesforce.apollo.utils.Utils; import io.grpc.StatusRuntimeException; +import io.netty.util.concurrent.ImmediateExecutor; import org.h2.mvstore.MVMap; import org.joou.ULong; import org.slf4j.Logger; @@ -73,30 +75,31 @@ public class CHOAM { private static final Logger log = LoggerFactory.getLogger(CHOAM.class); - private final Map cachedCheckpoints = new ConcurrentHashMap<>(); - private final AtomicReference checkpoint = new AtomicReference<>(); - private final ReliableBroadcaster combine; - private final CommonCommunications comm; - private final AtomicReference current = new AtomicReference<>(); - private final ExecutorService executions; - private final AtomicReference> futureBootstrap = new AtomicReference<>(); - private final AtomicReference> futureSynchronization = new AtomicReference<>(); - private final AtomicReference genesis = new AtomicReference<>(); - private final AtomicReference head = new AtomicReference<>(); - private final ExecutorService linear; - private final AtomicReference next = new AtomicReference<>(); - private final AtomicReference nextViewId = new AtomicReference<>(); - private final Parameters params; - private final PriorityBlockingQueue pending = new PriorityBlockingQueue<>(); - private final RoundScheduler roundScheduler; - private final Session session; - private final AtomicBoolean started = new AtomicBoolean(); - private final Store store; - private final CommonCommunications submissionComm; - private final Combine.Transitions transitions; - private final TransSubmission txnSubmission = new TransSubmission(); - private final AtomicReference view = new AtomicReference<>(); - private final PendingViews pendingViews = new PendingViews(); + private final Map cachedCheckpoints = new ConcurrentHashMap<>(); + private final AtomicReference checkpoint = new AtomicReference<>(); + private final ReliableBroadcaster combine; + private final CommonCommunications comm; + private final AtomicReference current = new AtomicReference<>(); + private final ExecutorService executions; + private final AtomicReference> futureBootstrap = new AtomicReference<>(); + private final AtomicReference> futureSynchronization = new AtomicReference<>(); + private final AtomicReference genesis = new AtomicReference<>(); + private final AtomicReference head = new AtomicReference<>(); + private final ExecutorService linear; + private final AtomicReference next = new AtomicReference<>(); + private final AtomicReference nextViewId = new AtomicReference<>(); + private final Parameters params; + private final PriorityBlockingQueue pending = new PriorityBlockingQueue<>(); + private final RoundScheduler roundScheduler; + private final Session session; + private final AtomicBoolean started = new AtomicBoolean(); + private final Store store; + private final CommonCommunications submissionComm; + private final Combine.Transitions transitions; + private final TransSubmission txnSubmission = new TransSubmission(); + private final AtomicReference view = new AtomicReference<>(); + private final PendingViews pendingViews = new PendingViews(); + private volatile AtomicBoolean ongoingJoin; public CHOAM(Parameters params) { this.store = new Store(params.digestAlgorithm(), params.mvBuilder().clone().build()); @@ -367,9 +370,15 @@ public void stop() { } final var c = current.get(); if (c != null) { - c.complete(); + try { + c.complete(); + } catch (Throwable e) { + } + } + try { + combine.stop(); + } catch (Throwable e) { } - combine.stop(); } private void accept(HashedCertifiedBlock next) { @@ -733,6 +742,12 @@ private void reconfigure(Digest hash, Reconfigure reconfigure) { } else { current.set(new Client(validators, getViewId())); } + final var oj = ongoingJoin; + ongoingJoin = null; + if (oj != null) { + log.trace("Halting ongoing join on: {}", params.member().getId()); + oj.set(true); + } log.info("Reconfigured to view: {} committee: {} validators: {} on: {}", new Digest(reconfigure.getId()), current.get().getClass().getSimpleName(), validators.entrySet() .stream() @@ -1263,6 +1278,7 @@ public Blocks fetchViewChain(BlockReplication request, Digest from) { @Override public Empty join(SignedViewMember nextView, Digest from) { + log.trace("Member: {} joining on: {}", from, params.member().getId()); CHOAM.this.join(nextView, from); return Empty.getDefaultInstance(); } @@ -1275,10 +1291,9 @@ public Initial sync(Synchronize request, Digest from) { /** abstract class to maintain the common state */ private abstract class Administration implements Committee { - protected final Digest viewId; - private final GroupIterator servers; - private final Map validators; - private volatile JoinState ongoingJoin; + protected final Digest viewId; + private final GroupIterator servers; + private final Map validators; public Administration(Map validators, Digest viewId) { this.validators = validators; @@ -1288,12 +1303,6 @@ public Administration(Map validators, Digest viewId) { @Override public void accept(HashedCertifiedBlock hb) { - final var oj = ongoingJoin; - ongoingJoin = null; - if (oj != null) { - oj.halt.set(true); - oj.joining.interrupt(); - } process(); } @@ -1386,28 +1395,44 @@ private void join(View view) { if (ongoingJoin != null) { throw new IllegalStateException("Ongoing join should have been cancelled"); } - log.info("Joining view: {} diadem: {} on: {}", nextViewId.get(), Digest.from(view.getDiadem()), - params.member().getId()); + log.trace("Joining view: {} diadem: {} on: {}", nextViewId.get(), Digest.from(view.getDiadem()), + params.member().getId()); var servers = new ConcurrentSkipListSet<>(validators.keySet()); var joined = new AtomicInteger(); var halt = new AtomicBoolean(false); - - ongoingJoin = new JoinState(halt, Thread.ofVirtual().start(Utils.wrapped(() -> { - log.error("Starting join of: {} diadem {} on: {}", nextViewId.get(), Digest.from(view.getDiadem()), + ongoingJoin = halt; + Thread.ofVirtual().start(Utils.wrapped(() -> { + log.trace("Starting join of: {} diadem {} on: {}", nextViewId.get(), Digest.from(view.getDiadem()), params.member().getId()); - while (!halt.get() & joined.get() < view.getMajority()) { - join(view, servers, joined); - } - log.info("Finishing join of: {} diadem: {} on: {}", nextViewId.get(), Digest.from(view.getDiadem()), - params.member().getId()); - ongoingJoin = null; - }, log()))); - } - private void join(View view, Collection servers, AtomicInteger joined) { + var scheduler = Executors.newSingleThreadScheduledExecutor(Thread.ofVirtual().factory()); + AtomicReference action = new AtomicReference<>(); + var attempts = new AtomicInteger(); + action.set(() -> { + log.trace("Join attempt: {} halt: {} joined: {} majority: {} on: {}", attempts.incrementAndGet(), + halt.get(), joined.get(), view.getMajority(), params.member().getId()); + if (!halt.get() & joined.get() < view.getMajority()) { + join(view, servers, joined); + if (joined.get() >= view.getMajority()) { + ongoingJoin = null; + log.trace("Finished join of: {} diadem: {} joins: {} on: {}", nextViewId.get(), + Digest.from(view.getDiadem()), joined.get(), params.member().getId()); + } else if (!halt.get()) { + log.trace("Rescheduling join of: {} diadem: {} joins: {} on: {}", nextViewId.get(), + Digest.from(view.getDiadem()), joined.get(), params.member().getId()); + scheduler.schedule(action.get(), 50, TimeUnit.MILLISECONDS); + } + } + }); + scheduler.schedule(action.get(), 50, TimeUnit.MILLISECONDS); + }, log())); + } + private void join(View view, Collection members, AtomicInteger joined) { + var sampled = new ArrayList<>(members); + Collections.shuffle(sampled); log.trace("Joining view: {} diadem: {} servers: {} on: {}", viewId, Digest.from(view.getDiadem()), - servers.stream().map(Member::getId).toList(), params.member().getId()); + sampled.stream().map(Member::getId).toList(), params.member().getId()); final var c = next.get(); var inView = ViewMember.newBuilder(c.member) .setDiadem(view.getDiadem()) @@ -1417,34 +1442,69 @@ private void join(View view, Collection servers, AtomicInteger joined) { .setVm(inView) .setSignature(params.member().sign(inView.toByteString()).toSig()) .build(); - var countdown = new CountDownLatch(servers.size()); - - servers.stream().map(comm::connect).filter(Objects::nonNull).forEach(t -> { - Thread.ofVirtual().start(Utils.wrapped(() -> { - try { - t.join(svm); - servers.remove(t.getMember()); - joined.incrementAndGet(); - countdown.countDown(); - log.trace("Joined with: {} view: {} diadem: {} on: {}", t.getMember().getId(), viewId, - Digest.from(view.getDiadem()), params.member().getId()); - } catch (StatusRuntimeException sre) { - log.trace("Failed join attempt: {} with: {} view: {} diadem: {} on: {}", sre.getStatus(), - t.getMember().getId(), nextViewId, Digest.from(view.getDiadem()), - params.member().getId(), sre); - } catch (Throwable throwable) { - log.trace("Failed join attempt with: {} view: {} diadem: {} on: {}", t.getMember().getId(), - nextViewId, Digest.from(view.getDiadem()), params.member().getId(), throwable); - } - }, log())); + var countdown = new CountDownLatch(sampled.size()); + sampled.stream().map(m -> { + var connection = comm.connect(m); + log.trace("connect to: {} is: {} on: {}", m.getId(), connection, params.member().getId()); + return connection; + }).map(t -> t == null ? null : join(view, t, svm)).forEach(t -> { + if (t == null) { + countdown.countDown(); + } else { + t.fs.addListener(() -> { + try { + t.fs.get(); + members.remove(t.m); + joined.incrementAndGet(); + log.trace("Joined with: {} view: {} diadem: {} on: {}", t.m.getId(), + Digest.from(inView.getId()), Digest.from(view.getDiadem()), + params.member().getId()); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } catch (ExecutionException e) { + log.error("Failed to join with: {} view: {} diadem: {} on: {}", t.m.getId(), viewId, + Digest.from(view.getDiadem()), params.member().getId(), e.getCause()); + } catch (Throwable e) { + log.error("Failed to join with: {} view: {} diadem: {} on: {}", t.m.getId(), viewId, + Digest.from(view.getDiadem()), params.member().getId(), e); + } finally { + countdown.countDown(); + } + }, ImmediateExecutor.INSTANCE); + } }); try { - countdown.await(2, TimeUnit.SECONDS); + countdown.await(5, TimeUnit.SECONDS); } catch (InterruptedException e) { Thread.currentThread().interrupt(); } } + private Attempt join(View view, Terminal t, SignedViewMember svm) { + try { + log.trace("Attempting to join with: {} context: {} diadem: {} on: {}", t.getMember().getId(), + context().getId(), Digest.from(view.getDiadem()), params.member().getId()); + return new Attempt(t.getMember(), t.join(svm)); + } catch (StatusRuntimeException sre) { + log.trace("Failed join attempt: {} with: {} view: {} diadem: {} on: {}", sre.getStatus(), + t.getMember().getId(), nextViewId, Digest.from(view.getDiadem()), params.member().getId(), + sre); + } catch (Throwable throwable) { + log.error("Failed join attempt with: {} view: {} diadem: {} on: {}", t.getMember().getId(), nextViewId, + Digest.from(view.getDiadem()), params.member().getId(), throwable); + } finally { + try { + t.close(); + } catch (IOException e) { + // ignored + } + } + return null; + } + + record Attempt(Member m, ListenableFuture fs) { + } + private record JoinState(AtomicBoolean halt, Thread joining) { } } diff --git a/choam/src/main/java/com/salesforce/apollo/choam/ViewAssembly.java b/choam/src/main/java/com/salesforce/apollo/choam/ViewAssembly.java index d922991fb..2e1d5cb11 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/ViewAssembly.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/ViewAssembly.java @@ -83,9 +83,6 @@ public Map getSlate() { public void joined(SignedViewMember viewMember) { final var mid = Digest.from(viewMember.getVm().getId()); - if (!validate(mid, viewMember)) { - return; - } joins.put(mid, SignedJoin.newBuilder() .setMember(params().member().getId().toDigeste()) .setJoin(viewMember) @@ -219,7 +216,7 @@ void join(List joins) { proposals.put(mid, svm); } } - checkAssembly(); + transitions.checkAssembly(); } void newEpoch() { @@ -237,15 +234,15 @@ private Map assemblyOf(List committee) { .collect(Collectors.toMap(Member::getId, m -> m)); } - private void checkAssembly() { + private boolean checkAssembly() { if (selected == null) { - return; + return false; } if (proposals.size() == selected.majority) { transitions.certified(); - } else if (proposals.size() >= selected.majority) { - transitions.gathered(); + return true; } + return false; } private Parameters params() { @@ -405,24 +402,40 @@ private class Recon implements Reconfiguration { @Override public void certify() { if (proposals.size() == selected.majority) { - log.debug("Certifying: {} majority: {} of: {} slate: {} on: {}", nextViewId, selected.majority, - nextViewId, proposals.keySet().stream().sorted().toList(), params().member().getId()); + log.info("Certifying: {} majority: {} of: {} slate: {} on: {}", nextViewId, selected.majority, + nextViewId, proposals.keySet().stream().sorted().toList(), params().member().getId()); transitions.certified(); } else { - countdown.set(3); - log.debug("Not certifying: {} majority: {} slate: {} of: {} on: {}", nextViewId, selected.majority, - proposals.keySet().stream().sorted().toList(), nextViewId, params().member().getId()); + countdown.set(4); + log.info("Not certifying: {} majority: {} slate: {} of: {} on: {}", nextViewId, selected.majority, + proposals.keySet().stream().sorted().toList(), nextViewId, params().member().getId()); } } public void checkAssembly() { - ViewAssembly.this.checkAssembly(); + if (ViewAssembly.this.checkAssembly()) { + return; + } + if (proposals.size() >= selected.majority) { + transitions.chill(); + } else { + log.info("Check assembly: {} on: {}", proposals.size(), params().member().getId()); + } } public void checkViews() { vote(); } + @Override + public void chill() { + if (ViewAssembly.this.checkAssembly()) { + transitions.certified(); + } else { + countdown.set(2); + } + } + @Override public void complete() { ViewAssembly.this.complete(); diff --git a/choam/src/main/java/com/salesforce/apollo/choam/comm/Terminal.java b/choam/src/main/java/com/salesforce/apollo/choam/comm/Terminal.java index 394f4664d..080aea7c8 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/comm/Terminal.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/comm/Terminal.java @@ -6,6 +6,8 @@ */ package com.salesforce.apollo.choam.comm; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.SettableFuture; import com.google.protobuf.Empty; import com.salesforce.apollo.archipelago.Link; import com.salesforce.apollo.choam.proto.*; @@ -47,8 +49,11 @@ public Member getMember() { } @Override - public Empty join(SignedViewMember join) { - return service.join(join, member.getId()); + public ListenableFuture join(SignedViewMember join) { + var j = service.join(join, member.getId()); + SettableFuture sf = SettableFuture.create(); + sf.set(j); + return sf; } @Override @@ -64,7 +69,7 @@ public Initial sync(Synchronize sync) { Blocks fetchViewChain(BlockReplication replication); - Empty join(SignedViewMember join); + ListenableFuture join(SignedViewMember join); Initial sync(Synchronize sync); } diff --git a/choam/src/main/java/com/salesforce/apollo/choam/comm/TerminalClient.java b/choam/src/main/java/com/salesforce/apollo/choam/comm/TerminalClient.java index 26ee81aa0..48fe85f76 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/comm/TerminalClient.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/comm/TerminalClient.java @@ -6,6 +6,7 @@ */ package com.salesforce.apollo.choam.comm; +import com.google.common.util.concurrent.ListenableFuture; import com.google.protobuf.Empty; import com.salesforce.apollo.archipelago.ManagedServerChannel; import com.salesforce.apollo.archipelago.ServerConnectionCache.CreateClientCommunications; @@ -20,12 +21,14 @@ public class TerminalClient implements Terminal { private final ManagedServerChannel channel; private final TerminalGrpc.TerminalBlockingStub client; + private final TerminalGrpc.TerminalFutureStub asyncClient; @SuppressWarnings("unused") private final ChoamMetrics metrics; public TerminalClient(ManagedServerChannel channel, ChoamMetrics metrics) { this.channel = channel; this.client = channel.wrap(TerminalGrpc.newBlockingStub(channel)); + this.asyncClient = channel.wrap(TerminalGrpc.newFutureStub(channel)); this.metrics = metrics; } @@ -60,8 +63,8 @@ public Member getMember() { } @Override - public Empty join(SignedViewMember vm) { - return client.join(vm); + public ListenableFuture join(SignedViewMember vm) { + return asyncClient.join(vm); } public void release() { diff --git a/choam/src/main/java/com/salesforce/apollo/choam/fsm/Reconfiguration.java b/choam/src/main/java/com/salesforce/apollo/choam/fsm/Reconfiguration.java index 01146befd..246daa51a 100644 --- a/choam/src/main/java/com/salesforce/apollo/choam/fsm/Reconfiguration.java +++ b/choam/src/main/java/com/salesforce/apollo/choam/fsm/Reconfiguration.java @@ -19,6 +19,8 @@ public interface Reconfiguration { void checkViews(); + void chill(); + void complete(); void failed(); @@ -60,10 +62,9 @@ public void certify() { context().certify(); } }, GATHER { - // We have a majority of the new committee Joins @Override - public Transitions gathered() { - return CERTIFICATION; + public Transitions chill() { + return CHILLIN; } // We have a full complement of the new committee Joins @@ -77,6 +78,29 @@ public Transitions certified() { public void gather() { context().checkAssembly(); } + + @Override + public Transitions checkAssembly() { + context().checkAssembly(); + return null; + } + }, CHILLIN { + @Override + public Transitions countdownCompleted() { + return certified(); + } + + // We have what we have + @Override + public Transitions certified() { + return CERTIFICATION; + } + + // Check to see if we already have a full complement of committee Joins + @Entry + public void chillin() { + context().chill(); + } }, PROTOCOL_FAILURE { @Override public Transitions certified() { @@ -139,6 +163,14 @@ default Transitions certified() { throw fsm().invalidTransitionOn(); } + default Transitions checkAssembly() { + throw fsm().invalidTransitionOn(); + } + + default Transitions chill() { + throw fsm().invalidTransitionOn(); + } + default Transitions complete() { throw fsm().invalidTransitionOn(); } @@ -151,10 +183,6 @@ default Transitions failed() { return Reconfigure.PROTOCOL_FAILURE; } - default Transitions gathered() { - throw fsm().invalidTransitionOn(); - } - default Transitions proposed() { throw fsm().invalidTransitionOn(); } diff --git a/choam/src/test/java/com/salesforce/apollo/choam/DynamicTest.java b/choam/src/test/java/com/salesforce/apollo/choam/DynamicTest.java index ebd508af6..fe29a7e45 100644 --- a/choam/src/test/java/com/salesforce/apollo/choam/DynamicTest.java +++ b/choam/src/test/java/com/salesforce/apollo/choam/DynamicTest.java @@ -3,7 +3,6 @@ import com.salesforce.apollo.archipelago.LocalServer; import com.salesforce.apollo.archipelago.Router; import com.salesforce.apollo.archipelago.ServerConnectionCache; -import com.salesforce.apollo.archipelago.UnsafeExecutors; import com.salesforce.apollo.choam.support.ExponentialBackoffPolicy; import com.salesforce.apollo.context.Context; import com.salesforce.apollo.context.DynamicContext; @@ -64,10 +63,9 @@ public void setUp() throws Exception { .toList(); final var prefix = UUID.randomUUID().toString(); - var executor = UnsafeExecutors.newVirtualThreadPerTaskExecutor(); routers = members.stream() .collect(Collectors.toMap(m -> m, m -> new LocalServer(prefix, m).router( - ServerConnectionCache.newBuilder().setTarget(cardinality * 2), executor))); + ServerConnectionCache.newBuilder().setTarget(cardinality * 2)))); var template = Parameters.newBuilder() .setGenerateGenesis(true) diff --git a/choam/src/test/java/com/salesforce/apollo/choam/MembershipTests.java b/choam/src/test/java/com/salesforce/apollo/choam/MembershipTests.java index febe8f415..effd854f5 100644 --- a/choam/src/test/java/com/salesforce/apollo/choam/MembershipTests.java +++ b/choam/src/test/java/com/salesforce/apollo/choam/MembershipTests.java @@ -9,7 +9,6 @@ import com.salesforce.apollo.archipelago.LocalServer; import com.salesforce.apollo.archipelago.Router; import com.salesforce.apollo.archipelago.ServerConnectionCache; -import com.salesforce.apollo.archipelago.UnsafeExecutors; import com.salesforce.apollo.choam.CHOAM.TransactionExecutor; import com.salesforce.apollo.choam.Parameters.BootstrapParameters; import com.salesforce.apollo.choam.Parameters.ProducerParameters; @@ -171,10 +170,9 @@ public SigningMember initialize(int checkpointBlockSize, int cardinality) throws SigningMember testSubject = new ControlledIdentifierMember(stereotomy.newIdentifier()); final var prefix = UUID.randomUUID().toString(); - var executor = UnsafeExecutors.newVirtualThreadPerTaskExecutor(); routers = members.stream() .collect(Collectors.toMap(Member::getId, m -> new LocalServer(prefix, m).router( - ServerConnectionCache.newBuilder().setTarget(cardinality), executor))); + ServerConnectionCache.newBuilder().setTarget(cardinality)))); routers.put(testSubject.getId(), new LocalServer(prefix, testSubject).router( ServerConnectionCache.newBuilder().setTarget(cardinality))); choams = new HashMap<>(); diff --git a/choam/src/test/java/com/salesforce/apollo/choam/TestCHOAM.java b/choam/src/test/java/com/salesforce/apollo/choam/TestCHOAM.java index 66d941748..391706bf2 100644 --- a/choam/src/test/java/com/salesforce/apollo/choam/TestCHOAM.java +++ b/choam/src/test/java/com/salesforce/apollo/choam/TestCHOAM.java @@ -8,7 +8,10 @@ import com.codahale.metrics.ConsoleReporter; import com.codahale.metrics.MetricRegistry; -import com.salesforce.apollo.archipelago.*; +import com.salesforce.apollo.archipelago.LocalServer; +import com.salesforce.apollo.archipelago.Router; +import com.salesforce.apollo.archipelago.ServerConnectionCache; +import com.salesforce.apollo.archipelago.ServerConnectionCacheMetricsImpl; import com.salesforce.apollo.choam.CHOAM.TransactionExecutor; import com.salesforce.apollo.choam.Parameters.ProducerParameters; import com.salesforce.apollo.choam.Parameters.RuntimeParameters; @@ -121,12 +124,11 @@ public void before() throws Exception { .toList(); var context = new StaticContext<>(origin, 0.2, members, 3); final var prefix = UUID.randomUUID().toString(); - var executor = UnsafeExecutors.newVirtualThreadPerTaskExecutor(); routers = members.stream() .collect(Collectors.toMap(m -> m.getId(), m -> new LocalServer(prefix, m).router( ServerConnectionCache.newBuilder() .setMetrics(new ServerConnectionCacheMetricsImpl(registry)) - .setTarget(CARDINALITY), executor))); + .setTarget(CARDINALITY)))); choams = members.stream().collect(Collectors.toMap(m -> m.getId(), m -> { var recording = new AtomicInteger(); blocks.put(m.getId(), recording); diff --git a/choam/src/test/resources/logback-test.xml b/choam/src/test/resources/logback-test.xml index 9f1825d70..d5be6a052 100644 --- a/choam/src/test/resources/logback-test.xml +++ b/choam/src/test/resources/logback-test.xml @@ -33,7 +33,7 @@ - + diff --git a/fireflies/src/test/java/com/salesforce/apollo/fireflies/ChurnTest.java b/fireflies/src/test/java/com/salesforce/apollo/fireflies/ChurnTest.java index dcfc99b4f..9825e45f6 100644 --- a/fireflies/src/test/java/com/salesforce/apollo/fireflies/ChurnTest.java +++ b/fireflies/src/test/java/com/salesforce/apollo/fireflies/ChurnTest.java @@ -273,7 +273,6 @@ private void initialize() { AtomicBoolean frist = new AtomicBoolean(true); final var prefix = UUID.randomUUID().toString(); final var gatewayPrefix = UUID.randomUUID().toString(); - var executor = UnsafeExecutors.newVirtualThreadPerTaskExecutor(); views = members.values().stream().map(node -> { DynamicContext context = ctxBuilder.build(); FireflyMetricsImpl metrics = new FireflyMetricsImpl(context.getId(), @@ -289,8 +288,8 @@ private void initialize() { .setMetrics( new ServerConnectionCacheMetricsImpl( frist.getAndSet(false) - ? node0Registry : registry)), - executor); + ? node0Registry + : registry))); comms.start(); communications.add(comms); diff --git a/fireflies/src/test/java/com/salesforce/apollo/fireflies/SwarmTest.java b/fireflies/src/test/java/com/salesforce/apollo/fireflies/SwarmTest.java index b18c32a3e..0e4f93dc6 100644 --- a/fireflies/src/test/java/com/salesforce/apollo/fireflies/SwarmTest.java +++ b/fireflies/src/test/java/com/salesforce/apollo/fireflies/SwarmTest.java @@ -226,7 +226,6 @@ private void initialize() { AtomicBoolean frist = new AtomicBoolean(true); final var prefix = UUID.randomUUID().toString(); final var gatewayPrefix = UUID.randomUUID().toString(); - var executor = UnsafeExecutors.newVirtualThreadPerTaskExecutor(); views = members.values().stream().map(node -> { DynamicContext context = ctxBuilder.build(); FireflyMetricsImpl metrics = new FireflyMetricsImpl(context.getId(), @@ -242,8 +241,8 @@ private void initialize() { .setMetrics( new ServerConnectionCacheMetricsImpl( frist.getAndSet(false) - ? node0Registry : registry)), - executor); + ? node0Registry + : registry))); comms.start(); communications.add(comms); diff --git a/memberships/src/main/java/com/salesforce/apollo/membership/messaging/rbc/ReliableBroadcaster.java b/memberships/src/main/java/com/salesforce/apollo/membership/messaging/rbc/ReliableBroadcaster.java index 290f5299a..a4b9f149f 100644 --- a/memberships/src/main/java/com/salesforce/apollo/membership/messaging/rbc/ReliableBroadcaster.java +++ b/memberships/src/main/java/com/salesforce/apollo/membership/messaging/rbc/ReliableBroadcaster.java @@ -328,10 +328,10 @@ public static Parameters.Builder newBuilder() { public static class Builder implements Cloneable { private int bufferSize = 1500; private int dedupBufferSize = 100; - private double dedupFpr = Math.pow(10, -9); + private double dedupFpr = Math.pow(10, -6); private int deliveredCacheSize = 100; private DigestAlgorithm digestAlgorithm = DigestAlgorithm.DEFAULT; - private double falsePositiveRate = 0.00125; + private double falsePositiveRate = 0.0000125; private int maxMessages = 500; public Parameters build() { diff --git a/model/pom.xml b/model/pom.xml index 2db95ab73..3b9acd54a 100644 --- a/model/pom.xml +++ b/model/pom.xml @@ -76,13 +76,6 @@ - - org.apache.maven.plugins - maven-surefire-plugin - - false - - org.apache.maven.plugins maven-antrun-plugin diff --git a/model/src/test/java/com/salesforce/apollo/model/ContainmentDomainTest.java b/model/src/test/java/com/salesforce/apollo/model/ContainmentDomainTest.java index 8df39f7d0..6494cbecb 100644 --- a/model/src/test/java/com/salesforce/apollo/model/ContainmentDomainTest.java +++ b/model/src/test/java/com/salesforce/apollo/model/ContainmentDomainTest.java @@ -6,7 +6,10 @@ */ package com.salesforce.apollo.model; -import com.salesforce.apollo.archipelago.*; +import com.salesforce.apollo.archipelago.EndpointProvider; +import com.salesforce.apollo.archipelago.LocalServer; +import com.salesforce.apollo.archipelago.Router; +import com.salesforce.apollo.archipelago.ServerConnectionCache; import com.salesforce.apollo.choam.Parameters; import com.salesforce.apollo.choam.Parameters.Builder; import com.salesforce.apollo.choam.Parameters.RuntimeParameters; @@ -79,11 +82,9 @@ public void before() throws Exception { var sealed = FoundationSeal.newBuilder().build(); final var group = DigestAlgorithm.DEFAULT.getOrigin(); - var executor = UnsafeExecutors.newVirtualThreadPerTaskExecutor(); identities.forEach((d, id) -> { final var member = new ControlledIdentifierMember(id); - var localRouter = new LocalServer(prefix, member).router(ServerConnectionCache.newBuilder().setTarget(30), - executor); + var localRouter = new LocalServer(prefix, member).router(ServerConnectionCache.newBuilder().setTarget(30)); routers.add(localRouter); var dbUrl = String.format("jdbc:h2:mem:sql-%s-%s;DB_CLOSE_DELAY=-1", member.getId(), UUID.randomUUID()); var pdParams = new ProcessDomain.ProcessDomainParameters(dbUrl, Duration.ofMinutes(1), diff --git a/model/src/test/java/com/salesforce/apollo/model/DomainTest.java b/model/src/test/java/com/salesforce/apollo/model/DomainTest.java index 3cb29dd42..dd5f0503f 100644 --- a/model/src/test/java/com/salesforce/apollo/model/DomainTest.java +++ b/model/src/test/java/com/salesforce/apollo/model/DomainTest.java @@ -6,7 +6,10 @@ */ package com.salesforce.apollo.model; -import com.salesforce.apollo.archipelago.*; +import com.salesforce.apollo.archipelago.EndpointProvider; +import com.salesforce.apollo.archipelago.LocalServer; +import com.salesforce.apollo.archipelago.Router; +import com.salesforce.apollo.archipelago.ServerConnectionCache; import com.salesforce.apollo.choam.Parameters; import com.salesforce.apollo.choam.Parameters.Builder; import com.salesforce.apollo.choam.Parameters.RuntimeParameters; @@ -235,11 +238,9 @@ public void before() throws Exception { var sealed = FoundationSeal.newBuilder().build(); final var group = DigestAlgorithm.DEFAULT.getOrigin(); - var executor = UnsafeExecutors.newVirtualThreadPerTaskExecutor(); identities.forEach((d, id) -> { final var member = new ControlledIdentifierMember(id); - var localRouter = new LocalServer(prefix, member).router(ServerConnectionCache.newBuilder().setTarget(30), - executor); + var localRouter = new LocalServer(prefix, member).router(ServerConnectionCache.newBuilder().setTarget(30)); routers.add(localRouter); var dbUrl = String.format("jdbc:h2:mem:sql-%s-%s;DB_CLOSE_DELAY=-1", member.getId(), UUID.randomUUID()); var pdParams = new ProcessDomain.ProcessDomainParameters(dbUrl, Duration.ofMinutes(1), diff --git a/model/src/test/java/com/salesforce/apollo/model/FireFliesTest.java b/model/src/test/java/com/salesforce/apollo/model/FireFliesTest.java index c6a48aa41..beb936b33 100644 --- a/model/src/test/java/com/salesforce/apollo/model/FireFliesTest.java +++ b/model/src/test/java/com/salesforce/apollo/model/FireFliesTest.java @@ -6,7 +6,10 @@ */ package com.salesforce.apollo.model; -import com.salesforce.apollo.archipelago.*; +import com.salesforce.apollo.archipelago.EndpointProvider; +import com.salesforce.apollo.archipelago.LocalServer; +import com.salesforce.apollo.archipelago.Router; +import com.salesforce.apollo.archipelago.ServerConnectionCache; import com.salesforce.apollo.choam.Parameters; import com.salesforce.apollo.choam.Parameters.Builder; import com.salesforce.apollo.choam.Parameters.RuntimeParameters; @@ -78,12 +81,10 @@ public void before() throws Exception { Digest group = DigestAlgorithm.DEFAULT.getOrigin(); var sealed = FoundationSeal.newBuilder().build(); - var executor = UnsafeExecutors.newVirtualThreadPerTaskExecutor(); identities.forEach((digest, id) -> { var context = new DynamicContextImpl<>(DigestAlgorithm.DEFAULT.getLast(), CARDINALITY, 0.2, 3); final var member = new ControlledIdentifierMember(id); - var localRouter = new LocalServer(prefix, member).router(ServerConnectionCache.newBuilder().setTarget(30), - executor); + var localRouter = new LocalServer(prefix, member).router(ServerConnectionCache.newBuilder().setTarget(30)); var dbUrl = String.format("jdbc:h2:mem:sql-%s-%s;DB_CLOSE_DELAY=-1", member.getId(), UUID.randomUUID()); var pdParams = new ProcessDomain.ProcessDomainParameters(dbUrl, Duration.ofSeconds(5), "jdbc:h2:mem:%s-state".formatted(digest), diff --git a/pom.xml b/pom.xml index 9e386ff13..416caaf06 100644 --- a/pom.xml +++ b/pom.xml @@ -783,7 +783,7 @@ 3.2.5 ${forks} - false + true -Xmx10G -Xms100M -Djdk.tracePinnedThreads=full diff --git a/sql-state/src/test/java/com/salesforce/apollo/state/AbstractLifecycleTest.java b/sql-state/src/test/java/com/salesforce/apollo/state/AbstractLifecycleTest.java index cc2d78819..028f4b67b 100644 --- a/sql-state/src/test/java/com/salesforce/apollo/state/AbstractLifecycleTest.java +++ b/sql-state/src/test/java/com/salesforce/apollo/state/AbstractLifecycleTest.java @@ -9,7 +9,6 @@ import com.salesforce.apollo.archipelago.LocalServer; import com.salesforce.apollo.archipelago.Router; import com.salesforce.apollo.archipelago.ServerConnectionCache; -import com.salesforce.apollo.archipelago.UnsafeExecutors; import com.salesforce.apollo.choam.CHOAM; import com.salesforce.apollo.choam.Parameters; import com.salesforce.apollo.choam.Parameters.BootstrapParameters; @@ -158,10 +157,8 @@ public void before() throws Exception { members.stream().filter(s -> s != testSubject).forEach(s -> context.activate(s)); final var prefix = UUID.randomUUID().toString(); - var executor = UnsafeExecutors.newVirtualThreadPerTaskExecutor(); routers = members.stream().collect(Collectors.toMap(m -> m.getId(), m -> { - var localRouter = new LocalServer(prefix, m).router(ServerConnectionCache.newBuilder().setTarget(30), - executor); + var localRouter = new LocalServer(prefix, m).router(ServerConnectionCache.newBuilder().setTarget(30)); return localRouter; })); routers.put(testSubject.getId(), diff --git a/sql-state/src/test/java/com/salesforce/apollo/state/CHOAMTest.java b/sql-state/src/test/java/com/salesforce/apollo/state/CHOAMTest.java index e3a526a70..e8cb71761 100644 --- a/sql-state/src/test/java/com/salesforce/apollo/state/CHOAMTest.java +++ b/sql-state/src/test/java/com/salesforce/apollo/state/CHOAMTest.java @@ -11,7 +11,6 @@ import com.salesforce.apollo.archipelago.LocalServer; import com.salesforce.apollo.archipelago.Router; import com.salesforce.apollo.archipelago.ServerConnectionCache; -import com.salesforce.apollo.archipelago.UnsafeExecutors; import com.salesforce.apollo.choam.CHOAM; import com.salesforce.apollo.choam.CHOAM.TransactionExecutor; import com.salesforce.apollo.choam.Parameters; @@ -155,10 +154,8 @@ public void before() throws Exception { }).map(cpk -> new ControlledIdentifierMember(cpk)).map(e -> (SigningMember) e).toList(); members.forEach(m -> context.activate(m)); final var prefix = UUID.randomUUID().toString(); - var executor = UnsafeExecutors.newVirtualThreadPerTaskExecutor(); routers = members.stream().collect(Collectors.toMap(m -> m.getId(), m -> { - var localRouter = new LocalServer(prefix, m).router(ServerConnectionCache.newBuilder().setTarget(30), - executor); + var localRouter = new LocalServer(prefix, m).router(ServerConnectionCache.newBuilder().setTarget(30)); return localRouter; })); choams = members.stream() From e87d6cac77d5a3532321511bdeccc7d5dbb8f533 Mon Sep 17 00:00:00 2001 From: Hellblazer Date: Thu, 6 Jun 2024 08:05:28 -0700 Subject: [PATCH 09/22] flew too close to the sun --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 416caaf06..9e386ff13 100644 --- a/pom.xml +++ b/pom.xml @@ -783,7 +783,7 @@ 3.2.5 ${forks} - true + false -Xmx10G -Xms100M -Djdk.tracePinnedThreads=full From 61dd978853f72e03eed091ee9153782b73a0b446 Mon Sep 17 00:00:00 2001 From: Hellblazer Date: Thu, 6 Jun 2024 09:36:30 -0700 Subject: [PATCH 10/22] use UE :: sigh :: --- .../java/com/salesforce/apollo/choam/DynamicTest.java | 4 +++- .../test/java/com/salesforce/apollo/choam/TestCHOAM.java | 8 +++----- .../salesforce/apollo/model/ContainmentDomainTest.java | 9 ++++----- .../java/com/salesforce/apollo/model/DomainTest.java | 9 ++++----- .../java/com/salesforce/apollo/model/FireFliesTest.java | 9 ++++----- 5 files changed, 18 insertions(+), 21 deletions(-) diff --git a/choam/src/test/java/com/salesforce/apollo/choam/DynamicTest.java b/choam/src/test/java/com/salesforce/apollo/choam/DynamicTest.java index fe29a7e45..ebd508af6 100644 --- a/choam/src/test/java/com/salesforce/apollo/choam/DynamicTest.java +++ b/choam/src/test/java/com/salesforce/apollo/choam/DynamicTest.java @@ -3,6 +3,7 @@ import com.salesforce.apollo.archipelago.LocalServer; import com.salesforce.apollo.archipelago.Router; import com.salesforce.apollo.archipelago.ServerConnectionCache; +import com.salesforce.apollo.archipelago.UnsafeExecutors; import com.salesforce.apollo.choam.support.ExponentialBackoffPolicy; import com.salesforce.apollo.context.Context; import com.salesforce.apollo.context.DynamicContext; @@ -63,9 +64,10 @@ public void setUp() throws Exception { .toList(); final var prefix = UUID.randomUUID().toString(); + var executor = UnsafeExecutors.newVirtualThreadPerTaskExecutor(); routers = members.stream() .collect(Collectors.toMap(m -> m, m -> new LocalServer(prefix, m).router( - ServerConnectionCache.newBuilder().setTarget(cardinality * 2)))); + ServerConnectionCache.newBuilder().setTarget(cardinality * 2), executor))); var template = Parameters.newBuilder() .setGenerateGenesis(true) diff --git a/choam/src/test/java/com/salesforce/apollo/choam/TestCHOAM.java b/choam/src/test/java/com/salesforce/apollo/choam/TestCHOAM.java index 391706bf2..66d941748 100644 --- a/choam/src/test/java/com/salesforce/apollo/choam/TestCHOAM.java +++ b/choam/src/test/java/com/salesforce/apollo/choam/TestCHOAM.java @@ -8,10 +8,7 @@ import com.codahale.metrics.ConsoleReporter; import com.codahale.metrics.MetricRegistry; -import com.salesforce.apollo.archipelago.LocalServer; -import com.salesforce.apollo.archipelago.Router; -import com.salesforce.apollo.archipelago.ServerConnectionCache; -import com.salesforce.apollo.archipelago.ServerConnectionCacheMetricsImpl; +import com.salesforce.apollo.archipelago.*; import com.salesforce.apollo.choam.CHOAM.TransactionExecutor; import com.salesforce.apollo.choam.Parameters.ProducerParameters; import com.salesforce.apollo.choam.Parameters.RuntimeParameters; @@ -124,11 +121,12 @@ public void before() throws Exception { .toList(); var context = new StaticContext<>(origin, 0.2, members, 3); final var prefix = UUID.randomUUID().toString(); + var executor = UnsafeExecutors.newVirtualThreadPerTaskExecutor(); routers = members.stream() .collect(Collectors.toMap(m -> m.getId(), m -> new LocalServer(prefix, m).router( ServerConnectionCache.newBuilder() .setMetrics(new ServerConnectionCacheMetricsImpl(registry)) - .setTarget(CARDINALITY)))); + .setTarget(CARDINALITY), executor))); choams = members.stream().collect(Collectors.toMap(m -> m.getId(), m -> { var recording = new AtomicInteger(); blocks.put(m.getId(), recording); diff --git a/model/src/test/java/com/salesforce/apollo/model/ContainmentDomainTest.java b/model/src/test/java/com/salesforce/apollo/model/ContainmentDomainTest.java index 6494cbecb..8df39f7d0 100644 --- a/model/src/test/java/com/salesforce/apollo/model/ContainmentDomainTest.java +++ b/model/src/test/java/com/salesforce/apollo/model/ContainmentDomainTest.java @@ -6,10 +6,7 @@ */ package com.salesforce.apollo.model; -import com.salesforce.apollo.archipelago.EndpointProvider; -import com.salesforce.apollo.archipelago.LocalServer; -import com.salesforce.apollo.archipelago.Router; -import com.salesforce.apollo.archipelago.ServerConnectionCache; +import com.salesforce.apollo.archipelago.*; import com.salesforce.apollo.choam.Parameters; import com.salesforce.apollo.choam.Parameters.Builder; import com.salesforce.apollo.choam.Parameters.RuntimeParameters; @@ -82,9 +79,11 @@ public void before() throws Exception { var sealed = FoundationSeal.newBuilder().build(); final var group = DigestAlgorithm.DEFAULT.getOrigin(); + var executor = UnsafeExecutors.newVirtualThreadPerTaskExecutor(); identities.forEach((d, id) -> { final var member = new ControlledIdentifierMember(id); - var localRouter = new LocalServer(prefix, member).router(ServerConnectionCache.newBuilder().setTarget(30)); + var localRouter = new LocalServer(prefix, member).router(ServerConnectionCache.newBuilder().setTarget(30), + executor); routers.add(localRouter); var dbUrl = String.format("jdbc:h2:mem:sql-%s-%s;DB_CLOSE_DELAY=-1", member.getId(), UUID.randomUUID()); var pdParams = new ProcessDomain.ProcessDomainParameters(dbUrl, Duration.ofMinutes(1), diff --git a/model/src/test/java/com/salesforce/apollo/model/DomainTest.java b/model/src/test/java/com/salesforce/apollo/model/DomainTest.java index dd5f0503f..3cb29dd42 100644 --- a/model/src/test/java/com/salesforce/apollo/model/DomainTest.java +++ b/model/src/test/java/com/salesforce/apollo/model/DomainTest.java @@ -6,10 +6,7 @@ */ package com.salesforce.apollo.model; -import com.salesforce.apollo.archipelago.EndpointProvider; -import com.salesforce.apollo.archipelago.LocalServer; -import com.salesforce.apollo.archipelago.Router; -import com.salesforce.apollo.archipelago.ServerConnectionCache; +import com.salesforce.apollo.archipelago.*; import com.salesforce.apollo.choam.Parameters; import com.salesforce.apollo.choam.Parameters.Builder; import com.salesforce.apollo.choam.Parameters.RuntimeParameters; @@ -238,9 +235,11 @@ public void before() throws Exception { var sealed = FoundationSeal.newBuilder().build(); final var group = DigestAlgorithm.DEFAULT.getOrigin(); + var executor = UnsafeExecutors.newVirtualThreadPerTaskExecutor(); identities.forEach((d, id) -> { final var member = new ControlledIdentifierMember(id); - var localRouter = new LocalServer(prefix, member).router(ServerConnectionCache.newBuilder().setTarget(30)); + var localRouter = new LocalServer(prefix, member).router(ServerConnectionCache.newBuilder().setTarget(30), + executor); routers.add(localRouter); var dbUrl = String.format("jdbc:h2:mem:sql-%s-%s;DB_CLOSE_DELAY=-1", member.getId(), UUID.randomUUID()); var pdParams = new ProcessDomain.ProcessDomainParameters(dbUrl, Duration.ofMinutes(1), diff --git a/model/src/test/java/com/salesforce/apollo/model/FireFliesTest.java b/model/src/test/java/com/salesforce/apollo/model/FireFliesTest.java index beb936b33..c6a48aa41 100644 --- a/model/src/test/java/com/salesforce/apollo/model/FireFliesTest.java +++ b/model/src/test/java/com/salesforce/apollo/model/FireFliesTest.java @@ -6,10 +6,7 @@ */ package com.salesforce.apollo.model; -import com.salesforce.apollo.archipelago.EndpointProvider; -import com.salesforce.apollo.archipelago.LocalServer; -import com.salesforce.apollo.archipelago.Router; -import com.salesforce.apollo.archipelago.ServerConnectionCache; +import com.salesforce.apollo.archipelago.*; import com.salesforce.apollo.choam.Parameters; import com.salesforce.apollo.choam.Parameters.Builder; import com.salesforce.apollo.choam.Parameters.RuntimeParameters; @@ -81,10 +78,12 @@ public void before() throws Exception { Digest group = DigestAlgorithm.DEFAULT.getOrigin(); var sealed = FoundationSeal.newBuilder().build(); + var executor = UnsafeExecutors.newVirtualThreadPerTaskExecutor(); identities.forEach((digest, id) -> { var context = new DynamicContextImpl<>(DigestAlgorithm.DEFAULT.getLast(), CARDINALITY, 0.2, 3); final var member = new ControlledIdentifierMember(id); - var localRouter = new LocalServer(prefix, member).router(ServerConnectionCache.newBuilder().setTarget(30)); + var localRouter = new LocalServer(prefix, member).router(ServerConnectionCache.newBuilder().setTarget(30), + executor); var dbUrl = String.format("jdbc:h2:mem:sql-%s-%s;DB_CLOSE_DELAY=-1", member.getId(), UUID.randomUUID()); var pdParams = new ProcessDomain.ProcessDomainParameters(dbUrl, Duration.ofSeconds(5), "jdbc:h2:mem:%s-state".formatted(digest), From 91424566919a4e529fe9ca47cc28ebd4f06d9398 Mon Sep 17 00:00:00 2001 From: Hellblazer Date: Thu, 6 Jun 2024 14:08:26 -0700 Subject: [PATCH 11/22] use UE --- .../java/com/salesforce/apollo/fireflies/ChurnTest.java | 8 +++++--- .../java/com/salesforce/apollo/fireflies/MtlsTest.java | 3 ++- .../java/com/salesforce/apollo/fireflies/SwarmTest.java | 8 +++++--- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/fireflies/src/test/java/com/salesforce/apollo/fireflies/ChurnTest.java b/fireflies/src/test/java/com/salesforce/apollo/fireflies/ChurnTest.java index 9825e45f6..64b249b3a 100644 --- a/fireflies/src/test/java/com/salesforce/apollo/fireflies/ChurnTest.java +++ b/fireflies/src/test/java/com/salesforce/apollo/fireflies/ChurnTest.java @@ -273,6 +273,7 @@ private void initialize() { AtomicBoolean frist = new AtomicBoolean(true); final var prefix = UUID.randomUUID().toString(); final var gatewayPrefix = UUID.randomUUID().toString(); + var executor = UnsafeExecutors.newVirtualThreadPerTaskExecutor(); views = members.values().stream().map(node -> { DynamicContext context = ctxBuilder.build(); FireflyMetricsImpl metrics = new FireflyMetricsImpl(context.getId(), @@ -282,14 +283,15 @@ private void initialize() { .setMetrics( new ServerConnectionCacheMetricsImpl( frist.getAndSet(false) ? node0Registry - : registry))); + : registry)), + executor); var gateway = new LocalServer(gatewayPrefix, node).router(ServerConnectionCache.newBuilder() .setTarget(200) .setMetrics( new ServerConnectionCacheMetricsImpl( frist.getAndSet(false) - ? node0Registry - : registry))); + ? node0Registry : registry)), + executor); comms.start(); communications.add(comms); diff --git a/fireflies/src/test/java/com/salesforce/apollo/fireflies/MtlsTest.java b/fireflies/src/test/java/com/salesforce/apollo/fireflies/MtlsTest.java index bc805d703..37cbf6ed7 100644 --- a/fireflies/src/test/java/com/salesforce/apollo/fireflies/MtlsTest.java +++ b/fireflies/src/test/java/com/salesforce/apollo/fireflies/MtlsTest.java @@ -118,6 +118,7 @@ public void smoke() throws Exception { var frist = new AtomicBoolean(true); var clientContextSupplier = clientContextSupplier(); + var executor = UnsafeExecutors.newVirtualThreadPerTaskExecutor(); views = members.stream().map(node -> { DynamicContext context = ctxBuilder.build(); FireflyMetricsImpl metrics = new FireflyMetricsImpl(context.getId(), @@ -127,7 +128,7 @@ public void smoke() throws Exception { builder.setMetrics(new ServerConnectionCacheMetricsImpl(frist.getAndSet(false) ? node0Registry : registry)); CertificateWithPrivateKey certWithKey = certs.get(node.getId()); Router comms = new MtlsServer(node, ep, clientContextSupplier, serverContextSupplier(certWithKey)).router( - builder); + builder, executor); communications.add(comms); return new View(context, node, endpoints.get(node.getId()), EventValidation.NONE, Verifiers.NONE, comms, parameters, DigestAlgorithm.DEFAULT, metrics); diff --git a/fireflies/src/test/java/com/salesforce/apollo/fireflies/SwarmTest.java b/fireflies/src/test/java/com/salesforce/apollo/fireflies/SwarmTest.java index 0e4f93dc6..51a88234d 100644 --- a/fireflies/src/test/java/com/salesforce/apollo/fireflies/SwarmTest.java +++ b/fireflies/src/test/java/com/salesforce/apollo/fireflies/SwarmTest.java @@ -230,19 +230,21 @@ private void initialize() { DynamicContext context = ctxBuilder.build(); FireflyMetricsImpl metrics = new FireflyMetricsImpl(context.getId(), frist.getAndSet(false) ? node0Registry : registry); + var executor = UnsafeExecutors.newVirtualThreadPerTaskExecutor(); var comms = new LocalServer(prefix, node).router(ServerConnectionCache.newBuilder() .setTarget(200) .setMetrics( new ServerConnectionCacheMetricsImpl( frist.getAndSet(false) ? node0Registry - : registry))); + : registry)), + executor); var gateway = new LocalServer(gatewayPrefix, node).router(ServerConnectionCache.newBuilder() .setTarget(200) .setMetrics( new ServerConnectionCacheMetricsImpl( frist.getAndSet(false) - ? node0Registry - : registry))); + ? node0Registry : registry)), + executor); comms.start(); communications.add(comms); From 1317613d23618531c4863bc1c0bb25eb3aabd21b Mon Sep 17 00:00:00 2001 From: Hellblazer Date: Thu, 6 Jun 2024 14:27:18 -0700 Subject: [PATCH 12/22] another attempt --- .../java/com/salesforce/apollo/choam/DynamicTest.java | 4 +--- .../test/java/com/salesforce/apollo/choam/TestCHOAM.java | 8 +++++--- .../java/com/salesforce/apollo/fireflies/ChurnTest.java | 8 +++----- .../java/com/salesforce/apollo/fireflies/SwarmTest.java | 8 +++----- .../java/com/salesforce/apollo/model/DomainTest.java | 9 +++++---- .../java/com/salesforce/apollo/model/FireFliesTest.java | 9 +++++---- pom.xml | 2 +- 7 files changed, 23 insertions(+), 25 deletions(-) diff --git a/choam/src/test/java/com/salesforce/apollo/choam/DynamicTest.java b/choam/src/test/java/com/salesforce/apollo/choam/DynamicTest.java index ebd508af6..fe29a7e45 100644 --- a/choam/src/test/java/com/salesforce/apollo/choam/DynamicTest.java +++ b/choam/src/test/java/com/salesforce/apollo/choam/DynamicTest.java @@ -3,7 +3,6 @@ import com.salesforce.apollo.archipelago.LocalServer; import com.salesforce.apollo.archipelago.Router; import com.salesforce.apollo.archipelago.ServerConnectionCache; -import com.salesforce.apollo.archipelago.UnsafeExecutors; import com.salesforce.apollo.choam.support.ExponentialBackoffPolicy; import com.salesforce.apollo.context.Context; import com.salesforce.apollo.context.DynamicContext; @@ -64,10 +63,9 @@ public void setUp() throws Exception { .toList(); final var prefix = UUID.randomUUID().toString(); - var executor = UnsafeExecutors.newVirtualThreadPerTaskExecutor(); routers = members.stream() .collect(Collectors.toMap(m -> m, m -> new LocalServer(prefix, m).router( - ServerConnectionCache.newBuilder().setTarget(cardinality * 2), executor))); + ServerConnectionCache.newBuilder().setTarget(cardinality * 2)))); var template = Parameters.newBuilder() .setGenerateGenesis(true) diff --git a/choam/src/test/java/com/salesforce/apollo/choam/TestCHOAM.java b/choam/src/test/java/com/salesforce/apollo/choam/TestCHOAM.java index 66d941748..391706bf2 100644 --- a/choam/src/test/java/com/salesforce/apollo/choam/TestCHOAM.java +++ b/choam/src/test/java/com/salesforce/apollo/choam/TestCHOAM.java @@ -8,7 +8,10 @@ import com.codahale.metrics.ConsoleReporter; import com.codahale.metrics.MetricRegistry; -import com.salesforce.apollo.archipelago.*; +import com.salesforce.apollo.archipelago.LocalServer; +import com.salesforce.apollo.archipelago.Router; +import com.salesforce.apollo.archipelago.ServerConnectionCache; +import com.salesforce.apollo.archipelago.ServerConnectionCacheMetricsImpl; import com.salesforce.apollo.choam.CHOAM.TransactionExecutor; import com.salesforce.apollo.choam.Parameters.ProducerParameters; import com.salesforce.apollo.choam.Parameters.RuntimeParameters; @@ -121,12 +124,11 @@ public void before() throws Exception { .toList(); var context = new StaticContext<>(origin, 0.2, members, 3); final var prefix = UUID.randomUUID().toString(); - var executor = UnsafeExecutors.newVirtualThreadPerTaskExecutor(); routers = members.stream() .collect(Collectors.toMap(m -> m.getId(), m -> new LocalServer(prefix, m).router( ServerConnectionCache.newBuilder() .setMetrics(new ServerConnectionCacheMetricsImpl(registry)) - .setTarget(CARDINALITY), executor))); + .setTarget(CARDINALITY)))); choams = members.stream().collect(Collectors.toMap(m -> m.getId(), m -> { var recording = new AtomicInteger(); blocks.put(m.getId(), recording); diff --git a/fireflies/src/test/java/com/salesforce/apollo/fireflies/ChurnTest.java b/fireflies/src/test/java/com/salesforce/apollo/fireflies/ChurnTest.java index 64b249b3a..9825e45f6 100644 --- a/fireflies/src/test/java/com/salesforce/apollo/fireflies/ChurnTest.java +++ b/fireflies/src/test/java/com/salesforce/apollo/fireflies/ChurnTest.java @@ -273,7 +273,6 @@ private void initialize() { AtomicBoolean frist = new AtomicBoolean(true); final var prefix = UUID.randomUUID().toString(); final var gatewayPrefix = UUID.randomUUID().toString(); - var executor = UnsafeExecutors.newVirtualThreadPerTaskExecutor(); views = members.values().stream().map(node -> { DynamicContext context = ctxBuilder.build(); FireflyMetricsImpl metrics = new FireflyMetricsImpl(context.getId(), @@ -283,15 +282,14 @@ private void initialize() { .setMetrics( new ServerConnectionCacheMetricsImpl( frist.getAndSet(false) ? node0Registry - : registry)), - executor); + : registry))); var gateway = new LocalServer(gatewayPrefix, node).router(ServerConnectionCache.newBuilder() .setTarget(200) .setMetrics( new ServerConnectionCacheMetricsImpl( frist.getAndSet(false) - ? node0Registry : registry)), - executor); + ? node0Registry + : registry))); comms.start(); communications.add(comms); diff --git a/fireflies/src/test/java/com/salesforce/apollo/fireflies/SwarmTest.java b/fireflies/src/test/java/com/salesforce/apollo/fireflies/SwarmTest.java index 51a88234d..0e4f93dc6 100644 --- a/fireflies/src/test/java/com/salesforce/apollo/fireflies/SwarmTest.java +++ b/fireflies/src/test/java/com/salesforce/apollo/fireflies/SwarmTest.java @@ -230,21 +230,19 @@ private void initialize() { DynamicContext context = ctxBuilder.build(); FireflyMetricsImpl metrics = new FireflyMetricsImpl(context.getId(), frist.getAndSet(false) ? node0Registry : registry); - var executor = UnsafeExecutors.newVirtualThreadPerTaskExecutor(); var comms = new LocalServer(prefix, node).router(ServerConnectionCache.newBuilder() .setTarget(200) .setMetrics( new ServerConnectionCacheMetricsImpl( frist.getAndSet(false) ? node0Registry - : registry)), - executor); + : registry))); var gateway = new LocalServer(gatewayPrefix, node).router(ServerConnectionCache.newBuilder() .setTarget(200) .setMetrics( new ServerConnectionCacheMetricsImpl( frist.getAndSet(false) - ? node0Registry : registry)), - executor); + ? node0Registry + : registry))); comms.start(); communications.add(comms); diff --git a/model/src/test/java/com/salesforce/apollo/model/DomainTest.java b/model/src/test/java/com/salesforce/apollo/model/DomainTest.java index 3cb29dd42..dd5f0503f 100644 --- a/model/src/test/java/com/salesforce/apollo/model/DomainTest.java +++ b/model/src/test/java/com/salesforce/apollo/model/DomainTest.java @@ -6,7 +6,10 @@ */ package com.salesforce.apollo.model; -import com.salesforce.apollo.archipelago.*; +import com.salesforce.apollo.archipelago.EndpointProvider; +import com.salesforce.apollo.archipelago.LocalServer; +import com.salesforce.apollo.archipelago.Router; +import com.salesforce.apollo.archipelago.ServerConnectionCache; import com.salesforce.apollo.choam.Parameters; import com.salesforce.apollo.choam.Parameters.Builder; import com.salesforce.apollo.choam.Parameters.RuntimeParameters; @@ -235,11 +238,9 @@ public void before() throws Exception { var sealed = FoundationSeal.newBuilder().build(); final var group = DigestAlgorithm.DEFAULT.getOrigin(); - var executor = UnsafeExecutors.newVirtualThreadPerTaskExecutor(); identities.forEach((d, id) -> { final var member = new ControlledIdentifierMember(id); - var localRouter = new LocalServer(prefix, member).router(ServerConnectionCache.newBuilder().setTarget(30), - executor); + var localRouter = new LocalServer(prefix, member).router(ServerConnectionCache.newBuilder().setTarget(30)); routers.add(localRouter); var dbUrl = String.format("jdbc:h2:mem:sql-%s-%s;DB_CLOSE_DELAY=-1", member.getId(), UUID.randomUUID()); var pdParams = new ProcessDomain.ProcessDomainParameters(dbUrl, Duration.ofMinutes(1), diff --git a/model/src/test/java/com/salesforce/apollo/model/FireFliesTest.java b/model/src/test/java/com/salesforce/apollo/model/FireFliesTest.java index c6a48aa41..beb936b33 100644 --- a/model/src/test/java/com/salesforce/apollo/model/FireFliesTest.java +++ b/model/src/test/java/com/salesforce/apollo/model/FireFliesTest.java @@ -6,7 +6,10 @@ */ package com.salesforce.apollo.model; -import com.salesforce.apollo.archipelago.*; +import com.salesforce.apollo.archipelago.EndpointProvider; +import com.salesforce.apollo.archipelago.LocalServer; +import com.salesforce.apollo.archipelago.Router; +import com.salesforce.apollo.archipelago.ServerConnectionCache; import com.salesforce.apollo.choam.Parameters; import com.salesforce.apollo.choam.Parameters.Builder; import com.salesforce.apollo.choam.Parameters.RuntimeParameters; @@ -78,12 +81,10 @@ public void before() throws Exception { Digest group = DigestAlgorithm.DEFAULT.getOrigin(); var sealed = FoundationSeal.newBuilder().build(); - var executor = UnsafeExecutors.newVirtualThreadPerTaskExecutor(); identities.forEach((digest, id) -> { var context = new DynamicContextImpl<>(DigestAlgorithm.DEFAULT.getLast(), CARDINALITY, 0.2, 3); final var member = new ControlledIdentifierMember(id); - var localRouter = new LocalServer(prefix, member).router(ServerConnectionCache.newBuilder().setTarget(30), - executor); + var localRouter = new LocalServer(prefix, member).router(ServerConnectionCache.newBuilder().setTarget(30)); var dbUrl = String.format("jdbc:h2:mem:sql-%s-%s;DB_CLOSE_DELAY=-1", member.getId(), UUID.randomUUID()); var pdParams = new ProcessDomain.ProcessDomainParameters(dbUrl, Duration.ofSeconds(5), "jdbc:h2:mem:%s-state".formatted(digest), diff --git a/pom.xml b/pom.xml index 9e386ff13..557face93 100644 --- a/pom.xml +++ b/pom.xml @@ -784,7 +784,7 @@ ${forks} false - -Xmx10G -Xms100M + -Xmx10G -Xms4G -Djdk.tracePinnedThreads=full From 23cf7ae1d81cf650e2a49e47c97c9971955ab7a7 Mon Sep 17 00:00:00 2001 From: Hellblazer Date: Thu, 6 Jun 2024 14:38:57 -0700 Subject: [PATCH 13/22] yet again --- .../java/com/salesforce/apollo/fireflies/MtlsTest.java | 3 +-- .../salesforce/apollo/model/ContainmentDomainTest.java | 9 +++++---- pom.xml | 4 +--- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/fireflies/src/test/java/com/salesforce/apollo/fireflies/MtlsTest.java b/fireflies/src/test/java/com/salesforce/apollo/fireflies/MtlsTest.java index 37cbf6ed7..bc805d703 100644 --- a/fireflies/src/test/java/com/salesforce/apollo/fireflies/MtlsTest.java +++ b/fireflies/src/test/java/com/salesforce/apollo/fireflies/MtlsTest.java @@ -118,7 +118,6 @@ public void smoke() throws Exception { var frist = new AtomicBoolean(true); var clientContextSupplier = clientContextSupplier(); - var executor = UnsafeExecutors.newVirtualThreadPerTaskExecutor(); views = members.stream().map(node -> { DynamicContext context = ctxBuilder.build(); FireflyMetricsImpl metrics = new FireflyMetricsImpl(context.getId(), @@ -128,7 +127,7 @@ public void smoke() throws Exception { builder.setMetrics(new ServerConnectionCacheMetricsImpl(frist.getAndSet(false) ? node0Registry : registry)); CertificateWithPrivateKey certWithKey = certs.get(node.getId()); Router comms = new MtlsServer(node, ep, clientContextSupplier, serverContextSupplier(certWithKey)).router( - builder, executor); + builder); communications.add(comms); return new View(context, node, endpoints.get(node.getId()), EventValidation.NONE, Verifiers.NONE, comms, parameters, DigestAlgorithm.DEFAULT, metrics); diff --git a/model/src/test/java/com/salesforce/apollo/model/ContainmentDomainTest.java b/model/src/test/java/com/salesforce/apollo/model/ContainmentDomainTest.java index 8df39f7d0..6494cbecb 100644 --- a/model/src/test/java/com/salesforce/apollo/model/ContainmentDomainTest.java +++ b/model/src/test/java/com/salesforce/apollo/model/ContainmentDomainTest.java @@ -6,7 +6,10 @@ */ package com.salesforce.apollo.model; -import com.salesforce.apollo.archipelago.*; +import com.salesforce.apollo.archipelago.EndpointProvider; +import com.salesforce.apollo.archipelago.LocalServer; +import com.salesforce.apollo.archipelago.Router; +import com.salesforce.apollo.archipelago.ServerConnectionCache; import com.salesforce.apollo.choam.Parameters; import com.salesforce.apollo.choam.Parameters.Builder; import com.salesforce.apollo.choam.Parameters.RuntimeParameters; @@ -79,11 +82,9 @@ public void before() throws Exception { var sealed = FoundationSeal.newBuilder().build(); final var group = DigestAlgorithm.DEFAULT.getOrigin(); - var executor = UnsafeExecutors.newVirtualThreadPerTaskExecutor(); identities.forEach((d, id) -> { final var member = new ControlledIdentifierMember(id); - var localRouter = new LocalServer(prefix, member).router(ServerConnectionCache.newBuilder().setTarget(30), - executor); + var localRouter = new LocalServer(prefix, member).router(ServerConnectionCache.newBuilder().setTarget(30)); routers.add(localRouter); var dbUrl = String.format("jdbc:h2:mem:sql-%s-%s;DB_CLOSE_DELAY=-1", member.getId(), UUID.randomUUID()); var pdParams = new ProcessDomain.ProcessDomainParameters(dbUrl, Duration.ofMinutes(1), diff --git a/pom.xml b/pom.xml index 557face93..3632fd893 100644 --- a/pom.xml +++ b/pom.xml @@ -783,9 +783,7 @@ 3.2.5 ${forks} - false - -Xmx10G -Xms4G - -Djdk.tracePinnedThreads=full + true From 96b163574aefe1d92bfd69e053820d32c6cd463e Mon Sep 17 00:00:00 2001 From: Hellblazer Date: Thu, 6 Jun 2024 14:49:30 -0700 Subject: [PATCH 14/22] ChurnTest --- .../test/java/com/salesforce/apollo/fireflies/ChurnTest.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/fireflies/src/test/java/com/salesforce/apollo/fireflies/ChurnTest.java b/fireflies/src/test/java/com/salesforce/apollo/fireflies/ChurnTest.java index 9825e45f6..b1e4cfd05 100644 --- a/fireflies/src/test/java/com/salesforce/apollo/fireflies/ChurnTest.java +++ b/fireflies/src/test/java/com/salesforce/apollo/fireflies/ChurnTest.java @@ -273,6 +273,7 @@ private void initialize() { AtomicBoolean frist = new AtomicBoolean(true); final var prefix = UUID.randomUUID().toString(); final var gatewayPrefix = UUID.randomUUID().toString(); + final var executor = UnsafeExecutors.newVirtualThreadPerTaskExecutor(); views = members.values().stream().map(node -> { DynamicContext context = ctxBuilder.build(); FireflyMetricsImpl metrics = new FireflyMetricsImpl(context.getId(), @@ -282,7 +283,8 @@ private void initialize() { .setMetrics( new ServerConnectionCacheMetricsImpl( frist.getAndSet(false) ? node0Registry - : registry))); + : registry)), + executor); var gateway = new LocalServer(gatewayPrefix, node).router(ServerConnectionCache.newBuilder() .setTarget(200) .setMetrics( From 07832b63133d1956037a1723e7b89f281879fbf0 Mon Sep 17 00:00:00 2001 From: Hellblazer Date: Thu, 6 Jun 2024 14:53:04 -0700 Subject: [PATCH 15/22] ahem no --- .../test/java/com/salesforce/apollo/fireflies/ChurnTest.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/fireflies/src/test/java/com/salesforce/apollo/fireflies/ChurnTest.java b/fireflies/src/test/java/com/salesforce/apollo/fireflies/ChurnTest.java index b1e4cfd05..9825e45f6 100644 --- a/fireflies/src/test/java/com/salesforce/apollo/fireflies/ChurnTest.java +++ b/fireflies/src/test/java/com/salesforce/apollo/fireflies/ChurnTest.java @@ -273,7 +273,6 @@ private void initialize() { AtomicBoolean frist = new AtomicBoolean(true); final var prefix = UUID.randomUUID().toString(); final var gatewayPrefix = UUID.randomUUID().toString(); - final var executor = UnsafeExecutors.newVirtualThreadPerTaskExecutor(); views = members.values().stream().map(node -> { DynamicContext context = ctxBuilder.build(); FireflyMetricsImpl metrics = new FireflyMetricsImpl(context.getId(), @@ -283,8 +282,7 @@ private void initialize() { .setMetrics( new ServerConnectionCacheMetricsImpl( frist.getAndSet(false) ? node0Registry - : registry)), - executor); + : registry))); var gateway = new LocalServer(gatewayPrefix, node).router(ServerConnectionCache.newBuilder() .setTarget(200) .setMetrics( From e14371d785307540a15cf259b85cf190b9073e8b Mon Sep 17 00:00:00 2001 From: Hellblazer Date: Thu, 6 Jun 2024 15:02:50 -0700 Subject: [PATCH 16/22] don't reuse forks --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 3632fd893..da01491d9 100644 --- a/pom.xml +++ b/pom.xml @@ -783,7 +783,7 @@ 3.2.5 ${forks} - true + false From 890d54990b0f81c6efd7f12df21e52aa03788788 Mon Sep 17 00:00:00 2001 From: Hellblazer Date: Thu, 6 Jun 2024 15:30:25 -0700 Subject: [PATCH 17/22] global platform threads --- .../apollo/archipelago/Enclave.java | 2 +- .../apollo/archipelago/LocalServer.java | 18 ++++++++++++----- .../apollo/archipelago/MtlsServer.java | 2 +- .../apollo/archipelago/RouterImpl.java | 20 ++++--------------- 4 files changed, 19 insertions(+), 23 deletions(-) diff --git a/memberships/src/main/java/com/salesforce/apollo/archipelago/Enclave.java b/memberships/src/main/java/com/salesforce/apollo/archipelago/Enclave.java index 08a830894..a7a9c6548 100644 --- a/memberships/src/main/java/com/salesforce/apollo/archipelago/Enclave.java +++ b/memberships/src/main/java/com/salesforce/apollo/archipelago/Enclave.java @@ -109,7 +109,7 @@ public Digest getAgent() { public Digest getFrom() { return Constants.SERVER_CLIENT_ID_KEY.get(); } - }, contextRegistration, validator, executor); + }, contextRegistration, validator); } private ManagedChannel connectTo(Member to) { diff --git a/memberships/src/main/java/com/salesforce/apollo/archipelago/LocalServer.java b/memberships/src/main/java/com/salesforce/apollo/archipelago/LocalServer.java index 781217edb..822b4c79f 100644 --- a/memberships/src/main/java/com/salesforce/apollo/archipelago/LocalServer.java +++ b/memberships/src/main/java/com/salesforce/apollo/archipelago/LocalServer.java @@ -27,6 +27,7 @@ import java.util.List; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.ThreadPoolExecutor; import java.util.function.Predicate; import java.util.function.Supplier; @@ -37,8 +38,16 @@ * @author hal.hildebrand */ public class LocalServer implements RouterSupplier { - private static final Logger log = LoggerFactory.getLogger(LocalServer.class); - private static final String NAME_TEMPLATE = "%s-%s"; + private static final Logger log = LoggerFactory.getLogger(LocalServer.class); + private static final String NAME_TEMPLATE = "%s-%s"; + private static final ExecutorService PLATFORM; + + static { + PLATFORM = Executors.newCachedThreadPool(); + var platform = (ThreadPoolExecutor) PLATFORM; + platform.setCorePoolSize(Runtime.getRuntime().availableProcessors()); + platform.prestartAllCoreThreads(); + } private final ClientInterceptor clientInterceptor; private final Member from; @@ -67,12 +76,11 @@ public Member getFrom() { return from; } - @Override public RouterImpl router(ServerConnectionCache.Builder cacheBuilder, Supplier serverLimit, LimitsRegistry limitsRegistry, List interceptors, Predicate validator, ExecutorService executor) { if (executor == null) { - executor = Executors.newVirtualThreadPerTaskExecutor(); + executor = PLATFORM; } String name = String.format(NAME_TEMPLATE, prefix, qb64(from.getId())); var limitsBuilder = new GrpcServerLimiterBuilder().limit(serverLimit.get()); @@ -97,7 +105,7 @@ public Digest getFrom() { return Constants.SERVER_CLIENT_ID_KEY.get(); } }, d -> { - }, validator, executor); + }, validator); } private ManagedChannel connectTo(Member to) { diff --git a/memberships/src/main/java/com/salesforce/apollo/archipelago/MtlsServer.java b/memberships/src/main/java/com/salesforce/apollo/archipelago/MtlsServer.java index 540d41119..88e79fc34 100644 --- a/memberships/src/main/java/com/salesforce/apollo/archipelago/MtlsServer.java +++ b/memberships/src/main/java/com/salesforce/apollo/archipelago/MtlsServer.java @@ -175,7 +175,7 @@ public Digest getFrom() { } }; return new RouterImpl(from, serverBuilder, cacheBuilder.setFactory(t -> connectTo(t)), identity, c -> { - }, validator, executor); + }, validator); } private ManagedChannel connectTo(Member to) { diff --git a/memberships/src/main/java/com/salesforce/apollo/archipelago/RouterImpl.java b/memberships/src/main/java/com/salesforce/apollo/archipelago/RouterImpl.java index 57705f081..b2f9d3734 100644 --- a/memberships/src/main/java/com/salesforce/apollo/archipelago/RouterImpl.java +++ b/memberships/src/main/java/com/salesforce/apollo/archipelago/RouterImpl.java @@ -23,8 +23,6 @@ import java.time.Duration; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Consumer; @@ -52,36 +50,27 @@ public class RouterImpl implements Router { private final Map> services = new ConcurrentHashMap<>(); private final AtomicBoolean started = new AtomicBoolean(); private final Predicate validator; - private final ExecutorService executor; public RouterImpl(Member from, ServerBuilder serverBuilder, ServerConnectionCache.Builder cacheBuilder, ClientIdentity clientIdentityProvider) { this(from, serverBuilder, cacheBuilder, clientIdentityProvider, d -> { - }, Executors.newVirtualThreadPerTaskExecutor()); - } - - public RouterImpl(Member from, ServerBuilder serverBuilder, ServerConnectionCache.Builder cacheBuilder, - ClientIdentity clientIdentityProvider, ExecutorService executor) { - this(from, serverBuilder, cacheBuilder, clientIdentityProvider, d -> { - }, executor); + }); } public RouterImpl(Member from, ServerBuilder serverBuilder, ServerConnectionCache.Builder cacheBuilder, - ClientIdentity clientIdentityProvider, Consumer contextRegistration, - ExecutorService executor) { - this(from, serverBuilder, cacheBuilder, clientIdentityProvider, contextRegistration, null, executor); + ClientIdentity clientIdentityProvider, Consumer contextRegistration) { + this(from, serverBuilder, cacheBuilder, clientIdentityProvider, contextRegistration, null); } public RouterImpl(Member from, ServerBuilder serverBuilder, ServerConnectionCache.Builder cacheBuilder, ClientIdentity clientIdentityProvider, Consumer contextRegistration, - Predicate validator, ExecutorService executor) { + Predicate validator) { this.server = serverBuilder.fallbackHandlerRegistry(registry).intercept(serverInterceptor()).build(); this.cache = cacheBuilder.clone().setMember(from.getId()).build(); this.clientIdentityProvider = clientIdentityProvider; this.contextRegistration = contextRegistration; this.from = from; this.validator = validator; - this.executor = executor; } public static ClientInterceptor clientInterceptor(Digest ctx) { @@ -135,7 +124,6 @@ public void close(Duration await) { } catch (InterruptedException e) { Thread.currentThread().interrupt(); } - executor.shutdown(); } @Override From e2d375eaf99b8f4d62dc155918ff829189afc4a7 Mon Sep 17 00:00:00 2001 From: Hellblazer Date: Thu, 6 Jun 2024 15:43:23 -0700 Subject: [PATCH 18/22] no globals --- .../apollo/archipelago/LocalServer.java | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/memberships/src/main/java/com/salesforce/apollo/archipelago/LocalServer.java b/memberships/src/main/java/com/salesforce/apollo/archipelago/LocalServer.java index 822b4c79f..530f0e90a 100644 --- a/memberships/src/main/java/com/salesforce/apollo/archipelago/LocalServer.java +++ b/memberships/src/main/java/com/salesforce/apollo/archipelago/LocalServer.java @@ -27,7 +27,6 @@ import java.util.List; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -import java.util.concurrent.ThreadPoolExecutor; import java.util.function.Predicate; import java.util.function.Supplier; @@ -38,16 +37,8 @@ * @author hal.hildebrand */ public class LocalServer implements RouterSupplier { - private static final Logger log = LoggerFactory.getLogger(LocalServer.class); - private static final String NAME_TEMPLATE = "%s-%s"; - private static final ExecutorService PLATFORM; - - static { - PLATFORM = Executors.newCachedThreadPool(); - var platform = (ThreadPoolExecutor) PLATFORM; - platform.setCorePoolSize(Runtime.getRuntime().availableProcessors()); - platform.prestartAllCoreThreads(); - } + private static final Logger log = LoggerFactory.getLogger(LocalServer.class); + private static final String NAME_TEMPLATE = "%s-%s"; private final ClientInterceptor clientInterceptor; private final Member from; @@ -80,7 +71,7 @@ public RouterImpl router(ServerConnectionCache.Builder cacheBuilder, Supplier
  • interceptors, Predicate validator, ExecutorService executor) { if (executor == null) { - executor = PLATFORM; + executor = Executors.newVirtualThreadPerTaskExecutor(); } String name = String.format(NAME_TEMPLATE, prefix, qb64(from.getId())); var limitsBuilder = new GrpcServerLimiterBuilder().limit(serverLimit.get()); From 743132454c9e8de7ed910592ec2b9fb5e7d817ae Mon Sep 17 00:00:00 2001 From: Hellblazer Date: Thu, 6 Jun 2024 16:00:02 -0700 Subject: [PATCH 19/22] lighter weight --- .../src/test/java/com/salesforce/apollo/choam/TestCHOAM.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/choam/src/test/java/com/salesforce/apollo/choam/TestCHOAM.java b/choam/src/test/java/com/salesforce/apollo/choam/TestCHOAM.java index 391706bf2..2a16ea860 100644 --- a/choam/src/test/java/com/salesforce/apollo/choam/TestCHOAM.java +++ b/choam/src/test/java/com/salesforce/apollo/choam/TestCHOAM.java @@ -106,7 +106,7 @@ public void before() throws Exception { .setProducer(ProducerParameters.newBuilder() .setMaxBatchCount(15_000) .setMaxBatchByteSize(200 * 1024 * 1024) - .setGossipDuration(Duration.ofMillis(30)) + .setGossipDuration(Duration.ofMillis(10)) .setBatchInterval(Duration.ofMillis(50)) .setEthereal(Config.newBuilder() .setNumberOfEpochs(3) @@ -171,7 +171,7 @@ public void submitMultiplTxn() throws Exception { final var transactioneers = new ArrayList(); final var clientCount = LARGE_TESTS ? 1_500 : 5; - final var max = LARGE_TESTS ? 100 : 10; + final var max = LARGE_TESTS ? 100 : 5; final var countdown = new CountDownLatch(clientCount * choams.size()); choams.values().forEach(c -> { for (int i = 0; i < clientCount; i++) { From 7c0724e23674a43180f0312b5ec5a113efb55d4c Mon Sep 17 00:00:00 2001 From: Hellblazer Date: Fri, 7 Jun 2024 07:43:17 -0700 Subject: [PATCH 20/22] =?UTF-8?q?test=20opti.=20reuse=20forks=20?= =?UTF-8?q?=F0=9F=A4=9E?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use UE with cached thread pools. tweak gossip durations, some epoch lengths, etc. --- .../salesforce/apollo/choam/DynamicTest.java | 16 ++++++++---- .../salesforce/apollo/choam/TestCHOAM.java | 22 ++++++++-------- .../apollo/fireflies/ChurnTest.java | 13 +++++++--- .../apollo/fireflies/SwarmTest.java | 13 +++++++--- .../apollo/archipelago/UnsafeExecutors.java | 5 +++- .../apollo/model/ContainmentDomainTest.java | 25 +++++++++++-------- .../salesforce/apollo/model/DomainTest.java | 24 ++++++++++-------- .../apollo/model/FireFliesTest.java | 24 ++++++++++-------- pom.xml | 2 +- .../apollo/state/AbstractLifecycleTest.java | 13 ++++++---- .../salesforce/apollo/state/CHOAMTest.java | 11 ++++++-- 11 files changed, 107 insertions(+), 61 deletions(-) diff --git a/choam/src/test/java/com/salesforce/apollo/choam/DynamicTest.java b/choam/src/test/java/com/salesforce/apollo/choam/DynamicTest.java index fe29a7e45..7cb99d6d1 100644 --- a/choam/src/test/java/com/salesforce/apollo/choam/DynamicTest.java +++ b/choam/src/test/java/com/salesforce/apollo/choam/DynamicTest.java @@ -3,6 +3,7 @@ import com.salesforce.apollo.archipelago.LocalServer; import com.salesforce.apollo.archipelago.Router; import com.salesforce.apollo.archipelago.ServerConnectionCache; +import com.salesforce.apollo.archipelago.UnsafeExecutors; import com.salesforce.apollo.choam.support.ExponentialBackoffPolicy; import com.salesforce.apollo.context.Context; import com.salesforce.apollo.context.DynamicContext; @@ -25,6 +26,7 @@ import java.util.List; import java.util.Map; import java.util.UUID; +import java.util.concurrent.ExecutorService; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -41,6 +43,7 @@ public class DynamicTest { private Map routers; private Map choams; private Map> contexts; + private ExecutorService executor; @BeforeEach public void setUp() throws Exception { @@ -61,21 +64,21 @@ public void setUp() throws Exception { .map(ControlledIdentifierMember::new) .map(e -> (Member) e) .toList(); - + executor = UnsafeExecutors.newVirtualThreadPerTaskExecutor(); final var prefix = UUID.randomUUID().toString(); routers = members.stream() .collect(Collectors.toMap(m -> m, m -> new LocalServer(prefix, m).router( - ServerConnectionCache.newBuilder().setTarget(cardinality * 2)))); + ServerConnectionCache.newBuilder().setTarget(cardinality * 2), executor))); var template = Parameters.newBuilder() .setGenerateGenesis(true) .setBootstrap(Parameters.BootstrapParameters.newBuilder() - .setGossipDuration(Duration.ofMillis(50)) + .setGossipDuration(Duration.ofMillis(20)) .build()) .setGenesisViewId(DigestAlgorithm.DEFAULT.getOrigin()) - .setGossipDuration(Duration.ofMillis(50)) + .setGossipDuration(Duration.ofMillis(20)) .setProducer(Parameters.ProducerParameters.newBuilder() - .setGossipDuration(Duration.ofMillis(50)) + .setGossipDuration(Duration.ofMillis(20)) .setBatchInterval(Duration.ofMillis(50)) .setMaxBatchByteSize(1024 * 1024) .setMaxBatchCount(10_000) @@ -216,6 +219,9 @@ public void tearDown() throws Exception { routers = null; } members = null; + if (executor != null) { + executor.shutdown(); + } } private CHOAM constructCHOAM(SigningMember m, Parameters.Builder params, Context context) { diff --git a/choam/src/test/java/com/salesforce/apollo/choam/TestCHOAM.java b/choam/src/test/java/com/salesforce/apollo/choam/TestCHOAM.java index 2a16ea860..64be54d3c 100644 --- a/choam/src/test/java/com/salesforce/apollo/choam/TestCHOAM.java +++ b/choam/src/test/java/com/salesforce/apollo/choam/TestCHOAM.java @@ -8,10 +8,7 @@ import com.codahale.metrics.ConsoleReporter; import com.codahale.metrics.MetricRegistry; -import com.salesforce.apollo.archipelago.LocalServer; -import com.salesforce.apollo.archipelago.Router; -import com.salesforce.apollo.archipelago.ServerConnectionCache; -import com.salesforce.apollo.archipelago.ServerConnectionCacheMetricsImpl; +import com.salesforce.apollo.archipelago.*; import com.salesforce.apollo.choam.CHOAM.TransactionExecutor; import com.salesforce.apollo.choam.Parameters.ProducerParameters; import com.salesforce.apollo.choam.Parameters.RuntimeParameters; @@ -71,6 +68,7 @@ public class TestCHOAM { private MetricRegistry registry; private Map routers; private ScheduledExecutorService scheduler; + private ExecutorService executor; @AfterEach public void after() throws Exception { @@ -85,13 +83,17 @@ public void after() throws Exception { if (scheduler != null) { scheduler.shutdown(); } + if (executor != null) { + executor.shutdown(); + } members = null; registry = null; } @BeforeEach public void before() throws Exception { - scheduler = Executors.newScheduledThreadPool(10); + scheduler = Executors.newScheduledThreadPool(10, Thread.ofVirtual().factory()); + executor = UnsafeExecutors.newVirtualThreadPerTaskExecutor(); var origin = DigestAlgorithm.DEFAULT.getOrigin(); registry = new MetricRegistry(); var metrics = new ChoamMetricsImpl(origin, registry); @@ -102,15 +104,15 @@ public void before() throws Exception { var params = Parameters.newBuilder() .setGenerateGenesis(true) .setGenesisViewId(origin.prefix(entropy.nextLong())) - .setGossipDuration(Duration.ofMillis(30)) + .setGossipDuration(Duration.ofMillis(20)) .setProducer(ProducerParameters.newBuilder() .setMaxBatchCount(15_000) .setMaxBatchByteSize(200 * 1024 * 1024) - .setGossipDuration(Duration.ofMillis(10)) + .setGossipDuration(Duration.ofMillis(20)) .setBatchInterval(Duration.ofMillis(50)) .setEthereal(Config.newBuilder() - .setNumberOfEpochs(3) - .setEpochLength(7)) + .setNumberOfEpochs(12) + .setEpochLength(15)) .build()) .setCheckpointBlockDelta(3); @@ -128,7 +130,7 @@ public void before() throws Exception { .collect(Collectors.toMap(m -> m.getId(), m -> new LocalServer(prefix, m).router( ServerConnectionCache.newBuilder() .setMetrics(new ServerConnectionCacheMetricsImpl(registry)) - .setTarget(CARDINALITY)))); + .setTarget(CARDINALITY), executor))); choams = members.stream().collect(Collectors.toMap(m -> m.getId(), m -> { var recording = new AtomicInteger(); blocks.put(m.getId(), recording); diff --git a/fireflies/src/test/java/com/salesforce/apollo/fireflies/ChurnTest.java b/fireflies/src/test/java/com/salesforce/apollo/fireflies/ChurnTest.java index 9825e45f6..fab24a1bd 100644 --- a/fireflies/src/test/java/com/salesforce/apollo/fireflies/ChurnTest.java +++ b/fireflies/src/test/java/com/salesforce/apollo/fireflies/ChurnTest.java @@ -29,6 +29,7 @@ import java.time.Duration; import java.util.*; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; @@ -52,6 +53,7 @@ public class ChurnTest { private MetricRegistry node0Registry; private MetricRegistry registry; private List views; + private ExecutorService executor; @BeforeAll public static void beforeClass() throws Exception { @@ -79,6 +81,9 @@ public void after() { gateways.forEach(e -> e.close(Duration.ofSeconds(0))); gateways.clear(); + if (executor != null) { + executor.shutdown(); + } } @Test @@ -260,6 +265,7 @@ public void churn() throws Exception { } private void initialize() { + executor = UnsafeExecutors.newVirtualThreadPerTaskExecutor(); var parameters = Parameters.newBuilder().setMaximumTxfr(20).build(); registry = new MetricRegistry(); node0Registry = new MetricRegistry(); @@ -282,14 +288,15 @@ private void initialize() { .setMetrics( new ServerConnectionCacheMetricsImpl( frist.getAndSet(false) ? node0Registry - : registry))); + : registry)), + executor); var gateway = new LocalServer(gatewayPrefix, node).router(ServerConnectionCache.newBuilder() .setTarget(200) .setMetrics( new ServerConnectionCacheMetricsImpl( frist.getAndSet(false) - ? node0Registry - : registry))); + ? node0Registry : registry)), + executor); comms.start(); communications.add(comms); diff --git a/fireflies/src/test/java/com/salesforce/apollo/fireflies/SwarmTest.java b/fireflies/src/test/java/com/salesforce/apollo/fireflies/SwarmTest.java index 0e4f93dc6..632cc0613 100644 --- a/fireflies/src/test/java/com/salesforce/apollo/fireflies/SwarmTest.java +++ b/fireflies/src/test/java/com/salesforce/apollo/fireflies/SwarmTest.java @@ -29,6 +29,7 @@ import java.time.Duration; import java.util.*; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; @@ -62,6 +63,7 @@ public class SwarmTest { private MetricRegistry node0Registry; private MetricRegistry registry; private List views; + private ExecutorService executor; @BeforeAll public static void beforeClass() throws Exception { @@ -89,6 +91,9 @@ public void after() { gateways.forEach(e -> e.close(Duration.ofSeconds(1))); gateways.clear(); + if (executor != null) { + executor.shutdown(); + } } @Test @@ -204,6 +209,7 @@ public void swarm() throws Exception { } private void initialize() { + executor = UnsafeExecutors.newVirtualThreadPerTaskExecutor(); var parameters = Parameters.newBuilder() .setMaxPending(50) .setMaximumTxfr(20) @@ -235,14 +241,15 @@ private void initialize() { .setMetrics( new ServerConnectionCacheMetricsImpl( frist.getAndSet(false) ? node0Registry - : registry))); + : registry)), + executor); var gateway = new LocalServer(gatewayPrefix, node).router(ServerConnectionCache.newBuilder() .setTarget(200) .setMetrics( new ServerConnectionCacheMetricsImpl( frist.getAndSet(false) - ? node0Registry - : registry))); + ? node0Registry : registry)), + executor); comms.start(); communications.add(comms); diff --git a/memberships/src/main/java/com/salesforce/apollo/archipelago/UnsafeExecutors.java b/memberships/src/main/java/com/salesforce/apollo/archipelago/UnsafeExecutors.java index 936f42498..5d4c19057 100644 --- a/memberships/src/main/java/com/salesforce/apollo/archipelago/UnsafeExecutors.java +++ b/memberships/src/main/java/com/salesforce/apollo/archipelago/UnsafeExecutors.java @@ -39,7 +39,10 @@ public class UnsafeExecutors { } public static ExecutorService newVirtualThreadPerTaskExecutor() { - return virtualThreadExecutor(Executors.newWorkStealingPool()); + var executor = (ThreadPoolExecutor) Executors.newCachedThreadPool(); + executor.setCorePoolSize(Runtime.getRuntime().availableProcessors()); + executor.prestartAllCoreThreads(); + return virtualThreadExecutor(executor); } public static B configureBuilderExecutor(B builder, Executor executor) { diff --git a/model/src/test/java/com/salesforce/apollo/model/ContainmentDomainTest.java b/model/src/test/java/com/salesforce/apollo/model/ContainmentDomainTest.java index 6494cbecb..641c211c8 100644 --- a/model/src/test/java/com/salesforce/apollo/model/ContainmentDomainTest.java +++ b/model/src/test/java/com/salesforce/apollo/model/ContainmentDomainTest.java @@ -6,10 +6,7 @@ */ package com.salesforce.apollo.model; -import com.salesforce.apollo.archipelago.EndpointProvider; -import com.salesforce.apollo.archipelago.LocalServer; -import com.salesforce.apollo.archipelago.Router; -import com.salesforce.apollo.archipelago.ServerConnectionCache; +import com.salesforce.apollo.archipelago.*; import com.salesforce.apollo.choam.Parameters; import com.salesforce.apollo.choam.Parameters.Builder; import com.salesforce.apollo.choam.Parameters.RuntimeParameters; @@ -36,6 +33,7 @@ import java.time.Duration; import java.util.ArrayList; import java.util.UUID; +import java.util.concurrent.ExecutorService; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -50,6 +48,7 @@ public class ContainmentDomainTest { "Give me food or give me slack or kill me".getBytes()); private final ArrayList domains = new ArrayList<>(); private final ArrayList routers = new ArrayList<>(); + private ExecutorService executor; @AfterEach public void after() { @@ -57,11 +56,14 @@ public void after() { domains.clear(); routers.forEach(r -> r.close(Duration.ofSeconds(0))); routers.clear(); + if (executor != null) { + executor.shutdown(); + } } @BeforeEach public void before() throws Exception { - + executor = UnsafeExecutors.newVirtualThreadPerTaskExecutor(); final var commsDirectory = Path.of("target/comms"); commsDirectory.toFile().mkdirs(); @@ -84,7 +86,8 @@ public void before() throws Exception { final var group = DigestAlgorithm.DEFAULT.getOrigin(); identities.forEach((d, id) -> { final var member = new ControlledIdentifierMember(id); - var localRouter = new LocalServer(prefix, member).router(ServerConnectionCache.newBuilder().setTarget(30)); + var localRouter = new LocalServer(prefix, member).router(ServerConnectionCache.newBuilder().setTarget(30), + executor); routers.add(localRouter); var dbUrl = String.format("jdbc:h2:mem:sql-%s-%s;DB_CLOSE_DELAY=-1", member.getId(), UUID.randomUUID()); var pdParams = new ProcessDomain.ProcessDomainParameters(dbUrl, Duration.ofMinutes(1), @@ -123,18 +126,18 @@ private Builder params() { .setGenerateGenesis(true) .setGenesisViewId(GENESIS_VIEW_ID) .setBootstrap(Parameters.BootstrapParameters.newBuilder() - .setGossipDuration(Duration.ofMillis(50)) + .setGossipDuration(Duration.ofMillis(20)) .build()) .setGenesisViewId(DigestAlgorithm.DEFAULT.getOrigin()) - .setGossipDuration(Duration.ofMillis(50)) + .setGossipDuration(Duration.ofMillis(20)) .setProducer(Parameters.ProducerParameters.newBuilder() - .setGossipDuration(Duration.ofMillis(50)) + .setGossipDuration(Duration.ofMillis(20)) .setBatchInterval(Duration.ofMillis(50)) .setMaxBatchByteSize(1024 * 1024) .setMaxBatchCount(10_000) .setEthereal(Config.newBuilder() - .setNumberOfEpochs(3) - .setEpochLength(7)) + .setNumberOfEpochs(12) + .setEpochLength(33)) .build()) .setCheckpointBlockDelta(200) .setDrainPolicy(ExponentialBackoffPolicy.newBuilder() diff --git a/model/src/test/java/com/salesforce/apollo/model/DomainTest.java b/model/src/test/java/com/salesforce/apollo/model/DomainTest.java index dd5f0503f..122956188 100644 --- a/model/src/test/java/com/salesforce/apollo/model/DomainTest.java +++ b/model/src/test/java/com/salesforce/apollo/model/DomainTest.java @@ -6,10 +6,7 @@ */ package com.salesforce.apollo.model; -import com.salesforce.apollo.archipelago.EndpointProvider; -import com.salesforce.apollo.archipelago.LocalServer; -import com.salesforce.apollo.archipelago.Router; -import com.salesforce.apollo.archipelago.ServerConnectionCache; +import com.salesforce.apollo.archipelago.*; import com.salesforce.apollo.choam.Parameters; import com.salesforce.apollo.choam.Parameters.Builder; import com.salesforce.apollo.choam.Parameters.RuntimeParameters; @@ -38,6 +35,7 @@ import java.util.Arrays; import java.util.UUID; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @@ -55,6 +53,7 @@ public class DomainTest { "Give me food or give me slack or kill me".getBytes()); private final ArrayList domains = new ArrayList<>(); private final ArrayList routers = new ArrayList<>(); + private ExecutorService executor; public static void smoke(Oracle oracle) throws Exception { // Namespace @@ -217,10 +216,14 @@ public void after() { domains.clear(); routers.forEach(r -> r.close(Duration.ofSeconds(0))); routers.clear(); + if (executor != null) { + executor.shutdown(); + } } @BeforeEach public void before() throws Exception { + executor = UnsafeExecutors.newVirtualThreadPerTaskExecutor(); var ffParams = com.salesforce.apollo.fireflies.Parameters.newBuilder(); var entropy = SecureRandom.getInstance("SHA1PRNG"); entropy.setSeed(new byte[] { 6, 6, 6 }); @@ -240,7 +243,8 @@ public void before() throws Exception { final var group = DigestAlgorithm.DEFAULT.getOrigin(); identities.forEach((d, id) -> { final var member = new ControlledIdentifierMember(id); - var localRouter = new LocalServer(prefix, member).router(ServerConnectionCache.newBuilder().setTarget(30)); + var localRouter = new LocalServer(prefix, member).router(ServerConnectionCache.newBuilder().setTarget(30), + executor); routers.add(localRouter); var dbUrl = String.format("jdbc:h2:mem:sql-%s-%s;DB_CLOSE_DELAY=-1", member.getId(), UUID.randomUUID()); var pdParams = new ProcessDomain.ProcessDomainParameters(dbUrl, Duration.ofMinutes(1), @@ -280,18 +284,18 @@ private Builder params() { .setGenerateGenesis(true) .setGenesisViewId(GENESIS_VIEW_ID) .setBootstrap(Parameters.BootstrapParameters.newBuilder() - .setGossipDuration(Duration.ofMillis(50)) + .setGossipDuration(Duration.ofMillis(20)) .build()) .setGenesisViewId(DigestAlgorithm.DEFAULT.getOrigin()) - .setGossipDuration(Duration.ofMillis(50)) + .setGossipDuration(Duration.ofMillis(20)) .setProducer(Parameters.ProducerParameters.newBuilder() - .setGossipDuration(Duration.ofMillis(50)) + .setGossipDuration(Duration.ofMillis(20)) .setBatchInterval(Duration.ofMillis(50)) .setMaxBatchByteSize(1024 * 1024) .setMaxBatchCount(10_000) .setEthereal(Config.newBuilder() - .setNumberOfEpochs(3) - .setEpochLength(7)) + .setNumberOfEpochs(12) + .setEpochLength(33)) .build()) .setCheckpointBlockDelta(200) .setDrainPolicy(ExponentialBackoffPolicy.newBuilder() diff --git a/model/src/test/java/com/salesforce/apollo/model/FireFliesTest.java b/model/src/test/java/com/salesforce/apollo/model/FireFliesTest.java index beb936b33..867d8d4d0 100644 --- a/model/src/test/java/com/salesforce/apollo/model/FireFliesTest.java +++ b/model/src/test/java/com/salesforce/apollo/model/FireFliesTest.java @@ -6,10 +6,7 @@ */ package com.salesforce.apollo.model; -import com.salesforce.apollo.archipelago.EndpointProvider; -import com.salesforce.apollo.archipelago.LocalServer; -import com.salesforce.apollo.archipelago.Router; -import com.salesforce.apollo.archipelago.ServerConnectionCache; +import com.salesforce.apollo.archipelago.*; import com.salesforce.apollo.choam.Parameters; import com.salesforce.apollo.choam.Parameters.Builder; import com.salesforce.apollo.choam.Parameters.RuntimeParameters; @@ -37,6 +34,7 @@ import java.time.Duration; import java.util.*; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; import java.util.function.BiConsumer; @@ -55,6 +53,7 @@ public class FireFliesTest { private final List domains = new ArrayList<>(); private final Map routers = new HashMap<>(); + private ExecutorService executor; @AfterEach public void after() { @@ -62,10 +61,14 @@ public void after() { domains.clear(); routers.values().forEach(r -> r.close(Duration.ofSeconds(0))); routers.clear(); + if (executor != null) { + executor.shutdown(); + } } @BeforeEach public void before() throws Exception { + executor = UnsafeExecutors.newVirtualThreadPerTaskExecutor(); var ffParams = com.salesforce.apollo.fireflies.Parameters.newBuilder(); var entropy = SecureRandom.getInstance("SHA1PRNG"); entropy.setSeed(new byte[] { 6, 6, 6 }); @@ -84,7 +87,8 @@ public void before() throws Exception { identities.forEach((digest, id) -> { var context = new DynamicContextImpl<>(DigestAlgorithm.DEFAULT.getLast(), CARDINALITY, 0.2, 3); final var member = new ControlledIdentifierMember(id); - var localRouter = new LocalServer(prefix, member).router(ServerConnectionCache.newBuilder().setTarget(30)); + var localRouter = new LocalServer(prefix, member).router(ServerConnectionCache.newBuilder().setTarget(30), + executor); var dbUrl = String.format("jdbc:h2:mem:sql-%s-%s;DB_CLOSE_DELAY=-1", member.getId(), UUID.randomUUID()); var pdParams = new ProcessDomain.ProcessDomainParameters(dbUrl, Duration.ofSeconds(5), "jdbc:h2:mem:%s-state".formatted(digest), @@ -175,18 +179,18 @@ private Builder params() { .setGenerateGenesis(true) .setGenesisViewId(GENESIS_VIEW_ID) .setBootstrap(Parameters.BootstrapParameters.newBuilder() - .setGossipDuration(Duration.ofMillis(50)) + .setGossipDuration(Duration.ofMillis(20)) .build()) .setGenesisViewId(DigestAlgorithm.DEFAULT.getOrigin()) - .setGossipDuration(Duration.ofMillis(50)) + .setGossipDuration(Duration.ofMillis(20)) .setProducer(Parameters.ProducerParameters.newBuilder() - .setGossipDuration(Duration.ofMillis(50)) + .setGossipDuration(Duration.ofMillis(20)) .setBatchInterval(Duration.ofMillis(50)) .setMaxBatchByteSize(1024 * 1024) .setMaxBatchCount(10_000) .setEthereal(Config.newBuilder() - .setNumberOfEpochs(3) - .setEpochLength(7)) + .setNumberOfEpochs(12) + .setEpochLength(33)) .build()) .setCheckpointBlockDelta(200) .setDrainPolicy(ExponentialBackoffPolicy.newBuilder() diff --git a/pom.xml b/pom.xml index da01491d9..3632fd893 100644 --- a/pom.xml +++ b/pom.xml @@ -783,7 +783,7 @@ 3.2.5 ${forks} - false + true diff --git a/sql-state/src/test/java/com/salesforce/apollo/state/AbstractLifecycleTest.java b/sql-state/src/test/java/com/salesforce/apollo/state/AbstractLifecycleTest.java index 028f4b67b..2ababf109 100644 --- a/sql-state/src/test/java/com/salesforce/apollo/state/AbstractLifecycleTest.java +++ b/sql-state/src/test/java/com/salesforce/apollo/state/AbstractLifecycleTest.java @@ -9,6 +9,7 @@ import com.salesforce.apollo.archipelago.LocalServer; import com.salesforce.apollo.archipelago.Router; import com.salesforce.apollo.archipelago.ServerConnectionCache; +import com.salesforce.apollo.archipelago.UnsafeExecutors; import com.salesforce.apollo.choam.CHOAM; import com.salesforce.apollo.choam.Parameters; import com.salesforce.apollo.choam.Parameters.BootstrapParameters; @@ -41,10 +42,7 @@ import java.sql.Statement; import java.time.Duration; import java.util.*; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; +import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Function; import java.util.stream.Collectors; @@ -75,6 +73,7 @@ abstract public class AbstractLifecycleTest { // } private final List GENESIS_DATA; private final Map parameters = new HashMap<>(); + protected ExecutorService executor; protected SecureRandom entropy; protected CountDownLatch checkpointOccurred; protected Map choams; @@ -122,6 +121,9 @@ public void after() throws Exception { scheduler.shutdownNow(); scheduler = null; } + if (executor != null) { + executor.shutdown(); + } updaters.values().forEach(up -> up.close()); updaters.clear(); parameters.clear(); @@ -132,7 +134,8 @@ public void after() throws Exception { @BeforeEach public void before() throws Exception { - scheduler = Executors.newScheduledThreadPool(10); + scheduler = Executors.newScheduledThreadPool(10, Thread.ofVirtual().factory()); + executor = UnsafeExecutors.newVirtualThreadPerTaskExecutor(); checkpointOccurred = new CountDownLatch(CARDINALITY); checkpointDirBase = new File("target/ct-chkpoints-" + Entropy.nextBitsStreamLong()); Utils.clean(checkpointDirBase); diff --git a/sql-state/src/test/java/com/salesforce/apollo/state/CHOAMTest.java b/sql-state/src/test/java/com/salesforce/apollo/state/CHOAMTest.java index e8cb71761..1ec5449d1 100644 --- a/sql-state/src/test/java/com/salesforce/apollo/state/CHOAMTest.java +++ b/sql-state/src/test/java/com/salesforce/apollo/state/CHOAMTest.java @@ -11,6 +11,7 @@ import com.salesforce.apollo.archipelago.LocalServer; import com.salesforce.apollo.archipelago.Router; import com.salesforce.apollo.archipelago.ServerConnectionCache; +import com.salesforce.apollo.archipelago.UnsafeExecutors; import com.salesforce.apollo.choam.CHOAM; import com.salesforce.apollo.choam.CHOAM.TransactionExecutor; import com.salesforce.apollo.choam.Parameters; @@ -82,6 +83,7 @@ public class CHOAMTest { private MetricRegistry registry; private Map routers; private ScheduledExecutorService scheduler; + private ExecutorService executor; private static Txn initialInsert() { return Txn.newBuilder() @@ -107,6 +109,9 @@ public void after() throws Exception { scheduler.shutdownNow(); scheduler = null; } + if (executor != null) { + executor.shutdown(); + } updaters.values().forEach(up -> up.close()); updaters.clear(); members = null; @@ -122,7 +127,8 @@ public void after() throws Exception { @BeforeEach public void before() throws Exception { - scheduler = Executors.newScheduledThreadPool(10); + scheduler = Executors.newScheduledThreadPool(10, Thread.ofVirtual().factory()); + executor = UnsafeExecutors.newVirtualThreadPerTaskExecutor(); registry = new MetricRegistry(); checkpointDirBase = new File("target/ct-chkpoints-" + Entropy.nextBitsStreamLong()); Utils.clean(checkpointDirBase); @@ -155,7 +161,8 @@ public void before() throws Exception { members.forEach(m -> context.activate(m)); final var prefix = UUID.randomUUID().toString(); routers = members.stream().collect(Collectors.toMap(m -> m.getId(), m -> { - var localRouter = new LocalServer(prefix, m).router(ServerConnectionCache.newBuilder().setTarget(30)); + var localRouter = new LocalServer(prefix, m).router(ServerConnectionCache.newBuilder().setTarget(30), + executor); return localRouter; })); choams = members.stream() From 1fdeaea4e17b453f9cc9a46252e25597f7be47cc Mon Sep 17 00:00:00 2001 From: Hellblazer Date: Fri, 7 Jun 2024 07:49:03 -0700 Subject: [PATCH 21/22] don't reuse forks --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 3632fd893..da01491d9 100644 --- a/pom.xml +++ b/pom.xml @@ -783,7 +783,7 @@ 3.2.5 ${forks} - true + false From 329f6424edfbe90080b977df8d1a804af5128f22 Mon Sep 17 00:00:00 2001 From: Hellblazer Date: Fri, 7 Jun 2024 08:19:33 -0700 Subject: [PATCH 22/22] mtls uses plat cached threads by default --- .../test/java/com/salesforce/apollo/fireflies/MtlsTest.java | 3 ++- .../java/com/salesforce/apollo/archipelago/MtlsServer.java | 3 +-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/fireflies/src/test/java/com/salesforce/apollo/fireflies/MtlsTest.java b/fireflies/src/test/java/com/salesforce/apollo/fireflies/MtlsTest.java index bc805d703..36a616dd3 100644 --- a/fireflies/src/test/java/com/salesforce/apollo/fireflies/MtlsTest.java +++ b/fireflies/src/test/java/com/salesforce/apollo/fireflies/MtlsTest.java @@ -39,6 +39,7 @@ import java.time.Instant; import java.util.*; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; @@ -127,7 +128,7 @@ public void smoke() throws Exception { builder.setMetrics(new ServerConnectionCacheMetricsImpl(frist.getAndSet(false) ? node0Registry : registry)); CertificateWithPrivateKey certWithKey = certs.get(node.getId()); Router comms = new MtlsServer(node, ep, clientContextSupplier, serverContextSupplier(certWithKey)).router( - builder); + builder, Executors.newVirtualThreadPerTaskExecutor()); communications.add(comms); return new View(context, node, endpoints.get(node.getId()), EventValidation.NONE, Verifiers.NONE, comms, parameters, DigestAlgorithm.DEFAULT, metrics); diff --git a/memberships/src/main/java/com/salesforce/apollo/archipelago/MtlsServer.java b/memberships/src/main/java/com/salesforce/apollo/archipelago/MtlsServer.java index 88e79fc34..4cf97b97a 100644 --- a/memberships/src/main/java/com/salesforce/apollo/archipelago/MtlsServer.java +++ b/memberships/src/main/java/com/salesforce/apollo/archipelago/MtlsServer.java @@ -44,7 +44,6 @@ import java.util.List; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; import java.util.function.Function; import java.util.function.Predicate; import java.util.function.Supplier; @@ -142,7 +141,7 @@ public RouterImpl router(ServerConnectionCache.Builder cacheBuilder, Supplier
  • interceptors, Predicate validator, ExecutorService executor) { if (executor == null) { - executor = Executors.newVirtualThreadPerTaskExecutor(); + executor = UnsafeExecutors.newVirtualThreadPerTaskExecutor(); } var limitsBuilder = new GrpcServerLimiterBuilder().limit(serverLimit.get()); if (limitsRegistry != null) {