From 06ffbee57c439760a15e80f523260945192206fe Mon Sep 17 00:00:00 2001 From: Koushik R Date: Tue, 9 Jul 2024 11:55:39 +0530 Subject: [PATCH 1/7] a) Making ServiceFinderHub Start and ServiceFinderAddition bounded. Pertaining to https://github.com/appform-io/ranger/pull/22/ b) Introduced hubStartDurationMs and finderRefreshDurationMs for defining the boundary. If this boundary were to breach, throw with an IllegalStateException c) Added tests, upgraded versions and added CHANGELOG --- CHANGELOG.md | 4 ++ pom.xml | 2 +- ranger-client/pom.xml | 2 +- .../client/AbstractRangerHubClient.java | 4 +- .../ranger/client/RangerHubClient.java | 3 +- ranger-core/pom.xml | 2 +- .../core/finderhub/ServiceFinderHub.java | 63 ++++++++++++++----- .../finderhub/ServiceFinderHubBuilder.java | 15 ++++- .../core/finderhub/ServiceFinderHubTest.java | 44 +++++++++---- ranger-http-client/pom.xml | 2 +- ranger-http-model/pom.xml | 2 +- ranger-http/pom.xml | 2 +- ranger-hub-server-bundle/pom.xml | 2 +- ranger-server-bundle/pom.xml | 2 +- ranger-server-common/pom.xml | 2 +- ranger-server/pom.xml | 2 +- ranger-zk-client/pom.xml | 2 +- ranger-zookeeper/pom.xml | 2 +- 18 files changed, 113 insertions(+), 44 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index da7a88c4..2ccdca2f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,10 @@ # Changelog All notable changes to this project will be documented in this file. +## [1.1-RC2] + +- Pertaining to PR : https://github.com/appform-io/ranger/pull/22/, building of a ServiceFinderHub and a ServiceFinder are bounded. + ## [1.0-RC18] - Version bump to release lexicographically higher version than 1.0-dw3-RC17 diff --git a/pom.xml b/pom.xml index c3a75300..ce453730 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ io.appform.ranger ranger pom - 1.1-RC1 + 1.1-RC2 Ranger https://github.com/appform-io/ranger Service Discovery for Java diff --git a/ranger-client/pom.xml b/ranger-client/pom.xml index 7967bf0a..e842fa5b 100644 --- a/ranger-client/pom.xml +++ b/ranger-client/pom.xml @@ -5,7 +5,7 @@ ranger io.appform.ranger - 1.1-RC1 + 1.1-RC2 4.0.0 diff --git a/ranger-client/src/main/java/io/appform/ranger/client/AbstractRangerHubClient.java b/ranger-client/src/main/java/io/appform/ranger/client/AbstractRangerHubClient.java index 66d20017..2003ba81 100644 --- a/ranger-client/src/main/java/io/appform/ranger/client/AbstractRangerHubClient.java +++ b/ranger-client/src/main/java/io/appform/ranger/client/AbstractRangerHubClient.java @@ -18,6 +18,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Preconditions; import io.appform.ranger.client.utils.CriteriaUtils; +import io.appform.ranger.core.finder.ServiceFinder; import io.appform.ranger.core.finderhub.ServiceDataSource; import io.appform.ranger.core.finderhub.ServiceFinderFactory; import io.appform.ranger.core.finderhub.ServiceFinderHub; @@ -31,7 +32,6 @@ import java.util.Collections; import java.util.List; import java.util.Optional; -import java.util.concurrent.CompletableFuture; import java.util.function.Predicate; import lombok.Getter; import lombok.experimental.SuperBuilder; @@ -160,7 +160,7 @@ public Collection getRegisteredServices() { * @return CompletableFuture which waits for hub to be ready for discovering the new service */ @Override - public CompletableFuture addService(Service service) { + public ServiceFinder addService(Service service) { if(hub == null) { throw new IllegalStateException("Hub not started yet. Call .start()"); } diff --git a/ranger-client/src/main/java/io/appform/ranger/client/RangerHubClient.java b/ranger-client/src/main/java/io/appform/ranger/client/RangerHubClient.java index 2094c2d8..7f2dc0fe 100644 --- a/ranger-client/src/main/java/io/appform/ranger/client/RangerHubClient.java +++ b/ranger-client/src/main/java/io/appform/ranger/client/RangerHubClient.java @@ -15,6 +15,7 @@ */ package io.appform.ranger.client; +import io.appform.ranger.core.finder.ServiceFinder; import io.appform.ranger.core.model.Service; import io.appform.ranger.core.model.ServiceNode; import io.appform.ranger.core.model.ServiceNodeSelector; @@ -33,7 +34,7 @@ public interface RangerHubClient> { Collection getRegisteredServices(); - CompletableFuture addService(Service service); + ServiceFinder addService(Service service); Optional> getNode(final Service service); diff --git a/ranger-core/pom.xml b/ranger-core/pom.xml index fbf825b6..2d043721 100644 --- a/ranger-core/pom.xml +++ b/ranger-core/pom.xml @@ -5,7 +5,7 @@ ranger io.appform.ranger - 1.1-RC1 + 1.1-RC2 4.0.0 diff --git a/ranger-core/src/main/java/io/appform/ranger/core/finderhub/ServiceFinderHub.java b/ranger-core/src/main/java/io/appform/ranger/core/finderhub/ServiceFinderHub.java index 4246dd0e..0c788bb9 100644 --- a/ranger-core/src/main/java/io/appform/ranger/core/finderhub/ServiceFinderHub.java +++ b/ranger-core/src/main/java/io/appform/ranger/core/finderhub/ServiceFinderHub.java @@ -16,7 +16,9 @@ package io.appform.ranger.core.finderhub; import com.github.rholder.retry.RetryerBuilder; +import com.github.rholder.retry.StopStrategies; import com.google.common.base.Stopwatch; +import com.google.common.base.Supplier; import io.appform.ranger.core.finder.ServiceFinder; import io.appform.ranger.core.model.Service; import io.appform.ranger.core.model.ServiceRegistry; @@ -66,11 +68,26 @@ public class ServiceFinderHub> { private final AtomicBoolean alreadyUpdating = new AtomicBoolean(false); private Future monitorFuture = null; + private final long serviceRefreshDurationMs; + private final long hubRefreshDurationMs; + public ServiceFinderHub( ServiceDataSource serviceDataSource, - ServiceFinderFactory finderFactory) { + ServiceFinderFactory finderFactory + ) { + this(serviceDataSource, finderFactory, + 10_000, 30_000); + } + + public ServiceFinderHub( + ServiceDataSource serviceDataSource, + ServiceFinderFactory finderFactory, + long serviceRefreshDurationMs, + long hubRefreshDurationMs) { this.serviceDataSource = serviceDataSource; this.finderFactory = finderFactory; + this.serviceRefreshDurationMs = serviceRefreshDurationMs; + this.hubRefreshDurationMs = hubRefreshDurationMs; this.refreshSignals.add(new ScheduledSignal<>("service-hub-updater", () -> null, Collections.emptyList(), @@ -81,22 +98,20 @@ public Optional> finder(final Service service) { return Optional.ofNullable(finders.get().get(service)); } - public CompletableFuture> buildFinder(final Service service) { + public ServiceFinder buildFinder(final Service service) { val finder = finders.get().get(service); - if (finder != null) { - return CompletableFuture.completedFuture(finder); + if (null != finder) { + return finder; } serviceDataSource.add(service); - return CompletableFuture.supplyAsync(() -> { - try { - updateAvailable(); - waitTillServiceIsReady(service); - return finders.get().get(service); - } catch(Exception e) { - log.warn("Exception whiling building finder", e); - throw e; - } - }); + try { + updateAvailable(); + waitTillServiceIsReady(service); + return finders.get().get(service); + } catch(Exception e) { + log.warn("Exception whiling building finder", e); + throw e; + } } public void start() { @@ -190,13 +205,31 @@ private void updateRegistry() { } private void waitTillHubIsReady() { - serviceDataSource.services().forEach(this::waitTillServiceIsReady); + val hubRefresher = CompletableFuture.allOf( + serviceDataSource.services() + .stream() + .map(service -> CompletableFuture.supplyAsync((Supplier) () -> { + waitTillServiceIsReady(service); + return null; + })).toArray(CompletableFuture[]::new) + ); + try { + hubRefresher.get(hubRefreshDurationMs, TimeUnit.MILLISECONDS); + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + Exceptions + .illegalState("Couldn't perform hub refresh at this time. Refresh exceeded the start up time specified"); + } catch (Exception e) { + Exceptions + .illegalState("Couldn't perform hub refresh at this time", e); + } } private void waitTillServiceIsReady(Service service) { try { RetryerBuilder.newBuilder() .retryIfResult(r -> !r) + .withStopStrategy(StopStrategies.stopAfterDelay(serviceRefreshDurationMs, TimeUnit.MILLISECONDS)) .build() .call(() -> Optional.ofNullable(getFinders().get().get(service)) .map(ServiceFinder::getServiceRegistry) diff --git a/ranger-core/src/main/java/io/appform/ranger/core/finderhub/ServiceFinderHubBuilder.java b/ranger-core/src/main/java/io/appform/ranger/core/finderhub/ServiceFinderHubBuilder.java index 8f2e483c..e2b554df 100644 --- a/ranger-core/src/main/java/io/appform/ranger/core/finderhub/ServiceFinderHubBuilder.java +++ b/ranger-core/src/main/java/io/appform/ranger/core/finderhub/ServiceFinderHubBuilder.java @@ -37,6 +37,8 @@ public abstract class ServiceFinderHubBuilder> { private final List> extraStartSignalConsumers = new ArrayList<>(); private final List> extraStopSignalConsumers = new ArrayList<>(); private final List> extraRefreshSignals = new ArrayList<>(); + private long serviceRefreshDurationMs = 10_000; + private long hubRefreshDurationMs = 30_000; public ServiceFinderHubBuilder withServiceDataSource(ServiceDataSource serviceDataSource) { this.serviceDataSource = serviceDataSource; @@ -68,12 +70,23 @@ public ServiceFinderHubBuilder withExtraRefreshSignal(Signal extraRe return this; } + public ServiceFinderHubBuilder withServiceRefreshDuration(long serviceRefreshDurationMs) { + this.serviceRefreshDurationMs = serviceRefreshDurationMs; + return this; + } + + public ServiceFinderHubBuilder withHubRefreshDuration(long hubRefreshDurationMs) { + this.hubRefreshDurationMs = hubRefreshDurationMs; + return this; + } + public ServiceFinderHub build() { preBuild(); Preconditions.checkNotNull(serviceDataSource, "Provide a non-null service data source"); Preconditions.checkNotNull(serviceFinderFactory, "Provide a non-null service finder factory"); - val hub = new ServiceFinderHub<>(serviceDataSource, serviceFinderFactory); + val hub = new ServiceFinderHub<>(serviceDataSource, serviceFinderFactory, + serviceRefreshDurationMs, hubRefreshDurationMs); final ScheduledSignal refreshSignal = new ScheduledSignal<>("service-hub-refresh-timer", () -> null, Collections.emptyList(), diff --git a/ranger-core/src/test/java/io/appform/ranger/core/finderhub/ServiceFinderHubTest.java b/ranger-core/src/test/java/io/appform/ranger/core/finderhub/ServiceFinderHubTest.java index cc76b11c..30f6bdc4 100644 --- a/ranger-core/src/test/java/io/appform/ranger/core/finderhub/ServiceFinderHubTest.java +++ b/ranger-core/src/test/java/io/appform/ranger/core/finderhub/ServiceFinderHubTest.java @@ -11,9 +11,12 @@ import io.appform.ranger.core.model.*; import io.appform.ranger.core.units.TestNodeData; import java.util.Optional; + +import io.appform.ranger.core.utils.RangerTestUtils; import lombok.val; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.function.Executable; import java.util.ArrayList; import java.util.HashSet; @@ -41,9 +44,7 @@ void testDynamicServiceAddition() { Assertions.assertEquals("HOST", node.get().getHost()); Assertions.assertEquals(0, node.get().getPort()); - serviceFinderHub.buildFinder(new Service("NS", "SERVICE")).join(); - val dynamicServiceFinder = serviceFinderHub.finder(new Service("NS", "SERVICE")) - .orElseThrow(() -> new IllegalStateException("Finder should be present")); + val dynamicServiceFinder = serviceFinderHub.buildFinder(new Service("NS", "SERVICE")); val dynamicServiceNode = dynamicServiceFinder.get(null, (criteria, serviceRegistry) -> serviceRegistry.nodeList()); Assertions.assertTrue(dynamicServiceNode.isPresent()); Assertions.assertEquals("HOST", dynamicServiceNode.get().getHost()); @@ -51,20 +52,31 @@ void testDynamicServiceAddition() { } @Test - void testDynamicServiceAdditionAsync() throws InterruptedException { + void testDelayedServiceAddition() { + val delayedHub = new ServiceFinderHub<>(new DynamicDataSource(Lists.newArrayList(new Service("NS", "SERVICE"))), + service -> new TestServiceFinderBuilder() + .withNamespace(service.getNamespace()) + .withServiceName(service.getServiceName()) + .withDeserializer(new Deserializer() {}) + .withSleepDuration(5) + .build(), 1_000, 5_000 + ); + Assertions.assertThrows(IllegalStateException.class, delayedHub::start); + val serviceFinderHub = new ServiceFinderHub<>(new DynamicDataSource(Lists.newArrayList(new Service("NS", "SERVICE"))), + service -> new TestServiceFinderBuilder() + .withNamespace(service.getNamespace()) + .withServiceName(service.getServiceName()) + .withDeserializer(new Deserializer() {}) + .withSleepDuration(1) + .build(), 2_000, 5_000 + ); serviceFinderHub.start(); - serviceFinderHub.buildFinder(new Service("NS", "SERVICE_NAME")); - val finderOpt = serviceFinderHub.finder(new Service("NS", "SERVICE_NAME")); - Assertions.assertFalse(finderOpt.isPresent(), "Finders will not be availbale immediately"); - Thread.sleep(1000); - val finderAfterWaitOpt = serviceFinderHub.finder(new Service("NS", "SERVICE_NAME")); - Assertions.assertTrue(finderAfterWaitOpt.isPresent(), "Finders should be availble after some time"); + Assertions.assertTrue(serviceFinderHub.finder(new Service("NS", "SERVICE")).isPresent()); } @Test void testDynamicServiceAdditionWithNonDynamicDataSource() { - val serviceFinderHub = new ServiceFinderHub<>(new StaticDataSource(new HashSet<>()), service -> new TestServiceFinderBuilder() .withNamespace(service.getNamespace()) .withServiceName(service.getServiceName()) @@ -73,8 +85,7 @@ void testDynamicServiceAdditionWithNonDynamicDataSource() { .build()); serviceFinderHub.start(); try { - val future = serviceFinderHub.buildFinder(new Service("NS", "SERVICE_NAME")); - future.join(); + serviceFinderHub.buildFinder(new Service("NS", "SERVICE_NAME")); Assertions.fail("Exception should have been thrown"); } catch (Exception exception) { Assertions.assertTrue(exception instanceof UnsupportedOperationException, "Unsupported exception should be thrown"); @@ -83,6 +94,8 @@ void testDynamicServiceAdditionWithNonDynamicDataSource() { private static class TestServiceFinderBuilder extends BaseServiceFinderBuilder, ServiceFinder>, TestServiceFinderBuilder, Deserializer> { + private int finderSleepDurationSeconds = 0; + @Override public ServiceFinder> build() { val bf = buildFinder(); @@ -97,11 +110,16 @@ protected NodeDataSource> dataSource(Se @Override protected ServiceFinder> buildFinder(Service service, ShardSelector> shardSelector, ServiceNodeSelector nodeSelector) { + RangerTestUtils.sleepUntil(finderSleepDurationSeconds); if (null == shardSelector) { shardSelector = new MatchingShardSelector<>(); } return new SimpleShardedServiceFinder<>(new MapBasedServiceRegistry<>(service), shardSelector, nodeSelector); + } + public TestServiceFinderBuilder withSleepDuration(final int finderSleepDurationSeconds) { + this.finderSleepDurationSeconds = finderSleepDurationSeconds; + return this; } private static class TestNodeDataSource implements NodeDataSource> { diff --git a/ranger-http-client/pom.xml b/ranger-http-client/pom.xml index 286dda72..3c6a66b3 100644 --- a/ranger-http-client/pom.xml +++ b/ranger-http-client/pom.xml @@ -5,7 +5,7 @@ ranger io.appform.ranger - 1.1-RC1 + 1.1-RC2 4.0.0 diff --git a/ranger-http-model/pom.xml b/ranger-http-model/pom.xml index 79f859cb..2c84e62f 100644 --- a/ranger-http-model/pom.xml +++ b/ranger-http-model/pom.xml @@ -5,7 +5,7 @@ ranger io.appform.ranger - 1.1-RC1 + 1.1-RC2 4.0.0 diff --git a/ranger-http/pom.xml b/ranger-http/pom.xml index 5356c651..5414ee04 100644 --- a/ranger-http/pom.xml +++ b/ranger-http/pom.xml @@ -5,7 +5,7 @@ ranger io.appform.ranger - 1.1-RC1 + 1.1-RC2 4.0.0 diff --git a/ranger-hub-server-bundle/pom.xml b/ranger-hub-server-bundle/pom.xml index d3efc881..72a57568 100644 --- a/ranger-hub-server-bundle/pom.xml +++ b/ranger-hub-server-bundle/pom.xml @@ -5,7 +5,7 @@ ranger io.appform.ranger - 1.0-RC18 + 1.1-RC2 diff --git a/ranger-server-bundle/pom.xml b/ranger-server-bundle/pom.xml index 56a04a42..f0322090 100644 --- a/ranger-server-bundle/pom.xml +++ b/ranger-server-bundle/pom.xml @@ -5,7 +5,7 @@ ranger io.appform.ranger - 1.1-RC1 + 1.1-RC2 4.0.0 diff --git a/ranger-server-common/pom.xml b/ranger-server-common/pom.xml index e496a4aa..e52ae92a 100644 --- a/ranger-server-common/pom.xml +++ b/ranger-server-common/pom.xml @@ -5,7 +5,7 @@ ranger io.appform.ranger - 1.1-RC1 + 1.1-RC2 4.0.0 diff --git a/ranger-server/pom.xml b/ranger-server/pom.xml index 1b6f2538..81df1daf 100644 --- a/ranger-server/pom.xml +++ b/ranger-server/pom.xml @@ -6,7 +6,7 @@ io.appform.ranger ranger - 1.0-RC18 + 1.1-RC2 ranger-server diff --git a/ranger-zk-client/pom.xml b/ranger-zk-client/pom.xml index 36ab7a01..72f422e0 100644 --- a/ranger-zk-client/pom.xml +++ b/ranger-zk-client/pom.xml @@ -5,7 +5,7 @@ ranger io.appform.ranger - 1.1-RC1 + 1.1-RC2 4.0.0 diff --git a/ranger-zookeeper/pom.xml b/ranger-zookeeper/pom.xml index c7829a76..fd172855 100644 --- a/ranger-zookeeper/pom.xml +++ b/ranger-zookeeper/pom.xml @@ -5,7 +5,7 @@ ranger io.appform.ranger - 1.1-RC1 + 1.1-RC2 4.0.0 From befb7a78c863fea8b9e3d6ff429a48c395cb41b5 Mon Sep 17 00:00:00 2001 From: Koushik R Date: Wed, 10 Jul 2024 12:42:24 +0530 Subject: [PATCH 2/7] Keeping the addService asynchrnous, as is : For users to be able to add a list of services without being blocked --- .../client/AbstractRangerHubClient.java | 9 +++---- .../ranger/client/RangerHubClient.java | 2 +- .../core/finderhub/ServiceFinderHub.java | 24 ++++++++++--------- .../core/finderhub/ServiceFinderHubTest.java | 4 ++-- 4 files changed, 21 insertions(+), 18 deletions(-) diff --git a/ranger-client/src/main/java/io/appform/ranger/client/AbstractRangerHubClient.java b/ranger-client/src/main/java/io/appform/ranger/client/AbstractRangerHubClient.java index 2003ba81..6ebca4b6 100644 --- a/ranger-client/src/main/java/io/appform/ranger/client/AbstractRangerHubClient.java +++ b/ranger-client/src/main/java/io/appform/ranger/client/AbstractRangerHubClient.java @@ -32,6 +32,7 @@ import java.util.Collections; import java.util.List; import java.util.Optional; +import java.util.concurrent.CompletableFuture; import java.util.function.Predicate; import lombok.Getter; import lombok.experimental.SuperBuilder; @@ -40,7 +41,7 @@ @Slf4j @Getter @SuperBuilder -public abstract class AbstractRangerHubClient, D extends Deserializer> implements RangerHubClient { +public abstract class AbstractRangerHubClient, D extends Deserializer> implements RangerHubClient { private final String namespace; private final ObjectMapper mapper; @@ -96,7 +97,7 @@ public Optional> getNode(final Service service, final Predicate> getNode( final Service service, final Predicate criteria, - final ShardSelector shardSelector) { + final ShardSelector shardSelector) { return getNode(service, criteria, shardSelector, null); } @@ -104,7 +105,7 @@ public Optional> getNode( public Optional> getNode( final Service service, final Predicate criteria, - final ShardSelector shardSelector, + final ShardSelector shardSelector, final ServiceNodeSelector nodeSelector) { return this.getHub() .finder(service) @@ -160,7 +161,7 @@ public Collection getRegisteredServices() { * @return CompletableFuture which waits for hub to be ready for discovering the new service */ @Override - public ServiceFinder addService(Service service) { + public CompletableFuture addService(Service service) { if(hub == null) { throw new IllegalStateException("Hub not started yet. Call .start()"); } diff --git a/ranger-client/src/main/java/io/appform/ranger/client/RangerHubClient.java b/ranger-client/src/main/java/io/appform/ranger/client/RangerHubClient.java index 7f2dc0fe..58063212 100644 --- a/ranger-client/src/main/java/io/appform/ranger/client/RangerHubClient.java +++ b/ranger-client/src/main/java/io/appform/ranger/client/RangerHubClient.java @@ -34,7 +34,7 @@ public interface RangerHubClient> { Collection getRegisteredServices(); - ServiceFinder addService(Service service); + CompletableFuture addService(Service service); Optional> getNode(final Service service); diff --git a/ranger-core/src/main/java/io/appform/ranger/core/finderhub/ServiceFinderHub.java b/ranger-core/src/main/java/io/appform/ranger/core/finderhub/ServiceFinderHub.java index 0c788bb9..f0963558 100644 --- a/ranger-core/src/main/java/io/appform/ranger/core/finderhub/ServiceFinderHub.java +++ b/ranger-core/src/main/java/io/appform/ranger/core/finderhub/ServiceFinderHub.java @@ -98,20 +98,22 @@ public Optional> finder(final Service service) { return Optional.ofNullable(finders.get().get(service)); } - public ServiceFinder buildFinder(final Service service) { + public CompletableFuture> buildFinder(final Service service) { val finder = finders.get().get(service); - if (null != finder) { - return finder; + if (finder != null) { + return CompletableFuture.completedFuture(finder); } serviceDataSource.add(service); - try { - updateAvailable(); - waitTillServiceIsReady(service); - return finders.get().get(service); - } catch(Exception e) { - log.warn("Exception whiling building finder", e); - throw e; - } + return CompletableFuture.supplyAsync(() -> { + try { + updateAvailable(); + waitTillServiceIsReady(service); + return finders.get().get(service); + } catch(Exception e) { + log.warn("Exception whiling building finder", e); + throw e; + } + }); } public void start() { diff --git a/ranger-core/src/test/java/io/appform/ranger/core/finderhub/ServiceFinderHubTest.java b/ranger-core/src/test/java/io/appform/ranger/core/finderhub/ServiceFinderHubTest.java index 30f6bdc4..b044cfaf 100644 --- a/ranger-core/src/test/java/io/appform/ranger/core/finderhub/ServiceFinderHubTest.java +++ b/ranger-core/src/test/java/io/appform/ranger/core/finderhub/ServiceFinderHubTest.java @@ -44,7 +44,7 @@ void testDynamicServiceAddition() { Assertions.assertEquals("HOST", node.get().getHost()); Assertions.assertEquals(0, node.get().getPort()); - val dynamicServiceFinder = serviceFinderHub.buildFinder(new Service("NS", "SERVICE")); + val dynamicServiceFinder = serviceFinderHub.buildFinder(new Service("NS", "SERVICE")).join(); val dynamicServiceNode = dynamicServiceFinder.get(null, (criteria, serviceRegistry) -> serviceRegistry.nodeList()); Assertions.assertTrue(dynamicServiceNode.isPresent()); Assertions.assertEquals("HOST", dynamicServiceNode.get().getHost()); @@ -85,7 +85,7 @@ void testDynamicServiceAdditionWithNonDynamicDataSource() { .build()); serviceFinderHub.start(); try { - serviceFinderHub.buildFinder(new Service("NS", "SERVICE_NAME")); + serviceFinderHub.buildFinder(new Service("NS", "SERVICE_NAME")).join(); Assertions.fail("Exception should have been thrown"); } catch (Exception exception) { Assertions.assertTrue(exception instanceof UnsupportedOperationException, "Unsupported exception should be thrown"); From 1dd67b60a6bb764ad44071541d6660ff317aaccb Mon Sep 17 00:00:00 2001 From: Koushik R Date: Wed, 10 Jul 2024 12:43:08 +0530 Subject: [PATCH 3/7] Removed an unnecessary import --- .../io/appform/ranger/core/finderhub/ServiceFinderHubTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/ranger-core/src/test/java/io/appform/ranger/core/finderhub/ServiceFinderHubTest.java b/ranger-core/src/test/java/io/appform/ranger/core/finderhub/ServiceFinderHubTest.java index b044cfaf..43a92024 100644 --- a/ranger-core/src/test/java/io/appform/ranger/core/finderhub/ServiceFinderHubTest.java +++ b/ranger-core/src/test/java/io/appform/ranger/core/finderhub/ServiceFinderHubTest.java @@ -16,7 +16,6 @@ import lombok.val; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.function.Executable; import java.util.ArrayList; import java.util.HashSet; From 4aa4057334fb5b0ca81d442281dafe33d5cc8048 Mon Sep 17 00:00:00 2001 From: Koushik R Date: Wed, 10 Jul 2024 12:46:59 +0530 Subject: [PATCH 4/7] Removed an unused import in AbstractRangerHubClient --- .../main/java/io/appform/ranger/client/RangerHubClient.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ranger-client/src/main/java/io/appform/ranger/client/RangerHubClient.java b/ranger-client/src/main/java/io/appform/ranger/client/RangerHubClient.java index 58063212..2094c2d8 100644 --- a/ranger-client/src/main/java/io/appform/ranger/client/RangerHubClient.java +++ b/ranger-client/src/main/java/io/appform/ranger/client/RangerHubClient.java @@ -15,7 +15,6 @@ */ package io.appform.ranger.client; -import io.appform.ranger.core.finder.ServiceFinder; import io.appform.ranger.core.model.Service; import io.appform.ranger.core.model.ServiceNode; import io.appform.ranger.core.model.ServiceNodeSelector; @@ -34,7 +33,7 @@ public interface RangerHubClient> { Collection getRegisteredServices(); - CompletableFuture addService(Service service); + CompletableFuture addService(Service service); Optional> getNode(final Service service); From d0aa16b05fd9fb487e9f4787fe762ae6feeb318c Mon Sep 17 00:00:00 2001 From: Koushik R Date: Tue, 13 Aug 2024 16:09:48 +0530 Subject: [PATCH 5/7] Moved refreshDuration to HubConstants, also moved all hub backed constants - the ones from RangerClientConstants to a single place --- .../client/AbstractRangerHubClient.java | 31 +++++++++++++------ .../core/finderhub/ServiceFinderHub.java | 3 +- .../finderhub/ServiceFinderHubBuilder.java | 7 +++-- .../ranger/core/model/HubConstants.java | 17 +++++----- .../http/AbstractRangerHttpHubClient.java | 2 ++ .../server/bundle/RangerHubServerBundle.java | 4 +-- .../RangerUpstreamConfiguration.java | 6 ++-- .../client/zk/AbstractRangerZKHubClient.java | 2 ++ .../client/zk/SimpleRangerZKClient.java | 10 +++--- 9 files changed, 52 insertions(+), 30 deletions(-) rename ranger-client/src/main/java/io/appform/ranger/client/RangerClientConstants.java => ranger-core/src/main/java/io/appform/ranger/core/model/HubConstants.java (63%) diff --git a/ranger-client/src/main/java/io/appform/ranger/client/AbstractRangerHubClient.java b/ranger-client/src/main/java/io/appform/ranger/client/AbstractRangerHubClient.java index 6ebca4b6..47b642a0 100644 --- a/ranger-client/src/main/java/io/appform/ranger/client/AbstractRangerHubClient.java +++ b/ranger-client/src/main/java/io/appform/ranger/client/AbstractRangerHubClient.java @@ -22,12 +22,8 @@ import io.appform.ranger.core.finderhub.ServiceDataSource; import io.appform.ranger.core.finderhub.ServiceFinderFactory; import io.appform.ranger.core.finderhub.ServiceFinderHub; -import io.appform.ranger.core.model.Deserializer; -import io.appform.ranger.core.model.Service; -import io.appform.ranger.core.model.ServiceNode; -import io.appform.ranger.core.model.ServiceNodeSelector; -import io.appform.ranger.core.model.ServiceRegistry; -import io.appform.ranger.core.model.ShardSelector; +import io.appform.ranger.core.model.*; + import java.util.Collection; import java.util.Collections; import java.util.List; @@ -51,6 +47,8 @@ public abstract class AbstractRangerHubClient, D private int nodeRefreshTimeMs; private ServiceFinderHub hub; private ServiceDataSource serviceDataSource; + private long serviceRefreshDurationMs; + private long hubRefreshDurationMs; @Override public void start() { @@ -58,12 +56,27 @@ public void start() { Preconditions.checkNotNull(namespace, "namespace can't be null"); Preconditions.checkNotNull(deserializer, "deserializer can't be null"); - if (this.nodeRefreshTimeMs < RangerClientConstants.MINIMUM_REFRESH_TIME) { + if (this.nodeRefreshTimeMs < HubConstants.MINIMUM_REFRESH_TIME_MS) { log.warn("Node info update interval too low: {} ms. Has been upgraded to {} ms ", this.nodeRefreshTimeMs, - RangerClientConstants.MINIMUM_REFRESH_TIME); + HubConstants.MINIMUM_REFRESH_TIME_MS); + } + this.nodeRefreshTimeMs = Math.max(HubConstants.MINIMUM_REFRESH_TIME_MS, this.nodeRefreshTimeMs); + + if (this.serviceRefreshDurationMs <= 0) { + log.warn("Service Refresh interval too low: {} ms. Has been upgraded to {} ms ", + this.serviceRefreshDurationMs, + HubConstants.SERVICE_REFRESH_DURATION_MS); + this.serviceRefreshDurationMs = HubConstants.SERVICE_REFRESH_DURATION_MS; } - this.nodeRefreshTimeMs = Math.max(RangerClientConstants.MINIMUM_REFRESH_TIME, this.nodeRefreshTimeMs); + + if (this.hubRefreshDurationMs <= 0) { + log.warn("Service Refresh interval too low: {} ms. Has been upgraded to {} ms ", + this.hubRefreshDurationMs, + HubConstants.HUB_REFRESH_DURATION_MS); + this.hubRefreshDurationMs = HubConstants.HUB_REFRESH_DURATION_MS; + } + if(null == this.serviceDataSource){ this.serviceDataSource = getDefaultDataSource(); } diff --git a/ranger-core/src/main/java/io/appform/ranger/core/finderhub/ServiceFinderHub.java b/ranger-core/src/main/java/io/appform/ranger/core/finderhub/ServiceFinderHub.java index f0963558..ef1a2083 100644 --- a/ranger-core/src/main/java/io/appform/ranger/core/finderhub/ServiceFinderHub.java +++ b/ranger-core/src/main/java/io/appform/ranger/core/finderhub/ServiceFinderHub.java @@ -20,6 +20,7 @@ import com.google.common.base.Stopwatch; import com.google.common.base.Supplier; import io.appform.ranger.core.finder.ServiceFinder; +import io.appform.ranger.core.model.HubConstants; import io.appform.ranger.core.model.Service; import io.appform.ranger.core.model.ServiceRegistry; import io.appform.ranger.core.signals.ExternalTriggeredSignal; @@ -76,7 +77,7 @@ public ServiceFinderHub( ServiceFinderFactory finderFactory ) { this(serviceDataSource, finderFactory, - 10_000, 30_000); + HubConstants.SERVICE_REFRESH_DURATION_MS, HubConstants.HUB_REFRESH_DURATION_MS); } public ServiceFinderHub( diff --git a/ranger-core/src/main/java/io/appform/ranger/core/finderhub/ServiceFinderHubBuilder.java b/ranger-core/src/main/java/io/appform/ranger/core/finderhub/ServiceFinderHubBuilder.java index e2b554df..2a32f11e 100644 --- a/ranger-core/src/main/java/io/appform/ranger/core/finderhub/ServiceFinderHubBuilder.java +++ b/ranger-core/src/main/java/io/appform/ranger/core/finderhub/ServiceFinderHubBuilder.java @@ -16,6 +16,7 @@ package io.appform.ranger.core.finderhub; import com.google.common.base.Preconditions; +import io.appform.ranger.core.model.HubConstants; import io.appform.ranger.core.model.ServiceRegistry; import io.appform.ranger.core.signals.ScheduledSignal; import io.appform.ranger.core.signals.Signal; @@ -33,12 +34,12 @@ public abstract class ServiceFinderHubBuilder> { private ServiceDataSource serviceDataSource; private ServiceFinderFactory serviceFinderFactory; - private long refreshFrequencyMs = 10_000; + private long refreshFrequencyMs = HubConstants.REFRESH_FREQUENCY_MS; private final List> extraStartSignalConsumers = new ArrayList<>(); private final List> extraStopSignalConsumers = new ArrayList<>(); private final List> extraRefreshSignals = new ArrayList<>(); - private long serviceRefreshDurationMs = 10_000; - private long hubRefreshDurationMs = 30_000; + private long serviceRefreshDurationMs = HubConstants.SERVICE_REFRESH_DURATION_MS; + private long hubRefreshDurationMs = HubConstants.HUB_REFRESH_DURATION_MS; public ServiceFinderHubBuilder withServiceDataSource(ServiceDataSource serviceDataSource) { this.serviceDataSource = serviceDataSource; diff --git a/ranger-client/src/main/java/io/appform/ranger/client/RangerClientConstants.java b/ranger-core/src/main/java/io/appform/ranger/core/model/HubConstants.java similarity index 63% rename from ranger-client/src/main/java/io/appform/ranger/client/RangerClientConstants.java rename to ranger-core/src/main/java/io/appform/ranger/core/model/HubConstants.java index 3b8a6fc9..bc0d4703 100644 --- a/ranger-client/src/main/java/io/appform/ranger/client/RangerClientConstants.java +++ b/ranger-core/src/main/java/io/appform/ranger/core/model/HubConstants.java @@ -1,24 +1,27 @@ /* * Copyright 2015 Flipkart Internet Pvt. Ltd. - * + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ -package io.appform.ranger.client; +package io.appform.ranger.core.model; import lombok.experimental.UtilityClass; @UtilityClass -public class RangerClientConstants { - public static final int CONNECTION_RETRY_TIME = 5000; - public static final int MINIMUM_REFRESH_TIME = 5000; +public class HubConstants { + public static long SERVICE_REFRESH_DURATION_MS = 10_000; + public static long HUB_REFRESH_DURATION_MS = 30_000; + public static long REFRESH_FREQUENCY_MS = 10_000; + public static final int CONNECTION_RETRY_TIME_MS = 5_000; + public static final int MINIMUM_REFRESH_TIME_MS = 5_000; } diff --git a/ranger-http-client/src/main/java/io/appform/ranger/client/http/AbstractRangerHttpHubClient.java b/ranger-http-client/src/main/java/io/appform/ranger/client/http/AbstractRangerHttpHubClient.java index 5dd8d92a..dc488e8c 100644 --- a/ranger-http-client/src/main/java/io/appform/ranger/client/http/AbstractRangerHttpHubClient.java +++ b/ranger-http-client/src/main/java/io/appform/ranger/client/http/AbstractRangerHttpHubClient.java @@ -51,6 +51,8 @@ protected ServiceFinderHub buildHub() { .withServiceDataSource(getServiceDataSource()) .withServiceFinderFactory(getFinderFactory()) .withRefreshFrequencyMs(getNodeRefreshTimeMs()) + .withHubRefreshDuration(getHubRefreshDurationMs()) + .withServiceRefreshDuration(getServiceRefreshDurationMs()) .build(); } } diff --git a/ranger-hub-server-bundle/src/main/java/io/appform/ranger/hub/server/bundle/RangerHubServerBundle.java b/ranger-hub-server-bundle/src/main/java/io/appform/ranger/hub/server/bundle/RangerHubServerBundle.java index 5ce111f0..60bbb15d 100644 --- a/ranger-hub-server-bundle/src/main/java/io/appform/ranger/hub/server/bundle/RangerHubServerBundle.java +++ b/ranger-hub-server-bundle/src/main/java/io/appform/ranger/hub/server/bundle/RangerHubServerBundle.java @@ -17,12 +17,12 @@ import com.codahale.metrics.health.HealthCheck; import com.fasterxml.jackson.core.type.TypeReference; -import io.appform.ranger.client.RangerClientConstants; import io.appform.ranger.client.RangerHubClient; import io.appform.ranger.client.http.UnshardedRangerHttpHubClient; import io.appform.ranger.client.zk.UnshardedRangerZKHubClient; import io.appform.ranger.common.server.ShardInfo; import io.appform.ranger.core.finder.serviceregistry.ListBasedServiceRegistry; +import io.appform.ranger.core.model.HubConstants; import io.appform.ranger.core.model.ServiceNode; import io.appform.ranger.core.signals.Signal; import io.appform.ranger.http.config.HttpClientConfig; @@ -88,7 +88,7 @@ private RangerHubClient> addCurat val curatorFramework = CuratorFrameworkFactory.builder() .connectString(zookeeper) .namespace(namespace) - .retryPolicy(new RetryForever(RangerClientConstants.CONNECTION_RETRY_TIME)) + .retryPolicy(new RetryForever(HubConstants.CONNECTION_RETRY_TIME_MS)) .build(); curatorFrameworks.add(curatorFramework); return UnshardedRangerZKHubClient.builder() diff --git a/ranger-hub-server-bundle/src/main/java/io/appform/ranger/hub/server/bundle/configuration/RangerUpstreamConfiguration.java b/ranger-hub-server-bundle/src/main/java/io/appform/ranger/hub/server/bundle/configuration/RangerUpstreamConfiguration.java index c8549e03..9a9bdf65 100644 --- a/ranger-hub-server-bundle/src/main/java/io/appform/ranger/hub/server/bundle/configuration/RangerUpstreamConfiguration.java +++ b/ranger-hub-server-bundle/src/main/java/io/appform/ranger/hub/server/bundle/configuration/RangerUpstreamConfiguration.java @@ -17,7 +17,7 @@ import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; -import io.appform.ranger.client.RangerClientConstants; +import io.appform.ranger.core.model.HubConstants; import io.appform.ranger.hub.server.bundle.models.BackendType; import lombok.AllArgsConstructor; import lombok.Getter; @@ -37,8 +37,8 @@ public abstract class RangerUpstreamConfiguration { @NotNull private BackendType type; - @Min(RangerClientConstants.MINIMUM_REFRESH_TIME) - private int nodeRefreshTimeMs = RangerClientConstants.MINIMUM_REFRESH_TIME; + @Min(HubConstants.MINIMUM_REFRESH_TIME_MS) + private int nodeRefreshTimeMs = HubConstants.MINIMUM_REFRESH_TIME_MS; protected RangerUpstreamConfiguration(BackendType type) { this.type = type; diff --git a/ranger-zk-client/src/main/java/io/appform/ranger/client/zk/AbstractRangerZKHubClient.java b/ranger-zk-client/src/main/java/io/appform/ranger/client/zk/AbstractRangerZKHubClient.java index 9322073c..56854f02 100644 --- a/ranger-zk-client/src/main/java/io/appform/ranger/client/zk/AbstractRangerZKHubClient.java +++ b/ranger-zk-client/src/main/java/io/appform/ranger/client/zk/AbstractRangerZKHubClient.java @@ -46,6 +46,8 @@ protected ServiceFinderHub buildHub() { .withRefreshFrequencyMs(getNodeRefreshTimeMs()) .withServiceDataSource(getServiceDataSource()) .withServiceFinderFactory(getFinderFactory()) + .withHubRefreshDuration(getHubRefreshDurationMs()) + .withServiceRefreshDuration(getServiceRefreshDurationMs()) .build(); } diff --git a/ranger-zk-client/src/main/java/io/appform/ranger/client/zk/SimpleRangerZKClient.java b/ranger-zk-client/src/main/java/io/appform/ranger/client/zk/SimpleRangerZKClient.java index ed8d245f..0206764c 100644 --- a/ranger-zk-client/src/main/java/io/appform/ranger/client/zk/SimpleRangerZKClient.java +++ b/ranger-zk-client/src/main/java/io/appform/ranger/client/zk/SimpleRangerZKClient.java @@ -18,10 +18,10 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Preconditions; import io.appform.ranger.client.AbstractRangerClient; -import io.appform.ranger.client.RangerClientConstants; import io.appform.ranger.core.finder.SimpleShardedServiceFinder; import io.appform.ranger.core.finder.serviceregistry.MapBasedServiceRegistry; import io.appform.ranger.core.finder.shardselector.MatchingShardSelector; +import io.appform.ranger.core.model.HubConstants; import io.appform.ranger.core.model.ShardSelector; import io.appform.ranger.zookeeper.ServiceFinderBuilders; import io.appform.ranger.zookeeper.serde.ZkNodeDataDeserializer; @@ -60,11 +60,11 @@ public void start() { int effectiveRefreshTime = nodeRefreshIntervalMs; - if (effectiveRefreshTime < RangerClientConstants.MINIMUM_REFRESH_TIME) { - effectiveRefreshTime = RangerClientConstants.MINIMUM_REFRESH_TIME; + if (effectiveRefreshTime < HubConstants.MINIMUM_REFRESH_TIME_MS) { + effectiveRefreshTime = HubConstants.MINIMUM_REFRESH_TIME_MS; log.warn("Node info update interval too low: {} ms. Has been upgraded to {} ms ", nodeRefreshIntervalMs, - RangerClientConstants.MINIMUM_REFRESH_TIME); + HubConstants.MINIMUM_REFRESH_TIME_MS); } if (null == curatorFramework) { @@ -72,7 +72,7 @@ public void start() { curatorFramework = CuratorFrameworkFactory.builder() .connectString(connectionString) .namespace(namespace) - .retryPolicy(new RetryForever(RangerClientConstants.CONNECTION_RETRY_TIME)) + .retryPolicy(new RetryForever(HubConstants.CONNECTION_RETRY_TIME_MS)) .build(); } From 2358fe0d7ebe9b5813e450ecc8891018a8bba64c Mon Sep 17 00:00:00 2001 From: Koushik R Date: Tue, 13 Aug 2024 16:16:55 +0530 Subject: [PATCH 6/7] added a log message when a null or 0 refresh interval is provided --- .../java/io/appform/ranger/client/AbstractRangerHubClient.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ranger-client/src/main/java/io/appform/ranger/client/AbstractRangerHubClient.java b/ranger-client/src/main/java/io/appform/ranger/client/AbstractRangerHubClient.java index 47b642a0..b48ccc53 100644 --- a/ranger-client/src/main/java/io/appform/ranger/client/AbstractRangerHubClient.java +++ b/ranger-client/src/main/java/io/appform/ranger/client/AbstractRangerHubClient.java @@ -71,7 +71,7 @@ public void start() { } if (this.hubRefreshDurationMs <= 0) { - log.warn("Service Refresh interval too low: {} ms. Has been upgraded to {} ms ", + log.warn("Hub Refresh interval too low: {} ms. Has been upgraded to {} ms ", this.hubRefreshDurationMs, HubConstants.HUB_REFRESH_DURATION_MS); this.hubRefreshDurationMs = HubConstants.HUB_REFRESH_DURATION_MS; From 76fc5061fd71aac6ad322e161da0c35ea28733af Mon Sep 17 00:00:00 2001 From: Koushik R Date: Tue, 13 Aug 2024 16:24:38 +0530 Subject: [PATCH 7/7] Removed an unnecessary import that crept in --- .../io/appform/ranger/client/AbstractRangerHubClient.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/ranger-client/src/main/java/io/appform/ranger/client/AbstractRangerHubClient.java b/ranger-client/src/main/java/io/appform/ranger/client/AbstractRangerHubClient.java index b48ccc53..10b69080 100644 --- a/ranger-client/src/main/java/io/appform/ranger/client/AbstractRangerHubClient.java +++ b/ranger-client/src/main/java/io/appform/ranger/client/AbstractRangerHubClient.java @@ -18,7 +18,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Preconditions; import io.appform.ranger.client.utils.CriteriaUtils; -import io.appform.ranger.core.finder.ServiceFinder; import io.appform.ranger.core.finderhub.ServiceDataSource; import io.appform.ranger.core.finderhub.ServiceFinderFactory; import io.appform.ranger.core.finderhub.ServiceFinderHub; @@ -110,7 +109,7 @@ public Optional> getNode(final Service service, final Predicate> getNode( final Service service, final Predicate criteria, - final ShardSelector shardSelector) { + final ShardSelector shardSelector) { return getNode(service, criteria, shardSelector, null); } @@ -118,7 +117,7 @@ public Optional> getNode( public Optional> getNode( final Service service, final Predicate criteria, - final ShardSelector shardSelector, + final ShardSelector shardSelector, final ServiceNodeSelector nodeSelector) { return this.getHub() .finder(service)