From 9f24172193803ed4bdb41516607faa3ec0d68fae Mon Sep 17 00:00:00 2001 From: Santanu Sinha Date: Tue, 17 Dec 2024 19:59:47 +0530 Subject: [PATCH] Deciding on replication only based on http client config --- .../ranger/client/AbstractRangerHubClient.java | 1 - .../io/appform/ranger/client/RangerHubClient.java | 4 +++- .../ranger/core/finderhub/ServiceFinderHub.java | 8 ++------ .../core/finderhub/ServiceFinderHubBuilder.java | 8 +------- .../core/finderhub/ServiceFinderHubTest.java | 4 ++-- ...bundle.id.IdGeneratorPerfTest.testGenerate.json | 2 +- ....id.IdGeneratorPerfTest.testGenerateBase36.json | 2 +- .../client/drove/AbstractRangerDroveHubClient.java | 1 - .../client/http/AbstractRangerHttpHubClient.java | 6 +++++- .../ranger/http/config/HttpClientConfig.java | 2 +- .../http/servicefinder/HttpApiCommunicator.java | 4 ++-- .../HttpServiceDataSourceTest.java | 2 +- .../hub/server/bundle/RangerHubServerBundle.java | 3 --- .../RangerHttpUpstreamConfiguration.java | 14 +++++++------- .../configuration/RangerUpstreamConfiguration.java | 2 -- .../server/bundle/resources/RangerResource.java | 3 ++- .../client/zk/AbstractRangerZKHubClient.java | 2 -- 17 files changed, 28 insertions(+), 40 deletions(-) diff --git a/ranger-client/src/main/java/io/appform/ranger/client/AbstractRangerHubClient.java b/ranger-client/src/main/java/io/appform/ranger/client/AbstractRangerHubClient.java index a84aefce..2ec971b0 100644 --- a/ranger-client/src/main/java/io/appform/ranger/client/AbstractRangerHubClient.java +++ b/ranger-client/src/main/java/io/appform/ranger/client/AbstractRangerHubClient.java @@ -56,7 +56,6 @@ public abstract class AbstractRangerHubClient, D */ private long hubStartTimeoutMs; private Set excludedServices; - private boolean replicationSource; @Override public void start() { diff --git a/ranger-client/src/main/java/io/appform/ranger/client/RangerHubClient.java b/ranger-client/src/main/java/io/appform/ranger/client/RangerHubClient.java index b0aca3d4..76963364 100644 --- a/ranger-client/src/main/java/io/appform/ranger/client/RangerHubClient.java +++ b/ranger-client/src/main/java/io/appform/ranger/client/RangerHubClient.java @@ -59,5 +59,7 @@ List> getAllNodes( final Predicate criteria, final ShardSelector shardSelector); - boolean isReplicationSource(); + default boolean isReplicationSource() { + return false; + } } diff --git a/ranger-core/src/main/java/io/appform/ranger/core/finderhub/ServiceFinderHub.java b/ranger-core/src/main/java/io/appform/ranger/core/finderhub/ServiceFinderHub.java index 54fa0d98..9feed7d9 100644 --- a/ranger-core/src/main/java/io/appform/ranger/core/finderhub/ServiceFinderHub.java +++ b/ranger-core/src/main/java/io/appform/ranger/core/finderhub/ServiceFinderHub.java @@ -78,8 +78,6 @@ public class ServiceFinderHub> { private final long hubStartTimeoutMs; private final Set excludedServices; - @Getter - private final boolean replicationSource; private final ForkJoinPool refresherPool; @@ -87,7 +85,7 @@ public ServiceFinderHub( ServiceDataSource serviceDataSource, ServiceFinderFactory finderFactory) { this(serviceDataSource, finderFactory, - HubConstants.SERVICE_REFRESH_TIMEOUT_MS, HubConstants.HUB_START_TIMEOUT_MS, Set.of(), false); + HubConstants.SERVICE_REFRESH_TIMEOUT_MS, HubConstants.HUB_START_TIMEOUT_MS, Set.of()); } public ServiceFinderHub( @@ -95,13 +93,11 @@ public ServiceFinderHub( ServiceFinderFactory finderFactory, long serviceRefreshTimeoutMs, long hubStartTimeoutMs, - final Set excludedServices, - boolean replicationSource) { + final Set excludedServices) { this.serviceDataSource = serviceDataSource; this.finderFactory = finderFactory; this.serviceRefreshTimeoutMs = serviceRefreshTimeoutMs == 0 ? HubConstants.SERVICE_REFRESH_TIMEOUT_MS : serviceRefreshTimeoutMs; this.hubStartTimeoutMs = hubStartTimeoutMs == 0 ? HubConstants.HUB_START_TIMEOUT_MS : hubStartTimeoutMs; - this.replicationSource = replicationSource; this.refreshSignals.add(new ScheduledSignal<>("service-hub-updater", () -> null, Collections.emptyList(), diff --git a/ranger-core/src/main/java/io/appform/ranger/core/finderhub/ServiceFinderHubBuilder.java b/ranger-core/src/main/java/io/appform/ranger/core/finderhub/ServiceFinderHubBuilder.java index db947669..0b41987d 100644 --- a/ranger-core/src/main/java/io/appform/ranger/core/finderhub/ServiceFinderHubBuilder.java +++ b/ranger-core/src/main/java/io/appform/ranger/core/finderhub/ServiceFinderHubBuilder.java @@ -38,7 +38,6 @@ public abstract class ServiceFinderHubBuilder> { private final List> extraRefreshSignals = new ArrayList<>(); private long serviceRefreshTimeoutMs = HubConstants.SERVICE_REFRESH_TIMEOUT_MS; private long hubStartTimeoutMs = HubConstants.HUB_START_TIMEOUT_MS; - private boolean replicationSource = false; private Set excludedServices = new HashSet<>(); @@ -87,18 +86,13 @@ public ServiceFinderHubBuilder withExcludedServices(Set excludedSe return this; } - public ServiceFinderHubBuilder withReplicationSource(boolean replicationSource) { - this.replicationSource = replicationSource; - return this; - } - public ServiceFinderHub build() { preBuild(); Preconditions.checkNotNull(serviceDataSource, "Provide a non-null service data source"); Preconditions.checkNotNull(serviceFinderFactory, "Provide a non-null service finder factory"); val hub = new ServiceFinderHub<>(serviceDataSource, serviceFinderFactory, serviceRefreshTimeoutMs, - hubStartTimeoutMs, excludedServices, replicationSource); + hubStartTimeoutMs, excludedServices); final ScheduledSignal refreshSignal = new ScheduledSignal<>("service-hub-refresh-timer", () -> null, Collections.emptyList(), diff --git a/ranger-core/src/test/java/io/appform/ranger/core/finderhub/ServiceFinderHubTest.java b/ranger-core/src/test/java/io/appform/ranger/core/finderhub/ServiceFinderHubTest.java index b9e76048..1ce78953 100644 --- a/ranger-core/src/test/java/io/appform/ranger/core/finderhub/ServiceFinderHubTest.java +++ b/ranger-core/src/test/java/io/appform/ranger/core/finderhub/ServiceFinderHubTest.java @@ -89,7 +89,7 @@ void testDelayedServiceAddition() { .withServiceName(service.getServiceName()) .withDeserializer(new Deserializer() {}) .withSleepDuration(5) - .build(), 1_000, 5_000, Set.of(), false); + .build(), 1_000, 5_000, Set.of()); Assertions.assertThrows(IllegalStateException.class, delayedHub::start); val serviceFinderHub = new ServiceFinderHub<>(new DynamicDataSource(Lists.newArrayList(new Service("NS", "SERVICE"))), service -> new TestServiceFinderBuilder() @@ -97,7 +97,7 @@ void testDelayedServiceAddition() { .withServiceName(service.getServiceName()) .withDeserializer(new Deserializer() {}) .withSleepDuration(1) - .build(), 5_000, 5_000, Set.of(), false); + .build(), 5_000, 5_000, Set.of()); serviceFinderHub.start(); Assertions.assertTrue(serviceFinderHub.finder(new Service("NS", "SERVICE")).isPresent()); } diff --git a/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.IdGeneratorPerfTest.testGenerate.json b/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.IdGeneratorPerfTest.testGenerate.json index ba2c090b..33e4b0fc 100644 --- a/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.IdGeneratorPerfTest.testGenerate.json +++ b/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.IdGeneratorPerfTest.testGenerate.json @@ -4,5 +4,5 @@ "iterations" : 4, "threads" : 1, "forks" : 3, - "mean_ops" : 780203.0578756403 + "mean_ops" : 659517.6790772061 } \ No newline at end of file diff --git a/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.IdGeneratorPerfTest.testGenerateBase36.json b/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.IdGeneratorPerfTest.testGenerateBase36.json index c6d9e714..fcd4e4bd 100644 --- a/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.IdGeneratorPerfTest.testGenerateBase36.json +++ b/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.IdGeneratorPerfTest.testGenerateBase36.json @@ -4,5 +4,5 @@ "iterations" : 4, "threads" : 1, "forks" : 3, - "mean_ops" : 610879.2316480416 + "mean_ops" : 494777.4621186971 } \ No newline at end of file diff --git a/ranger-drove-client/src/main/java/io/appform/ranger/client/drove/AbstractRangerDroveHubClient.java b/ranger-drove-client/src/main/java/io/appform/ranger/client/drove/AbstractRangerDroveHubClient.java index dde199a4..b205f9d8 100644 --- a/ranger-drove-client/src/main/java/io/appform/ranger/client/drove/AbstractRangerDroveHubClient.java +++ b/ranger-drove-client/src/main/java/io/appform/ranger/client/drove/AbstractRangerDroveHubClient.java @@ -58,7 +58,6 @@ protected ServiceFinderHub buildHub() { .withHubStartTimeout(getHubStartTimeoutMs()) .withServiceRefreshTimeout(getServiceRefreshTimeoutMs()) .withExcludedServices(getExcludedServices()) - .withReplicationSource(isReplicationSource()) .build(); } } diff --git a/ranger-http-client/src/main/java/io/appform/ranger/client/http/AbstractRangerHttpHubClient.java b/ranger-http-client/src/main/java/io/appform/ranger/client/http/AbstractRangerHttpHubClient.java index 654bee9e..5dd6910b 100644 --- a/ranger-http-client/src/main/java/io/appform/ranger/client/http/AbstractRangerHttpHubClient.java +++ b/ranger-http-client/src/main/java/io/appform/ranger/client/http/AbstractRangerHttpHubClient.java @@ -66,7 +66,11 @@ protected ServiceFinderHub buildHub() { .withHubStartTimeout(getHubStartTimeoutMs()) .withServiceRefreshTimeout(getServiceRefreshTimeoutMs()) .withExcludedServices(getExcludedServices()) - .withReplicationSource(isReplicationSource()) .build(); } + + @Override + public boolean isReplicationSource() { + return clientConfig.isReplicationSource(); + } } diff --git a/ranger-http/src/main/java/io/appform/ranger/http/config/HttpClientConfig.java b/ranger-http/src/main/java/io/appform/ranger/http/config/HttpClientConfig.java index 02544d40..dbcac3a8 100644 --- a/ranger-http/src/main/java/io/appform/ranger/http/config/HttpClientConfig.java +++ b/ranger-http/src/main/java/io/appform/ranger/http/config/HttpClientConfig.java @@ -34,5 +34,5 @@ public class HttpClientConfig { long connectionTimeoutMs; long operationTimeoutMs; long refreshIntervalMillis; - boolean skipReplicatedData; + boolean replicationSource; } diff --git a/ranger-http/src/main/java/io/appform/ranger/http/servicefinder/HttpApiCommunicator.java b/ranger-http/src/main/java/io/appform/ranger/http/servicefinder/HttpApiCommunicator.java index 58ccfae8..2f0eff55 100644 --- a/ranger-http/src/main/java/io/appform/ranger/http/servicefinder/HttpApiCommunicator.java +++ b/ranger-http/src/main/java/io/appform/ranger/http/servicefinder/HttpApiCommunicator.java @@ -73,7 +73,7 @@ public Set services() { .host(config.getHost()) .port(config.getPort() == 0 ? defaultPort() : config.getPort()) .encodedPath("/ranger/services/v1") - .addQueryParameter("skipReplicationSources", Objects.toString(config.isSkipReplicatedData())) + .addQueryParameter("skipReplicationSources", Objects.toString(config.isReplicationSource())) .build(); val request = new Request.Builder() .url(httpUrl) @@ -110,7 +110,7 @@ public List> listNodes( .host(config.getHost()) .port(config.getPort() == 0 ? defaultPort() : config.getPort()) .encodedPath(url) - .addQueryParameter("skipReplicationSources", Objects.toString(config.isSkipReplicatedData())) + .addQueryParameter("skipReplicationSources", Objects.toString(config.isReplicationSource())) .build(); val request = new Request.Builder() .url(httpUrl) diff --git a/ranger-http/src/test/java/io/appform/ranger/http/servicefinderhub/HttpServiceDataSourceTest.java b/ranger-http/src/test/java/io/appform/ranger/http/servicefinderhub/HttpServiceDataSourceTest.java index f9e700e0..397ee62f 100644 --- a/ranger-http/src/test/java/io/appform/ranger/http/servicefinderhub/HttpServiceDataSourceTest.java +++ b/ranger-http/src/test/java/io/appform/ranger/http/servicefinderhub/HttpServiceDataSourceTest.java @@ -84,7 +84,7 @@ void testServiceDataSource(WireMockRuntimeInfo wireMockRuntimeInfo) throws IOExc .port(wireMockRuntimeInfo.getHttpPort()) .connectionTimeoutMs(30_000) .operationTimeoutMs(30_000) - .skipReplicatedData(true) + .replicationSource(true) .build(); val httpServiceDataSource = new HttpServiceDataSource<>(clientConfig, RangerHttpUtils.httpClient(clientConfig, MAPPER)); diff --git a/ranger-hub-server-bundle/src/main/java/io/appform/ranger/hub/server/bundle/RangerHubServerBundle.java b/ranger-hub-server-bundle/src/main/java/io/appform/ranger/hub/server/bundle/RangerHubServerBundle.java index 63ce587e..8a3eb670 100644 --- a/ranger-hub-server-bundle/src/main/java/io/appform/ranger/hub/server/bundle/RangerHubServerBundle.java +++ b/ranger-hub-server-bundle/src/main/java/io/appform/ranger/hub/server/bundle/RangerHubServerBundle.java @@ -109,7 +109,6 @@ private RangerHubClient> addCurat .hubStartTimeoutMs(zkConfiguration.getHubStartTimeoutMs()) .nodeRefreshTimeMs(zkConfiguration.getNodeRefreshTimeMs()) .excludedServices(excludedServices) - .replicationSource(zkConfiguration.isReplicationSource()) .deserializer(data -> { try { return getMapper().readValue(data, new TypeReference>() { @@ -134,7 +133,6 @@ private RangerHubClient> getHttpH .hubStartTimeoutMs(httpConfiguration.getHubStartTimeoutMs()) .nodeRefreshTimeMs(httpConfiguration.getNodeRefreshTimeMs()) .excludedServices(excludedServices) - .replicationSource(httpConfiguration.isReplicationSource()) .deserializer(data -> { try { return getMapper().readValue(data, new TypeReference<>() {}); @@ -163,7 +161,6 @@ private RangerHubClient> getDrove .hubStartTimeoutMs(droveUpstreamConfiguration.getHubStartTimeoutMs()) .nodeRefreshTimeMs(droveUpstreamConfiguration.getNodeRefreshTimeMs()) .excludedServices(excludedServices) - .replicationSource(droveUpstreamConfiguration.isReplicationSource()) .deserializer(new DroveResponseDataDeserializer<>() { @Override protected ShardInfo translate(ExposedAppInfo appInfo, ExposedAppInfo.ExposedHost host) { diff --git a/ranger-hub-server-bundle/src/main/java/io/appform/ranger/hub/server/bundle/configuration/RangerHttpUpstreamConfiguration.java b/ranger-hub-server-bundle/src/main/java/io/appform/ranger/hub/server/bundle/configuration/RangerHttpUpstreamConfiguration.java index 12584251..96b9c423 100644 --- a/ranger-hub-server-bundle/src/main/java/io/appform/ranger/hub/server/bundle/configuration/RangerHttpUpstreamConfiguration.java +++ b/ranger-hub-server-bundle/src/main/java/io/appform/ranger/hub/server/bundle/configuration/RangerHttpUpstreamConfiguration.java @@ -33,16 +33,16 @@ @JsonIgnoreProperties(ignoreUnknown = true) public class RangerHttpUpstreamConfiguration extends RangerUpstreamConfiguration { - @NotEmpty - @Valid - private List httpClientConfigs; + @NotEmpty + @Valid + private List httpClientConfigs; public RangerHttpUpstreamConfiguration() { super(BackendType.HTTP); } - @Override - public T accept(RangerConfigurationVisitor visitor) { - return visitor.visit(this); - } + @Override + public T accept(RangerConfigurationVisitor visitor) { + return visitor.visit(this); + } } diff --git a/ranger-hub-server-bundle/src/main/java/io/appform/ranger/hub/server/bundle/configuration/RangerUpstreamConfiguration.java b/ranger-hub-server-bundle/src/main/java/io/appform/ranger/hub/server/bundle/configuration/RangerUpstreamConfiguration.java index 4061d498..06ec4b43 100644 --- a/ranger-hub-server-bundle/src/main/java/io/appform/ranger/hub/server/bundle/configuration/RangerUpstreamConfiguration.java +++ b/ranger-hub-server-bundle/src/main/java/io/appform/ranger/hub/server/bundle/configuration/RangerUpstreamConfiguration.java @@ -47,8 +47,6 @@ public abstract class RangerUpstreamConfiguration { @Min(HubConstants.MINIMUM_HUB_START_TIMEOUT_MS) private int hubStartTimeoutMs = HubConstants.HUB_START_TIMEOUT_MS; - private boolean replicationSource; - protected RangerUpstreamConfiguration(BackendType type) { this.type = type; } diff --git a/ranger-server-bundle/src/main/java/io/appform/ranger/server/bundle/resources/RangerResource.java b/ranger-server-bundle/src/main/java/io/appform/ranger/server/bundle/resources/RangerResource.java index e82f658d..bb38e6e3 100644 --- a/ranger-server-bundle/src/main/java/io/appform/ranger/server/bundle/resources/RangerResource.java +++ b/ranger-server-bundle/src/main/java/io/appform/ranger/server/bundle/resources/RangerResource.java @@ -55,6 +55,7 @@ public GenericResponse> getServices( @QueryParam("skipReplicationSources") @DefaultValue("false") boolean skipReplicationSources) { return GenericResponse.>builder() .data(rangerHubs.stream() + .filter(hub -> !skipReplicationSources || !hub.isReplicationSource()) .map(RangerHubClient::getRegisteredServices) .flatMap(Collection::stream) .collect(Collectors.toSet())) @@ -71,7 +72,7 @@ public GenericResponse>> getNodes( val service = Service.builder().namespace(namespace).serviceName(serviceName).build(); return GenericResponse.>>builder() .data(rangerHubs.stream() - .filter(hub -> skipReplicationSources && !hub.isReplicationSource()) + .filter(hub -> !skipReplicationSources || !hub.isReplicationSource()) .map(hub -> hub.getAllNodes(service)) .flatMap(List::stream) .collect(Collectors.toMap(node -> node.getHost() + ":" + node.getPort(), diff --git a/ranger-zk-client/src/main/java/io/appform/ranger/client/zk/AbstractRangerZKHubClient.java b/ranger-zk-client/src/main/java/io/appform/ranger/client/zk/AbstractRangerZKHubClient.java index d64bb060..41872e14 100644 --- a/ranger-zk-client/src/main/java/io/appform/ranger/client/zk/AbstractRangerZKHubClient.java +++ b/ranger-zk-client/src/main/java/io/appform/ranger/client/zk/AbstractRangerZKHubClient.java @@ -36,7 +36,6 @@ public abstract class AbstractRangerZKHubClient, private final boolean disablePushUpdaters; private final String connectionString; private final CuratorFramework curatorFramework; - private boolean replicationSource; @Override protected ServiceFinderHub buildHub() { @@ -50,7 +49,6 @@ protected ServiceFinderHub buildHub() { .withHubStartTimeout(getHubStartTimeoutMs()) .withServiceRefreshTimeout(getServiceRefreshTimeoutMs()) .withExcludedServices(getExcludedServices()) - .withReplicationSource(isReplicationSource()) .build(); }