Implement eventual consistency enforcement for the datadir catalog ba…

…ckend

When performing several catalog updates, for example through the REST
API, and especially if the rabbitmq event bus is under stress, catalog
events may arrive out of order.

This is troublesome for the datadir catalog backend, as the add/modify
event for a given CatalogInfo may arrive before the event for one of
its dependencies. For example, the WorkspaceInfo for a StoreInfo does
not yet exist in the local catalog.

The `EventuallyConsistentCatalogFacade` will delay such changes with
missing dependencies until they're resolved. This is usually a matter of
a few milliseconds.

Also, when a web request is in progress, and an object is not found, the
catalog facade will retry the operation for a configurable number of
times and milliseconds, giving the eventual consistency enforcer a
chance to having resolved a pending update.

The default configuration properties are as follow:

```
geoserver:
  backend:
    data-directory:
      eventual-consistency:
        # eventual consistency enfocement. Bus events may come out of order under stress
        enabled: true
        # milliseconds to wait before retries. The list size determines the number of retries. The values the milliseconds to wait
        retries: 25, 25, 50
```

compose/infra.yml

@@ -34,7 +34,7 @@ services:
       resources:
         limits:
           cpus: '4.0'
-          memory: 1G
+          memory: 2G
     restart: unless-stopped
     healthcheck:
       test: rabbitmq-diagnostics is_running

src/catalog/backends/datadir/src/main/java/org/geoserver/cloud/autoconfigure/catalog/backend/datadir/RemoteEventDataDirectoryAutoConfiguration.java

@@ -4,23 +4,32 @@
  */
 package org.geoserver.cloud.autoconfigure.catalog.backend.datadir;
 
-import org.geoserver.catalog.plugin.CatalogPlugin;
 import org.geoserver.cloud.autoconfigure.catalog.event.ConditionalOnCatalogEvents;
-import org.geoserver.cloud.event.remote.datadir.RemoteEventDataDirectoryProcessor;
-import org.geoserver.config.plugin.RepositoryGeoServerFacade;
-import org.springframework.beans.factory.annotation.Qualifier;
+import org.geoserver.cloud.catalog.backend.datadir.EventualConsistencyEnforcer;
+import org.geoserver.cloud.event.remote.datadir.RemoteEventDataDirectoryConfiguration;
 import org.springframework.boot.autoconfigure.AutoConfiguration;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
 import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Import;
 
+/**
+ * {@link AutoConfiguration @AutoConfiguration} to contribute beans related to handling remotely
+ * produced catalog and config events
+ *
+ * @see RemoteEventDataDirectoryConfiguration
+ */
 @AutoConfiguration
 @ConditionalOnDataDirectoryEnabled
 @ConditionalOnCatalogEvents
+@Import(RemoteEventDataDirectoryConfiguration.class)
 public class RemoteEventDataDirectoryAutoConfiguration {
 
     @Bean
-    RemoteEventDataDirectoryProcessor dataDirectoryRemoteEventProcessor(
-            @Qualifier("geoserverFacade") RepositoryGeoServerFacade configFacade,
-            @Qualifier("rawCatalog") CatalogPlugin rawCatalog) {
-        return new RemoteEventDataDirectoryProcessor(configFacade, rawCatalog);
+    @ConditionalOnProperty(
+            name = "geoserver.backend.data-directory.eventual-consistency.enabled",
+            havingValue = "true",
+            matchIfMissing = true)
+    EventualConsistencyEnforcer eventualConsistencyEnforcer() {
+        return new EventualConsistencyEnforcer();
     }
 }