Refactor Shared Settings, Introduce Dynamic Forecast Settings, Rename

build.gradle

@@ -668,8 +668,9 @@ List<String> jacocoExclusions = [
         'org.opensearch.ad.constant.*',
         'org.opensearch.forecast.constant.*',
         'org.opensearch.timeseries.constant.*',
+        'org.opensearch.timeseries.settings.TimeSeriesSettings',
+        'org.opensearch.forecast.settings.ForecastSettings',
 
-        //'org.opensearch.ad.common.exception.AnomalyDetectionException',
         'org.opensearch.ad.util.ClientUtil',
 
         'org.opensearch.ad.transport.CronRequest',

src/main/java/org/opensearch/ad/AnomalyDetectorPlugin.java

@@ -873,8 +873,8 @@ public List<Setting<?>> getSettings() {
                 AnomalyDetectorSettings.MAX_RETRY_FOR_UNRESPONSIVE_NODE,
                 AnomalyDetectorSettings.COOLDOWN_MINUTES,
                 AnomalyDetectorSettings.BACKOFF_MINUTES,
-                AnomalyDetectorSettings.BACKOFF_INITIAL_DELAY,
-                AnomalyDetectorSettings.MAX_RETRY_FOR_BACKOFF,
+                AnomalyDetectorSettings.AD_BACKOFF_INITIAL_DELAY,
+                AnomalyDetectorSettings.AD_MAX_RETRY_FOR_BACKOFF,
                 // result index rollover
                 LegacyOpenDistroAnomalyDetectorSettings.AD_RESULT_HISTORY_ROLLOVER_PERIOD,
                 LegacyOpenDistroAnomalyDetectorSettings.AD_RESULT_HISTORY_MAX_DOCS,
@@ -891,8 +891,8 @@ public List<Setting<?>> getSettings() {
                 AnomalyDetectorSettings.MODEL_MAX_SIZE_PERCENTAGE,
                 AnomalyDetectorSettings.MAX_SINGLE_ENTITY_ANOMALY_DETECTORS,
                 AnomalyDetectorSettings.MAX_MULTI_ENTITY_ANOMALY_DETECTORS,
-                AnomalyDetectorSettings.INDEX_PRESSURE_SOFT_LIMIT,
-                AnomalyDetectorSettings.INDEX_PRESSURE_HARD_LIMIT,
+                AnomalyDetectorSettings.AD_INDEX_PRESSURE_SOFT_LIMIT,
+                AnomalyDetectorSettings.AD_INDEX_PRESSURE_HARD_LIMIT,
                 AnomalyDetectorSettings.MAX_PRIMARY_SHARDS,
                 // Security
                 LegacyOpenDistroAnomalyDetectorSettings.FILTER_BY_BACKEND_ROLES,
@@ -910,21 +910,21 @@ public List<Setting<?>> getSettings() {
                 AnomalyDetectorSettings.MAX_RUNNING_ENTITIES_PER_DETECTOR_FOR_HISTORICAL_ANALYSIS,
                 AnomalyDetectorSettings.MAX_CACHED_DELETED_TASKS,
                 // rate limiting
-                AnomalyDetectorSettings.CHECKPOINT_READ_QUEUE_CONCURRENCY,
-                AnomalyDetectorSettings.CHECKPOINT_WRITE_QUEUE_CONCURRENCY,
+                AnomalyDetectorSettings.AD_CHECKPOINT_READ_QUEUE_CONCURRENCY,
+                AnomalyDetectorSettings.AD_CHECKPOINT_WRITE_QUEUE_CONCURRENCY,
                 AnomalyDetectorSettings.ENTITY_COLD_START_QUEUE_CONCURRENCY,
-                AnomalyDetectorSettings.RESULT_WRITE_QUEUE_CONCURRENCY,
-                AnomalyDetectorSettings.CHECKPOINT_READ_QUEUE_BATCH_SIZE,
-                AnomalyDetectorSettings.CHECKPOINT_WRITE_QUEUE_BATCH_SIZE,
-                AnomalyDetectorSettings.RESULT_WRITE_QUEUE_BATCH_SIZE,
+                AnomalyDetectorSettings.AD_RESULT_WRITE_QUEUE_CONCURRENCY,
+                AnomalyDetectorSettings.AD_CHECKPOINT_READ_QUEUE_BATCH_SIZE,
+                AnomalyDetectorSettings.AD_CHECKPOINT_WRITE_QUEUE_BATCH_SIZE,
+                AnomalyDetectorSettings.AD_RESULT_WRITE_QUEUE_BATCH_SIZE,
                 AnomalyDetectorSettings.COLD_ENTITY_QUEUE_MAX_HEAP_PERCENT,
                 AnomalyDetectorSettings.CHECKPOINT_READ_QUEUE_MAX_HEAP_PERCENT,
                 AnomalyDetectorSettings.CHECKPOINT_WRITE_QUEUE_MAX_HEAP_PERCENT,
                 AnomalyDetectorSettings.RESULT_WRITE_QUEUE_MAX_HEAP_PERCENT,
                 AnomalyDetectorSettings.CHECKPOINT_MAINTAIN_QUEUE_MAX_HEAP_PERCENT,
                 AnomalyDetectorSettings.ENTITY_COLD_START_QUEUE_MAX_HEAP_PERCENT,
                 AnomalyDetectorSettings.EXPECTED_COLD_ENTITY_EXECUTION_TIME_IN_MILLISECS,
-                AnomalyDetectorSettings.EXPECTED_CHECKPOINT_MAINTAIN_TIME_IN_MILLISECS,
+                AnomalyDetectorSettings.AD_EXPECTED_CHECKPOINT_MAINTAIN_TIME_IN_MILLISECS,
                 AnomalyDetectorSettings.CHECKPOINT_SAVING_FREQ,
                 AnomalyDetectorSettings.CHECKPOINT_TTL,
                 // query limit

src/main/java/org/opensearch/ad/caching/PriorityCache.java

@@ -53,7 +53,6 @@
 import org.opensearch.ad.model.ModelProfile;
 import org.opensearch.ad.ratelimit.CheckpointMaintainWorker;
 import org.opensearch.ad.ratelimit.CheckpointWriteWorker;
-import org.opensearch.ad.settings.AnomalyDetectorSettings;
 import org.opensearch.ad.settings.EnabledSetting;
 import org.opensearch.ad.util.DateUtils;
 import org.opensearch.cluster.service.ClusterService;
@@ -63,6 +62,7 @@
 import org.opensearch.core.common.Strings;
 import org.opensearch.threadpool.ThreadPool;
 import org.opensearch.timeseries.constant.CommonMessages;
+import org.opensearch.timeseries.settings.TimeSeriesSettings;
 
 import com.google.common.cache.Cache;
 import com.google.common.cache.CacheBuilder;
@@ -169,9 +169,9 @@ public ModelState<EntityModel> get(String modelId, AnomalyDetector detector) {
                         id -> {
                             // reset every 60 intervals
                             return new DoorKeeper(
-                                AnomalyDetectorSettings.DOOR_KEEPER_FOR_CACHE_MAX_INSERTION,
-                                AnomalyDetectorSettings.DOOR_KEEPER_FAULSE_POSITIVE_RATE,
-                                detector.getDetectionIntervalDuration().multipliedBy(AnomalyDetectorSettings.DOOR_KEEPER_MAINTENANCE_FREQ),
+                                TimeSeriesSettings.DOOR_KEEPER_FOR_CACHE_MAX_INSERTION,
+                                TimeSeriesSettings.DOOR_KEEPER_FAULSE_POSITIVE_RATE,
+                                detector.getDetectionIntervalDuration().multipliedBy(TimeSeriesSettings.DOOR_KEEPER_MAINTENANCE_FREQ),
                                 clock
                             );
                         }
@@ -501,7 +501,7 @@ private long getRequiredMemory(AnomalyDetector detector, int numberOfEntity) {
             .estimateTRCFModelSize(
                 dimension,
                 numberOfTrees,
-                AnomalyDetectorSettings.REAL_TIME_BOUNDING_BOX_CACHE_RATIO,
+                TimeSeriesSettings.REAL_TIME_BOUNDING_BOX_CACHE_RATIO,
                 detector.getShingleSize().intValue(),
                 true
             );

src/main/java/org/opensearch/ad/indices/AnomalyDetectionIndices.java

@@ -16,12 +16,12 @@
 import static org.opensearch.ad.settings.AnomalyDetectorSettings.AD_RESULT_HISTORY_RETENTION_PERIOD;
 import static org.opensearch.ad.settings.AnomalyDetectorSettings.AD_RESULT_HISTORY_ROLLOVER_PERIOD;
 import static org.opensearch.ad.settings.AnomalyDetectorSettings.ANOMALY_DETECTION_STATE_INDEX_MAPPING_FILE;
-import static org.opensearch.ad.settings.AnomalyDetectorSettings.ANOMALY_DETECTORS_INDEX_MAPPING_FILE;
-import static org.opensearch.ad.settings.AnomalyDetectorSettings.ANOMALY_DETECTOR_JOBS_INDEX_MAPPING_FILE;
 import static org.opensearch.ad.settings.AnomalyDetectorSettings.ANOMALY_RESULTS_INDEX_MAPPING_FILE;
 import static org.opensearch.ad.settings.AnomalyDetectorSettings.CHECKPOINT_INDEX_MAPPING_FILE;
 import static org.opensearch.ad.settings.AnomalyDetectorSettings.MAX_PRIMARY_SHARDS;
 import static org.opensearch.timeseries.constant.CommonMessages.CAN_NOT_FIND_RESULT_INDEX;
+import static org.opensearch.timeseries.settings.TimeSeriesSettings.ANOMALY_DETECTORS_INDEX_MAPPING_FILE;
+import static org.opensearch.timeseries.settings.TimeSeriesSettings.ANOMALY_DETECTOR_JOBS_INDEX_MAPPING_FILE;
 
 import java.io.IOException;
 import java.net.URL;

src/main/java/org/opensearch/ad/ml/EntityColdStarter.java

@@ -51,12 +51,12 @@
 import org.opensearch.ad.model.IntervalTimeConfiguration;
 import org.opensearch.ad.ratelimit.CheckpointWriteWorker;
 import org.opensearch.ad.ratelimit.RequestPriority;
-import org.opensearch.ad.settings.AnomalyDetectorSettings;
 import org.opensearch.ad.settings.EnabledSetting;
 import org.opensearch.ad.util.ExceptionUtil;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.threadpool.ThreadPool;
 import org.opensearch.timeseries.dataprocessor.Imputer;
+import org.opensearch.timeseries.settings.TimeSeriesSettings;
 
 import com.amazon.randomcutforest.config.Precision;
 import com.amazon.randomcutforest.parkservices.ThresholdedRandomCutForest;
@@ -251,9 +251,9 @@ private void coldStart(
                     id -> {
                         // reset every 60 intervals
                         return new DoorKeeper(
-                            AnomalyDetectorSettings.DOOR_KEEPER_FOR_COLD_STARTER_MAX_INSERTION,
-                            AnomalyDetectorSettings.DOOR_KEEPER_FAULSE_POSITIVE_RATE,
-                            detector.getDetectionIntervalDuration().multipliedBy(AnomalyDetectorSettings.DOOR_KEEPER_MAINTENANCE_FREQ),
+                            TimeSeriesSettings.DOOR_KEEPER_FOR_COLD_STARTER_MAX_INSERTION,
+                            TimeSeriesSettings.DOOR_KEEPER_FAULSE_POSITIVE_RATE,
+                            detector.getDetectionIntervalDuration().multipliedBy(TimeSeriesSettings.DOOR_KEEPER_MAINTENANCE_FREQ),
                             clock
                         );
                     }
@@ -365,7 +365,7 @@ private void trainModelFromDataSegments(
             .parallelExecutionEnabled(false)
             .compact(true)
             .precision(Precision.FLOAT_32)
-            .boundingBoxCacheFraction(AnomalyDetectorSettings.REAL_TIME_BOUNDING_BOX_CACHE_RATIO)
+            .boundingBoxCacheFraction(TimeSeriesSettings.REAL_TIME_BOUNDING_BOX_CACHE_RATIO)
             // same with dimension for opportunistic memory saving
             // Usually, we use it as shingleSize(dimension). When a new point comes in, we will
             // look at the point store if there is any overlapping. Say the previously-stored

src/main/java/org/opensearch/ad/ml/ModelManager.java

@@ -46,6 +46,7 @@
 import org.opensearch.common.settings.Setting;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.unit.TimeValue;
+import org.opensearch.timeseries.settings.TimeSeriesSettings;
 
 import com.amazon.randomcutforest.RandomCutForest;
 import com.amazon.randomcutforest.config.Precision;
@@ -527,7 +528,7 @@ private void trainModelForStep(
             .parallelExecutionEnabled(false)
             .compact(true)
             .precision(Precision.FLOAT_32)
-            .boundingBoxCacheFraction(AnomalyDetectorSettings.REAL_TIME_BOUNDING_BOX_CACHE_RATIO)
+            .boundingBoxCacheFraction(TimeSeriesSettings.REAL_TIME_BOUNDING_BOX_CACHE_RATIO)
             .shingleSize(detector.getShingleSize())
             .anomalyRate(1 - thresholdMinPvalue)
             .build();

src/main/java/org/opensearch/ad/model/AnomalyDetector.java

@@ -15,10 +15,10 @@
 import static org.opensearch.ad.constant.ADCommonName.CUSTOM_RESULT_INDEX_PREFIX;
 import static org.opensearch.ad.model.AnomalyDetectorType.MULTI_ENTITY;
 import static org.opensearch.ad.model.AnomalyDetectorType.SINGLE_ENTITY;
-import static org.opensearch.ad.settings.AnomalyDetectorSettings.DEFAULT_SHINGLE_SIZE;
 import static org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
 import static org.opensearch.index.query.AbstractQueryBuilder.parseInnerQueryBuilder;
 import static org.opensearch.timeseries.constant.CommonMessages.INVALID_CHAR_IN_RESULT_INDEX_NAME;
+import static org.opensearch.timeseries.settings.TimeSeriesSettings.DEFAULT_SHINGLE_SIZE;
 
 import java.io.IOException;
 import java.time.Duration;
@@ -33,7 +33,6 @@
 import org.opensearch.ad.common.exception.ADValidationException;
 import org.opensearch.ad.constant.ADCommonMessages;
 import org.opensearch.ad.constant.CommonValue;
-import org.opensearch.ad.settings.AnomalyDetectorSettings;
 import org.opensearch.ad.settings.NumericSetting;
 import org.opensearch.ad.util.ParseUtils;
 import org.opensearch.common.ParsingException;
@@ -54,6 +53,7 @@
 import org.opensearch.timeseries.annotation.Generated;
 import org.opensearch.timeseries.constant.CommonMessages;
 import org.opensearch.timeseries.constant.CommonName;
+import org.opensearch.timeseries.settings.TimeSeriesSettings;
 
 import com.google.common.base.Objects;
 import com.google.common.collect.ImmutableList;
@@ -191,10 +191,7 @@ public AnomalyDetector(
         }
         if (invalidShingleSizeRange(shingleSize)) {
             throw new ADValidationException(
-                "Shingle size must be a positive integer no larger than "
-                    + AnomalyDetectorSettings.MAX_SHINGLE_SIZE
-                    + ". Got "
-                    + shingleSize,
+                "Shingle size must be a positive integer no larger than " + TimeSeriesSettings.MAX_SHINGLE_SIZE + ". Got " + shingleSize,
                 DetectorValidationIssueType.SHINGLE_SIZE_FIELD,
                 ValidationAspect.DETECTOR
             );
@@ -756,6 +753,6 @@ private static boolean isMultientityDetector(List<String> categoryFields) {
     }
 
     public boolean invalidShingleSizeRange(Integer shingleSizeToTest) {
-        return shingleSizeToTest != null && (shingleSizeToTest < 1 || shingleSizeToTest > AnomalyDetectorSettings.MAX_SHINGLE_SIZE);
+        return shingleSizeToTest != null && (shingleSizeToTest < 1 || shingleSizeToTest > TimeSeriesSettings.MAX_SHINGLE_SIZE);
     }
 }

src/main/java/org/opensearch/ad/ratelimit/CheckpointMaintainWorker.java

@@ -11,8 +11,8 @@
 
 package org.opensearch.ad.ratelimit;
 
-import static org.opensearch.ad.settings.AnomalyDetectorSettings.CHECKPOINT_WRITE_QUEUE_BATCH_SIZE;
-import static org.opensearch.ad.settings.AnomalyDetectorSettings.EXPECTED_CHECKPOINT_MAINTAIN_TIME_IN_MILLISECS;
+import static org.opensearch.ad.settings.AnomalyDetectorSettings.AD_CHECKPOINT_WRITE_QUEUE_BATCH_SIZE;
+import static org.opensearch.ad.settings.AnomalyDetectorSettings.AD_EXPECTED_CHECKPOINT_MAINTAIN_TIME_IN_MILLISECS;
 
 import java.time.Clock;
 import java.time.Duration;
@@ -76,15 +76,15 @@ public CheckpointMaintainWorker(
             nodeStateManager
         );
 
-        this.batchSize = CHECKPOINT_WRITE_QUEUE_BATCH_SIZE.get(settings);
-        clusterService.getClusterSettings().addSettingsUpdateConsumer(CHECKPOINT_WRITE_QUEUE_BATCH_SIZE, it -> this.batchSize = it);
+        this.batchSize = AD_CHECKPOINT_WRITE_QUEUE_BATCH_SIZE.get(settings);
+        clusterService.getClusterSettings().addSettingsUpdateConsumer(AD_CHECKPOINT_WRITE_QUEUE_BATCH_SIZE, it -> this.batchSize = it);
 
-        this.expectedExecutionTimeInMilliSecsPerRequest = AnomalyDetectorSettings.EXPECTED_CHECKPOINT_MAINTAIN_TIME_IN_MILLISECS
+        this.expectedExecutionTimeInMilliSecsPerRequest = AnomalyDetectorSettings.AD_EXPECTED_CHECKPOINT_MAINTAIN_TIME_IN_MILLISECS
             .get(settings);
         clusterService
             .getClusterSettings()
             .addSettingsUpdateConsumer(
-                EXPECTED_CHECKPOINT_MAINTAIN_TIME_IN_MILLISECS,
+                AD_EXPECTED_CHECKPOINT_MAINTAIN_TIME_IN_MILLISECS,
                 it -> this.expectedExecutionTimeInMilliSecsPerRequest = it
             );
         this.adapter = adapter;

src/main/java/org/opensearch/ad/ratelimit/CheckpointReadWorker.java

@@ -11,8 +11,8 @@
 
 package org.opensearch.ad.ratelimit;
 
-import static org.opensearch.ad.settings.AnomalyDetectorSettings.CHECKPOINT_READ_QUEUE_BATCH_SIZE;
-import static org.opensearch.ad.settings.AnomalyDetectorSettings.CHECKPOINT_READ_QUEUE_CONCURRENCY;
+import static org.opensearch.ad.settings.AnomalyDetectorSettings.AD_CHECKPOINT_READ_QUEUE_BATCH_SIZE;
+import static org.opensearch.ad.settings.AnomalyDetectorSettings.AD_CHECKPOINT_READ_QUEUE_CONCURRENCY;
 
 import java.time.Clock;
 import java.time.Duration;
@@ -124,9 +124,9 @@ public CheckpointReadWorker(
             mediumSegmentPruneRatio,
             lowSegmentPruneRatio,
             maintenanceFreqConstant,
-            CHECKPOINT_READ_QUEUE_CONCURRENCY,
+            AD_CHECKPOINT_READ_QUEUE_CONCURRENCY,
             executionTtl,
-            CHECKPOINT_READ_QUEUE_BATCH_SIZE,
+            AD_CHECKPOINT_READ_QUEUE_BATCH_SIZE,
             stateTtl,
             stateManager
         );

src/main/java/org/opensearch/ad/ratelimit/CheckpointWriteWorker.java

@@ -11,8 +11,8 @@
 
 package org.opensearch.ad.ratelimit;
 
-import static org.opensearch.ad.settings.AnomalyDetectorSettings.CHECKPOINT_WRITE_QUEUE_BATCH_SIZE;
-import static org.opensearch.ad.settings.AnomalyDetectorSettings.CHECKPOINT_WRITE_QUEUE_CONCURRENCY;
+import static org.opensearch.ad.settings.AnomalyDetectorSettings.AD_CHECKPOINT_WRITE_QUEUE_BATCH_SIZE;
+import static org.opensearch.ad.settings.AnomalyDetectorSettings.AD_CHECKPOINT_WRITE_QUEUE_CONCURRENCY;
 
 import java.time.Clock;
 import java.time.Duration;
@@ -88,9 +88,9 @@ public CheckpointWriteWorker(
             mediumSegmentPruneRatio,
             lowSegmentPruneRatio,
             maintenanceFreqConstant,
-            CHECKPOINT_WRITE_QUEUE_CONCURRENCY,
+            AD_CHECKPOINT_WRITE_QUEUE_CONCURRENCY,
             executionTtl,
-            CHECKPOINT_WRITE_QUEUE_BATCH_SIZE,
+            AD_CHECKPOINT_WRITE_QUEUE_BATCH_SIZE,
             stateTtl,
             stateManager
         );

src/main/java/org/opensearch/ad/ratelimit/ColdEntityWorker.java

@@ -11,7 +11,7 @@
 
 package org.opensearch.ad.ratelimit;
 
-import static org.opensearch.ad.settings.AnomalyDetectorSettings.CHECKPOINT_READ_QUEUE_BATCH_SIZE;
+import static org.opensearch.ad.settings.AnomalyDetectorSettings.AD_CHECKPOINT_READ_QUEUE_BATCH_SIZE;
 import static org.opensearch.ad.settings.AnomalyDetectorSettings.EXPECTED_COLD_ENTITY_EXECUTION_TIME_IN_MILLISECS;
 
 import java.time.Clock;
@@ -84,8 +84,8 @@ public ColdEntityWorker(
             nodeStateManager
         );
 
-        this.batchSize = CHECKPOINT_READ_QUEUE_BATCH_SIZE.get(settings);
-        clusterService.getClusterSettings().addSettingsUpdateConsumer(CHECKPOINT_READ_QUEUE_BATCH_SIZE, it -> this.batchSize = it);
+        this.batchSize = AD_CHECKPOINT_READ_QUEUE_BATCH_SIZE.get(settings);
+        clusterService.getClusterSettings().addSettingsUpdateConsumer(AD_CHECKPOINT_READ_QUEUE_BATCH_SIZE, it -> this.batchSize = it);
 
         this.expectedExecutionTimeInMilliSecsPerRequest = AnomalyDetectorSettings.EXPECTED_COLD_ENTITY_EXECUTION_TIME_IN_MILLISECS
             .get(settings);

src/main/java/org/opensearch/ad/ratelimit/ResultWriteWorker.java

@@ -11,8 +11,8 @@
 
 package org.opensearch.ad.ratelimit;
 
-import static org.opensearch.ad.settings.AnomalyDetectorSettings.RESULT_WRITE_QUEUE_BATCH_SIZE;
-import static org.opensearch.ad.settings.AnomalyDetectorSettings.RESULT_WRITE_QUEUE_CONCURRENCY;
+import static org.opensearch.ad.settings.AnomalyDetectorSettings.AD_RESULT_WRITE_QUEUE_BATCH_SIZE;
+import static org.opensearch.ad.settings.AnomalyDetectorSettings.AD_RESULT_WRITE_QUEUE_CONCURRENCY;
 
 import java.time.Clock;
 import java.time.Duration;
@@ -87,9 +87,9 @@ public ResultWriteWorker(
             mediumSegmentPruneRatio,
             lowSegmentPruneRatio,
             maintenanceFreqConstant,
-            RESULT_WRITE_QUEUE_CONCURRENCY,
+            AD_RESULT_WRITE_QUEUE_CONCURRENCY,
             executionTtl,
-            RESULT_WRITE_QUEUE_BATCH_SIZE,
+            AD_RESULT_WRITE_QUEUE_BATCH_SIZE,
             stateTtl,
             stateManager
         );