From 0509e8e1ce6041f37554c40edf7f6f42919387df Mon Sep 17 00:00:00 2001
From: Daniel-Weinshenker <9273337+dweinshenker@users.noreply.github.com>
Date: Wed, 31 Jul 2024 07:19:42 -0700
Subject: [PATCH] Add OpenSearch Advanced Config for DBAAS (#900)

* Add OpenSearch advanced config settings

* refactor files into appropriate directories

* add database advanced config to response samples for each engine in dbaas
---
 specification/DigitalOcean-public.v2.yaml     |   5 +-
 .../kafka_advanced_config.yml}                |   0
 .../mongo_advanced_config.yml}                |   0
 .../mysql_advanced_config.yml}                |   0
 .../opensearch_advanced_config.yml            | 284 ++++++++++++++++++
 .../pgbouncer_advanced_config.yml}            |   0
 .../postgres_advanced_config.yml}             |   4 +-
 .../redis_advanced_config.yml}                |   2 +-
 .../timescaledb_advanced_config.yml}          |   0
 .../databases/models/database_config.yml      |  11 +-
 .../elasticsearch_logsink.yml}                |   0
 .../opensearch_logsink.yml}                   |   0
 .../rsyslog_logsink.yml}                      |   0
 .../databases/models/logsink_create.yml       |   6 +-
 .../databases/models/logsink_update.yml       |   6 +-
 .../databases/models/logsink_verbose.yml      |   6 +-
 .../databases/responses/database_config.yml   |   9 +-
 17 files changed, 311 insertions(+), 22 deletions(-)
 rename specification/resources/databases/models/{kafka.yml => advanced_config/kafka_advanced_config.yml} (100%)
 rename specification/resources/databases/models/{mongo.yml => advanced_config/mongo_advanced_config.yml} (100%)
 rename specification/resources/databases/models/{mysql.yml => advanced_config/mysql_advanced_config.yml} (100%)
 create mode 100644 specification/resources/databases/models/advanced_config/opensearch_advanced_config.yml
 rename specification/resources/databases/models/{pgbouncer.yml => advanced_config/pgbouncer_advanced_config.yml} (100%)
 rename specification/resources/databases/models/{postgres.yml => advanced_config/postgres_advanced_config.yml} (99%)
 rename specification/resources/databases/models/{redis.yml => advanced_config/redis_advanced_config.yml} (98%)
 rename specification/resources/databases/models/{timescaledb.yml => advanced_config/timescaledb_advanced_config.yml} (100%)
 rename specification/resources/databases/models/{elasticsearch.yml => logsink/elasticsearch_logsink.yml} (100%)
 rename specification/resources/databases/models/{opensearch.yml => logsink/opensearch_logsink.yml} (100%)
 rename specification/resources/databases/models/{rsyslog.yml => logsink/rsyslog_logsink.yml} (100%)

diff --git a/specification/DigitalOcean-public.v2.yaml b/specification/DigitalOcean-public.v2.yaml
index 09b4acfbe..55f73fc58 100644
--- a/specification/DigitalOcean-public.v2.yaml
+++ b/specification/DigitalOcean-public.v2.yaml
@@ -144,8 +144,9 @@ tags:
       simplifies the creation and management of highly available database clusters. Currently, it
       offers support for [PostgreSQL](http://www.digitalocean.com/docs/databases/postgresql/),
       [Redis](https://www.digitalocean.com/docs/databases/redis/),
-      [MySQL](https://www.digitalocean.com/docs/databases/mysql/), and
-      [MongoDB](https://www.digitalocean.com/docs/databases/mongodb/).
+      [MySQL](https://www.digitalocean.com/docs/databases/mysql/), 
+      [MongoDB](https://www.digitalocean.com/docs/databases/mongodb/), and
+      [OpenSearch](https://docs.digitalocean.com/products/databases/opensearch/).
 
       By sending requests to the `/v2/databases` endpoint, you can list, create, or delete
       database clusters as well as scale the size of a cluster, add or remove read-only replicas,
diff --git a/specification/resources/databases/models/kafka.yml b/specification/resources/databases/models/advanced_config/kafka_advanced_config.yml
similarity index 100%
rename from specification/resources/databases/models/kafka.yml
rename to specification/resources/databases/models/advanced_config/kafka_advanced_config.yml
diff --git a/specification/resources/databases/models/mongo.yml b/specification/resources/databases/models/advanced_config/mongo_advanced_config.yml
similarity index 100%
rename from specification/resources/databases/models/mongo.yml
rename to specification/resources/databases/models/advanced_config/mongo_advanced_config.yml
diff --git a/specification/resources/databases/models/mysql.yml b/specification/resources/databases/models/advanced_config/mysql_advanced_config.yml
similarity index 100%
rename from specification/resources/databases/models/mysql.yml
rename to specification/resources/databases/models/advanced_config/mysql_advanced_config.yml
diff --git a/specification/resources/databases/models/advanced_config/opensearch_advanced_config.yml b/specification/resources/databases/models/advanced_config/opensearch_advanced_config.yml
new file mode 100644
index 000000000..b72e7b1de
--- /dev/null
+++ b/specification/resources/databases/models/advanced_config/opensearch_advanced_config.yml
@@ -0,0 +1,284 @@
+type: object
+
+properties:
+  http_max_content_length_bytes:
+    description: >-
+      Maximum content length for HTTP requests to the OpenSearch HTTP API, in bytes.
+    type: integer
+    example: 100000000
+    minimum: 1
+    maximum: 2147483647
+    default: 100000000
+  http_max_header_size_bytes:
+    description: >-
+      Maximum size of allowed headers, in bytes.
+    type: integer
+    example: 8192
+    minimum: 1024
+    maximum: 262144
+    default: 8192
+  http_max_initial_line_length_bytes:
+    description: >-
+      Maximum length of an HTTP URL, in bytes.
+    type: integer
+    example: 4096
+    minimum: 1024
+    maximum: 65536
+    default: 4096
+  indices_query_bool_max_clause_count:
+    description: >-
+      Maximum number of clauses Lucene BooleanQuery can have. 
+      Only increase it if necessary, as it may cause performance issues.
+    type: integer
+    example: 1024
+    minimum: 64
+    maximum: 4096
+    default: 1024
+  indices_fielddata_cache_size_percentage:
+    description: >-
+      Maximum amount of heap memory used for field data cache, expressed as a percentage.
+      Decreasing the value too much will increase overhead of loading field data.
+      Increasing the value too much will decrease amount of heap available for other operations.
+    type: integer
+    example: 3
+    minimum: 3
+    maximum: 100
+  indices_memory_index_buffer_size_percentage:
+    description: >-
+      Total amount of heap used for indexing buffer before writing segments to disk, expressed as a percentage.
+      Too low value will slow down indexing; too high value will increase indexing performance but causes performance issues for query performance.
+    type: integer
+    example: 10
+    minimum: 3
+    maximum: 40
+    default: 10
+  indices_memory_min_index_buffer_size_mb:
+    description: >-
+      Minimum amount of heap used for indexing buffer before writing segments to disk, in mb.
+      Works in conjunction with indices_memory_index_buffer_size_percentage, each being enforced.
+    type: integer
+    example: 48
+    minimum: 3
+    maximum: 2048
+    default: 48
+  indices_memory_max_index_buffer_size_mb:
+    description: >-
+      Maximum amount of heap used for indexing buffer before writing segments to disk, in mb.
+      Works in conjunction with indices_memory_index_buffer_size_percentage, each being enforced.
+      The default is unbounded.
+    type: integer
+    example: 48
+    minimum: 3
+    maximum: 2048
+  indices_queries_cache_size_percentage:
+    description: >-
+      Maximum amount of heap used for query cache. 
+      Too low value will decrease query performance and increase performance for other operations; too high value will cause issues with other functionality.
+    type: integer
+    example: 10
+    minimum: 3
+    maximum: 40
+    default: 10
+  indices_recovery_max_mb_per_sec:
+    description: >-
+      Limits total inbound and outbound recovery traffic for each node, expressed in mb per second.
+      Applies to both peer recoveries as well as snapshot recoveries (i.e., restores from a snapshot).
+    type: integer
+    example: 40
+    minimum: 40
+    maximum: 400
+    default: 40
+  indices_recovery_max_concurrent_file_chunks:
+    description: >-
+      Maximum number of file chunks sent in parallel for each recovery.
+    type: integer
+    example: 2
+    minimum: 2
+    maximum: 5
+    default: 2
+  thread_pool_search_size:
+    description: >-
+      Number of workers in the search operation thread pool. 
+      Do note this may have maximum value depending on CPU count - value is automatically lowered if set to higher than maximum value.
+    type: integer
+    example: 1
+    minimum: 1
+    maximum: 128
+  thread_pool_search_throttled_size:
+    description: >-
+      Number of workers in the search throttled operation thread pool. This pool is used for searching frozen indices.
+      Do note this may have maximum value depending on CPU count - value is automatically lowered if set to higher than maximum value.
+    type: integer
+    example: 1
+    minimum: 1
+    maximum: 128
+  thread_pool_get_size:
+    description: >-
+      Number of workers in the get operation thread pool. 
+      Do note this may have maximum value depending on CPU count - value is automatically lowered if set to higher than maximum value.
+    type: integer
+    example: 1
+    minimum: 1
+    maximum: 128
+  thread_pool_analyze_size:
+    description: >-
+      Number of workers in the analyze operation thread pool. 
+      Do note this may have maximum value depending on CPU count - value is automatically lowered if set to higher than maximum value.
+    type: integer
+    example: 1
+    minimum: 1
+    maximum: 128
+  thread_pool_write_size:
+    description: >-
+      Number of workers in the write operation thread pool. 
+      Do note this may have maximum value depending on CPU count - value is automatically lowered if set to higher than maximum value.
+    type: integer
+    example: 1
+    minimum: 1
+    maximum: 128
+  thread_pool_force_merge_size:
+    description: >-
+      Number of workers in the force merge operation thread pool. This pool is used for forcing a merge between shards of one or more indices.
+      Do note this may have maximum value depending on CPU count - value is automatically lowered if set to higher than maximum value.
+    type: integer
+    example: 1
+    minimum: 1
+    maximum: 128
+  thread_pool_search_queue_size:
+    description: >-
+      Size of queue for operations in the search thread pool.
+    type: integer
+    example: 10
+    minimum: 10
+    maximum: 2000
+  thread_pool_search_throttled_queue_size:
+    description: >-
+      Size of queue for operations in the search throttled thread pool.
+    type: integer
+    example: 10
+    minimum: 10
+    maximum: 2000
+  thread_pool_get_queue_size:
+    description: >-
+      Size of queue for operations in the get thread pool.
+    type: integer
+    example: 10
+    minimum: 10
+    maximum: 2000
+  thread_pool_analyze_queue_size:
+    description: >-
+      Size of queue for operations in the analyze thread pool.
+    type: integer
+    example: 10
+    minimum: 10
+    maximum: 2000
+  thread_pool_write_queue_size:
+    description: >-
+      Size of queue for operations in the write thread pool.
+    type: integer
+    example: 10
+    minimum: 10
+    maximum: 2000
+  ism_enabled:
+    description: >-
+      Specifies whether ISM is enabled or not.
+    type: boolean
+    example: true
+    default: true
+  ism_history_enabled:
+    description: >-
+      Specifies whether audit history is enabled or not. The logs from ISM are automatically indexed to a logs document.
+    type: boolean
+    example: true
+    default: true
+  ism_history_max_age_hours:
+    description: >-
+      Maximum age before rolling over the audit history index, in hours.
+    type: integer
+    example: 24
+    minimum: 1
+    maximum: 2147483647
+    default: 24
+  ism_history_max_docs:
+    description: >-
+      Maximum number of documents before rolling over the audit history index.
+    type: integer
+    example: 2500000
+    minimum: 1
+    maximum: 9223372036854776000
+    default: 2500000
+  ism_history_rollover_check_period_hours:
+    description: >-
+      The time between rollover checks for the audit history index, in hours.
+    type: integer
+    example: 8
+    minimum: 1
+    maximum: 2147483647
+    default: 8
+  ism_history_rollover_retention_period_days:
+    description: >-
+      Length of time long audit history indices are kept, in days.
+    type: integer
+    example: 30
+    minimum: 1
+    maximum: 2147483647
+    default: 30
+  search_max_buckets:
+    description: >-
+      Maximum number of aggregation buckets allowed in a single response.
+    type: integer
+    example: 10000
+    minimum: 1
+    maximum: 1000000
+    default: 10000
+  action_auto_create_index_enabled:
+    description: >-
+      Specifices whether to allow automatic creation of indices.
+    type: boolean
+    example: true
+    default: true
+  enable_security_audit:
+    description: >-
+      Specifies whether to allow security audit logging.
+    type: boolean
+    example: false
+    default: false
+  action_destructive_requires_name:
+    description: >-
+      Specifies whether to require explicit index names when deleting indices.
+    type: boolean
+    example: false
+  cluster_max_shards_per_node:
+    description: >-
+      Maximum number of shards allowed per data node.
+    type: integer
+    example: 100
+    minimum: 100
+    maximum: 10000
+  override_main_response_version:
+    description: >-
+      Compatibility mode sets OpenSearch to report its version as 7.10 so clients continue to work.
+    type: boolean
+    example: false
+    default: false
+  script_max_compilations_rate:
+    description: >-
+      Limits the number of inline script compilations within a period of time. Default is use-context
+    type: string
+    example: 75/5m
+    default: use-context
+  cluster_routing_allocation_node_concurrent_recoveries:
+    description: >-
+      Maximum concurrent incoming/outgoing shard recoveries (normally replicas) are allowed to happen per node . 
+    type: integer
+    example: 2
+    minimum: 2
+    maximum: 16
+    default: 2
+  reindex_remote_whitelist:
+    description: >-
+      Allowlist of remote IP addresses for reindexing. Changing this value will cause all OpenSearch instances to restart.
+    type: array
+    items: 
+      type: string
+    example: ["255.255.223.233:9200", "222.33.222.222:6300"]
\ No newline at end of file
diff --git a/specification/resources/databases/models/pgbouncer.yml b/specification/resources/databases/models/advanced_config/pgbouncer_advanced_config.yml
similarity index 100%
rename from specification/resources/databases/models/pgbouncer.yml
rename to specification/resources/databases/models/advanced_config/pgbouncer_advanced_config.yml
diff --git a/specification/resources/databases/models/postgres.yml b/specification/resources/databases/models/advanced_config/postgres_advanced_config.yml
similarity index 99%
rename from specification/resources/databases/models/postgres.yml
rename to specification/resources/databases/models/advanced_config/postgres_advanced_config.yml
index 6e32228cc..dc77b86cc 100644
--- a/specification/resources/databases/models/postgres.yml
+++ b/specification/resources/databases/models/advanced_config/postgres_advanced_config.yml
@@ -384,7 +384,7 @@ properties:
     maximum: 60.0
     example: 41.5
   pgbouncer:
-    $ref: './pgbouncer.yml'
+    $ref: './pgbouncer_advanced_config.yml'
   work_mem:
     description: >-
       The maximum amount of memory, in MB, used by a query operation (such as a sort or hash table) before writing to temporary disk files.
@@ -394,7 +394,7 @@ properties:
     maximum: 1024
     example: 4
   timescaledb:
-    $ref: './timescaledb.yml'
+    $ref: './timescaledb_advanced_config.yml'
   synchronous_replication:
     description: Synchronous replication type. Note that the service plan also needs to support synchronous replication.
     type: string
diff --git a/specification/resources/databases/models/redis.yml b/specification/resources/databases/models/advanced_config/redis_advanced_config.yml
similarity index 98%
rename from specification/resources/databases/models/redis.yml
rename to specification/resources/databases/models/advanced_config/redis_advanced_config.yml
index fb80af11a..9e418eb60 100644
--- a/specification/resources/databases/models/redis.yml
+++ b/specification/resources/databases/models/advanced_config/redis_advanced_config.yml
@@ -2,7 +2,7 @@ type: object
 
 properties:
   redis_maxmemory_policy:
-    $ref: 'eviction_policy_model.yml'
+    $ref: '../eviction_policy_model.yml'
   redis_pubsub_client_output_buffer_limit:
     description: >-
       Set output buffer limit for pub / sub clients in MB. The value is the hard
diff --git a/specification/resources/databases/models/timescaledb.yml b/specification/resources/databases/models/advanced_config/timescaledb_advanced_config.yml
similarity index 100%
rename from specification/resources/databases/models/timescaledb.yml
rename to specification/resources/databases/models/advanced_config/timescaledb_advanced_config.yml
diff --git a/specification/resources/databases/models/database_config.yml b/specification/resources/databases/models/database_config.yml
index 61282ddbf..4143b1ba2 100644
--- a/specification/resources/databases/models/database_config.yml
+++ b/specification/resources/databases/models/database_config.yml
@@ -3,8 +3,9 @@ type: object
 properties:
   config:
     anyOf:
-      - $ref: './mysql.yml'
-      - $ref: './postgres.yml'
-      - $ref: './redis.yml'
-      - $ref: './mongo.yml'
-      - $ref: './kafka.yml'
+      - $ref: './advanced_config/mysql_advanced_config.yml'
+      - $ref: './advanced_config/postgres_advanced_config.yml'
+      - $ref: './advanced_config/redis_advanced_config.yml'
+      - $ref: './advanced_config/mongo_advanced_config.yml'
+      - $ref: './advanced_config/kafka_advanced_config.yml'
+      - $ref: './advanced_config/opensearch_advanced_config.yml'
diff --git a/specification/resources/databases/models/elasticsearch.yml b/specification/resources/databases/models/logsink/elasticsearch_logsink.yml
similarity index 100%
rename from specification/resources/databases/models/elasticsearch.yml
rename to specification/resources/databases/models/logsink/elasticsearch_logsink.yml
diff --git a/specification/resources/databases/models/opensearch.yml b/specification/resources/databases/models/logsink/opensearch_logsink.yml
similarity index 100%
rename from specification/resources/databases/models/opensearch.yml
rename to specification/resources/databases/models/logsink/opensearch_logsink.yml
diff --git a/specification/resources/databases/models/rsyslog.yml b/specification/resources/databases/models/logsink/rsyslog_logsink.yml
similarity index 100%
rename from specification/resources/databases/models/rsyslog.yml
rename to specification/resources/databases/models/logsink/rsyslog_logsink.yml
diff --git a/specification/resources/databases/models/logsink_create.yml b/specification/resources/databases/models/logsink_create.yml
index 5d65a4700..3a44034f5 100644
--- a/specification/resources/databases/models/logsink_create.yml
+++ b/specification/resources/databases/models/logsink_create.yml
@@ -5,6 +5,6 @@ allOf:
 - properties:
     config:
       anyOf:
-      - $ref: './rsyslog.yml'
-      - $ref: './elasticsearch.yml'
-      - $ref: './opensearch.yml'
\ No newline at end of file
+      - $ref: './logsink/rsyslog_logsink.yml'
+      - $ref: './logsink/elasticsearch_logsink.yml'
+      - $ref: './logsink/opensearch_logsink.yml'
\ No newline at end of file
diff --git a/specification/resources/databases/models/logsink_update.yml b/specification/resources/databases/models/logsink_update.yml
index d2bb32020..2174051d8 100644
--- a/specification/resources/databases/models/logsink_update.yml
+++ b/specification/resources/databases/models/logsink_update.yml
@@ -3,9 +3,9 @@ type: object
 properties:
   config:
     anyOf:
-      - $ref: './rsyslog.yml'
-      - $ref: './elasticsearch.yml'
-      - $ref: './opensearch.yml'
+      - $ref: './logsink/rsyslog_logsink.yml'
+      - $ref: './logsink/elasticsearch_logsink.yml'
+      - $ref: './logsink/opensearch_logsink.yml'
 
 required:
   - config
\ No newline at end of file
diff --git a/specification/resources/databases/models/logsink_verbose.yml b/specification/resources/databases/models/logsink_verbose.yml
index e8374c3ac..2215096a6 100644
--- a/specification/resources/databases/models/logsink_verbose.yml
+++ b/specification/resources/databases/models/logsink_verbose.yml
@@ -5,9 +5,9 @@ allOf:
   - properties:
       config:
         anyOf:
-          - $ref: "./rsyslog.yml"
-          - $ref: "./elasticsearch.yml"
-          - $ref: "./opensearch.yml"
+          - $ref: './logsink/rsyslog_logsink.yml'
+          - $ref: './logsink/elasticsearch_logsink.yml'
+          - $ref: './logsink/opensearch_logsink.yml'
         example:
           config:
             server: 192.168.0.1
diff --git a/specification/resources/databases/responses/database_config.yml b/specification/resources/databases/responses/database_config.yml
index 27e469e9c..6b0f36817 100644
--- a/specification/resources/databases/responses/database_config.yml
+++ b/specification/resources/databases/responses/database_config.yml
@@ -15,9 +15,12 @@ content:
       properties:
         config:
           anyOf:
-            - $ref: '../models/mysql.yml'
-            - $ref: '../models/postgres.yml'
-            - $ref: '../models/redis.yml'
+            - $ref: '../models/advanced_config/mysql_advanced_config.yml'
+            - $ref: '../models/advanced_config/postgres_advanced_config.yml'
+            - $ref: '../models/advanced_config/redis_advanced_config.yml'
+            - $ref: '../models/advanced_config/kafka_advanced_config.yml'
+            - $ref: '../models/advanced_config/opensearch_advanced_config.yml'
+            - $ref: '../models/advanced_config/mongo_advanced_config.yml'
       required:
         - config
     example: