Merge branch 'apecloud:main' into main

yabinji · Mar 4, 2024 · fc71931 · fc71931
2 parents 56ee827 + 4d191f8
commit fc71931
Show file tree

Hide file tree

Showing 305 changed files with 17,227 additions and 2,394 deletions.
diff --git a/.github/workflows/release-chart.yml b/.github/workflows/release-chart.yml
@@ -52,8 +52,8 @@ on:
           - nebula-cluster
           - neon
           - neon-cluster
-          - oceanbase
-          - oceanbase-cluster
+          - oceanbase-ce
+          - oceanbase-ce-cluster
           - official-postgresql
           - official-postgresql-cluster
           - openldap
@@ -78,8 +78,8 @@ on:
           - redis-cluster
           - risingwave
           - risingwave-cluster
-          - starrocks
-          - starrocks-cluster
+          - starrocks-ce
+          - starrocks-ce-cluster
           - tdengine
           - tdengine-cluster
           - tidb
@@ -88,12 +88,15 @@ on:
           - weaviate-cluster
           - xinference
           - xinference-cluster
+          - yashandb
+          - yashandb-cluster
           - zookeeper
           - zookeeper-cluster
       chart_version:
         description: 'release specify chart version or empty to release default'
         required: false
         default: ''
+
 env:
   GH_TOKEN: ${{ github.token }}
 
@@ -142,7 +145,7 @@ jobs:
           done
           echo "$ADDONS_DIR"
           echo "matrix={\"include\":[$ADDONS_DIR]}" >> $GITHUB_OUTPUT
-    
+
 
   release-addons-chart-jihu:
     needs: [ get-addons-chart-dir ]

diff --git a/.github/workflows/release-image.yml b/.github/workflows/release-image.yml
@@ -0,0 +1,30 @@
+name: Release Chart Image
+
+on:
+  workflow_dispatch:
+    inputs:
+      VERSION:
+        description: 'The version of release image'
+        required: false
+        default: ''
+  workflow_call:
+    inputs:
+      VERSION:
+        description: 'The version of release image'
+        type: string
+        required: false
+        default: ''
+
+run-name: release image ${{ inputs.VERSION }}
+
+jobs:
+  release-charts-image:
+    uses: apecloud/apecloud-cd/.github/workflows/[email protected]
+    with:
+      IMG: "apecloud/apecloud-charts"
+      VERSION: "${{ inputs.VERSION }}"
+      APECD_REF: "v0.1.55"
+      DOCKERFILE_PATH: "./docker/Dockerfile-charts"
+      REMOVE_PREFIX: false
+      GITHUB_REPO: "apecloud/apecloud-addons"
+    secrets: inherit
diff --git a/README.md b/README.md
@@ -49,3 +49,4 @@ KubeBlocks add-ons.
 | weaviate | 1.18.0                                                                                                                             | Weaviate is an open-source vector database. It allows you to store data objects and vector embeddings from your favorite ML-models, and scale seamlessly into billions of data objects.                                                                                                                                                        
 | xinference | 1.16.0<br>cpu-latest                                                                                                               | Xorbits Inference(Xinference) is a powerful and versatile library designed to serve language, speech recognition, and multimodal models.                                                                                                                                                                                                       
 | zookeeper | 3.7.1                                                                                                                              | Apache ZooKeeper is a centralized service for maintaining configuration information, naming, providing distributed synchronization, and providing group services.                                                                                                                                                                              
+| yashandb | personal-23.1.1.100                                                                                                                              | YashanDB is a new database system completely independently designed and developed by SICS. Based on classical database theories, it incorporates original Bounded Evaluation theory, Approximation theory, Parallel Scalability theory and Cross-Modal Fusion Computation theory, supports multiple deployment methods such as stand-alone/primary-standby, shared cluster, and distributed ones, covers OLTP/HTAP/OLAP transactions and analyzes mixed load scenarios.                                                                                                                                                                              
diff --git a/addons/apecloud-mysql-cluster/Chart.yaml b/addons/apecloud-mysql-cluster/Chart.yaml
@@ -3,7 +3,7 @@ annotations:
 apiVersion: v2
 name: apecloud-mysql-cluster
 type: application
-version: 0.7.0
+version: 0.8.0
 description: ApeCloud MySQL is a database that is compatible with MySQL syntax and achieves high availability
   through the utilization of the RAFT consensus protocol.
 dependencies:

diff --git a/addons/apecloud-mysql-cluster/templates/cluster.yaml b/addons/apecloud-mysql-cluster/templates/cluster.yaml
@@ -27,4 +27,4 @@ spec:
       {{- include "kblib.componentServices" . | indent 6 }}
     {{- if and (eq .Values.mode "raftGroup") .Values.proxyEnabled }}
     {{- include "apecloud-mysql-cluster.proxyComponents" . | indent 4 }}
-    {{- end }}
+    {{- end }}
diff --git a/addons/apecloud-mysql/Chart.yaml b/addons/apecloud-mysql/Chart.yaml
@@ -5,7 +5,7 @@ description: ApeCloud MySQL is a database that is compatible with MySQL syntax a
 
 type: application
 
-version: 0.8.0-beta.5
+version: 0.9.0-beta.9
 
 # This is the version number of the ApeCloud MySQL being deployed,
 # rather than the version number of ApeCloud MySQL-Scale itself.

diff --git a/addons/apecloud-mysql/config/mysql-scale-vtgate-config-constraint.cue b/addons/apecloud-mysql/config/mysql-scale-vtgate-config-constraint.cue
@@ -77,12 +77,27 @@
 	//Path to ssl key for mysql server plugin SSL
 	mysql_server_ssl_key: string
 
+	//Path to the ssl ca for mysql server plugin SSL
+	mysql_server_ssl_ca: string
+
 	//Path to the ssl cert for mysql server plugin SSL
 	mysql_server_ssl_cert: string
 
 	//Reject insecure connections but only if mysql_server_ssl_cert and mysql_server_ssl_key are provided.(default "false")
 	mysql_server_require_secure_transport: bool
 
+	// Set default strategy for DDL statements. Override with @@ddl_strategy session variable
+	ddl_strategy: string & "direct" | "online" | "mysql"
+
+    // Enable or disable the feature of showing information about the vttablet node which executing the SQL. (default false)
+    enable_display_sql_execution_vttablets: bool
+
+    // Enable or disable the feature of read write splitting for read only txn (default false)
+    enable_read_write_split_for_read_only_txn: bool
+
+    // Enable or disable the feature of interception for DML without where clause (default true)
+    enable_interception_for_dml_without_where: bool
+
 	...
 }
 

diff --git a/addons/apecloud-mysql/config/mysql-scale-vtgate-config-effect-scope.yaml b/addons/apecloud-mysql/config/mysql-scale-vtgate-config-effect-scope.yaml
@@ -17,15 +17,20 @@ staticParameters:
   - buffer_max_failover_duration
   - buffer_min_time_between_failovers
   - mysql_auth_server_impl
-  - mysql_server_require_secure_transport
   - mysql_auth_server_static_file
-  - mysql_server_ssl_key
-  - mysql_server_ssl_cert
   - enable_logs
   - enable_query_log
 
 dynamicParameters:
   - read_write_splitting_policy
   - read_write_splitting_ratio
   - read_after_write_consistency
-  - read_after_write_timeout
+  - read_after_write_timeout
+  - ddl_strategy
+  - enable_display_sql_execution_vttablets
+  - enable_read_write_split_for_read_only_txn
+  - mysql_server_ssl_key
+  - mysql_server_ssl_cert
+  - mysql_server_ssl_ca
+  - mysql_server_require_secure_transport
+  - enable_interception_for_dml_without_where
diff --git a/addons/apecloud-mysql/config/mysql-scale-vtgate-config.tpl b/addons/apecloud-mysql/config/mysql-scale-vtgate-config.tpl
@@ -14,11 +14,9 @@ buffer_size=10000
 buffer_window=30s
 buffer_max_failover_duration=60s
 buffer_min_time_between_failovers=60s
-mysql_auth_server_impl=none
-mysql_server_require_secure_transport=false
+mysql_auth_server_impl=mysqlbased
 mysql_auth_server_static_file=
-mysql_server_ssl_key=
-mysql_server_ssl_cert=
+mysql_server_require_secure_transport=false
 {{ block "logsBlock" . }}
 {{- if hasKey $.component "enabledLogs" }}
 enable_logs=true
@@ -27,3 +25,21 @@ enable_query_log=true
 {{- end }}
 {{- end }}
 {{ end }}
+ddl_strategy=direct
+enable_display_sql_execution_vttablets=false
+enable_read_write_split_for_read_only_txn=false
+enable_interception_for_dml_without_where=true
+
+{{- if $.component.tlsConfig }}
+{{- $ca_file := getCAFile }}
+{{- $cert_file := getCertFile }}
+{{- $key_file := getKeyFile }}
+mysql_server_ssl_ca={{ $ca_file }}
+mysql_server_ssl_cert={{ $cert_file }}
+mysql_server_ssl_key={{ $key_file }}
+# tls
+{{- else }}
+mysql_server_ssl_ca=
+mysql_server_ssl_cert=
+mysql_server_ssl_key=
+{{- end }}
diff --git a/addons/apecloud-mysql/config/mysql-scale-vttablet-config-constraint.cue b/addons/apecloud-mysql/config/mysql-scale-vttablet-config-constraint.cue
@@ -59,6 +59,30 @@
 	// query server transaction cap is the maximum number of transactions allowed to happen at any given point of a time for a single vttablet. E.g. by setting transaction cap to 100, there are at most 100 transactions will be processed by a vttablet and the 101th transaction will be blocked (and fail if it cannot get connection within specified timeout)
 	queryserver_config_transaction_cap: int & >=0
 
+	// the size of database connection pool in non transaction dml
+    non_transactional_dml_database_pool_size: int & >=1
+
+    // the number of rows to be processed in one batch by default
+    non_transactional_dml_default_batch_size: int & >=1
+
+    // the interval of batch processing in milliseconds by default
+    non_transactional_dml_default_batch_interval: int & >=1
+
+    // the interval of table GC in hours
+    non_transactional_dml_table_gc_interval: int & >=1
+
+    // the interval of job scheduler running in seconds
+    non_transactional_dml_job_manager_running_interval: int & >=1
+
+    // the interval of throttle check in milliseconds
+    non_transactional_dml_throttle_check_interval: int & >=1
+
+    // the threshold of batch size
+    non_transactional_dml_batch_size_threshold: int & >=1 & <=1000000
+
+    // final threshold = ratio * non_transactional_dml_batch_size_threshold / table index numbers
+    non_transactional_dml_batch_size_threshold_ratio: float & >=0 & <=1
+
 	...
 }
 

diff --git a/addons/apecloud-mysql/config/mysql-scale-vttablet-config-effect-scope.yaml b/addons/apecloud-mysql/config/mysql-scale-vttablet-config-effect-scope.yaml
@@ -20,4 +20,12 @@ staticParameters:
 dynamicParameters:
   - queryserver_config_pool_size
   - queryserver_config_stream_pool_size
-  - queryserver_config_transaction_cap
+  - queryserver_config_transaction_cap
+  - non_transactional_dml_database_pool_size
+  - non_transactional_dml_default_batch_size
+  - non_transactional_dml_default_batch_interval
+  - non_transactional_dml_table_gc_interval
+  - non_transactional_dml_job_manager_running_interval
+  - non_transactional_dml_throttle_check_interval
+  - non_transactional_dml_batch_size_threshold
+  - non_transactional_dml_batch_size_threshold_ratio
diff --git a/addons/apecloud-mysql/config/mysql-scale-vttablet-config.tpl b/addons/apecloud-mysql/config/mysql-scale-vttablet-config.tpl
@@ -6,16 +6,57 @@ db_connect_timeout_ms=500
 table_acl_config_mode=simple
 enable_logs=true
 enable_query_log=true
-table_acl_config=
-queryserver_config_strict_table_acl=false
-table_acl_config_reload_interval=30s
+table_acl_config=mysqlbased
+queryserver_config_strict_table_acl=true
+table_acl_config_reload_interval=5s
 enforce_tableacl_config=false
 
+{{- $phy_memory := getContainerMemory ( index $.podSpec.containers 0 ) }}
+{{- $thread_stack := 262144 }}
+{{- $binlog_cache_size := 32768 }}
+{{- $join_buffer_size := 262144 }}
+{{- $sort_buffer_size := 262144 }}
+{{- $read_buffer_size := 262144 }}
+{{- $read_rnd_buffer_size := 524288 }}
+{{- $single_thread_memory := add $thread_stack $binlog_cache_size $join_buffer_size $sort_buffer_size $read_buffer_size $read_rnd_buffer_size }}
+{{- if gt $phy_memory 0 }}
+# max_connections={{ div ( div $phy_memory 4 ) $single_thread_memory }}
+{{- end}}
+
+{{- $max_connections := div ( div $phy_memory 4 ) $single_thread_memory }}
+# 10 percentage
+{{- $pool_k := max 1 ( div (sub $max_connections 35) 10 ) }}
+
+# TxPool
+queryserver_config_transaction_cap={{ mul 5 $pool_k }}
+
 # OltpReadPool
-queryserver_config_pool_size=30
+queryserver_config_pool_size={{ mul 4 $pool_k }}
 
 # OlapReadPool
-queryserver_config_stream_pool_size=30
+queryserver_config_stream_pool_size={{ mul $pool_k }}
 
-# TxPool
-queryserver_config_transaction_cap=50
+
+# the size of database connection pool in non transaction dml
+non_transactional_dml_database_pool_size=3
+
+# the number of rows to be processed in one batch by default
+non_transactional_dml_default_batch_size=2000
+
+# the interval of batch processing in milliseconds by default
+non_transactional_dml_default_batch_interval=1
+
+# the interval of table GC in hours
+non_transactional_dml_table_gc_interval=24
+
+# the interval of job scheduler running in seconds
+non_transactional_dml_job_manager_running_interval=24
+
+# the interval of throttle check in milliseconds
+non_transactional_dml_throttle_check_interval=250
+
+# the threshold of batch size
+non_transactional_dml_batch_size_threshold=10000
+
+# final threshold = ratio * non_transactional_dml_batch_size_threshold / table index numbers
+non_transactional_dml_batch_size_threshold_ratio=0.5
diff --git a/addons/apecloud-mysql/dashboards/mysql-proxy.json b/addons/apecloud-mysql/dashboards/mysql-proxy.json
@@ -4214,6 +4214,7 @@
    "style": "dark",
    "tags": [
       "db",
+      "apps",
       "MySQL-Proxy"
    ],
    "templating": {
@@ -4374,7 +4375,7 @@
       ]
    },
    "timezone": "browser",
-   "title": "MySQL-Proxy",
+   "title": "APPS / MySQL-Proxy",
    "uid": "mysql-proxy-cluster-overview",
    "version": 0
 }
diff --git a/addons/apecloud-mysql/dashboards/mysql.json b/addons/apecloud-mysql/dashboards/mysql.json
@@ -4779,7 +4779,8 @@
   "style": "dark",
   "tags": [
     "mysql",
-    "db"
+    "db",
+    "apps"
   ],
   "templating": {
     "list": [
@@ -4968,8 +4969,8 @@
     "type": "timepicker"
   },
   "timezone": "",
-  "title": "MySQL",
-  "uid": "mysql",
+  "title": "APPS / MySQL",
+  "uid": "apecloud-mysql",
   "version": 1,
   "weekStart": ""
 }
diff --git a/addons/apecloud-mysql/scripts/vtgate.sh b/addons/apecloud-mysql/scripts/vtgate.sh
@@ -13,11 +13,6 @@ su vitess <<EOF
 exec vtgate \
   $TOPOLOGY_FLAGS \
   --alsologtostderr \
-  --gateway_initial_tablet_timeout $gateway_initial_tablet_timeout \
-  --healthcheck_timeout $healthcheck_timeout \
-  --srv_topo_timeout $srv_topo_timeout \
-  --grpc_keepalive_time $grpc_keepalive_time \
-  --grpc_keepalive_timeout $grpc_keepalive_timeout \
   $(if [ "$enable_logs" == "true" ]; then echo "--log_dir $VTDATAROOT"; fi) \
   $(if [ "$enable_query_log" == "true" ]; then echo "--log_queries_to_file $VTDATAROOT/vtgate_querylog.txt"; fi) \
   --port $web_port \
@@ -27,21 +22,6 @@ exec vtgate \
   --cell $cell \
   --cells_to_watch $cell \
   --tablet_types_to_wait PRIMARY,REPLICA \
-  --tablet_refresh_interval $tablet_refresh_interval \
   --service_map 'grpc-vtgateservice' \
-  --pid_file $VTDATAROOT/vtgate.pid \
-  --read_write_splitting_policy $read_write_splitting_policy  \
-  --read_write_splitting_ratio $read_write_splitting_ratio  \
-  --read_after_write_consistency $read_after_write_consistency \
-  --read_after_write_timeout $read_after_write_timeout \
-  --enable_buffer=$enable_buffer \
-  --buffer_size $buffer_size \
-  --buffer_window $buffer_window \
-  --buffer_max_failover_duration $buffer_max_failover_duration \
-  --buffer_min_time_between_failovers $buffer_min_time_between_failovers \
-  $(if [ "$mysql_server_require_secure_transport" == "true" ]; then echo "--mysql_server_require_secure_transport"; fi) \
-  $(if [ -n "$mysql_server_ssl_cert" ]; then echo "--mysql_server_ssl_cert $mysql_server_ssl_cert"; fi) \
-  $(if [ -n "$mysql_server_ssl_key" ]; then echo "--mysql_server_ssl_key $mysql_server_ssl_key"; fi) \
-  $(if [ -n "$mysql_auth_server_static_file" ]; then echo "--mysql_auth_server_static_file $mysql_auth_server_static_file"; fi) \
---mysql_auth_server_impl $mysql_auth_server_impl
+  --pid_file $VTDATAROOT/vtgate.pid
 EOF