Merge pull request #347 from everpcpc/feat-databend

Upgrade databend to v1.2.279

src/stable/databend-meta/Chart.yaml

@@ -26,13 +26,13 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.6.1
+version: 0.7.3
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
 # It is recommended to use it with quotes.
-appVersion: "v1.0.3-nightly"
+appVersion: "v1.2.279"
 
 dependencies:
 - name: common

src/stable/databend-meta/README.md

@@ -15,7 +15,7 @@ helm install my-release databend/databend-meta --namespace databend --create-nam
 
 Note that for a production cluster, you will likely want to override the following parameters in [values.yaml](values.yaml) with your own values.
 
-- `resources.requests.memory` and `resources.limit.memory` allocate memory resource to query pods in your cluser.
+- `resources.requests.memory` and `resources.limit.memory` allocate memory resource to query pods in your cluster.
 - `replicaCount` defaults to `1`, We strongly recommend that you set to `3` for HA.
 - `persistence.size` defaults to `10Gi` of disk space per pod, which you may increase or decrease for your use case.
 - `persistence.storageClass` uses the default storage class for your environment.

src/stable/databend-meta/templates/poddisruptionbudget.yaml

@@ -0,0 +1,11 @@
+{{- if gt (int .Values.replicaCount) 1 }}
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+  name: {{ include "databend-meta.fullname" . }}
+spec:
+  maxUnavailable: 1
+  selector:
+    matchLabels:
+      {{- include "databend-meta.selectorLabels" . | nindent 6 }}
+{{- end }}

src/stable/databend-meta/templates/statefulset.yaml

@@ -10,11 +10,7 @@ metadata:
 spec:
   serviceName: {{ $fullName }}
   replicas: {{ .Values.replicaCount }}
-  {{- if $bootstrap }}
-  podManagementPolicy: OrderedReady
-  {{- else }}
   podManagementPolicy: Parallel
-  {{- end }}
   selector:
     matchLabels:
       {{- include "databend-meta.selectorLabels" . | nindent 6 }}
@@ -147,6 +143,9 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: status.podIP
+            {{- with .Values.envs }}
+              {{- toYaml . | nindent 12 }}
+            {{- end }}
           volumeMounts:
             {{- if .Values.persistence.enabled }}
             - name: data

src/stable/databend-meta/values.yaml

@@ -44,6 +44,8 @@ config:
   raft:
     dir: /data/databend-meta/raft
 
+envs: []
+
 persistence:
   enabled: true
   storageClass: ""

src/stable/databend-query/Chart.yaml

@@ -25,19 +25,15 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.5.1
+version: 0.8.3
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
 # It is recommended to use it with quotes.
-appVersion: "v1.0.3-nightly"
+appVersion: "v1.2.279"
 
 dependencies:
-- name: minio
-  version: 3.6.3
-  repository: https://charts.min.io/
-  condition: minio.enabled
 - name: common
   version: 1.x.x
   repository: https://charts.bitnami.com/bitnami

src/stable/databend-query/README.md

@@ -15,7 +15,7 @@ helm install my-release databend/databend-query --namespace databend --create-na
 
 Note that for a production cluster, you will likely want to override the following parameters in [values.yaml](values.yaml) with your own values.
 
-- `resources.requests.memory` and `resources.limit.memory` allocate memory resource to query pods in your cluser.
+- `resources.requests.memory` and `resources.limit.memory` allocate memory resource to query pods in your cluster.
 - `config.meta.address` indicates the grpc address of a [Databend Meta](../databend-meta) service.
 - `config.storage.type` defaults to `fs` for testing only, `s3` is recommended in production.
 - `config.storage.s3.accessKeyId` and `config.storage.s3.secretAccessKey` should be set when using `s3` storage, `config.storage.s3.endpointUrl` defaults to `https://s3.amazonaws.com`.

src/stable/databend-query/templates/configmap.yaml

@@ -1,3 +1,4 @@
+{{- $storageType := .Values.config.storage.type }}
 apiVersion: v1
 kind: ConfigMap
 metadata:
@@ -17,6 +18,17 @@ data:
       management_mode = {{ .Values.config.query.managementMode }}
       jwt_key_file = {{ .Values.config.query.jwtKeyFile | quote }}
 
+      max_server_memory_usage = {{ .Values.config.query.maxServerMemoryUsage | default 0 }}
+      max_memory_limit_enabled = {{ .Values.config.query.maxMemoryLimitEnabled | default false }}
+
+      {{- range $field, $value := .Values.config.query.extra }}
+      {{- if (kindIs "string" $value) }}
+      {{ $field }} = {{ $value | quote }}
+      {{- else }}
+      {{ $field }} = {{ $value }}
+      {{- end }}
+      {{- end }}
+
       {{- range .Values.config.query.users }}
       [[query.users]]
         name = {{ .name | quote }}
@@ -28,19 +40,11 @@ data:
         {{- end }}
       {{- end }}
 
-      {{- range $field, $value := .Values.config.query.extra }}
-      {{- if (kindIs "string" $value) }}
-      {{ $field }} = {{ $value | quote }}
-      {{- else }}
-      {{ $field }} = {{ $value }}
-      {{- end }}
-      {{- end }}
-
     [log]
       [log.file]
         on = {{ .Values.config.log.file.enabled }}
         level = {{ .Values.config.log.file.level | default "INFO" | quote }}
-        dir = "/var/log/databend"
+        dir = {{ .Values.config.log.file.dir | default "/var/log/databend" | quote }}
       [log.stderr]
         on = {{ .Values.config.log.stderr.enabled }}
         level = {{ .Values.config.log.stderr.level | default "WARN" | quote }}
@@ -52,48 +56,37 @@ data:
       client_timeout_in_second = {{ .Values.config.meta.clientTimeoutInSecond | default 60 }}
 
     [storage]
-      storage_type = {{ .Values.config.storage.type | quote }}
+      type = {{ $storageType | quote }}
+      allow_insecure = {{ .Values.config.storage.allow_insecure | default false }}
 
-      {{- if eq .Values.config.storage.type "fs" }}
-      [storage.fs]
-        {{- range $field, $value := .Values.config.storage.fs }}
-        {{- if (kindIs "string" $value) }}
-        {{ $field }} = {{ $value | quote }}
-        {{- else }}
-        {{ $field }} = {{ $value }}
-        {{- end }}
-        {{- end }}
-      {{- else if eq .Values.config.storage.type "s3"}}
+      {{- if eq $storageType "s3"}}
       [storage.s3]
-        {{- if .Values.minio.enabled }}
-        bucket = {{ .Values.config.storage.s3.bucket | quote }}
-        endpoint_url = "http://{{ printf "%s-%s.%s" .Release.Name "minio" .Release.Namespace }}:{{ .Values.minio.minioAPIPort }}"
-        access_key_id = {{ .Values.minio.rootUser | quote }}
-        secret_access_key = {{ .Values.minio.rootPassword | quote }}
-        {{- else }}
         {{- range $field, $value := .Values.config.storage.s3 }}
         {{- if (kindIs "string" $value) }}
         {{ $field }} = {{ $value | quote }}
         {{- else }}
         {{ $field }} = {{ $value }}
         {{- end }}
         {{- end }}
-        {{- end }}
-      {{- else if eq .Values.config.storage.type "gcs"}}
-        {{- range $field, $value := .Values.config.storage.gcs }}
+      {{- else }}
+      [storage.{{ $storageType }}]
+        {{- range $name, $configs := .Values.config.storage }}
+        {{- if eq $name $storageType }}
+        {{- range $field, $value := $configs }}
         {{- if (kindIs "string" $value) }}
         {{ $field }} = {{ $value | quote }}
         {{- else }}
         {{ $field }} = {{ $value }}
         {{- end }}
         {{- end }}
-      {{- else if eq .Values.config.storage.type "oss"}}
-      [storage.oss]
-        {{- range $field, $value := .Values.config.storage.oss }}
-        {{- if (kindIs "string" $value) }}
-        {{ $field }} = {{ $value | quote }}
-        {{- else }}
-        {{ $field }} = {{ $value }}
         {{- end }}
         {{- end }}
       {{- end }}
+
+    {{- if .Values.cache.enabled }}
+    [cache]
+      data_cache_storage = "disk"
+      [cache.disk]
+        path = {{ .Values.cache.path | default "/var/lib/databend/cache" | quote }}
+        max_bytes = {{ .Values.cache.maxBytes | default 21474836480 | int64 }}
+    {{- end }}

src/stable/databend-query/templates/service.yaml

@@ -19,3 +19,4 @@ spec:
     {{- end }}
   selector:
     {{- include "databend-query.selectorLabels" . | nindent 4 }}
+    statefulset.kubernetes.io/pod-name: {{ include "databend-query.fullname" . }}-0

src/stable/databend-query/templates/deployment.yaml → src/stable/databend-query/templates/statefulset.yaml

@@ -1,11 +1,13 @@
 apiVersion: apps/v1
-kind: Deployment
+kind: StatefulSet
 metadata:
   name: {{ include "databend-query.fullname" . }}
   labels:
     {{- include "databend-query.labels" . | nindent 4 }}
 spec:
+  serviceName: {{ include "databend-query.fullname" . }}
   replicas: {{ .Values.replicaCount }}
+  podManagementPolicy: Parallel
   selector:
     matchLabels:
       {{- include "databend-query.selectorLabels" . | nindent 6 }}
@@ -41,18 +43,10 @@ spec:
           image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
           imagePullPolicy: {{ .Values.image.pullPolicy }}
           ports:
-            - name: metric
-              containerPort: 7070
-            - name: admin
-              containerPort: 8080
-            - name: flight
-              containerPort: 9090
-            - name: mysql
-              containerPort: 3307
-            - name: clickhouse
-              containerPort: 9000
-            - name: http
-              containerPort: 8000
+            {{- range $key, $val := .Values.service.ports }}
+            - name: {{ $key }}
+              containerPort: {{ $val}}
+            {{- end }}
           livenessProbe:
             httpGet:
               path: /v1/health
@@ -80,30 +74,37 @@ spec:
                 fieldRef:
                   fieldPath: status.podIP
             - name: QUERY_METRIC_API_ADDRESS
-              value: "$(POD_IP):7070"
+              value: "$(POD_IP):{{ .Values.service.ports.metric | default 7070 }}"
             - name: QUERY_ADMIN_API_ADDRESS
-              value: "$(POD_IP):8080"
+              value: "$(POD_IP):{{ .Values.service.ports.admin | default 8080 }}"
             - name: QUERY_FLIGHT_API_ADDRESS
-              value: "$(POD_IP):9090"
+              value: "$(POD_IP):{{ .Values.service.ports.flight | default 9090 }}"
+            - name: QUERY_HTTP_HANDLER_HOST
+              value: 0.0.0.0
+            - name: QUERY_HTTP_HANDLER_PORT
+              value: {{ .Values.service.ports.http | default 8000 | quote }}
+            - name: QUERY_FLIGHT_SQL_HANDLER_HOST
+              value: 0.0.0.0
+            - name: QUERY_FLIGHT_SQL_HANDLER_PORT
+              value: {{ .Values.service.ports.flightsql | default 8900 | quote }}
             - name: QUERY_MYSQL_HANDLER_HOST
               value: 0.0.0.0
             - name: QUERY_MYSQL_HANDLER_PORT
-              value: "3307"
-            - name: QUERY_CLICKHOUSE_HANDLER_HOST
+              value: {{ .Values.service.ports.mysql | default 3307 | quote }}
+            - name: QUERY_CLICKHOUSE_HTTP_HANDLER_HOST
               value: 0.0.0.0
-            - name: QUERY_CLICKHOUSE_HANDLER_PORT
-              value: "9000"
-            - name: QUERY_HTTP_HANDLER_HOST
-              value: 0.0.0.0
-            - name: QUERY_HTTP_HANDLER_PORT
-              value: "8000"
+            - name: QUERY_CLICKHOUSE_HTTP_HANDLER_PORT
+              value: {{ .Values.service.ports.ckhttp |default 8124 | quote }}
+            {{- with .Values.envs }}
+              {{- toYaml . | nindent 12 }}
+            {{- end }}
           volumeMounts:
             - name: config
               # Note: subPath volume mount will not receive ConfigMap update.
               mountPath: /etc/databend-query
-            {{- if .Values.persistence.enabled }}
-            - name: data
-              mountPath: {{ .Values.persistence.mountPath }}
+            {{- if .Values.cache.enabled }}
+            - name: cache
+              mountPath: {{ .Values.cache.path | default "/var/lib/databend/cache" | quote }}
             {{- end }}
         {{- if .Values.sidecars }}
         {{- include "common.tplvalues.render" (dict "value" .Values.sidecars "context" $) | nindent 8 }}
@@ -112,15 +113,6 @@ spec:
         - name: config
           configMap:
             name: {{ include "databend-query.fullname" .}}
-        {{- if .Values.persistence.enabled }}
-        - name: data
-          persistentVolumeClaim:
-          {{- if .Values.persistence.existingClaim }}
-            claimName: {{ .Values.persistence.existingClaim }}
-          {{- else }}
-            claimName: {{ include "databend-query.fullname" . }}
-          {{- end }}
-        {{- end }}
       {{- with .Values.nodeSelector }}
       nodeSelector:
         {{- toYaml . | nindent 8 }}
@@ -133,3 +125,14 @@ spec:
       tolerations:
         {{- toYaml . | nindent 8 }}
       {{- end }}
+  {{- if .Values.cache.enabled }}
+  volumeClaimTemplates:
+  - metadata:
+      name: cache
+    spec:
+      accessModes: [ "ReadWriteOnce" ]
+      storageClassName: {{ .Values.cache.storageClass | quote }}
+      resources:
+        requests:
+          storage: {{ .Values.cache.maxBytes | quote }}
+  {{- end }}