Merge branch 'acryldata:kafka-message-size-options' into kafka-messag…

…e-size-options
acryldata · Oct 30, 2023 · a1a6899 · a1a6899
2 parents 9e9579f + 69c2337
commit a1a6899
Show file tree

Hide file tree

Showing 3 changed files with 22 additions and 7 deletions.
diff --git a/charts/datahub/templates/datahub-upgrade/_upgrade.tpl b/charts/datahub/templates/datahub-upgrade/_upgrade.tpl
@@ -41,6 +41,18 @@ Return the env variables for upgrade jobs
   value: "{{ .Values.global.sql.datasource.driver }}"
 - name: KAFKA_BOOTSTRAP_SERVER
   value: "{{ .Values.global.kafka.bootstrap.server }}"
+{{- with .Values.global.kafka.producer.compressionType }}
+- name: KAFKA_PRODUCER_COMPRESSION_TYPE
+  value: "{{ . }}"
+{{- end }}
+{{- with .Values.global.kafka.producer.maxRequestSize }}
+- name: KAFKA_PRODUCER_MAX_REQUEST_SIZE
+  value: "{{ . }}"
+{{- end }}
+{{- with .Values.global.kafka.consumer.maxPartitionFetchBytes }}
+- name: KAFKA_CONSUMER_MAX_PARTITION_FETCH_BYTES
+  value: "{{ . }}"
+{{- end }}
 {{- if eq .Values.global.kafka.schemaregistry.type "INTERNAL" }}
 - name: KAFKA_SCHEMAREGISTRY_URL
   value: {{ printf "http://%s-%s:%s/schema-registry/api/" .Release.Name "datahub-gms" .Values.global.datahub.gms.port }}

diff --git a/charts/datahub/templates/kafka-setup-job.yml b/charts/datahub/templates/kafka-setup-job.yml
@@ -62,6 +62,10 @@ spec:
               value: {{ .Values.global.kafka.zookeeper.server | quote }}
             - name: KAFKA_BOOTSTRAP_SERVER
               value: {{ .Values.global.kafka.bootstrap.server | quote }}
+            {{- with .Values.global.kafka.maxMessageBytes }}
+            - name: MAX_MESSAGE_BYTES
+              value: "{{ . }}"
+            {{- end }}
             {{- if eq .Values.global.kafka.schemaregistry.type "INTERNAL" }}
             - name: USE_CONFLUENT_SCHEMA_REGISTRY
               value: "false"

diff --git a/charts/datahub/values.yaml b/charts/datahub/values.yaml
@@ -499,6 +499,12 @@ global:
       metadata_change_log_timeseries_topic_name: "MetadataChangeLog_Timeseries_v1"
       platform_event_topic_name: "PlatformEvent_v1"
       datahub_upgrade_history_topic_name: "DataHubUpgradeHistory_v1"
+    maxMessageBytes: 5242880  # 5MB
+    producer:
+      compressionType: snappy
+      maxRequestSize: 5242880  # 5MB
+    consumer:
+      maxPartitionFetchBytes: 5242880  # 5MB
     ## For AWS MSK set this to a number larger than 1
     # partitions: 3
     # replicationFactor: 3
@@ -515,13 +521,6 @@ global:
       #   region: us-east-1
       #   registry: datahub
 
-    ## Kafka producer and consumer settings
-    producer:
-      compressionType: snappy
-      maxRequestSize: 5242880
-    consumer:
-      maxPartitionFetchBytes: 5242880
-
   neo4j:
     host: "prerequisites-neo4j:7474"
     uri: "bolt://prerequisites-neo4j"