acryldata · david-leifker · Nov 28, 2024 · Nov 27, 2024
diff --git a/charts/datahub/Chart.yaml b/charts/datahub/Chart.yaml
@@ -4,13 +4,13 @@ description: A Helm chart for DataHub
 type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
-version: 0.4.36
+version: 0.4.37
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application.
 appVersion: 0.14.1
 dependencies:
   - name: datahub-gms
-    version: 0.2.175
+    version: 0.2.176
     repository: file://./subcharts/datahub-gms
     condition: datahub-gms.enabled
   - name: datahub-frontend
@@ -22,7 +22,7 @@ dependencies:
     repository: file://./subcharts/datahub-mae-consumer
     condition: global.datahub_standalone_consumers_enabled
   - name: datahub-mce-consumer
-    version: 0.2.166
+    version: 0.2.167
     repository: file://./subcharts/datahub-mce-consumer
     condition: global.datahub_standalone_consumers_enabled
   - name: datahub-ingestion-cron

diff --git a/charts/datahub/subcharts/datahub-gms/Chart.yaml b/charts/datahub/subcharts/datahub-gms/Chart.yaml
@@ -12,7 +12,7 @@ description: A Helm chart for DataHub's datahub-gms component
 type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
-version: 0.2.175
+version: 0.2.176
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application.
 appVersion: v0.14.1
diff --git a/charts/datahub/subcharts/datahub-gms/templates/deployment.yaml b/charts/datahub/subcharts/datahub-gms/templates/deployment.yaml
@@ -506,6 +506,12 @@ spec:
             - name: FORMS_HOOK_CONSUMER_GROUP_SUFFIX
               value: "{{ .consumerGroupSuffix }}"
             {{- end }}
+            {{- with .Values.global.datahub.strictUrnValidation }}
+            - name: STRICT_URN_VALIDATION_ENABLED
+              value: "{{ . }}"
+            {{- end }}
+            - name: MCP_CONSUMER_BATCH_ENABLED
+              value: {{ .Values.global.datahub.metadataChangeProposal.consumer.batch.enabled | quote }}
           {{- with .Values.extraEnvs }}
             {{- toYaml . | nindent 12 }}
           {{- end }}

diff --git a/charts/datahub/subcharts/datahub-gms/values.yaml b/charts/datahub/subcharts/datahub-gms/values.yaml
@@ -256,6 +256,9 @@ global:
     ## Enables diff mode for graph writes, uses a different code path that produces a diff from previous to next to write relationships instead of wholesale deleting edges and reading
     enableGraphDiffMode: true
 
+    ## Enable stricter URN validation logic
+    strictUrnValidation: false
+
     cache:
       search:
         ## Enable general search caching
@@ -275,6 +278,11 @@ global:
           ttlSeconds: 86400
           lightningThreshold: 300
 
+    metadataChangeProposal:
+      consumer:
+        batch:
+          enabled: false
+
   hostAliases:
     - ip: "192.168.0.104"
       hostnames:

diff --git a/charts/datahub/subcharts/datahub-mce-consumer/Chart.yaml b/charts/datahub/subcharts/datahub-mce-consumer/Chart.yaml
@@ -12,7 +12,7 @@ description: A Helm chart for Kubernetes
 type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
-version: 0.2.166
+version: 0.2.167
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application.
 appVersion: v0.14.1
diff --git a/charts/datahub/subcharts/datahub-mce-consumer/templates/deployment.yaml b/charts/datahub/subcharts/datahub-mce-consumer/templates/deployment.yaml
@@ -339,6 +339,12 @@ spec:
             {{- end }}
             {{- end }}
             {{- end }}
+            {{- with .Values.global.datahub.strictUrnValidation }}
+            - name: STRICT_URN_VALIDATION_ENABLED
+              value: "{{ . }}"
+            {{- end }}
+            - name: MCP_CONSUMER_BATCH_ENABLED
+              value: {{ .Values.global.datahub.metadataChangeProposal.consumer.batch.enabled | quote }}
           {{- with .Values.extraEnvs }}
             {{- toYaml . | nindent 12 }}
           {{- end }}

diff --git a/charts/datahub/subcharts/datahub-mce-consumer/values.yaml b/charts/datahub/subcharts/datahub-mce-consumer/values.yaml
@@ -184,6 +184,9 @@ global:
     ## Enables diff mode for graph writes, uses a different code path that produces a diff from previous to next to write relationships instead of wholesale deleting edges and reading
     enableGraphDiffMode: true
 
+    ## Enable stricter URN validation logic
+    strictUrnValidation: false
+
   hostAliases:
     - ip: "192.168.0.104"
       hostnames:

diff --git a/charts/datahub/templates/datahub-upgrade/datahub-system-update-job.yml b/charts/datahub/templates/datahub-upgrade/datahub-system-update-job.yml
@@ -151,7 +151,7 @@ spec:
               value: {{ $result | toJson | quote }}
             {{- with $v.revision_env }}
             - name: {{ . }}
-              value: '{"version":"{{ $.Chart.Version }}-{{ $.Release.Revision }}"}'
+              value: '{"version":"{{ $.Values.global.datahub.version }}-{{ $result | toJson | sha256sum | trunc 7 }}"}'
             {{- end }}
             {{- end }}
             {{- end }}
@@ -334,7 +334,7 @@ spec:
               value: {{ $result | toJson | quote }}
             {{- with $v.revision_env }}
             - name: {{ . }}
-              value: '{"version":"{{ $.Chart.Version }}-{{ $.Release.Revision }}"}'
+              value: '{"version":"{{ $.Values.global.datahub.version }}-{{ $result | toJson | sha256sum | trunc 7 }}"}'
             {{- end }}
             {{- end }}
             {{- end }}

diff --git a/charts/datahub/values.yaml b/charts/datahub/values.yaml
@@ -415,15 +415,16 @@ datahubSystemUpdate:
       values:
         ingestion:
           name: datahub-gc
-          # override global.datahub.managed_ingestion.defaultCliVersion
-          version: "0.14.1.7rc2"
+          # # override global.datahub.managed_ingestion.defaultCliVersion
+          # version: "0.14.1.7rc2"
         # schedule:
           # # override global.datahub.timezone
           # timezone: "UTC"
           # # override dailyCronWindow
           # interval: "0 1 * * *"
         cleanup_expired_tokens: "false"
         truncate_indices: "true"
+        truncate_indices_retention_days: 30
         dataprocess_cleanup:
           retention_days: 30
           delete_empty_data_jobs: "true"
@@ -819,6 +820,9 @@ global:
     ## Enables diff mode for graph writes, uses a different code path that produces a diff from previous to next to write relationships instead of wholesale deleting edges and reading
     enableGraphDiffMode: true
 
+    ## Enable stricter URN validation logic
+    strictUrnValidation: false
+
     ## Values specific to the unified search and browse feature.
     search_and_browse:
       show_search_v2: true  # If on, show the new search filters experience as of v0.10.5