From 127a24f97b6a96381897df4b27234a8fb7531afc Mon Sep 17 00:00:00 2001 From: dspeck1 Date: Wed, 2 Oct 2024 10:21:06 -0500 Subject: [PATCH 01/28] Add authorization for prompt processing user to the test.next-visit-job topic. --- .../sasquatch/charts/strimzi-kafka/templates/users.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/applications/sasquatch/charts/strimzi-kafka/templates/users.yaml b/applications/sasquatch/charts/strimzi-kafka/templates/users.yaml index 75b9433255..8e73565705 100644 --- a/applications/sasquatch/charts/strimzi-kafka/templates/users.yaml +++ b/applications/sasquatch/charts/strimzi-kafka/templates/users.yaml @@ -267,6 +267,13 @@ spec: type: allow host: "*" operation: Read + - resource: + type: topic + name: "test.next-visit-job" + patternType: literal + type: allow + host: "*" + operation: Read {{- end }} {{- if .Values.users.consdb.enabled }} --- From 00da76d22f2537726cdde00e2bdd95d0428db486 Mon Sep 17 00:00:00 2001 From: dspeck1 Date: Tue, 15 Oct 2024 11:04:15 -0500 Subject: [PATCH 02/28] Initial commit of prompt processing kafka templates --- .../prompt-processing-kafka/Chart.yaml | 15 ++ .../charts/strimzi-kafka/Chart.yaml | 5 + .../charts/strimzi-kafka/README.md | 70 +++++ .../strimzi-kafka/templates/certificates.yaml | 19 ++ .../templates/kafka-metrics-configmap.yaml | 156 ++++++++++++ .../charts/strimzi-kafka/templates/kafka.yaml | 163 ++++++++++++ .../templates/schema-registry-ingress.yaml | 25 ++ .../templates/schema-registry.yaml | 73 ++++++ .../strimzi-kafka/templates/superusers.yaml | 27 ++ .../charts/strimzi-kafka/templates/users.yaml | 91 +++++++ .../charts/strimzi-kafka/values.yaml | 239 ++++++++++++++++++ .../prompt-processing-kafka/topic.yaml | 12 + .../values-usdfdev-prompt-processing.yaml | 8 + .../prompt-processing-kafka/values.yaml | 35 +++ .../prompt-processing-kafka/index.rst | 18 ++ .../prompt-processing-kafka/values.md | 12 + environments/README.md | 1 + .../prompt/prompt-processing-kafka.yaml | 34 +++ .../values-usdfdev-prompt-processing.yaml | 1 + environments/values.yaml | 3 + 20 files changed, 1007 insertions(+) create mode 100644 applications/prompt-processing-kafka/Chart.yaml create mode 100644 applications/prompt-processing-kafka/charts/strimzi-kafka/Chart.yaml create mode 100644 applications/prompt-processing-kafka/charts/strimzi-kafka/README.md create mode 100644 applications/prompt-processing-kafka/charts/strimzi-kafka/templates/certificates.yaml create mode 100644 applications/prompt-processing-kafka/charts/strimzi-kafka/templates/kafka-metrics-configmap.yaml create mode 100644 applications/prompt-processing-kafka/charts/strimzi-kafka/templates/kafka.yaml create mode 100644 applications/prompt-processing-kafka/charts/strimzi-kafka/templates/schema-registry-ingress.yaml create mode 100644 applications/prompt-processing-kafka/charts/strimzi-kafka/templates/schema-registry.yaml create mode 100644 applications/prompt-processing-kafka/charts/strimzi-kafka/templates/superusers.yaml create mode 100644 applications/prompt-processing-kafka/charts/strimzi-kafka/templates/users.yaml create mode 100644 applications/prompt-processing-kafka/charts/strimzi-kafka/values.yaml create mode 100644 applications/prompt-processing-kafka/topic.yaml create mode 100644 applications/prompt-processing-kafka/values-usdfdev-prompt-processing.yaml create mode 100644 applications/prompt-processing-kafka/values.yaml create mode 100644 docs/applications/prompt-processing-kafka/index.rst create mode 100644 docs/applications/prompt-processing-kafka/values.md create mode 100644 environments/templates/applications/prompt/prompt-processing-kafka.yaml diff --git a/applications/prompt-processing-kafka/Chart.yaml b/applications/prompt-processing-kafka/Chart.yaml new file mode 100644 index 0000000000..6d794a6094 --- /dev/null +++ b/applications/prompt-processing-kafka/Chart.yaml @@ -0,0 +1,15 @@ +apiVersion: v2 +name: prompt-processing-kafka +version: 1.0.0 +description: "Kafka environment for prompt processing fan out events and bucket notifications" +sources: + - "https://github.com/lsst-dm/prompt_processing" + - "https://github.com/lsst-dm/next_visit_fan_out" +appVersion: 0.1.0 + +dependencies: + - name: strimzi-kafka + version: 1.0.0 + - name: strimzi-registry-operator + version: 2.1.0 + repository: https://lsst-sqre.github.io/charts/ diff --git a/applications/prompt-processing-kafka/charts/strimzi-kafka/Chart.yaml b/applications/prompt-processing-kafka/charts/strimzi-kafka/Chart.yaml new file mode 100644 index 0000000000..e2a92505ab --- /dev/null +++ b/applications/prompt-processing-kafka/charts/strimzi-kafka/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v2 +name: strimzi-kafka +version: 1.0.0 +description: A subchart to deploy Strimzi Kafka components for Prompt Processing. +appVersion: 0.42.0 diff --git a/applications/prompt-processing-kafka/charts/strimzi-kafka/README.md b/applications/prompt-processing-kafka/charts/strimzi-kafka/README.md new file mode 100644 index 0000000000..fe66fffbee --- /dev/null +++ b/applications/prompt-processing-kafka/charts/strimzi-kafka/README.md @@ -0,0 +1,70 @@ +# strimzi-kafka + +A subchart to deploy Strimzi Kafka components for Prompt Processing. + +## Values + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| cluster.monitorLabel | object | `{}` | Site wide label required for gathering Prometheus metrics if they are enabled | +| cluster.name | string | `"prompt-processing-kafka"` | Name used for the Kafka cluster, and used by Strimzi for many annotations | +| connect.config."key.converter" | string | `"io.confluent.connect.avro.AvroConverter"` | Set the converter for the message ke | +| connect.config."key.converter.schema.registry.url" | string | `"http://sasquatch-schema-registry.sasquatch:8081"` | URL for the schema registry | +| connect.config."key.converter.schemas.enable" | bool | `true` | Enable converted schemas for the message key | +| connect.config."value.converter" | string | `"io.confluent.connect.avro.AvroConverter"` | Converter for the message value | +| connect.config."value.converter.schema.registry.url" | string | `"http://sasquatch-schema-registry.sasquatch:8081"` | URL for the schema registry | +| connect.config."value.converter.schemas.enable" | bool | `true` | Enable converted schemas for the message value | +| connect.enabled | bool | `false` | Enable Kafka Connect | +| connect.image | string | `"ghcr.io/lsst-sqre/strimzi-0.40.0-kafka-3.7.0:tickets-DM-43491"` | Custom strimzi-kafka image with connector plugins used by sasquatch | +| connect.replicas | int | `3` | Number of Kafka Connect replicas to run | +| cruiseControl.enabled | bool | `false` | | +| kafka.affinity | object | See `values.yaml` | Affinity for Kafka pod assignment | +| kafka.config."log.retention.bytes" | string | `"350000000000"` | How much disk space Kafka will ensure is available, set to 70% of the data partition size | +| kafka.config."log.retention.hours" | int | `48` | Number of days for a topic's data to be retained | +| kafka.config."message.max.bytes" | int | `10485760` | The largest record batch size allowed by Kafka | +| kafka.config."offsets.retention.minutes" | int | `2880` | Number of minutes for a consumer group's offsets to be retained | +| kafka.config."replica.fetch.max.bytes" | int | `10485760` | The number of bytes of messages to attempt to fetch for each partition | +| kafka.externalListener.bootstrap.annotations | object | `{}` | Annotations that will be added to the Ingress, Route, or Service resource | +| kafka.externalListener.bootstrap.host | string | Do not configure TLS | Name used for TLS hostname verification | +| kafka.externalListener.bootstrap.loadBalancerIP | string | Do not request a load balancer IP | Request this load balancer IP. See `values.yaml` for more discussion | +| kafka.externalListener.brokers | list | `[]` | Brokers configuration. _host_ is used in the brokers' advertised.brokers configuration and for TLS hostname verification. The format is a list of maps. | +| kafka.externalListener.tls.certIssuerName | string | `"letsencrypt-dns"` | Name of a ClusterIssuer capable of provisioning a TLS certificate for the broker | +| kafka.externalListener.tls.enabled | bool | `false` | Whether TLS encryption is enabled | +| kafka.listeners.external.enabled | bool | `false` | Whether external listener is enabled | +| kafka.listeners.noauth.enabled | bool | `false` | Whether internal noauth listener is enabled | +| kafka.listeners.plain.enabled | bool | `false` | Whether internal plaintext listener is enabled | +| kafka.listeners.tls.enabled | bool | `false` | Whether internal TLS listener is enabled | +| kafka.metricsConfig.enabled | bool | `false` | Whether metric configuration is enabled | +| kafka.minInsyncReplicas | int | `2` | The minimum number of in-sync replicas that must be available for the producer to successfully send records Cannot be greater than the number of replicas. | +| kafka.replicas | int | `3` | Number of Kafka broker replicas to run | +| kafka.resources | object | See `values.yaml` | Kubernetes requests and limits for the Kafka brokers | +| kafka.storage.size | string | `"500Gi"` | Size of the backing storage disk for each of the Kafka brokers | +| kafka.storage.storageClassName | string | `""` | Name of a StorageClass to use when requesting persistent volumes | +| kafka.tolerations | list | `[]` | Tolerations for Kafka broker pod assignment | +| kafka.version | string | `"3.8.0"` | Version of Kafka to deploy | +| kafkaController.enabled | bool | `false` | Enable Kafka Controller | +| kafkaController.resources | object | See `values.yaml` | Kubernetes requests and limits for the Kafka Controller | +| kafkaController.storage.size | string | `"20Gi"` | Size of the backing storage disk for each of the Kafka controllers | +| kafkaController.storage.storageClassName | string | `""` | Name of a StorageClass to use when requesting persistent volumes | +| kafkaExporter.enableSaramaLogging | bool | `false` | Enable Sarama logging for pod | +| kafkaExporter.enabled | bool | `false` | Enable Kafka exporter | +| kafkaExporter.groupRegex | string | `".*"` | Consumer groups to monitor | +| kafkaExporter.logging | string | `"info"` | Logging level | +| kafkaExporter.resources | object | See `values.yaml` | Kubernetes requests and limits for the Kafka exporter | +| kafkaExporter.topicRegex | string | `".*"` | Kafka topics to monitor | +| kraft.enabled | bool | `false` | Enable KRaft mode for Kafka | +| registry.ingress.annotations | object | `{}` | Annotations that will be added to the Ingress resource | +| registry.ingress.enabled | bool | `false` | Whether to enable an ingress for the Schema Registry | +| registry.ingress.hostname | string | None, must be set if ingress is enabled | Hostname for the Schema Registry | +| registry.resources | object | See `values.yaml` | Kubernetes requests and limits for the Schema Registry | +| registry.schemaTopic | string | `"registry-schemas"` | Name of the topic used by the Schema Registry | +| superusers | list | `["kafka-admin"]` | A list of usernames for users who should have global admin permissions. These users will be created, along with their credentials. | +| users.appmetrics.enabled | bool | `false` | Enable user appmetrics | +| users.camera.enabled | bool | `false` | Enable user camera, used at the camera environments | +| users.consdb.enabled | bool | `false` | Enable user consdb | +| users.kafdrop.enabled | bool | `false` | Enable user Kafdrop (deployed by parent Sasquatch chart). | +| users.kafkaConnectManager.enabled | bool | `false` | Enable user kafka-connect-manager | +| users.promptProcessing.enabled | bool | `false` | Enable user prompt-processing | +| users.replicator.enabled | bool | `false` | Enable user replicator (used by Mirror Maker 2 and required at both source and target clusters) | +| users.telegraf.enabled | bool | `false` | Enable user telegraf (deployed by parent Sasquatch chart) | +| users.tsSalKafka.enabled | bool | `false` | Enable user ts-salkafka, used at the telescope environments | diff --git a/applications/prompt-processing-kafka/charts/strimzi-kafka/templates/certificates.yaml b/applications/prompt-processing-kafka/charts/strimzi-kafka/templates/certificates.yaml new file mode 100644 index 0000000000..71f39d59a5 --- /dev/null +++ b/applications/prompt-processing-kafka/charts/strimzi-kafka/templates/certificates.yaml @@ -0,0 +1,19 @@ +{{- if and (.Values.kafka.externalListener.tls.enabled) (.Values.kafka.externalListener.bootstrap.host) }} +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ .Values.cluster.name }}-external-tls + +spec: + secretName: {{ .Values.cluster.name }}-external-tls + + issuerRef: + name: {{ .Values.kafka.externalListener.tls.certIssuerName }} + kind: ClusterIssuer + + dnsNames: + - {{ .Values.kafka.externalListener.bootstrap.host }} + {{- range $broker := .Values.kafka.externalListener.brokers }} + - {{ $broker.host }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/applications/prompt-processing-kafka/charts/strimzi-kafka/templates/kafka-metrics-configmap.yaml b/applications/prompt-processing-kafka/charts/strimzi-kafka/templates/kafka-metrics-configmap.yaml new file mode 100644 index 0000000000..83fa415a51 --- /dev/null +++ b/applications/prompt-processing-kafka/charts/strimzi-kafka/templates/kafka-metrics-configmap.yaml @@ -0,0 +1,156 @@ +{{- if .Values.kafka.metricsConfig.enabled }} +kind: ConfigMap +apiVersion: v1 +metadata: + name: kafka-metrics + labels: + app: prompt-processing-kafka-metrics +data: + kafka-metrics-config.yml: | + # See https://github.com/prometheus/jmx_exporter for more info about JMX Prometheus Exporter metrics + lowercaseOutputName: true + rules: + # Special cases and very specific rules + - pattern: kafka.server<>Value + name: kafka_server_$1_$2 + type: GAUGE + labels: + clientId: "$3" + topic: "$4" + partition: "$5" + - pattern: kafka.server<>Value + name: kafka_server_$1_$2 + type: GAUGE + labels: + clientId: "$3" + broker: "$4:$5" + - pattern: kafka.server<>connections + name: kafka_server_$1_connections_tls_info + type: GAUGE + labels: + cipher: "$2" + protocol: "$3" + listener: "$4" + networkProcessor: "$5" + - pattern: kafka.server<>connections + name: kafka_server_$1_connections_software + type: GAUGE + labels: + clientSoftwareName: "$2" + clientSoftwareVersion: "$3" + listener: "$4" + networkProcessor: "$5" + - pattern: "kafka.server<>(.+):" + name: kafka_server_$1_$4 + type: GAUGE + labels: + listener: "$2" + networkProcessor: "$3" + - pattern: kafka.server<>(.+) + name: kafka_server_$1_$4 + type: GAUGE + labels: + listener: "$2" + networkProcessor: "$3" + # Some percent metrics use MeanRate attribute + # Ex) kafka.server<>MeanRate + - pattern: kafka.(\w+)<>MeanRate + name: kafka_$1_$2_$3_percent + type: GAUGE + # Generic gauges for percents + - pattern: kafka.(\w+)<>Value + name: kafka_$1_$2_$3_percent + type: GAUGE + - pattern: kafka.(\w+)<>Value + name: kafka_$1_$2_$3_percent + type: GAUGE + labels: + "$4": "$5" + # Generic per-second counters with 0-2 key/value pairs + - pattern: kafka.(\w+)<>Count + name: kafka_$1_$2_$3_total + type: COUNTER + labels: + "$4": "$5" + "$6": "$7" + - pattern: kafka.(\w+)<>Count + name: kafka_$1_$2_$3_total + type: COUNTER + labels: + "$4": "$5" + - pattern: kafka.(\w+)<>Count + name: kafka_$1_$2_$3_total + type: COUNTER + # Generic gauges with 0-2 key/value pairs + - pattern: kafka.(\w+)<>Value + name: kafka_$1_$2_$3 + type: GAUGE + labels: + "$4": "$5" + "$6": "$7" + - pattern: kafka.(\w+)<>Value + name: kafka_$1_$2_$3 + type: GAUGE + labels: + "$4": "$5" + - pattern: kafka.(\w+)<>Value + name: kafka_$1_$2_$3 + type: GAUGE + # Emulate Prometheus 'Summary' metrics for the exported 'Histogram's. + # Note that these are missing the '_sum' metric! + - pattern: kafka.(\w+)<>Count + name: kafka_$1_$2_$3_count + type: COUNTER + labels: + "$4": "$5" + "$6": "$7" + - pattern: kafka.(\w+)<>(\d+)thPercentile + name: kafka_$1_$2_$3 + type: GAUGE + labels: + "$4": "$5" + "$6": "$7" + quantile: "0.$8" + - pattern: kafka.(\w+)<>Count + name: kafka_$1_$2_$3_count + type: COUNTER + labels: + "$4": "$5" + - pattern: kafka.(\w+)<>(\d+)thPercentile + name: kafka_$1_$2_$3 + type: GAUGE + labels: + "$4": "$5" + quantile: "0.$6" + - pattern: kafka.(\w+)<>Count + name: kafka_$1_$2_$3_count + type: COUNTER + - pattern: kafka.(\w+)<>(\d+)thPercentile + name: kafka_$1_$2_$3 + type: GAUGE + labels: + quantile: "0.$4" + {{- if .Values.kraft.enabled }} + # KRaft mode: uncomment the following lines to export KRaft related metrics + # KRaft overall related metrics + # distinguish between always increasing COUNTER (total and max) and variable GAUGE (all others) metrics + - pattern: "kafka.server<>(.+-total|.+-max):" + name: kafka_server_raftmetrics_$1 + type: COUNTER + - pattern: "kafka.server<>(.+):" + name: kafka_server_raftmetrics_$1 + type: GAUGE + # KRaft "low level" channels related metrics + # distinguish between always increasing COUNTER (total and max) and variable GAUGE (all others) metrics + - pattern: "kafka.server<>(.+-total|.+-max):" + name: kafka_server_raftchannelmetrics_$1 + type: COUNTER + - pattern: "kafka.server<>(.+):" + name: kafka_server_raftchannelmetrics_$1 + type: GAUGE + # Broker metrics related to fetching metadata topic records in KRaft mode + - pattern: "kafka.server<>(.+):" + name: kafka_server_brokermetadatametrics_$1 + type: GAUGE + {{- end }} +{{- end}} diff --git a/applications/prompt-processing-kafka/charts/strimzi-kafka/templates/kafka.yaml b/applications/prompt-processing-kafka/charts/strimzi-kafka/templates/kafka.yaml new file mode 100644 index 0000000000..7a28269717 --- /dev/null +++ b/applications/prompt-processing-kafka/charts/strimzi-kafka/templates/kafka.yaml @@ -0,0 +1,163 @@ +apiVersion: kafka.strimzi.io/v1beta2 +kind: Kafka +metadata: + name: {{ .Values.cluster.name }} + annotations: + {{- if .Values.kraft.enabled }} + strimzi.io/kraft: enabled + {{- end }} + strimzi.io/node-pools: enabled +spec: + kafka: + template: + persistentVolumeClaim: + metadata: + annotations: + argocd.argoproj.io/compare-options: IgnoreExtraneous + argocd.argoproj.io/sync-options: Prune=false + pod: + {{- with .Values.kafka.affinity }} + affinity: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with .Values.kafka.tolerations }} + tolerations: + {{- toYaml . | nindent 10 }} + {{- end}} + version: {{ .Values.kafka.version | quote }} + replicas: {{ .Values.kafka.replicas }} + listeners: + {{- if .Values.kafka.listeners.noauth.enabled }} + # internal listener without tls encryption and without authentication + # used by ceph bucket notifications as authentication does not work with + - name: plain + port: 9092 + type: internal + tls: false + {{- end }} + {{- if .Values.kafka.listeners.plain.enabled }} + # internal listener without tls encryption and with scram-sha-512 authentication + # used by clients inside the Kubernetes cluster + - name: plain + port: 9092 + type: internal + tls: false + authentication: + type: scram-sha-512 + {{- end }} + {{- if .Values.kafka.listeners.tls.enabled }} + # internal listener with tls encryption and mutual tls authentication + # used by the schema registry and kafka connect clients + - name: tls + port: 9093 + type: internal + tls: true + authentication: + type: tls + {{- end }} + {{- if .Values.kafka.listeners.external.enabled }} + # external listener of type loadbalancer with tls encryption and scram-sha-512 + # authentication used by clients outside the Kubernetes cluster + - name: external + port: 9094 + type: loadbalancer + tls: {{ .Values.kafka.externalListener.tls.enabled }} + authentication: + type: scram-sha-512 + configuration: + bootstrap: + {{- if .Values.kafka.externalListener.bootstrap.loadBalancerIP }} + loadBalancerIP: {{ .Values.kafka.externalListener.bootstrap.loadBalancerIP }} + {{- end }} + {{- if .Values.kafka.externalListener.bootstrap.annotations }} + annotations: {{ .Values.kafka.externalListener.bootstrap.annotations }} + {{- end }} + {{- if .Values.kafka.externalListener.brokers }} + brokers: + {{- range $broker := .Values.kafka.externalListener.brokers }} + - broker: {{ $broker.broker }} + loadBalancerIP: {{ $broker.loadBalancerIP }} + advertisedHost: {{ $broker.host }} + advertisedPort: 9094 + annotations: + {{- range $key, $value := $broker.annotations }} + {{ $key }}: {{ $value }} + {{- end}} + {{- end }} + {{- end }} + {{- if and (.Values.kafka.externalListener.tls.enabled) (.Values.kafka.externalListener.bootstrap.host) }} + brokerCertChainAndKey: + secretName: {{ .Values.cluster.name }}-external-tls + certificate: tls.crt + key: tls.key + {{- end }} + {{- end }} + + authorization: + type: simple +{{- if .Values.superusers }} + superUsers: +{{- range .Values.superusers }} + - {{ . }} +{{- end }} +{{- end }} + config: + offsets.topic.replication.factor: {{ .Values.kafka.replicas }} + transaction.state.log.replication.factor: {{ .Values.kafka.replicas }} + transaction.state.log.min.isr: {{ .Values.kafka.minInsyncReplicas }} + default.replication.factor: {{ .Values.kafka.replicas }} + min.insync.replicas: {{ .Values.kafka.minInsyncReplicas }} + {{- range $key, $value := .Values.kafka.config }} + {{ $key }}: {{ $value }} + {{- end }} + {{- if .Values.kafka.metricsConfig.enabled }} + metricsConfig: + type: jmxPrometheusExporter + valueFrom: + configMapKeyRef: + name: kafka-metrics + key: kafka-metrics-config.yml + {{- end }} + storage: + type: jbod + volumes: + # Note that storage is configured per replica. If there are 3 replicas, + # and 2 volumes in this array, each replica will get 2 + # PersistentVolumeClaims for the configured size, for a total of 6 + # volumes. + - id: 0 + type: persistent-claim + size: {{ .Values.kafka.storage.size }} + {{- if .Values.kafka.storage.storageClassName }} + class: {{ .Values.kafka.storage.storageClassName }} + {{- end}} + deleteClaim: false + entityOperator: + topicOperator: {} + userOperator: {} + template: + topicOperatorContainer: + env: + - name: STRIMZI_MAX_QUEUE_SIZE + value: "8192" + {{- if .Values.kafkaExporter.enabled }} + kafkaExporter: + topicRegex: {{ .Values.kafkaExporter.topicRegex }} + groupRegex: {{ .Values.kafkaExporter.groupRegex }} + logging: {{ .Values.kafkaExporter.logging }} + enableSaramaLogging: {{ .Values.kafkaExporter.enableSaramaLogging }} + {{- with .Values.kafkaExporter.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.cluster.monitorLabel }} + template: + pod: + metadata: + labels: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- end }} + {{ if .Values.cruiseControl.enabled }} + cruiseControl: {} + {{- end }} diff --git a/applications/prompt-processing-kafka/charts/strimzi-kafka/templates/schema-registry-ingress.yaml b/applications/prompt-processing-kafka/charts/strimzi-kafka/templates/schema-registry-ingress.yaml new file mode 100644 index 0000000000..3ce98b48d5 --- /dev/null +++ b/applications/prompt-processing-kafka/charts/strimzi-kafka/templates/schema-registry-ingress.yaml @@ -0,0 +1,25 @@ +{{- if .Values.registry.ingress.enabled }} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: prompt-processing-kafka-schema-registry + labels: + name: prompt-processing-kafka-schema-registry + annotations: + {{- with .Values.registry.ingress.annotations }} + {{ toYaml . | indent 4 }} + {{- end }} +spec: + ingressClassName: "nginx" + rules: + - host: {{ .Values.registry.ingress.hostname | quote }} + http: + paths: + - path: {{ .Values.registry.ingress.path }} + pathType: Prefix + backend: + service: + name: prompt-processing-kafka-schema-registry + port: + number: 8081 +{{- end }} diff --git a/applications/prompt-processing-kafka/charts/strimzi-kafka/templates/schema-registry.yaml b/applications/prompt-processing-kafka/charts/strimzi-kafka/templates/schema-registry.yaml new file mode 100644 index 0000000000..8f15e429a5 --- /dev/null +++ b/applications/prompt-processing-kafka/charts/strimzi-kafka/templates/schema-registry.yaml @@ -0,0 +1,73 @@ +apiVersion: roundtable.lsst.codes/v1beta1 +kind: StrimziSchemaRegistry +metadata: + name: {{ .Values.cluster.name }}-schema-registry +spec: + listener: tls + compatibilityLevel: none + cpuLimit: {{ .Values.registry.resources.limits.cpu | quote }} + cpuRequest: {{ .Values.registry.resources.requests.cpu | quote }} + memoryLimit: {{ .Values.registry.resources.limits.memory | quote }} + memoryRequest: {{ .Values.registry.resources.requests.memory | quote }} +--- +apiVersion: kafka.strimzi.io/v1beta2 +kind: KafkaTopic +metadata: + name: {{ .Values.registry.schemaTopic }} + labels: + strimzi.io/cluster: {{ .Values.cluster.name }} +spec: + partitions: 1 + replicas: 3 + config: + cleanup.policy: compact +--- +apiVersion: kafka.strimzi.io/v1beta2 +kind: KafkaUser +metadata: + name: {{ .Values.cluster.name }}-schema-registry + labels: + strimzi.io/cluster: {{ .Values.cluster.name }} +spec: + authentication: + type: tls + authorization: + # Official docs on authorizations required for the Schema Registry: + # https://docs.confluent.io/current/schema-registry/security/index.html#authorizing-access-to-the-schemas-topic + type: simple + acls: + # Allow Read, Write and DescribeConfigs operations on the + # schemas topic + - resource: + type: topic + name: {{ .Values.registry.schemaTopic }} + patternType: literal + operation: Read + type: allow + - resource: + type: topic + name: {{ .Values.registry.schemaTopic }} + patternType: literal + operation: Write + type: allow + - resource: + type: topic + name: {{ .Values.registry.schemaTopic }} + patternType: literal + operation: DescribeConfigs + type: allow + # Allow all operations on the schema-registry* group + - resource: + type: group + name: schema-registry + patternType: prefix + operation: All + type: allow + # Allow Describe on the __consumer_offsets topic + # (The official docs also mention DescribeConfigs?) + - resource: + type: topic + name: "__consumer_offsets" + patternType: literal + operation: Describe + type: allow diff --git a/applications/prompt-processing-kafka/charts/strimzi-kafka/templates/superusers.yaml b/applications/prompt-processing-kafka/charts/strimzi-kafka/templates/superusers.yaml new file mode 100644 index 0000000000..4da1e5b269 --- /dev/null +++ b/applications/prompt-processing-kafka/charts/strimzi-kafka/templates/superusers.yaml @@ -0,0 +1,27 @@ +{{ range $idx, $username := .Values.superusers }} +--- +apiVersion: kafka.strimzi.io/v1beta2 +kind: KafkaUser +metadata: + name: {{ $username }} + labels: + strimzi.io/cluster: {{ $.Values.cluster.name }} +spec: + authentication: + type: tls + authorization: + type: simple + acls: + - resource: + type: group + name: "*" + patternType: literal + operation: All + - resource: + type: topic + name: "*" + patternType: literal + type: allow + host: "*" + operation: All +{{ end }} diff --git a/applications/prompt-processing-kafka/charts/strimzi-kafka/templates/users.yaml b/applications/prompt-processing-kafka/charts/strimzi-kafka/templates/users.yaml new file mode 100644 index 0000000000..a7ac889f3e --- /dev/null +++ b/applications/prompt-processing-kafka/charts/strimzi-kafka/templates/users.yaml @@ -0,0 +1,91 @@ +apiVersion: kafka.strimzi.io/v1beta2 +kind: KafkaUser +metadata: + name: fan-out-producer + labels: + strimzi.io/cluster: {{ .Values.cluster.name }} +spec: + authentication: + type: scram-sha-512 + password: + valueFrom: + secretKeyRef: + name: prompt-processing-kafka + key: fan-out-producer-password + authorization: + type: simple + acls: + - resource: + type: group + name: "*" + patternType: literal + operation: All + - resource: + type: topic + name: "fanned-out-events" + patternType: literal + type: allow + host: "*" + operation: Write +--- +apiVersion: kafka.strimzi.io/v1beta2 +kind: KafkaUser +metadata: + name: fan-out-consumer + labels: + strimzi.io/cluster: {{ .Values.cluster.name }} +spec: + authentication: + type: scram-sha-512 + password: + valueFrom: + secretKeyRef: + name: prompt-processing-kafka + key: fan-out-consumer-password + authorization: + type: simple + acls: + - resource: + type: group + name: "*" + patternType: literal + operation: All + - resource: + type: topic + name: "fanned-out-events" + patternType: literal + type: allow + host: "*" + operation: Read +{{- if .Values.users.kafdrop.enabled }} +--- +apiVersion: kafka.strimzi.io/v1beta2 +kind: KafkaUser +metadata: + name: kafdrop + labels: + strimzi.io/cluster: {{ .Values.cluster.name }} +spec: + authentication: + type: scram-sha-512 + password: + valueFrom: + secretKeyRef: + name: prompt-processing-kafka + key: kafdrop-password + authorization: + type: simple + acls: + - resource: + type: group + name: "*" + patternType: literal + operation: All + - resource: + type: topic + name: "*" + patternType: literal + type: allow + host: "*" + operation: All +{{- end }} diff --git a/applications/prompt-processing-kafka/charts/strimzi-kafka/values.yaml b/applications/prompt-processing-kafka/charts/strimzi-kafka/values.yaml new file mode 100644 index 0000000000..2929061915 --- /dev/null +++ b/applications/prompt-processing-kafka/charts/strimzi-kafka/values.yaml @@ -0,0 +1,239 @@ +# Default values for Kafka used in Prompt Processing Kafka. + +cluster: + # -- Name used for the Kafka cluster, and used by Strimzi for many + # annotations + name: prompt-processing-kafka + + # -- Site wide label required for gathering Prometheus metrics if they are + # enabled + monitorLabel: {} + +# TODO. update structure. temporary set. +cruiseControl: + enabled: false + +kafka: + # -- Version of Kafka to deploy + version: "3.8.0" + + # -- Number of Kafka broker replicas to run + replicas: 3 + + # -- The minimum number of in-sync replicas that must be available for the producer to successfully send records + # Cannot be greater than the number of replicas. + minInsyncReplicas: 2 + + storage: + # -- Size of the backing storage disk for each of the Kafka brokers + size: 500Gi + + # -- Name of a StorageClass to use when requesting persistent volumes + storageClassName: "" + + config: + # -- Number of minutes for a consumer group's offsets to be retained + offsets.retention.minutes: 2880 + + # -- Number of days for a topic's data to be retained + log.retention.hours: 48 + + # -- How much disk space Kafka will ensure is available, set to 70% of the + # data partition size + log.retention.bytes: "350000000000" + + # -- The largest record batch size allowed by Kafka + message.max.bytes: 10485760 + + # -- The number of bytes of messages to attempt to fetch for each partition + replica.fetch.max.bytes: 10485760 + + metricsConfig: + # -- Whether metric configuration is enabled + enabled: false + + listeners: + noauth: + # -- Whether internal noauth listener is enabled + enabled: false + + plain: + # -- Whether internal plaintext listener is enabled + enabled: false + + tls: + # -- Whether internal TLS listener is enabled + enabled: false + + external: + # -- Whether external listener is enabled + enabled: false + + externalListener: + tls: + # -- Whether TLS encryption is enabled + enabled: false + + # -- Name of a ClusterIssuer capable of provisioning a TLS certificate + # for the broker + certIssuerName: "letsencrypt-dns" + + bootstrap: + # Load balancer IP configuration + # + # The loadbalancer is requested with the IP address specified in this + # field. This feature depends on whether the underlying cloud provider + # supports specifying the loadBalancerIP when a load balancer is + # created. This field is ignored if the cloud provider does not support + # the feature. + # + # Once the IP address is provisioned this option make it possible to pin + # the IP address. We can request the same IP next time it is + # provisioned. This is important because it lets us configure a DNS + # record, associating a hostname with that pinned IP address. + # + # -- Request this load balancer IP. See `values.yaml` for more + # discussion + # @default -- Do not request a load balancer IP + loadBalancerIP: "" + + # -- Name used for TLS hostname verification + # @default -- Do not configure TLS + host: "" + + # -- Annotations that will be added to the Ingress, Route, or Service + # resource + annotations: {} + + # -- Brokers configuration. _host_ is used in the brokers' + # advertised.brokers configuration and for TLS hostname verification. The + # format is a list of maps. + brokers: [] + # For example: + # brokers: + # - broker: 0 + # loadBalancerIP: "192.168.1.1" + # host: broker-0.example + # annotations: + # metallb.universe.tf/address-pool: sdf-dmz + # - broker: 1 + # loadBalancerIP: "192.168.1.2" + # host: broker-1.example + # annotations: + # metallb.universe.tf/address-pool: sdf-dmz + + # -- Affinity for Kafka pod assignment + # @default -- See `values.yaml` + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: "app.kubernetes.io/name" + operator: In + values: + - kafka + topologyKey: "kubernetes.io/hostname" + + # -- Tolerations for Kafka broker pod assignment + tolerations: [] + + # -- Kubernetes requests and limits for the Kafka brokers + # @default -- See `values.yaml` + resources: + requests: + memory: 32Gi + cpu: "4" + limits: + memory: 64Gi + cpu: "8" + +kraft: + # -- Enable KRaft mode for Kafka + enabled: false + +kafkaController: + # -- Enable Kafka Controller + enabled: false + + storage: + # -- Size of the backing storage disk for each of the Kafka controllers + size: 20Gi + + # -- Name of a StorageClass to use when requesting persistent volumes + storageClassName: "" + + # -- Kubernetes requests and limits for the Kafka Controller + # @default -- See `values.yaml` + resources: + requests: + memory: 32Gi + cpu: "4" + limits: + memory: 64Gi + cpu: "8" + +kafkaExporter: + # -- Enable Kafka exporter + enabled: false + + # -- Kafka topics to monitor + topicRegex: ".*" + + # -- Consumer groups to monitor + groupRegex: ".*" + + # -- Logging level + logging: info + + # -- Enable Sarama logging for pod + enableSaramaLogging: false + + # -- Kubernetes requests and limits for the Kafka exporter + # @default -- See `values.yaml` + resources: + limits: + cpu: "1" + memory: "256Mi" + requests: + cpu: "375m" + memory: "100Mi" + +registry: + ingress: + # -- Whether to enable an ingress for the Schema Registry + enabled: false + + # -- Hostname for the Schema Registry + # @default -- None, must be set if ingress is enabled + hostname: "" + + # -- Annotations that will be added to the Ingress resource + annotations: {} + + # -- Name of the topic used by the Schema Registry + schemaTopic: registry-schemas + + # -- Kubernetes requests and limits for the Schema Registry + # @default -- See `values.yaml` + resources: + requests: + memory: "128Mi" + cpu: "5m" + limits: + memory: "2Gi" + cpu: "1" + +# -- A list of usernames for users who should have global admin permissions. +# These users will be created, along with their credentials. +superusers: + - kafka-admin + +users: + kafdrop: + # -- Enable user Kafdrop (deployed by parent Prompt Processing Kafka chart). + enabled: false + + promptProcessing: + # -- Enable user prompt-processing + enabled: false diff --git a/applications/prompt-processing-kafka/topic.yaml b/applications/prompt-processing-kafka/topic.yaml new file mode 100644 index 0000000000..d034b9f140 --- /dev/null +++ b/applications/prompt-processing-kafka/topic.yaml @@ -0,0 +1,12 @@ +apiVersion: kafka.strimzi.io/v1beta2 +kind: KafkaTopic +metadata: + labels: + strimzi.io/cluster: prompt-processing + name: registry-schemas + namespace: kafka +spec: + config: + cleanup.policy: compact + partitions: 1 + replicas: 1 diff --git a/applications/prompt-processing-kafka/values-usdfdev-prompt-processing.yaml b/applications/prompt-processing-kafka/values-usdfdev-prompt-processing.yaml new file mode 100644 index 0000000000..5448eb667f --- /dev/null +++ b/applications/prompt-processing-kafka/values-usdfdev-prompt-processing.yaml @@ -0,0 +1,8 @@ +strimzi-kafka: + users: + kafdrop: + enabled: false + kraft: + enabled: true + kafkaController: + enabled: true diff --git a/applications/prompt-processing-kafka/values.yaml b/applications/prompt-processing-kafka/values.yaml new file mode 100644 index 0000000000..bb7781272a --- /dev/null +++ b/applications/prompt-processing-kafka/values.yaml @@ -0,0 +1,35 @@ +# Default values for Sasquatch. + +# Override strimzi-kafka subchart configuration. +strimzi-kafka: + kafka: + listeners: + noauth: + # -- Whether internal no authentication listener is enabled + enabled: true + + tls: + # -- Whether internal TLS listener is enabled + enabled: true + + plain: + # -- Whether internal plaintext listener is enabled + enabled: true + + external: + # -- Whether external listener is enabled + enabled: false + +strimzi-registry-operator: + # -- Name of the Strimzi Kafka cluster + clusterName: prompt-processing-kafka + + # -- Namespace where the Strimzi Kafka cluster is deployed + clusterNamespace: prompt-processing-kafka + + # -- Namespace where the strimzi-registry-operator is deployed + operatorNamespace: prompt-processing-kafka + +kafdrop: + # -- Whether Kafdrop is enabled + enabled: false diff --git a/docs/applications/prompt-processing-kafka/index.rst b/docs/applications/prompt-processing-kafka/index.rst new file mode 100644 index 0000000000..baec9eb5a3 --- /dev/null +++ b/docs/applications/prompt-processing-kafka/index.rst @@ -0,0 +1,18 @@ +.. px-app:: prompt-processing-kafka + +########################################################### +pprompt-processing-kafka — Kafka cluster for prompt processing +########################################################### + +Kafka cluster for fan out events and bucket notifications. + +.. jinja:: pprompt-processing-kafka + :file: applications/_summary.rst.jinja + +Guides +====== + +.. toctree:: + :maxdepth: 2 + + values diff --git a/docs/applications/prompt-processing-kafka/values.md b/docs/applications/prompt-processing-kafka/values.md new file mode 100644 index 0000000000..21bce376e0 --- /dev/null +++ b/docs/applications/prompt-processing-kafka/values.md @@ -0,0 +1,12 @@ +```{px-app-values} prompt-proto-service-hsc +``` + +# Prompt Processing Kafka Helm values reference + +Helm values reference table for the {px-app}`prompt-processing-kafka` application. + +```{include} ../../../applications/prompt-processing-kafka/README.md +--- +start-after: "## Values" +--- +``` diff --git a/environments/README.md b/environments/README.md index 9ec20b4fda..e348ff3cd5 100644 --- a/environments/README.md +++ b/environments/README.md @@ -44,6 +44,7 @@ | applications.portal | bool | `false` | Enable the portal application | | applications.postgres | bool | `false` | Enable the in-cluster PostgreSQL server. Use of this server is discouraged in favor of using infrastructure SQL, but will remain supported for use cases such as minikube test deployments. | | applications.production-tools | bool | `false` | Enable the production-tools application | +| applications.prompt-processing-kafka | bool | `false` | Enable the prompt-processing-kafka application | | applications.prompt-proto-service-hsc | bool | `false` | Enable the prompt-proto-service-hsc application | | applications.prompt-proto-service-hsc-gpu | bool | `false` | Enable the prompt-proto-service-hsc-gpu application | | applications.prompt-proto-service-latiss | bool | `false` | Enable the prompt-proto-service-latiss application | diff --git a/environments/templates/applications/prompt/prompt-processing-kafka.yaml b/environments/templates/applications/prompt/prompt-processing-kafka.yaml new file mode 100644 index 0000000000..14b76d5207 --- /dev/null +++ b/environments/templates/applications/prompt/prompt-processing-kafka.yaml @@ -0,0 +1,34 @@ +{{- if (index .Values "applications" "prompt-processing-kafka") -}} +apiVersion: v1 +kind: Namespace +metadata: + name: "prompt-processing-kafka" +--- +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: "prompt-processing-kafka" + namespace: "argocd" + finalizers: + - "resources-finalizer.argocd.argoproj.io" +spec: + destination: + namespace: "prompt-processing-kafka" + server: "https://kubernetes.default.svc" + project: "prompt" + source: + path: "applications/prompt-processing-kafka" + repoURL: {{ .Values.repoUrl | quote }} + targetRevision: {{ .Values.targetRevision | quote }} + helm: + parameters: + - name: "global.host" + value: {{ .Values.fqdn | quote }} + - name: "global.baseUrl" + value: "https://{{ .Values.fqdn }}" + - name: "global.vaultSecretsPath" + value: {{ .Values.vaultPathPrefix | quote }} + valueFiles: + - "values.yaml" + - "values-{{ .Values.name }}.yaml" +{{- end -}} diff --git a/environments/values-usdfdev-prompt-processing.yaml b/environments/values-usdfdev-prompt-processing.yaml index 236be82dac..40888107cb 100644 --- a/environments/values-usdfdev-prompt-processing.yaml +++ b/environments/values-usdfdev-prompt-processing.yaml @@ -8,6 +8,7 @@ applications: gafaelfawr: false ingress-nginx: false next-visit-fan-out: true + prompt-processing-kafka: true prompt-proto-service-hsc: true prompt-proto-service-hsc-gpu: true prompt-proto-service-latiss: true diff --git a/environments/values.yaml b/environments/values.yaml index b7774a1d1e..73d07b046d 100644 --- a/environments/values.yaml +++ b/environments/values.yaml @@ -183,6 +183,9 @@ applications: # -- Enable the production-tools application production-tools: false + # -- Enable the prompt-processing-kafka application + prompt-processing-kafka: false + # -- Enable the prompt-proto-service-hsc application prompt-proto-service-hsc: false From b22799701f0423dceddb79a300848b775931bfcb Mon Sep 17 00:00:00 2001 From: dspeck1 Date: Tue, 15 Oct 2024 11:14:57 -0500 Subject: [PATCH 03/28] fix docs and typo --- .../prompt-processing-kafka/README.md | 28 +++++++++++++++++++ .../charts/strimzi-kafka/README.md | 18 +----------- .../prompt-processing-kafka/index.rst | 4 +-- 3 files changed, 31 insertions(+), 19 deletions(-) create mode 100644 applications/prompt-processing-kafka/README.md diff --git a/applications/prompt-processing-kafka/README.md b/applications/prompt-processing-kafka/README.md new file mode 100644 index 0000000000..49617ed11b --- /dev/null +++ b/applications/prompt-processing-kafka/README.md @@ -0,0 +1,28 @@ +# prompt-processing-kafka + +Kafka environment for prompt processing fan out events and bucket notifications + +## Source Code + +* +* + +## Requirements + +| Repository | Name | Version | +|------------|------|---------| +| | strimzi-kafka | 1.0.0 | +| https://lsst-sqre.github.io/charts/ | strimzi-registry-operator | 2.1.0 | + +## Values + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| kafdrop.enabled | bool | `false` | Whether Kafdrop is enabled | +| strimzi-kafka.kafka.listeners.external.enabled | bool | `false` | Whether external listener is enabled | +| strimzi-kafka.kafka.listeners.noauth.enabled | bool | `true` | Whether internal no authentication listener is enabled | +| strimzi-kafka.kafka.listeners.plain.enabled | bool | `true` | Whether internal plaintext listener is enabled | +| strimzi-kafka.kafka.listeners.tls.enabled | bool | `true` | Whether internal TLS listener is enabled | +| strimzi-registry-operator.clusterName | string | `"prompt-processing-kafka"` | Name of the Strimzi Kafka cluster | +| strimzi-registry-operator.clusterNamespace | string | `"prompt-processing-kafka"` | Namespace where the Strimzi Kafka cluster is deployed | +| strimzi-registry-operator.operatorNamespace | string | `"prompt-processing-kafka"` | Namespace where the strimzi-registry-operator is deployed | diff --git a/applications/prompt-processing-kafka/charts/strimzi-kafka/README.md b/applications/prompt-processing-kafka/charts/strimzi-kafka/README.md index fe66fffbee..7e00d9d4dd 100644 --- a/applications/prompt-processing-kafka/charts/strimzi-kafka/README.md +++ b/applications/prompt-processing-kafka/charts/strimzi-kafka/README.md @@ -8,15 +8,6 @@ A subchart to deploy Strimzi Kafka components for Prompt Processing. |-----|------|---------|-------------| | cluster.monitorLabel | object | `{}` | Site wide label required for gathering Prometheus metrics if they are enabled | | cluster.name | string | `"prompt-processing-kafka"` | Name used for the Kafka cluster, and used by Strimzi for many annotations | -| connect.config."key.converter" | string | `"io.confluent.connect.avro.AvroConverter"` | Set the converter for the message ke | -| connect.config."key.converter.schema.registry.url" | string | `"http://sasquatch-schema-registry.sasquatch:8081"` | URL for the schema registry | -| connect.config."key.converter.schemas.enable" | bool | `true` | Enable converted schemas for the message key | -| connect.config."value.converter" | string | `"io.confluent.connect.avro.AvroConverter"` | Converter for the message value | -| connect.config."value.converter.schema.registry.url" | string | `"http://sasquatch-schema-registry.sasquatch:8081"` | URL for the schema registry | -| connect.config."value.converter.schemas.enable" | bool | `true` | Enable converted schemas for the message value | -| connect.enabled | bool | `false` | Enable Kafka Connect | -| connect.image | string | `"ghcr.io/lsst-sqre/strimzi-0.40.0-kafka-3.7.0:tickets-DM-43491"` | Custom strimzi-kafka image with connector plugins used by sasquatch | -| connect.replicas | int | `3` | Number of Kafka Connect replicas to run | | cruiseControl.enabled | bool | `false` | | | kafka.affinity | object | See `values.yaml` | Affinity for Kafka pod assignment | | kafka.config."log.retention.bytes" | string | `"350000000000"` | How much disk space Kafka will ensure is available, set to 70% of the data partition size | @@ -59,12 +50,5 @@ A subchart to deploy Strimzi Kafka components for Prompt Processing. | registry.resources | object | See `values.yaml` | Kubernetes requests and limits for the Schema Registry | | registry.schemaTopic | string | `"registry-schemas"` | Name of the topic used by the Schema Registry | | superusers | list | `["kafka-admin"]` | A list of usernames for users who should have global admin permissions. These users will be created, along with their credentials. | -| users.appmetrics.enabled | bool | `false` | Enable user appmetrics | -| users.camera.enabled | bool | `false` | Enable user camera, used at the camera environments | -| users.consdb.enabled | bool | `false` | Enable user consdb | -| users.kafdrop.enabled | bool | `false` | Enable user Kafdrop (deployed by parent Sasquatch chart). | -| users.kafkaConnectManager.enabled | bool | `false` | Enable user kafka-connect-manager | +| users.kafdrop.enabled | bool | `false` | Enable user Kafdrop (deployed by parent Prompt Processing Kafka chart). | | users.promptProcessing.enabled | bool | `false` | Enable user prompt-processing | -| users.replicator.enabled | bool | `false` | Enable user replicator (used by Mirror Maker 2 and required at both source and target clusters) | -| users.telegraf.enabled | bool | `false` | Enable user telegraf (deployed by parent Sasquatch chart) | -| users.tsSalKafka.enabled | bool | `false` | Enable user ts-salkafka, used at the telescope environments | diff --git a/docs/applications/prompt-processing-kafka/index.rst b/docs/applications/prompt-processing-kafka/index.rst index baec9eb5a3..243fd6bc74 100644 --- a/docs/applications/prompt-processing-kafka/index.rst +++ b/docs/applications/prompt-processing-kafka/index.rst @@ -1,12 +1,12 @@ .. px-app:: prompt-processing-kafka ########################################################### -pprompt-processing-kafka — Kafka cluster for prompt processing +prompt-processing-kafka — Kafka cluster for prompt processing ########################################################### Kafka cluster for fan out events and bucket notifications. -.. jinja:: pprompt-processing-kafka +.. jinja:: prompt-processing-kafka :file: applications/_summary.rst.jinja Guides From 6a285e978baef250e2fd7a71716408c2ad06ca2a Mon Sep 17 00:00:00 2001 From: dspeck1 Date: Tue, 15 Oct 2024 11:38:05 -0500 Subject: [PATCH 04/28] Enable strimzi and fix doc issue. --- .../prompt-processing-kafka/README.md | 53 ++++++++++++++++--- .../values-usdfdev-prompt-processing.yaml | 10 ++++ .../values-usdfdev-prompt-processing.yaml | 1 + 3 files changed, 57 insertions(+), 7 deletions(-) create mode 100644 applications/strimzi/values-usdfdev-prompt-processing.yaml diff --git a/applications/prompt-processing-kafka/README.md b/applications/prompt-processing-kafka/README.md index 49617ed11b..428b089a02 100644 --- a/applications/prompt-processing-kafka/README.md +++ b/applications/prompt-processing-kafka/README.md @@ -7,13 +7,6 @@ Kafka environment for prompt processing fan out events and bucket notifications * * -## Requirements - -| Repository | Name | Version | -|------------|------|---------| -| | strimzi-kafka | 1.0.0 | -| https://lsst-sqre.github.io/charts/ | strimzi-registry-operator | 2.1.0 | - ## Values | Key | Type | Default | Description | @@ -26,3 +19,49 @@ Kafka environment for prompt processing fan out events and bucket notifications | strimzi-registry-operator.clusterName | string | `"prompt-processing-kafka"` | Name of the Strimzi Kafka cluster | | strimzi-registry-operator.clusterNamespace | string | `"prompt-processing-kafka"` | Namespace where the Strimzi Kafka cluster is deployed | | strimzi-registry-operator.operatorNamespace | string | `"prompt-processing-kafka"` | Namespace where the strimzi-registry-operator is deployed | +| strimzi-kafka.cluster.monitorLabel | object | `{}` | Site wide label required for gathering Prometheus metrics if they are enabled | +| strimzi-kafka.cluster.name | string | `"prompt-processing-kafka"` | Name used for the Kafka cluster, and used by Strimzi for many annotations | +| strimzi-kafka.cruiseControl.enabled | bool | `false` | | +| strimzi-kafka.kafka.affinity | object | See `values.yaml` | Affinity for Kafka pod assignment | +| strimzi-kafka.kafka.config."log.retention.bytes" | string | `"350000000000"` | How much disk space Kafka will ensure is available, set to 70% of the data partition size | +| strimzi-kafka.kafka.config."log.retention.hours" | int | `48` | Number of days for a topic's data to be retained | +| strimzi-kafka.kafka.config."message.max.bytes" | int | `10485760` | The largest record batch size allowed by Kafka | +| strimzi-kafka.kafka.config."offsets.retention.minutes" | int | `2880` | Number of minutes for a consumer group's offsets to be retained | +| strimzi-kafka.kafka.config."replica.fetch.max.bytes" | int | `10485760` | The number of bytes of messages to attempt to fetch for each partition | +| strimzi-kafka.kafka.externalListener.bootstrap.annotations | object | `{}` | Annotations that will be added to the Ingress, Route, or Service resource | +| strimzi-kafka.kafka.externalListener.bootstrap.host | string | Do not configure TLS | Name used for TLS hostname verification | +| strimzi-kafka.kafka.externalListener.bootstrap.loadBalancerIP | string | Do not request a load balancer IP | Request this load balancer IP. See `values.yaml` for more discussion | +| strimzi-kafka.kafka.externalListener.brokers | list | `[]` | Brokers configuration. _host_ is used in the brokers' advertised.brokers configuration and for TLS hostname verification. The format is a list of maps. | +| strimzi-kafka.kafka.externalListener.tls.certIssuerName | string | `"letsencrypt-dns"` | Name of a ClusterIssuer capable of provisioning a TLS certificate for the broker | +| strimzi-kafka.kafka.externalListener.tls.enabled | bool | `false` | Whether TLS encryption is enabled | +| strimzi-kafka.kafka.listeners.external.enabled | bool | `false` | Whether external listener is enabled | +| strimzi-kafka.kafka.listeners.noauth.enabled | bool | `false` | Whether internal noauth listener is enabled | +| strimzi-kafka.kafka.listeners.plain.enabled | bool | `false` | Whether internal plaintext listener is enabled | +| strimzi-kafka.kafka.listeners.tls.enabled | bool | `false` | Whether internal TLS listener is enabled | +| strimzi-kafka.kafka.metricsConfig.enabled | bool | `false` | Whether metric configuration is enabled | +| strimzi-kafka.kafka.minInsyncReplicas | int | `2` | The minimum number of in-sync replicas that must be available for the producer to successfully send records Cannot be greater than the number of replicas. | +| strimzi-kafka.kafka.replicas | int | `3` | Number of Kafka broker replicas to run | +| strimzi-kafka.kafka.resources | object | See `values.yaml` | Kubernetes requests and limits for the Kafka brokers | +| strimzi-kafka.kafka.storage.size | string | `"500Gi"` | Size of the backing storage disk for each of the Kafka brokers | +| strimzi-kafka.kafka.storage.storageClassName | string | `""` | Name of a StorageClass to use when requesting persistent volumes | +| strimzi-kafka.kafka.tolerations | list | `[]` | Tolerations for Kafka broker pod assignment | +| strimzi-kafka.kafka.version | string | `"3.8.0"` | Version of Kafka to deploy | +| strimzi-kafka.kafkaController.enabled | bool | `false` | Enable Kafka Controller | +| strimzi-kafka.kafkaController.resources | object | See `values.yaml` | Kubernetes requests and limits for the Kafka Controller | +| strimzi-kafka.kafkaController.storage.size | string | `"20Gi"` | Size of the backing storage disk for each of the Kafka controllers | +| strimzi-kafka.kafkaController.storage.storageClassName | string | `""` | Name of a StorageClass to use when requesting persistent volumes | +| strimzi-kafka.kafkaExporter.enableSaramaLogging | bool | `false` | Enable Sarama logging for pod | +| strimzi-kafka.kafkaExporter.enabled | bool | `false` | Enable Kafka exporter | +| strimzi-kafka.kafkaExporter.groupRegex | string | `".*"` | Consumer groups to monitor | +| strimzi-kafka.kafkaExporter.logging | string | `"info"` | Logging level | +| strimzi-kafka.kafkaExporter.resources | object | See `values.yaml` | Kubernetes requests and limits for the Kafka exporter | +| strimzi-kafka.kafkaExporter.topicRegex | string | `".*"` | Kafka topics to monitor | +| strimzi-kafka.kraft.enabled | bool | `false` | Enable KRaft mode for Kafka | +| strimzi-kafka.registry.ingress.annotations | object | `{}` | Annotations that will be added to the Ingress resource | +| strimzi-kafka.registry.ingress.enabled | bool | `false` | Whether to enable an ingress for the Schema Registry | +| strimzi-kafka.registry.ingress.hostname | string | None, must be set if ingress is enabled | Hostname for the Schema Registry | +| strimzi-kafka.registry.resources | object | See `values.yaml` | Kubernetes requests and limits for the Schema Registry | +| strimzi-kafka.registry.schemaTopic | string | `"registry-schemas"` | Name of the topic used by the Schema Registry | +| strimzi-kafka.superusers | list | `["kafka-admin"]` | A list of usernames for users who should have global admin permissions. These users will be created, along with their credentials. | +| strimzi-kafka.users.kafdrop.enabled | bool | `false` | Enable user Kafdrop (deployed by parent Prompt Processing Kafka chart). | +| strimzi-kafka.users.promptProcessing.enabled | bool | `false` | Enable user prompt-processing | diff --git a/applications/strimzi/values-usdfdev-prompt-processing.yaml b/applications/strimzi/values-usdfdev-prompt-processing.yaml new file mode 100644 index 0000000000..3777b2d0eb --- /dev/null +++ b/applications/strimzi/values-usdfdev-prompt-processing.yaml @@ -0,0 +1,10 @@ +strimzi-kafka-operator: + resources: + limits: + memory: "1Gi" + requests: + memory: "512Mi" + watchNamespaces: + - "kafka" + - "prompt-processing-kafka" + logLevel: "INFO" diff --git a/environments/values-usdfdev-prompt-processing.yaml b/environments/values-usdfdev-prompt-processing.yaml index 40888107cb..ccbb612439 100644 --- a/environments/values-usdfdev-prompt-processing.yaml +++ b/environments/values-usdfdev-prompt-processing.yaml @@ -15,4 +15,5 @@ applications: prompt-proto-service-lsstcam: true prompt-proto-service-lsstcomcam: true prompt-proto-service-lsstcomcamsim: true + strimzi: true vault-secrets-operator: false From 46bf02c9a01821a118dfe76b155d8ef79e20cf12 Mon Sep 17 00:00:00 2001 From: dspeck1 Date: Tue, 15 Oct 2024 11:49:34 -0500 Subject: [PATCH 05/28] Removed strimzi node pool setting --- .../charts/strimzi-kafka/templates/kafka.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/applications/prompt-processing-kafka/charts/strimzi-kafka/templates/kafka.yaml b/applications/prompt-processing-kafka/charts/strimzi-kafka/templates/kafka.yaml index 7a28269717..a5f018c23c 100644 --- a/applications/prompt-processing-kafka/charts/strimzi-kafka/templates/kafka.yaml +++ b/applications/prompt-processing-kafka/charts/strimzi-kafka/templates/kafka.yaml @@ -6,7 +6,6 @@ metadata: {{- if .Values.kraft.enabled }} strimzi.io/kraft: enabled {{- end }} - strimzi.io/node-pools: enabled spec: kafka: template: From e6fc98a550594f8ab63e3bfef82c0097a913f7a6 Mon Sep 17 00:00:00 2001 From: dspeck1 Date: Tue, 15 Oct 2024 12:57:42 -0500 Subject: [PATCH 06/28] add secrets configuration --- applications/prompt-processing-kafka/secrets.yaml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 applications/prompt-processing-kafka/secrets.yaml diff --git a/applications/prompt-processing-kafka/secrets.yaml b/applications/prompt-processing-kafka/secrets.yaml new file mode 100644 index 0000000000..44d8764493 --- /dev/null +++ b/applications/prompt-processing-kafka/secrets.yaml @@ -0,0 +1,6 @@ +fan-out-consumer-password: + description: >- + fan out kafka consumer password used by prompt-processing +fan-out-producer-password: + description: >- + fan out kafka producer password used by fan out to create fanned out events From 811ebd3c1823e2ecd2202cf79547cc7a8f194178 Mon Sep 17 00:00:00 2001 From: dspeck1 Date: Tue, 15 Oct 2024 13:27:56 -0500 Subject: [PATCH 07/28] Add kafka node pool --- .../charts/strimzi-kafka/templates/kafka.yaml | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/applications/prompt-processing-kafka/charts/strimzi-kafka/templates/kafka.yaml b/applications/prompt-processing-kafka/charts/strimzi-kafka/templates/kafka.yaml index a5f018c23c..20e7d188de 100644 --- a/applications/prompt-processing-kafka/charts/strimzi-kafka/templates/kafka.yaml +++ b/applications/prompt-processing-kafka/charts/strimzi-kafka/templates/kafka.yaml @@ -1,4 +1,54 @@ apiVersion: kafka.strimzi.io/v1beta2 +kind: KafkaNodePool +metadata: + name: controller + labels: + strimzi.io/cluster: {{ .Values.cluster.name }} +spec: + replicas: {{ .Values.kafka.replicas }} + roles: + - controller + storage: + type: jbod + volumes: + - id: 0 + type: persistent-claim + size: {{ .Values.kafkaController.storage.size }} + class: {{ .Values.kafkaController.storage.storageClassName }} + deleteClaim: false + {{- with .Values.kafkaController.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} +--- +apiVersion: kafka.strimzi.io/v1beta2 +kind: KafkaNodePool +metadata: + name: kafka + labels: + strimzi.io/cluster: {{ .Values.cluster.name }} + annotations: + strimzi.io/next-node-ids: "[0-99]" +spec: + replicas: {{ .Values.kafka.replicas }} + roles: + - broker + storage: + type: jbod + volumes: + - id: 0 + type: persistent-claim + size: {{ .Values.kafka.storage.size }} + {{- if .Values.kafka.storage.storageClassName }} + class: {{ .Values.kafka.storage.storageClassName }} + {{- end}} + deleteClaim: false + {{- with .Values.kafka.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} +--- +apiVersion: kafka.strimzi.io/v1beta2 kind: Kafka metadata: name: {{ .Values.cluster.name }} @@ -6,6 +56,7 @@ metadata: {{- if .Values.kraft.enabled }} strimzi.io/kraft: enabled {{- end }} + strimzi.io/node-pools: enabled spec: kafka: template: From 09f5dfbca88904c0c784f53c681c2ad33ee8f643 Mon Sep 17 00:00:00 2001 From: dspeck1 Date: Tue, 15 Oct 2024 13:30:29 -0500 Subject: [PATCH 08/28] Change due to port conflict --- .../charts/strimzi-kafka/templates/kafka.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/applications/prompt-processing-kafka/charts/strimzi-kafka/templates/kafka.yaml b/applications/prompt-processing-kafka/charts/strimzi-kafka/templates/kafka.yaml index 20e7d188de..9a74372e97 100644 --- a/applications/prompt-processing-kafka/charts/strimzi-kafka/templates/kafka.yaml +++ b/applications/prompt-processing-kafka/charts/strimzi-kafka/templates/kafka.yaml @@ -81,7 +81,7 @@ spec: # internal listener without tls encryption and without authentication # used by ceph bucket notifications as authentication does not work with - name: plain - port: 9092 + port: 9091 type: internal tls: false {{- end }} From 9f4b9542397efd5088ee24a27659a0e1402430c4 Mon Sep 17 00:00:00 2001 From: dspeck1 Date: Tue, 15 Oct 2024 13:31:50 -0500 Subject: [PATCH 09/28] change port due to requirement of higher than 9092 --- .../charts/strimzi-kafka/templates/kafka.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/applications/prompt-processing-kafka/charts/strimzi-kafka/templates/kafka.yaml b/applications/prompt-processing-kafka/charts/strimzi-kafka/templates/kafka.yaml index 9a74372e97..0db88aef9e 100644 --- a/applications/prompt-processing-kafka/charts/strimzi-kafka/templates/kafka.yaml +++ b/applications/prompt-processing-kafka/charts/strimzi-kafka/templates/kafka.yaml @@ -81,7 +81,7 @@ spec: # internal listener without tls encryption and without authentication # used by ceph bucket notifications as authentication does not work with - name: plain - port: 9091 + port: 9095 type: internal tls: false {{- end }} From 404677fecf870a643dfa6890fe4fceb470978426 Mon Sep 17 00:00:00 2001 From: dspeck1 Date: Tue, 15 Oct 2024 13:32:33 -0500 Subject: [PATCH 10/28] change listener name --- .../charts/strimzi-kafka/templates/kafka.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/applications/prompt-processing-kafka/charts/strimzi-kafka/templates/kafka.yaml b/applications/prompt-processing-kafka/charts/strimzi-kafka/templates/kafka.yaml index 0db88aef9e..80a6304bef 100644 --- a/applications/prompt-processing-kafka/charts/strimzi-kafka/templates/kafka.yaml +++ b/applications/prompt-processing-kafka/charts/strimzi-kafka/templates/kafka.yaml @@ -80,7 +80,7 @@ spec: {{- if .Values.kafka.listeners.noauth.enabled }} # internal listener without tls encryption and without authentication # used by ceph bucket notifications as authentication does not work with - - name: plain + - name: noauth port: 9095 type: internal tls: false From 71c23b184b6337e5a679b65ee0892f6a9f5f8bce Mon Sep 17 00:00:00 2001 From: dspeck1 Date: Wed, 16 Oct 2024 11:42:13 -0500 Subject: [PATCH 11/28] Add authorization for additional fan out topics --- .../charts/strimzi-kafka/templates/users.yaml | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/applications/prompt-processing-kafka/charts/strimzi-kafka/templates/users.yaml b/applications/prompt-processing-kafka/charts/strimzi-kafka/templates/users.yaml index a7ac889f3e..7e9edb3f7e 100644 --- a/applications/prompt-processing-kafka/charts/strimzi-kafka/templates/users.yaml +++ b/applications/prompt-processing-kafka/charts/strimzi-kafka/templates/users.yaml @@ -22,11 +22,25 @@ spec: operation: All - resource: type: topic - name: "fanned-out-events" + name: "fanned-out-comcamsim-events" + patternType: literal + type: allow + host: "*" + operation: All + - resource: + type: topic + name: "fanned-out-hsc-events" patternType: literal type: allow host: "*" - operation: Write + operation: All + - resource: + type: topic + name: "fanned-out-latiss-events" + patternType: literal + type: allow + host: "*" + operation: All --- apiVersion: kafka.strimzi.io/v1beta2 kind: KafkaUser From ed09005a9e6e6915cffc10489f540222d3490619 Mon Sep 17 00:00:00 2001 From: dspeck1 Date: Fri, 18 Oct 2024 14:26:28 -0500 Subject: [PATCH 12/28] Add authorization for fan out topics for consumer --- .../charts/strimzi-kafka/templates/users.yaml | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/applications/prompt-processing-kafka/charts/strimzi-kafka/templates/users.yaml b/applications/prompt-processing-kafka/charts/strimzi-kafka/templates/users.yaml index 7e9edb3f7e..fe3cf777fb 100644 --- a/applications/prompt-processing-kafka/charts/strimzi-kafka/templates/users.yaml +++ b/applications/prompt-processing-kafka/charts/strimzi-kafka/templates/users.yaml @@ -66,7 +66,21 @@ spec: operation: All - resource: type: topic - name: "fanned-out-events" + name: "fanned-out-comcamsim-events" + patternType: literal + type: allow + host: "*" + operation: Read + - resource: + type: topic + name: "fanned-out-hsc-events" + patternType: literal + type: allow + host: "*" + operation: Read + - resource: + type: topic + name: "fanned-out-latiss-events" patternType: literal type: allow host: "*" From adcef72fb3806b2e591bef2d805b920ae566ba12 Mon Sep 17 00:00:00 2001 From: dspeck1 Date: Wed, 23 Oct 2024 12:30:40 -0500 Subject: [PATCH 13/28] Add kafdrop. Remove prompt processing user flag as that user is not used on this cluster. --- .../prompt-processing-kafka/README.md | 3 +- .../charts/kafdrop/Chart.yaml | 7 ++ .../charts/kafdrop/README.md | 36 +++++++ .../charts/kafdrop/templates/NOTES.txt | 9 ++ .../charts/kafdrop/templates/_helpers.tpl | 52 ++++++++++ .../charts/kafdrop/templates/deployment.yaml | 95 +++++++++++++++++++ .../charts/kafdrop/templates/ingress.yaml | 27 ++++++ .../charts/kafdrop/templates/service.yaml | 15 +++ .../charts/kafdrop/values.yaml | 93 ++++++++++++++++++ .../charts/strimzi-kafka/README.md | 3 +- .../charts/strimzi-kafka/values.yaml | 6 +- .../values-usdfdev-prompt-processing.yaml | 10 ++ 12 files changed, 347 insertions(+), 9 deletions(-) create mode 100644 applications/prompt-processing-kafka/charts/kafdrop/Chart.yaml create mode 100644 applications/prompt-processing-kafka/charts/kafdrop/README.md create mode 100644 applications/prompt-processing-kafka/charts/kafdrop/templates/NOTES.txt create mode 100644 applications/prompt-processing-kafka/charts/kafdrop/templates/_helpers.tpl create mode 100644 applications/prompt-processing-kafka/charts/kafdrop/templates/deployment.yaml create mode 100644 applications/prompt-processing-kafka/charts/kafdrop/templates/ingress.yaml create mode 100644 applications/prompt-processing-kafka/charts/kafdrop/templates/service.yaml create mode 100644 applications/prompt-processing-kafka/charts/kafdrop/values.yaml diff --git a/applications/prompt-processing-kafka/README.md b/applications/prompt-processing-kafka/README.md index 428b089a02..509ed4d613 100644 --- a/applications/prompt-processing-kafka/README.md +++ b/applications/prompt-processing-kafka/README.md @@ -63,5 +63,4 @@ Kafka environment for prompt processing fan out events and bucket notifications | strimzi-kafka.registry.resources | object | See `values.yaml` | Kubernetes requests and limits for the Schema Registry | | strimzi-kafka.registry.schemaTopic | string | `"registry-schemas"` | Name of the topic used by the Schema Registry | | strimzi-kafka.superusers | list | `["kafka-admin"]` | A list of usernames for users who should have global admin permissions. These users will be created, along with their credentials. | -| strimzi-kafka.users.kafdrop.enabled | bool | `false` | Enable user Kafdrop (deployed by parent Prompt Processing Kafka chart). | -| strimzi-kafka.users.promptProcessing.enabled | bool | `false` | Enable user prompt-processing | +| strimzi-kafka.users.kafdrop.enabled | bool | `true` | Enable user Kafdrop (deployed by parent Prompt Processing Kafka chart). | diff --git a/applications/prompt-processing-kafka/charts/kafdrop/Chart.yaml b/applications/prompt-processing-kafka/charts/kafdrop/Chart.yaml new file mode 100644 index 0000000000..09bb251780 --- /dev/null +++ b/applications/prompt-processing-kafka/charts/kafdrop/Chart.yaml @@ -0,0 +1,7 @@ +apiVersion: v2 +name: kafdrop +version: 1.0.0 +description: A subchart to deploy the Kafdrop UI for Sasquatch. +sources: + - https://github.com/obsidiandynamics/kafdrop +appVersion: 3.30.0 diff --git a/applications/prompt-processing-kafka/charts/kafdrop/README.md b/applications/prompt-processing-kafka/charts/kafdrop/README.md new file mode 100644 index 0000000000..b188d3b3c5 --- /dev/null +++ b/applications/prompt-processing-kafka/charts/kafdrop/README.md @@ -0,0 +1,36 @@ +# kafdrop + +A subchart to deploy the Kafdrop UI for Sasquatch. + +## Source Code + +* + +## Values + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| affinity | object | `{}` | Affinity configuration | +| cmdArgs | string | See `values.yaml` | Command line arguments to Kafdrop | +| existingSecret | string | Do not use a secret | Existing Kubernetes secrect use to set kafdrop environment variables. Set `SCHEMAREGISTRY_AUTH` for basic auth credentials in the form `:` | +| host | string | `"localhost"` | The hostname to report for the RMI registry (used for JMX) | +| image.pullPolicy | string | `"IfNotPresent"` | Image pull policy | +| image.repository | string | `"obsidiandynamics/kafdrop"` | Kafdrop Docker image repository | +| image.tag | string | `"4.0.2"` | Kafdrop image version | +| ingress.annotations | object | `{}` | Additional ingress annotations | +| ingress.enabled | bool | `false` | Whether to enable the ingress | +| ingress.hostname | string | None, must be set if ingress is enabled | Ingress hostname | +| ingress.path | string | `"/kafdrop"` | Ingress path | +| jmx.port | int | `8686` | Port to use for JMX. If unspecified, JMX will not be exposed. | +| jvm.opts | string | `""` | JVM options | +| kafka.broker | string | `""` | Bootstrap list of Kafka host/port pairs | +| nodeSelector | object | `{}` | Node selector configuration | +| podAnnotations | object | `{}` | Pod annotations | +| replicaCount | int | `1` | Number of kafdrop pods to run in the deployment. | +| resources | object | See `values.yaml` | Kubernetes requests and limits for Kafdrop | +| schemaregistry | string | `""` | The endpoint of Schema Registry | +| server.port | int | `9000` | The web server port to listen on | +| server.servlet.contextPath | string | `"/kafdrop"` | The context path to serve requests on | +| service.annotations | object | `{}` | Additional annotations to add to the service | +| service.port | int | `9000` | Service port | +| tolerations | list | `[]` | Tolerations configuration | diff --git a/applications/prompt-processing-kafka/charts/kafdrop/templates/NOTES.txt b/applications/prompt-processing-kafka/charts/kafdrop/templates/NOTES.txt new file mode 100644 index 0000000000..b6244d54c8 --- /dev/null +++ b/applications/prompt-processing-kafka/charts/kafdrop/templates/NOTES.txt @@ -0,0 +1,9 @@ +1. Get the application URL by running these commands: +{{- if .Values.ingress.enabled }} +{{- range .Values.ingress.hosts }} + http{{ if $.Values.ingress.tls }}s{{ end }}://{{ . }}{{ $.Values.ingress.path }} +{{- end }} + export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "chart.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") + echo "Visit http://127.0.0.1:8080 to use your application" + kubectl port-forward $POD_NAME 8080:80 +{{- end }} diff --git a/applications/prompt-processing-kafka/charts/kafdrop/templates/_helpers.tpl b/applications/prompt-processing-kafka/charts/kafdrop/templates/_helpers.tpl new file mode 100644 index 0000000000..ffeac36252 --- /dev/null +++ b/applications/prompt-processing-kafka/charts/kafdrop/templates/_helpers.tpl @@ -0,0 +1,52 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "chart.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "chart.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "chart.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Common labels +*/}} +{{- define "kafdrop.labels" -}} +helm.sh/chart: {{ include "chart.name" . }} +{{ include "kafdrop.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "kafdrop.selectorLabels" -}} +app.kubernetes.io/name: {{ include "chart.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} diff --git a/applications/prompt-processing-kafka/charts/kafdrop/templates/deployment.yaml b/applications/prompt-processing-kafka/charts/kafdrop/templates/deployment.yaml new file mode 100644 index 0000000000..cbd51dedc4 --- /dev/null +++ b/applications/prompt-processing-kafka/charts/kafdrop/templates/deployment.yaml @@ -0,0 +1,95 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "chart.fullname" . }} + labels: + {{- include "kafdrop.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + {{- include "kafdrop.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + {{- include "kafdrop.selectorLabels" . | nindent 8 }} + annotations: + {{- with .Values.podAnnotations }} + {{ toYaml . | indent 8 }} + {{- end }} + spec: + automountServiceAccountToken: false + securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + containers: + - name: {{ .Chart.Name }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + {{- if .Values.existingSecret -}} + envFrom: + - secretRef: + name: {{ .Values.existingSecret }} + {{- end }} + env: + - name: KAFKA_BROKERCONNECT + value: {{ .Values.kafka.broker | quote }} + - name: JVM_OPTS + value: {{ .Values.jvm.opts | quote }} + - name: HOST + value: {{ .Values.host | quote }} + - name: JMX_PORT + value: {{ .Values.jmx.port | quote }} + - name: SERVER_SERVLET_CONTEXTPATH + value: {{ .Values.server.servlet.contextPath | trimSuffix "/" | quote }} + - name: SERVER_PORT + value: {{ .Values.server.port | quote }} + - name: CMD_ARGS + value: {{ .Values.cmdArgs | quote }} + - name: SCHEMAREGISTRY_CONNECT + value: {{ .Values.schemaregistry | quote }} + - name: KAFKA_PROPERTIES_FILE + value: "/tmp/kafka.properties" + - name: KAFKA_PROPERTIES + valueFrom: + secretKeyRef: + name: sasquatch + key: kafdrop-kafka-properties + ports: + - name: http + containerPort: {{ .Values.server.port }} + protocol: TCP + livenessProbe: + httpGet: + path: "{{ .Values.server.servlet.contextPath | trimSuffix "/" }}/actuator/health" + port: http + initialDelaySeconds: 180 + periodSeconds: 30 + timeoutSeconds: 10 + readinessProbe: + httpGet: + path: "{{ .Values.server.servlet.contextPath | trimSuffix "/" }}/actuator/health" + port: http + initialDelaySeconds: 20 + periodSeconds: 5 + timeoutSeconds: 10 + resources: + {{- toYaml .Values.resources | nindent 12 }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{ toYaml . | indent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{ toYaml . | indent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{ toYaml . | indent 8 }} + {{- end }} diff --git a/applications/prompt-processing-kafka/charts/kafdrop/templates/ingress.yaml b/applications/prompt-processing-kafka/charts/kafdrop/templates/ingress.yaml new file mode 100644 index 0000000000..7305fbf063 --- /dev/null +++ b/applications/prompt-processing-kafka/charts/kafdrop/templates/ingress.yaml @@ -0,0 +1,27 @@ +{{- if .Values.ingress.enabled -}} +{{- $fullName := include "chart.fullname" . -}} +{{- $ingressPath := .Values.ingress.path -}} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: {{ $fullName }} + labels: + {{- include "kafdrop.labels" . | nindent 4 }} + annotations: + {{- with .Values.ingress.annotations }} + {{ toYaml . | indent 4 }} + {{- end }} +spec: + ingressClassName: "nginx" + rules: + - host: {{ .Values.ingress.hostname | quote }} + http: + paths: + - path: {{ $ingressPath }} + pathType: Prefix + backend: + service: + name: {{ $fullName }} + port: + number: {{ .Values.service.port }} +{{- end }} diff --git a/applications/prompt-processing-kafka/charts/kafdrop/templates/service.yaml b/applications/prompt-processing-kafka/charts/kafdrop/templates/service.yaml new file mode 100644 index 0000000000..720eb65e0d --- /dev/null +++ b/applications/prompt-processing-kafka/charts/kafdrop/templates/service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "chart.fullname" . }} + labels: + {{- include "kafdrop.labels" . | nindent 4 }} +spec: + type: ClusterIP + ports: + - port: {{ .Values.service.port }} + targetPort: http + protocol: TCP + name: http + selector: + {{- include "kafdrop.selectorLabels" . | nindent 4 }} diff --git a/applications/prompt-processing-kafka/charts/kafdrop/values.yaml b/applications/prompt-processing-kafka/charts/kafdrop/values.yaml new file mode 100644 index 0000000000..74056f8f84 --- /dev/null +++ b/applications/prompt-processing-kafka/charts/kafdrop/values.yaml @@ -0,0 +1,93 @@ +# Default values for Kafdrop + +# -- Number of kafdrop pods to run in the deployment. +replicaCount: 1 + +image: + # -- Kafdrop Docker image repository + repository: obsidiandynamics/kafdrop + + # -- Image pull policy + pullPolicy: IfNotPresent + + # -- Kafdrop image version + tag: 4.0.2 + +kafka: + # -- Bootstrap list of Kafka host/port pairs + broker: "" + +jvm: + # -- JVM options + opts: "" + +# -- The hostname to report for the RMI registry (used for JMX) +host: localhost + +jmx: + # -- Port to use for JMX. If unspecified, JMX will not be exposed. + port: 8686 + +server: + servlet: + # -- The context path to serve requests on + contextPath: /kafdrop + + # -- The web server port to listen on + port: 9000 + +# -- The endpoint of Schema Registry +schemaregistry: "" + +# -- Existing Kubernetes secrect use to set kafdrop environment variables. +# Set `SCHEMAREGISTRY_AUTH` for basic auth credentials in the form +# `:` +# @default -- Do not use a secret +existingSecret: "" + +# -- Command line arguments to Kafdrop +# @default -- See `values.yaml` +cmdArgs: "--message.format=AVRO --topic.deleteEnabled=false --topic.createEnabled=false" + +service: + # -- Additional annotations to add to the service + annotations: {} + + # -- Service port + port: 9000 + +ingress: + # -- Whether to enable the ingress + enabled: false + + # -- Additional ingress annotations + annotations: {} + + # -- Ingress hostname + # @default -- None, must be set if ingress is enabled + hostname: "" + + # -- Ingress path + path: /kafdrop + +# -- Kubernetes requests and limits for Kafdrop +# @default -- See `values.yaml` +resources: + requests: + memory: 200Mi + cpu: 1 + limits: + memory: 4Gi + cpu: 2 + +# -- Node selector configuration +nodeSelector: {} + +# -- Tolerations configuration +tolerations: [] + +# -- Affinity configuration +affinity: {} + +# -- Pod annotations +podAnnotations: {} diff --git a/applications/prompt-processing-kafka/charts/strimzi-kafka/README.md b/applications/prompt-processing-kafka/charts/strimzi-kafka/README.md index 7e00d9d4dd..0df8166c2c 100644 --- a/applications/prompt-processing-kafka/charts/strimzi-kafka/README.md +++ b/applications/prompt-processing-kafka/charts/strimzi-kafka/README.md @@ -50,5 +50,4 @@ A subchart to deploy Strimzi Kafka components for Prompt Processing. | registry.resources | object | See `values.yaml` | Kubernetes requests and limits for the Schema Registry | | registry.schemaTopic | string | `"registry-schemas"` | Name of the topic used by the Schema Registry | | superusers | list | `["kafka-admin"]` | A list of usernames for users who should have global admin permissions. These users will be created, along with their credentials. | -| users.kafdrop.enabled | bool | `false` | Enable user Kafdrop (deployed by parent Prompt Processing Kafka chart). | -| users.promptProcessing.enabled | bool | `false` | Enable user prompt-processing | +| users.kafdrop.enabled | bool | `true` | Enable user Kafdrop (deployed by parent Prompt Processing Kafka chart). | diff --git a/applications/prompt-processing-kafka/charts/strimzi-kafka/values.yaml b/applications/prompt-processing-kafka/charts/strimzi-kafka/values.yaml index 2929061915..a36d2115b7 100644 --- a/applications/prompt-processing-kafka/charts/strimzi-kafka/values.yaml +++ b/applications/prompt-processing-kafka/charts/strimzi-kafka/values.yaml @@ -232,8 +232,4 @@ superusers: users: kafdrop: # -- Enable user Kafdrop (deployed by parent Prompt Processing Kafka chart). - enabled: false - - promptProcessing: - # -- Enable user prompt-processing - enabled: false + enabled: true diff --git a/applications/prompt-processing-kafka/values-usdfdev-prompt-processing.yaml b/applications/prompt-processing-kafka/values-usdfdev-prompt-processing.yaml index 5448eb667f..259b470bed 100644 --- a/applications/prompt-processing-kafka/values-usdfdev-prompt-processing.yaml +++ b/applications/prompt-processing-kafka/values-usdfdev-prompt-processing.yaml @@ -6,3 +6,13 @@ strimzi-kafka: enabled: true kafkaController: enabled: true + kafka: + storage: + size: 100Gi + storageClassName: wekafs--sdf-k8s01 +kafdrop: + ingress: + enabled: false + kafka: + broker: 10.103.195.233:9092 + schemaregistry: http://10.104.75.248:8081 From 66b08ef03788e2a057535fa2e5ffe0011a3de093 Mon Sep 17 00:00:00 2001 From: dspeck1 Date: Wed, 23 Oct 2024 12:33:45 -0500 Subject: [PATCH 14/28] Add authorization for fan out topic. --- .../charts/strimzi-kafka/templates/users.yaml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/applications/prompt-processing-kafka/charts/strimzi-kafka/templates/users.yaml b/applications/prompt-processing-kafka/charts/strimzi-kafka/templates/users.yaml index fe3cf777fb..535a51d748 100644 --- a/applications/prompt-processing-kafka/charts/strimzi-kafka/templates/users.yaml +++ b/applications/prompt-processing-kafka/charts/strimzi-kafka/templates/users.yaml @@ -41,6 +41,13 @@ spec: type: allow host: "*" operation: All + - resource: + type: topic + name: "fanned-out-latiss-events-2" + patternType: literal + type: allow + host: "*" + operation: All --- apiVersion: kafka.strimzi.io/v1beta2 kind: KafkaUser @@ -85,6 +92,13 @@ spec: type: allow host: "*" operation: Read + - resource: + type: topic + name: "fanned-out-latiss-events-2" + patternType: literal + type: allow + host: "*" + operation: Read {{- if .Values.users.kafdrop.enabled }} --- apiVersion: kafka.strimzi.io/v1beta2 From 466fcc0634329f7cb45d1a1e2679da0a759eaa0a Mon Sep 17 00:00:00 2001 From: dspeck1 Date: Wed, 23 Oct 2024 12:38:12 -0500 Subject: [PATCH 15/28] update secret name for kafdrop. --- .../charts/kafdrop/templates/deployment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/applications/prompt-processing-kafka/charts/kafdrop/templates/deployment.yaml b/applications/prompt-processing-kafka/charts/kafdrop/templates/deployment.yaml index cbd51dedc4..99c520975e 100644 --- a/applications/prompt-processing-kafka/charts/kafdrop/templates/deployment.yaml +++ b/applications/prompt-processing-kafka/charts/kafdrop/templates/deployment.yaml @@ -59,7 +59,7 @@ spec: - name: KAFKA_PROPERTIES valueFrom: secretKeyRef: - name: sasquatch + name: prompt-processing-kafka-secret key: kafdrop-kafka-properties ports: - name: http From f73cf63bebd0e96d84aff86890836240f44f4f88 Mon Sep 17 00:00:00 2001 From: dspeck1 Date: Wed, 23 Oct 2024 13:00:34 -0500 Subject: [PATCH 16/28] Enable kafdrop user --- applications/prompt-processing-kafka/README.md | 2 +- .../prompt-processing-kafka/charts/strimzi-kafka/README.md | 2 +- .../prompt-processing-kafka/charts/strimzi-kafka/values.yaml | 2 +- .../values-usdfdev-prompt-processing.yaml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/applications/prompt-processing-kafka/README.md b/applications/prompt-processing-kafka/README.md index 509ed4d613..48cde6b00a 100644 --- a/applications/prompt-processing-kafka/README.md +++ b/applications/prompt-processing-kafka/README.md @@ -63,4 +63,4 @@ Kafka environment for prompt processing fan out events and bucket notifications | strimzi-kafka.registry.resources | object | See `values.yaml` | Kubernetes requests and limits for the Schema Registry | | strimzi-kafka.registry.schemaTopic | string | `"registry-schemas"` | Name of the topic used by the Schema Registry | | strimzi-kafka.superusers | list | `["kafka-admin"]` | A list of usernames for users who should have global admin permissions. These users will be created, along with their credentials. | -| strimzi-kafka.users.kafdrop.enabled | bool | `true` | Enable user Kafdrop (deployed by parent Prompt Processing Kafka chart). | +| strimzi-kafka.users.kafdrop.enabled | bool | `false` | Enable user Kafdrop (deployed by parent Prompt Processing Kafka chart). | diff --git a/applications/prompt-processing-kafka/charts/strimzi-kafka/README.md b/applications/prompt-processing-kafka/charts/strimzi-kafka/README.md index 0df8166c2c..612279a7f2 100644 --- a/applications/prompt-processing-kafka/charts/strimzi-kafka/README.md +++ b/applications/prompt-processing-kafka/charts/strimzi-kafka/README.md @@ -50,4 +50,4 @@ A subchart to deploy Strimzi Kafka components for Prompt Processing. | registry.resources | object | See `values.yaml` | Kubernetes requests and limits for the Schema Registry | | registry.schemaTopic | string | `"registry-schemas"` | Name of the topic used by the Schema Registry | | superusers | list | `["kafka-admin"]` | A list of usernames for users who should have global admin permissions. These users will be created, along with their credentials. | -| users.kafdrop.enabled | bool | `true` | Enable user Kafdrop (deployed by parent Prompt Processing Kafka chart). | +| users.kafdrop.enabled | bool | `false` | Enable user Kafdrop (deployed by parent Prompt Processing Kafka chart). | diff --git a/applications/prompt-processing-kafka/charts/strimzi-kafka/values.yaml b/applications/prompt-processing-kafka/charts/strimzi-kafka/values.yaml index a36d2115b7..5de8a4360f 100644 --- a/applications/prompt-processing-kafka/charts/strimzi-kafka/values.yaml +++ b/applications/prompt-processing-kafka/charts/strimzi-kafka/values.yaml @@ -232,4 +232,4 @@ superusers: users: kafdrop: # -- Enable user Kafdrop (deployed by parent Prompt Processing Kafka chart). - enabled: true + enabled: false diff --git a/applications/prompt-processing-kafka/values-usdfdev-prompt-processing.yaml b/applications/prompt-processing-kafka/values-usdfdev-prompt-processing.yaml index 259b470bed..9be5a47ba1 100644 --- a/applications/prompt-processing-kafka/values-usdfdev-prompt-processing.yaml +++ b/applications/prompt-processing-kafka/values-usdfdev-prompt-processing.yaml @@ -1,7 +1,7 @@ strimzi-kafka: users: kafdrop: - enabled: false + enabled: true kraft: enabled: true kafkaController: From fb84fc8defbe9acefad598d27ad9c9f70308aaa4 Mon Sep 17 00:00:00 2001 From: dspeck1 Date: Wed, 23 Oct 2024 13:05:10 -0500 Subject: [PATCH 17/28] Update secret path --- .../charts/strimzi-kafka/templates/users.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/applications/prompt-processing-kafka/charts/strimzi-kafka/templates/users.yaml b/applications/prompt-processing-kafka/charts/strimzi-kafka/templates/users.yaml index 535a51d748..16a37c8d2d 100644 --- a/applications/prompt-processing-kafka/charts/strimzi-kafka/templates/users.yaml +++ b/applications/prompt-processing-kafka/charts/strimzi-kafka/templates/users.yaml @@ -10,7 +10,7 @@ spec: password: valueFrom: secretKeyRef: - name: prompt-processing-kafka + name: prompt-processing-kafka-secret key: fan-out-producer-password authorization: type: simple @@ -61,7 +61,7 @@ spec: password: valueFrom: secretKeyRef: - name: prompt-processing-kafka + name: prompt-processing-kafka-secret key: fan-out-consumer-password authorization: type: simple @@ -113,7 +113,7 @@ spec: password: valueFrom: secretKeyRef: - name: prompt-processing-kafka + name: prompt-processing-kafka-secret key: kafdrop-password authorization: type: simple From dcd5ccc4f260298d21568773871644165c0cbaef Mon Sep 17 00:00:00 2001 From: dspeck1 Date: Wed, 23 Oct 2024 15:23:06 -0500 Subject: [PATCH 18/28] Add fanned out events topic configuration. --- .../prompt-processing-kafka/README.md | 20 ++++++ .../charts/strimzi-kafka/README.md | 20 ++++++ .../strimzi-kafka/templates/topics.yaml | 70 +++++++++++++++++++ .../charts/strimzi-kafka/values.yaml | 47 +++++++++++++ .../values-usdfdev-prompt-processing.yaml | 11 +++ 5 files changed, 168 insertions(+) create mode 100644 applications/prompt-processing-kafka/charts/strimzi-kafka/templates/topics.yaml diff --git a/applications/prompt-processing-kafka/README.md b/applications/prompt-processing-kafka/README.md index 48cde6b00a..90ceae29d3 100644 --- a/applications/prompt-processing-kafka/README.md +++ b/applications/prompt-processing-kafka/README.md @@ -63,4 +63,24 @@ Kafka environment for prompt processing fan out events and bucket notifications | strimzi-kafka.registry.resources | object | See `values.yaml` | Kubernetes requests and limits for the Schema Registry | | strimzi-kafka.registry.schemaTopic | string | `"registry-schemas"` | Name of the topic used by the Schema Registry | | strimzi-kafka.superusers | list | `["kafka-admin"]` | A list of usernames for users who should have global admin permissions. These users will be created, along with their credentials. | +| strimzi-kafka.topics | object | `{"hsc":{"enabled":false,"partitions":1,"replicas":1,"retention":3600000},"latiss":{"enabled":false,"partitions":1,"replicas":1,"retention":3600000},"lsstcam":{"enabled":false,"partitions":1,"replicas":1,"retention":3600000},"lsstcomcam":{"enabled":false,"partitions":1,"replicas":1,"retention":3600000},"lsstcomcamsim":{"enabled":false,"partitions":1,"replicas":1,"retention":3600000}}` | Topic configuration. Enable for supporting certain instruments. | +| strimzi-kafka.topics.hsc.enabled | bool | `false` | Enable hsc topic | +| strimzi-kafka.topics.hsc.partitions | int | `1` | Number of partitions on topic | +| strimzi-kafka.topics.hsc.replicas | int | `1` | Number of replicas | +| strimzi-kafka.topics.latiss.enabled | bool | `false` | Enable latiss topic | +| strimzi-kafka.topics.latiss.partitions | int | `1` | Number of partitions on topic | +| strimzi-kafka.topics.latiss.replicas | int | `1` | Number of replicas | +| strimzi-kafka.topics.latiss.retention | int | `3600000` | Retention time of events in milliseconds | +| strimzi-kafka.topics.lsstcam.enabled | bool | `false` | Enable lsstcam topic | +| strimzi-kafka.topics.lsstcam.partitions | int | `1` | Number of partitions on topic | +| strimzi-kafka.topics.lsstcam.replicas | int | `1` | Number of replicas | +| strimzi-kafka.topics.lsstcam.retention | int | `3600000` | Retention time of events in milliseconds | +| strimzi-kafka.topics.lsstcomcam.enabled | bool | `false` | Enable lsstcomcam topic | +| strimzi-kafka.topics.lsstcomcam.partitions | int | `1` | Number of partitions on topic | +| strimzi-kafka.topics.lsstcomcam.replicas | int | `1` | Number of replicas | +| strimzi-kafka.topics.lsstcomcam.retention | int | `3600000` | Retention time of events in milliseconds | +| strimzi-kafka.topics.lsstcomcamsim.enabled | bool | `false` | Enable lsstcomcamsim topic | +| strimzi-kafka.topics.lsstcomcamsim.partitions | int | `1` | Number of partitions on topic | +| strimzi-kafka.topics.lsstcomcamsim.replicas | int | `1` | Number of replicas | +| strimzi-kafka.topics.lsstcomcamsim.retention | int | `3600000` | Retention time of events in milliseconds | | strimzi-kafka.users.kafdrop.enabled | bool | `false` | Enable user Kafdrop (deployed by parent Prompt Processing Kafka chart). | diff --git a/applications/prompt-processing-kafka/charts/strimzi-kafka/README.md b/applications/prompt-processing-kafka/charts/strimzi-kafka/README.md index 612279a7f2..e369ab8c31 100644 --- a/applications/prompt-processing-kafka/charts/strimzi-kafka/README.md +++ b/applications/prompt-processing-kafka/charts/strimzi-kafka/README.md @@ -50,4 +50,24 @@ A subchart to deploy Strimzi Kafka components for Prompt Processing. | registry.resources | object | See `values.yaml` | Kubernetes requests and limits for the Schema Registry | | registry.schemaTopic | string | `"registry-schemas"` | Name of the topic used by the Schema Registry | | superusers | list | `["kafka-admin"]` | A list of usernames for users who should have global admin permissions. These users will be created, along with their credentials. | +| topics | object | `{"hsc":{"enabled":false,"partitions":1,"replicas":1,"retention":3600000},"latiss":{"enabled":false,"partitions":1,"replicas":1,"retention":3600000},"lsstcam":{"enabled":false,"partitions":1,"replicas":1,"retention":3600000},"lsstcomcam":{"enabled":false,"partitions":1,"replicas":1,"retention":3600000},"lsstcomcamsim":{"enabled":false,"partitions":1,"replicas":1,"retention":3600000}}` | Topic configuration. Enable for supporting certain instruments. | +| topics.hsc.enabled | bool | `false` | Enable hsc topic | +| topics.hsc.partitions | int | `1` | Number of partitions on topic | +| topics.hsc.replicas | int | `1` | Number of replicas | +| topics.latiss.enabled | bool | `false` | Enable latiss topic | +| topics.latiss.partitions | int | `1` | Number of partitions on topic | +| topics.latiss.replicas | int | `1` | Number of replicas | +| topics.latiss.retention | int | `3600000` | Retention time of events in milliseconds | +| topics.lsstcam.enabled | bool | `false` | Enable lsstcam topic | +| topics.lsstcam.partitions | int | `1` | Number of partitions on topic | +| topics.lsstcam.replicas | int | `1` | Number of replicas | +| topics.lsstcam.retention | int | `3600000` | Retention time of events in milliseconds | +| topics.lsstcomcam.enabled | bool | `false` | Enable lsstcomcam topic | +| topics.lsstcomcam.partitions | int | `1` | Number of partitions on topic | +| topics.lsstcomcam.replicas | int | `1` | Number of replicas | +| topics.lsstcomcam.retention | int | `3600000` | Retention time of events in milliseconds | +| topics.lsstcomcamsim.enabled | bool | `false` | Enable lsstcomcamsim topic | +| topics.lsstcomcamsim.partitions | int | `1` | Number of partitions on topic | +| topics.lsstcomcamsim.replicas | int | `1` | Number of replicas | +| topics.lsstcomcamsim.retention | int | `3600000` | Retention time of events in milliseconds | | users.kafdrop.enabled | bool | `false` | Enable user Kafdrop (deployed by parent Prompt Processing Kafka chart). | diff --git a/applications/prompt-processing-kafka/charts/strimzi-kafka/templates/topics.yaml b/applications/prompt-processing-kafka/charts/strimzi-kafka/templates/topics.yaml new file mode 100644 index 0000000000..d2397f8e9c --- /dev/null +++ b/applications/prompt-processing-kafka/charts/strimzi-kafka/templates/topics.yaml @@ -0,0 +1,70 @@ +{{- if .Values.topics.hsc.enabled }} +apiVersion: kafka.strimzi.io/v1beta2 +kind: KafkaTopic +metadata: + name: fanned-out-hsc-events + labels: + strimzi.io/cluster: {{ .Values.cluster.name }} +spec: + partitions: {{ .Values.topics.hsc.partitions }} + replicas: {{ .Values.topics.hsc.replicas }} + config: + retention.ms: {{ .Values.topics.hsc.retention }} +{{- end }} +--- +{{- if .Values.topics.latiss.enabled }} +apiVersion: kafka.strimzi.io/v1beta2 +kind: KafkaTopic +metadata: + name: fanned-out-latiss-events + labels: + strimzi.io/cluster: {{ .Values.cluster.name }} +spec: + partitions: {{ .Values.topics.latiss.partitions }} + replicas: {{ .Values.topics.latiss.replicas }} + config: + retention.ms: {{ .Values.topics.latiss.retention }} + {{- end }} +--- +{{- if .Values.topics.lsstcam.enabled }} +apiVersion: kafka.strimzi.io/v1beta2 +kind: KafkaTopic +metadata: + name: fanned-out-lsstcam-events + labels: + strimzi.io/cluster: {{ .Values.cluster.name }} +spec: + partitions: {{ .Values.topics.lsstcam.partitions }} + replicas: {{ .Values.topics.lsstcam.replicas }} + config: + retention.ms: {{ .Values.topics.lsstcam.retention }} + {{- end }} +--- +{{- if .Values.topics.lsstcomcam.enabled }} +apiVersion: kafka.strimzi.io/v1beta2 +kind: KafkaTopic +metadata: + name: fanned-out-lsstcomcam-events + labels: + strimzi.io/cluster: {{ .Values.cluster.name }} +spec: + partitions: {{ .Values.topics.lsstcomcam.partitions }} + replicas: {{ .Values.topics.lsstcomcam.replicas }} + config: + retention.ms: {{ .Values.topics.lsstcomcam.retention }} + {{- end }} +--- +{{- if .Values.topics.lsstcomcamsim.enabled }} +apiVersion: kafka.strimzi.io/v1beta2 +kind: KafkaTopic +metadata: + name: fanned-out-lsstcomcamsim-events + labels: + strimzi.io/cluster: {{ .Values.cluster.name }} +spec: + partitions: {{ .Values.topics.lsstcomcamsim.partitions }} + replicas: {{ .Values.topics.lsstcomcamsim.replicas }} + config: + retention.ms: {{ .Values.topics.lsstcomcamsimretention }} + {{- end }} +--- \ No newline at end of file diff --git a/applications/prompt-processing-kafka/charts/strimzi-kafka/values.yaml b/applications/prompt-processing-kafka/charts/strimzi-kafka/values.yaml index 5de8a4360f..973a7b6cca 100644 --- a/applications/prompt-processing-kafka/charts/strimzi-kafka/values.yaml +++ b/applications/prompt-processing-kafka/charts/strimzi-kafka/values.yaml @@ -229,6 +229,53 @@ registry: superusers: - kafka-admin +# -- Topic configuration. Enable for supporting certain instruments. +topics: + hsc: + # -- Enable hsc topic + enabled: false + # -- Number of partitions on topic + partitions: 1 + # -- Number of replicas + replicas: 1 + retention: 3600000 + latiss: + # -- Enable latiss topic + enabled: false + # -- Number of partitions on topic + partitions: 1 + # -- Number of replicas + replicas: 1 + # -- Retention time of events in milliseconds + retention: 3600000 + lsstcam: + # -- Enable lsstcam topic + enabled: false + # -- Number of partitions on topic + partitions: 1 + # -- Number of replicas + replicas: 1 + # -- Retention time of events in milliseconds + retention: 3600000 + lsstcomcam: + # -- Enable lsstcomcam topic + enabled: false + # -- Number of partitions on topic + partitions: 1 + # -- Number of replicas + replicas: 1 + # -- Retention time of events in milliseconds + retention: 3600000 + lsstcomcamsim: + # -- Enable lsstcomcamsim topic + enabled: false + # -- Number of partitions on topic + partitions: 1 + # -- Number of replicas + replicas: 1 + # -- Retention time of events in milliseconds + retention: 3600000 + users: kafdrop: # -- Enable user Kafdrop (deployed by parent Prompt Processing Kafka chart). diff --git a/applications/prompt-processing-kafka/values-usdfdev-prompt-processing.yaml b/applications/prompt-processing-kafka/values-usdfdev-prompt-processing.yaml index 9be5a47ba1..6e1a953a5a 100644 --- a/applications/prompt-processing-kafka/values-usdfdev-prompt-processing.yaml +++ b/applications/prompt-processing-kafka/values-usdfdev-prompt-processing.yaml @@ -10,6 +10,17 @@ strimzi-kafka: storage: size: 100Gi storageClassName: wekafs--sdf-k8s01 + topics: + hsc: + enabled: true + partitions: 200 + replicas: 1 + retention: 3600000 + latiss: + enabled: true + partitions: 200 + replicas: 1 + retention: 3600000 kafdrop: ingress: enabled: false From 135fbbb9971c84ba36e87ddde885e05a60b3228e Mon Sep 17 00:00:00 2001 From: dspeck1 Date: Wed, 23 Oct 2024 15:28:16 -0500 Subject: [PATCH 19/28] Adjust replicas on topics. --- .../values-usdfdev-prompt-processing.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/applications/prompt-processing-kafka/values-usdfdev-prompt-processing.yaml b/applications/prompt-processing-kafka/values-usdfdev-prompt-processing.yaml index 6e1a953a5a..2f8592781d 100644 --- a/applications/prompt-processing-kafka/values-usdfdev-prompt-processing.yaml +++ b/applications/prompt-processing-kafka/values-usdfdev-prompt-processing.yaml @@ -14,12 +14,12 @@ strimzi-kafka: hsc: enabled: true partitions: 200 - replicas: 1 + replicas: 3 retention: 3600000 latiss: enabled: true partitions: 200 - replicas: 1 + replicas: 3 retention: 3600000 kafdrop: ingress: From c6c135e195d9d256556520e96c1172a1182fb2f3 Mon Sep 17 00:00:00 2001 From: dspeck1 Date: Mon, 2 Dec 2024 13:10:01 -0600 Subject: [PATCH 20/28] Add prod and additional instruments. --- .../charts/strimzi-kafka/templates/users.yaml | 51 +++++++++++++++++-- .../values-usdfdev-prompt-processing.yaml | 23 +++++++-- .../values-usdfprod-prompt-processing.yaml | 18 +++++++ .../values-usdfprod-prompt-processing.yaml | 1 + 4 files changed, 84 insertions(+), 9 deletions(-) create mode 100644 applications/prompt-processing-kafka/values-usdfprod-prompt-processing.yaml diff --git a/applications/prompt-processing-kafka/charts/strimzi-kafka/templates/users.yaml b/applications/prompt-processing-kafka/charts/strimzi-kafka/templates/users.yaml index 16a37c8d2d..9e095a18d5 100644 --- a/applications/prompt-processing-kafka/charts/strimzi-kafka/templates/users.yaml +++ b/applications/prompt-processing-kafka/charts/strimzi-kafka/templates/users.yaml @@ -20,34 +20,51 @@ spec: name: "*" patternType: literal operation: All + {{- if .Values.topics.hsc.enabled }} - resource: type: topic - name: "fanned-out-comcamsim-events" + name: "fanned-out-hsc-events" patternType: literal type: allow host: "*" operation: All + {{- end }} + {{- if .Values.topics.latiss.enabled }} - resource: type: topic - name: "fanned-out-hsc-events" + name: "fanned-out-latiss-events" patternType: literal type: allow host: "*" operation: All + {{- end }} + {{- if .Values.topics.lsstcam.enabled }} - resource: type: topic - name: "fanned-out-latiss-events" + name: "fanned-out-lsstcam-events" + patternType: literal + type: allow + host: "*" + operation: All + {{- end }} + {{- if .Values.topics.lsstcomcam.enabled }} + - resource: + type: topic + name: "fanned-out-lsstcomcam-events" patternType: literal type: allow host: "*" operation: All + {{- end }} + {{- if .Values.topics.lsstcomcamsim.enabled }} - resource: type: topic - name: "fanned-out-latiss-events-2" + name: "fanned-out-lsstcomcamsim-events" patternType: literal type: allow host: "*" operation: All + {{- end }} --- apiVersion: kafka.strimzi.io/v1beta2 kind: KafkaUser @@ -78,6 +95,7 @@ spec: type: allow host: "*" operation: Read + {{- if .Values.topics.hsc.enabled }} - resource: type: topic name: "fanned-out-hsc-events" @@ -85,6 +103,8 @@ spec: type: allow host: "*" operation: Read + {{- end }} + {{- if .Values.topics.latiss.enabled }} - resource: type: topic name: "fanned-out-latiss-events" @@ -92,13 +112,34 @@ spec: type: allow host: "*" operation: Read + {{- end }} + {{- if .Values.topics.lsstcam.enabled }} + - resource: + type: topic + name: "fanned-out-lsstcam-events" + patternType: literal + type: allow + host: "*" + operation: Read + {{- end }} + {{- if .Values.topics.lsstcomcam.enabled }} + - resource: + type: topic + name: "fanned-out-lsstcomcam-events" + patternType: literal + type: allow + host: "*" + operation: Read + {{- end }} + {{- if .Values.topics.lsstcomcamsim.enabled }} - resource: type: topic - name: "fanned-out-latiss-events-2" + name: "fanned-out-lsstcomcamsim-events" patternType: literal type: allow host: "*" operation: Read + {{- end }} {{- if .Values.users.kafdrop.enabled }} --- apiVersion: kafka.strimzi.io/v1beta2 diff --git a/applications/prompt-processing-kafka/values-usdfdev-prompt-processing.yaml b/applications/prompt-processing-kafka/values-usdfdev-prompt-processing.yaml index 2f8592781d..67d02b9216 100644 --- a/applications/prompt-processing-kafka/values-usdfdev-prompt-processing.yaml +++ b/applications/prompt-processing-kafka/values-usdfdev-prompt-processing.yaml @@ -13,14 +13,29 @@ strimzi-kafka: topics: hsc: enabled: true - partitions: 200 + partitions: 50 replicas: 3 - retention: 3600000 + retention: 300000 latiss: enabled: true - partitions: 200 + partitions: 10 replicas: 3 - retention: 3600000 + retention: 300000 + lsstcam: + enabled: true + partitions: 50 + replicas: 3 + retention: 300000 + lsstcomcam: + enabled: true + partitions: 50 + replicas: 3 + retention: 300000 + lsstcomcamsim: + enabled: true + partitions: 50 + replicas: 3 + retention: 300000 kafdrop: ingress: enabled: false diff --git a/applications/prompt-processing-kafka/values-usdfprod-prompt-processing.yaml b/applications/prompt-processing-kafka/values-usdfprod-prompt-processing.yaml new file mode 100644 index 0000000000..e749fd1c50 --- /dev/null +++ b/applications/prompt-processing-kafka/values-usdfprod-prompt-processing.yaml @@ -0,0 +1,18 @@ +strimzi-kafka: + users: + kafdrop: + enabled: false + kraft: + enabled: true + kafkaController: + enabled: true + kafka: + storage: + size: 100Gi + storageClassName: wekafs--sdf-k8s01 + topics: + lsstcomcam: + enabled: true + partitions: 50 + replicas: 3 + retention: 300000 diff --git a/environments/values-usdfprod-prompt-processing.yaml b/environments/values-usdfprod-prompt-processing.yaml index b1c1ce92d9..340df783d7 100644 --- a/environments/values-usdfprod-prompt-processing.yaml +++ b/environments/values-usdfprod-prompt-processing.yaml @@ -8,6 +8,7 @@ applications: gafaelfawr: false ingress-nginx: false next-visit-fan-out: true + prompt-processing-kafka: true prompt-proto-service-hsc: false prompt-proto-service-latiss: true prompt-proto-service-lsstcam: false From a64defb9ec40bf6eb951541cb730d922e00288a7 Mon Sep 17 00:00:00 2001 From: dspeck1 Date: Mon, 2 Dec 2024 13:19:37 -0600 Subject: [PATCH 21/28] Enable missing ComCam in Prod --- environments/values-usdfprod-prompt-processing.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environments/values-usdfprod-prompt-processing.yaml b/environments/values-usdfprod-prompt-processing.yaml index 340df783d7..06e9186fa5 100644 --- a/environments/values-usdfprod-prompt-processing.yaml +++ b/environments/values-usdfprod-prompt-processing.yaml @@ -12,6 +12,6 @@ applications: prompt-proto-service-hsc: false prompt-proto-service-latiss: true prompt-proto-service-lsstcam: false - prompt-proto-service-lsstcomcam: false + prompt-proto-service-lsstcomcam: true prompt-proto-service-lsstcomcamsim: false vault-secrets-operator: false From e18e19c6e5bcf4021fad754cf21224ef510a6301 Mon Sep 17 00:00:00 2001 From: dspeck1 Date: Mon, 2 Dec 2024 14:24:16 -0600 Subject: [PATCH 22/28] fix enabling of app --- .../applications/prompt/prompt-proto-service-lsstcomcam.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environments/templates/applications/prompt/prompt-proto-service-lsstcomcam.yaml b/environments/templates/applications/prompt/prompt-proto-service-lsstcomcam.yaml index 96017e4d65..842abfdef9 100644 --- a/environments/templates/applications/prompt/prompt-proto-service-lsstcomcam.yaml +++ b/environments/templates/applications/prompt/prompt-proto-service-lsstcomcam.yaml @@ -1,4 +1,4 @@ -{{- if (index .Values "applications" "prompt-proto-service-lsstcam") -}} +{{- if (index .Values "applications" "prompt-proto-service-lsstcomcam") -}} apiVersion: argoproj.io/v1alpha1 kind: Application metadata: From e2ae9d91ddf72f5a0d7c3afe8418171bbf16c863 Mon Sep 17 00:00:00 2001 From: dspeck1 Date: Mon, 2 Dec 2024 14:39:49 -0600 Subject: [PATCH 23/28] Add strimzi app and configuration. --- .../strimzi/values-usdfprod-prompt-processing.yaml | 10 ++++++++++ environments/values-usdfprod-prompt-processing.yaml | 1 + 2 files changed, 11 insertions(+) create mode 100644 applications/strimzi/values-usdfprod-prompt-processing.yaml diff --git a/applications/strimzi/values-usdfprod-prompt-processing.yaml b/applications/strimzi/values-usdfprod-prompt-processing.yaml new file mode 100644 index 0000000000..3777b2d0eb --- /dev/null +++ b/applications/strimzi/values-usdfprod-prompt-processing.yaml @@ -0,0 +1,10 @@ +strimzi-kafka-operator: + resources: + limits: + memory: "1Gi" + requests: + memory: "512Mi" + watchNamespaces: + - "kafka" + - "prompt-processing-kafka" + logLevel: "INFO" diff --git a/environments/values-usdfprod-prompt-processing.yaml b/environments/values-usdfprod-prompt-processing.yaml index 06e9186fa5..a915074894 100644 --- a/environments/values-usdfprod-prompt-processing.yaml +++ b/environments/values-usdfprod-prompt-processing.yaml @@ -14,4 +14,5 @@ applications: prompt-proto-service-lsstcam: false prompt-proto-service-lsstcomcam: true prompt-proto-service-lsstcomcamsim: false + strimzi: true vault-secrets-operator: false From 26d8ff0564413c3d798fadcf596875889e7f1cd8 Mon Sep 17 00:00:00 2001 From: dspeck1 Date: Mon, 2 Dec 2024 16:29:56 -0600 Subject: [PATCH 24/28] enable kafdrop in prod --- .../values-usdfprod-prompt-processing.yaml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/applications/prompt-processing-kafka/values-usdfprod-prompt-processing.yaml b/applications/prompt-processing-kafka/values-usdfprod-prompt-processing.yaml index e749fd1c50..36e8f716cf 100644 --- a/applications/prompt-processing-kafka/values-usdfprod-prompt-processing.yaml +++ b/applications/prompt-processing-kafka/values-usdfprod-prompt-processing.yaml @@ -1,7 +1,7 @@ strimzi-kafka: users: kafdrop: - enabled: false + enabled: true kraft: enabled: true kafkaController: @@ -16,3 +16,9 @@ strimzi-kafka: partitions: 50 replicas: 3 retention: 300000 +kafdrop: + ingress: + enabled: false + kafka: + broker: prompt-processing-kafka-kafka-bootstrap.prompt-processing-kafka:9092 + schemaregistry: http://10.106.68.124:8081 From b126ac03013496be5f2d0d5305710ea2300dde1d Mon Sep 17 00:00:00 2001 From: dspeck1 Date: Thu, 5 Dec 2024 10:21:14 -0600 Subject: [PATCH 25/28] Test adding argocd env variables. --- .../charts/kafdrop/templates/deployment.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/applications/prompt-processing-kafka/charts/kafdrop/templates/deployment.yaml b/applications/prompt-processing-kafka/charts/kafdrop/templates/deployment.yaml index 99c520975e..94b09e17e2 100644 --- a/applications/prompt-processing-kafka/charts/kafdrop/templates/deployment.yaml +++ b/applications/prompt-processing-kafka/charts/kafdrop/templates/deployment.yaml @@ -38,6 +38,9 @@ spec: name: {{ .Values.existingSecret }} {{- end }} env: + - name: ARGOCD_APP_REVISION + - name: ARGOCD_APP_REVISION_SHORT + - name: KUBE_VERSION - name: KAFKA_BROKERCONNECT value: {{ .Values.kafka.broker | quote }} - name: JVM_OPTS From b16227eb2544bd6ebf986649c1bd6648af5e43ba Mon Sep 17 00:00:00 2001 From: dspeck1 Date: Thu, 5 Dec 2024 10:25:30 -0600 Subject: [PATCH 26/28] add values --- .../charts/kafdrop/templates/deployment.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/applications/prompt-processing-kafka/charts/kafdrop/templates/deployment.yaml b/applications/prompt-processing-kafka/charts/kafdrop/templates/deployment.yaml index 94b09e17e2..ff31e84c62 100644 --- a/applications/prompt-processing-kafka/charts/kafdrop/templates/deployment.yaml +++ b/applications/prompt-processing-kafka/charts/kafdrop/templates/deployment.yaml @@ -39,8 +39,10 @@ spec: {{- end }} env: - name: ARGOCD_APP_REVISION + value: $ARGOCD_APP_REVISION - name: ARGOCD_APP_REVISION_SHORT - name: KUBE_VERSION + value: $KUBE_VERSION - name: KAFKA_BROKERCONNECT value: {{ .Values.kafka.broker | quote }} - name: JVM_OPTS From a105ad678782ed714c5e48b7bb1c55f719918730 Mon Sep 17 00:00:00 2001 From: dspeck1 Date: Thu, 5 Dec 2024 10:41:12 -0600 Subject: [PATCH 27/28] Test adding argocd_revsion to app def. --- .../charts/kafdrop/templates/deployment.yaml | 5 +---- .../applications/prompt/prompt-processing-kafka.yaml | 2 ++ 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/applications/prompt-processing-kafka/charts/kafdrop/templates/deployment.yaml b/applications/prompt-processing-kafka/charts/kafdrop/templates/deployment.yaml index ff31e84c62..e8f7f1f595 100644 --- a/applications/prompt-processing-kafka/charts/kafdrop/templates/deployment.yaml +++ b/applications/prompt-processing-kafka/charts/kafdrop/templates/deployment.yaml @@ -39,10 +39,7 @@ spec: {{- end }} env: - name: ARGOCD_APP_REVISION - value: $ARGOCD_APP_REVISION - - name: ARGOCD_APP_REVISION_SHORT - - name: KUBE_VERSION - value: $KUBE_VERSION + value: {{.Values.global.argo_revision}} - name: KAFKA_BROKERCONNECT value: {{ .Values.kafka.broker | quote }} - name: JVM_OPTS diff --git a/environments/templates/applications/prompt/prompt-processing-kafka.yaml b/environments/templates/applications/prompt/prompt-processing-kafka.yaml index 14b76d5207..7b7ff4d5e6 100644 --- a/environments/templates/applications/prompt/prompt-processing-kafka.yaml +++ b/environments/templates/applications/prompt/prompt-processing-kafka.yaml @@ -22,6 +22,8 @@ spec: targetRevision: {{ .Values.targetRevision | quote }} helm: parameters: + - name: "global.argo_revision" + value: $ARGOCD_APP_NAME - name: "global.host" value: {{ .Values.fqdn | quote }} - name: "global.baseUrl" From 7fb8501c4f4af2a50a765922af8d7ba89be8298d Mon Sep 17 00:00:00 2001 From: dspeck1 Date: Thu, 5 Dec 2024 11:03:07 -0600 Subject: [PATCH 28/28] fix spacing and variable name for argocd revision variable --- .../charts/kafdrop/templates/deployment.yaml | 2 +- .../applications/prompt/prompt-processing-kafka.yaml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/applications/prompt-processing-kafka/charts/kafdrop/templates/deployment.yaml b/applications/prompt-processing-kafka/charts/kafdrop/templates/deployment.yaml index e8f7f1f595..fcaed235be 100644 --- a/applications/prompt-processing-kafka/charts/kafdrop/templates/deployment.yaml +++ b/applications/prompt-processing-kafka/charts/kafdrop/templates/deployment.yaml @@ -39,7 +39,7 @@ spec: {{- end }} env: - name: ARGOCD_APP_REVISION - value: {{.Values.global.argo_revision}} + value: {{ .Values.global.argocd_revision }} - name: KAFKA_BROKERCONNECT value: {{ .Values.kafka.broker | quote }} - name: JVM_OPTS diff --git a/environments/templates/applications/prompt/prompt-processing-kafka.yaml b/environments/templates/applications/prompt/prompt-processing-kafka.yaml index 7b7ff4d5e6..16cfd8bf61 100644 --- a/environments/templates/applications/prompt/prompt-processing-kafka.yaml +++ b/environments/templates/applications/prompt/prompt-processing-kafka.yaml @@ -22,8 +22,8 @@ spec: targetRevision: {{ .Values.targetRevision | quote }} helm: parameters: - - name: "global.argo_revision" - value: $ARGOCD_APP_NAME + - name: "global.argocd_revision" + value: $ARGOCD_APP_REVISION - name: "global.host" value: {{ .Values.fqdn | quote }} - name: "global.baseUrl"