Merge pull request #11 from lsst-uk/main

Update

.github/workflows/docs.yaml

@@ -40,6 +40,7 @@ jobs:
               - "applications/argocd/values-*.yaml"
               - "applications/gafaelfawr/values-*.yaml"
               - "environments/values-*.yaml"
+              - "requirements/*.txt"
               - "src/phalanx/**"
             docsSpecific:
               - "docs/**"

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.6.0
+    rev: v5.0.0
     hooks:
       - id: check-merge-conflict
       - id: check-toml
@@ -14,7 +14,7 @@ repos:
           - -c=.yamllint.yml
 
   - repo: https://github.com/python-jsonschema/check-jsonschema
-    rev: 0.29.1
+    rev: 0.30.0
     hooks:
       - id: check-jsonschema
         files: ^applications/.*/secrets(-[^./-]+)?\.yaml
@@ -46,14 +46,14 @@ repos:
           - --template-files=../helm-docs.md.gotmpl
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.5.7
+    rev: v0.8.1
     hooks:
       - id: ruff
         args: [--fix, --exit-non-zero-on-fix]
       - id: ruff-format
 
   - repo: https://github.com/adamchainz/blacken-docs
-    rev: 1.18.0
+    rev: 1.19.1
     hooks:
       - id: blacken-docs
         additional_dependencies: [black==23.7.0]

Makefile

@@ -41,20 +41,20 @@ update-deps:
 	pip install --upgrade pip uv
 	uv pip install --upgrade pre-commit
 	pre-commit autoupdate
-	uv pip compile --upgrade --generate-hashes			\
-	    --output-file requirements/main.txt requirements/main.in
-	uv pip compile --upgrade --generate-hashes			\
+	uv pip compile --upgrade --universal --generate-hashes		\
+	    --output-file requirements/main.txt pyproject.toml
+	uv pip compile --upgrade --universal --generate-hashes		\
 	    --output-file requirements/dev.txt requirements/dev.in
-	uv pip compile --upgrade --generate-hashes			\
+	uv pip compile --upgrade --universal --generate-hashes		\
 	    --output-file requirements/tox.txt requirements/tox.in
 
 # Useful for testing against a Git version of Safir.
 .PHONY: update-deps-no-hashes
 update-deps-no-hashes:
 	pip install --upgrade uv
-	uv pip compile --upgrade					\
-	    --output-file requirements/main.txt requirements/main.in
-	uv pip compile --upgrade					\
+	uv pip compile --upgrade --universal				\
+	    --output-file requirements/main.txt pyproject.toml
+	uv pip compile --upgrade --universal				\
 	    --output-file requirements/dev.txt requirements/dev.in
-	uv pip compile --upgrade					\
+	uv pip compile --upgrade --universal				\
 	    --output-file requirements/tox.txt requirements/tox.in

applications/alert-stream-broker/README.md

@@ -32,7 +32,6 @@ Alert transmission to community brokers
 | alert-database.ingester.serviceAccountName | string | `"alert-database-ingester"` | The name of the Kubernetes ServiceAccount (*not* the Google Cloud IAM service account!) which is used by the alert database ingester. |
 | alert-database.ingress.annotations | object | `{}` |  |
 | alert-database.ingress.enabled | bool | `true` | Whether to create an ingress |
-| alert-database.ingress.gafaelfawrAuthQuery | string | `"scope=read:alertdb"` | Query string for Gafaelfawr to authorize access |
 | alert-database.ingress.host | string | None, must be set if the ingress is enabled | Hostname for the ingress |
 | alert-database.ingress.path | string | `"/alertdb"` | Subpath to host the alert database application under the ingress |
 | alert-database.ingress.tls | list | `[]` | Configures TLS for the ingress if needed. If multiple ingresses share the same hostname, only one of them needs a TLS configuration. |
@@ -72,12 +71,17 @@ Alert transmission to community brokers
 | alert-stream-broker.kafka.storage.size | string | `"1000Gi"` | Size of the backing storage disk for each of the Kafka brokers. |
 | alert-stream-broker.kafka.storage.storageClassName | string | `"standard"` | Name of a StorageClass to use when requesting persistent volumes. |
 | alert-stream-broker.kafka.version | string | `"3.4.0"` | Version of Kafka to deploy. |
+| alert-stream-broker.kafkaController.enabled | bool | `false` | Enable Kafka Controller |
+| alert-stream-broker.kafkaController.resources | object | See `values.yaml` | Kubernetes requests and limits for the Kafka Controller |
+| alert-stream-broker.kafkaController.storage.size | string | `"20Gi"` | Size of the backing storage disk for each of the Kafka controllers |
+| alert-stream-broker.kafkaController.storage.storageClassName | string | `""` | Name of a StorageClass to use when requesting persistent volumes |
 | alert-stream-broker.kafkaExporter | object | `{"enableSaramaLogging":false,"enabled":false,"groupRegex":".*","logLevel":"warning","topicRegex":".*"}` | Kafka JMX Exporter for more detailed diagnostic metrics. |
 | alert-stream-broker.kafkaExporter.enableSaramaLogging | bool | `false` | Enable Sarama logging |
 | alert-stream-broker.kafkaExporter.enabled | bool | `false` | Enable Kafka exporter. |
 | alert-stream-broker.kafkaExporter.groupRegex | string | `".*"` | Consumer groups to monitor |
 | alert-stream-broker.kafkaExporter.logLevel | string | `"warning"` | Log level for Sarama logging |
 | alert-stream-broker.kafkaExporter.topicRegex | string | `".*"` | Kafka topics to monitor |
+| alert-stream-broker.kraft | bool | `true` |  |
 | alert-stream-broker.maxBytesRetained | string | `"100000000000"` | Maximum number of bytes for the replay topic, per partition, per replica. Default is 100GB, but should be lower to not fill storage. |
 | alert-stream-broker.maxMillisecondsRetained | string | `"5259492000"` | Maximum amount of time to save alerts in the replay topic, in milliseconds. Default is 7 days (604800000). |
 | alert-stream-broker.nameOverride | string | `""` |  |
@@ -95,10 +99,8 @@ Alert transmission to community brokers
 | alert-stream-broker.users[0].readonlyTopics | list | `["alert-stream","alerts-simulated","alert-stream-test"]` | A list of topics that the user should get read-only access to. |
 | alert-stream-broker.users[0].username | string | `"rubin-testing"` | The username for the user that should be created. |
 | alert-stream-broker.vaultSecretsPath | string | `""` | Path to the secret resource in Vault |
-| alert-stream-broker.zookeeper.replicas | int | `3` | Number of Zookeeper replicas to run. |
-| alert-stream-broker.zookeeper.storage.size | string | `"1000Gi"` | Size of the backing storage disk for each of the Zookeeper instances. |
-| alert-stream-broker.zookeeper.storage.storageClassName | string | `"standard"` | Name of a StorageClass to use when requesting persistent volumes. |
 | alert-stream-schema-registry.clusterName | string | `"alert-broker"` | Strimzi "cluster name" of the broker to use as a backend. |
+| alert-stream-schema-registry.compatibilityLevel | string | `"None"` |  |
 | alert-stream-schema-registry.hostname | string | `"usdf-alert-schemas-dev.slac.stanford.edu"` | Hostname for an ingress which sends traffic to the Schema Registry. |
 | alert-stream-schema-registry.name | string | `"alert-schema-registry"` | Name used by the registry, and by its users. |
 | alert-stream-schema-registry.port | int | `8081` | Port where the registry is listening. NOTE: Not actually configurable in strimzi-registry-operator, so this basically cannot be changed. |

applications/alert-stream-broker/charts/alert-database/README.md

@@ -23,7 +23,6 @@ Archival database of alerts sent through the alert stream.
 | ingester.serviceAccountName | string | `"alert-database-ingester"` | The name of the Kubernetes ServiceAccount (*not* the Google Cloud IAM service account!) which is used by the alert database ingester. |
 | ingress.annotations | object | `{}` |  |
 | ingress.enabled | bool | `true` | Whether to create an ingress |
-| ingress.gafaelfawrAuthQuery | string | `"scope=read:alertdb"` | Query string for Gafaelfawr to authorize access |
 | ingress.host | string | None, must be set if the ingress is enabled | Hostname for the ingress |
 | ingress.path | string | `"/alertdb"` | Subpath to host the alert database application under the ingress |
 | ingress.tls | list | `[]` | Configures TLS for the ingress if needed. If multiple ingresses share the same hostname, only one of them needs a TLS configuration. |

applications/alert-stream-broker/charts/alert-database/templates/ingress.yaml

@@ -1,38 +1,46 @@
 {{- if .Values.ingress.enabled -}}
-apiVersion: networking.k8s.io/v1
-kind: Ingress
+apiVersion: gafaelfawr.lsst.io/v1alpha1
+kind: GafaelfawrIngress
 metadata:
-  annotations:
-    kubernetes.io/ingress.class: "nginx"
-    nginx.ingress.kubernetes.io/rewrite-target: /$2
-    nginx.ingress.kubernetes.io/auth-method: "GET"
-    nginx.ingress.kubernetes.io/auth-url: "http://gafaelfawr.gafaelfawr.svc.cluster.local:8080/auth?{{ required "ingress.gafaelfawrAuthQuery must be set" .Values.ingress.gafaelfawrAuthQuery }}"
-    {{- with .Values.ingress.annotations }}
-    {{- toYaml . | nindent 4 }}
-    {{- end }}
   name: {{ template "alertDatabase.fullname" . }}
   labels:
     {{- include "alertDatabase.labels" . | nindent 4 }}
-spec:
-  rules:
-    - host: {{ required "ingress.host must be set" .Values.ingress.host | quote }}
-      http:
-        paths:
-          - path: "{{ .Values.ingress.path }}(/|$)(.*)"
-            pathType: Prefix
-            backend:
-              service:
-                name: {{ template "alertDatabase.fullname" . }}
-                port:
-                  name: http
-  {{- if .Values.ingress.tls }}
-  tls:
-    {{- range .Values.ingress.tls }}
-    - hosts:
-        {{- range .hosts }}
-        - {{ . | quote }}
-        {{- end }}
-      secretName: {{ .secretName }}
+config:
+  baseUrl: {{ .Values.global.baseUrl | quote }}
+  scopes:
+    all:
+      - "read:alertdb"
+  service: "alert-stream-broker"
+template:
+  metadata:
+    name: {{ template "alertDatabase.fullname" . }}
+    annotations:
+      nginx.ingress.kubernetes.io/rewrite-target: "/$2"
+      {{- with .Values.ingress.annotations }}
+      {{- toYaml . | nindent 6 }}
+      {{- end }}
+    labels:
+      {{- include "alertDatabase.labels" . | nindent 4 }}
+  spec:
+    rules:
+      - host: {{ required "ingress.host must be set" .Values.ingress.host | quote }}
+        http:
+          paths:
+            - path: "{{ .Values.ingress.path }}(/|$)(.*)"
+              pathType: ImplementationSpecific
+              backend:
+                service:
+                  name: {{ template "alertDatabase.fullname" . }}
+                  port:
+                    name: http
+    {{- if .Values.ingress.tls }}
+    tls:
+      {{- range .Values.ingress.tls }}
+      - hosts:
+          {{- range .hosts }}
+          - {{ . | quote }}
+          {{- end }}
+        secretName: {{ .secretName }}
+      {{- end }}
     {{- end }}
-  {{- end }}
 {{- end }}

applications/alert-stream-broker/charts/alert-database/values.yaml

@@ -101,6 +101,3 @@ ingress:
 
   # -- Subpath to host the alert database application under the ingress
   path: "/alertdb"
-
-  # -- Query string for Gafaelfawr to authorize access
-  gafaelfawrAuthQuery: "scope=read:alertdb"

applications/alert-stream-broker/charts/alert-stream-broker/README.md

@@ -29,12 +29,17 @@ Kafka broker cluster for distributing alerts
 | kafka.storage.size | string | `"1000Gi"` | Size of the backing storage disk for each of the Kafka brokers. |
 | kafka.storage.storageClassName | string | `"standard"` | Name of a StorageClass to use when requesting persistent volumes. |
 | kafka.version | string | `"3.4.0"` | Version of Kafka to deploy. |
+| kafkaController.enabled | bool | `false` | Enable Kafka Controller |
+| kafkaController.resources | object | See `values.yaml` | Kubernetes requests and limits for the Kafka Controller |
+| kafkaController.storage.size | string | `"20Gi"` | Size of the backing storage disk for each of the Kafka controllers |
+| kafkaController.storage.storageClassName | string | `""` | Name of a StorageClass to use when requesting persistent volumes |
 | kafkaExporter | object | `{"enableSaramaLogging":false,"enabled":false,"groupRegex":".*","logLevel":"warning","topicRegex":".*"}` | Kafka JMX Exporter for more detailed diagnostic metrics. |
 | kafkaExporter.enableSaramaLogging | bool | `false` | Enable Sarama logging |
 | kafkaExporter.enabled | bool | `false` | Enable Kafka exporter. |
 | kafkaExporter.groupRegex | string | `".*"` | Consumer groups to monitor |
 | kafkaExporter.logLevel | string | `"warning"` | Log level for Sarama logging |
 | kafkaExporter.topicRegex | string | `".*"` | Kafka topics to monitor |
+| kraft | bool | `true` |  |
 | maxBytesRetained | string | `"100000000000"` | Maximum number of bytes for the replay topic, per partition, per replica. Default is 100GB, but should be lower to not fill storage. |
 | maxMillisecondsRetained | string | `"5259492000"` | Maximum amount of time to save alerts in the replay topic, in milliseconds. Default is 7 days (604800000). |
 | nameOverride | string | `""` |  |
@@ -52,6 +57,3 @@ Kafka broker cluster for distributing alerts
 | users[0].readonlyTopics | list | `["alert-stream","alerts-simulated","alert-stream-test"]` | A list of topics that the user should get read-only access to. |
 | users[0].username | string | `"rubin-testing"` | The username for the user that should be created. |
 | vaultSecretsPath | string | `""` | Path to the secret resource in Vault |
-| zookeeper.replicas | int | `3` | Number of Zookeeper replicas to run. |
-| zookeeper.storage.size | string | `"1000Gi"` | Size of the backing storage disk for each of the Zookeeper instances. |
-| zookeeper.storage.storageClassName | string | `"standard"` | Name of a StorageClass to use when requesting persistent volumes. |

applications/alert-stream-broker/charts/alert-stream-broker/templates/kafka-topics.yaml

@@ -26,6 +26,7 @@ spec:
     cleanup.policy: "delete"
     retention.ms: {{ .Values.maxMillisecondsRetained }} # 7 days
     retention.bytes: {{ .Values.maxBytesRetained }}
+    compression.type: {{ .Values.topicCompression}}
     # The default timestamp is the creation time of the alert.
     # To get the ingestion rate, we need this to be the log
     # append time, and the header will contain the producer
@@ -45,6 +46,7 @@ spec:
     cleanup.policy: "delete"
     retention.ms: {{ .Values.maxMillisecondsRetained }} # 7 days
     retention.bytes: {{ .Values.maxBytesRetained }}
+    compression.type: {{ .Values.topicCompression}}
     # The default timestamp is the creation time of the alert.
     # To get the ingestion rate, we need this to be the log
     # append time, and the header will contain the producer
@@ -64,11 +66,51 @@ spec:
     cleanup.policy: "delete"
     retention.ms: {{ .Values.maxMillisecondsRetained }} # 7 days
     retention.bytes: {{ .Values.maxBytesRetained }}
-    compression.type: {{ .Values.devTopicCompression}}
+    compression.type: {{ .Values.topicCompression}}
     # The default timestamp is the creation time of the alert.
     # To get the ingestion rate, we need this to be the log
     # append time, and the header will contain the producer
     # timestamp instead
     message.timestamp.type: 'LogAppendTime'
   partitions: {{ .Values.devTopicPartitions }}
-  replicas: {{ .Values.devTopicReplicas }}
+  replicas: {{ .Values.devTopicReplicas }}
+---
+apiVersion: "kafka.strimzi.io/{{ .Values.strimziAPIVersion }}"
+kind: KafkaTopic
+metadata:
+  labels:
+    strimzi.io/cluster: "{{ .Values.clusterName }}"
+  name: "{{ .Values.latissTopicName}}"
+spec:
+  config:
+    cleanup.policy: "delete"
+    retention.ms: {{ .Values.maxMillisecondsRetained }} # 7 days
+    retention.bytes: {{ .Values.maxBytesRetained }}
+    compression.type: {{ .Values.topicCompression}}
+    # The default timestamp is the creation time of the alert.
+    # To get the ingestion rate, we need this to be the log
+    # append time, and the header will contain the producer
+    # timestamp instead
+    message.timestamp.type: 'LogAppendTime'
+  partitions: {{ .Values.latissTopicPartitions }}
+  replicas: {{ .Values.latissTopicReplicas }}
+---
+apiVersion: "kafka.strimzi.io/{{ .Values.strimziAPIVersion }}"
+kind: KafkaTopic
+metadata:
+  labels:
+    strimzi.io/cluster: "{{ .Values.clusterName }}"
+  name: "{{ .Values.comcamTopicName}}"
+spec:
+  config:
+    cleanup.policy: "delete"
+    retention.ms: {{ .Values.maxMillisecondsRetained }} # 7 days
+    retention.bytes: {{ .Values.maxBytesRetained }}
+    compression.type: {{ .Values.topicCompression }}
+    # The default timestamp is the creation time of the alert.
+    # To get the ingestion rate, we need this to be the log
+    # append time, and the header will contain the producer
+    # timestamp instead
+    message.timestamp.type: 'LogAppendTime'
+  partitions: {{ .Values.comcamTopicPartitions }}
+  replicas: {{ .Values.comcamTopicReplicas }}