Merge branch 'lsst-sqre:main' into rsptest

lsst-uk · Nov 20, 2024 · 0fdeb25 · 0fdeb25
2 parents ab5c649 + fefd969
commit 0fdeb25
Show file tree

Hide file tree

Showing 330 changed files with 6,036 additions and 1,776 deletions.
diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml
@@ -40,6 +40,7 @@ jobs:
               - "applications/argocd/values-*.yaml"
               - "applications/gafaelfawr/values-*.yaml"
               - "environments/values-*.yaml"
+              - "requirements/*.txt"
               - "src/phalanx/**"
             docsSpecific:
               - "docs/**"

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -1,6 +1,6 @@
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.6.0
+    rev: v5.0.0
     hooks:
       - id: check-merge-conflict
       - id: check-toml
@@ -14,7 +14,7 @@ repos:
           - -c=.yamllint.yml
 
   - repo: https://github.com/python-jsonschema/check-jsonschema
-    rev: 0.29.2
+    rev: 0.29.4
     hooks:
       - id: check-jsonschema
         files: ^applications/.*/secrets(-[^./-]+)?\.yaml
@@ -46,14 +46,14 @@ repos:
           - --template-files=../helm-docs.md.gotmpl
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.6.7
+    rev: v0.7.4
     hooks:
       - id: ruff
         args: [--fix, --exit-non-zero-on-fix]
       - id: ruff-format
 
   - repo: https://github.com/adamchainz/blacken-docs
-    rev: 1.18.0
+    rev: 1.19.1
     hooks:
       - id: blacken-docs
         additional_dependencies: [black==23.7.0]
diff --git a/applications/alert-stream-broker/README.md b/applications/alert-stream-broker/README.md
@@ -32,7 +32,6 @@ Alert transmission to community brokers
 | alert-database.ingester.serviceAccountName | string | `"alert-database-ingester"` | The name of the Kubernetes ServiceAccount (*not* the Google Cloud IAM service account!) which is used by the alert database ingester. |
 | alert-database.ingress.annotations | object | `{}` |  |
 | alert-database.ingress.enabled | bool | `true` | Whether to create an ingress |
-| alert-database.ingress.gafaelfawrAuthQuery | string | `"scope=read:alertdb"` | Query string for Gafaelfawr to authorize access |
 | alert-database.ingress.host | string | None, must be set if the ingress is enabled | Hostname for the ingress |
 | alert-database.ingress.path | string | `"/alertdb"` | Subpath to host the alert database application under the ingress |
 | alert-database.ingress.tls | list | `[]` | Configures TLS for the ingress if needed. If multiple ingresses share the same hostname, only one of them needs a TLS configuration. |

diff --git a/applications/alert-stream-broker/charts/alert-database/README.md b/applications/alert-stream-broker/charts/alert-database/README.md
@@ -23,7 +23,6 @@ Archival database of alerts sent through the alert stream.
 | ingester.serviceAccountName | string | `"alert-database-ingester"` | The name of the Kubernetes ServiceAccount (*not* the Google Cloud IAM service account!) which is used by the alert database ingester. |
 | ingress.annotations | object | `{}` |  |
 | ingress.enabled | bool | `true` | Whether to create an ingress |
-| ingress.gafaelfawrAuthQuery | string | `"scope=read:alertdb"` | Query string for Gafaelfawr to authorize access |
 | ingress.host | string | None, must be set if the ingress is enabled | Hostname for the ingress |
 | ingress.path | string | `"/alertdb"` | Subpath to host the alert database application under the ingress |
 | ingress.tls | list | `[]` | Configures TLS for the ingress if needed. If multiple ingresses share the same hostname, only one of them needs a TLS configuration. |

diff --git a/applications/alert-stream-broker/charts/alert-database/templates/ingress.yaml b/applications/alert-stream-broker/charts/alert-database/templates/ingress.yaml
@@ -1,38 +1,45 @@
 {{- if .Values.ingress.enabled -}}
-apiVersion: networking.k8s.io/v1
-kind: Ingress
+apiVersion: gafaelfawr.lsst.io/v1alpha1
+kind: GafaelfawrIngress
 metadata:
-  annotations:
-    kubernetes.io/ingress.class: "nginx"
-    nginx.ingress.kubernetes.io/rewrite-target: /$2
-    nginx.ingress.kubernetes.io/auth-method: "GET"
-    nginx.ingress.kubernetes.io/auth-url: "http://gafaelfawr.gafaelfawr.svc.cluster.local:8080/auth?{{ required "ingress.gafaelfawrAuthQuery must be set" .Values.ingress.gafaelfawrAuthQuery }}"
-    {{- with .Values.ingress.annotations }}
-    {{- toYaml . | nindent 4 }}
-    {{- end }}
   name: {{ template "alertDatabase.fullname" . }}
   labels:
     {{- include "alertDatabase.labels" . | nindent 4 }}
-spec:
-  rules:
-    - host: {{ required "ingress.host must be set" .Values.ingress.host | quote }}
-      http:
-        paths:
-          - path: "{{ .Values.ingress.path }}(/|$)(.*)"
-            pathType: Prefix
-            backend:
-              service:
-                name: {{ template "alertDatabase.fullname" . }}
-                port:
-                  name: http
-  {{- if .Values.ingress.tls }}
-  tls:
-    {{- range .Values.ingress.tls }}
-    - hosts:
-        {{- range .hosts }}
-        - {{ . | quote }}
-        {{- end }}
-      secretName: {{ .secretName }}
+config:
+  baseUrl: {{ .Values.global.baseUrl | quote }}
+  scopes:
+    all:
+      - "read:alertdb"
+template:
+  metadata:
+    name: {{ template "alertDatabase.fullname" . }}
+    annotations:
+      nginx.ingress.kubernetes.io/rewrite-target: "/$2"
+      {{- with .Values.ingress.annotations }}
+      {{- toYaml . | nindent 6 }}
+      {{- end }}
+    labels:
+      {{- include "alertDatabase.labels" . | nindent 4 }}
+  spec:
+    rules:
+      - host: {{ required "ingress.host must be set" .Values.ingress.host | quote }}
+        http:
+          paths:
+            - path: "{{ .Values.ingress.path }}(/|$)(.*)"
+              pathType: ImplementationSpecific
+              backend:
+                service:
+                  name: {{ template "alertDatabase.fullname" . }}
+                  port:
+                    name: http
+    {{- if .Values.ingress.tls }}
+    tls:
+      {{- range .Values.ingress.tls }}
+      - hosts:
+          {{- range .hosts }}
+          - {{ . | quote }}
+          {{- end }}
+        secretName: {{ .secretName }}
+      {{- end }}
     {{- end }}
-  {{- end }}
 {{- end }}
diff --git a/applications/alert-stream-broker/charts/alert-database/values.yaml b/applications/alert-stream-broker/charts/alert-database/values.yaml
@@ -101,6 +101,3 @@ ingress:
 
   # -- Subpath to host the alert database application under the ingress
   path: "/alertdb"
-
-  # -- Query string for Gafaelfawr to authorize access
-  gafaelfawrAuthQuery: "scope=read:alertdb"
diff --git a/applications/alert-stream-broker/charts/alert-stream-broker/templates/kafka-topics.yaml b/applications/alert-stream-broker/charts/alert-stream-broker/templates/kafka-topics.yaml
@@ -93,4 +93,24 @@ spec:
     # timestamp instead
     message.timestamp.type: 'LogAppendTime'
   partitions: {{ .Values.latissTopicPartitions }}
-  replicas: {{ .Values.latissTopicReplicas }}
+  replicas: {{ .Values.latissTopicReplicas }}
+---
+apiVersion: "kafka.strimzi.io/{{ .Values.strimziAPIVersion }}"
+kind: KafkaTopic
+metadata:
+  labels:
+    strimzi.io/cluster: "{{ .Values.clusterName }}"
+  name: "{{ .Values.comcamTopicName}}"
+spec:
+  config:
+    cleanup.policy: "delete"
+    retention.ms: {{ .Values.maxMillisecondsRetained }} # 7 days
+    retention.bytes: {{ .Values.maxBytesRetained }}
+    compression.type: {{ .Values.topicCompression }}
+    # The default timestamp is the creation time of the alert.
+    # To get the ingestion rate, we need this to be the log
+    # append time, and the header will contain the producer
+    # timestamp instead
+    message.timestamp.type: 'LogAppendTime'
+  partitions: {{ .Values.comcamTopicPartitions }}
+  replicas: {{ .Values.comcamTopicReplicas }}
diff --git a/applications/alert-stream-broker/values-usdfdev-alert-stream-broker.yaml b/applications/alert-stream-broker/values-usdfdev-alert-stream-broker.yaml
@@ -17,7 +17,7 @@ alert-stream-broker:
         enabled: false
       bootstrap:
         host: usdf-alert-stream-dev.lsst.cloud
-        ip: ""
+        ip: "134.79.23.185"
         annotations:
           metallb.universe.tf/address-pool: 'sdf-dmz'
       brokers:
@@ -121,6 +121,10 @@ alert-stream-broker:
   latissTopicPartitions: 45
   latissTopicReplicas: 1
 
+  comcamTopicName: lsstcomcam-alerts
+  comcamTopicPartitions: 45
+  comcamTopicReplicas: 1
+
   # Compression set to snappy to balance alert packet compression speed and size.
   topicCompression: snappy
 

diff --git a/applications/alert-stream-broker/values-usdfdev.yaml b/applications/alert-stream-broker/values-usdfdev.yaml
@@ -110,7 +110,6 @@ alert-database:
   ingress:
     enabled: true
     host: "usdf-rsp-dev.slac.stanford.edu"
-    gafaelfawrAuthQuery: "scope=read:alertdb"
 
   storage:
     gcp:

diff --git a/applications/argo-workflows/Chart.yaml b/applications/argo-workflows/Chart.yaml
@@ -8,5 +8,5 @@ sources:
   - https://github.com/argoproj/argo-helm
 dependencies:
   - name: argo-workflows
-    version: 0.42.3
+    version: 0.42.7
     repository: https://argoproj.github.io/argo-helm
diff --git a/applications/argocd/Chart.yaml b/applications/argocd/Chart.yaml
@@ -8,5 +8,5 @@ sources:
   - https://github.com/argoproj/argo-helm
 dependencies:
   - name: argo-cd
-    version: 7.6.1
+    version: 7.7.3
     repository: https://argoproj.github.io/argo-helm
diff --git a/applications/argocd/values-usdf-cm-dev.yaml b/applications/argocd/values-usdf-cm-dev.yaml
@@ -29,6 +29,7 @@ argo-cd:
         g, [email protected], role:admin
         g, [email protected], role:admin
         g, [email protected], role:admin
+        g, [email protected], role:admin
       scopes: "[email]"
 
   server:

diff --git a/applications/argocd/values-usdfint.yaml b/applications/argocd/values-usdfint.yaml
@@ -58,6 +58,7 @@ argo-cd:
         g, [email protected], role:developer
         g, [email protected], role:developer
         g, [email protected], role:developer
+        g, [email protected], role:developer
       scopes: "[email]"
 
   server:

diff --git a/applications/argocd/values-usdfprod.yaml b/applications/argocd/values-usdfprod.yaml
@@ -34,6 +34,7 @@ argo-cd:
 
         g, [email protected], role:admin
         g, [email protected], role:admin
+        g, [email protected], role:admin
         g, [email protected], role:admin
         g, [email protected], role:admin
         g, [email protected], role:admin
@@ -54,6 +55,7 @@ argo-cd:
         g, [email protected], role:developer
         g, [email protected], role:developer
         g, [email protected], role:developer
+        g, [email protected], role:developer
       scopes: "[email]"
 
   server:

diff --git a/applications/butler/Chart.yaml b/applications/butler/Chart.yaml
@@ -4,4 +4,4 @@ version: 1.0.0
 description: Server for Butler data abstraction service
 sources:
   - https://github.com/lsst/daf_butler
-appVersion: server-2.1.0
+appVersion: server-2.3.0
diff --git a/applications/butler/README.md b/applications/butler/README.md
@@ -15,13 +15,13 @@ Server for Butler data abstraction service
 | autoscaling.maxReplicas | int | `100` | Maximum number of butler deployment pods |
 | autoscaling.minReplicas | int | `1` | Minimum number of butler deployment pods |
 | autoscaling.targetCPUUtilizationPercentage | int | `80` | Target CPU utilization of butler deployment pods |
-| config.additionalS3ProfileName | string | No second S3 profile is available. | Profile name identifying a second S3 endpoint and set of credentials to use for accessing files in the datastore. |
+| config.additionalS3EndpointUrls | object | No additional URLs | Endpoint URLs for additional S3 services used by the Butler, as a mapping from profile name to URL. |
 | config.dp02ClientServerIsDefault | bool | `false` | True if the 'dp02' Butler repository alias should use client/server Butler.  False if it should use DirectButler. |
 | config.dp02PostgresUri | string | No configuration file for DP02 will be generated. | Postgres connection string pointing to the registry database hosting Data Preview 0.2 data. |
 | config.pathPrefix | string | `"/api/butler"` | The prefix of the path portion of the URL where the Butler service will be exposed.  For example, if the service should be exposed at `https://data.lsst.cloud/api/butler`, this should be set to `/api/butler` |
 | config.pguser | string | Use values specified in per-repository Butler config files. | Postgres username used to connect to the Butler DB |
 | config.repositories | object | `{}` | Mapping from Butler repository label to Butler configuration URI for repositories which will be hosted by this server. |
-| config.s3EndpointUrl | string | `""` | URL for the S3 service where files for datasets are stored by Butler. |
+| config.s3EndpointUrl | string | `""` | URL for the primary S3 service where files for datasets are stored by Butler. |
 | global.baseUrl | string | Set by Argo CD | Base URL for the environment |
 | global.host | string | Set by Argo CD | Host name for ingress |
 | global.vaultSecretsPath | string | Set by Argo CD | Base path for Vault secrets |

diff --git a/applications/butler/secrets.yaml b/applications/butler/secrets.yaml
@@ -18,9 +18,3 @@
   copy:
     application: nublado
     key: "postgres-credentials.txt"
-"additional-s3-profile":
-  description: >-
-    Credentials and endpoint for a second S3 profile to use, in addition to the
-    default endpoint.  For docs on format see
-    https://github.com/lsst/resources/blob/a34598e125919799d3db4bd8a2363087c3de434e/python/lsst/resources/s3utils.py#L201
-  if: config.additionalS3ProfileName
diff --git a/applications/butler/templates/configmap.yaml b/applications/butler/templates/configmap.yaml
@@ -13,6 +13,12 @@ data:
       datastore_constraints:
         # One entry per datastore in datastores section
         # Use empty `-` if no constraint override required
+        - constraints:
+            reject:
+              - all
+        - constraints:
+            reject:
+              - all
         - constraints:
             reject:
               - all
@@ -24,6 +30,22 @@ data:
             name: FileDatastore@s3://butler-us-central1-panda-dev/dc2
             cls: lsst.daf.butler.datastores.fileDatastore.FileDatastore
             root: s3://butler-us-central1-panda-dev/dc2
+        - datastore:
+            # Datasets of type 'raw' are stored in a separate bucket for
+            # historical reasons.
+            name: FileDatastore@s3://curation-us-central1-desc-dc2-run22i
+            cls: lsst.daf.butler.datastores.fileDatastore.FileDatastore
+            root: s3://curation-us-central1-desc-dc2-run22i/
+            records:
+              table: raw_datastore_records
+        - datastore:
+            # Also for historical reasons, some files that originated in DP01
+            # are kept in a separate bucket.
+            name: FileDatastore@s3://butler-us-central1-dp01-desc-dr6
+            cls: lsst.daf.butler.datastores.fileDatastore.FileDatastore
+            root: s3://butler-us-central1-dp01-desc-dr6/
+            records:
+              table: dp01_datastore_records
         - datastore:
             name: FileDatastore@s3://butler-us-central1-dp02-user
             cls: lsst.daf.butler.datastores.fileDatastore.FileDatastore

diff --git a/applications/butler/templates/deployment.yaml b/applications/butler/templates/deployment.yaml
@@ -52,6 +52,10 @@ spec:
               value: "/opt/lsst/butler/secrets/butler-gcs-creds.json"
             - name: S3_ENDPOINT_URL
               value: {{ .Values.config.s3EndpointUrl | quote }}
+            {{- range $name, $url := .Values.config.additionalS3EndpointUrls }}
+            - name: LSST_RESOURCES_S3_PROFILE_{{ $name }}
+              value: {{ $url }}
+            {{ end }}
             - name: DAF_BUTLER_REPOSITORIES
               value: {{ .Values.config.repositories | toJson | quote }}
             # Serve the configuration files generated by configmap.yaml via
@@ -65,13 +69,6 @@ spec:
             - name: PGUSER
               value: {{ .Values.config.pguser | quote }}
             {{ end }}
-            {{ if .Values.config.additionalS3ProfileName }}
-            - name: LSST_RESOURCES_S3_PROFILE_{{ .Values.config.additionalS3ProfileName }}
-              valueFrom:
-                secretKeyRef:
-                  name: {{ include "butler.fullname" . }}
-                  key: additional-s3-profile
-            {{ end }}
           volumeMounts:
             - name: "butler-secrets"
               mountPath: "/opt/lsst/butler/secrets"

diff --git a/applications/butler/values-idfdev.yaml b/applications/butler/values-idfdev.yaml
@@ -5,7 +5,5 @@ config:
   dp02ClientServerIsDefault: true
   dp02PostgresUri: postgresql://[email protected]:5432/dp02
   s3EndpointUrl: "https://storage.googleapis.com"
-  additionalS3ProfileName: "ir2"
   repositories:
     dp02: "file:///opt/lsst/butler/config/dp02.yaml"
-    ir2: "s3://butler-us-central1-panda-dev/ir2/butler-ir2.yaml"
diff --git a/applications/butler/values-idfprod.yaml b/applications/butler/values-idfprod.yaml
@@ -1,4 +1,5 @@
 config:
+  dp02ClientServerIsDefault: true
   dp02PostgresUri: postgresql://[email protected]/idfdp02
   s3EndpointUrl: "https://storage.googleapis.com"
   repositories:

diff --git a/applications/butler/values-usdfdev.yaml b/applications/butler/values-usdfdev.yaml
@@ -2,4 +2,6 @@ config:
   pguser: "rubin"
   s3EndpointUrl: "https://s3dfrgw.slac.stanford.edu"
   repositories:
-    embargo: "s3://rubin-summit-users/butler.yaml"
+    embargo: s3://embargo@rubin-summit-users/butler.yaml
+  additionalS3EndpointUrls:
+    embargo: "https://sdfembs3.sdf.slac.stanford.edu"
diff --git a/applications/butler/values-usdfint.yaml b/applications/butler/values-usdfint.yaml
@@ -2,4 +2,6 @@ config:
   pguser: "rubin"
   s3EndpointUrl: "https://s3dfrgw.slac.stanford.edu"
   repositories:
-    embargo: "s3://rubin-summit-users/butler.yaml"
+    embargo: s3://embargo@rubin-summit-users/butler.yaml
+  additionalS3EndpointUrls:
+    embargo: "https://sdfembs3.sdf.slac.stanford.edu"
diff --git a/applications/butler/values-usdfprod.yaml b/applications/butler/values-usdfprod.yaml
@@ -2,4 +2,6 @@ config:
   pguser: "rubin"
   s3EndpointUrl: "https://s3dfrgw.slac.stanford.edu"
   repositories:
-    embargo: "s3://rubin-summit-users/butler.yaml"
+    embargo: s3://embargo@rubin-summit-users/butler.yaml
+  additionalS3EndpointUrls:
+    embargo: "https://sdfembs3.sdf.slac.stanford.edu"
diff --git a/applications/butler/values.yaml b/applications/butler/values.yaml
@@ -93,13 +93,13 @@ config:
   # @default -- Use values specified in per-repository Butler config files.
   pguser: ""
 
-  # -- URL for the S3 service where files for datasets are stored by Butler.
+  # -- URL for the primary S3 service where files for datasets are stored by Butler.
   s3EndpointUrl: ""
 
-  # -- Profile name identifying a second S3 endpoint and set of credentials
-  # to use for accessing files in the datastore.
-  # @default -- No second S3 profile is available.
-  additionalS3ProfileName: ""
+  # -- Endpoint URLs for additional S3 services used by the Butler, as a
+  # mapping from profile name to URL.
+  # @default -- No additional URLs
+  additionalS3EndpointUrls: {}
 
   # -- The prefix of the path portion of the URL where the Butler service will
   # be exposed.  For example, if the service should be exposed at

diff --git a/applications/cert-manager/Chart.yaml b/applications/cert-manager/Chart.yaml
@@ -7,5 +7,5 @@ sources:
   - https://github.com/cert-manager/cert-manager
 dependencies:
   - name: cert-manager
-    version: v1.15.3
+    version: v1.16.1
     repository: https://charts.jetstack.io