Merge updated main into rspprod

.github/workflows/docs.yaml

@@ -40,6 +40,7 @@ jobs:
               - "applications/argocd/values-*.yaml"
               - "applications/gafaelfawr/values-*.yaml"
               - "environments/values-*.yaml"
+              - "requirements/*.txt"
               - "src/phalanx/**"
             docsSpecific:
               - "docs/**"

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.6.0
+    rev: v5.0.0
     hooks:
       - id: check-merge-conflict
       - id: check-toml
@@ -14,7 +14,7 @@ repos:
           - -c=.yamllint.yml
 
   - repo: https://github.com/python-jsonschema/check-jsonschema
-    rev: 0.29.2
+    rev: 0.30.0
     hooks:
       - id: check-jsonschema
         files: ^applications/.*/secrets(-[^./-]+)?\.yaml
@@ -46,14 +46,14 @@ repos:
           - --template-files=../helm-docs.md.gotmpl
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.6.7
+    rev: v0.8.1
     hooks:
       - id: ruff
         args: [--fix, --exit-non-zero-on-fix]
       - id: ruff-format
 
   - repo: https://github.com/adamchainz/blacken-docs
-    rev: 1.18.0
+    rev: 1.19.1
     hooks:
       - id: blacken-docs
         additional_dependencies: [black==23.7.0]

applications/alert-stream-broker/README.md

@@ -32,7 +32,6 @@ Alert transmission to community brokers
 | alert-database.ingester.serviceAccountName | string | `"alert-database-ingester"` | The name of the Kubernetes ServiceAccount (*not* the Google Cloud IAM service account!) which is used by the alert database ingester. |
 | alert-database.ingress.annotations | object | `{}` |  |
 | alert-database.ingress.enabled | bool | `true` | Whether to create an ingress |
-| alert-database.ingress.gafaelfawrAuthQuery | string | `"scope=read:alertdb"` | Query string for Gafaelfawr to authorize access |
 | alert-database.ingress.host | string | None, must be set if the ingress is enabled | Hostname for the ingress |
 | alert-database.ingress.path | string | `"/alertdb"` | Subpath to host the alert database application under the ingress |
 | alert-database.ingress.tls | list | `[]` | Configures TLS for the ingress if needed. If multiple ingresses share the same hostname, only one of them needs a TLS configuration. |

applications/alert-stream-broker/charts/alert-database/README.md

@@ -23,7 +23,6 @@ Archival database of alerts sent through the alert stream.
 | ingester.serviceAccountName | string | `"alert-database-ingester"` | The name of the Kubernetes ServiceAccount (*not* the Google Cloud IAM service account!) which is used by the alert database ingester. |
 | ingress.annotations | object | `{}` |  |
 | ingress.enabled | bool | `true` | Whether to create an ingress |
-| ingress.gafaelfawrAuthQuery | string | `"scope=read:alertdb"` | Query string for Gafaelfawr to authorize access |
 | ingress.host | string | None, must be set if the ingress is enabled | Hostname for the ingress |
 | ingress.path | string | `"/alertdb"` | Subpath to host the alert database application under the ingress |
 | ingress.tls | list | `[]` | Configures TLS for the ingress if needed. If multiple ingresses share the same hostname, only one of them needs a TLS configuration. |

applications/alert-stream-broker/charts/alert-database/templates/ingress.yaml

@@ -1,38 +1,46 @@
 {{- if .Values.ingress.enabled -}}
-apiVersion: networking.k8s.io/v1
-kind: Ingress
+apiVersion: gafaelfawr.lsst.io/v1alpha1
+kind: GafaelfawrIngress
 metadata:
-  annotations:
-    kubernetes.io/ingress.class: "nginx"
-    nginx.ingress.kubernetes.io/rewrite-target: /$2
-    nginx.ingress.kubernetes.io/auth-method: "GET"
-    nginx.ingress.kubernetes.io/auth-url: "http://gafaelfawr.gafaelfawr.svc.cluster.local:8080/auth?{{ required "ingress.gafaelfawrAuthQuery must be set" .Values.ingress.gafaelfawrAuthQuery }}"
-    {{- with .Values.ingress.annotations }}
-    {{- toYaml . | nindent 4 }}
-    {{- end }}
   name: {{ template "alertDatabase.fullname" . }}
   labels:
     {{- include "alertDatabase.labels" . | nindent 4 }}
-spec:
-  rules:
-    - host: {{ required "ingress.host must be set" .Values.ingress.host | quote }}
-      http:
-        paths:
-          - path: "{{ .Values.ingress.path }}(/|$)(.*)"
-            pathType: Prefix
-            backend:
-              service:
-                name: {{ template "alertDatabase.fullname" . }}
-                port:
-                  name: http
-  {{- if .Values.ingress.tls }}
-  tls:
-    {{- range .Values.ingress.tls }}
-    - hosts:
-        {{- range .hosts }}
-        - {{ . | quote }}
-        {{- end }}
-      secretName: {{ .secretName }}
+config:
+  baseUrl: {{ .Values.global.baseUrl | quote }}
+  scopes:
+    all:
+      - "read:alertdb"
+  service: "alert-stream-broker"
+template:
+  metadata:
+    name: {{ template "alertDatabase.fullname" . }}
+    annotations:
+      nginx.ingress.kubernetes.io/rewrite-target: "/$2"
+      {{- with .Values.ingress.annotations }}
+      {{- toYaml . | nindent 6 }}
+      {{- end }}
+    labels:
+      {{- include "alertDatabase.labels" . | nindent 4 }}
+  spec:
+    rules:
+      - host: {{ required "ingress.host must be set" .Values.ingress.host | quote }}
+        http:
+          paths:
+            - path: "{{ .Values.ingress.path }}(/|$)(.*)"
+              pathType: ImplementationSpecific
+              backend:
+                service:
+                  name: {{ template "alertDatabase.fullname" . }}
+                  port:
+                    name: http
+    {{- if .Values.ingress.tls }}
+    tls:
+      {{- range .Values.ingress.tls }}
+      - hosts:
+          {{- range .hosts }}
+          - {{ . | quote }}
+          {{- end }}
+        secretName: {{ .secretName }}
+      {{- end }}
     {{- end }}
-  {{- end }}
 {{- end }}

applications/alert-stream-broker/charts/alert-database/values.yaml

@@ -101,6 +101,3 @@ ingress:
 
   # -- Subpath to host the alert database application under the ingress
   path: "/alertdb"
-
-  # -- Query string for Gafaelfawr to authorize access
-  gafaelfawrAuthQuery: "scope=read:alertdb"

applications/alert-stream-broker/charts/alert-stream-broker/templates/kafka-topics.yaml

@@ -93,4 +93,24 @@ spec:
     # timestamp instead
     message.timestamp.type: 'LogAppendTime'
   partitions: {{ .Values.latissTopicPartitions }}
-  replicas: {{ .Values.latissTopicReplicas }}
+  replicas: {{ .Values.latissTopicReplicas }}
+---
+apiVersion: "kafka.strimzi.io/{{ .Values.strimziAPIVersion }}"
+kind: KafkaTopic
+metadata:
+  labels:
+    strimzi.io/cluster: "{{ .Values.clusterName }}"
+  name: "{{ .Values.comcamTopicName}}"
+spec:
+  config:
+    cleanup.policy: "delete"
+    retention.ms: {{ .Values.maxMillisecondsRetained }} # 7 days
+    retention.bytes: {{ .Values.maxBytesRetained }}
+    compression.type: {{ .Values.topicCompression }}
+    # The default timestamp is the creation time of the alert.
+    # To get the ingestion rate, we need this to be the log
+    # append time, and the header will contain the producer
+    # timestamp instead
+    message.timestamp.type: 'LogAppendTime'
+  partitions: {{ .Values.comcamTopicPartitions }}
+  replicas: {{ .Values.comcamTopicReplicas }}

applications/alert-stream-broker/values-usdfdev-alert-stream-broker.yaml

@@ -17,7 +17,7 @@ alert-stream-broker:
         enabled: false
       bootstrap:
         host: usdf-alert-stream-dev.lsst.cloud
-        ip: ""
+        ip: "134.79.23.185"
         annotations:
           metallb.universe.tf/address-pool: 'sdf-dmz'
       brokers:
@@ -121,6 +121,10 @@ alert-stream-broker:
   latissTopicPartitions: 45
   latissTopicReplicas: 1
 
+  comcamTopicName: lsstcomcam-alerts
+  comcamTopicPartitions: 45
+  comcamTopicReplicas: 1
+
   # Compression set to snappy to balance alert packet compression speed and size.
   topicCompression: snappy
 

applications/alert-stream-broker/values-usdfdev.yaml

@@ -110,7 +110,6 @@ alert-database:
   ingress:
     enabled: true
     host: "usdf-rsp-dev.slac.stanford.edu"
-    gafaelfawrAuthQuery: "scope=read:alertdb"
 
   storage:
     gcp:

applications/argo-workflows/Chart.yaml

@@ -8,5 +8,5 @@ sources:
   - https://github.com/argoproj/argo-helm
 dependencies:
   - name: argo-workflows
-    version: 0.42.3
+    version: 0.45.0
     repository: https://argoproj.github.io/argo-helm

applications/argo-workflows/templates/ingress.yaml

@@ -4,9 +4,10 @@ metadata:
   name: argo-workflows
 config:
   baseUrl: {{ .Values.global.baseUrl | quote }}
+  loginRedirect: true
   scopes:
     all: {{ .Values.ingress.scopes }}
-  loginRedirect: true
+  service: "argo-workflows"
 template:
   metadata:
     name: argo-workflows

applications/argocd/Chart.yaml

@@ -8,5 +8,5 @@ sources:
   - https://github.com/argoproj/argo-helm
 dependencies:
   - name: argo-cd
-    version: 7.6.1
+    version: 7.7.7
     repository: https://argoproj.github.io/argo-helm

applications/argocd/values-usdf-cm-dev.yaml

@@ -29,6 +29,8 @@ argo-cd:
         g, [email protected], role:admin
         g, [email protected], role:admin
         g, [email protected], role:admin
+        g, [email protected], role:admin
+        g, [email protected], role:admin
       scopes: "[email]"
 
   server:

applications/argocd/values-usdf-cm.yaml

@@ -29,6 +29,7 @@ argo-cd:
         g, [email protected], role:admin
         g, [email protected], role:admin
         g, [email protected], role:admin
+        g, [email protected], role:admin
       scopes: "[email]"
 
   server:

applications/argocd/values-usdfint.yaml

@@ -58,6 +58,7 @@ argo-cd:
         g, [email protected], role:developer
         g, [email protected], role:developer
         g, [email protected], role:developer
+        g, [email protected], role:developer
       scopes: "[email]"
 
   server:

applications/argocd/values-usdfprod.yaml

@@ -34,6 +34,7 @@ argo-cd:
 
         g, [email protected], role:admin
         g, [email protected], role:admin
+        g, [email protected], role:admin
         g, [email protected], role:admin
         g, [email protected], role:admin
         g, [email protected], role:admin
@@ -54,6 +55,7 @@ argo-cd:
         g, [email protected], role:developer
         g, [email protected], role:developer
         g, [email protected], role:developer
+        g, [email protected], role:developer
       scopes: "[email]"
 
   server:

applications/auxtel/README.md

@@ -28,8 +28,12 @@ Deployment for the Auxiliary Telescope CSCs
 | atspectrograph-sim.enabled | bool | `false` | Enable the ATSpectograph simulator CSC |
 | atspectrograph.enabled | bool | `false` | Enable the ATSpectrograph CSC |
 | hexapod-sim.enabled | bool | `false` | Enable the hexapod controller simulator |
+| hexapod-sim.affinity | object | `{}` | This specifies the scheduling constraints of the pod |
 | hexapod-sim.image | object | `{"pullPolicy":"Always","repository":"ts-dockerhub.lsst.org/hexapod_simulator","tag":"latest"}` | This section holds the configuration of the container image |
 | hexapod-sim.image.pullPolicy | string | `"Always"` | The policy to apply when pulling an image for deployment |
 | hexapod-sim.image.repository | string | `"ts-dockerhub.lsst.org/hexapod_simulator"` | The Docker registry name of the container image |
 | hexapod-sim.image.tag | string | `"latest"` | The tag of the container image |
 | hexapod-sim.namespace | string | `"auxtel"` | This is the namespace in which the hexapod controller simulator will be placed |
+| hexapod-sim.nodeSelector | object | `{}` | This allows the specification of using specific nodes to run the pod |
+| hexapod-sim.resources | object | `{}` | This allows the specification of resources (CPU, memory) requires to run the container |
+| hexapod-sim.tolerations | list | `[]` | This specifies the tolerations of the pod for any system taints |

applications/auxtel/charts/hexapod-sim/README.md

@@ -6,8 +6,12 @@ Chart for the hexapod simulator that supports the ATHexapod
 
 | Key | Type | Default | Description |
 |-----|------|---------|-------------|
+| affinity | object | `{}` | This specifies the scheduling constraints of the pod |
 | image | object | `{"pullPolicy":"Always","repository":"ts-dockerhub.lsst.org/hexapod_simulator","tag":"latest"}` | This section holds the configuration of the container image |
 | image.pullPolicy | string | `"Always"` | The policy to apply when pulling an image for deployment |
 | image.repository | string | `"ts-dockerhub.lsst.org/hexapod_simulator"` | The Docker registry name of the container image |
 | image.tag | string | `"latest"` | The tag of the container image |
 | namespace | string | `"auxtel"` | This is the namespace in which the hexapod controller simulator will be placed |
+| nodeSelector | object | `{}` | This allows the specification of using specific nodes to run the pod |
+| resources | object | `{}` | This allows the specification of resources (CPU, memory) requires to run the container |
+| tolerations | list | `[]` | This specifies the tolerations of the pod for any system taints |

applications/auxtel/charts/hexapod-sim/templates/deployment.yaml

@@ -23,5 +23,21 @@ spec:
           imagePullPolicy: {{ .Values.image.pullPolicy }}
           stdin: true
           tty: true
+          {{- with .Values.resources }}
+          resources:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
       imagePullSecrets:
         - name: pull-secret
+      {{- with .Values.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}

applications/auxtel/charts/hexapod-sim/values.yaml

@@ -8,3 +8,11 @@ image:
   tag: latest
   # -- The policy to apply when pulling an image for deployment
   pullPolicy: Always
+# -- This allows the specification of resources (CPU, memory) requires to run the container
+resources: {}
+# -- This allows the specification of using specific nodes to run the pod
+nodeSelector: {}
+# -- This specifies the tolerations of the pod for any system taints
+tolerations: []
+# -- This specifies the scheduling constraints of the pod
+affinity: {}

applications/auxtel/values-base.yaml

@@ -45,6 +45,8 @@ atmcs-sim:
   image:
     repository: ts-dockerhub.lsst.org/atmcs_sim
     pullPolicy: Always
+  env:
+    RUN_ARG: --simulate
 
 atoods:
   image:
@@ -126,6 +128,8 @@ atpneumatics-sim:
   image:
     repository: ts-dockerhub.lsst.org/at_pneumatics_sim
     pullPolicy: Always
+  env:
+    RUN_ARG: --simulate
 
 atptg:
   image:
@@ -144,3 +148,10 @@ atspectrograph-sim:
 
 hexapod-sim:
   enabled: true
+  resources:
+    limits:
+      cpu: 100m
+      memory: 500Mi
+    requests:
+      cpu: 20m
+      memory: 200Mi

applications/auxtel/values-tucson-teststand.yaml

@@ -45,6 +45,8 @@ atmcs-sim:
   image:
     repository: ts-dockerhub.lsst.org/atmcs_sim
     pullPolicy: Always
+  env:
+    RUN_ARG: --simulate
 
 atoods:
   image:
@@ -126,6 +128,8 @@ atpneumatics-sim:
   image:
     repository: ts-dockerhub.lsst.org/at_pneumatics_sim
     pullPolicy: Always
+  env:
+    RUN_ARG: --simulate
 
 atptg:
   image:

applications/butler/Chart.yaml

@@ -4,4 +4,4 @@ version: 1.0.0
 description: Server for Butler data abstraction service
 sources:
   - https://github.com/lsst/daf_butler
-appVersion: server-2.1.0
+appVersion: server-2.3.0

applications/butler/README.md

@@ -15,13 +15,13 @@ Server for Butler data abstraction service
 | autoscaling.maxReplicas | int | `100` | Maximum number of butler deployment pods |
 | autoscaling.minReplicas | int | `1` | Minimum number of butler deployment pods |
 | autoscaling.targetCPUUtilizationPercentage | int | `80` | Target CPU utilization of butler deployment pods |
-| config.additionalS3ProfileName | string | No second S3 profile is available. | Profile name identifying a second S3 endpoint and set of credentials to use for accessing files in the datastore. |
+| config.additionalS3EndpointUrls | object | No additional URLs | Endpoint URLs for additional S3 services used by the Butler, as a mapping from profile name to URL. |
 | config.dp02ClientServerIsDefault | bool | `false` | True if the 'dp02' Butler repository alias should use client/server Butler.  False if it should use DirectButler. |
 | config.dp02PostgresUri | string | No configuration file for DP02 will be generated. | Postgres connection string pointing to the registry database hosting Data Preview 0.2 data. |
 | config.pathPrefix | string | `"/api/butler"` | The prefix of the path portion of the URL where the Butler service will be exposed.  For example, if the service should be exposed at `https://data.lsst.cloud/api/butler`, this should be set to `/api/butler` |
 | config.pguser | string | Use values specified in per-repository Butler config files. | Postgres username used to connect to the Butler DB |
 | config.repositories | object | `{}` | Mapping from Butler repository label to Butler configuration URI for repositories which will be hosted by this server. |
-| config.s3EndpointUrl | string | `""` | URL for the S3 service where files for datasets are stored by Butler. |
+| config.s3EndpointUrl | string | `""` | URL for the primary S3 service where files for datasets are stored by Butler. |
 | global.baseUrl | string | Set by Argo CD | Base URL for the environment |
 | global.host | string | Set by Argo CD | Host name for ingress |
 | global.vaultSecretsPath | string | Set by Argo CD | Base path for Vault secrets |