Changes for in-kind contribution JAP-JPG-S3 from NAOJ

.pre-commit-config.yaml

@@ -16,9 +16,6 @@ repos:
   - repo: https://github.com/python-jsonschema/check-jsonschema
     rev: 0.30.0
     hooks:
-      - id: check-jsonschema
-        files: ^applications/.*/secrets(-[^./-]+)?\.yaml
-        args: [--schemafile, docs/extras/schemas/secrets.json]
       - id: check-jsonschema
         files: ^environments/values(-[^./-]+)?\.yaml
         args: [--schemafile, docs/extras/schemas/environment.json]

applications/fov-quicklook/.helmignore

@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

applications/fov-quicklook/Chart.yaml

@@ -0,0 +1,8 @@
+apiVersion: v2
+appVersion: 0.1.0
+description: Full focal plane viewer
+name: fov-quicklook
+sources:
+- https://github.com/michitaro/rubin-fov-quicklook
+type: application
+version: 1.0.0

applications/fov-quicklook/README.md

@@ -0,0 +1,31 @@
+# fov-quicklook
+
+Full focal plane viewer
+
+## Source Code
+
+* <https://github.com/michitaro/rubin-fov-quicklook>
+
+## Values
+
+| Key | Type | Default | Description |
+|-----|------|---------|-------------|
+| config.pathPrefix | string | `"/fov-quicklook"` | URL path prefix |
+| coordinator.resources.limits | object | `{"cpu":"4000m","memory":"256Mi"}` | Resource limits for the coordinator |
+| coordinator.resources.requests | object | `{"cpu":"100m","memory":"256Mi"}` | Resource requests for the coordinator |
+| db.resources.limits | object | `{"cpu":"2000m","memory":"256Mi"}` | Resource limits for the database |
+| db.resources.requests | object | `{"cpu":"100m","memory":"256Mi"}` | Resource requests for the database |
+| db_storage_class | string | `nil` | Storage class to use for the database |
+| frontend.resources.limits | object | `{"cpu":"8000m","memory":"256Mi"}` | Resource limits for the frontend |
+| frontend.resources.requests | object | `{"cpu":"100m","memory":"256Mi"}` | Resource requests for the frontend |
+| generator.replicas | int | `8` | Number of replicas for the generator |
+| generator.resources.limits | object | `{"cpu":"16000m","memory":"32Gi"}` | Resource limits for the generator |
+| generator.resources.requests | object | `{"cpu":"100m","memory":"32Gi"}` | Resource requests for the generator |
+| generator.workdir.medium | string | `"Memory"` | Work directory type for the generator |
+| image.pullPolicy | string | `"Always"` | Pull policy for the fov-quicklook image |
+| image.repository | string | `"ghcr.io/michitaro/rubin-fov-viewer"` | Image to use in the fov-quicklook deployment |
+| image.tag | string | `"latest"` | Tag of image to use |
+| s3_repository | object | `{"bucket":"fov-quicklook-repository","endpoint":"sdfembs3.sdf.slac.stanford.edu:443","secure":true}` | S3 configuration for the repository |
+| s3_tile | object | `{"bucket":"fov-quicklook-tile","endpoint":"sdfembs3.sdf.slac.stanford.edu:443","secure":true}` | S3 configuration for the tile storage |
+| use_gafaelfawr | bool | `true` | Use gafaelfawr to authenticate |
+| use_vault | bool | `true` | Use vault to store secrets |

applications/fov-quicklook/secrets.yaml

@@ -0,0 +1,30 @@
+s3_tile_access_key:
+  description: >-
+    The access key for the S3 bucket that contains the tile data.
+s3_tile_secret_key:
+  description: >-
+    The secret key for the S3 bucket that contains the tile data.
+s3_repository_access_key:
+  description: >-
+    The access key for the S3 bucket that contains the repository data.
+s3_repository_secret_key:
+  description: >-
+    The secret key for the S3 bucket that contains the repository data.
+db_password:
+  description: >-
+    The password for the database user.
+  generate:
+    type: password
+"aws-credentials.ini":
+  description: >-
+    Google Cloud Storage credentials to the Butler data store, formatted using
+    AWS syntax for use with boto.
+  copy:
+    application: nublado
+    key: aws-credentials.ini
+"postgres-credentials.txt":
+  description: >-
+    PostgreSQL credentials in its pgpass format for the Butler database.
+  copy:
+    application: nublado
+    key: postgres-credentials.txt

applications/fov-quicklook/templates/_helpers.tpl

@@ -0,0 +1,70 @@
+{{- define "fov-quicklook.env.s3_tile" -}}
+- name: QUICKLOOK_s3_tile
+  value: {{ .Values.s3_tile | toJson | quote }}
+- name: QUICKLOOK_s3_tile__access_key
+  valueFrom:
+    secretKeyRef:
+      name: fov-quicklook
+      key: s3_tile_access_key
+- name: QUICKLOOK_s3_tile__secret_key
+  valueFrom:
+    secretKeyRef:
+      name: fov-quicklook
+      key: s3_tile_secret_key
+{{- end }}
+
+{{- define "fov-quicklook.env.s3_repository" -}}
+- name: QUICKLOOK_s3_repository
+  value: {{ .Values.s3_repository | toJson | quote }}
+- name: QUICKLOOK_s3_repository__access_key
+  valueFrom:
+    secretKeyRef:
+      name: fov-quicklook
+      key: s3_repository_access_key
+- name: QUICKLOOK_s3_repository__secret_key
+  valueFrom:
+    secretKeyRef:
+      name: fov-quicklook
+      key: s3_repository_secret_key
+{{- end }}
+
+{{- define "quicklook.ingress.spec" -}}
+rules:
+  - http:
+      paths:
+        - path: {{ .Values.config.pathPrefix }}
+          pathType: Prefix
+          backend:
+            service:
+              name: fov-quicklook-frontend
+              port:
+                number: 9500
+{{- end -}}
+
+{{- define "quicklook.butler-settings.env" -}}
+- name: AWS_SHARED_CREDENTIALS_FILE
+  value: /var/run/secrets/aws-credentials.ini
+- name: PGPASSFILE
+  value: /var/run/secrets/postgres-credentials.txt
+- name: PGUSER
+  value: rubin
+- name: LSST_RESOURCES_S3_PROFILE_embargo
+  value: https://sdfembs3.sdf.slac.stanford.edu
+- name: DAF_BUTLER_REPOSITORY_INDEX
+  value: s3://embargo@rubin-summit-users/data-repos.yaml
+{{- end -}}
+
+{{- define "quicklook.butler-settings.volumes" -}}
+- name: butler-settings
+  secret:
+    secretName: fov-quicklook
+{{- end -}}
+
+{{- define "quicklook.butler-settings.volumeMounts" -}}
+- name: butler-settings
+  mountPath: /var/run/secrets/aws-credentials.ini
+  subPath: aws-credentials.ini
+- name: butler-settings
+  mountPath: /var/run/secrets/postgres-credentials.txt
+  subPath: postgres-credentials.txt
+{{- end -}}

applications/fov-quicklook/templates/coordinator.yaml

@@ -0,0 +1,84 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: fov-quicklook-coordinator
+spec:
+  strategy:
+    type: Recreate
+  replicas: 1
+  selector:
+    matchLabels:
+      app: fov-quicklook-coordinator
+  template:
+    metadata:
+      labels:
+        app: fov-quicklook-coordinator
+    spec:
+      containers:
+        - name: fov-quicklook-coordinator
+          image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+          imagePullPolicy: {{ .Values.image.pullPolicy | quote }}
+          command:
+            - sh
+            - -c
+            - |
+              set -e
+              alembic upgrade head
+              exec python -m quicklook.coordinator.api
+          env:
+            - name: DB_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: fov-quicklook
+                  key: db_password
+            - name: QUICKLOOK_db_url
+              value: postgresql://quicklook:$(DB_PASSWORD)@fov-quicklook-db:5432/quicklook
+            {{- include "fov-quicklook.env.s3_tile" . | nindent 12 }}
+            {{- include "fov-quicklook.env.s3_repository" . | nindent 12 }}
+            {{- include "quicklook.butler-settings.env" . | nindent 12 }}
+          ports:
+            - containerPort: 9501
+          securityContext:
+            runAsNonRoot: true
+            runAsUser: 1000
+            runAsGroup: 1000
+          resources: {{ toYaml .Values.coordinator.resources | nindent 12 }}
+          volumeMounts:
+            {{- include "quicklook.butler-settings.volumeMounts" . | nindent 12 }}
+      volumes:
+        {{- include "quicklook.butler-settings.volumes" . | nindent 8 }}
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: fov-quicklook-coordinator
+spec:
+  selector:
+    app: fov-quicklook-coordinator
+  type: ClusterIP
+  ports:
+    - name: http
+      protocol: TCP
+      port: 9501
+      targetPort: 9501
+---
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: fov-quicklook-coordinator-policy
+spec:
+  podSelector:
+    matchLabels:
+      app: fov-quicklook-coordinator
+  ingress:
+    - from:
+        - podSelector:
+            matchLabels:
+              app: fov-quicklook-generator
+        - podSelector:
+            matchLabels:
+              app: fov-quicklook-frontend
+      ports:
+        - protocol: TCP
+          port: 9501

applications/fov-quicklook/templates/db.yaml

@@ -0,0 +1,94 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: fov-quicklook-db
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: fov-quicklook-db
+  template:
+    metadata:
+      labels:
+        app: fov-quicklook-db
+    spec:
+      initContainers:
+        - name: init-permissions
+          image: 'busybox'
+          command: ['sh', '-c', 'chown 999:999 /var/lib/postgresql/data']
+          volumeMounts:
+            - name: fov-quicklook-db
+              mountPath: /var/lib/postgresql/data
+      containers:
+        - name: fov-quicklook-db
+          image: 'postgres:16'
+          env:
+            - name: POSTGRES_USER
+              value: quicklook
+            - name: POSTGRES_DB
+              value: quicklook
+            - name: POSTGRES_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: fov-quicklook
+                  key: db_password
+          ports:
+            - containerPort: 5432
+          volumeMounts:
+            - name: fov-quicklook-db
+              mountPath: /var/lib/postgresql/data
+          securityContext:
+            runAsNonRoot: true
+            runAsUser: 999
+            runAsGroup: 999
+          resources: {{ toYaml .Values.db.resources | nindent 12 }}
+      volumes:
+        - name: fov-quicklook-db
+          persistentVolumeClaim:
+            claimName: fov-quicklook-db
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: fov-quicklook-db
+spec:
+  selector:
+    app: fov-quicklook-db
+  type: ClusterIP
+  clusterIP: None
+  ports:
+    - name: postgres
+      protocol: TCP
+      port: 5432
+      targetPort: 5432
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: fov-quicklook-db
+spec:
+  {{- if .Values.db_storage_class }}
+  storageClassName: {{ .Values.db_storage_class }}
+  {{- end }}
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 10Gi
+---
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: fov-quicklook-db-policy
+spec:
+  podSelector:
+    matchLabels:
+      app: fov-quicklook-db
+  ingress:
+    - from:
+        - podSelector:
+            matchLabels:
+              app: fov-quicklook-coordinator
+      ports:
+        - protocol: TCP
+          port: 5432