Auto helm test

.github/workflows/docker.yaml

@@ -7,6 +7,7 @@ on:
       - '*docker*'
     tags:
       - 'v[0-9]+.[0-9]+.[0-9]+*' # Semver matching pattern with optional suffix
+  pull_request:
 
 permissions:
   packages: write
@@ -46,40 +47,42 @@ jobs:
     steps:
       # Get the repository's code
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
       - name: Login to Docker Hub
         uses: docker/login-action@v2
+        if:  ${{ github.event_name != 'pull_request' }}
         with:
           username: ${{ secrets.DOCKER_HUB_USERNAME }}
           password: ${{ secrets.DOCKER_HUB_PASSWORD }}
       - name: Login to GHCR
-        if: github.event_name != 'pull_request'
         uses: docker/login-action@v1
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
           password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
       - name: Docker meta
         id: arroyo-docker # you'll use this in the next step
-        uses: docker/metadata-action@v3
+        uses: docker/metadata-action@v4.6.0
         with:
           images: |
-            ghcr.io/arroyosystems/arroyo-${{ matrix.image_type }}
+            ghcr.io/${{ github.repository_owner }}/arroyo-${{ matrix.image_type }}
           tags: |
             type=schedule
             type=ref,event=branch
-            type=ref,event=pr
+            type=ref,event=pr,prefix=${{ matrix.prom_arch }}-pr-
             type=semver,pattern={{version}}
             type=semver,pattern={{major}}.{{minor}}
             type=semver,pattern={{major}}
-            type=sha
+            type=sha,prefix=${{ matrix.prom_arch }}-sha-
           flavor: |
             latest=false
             prefix=${{ matrix.prom_arch }}-
       - name: Set env
         run: echo "GIT_SHA=$(git rev-parse HEAD)" >> $GITHUB_ENV
       - name: Build and push
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v4.1.1
         with:
           context: .
           file: ${{ matrix.dockerfile }}
@@ -89,11 +92,14 @@ jobs:
             PROTO_ARCH=${{ matrix.proto_arch }}
             PROM_ARCH=${{ matrix.prom_arch }}
             GIT_SHA=${{ env.GIT_SHA }}
-          push: ${{ github.event_name != 'pull_request' }}
+          push: true
+          cache-from: type=registry,ref=user/app:latest
+          cache-to: type=inline
           tags: ${{ steps.arroyo-docker.outputs.tags }}
           labels: ${{ steps.arroyo-docker.outputs.labels }}
   manifest:
     needs: build
+    if:  ${{ github.event_name != 'pull_request' }}
     strategy:
       matrix:
         image_type:
@@ -114,7 +120,7 @@ jobs:
         uses: docker/metadata-action@v3
         with:
           images: |
-            ghcr.io/arroyosystems/arroyo-${{ matrix.image_type }}
+            ghcr.io/${{ github.repository_owner }}/arroyo-${{ matrix.image_type }}
           tags: |
             type=schedule
             type=ref,event=branch
@@ -126,7 +132,7 @@ jobs:
       - name: Create and push Docker manifest
         run: |
           TAG=${{ steps.arroyo-docker.outputs.version }}
-          IMAGE=ghcr.io/arroyosystems/arroyo-${{ matrix.image_type }}
+          IMAGE=ghcr.io/${{ github.repository_owner }}/arroyo-${{ matrix.image_type }}
           docker manifest create   ${IMAGE}:${TAG} ${IMAGE}:arm64-${TAG} ${IMAGE}:amd64-${TAG}
           docker manifest annotate ${IMAGE}:${TAG} ${IMAGE}:arm64-${TAG} --arch arm64
           docker manifest annotate ${IMAGE}:${TAG} ${IMAGE}:amd64-${TAG} --arch amd64
@@ -135,8 +141,74 @@ jobs:
         if: github.ref == 'refs/heads/master'
         run: |
           TAG=${{ steps.arroyo-docker.outputs.version }}
-          IMAGE=ghcr.io/arroyosystems/arroyo-${{ matrix.image_type }}
+          IMAGE=ghcr.io/${{ github.repository_owner }}/arroyo-${{ matrix.image_type }}
           docker manifest create   ${IMAGE}:tip ${IMAGE}:arm64-${TAG} ${IMAGE}:amd64-${TAG}
           docker manifest annotate ${IMAGE}:tip ${IMAGE}:arm64-${TAG} --arch arm64
           docker manifest annotate ${IMAGE}:tip ${IMAGE}:amd64-${TAG} --arch amd64
           docker manifest push     ${IMAGE}:tip
+  helm:
+    strategy:
+      matrix:
+        config:
+          - {runner: 'buildjet-8vcpu-ubuntu-2204', arch: 'amd64'}
+          - {runner: 'buildjet-8vcpu-ubuntu-2204-arm', arch: 'arm64'}
+    needs: build
+    runs-on: ${{ matrix.config.runner }}
+    if:  ${{ github.event_name == 'pull_request' }}
+    defaults:
+      run:
+        working-directory: k8s/arroyo
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Set up Helm
+        uses: azure/[email protected]
+        with:
+          version: v3.8.1
+
+      # Python is required because `ct lint` runs Yamale (https://github.com/23andMe/Yamale) and
+      # yamllint (https://github.com/adrienverge/yamllint) which require Python
+      # actions/setup-python doesn't yet support ARM
+      - if: ${{ !endsWith(matrix.config.runner, '-arm') }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.11"
+      - if: ${{ endsWith(matrix.config.runner, '-arm') }}
+        uses: deadsnakes/[email protected]
+        with:
+          python-version: "3.11"
+
+      - name: Set up chart-testing
+        ## 2.4.0 latest release doesn't work well with Arm
+        uses: helm/chart-testing-action@a629e844a9d081a827cf96486ef29f60afef87e4
+        with:
+          version: v3.9.0
+
+      - name: Lint chart
+        run: ct lint --all 
+
+      - name: Create kind cluster
+        uses: helm/[email protected]
+
+      - uses: benjlevesque/[email protected]
+        id: short-sha
+        with:
+          length: 7
+
+      - name: Run chart-testing (install)
+        run: |
+          ct install --charts ./ --config ct.yaml \
+          --helm-extra-set-args="\
+          --set=api.image.repository=ghcr.io/${{ github.repository_owner }}/arroyo-services \
+          --set=api.image.tag=${{ matrix.config.arch }}-sha-${{ steps.short-sha.outputs.sha }} \
+          --set=compiler.image.repository=ghcr.io/${{ github.repository_owner }}/arroyo-compiler \
+          --set=compiler.image.tag=${{ matrix.config.arch }}-sha-${{ steps.short-sha.outputs.sha }} \
+          --set=controller.image.repository=ghcr.io/${{ github.repository_owner }}/arroyo-services \
+          --set=controller.image.tag=${{ matrix.config.arch }}-sha-${{ steps.short-sha.outputs.sha }} \
+          --set=worker.image.repository=ghcr.io/${{ github.repository_owner }}/arroyo-worker \
+          --set=worker.image.tag=${{ matrix.config.arch }}-sha-${{ steps.short-sha.outputs.sha }}"
+

k8s/arroyo/Chart.lock

@@ -4,6 +4,6 @@ dependencies:
   version: 21.1.2
 - name: postgresql
   repository: https://charts.bitnami.com/bitnami
-  version: 12.4.2
-digest: sha256:4e49dd164fda3077ac0d4ff875ac9353c83ac71fcf5bbf62e2c0fa4dd003441f
-generated: "2023-04-30T16:36:45.355074578-07:00"
+  version: 12.6.8
+digest: sha256:cda91c8cc63c3f994cf54a9c6957af1e6617b5adf73e2f26afd601391c639c65
+generated: "2023-07-22T15:33:09.406298-07:00"

k8s/arroyo/ci/ci-test-values.yaml

@@ -0,0 +1 @@
+outputDir: /home

k8s/arroyo/ct.yaml

@@ -0,0 +1,3 @@
+chart-repos:
+  - prometheus=https://prometheus-community.github.io/helm-charts
+  - bitnami=https://charts.bitnami.com/bitnami

k8s/arroyo/templates/_helpers.tpl

@@ -127,18 +127,18 @@ Database environment variables
 Checkpoint / artifact storage env vars
 */}}
 {{- define "arroyo.storageEnvVars" -}}
-{{- if .Values.outputDir }}
+{{- if .Values.outputDir -}}
 - name: OUTPUT_DIR
   value: {{ .Values.outputDir }}
-{{- else if .Values.s3.bucket }}
+{{- else if .Values.s3.bucket -}}
 - name: S3_BUCKET
   value: {{ .Values.s3.bucket }}
-{{- if .Values.s3.region }}
+{{- if .Values.s3.region -}}
 - name: S3_REGION
   value: {{ .Values.s3.region }}
-{{- end }}
-{{- end }}
-{{- end }}
+{{- end -}}
+{{- end -}}
+{{- end -}}
 
 {{- define "tplvalues.render" -}}
     {{- if typeIs "string" .value }}

k8s/arroyo/templates/api.yaml

@@ -59,10 +59,12 @@ spec:
           httpGet:
             path: /status
             port: admin
+          initialDelaySeconds: 5
         readinessProbe:
           httpGet:
             path: /status
             port: admin
+          initialDelaySeconds: 5
         {{- if .Values.api.resources }}
         resources: {{- toYaml .Values.api.resources | nindent 12 }}
         {{- end }}
@@ -89,9 +91,9 @@ spec:
   ports:
     - name: http
       protocol: TCP
-      port: 80
+      port: {{ .Values.service.apiPort }}
       targetPort: 8000
     - name: grpc
       protocol: TCP
-      port: 8001
+      port: {{ .Values.service.grpcPort }}
       targetPort: 8001

k8s/arroyo/templates/compiler.yaml

@@ -48,10 +48,12 @@ spec:
           httpGet:
             path: /status
             port: admin
+          initialDelaySeconds: 5
         readinessProbe:
           httpGet:
             path: /status
             port: admin
+          initialDelaySeconds: 5
         {{- if .Values.compiler.resources }}
         resources: {{- toYaml .Values.compiler.resources | nindent 10 }}
         {{- end }}

k8s/arroyo/templates/controller.yaml

@@ -5,7 +5,6 @@ metadata:
   labels:
     {{- include "arroyo.labels" . | nindent 4 }}
     app: {{ include "arroyo.fullname" . }}-controller
-spec:
 spec:
   replicas: 1
   selector:
@@ -99,10 +98,12 @@ spec:
           httpGet:
             path: /status
             port: admin
+          initialDelaySeconds: 5
         readinessProbe:
           httpGet:
             path: /status
             port: admin
+          initialDelaySeconds: 5
         {{- if .Values.compiler.resources }}
         resources: {{- toYaml .Values.compiler.resources | nindent 10 }}
         {{- end }}

k8s/arroyo/templates/tests/test-connection.yaml → k8s/arroyo/templates/tests/test-console-connection.yaml

@@ -11,5 +11,5 @@ spec:
     - name: wget
       image: busybox
       command: ['wget']
-      args: ['{{ include "arroyo.fullname" . }}-api:{{ .Values.service.port }}']
+      args: ['{{ include "arroyo.fullname" . }}-api:{{ .Values.service.apiPort }}']
   restartPolicy: Never

k8s/arroyo/templates/tests/test-grpc-connection.yaml

@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: "{{ include "arroyo.fullname" . }}-test-connection"
+  labels:
+    {{- include "arroyo.labels" . | nindent 4 }}
+  annotations:
+    "helm.sh/hook": test
+spec:
+  containers:
+    - name: grpcurl
+      image: fullstorydev/grpcurl:v1.8.7-alpine
+      args: ['-plaintext', '{{ include "arroyo.fullname" . }}-api:{{ .Values.service.grpcPort }}','list']
+  restartPolicy: Never

k8s/arroyo/values.yaml

@@ -140,7 +140,9 @@ securityContext: {}
 
 service:
   type: ClusterIP
-  port: 80
+  apiPort: 80
+  grpcPort: 8001
+
 
 apiResources: {}
 controllerResources: {}