From 70a7baf41a447c1f08b97955c33bc6338a82e23a Mon Sep 17 00:00:00 2001
From: Daichi Sakaue <daichi-sakaue@cybozu.co.jp>
Date: Wed, 27 Nov 2024 15:20:07 +0900
Subject: [PATCH] wip: dynamic CI matrix

Signed-off-by: Daichi Sakaue <daichi-sakaue@cybozu.co.jp>
---
 .github/actions/prepare_matrix/action.yaml |  31 ++
 .github/workflows/main.yaml                | 513 ++++++++++-----------
 generate_matrix                            |  27 ++
 squid/TAG                                  |   2 +-
 4 files changed, 304 insertions(+), 269 deletions(-)
 create mode 100644 .github/actions/prepare_matrix/action.yaml
 create mode 100755 generate_matrix

diff --git a/.github/actions/prepare_matrix/action.yaml b/.github/actions/prepare_matrix/action.yaml
new file mode 100644
index 000000000..2a2cc5ca3
--- /dev/null
+++ b/.github/actions/prepare_matrix/action.yaml
@@ -0,0 +1,31 @@
+name: "Prepare build matrix"
+description: "Prepare build matrix"
+inputs:
+  matrix:
+    description: "Full matrix candidate"
+    required: true
+    default: ""
+outputs:
+  jobs:
+    description: "matrix"
+    value: ${{ steps.prepare-matrix.outputs.jobs }}
+
+runs:
+  using: composite
+  steps:
+    - name: Check and output changed files
+      id: changed-files
+      uses: tj-actions/changed-files@a29e8b565651ce417abb5db7164b4a2ad8b6155c # v44.4.0
+      with:
+        diff_relative: 'false'
+        files_ignore: |
+          **/*.md
+        separator: ','
+        write_output_files: 'true'
+    - id: prepare-matrix
+      shell: bash
+      env:
+        INPUT_MATRIX: ${{ inputs.matrix }}
+      run: |
+        ./generate_matrix
+        echo "jobs=$(cat .github/outputs/__matrix.json)" >> $GITHUB_OUTPUT
diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml
index f1160915f..e50754477 100644
--- a/.github/workflows/main.yaml
+++ b/.github/workflows/main.yaml
@@ -6,244 +6,254 @@ on:
       - "main"
       - "ceph-v*"
 jobs:
+  prepare_matrix:
+    runs-on: ubuntu-22.04
+    outputs:
+      jobs: ${{ steps.prepare-matrix.outputs.jobs }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/prepare_matrix
+        id: prepare-matrix
+        with:
+          matrix: |
+            - dir: "./admission"
+              container-image: "neco-admission"
+              make-targets: "check-generate test"
+            - dir: "./alertmanager"
+              container-image: "alertmanager"
+            - dir: "./argocd"
+              container-image: "argocd"
+            - dir: "./argocd-image-updater"
+              container-image: "argocd-image-updater"
+            - dir: "./bird"
+              container-image: "bird"
+            - dir: "./blackbox_exporter"
+              container-image: "blackbox_exporter"
+            - dir: "./bmc-reverse-proxy"
+              container-image: "bmc-reverse-proxy"
+              make-targets: "setup check-generate test"
+            - dir: "./bmc-log-collector"
+              container-image: "bmc-log-collector"
+              make-targets: "setup check-generate test"
+            - dir: "./bpf-map-pressure-exporter"
+              container-image: "bpf-map-pressure-exporter"
+              make-targets: "check-generate test"
+            - dir: "./cadvisor"
+              container-image: "cadvisor"
+            - dir: "./cephcsi"
+              container-image: "cephcsi"
+              request-scan: "false"
+            - dir: "./cep-checker"
+              container-image: "cep-checker"
+              make-targets: "test"
+            - dir: "./ceph-extra-exporter"
+              container-image: "ceph-extra-exporter"
+              make-targets: "check-generate test"
+            - dir: "./cert-manager"
+              container-image: "cert-manager"
+            - dir: "./chrony"
+              container-image: "chrony"
+            - dir: "./cilium-certgen"
+              container-image: "cilium-certgen"
+              request-scan: "false"
+            - dir: "./cilium-operator-generic"
+              container-image: "cilium-operator-generic"
+              request-scan: "false"
+            - dir: "./configmap-reload"
+              container-image: "configmap-reload"
+            - dir: "./contour"
+              container-image: "contour"
+            - dir: "./coredns"
+              container-image: "coredns"
+            - dir: "csi-attacher"
+              container-image: "csi-attacher"
+            - dir: "csi-node-driver-registrar"
+              container-image: "csi-node-driver-registrar"
+            - dir: "csi-provisioner"
+              container-image: "csi-provisioner"
+            - dir: "csi-resizer"
+              container-image: "csi-resizer"
+            - dir: "csi-snapshotter"
+              container-image: "csi-snapshotter"
+            - dir: "./dex"
+              container-image: "dex"
+            - dir: "./etcd"
+              container-image: "etcd"
+            - dir: "./external-dns"
+              container-image: "external-dns"
+            - dir: "./fluent-bit"
+              container-image: "fluent-bit"
+            - dir: "./golang-all/golang-1.22-jammy"
+              container-image: "golang"
+              enable_arm64: true
+              request-scan: "true"
+            - dir: "./golang-all/golang-1.23-jammy"
+              container-image: "golang"
+              enable_arm64: true
+              request-scan: "true"
+            - dir: "./golang-all/golang-1.22-noble"
+              container-image: "golang"
+              enable_arm64: true
+              request-scan: "true"
+            - dir: "./golang-all/golang-1.23-noble"
+              container-image: "golang"
+              enable_arm64: true
+              request-scan: "true"
+            - dir: "./grafana"
+              container-image: "grafana"
+            - dir: "./grafana-operator"
+              container-image: "grafana-operator"
+            - dir: "./haproxy"
+              container-image: "haproxy"
+            - dir: "./heartbeat"
+              container-image: "heartbeat"
+              make-targets: "setup check-generate test"
+            - dir: "./kube-metrics-adapter"
+              container-image: "kube-metrics-adapter"
+            - dir: "./kube-state-metrics"
+              container-image: "kube-state-metrics"
+            - dir: "./kube-storage-version-migrator"
+              container-image: "storage-version-migration-initializer"
+              target: "initializer"
+            - dir: "./kube-storage-version-migrator"
+              container-image: "storage-version-migration-migrator"
+              target: "migrator"
+            - dir: "./kube-storage-version-migrator"
+              container-image: "storage-version-migration-trigger"
+              target: "trigger"
+            - dir: "./kubernetes"
+              container-image: "kubernetes"
+            - dir: "./local-pv-provisioner"
+              container-image: "local-pv-provisioner"
+              make-targets: "check-generate test ci-e2e-test"
+            - dir: "./loki"
+              container-image: "loki"
+            - dir: "./machines-endpoints"
+              container-image: "machines-endpoints"
+              make-targets: "setup check-generate test"
+            - dir: "./memcached"
+              container-image: "memcached"
+            - dir: "./memcached-exporter"
+              container-image: "memcached-exporter"
+            - dir: "./meows-dctest-runner"
+              container-image: "meows-dctest-runner"
+              request-scan: "true"
+            - dir: "./meows-neco-runner"
+              container-image: "meows-neco-runner"
+              request-scan: "true"
+            - dir: "./moco-switchover-downtime-monitor"
+              container-image: "moco-switchover-downtime-monitor"
+            - dir: "./opentelemetry-collector"
+              container-image: "opentelemetry-collector"
+            - dir: "./hubble"
+              container-image: "hubble"
+            - dir: "./hubble-relay"
+              container-image: "hubble-relay"
+              request-scan: "false"
+            - dir: "./hubble-ui"
+              container-image: "hubble-ui-frontend"
+              request-scan: "true"
+              target: "frontend"
+            - dir: "./hubble-ui"
+              container-image: "hubble-ui-backend"
+              request-scan: "true"
+              target: "backend"
+            - dir: "./pause"
+              container-image: "pause"
+            - dir: "./pod-delete-rate-limiter"
+              container-image: "pod-delete-rate-limiter"
+              make-targets: "test"
+            - dir: "./pomerium"
+              container-image: "pomerium"
+            - dir: "./prometheus-adapter"
+              container-image: "prometheus-adapter"
+            - dir: "./prometheus-config-reloader"
+              container-image: "prometheus-config-reloader"
+            - dir: "./promtail"
+              container-image: "promtail"
+            - dir: "./promtail-debug"
+              container-image: "promtail-debug"
+            - dir: "./pushgateway"
+              container-image: "pushgateway"
+            - dir: "./redis"
+              container-image: "redis"
+            - dir: "./registry"
+              container-image: "registry"
+            - dir: "./rook"
+              container-image: "rook"
+            - dir: "./sealed-secrets"
+              container-image: "sealed-secrets"
+            - dir: "./s3gw"
+              container-image: "s3gw"
+              make-targets: "setup check-generate test"
+            - dir: "./serf"
+              container-image: "serf"
+            - dir: "./stakater-reloader"
+              container-image: "stakater-reloader"
+            - dir: "./spegel"
+              container-image: "spegel"
+            - dir: "./squid"
+              container-image: "squid"
+            - dir: "./squid-exporter"
+              container-image: "squid-exporter"
+              make-targets: "test"
+            - dir: "./tcp-keepalive"
+              container-image: "tcp-keepalive"
+              make-targets: "check-diff test"
+            - dir: "./teleport-node"
+              container-image: "teleport-node"
+            - dir: "./tempo"
+              container-image: "tempo"
+            - dir: "./testhttpd"
+              container-image: "testhttpd"
+              make-targets: "setup check-generate test"
+            - dir: "./trust-manager"
+              container-image: "trust-manager"
+            - dir: "./trust-packages"
+              container-image: "trust-packages"
+              make-targets: "download-certificates"
+            - dir: "./unbound"
+              container-image: "unbound"
+              make-targets: "check-generate"
+              make-post-targets: "validate"
+            - dir: "./unbound_exporter"
+              container-image: "unbound_exporter"
+            - dir: "./vault"
+              container-image: "vault"
+            - dir: "./victoriametrics"
+              container-image: "victoriametrics-vmagent"
+              target: "vmagent"
+            - dir: "./victoriametrics"
+              container-image: "victoriametrics-vmalert"
+              target: "vmalert"
+            - dir: "./victoriametrics"
+              container-image: "victoriametrics-vmbackup"
+              target: "vmbackup"
+            - dir: "./victoriametrics"
+              container-image: "victoriametrics-vmctl"
+              target: "vmctl"
+            - dir: "./victoriametrics"
+              container-image: "victoriametrics-vminsert"
+              target: "vminsert"
+            - dir: "./victoriametrics"
+              container-image: "victoriametrics-vmrestore"
+              target: "vmrestore"
+            - dir: "./victoriametrics"
+              container-image: "victoriametrics-vmselect"
+              target: "vmselect"
+            - dir: "./victoriametrics"
+              container-image: "victoriametrics-vmsingle"
+              target: "vmsingle"
+            - dir: "./victoriametrics"
+              container-image: "victoriametrics-vmstorage"
+              target: "vmstorage"
+            - dir: "./victoriametrics-operator"
+              container-image: "victoriametrics-operator"
   build_container:
+    needs: prepare_matrix
     strategy:
       matrix:
-        job:
-          - dir: "./admission"
-            container-image: "neco-admission"
-            make-targets: "check-generate test"
-          - dir: "./alertmanager"
-            container-image: "alertmanager"
-          - dir: "./argocd"
-            container-image: "argocd"
-          - dir: "./argocd-image-updater"
-            container-image: "argocd-image-updater"
-          - dir: "./bird"
-            container-image: "bird"
-          - dir: "./blackbox_exporter"
-            container-image: "blackbox_exporter"
-          - dir: "./bmc-reverse-proxy"
-            container-image: "bmc-reverse-proxy"
-            make-targets: "setup check-generate test"
-          - dir: "./bmc-log-collector"
-            container-image: "bmc-log-collector"
-            make-targets: "setup check-generate test"
-          - dir: "./bpf-map-pressure-exporter"
-            container-image: "bpf-map-pressure-exporter"
-            make-targets: "check-generate test"
-          - dir: "./cadvisor"
-            container-image: "cadvisor"
-          - dir: "./cephcsi"
-            container-image: "cephcsi"
-            request-scan: "false"
-          - dir: "./cep-checker"
-            container-image: "cep-checker"
-            make-targets: "test"
-          - dir: "./ceph-extra-exporter"
-            container-image: "ceph-extra-exporter"
-            make-targets: "check-generate test"
-          - dir: "./cert-manager"
-            container-image: "cert-manager"
-          - dir: "./chrony"
-            container-image: "chrony"
-          - dir: "./cilium-certgen"
-            container-image: "cilium-certgen"
-            request-scan: "false"
-          - dir: "./cilium-operator-generic"
-            container-image: "cilium-operator-generic"
-            request-scan: "false"
-          - dir: "./configmap-reload"
-            container-image: "configmap-reload"
-          - dir: "./contour"
-            container-image: "contour"
-          - dir: "./coredns"
-            container-image: "coredns"
-          - dir: "csi-attacher"
-            container-image: "csi-attacher"
-          - dir: "csi-node-driver-registrar"
-            container-image: "csi-node-driver-registrar"
-          - dir: "csi-provisioner"
-            container-image: "csi-provisioner"
-          - dir: "csi-resizer"
-            container-image: "csi-resizer"
-          - dir: "csi-snapshotter"
-            container-image: "csi-snapshotter"
-          - dir: "./dex"
-            container-image: "dex"
-          - dir: "./etcd"
-            container-image: "etcd"
-          - dir: "./external-dns"
-            container-image: "external-dns"
-          - dir: "./fluent-bit"
-            container-image: "fluent-bit"
-          - dir: "./golang-all/golang-1.22-jammy"
-            container-image: "golang"
-            enable_arm64: true
-            request-scan: "true"
-          - dir: "./golang-all/golang-1.23-jammy"
-            container-image: "golang"
-            enable_arm64: true
-            request-scan: "true"
-          - dir: "./golang-all/golang-1.22-noble"
-            container-image: "golang"
-            enable_arm64: true
-            request-scan: "true"
-          - dir: "./golang-all/golang-1.23-noble"
-            container-image: "golang"
-            enable_arm64: true
-            request-scan: "true"
-          - dir: "./grafana"
-            container-image: "grafana"
-          - dir: "./grafana-operator"
-            container-image: "grafana-operator"
-          - dir: "./haproxy"
-            container-image: "haproxy"
-          - dir: "./heartbeat"
-            container-image: "heartbeat"
-            make-targets: "setup check-generate test"
-          - dir: "./kube-metrics-adapter"
-            container-image: "kube-metrics-adapter"
-          - dir: "./kube-state-metrics"
-            container-image: "kube-state-metrics"
-          - dir: "./kube-storage-version-migrator"
-            container-image: "storage-version-migration-initializer"
-            target: "initializer"
-          - dir: "./kube-storage-version-migrator"
-            container-image: "storage-version-migration-migrator"
-            target: "migrator"
-          - dir: "./kube-storage-version-migrator"
-            container-image: "storage-version-migration-trigger"
-            target: "trigger"
-          - dir: "./kubernetes"
-            container-image: "kubernetes"
-          - dir: "./local-pv-provisioner"
-            container-image: "local-pv-provisioner"
-            make-targets: "check-generate test ci-e2e-test"
-          - dir: "./loki"
-            container-image: "loki"
-          - dir: "./machines-endpoints"
-            container-image: "machines-endpoints"
-            make-targets: "setup check-generate test"
-          - dir: "./memcached"
-            container-image: "memcached"
-          - dir: "./memcached-exporter"
-            container-image: "memcached-exporter"
-          - dir: "./meows-dctest-runner"
-            container-image: "meows-dctest-runner"
-            request-scan: "true"
-          - dir: "./meows-neco-runner"
-            container-image: "meows-neco-runner"
-            request-scan: "true"
-          - dir: "./moco-switchover-downtime-monitor"
-            container-image: "moco-switchover-downtime-monitor"
-          - dir: "./opentelemetry-collector"
-            container-image: "opentelemetry-collector"
-          - dir: "./hubble"
-            container-image: "hubble"
-          - dir: "./hubble-relay"
-            container-image: "hubble-relay"
-            request-scan: "false"
-          - dir: "./hubble-ui"
-            container-image: "hubble-ui-frontend"
-            request-scan: "true"
-            target: "frontend"
-          - dir: "./hubble-ui"
-            container-image: "hubble-ui-backend"
-            request-scan: "true"
-            target: "backend"
-          - dir: "./pause"
-            container-image: "pause"
-          - dir: "./pod-delete-rate-limiter"
-            container-image: "pod-delete-rate-limiter"
-            make-targets: "test"
-          - dir: "./pomerium"
-            container-image: "pomerium"
-          - dir: "./prometheus-adapter"
-            container-image: "prometheus-adapter"
-          - dir: "./prometheus-config-reloader"
-            container-image: "prometheus-config-reloader"
-          - dir: "./promtail"
-            container-image: "promtail"
-          - dir: "./promtail-debug"
-            container-image: "promtail-debug"
-          - dir: "./pushgateway"
-            container-image: "pushgateway"
-          - dir: "./redis"
-            container-image: "redis"
-          - dir: "./registry"
-            container-image: "registry"
-          - dir: "./rook"
-            container-image: "rook"
-          - dir: "./sealed-secrets"
-            container-image: "sealed-secrets"
-          - dir: "./s3gw"
-            container-image: "s3gw"
-            make-targets: "setup check-generate test"
-          - dir: "./serf"
-            container-image: "serf"
-          - dir: "./stakater-reloader"
-            container-image: "stakater-reloader"
-          - dir: "./spegel"
-            container-image: "spegel"
-          - dir: "./squid"
-            container-image: "squid"
-          - dir: "./squid-exporter"
-            container-image: "squid-exporter"
-            make-targets: "test"
-          - dir: "./tcp-keepalive"
-            container-image: "tcp-keepalive"
-            make-targets: "check-diff test"
-          - dir: "./teleport-node"
-            container-image: "teleport-node"
-          - dir: "./tempo"
-            container-image: "tempo"
-          - dir: "./testhttpd"
-            container-image: "testhttpd"
-            make-targets: "setup check-generate test"
-          - dir: "./trust-manager"
-            container-image: "trust-manager"
-          - dir: "./trust-packages"
-            container-image: "trust-packages"
-            make-targets: "download-certificates"
-          - dir: "./unbound"
-            container-image: "unbound"
-            make-targets: "check-generate"
-            make-post-targets: "validate"
-          - dir: "./unbound_exporter"
-            container-image: "unbound_exporter"
-          - dir: "./vault"
-            container-image: "vault"
-          - dir: "./victoriametrics"
-            container-image: "victoriametrics-vmagent"
-            target: "vmagent"
-          - dir: "./victoriametrics"
-            container-image: "victoriametrics-vmalert"
-            target: "vmalert"
-          - dir: "./victoriametrics"
-            container-image: "victoriametrics-vmbackup"
-            target: "vmbackup"
-          - dir: "./victoriametrics"
-            container-image: "victoriametrics-vmctl"
-            target: "vmctl"
-          - dir: "./victoriametrics"
-            container-image: "victoriametrics-vminsert"
-            target: "vminsert"
-          - dir: "./victoriametrics"
-            container-image: "victoriametrics-vmrestore"
-            target: "vmrestore"
-          - dir: "./victoriametrics"
-            container-image: "victoriametrics-vmselect"
-            target: "vmselect"
-          - dir: "./victoriametrics"
-            container-image: "victoriametrics-vmsingle"
-            target: "vmsingle"
-          - dir: "./victoriametrics"
-            container-image: "victoriametrics-vmstorage"
-            target: "vmstorage"
-          - dir: "./victoriametrics-operator"
-            container-image: "victoriametrics-operator"
-
+        job: ${{ fromJson(needs.prepare_matrix.outputs.jobs) }}
     runs-on: ubuntu-22.04
     env:
       go-version: "1.23.3"
@@ -266,36 +276,3 @@ jobs:
           make-post-targets: ${{ matrix.job.make-post-targets }}
           request-scan: ${{ matrix.job.request-scan }}
           target: ${{ matrix.job.target }}
-
-  build_envoy:
-    runs-on: neco-containers-ubuntu-22.04
-    steps:
-      - uses: actions/checkout@v4
-      - name: Build container image
-        uses: ./.github/actions/build_envoy
-        with:
-          clang_archive_path: https://github.com/llvm/llvm-project/releases/download/llvmorg-14.0.0/clang+llvm-14.0.0-x86_64-linux-gnu-ubuntu-18.04.tar.xz
-          github_token: ${{ secrets.GITHUB_TOKEN }}
-          yamory_token: ${{ secrets.YAMORY_ACCESS_TOKEN }}
-
-  build_ceph:
-    runs-on: neco-containers-ubuntu-22.04
-    steps:
-    - uses: actions/checkout@v4
-    - name: Build ceph image
-      uses: ./.github/actions/build_ceph
-      with:
-        github_token: ${{ secrets.GITHUB_TOKEN }}
-        version: 18.2.4
-
-  build_cilium:
-    runs-on: neco-containers-ubuntu-22.04
-    steps:
-    - uses: actions/checkout@v4
-    - name: Build cilium-agent image
-      uses: ./.github/actions/build_cilium-agent
-      with:
-        github_token: ${{ secrets.GITHUB_TOKEN }}
-        yamory_token: ${{ secrets.YAMORY_ACCESS_TOKEN }}
-        cilium-proxy_version: 39a2a56bbd5b3a591f69dbca51d3e30ef97e0e51
-        image-tools_version: ff22ba3bff1010f4a2dd76ede789663c3beaf8d2
diff --git a/generate_matrix b/generate_matrix
new file mode 100755
index 000000000..6777102f6
--- /dev/null
+++ b/generate_matrix
@@ -0,0 +1,27 @@
+#!/bin/bash -e
+
+# https://github.com/tj-actions/changed-files?tab=readme-ov-file#outputs-
+cat .github/outputs/all_changed_and_modified_files.txt | tr ',' '\n' | xargs realpath > .github/outputs/__diff.txt
+
+echo "Changed Files:"
+cat .github/outputs/__diff.txt | jq -n --raw-input '[inputs]' | yq -P
+echo
+
+JSON_MATRIX=$(echo "${INPUT_MATRIX}" | yq -ojson)
+OUTPUT_MATRIX='[]'
+for i in $(seq $(echo "${JSON_MATRIX}" | jq length)); do
+    ENTRY=$(echo "${JSON_MATRIX}" | jq ".[$i]")
+    CONTAINER_DIR=$(echo "${ENTRY}" | jq -r '.dir')
+    CONTAINER_NAME=$(echo "${ENTRY}" | jq -r '."container-image"')
+    echo "Checking ${CONTAINER_NAME}..."
+    if grep -F $(realpath ${CONTAINER_DIR}) .github/outputs/__diff.txt > /dev/null; then
+        OUTPUT_MATRIX=$(jq -n '[inputs][0] + [[inputs][1]]' <(echo "${OUTPUT_MATRIX}") <(echo "${ENTRY}"))
+    fi
+done
+
+echo
+echo "Result:"
+echo ${OUTPUT_MATRIX} | yq -P
+
+# Render matrix
+echo "${OUTPUT_MATRIX}" | jq -c > .github/outputs/__matrix.json
diff --git a/squid/TAG b/squid/TAG
index 81f1df119..3ae64ac5d 100644
--- a/squid/TAG
+++ b/squid/TAG
@@ -1 +1 @@
-6.12.0.1
+6.12.0.2