diff --git a/.gitattributes b/.gitattributes index 3f5563f4c..983ee6ebd 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,2 +1,3 @@ * text=auto eol=lf *.sops.* diff=sopsdiffer +*.yaml.j2 linguist-language=YAML diff --git a/.github/labeler.yaml b/.github/labeler.yaml new file mode 100644 index 000000000..af6ba72e0 --- /dev/null +++ b/.github/labeler.yaml @@ -0,0 +1,22 @@ +--- +# Areas +area/docs: + - changed-files: + - any-glob-to-any-file: + - "docs/**/*" + - "README.md" +area/github: + - changed-files: + - any-glob-to-any-file: ".github/**/*" +area/kubernetes: + - changed-files: + - any-glob-to-any-file: "kubernetes/**/*" +area/taskfile: + - changed-files: + - any-glob-to-any-file: + - ".taskfiles/**/*" + - "Taskfile.yaml" +# Clusters +cluster/main: + - changed-files: + - any-glob-to-any-file: "kubernetes/main/**/*" diff --git a/.github/labels.yaml b/.github/labels.yaml new file mode 100644 index 000000000..86f42d1d9 --- /dev/null +++ b/.github/labels.yaml @@ -0,0 +1,38 @@ +--- +# Areas +- name: area/docs + color: "0e8a16" +- name: area/github + color: "0e8a16" +- name: area/kubernetes + color: "0e8a16" +- name: area/taskfile + color: "0e8a16" +# Clusters +- name: cluster/main + color: "ffc300" +# Renovate Types +- name: renovate/container + color: "027fa0" +- name: renovate/github-action + color: "027fa0" +- name: renovate/grafana-dashboard + color: "027fa0" +- name: renovate/github-release + color: "027fa0" +- name: renovate/helm + color: "027fa0" +# Semantic Types +- name: type/digest + color: "ffeC19" +- name: type/patch + color: "ffeC19" +- name: type/minor + color: "ff9800" +- name: type/major + color: "f6412d" +# Uncategorized +- name: community + color: "370fb2" +- name: hold + color: "ee0701" diff --git a/.github/renovate.json5 b/.github/renovate.json5 new file mode 100644 index 000000000..5302f242f --- /dev/null +++ b/.github/renovate.json5 @@ -0,0 +1,45 @@ +{ + "$schema": "https://docs.renovatebot.com/renovate-schema.json", + "extends": [ + "config:recommended", + "docker:enableMajor", + "replacements:k8s-registry-move", + ":automergeBranch", + ":disableRateLimiting", + ":dependencyDashboard", + ":semanticCommits", + ":skipStatusChecks", + ":timezone(America/New_York)", + "github>onedr0p/home-ops//.github/renovate/allowedVersions.json5", + "github>onedr0p/home-ops//.github/renovate/autoMerge.json5", + "github>onedr0p/home-ops//.github/renovate/clusters.json5", + "github>onedr0p/home-ops//.github/renovate/commitMessage.json5", + "github>onedr0p/home-ops//.github/renovate/customManagers.json5", + "github>onedr0p/home-ops//.github/renovate/grafanaDashboards.json5", + "github>onedr0p/home-ops//.github/renovate/groups.json5", + "github>onedr0p/home-ops//.github/renovate/labels.json5", + "github>onedr0p/home-ops//.github/renovate/packageRules.json5", + "github>onedr0p/home-ops//.github/renovate/semanticCommits.json5" + ], + "dependencyDashboardTitle": "Renovate Dashboard 🤖", + "suppressNotifications": ["prEditedNotification", "prIgnoreNotification"], + "onboarding": false, + "requireConfig": "ignored", + "ignorePaths": ["**/*.sops.*", "**/.archive/**", "**/resources/**"], + "flux": { + "fileMatch": [ + "(^|/)kubernetes/.+\\.ya?ml(?:\\.j2)?$" + ] + }, + "helm-values": { + "fileMatch": [ + "(^|/)kubernetes/.+\\.ya?ml(?:\\.j2)?$" + ] + }, + "kubernetes": { + "fileMatch": [ + "(^|/)\\.taskfiles/.+\\.ya?ml(?:\\.j2)?$", + "(^|/)kubernetes/.+\\.ya?ml(?:\\.j2)?$" + ] + } +} diff --git a/.github/renovate/allowedVersions.json5 b/.github/renovate/allowedVersions.json5 new file mode 100644 index 000000000..7374aaf7c --- /dev/null +++ b/.github/renovate/allowedVersions.json5 @@ -0,0 +1,15 @@ +{ + "$schema": "https://docs.renovatebot.com/renovate-schema.json", + "packageRules": [ + { + "matchDatasources": ["docker"], + "matchPackagePatterns": ["kopia"], + "allowedVersions": "<999" + }, + { + "matchDatasources": ["docker"], + "matchPackagePatterns": ["postgresql"], + "allowedVersions": "<18" + } + ] +} diff --git a/.github/renovate/autoMerge.json5 b/.github/renovate/autoMerge.json5 new file mode 100644 index 000000000..73d3cdc17 --- /dev/null +++ b/.github/renovate/autoMerge.json5 @@ -0,0 +1,21 @@ +{ + "$schema": "https://docs.renovatebot.com/renovate-schema.json", + "packageRules": [ + { + "description": ["Auto-merge container digests updates for trusted containers"], + "matchDatasources": ["docker"], + "automerge": true, + "automergeType": "branch", + "matchUpdateTypes": ["digest"], + "matchPackagePatterns": ["ghcr.io/bjw-s", "ghcr.io/onedr0p"] + }, + { + "description": ["Auto-merge GitHub Actions for minor and patch"], + "matchManagers": ["github-actions"], + "matchDatasources": ["github-tags"], + "automerge": true, + "automergeType": "branch", + "matchUpdateTypes": ["minor", "patch"] + } + ] +} diff --git a/.github/renovate/clusters.json5 b/.github/renovate/clusters.json5 new file mode 100644 index 000000000..7ceb227b0 --- /dev/null +++ b/.github/renovate/clusters.json5 @@ -0,0 +1,10 @@ +{ + "$schema": "https://docs.renovatebot.com/renovate-schema.json", + "packageRules": [ + { + "description": ["Separate PRs for main cluster"], + "matchFileNames": ["**/kubernetes/main/**"], + "additionalBranchPrefix": "main-" + } + ] +} diff --git a/.github/renovate/commitMessage.json5 b/.github/renovate/commitMessage.json5 new file mode 100644 index 000000000..3fea62872 --- /dev/null +++ b/.github/renovate/commitMessage.json5 @@ -0,0 +1,16 @@ +{ + "$schema": "https://docs.renovatebot.com/renovate-schema.json", + "commitMessageTopic": "{{depName}}", + "commitMessageExtra": "to {{newVersion}}", + "commitMessageSuffix": "", + "packageRules": [ + { + "matchDatasources": ["helm"], + "commitMessageTopic": "chart {{depName}}" + }, + { + "matchDatasources": ["docker"], + "commitMessageTopic": "image {{depName}}" + } + ] +} diff --git a/.github/renovate/customManagers.json5 b/.github/renovate/customManagers.json5 new file mode 100644 index 000000000..d3ab54f1e --- /dev/null +++ b/.github/renovate/customManagers.json5 @@ -0,0 +1,29 @@ +{ + "$schema": "https://docs.renovatebot.com/renovate-schema.json", + "customManagers": [ + { + "customType": "regex", + "description": ["Process custom dependencies"], + "fileMatch": ["(^|/)kubernetes/.+\\.ya?ml(?:\\.j2)?$"], + "matchStrings": [ + // # renovate: datasource=github-releases depName=k3s-io/k3s + // k3s_release_version: &version v1.29.0+k3s1 + // # renovate: datasource=helm depName=cilium repository=https://helm.cilium.io + // version: 1.15.1 + "datasource=(?\\S+) depName=(?\\S+)( repository=(?\\S+))?\\n.+: (&\\S+\\s)?(?\\S+)", + // # renovate: datasource=github-releases depName=rancher/system-upgrade-controller + // https://github.com/rancher/system-upgrade-controller/releases/download/v0.13.2/crd.yaml + "datasource=(?\\S+) depName=(?\\S+)\\n.+/(?(v|\\d)[^/]+)" + ], + "datasourceTemplate": "{{#if datasource}}{{{datasource}}}{{else}}github-releases{{/if}}" + }, + { + "customType": "regex", + "description": ["Process CloudnativePG Postgresql version"], + "fileMatch": ["(^|/)kubernetes/.+\\.ya?ml(?:\\.j2)?$"], + "matchStrings": ["imageName: (?\\S+):(?.*\\-.*)"], + "datasourceTemplate": "docker", + "versioningTemplate": "redhat" + } + ] +} diff --git a/.github/renovate/grafanaDashboards.json5 b/.github/renovate/grafanaDashboards.json5 new file mode 100644 index 000000000..08e5fb3b4 --- /dev/null +++ b/.github/renovate/grafanaDashboards.json5 @@ -0,0 +1,34 @@ +{ + "$schema": "https://docs.renovatebot.com/renovate-schema.json", + "customDatasources": { + "grafana-dashboards": { + "defaultRegistryUrlTemplate": "https://grafana.com/api/dashboards/{{packageName}}", + "format": "json", + "transformTemplates": ["{\"releases\":[{\"version\": $string(revision)}]}"] + } + }, + "customManagers": [ + { + "customType": "regex", + "description": ["Process Grafana dashboards"], + "fileMatch": ["(^|/)kubernetes/.+\\.ya?ml(?:\\.j2)?$"], + "matchStrings": ["depName=\"(?.*)\"\\n(?\\s+)gnetId: (?\\d+)\\n.+revision: (?\\d+)"], + "autoReplaceStringTemplate": "depName=\"{{{depName}}}\"\n{{{indentation}}}gnetId: {{{packageName}}}\n{{{indentation}}}revision: {{{newValue}}}", + "datasourceTemplate": "custom.grafana-dashboards", + "versioningTemplate": "regex:^(?\\d+)$" + } + ], + "packageRules": [ + { + "addLabels": ["renovate/grafana-dashboard"], + "automerge": true, + "automergeType": "branch", + "matchDatasources": ["custom.grafana-dashboards"], + "matchUpdateTypes": ["major"], + "semanticCommitType": "chore", + "semanticCommitScope": "grafana-dashboards", + "commitMessageTopic": "dashboard {{depName}}", + "commitMessageExtra": "( {{currentVersion}} → {{newVersion}} )" + } + ] +} diff --git a/.github/renovate/groups.json5 b/.github/renovate/groups.json5 new file mode 100644 index 000000000..79a05f8e7 --- /dev/null +++ b/.github/renovate/groups.json5 @@ -0,0 +1,66 @@ +{ + "$schema": "https://docs.renovatebot.com/renovate-schema.json", + "packageRules": [ + { + "description": ["1Password Connect Group"], + "groupName": "1Password Connnect", + "matchPackagePatterns": ["1password/connect"], + "matchDatasources": ["docker"], + "group": { + "commitMessageTopic": "{{{groupName}}} group" + }, + "separateMinorPatch": true + }, + { + "description": ["Actions Runner Controller Group"], + "groupName": "Actions Runner Controller", + "matchPackagePatterns": ["gha-runner-scale-set"], + "matchDatasources": ["docker", "helm"], + "group": { + "commitMessageTopic": "{{{groupName}}} group" + }, + "separateMinorPatch": true + }, + { + "description": ["Flux Group"], + "groupName": "Flux", + "matchPackagePatterns": ["fluxcd"], + "matchDatasources": ["docker", "github-tags"], + "versioning": "semver", + "group": { + "commitMessageTopic": "{{{groupName}}} group" + }, + "separateMinorPatch": true + }, + { + "description": ["Intel Device Plugins Group"], + "groupName": "Intel-Device-Plugins", + "matchPackagePatterns": ["intel-device-plugins"], + "matchDatasources": ["helm"], + "group": { + "commitMessageTopic": "{{{groupName}}} group" + }, + "separateMinorPatch": true + }, + { + "description": ["Rook-Ceph Group"], + "groupName": "Rook-Ceph", + "matchPackagePatterns": ["rook.ceph"], + "matchDatasources": ["helm"], + "group": { + "commitMessageTopic": "{{{groupName}}} group" + }, + "separateMinorPatch": true + }, + { + "description": ["Talos Group"], + "groupName": "Talos", + "matchPackagePatterns": ["siderolabs/talosctl", "siderolabs/installer"], + "matchDatasources": ["docker"], + "group": { + "commitMessageTopic": "{{{groupName}}} group" + }, + "separateMinorPatch": true + } + ] +} diff --git a/.github/renovate/labels.json5 b/.github/renovate/labels.json5 new file mode 100644 index 000000000..641ea6e98 --- /dev/null +++ b/.github/renovate/labels.json5 @@ -0,0 +1,37 @@ +{ + "$schema": "https://docs.renovatebot.com/renovate-schema.json", + "packageRules": [ + { + "matchUpdateTypes": ["major"], + "labels": ["type/major"] + }, + { + "matchUpdateTypes": ["minor"], + "labels": ["type/minor"] + }, + { + "matchUpdateTypes": ["patch"], + "labels": ["type/patch"] + }, + { + "matchUpdateTypes": ["digest"], + "labels": ["type/digest"] + }, + { + "matchDatasources": ["docker"], + "addLabels": ["renovate/container"] + }, + { + "matchDatasources": ["helm"], + "addLabels": ["renovate/helm"] + }, + { + "matchDatasources": ["github-releases", "github-tags"], + "addLabels": ["renovate/github-release"] + }, + { + "matchManagers": ["github-actions"], + "addLabels": ["renovate/github-action"] + } + ] +} diff --git a/.github/renovate/packageRules.json5 b/.github/renovate/packageRules.json5 new file mode 100644 index 000000000..717eead3d --- /dev/null +++ b/.github/renovate/packageRules.json5 @@ -0,0 +1,29 @@ +{ + "$schema": "https://docs.renovatebot.com/renovate-schema.json", + "packageRules": [ + { + "description": ["Loose versioning for non-semver packages"], + "matchDatasources": ["docker"], + "matchPackagePatterns": ["cross-seed", "plex"], + "versioning": "loose" + }, + { + "description": ["Custom schedule for frequently updated packages"], + "matchDataSources": ["docker", "helm"], + "matchPackagePatterns": ["minio", "postgresql", "reloader"], + "schedule": ["on the first day of the month"] + }, + { + "description": ["Custom versioning for k3s"], + "matchDatasources": ["github-releases"], + "matchPackagePatterns": ["k3s"], + "versioning": "regex:^v(?\\d+)\\.(?\\d+)\\.(?\\d+)(?\\+k3s)(?\\d+)$" + }, + { + "description": ["Custom versioning for minio"], + "matchDatasources": ["docker"], + "matchPackagePatterns": ["minio"], + "versioning": "regex:^RELEASE\\.(?\\d+)-(?\\d+)-(?\\d+)T.*Z$" + } + ] +} diff --git a/.github/renovate/semanticCommits.json5 b/.github/renovate/semanticCommits.json5 new file mode 100644 index 000000000..0d88d8db6 --- /dev/null +++ b/.github/renovate/semanticCommits.json5 @@ -0,0 +1,105 @@ +{ + "$schema": "https://docs.renovatebot.com/renovate-schema.json", + "packageRules": [ + { + "matchDatasources": ["docker"], + "matchUpdateTypes": ["major"], + "commitMessagePrefix": "feat(container)!: ", + "commitMessageTopic": "{{depName}}", + "commitMessageExtra": " ( {{currentVersion}} → {{newVersion}} )" + }, + { + "matchDatasources": ["docker"], + "matchUpdateTypes": ["minor"], + "semanticCommitType": "feat", + "semanticCommitScope": "container", + "commitMessageTopic": "{{depName}}", + "commitMessageExtra": "( {{currentVersion}} → {{newVersion}} )" + }, + { + "matchDatasources": ["docker"], + "matchUpdateTypes": ["patch"], + "semanticCommitType": "fix", + "semanticCommitScope": "container", + "commitMessageTopic": "{{depName}}", + "commitMessageExtra": "( {{currentVersion}} → {{newVersion}} )" + }, + { + "matchDatasources": ["docker"], + "matchUpdateTypes": ["digest"], + "semanticCommitType": "chore", + "semanticCommitScope": "container", + "commitMessageTopic": "{{depName}}", + "commitMessageExtra": "( {{currentDigestShort}} → {{newDigestShort}} )" + }, + { + "matchDatasources": ["helm"], + "matchUpdateTypes": ["major"], + "commitMessagePrefix": "feat(helm)!: ", + "commitMessageTopic": "{{depName}}", + "commitMessageExtra": "( {{currentVersion}} → {{newVersion}} )" + }, + { + "matchDatasources": ["helm"], + "matchUpdateTypes": ["minor"], + "semanticCommitType": "feat", + "semanticCommitScope": "helm", + "commitMessageTopic": "{{depName}}", + "commitMessageExtra": "( {{currentVersion}} → {{newVersion}} )" + }, + { + "matchDatasources": ["helm"], + "matchUpdateTypes": ["patch"], + "semanticCommitType": "fix", + "semanticCommitScope": "helm", + "commitMessageTopic": "{{depName}}", + "commitMessageExtra": "( {{currentVersion}} → {{newVersion}} )" + }, + { + "matchDatasources": ["github-releases", "github-tags"], + "matchUpdateTypes": ["major"], + "commitMessagePrefix": "feat(github-release)!: ", + "commitMessageTopic": "{{depName}}", + "commitMessageExtra": "( {{currentVersion}} → {{newVersion}} )" + }, + { + "matchDatasources": ["github-releases", "github-tags"], + "matchUpdateTypes": ["minor"], + "semanticCommitType": "feat", + "semanticCommitScope": "github-release", + "commitMessageTopic": "{{depName}}", + "commitMessageExtra": "( {{currentVersion}} → {{newVersion}} )" + }, + { + "matchDatasources": ["github-releases", "github-tags"], + "matchUpdateTypes": ["patch"], + "semanticCommitType": "fix", + "semanticCommitScope": "github-release", + "commitMessageTopic": "{{depName}}", + "commitMessageExtra": "( {{currentVersion}} → {{newVersion}} )" + }, + { + "matchManagers": ["github-actions"], + "matchUpdateTypes": ["major"], + "commitMessagePrefix": "feat(github-action)!: ", + "commitMessageTopic": "{{depName}}", + "commitMessageExtra": "( {{currentVersion}} → {{newVersion}} )" + }, + { + "matchManagers": ["github-actions"], + "matchUpdateTypes": ["minor"], + "semanticCommitType": "feat", + "semanticCommitScope": "github-action", + "commitMessageTopic": "{{depName}}", + "commitMessageExtra": "( {{currentVersion}} → {{newVersion}} )" + }, + { + "matchManagers": ["github-actions"], + "matchUpdateTypes": ["patch"], + "semanticCommitType": "fix", + "semanticCommitScope": "github-action", + "commitMessageTopic": "{{depName}}", + "commitMessageExtra": "( {{currentVersion}} → {{newVersion}} )" + } + ] +} diff --git a/.github/workflows/flux-diff.yaml b/.github/workflows/flux-diff.yaml new file mode 100644 index 000000000..b01b68f9a --- /dev/null +++ b/.github/workflows/flux-diff.yaml @@ -0,0 +1,125 @@ +--- +# yaml-language-server: $schema=https://json.schemastore.org/github-workflow.json +name: "Flux Diff" + +on: + pull_request: + branches: ["main"] + paths: ["kubernetes/**"] + +concurrency: + group: ${{ github.workflow }}-${{ github.event.number || github.ref }} + cancel-in-progress: true + +jobs: + changed-clusters: + name: Changed Clusters + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.changed-clusters.outputs.all_changed_and_modified_files }} + steps: + - name: Generate Token + uses: actions/create-github-app-token@v1 + id: app-token + with: + app-id: "${{ secrets.BOT_APP_ID }}" + private-key: "${{ secrets.BOT_APP_PRIVATE_KEY }}" + + - name: Checkout Default Branch + uses: actions/checkout@v4 + with: + token: "${{ steps.app-token.outputs.token }}" + fetch-depth: 0 + + - name: Get Changed Clusters + id: changed-clusters + uses: tj-actions/changed-files@v45 + with: + files: kubernetes/** + dir_names: true + dir_names_max_depth: 2 + matrix: true + + - name: List All Changed Clusters + run: echo "${{ steps.changed-clusters.outputs.all_changed_and_modified_files }}" + + flux-diff: + name: Flux Diff + runs-on: ubuntu-latest + needs: ["changed-clusters"] + permissions: + pull-requests: write + strategy: + matrix: + paths: ${{ fromJSON(needs.changed-clusters.outputs.matrix) }} + resources: ["helmrelease", "kustomization"] + max-parallel: 4 + fail-fast: false + steps: + - name: Generate Token + uses: actions/create-github-app-token@v1 + id: app-token + with: + app-id: "${{ secrets.BOT_APP_ID }}" + private-key: "${{ secrets.BOT_APP_PRIVATE_KEY }}" + + - name: Checkout + uses: actions/checkout@v4 + with: + token: "${{ steps.app-token.outputs.token }}" + path: pull + + - name: Checkout Default Branch + uses: actions/checkout@v4 + with: + token: "${{ steps.app-token.outputs.token }}" + ref: "${{ github.event.repository.default_branch }}" + path: default + + - name: Diff Resources + uses: docker://ghcr.io/allenporter/flux-local:v5.5.1 + with: + args: >- + diff ${{ matrix.resources }} + --unified 6 + --path /github/workspace/pull/${{ matrix.paths }}/flux + --path-orig /github/workspace/default/${{ matrix.paths }}/flux + --strip-attrs "helm.sh/chart,checksum/config,app.kubernetes.io/version,chart" + --limit-bytes 10000 + --all-namespaces + --sources "home-kubernetes" + --output-file diff.patch + + - name: Generate Diff + id: diff + run: | + echo "diff<> $GITHUB_OUTPUT + cat diff.patch >> $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT + echo "### Diff" >> $GITHUB_STEP_SUMMARY + echo '```diff' >> $GITHUB_STEP_SUMMARY + cat diff.patch >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + + - if: ${{ steps.diff.outputs.diff != '' }} + name: Add comment + uses: mshick/add-pr-comment@v2 + with: + repo-token: "${{ steps.app-token.outputs.token }}" + message-id: "${{ github.event.pull_request.number }}/${{ matrix.paths }}/${{ matrix.resources }}" + message-failure: Diff was not successful + message: | + ```diff + ${{ steps.diff.outputs.diff }} + ``` + + # Summarize matrix https://github.community/t/status-check-for-a-matrix-jobs/127354/7 + flux-diff-success: + if: ${{ always() }} + needs: ["flux-diff"] + name: Flux Diff Successful + runs-on: ubuntu-latest + steps: + - if: ${{ contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') }} + name: Check matrix status + run: exit 1 diff --git a/.github/workflows/flux-hr-sync.yaml b/.github/workflows/flux-hr-sync.yaml new file mode 100644 index 000000000..67887c0d9 --- /dev/null +++ b/.github/workflows/flux-hr-sync.yaml @@ -0,0 +1,98 @@ +--- +# yaml-language-server: $schema=https://json.schemastore.org/github-workflow.json +name: "Flux Helm Repository Sync" + +on: + workflow_dispatch: + inputs: + clusterName: + description: Cluster Name + default: main + required: true + helmRepoNamespace: + description: Helm Repository Namespace + default: flux-system + required: true + helmRepoName: + description: Helm Repository Name + required: true + pull_request: + branches: ["main"] + paths: ["kubernetes/**/helmrelease.yaml"] + +jobs: + sync: + name: Flux Helm Repository Sync + runs-on: ["gha-runner-scale-set"] + steps: + - name: Generate Token + uses: actions/create-github-app-token@v1 + id: app-token + with: + app-id: "${{ secrets.BOT_APP_ID }}" + private-key: "${{ secrets.BOT_APP_PRIVATE_KEY }}" + + - name: Checkout + uses: actions/checkout@v4 + with: + token: "${{ steps.app-token.outputs.token }}" + fetch-depth: 0 + + - name: Setup Homebrew + uses: Homebrew/actions/setup-homebrew@master + + - name: Setup Workflow Tools + shell: bash + run: brew install fluxcd/tap/flux yq + + - name: Write kubeconfig + id: kubeconfig + uses: timheuer/base64-to-file@v1 + with: + encodedString: "${{ secrets.KUBECONFIG }}" + fileName: kubeconfig + + - if: ${{ github.event.inputs.clusterName == '' && github.event.inputs.helmRepoNamespace == '' && github.event.inputs.helmRepoName == '' }} + name: Get Changed Files + id: changed-files + uses: tj-actions/changed-files@v45 + with: + files: kubernetes/**/helmrelease.yaml + safe_output: false + + - if: ${{ github.event.inputs.clusterName == '' && github.event.inputs.helmRepoNamespace == '' && github.event.inputs.helmRepoName == '' }} + name: List All Changed Files + run: echo "${{ steps.changed-files.outputs.all_changed_and_modified_files }}" + + - if: ${{ github.event.inputs.clusterName == '' && github.event.inputs.helmRepoNamespace == '' && github.event.inputs.helmRepoName == '' }} + name: Sync Helm Repository + env: + KUBECONFIG: "${{ steps.kubeconfig.outputs.filePath }}" + shell: bash + run: | + declare -a repos=() + for f in ${{ steps.changed-files.outputs.all_changed_and_modified_files }}; do + cluster_name=$(echo "${f}" | awk -F'/' '{print $2}') + repo_namespace="$(yq -r '.spec.chart.spec.sourceRef.namespace' "${f}")" + repo_name="$(yq -r '.spec.chart.spec.sourceRef.name' "${f}")" + repos+=("${cluster_name}:${repo_namespace}:${repo_name}") + done + mapfile -t repos < <(printf "%s\n" "${repos[@]}" | sort -u) + for r in "${repos[@]}"; do + IFS=':' read -r cluster_name repo_namespace repo_name <<< "${r}" + flux \ + --context ${cluster_name} \ + --namespace ${repo_namespace} \ + reconcile source helm ${repo_name} + done + + - if: ${{ github.event.inputs.clusterName != '' && github.event.inputs.helmRepoNamespace != '' && github.event.inputs.helmRepoName != '' }} + name: Sync Helm Repository + env: + KUBECONFIG: ${{ steps.kubeconfig.outputs.filePath }} + shell: bash + run: | + flux \ + --context ${{ github.event.inputs.clusterName }} \ + --namespace ${{ github.event.inputs.helmRepoNamespace }} \ + reconcile source helm ${{ github.event.inputs.helmRepoName }} diff --git a/.github/workflows/flux-image-test.yaml b/.github/workflows/flux-image-test.yaml new file mode 100644 index 000000000..0407f4d64 --- /dev/null +++ b/.github/workflows/flux-image-test.yaml @@ -0,0 +1,152 @@ +--- +# yaml-language-server: $schema=https://json.schemastore.org/github-workflow.json +name: "Flux Image Test" + +on: + pull_request: + branches: ["main"] + paths: ["kubernetes/**"] + +concurrency: + group: ${{ github.workflow }}-${{ github.event.number || github.ref }} + cancel-in-progress: true + +jobs: + changed-clusters: + name: Changed Clusters + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.changed-clusters.outputs.all_changed_and_modified_files }} + steps: + - name: Generate Token + uses: actions/create-github-app-token@v1 + id: app-token + with: + app-id: "${{ secrets.BOT_APP_ID }}" + private-key: "${{ secrets.BOT_APP_PRIVATE_KEY }}" + + - name: Checkout + uses: actions/checkout@v4 + with: + token: "${{ steps.app-token.outputs.token }}" + fetch-depth: 0 + + - name: Get Changed Clusters + id: changed-clusters + uses: tj-actions/changed-files@v45 + with: + files: kubernetes/** + dir_names: true + dir_names_max_depth: 2 + matrix: true + + - name: List All Changed Clusters + run: echo "${{ steps.changed-clusters.outputs.all_changed_and_modified_files }}" + + extract-images: + name: Extract Images + runs-on: ubuntu-latest + needs: ["changed-clusters"] + permissions: + pull-requests: write + strategy: + matrix: + paths: ${{ fromJSON(needs.changed-clusters.outputs.matrix) }} + max-parallel: 4 + fail-fast: false + outputs: + matrix: ${{ steps.extract-images.outputs.images }} + steps: + - name: Generate Token + uses: actions/create-github-app-token@v1 + id: app-token + with: + app-id: "${{ secrets.BOT_APP_ID }}" + private-key: "${{ secrets.BOT_APP_PRIVATE_KEY }}" + + - name: Setup Homebrew + uses: Homebrew/actions/setup-homebrew@master + + - name: Setup Workflow Tools + shell: bash + run: brew install jo yq + + - name: Checkout Default Branch + uses: actions/checkout@v4 + with: + token: "${{ steps.app-token.outputs.token }}" + ref: "${{ github.event.repository.default_branch }}" + path: default + + - name: Checkout Pull Request Branch + uses: actions/checkout@v4 + with: + token: "${{ steps.app-token.outputs.token }}" + path: pull + + - name: Gather Images in Default Branch + uses: docker://ghcr.io/allenporter/flux-local:v5.5.1 + with: + args: >- + get cluster + --path /github/workspace/default/${{ matrix.paths }}/flux + --enable-images + --output yaml + --output-file default.yaml + + - name: Gather Images in Pull Request Branch + uses: docker://ghcr.io/allenporter/flux-local:v5.5.1 + with: + args: >- + get cluster + --path /github/workspace/pull/${{ matrix.paths }}/flux + --enable-images + --output yaml + --output-file pull.yaml + + - name: Filter Default Branch Results + shell: bash + run: | + yq -r '[.. | .images? | select(. != null)] | flatten | sort | unique | .[]' \ + default.yaml > default.txt + + - name: Filter Pull Request Branch Results + shell: bash + run: | + yq -r '[.. | .images? | select(. != null)] | flatten | sort | unique | .[]' \ + pull.yaml > pull.txt + + - name: Compare Default and Pull Request Images + id: extract-images + shell: bash + run: | + images=$(jo -a $(grep -vf default.txt pull.txt)) + echo "images=${images}" >> $GITHUB_OUTPUT + echo "${images}" + echo "### Images" >> $GITHUB_STEP_SUMMARY + echo "${images}" | jq -r 'to_entries[] | "* \(.value)"' >> $GITHUB_STEP_SUMMARY + + test-images: + if: ${{ needs.extract-images.outputs.matrix != '[]' }} + name: Test images + runs-on: ubuntu-latest + needs: ["extract-images"] + strategy: + matrix: + images: ${{ fromJSON(needs.extract-images.outputs.matrix) }} + max-parallel: 4 + fail-fast: false + steps: + - name: Inspect Image + run: docker buildx imagetools inspect ${{ matrix.images }} + + # Summarize matrix https://github.community/t/status-check-for-a-matrix-jobs/127354/7 + test-images-success: + if: ${{ always() }} + needs: ["test-images"] + name: Test Images Successful + runs-on: ubuntu-latest + steps: + - if: ${{ contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') }} + name: Check matrix status + run: exit 1 diff --git a/.github/workflows/label-sync.yaml b/.github/workflows/label-sync.yaml new file mode 100644 index 000000000..eeb26ef15 --- /dev/null +++ b/.github/workflows/label-sync.yaml @@ -0,0 +1,30 @@ +--- +# yaml-language-server: $schema=https://json.schemastore.org/github-workflow.json +name: "Label Sync" + +on: + workflow_dispatch: + push: + branches: ["main"] + paths: [".github/labels.yaml"] + schedule: + - cron: "0 0 * * *" # Every day at midnight + +permissions: + issues: write + +jobs: + label-sync: + name: Label Sync + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + sparse-checkout: .github/labels.yaml + + - name: Sync Labels + uses: EndBug/label-sync@v2 + with: + config-file: .github/labels.yaml + delete-other-labels: true diff --git a/.github/workflows/labeler.yaml b/.github/workflows/labeler.yaml new file mode 100644 index 000000000..d658c1d96 --- /dev/null +++ b/.github/workflows/labeler.yaml @@ -0,0 +1,21 @@ +--- +# yaml-language-server: $schema=https://json.schemastore.org/github-workflow.json +name: "Labeler" + +on: + workflow_dispatch: + pull_request_target: + branches: ["main"] + +jobs: + labeler: + name: Labeler + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: write + steps: + - name: Labeler + uses: actions/labeler@v5 + with: + configuration-path: .github/labeler.yaml diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml new file mode 100644 index 000000000..ab809acf3 --- /dev/null +++ b/.github/workflows/release.yaml @@ -0,0 +1,52 @@ +--- +# yaml-language-server: $schema=https://json.schemastore.org/github-workflow.json +name: "Release" + +on: + workflow_dispatch: + schedule: + - cron: "0 0 1 * *" # 1st of every month at midnight + +jobs: + release: + name: Release + runs-on: ubuntu-latest + steps: + - name: Generate Token + uses: actions/create-github-app-token@v1 + id: app-token + with: + app-id: "${{ secrets.BOT_APP_ID }}" + private-key: "${{ secrets.BOT_APP_PRIVATE_KEY }}" + + - name: Checkout + uses: actions/checkout@v4 + with: + token: "${{ steps.app-token.outputs.token }}" + + - name: Create Release + shell: bash + env: + GITHUB_TOKEN: "${{ steps.app-token.outputs.token }}" + run: | + # Retrieve previous release tag + previous_tag="$(gh release list --limit 1 | awk '{ print $1 }')" + previous_major="${previous_tag%%\.*}" + previous_minor="${previous_tag#*.}" + previous_minor="${previous_minor%.*}" + previous_patch="${previous_tag##*.}" + # Determine next release tag + next_major_minor="$(date +'%Y').$(date +'%-m')" + if [[ "${previous_major}.${previous_minor}" == "${next_major_minor}" ]]; then + echo "Month release already exists for year, incrementing patch number by 1" + next_patch="$((previous_patch + 1))" + else + echo "Month release does not exist for year, setting patch number to 0" + next_patch="0" + fi + # Create release + release_tag="${next_major_minor}.${next_patch}" + gh release create "${release_tag}" \ + --repo="${GITHUB_REPOSITORY}" \ + --title="${release_tag}" \ + --generate-notes diff --git a/.github/workflows/renovate.yaml b/.github/workflows/renovate.yaml new file mode 100644 index 000000000..55b4a457f --- /dev/null +++ b/.github/workflows/renovate.yaml @@ -0,0 +1,63 @@ +--- +# yaml-language-server: $schema=https://json.schemastore.org/github-workflow.json +name: "Renovate" + +on: + workflow_dispatch: + inputs: + dryRun: + description: Dry Run + default: "false" + required: false + logLevel: + description: Log Level + default: debug + required: false + version: + description: Renovate version + default: latest + required: false + schedule: + - cron: "0 * * * *" # Every hour + push: + branches: ["main"] + paths: + - .github/renovate.json5 + - .github/renovate/**.json5 + +concurrency: + group: ${{ github.workflow }}-${{ github.event.number || github.ref }} + cancel-in-progress: true + +env: + LOG_LEVEL: "${{ inputs.logLevel || 'debug' }}" + RENOVATE_AUTODISCOVER: true + RENOVATE_AUTODISCOVER_FILTER: "${{ github.repository }}" + RENOVATE_DRY_RUN: "${{ inputs.dryRun == true }}" + RENOVATE_PLATFORM: github + RENOVATE_PLATFORM_COMMIT: true + WORKFLOW_RENOVATE_VERSION: "${{ inputs.version || 'latest' }}" + +jobs: + renovate: + name: Renovate + runs-on: ubuntu-latest + steps: + - name: Generate Token + uses: actions/create-github-app-token@v1 + id: app-token + with: + app-id: "${{ secrets.BOT_APP_ID }}" + private-key: "${{ secrets.BOT_APP_PRIVATE_KEY }}" + + - name: Checkout + uses: actions/checkout@v4 + with: + token: "${{ steps.app-token.outputs.token }}" + + - name: Renovate + uses: renovatebot/github-action@v40.3.1 + with: + configurationFile: .github/renovate.json5 + token: "${{ steps.app-token.outputs.token }}" + renovate-version: "${{ env.WORKFLOW_RENOVATE_VERSION }}" diff --git a/kubernetes/main/apps/actions-runner-system/gha-runner-scale-set-controller/app/externalsecret.yaml b/kubernetes/main/apps/actions-runner-system/gha-runner-scale-set-controller/app/externalsecret.yaml new file mode 100644 index 000000000..90b58c598 --- /dev/null +++ b/kubernetes/main/apps/actions-runner-system/gha-runner-scale-set-controller/app/externalsecret.yaml @@ -0,0 +1,26 @@ +--- +# yaml-language-server: $schema=https://kubernetes-schemas.pages.dev/external-secrets.io/externalsecret_v1beta1.json +apiVersion: external-secrets.io/v1beta1 +kind: ExternalSecret +metadata: + name: actions-runner-controller-auth +spec: + secretStoreRef: + kind: ClusterSecretStore + name: vault-backend + target: + name: actions-runner-controller-auth-secret + template: + engineVersion: v2 + data: + ACTION_RUNNER_CONTROLLER_GITHUB_APP_ID: |- + {{ .ACTION_RUNNER_CONTROLLER_GITHUB_APP_ID }} + ACTION_RUNNER_CONTROLLER_GITHUB_INSTALLATION_ID: |- + {{ .ACTION_RUNNER_CONTROLLER_GITHUB_INSTALLATION_ID }} + ACTION_RUNNER_CONTROLLER_GITHUB_PRIVATE_KEY: |- + {{ .ACTION_RUNNER_CONTROLLER_GITHUB_PRIVATE_KEY }} + ACTION_RUNNER_CONTROLLER_GITHUB_WEBHOOK_SECRET_TOKEN: |- + {{ .ACTION_RUNNER_CONTROLLER_GITHUB_WEBHOOK_SECRET_TOKEN }} + dataFrom: + - extract: + key: secrets/actions-runner-controller diff --git a/kubernetes/main/apps/actions-runner-system/gha-runner-scale-set-controller/app/helmrelease.yaml b/kubernetes/main/apps/actions-runner-system/gha-runner-scale-set-controller/app/helmrelease.yaml new file mode 100644 index 000000000..64cd7d43d --- /dev/null +++ b/kubernetes/main/apps/actions-runner-system/gha-runner-scale-set-controller/app/helmrelease.yaml @@ -0,0 +1,28 @@ +--- +# yaml-language-server: $schema=https://kubernetes-schemas.pages.dev/helm.toolkit.fluxcd.io/helmrelease_v2.json +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: gha-runner-scale-set-controller +spec: + interval: 30m + chart: + spec: + chart: gha-runner-scale-set-controller + version: 0.9.2 + sourceRef: + kind: HelmRepository + name: actions-runner-controller + namespace: flux-system + install: + crds: CreateReplace + remediation: + retries: 3 + upgrade: + cleanupOnFail: true + crds: CreateReplace + remediation: + strategy: rollback + retries: 3 + values: + fullnameOverride: gha-runner-scale-set-controller diff --git a/kubernetes/main/apps/actions-runner-system/gha-runner-scale-set-controller/app/kustomization.yaml b/kubernetes/main/apps/actions-runner-system/gha-runner-scale-set-controller/app/kustomization.yaml new file mode 100644 index 000000000..4eed917b9 --- /dev/null +++ b/kubernetes/main/apps/actions-runner-system/gha-runner-scale-set-controller/app/kustomization.yaml @@ -0,0 +1,7 @@ +--- +# yaml-language-server: $schema=https://json.schemastore.org/kustomization +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ./externalsecret.yaml + - ./helmrelease.yaml diff --git a/kubernetes/main/apps/actions-runner-system/gha-runner-scale-set-controller/ks.yaml b/kubernetes/main/apps/actions-runner-system/gha-runner-scale-set-controller/ks.yaml new file mode 100644 index 000000000..7c5ae7998 --- /dev/null +++ b/kubernetes/main/apps/actions-runner-system/gha-runner-scale-set-controller/ks.yaml @@ -0,0 +1,23 @@ +--- +# yaml-language-server: $schema=https://kubernetes-schemas.pages.dev/kustomize.toolkit.fluxcd.io/kustomization_v1.json +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: &app gha-runner-scale-set-controller + namespace: flux-system +spec: + targetNamespace: actions-runner-system + commonMetadata: + labels: + app.kubernetes.io/name: *app + dependsOn: + - name: external-secrets-stores + path: ./kubernetes/main/apps/actions-runner-system/gha-runner-scale-set-controller/app + prune: true + sourceRef: + kind: GitRepository + name: home-kubernetes + wait: false + interval: 30m + retryInterval: 1m + timeout: 5m diff --git a/kubernetes/main/apps/actions-runner-system/gha-runner-scale-set/app/helmrelease.yaml b/kubernetes/main/apps/actions-runner-system/gha-runner-scale-set/app/helmrelease.yaml new file mode 100644 index 000000000..0551df9eb --- /dev/null +++ b/kubernetes/main/apps/actions-runner-system/gha-runner-scale-set/app/helmrelease.yaml @@ -0,0 +1,57 @@ +--- +# yaml-language-server: $schema=https://kubernetes-schemas.pages.dev/helm.toolkit.fluxcd.io/helmrelease_v2.json +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: gha-runner-scale-set +spec: + interval: 30m + chart: + spec: + chart: gha-runner-scale-set + version: 0.9.2 + sourceRef: + kind: HelmRepository + name: actions-runner-controller + namespace: flux-system + install: + remediation: + retries: 3 + upgrade: + cleanupOnFail: true + remediation: + strategy: rollback + retries: 3 + dependsOn: + - name: gha-runner-scale-set-controller + namespace: actions-runner-system + valuesFrom: + - targetPath: githubConfigSecret.github_app_id + kind: Secret + name: actions-runner-controller-auth-secret + valuesKey: ACTION_RUNNER_CONTROLLER_GITHUB_APP_ID + - targetPath: githubConfigSecret.github_app_installation_id + kind: Secret + name: actions-runner-controller-auth-secret + valuesKey: ACTION_RUNNER_CONTROLLER_GITHUB_INSTALLATION_ID + - targetPath: githubConfigSecret.github_app_private_key + kind: Secret + name: actions-runner-controller-auth-secret + valuesKey: ACTION_RUNNER_CONTROLLER_GITHUB_PRIVATE_KEY + values: + nameOverride: gha-runner-scale-set + runnerScaleSetName: gha-runner-scale-set + githubConfigUrl: https://github.com/Darkfella91/home-ops + minRunners: 1 + maxRunners: 6 + containerMode: + type: dind + template: + spec: + containers: + - name: runner + image: ghcr.io/onedr0p/actions-runner:2.319.1@sha256:a4089b96bb4561051c954cc1f9019497dcc166c027b8e1474da7246a16796b43 + command: ["/home/runner/run.sh"] + controllerServiceAccount: + name: gha-runner-scale-set-controller + namespace: actions-runner-system diff --git a/kubernetes/main/apps/actions-runner-system/gha-runner-scale-set/app/kustomization.yaml b/kubernetes/main/apps/actions-runner-system/gha-runner-scale-set/app/kustomization.yaml new file mode 100644 index 000000000..17cbc72b2 --- /dev/null +++ b/kubernetes/main/apps/actions-runner-system/gha-runner-scale-set/app/kustomization.yaml @@ -0,0 +1,6 @@ +--- +# yaml-language-server: $schema=https://json.schemastore.org/kustomization +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ./helmrelease.yaml diff --git a/kubernetes/main/apps/actions-runner-system/gha-runner-scale-set/ks.yaml b/kubernetes/main/apps/actions-runner-system/gha-runner-scale-set/ks.yaml new file mode 100644 index 000000000..293feafa2 --- /dev/null +++ b/kubernetes/main/apps/actions-runner-system/gha-runner-scale-set/ks.yaml @@ -0,0 +1,21 @@ +--- +# yaml-language-server: $schema=https://kubernetes-schemas.pages.dev/kustomize.toolkit.fluxcd.io/kustomization_v1.json +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: &app gha-runner-scale-set + namespace: flux-system +spec: + targetNamespace: actions-runner-system + commonMetadata: + labels: + app.kubernetes.io/name: *app + path: ./kubernetes/main/apps/actions-runner-system/gha-runner-scale-set/app + prune: true + sourceRef: + kind: GitRepository + name: home-kubernetes + wait: false + interval: 30m + retryInterval: 1m + timeout: 5m diff --git a/kubernetes/main/apps/actions-runner-system/kustomization.yaml b/kubernetes/main/apps/actions-runner-system/kustomization.yaml new file mode 100644 index 000000000..98183e38e --- /dev/null +++ b/kubernetes/main/apps/actions-runner-system/kustomization.yaml @@ -0,0 +1,10 @@ +--- +# yaml-language-server: $schema=https://json.schemastore.org/kustomization +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + # Pre Flux-Kustomizations + - ./namespace.yaml + # Flux-Kustomizations + - ./gha-runner-scale-set-controller/ks.yaml + - ./gha-runner-scale-set/ks.yaml diff --git a/kubernetes/main/apps/actions-runner-system/namespace.yaml b/kubernetes/main/apps/actions-runner-system/namespace.yaml new file mode 100644 index 000000000..7bdef02e2 --- /dev/null +++ b/kubernetes/main/apps/actions-runner-system/namespace.yaml @@ -0,0 +1,38 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: actions-runner-system + annotations: + kustomize.toolkit.fluxcd.io/prune: disabled + volsync.backube/privileged-movers: "true" +--- +# yaml-language-server: $schema=https://kubernetes-schemas.pages.dev/notification.toolkit.fluxcd.io/provider_v1beta3.json +apiVersion: notification.toolkit.fluxcd.io/v1beta3 +kind: Provider +metadata: + name: alert-manager + namespace: actions-runner-system +spec: + type: alertmanager + address: http://alertmanager-operated.observability.svc.cluster.local:9093/api/v2/alerts/ +--- +# yaml-language-server: $schema=https://kubernetes-schemas.pages.dev/notification.toolkit.fluxcd.io/alert_v1beta3.json +apiVersion: notification.toolkit.fluxcd.io/v1beta3 +kind: Alert +metadata: + name: alert-manager + namespace: actions-runner-system +spec: + providerRef: + name: alert-manager + eventSeverity: error + eventSources: + - kind: HelmRelease + name: "*" + exclusionList: + - "error.*lookup github\\.com" + - "error.*lookup raw\\.githubusercontent\\.com" + - "dial.*tcp.*timeout" + - "waiting.*socket" + suspend: false diff --git a/kubernetes/main/flux/repositories/helm/actions-runner-controller.yaml b/kubernetes/main/flux/repositories/helm/actions-runner-controller.yaml new file mode 100644 index 000000000..c3ad49056 --- /dev/null +++ b/kubernetes/main/flux/repositories/helm/actions-runner-controller.yaml @@ -0,0 +1,12 @@ +--- +# yaml-language-server: $schema=https://kubernetes-schemas.pages.dev/source.toolkit.fluxcd.io/helmrepository_v1.json +apiVersion: source.toolkit.fluxcd.io/v1 +kind: HelmRepository +metadata: + name: bitnami + namespace: flux-system +spec: + type: oci + interval: 5m + url: oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set-controller + diff --git a/kubernetes/main/flux/repositories/helm/kustomization.yaml b/kubernetes/main/flux/repositories/helm/kustomization.yaml index 7d24bbbe7..64d1dcb8d 100644 --- a/kubernetes/main/flux/repositories/helm/kustomization.yaml +++ b/kubernetes/main/flux/repositories/helm/kustomization.yaml @@ -3,6 +3,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: + - ./actions-runner-controller.yaml - ./authentik.yaml - ./backube.yaml - ./bitnami.yaml