From 31613df2b30784476a9c0c2d7102fa79abeac1d0 Mon Sep 17 00:00:00 2001 From: Joseph Richardson <49208786+JoeHCQ1@users.noreply.github.com> Date: Fri, 6 Dec 2024 13:45:44 -0500 Subject: [PATCH] fix: added gitaly cgroups init image to package (#246) ## Description Downstream we use this image to modify cgroups on gitaly nodes to improve resiliency. This isn't in the package right now, so we're having to add it to a local package so it is available. By putting it here in the original package we ensure the version always matches the rest of the install, and save others who do similar work time later. ## Type of change - [X] Bug fix (non-breaking change which fixes an issue) ## Checklist before merging - [X] Test, docs, adr added or updated as needed - [X] [Contributor Guide Steps](https://github.com/defenseunicorns/uds-package-gitlab/blob/main/CONTRIBUTING.md#developer-workflow) followed --------- Co-authored-by: Rob Co-authored-by: Wayne Starr Co-authored-by: Rob McElvenny Release-As: v17.6.1-uds.1 --- .../templates/gitaly-cgroups-exemption.yaml | 23 +++++++++++++++++++ .../templates/gitlab-object-store-secret.yaml | 2 +- charts/config/values.yaml | 4 ++++ .../settings/templates/settings-secret.yaml | 2 +- docs/configuration.md | 13 ++++++++++- values/common-values.yaml | 9 ++++++++ values/registry1-values.yaml | 1 + values/unicorn-values.yaml | 5 ++++ values/upstream-values.yaml | 5 ++++ zarf.yaml | 2 ++ 10 files changed, 63 insertions(+), 3 deletions(-) create mode 100644 charts/config/templates/gitaly-cgroups-exemption.yaml diff --git a/charts/config/templates/gitaly-cgroups-exemption.yaml b/charts/config/templates/gitaly-cgroups-exemption.yaml new file mode 100644 index 00000000..9f30ac02 --- /dev/null +++ b/charts/config/templates/gitaly-cgroups-exemption.yaml @@ -0,0 +1,23 @@ +# Copyright 2024 Defense Unicorns +# SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Defense-Unicorns-Commercial + +{{- if .Values.gitalyCgroupsInit.enabled }} + +apiVersion: uds.dev/v1alpha1 +kind: Exemption +metadata: + name: gitaly-cgroups-init-container + namespace: uds-policy-exemptions +spec: + exemptions: + - policies: + - RestrictHostPathWrite + - RestrictVolumeTypes + - RequireNonRootUser + - DisallowPrivileged + matcher: + namespace: {{ .Release.Namespace }} + name: "gitlab-gitaly.*" + title: "gitlab gitaly exemptions" + description: "Exemption allows cgroup modification by init container. See https://docs.gitlab.com/ee/administration/gitaly/kubernetes.html#constrain-git-processes-resource-usage" +{{- end }} diff --git a/charts/config/templates/gitlab-object-store-secret.yaml b/charts/config/templates/gitlab-object-store-secret.yaml index 80e02c26..e73a131e 100644 --- a/charts/config/templates/gitlab-object-store-secret.yaml +++ b/charts/config/templates/gitlab-object-store-secret.yaml @@ -6,7 +6,7 @@ apiVersion: v1 kind: Secret metadata: name: gitlab-object-store - namespace: gitlab + namespace: {{ .Release.Namespace }} type: kubernetes.io/opaque stringData: {{- $awsAccessKey := "" }} diff --git a/charts/config/values.yaml b/charts/config/values.yaml index ccccba94..b5c7d246 100644 --- a/charts/config/values.yaml +++ b/charts/config/values.yaml @@ -81,6 +81,10 @@ mirroring: ports: - 443 +# Add the exemption for the gitaly cgroups init container +gitalyCgroupsInit: + enabled: false + # custom: # # Notice no `remoteGenerated` field here on custom internal rule # - direction: Ingress diff --git a/charts/settings/templates/settings-secret.yaml b/charts/settings/templates/settings-secret.yaml index 57d9dde4..93462704 100644 --- a/charts/settings/templates/settings-secret.yaml +++ b/charts/settings/templates/settings-secret.yaml @@ -6,7 +6,7 @@ apiVersion: v1 kind: Secret metadata: name: gitlab-settings-secret - namespace: gitlab + namespace: {{ .Release.Namespace }} type: Opaque stringData: application.json: {{ .Values.settingsJob.application | toJson | quote }} diff --git a/docs/configuration.md b/docs/configuration.md index 822d148a..b22055e2 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -83,7 +83,7 @@ By default, the application is configured to work with `uds-package-minio-operat If you are not using in-cluster MinIO, but rather are using an external cloud providers object storage, you have two options. You can either create an object storage secret manually and disable the generation of the secret or have the helm chart generate one for you based on a set of input values. > [!NOTE] -> If you would like to opt out of the in-chart secret generation process, you may disable it by setting the zarf variable GENERATE_STORAGE_SECRET to false. Then you can provide your own object store secret, named gitlab-object-store, as needed following GitLab's documentation. +> If you would like to opt out of the in-chart secret generation process, you may disable it by setting the zarf variable `GENERATE_STORAGE_SECRET` to false. Then you can provide your own object store secret, named `gitlab-object-store`, as needed following GitLab's documentation. When configuring the GitLab to connect to S3 storage in AWS, it is assumed IRSA will be used to connect to the buckets. The prerequisites for this are the buckets created with the appropriate iam roles and policies. Once those are created, two values need to be overridden in the config chart for secret generation: `storage.createSecret.provider` needs to be set to `aws` and `storage.createSecret.region` needs to be set to your AWS regions (i.e `us-gov-west-1`). From there, additional overrides are required in the gitlab chart to finish this setup. Specifically, the gitlab service accounts need to be overridden to have the annotations that are required for IRSA. Below is an example of how you would define the variable overrides where you would then pass in the IAM role ARNs on deploy. @@ -327,3 +327,14 @@ This will configure a bot account named `renovatebot` and create a PAT with scop > [!NOTE] > If the GitLab instance is configured with a license for Premium or Ultimate, [Gitlab Service Accounts](https://docs.gitlab.com/ee/user/profile/service_accounts.html) will be created. Otherwise, standard user accounts will be created. + +## Gitaly HA + +To use [custom cgroup sizes for Gitaly](https://docs.gitlab.com/ee/administration/gitaly/kubernetes.html#constrain-git-processes-resource-usage): + +1. Set `gitlab.gitaly.cgroups.enabled` to `true` in the `gitlab` chart. +2. Set the cgroup permissions under the pod's resource limits as shown in the [GitLab docs](https://docs.gitlab.com/ee/administration/gitaly/kubernetes.html#constrain-git-processes-resource-usage). +3. Set `gitalyCgroupsInit` to `true` in the `uds-gitlab-config` chart. This causes a policy exemption to be created allowing the init container privileged access to the host nodes, required to customize the cgroups. + +> [!NOTE] +> Only the `upstream` and `unicorn` flavors include the Gitaly init container required for this configuration. It will not work if using the `registry1` flavor. diff --git a/values/common-values.yaml b/values/common-values.yaml index c98c2d54..b06cb55e 100644 --- a/values/common-values.yaml +++ b/values/common-values.yaml @@ -186,6 +186,15 @@ gitlab: enabled: true serviceMonitor: enabled: true + cgroups: + enabled: false + initContainer: + securityContext: + # This multi-line oddity is a hacky way to bypass: https://gitlab.com/gitlab-org/gitlab/-/issues/507883 + runAsUser: | + 0 + privileged: true + runAsGroup: 0 gitlab-shell: # override to enable ssh diff --git a/values/registry1-values.yaml b/values/registry1-values.yaml index 6eee4fba..e5aa0254 100644 --- a/values/registry1-values.yaml +++ b/values/registry1-values.yaml @@ -22,6 +22,7 @@ gitlab: image: repository: registry1.dso.mil/ironbank/gitlab/gitlab/gitaly tag: 17.6.1 + # Note, the registry1 flavor is missing the cgroups init image because it's not in ironbank gitlab-exporter: image: repository: registry1.dso.mil/ironbank/gitlab/gitlab/gitlab-exporter diff --git a/values/unicorn-values.yaml b/values/unicorn-values.yaml index f7340e00..c01b8b9b 100644 --- a/values/unicorn-values.yaml +++ b/values/unicorn-values.yaml @@ -22,6 +22,11 @@ gitlab: image: repository: registry.gitlab.com/gitlab-org/build/cng/gitaly tag: v17.6.1 + cgroups: + initContainer: + image: + repository: registry.gitlab.com/gitlab-org/build/cng/gitaly-init-cgroups + tag: v17.6.1 gitlab-exporter: image: repository: registry.gitlab.com/gitlab-org/build/cng/gitlab-exporter diff --git a/values/upstream-values.yaml b/values/upstream-values.yaml index f7340e00..c01b8b9b 100644 --- a/values/upstream-values.yaml +++ b/values/upstream-values.yaml @@ -22,6 +22,11 @@ gitlab: image: repository: registry.gitlab.com/gitlab-org/build/cng/gitaly tag: v17.6.1 + cgroups: + initContainer: + image: + repository: registry.gitlab.com/gitlab-org/build/cng/gitaly-init-cgroups + tag: v17.6.1 gitlab-exporter: image: repository: registry.gitlab.com/gitlab-org/build/cng/gitlab-exporter diff --git a/zarf.yaml b/zarf.yaml index 817797c4..1cbce058 100644 --- a/zarf.yaml +++ b/zarf.yaml @@ -88,6 +88,7 @@ components: - "registry.gitlab.com/gitlab-org/build/cng/certificates:v17.6.1" - "registry.gitlab.com/gitlab-org/build/cng/cfssl-self-sign:v17.6.1" - "registry.gitlab.com/gitlab-org/build/cng/gitaly:v17.6.1" + - "registry.gitlab.com/gitlab-org/build/cng/gitaly-init-cgroups:v17.6.1" - "registry.gitlab.com/gitlab-org/build/cng/gitlab-container-registry:v17.6.1" - "registry.gitlab.com/gitlab-org/build/cng/gitlab-pages:v17.6.1" - "registry.gitlab.com/gitlab-org/build/cng/gitlab-shell:v17.6.1" @@ -118,6 +119,7 @@ components: - "registry.gitlab.com/gitlab-org/build/cng/certificates:v17.6.1" - "registry.gitlab.com/gitlab-org/build/cng/cfssl-self-sign:v17.6.1" - "registry.gitlab.com/gitlab-org/build/cng/gitaly:v17.6.1" + - "registry.gitlab.com/gitlab-org/build/cng/gitaly-init-cgroups:v17.6.1" - "registry.gitlab.com/gitlab-org/build/cng/gitlab-container-registry:v17.6.1" - "registry.gitlab.com/gitlab-org/build/cng/gitlab-pages:v17.6.1" - "registry.gitlab.com/gitlab-org/build/cng/gitlab-shell:v17.6.1"