Merge branch 'main' into 458_lula-integration-istio

src/keycloak/chart/README.md

@@ -10,7 +10,11 @@ For more information on Keycloak and its capabilities, see its [documentation](h
 
 ### Dev Mode
 
-When `devMode: true` is set, the chart will deploy a single Keycloak Pod with an in-memory database and scaling turned off. Devmode also leverages PVCs by default for `data` and `themes`.
+When `devMode: true` is set, the chart will deploy a single Keycloak Pod with an in-memory database and scaling turned off. Dev Mode also leverages PVCs by default for `data` and `themes`. 
+
+Using an external database with Dev Mode enabled is not supported.
+
+Dev Mode enables debug logging for Keycloak. To configure debug logging outside of Dev Mode, set `debugMode: true` in your values.
 
 ### Autoscaling
 

src/keycloak/chart/templates/_helpers.tpl

@@ -74,3 +74,24 @@ Create the service DNS name.
 {{- define "keycloak.serviceDnsName" -}}
 {{ include "keycloak.fullname" . }}-headless.{{ .Release.Namespace }}.svc.{{ .Values.clusterDomain }}
 {{- end }}
+
+{{/*
+Check external PostgreSQL connection information. Fails when required values are missing or if PostgreSQL is configured when devMode is enabled.
+*/}}
+
+{{- define "keycloak.postgresql.config" -}}
+{{- if not .Values.devMode -}}
+{{- if .Values.postgresql -}}
+{{ $requiredKeys := list "username" "password" "database" "host" "port" }}
+{{- range $k := $requiredKeys -}}
+{{ if empty (get $.Values.postgresql $k) }}{{- fail (printf "Missing value for \"postgresql.%s\"." $k ) -}}{{- end }}
+{{- end }}
+{{- else -}}{{fail "You must define \"username\", \"password\", \"database\", \"host\", and \"port\" for \"postgresql\"."}}
+{{- end -}}
+{{- default "true" "" }}
+{{- else if not (empty (compact (values (omit .Values.postgresql "port")))) -}}
+{{ fail "Cannot use an external PostgreSQL Database when devMode is enabled." -}}
+{{- else -}}
+{{ default "false" "" }}
+{{- end }}
+{{- end }}

src/keycloak/chart/templates/secret-postgresql.yaml

@@ -1,4 +1,4 @@
-{{- if not .Values.devMode }}
+{{- if eq (include "keycloak.postgresql.config" .) "true" }}
 apiVersion: v1
 kind: Secret
 metadata:
@@ -13,4 +13,4 @@ data:
   password: {{ .Values.postgresql.password | b64enc }}
   host: {{ .Values.postgresql.host | b64enc }}
   port: {{ .Values.postgresql.port | toString | b64enc }}
-{{- end }}
+{{- end }}

src/keycloak/chart/templates/statefulset.yaml

@@ -113,15 +113,16 @@ spec:
             # Dumb value (not used in the nginx provider, but required by the SPI)
             - name: KC_SPI_X509CERT_LOOKUP_NGINX_SSL_CLIENT_CERT_CHAIN_PREFIX
               value: UNUSED
-          {{- if .Values.devMode }}
-            # Enable dubug logs in dev mode
+          {{- if or .Values.devMode .Values.debugMode }}
+            # Enable debug logs
             - name: KC_LOG_LEVEL
               value: DEBUG
             - name: QUARKUS_LOG_CATEGORY__ORG_APACHE_HTTP__LEVEL
               value: DEBUG
             - name: QUARKUS_LOG_CATEGORY__ORG_KEYCLOAK_SERVICES_X509__LEVEL
               value: TRACE
-          {{- else }}
+          {{- end }}
+          {{- if eq (include "keycloak.postgresql.config" .) "true" }}
             # Infinispan cache configuration
             - name: KC_CACHE
               value: ispn
@@ -168,7 +169,7 @@ spec:
             - name: JAVA_TOOL_OPTIONS
               value: "-Dcom.redhat.fips=true"
             {{- end }}
-          {{- end }}
+           {{- end }}  
           {{- if .Values.insecureAdminPasswordGeneration.enabled }}
             - name: KEYCLOAK_ADMIN
               valueFrom:

src/keycloak/chart/templates/uds-package.yaml

@@ -52,8 +52,8 @@ spec:
         remoteGenerated: Anywhere
       {{- end }}
 
-      {{- if not .Values.devMode }}
-      - description: "PostgresQL Database access"
+      {{- if eq (include "keycloak.postgresql.config" .) "true" }}
+      - description: "PostgreSQL Database access"
         direction: Egress
         selector:
           app.kubernetes.io/name: keycloak

src/keycloak/chart/values.yaml

@@ -55,8 +55,12 @@ terminationGracePeriodSeconds: 5
 clusterDomain: cluster.local
 
 # Sets development mode for Keycloak. This disables caching, Postgres and HPAs and should only be used for testing
+# Must have no values populated for `postgresql` in order to use
 devMode: true
 
+# Enable debug logging for keycloak and quarkus
+debugMode: false
+
 # Enable SMTP networkPolicy and config
 smtp:
   enabled: false
@@ -149,16 +153,17 @@ service:
   # Session affinity config
   sessionAffinityConfig: {}
 
+# Connection information for external postgres database
 postgresql:
-  # PostgreSQL User to create
-  username: keycloak
-  # PostgreSQL Password for the new user
-  password: keycloak
-  # PostgreSQL Database to create
-  database: keycloak
-  # PostgreSQL host
-  host: postgresql
-  # PostgreSQL port
+  # The username of the database user
+  username: ""
+  # The password of the database user
+  password: ""
+  # Database name
+  database: ""
+  # URL for the database
+  host: ""
+  # Port the database is listening on
   port: 5432
 
 serviceMonitor:

src/pepr/operator/controllers/istio/istio-resources.ts

@@ -2,7 +2,7 @@ import { K8s } from "pepr";
 
 import { Component, setupLogger } from "../../../logger";
 import { IstioServiceEntry, IstioVirtualService, UDSPackage } from "../../crd";
-import { getOwnerRef } from "../utils";
+import { getOwnerRef, purgeOrphans } from "../utils";
 import { generateServiceEntry } from "./service-entry";
 import { generateVirtualService } from "./virtual-service";
 
@@ -57,39 +57,8 @@ export async function istioResources(pkg: UDSPackage, namespace: string) {
     serviceEntryNames.set(sePayload.metadata!.name!, true);
   }
 
-  // Get all related VirtualServices in the namespace
-  const virtualServices = await K8s(IstioVirtualService)
-    .InNamespace(namespace)
-    .WithLabel("uds/package", pkgName)
-    .Get();
-
-  // Find any orphaned VirtualServices (not matching the current generation)
-  const orphanedVS = virtualServices.items.filter(
-    vs => vs.metadata?.labels?.["uds/generation"] !== generation,
-  );
-
-  // Delete any orphaned VirtualServices
-  for (const vs of orphanedVS) {
-    log.debug(vs, `Deleting orphaned VirtualService ${vs.metadata!.name}`);
-    await K8s(IstioVirtualService).Delete(vs);
-  }
-
-  // Get all related ServiceEntries in the namespace
-  const serviceEntries = await K8s(IstioServiceEntry)
-    .InNamespace(namespace)
-    .WithLabel("uds/package", pkgName)
-    .Get();
-
-  // Find any orphaned ServiceEntries (not matching the current generation)
-  const orphanedSE = serviceEntries.items.filter(
-    se => se.metadata?.labels?.["uds/generation"] !== generation,
-  );
-
-  // Delete any orphaned ServiceEntries
-  for (const se of orphanedSE) {
-    log.debug(se, `Deleting orphaned ServiceEntry ${se.metadata!.name}`);
-    await K8s(IstioServiceEntry).Delete(se);
-  }
+  await purgeOrphans(generation, namespace, pkgName, IstioVirtualService, log);
+  await purgeOrphans(generation, namespace, pkgName, IstioServiceEntry, log);
 
   // Return the list of unique hostnames
   return [...hosts];

src/pepr/operator/controllers/keycloak/authservice/authorization-policy.ts

@@ -6,7 +6,7 @@ import {
   IstioRequestAuthentication,
   UDSPackage,
 } from "../../../crd";
-import { getOwnerRef } from "../../utils";
+import { getOwnerRef, purgeOrphans } from "../../utils";
 import { log } from "./authservice";
 import { Action as AuthServiceAction, AuthServiceEvent } from "./types";
 
@@ -155,17 +155,7 @@ async function updatePolicy(
 
 async function purgeOrphanPolicies(generation: string, namespace: string, pkgName: string) {
   for (const kind of [IstioAuthorizationPolicy, IstioRequestAuthentication]) {
-    const resources = await K8s(kind)
-      .InNamespace(namespace)
-      .WithLabel("uds/package", pkgName)
-      .Get();
-
-    for (const resource of resources.items) {
-      if (resource.metadata?.labels?.["uds/generation"] !== generation) {
-        log.debug(resource, `Deleting orphaned ${resource.kind!} ${resource.metadata!.name}`);
-        await K8s(kind).Delete(resource);
-      }
-    }
+    await purgeOrphans(generation, namespace, pkgName, kind, log);
   }
 }
 

src/pepr/operator/controllers/monitoring/pod-monitor.ts

@@ -3,7 +3,7 @@ import { K8s } from "pepr";
 import { Component, setupLogger } from "../../../logger";
 import { Monitor, PrometheusPodMonitor, UDSPackage } from "../../crd";
 import { Kind } from "../../crd/generated/package-v1alpha1";
-import { getOwnerRef } from "../utils";
+import { getOwnerRef, purgeOrphans } from "../utils";
 import { generateMonitorName } from "./common";
 
 // configure subproject logger
@@ -42,22 +42,7 @@ export async function podMonitor(pkg: UDSPackage, namespace: string) {
       }
     }
 
-    // Get all related PodMonitors in the namespace
-    const podMonitors = await K8s(PrometheusPodMonitor)
-      .InNamespace(namespace)
-      .WithLabel("uds/package", pkgName)
-      .Get();
-
-    // Find any orphaned PodMonitors (not matching the current generation)
-    const orphanedMonitor = podMonitors.items.filter(
-      m => m.metadata?.labels?.["uds/generation"] !== generation,
-    );
-
-    // Delete any orphaned PodMonitors
-    for (const m of orphanedMonitor) {
-      log.debug(m, `Deleting orphaned PodMonitor ${m.metadata!.name}`);
-      await K8s(PrometheusPodMonitor).Delete(m);
-    }
+    await purgeOrphans(generation, namespace, pkgName, PrometheusPodMonitor, log);
   } catch (err) {
     throw new Error(`Failed to process PodMonitors for ${pkgName}, cause: ${JSON.stringify(err)}`);
   }

src/pepr/operator/controllers/monitoring/service-monitor.ts

@@ -4,7 +4,7 @@ import { V1OwnerReference } from "@kubernetes/client-node";
 import { Component, setupLogger } from "../../../logger";
 import { Monitor, PrometheusServiceMonitor, UDSPackage } from "../../crd";
 import { Kind } from "../../crd/generated/package-v1alpha1";
-import { getOwnerRef } from "../utils";
+import { getOwnerRef, purgeOrphans } from "../utils";
 import { generateMonitorName } from "./common";
 
 // configure subproject logger
@@ -43,22 +43,7 @@ export async function serviceMonitor(pkg: UDSPackage, namespace: string) {
       }
     }
 
-    // Get all related ServiceMonitors in the namespace
-    const serviceMonitors = await K8s(PrometheusServiceMonitor)
-      .InNamespace(namespace)
-      .WithLabel("uds/package", pkgName)
-      .Get();
-
-    // Find any orphaned ServiceMonitors (not matching the current generation)
-    const orphanedMonitor = serviceMonitors.items.filter(
-      m => m.metadata?.labels?.["uds/generation"] !== generation,
-    );
-
-    // Delete any orphaned ServiceMonitors
-    for (const m of orphanedMonitor) {
-      log.debug(m, `Deleting orphaned ServiceMonitor ${m.metadata!.name}`);
-      await K8s(PrometheusServiceMonitor).Delete(m);
-    }
+    await purgeOrphans(generation, namespace, pkgName, PrometheusServiceMonitor, log);
   } catch (err) {
     throw new Error(
       `Failed to process ServiceMonitors for ${pkgName}, cause: ${JSON.stringify(err)}`,

src/pepr/operator/controllers/network/policies.ts

@@ -2,7 +2,7 @@ import { K8s, kind } from "pepr";
 
 import { Component, setupLogger } from "../../../logger";
 import { Allow, Direction, Gateway, UDSPackage } from "../../crd";
-import { getOwnerRef, sanitizeResourceName } from "../utils";
+import { getOwnerRef, purgeOrphans, sanitizeResourceName } from "../utils";
 import { allowEgressDNS } from "./defaults/allow-egress-dns";
 import { allowEgressIstiod } from "./defaults/allow-egress-istiod";
 import { allowIngressSidecarMonitoring } from "./defaults/allow-ingress-sidecar-monitoring";
@@ -146,22 +146,7 @@ export async function networkPolicies(pkg: UDSPackage, namespace: string) {
     await K8s(kind.NetworkPolicy).Apply(policy, { force: true });
   }
 
-  // Delete any policies that are no longer needed
-  const policyList = await K8s(kind.NetworkPolicy)
-    .InNamespace(namespace)
-    .WithLabel("uds/package", pkgName)
-    .Get();
-
-  // Find any orphaned polices (not matching the current generation)
-  const orphanedNetPol = policyList.items.filter(
-    netPol => netPol.metadata?.labels?.["uds/generation"] !== generation,
-  );
-
-  // Delete any orphaned policies
-  for (const netPol of orphanedNetPol) {
-    log.debug(netPol, `Deleting orphaned NetworkPolicy ${netPol.metadata!.name}`);
-    await K8s(kind.NetworkPolicy).Delete(netPol);
-  }
+  await purgeOrphans(generation, namespace, pkgName, kind.NetworkPolicy, log);
 
   // Return the list of policies
   return policies;

src/pepr/operator/controllers/utils.ts

@@ -1,5 +1,7 @@
 import { V1OwnerReference } from "@kubernetes/client-node";
-import { GenericKind } from "kubernetes-fluent-client";
+import { GenericClass, GenericKind } from "kubernetes-fluent-client";
+import { K8s } from "pepr";
+import { Logger } from "pino";
 
 /**
  * Sanitize a resource name to make it a valid Kubernetes resource name.
@@ -38,3 +40,20 @@ export function getOwnerRef(cr: GenericKind): V1OwnerReference[] {
     },
   ];
 }
+
+export async function purgeOrphans<T extends GenericClass>(
+  generation: string,
+  namespace: string,
+  pkgName: string,
+  kind: T,
+  log: Logger,
+) {
+  const resources = await K8s(kind).InNamespace(namespace).WithLabel("uds/package", pkgName).Get();
+
+  for (const resource of resources.items) {
+    if (resource.metadata?.labels?.["uds/generation"] !== generation) {
+      log.debug(resource, `Deleting orphaned ${resource.kind!} ${resource.metadata!.name}`);
+      await K8s(kind).Delete(resource);
+    }
+  }
+}

src/promtail/tasks.yaml

@@ -1,7 +1,7 @@
 tasks:
   - name: validate
     actions:
-      - description: Validate promail
+      - description: Validate promtail
         wait:
           cluster:
             kind: Pod

src/velero/tasks.yaml

@@ -54,7 +54,7 @@ tasks:
             echo "Status is '$STATUS'... waiting to see if it changes"
 
             # local testing indicates the status is "Finalizing" for a few seconds after completion
-            sleep 15
+            sleep 30
 
             # check again...
             STATUS=$(uds zarf tools kubectl get backups -n velero ${BACKUP_NAME} -o jsonpath='{.status.phase}')