chore(deps): update prometheus-stack (prometheus 3.0.1, operator 0.79…

….2) (#949)

Note: This PR was initially generated by Renovate but due to changes
needed in the operator/other places to support the latest Prometheus
version this PR description has been scrubbed of the original contents.

## Description

This PR contains a number of application updates to the prometheus
stack:
- Prometheus: `2.54.1` -> `3.0.1`
- Prometheus Operator: `0.77.1` -> `0.79.2`
- Prometheus Config Reloader: `0.77.1` -> `0.79.2`
- Kube State Metrics: `2.13.0` -> `2.14.0`

Two helm chart updates are also included for the prometheus stack:
- Kube-Prometheus-Stack: `65.3.1` -> `67.3.1`
- Prometheus-Operator-CRDs: `15.0.0` -> `17.0.2`

In addition to provide a seamless upgrade experience with the new
version of Prometheus 3.0 (specifically the change in behavior around
[scrape
protocol](https://prometheus.io/docs/prometheus/3.0/migration/#scrape-protocols)),
the following changes were made:
- Addition of support for setting `fallbackScrapeProtocol` in the
`monitor` section of the `Package` CR spec
- New mutation added to PodMonitor and ServiceMonitor code to set the
`fallbackScrapeProtocol` to `PrometheusText004` by default (this matches
the pre-3.x behavior in Prometheus)
- "One-time" `onDeploy.after` task to annotate all existing pod and
service monitors: This forces all monitors to go through the mutation
cycle again and have the `fallbackScrapeProtocol` added immediately,
without operator intervention. Note that other options were considered
(i.e. doing this via Pepr) but this was seen as the easiest and most
lightweight solution.
- In order to support the above changes in the operator code the types
for Prometheus CRDs were regenerated

Also note that since this update includes a major version of Prometheus
there are additional notes in the [upstream migration
guide](https://prometheus.io/docs/prometheus/3.0/migration/). In testing
with UDS Core and several apps on top of core no immediate issues were
identified with this upgrade.

## Related Issue

N/A

## Type of change

- [ ] Bug fix (non-breaking change which fixes an issue)
- [ ] New feature (non-breaking change which adds functionality)
- [x] Other (security config, docs update, etc)

## Steps to Validate

The issue with metrics and scrape protocol can be seen with GitLab. To
validate the proper behavior across upgrades I ran through the below
steps:
```console
# Using Colima the below VM was a good size to handle the GitLab workloads
colima start --cpu 12 --memory 20 -t vz --disk 100

# Deploy part of core FROM `main`
uds run test-single-layer --set LAYER=monitoring --set FLAVOR=unicorn

# Deploy gitlab test bundle
uds deploy ghcr.io/defenseunicorns/packages/uds/bundles/gitlab-test:17.6.2-uds.0 --confirm

# Check metrics status using the e2e test (this is with main so all should pass, just here as a sanity check)
uds run -f src/prometheus-stack/tasks.yaml e2e-test

# Deploy core base, then monitoring FROM `renovate/prometheus-stack` branch
uds run test:single-layer --set LAYER=base --set FLAVOR=unicorn
uds run test:single-layer --set LAYER=monitoring --set FLAVOR=unicorn 
# Note that the tests may fail here due to timing, Prometheus takes upwards of 1 min to read in the new config, re-run them after a short wait
uds run -f src/prometheus-stack/tasks.yaml e2e-test
# Expected behavior here is that all metrics continue to work, simply by deploying latest core, despite upstream issues with GitLab's content type on some metrics

# Check metrics status (manual version of the e2e test) if desired
kubectl port-forward svc/kube-prometheus-stack-prometheus 9090:9090 -n monitoring
# Navigate to http://localhost:9090/targets?pool= and validate all show as up/no targets
```

To more generally validate core:
```console
# Test a complete install
uds run test-uds-core --set FLAVOR=unicorn
# Run the e2e tests for Prom
uds run -f src/prometheus-stack/tasks.yaml e2e-test

# Test a complete upgrade (e2e tests will run as part of this)
uds run test-uds-core-upgrade --set FLAVOR=unicorn
```

## Checklist before merging

- [x] Test, docs, adr added or updated as needed
- [x] [Contributor
Guide](https://github.com/defenseunicorns/uds-template-capability/blob/main/CONTRIBUTING.md)
followed

---------

Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com>
Co-authored-by: Micah Nagel <[email protected]>

.codespellrc

@@ -1,6 +1,6 @@
 # Lint Codespell configurations
 [codespell]
-skip = .codespellrc,.git,node_modules,build,dist,*.zst,CHANGELOG.md,.playwright,.terraform,*.svg
+skip = .codespellrc,.git,node_modules,build,dist,*.zst,CHANGELOG.md,.playwright,.terraform,*.svg,**/pepr/operator/crd/generated/*.ts
 ignore-words-list = NotIn,AKS,LICENS,aks,afterAll
 enable-colors =
 check-hidden =

docs/reference/configuration/custom resources/packages-v1alpha1-cr.md

@@ -52,7 +52,7 @@ tableOfContents:
     </tr>
   </thead>
   <tbody>
-    <tr><td style="white-space: nowrap;">authorization</td><td style="white-space: nowrap;"><a href="#Authorization">Authorization</a></td><td>Authorization settings.</td></tr><tr><td style="white-space: nowrap;">description</td><td style="white-space: nowrap;">string</td><td>A description of this monitor entry, this will become part of the ServiceMonitor name</td></tr><tr><td style="white-space: nowrap;">kind</td><td style="white-space: nowrap;">string (enum):<ul><li><code>PodMonitor</code></li><li><code>ServiceMonitor</code></li></ul></td><td>The type of monitor to create; PodMonitor or ServiceMonitor. ServiceMonitor is the default.</td></tr><tr><td style="white-space: nowrap;">path</td><td style="white-space: nowrap;">string</td><td>HTTP path from which to scrape for metrics, defaults to `/metrics`</td></tr><tr><td style="white-space: nowrap;">podSelector</td><td style="white-space: nowrap;"></td><td>Labels to match pods in the namespace to apply the policy to. Leave empty to apply to all pods in the namespace</td></tr><tr><td style="white-space: nowrap;">portName</td><td style="white-space: nowrap;">string</td><td>The port name for the serviceMonitor</td></tr><tr><td style="white-space: nowrap;">selector</td><td style="white-space: nowrap;"></td><td>Labels to match pods in the namespace to apply the policy to. Leave empty to apply to all pods in the namespace</td></tr><tr><td style="white-space: nowrap;">targetPort</td><td style="white-space: nowrap;">number</td><td>The service targetPort. This is required so the NetworkPolicy can be generated correctly.</td></tr>
+    <tr><td style="white-space: nowrap;">authorization</td><td style="white-space: nowrap;"><a href="#Authorization">Authorization</a></td><td>Authorization settings.</td></tr><tr><td style="white-space: nowrap;">description</td><td style="white-space: nowrap;">string</td><td>A description of this monitor entry, this will become part of the ServiceMonitor name</td></tr><tr><td style="white-space: nowrap;">fallbackScrapeProtocol</td><td style="white-space: nowrap;">string (enum):<ul><li><code>OpenMetricsText0.0.1</code></li><li><code>OpenMetricsText1.0.0</code></li><li><code>PrometheusProto</code></li><li><code>PrometheusText0.0.4</code></li><li><code>PrometheusText1.0.0</code></li></ul></td><td>The protocol for Prometheus to use if a scrape returns a blank, unparsable, or otherwise invalid Content-Type</td></tr><tr><td style="white-space: nowrap;">kind</td><td style="white-space: nowrap;">string (enum):<ul><li><code>PodMonitor</code></li><li><code>ServiceMonitor</code></li></ul></td><td>The type of monitor to create; PodMonitor or ServiceMonitor. ServiceMonitor is the default.</td></tr><tr><td style="white-space: nowrap;">path</td><td style="white-space: nowrap;">string</td><td>HTTP path from which to scrape for metrics, defaults to `/metrics`</td></tr><tr><td style="white-space: nowrap;">podSelector</td><td style="white-space: nowrap;"></td><td>Labels to match pods in the namespace to apply the policy to. Leave empty to apply to all pods in the namespace</td></tr><tr><td style="white-space: nowrap;">portName</td><td style="white-space: nowrap;">string</td><td>The port name for the serviceMonitor</td></tr><tr><td style="white-space: nowrap;">selector</td><td style="white-space: nowrap;"></td><td>Labels to match pods in the namespace to apply the policy to. Leave empty to apply to all pods in the namespace</td></tr><tr><td style="white-space: nowrap;">targetPort</td><td style="white-space: nowrap;">number</td><td>The service targetPort. This is required so the NetworkPolicy can be generated correctly.</td></tr>
   </tbody>
 </table>
 </div>

package.json

@@ -1,6 +1,6 @@
 {
   "name": "uds-core",
-  "version": "0.5.0",
+  "version": "0.6.0",
   "description": "A collection of capabilities for UDS Core",
   "keywords": [
     "pepr",

pepr.ts

@@ -7,12 +7,10 @@ import { PeprModule } from "pepr";
 
 import cfg from "./package.json";
 
-import { DataStore } from "pepr/dist/lib/storage";
 import { istio } from "./src/pepr/istio";
 import { Component, setupLogger } from "./src/pepr/logger";
 import { operator } from "./src/pepr/operator";
 import { setupAuthserviceSecret } from "./src/pepr/operator/controllers/keycloak/authservice/config";
-import { Policy } from "./src/pepr/operator/crd";
 import { registerCRDs } from "./src/pepr/operator/crd/register";
 import { patches } from "./src/pepr/patches";
 import { policies, startExemptionWatch } from "./src/pepr/policies";
@@ -42,23 +40,6 @@ const log = setupLogger(Component.STARTUP);
     // Patches for specific components
     patches,
   ]);
-  // Remove legacy policy entries from the pepr store for the 0.5.0 upgrade
-  if (
-    process.env.PEPR_MODE === "dev" ||
-    (process.env.PEPR_WATCH_MODE === "true" && cfg.version === "0.5.0")
-  ) {
-    log.debug("Clearing legacy pepr store exemption entries...");
-    policies.Store.onReady((data: DataStore) => {
-      const policiesList = Object.values(Policy);
-      for (const p of Object.keys(data)) {
-        // if p matches a Policy key, remove it
-        if (policiesList.includes(p as Policy)) {
-          log.debug(`Removing legacy storage of ${p} policy exemptions...`);
-          policies.Store.removeItem(p);
-        }
-      }
-    });
-  }
 })().catch(err => {
   log.error(err, "Critical error during startup. Exiting...");
   process.exit(1);

src/pepr/operator/controllers/monitoring/pod-monitor.ts

@@ -8,6 +8,7 @@ import { K8s } from "pepr";
 import { Component, setupLogger } from "../../../logger";
 import { Monitor, PrometheusPodMonitor, UDSPackage } from "../../crd";
 import { Kind } from "../../crd/generated/package-v1alpha1";
+import { FallbackScrapeProtocol } from "../../crd/generated/prometheus/podmonitor-v1";
 import { getOwnerRef, purgeOrphans } from "../utils";
 import { generateMonitorName } from "./common";
 
@@ -86,6 +87,9 @@ export function generatePodMonitor(
       selector: {
         matchLabels: podSelector ?? selector,
       },
+      // Fallback to the Prometheus 2.x default if not defined
+      fallbackScrapeProtocol:
+        monitor.fallbackScrapeProtocol || FallbackScrapeProtocol.PrometheusText004,
     },
   };
 

src/pepr/operator/controllers/monitoring/service-monitor.ts

@@ -9,6 +9,7 @@ import { V1OwnerReference } from "@kubernetes/client-node";
 import { Component, setupLogger } from "../../../logger";
 import { Monitor, PrometheusServiceMonitor, UDSPackage } from "../../crd";
 import { Kind } from "../../crd/generated/package-v1alpha1";
+import { FallbackScrapeProtocol } from "../../crd/generated/prometheus/servicemonitor-v1";
 import { getOwnerRef, purgeOrphans } from "../utils";
 import { generateMonitorName } from "./common";
 
@@ -89,6 +90,9 @@ export function generateServiceMonitor(
       selector: {
         matchLabels: selector,
       },
+      // Fallback to the Prometheus 2.x default if not defined
+      fallbackScrapeProtocol:
+        monitor.fallbackScrapeProtocol || FallbackScrapeProtocol.PrometheusText004,
     },
   };
 

src/pepr/operator/crd/generated/package-v1alpha1.ts

@@ -34,6 +34,11 @@ export interface Monitor {
    * A description of this monitor entry, this will become part of the ServiceMonitor name
    */
   description?: string;
+  /**
+   * The protocol for Prometheus to use if a scrape returns a blank, unparsable, or otherwise
+   * invalid Content-Type
+   */
+  fallbackScrapeProtocol?: FallbackScrapeProtocol;
   /**
    * The type of monitor to create; PodMonitor or ServiceMonitor. ServiceMonitor is the
    * default.
@@ -99,6 +104,18 @@ export interface Credentials {
   optional?: boolean;
 }
 
+/**
+ * The protocol for Prometheus to use if a scrape returns a blank, unparsable, or otherwise
+ * invalid Content-Type
+ */
+export enum FallbackScrapeProtocol {
+  OpenMetricsText001 = "OpenMetricsText0.0.1",
+  OpenMetricsText100 = "OpenMetricsText1.0.0",
+  PrometheusProto = "PrometheusProto",
+  PrometheusText004 = "PrometheusText0.0.4",
+  PrometheusText100 = "PrometheusText1.0.0",
+}
+
 /**
  * The type of monitor to create; PodMonitor or ServiceMonitor. ServiceMonitor is the
  * default.