From 7af72bb539fc444c56e1c6c257fac25b80ef9d56 Mon Sep 17 00:00:00 2001 From: Wouter Verlaek Date: Wed, 8 Nov 2023 09:45:45 +0100 Subject: [PATCH] Add GiB left in storage warning (#19035) --- .../mixins/workspace/rules/central/ipfs.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/operations/observability/mixins/workspace/rules/central/ipfs.yaml b/operations/observability/mixins/workspace/rules/central/ipfs.yaml index 8ba4bc59398003..c9d403c6fe3692 100644 --- a/operations/observability/mixins/workspace/rules/central/ipfs.yaml +++ b/operations/observability/mixins/workspace/rules/central/ipfs.yaml @@ -21,8 +21,8 @@ spec: annotations: runbook_url: https://github.com/gitpod-io/runbooks/blob/main/runbooks/IPFSAlmostOutOfStorage.md summary: IPFS is running out of storage within a few days in a workspace cluster. Create and shift to a new workspace cluster with a fresh IPFS. - description: IPFS in cluster {{ $labels.cluster }} is running out of storage. This happens naturally for workspace clusters over time, and is prevented by recreating the clusters regularly. Once out of storage, IPFS will stop working, and we fall back to the GCP registry for workspace image pulls, resulting in slower workspace startup times. - expr: min(node_filesystem_avail_bytes{device="/dev/mapper/lvm--disk-ipfs", node=~"services-.*", cluster!~"ephemeral.*"}) by (cluster) < 600*1024*1024*1024 + description: IPFS in cluster {{ $labels.cluster }} is running out of storage ({{ $value }}GiB left). This happens naturally for workspace clusters over time, and is prevented by recreating the clusters regularly. Once out of storage, IPFS will stop working, and we fall back to the GCP registry for workspace image pulls, resulting in slower workspace startup times. + expr: min(node_filesystem_avail_bytes{device="/dev/mapper/lvm--disk-ipfs", node=~"services-.*", cluster!~"ephemeral.*"}) by (cluster) / (1024*1024*1024) < 600 - alert: IPFSStorageCritical labels: severity: critical @@ -31,5 +31,5 @@ spec: annotations: runbook_url: https://github.com/gitpod-io/runbooks/blob/main/runbooks/IPFSAlmostOutOfStorage.md summary: IPFS is now very close to running out of storage in a workspace cluster. Create and shift to a new workspace cluster with a fresh IPFS. - description: IPFS in cluster {{ $labels.cluster }} has almost ran out of storage. This happens naturally for workspace clusters over time, and is prevented by recreating the clusters regularly. Once out of storage, IPFS will stop working, and we fall back to the GCP registry for workspace image pulls, resulting in slower workspace startup times. - expr: min(node_filesystem_avail_bytes{device="/dev/mapper/lvm--disk-ipfs", node=~"services-.*", cluster!~"ephemeral.*"}) by (cluster) < 200*1024*1024*1024 + description: IPFS in cluster {{ $labels.cluster }} has almost ran out of storage ({{ $value }}GiB left). This happens naturally for workspace clusters over time, and is prevented by recreating the clusters regularly. Once out of storage, IPFS will stop working, and we fall back to the GCP registry for workspace image pulls, resulting in slower workspace startup times. + expr: min(node_filesystem_avail_bytes{device="/dev/mapper/lvm--disk-ipfs", node=~"services-.*", cluster!~"ephemeral.*"}) by (cluster) / (1024*1024*1024) < 200