From db21db3b92f71284e6b395a9d430190ac133a571 Mon Sep 17 00:00:00 2001 From: Jason Harmon Date: Wed, 20 Jan 2021 09:54:44 -0500 Subject: [PATCH 1/3] Implemented configuration via annotations --- README.md | 63 ++++++++++++++++++++++++++++++++++ rules/chaos.go | 33 ++++++++++++++---- rules/chaos_test.go | 43 ++++++++++++++++++++++- rules/container_status.go | 32 +++++++++++++---- rules/container_status_test.go | 32 ++++++++++++++++- rules/duration.go | 39 +++++++++++++++++---- rules/duration_test.go | 45 ++++++++++++++++++++++++ rules/pod_status.go | 32 +++++++++++++---- rules/pod_status_test.go | 32 ++++++++++++++++- rules/rules.go | 22 +++++++++++- rules/unready.go | 35 +++++++++++++++---- rules/unready_test.go | 45 ++++++++++++++++++++++++ 12 files changed, 415 insertions(+), 38 deletions(-) diff --git a/README.md b/README.md index 5135247..29d8148 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,7 @@ Pod-Reaper is configurable through environment variables. The pod-reaper specifi - `REQUIRE_LABEL_VALUES` comma-separated list of metadata label values (of key-value pair) that pod-reaper should require - `REQUIRE_ANNOTATION_KEY` pod metadata annotation (of key-value pair) that pod-reaper should require - `REQUIRE_ANNOTATION_VALUES` comma-separated list of metadata annotation values (of key-value pair) that pod-reaper should require +- `RULES` comma-separated list of rules to load regardless of default Additionally, at least one rule must be enabled, or the pod-reaper will error and exit. See the Rules section below for configuring and enabling rules. @@ -37,6 +38,52 @@ EXCLUDE_LABEL_VALUES=disabled,false CHAOS_CHANCE=.001 ``` +#### Annotations + +Rule configuration may be overridden by annotations on individual pods. For single-value rules, the configured rule value will be replaced by the annotation value. For multi-value rules, annotations will be added to the configured rule values. See [Implemented Rules](#implemented-rules) for available annotations. + +Example environment variables with annotations: + +```sh +# pod-reaper configuration +NAMESPACE=test +SCHEDULE=@every 30s + +# enable at least one rule +MAX_UNREADY=5m +RULES=duration,unready +``` + +Pods + +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: a + annotations: + pod-reaper/max-duration: 1d +spec: + containers: + - name: nginx + image: nginx:1.14.2 + ports: + - containerPort: 80 +--- +apiVersion: v1 +kind: Pod +metadata: + name: b +spec: + containers: + - name: nginx + image: nginx:1.14.2 + ports: + - containerPort: 80 +``` + +In this configuration, the Duration, and Unready rules will be loaded. Either pod will be reaped if it is unready for more than 5 minutes. Pod _a_ will be reaped after 1 day, regardless of status. + ### `NAMESPACE` Default value: "" (which will look at ALL namespaces) @@ -134,10 +181,18 @@ Default value: Logrus This environment variable modifies the structured log format for easy ingestion into different logging systems, including Stackdriver via the Fluentd format. Available formats: Logrus, Fluentd +### `RULES` + +This is an optional, comma-separated list of rules which should be loaded. If a rule is specified here, it will be loaded even if it does not have a configuration defined in an environment variable. This is used to load rules which only operate on annotations. + +Available rules: chaos, container_status, duration, pod_status, unready + ## Implemented Rules ### Chaos Chance +Annotation: `pod-reaper/chaos-chance` + Flags a pod for reaping based on a random number generator. Enabled and configured by setting the environment variable `CHAOS_CHANCE` with a floating point value. A random number generator will generate a value in range `[0,1)` and if the the generated value is below the configured chaos chance, the pod will be flagged for reaping. @@ -154,6 +209,8 @@ Remember that pods can be excluded from reaping if the pod has a label matching ### Container Status +Annotation: `pod-reaper/container-statuses` + Flags a pod for reaping based on a container within a pod having a specific container status. Enabled and configured by setting the environment variable `CONTAINER_STATUSES` with a coma separated list (no whitespace) of statuses. If a pod is in either a waiting or terminated state with a status in the specified list of status, the pod will be flagged for reaping. @@ -169,6 +226,8 @@ Note that this will not catch statuses that are describing the entire pod like t ### Pod Status +Annotation: `pod-reaper/pod-statuses` + Flags a pod for reaping based on the pod status. Enabled and configured by setting the environment variable `POD_STATUSES` with a coma separated list (no whitespace) of statuses. If the pod status in the specified list of status, the pod will be flagged for reaping. @@ -184,12 +243,16 @@ Note that pod status is different than container statuses as it checks the statu ### Duration +Annotation: `pod-reaper/max-duration` + Flags a pod for reaping based on the pods current run duration. Enabled and configured by setting the environment variable `MAX_DURATION` with a valid go-lang `time.duration` format (example: "1h15m30s"). If a pod has been running longer than the specified duration, the pod will be flagged for reaping. ### Unready +Annotation: `pod-reaper/max-unready` + Flags a pod for reaping based on the time the pod has been unready. Enabled and configured by setting the environment variable `MAX_UNREADY` with a valid go-lang `time.duration` format (example: "10m"). If a pod has been unready longer than the specified duration, the pod will be flagged for reaping. diff --git a/rules/chaos.go b/rules/chaos.go index 537f715..3cf645c 100644 --- a/rules/chaos.go +++ b/rules/chaos.go @@ -7,10 +7,15 @@ import ( "strconv" "time" + "github.com/sirupsen/logrus" v1 "k8s.io/api/core/v1" ) -const envChaosChance = "CHAOS_CHANCE" +const ( + ruleChaos = "chaos" + envChaosChance = "CHAOS_CHANCE" + annotationChaosChance = annotationPrefix + "/chaos-chance" +) var _ Rule = (*chaos)(nil) @@ -23,18 +28,34 @@ func init() { } func (rule *chaos) load() (bool, string, error) { - value, active := os.LookupEnv(envChaosChance) - if !active { + explicit := explicitLoad(ruleChaos) + value, hasDefault := os.LookupEnv(envChaosChance) + if !explicit && !hasDefault { return false, "", nil } chance, err := strconv.ParseFloat(value, 64) - if err != nil { + if !explicit && err != nil { return false, "", fmt.Errorf("invalid chaos chance %s", err) } rule.chance = chance - return true, fmt.Sprintf("chaos chance %s", value), nil + + if rule.chance != 0 { + return true, fmt.Sprintf("chaos chance %s", value), nil + } + return true, fmt.Sprint("chaos loaded explicitly"), nil } func (rule *chaos) ShouldReap(pod v1.Pod) (bool, string) { - return rand.Float64() < rule.chance, "was flagged for chaos" + chance := rule.chance + annotationValue := pod.Annotations[annotationChaosChance] + if annotationValue != "" { + annotationChance, err := strconv.ParseFloat(annotationValue, 64) + if err == nil { + chance = annotationChance + } else { + logrus.Warnf("pod %s has invalid chaos chance: %s", pod.Name, err) + } + } + + return rand.Float64() < chance, "was flagged for chaos" } diff --git a/rules/chaos_test.go b/rules/chaos_test.go index 65a397d..9200a6b 100644 --- a/rules/chaos_test.go +++ b/rules/chaos_test.go @@ -5,7 +5,7 @@ import ( "testing" "github.com/stretchr/testify/assert" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" ) func TestChaosLoad(t *testing.T) { @@ -32,6 +32,14 @@ func TestChaosLoad(t *testing.T) { assert.Equal(t, "", message) assert.False(t, loaded) }) + t.Run("explicit load without default", func(t *testing.T) { + os.Clearenv() + os.Setenv(envExplicitLoad, ruleChaos) + loaded, message, err := (&chaos{}).load() + assert.NoError(t, err) + assert.Equal(t, "chaos loaded explicitly", message) + assert.True(t, loaded) + }) } func TestChaosShouldReap(t *testing.T) { @@ -52,4 +60,37 @@ func TestChaosShouldReap(t *testing.T) { shouldReap, _ := chaos.ShouldReap(v1.Pod{}) assert.False(t, shouldReap) }) + t.Run("annotation override reap", func(t *testing.T) { + os.Clearenv() + os.Setenv(envChaosChance, "0.0") // default never + chaos := chaos{} + chaos.load() + pod := v1.Pod{} + pod.Annotations = map[string]string{ + annotationChaosChance: "1.0", // override always + } + shouldReap, message := chaos.ShouldReap(pod) + assert.True(t, shouldReap) + assert.Equal(t, "was flagged for chaos", message) + }) + t.Run("annotation override no reap", func(t *testing.T) { + os.Clearenv() + os.Setenv(envChaosChance, "1.0") // default always + chaos := chaos{} + chaos.load() + pod := v1.Pod{} + pod.Annotations = map[string]string{ + annotationChaosChance: "0.0", // override never + } + shouldReap, _ := chaos.ShouldReap(pod) + assert.False(t, shouldReap) + }) + t.Run("explicit load no annotation", func(t *testing.T) { + os.Clearenv() + os.Setenv(envExplicitLoad, ruleChaos) + chaos := chaos{} + chaos.load() + shouldReap, _ := chaos.ShouldReap(v1.Pod{}) + assert.False(t, shouldReap) + }) } diff --git a/rules/container_status.go b/rules/container_status.go index 57fc7e0..d6a166d 100644 --- a/rules/container_status.go +++ b/rules/container_status.go @@ -5,10 +5,14 @@ import ( "os" "strings" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" ) -const envContainerStatus = "CONTAINER_STATUSES" +const ( + containerStatusName = "container_status" + envContainerStatus = "CONTAINER_STATUSES" + annotationContainerStatus = annotationPrefix + "/container-statuses" +) var _ Rule = (*containerStatus)(nil) @@ -17,16 +21,30 @@ type containerStatus struct { } func (rule *containerStatus) load() (bool, string, error) { - value, active := os.LookupEnv(envContainerStatus) - if !active { + explicit := explicitLoad(containerStatusName) + value, hasDefault := os.LookupEnv(envContainerStatus) + if !explicit && !hasDefault { return false, "", nil } - rule.reapStatuses = strings.Split(value, ",") - return true, fmt.Sprintf("container status in [%s]", value), nil + if value != "" { + rule.reapStatuses = strings.Split(value, ",") + } + + if len(rule.reapStatuses) != 0 { + return true, fmt.Sprintf("container status in [%s]", value), nil + } + return true, "container status loaded explicitly", nil } func (rule *containerStatus) ShouldReap(pod v1.Pod) (bool, string) { - for _, reapStatus := range rule.reapStatuses { + reapStatuses := rule.reapStatuses + annotationValue := pod.Annotations[annotationContainerStatus] + if annotationValue != "" { + annotationValues := strings.Split(annotationValue, ",") + reapStatuses = append(reapStatuses, annotationValues...) + } + + for _, reapStatus := range reapStatuses { for _, containerStatus := range pod.Status.ContainerStatuses { state := containerStatus.State // check both waiting and terminated conditions diff --git a/rules/container_status_test.go b/rules/container_status_test.go index 5b8515f..e6edae1 100644 --- a/rules/container_status_test.go +++ b/rules/container_status_test.go @@ -5,7 +5,7 @@ import ( "testing" "github.com/stretchr/testify/assert" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" ) func testWaitContainerState(reason string) v1.ContainerState { @@ -65,6 +65,14 @@ func TestContainerStatusLoad(t *testing.T) { assert.Equal(t, "", message) assert.False(t, loaded) }) + t.Run("explicit load without default", func(t *testing.T) { + os.Clearenv() + os.Setenv(envExplicitLoad, containerStatusName) + loaded, message, err := (&containerStatus{}).load() + assert.NoError(t, err) + assert.Equal(t, "container status loaded explicitly", message) + assert.True(t, loaded) + }) } func TestContainerStatusShouldReap(t *testing.T) { @@ -87,4 +95,26 @@ func TestContainerStatusShouldReap(t *testing.T) { shouldReap, _ := containerStatus.ShouldReap(pod) assert.False(t, shouldReap) }) + t.Run("annotation reap", func(t *testing.T) { + os.Clearenv() + os.Setenv(envContainerStatus, "test-status") + containerStatus := containerStatus{} + containerStatus.load() + pod := testStatusPod(testWaitContainerState("another-status")) + pod.Annotations = map[string]string{ + annotationContainerStatus: "another-status", + } + shouldReap, reason := containerStatus.ShouldReap(pod) + assert.True(t, shouldReap) + assert.Regexp(t, ".*another-status.*", reason) + }) + t.Run("explicit load no annotation", func(t *testing.T) { + os.Clearenv() + os.Setenv(envExplicitLoad, containerStatusName) + containerStatus := containerStatus{} + containerStatus.load() + pod := testStatusPod(testWaitContainerState("not-present")) + shouldReap, _ := containerStatus.ShouldReap(pod) + assert.False(t, shouldReap) + }) } diff --git a/rules/duration.go b/rules/duration.go index 95b0359..58e2f62 100644 --- a/rules/duration.go +++ b/rules/duration.go @@ -5,10 +5,15 @@ import ( "os" "time" - "k8s.io/api/core/v1" + "github.com/sirupsen/logrus" + v1 "k8s.io/api/core/v1" ) -const envMaxDuration = "MAX_DURATION" +const ( + ruleDuration = "duration" + envMaxDuration = "MAX_DURATION" + annotationMaxDuration = annotationPrefix + "/max-duration" +) var _ Rule = (*duration)(nil) @@ -17,25 +22,45 @@ type duration struct { } func (rule *duration) load() (bool, string, error) { - value, active := os.LookupEnv(envMaxDuration) - if !active { + explicit := explicitLoad(ruleDuration) + value, hasDefault := os.LookupEnv(envMaxDuration) + if !explicit && !hasDefault { return false, "", nil } duration, err := time.ParseDuration(value) - if err != nil { + if !explicit && err != nil { return false, "", fmt.Errorf("invalid max duration: %s", err) } rule.duration = duration - return true, fmt.Sprintf("maximum run duration %s", value), nil + + if rule.duration != 0 { + return true, fmt.Sprintf("maximum run duration %s", value), nil + } + return true, fmt.Sprint("maximum run duration loaded explicitly"), nil } func (rule *duration) ShouldReap(pod v1.Pod) (bool, string) { + duration := rule.duration + annotationValue := pod.Annotations[annotationMaxDuration] + if annotationValue != "" { + annotationDuration, err := time.ParseDuration(annotationValue) + if err == nil { + duration = annotationDuration + } else { + logrus.Warnf("pod %s has invalid max duration annotation: %s", pod.Name, err) + } + } + if duration == 0 { + return false, "" + } + podStartTime := pod.Status.StartTime if podStartTime == nil { return false, "" } + startTime := time.Unix(podStartTime.Unix(), 0) // convert to standard go time - cutoffTime := time.Now().Add(-1 * rule.duration) + cutoffTime := time.Now().Add(-1 * duration) runningDuration := time.Now().Sub(startTime) message := fmt.Sprintf("has been running for %s", runningDuration.String()) return startTime.Before(cutoffTime), message diff --git a/rules/duration_test.go b/rules/duration_test.go index e703f5a..97d3dd7 100644 --- a/rules/duration_test.go +++ b/rules/duration_test.go @@ -43,6 +43,14 @@ func TestDurationLoad(t *testing.T) { assert.Equal(t, "", message) assert.False(t, loaded) }) + t.Run("explicit load without default", func(t *testing.T) { + os.Clearenv() + os.Setenv(envExplicitLoad, ruleDuration) + loaded, message, err := (&duration{}).load() + assert.NoError(t, err) + assert.Equal(t, "maximum run duration loaded explicitly", message) + assert.True(t, loaded) + }) } func TestDurationShouldReap(t *testing.T) { @@ -76,4 +84,41 @@ func TestDurationShouldReap(t *testing.T) { shouldReap, _ := duration.ShouldReap(pod) assert.False(t, shouldReap) }) + t.Run("annotation override reap", func(t *testing.T) { + os.Clearenv() + os.Setenv(envMaxDuration, "5m") + duration := duration{} + duration.load() + startTime := time.Now().Add(-2 * time.Minute) + pod := testDurationPod(&startTime) + pod.Annotations = map[string]string{ + annotationMaxDuration: "2m", + } + shouldReap, reason := duration.ShouldReap(pod) + assert.True(t, shouldReap) + assert.Regexp(t, ".*has been running.*", reason) + }) + t.Run("annotation override no reap", func(t *testing.T) { + os.Clearenv() + os.Setenv(envMaxDuration, "1m59s") + duration := duration{} + duration.load() + startTime := time.Now().Add(-2 * time.Minute) + pod := testDurationPod(&startTime) + pod.Annotations = map[string]string{ + annotationMaxDuration: "2m1s", + } + shouldReap, _ := duration.ShouldReap(pod) + assert.False(t, shouldReap) + }) + t.Run("explicit load no annotation", func(t *testing.T) { + os.Clearenv() + os.Setenv(envExplicitLoad, ruleDuration) + duration := duration{} + duration.load() + startTime := time.Now().Add(-2 * time.Minute) + pod := testDurationPod(&startTime) + shouldReap, _ := duration.ShouldReap(pod) + assert.False(t, shouldReap) + }) } diff --git a/rules/pod_status.go b/rules/pod_status.go index eb5ec3a..ef907a0 100644 --- a/rules/pod_status.go +++ b/rules/pod_status.go @@ -5,10 +5,14 @@ import ( "os" "strings" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" ) -const envPodStatus = "POD_STATUSES" +const ( + podStatusName = "pod_status" + envPodStatus = "POD_STATUSES" + annotationPodStatus = annotationPrefix + "/pod-statuses" +) var _ Rule = (*podStatus)(nil) @@ -17,17 +21,31 @@ type podStatus struct { } func (rule *podStatus) load() (bool, string, error) { - value, active := os.LookupEnv(envPodStatus) - if !active { + explicit := explicitLoad(podStatusName) + value, hasDefault := os.LookupEnv(envPodStatus) + if !explicit && !hasDefault { return false, "", nil } - rule.reapStatuses = strings.Split(value, ",") - return true, fmt.Sprintf("pod status in [%s]", value), nil + if value != "" { + rule.reapStatuses = strings.Split(value, ",") + } + + if len(rule.reapStatuses) != 0 { + return true, fmt.Sprintf("pod status in [%s]", value), nil + } + return true, "pod status loaded explicitly", nil } func (rule *podStatus) ShouldReap(pod v1.Pod) (bool, string) { + reapStatuses := rule.reapStatuses + annotationValue := pod.Annotations[annotationPodStatus] + if annotationValue != "" { + annotationValues := strings.Split(annotationValue, ",") + reapStatuses = append(reapStatuses, annotationValues...) + } + status := pod.Status.Reason - for _, reapStatus := range rule.reapStatuses { + for _, reapStatus := range reapStatuses { if status == reapStatus { return true, fmt.Sprintf("has pod status %s", reapStatus) } diff --git a/rules/pod_status_test.go b/rules/pod_status_test.go index 4b83b27..61c4599 100644 --- a/rules/pod_status_test.go +++ b/rules/pod_status_test.go @@ -5,7 +5,7 @@ import ( "testing" "github.com/stretchr/testify/assert" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" ) func testPodFromReason(reason string) v1.Pod { @@ -44,6 +44,14 @@ func TestPodStatusLoad(t *testing.T) { assert.Equal(t, "", message) assert.False(t, loaded) }) + t.Run("explicit load without default", func(t *testing.T) { + os.Clearenv() + os.Setenv(envExplicitLoad, podStatusName) + loaded, message, err := (&podStatus{}).load() + assert.NoError(t, err) + assert.Equal(t, "pod status loaded explicitly", message) + assert.True(t, loaded) + }) } func TestPodStatusShouldReap(t *testing.T) { @@ -66,4 +74,26 @@ func TestPodStatusShouldReap(t *testing.T) { shouldReap, _ := podStatus.ShouldReap(pod) assert.False(t, shouldReap) }) + t.Run("annotation reap", func(t *testing.T) { + os.Clearenv() + os.Setenv(envPodStatus, "test-status") + podStatus := podStatus{} + podStatus.load() + pod := testPodFromReason("another-status") + pod.Annotations = map[string]string{ + annotationPodStatus: "another-status", + } + shouldReap, reason := podStatus.ShouldReap(pod) + assert.True(t, shouldReap) + assert.Regexp(t, ".*another-status.*", reason) + }) + t.Run("explicit load no annotation", func(t *testing.T) { + os.Clearenv() + os.Setenv(envExplicitLoad, podStatusName) + containerStatus := containerStatus{} + containerStatus.load() + pod := testPodFromReason("another-status") + shouldReap, _ := containerStatus.ShouldReap(pod) + assert.False(t, shouldReap) + }) } diff --git a/rules/rules.go b/rules/rules.go index 8307532..ffeceac 100644 --- a/rules/rules.go +++ b/rules/rules.go @@ -2,11 +2,16 @@ package rules import ( "errors" + "os" + "strings" "github.com/sirupsen/logrus" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" ) +const envExplicitLoad = "RULES" +const annotationPrefix = "pod-reaper" + // Rule is an interface defining the two functions needed for pod reaper to use the rule. type Rule interface { @@ -65,3 +70,18 @@ func (rules Rules) ShouldReap(pod v1.Pod) (bool, []string) { } return true, reasons } + +func explicitLoad(ruleName string) bool { + value, exists := os.LookupEnv(envExplicitLoad) + if !exists { + return false + } + + values := strings.Split(value, ",") + for _, v := range values { + if v == ruleName { + return true + } + } + return false +} diff --git a/rules/unready.go b/rules/unready.go index 511873d..145724e 100644 --- a/rules/unready.go +++ b/rules/unready.go @@ -5,10 +5,15 @@ import ( "os" "time" - "k8s.io/api/core/v1" + "github.com/sirupsen/logrus" + v1 "k8s.io/api/core/v1" ) -const envMaxUnready = "MAX_UNREADY" +const ( + ruleUnready = "unready" + envMaxUnready = "MAX_UNREADY" + annotationMaxUnready = annotationPrefix + "/max-unready" +) var _ Rule = (*unready)(nil) @@ -17,26 +22,42 @@ type unready struct { } func (rule *unready) load() (bool, string, error) { - value, active := os.LookupEnv(envMaxUnready) - if !active { + explicit := explicitLoad(ruleUnready) + value, hasDefault := os.LookupEnv(envMaxUnready) + if !explicit && !hasDefault { return false, "", nil } duration, err := time.ParseDuration(value) - if err != nil { + if !explicit && err != nil { return false, "", fmt.Errorf("invalid max unready duration: %s", err) } rule.duration = duration - return true, fmt.Sprintf("maximum unready %s", value), nil + + if rule.duration != 0 { + return true, fmt.Sprintf("maximum unready %s", value), nil + } + return true, fmt.Sprint("maximum unready duration loaded explicitly"), nil } func (rule *unready) ShouldReap(pod v1.Pod) (bool, string) { + duration := rule.duration + annotationValue := pod.Annotations[annotationMaxUnready] + if annotationValue != "" { + annotationDuration, err := time.ParseDuration(annotationValue) + if err == nil { + duration = annotationDuration + } else { + logrus.Warnf("invalid max unready duration annotation: %s", err) + } + } + condition := getCondition(pod, v1.PodReady) if condition == nil || condition.Status == "True" { return false, "" } transitionTime := time.Unix(condition.LastTransitionTime.Unix(), 0) // convert to standard go time - cutoffTime := time.Now().Add(-1 * rule.duration) + cutoffTime := time.Now().Add(-1 * duration) unreadyDuration := time.Now().Sub(transitionTime) message := fmt.Sprintf("has been unready for %s", unreadyDuration.String()) return transitionTime.Before(cutoffTime), message diff --git a/rules/unready_test.go b/rules/unready_test.go index 6be44ee..c1aeec2 100644 --- a/rules/unready_test.go +++ b/rules/unready_test.go @@ -50,6 +50,14 @@ func TestUnreadyLoad(t *testing.T) { assert.Equal(t, "", message) assert.False(t, loaded) }) + t.Run("explicit load without default", func(t *testing.T) { + os.Clearenv() + os.Setenv(envExplicitLoad, ruleUnready) + loaded, message, err := (&unready{}).load() + assert.NoError(t, err) + assert.Equal(t, "maximum unready duration loaded explicitly", message) + assert.True(t, loaded) + }) } func TestUnreadyShouldReap(t *testing.T) { @@ -83,4 +91,41 @@ func TestUnreadyShouldReap(t *testing.T) { shouldReap, _ := unready.ShouldReap(pod) assert.False(t, shouldReap) }) + t.Run("annotation override reap", func(t *testing.T) { + os.Clearenv() + os.Setenv(envMaxUnready, "1h") + unready := unready{} + unready.load() + lastTransitionTime := time.Now().Add(-10 * time.Minute) + pod := testUnreadyPod(&lastTransitionTime) + pod.Annotations = map[string]string{ + annotationMaxUnready: "9m59s", + } + shouldReap, reason := unready.ShouldReap(pod) + assert.True(t, shouldReap) + assert.Regexp(t, ".*has been unready.*", reason) + }) + t.Run("annotation override no reap", func(t *testing.T) { + os.Clearenv() + os.Setenv(envMaxUnready, "9m59s") + unready := unready{} + unready.load() + lastTransitionTime := time.Now().Add(-10 * time.Minute) + pod := testUnreadyPod(&lastTransitionTime) + pod.Annotations = map[string]string{ + annotationMaxUnready: "20m", + } + shouldReap, _ := unready.ShouldReap(pod) + assert.False(t, shouldReap) + }) + t.Run("explicit load no annotation", func(t *testing.T) { + os.Clearenv() + os.Setenv(envExplicitLoad, ruleUnready) + duration := duration{} + duration.load() + startTime := time.Now().Add(-2 * time.Minute) + pod := testUnreadyPod(&startTime) + shouldReap, _ := duration.ShouldReap(pod) + assert.False(t, shouldReap) + }) } From 54a76299d02a873d8347c64b92fd57d5a3fdcdb1 Mon Sep 17 00:00:00 2001 From: Jason Harmon Date: Wed, 20 Jan 2021 13:16:02 -0500 Subject: [PATCH 2/3] Clean up loading logs --- rules/chaos.go | 2 +- rules/chaos_test.go | 2 +- rules/container_status.go | 2 +- rules/container_status_test.go | 2 +- rules/duration.go | 2 +- rules/duration_test.go | 2 +- rules/pod_status.go | 2 +- rules/pod_status_test.go | 2 +- rules/unready.go | 2 +- rules/unready_test.go | 2 +- 10 files changed, 10 insertions(+), 10 deletions(-) diff --git a/rules/chaos.go b/rules/chaos.go index 3cf645c..eed172b 100644 --- a/rules/chaos.go +++ b/rules/chaos.go @@ -42,7 +42,7 @@ func (rule *chaos) load() (bool, string, error) { if rule.chance != 0 { return true, fmt.Sprintf("chaos chance %s", value), nil } - return true, fmt.Sprint("chaos loaded explicitly"), nil + return true, fmt.Sprint("chaos (no default)"), nil } func (rule *chaos) ShouldReap(pod v1.Pod) (bool, string) { diff --git a/rules/chaos_test.go b/rules/chaos_test.go index 9200a6b..1913aa2 100644 --- a/rules/chaos_test.go +++ b/rules/chaos_test.go @@ -37,7 +37,7 @@ func TestChaosLoad(t *testing.T) { os.Setenv(envExplicitLoad, ruleChaos) loaded, message, err := (&chaos{}).load() assert.NoError(t, err) - assert.Equal(t, "chaos loaded explicitly", message) + assert.Equal(t, "chaos (no default)", message) assert.True(t, loaded) }) } diff --git a/rules/container_status.go b/rules/container_status.go index d6a166d..7c6540c 100644 --- a/rules/container_status.go +++ b/rules/container_status.go @@ -33,7 +33,7 @@ func (rule *containerStatus) load() (bool, string, error) { if len(rule.reapStatuses) != 0 { return true, fmt.Sprintf("container status in [%s]", value), nil } - return true, "container status loaded explicitly", nil + return true, "container status (no default)", nil } func (rule *containerStatus) ShouldReap(pod v1.Pod) (bool, string) { diff --git a/rules/container_status_test.go b/rules/container_status_test.go index e6edae1..6e98076 100644 --- a/rules/container_status_test.go +++ b/rules/container_status_test.go @@ -70,7 +70,7 @@ func TestContainerStatusLoad(t *testing.T) { os.Setenv(envExplicitLoad, containerStatusName) loaded, message, err := (&containerStatus{}).load() assert.NoError(t, err) - assert.Equal(t, "container status loaded explicitly", message) + assert.Equal(t, "container status (no default)", message) assert.True(t, loaded) }) } diff --git a/rules/duration.go b/rules/duration.go index 58e2f62..4b957e1 100644 --- a/rules/duration.go +++ b/rules/duration.go @@ -36,7 +36,7 @@ func (rule *duration) load() (bool, string, error) { if rule.duration != 0 { return true, fmt.Sprintf("maximum run duration %s", value), nil } - return true, fmt.Sprint("maximum run duration loaded explicitly"), nil + return true, fmt.Sprint("maximum run duration (no default)"), nil } func (rule *duration) ShouldReap(pod v1.Pod) (bool, string) { diff --git a/rules/duration_test.go b/rules/duration_test.go index 97d3dd7..6446570 100644 --- a/rules/duration_test.go +++ b/rules/duration_test.go @@ -48,7 +48,7 @@ func TestDurationLoad(t *testing.T) { os.Setenv(envExplicitLoad, ruleDuration) loaded, message, err := (&duration{}).load() assert.NoError(t, err) - assert.Equal(t, "maximum run duration loaded explicitly", message) + assert.Equal(t, "maximum run duration (no default)", message) assert.True(t, loaded) }) } diff --git a/rules/pod_status.go b/rules/pod_status.go index ef907a0..e5255eb 100644 --- a/rules/pod_status.go +++ b/rules/pod_status.go @@ -33,7 +33,7 @@ func (rule *podStatus) load() (bool, string, error) { if len(rule.reapStatuses) != 0 { return true, fmt.Sprintf("pod status in [%s]", value), nil } - return true, "pod status loaded explicitly", nil + return true, "pod status (no default)", nil } func (rule *podStatus) ShouldReap(pod v1.Pod) (bool, string) { diff --git a/rules/pod_status_test.go b/rules/pod_status_test.go index 61c4599..eb74f7f 100644 --- a/rules/pod_status_test.go +++ b/rules/pod_status_test.go @@ -49,7 +49,7 @@ func TestPodStatusLoad(t *testing.T) { os.Setenv(envExplicitLoad, podStatusName) loaded, message, err := (&podStatus{}).load() assert.NoError(t, err) - assert.Equal(t, "pod status loaded explicitly", message) + assert.Equal(t, "pod status (no default)", message) assert.True(t, loaded) }) } diff --git a/rules/unready.go b/rules/unready.go index 145724e..7b72445 100644 --- a/rules/unready.go +++ b/rules/unready.go @@ -36,7 +36,7 @@ func (rule *unready) load() (bool, string, error) { if rule.duration != 0 { return true, fmt.Sprintf("maximum unready %s", value), nil } - return true, fmt.Sprint("maximum unready duration loaded explicitly"), nil + return true, fmt.Sprint("maximum unready duration (no default)"), nil } func (rule *unready) ShouldReap(pod v1.Pod) (bool, string) { diff --git a/rules/unready_test.go b/rules/unready_test.go index c1aeec2..342b668 100644 --- a/rules/unready_test.go +++ b/rules/unready_test.go @@ -55,7 +55,7 @@ func TestUnreadyLoad(t *testing.T) { os.Setenv(envExplicitLoad, ruleUnready) loaded, message, err := (&unready{}).load() assert.NoError(t, err) - assert.Equal(t, "maximum unready duration loaded explicitly", message) + assert.Equal(t, "maximum unready duration (no default)", message) assert.True(t, loaded) }) } From d57575e4e2be4021b2c6a3b8fba5a70df2f64eff Mon Sep 17 00:00:00 2001 From: Jason Harmon Date: Wed, 20 Jan 2021 13:32:26 -0500 Subject: [PATCH 3/3] Updated README to reflect multiple rule behavior --- README.md | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 29d8148..06c9aaf 100644 --- a/README.md +++ b/README.md @@ -60,20 +60,9 @@ Pods apiVersion: v1 kind: Pod metadata: - name: a + name: test annotations: - pod-reaper/max-duration: 1d -spec: - containers: - - name: nginx - image: nginx:1.14.2 - ports: - - containerPort: 80 ---- -apiVersion: v1 -kind: Pod -metadata: - name: b + pod-reaper/max-duration: 1h spec: containers: - name: nginx @@ -82,7 +71,7 @@ spec: - containerPort: 80 ``` -In this configuration, the Duration, and Unready rules will be loaded. Either pod will be reaped if it is unready for more than 5 minutes. Pod _a_ will be reaped after 1 day, regardless of status. +In this configuration, the Duration, and Unready rules will be loaded. The pod will be reaped if it is older than 1 hour and unready for 5 minutes. ### `NAMESPACE`