Skip to content

Commit

Permalink
Delete waves, status hints (#77)
Browse files Browse the repository at this point in the history
* [INCOMPATIBLE CHANGE] add delete waves
This commit adds delete waves, which can be controlled through the
new annotation <operator-name>/delete-order; by default, all objects
have a delete order of zero, and therefore are deleted in one wave.
Note that delete waves are independent of apply waves.
This commit brings some incompatible changes:
1. The annotation <operator-name>/order was renamed to
   <operator-name>/apply-order; the according constant in pkg/types
   was renamed as well.
2. The InventoryItem type was enhanced with new fields; consumers
   therefore *must* regenerate their CRDs.

* exlude purge order from inventory for now

* add new annotation <operator-name>/status-hint
this can be used to tweak the status detection (done by kstatus);
currently there are two options available that can be passed as
value of this annotation (comma-separated): 'has-observed-generation',
and 'has-ready-condition', which tells kstatus that the object
does have an observedGeneration field in the status, resp. a condition
of type Ready, even if it is not (yet) present in the status.

* bump controller-tools

* update generated artifacts

* update docs
  • Loading branch information
cbarbian-sap authored Mar 12, 2024
1 parent 97834ce commit b86915b
Show file tree
Hide file tree
Showing 16 changed files with 650 additions and 302 deletions.
2 changes: 1 addition & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ $(LOCALBIN):
CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen

## Tool Versions
CONTROLLER_TOOLS_VERSION ?= v0.9.2
CONTROLLER_TOOLS_VERSION ?= v0.14.0

.PHONY: controller-gen
controller-gen: $(CONTROLLER_GEN) ## Download controller-gen locally if necessary.
Expand Down
7 changes: 4 additions & 3 deletions go.mod
Original file line number Diff line number Diff line change
@@ -1,13 +1,15 @@
module github.com/sap/component-operator-runtime

go 1.21.7
go 1.22.1

require (
github.com/Masterminds/sprig/v3 v3.2.3
github.com/hashicorp/go-multierror v1.1.1
github.com/iancoleman/strcase v0.3.0
github.com/pkg/errors v0.9.1
github.com/sap/go-generics v0.2.0
github.com/sap/go-generics v0.2.3
github.com/spf13/pflag v1.0.5
golang.org/x/time v0.5.0
k8s.io/api v0.29.2
k8s.io/apiextensions-apiserver v0.29.2
k8s.io/apimachinery v0.29.2
Expand Down Expand Up @@ -71,7 +73,6 @@ require (
golang.org/x/sys v0.17.0 // indirect
golang.org/x/term v0.17.0 // indirect
golang.org/x/text v0.14.0 // indirect
golang.org/x/time v0.5.0 // indirect
gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect
google.golang.org/appengine v1.6.8 // indirect
google.golang.org/protobuf v1.32.0 // indirect
Expand Down
10 changes: 6 additions & 4 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,8 @@ github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9
github.com/huandu/xstrings v1.3.3/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE=
github.com/huandu/xstrings v1.4.0 h1:D17IlohoQq4UcpqD7fDk80P7l+lwAmlFaBHgOipl2FU=
github.com/huandu/xstrings v1.4.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE=
github.com/iancoleman/strcase v0.3.0 h1:nTXanmYxhfFAMjZL34Ov6gkzEsSJZ5DbhxWjvSASxEI=
github.com/iancoleman/strcase v0.3.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho=
github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4=
github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY=
Expand Down Expand Up @@ -98,8 +100,8 @@ github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 h1:n6/
github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
github.com/onsi/ginkgo/v2 v2.15.0 h1:79HwNRBAZHOEwrczrgSOPy+eFTTlIGELKy5as+ClttY=
github.com/onsi/ginkgo/v2 v2.15.0/go.mod h1:HlxMHtYF57y6Dpf+mc5529KKmSq9h2FpCF+/ZkwUxKM=
github.com/onsi/ginkgo/v2 v2.16.0 h1:7q1w9frJDzninhXxjZd+Y/x54XNjG/UlRLIYPZafsPM=
github.com/onsi/ginkgo/v2 v2.16.0/go.mod h1:llBI3WDLL9Z6taip6f33H76YcWtJv+7R3HigUjbIBOs=
github.com/onsi/gomega v1.31.1 h1:KYppCUK+bUgAZwHOu7EXVBKyQA6ILvOESHkn/tgoqvo=
github.com/onsi/gomega v1.31.1/go.mod h1:y40C95dwAD1Nz36SsEnxvfFe8FFfNxzI5eJ0EYGyAy0=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
Expand All @@ -116,8 +118,8 @@ github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k
github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo=
github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M=
github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA=
github.com/sap/go-generics v0.2.0 h1:uXjK6eZDj4XFe52KiMfX7YsHJ+YyOrhUgohe1hNT/78=
github.com/sap/go-generics v0.2.0/go.mod h1:LPjEUR4matw9C7GZdHYMExVN8+LeNK5LmrL24JKr8eg=
github.com/sap/go-generics v0.2.3 h1:cEY63YaVIqvOu2347drCilMvdgM1p2we2QwY4k/Nas0=
github.com/sap/go-generics v0.2.3/go.mod h1:eBhccCEzOiM5dn1W2kupUMOAm4uS9CfKHzQsDlZHQzc=
github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0=
github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
Expand Down
10 changes: 9 additions & 1 deletion internal/backoff/backoff.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (
"sync"
"time"

"golang.org/x/time/rate"
"k8s.io/client-go/util/workqueue"
)

Expand All @@ -21,7 +22,14 @@ type Backoff struct {
func NewBackoff(maxDelay time.Duration) *Backoff {
return &Backoff{
activities: make(map[any]any),
limiter: workqueue.NewItemExponentialFailureRateLimiter(20*time.Millisecond, maxDelay),
// resulting per-item backoff is the maximum of a 300-times-20ms-then-maxDelay per-item limiter,
// and an overall 10-per-second-burst-20 bucket limiter;
// as a consequence, we have up to 20 almost immediate retries, then a phase of 10 retries per seconnd
// for approximately 30s, and then slow retries at the rate given by maxDelay
limiter: workqueue.NewMaxOfRateLimiter(
workqueue.NewItemFastSlowRateLimiter(20*time.Millisecond, maxDelay, 300),
&workqueue.BucketRateLimiter{Limiter: rate.NewLimiter(rate.Limit(10), 20)},
),
}
}

Expand Down
115 changes: 115 additions & 0 deletions internal/kstatus/analyzer.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,115 @@
/*
SPDX-FileCopyrightText: 2023 SAP SE or an SAP affiliate company and component-operator-runtime contributors
SPDX-License-Identifier: Apache-2.0
*/

package kstatus

import (
"strings"

"github.com/iancoleman/strcase"

batchv1 "k8s.io/api/batch/v1"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime/schema"
kstatus "sigs.k8s.io/cli-utils/pkg/kstatus/status"

"github.com/sap/component-operator-runtime/pkg/types"
)

const conditionTypeReady = "Ready"

type statusAnalyzer struct {
reconcilerName string
}

func NewStatusAnalyzer(reconcilerName string) StatusAnalyzer {
return &statusAnalyzer{
reconcilerName: reconcilerName,
}
}

func (s *statusAnalyzer) ComputeStatus(object *unstructured.Unstructured) (Status, error) {
if hint, ok := object.GetAnnotations()[s.reconcilerName+"/"+types.AnnotationKeySuffixStatusHint]; ok {
object = object.DeepCopy()

for _, hint := range strings.Split(hint, ",") {
switch strcase.ToKebab(hint) {
case types.StatusHintHasObservedGeneration:
_, found, err := unstructured.NestedInt64(object.Object, "status", "observedGeneration")
if err != nil {
return UnknownStatus, err
}
if !found {
if err := unstructured.SetNestedField(object.Object, -1, "status", "observedGeneration"); err != nil {
return UnknownStatus, err
}
}
case types.StatusHintHasReadyCondition:
foundReadyCondition := false
conditions, found, err := unstructured.NestedSlice(object.Object, "status", "conditions")
if err != nil {
return UnknownStatus, err
}
if !found {
conditions = make([]any, 0)
}
for _, condition := range conditions {
if condition, ok := condition.(map[string]any); ok {
condType, found, err := unstructured.NestedString(condition, "type")
if err != nil {
return UnknownStatus, err
}
if found && condType == conditionTypeReady {
foundReadyCondition = true
break
}
}
}
if !foundReadyCondition {
conditions = append(conditions, map[string]any{
"type": conditionTypeReady,
"status": string(corev1.ConditionUnknown),
})
if err := unstructured.SetNestedSlice(object.Object, conditions, "status", "conditions"); err != nil {
return UnknownStatus, err
}
}
}
}
}

res, err := kstatus.Compute(object)
if err != nil {
return UnknownStatus, err
}

switch object.GroupVersionKind() {
case schema.GroupVersionKind{Group: "batch", Version: "v1", Kind: "Job"}:
// other than kstatus we want to consider jobs as InProgress if its pods are still running, resp. did not (yet) finish successfully
if res.Status == kstatus.CurrentStatus {
done := false
objc, err := kstatus.GetObjectWithConditions(object.UnstructuredContent())
if err != nil {
return UnknownStatus, err
}
for _, cond := range objc.Status.Conditions {
if cond.Type == string(batchv1.JobComplete) && cond.Status == corev1.ConditionTrue {
done = true
break
}
if cond.Type == string(batchv1.JobFailed) && cond.Status == corev1.ConditionTrue {
done = true
break
}
}
if !done {
res.Status = kstatus.InProgressStatus
}
}
}

return Status(res.Status), nil
}
10 changes: 10 additions & 0 deletions internal/kstatus/status.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
/*
SPDX-FileCopyrightText: 2023 SAP SE or an SAP affiliate company and component-operator-runtime contributors
SPDX-License-Identifier: Apache-2.0
*/

package kstatus

func (s Status) String() string {
return string(s)
}
29 changes: 29 additions & 0 deletions internal/kstatus/types.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
/*
SPDX-FileCopyrightText: 2023 SAP SE or an SAP affiliate company and component-operator-runtime contributors
SPDX-License-Identifier: Apache-2.0
*/

package kstatus

import (
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
kstatus "sigs.k8s.io/cli-utils/pkg/kstatus/status"
)

// TODO: the StatusAnalyzer interface should be public.

// The StatusAnalyzer interface models types which allow to extract a kstatus-compatible status from an object.
type StatusAnalyzer interface {
ComputeStatus(object *unstructured.Unstructured) (Status, error)
}

type Status kstatus.Status

const (
InProgressStatus Status = Status(kstatus.InProgressStatus)
FailedStatus Status = Status(kstatus.FailedStatus)
CurrentStatus Status = Status(kstatus.CurrentStatus)
TerminatingStatus Status = Status(kstatus.TerminatingStatus)
NotFoundStatus Status = Status(kstatus.NotFoundStatus)
UnknownStatus Status = Status(kstatus.UnknownStatus)
)
84 changes: 9 additions & 75 deletions pkg/component/reconcile.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ import (

"github.com/sap/component-operator-runtime/internal/backoff"
"github.com/sap/component-operator-runtime/internal/cluster"
"github.com/sap/component-operator-runtime/internal/kstatus"
"github.com/sap/component-operator-runtime/pkg/manifests"
"github.com/sap/component-operator-runtime/pkg/types"
)
Expand Down Expand Up @@ -86,86 +87,13 @@ type ReconcilerOptions struct {
SchemeBuilder types.SchemeBuilder
}

// AdoptionPolicy defines how the reconciler reacts if a dependent object exists but has no or a different owner.
type AdoptionPolicy string

const (
// Fail if the dependent object exists but has no or a different owner.
AdoptionPolicyNever AdoptionPolicy = "Never"
// Adopt existing dependent objects if they have no owner set.
AdoptionPolicyIfUnowned AdoptionPolicy = "IfUnowned"
// Adopt existing dependent objects, even if they have a conflicting owner.
AdoptionPolicyAlways AdoptionPolicy = "Always"
)

var adoptionPolicyByAnnotation = map[string]AdoptionPolicy{
types.AdoptionPolicyNever: AdoptionPolicyNever,
types.AdoptionPolicyIfUnowned: AdoptionPolicyIfUnowned,
types.AdoptionPolicyAlways: AdoptionPolicyAlways,
}

// ReconcilePolicy defines when the reconciler will reconcile the dependent object.
type ReconcilePolicy string

const (
// Reconcile the dependent object if its manifest, as produced by the generator, changes.
ReconcilePolicyOnObjectChange ReconcilePolicy = "OnObjectChange"
// Reconcile the dependent object if its manifest, as produced by the generator, changes, or if the owning
// component changes (identified by a change of its metadata.generation).
ReconcilePolicyOnObjectOrComponentChange ReconcilePolicy = "OnObjectOrComponentChange"
// Reconcile the dependent object only once; afterwards it will never be touched again by the reconciler.
ReconcilePolicyOnce ReconcilePolicy = "Once"
)

var reconcilePolicyByAnnotation = map[string]ReconcilePolicy{
types.ReconcilePolicyOnObjectChange: ReconcilePolicyOnObjectChange,
types.ReconcilePolicyOnObjectOrComponentChange: ReconcilePolicyOnObjectOrComponentChange,
types.ReconcilePolicyOnce: ReconcilePolicyOnce,
}

// UpdatePolicy defines how the reconciler will update dependent objects.
type UpdatePolicy string

const (
// Recreate (that is: delete and create) existing dependent objects.
UpdatePolicyRecreate UpdatePolicy = "Recreate"
// Replace existing dependent objects.
UpdatePolicyReplace UpdatePolicy = "Replace"
// Use server side apply to update existing dependents.
UpdatePolicySsaMerge UpdatePolicy = "SsaMerge"
// Use server side apply to update existing dependents and, in addition, reclaim fields owned by certain
// field owners, such as kubectl or helm.
UpdatePolicySsaOverride UpdatePolicy = "SsaOverride"
)

var updatePolicyByAnnotation = map[string]UpdatePolicy{
types.UpdatePolicyRecreate: UpdatePolicyRecreate,
types.UpdatePolicyReplace: UpdatePolicyReplace,
types.UpdatePolicySsaMerge: UpdatePolicySsaMerge,
types.UpdatePolicySsaOverride: UpdatePolicySsaOverride,
}

// DeletePolicy defines how the reconciler will delete dependent objects.
type DeletePolicy string

const (
// Delete dependent objects.
DeletePolicyDelete DeletePolicy = "Delete"
// Orphan dependent objects.
DeletePolicyOrphan DeletePolicy = "Orphan"
)

var deletePolicyByAnnotation = map[string]DeletePolicy{
types.DeletePolicyDelete: DeletePolicyDelete,
types.DeletePolicyOrphan: DeletePolicyOrphan,
}

// Reconciler provides the implementation of controller-runtime's Reconciler interface, for a given Component type T.
type Reconciler[T Component] struct {
name string
id string
client cluster.Client
resourceGenerator manifests.Generator
statusAnalyzer kstatus.StatusAnalyzer
options ReconcilerOptions
clients *cluster.ClientFactory
backoff *backoff.Backoff
Expand Down Expand Up @@ -196,6 +124,7 @@ func NewReconciler[T Component](name string, resourceGenerator manifests.Generat
return &Reconciler[T]{
name: name,
resourceGenerator: resourceGenerator,
statusAnalyzer: kstatus.NewStatusAnalyzer(name),
options: options,
backoff: backoff.NewBackoff(10 * time.Second),
postReadHooks: []HookFunc[T]{resolveReferences[T]},
Expand Down Expand Up @@ -249,6 +178,11 @@ func (r *Reconciler[T]) Reconcile(ctx context.Context, req ctrl.Request) (result
// always attempt to update the status
skipStatusUpdate := false
defer func() {
if r := recover(); r != nil {
log.Error(fmt.Errorf("panic occurred during reconcile"), "panic", r)
// re-panic in order skip the remaining steps
panic(r)
}
log.V(1).Info("reconcile done", "withError", err != nil, "requeue", result.Requeue || result.RequeueAfter > 0, "requeueAfter", result.RequeueAfter.String())
if status.State == StateReady || err != nil {
r.backoff.Forget(req)
Expand Down Expand Up @@ -314,7 +248,7 @@ func (r *Reconciler[T]) Reconcile(ctx context.Context, req ctrl.Request) (result
if err != nil {
return ctrl.Result{}, errors.Wrap(err, "error getting client for component")
}
target := newReconcileTarget[T](r.name, r.id, targetClient, r.resourceGenerator, *r.options.CreateMissingNamespaces, *r.options.AdoptionPolicy, *r.options.UpdatePolicy)
target := newReconcileTarget[T](r.name, r.id, targetClient, r.resourceGenerator, r.statusAnalyzer, *r.options.CreateMissingNamespaces, *r.options.AdoptionPolicy, *r.options.UpdatePolicy)
hookCtx := newContext(ctx).WithClient(targetClient)

// do the reconciliation
Expand Down
Loading

0 comments on commit b86915b

Please sign in to comment.