Skip to content

Commit

Permalink
chore: unified role probe framework (#5551)
Browse files Browse the repository at this point in the history
  • Loading branch information
@free6om
free6om authored Oct 20, 2023
1 parent 9bd65ca commit a9fd7ea
Show file tree
Hide file tree
Showing 21 changed files with 319 additions and 330 deletions.
3 changes: 3 additions & 0 deletions apis/apps/v1alpha1/clusterdefinition_types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -330,6 +330,9 @@ type ClusterComponentDefinition struct {

// characterType defines well-known database component name, such as mongos(mongodb), proxy(redis), mariadb(mysql)
// KubeBlocks will generate proper monitor configs for well-known characterType when builtIn is true.
//
// CharacterType will also be used in role probe to decide which probe engine to use.
// current available candidates are: mysql, postgres, mongodb, redis, etcd, kafka.
// +optional
CharacterType string `json:"characterType,omitempty"`

Expand Down
22 changes: 16 additions & 6 deletions apis/workloads/v1alpha1/replicatedstatemachine_types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -217,20 +217,30 @@ type RoleUpdateMechanism string
const (
ReadinessProbeEventUpdate RoleUpdateMechanism = "ReadinessProbeEventUpdate"
DirectAPIServerEventUpdate RoleUpdateMechanism = "DirectAPIServerEventUpdate"
NoneUpdate RoleUpdateMechanism = "None"
)

// RoleProbe defines how to observe role
type RoleProbe struct {
// ProbeActions define Actions to be taken in serial.
// BuiltinHandler specifies the builtin handler name to use to probe the role of the main container.
// current available handlers: mysql, postgres, mongodb, redis, etcd, kafka.
// use CustomHandler to define your own role probe function if none of them satisfies the requirement.
// +optional
BuiltinHandler *string `json:"builtinHandlerName,omitempty"`

// CustomHandler defines the custom way to do role probe.
// if the BuiltinHandler satisfies the requirement, use it instead.
//
// how the actions defined here works:
//
// Actions will be taken in serial.
// after all actions done, the final output should be a single string of the role name defined in spec.Roles
// latest [BusyBox](https://busybox.net/) image will be used if Image not configured
// Environment variables can be used in Command:
// - v_KB_RSM_LAST_STDOUT stdout from last action, watch 'v_' prefixed
// - KB_RSM_USERNAME username part of credential
// - KB_RSM_PASSWORD password part of credential
// +kubebuilder:validation:Required
ProbeActions []Action `json:"probeActions"`
// +optional
CustomHandler []Action `json:"customHandler,omitempty"`

// Number of seconds after the container has started before role probe has started.
// +kubebuilder:default=0
Expand Down Expand Up @@ -267,8 +277,8 @@ type RoleProbe struct {
FailureThreshold int32 `json:"failureThreshold,omitempty"`

// RoleUpdateMechanism specifies the way how pod role label being updated.
// +kubebuilder:default=None
// +kubebuilder:validation:Enum={ReadinessProbeEventUpdate, DirectAPIServerEventUpdate, None}
// +kubebuilder:default=ReadinessProbeEventUpdate
// +kubebuilder:validation:Enum={ReadinessProbeEventUpdate, DirectAPIServerEventUpdate}
// +optional
RoleUpdateMechanism RoleUpdateMechanism `json:"roleUpdateMechanism,omitempty"`
}
Expand Down
2 changes: 1 addition & 1 deletion apis/workloads/v1alpha1/replicatedstatemachine_webhook_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ var _ = Describe("ReplicatedStateMachine Webhook", func() {
},
Service: &corev1.Service{},
RoleProbe: &RoleProbe{
ProbeActions: []Action{
CustomHandler: []Action{
{
Image: "foo",
Command: []string{"bar"},
Expand Down
9 changes: 7 additions & 2 deletions apis/workloads/v1alpha1/zz_generated.deepcopy.go
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

75 changes: 43 additions & 32 deletions config/crd/bases/apps.kubeblocks.io_clusterdefinitions.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,10 +59,13 @@ spec:
stateful workloads and day-2 operations behaviors.
properties:
characterType:
description: characterType defines well-known database component
description: "characterType defines well-known database component
name, such as mongos(mongodb), proxy(redis), mariadb(mysql)
KubeBlocks will generate proper monitor configs for well-known
characterType when builtIn is true.
characterType when builtIn is true. \n CharacterType will
also be used in role probe to decide which probe engine to
use. current available candidates are: mysql, postgres, mongodb,
redis, etcd, kafka."
type: string
componentDefRef:
description: componentDefRef is used to inject values from other
Expand Down Expand Up @@ -8635,6 +8638,43 @@ spec:
roleProbe:
description: RoleProbe provides method to probe role.
properties:
builtinHandlerName:
description: 'BuiltinHandler specifies the builtin handler
name to use to probe the role of the main container.
current available handlers: mysql, postgres, mongodb,
redis, etcd, kafka. use CustomHandler to define your
own role probe function if none of them satisfies
the requirement.'
type: string
customHandler:
description: "CustomHandler defines the custom way to
do role probe. if the BuiltinHandler satisfies the
requirement, use it instead. \n how the actions defined
here works: \n Actions will be taken in serial. after
all actions done, the final output should be a single
string of the role name defined in spec.Roles latest
[BusyBox](https://busybox.net/) image will be used
if Image not configured Environment variables can
be used in Command: - v_KB_RSM_LAST_STDOUT stdout
from last action, watch 'v_' prefixed - KB_RSM_USERNAME
username part of credential - KB_RSM_PASSWORD password
part of credential"
items:
properties:
command:
description: Command will be executed in Container
to retrieve or process role info
items:
type: string
type: array
image:
description: utility image contains command that
can be used to retrieve of process role info
type: string
required:
- command
type: object
type: array
failureThreshold:
default: 3
description: Minimum consecutive failures for the probe
Expand All @@ -8657,40 +8697,13 @@ spec:
format: int32
minimum: 1
type: integer
probeActions:
description: 'ProbeActions define Actions to be taken
in serial. after all actions done, the final output
should be a single string of the role name defined
in spec.Roles latest [BusyBox](https://busybox.net/)
image will be used if Image not configured Environment
variables can be used in Command: - v_KB_RSM_LAST_STDOUT
stdout from last action, watch ''v_'' prefixed - KB_RSM_USERNAME
username part of credential - KB_RSM_PASSWORD password
part of credential'
items:
properties:
command:
description: Command will be executed in Container
to retrieve or process role info
items:
type: string
type: array
image:
description: utility image contains command that
can be used to retrieve of process role info
type: string
required:
- command
type: object
type: array
roleUpdateMechanism:
default: None
default: ReadinessProbeEventUpdate
description: RoleUpdateMechanism specifies the way how
pod role label being updated.
enum:
- ReadinessProbeEventUpdate
- DirectAPIServerEventUpdate
- None
type: string
successThreshold:
default: 1
Expand All @@ -8708,8 +8721,6 @@ spec:
format: int32
minimum: 1
type: integer
required:
- probeActions
type: object
roles:
description: Roles, a list of roles defined in the system.
Expand Down
63 changes: 35 additions & 28 deletions config/crd/bases/workloads.kubeblocks.io_replicatedstatemachines.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -939,6 +939,40 @@ spec:
roleProbe:
description: RoleProbe provides method to probe role.
properties:
builtinHandlerName:
description: 'BuiltinHandler specifies the builtin handler name
to use to probe the role of the main container. current available
handlers: mysql, postgres, mongodb, redis, etcd, kafka. use
CustomHandler to define your own role probe function if none
of them satisfies the requirement.'
type: string
customHandler:
description: "CustomHandler defines the custom way to do role
probe. if the BuiltinHandler satisfies the requirement, use
it instead. \n how the actions defined here works: \n Actions
will be taken in serial. after all actions done, the final output
should be a single string of the role name defined in spec.Roles
latest [BusyBox](https://busybox.net/) image will be used if
Image not configured Environment variables can be used in Command:
- v_KB_RSM_LAST_STDOUT stdout from last action, watch 'v_' prefixed
- KB_RSM_USERNAME username part of credential - KB_RSM_PASSWORD
password part of credential"
items:
properties:
command:
description: Command will be executed in Container to retrieve
or process role info
items:
type: string
type: array
image:
description: utility image contains command that can be
used to retrieve of process role info
type: string
required:
- command
type: object
type: array
failureThreshold:
default: 3
description: Minimum consecutive failures for the probe to be
Expand All @@ -961,38 +995,13 @@ spec:
format: int32
minimum: 1
type: integer
probeActions:
description: 'ProbeActions define Actions to be taken in serial.
after all actions done, the final output should be a single
string of the role name defined in spec.Roles latest [BusyBox](https://busybox.net/)
image will be used if Image not configured Environment variables
can be used in Command: - v_KB_RSM_LAST_STDOUT stdout from last
action, watch ''v_'' prefixed - KB_RSM_USERNAME username part
of credential - KB_RSM_PASSWORD password part of credential'
items:
properties:
command:
description: Command will be executed in Container to retrieve
or process role info
items:
type: string
type: array
image:
description: utility image contains command that can be
used to retrieve of process role info
type: string
required:
- command
type: object
type: array
roleUpdateMechanism:
default: None
default: ReadinessProbeEventUpdate
description: RoleUpdateMechanism specifies the way how pod role
label being updated.
enum:
- ReadinessProbeEventUpdate
- DirectAPIServerEventUpdate
- None
type: string
successThreshold:
default: 1
Expand All @@ -1009,8 +1018,6 @@ spec:
format: int32
minimum: 1
type: integer
required:
- probeActions
type: object
roles:
description: Roles, a list of roles defined in the system.
Expand Down
25 changes: 18 additions & 7 deletions controllers/apps/components/component.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -141,7 +141,7 @@ func (c *rsmComponent) Delete(reqCtx intctrlutil.RequestCtx, cli client.Client)
}

func (c *rsmComponent) Status(reqCtx intctrlutil.RequestCtx, cli client.Client) error {
return c.status(reqCtx, cli, c.newBuilder(reqCtx, cli, model.ActionNoopPtr()))
return c.status(reqCtx, cli, c.newBuilder(reqCtx, cli, nil))
}

func (c *rsmComponent) newBuilder(reqCtx intctrlutil.RequestCtx, cli client.Client,
Expand Down Expand Up @@ -276,6 +276,7 @@ func (c *rsmComponent) status(reqCtx intctrlutil.RequestCtx, cli client.Client,
if c.runningWorkload == nil {
return nil
}
c.noopAllNoneWorkloadObjects()

isDeleting := func() bool {
return !c.runningWorkload.DeletionTimestamp.IsZero()
Expand Down Expand Up @@ -388,6 +389,11 @@ func (c *rsmComponent) status(reqCtx intctrlutil.RequestCtx, cli client.Client,
return err
}

graphCli := model.NewGraphClient(c.Client)
if graphCli.IsAction(c.dag, c.workload, nil) {
graphCli.Noop(c.dag, c.workload)
}

return nil
}

Expand Down Expand Up @@ -478,23 +484,28 @@ func (c *rsmComponent) resolveObjectsAction(reqCtx intctrlutil.RequestCtx, cli c
switch action, err := resolveObjectAction(snapshot, object, cli.Scheme()); {
case err != nil:
return err
case *action == model.UPDATE:
graphCli.Update(c.dag, nil, object)
case *action == model.CREATE:
graphCli.Create(c.dag, object)
default:
graphCli.Noop(c.dag, object)

}
}
if c.GetCluster().IsStatusUpdating() {
// TODO(refactor): fix me, this is a workaround for h-scaling to update stateful set.
objects = graphCli.FindAll(c.dag, &workloads.ReplicatedStateMachine{}, model.HaveDifferentTypeWithOption)
for _, object := range objects {
graphCli.Noop(c.dag, object)
}
c.noopAllNoneWorkloadObjects()
}
return c.validateObjectsAction()
}

func (c *rsmComponent) noopAllNoneWorkloadObjects() {
graphCli := model.NewGraphClient(c.Client)
objects := graphCli.FindAll(c.dag, &workloads.ReplicatedStateMachine{}, model.HaveDifferentTypeWithOption)
for _, object := range objects {
graphCli.Noop(c.dag, object)
}
}

// setStatusPhase sets the cluster component phase and messages conditionally.
func (c *rsmComponent) setStatusPhase(phase appsv1alpha1.ClusterComponentPhase,
statusMessage appsv1alpha1.ComponentMessageMap, phaseTransitionMsg string) {
Expand Down
2 changes: 1 addition & 1 deletion controllers/workloads/replicatedstatemachine_controller_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,7 @@ var _ = Describe("ReplicatedStateMachine Controller", func() {
AddMatchLabelsInMap(commonLabels).
SetService(service).
SetTemplate(template).
AddProbeAction(action).
AddCustomHandler(action).
GetObject()
Expect(k8sClient.Create(ctx, rsm)).Should(Succeed())
Eventually(testapps.CheckObj(&testCtx, client.ObjectKeyFromObject(rsm),
Expand Down
Loading

0 comments on commit a9fd7ea

Please sign in to comment.