diff --git a/apis/apps/v1alpha1/clusterdefinition_types.go b/apis/apps/v1alpha1/clusterdefinition_types.go index 63b66e095cf..e4a35a20137 100644 --- a/apis/apps/v1alpha1/clusterdefinition_types.go +++ b/apis/apps/v1alpha1/clusterdefinition_types.go @@ -330,6 +330,9 @@ type ClusterComponentDefinition struct { // characterType defines well-known database component name, such as mongos(mongodb), proxy(redis), mariadb(mysql) // KubeBlocks will generate proper monitor configs for well-known characterType when builtIn is true. + // + // CharacterType will also be used in role probe to decide which probe engine to use. + // current available candidates are: mysql, postgres, mongodb, redis, etcd, kafka. // +optional CharacterType string `json:"characterType,omitempty"` diff --git a/apis/workloads/v1alpha1/replicatedstatemachine_types.go b/apis/workloads/v1alpha1/replicatedstatemachine_types.go index 9f1fdd2146b..8743c059a58 100644 --- a/apis/workloads/v1alpha1/replicatedstatemachine_types.go +++ b/apis/workloads/v1alpha1/replicatedstatemachine_types.go @@ -217,20 +217,30 @@ type RoleUpdateMechanism string const ( ReadinessProbeEventUpdate RoleUpdateMechanism = "ReadinessProbeEventUpdate" DirectAPIServerEventUpdate RoleUpdateMechanism = "DirectAPIServerEventUpdate" - NoneUpdate RoleUpdateMechanism = "None" ) // RoleProbe defines how to observe role type RoleProbe struct { - // ProbeActions define Actions to be taken in serial. + // BuiltinHandler specifies the builtin handler name to use to probe the role of the main container. + // current available handlers: mysql, postgres, mongodb, redis, etcd, kafka. + // use CustomHandler to define your own role probe function if none of them satisfies the requirement. + // +optional + BuiltinHandler *string `json:"builtinHandlerName,omitempty"` + + // CustomHandler defines the custom way to do role probe. + // if the BuiltinHandler satisfies the requirement, use it instead. + // + // how the actions defined here works: + // + // Actions will be taken in serial. // after all actions done, the final output should be a single string of the role name defined in spec.Roles // latest [BusyBox](https://busybox.net/) image will be used if Image not configured // Environment variables can be used in Command: // - v_KB_RSM_LAST_STDOUT stdout from last action, watch 'v_' prefixed // - KB_RSM_USERNAME username part of credential // - KB_RSM_PASSWORD password part of credential - // +kubebuilder:validation:Required - ProbeActions []Action `json:"probeActions"` + // +optional + CustomHandler []Action `json:"customHandler,omitempty"` // Number of seconds after the container has started before role probe has started. // +kubebuilder:default=0 @@ -267,8 +277,8 @@ type RoleProbe struct { FailureThreshold int32 `json:"failureThreshold,omitempty"` // RoleUpdateMechanism specifies the way how pod role label being updated. - // +kubebuilder:default=None - // +kubebuilder:validation:Enum={ReadinessProbeEventUpdate, DirectAPIServerEventUpdate, None} + // +kubebuilder:default=ReadinessProbeEventUpdate + // +kubebuilder:validation:Enum={ReadinessProbeEventUpdate, DirectAPIServerEventUpdate} // +optional RoleUpdateMechanism RoleUpdateMechanism `json:"roleUpdateMechanism,omitempty"` } diff --git a/apis/workloads/v1alpha1/replicatedstatemachine_webhook_test.go b/apis/workloads/v1alpha1/replicatedstatemachine_webhook_test.go index 768c3347d25..3d6648289bd 100644 --- a/apis/workloads/v1alpha1/replicatedstatemachine_webhook_test.go +++ b/apis/workloads/v1alpha1/replicatedstatemachine_webhook_test.go @@ -55,7 +55,7 @@ var _ = Describe("ReplicatedStateMachine Webhook", func() { }, Service: &corev1.Service{}, RoleProbe: &RoleProbe{ - ProbeActions: []Action{ + CustomHandler: []Action{ { Image: "foo", Command: []string{"bar"}, diff --git a/apis/workloads/v1alpha1/zz_generated.deepcopy.go b/apis/workloads/v1alpha1/zz_generated.deepcopy.go index 563b051a208..f2a4dfdf03d 100644 --- a/apis/workloads/v1alpha1/zz_generated.deepcopy.go +++ b/apis/workloads/v1alpha1/zz_generated.deepcopy.go @@ -312,8 +312,13 @@ func (in *ReplicatedStateMachineStatus) DeepCopy() *ReplicatedStateMachineStatus // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *RoleProbe) DeepCopyInto(out *RoleProbe) { *out = *in - if in.ProbeActions != nil { - in, out := &in.ProbeActions, &out.ProbeActions + if in.BuiltinHandler != nil { + in, out := &in.BuiltinHandler, &out.BuiltinHandler + *out = new(string) + **out = **in + } + if in.CustomHandler != nil { + in, out := &in.CustomHandler, &out.CustomHandler *out = make([]Action, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) diff --git a/config/crd/bases/apps.kubeblocks.io_clusterdefinitions.yaml b/config/crd/bases/apps.kubeblocks.io_clusterdefinitions.yaml index 538dc69bc71..8080c8950aa 100644 --- a/config/crd/bases/apps.kubeblocks.io_clusterdefinitions.yaml +++ b/config/crd/bases/apps.kubeblocks.io_clusterdefinitions.yaml @@ -59,10 +59,13 @@ spec: stateful workloads and day-2 operations behaviors. properties: characterType: - description: characterType defines well-known database component + description: "characterType defines well-known database component name, such as mongos(mongodb), proxy(redis), mariadb(mysql) KubeBlocks will generate proper monitor configs for well-known - characterType when builtIn is true. + characterType when builtIn is true. \n CharacterType will + also be used in role probe to decide which probe engine to + use. current available candidates are: mysql, postgres, mongodb, + redis, etcd, kafka." type: string componentDefRef: description: componentDefRef is used to inject values from other @@ -8635,6 +8638,43 @@ spec: roleProbe: description: RoleProbe provides method to probe role. properties: + builtinHandlerName: + description: 'BuiltinHandler specifies the builtin handler + name to use to probe the role of the main container. + current available handlers: mysql, postgres, mongodb, + redis, etcd, kafka. use CustomHandler to define your + own role probe function if none of them satisfies + the requirement.' + type: string + customHandler: + description: "CustomHandler defines the custom way to + do role probe. if the BuiltinHandler satisfies the + requirement, use it instead. \n how the actions defined + here works: \n Actions will be taken in serial. after + all actions done, the final output should be a single + string of the role name defined in spec.Roles latest + [BusyBox](https://busybox.net/) image will be used + if Image not configured Environment variables can + be used in Command: - v_KB_RSM_LAST_STDOUT stdout + from last action, watch 'v_' prefixed - KB_RSM_USERNAME + username part of credential - KB_RSM_PASSWORD password + part of credential" + items: + properties: + command: + description: Command will be executed in Container + to retrieve or process role info + items: + type: string + type: array + image: + description: utility image contains command that + can be used to retrieve of process role info + type: string + required: + - command + type: object + type: array failureThreshold: default: 3 description: Minimum consecutive failures for the probe @@ -8657,40 +8697,13 @@ spec: format: int32 minimum: 1 type: integer - probeActions: - description: 'ProbeActions define Actions to be taken - in serial. after all actions done, the final output - should be a single string of the role name defined - in spec.Roles latest [BusyBox](https://busybox.net/) - image will be used if Image not configured Environment - variables can be used in Command: - v_KB_RSM_LAST_STDOUT - stdout from last action, watch ''v_'' prefixed - KB_RSM_USERNAME - username part of credential - KB_RSM_PASSWORD password - part of credential' - items: - properties: - command: - description: Command will be executed in Container - to retrieve or process role info - items: - type: string - type: array - image: - description: utility image contains command that - can be used to retrieve of process role info - type: string - required: - - command - type: object - type: array roleUpdateMechanism: - default: None + default: ReadinessProbeEventUpdate description: RoleUpdateMechanism specifies the way how pod role label being updated. enum: - ReadinessProbeEventUpdate - DirectAPIServerEventUpdate - - None type: string successThreshold: default: 1 @@ -8708,8 +8721,6 @@ spec: format: int32 minimum: 1 type: integer - required: - - probeActions type: object roles: description: Roles, a list of roles defined in the system. diff --git a/config/crd/bases/workloads.kubeblocks.io_replicatedstatemachines.yaml b/config/crd/bases/workloads.kubeblocks.io_replicatedstatemachines.yaml index 6e84668baf6..d6e7c44de27 100644 --- a/config/crd/bases/workloads.kubeblocks.io_replicatedstatemachines.yaml +++ b/config/crd/bases/workloads.kubeblocks.io_replicatedstatemachines.yaml @@ -939,6 +939,40 @@ spec: roleProbe: description: RoleProbe provides method to probe role. properties: + builtinHandlerName: + description: 'BuiltinHandler specifies the builtin handler name + to use to probe the role of the main container. current available + handlers: mysql, postgres, mongodb, redis, etcd, kafka. use + CustomHandler to define your own role probe function if none + of them satisfies the requirement.' + type: string + customHandler: + description: "CustomHandler defines the custom way to do role + probe. if the BuiltinHandler satisfies the requirement, use + it instead. \n how the actions defined here works: \n Actions + will be taken in serial. after all actions done, the final output + should be a single string of the role name defined in spec.Roles + latest [BusyBox](https://busybox.net/) image will be used if + Image not configured Environment variables can be used in Command: + - v_KB_RSM_LAST_STDOUT stdout from last action, watch 'v_' prefixed + - KB_RSM_USERNAME username part of credential - KB_RSM_PASSWORD + password part of credential" + items: + properties: + command: + description: Command will be executed in Container to retrieve + or process role info + items: + type: string + type: array + image: + description: utility image contains command that can be + used to retrieve of process role info + type: string + required: + - command + type: object + type: array failureThreshold: default: 3 description: Minimum consecutive failures for the probe to be @@ -961,38 +995,13 @@ spec: format: int32 minimum: 1 type: integer - probeActions: - description: 'ProbeActions define Actions to be taken in serial. - after all actions done, the final output should be a single - string of the role name defined in spec.Roles latest [BusyBox](https://busybox.net/) - image will be used if Image not configured Environment variables - can be used in Command: - v_KB_RSM_LAST_STDOUT stdout from last - action, watch ''v_'' prefixed - KB_RSM_USERNAME username part - of credential - KB_RSM_PASSWORD password part of credential' - items: - properties: - command: - description: Command will be executed in Container to retrieve - or process role info - items: - type: string - type: array - image: - description: utility image contains command that can be - used to retrieve of process role info - type: string - required: - - command - type: object - type: array roleUpdateMechanism: - default: None + default: ReadinessProbeEventUpdate description: RoleUpdateMechanism specifies the way how pod role label being updated. enum: - ReadinessProbeEventUpdate - DirectAPIServerEventUpdate - - None type: string successThreshold: default: 1 @@ -1009,8 +1018,6 @@ spec: format: int32 minimum: 1 type: integer - required: - - probeActions type: object roles: description: Roles, a list of roles defined in the system. diff --git a/controllers/apps/components/component.go b/controllers/apps/components/component.go index fb3617941d2..16e46ab1090 100644 --- a/controllers/apps/components/component.go +++ b/controllers/apps/components/component.go @@ -141,7 +141,7 @@ func (c *rsmComponent) Delete(reqCtx intctrlutil.RequestCtx, cli client.Client) } func (c *rsmComponent) Status(reqCtx intctrlutil.RequestCtx, cli client.Client) error { - return c.status(reqCtx, cli, c.newBuilder(reqCtx, cli, model.ActionNoopPtr())) + return c.status(reqCtx, cli, c.newBuilder(reqCtx, cli, nil)) } func (c *rsmComponent) newBuilder(reqCtx intctrlutil.RequestCtx, cli client.Client, @@ -276,6 +276,7 @@ func (c *rsmComponent) status(reqCtx intctrlutil.RequestCtx, cli client.Client, if c.runningWorkload == nil { return nil } + c.noopAllNoneWorkloadObjects() isDeleting := func() bool { return !c.runningWorkload.DeletionTimestamp.IsZero() @@ -388,6 +389,11 @@ func (c *rsmComponent) status(reqCtx intctrlutil.RequestCtx, cli client.Client, return err } + graphCli := model.NewGraphClient(c.Client) + if graphCli.IsAction(c.dag, c.workload, nil) { + graphCli.Noop(c.dag, c.workload) + } + return nil } @@ -478,8 +484,8 @@ func (c *rsmComponent) resolveObjectsAction(reqCtx intctrlutil.RequestCtx, cli c switch action, err := resolveObjectAction(snapshot, object, cli.Scheme()); { case err != nil: return err - case *action == model.UPDATE: - graphCli.Update(c.dag, nil, object) + case *action == model.CREATE: + graphCli.Create(c.dag, object) default: graphCli.Noop(c.dag, object) @@ -487,14 +493,19 @@ func (c *rsmComponent) resolveObjectsAction(reqCtx intctrlutil.RequestCtx, cli c } if c.GetCluster().IsStatusUpdating() { // TODO(refactor): fix me, this is a workaround for h-scaling to update stateful set. - objects = graphCli.FindAll(c.dag, &workloads.ReplicatedStateMachine{}, model.HaveDifferentTypeWithOption) - for _, object := range objects { - graphCli.Noop(c.dag, object) - } + c.noopAllNoneWorkloadObjects() } return c.validateObjectsAction() } +func (c *rsmComponent) noopAllNoneWorkloadObjects() { + graphCli := model.NewGraphClient(c.Client) + objects := graphCli.FindAll(c.dag, &workloads.ReplicatedStateMachine{}, model.HaveDifferentTypeWithOption) + for _, object := range objects { + graphCli.Noop(c.dag, object) + } +} + // setStatusPhase sets the cluster component phase and messages conditionally. func (c *rsmComponent) setStatusPhase(phase appsv1alpha1.ClusterComponentPhase, statusMessage appsv1alpha1.ComponentMessageMap, phaseTransitionMsg string) { diff --git a/controllers/workloads/replicatedstatemachine_controller_test.go b/controllers/workloads/replicatedstatemachine_controller_test.go index 21115e7fa71..86b6689042f 100644 --- a/controllers/workloads/replicatedstatemachine_controller_test.go +++ b/controllers/workloads/replicatedstatemachine_controller_test.go @@ -80,7 +80,7 @@ var _ = Describe("ReplicatedStateMachine Controller", func() { AddMatchLabelsInMap(commonLabels). SetService(service). SetTemplate(template). - AddProbeAction(action). + AddCustomHandler(action). GetObject() Expect(k8sClient.Create(ctx, rsm)).Should(Succeed()) Eventually(testapps.CheckObj(&testCtx, client.ObjectKeyFromObject(rsm), diff --git a/deploy/helm/crds/apps.kubeblocks.io_clusterdefinitions.yaml b/deploy/helm/crds/apps.kubeblocks.io_clusterdefinitions.yaml index 538dc69bc71..8080c8950aa 100644 --- a/deploy/helm/crds/apps.kubeblocks.io_clusterdefinitions.yaml +++ b/deploy/helm/crds/apps.kubeblocks.io_clusterdefinitions.yaml @@ -59,10 +59,13 @@ spec: stateful workloads and day-2 operations behaviors. properties: characterType: - description: characterType defines well-known database component + description: "characterType defines well-known database component name, such as mongos(mongodb), proxy(redis), mariadb(mysql) KubeBlocks will generate proper monitor configs for well-known - characterType when builtIn is true. + characterType when builtIn is true. \n CharacterType will + also be used in role probe to decide which probe engine to + use. current available candidates are: mysql, postgres, mongodb, + redis, etcd, kafka." type: string componentDefRef: description: componentDefRef is used to inject values from other @@ -8635,6 +8638,43 @@ spec: roleProbe: description: RoleProbe provides method to probe role. properties: + builtinHandlerName: + description: 'BuiltinHandler specifies the builtin handler + name to use to probe the role of the main container. + current available handlers: mysql, postgres, mongodb, + redis, etcd, kafka. use CustomHandler to define your + own role probe function if none of them satisfies + the requirement.' + type: string + customHandler: + description: "CustomHandler defines the custom way to + do role probe. if the BuiltinHandler satisfies the + requirement, use it instead. \n how the actions defined + here works: \n Actions will be taken in serial. after + all actions done, the final output should be a single + string of the role name defined in spec.Roles latest + [BusyBox](https://busybox.net/) image will be used + if Image not configured Environment variables can + be used in Command: - v_KB_RSM_LAST_STDOUT stdout + from last action, watch 'v_' prefixed - KB_RSM_USERNAME + username part of credential - KB_RSM_PASSWORD password + part of credential" + items: + properties: + command: + description: Command will be executed in Container + to retrieve or process role info + items: + type: string + type: array + image: + description: utility image contains command that + can be used to retrieve of process role info + type: string + required: + - command + type: object + type: array failureThreshold: default: 3 description: Minimum consecutive failures for the probe @@ -8657,40 +8697,13 @@ spec: format: int32 minimum: 1 type: integer - probeActions: - description: 'ProbeActions define Actions to be taken - in serial. after all actions done, the final output - should be a single string of the role name defined - in spec.Roles latest [BusyBox](https://busybox.net/) - image will be used if Image not configured Environment - variables can be used in Command: - v_KB_RSM_LAST_STDOUT - stdout from last action, watch ''v_'' prefixed - KB_RSM_USERNAME - username part of credential - KB_RSM_PASSWORD password - part of credential' - items: - properties: - command: - description: Command will be executed in Container - to retrieve or process role info - items: - type: string - type: array - image: - description: utility image contains command that - can be used to retrieve of process role info - type: string - required: - - command - type: object - type: array roleUpdateMechanism: - default: None + default: ReadinessProbeEventUpdate description: RoleUpdateMechanism specifies the way how pod role label being updated. enum: - ReadinessProbeEventUpdate - DirectAPIServerEventUpdate - - None type: string successThreshold: default: 1 @@ -8708,8 +8721,6 @@ spec: format: int32 minimum: 1 type: integer - required: - - probeActions type: object roles: description: Roles, a list of roles defined in the system. diff --git a/deploy/helm/crds/workloads.kubeblocks.io_replicatedstatemachines.yaml b/deploy/helm/crds/workloads.kubeblocks.io_replicatedstatemachines.yaml index 6e84668baf6..d6e7c44de27 100644 --- a/deploy/helm/crds/workloads.kubeblocks.io_replicatedstatemachines.yaml +++ b/deploy/helm/crds/workloads.kubeblocks.io_replicatedstatemachines.yaml @@ -939,6 +939,40 @@ spec: roleProbe: description: RoleProbe provides method to probe role. properties: + builtinHandlerName: + description: 'BuiltinHandler specifies the builtin handler name + to use to probe the role of the main container. current available + handlers: mysql, postgres, mongodb, redis, etcd, kafka. use + CustomHandler to define your own role probe function if none + of them satisfies the requirement.' + type: string + customHandler: + description: "CustomHandler defines the custom way to do role + probe. if the BuiltinHandler satisfies the requirement, use + it instead. \n how the actions defined here works: \n Actions + will be taken in serial. after all actions done, the final output + should be a single string of the role name defined in spec.Roles + latest [BusyBox](https://busybox.net/) image will be used if + Image not configured Environment variables can be used in Command: + - v_KB_RSM_LAST_STDOUT stdout from last action, watch 'v_' prefixed + - KB_RSM_USERNAME username part of credential - KB_RSM_PASSWORD + password part of credential" + items: + properties: + command: + description: Command will be executed in Container to retrieve + or process role info + items: + type: string + type: array + image: + description: utility image contains command that can be + used to retrieve of process role info + type: string + required: + - command + type: object + type: array failureThreshold: default: 3 description: Minimum consecutive failures for the probe to be @@ -961,38 +995,13 @@ spec: format: int32 minimum: 1 type: integer - probeActions: - description: 'ProbeActions define Actions to be taken in serial. - after all actions done, the final output should be a single - string of the role name defined in spec.Roles latest [BusyBox](https://busybox.net/) - image will be used if Image not configured Environment variables - can be used in Command: - v_KB_RSM_LAST_STDOUT stdout from last - action, watch ''v_'' prefixed - KB_RSM_USERNAME username part - of credential - KB_RSM_PASSWORD password part of credential' - items: - properties: - command: - description: Command will be executed in Container to retrieve - or process role info - items: - type: string - type: array - image: - description: utility image contains command that can be - used to retrieve of process role info - type: string - required: - - command - type: object - type: array roleUpdateMechanism: - default: None + default: ReadinessProbeEventUpdate description: RoleUpdateMechanism specifies the way how pod role label being updated. enum: - ReadinessProbeEventUpdate - DirectAPIServerEventUpdate - - None type: string successThreshold: default: 1 @@ -1009,8 +1018,6 @@ spec: format: int32 minimum: 1 type: integer - required: - - probeActions type: object roles: description: Roles, a list of roles defined in the system. diff --git a/deploy/polardbx/templates/clusterDefintion.yaml b/deploy/polardbx/templates/clusterDefintion.yaml index 2f64dd8a5d6..4cf96e4fdc9 100644 --- a/deploy/polardbx/templates/clusterDefintion.yaml +++ b/deploy/polardbx/templates/clusterDefintion.yaml @@ -22,13 +22,6 @@ spec: defaultMode: 0555 workloadType: Consensus characterType: polardbx - consensusSpec: - leader: - name: "leader" - accessMode: ReadWrite - followers: - - name: "follower" - accessMode: Readonly rsmSpec: roles: - name: "leader" @@ -40,7 +33,7 @@ spec: canVote: true roleProbe: roleUpdateMechanism: DirectAPIServerEventUpdate - probeActions: + customHandler: - image: "arey/mysql-client:latest" command: - mysql @@ -243,13 +236,6 @@ spec: - name: GMS_SVC_NAME valueFrom: type: ServiceRef - consensusSpec: - leader: - name: "leader" - accessMode: ReadWrite - followers: - - name: "follower" - accessMode: Readonly rsmSpec: roles: - name: "leader" @@ -261,7 +247,7 @@ spec: canVote: true roleProbe: roleUpdateMechanism: DirectAPIServerEventUpdate - probeActions: + customHandler: - image: "arey/mysql-client:latest" command: - mysql diff --git a/pkg/common/types.go b/pkg/common/types.go index f4279a1c9f4..54b073ce549 100644 --- a/pkg/common/types.go +++ b/pkg/common/types.go @@ -37,3 +37,16 @@ type GlobalRoleSnapshot struct { Version string `json:"term,omitempty"` PodRoleNamePairs []PodRoleNamePair `json:"PodRoleNamePairs,omitempty"` } + +// BuiltinHandler defines builtin role probe handler name. +type BuiltinHandler string + +const ( + MySQLHandler BuiltinHandler = "mysql" + PostgresHandler BuiltinHandler = "postgres" + MongoDBHandler BuiltinHandler = "mongodb" + RedisHandler BuiltinHandler = "redis" + ETCDHandler BuiltinHandler = "etcd" + KafkaHandler BuiltinHandler = "kafka" + WeSQLHandler BuiltinHandler = "wesql" +) diff --git a/pkg/controller/builder/builder_replicated_state_machine.go b/pkg/controller/builder/builder_replicated_state_machine.go index ebb945910c3..695e113ed9d 100644 --- a/pkg/controller/builder/builder_replicated_state_machine.go +++ b/pkg/controller/builder/builder_replicated_state_machine.go @@ -117,24 +117,24 @@ func (builder *ReplicatedStateMachineBuilder) SetUpdateStrategyType(strategyType return builder } -func (builder *ReplicatedStateMachineBuilder) SetProbeActions(actions []workloads.Action) *ReplicatedStateMachineBuilder { +func (builder *ReplicatedStateMachineBuilder) SetCustomHandler(handler []workloads.Action) *ReplicatedStateMachineBuilder { roleProbe := builder.get().Spec.RoleProbe if roleProbe == nil { roleProbe = &workloads.RoleProbe{} } - roleProbe.ProbeActions = actions + roleProbe.CustomHandler = handler builder.get().Spec.RoleProbe = roleProbe return builder } -func (builder *ReplicatedStateMachineBuilder) AddProbeAction(action workloads.Action) *ReplicatedStateMachineBuilder { +func (builder *ReplicatedStateMachineBuilder) AddCustomHandler(handler workloads.Action) *ReplicatedStateMachineBuilder { roleProbe := builder.get().Spec.RoleProbe if roleProbe == nil { roleProbe = &workloads.RoleProbe{} } - actions := roleProbe.ProbeActions - actions = append(actions, action) - roleProbe.ProbeActions = actions + handlers := roleProbe.CustomHandler + handlers = append(handlers, handler) + roleProbe.CustomHandler = handlers builder.get().Spec.RoleProbe = roleProbe return builder } diff --git a/pkg/controller/builder/builder_replicated_state_machine_test.go b/pkg/controller/builder/builder_replicated_state_machine_test.go index fcdb803d9ef..396193da9f8 100644 --- a/pkg/controller/builder/builder_replicated_state_machine_test.go +++ b/pkg/controller/builder/builder_replicated_state_machine_test.go @@ -173,8 +173,8 @@ var _ = Describe("replicated_state_machine builder", func() { SetUpdateStrategy(strategy). SetUpdateStrategyType(strategyType). SetRoleProbe(&roleProbe). - SetProbeActions(actions). - AddProbeAction(action). + SetCustomHandler(actions). + AddCustomHandler(action). SetMemberUpdateStrategy(&memberUpdateStrategy). SetService(service). SetAlternativeServices(alternativeServices). @@ -209,9 +209,9 @@ var _ = Describe("replicated_state_machine builder", func() { Expect(rsm.Spec.UpdateStrategy.RollingUpdate.MaxUnavailable).ShouldNot(Equal(maxUnavailable)) Expect(rsm.Spec.RoleProbe).ShouldNot(BeNil()) Expect(rsm.Spec.RoleProbe.InitialDelaySeconds).Should(Equal(delay)) - Expect(rsm.Spec.RoleProbe.ProbeActions).Should(HaveLen(2)) - Expect(rsm.Spec.RoleProbe.ProbeActions[0]).Should(Equal(actions[0])) - Expect(rsm.Spec.RoleProbe.ProbeActions[1]).Should(Equal(action)) + Expect(rsm.Spec.RoleProbe.CustomHandler).Should(HaveLen(2)) + Expect(rsm.Spec.RoleProbe.CustomHandler[0]).Should(Equal(actions[0])) + Expect(rsm.Spec.RoleProbe.CustomHandler[1]).Should(Equal(action)) Expect(rsm.Spec.MemberUpdateStrategy).ShouldNot(BeNil()) Expect(*rsm.Spec.MemberUpdateStrategy).Should(Equal(memberUpdateStrategy)) Expect(rsm.Spec.Service).ShouldNot(BeNil()) diff --git a/pkg/controller/component/lorry_utils.go b/pkg/controller/component/lorry_utils.go index a89093e332b..0ad4a42f871 100644 --- a/pkg/controller/component/lorry_utils.go +++ b/pkg/controller/component/lorry_utils.go @@ -73,6 +73,7 @@ func buildLorryContainers(reqCtx intctrlutil.RequestCtx, component *SynthesizedC reqCtx.Log.Info("get lorry container port failed", "error", err) return err } + lorrySvcGRPCPort := viper.GetInt("PROBE_SERVICE_GRPC_PORT") if componentProbes.RoleProbe != nil && (component.RSMSpec == nil || component.RSMSpec.RoleProbe == nil) { roleChangedContainer := container.DeepCopy() @@ -107,7 +108,7 @@ func buildLorryContainers(reqCtx intctrlutil.RequestCtx, component *SynthesizedC lorryContainers = append(lorryContainers, *weSyncerContainer) } - buildLorryServiceContainer(component, &lorryContainers[0], int(lorrySvcHTTPPort)) + buildLorryServiceContainer(component, &lorryContainers[0], int(lorrySvcHTTPPort), lorrySvcGRPCPort) reqCtx.Log.V(1).Info("lorry", "containers", lorryContainers) component.PodSpec.Containers = append(component.PodSpec.Containers, lorryContainers...) @@ -143,11 +144,12 @@ func buildBasicContainer() *corev1.Container { GetObject() } -func buildLorryServiceContainer(component *SynthesizedComponent, container *corev1.Container, lorrySvcHTTPPort int) { +func buildLorryServiceContainer(component *SynthesizedComponent, container *corev1.Container, lorrySvcHTTPPort, lorrySvcGRPCPort int) { container.Image = viper.GetString(constant.KBToolsImage) container.ImagePullPolicy = corev1.PullPolicy(viper.GetString(constant.KBImagePullPolicy)) container.Command = []string{"lorry", "--port", strconv.Itoa(lorrySvcHTTPPort), + "--grpcport", strconv.Itoa(lorrySvcGRPCPort), } if len(component.PodSpec.Containers) > 0 { @@ -222,7 +224,13 @@ func buildLorryServiceContainer(component *SynthesizedComponent, container *core ContainerPort: int32(lorrySvcHTTPPort), Name: constant.LorryHTTPPortName, Protocol: "TCP", - }} + }, + { + ContainerPort: int32(lorrySvcGRPCPort), + Name: constant.LorryGRPCPortName, + Protocol: "TCP", + }, + } // pass the volume protection spec to lorry container through env. if volumeProtectionEnabled(component) { diff --git a/pkg/controller/component/lorry_utils_test.go b/pkg/controller/component/lorry_utils_test.go index a2dd4d25a22..4241ede273b 100644 --- a/pkg/controller/component/lorry_utils_test.go +++ b/pkg/controller/component/lorry_utils_test.go @@ -41,11 +41,13 @@ var _ = Describe("probe_utils", func() { var container *corev1.Container var component *SynthesizedComponent var probeServiceHTTPPort int + var probeServiceGRPCPort int var clusterDefProbe *appsv1alpha1.ClusterDefinitionProbe BeforeEach(func() { container = buildBasicContainer() probeServiceHTTPPort = 3501 + probeServiceGRPCPort = 50001 clusterDefProbe = &appsv1alpha1.ClusterDefinitionProbe{} clusterDefProbe.PeriodSeconds = 1 @@ -99,7 +101,7 @@ var _ = Describe("probe_utils", func() { }) It("should build role service container", func() { - buildLorryServiceContainer(component, container, probeServiceHTTPPort) + buildLorryServiceContainer(component, container, probeServiceHTTPPort, probeServiceGRPCPort) Expect(container.Command).ShouldNot(BeEmpty()) }) diff --git a/pkg/controller/factory/builder.go b/pkg/controller/factory/builder.go index 8634d7a65e7..cc574cdc557 100644 --- a/pkg/controller/factory/builder.go +++ b/pkg/controller/factory/builder.go @@ -42,6 +42,7 @@ import ( appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" dpv1alpha1 "github.com/apecloud/kubeblocks/apis/dataprotection/v1alpha1" workloads "github.com/apecloud/kubeblocks/apis/workloads/v1alpha1" + "github.com/apecloud/kubeblocks/pkg/common" cfgcm "github.com/apecloud/kubeblocks/pkg/configuration/config_manager" "github.com/apecloud/kubeblocks/pkg/constant" "github.com/apecloud/kubeblocks/pkg/controller/builder" @@ -465,9 +466,11 @@ func buildRoleInfo(component *component.SynthesizedComponent) ([]workloads.Repli strategy *workloads.MemberUpdateStrategy ) - actions := buildActionFromCharacterType(component.CharacterType, component.WorkloadType == appsv1alpha1.Consensus) - if actions != nil && component.Probes != nil && component.Probes.RoleProbe != nil { - probe = &workloads.RoleProbe{ProbeActions: actions} + handler := convertCharacterTypeToHandler(component.CharacterType, component.WorkloadType == appsv1alpha1.Consensus) + + if handler != nil && component.Probes != nil && component.Probes.RoleProbe != nil { + probe = &workloads.RoleProbe{} + probe.BuiltinHandler = (*string)(handler) roleProbe := component.Probes.RoleProbe probe.PeriodSeconds = roleProbe.PeriodSeconds probe.TimeoutSeconds = roleProbe.TimeoutSeconds @@ -477,7 +480,6 @@ func buildRoleInfo(component *component.SynthesizedComponent) ([]workloads.Repli probe.RoleUpdateMechanism = workloads.DirectAPIServerEventUpdate } - // TODO(free6om): set default reconfiguration actions after relative addon refactored reconfiguration = nil switch component.WorkloadType { @@ -554,104 +556,30 @@ func buildRoleInfoFromConsensus(consensusSpec *appsv1alpha1.ConsensusSetSpec) ([ return roles, strategy } -// buildActionFromCharacterType is a temporary workaround to provide vary engines' role probe functionality, -// as there is no way to configure these fields in Cluster API currently. -// TODO(free6om): remove this after ComponentDefinition API re-designed. -func buildActionFromCharacterType(characterType string, isConsensus bool) []workloads.Action { +func convertCharacterTypeToHandler(characterType string, isConsensus bool) *common.BuiltinHandler { + var handler common.BuiltinHandler kind := strings.ToLower(characterType) switch kind { case "mysql": //nolint:goconst if isConsensus { - return []workloads.Action{ - { - Image: "arey/mysql-client:latest", - Command: []string{ - "mysql", - "-h127.0.0.1", - "-P3306", - "-u$KB_RSM_USERNAME", - "-p$KB_RSM_PASSWORD", - "-N", - "-B", - "-e", - "\"select role from information_schema.wesql_cluster_local\"", - "|", - "xargs echo -n", - }, - }, - { - Command: []string{"echo $v_KB_RSM_LAST_STDOUT | tr '[:upper:]' '[:lower:]' | xargs echo -n"}, - }, - } - } - return []workloads.Action{ - { - Image: "free6om/kubeblocks:latest", - Command: []string{ - "curl http://127.0.0.1:3501/v1.0/bindings/mysql?operation=checkRole&workloadType=Replication", - }, - }, - { - Image: "jetbrainsinfra/jq:latest", - Command: []string{ - "echo $v_KB_RSM_LAST_STDOUT | jq -r '.role' | tr '[:upper:]' '[:lower:]' | xargs echo -n", - }, - }, + handler = common.WeSQLHandler + } else { + handler = common.MySQLHandler } case "postgres", "postgresql": - return []workloads.Action{ - { - Image: "governmentpaas/psql:latest", - Command: []string{ - "PGPASSWORD=$KB_RSM_PASSWORD psql", - "-h 127.0.0.1", - "-p 5432", - "-U $KB_RSM_USERNAME", - "-w", - "-t", - "-c", - "\"select pg_is_in_recovery();\"", - "|", - "xargs echo -n", - }, - }, - { - Command: []string{"if [ \"f\" = \"$v_KB_RSM_LAST_STDOUT\" ]; then echo -n \"primary\"; else echo -n \"secondary\"; fi"}, - }, - } + handler = common.PostgresHandler case "mongodb": - return []workloads.Action{ - { - Image: "infracreate-registry.cn-zhangjiakou.cr.aliyuncs.com/apecloud/mongo:5.0.14", - Command: []string{ - "Status=$(export CLIENT=`which mongosh>/dev/null&&echo mongosh||echo mongo`; $CLIENT -u $KB_RSM_USERNAME -p $KB_RSM_PASSWORD 127.0.0.1:27017 --authenticationDatabase admin --quiet --eval \"JSON.stringify(rs.status())\") &&", - "MyState=$(echo $Status | jq '.myState') &&", - "echo $Status | jq \".members[] | select(.state == ($MyState | tonumber)) | .stateStr\" |tr '[:upper:]' '[:lower:]' | xargs echo -n", - }, - }, - } + + handler = common.MongoDBHandler case "etcd": - return []workloads.Action{ - { - Image: "quay.io/coreos/etcd:v3.5.6", - Command: []string{ - "Status=$(etcdctl --endpoints=127.0.0.1:2379 endpoint status -w simple --command-timeout=300ms --dial-timeout=100m) &&", - "IsLeader=$(echo $Status | awk -F ', ' '{print $5}') &&", - "IsLearner=$(echo $Status | awk -F ', ' '{print $6}') &&", - "if [ \"true\" = \"$IsLeader\" ]; then echo -n \"leader\"; elif [ \"true\" = \"$IsLearner\" ]; then echo -n \"learner\"; else echo -n \"follower\"; fi", - }, - }, - } + handler = common.ETCDHandler case "redis": - return []workloads.Action{ - { - Image: "infracreate-registry.cn-zhangjiakou.cr.aliyuncs.com/apecloud/redis-stack-server:7.0.6-RC8", - Command: []string{ - "Role=$(redis-cli --user $KB_RSM_USERNAME --pass $KB_RSM_PASSWORD --no-auth-warning info | grep role | awk -F ':' '{print $2}' | tr '[:upper:]' '[:lower:]' | tr -d '\r' | tr -d '\n') &&", - "if [ \"master\" = \"$Role\" ]; then echo -n \"primary\"; else echo -n \"secondary\"; fi", - }, - }, - } + handler = common.RedisHandler + case "kafka": + handler = common.KafkaHandler + } + if handler != "" { + return &handler } return nil } diff --git a/pkg/controller/rsm/suite_test.go b/pkg/controller/rsm/suite_test.go index dbe2c34690a..150fecd7dc3 100644 --- a/pkg/controller/rsm/suite_test.go +++ b/pkg/controller/rsm/suite_test.go @@ -100,7 +100,7 @@ var ( } roleProbe = &workloads.RoleProbe{ - ProbeActions: []workloads.Action{{Command: []string{"cmd"}}}, + CustomHandler: []workloads.Action{{Command: []string{"cmd"}}}, } reconfiguration = workloads.MembershipReconfiguration{ diff --git a/pkg/controller/rsm/transformer_object_generation.go b/pkg/controller/rsm/transformer_object_generation.go index f0173c4d5c6..10abd16efdf 100644 --- a/pkg/controller/rsm/transformer_object_generation.go +++ b/pkg/controller/rsm/transformer_object_generation.go @@ -351,53 +351,63 @@ func injectRoleProbeContainer(rsm workloads.ReplicatedStateMachine, template *co ValueFrom: credential.Password.ValueFrom, }) } - allUsedPorts := findAllUsedPorts(template) - svcPort := actionSvcPortBase - var actionSvcPorts []int32 - for range roleProbe.ProbeActions { - svcPort = findNextAvailablePort(svcPort, allUsedPorts) - actionSvcPorts = append(actionSvcPorts, svcPort) - } - injectProbeActionContainer(rsm, template, actionSvcPorts, credentialEnv) + + actionSvcPorts := buildActionSvcPorts(template, roleProbe.CustomHandler) + actionSvcList, _ := json.Marshal(actionSvcPorts) - injectRoleProbeAgentContainer(rsm, template, string(actionSvcList), credentialEnv) + injectRoleProbeBaseContainer(rsm, template, string(actionSvcList), credentialEnv) + + if roleProbe.CustomHandler != nil { + injectCustomRoleProbeContainer(rsm, template, actionSvcPorts, credentialEnv) + } } -func findNextAvailablePort(base int32, allUsedPorts []int32) int32 { - for port := base + 1; port < 65535; port++ { - available := true - for _, usedPort := range allUsedPorts { - if port == usedPort { - available = false - break +func buildActionSvcPorts(template *corev1.PodTemplateSpec, actions []workloads.Action) []int32 { + findAllUsedPorts := func() []int32 { + allUsedPorts := make([]int32, 0) + for _, container := range template.Spec.Containers { + for _, port := range container.Ports { + allUsedPorts = append(allUsedPorts, port.ContainerPort) + allUsedPorts = append(allUsedPorts, port.HostPort) } } - if available { - return port - } + return allUsedPorts } - return 0 -} -func findAllUsedPorts(template *corev1.PodTemplateSpec) []int32 { - allUsedPorts := make([]int32, 0) - for _, container := range template.Spec.Containers { - for _, port := range container.Ports { - allUsedPorts = append(allUsedPorts, port.ContainerPort) - allUsedPorts = append(allUsedPorts, port.HostPort) + findNextAvailablePort := func(base int32, allUsedPorts []int32) int32 { + for port := base + 1; port < 65535; port++ { + available := true + for _, usedPort := range allUsedPorts { + if port == usedPort { + available = false + break + } + } + if available { + return port + } } + return 0 + } + + allUsedPorts := findAllUsedPorts() + svcPort := actionSvcPortBase + var actionSvcPorts []int32 + for range actions { + svcPort = findNextAvailablePort(svcPort, allUsedPorts) + actionSvcPorts = append(actionSvcPorts, svcPort) } - return allUsedPorts + return actionSvcPorts } -func injectRoleProbeAgentContainer(rsm workloads.ReplicatedStateMachine, template *corev1.PodTemplateSpec, actionSvcList string, credentialEnv []corev1.EnvVar) { - // compute parameters for role probe agent container +func injectRoleProbeBaseContainer(rsm workloads.ReplicatedStateMachine, template *corev1.PodTemplateSpec, actionSvcList string, credentialEnv []corev1.EnvVar) { + // compute parameters for role probe base container roleProbe := rsm.Spec.RoleProbe if roleProbe == nil { return } credential := rsm.Spec.Credential - image := viper.GetString("ROLE_PROBE_AGENT_IMAGE") + image := viper.GetString("ROLE_PROBE_IMAGE") if len(image) == 0 { image = defaultRoleProbeAgentImage } @@ -494,28 +504,14 @@ func injectRoleProbeAgentContainer(rsm workloads.ReplicatedStateMachine, templat }, ) - getCharacterTypeAndWorkload := func(template *corev1.PodTemplateSpec) (characterType *corev1.EnvVar) { - for _, container := range template.Spec.Containers { - if !(len(container.Command) > 0 && container.Command[0] == "lorry") { - continue - } - envs := container.Env - for i, e := range envs { - if e.Name == kBEnvCharacterType { - characterType = &envs[i] - } - } - if characterType != nil { - break - } - } - return characterType - } - - characterType := getCharacterTypeAndWorkload(template) - if characterType != nil { - env = append(env, *characterType) + characterType := "custom" + if roleProbe.BuiltinHandler != nil { + characterType = *roleProbe.BuiltinHandler } + env = append(env, corev1.EnvVar{ + Name: constant.KBEnvCharacterType, + Value: characterType, + }) readinessProbe := &corev1.Probe{ ProbeHandler: corev1.ProbeHandler{ @@ -535,17 +531,9 @@ func injectRoleProbeAgentContainer(rsm workloads.ReplicatedStateMachine, templat tryToGetRoleProbeContainer := func() *corev1.Container { for i, container := range template.Spec.Containers { - if container.Image != image { - continue - } - if len(container.Command) == 0 || container.Command[0] != roleProbeBinaryName { - continue - } - if container.ReadinessProbe != nil { - continue + if container.Name == constant.RoleProbeContainerName { + return &template.Spec.Containers[i] } - // if all the above conditions satisfied, container that can do the role probe job found - return &template.Spec.Containers[i] } return nil } @@ -554,11 +542,12 @@ func injectRoleProbeAgentContainer(rsm workloads.ReplicatedStateMachine, templat if container := tryToGetRoleProbeContainer(); container != nil { // presume the second port is the grpc port. // this is an easily broken contract between rsm controller and cluster controller. - // TODO(free6om): design a better way to do this after Lorry-WeSyncer separation done + // TODO(free6om): design a better way to do this readinessProbe.Exec.Command = []string{ grpcHealthProbeBinaryPath, fmt.Sprintf(grpcHealthProbeArgsFormat, int(container.Ports[1].ContainerPort)), } + readinessProbe.HTTPGet = nil container.ReadinessProbe = readinessProbe for _, e := range env { if slices.IndexFunc(container.Env, func(v corev1.EnvVar) bool { @@ -601,7 +590,7 @@ func injectRoleProbeAgentContainer(rsm workloads.ReplicatedStateMachine, templat template.Spec.Containers = append(template.Spec.Containers, *container) } -func injectProbeActionContainer(rsm workloads.ReplicatedStateMachine, template *corev1.PodTemplateSpec, actionSvcPorts []int32, credentialEnv []corev1.EnvVar) { +func injectCustomRoleProbeContainer(rsm workloads.ReplicatedStateMachine, template *corev1.PodTemplateSpec, actionSvcPorts []int32, credentialEnv []corev1.EnvVar) { if rsm.Spec.RoleProbe == nil { return } @@ -635,7 +624,7 @@ func injectProbeActionContainer(rsm workloads.ReplicatedStateMachine, template * template.Spec.InitContainers = append(template.Spec.InitContainers, initContainer) // inject action containers based on utility images - for i, action := range rsm.Spec.RoleProbe.ProbeActions { + for i, action := range rsm.Spec.RoleProbe.CustomHandler { image := action.Image if len(image) == 0 { image = defaultActionImage diff --git a/pkg/controller/rsm/transformer_objection_generation_test.go b/pkg/controller/rsm/transformer_objection_generation_test.go index ba9d390f0ff..044bedf59e6 100644 --- a/pkg/controller/rsm/transformer_objection_generation_test.go +++ b/pkg/controller/rsm/transformer_objection_generation_test.go @@ -47,7 +47,7 @@ var _ = Describe("object generation transformer test.", func() { SetService(service). SetCredential(credential). SetTemplate(template). - SetProbeActions(observeActions). + SetCustomHandler(observeActions). GetObject() transCtx = &rsmTransformContext{ diff --git a/pkg/controller/rsm/types.go b/pkg/controller/rsm/types.go index 80f8a1b3f2e..dfe92d51983 100644 --- a/pkg/controller/rsm/types.go +++ b/pkg/controller/rsm/types.go @@ -94,8 +94,6 @@ const ( RoleUpdateMechanismVarName = "KB_RSM_ROLE_UPDATE_MECHANISM" roleProbeTimeoutVarName = "KB_RSM_ROLE_PROBE_TIMEOUT" directAPIServerEventFieldPath = "spec.containers{sqlchannel}" - kBEnvCharacterType = "KB_SERVICE_CHARACTER_TYPE" - kBEnvWorkloadType = "KB_WORKLOAD_TYPE" readinessProbeEventFieldPath = "spec.containers{" + roleProbeContainerName + "}" legacyEventFieldPath = "spec.containers{kb-checkrole}" checkRoleEventReason = "checkRole"