Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: refactor cmpd's role definition #8416

Open
wants to merge 28 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
56 changes: 14 additions & 42 deletions apis/apps/v1/componentdefinition_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,8 @@ import (
corev1 "k8s.io/api/core/v1"
rbacv1 "k8s.io/api/rbac/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

basev1 "github.com/apecloud/kubeblocks/apis/base/v1"
)

// +genclient
Expand Down Expand Up @@ -408,11 +410,16 @@ type ComponentDefinitionSpec struct {
// +optional
Available *ComponentAvailable `json:"available,omitempty"`

// FIXME: there's a bug in CEL's cost estimation when chaining .filter() and .map().
// It was fixed in k8s 1.30, see: https://github.com/kubernetes/kubernetes/pull/123562.
// Maybe we can add this back later.
// TODO +kubebuilder:validation:XValidation:rule="self.filter(x, x.participatesInQuorum == true).map(x, x.updatePriority).min() > self.filter(x, x.participatesInQuorum == false).map(x, x.updatePriority).max()",message="Roles participate in quorum should have higher update priority than roles do not participate in quorum."

// Enumerate all possible roles assigned to each replica of the Component, influencing its behavior.
//
// A replica can have zero to multiple roles.
// KubeBlocks operator determines the roles of each replica by invoking the `lifecycleActions.roleProbe` method.
// This action returns a list of roles for each replica, and the returned roles must be predefined in the `roles` field.
// A replica can have zero or one role.
// KubeBlocks operator determines the role of each replica by invoking the `lifecycleActions.roleProbe` method.
// This action returns the role for each replica, and the returned role must be predefined here.
//
// The roles assigned to a replica can influence various aspects of the Component's behavior, such as:
//
Expand All @@ -423,6 +430,7 @@ type ComponentDefinitionSpec struct {
//
// This field is immutable.
//
// +kubebuilder:validation:MaxItems=128
// +optional
Roles []ReplicaRole `json:"roles,omitempty"`

Expand Down Expand Up @@ -1370,45 +1378,9 @@ type ComponentAvailableProbeAssertion struct {
Strict *bool `json:"strict,omitempty"`
}

// ReplicaRole represents a role that can be assumed by a component instance.
type ReplicaRole struct {
// Defines the role's identifier. It is used to set the "apps.kubeblocks.io/role" label value
// on the corresponding object.
//
// This field is immutable once set.
//
// +kubebuilder:validation:Required
// +kubebuilder:validation:MaxLength=32
// +kubebuilder:validation:Pattern=`^.*[^\s]+.*$`
Name string `json:"name"`

// Indicates whether a replica assigned this role is capable of providing services.
//
// This field is immutable once set.
//
// +kubebuilder:default=false
// +optional
Serviceable bool `json:"serviceable,omitempty"`

// Determines if a replica in this role has the authority to perform write operations.
// A writable replica can modify data, handle update operations.
//
// This field is immutable once set.
//
// +kubebuilder:default=false
// +optional
Writable bool `json:"writable,omitempty"`

// Specifies whether a replica with this role has voting rights.
// In distributed systems, this typically means the replica can participate in consensus decisions,
// configuration changes, or other processes that require a quorum.
//
// This field is immutable once set.
//
// +kubebuilder:default=false
// +optional
Votable bool `json:"votable,omitempty"`
}
// ReplicaRole represents a role that can be assigned to a component instance, defining its behavior and responsibilities.
// +kubebuilder:object:generate=false
type ReplicaRole = basev1.ReplicaRole

// UpdateStrategy defines the update strategy for cluster components. This strategy determines how updates are applied
// across the cluster.
Expand Down
18 changes: 2 additions & 16 deletions apis/apps/v1/zz_generated.deepcopy.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

39 changes: 39 additions & 0 deletions apis/base/v1/groupversion_info.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
/*
Copyright (C) 2022-2024 ApeCloud Co., Ltd

This file is part of KubeBlocks project

This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.

This program is distributed in the hope that it will be useful
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.

You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/

// Package v1 contains API Schema definitions for the base v1 API group
// +kubebuilder:object:generate=true
// +groupName=base.kubeblocks.io
package v1

import (
"k8s.io/apimachinery/pkg/runtime/schema"
"sigs.k8s.io/controller-runtime/pkg/scheme"
)

var (
// GroupVersion is group version used to register these objects
GroupVersion = schema.GroupVersion{Group: "base.kubeblocks.io", Version: "v1"}

// SchemeBuilder is used to add go types to the GroupVersionKind scheme
SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion}

// AddToScheme adds the types in this group-version to the given scheme.
AddToScheme = SchemeBuilder.AddToScheme
)
83 changes: 83 additions & 0 deletions apis/base/v1/types.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
/*
Copyright (C) 2022-2024 ApeCloud Co., Ltd

This file is part of KubeBlocks project

This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.

This program is distributed in the hope that it will be useful
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.

You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/

package v1

// ReplicaRole represents a role that can be assigned to a component instance, defining its behavior and responsibilities.
type ReplicaRole struct {
// Name defines the role's unique identifier. This value is used to set the "apps.kubeblocks.io/role" label
// on the corresponding object to identify its role.
//
// For example, common role names include:
// - "leader": The primary/master instance that handles write operations
// - "follower": Secondary/replica instances that replicate data from the leader
// - "learner": Read-only instances that don't participate in elections
//
// This field is immutable once set.
//
// +kubebuilder:validation:Required
// +kubebuilder:validation:MaxLength=32
// +kubebuilder:validation:Pattern=`^.*[^\s]+.*$`
Name string `json:"name"`

// UpdatePriority determines the order in which pods with different roles are updated.
// Pods are sorted by this priority (higher numbers = higher priority) and updated accordingly.
// Roles with the highest priority will be updated last.
// The default priority is 0.
//
// For example:
// - Leader role may have priority 2 (updated last)
// - Follower role may have priority 1 (updated before leader)
// - Learner role may have priority 0 (updated first)
//
// This field is immutable once set.
//
// +kubebuilder:default=0
// +optional
UpdatePriority int `json:"updatePriority"`
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

maybe MaintenanceOrder


// ParticipatesInQuorum indicates if pods with this role are counted when determining quorum.
// This affects update strategies that need to maintain quorum for availability. Roles participate
// in quorum should have higher update priority than roles do not participate in quorum.
// The default value is false.
//
// For example, in a 5-pod component where:
// - 2 learner pods (participatesInQuorum=false)
// - 2 follower pods (participatesInQuorum=true)
// - 1 leader pod (participatesInQuorum=true)
// The quorum size would be 3 (based on the 3 participating pods), allowing parallel updates
// of 2 learners and 1 follower while maintaining quorum.
//
// This field is immutable once set.
//
// +kubebuilder:default=false
// +optional
ParticipatesInQuorum bool `json:"participatesInQuorum"`
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

maybe simplify to InQuorum


// SwitchoverBeforeUpdate indicates if a role switchover operation should be performed before
// updating or scaling in pods with this role. This is typically used for leader roles to
// ensure minimal disruption during updates.
// The default value is false.
//
// This field is immutable once set.
//
// +kubebuilder:default=false
// +optional
SwitchoverBeforeUpdate bool `json:"switchoverBeforeUpdate"`
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

maybe SwitchoverBeforeMaintenance or NeedSwitchover

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

How about remove this option and decide within the switchover action whether a switchover is necessary based on the role of target pod?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What you mean is to use lifecycleaction's TargetPodSelector and MatchingKey to select the role to perform switchover?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If so, then this part of code can be removed. And there's only one place that needs role definition: instanceset controller uses roles to determine pod update sequence.

}
41 changes: 41 additions & 0 deletions apis/base/v1/zz_generated.deepcopy.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

5 changes: 3 additions & 2 deletions apis/operations/v1alpha1/opsrequest_validation.go
Original file line number Diff line number Diff line change
Expand Up @@ -799,9 +799,10 @@ func validateSwitchoverResourceList(ctx context.Context, cli client.Client, clus
return targetRole, errors.New("component has no roles definition, does not support switchover")
}
for _, role := range roles {
if role.Serviceable && role.Writable {
// FIXME: the assumption that only one role supports switchover may change in the future
if role.SwitchoverBeforeUpdate {
if targetRole != "" {
return targetRole, errors.New("componentDefinition has more than role is serviceable and writable, does not support switchover")
return targetRole, errors.New("componentDefinition has more than role that needs switchover before, does not support switchover")
}
targetRole = role.Name
}
Expand Down
37 changes: 5 additions & 32 deletions apis/workloads/v1/instanceset_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@ import (
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/intstr"

basev1 "github.com/apecloud/kubeblocks/apis/base/v1"
)

// +genclient
Expand Down Expand Up @@ -289,11 +291,6 @@ type InstanceSetStatus struct {
// +optional
MembersStatus []MemberStatus `json:"membersStatus,omitempty"`

// Indicates whether it is required for the InstanceSet to have at least one primary instance ready.
//
// +optional
ReadyWithoutPrimary bool `json:"readyWithoutPrimary,omitempty"`

// currentRevisions, if not empty, indicates the old version of the InstanceSet used to generate the underlying workload.
// key is the pod name, value is the revision.
//
Expand Down Expand Up @@ -468,33 +465,9 @@ const (
PreferInPlacePodUpdatePolicyType PodUpdatePolicyType = "PreferInPlace"
)

type ReplicaRole struct {

// Defines the role name of the replica.
//
// +kubebuilder:validation:Required
// +kubebuilder:default=leader
Name string `json:"name"`

// Specifies the service capabilities of this member.
//
// +kubebuilder:validation:Required
// +kubebuilder:default=ReadWrite
// +kubebuilder:validation:Enum={None, Readonly, ReadWrite}
AccessMode AccessMode `json:"accessMode"`

// Indicates if this member has voting rights.
//
// +kubebuilder:default=true
// +optional
CanVote bool `json:"canVote"`

// Determines if this member is the leader.
//
// +kubebuilder:default=false
// +optional
IsLeader bool `json:"isLeader"`
}
// ReplicaRole represents a role that can be assigned to a component instance, defining its behavior and responsibilities.
// +kubebuilder:object:generate=false
type ReplicaRole = basev1.ReplicaRole

// AccessMode defines SVC access mode enums.
// +enum
Expand Down
Loading
Loading