From 04e87ceb0c93701eb5e9d3d1b8fd49aaf9d4d659 Mon Sep 17 00:00:00 2001 From: Ari Ekmekji Date: Tue, 16 Jul 2024 12:35:13 -0700 Subject: [PATCH] Add generation to all components statuses --- docs/api-references/docs.md | 90 ++++++++++++++ manifests/crd.yaml | 33 ++++- manifests/crd/v1/pingcap.com_dmclusters.yaml | 6 + .../crd/v1/pingcap.com_tidbclusters.yaml | 27 +++- pkg/apis/pingcap/v1alpha1/component_status.go | 32 +++++ pkg/apis/pingcap/v1alpha1/types.go | 117 ++++++++++-------- .../member/dm_master_member_manager.go | 2 + pkg/manager/member/dm_master_upgrader.go | 2 + .../member/dm_worker_member_manager.go | 1 + pkg/manager/member/pd_member_manager.go | 3 + pkg/manager/member/pd_member_manager_test.go | 7 +- pkg/manager/member/pd_ms_member_manager.go | 2 + pkg/manager/member/pd_upgrader.go | 2 + pkg/manager/member/pump_member_manager.go | 1 + pkg/manager/member/ticdc_member_manager.go | 1 + pkg/manager/member/ticdc_upgrader.go | 1 + pkg/manager/member/tidb_member_manager.go | 2 + .../member/tidb_member_manager_test.go | 5 + pkg/manager/member/tidb_upgrader.go | 2 + pkg/manager/member/tiflash_member_manager.go | 1 + pkg/manager/member/tiflash_upgrader.go | 1 + pkg/manager/member/tikv_member_manager.go | 2 + pkg/manager/member/tikv_upgrader.go | 3 + pkg/manager/member/tiproxy_member_manager.go | 1 + pkg/manager/member/tiproxy_upgrader.go | 1 + pkg/manager/suspender/suspender.go | 2 + 26 files changed, 289 insertions(+), 58 deletions(-) diff --git a/docs/api-references/docs.md b/docs/api-references/docs.md index f1cad33920..a2cbcdbaed 100644 --- a/docs/api-references/docs.md +++ b/docs/api-references/docs.md @@ -9592,6 +9592,16 @@ MemberPhase +observedGeneration
+ +int64 + + + + + + + statefulSet
@@ -11174,6 +11184,16 @@ MemberPhase +observedGeneration
+ +int64 + + + + + + + statefulSet
 @@ -12389,6 +12409,16 @@ MemberPhase +observedGeneration
+ +int64 + + + + + + + statefulSet
 @@ -13812,6 +13842,16 @@ MemberPhase +observedGeneration
+ +int64 + + + + + + + statefulSet
 @@ -16630,6 +16670,16 @@ MemberPhase +observedGeneration
+ +int64 + + + + + + + statefulSet
 @@ -18012,6 +18062,16 @@ MemberPhase +observedGeneration
+ +int64 + + + + + + + statefulSet
 @@ -22725,6 +22785,16 @@ MemberPhase +observedGeneration
+ +int64 + + + + + + + bootStrapped
bool @@ -23738,6 +23808,16 @@ MemberPhase +observedGeneration
+ +int64 + + + + + + + members
 @@ -27283,6 +27363,16 @@ MemberPhase +observedGeneration
+ +int64 + + + + + + + statefulSet
 diff --git a/manifests/crd.yaml b/manifests/crd.yaml index 4bad64b956..04402d1697 100644 --- a/manifests/crd.yaml +++ b/manifests/crd.yaml @@ -15484,6 +15484,9 @@ spec: - name type: object type: object + observedGeneration: + format: int64 + type: integer phase: type: string statefulSet: @@ -15665,6 +15668,9 @@ spec: - stage type: object type: object + observedGeneration: + format: int64 + type: integer phase: type: string statefulSet: @@ -44077,6 +44083,9 @@ spec: - name type: object type: object + observedGeneration: + format: int64 + type: integer peerMembers: additionalProperties: properties: @@ -44259,6 +44268,9 @@ spec: type: array name: type: string + observedGeneration: + format: int64 + type: integer phase: type: string statefulSet: @@ -44407,6 +44419,9 @@ spec: - state type: object type: array + observedGeneration: + format: int64 + type: integer phase: type: string statefulSet: @@ -44552,6 +44567,9 @@ spec: type: object nullable: true type: array + observedGeneration: + format: int64 + type: integer phase: type: string statefulSet: @@ -44715,6 +44733,9 @@ spec: - name type: object type: object + observedGeneration: + format: int64 + type: integer passwordInitialized: type: boolean phase: @@ -44877,6 +44898,9 @@ spec: type: object image: type: string + observedGeneration: + format: int64 + type: integer peerStores: additionalProperties: properties: @@ -45136,6 +45160,9 @@ spec: type: object image: type: string + observedGeneration: + format: int64 + type: integer peerStores: additionalProperties: properties: @@ -45375,6 +45402,9 @@ spec: - name type: object type: object + observedGeneration: + format: int64 + type: integer phase: type: string statefulSet: @@ -45475,7 +45505,8 @@ spec: type: object served: true storage: true - subresources: {} + subresources: + status: {} status: acceptedNames: kind: "" diff --git a/manifests/crd/v1/pingcap.com_dmclusters.yaml b/manifests/crd/v1/pingcap.com_dmclusters.yaml index 0982133f0d..ec5b41dca4 100644 --- a/manifests/crd/v1/pingcap.com_dmclusters.yaml +++ b/manifests/crd/v1/pingcap.com_dmclusters.yaml @@ -8635,6 +8635,9 @@ spec: - name type: object type: object + observedGeneration: + format: int64 + type: integer phase: type: string statefulSet: @@ -8816,6 +8819,9 @@ spec: - stage type: object type: object + observedGeneration: + format: int64 + type: integer phase: type: string statefulSet: diff --git a/manifests/crd/v1/pingcap.com_tidbclusters.yaml b/manifests/crd/v1/pingcap.com_tidbclusters.yaml index adf7a3551c..ac6ecc38b9 100644 --- a/manifests/crd/v1/pingcap.com_tidbclusters.yaml +++ b/manifests/crd/v1/pingcap.com_tidbclusters.yaml @@ -25023,6 +25023,9 @@ spec: - name type: object type: object + observedGeneration: + format: int64 + type: integer peerMembers: additionalProperties: properties: @@ -25205,6 +25208,9 @@ spec: type: array name: type: string + observedGeneration: + format: int64 + type: integer phase: type: string statefulSet: @@ -25353,6 +25359,9 @@ spec: - state type: object type: array + observedGeneration: + format: int64 + type: integer phase: type: string statefulSet: @@ -25498,6 +25507,9 @@ spec: type: object nullable: true type: array + observedGeneration: + format: int64 + type: integer phase: type: string statefulSet: @@ -25661,6 +25673,9 @@ spec: - name type: object type: object + observedGeneration: + format: int64 + type: integer passwordInitialized: type: boolean phase: @@ -25823,6 +25838,9 @@ spec: type: object image: type: string + observedGeneration: + format: int64 + type: integer peerStores: additionalProperties: properties: @@ -26082,6 +26100,9 @@ spec: type: object image: type: string + observedGeneration: + format: int64 + type: integer peerStores: additionalProperties: properties: @@ -26321,6 +26342,9 @@ spec: - name type: object type: object + observedGeneration: + format: int64 + type: integer phase: type: string statefulSet: @@ -26421,7 +26445,8 @@ spec: type: object served: true storage: true - subresources: {} + subresources: + status: {} status: acceptedNames: kind: "" diff --git a/pkg/apis/pingcap/v1alpha1/component_status.go b/pkg/apis/pingcap/v1alpha1/component_status.go index 1d1cbc9492..55877a1b23 100644 --- a/pkg/apis/pingcap/v1alpha1/component_status.go +++ b/pkg/apis/pingcap/v1alpha1/component_status.go @@ -58,6 +58,8 @@ type ComponentStatus interface { // // Not supported for tidb and pump SetSynced(bool) + // SetObservedGeneration sets the generation of the component. + SetObservedGeneration(generation int64) // SetPhase sets the phase of the component. SetPhase(phase MemberPhase) // SetVolumes sets the `status.volumes` @@ -173,6 +175,9 @@ func (s *PDStatus) RemoveCondition(conditionType string) { meta.RemoveStatusCondition(&conditions, conditionType) s.Conditions = conditions } +func (s *PDStatus) SetObservedGeneration(generation int64) { + s.ObservedGeneration = generation +} func (s *PDStatus) SetPhase(phase MemberPhase) { s.Phase = phase } @@ -223,6 +228,9 @@ func (s *PDMSStatus) RemoveCondition(conditionType string) { meta.RemoveStatusCondition(&conditions, conditionType) s.Conditions = conditions } +func (s *PDMSStatus) SetObservedGeneration(generation int64) { + s.ObservedGeneration = generation +} func (s *PDMSStatus) SetPhase(phase MemberPhase) { s.Phase = phase } @@ -271,6 +279,9 @@ func (s *TiKVStatus) RemoveCondition(conditionType string) { meta.RemoveStatusCondition(&conditions, conditionType) s.Conditions = conditions } +func (s *TiKVStatus) SetObservedGeneration(generation int64) { + s.ObservedGeneration = generation +} func (s *TiKVStatus) SetPhase(phase MemberPhase) { s.Phase = phase } @@ -322,6 +333,9 @@ func (s *TiDBStatus) RemoveCondition(conditionType string) { meta.RemoveStatusCondition(&conditions, conditionType) s.Conditions = conditions } +func (s *TiDBStatus) SetObservedGeneration(generation int64) { + s.ObservedGeneration = generation +} func (s *TiDBStatus) SetPhase(phase MemberPhase) { s.Phase = phase } @@ -373,6 +387,9 @@ func (s *PumpStatus) RemoveCondition(conditionType string) { meta.RemoveStatusCondition(&conditions, conditionType) s.Conditions = conditions } +func (s *PumpStatus) SetObservedGeneration(generation int64) { + s.ObservedGeneration = generation +} func (s *PumpStatus) SetPhase(phase MemberPhase) { s.Phase = phase } @@ -424,6 +441,9 @@ func (s *TiFlashStatus) RemoveCondition(conditionType string) { meta.RemoveStatusCondition(&conditions, conditionType) s.Conditions = conditions } +func (s *TiFlashStatus) SetObservedGeneration(generation int64) { + s.ObservedGeneration = generation +} func (s *TiFlashStatus) SetPhase(phase MemberPhase) { s.Phase = phase } @@ -475,6 +495,9 @@ func (s *TiCDCStatus) RemoveCondition(conditionType string) { meta.RemoveStatusCondition(&conditions, conditionType) s.Conditions = conditions } +func (s *TiCDCStatus) SetObservedGeneration(generation int64) { + s.ObservedGeneration = generation +} func (s *TiCDCStatus) SetPhase(phase MemberPhase) { s.Phase = phase } @@ -526,6 +549,9 @@ func (s *MasterStatus) RemoveCondition(conditionType string) { meta.RemoveStatusCondition(&conditions, conditionType) s.Conditions = conditions } +func (s *MasterStatus) SetObservedGeneration(generation int64) { + s.ObservedGeneration = generation +} func (s *MasterStatus) SetPhase(phase MemberPhase) { s.Phase = phase } @@ -577,6 +603,9 @@ func (s *WorkerStatus) RemoveCondition(conditionType string) { meta.RemoveStatusCondition(&conditions, conditionType) s.Conditions = conditions } +func (s *WorkerStatus) SetObservedGeneration(generation int64) { + s.ObservedGeneration = generation +} func (s *WorkerStatus) SetPhase(phase MemberPhase) { s.Phase = phase } @@ -628,6 +657,9 @@ func (s *TiProxyStatus) RemoveCondition(conditionType string) { meta.RemoveStatusCondition(&conditions, conditionType) s.Conditions = conditions } +func (s *TiProxyStatus) SetObservedGeneration(generation int64) { + s.ObservedGeneration = generation +} func (s *TiProxyStatus) SetPhase(phase MemberPhase) { s.Phase = phase } diff --git a/pkg/apis/pingcap/v1alpha1/types.go b/pkg/apis/pingcap/v1alpha1/types.go index f4063a5601..fb5a84b35d 100644 --- a/pkg/apis/pingcap/v1alpha1/types.go +++ b/pkg/apis/pingcap/v1alpha1/types.go @@ -168,6 +168,7 @@ const ( // // +k8s:openapi-gen=true // +kubebuilder:resource:shortName="tc" +// +kubebuilder:subresource:status // +kubebuilder:printcolumn:name="Ready",type=string,JSONPath=`.status.conditions[?(@.type=="Ready")].status` // +kubebuilder:printcolumn:name="PD",type=string,JSONPath=`.status.pd.image`,description="The image for PD cluster" // +kubebuilder:printcolumn:name="Storage",type=string,JSONPath=`.spec.pd.requests.storage`,description="The storage size specified for PD node" @@ -1470,9 +1471,10 @@ type SuspendAction struct { // PDStatus is PD status type PDStatus struct { // +optional - Synced bool `json:"synced"` - Phase MemberPhase `json:"phase,omitempty"` - StatefulSet *apps.StatefulSetStatus `json:"statefulSet,omitempty"` + Synced bool `json:"synced"` + Phase MemberPhase `json:"phase,omitempty"` + ObservedGeneration int64 `json:"observedGeneration,omitempty"` + StatefulSet *apps.StatefulSetStatus `json:"statefulSet,omitempty"` // Members contains PDs in current TidbCluster Members map[string]PDMember `json:"members,omitempty"` // PeerMembers contains PDs NOT in current TidbCluster @@ -1495,9 +1497,10 @@ type PDStatus struct { type PDMSStatus struct { Name string `json:"name,omitempty"` // +optional - Synced bool `json:"synced"` - Phase MemberPhase `json:"phase,omitempty"` - StatefulSet *apps.StatefulSetStatus `json:"statefulSet,omitempty"` + Synced bool `json:"synced"` + Phase MemberPhase `json:"phase,omitempty"` + ObservedGeneration int64 `json:"observedGeneration,omitempty"` + StatefulSet *apps.StatefulSetStatus `json:"statefulSet,omitempty"` // Volumes contains the status of all volumes. Volumes map[StorageVolumeName]*StorageVolumeStatus `json:"volumes,omitempty"` // Members contains other service in current TidbCluster @@ -1551,6 +1554,7 @@ type UnjoinedMember struct { // TiDBStatus is TiDB status type TiDBStatus struct { Phase MemberPhase `json:"phase,omitempty"` + ObservedGeneration int64 `json:"observedGeneration,omitempty"` StatefulSet *apps.StatefulSetStatus `json:"statefulSet,omitempty"` Members map[string]TiDBMember `json:"members,omitempty"` FailureMembers map[string]TiDBFailureMember `json:"failureMembers,omitempty"` @@ -1654,17 +1658,18 @@ const ( // TiKVStatus is TiKV status type TiKVStatus struct { - Synced bool `json:"synced,omitempty"` - Phase MemberPhase `json:"phase,omitempty"` - BootStrapped bool `json:"bootStrapped,omitempty"` - StatefulSet *apps.StatefulSetStatus `json:"statefulSet,omitempty"` - Stores map[string]TiKVStore `json:"stores,omitempty"` // key: store id - PeerStores map[string]TiKVStore `json:"peerStores,omitempty"` - TombstoneStores map[string]TiKVStore `json:"tombstoneStores,omitempty"` - FailureStores map[string]TiKVFailureStore `json:"failureStores,omitempty"` - FailoverUID types.UID `json:"failoverUID,omitempty"` - Image string `json:"image,omitempty"` - EvictLeader map[string]*EvictLeaderStatus `json:"evictLeader,omitempty"` + Synced bool `json:"synced,omitempty"` + Phase MemberPhase `json:"phase,omitempty"` + ObservedGeneration int64 `json:"observedGeneration,omitempty"` + BootStrapped bool `json:"bootStrapped,omitempty"` + StatefulSet *apps.StatefulSetStatus `json:"statefulSet,omitempty"` + Stores map[string]TiKVStore `json:"stores,omitempty"` // key: store id + PeerStores map[string]TiKVStore `json:"peerStores,omitempty"` + TombstoneStores map[string]TiKVStore `json:"tombstoneStores,omitempty"` + FailureStores map[string]TiKVFailureStore `json:"failureStores,omitempty"` + FailoverUID types.UID `json:"failoverUID,omitempty"` + Image string `json:"image,omitempty"` + EvictLeader map[string]*EvictLeaderStatus `json:"evictLeader,omitempty"` // Volumes contains the status of all volumes. Volumes map[StorageVolumeName]*StorageVolumeStatus `json:"volumes,omitempty"` // Represents the latest available observations of a component's state. @@ -1677,15 +1682,16 @@ type TiKVStatus struct { // TiFlashStatus is TiFlash status type TiFlashStatus struct { - Synced bool `json:"synced,omitempty"` - Phase MemberPhase `json:"phase,omitempty"` - StatefulSet *apps.StatefulSetStatus `json:"statefulSet,omitempty"` - Stores map[string]TiKVStore `json:"stores,omitempty"` - PeerStores map[string]TiKVStore `json:"peerStores,omitempty"` - TombstoneStores map[string]TiKVStore `json:"tombstoneStores,omitempty"` - FailureStores map[string]TiKVFailureStore `json:"failureStores,omitempty"` - FailoverUID types.UID `json:"failoverUID,omitempty"` - Image string `json:"image,omitempty"` + Synced bool `json:"synced,omitempty"` + Phase MemberPhase `json:"phase,omitempty"` + ObservedGeneration int64 `json:"observedGeneration,omitempty"` + StatefulSet *apps.StatefulSetStatus `json:"statefulSet,omitempty"` + Stores map[string]TiKVStore `json:"stores,omitempty"` + PeerStores map[string]TiKVStore `json:"peerStores,omitempty"` + TombstoneStores map[string]TiKVStore `json:"tombstoneStores,omitempty"` + FailureStores map[string]TiKVFailureStore `json:"failureStores,omitempty"` + FailoverUID types.UID `json:"failoverUID,omitempty"` + Image string `json:"image,omitempty"` // Volumes contains the status of all volumes. Volumes map[StorageVolumeName]*StorageVolumeStatus `json:"volumes,omitempty"` // Represents the latest available observations of a component's state. @@ -1709,11 +1715,12 @@ type TiProxyMember struct { // TiProxyStatus is TiProxy status type TiProxyStatus struct { - Synced bool `json:"synced,omitempty"` - Phase MemberPhase `json:"phase,omitempty"` - Members map[string]TiProxyMember `json:"members,omitempty"` - StatefulSet *apps.StatefulSetStatus `json:"statefulSet,omitempty"` - Volumes map[StorageVolumeName]*StorageVolumeStatus `json:"volumes,omitempty"` + Synced bool `json:"synced,omitempty"` + Phase MemberPhase `json:"phase,omitempty"` + ObservedGeneration int64 `json:"observedGeneration,omitempty"` + Members map[string]TiProxyMember `json:"members,omitempty"` + StatefulSet *apps.StatefulSetStatus `json:"statefulSet,omitempty"` + Volumes map[StorageVolumeName]*StorageVolumeStatus `json:"volumes,omitempty"` // Represents the latest available observations of a component's state. // +optional // +nullable @@ -1722,10 +1729,11 @@ type TiProxyStatus struct { // TiCDCStatus is TiCDC status type TiCDCStatus struct { - Synced bool `json:"synced,omitempty"` - Phase MemberPhase `json:"phase,omitempty"` - StatefulSet *apps.StatefulSetStatus `json:"statefulSet,omitempty"` - Captures map[string]TiCDCCapture `json:"captures,omitempty"` + Synced bool `json:"synced,omitempty"` + Phase MemberPhase `json:"phase,omitempty"` + ObservedGeneration int64 `json:"observedGeneration,omitempty"` + StatefulSet *apps.StatefulSetStatus `json:"statefulSet,omitempty"` + Captures map[string]TiCDCCapture `json:"captures,omitempty"` // Volumes contains the status of all volumes. Volumes map[StorageVolumeName]*StorageVolumeStatus `json:"volumes,omitempty"` // Represents the latest available observations of a component's state. @@ -1788,9 +1796,10 @@ type PumpNodeStatus struct { // PumpStatus is Pump status type PumpStatus struct { - Phase MemberPhase `json:"phase,omitempty"` - StatefulSet *apps.StatefulSetStatus `json:"statefulSet,omitempty"` - Members []*PumpNodeStatus `json:"members,omitempty"` + Phase MemberPhase `json:"phase,omitempty"` + ObservedGeneration int64 `json:"observedGeneration,omitempty"` + StatefulSet *apps.StatefulSetStatus `json:"statefulSet,omitempty"` + Members []*PumpNodeStatus `json:"members,omitempty"` // Volumes contains the status of all volumes. Volumes map[StorageVolumeName]*StorageVolumeStatus `json:"volumes,omitempty"` // Represents the latest available observations of a component's state. @@ -3171,14 +3180,15 @@ const ( // MasterStatus is dm-master status type MasterStatus struct { - Synced bool `json:"synced,omitempty"` - Phase MemberPhase `json:"phase,omitempty"` - StatefulSet *apps.StatefulSetStatus `json:"statefulSet,omitempty"` - Members map[string]MasterMember `json:"members,omitempty"` - Leader MasterMember `json:"leader,omitempty"` - FailureMembers map[string]MasterFailureMember `json:"failureMembers,omitempty"` - UnjoinedMembers map[string]UnjoinedMember `json:"unjoinedMembers,omitempty"` - Image string `json:"image,omitempty"` + Synced bool `json:"synced,omitempty"` + Phase MemberPhase `json:"phase,omitempty"` + ObservedGeneration int64 `json:"observedGeneration,omitempty"` + StatefulSet *apps.StatefulSetStatus `json:"statefulSet,omitempty"` + Members map[string]MasterMember `json:"members,omitempty"` + Leader MasterMember `json:"leader,omitempty"` + FailureMembers map[string]MasterFailureMember `json:"failureMembers,omitempty"` + UnjoinedMembers map[string]UnjoinedMember `json:"unjoinedMembers,omitempty"` + Image string `json:"image,omitempty"` // Volumes contains the status of all volumes. Volumes map[StorageVolumeName]*StorageVolumeStatus `json:"volumes,omitempty"` // Represents the latest available observations of a component's state. @@ -3212,13 +3222,14 @@ type MasterFailureMember struct { // WorkerStatus is dm-worker status type WorkerStatus struct { - Synced bool `json:"synced,omitempty"` - Phase MemberPhase `json:"phase,omitempty"` - StatefulSet *apps.StatefulSetStatus `json:"statefulSet,omitempty"` - Members map[string]WorkerMember `json:"members,omitempty"` - FailureMembers map[string]WorkerFailureMember `json:"failureMembers,omitempty"` - FailoverUID types.UID `json:"failoverUID,omitempty"` - Image string `json:"image,omitempty"` + Synced bool `json:"synced,omitempty"` + Phase MemberPhase `json:"phase,omitempty"` + ObservedGeneration int64 `json:"observedGeneration,omitempty"` + StatefulSet *apps.StatefulSetStatus `json:"statefulSet,omitempty"` + Members map[string]WorkerMember `json:"members,omitempty"` + FailureMembers map[string]WorkerFailureMember `json:"failureMembers,omitempty"` + FailoverUID types.UID `json:"failoverUID,omitempty"` + Image string `json:"image,omitempty"` // Volumes contains the status of all volumes. Volumes map[StorageVolumeName]*StorageVolumeStatus `json:"volumes,omitempty"` // Represents the latest available observations of a component's state. diff --git a/pkg/manager/member/dm_master_member_manager.go b/pkg/manager/member/dm_master_member_manager.go index d881deb45a..645aaa1855 100644 --- a/pkg/manager/member/dm_master_member_manager.go +++ b/pkg/manager/member/dm_master_member_manager.go @@ -212,6 +212,7 @@ func (m *masterMemberManager) syncMasterStatefulSetForDMCluster(dc *v1alpha1.DMC // Force update takes precedence over scaling because force upgrade won't take effect when cluster gets stuck at scaling if !dc.Status.Master.Synced && NeedForceUpgrade(dc.Annotations) { + dc.Status.Master.ObservedGeneration = dc.Generation dc.Status.Master.Phase = v1alpha1.UpgradePhase mngerutils.SetUpgradePartition(newMasterSet, 0) errSTS := mngerutils.UpdateStatefulSet(m.deps.StatefulSetControl, dc, newMasterSet, oldMasterSet) @@ -292,6 +293,7 @@ func (m *masterMemberManager) syncDMClusterStatus(dc *v1alpha1.DMCluster, set *a return err } + dc.Status.Master.ObservedGeneration = dc.Generation // Scaling takes precedence over upgrading. if dc.MasterStsDesiredReplicas() != *set.Spec.Replicas { dc.Status.Master.Phase = v1alpha1.ScalePhase diff --git a/pkg/manager/member/dm_master_upgrader.go b/pkg/manager/member/dm_master_upgrader.go index 6be5544aa4..8fd11f6f77 100644 --- a/pkg/manager/member/dm_master_upgrader.go +++ b/pkg/manager/member/dm_master_upgrader.go @@ -57,6 +57,7 @@ func (u *masterUpgrader) gracefulUpgrade(dc *v1alpha1.DMCluster, oldSet *apps.St return nil } + dc.Status.Master.ObservedGeneration = dc.Generation dc.Status.Master.Phase = v1alpha1.UpgradePhase if !templateEqual(newSet, oldSet) { return nil @@ -136,6 +137,7 @@ func (u *fakeMasterUpgrader) Upgrade(dc *v1alpha1.DMCluster, _ *apps.StatefulSet if !dc.Status.Master.Synced { return fmt.Errorf("dmcluster: dm-master status sync failed,can not to be upgraded") } + dc.Status.Master.ObservedGeneration = dc.Generation dc.Status.Master.Phase = v1alpha1.UpgradePhase return nil } diff --git a/pkg/manager/member/dm_worker_member_manager.go b/pkg/manager/member/dm_worker_member_manager.go index 221cc7ec32..54d5ae8676 100644 --- a/pkg/manager/member/dm_worker_member_manager.go +++ b/pkg/manager/member/dm_worker_member_manager.go @@ -250,6 +250,7 @@ func (m *workerMemberManager) syncDMClusterStatus(dc *v1alpha1.DMCluster, set *a if err != nil { return err } + dc.Status.Worker.ObservedGeneration = dc.Generation if upgrading { dc.Status.Worker.Phase = v1alpha1.UpgradePhase } else if dc.WorkerStsDesiredReplicas() != *set.Spec.Replicas { diff --git a/pkg/manager/member/pd_member_manager.go b/pkg/manager/member/pd_member_manager.go index f4d14d9fe6..782dbe1663 100644 --- a/pkg/manager/member/pd_member_manager.go +++ b/pkg/manager/member/pd_member_manager.go @@ -233,6 +233,7 @@ func (m *pdMemberManager) syncPDStatefulSetForTidbCluster(tc *v1alpha1.TidbClust onlyOnePD := *oldPDSet.Spec.Replicas < 2 && len(tc.Status.PD.PeerMembers) == 0 // it's acceptable to use old record about peer members if forceUpgradeAnnoSet || onlyOnePD { + tc.Status.PD.ObservedGeneration = tc.Generation tc.Status.PD.Phase = v1alpha1.UpgradePhase mngerutils.SetUpgradePartition(newPDSet, 0) errSTS := mngerutils.UpdateStatefulSet(m.deps.StatefulSetControl, tc, newPDSet, oldPDSet) @@ -330,6 +331,8 @@ func (m *pdMemberManager) syncTidbClusterStatus(tc *v1alpha1.TidbCluster, set *a return err } + tc.Status.PD.ObservedGeneration = tc.Generation + // Scaling takes precedence over upgrading. if tc.PDStsDesiredReplicas() != *set.Spec.Replicas { tc.Status.PD.Phase = v1alpha1.ScalePhase diff --git a/pkg/manager/member/pd_member_manager_test.go b/pkg/manager/member/pd_member_manager_test.go index 149b1b92d4..a05c04a1d0 100644 --- a/pkg/manager/member/pd_member_manager_test.go +++ b/pkg/manager/member/pd_member_manager_test.go @@ -949,9 +949,10 @@ func newTidbClusterForPD() *v1alpha1.TidbCluster { APIVersion: "pingcap.com/v1alpha1", }, ObjectMeta: metav1.ObjectMeta{ - Name: "test", - Namespace: corev1.NamespaceDefault, - UID: types.UID("test"), + Name: "test", + Namespace: corev1.NamespaceDefault, + UID: types.UID("test"), + Generation: 10, }, Spec: v1alpha1.TidbClusterSpec{ PD: &v1alpha1.PDSpec{ diff --git a/pkg/manager/member/pd_ms_member_manager.go b/pkg/manager/member/pd_ms_member_manager.go index 7ad6189cbd..3ac6789e4c 100644 --- a/pkg/manager/member/pd_ms_member_manager.go +++ b/pkg/manager/member/pd_ms_member_manager.go @@ -345,6 +345,8 @@ func (m *pdMSMemberManager) syncStatus(tc *v1alpha1.TidbCluster, sts *apps.State return err } + tc.Status.PDMS[curService].ObservedGeneration = tc.Generation + // Scaling takes precedence over upgrading. if tc.PDMSStsDesiredReplicas(curService) != *sts.Spec.Replicas { tc.Status.PDMS[curService].Phase = v1alpha1.ScalePhase diff --git a/pkg/manager/member/pd_upgrader.go b/pkg/manager/member/pd_upgrader.go index a7327f2505..9b069256f1 100644 --- a/pkg/manager/member/pd_upgrader.go +++ b/pkg/manager/member/pd_upgrader.go @@ -64,6 +64,7 @@ func (u *pdUpgrader) gracefulUpgrade(tc *v1alpha1.TidbCluster, oldSet *apps.Stat return nil } + tc.Status.PD.ObservedGeneration = tc.Generation tc.Status.PD.Phase = v1alpha1.UpgradePhase if !templateEqual(newSet, oldSet) { return nil @@ -234,6 +235,7 @@ func (u *fakePDUpgrader) Upgrade(tc *v1alpha1.TidbCluster, _ *apps.StatefulSet, if !tc.Status.PD.Synced { return fmt.Errorf("tidbcluster: pd status sync failed, can not to be upgraded") } + tc.Status.PD.ObservedGeneration = tc.Generation tc.Status.PD.Phase = v1alpha1.UpgradePhase return nil } diff --git a/pkg/manager/member/pump_member_manager.go b/pkg/manager/member/pump_member_manager.go index abe686afc0..12885d8cf8 100644 --- a/pkg/manager/member/pump_member_manager.go +++ b/pkg/manager/member/pump_member_manager.go @@ -200,6 +200,7 @@ func (m *pumpMemberManager) syncTiDBClusterStatus(tc *v1alpha1.TidbCluster, set return err } + tc.Status.Pump.ObservedGeneration = tc.Generation if upgrading { tc.Status.Pump.Phase = v1alpha1.UpgradePhase } else { diff --git a/pkg/manager/member/ticdc_member_manager.go b/pkg/manager/member/ticdc_member_manager.go index 3c29259fed..7f1fdb046f 100644 --- a/pkg/manager/member/ticdc_member_manager.go +++ b/pkg/manager/member/ticdc_member_manager.go @@ -242,6 +242,7 @@ func (m *ticdcMemberManager) syncTiCDCStatus(tc *v1alpha1.TidbCluster, sts *apps tc.Status.TiCDC.Synced = false return err } + tc.Status.TiCDC.ObservedGeneration = tc.Generation if upgrading { tc.Status.TiCDC.Phase = v1alpha1.UpgradePhase } else { diff --git a/pkg/manager/member/ticdc_upgrader.go b/pkg/manager/member/ticdc_upgrader.go index 3515792568..fdb3925ceb 100644 --- a/pkg/manager/member/ticdc_upgrader.go +++ b/pkg/manager/member/ticdc_upgrader.go @@ -66,6 +66,7 @@ func (u *ticdcUpgrader) Upgrade(tc *v1alpha1.TidbCluster, oldSet *apps.StatefulS return nil } + tc.Status.TiCDC.ObservedGeneration = tc.Generation tc.Status.TiCDC.Phase = v1alpha1.UpgradePhase if !templateEqual(newSet, oldSet) { return nil diff --git a/pkg/manager/member/tidb_member_manager.go b/pkg/manager/member/tidb_member_manager.go index 5780fc78e4..cfcecaacc8 100644 --- a/pkg/manager/member/tidb_member_manager.go +++ b/pkg/manager/member/tidb_member_manager.go @@ -1147,6 +1147,8 @@ func (m *tidbMemberManager) syncTidbClusterStatus(tc *v1alpha1.TidbCluster, set return err } + tc.Status.TiDB.ObservedGeneration = tc.Generation + if tc.TiDBStsDesiredReplicas() != *set.Spec.Replicas { tc.Status.TiDB.Phase = v1alpha1.ScalePhase } else if upgrading && tc.Status.TiKV.Phase != v1alpha1.UpgradePhase && diff --git a/pkg/manager/member/tidb_member_manager_test.go b/pkg/manager/member/tidb_member_manager_test.go index 316071bc2e..2ec7dfd0ef 100644 --- a/pkg/manager/member/tidb_member_manager_test.go +++ b/pkg/manager/member/tidb_member_manager_test.go @@ -404,6 +404,7 @@ func TestTiDBMemberManagerSyncTidbClusterStatus(t *testing.T) { } if test.updateSts != nil { test.updateSts(set) + tc.Generation = tc.Generation + 1 } pmm, _, tidbControl, _ := newFakeTiDBMemberManager() @@ -449,6 +450,7 @@ func TestTiDBMemberManagerSyncTidbClusterStatus(t *testing.T) { tcExpectFn: func(g *GomegaWithT, tc *v1alpha1.TidbCluster) { g.Expect(tc.Status.TiDB.StatefulSet.Replicas).To(Equal(int32(3))) g.Expect(tc.Status.TiDB.Phase).To(Equal(v1alpha1.UpgradePhase)) + g.Expect(tc.Status.TiDB.ObservedGeneration).To(Equal(tc.Generation)) }, }, { @@ -464,6 +466,7 @@ func TestTiDBMemberManagerSyncTidbClusterStatus(t *testing.T) { tcExpectFn: func(g *GomegaWithT, tc *v1alpha1.TidbCluster) { g.Expect(tc.Status.TiDB.StatefulSet.Replicas).To(Equal(int32(3))) g.Expect(tc.Status.TiDB.Phase).To(Equal(v1alpha1.NormalPhase)) + g.Expect(tc.Status.TiDB.ObservedGeneration).To(Equal(tc.Generation)) }, }, { @@ -479,6 +482,7 @@ func TestTiDBMemberManagerSyncTidbClusterStatus(t *testing.T) { tcExpectFn: func(g *GomegaWithT, tc *v1alpha1.TidbCluster) { g.Expect(tc.Status.TiDB.StatefulSet.Replicas).To(Equal(int32(3))) g.Expect(tc.Status.TiDB.Phase).To(Equal(v1alpha1.NormalPhase)) + g.Expect(tc.Status.TiDB.ObservedGeneration).To(Equal(tc.Generation)) }, }, { @@ -492,6 +496,7 @@ func TestTiDBMemberManagerSyncTidbClusterStatus(t *testing.T) { tcExpectFn: func(g *GomegaWithT, tc *v1alpha1.TidbCluster) { g.Expect(tc.Status.TiDB.StatefulSet.Replicas).To(Equal(int32(3))) g.Expect(tc.Status.TiDB.Phase).To(Equal(v1alpha1.NormalPhase)) + g.Expect(tc.Status.TiDB.ObservedGeneration).To(Equal(tc.Generation)) }, }, { diff --git a/pkg/manager/member/tidb_upgrader.go b/pkg/manager/member/tidb_upgrader.go index 685b6f3cb4..991dcd69a7 100644 --- a/pkg/manager/member/tidb_upgrader.go +++ b/pkg/manager/member/tidb_upgrader.go @@ -75,6 +75,7 @@ func (u *tidbUpgrader) Upgrade(tc *v1alpha1.TidbCluster, oldSet *apps.StatefulSe return nil } + tc.Status.TiDB.ObservedGeneration = tc.Generation tc.Status.TiDB.Phase = v1alpha1.UpgradePhase if !templateEqual(newSet, oldSet) { return nil @@ -148,6 +149,7 @@ func NewFakeTiDBUpgrader() Upgrader { } func (u *fakeTiDBUpgrader) Upgrade(tc *v1alpha1.TidbCluster, _ *apps.StatefulSet, _ *apps.StatefulSet) error { + tc.Status.TiDB.ObservedGeneration = tc.Generation tc.Status.TiDB.Phase = v1alpha1.UpgradePhase return nil } diff --git a/pkg/manager/member/tiflash_member_manager.go b/pkg/manager/member/tiflash_member_manager.go index c9bcf5451b..1689d74928 100644 --- a/pkg/manager/member/tiflash_member_manager.go +++ b/pkg/manager/member/tiflash_member_manager.go @@ -743,6 +743,7 @@ func (m *tiflashMemberManager) syncTidbClusterStatus(tc *v1alpha1.TidbCluster, s if err != nil { return err } + tc.Status.TiFlash.ObservedGeneration = tc.Generation if tc.TiFlashStsDesiredReplicas() != *set.Spec.Replicas { tc.Status.TiFlash.Phase = v1alpha1.ScalePhase } else if upgrading { diff --git a/pkg/manager/member/tiflash_upgrader.go b/pkg/manager/member/tiflash_upgrader.go index 2799be8262..ae2c08c062 100644 --- a/pkg/manager/member/tiflash_upgrader.go +++ b/pkg/manager/member/tiflash_upgrader.go @@ -75,6 +75,7 @@ func (u *tiflashUpgrader) Upgrade(tc *v1alpha1.TidbCluster, oldSet *apps.Statefu return fmt.Errorf("cluster: [%s/%s]'s TiFlash status is not synced, can not upgrade", ns, tcName) } + tc.Status.TiFlash.ObservedGeneration = tc.Generation tc.Status.TiFlash.Phase = v1alpha1.UpgradePhase if !templateEqual(newSet, oldSet) { return nil diff --git a/pkg/manager/member/tikv_member_manager.go b/pkg/manager/member/tikv_member_manager.go index 5da033d984..40880e1338 100644 --- a/pkg/manager/member/tikv_member_manager.go +++ b/pkg/manager/member/tikv_member_manager.go @@ -834,6 +834,8 @@ func (m *tikvMemberManager) syncTiKVClusterStatus(tc *v1alpha1.TidbCluster, set } } + tc.Status.TiKV.ObservedGeneration = tc.Generation + // Scaling takes precedence over upgrading. if tc.TiKVStsDesiredReplicas() != *set.Spec.Replicas { tc.Status.TiKV.Phase = v1alpha1.ScalePhase diff --git a/pkg/manager/member/tikv_upgrader.go b/pkg/manager/member/tikv_upgrader.go index bf2fe58538..fab6724da5 100644 --- a/pkg/manager/member/tikv_upgrader.go +++ b/pkg/manager/member/tikv_upgrader.go @@ -91,6 +91,7 @@ func (u *tikvUpgrader) Upgrade(meta metav1.Object, oldSet *apps.StatefulSet, new // NOTE: If `TiKVStatus.Synced`` is false, it's acceptable to use old record about peer stores if *oldSet.Spec.Replicas < 2 && len(tc.Status.TiKV.PeerStores) == 0 { klog.Infof("TiKV statefulset replicas are less than 2, skip evicting region leader for tc %s/%s", ns, tcName) + status.ObservedGeneration = tc.Generation status.Phase = v1alpha1.UpgradePhase mngerutils.SetUpgradePartition(newSet, 0) return nil @@ -100,6 +101,7 @@ func (u *tikvUpgrader) Upgrade(meta metav1.Object, oldSet *apps.StatefulSet, new return fmt.Errorf("cluster: [%s/%s]'s tikv status sync failed, can not to be upgraded", ns, tcName) } + status.ObservedGeneration = tc.Generation status.Phase = v1alpha1.UpgradePhase if !templateEqual(newSet, oldSet) { return nil @@ -575,6 +577,7 @@ func NewFakeTiKVUpgrader() TiKVUpgrader { func (u *fakeTiKVUpgrader) Upgrade(meta metav1.Object, _ *apps.StatefulSet, _ *apps.StatefulSet) error { tc := meta.(*v1alpha1.TidbCluster) + tc.Status.TiKV.ObservedGeneration = tc.Generation tc.Status.TiKV.Phase = v1alpha1.UpgradePhase return nil } diff --git a/pkg/manager/member/tiproxy_member_manager.go b/pkg/manager/member/tiproxy_member_manager.go index 44ae578815..044488aea0 100644 --- a/pkg/manager/member/tiproxy_member_manager.go +++ b/pkg/manager/member/tiproxy_member_manager.go @@ -288,6 +288,7 @@ func (m *tiproxyMemberManager) syncStatus(tc *v1alpha1.TidbCluster, sts *apps.St tc.Status.TiProxy.Synced = false return err } + tc.Status.TiProxy.ObservedGeneration = tc.Generation if tc.Spec.TiProxy.Replicas != *sts.Spec.Replicas { tc.Status.TiProxy.Phase = v1alpha1.ScalePhase } else if upgrading { diff --git a/pkg/manager/member/tiproxy_upgrader.go b/pkg/manager/member/tiproxy_upgrader.go index 85a3322e12..a5e0bdbdd8 100644 --- a/pkg/manager/member/tiproxy_upgrader.go +++ b/pkg/manager/member/tiproxy_upgrader.go @@ -60,6 +60,7 @@ func (u *tiproxyUpgrader) Upgrade(tc *v1alpha1.TidbCluster, oldSet *apps.Statefu return nil } + tc.Status.TiProxy.ObservedGeneration = tc.Generation tc.Status.TiProxy.Phase = v1alpha1.UpgradePhase if !templateEqual(newSet, oldSet) { return nil diff --git a/pkg/manager/suspender/suspender.go b/pkg/manager/suspender/suspender.go index 045928d874..618ed92d0d 100644 --- a/pkg/manager/suspender/suspender.go +++ b/pkg/manager/suspender/suspender.go @@ -194,6 +194,7 @@ func (s *suspender) begin(ctx *suspendComponentCtx) error { phase := v1alpha1.SuspendPhase klog.Infof("begin to suspend component %s and transfer phase from %s to %s", ctx.ComponentID(), status.GetPhase(), phase) + ctx.status.SetObservedGeneration(ctx.cluster.GetGeneration()) ctx.status.SetPhase(phase) return nil } @@ -203,6 +204,7 @@ func (s *suspender) end(ctx *suspendComponentCtx) error { phase := v1alpha1.NormalPhase klog.Infof("end to suspend component %s and transfer phase from %s to %s", ctx.ComponentID(), status.GetPhase(), phase) + ctx.status.SetObservedGeneration(ctx.cluster.GetGeneration()) ctx.status.SetPhase(phase) return nil }