fix(*): empty cpuset.mems will lead to failure of tmo

Signed-off-by: linzhecheng <[email protected]>
kubewharf · Nov 14, 2024 · c847f11 · c847f11
1 parent 3269ac6
commit c847f11
Show file tree

Hide file tree

Showing 5 changed files with 64 additions and 11 deletions.
diff --git a/pkg/agent/qrm-plugins/memory/dynamicpolicy/policy_advisor_handler.go b/pkg/agent/qrm-plugins/memory/dynamicpolicy/policy_advisor_handler.go
@@ -736,11 +736,7 @@ func (p *DynamicPolicy) handleAdvisorMemoryOffloading(_ *config.Configuration,
 		}
 	}
 
-	cpuSetStats, err := cgroupmgr.GetCPUSetWithAbsolutePath(absCGPath)
-	if err != nil {
-		return fmt.Errorf("GetCPUSetWithAbsolutePath failed with error: %v", err)
-	}
-	mems, err := machine.Parse(cpuSetStats.Mems)
+	_, mems, err := cgroupmgr.GetEffectiveCPUSetWithAbsolutePath(absCGPath)
 	if err != nil {
 		return fmt.Errorf("parse cpuSetStats failed with error: %v", err)
 	}

diff --git a/pkg/util/cgroup/common/types.go b/pkg/util/cgroup/common/types.go
@@ -184,8 +184,10 @@ type CPUStats struct {
 
 // CPUSetStats get cgroup cpuset data
 type CPUSetStats struct {
-	CPUs string
-	Mems string
+	CPUs          string
+	EffectiveCPUs string
+	Mems          string
+	EffectiveMems string
 }
 
 // MemoryMetrics get memory cgroup metrics

diff --git a/pkg/util/cgroup/manager/cgroup.go b/pkg/util/cgroup/manager/cgroup.go
@@ -21,6 +21,7 @@ import (
 	"fmt"
 	"io/fs"
 	"math"
+	"os"
 	"os/exec"
 	"path/filepath"
 	"strconv"
@@ -514,3 +515,41 @@ func MemoryOffloadingWithAbsolutePath(ctx context.Context, absCgroupPath string,
 
 	return err
 }
+
+func GetEffectiveCPUSetWithAbsolutePath(absCgroupPath string) (machine.CPUSet, machine.CPUSet, error) {
+	_, err := os.Stat(absCgroupPath)
+	if err != nil {
+		return machine.CPUSet{}, machine.CPUSet{}, err
+	}
+
+	cpusetStat, err := GetCPUSetWithAbsolutePath(absCgroupPath)
+	if err != nil {
+		// if controller is disabled, we should walk the parent's dir.
+		if os.IsNotExist(err) {
+			return GetEffectiveCPUSetWithAbsolutePath(filepath.Dir(absCgroupPath))
+		}
+		return machine.CPUSet{}, machine.CPUSet{}, err
+	}
+	// if the cpus or mems is empty, they will inherit the parent's mask.
+	cpus, err := machine.Parse(cpusetStat.EffectiveCPUs)
+	if err != nil {
+		return machine.CPUSet{}, machine.CPUSet{}, err
+	}
+	if cpus.IsEmpty() {
+		cpus, _, err = GetEffectiveCPUSetWithAbsolutePath(filepath.Dir(absCgroupPath))
+		if err != nil {
+			return machine.CPUSet{}, machine.CPUSet{}, err
+		}
+	}
+	mems, err := machine.Parse(cpusetStat.EffectiveMems)
+	if err != nil {
+		return machine.CPUSet{}, machine.CPUSet{}, err
+	}
+	if mems.IsEmpty() {
+		_, mems, err = GetEffectiveCPUSetWithAbsolutePath(filepath.Dir(absCgroupPath))
+		if err != nil {
+			return machine.CPUSet{}, machine.CPUSet{}, err
+		}
+	}
+	return cpus, mems, nil
+}
diff --git a/pkg/util/cgroup/manager/v1/fs_linux.go b/pkg/util/cgroup/manager/v1/fs_linux.go
@@ -308,12 +308,20 @@ func (m *manager) GetCPUSet(absCgroupPath string) (*common.CPUSetStats, error) {
 	var err error
 	cpusetStats.CPUs, err = fscommon.GetCgroupParamString(absCgroupPath, "cpuset.cpus")
 	if err != nil {
-		return nil, fmt.Errorf("read cpuset.cpus failed with error: %v", err)
+		return nil, err
+	}
+	cpusetStats.EffectiveCPUs, err = fscommon.GetCgroupParamString(absCgroupPath, "cpuset.effective_cpus")
+	if err != nil {
+		return nil, err
 	}
 
 	cpusetStats.Mems, err = fscommon.GetCgroupParamString(absCgroupPath, "cpuset.mems")
 	if err != nil {
-		return nil, fmt.Errorf("read cpuset.mems failed with error: %v", err)
+		return nil, err
+	}
+	cpusetStats.EffectiveMems, err = fscommon.GetCgroupParamString(absCgroupPath, "cpuset.effective_mems")
+	if err != nil {
+		return nil, err
 	}
 
 	return cpusetStats, nil

diff --git a/pkg/util/cgroup/manager/v2/fs_linux.go b/pkg/util/cgroup/manager/v2/fs_linux.go
@@ -432,12 +432,20 @@ func (m *manager) GetCPUSet(absCgroupPath string) (*common.CPUSetStats, error) {
 	var err error
 	cpusetStats.CPUs, err = fscommon.GetCgroupParamString(absCgroupPath, "cpuset.cpus")
 	if err != nil {
-		return nil, fmt.Errorf("read cpuset.cpus failed with error: %v", err)
+		return nil, err
+	}
+	cpusetStats.EffectiveCPUs, err = fscommon.GetCgroupParamString(absCgroupPath, "cpuset.cpus.effective")
+	if err != nil {
+		return nil, err
 	}
 
 	cpusetStats.Mems, err = fscommon.GetCgroupParamString(absCgroupPath, "cpuset.mems")
 	if err != nil {
-		return nil, fmt.Errorf("read cpuset.mems failed with error: %v", err)
+		return nil, err
+	}
+	cpusetStats.EffectiveMems, err = fscommon.GetCgroupParamString(absCgroupPath, "cpuset.mems.effective")
+	if err != nil {
+		return nil, err
 	}
 
 	return cpusetStats, nil