Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add a node parameter to the FIO benchmark #105

Merged
merged 1 commit into from
Aug 16, 2022
Merged

Add a node parameter to the FIO benchmark #105

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 5 additions & 2 deletions cmd/rootCmd.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@ var (
containerImage string

fioCheckerSize string
fioNodeSelector map[string]string
fioCheckerFilePath string
fioCheckerTestName string
fioCmd = &cobra.Command{
Expand All @@ -59,7 +60,7 @@ var (
RunE: func(cmd *cobra.Command, args []string) error {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
defer cancel()
return Fio(ctx, output, outfile, storageClass, fioCheckerSize, namespace, fioCheckerTestName, fioCheckerFilePath, containerImage)
return Fio(ctx, output, outfile, storageClass, fioCheckerSize, namespace, fioNodeSelector, fioCheckerTestName, fioCheckerFilePath, containerImage)
},
}

Expand Down Expand Up @@ -104,6 +105,7 @@ func init() {
_ = fioCmd.MarkFlagRequired("storageclass")
fioCmd.Flags().StringVarP(&fioCheckerSize, "size", "z", fio.DefaultPVCSize, "The size of the volume used to run FIO. Note that the FIO job definition is not scaled accordingly.")
fioCmd.Flags().StringVarP(&namespace, "namespace", "n", fio.DefaultNS, "The namespace used to run FIO.")
fioCmd.Flags().StringToStringVarP(&fioNodeSelector, "nodeselector", "N", map[string]string{}, "Node selector applied to pod.")
fioCmd.Flags().StringVarP(&fioCheckerFilePath, "fiofile", "f", "", "The path to a an fio config file.")
fioCmd.Flags().StringVarP(&fioCheckerTestName, "testname", "t", "", "The Name of a predefined kubestr fio test. Options(default-fio)")
fioCmd.Flags().StringVarP(&containerImage, "image", "i", "", "The container image used to create a pod.")
Expand Down Expand Up @@ -189,7 +191,7 @@ func PrintAndJsonOutput(result []*kubestr.TestOutput, output string, outfile str
}

// Fio executes the FIO test.
func Fio(ctx context.Context, output, outfile, storageclass, size, namespace, jobName, fioFilePath string, containerImage string) error {
func Fio(ctx context.Context, output, outfile, storageclass, size, namespace string, nodeSelector map[string]string, jobName, fioFilePath string, containerImage string) error {
cli, err := kubestr.LoadKubeCli()
if err != nil {
fmt.Println(err.Error())
Expand All @@ -204,6 +206,7 @@ func Fio(ctx context.Context, output, outfile, storageclass, size, namespace, jo
StorageClass: storageclass,
Size: size,
Namespace: namespace,
NodeSelector: nodeSelector,
FIOJobName: jobName,
FIOJobFilepath: fioFilePath,
Image: containerImage,
Expand Down
29 changes: 26 additions & 3 deletions pkg/fio/fio.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"encoding/json"
"fmt"
"os"
"k8s.io/apimachinery/pkg/labels"
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[nit] this should probably move to the list below with other third-party imports

Not blocking the PR, we can fix it as a follow-up

"path/filepath"
"time"

Expand Down Expand Up @@ -63,6 +64,7 @@ type RunFIOArgs struct {
StorageClass string
Size string
Namespace string
NodeSelector map[string]string
FIOJobFilepath string
FIOJobName string
Image string
Expand Down Expand Up @@ -106,6 +108,10 @@ func (f *FIOrunner) RunFioHelper(ctx context.Context, args *RunFIOArgs) (*RunFIO
return nil, errors.Wrapf(err, "Unable to find namespace (%s)", args.Namespace)
}

if err := f.fioSteps.validateNodeSelector(ctx, args.NodeSelector); err != nil {
return nil, errors.Wrapf(err, "Unable to find nodes satisfying node selector (%v)", args.NodeSelector)
}

sc, err := f.fioSteps.storageClassExists(ctx, args.StorageClass)
if err != nil {
return nil, errors.Wrap(err, "Cannot find StorageClass")
Expand Down Expand Up @@ -133,7 +139,7 @@ func (f *FIOrunner) RunFioHelper(ctx context.Context, args *RunFIOArgs) (*RunFIO
}()
fmt.Println("PVC created", pvc.Name)

pod, err := f.fioSteps.createPod(ctx, pvc.Name, configMap.Name, testFileName, args.Namespace, args.Image)
pod, err := f.fioSteps.createPod(ctx, pvc.Name, configMap.Name, testFileName, args.Namespace, args.NodeSelector, args.Image)
if err != nil {
return nil, errors.Wrap(err, "Failed to create POD")
}
Expand All @@ -156,11 +162,12 @@ func (f *FIOrunner) RunFioHelper(ctx context.Context, args *RunFIOArgs) (*RunFIO

type fioSteps interface {
validateNamespace(ctx context.Context, namespace string) error
validateNodeSelector(ctx context.Context, selector map[string]string) error
storageClassExists(ctx context.Context, storageClass string) (*sv1.StorageClass, error)
loadConfigMap(ctx context.Context, args *RunFIOArgs) (*v1.ConfigMap, error)
createPVC(ctx context.Context, storageclass, size, namespace string) (*v1.PersistentVolumeClaim, error)
deletePVC(ctx context.Context, pvcName, namespace string) error
createPod(ctx context.Context, pvcName, configMapName, testFileName, namespace string, image string) (*v1.Pod, error)
createPod(ctx context.Context, pvcName, configMapName, testFileName, namespace string, nodeSelector map[string]string, image string) (*v1.Pod, error)
deletePod(ctx context.Context, podName, namespace string) error
runFIOCommand(ctx context.Context, podName, containerName, testFileName, namespace string) (FioResult, error)
deleteConfigMap(ctx context.Context, configMap *v1.ConfigMap, namespace string) error
Expand All @@ -179,6 +186,21 @@ func (s *fioStepper) validateNamespace(ctx context.Context, namespace string) er
return nil
}

func (s *fioStepper) validateNodeSelector(ctx context.Context, selector map[string]string) error {
nodes, err := s.cli.CoreV1().Nodes().List(ctx, metav1.ListOptions{
LabelSelector: labels.SelectorFromSet(selector).String(),
})
if err != nil {
return err
}

if len(nodes.Items) == 0 {
return fmt.Errorf("No nodes match selector")
}

return nil
}

func (s *fioStepper) storageClassExists(ctx context.Context, storageClass string) (*sv1.StorageClass, error) {
return s.cli.StorageV1().StorageClasses().Get(ctx, storageClass, metav1.GetOptions{})
}
Expand Down Expand Up @@ -234,7 +256,7 @@ func (s *fioStepper) deletePVC(ctx context.Context, pvcName, namespace string) e
return s.cli.CoreV1().PersistentVolumeClaims(namespace).Delete(ctx, pvcName, metav1.DeleteOptions{})
}

func (s *fioStepper) createPod(ctx context.Context, pvcName, configMapName, testFileName, namespace string, image string) (*v1.Pod, error) {
func (s *fioStepper) createPod(ctx context.Context, pvcName, configMapName, testFileName, namespace string, nodeSelector map[string]string, image string) (*v1.Pod, error) {
if pvcName == "" || configMapName == "" || testFileName == "" {
return nil, fmt.Errorf("Create pod missing required arguments.")
}
Expand Down Expand Up @@ -277,6 +299,7 @@ func (s *fioStepper) createPod(ctx context.Context, pvcName, configMapName, test
},
},
},
NodeSelector: nodeSelector,
},
}
podRes, err := s.cli.CoreV1().Pods(namespace).Create(ctx, pod, metav1.CreateOptions{})
Expand Down
95 changes: 85 additions & 10 deletions pkg/fio/fio_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,19 @@ func (s *FIOTestSuite) TestRunFioHelper(c *C) {
checker: NotNil,
expectedSteps: []string{"VN"},
},
{ // no node satisfies selector
cli: fake.NewSimpleClientset(),
stepper: &fakeFioStepper{
vnsErr: fmt.Errorf("node selector Err"),
},
args: &RunFIOArgs{
StorageClass: "sc",
Size: "100Gi",
Namespace: "foo",
},
checker: NotNil,
expectedSteps: []string{"VN", "VNS"},
},
{ // storageclass not found
cli: fake.NewSimpleClientset(),
stepper: &fakeFioStepper{
Expand All @@ -96,7 +109,7 @@ func (s *FIOTestSuite) TestRunFioHelper(c *C) {
Namespace: "foo",
},
checker: NotNil,
expectedSteps: []string{"VN", "SCE"},
expectedSteps: []string{"VN", "VNS", "SCE"},
},
{ // success
cli: fake.NewSimpleClientset(),
Expand Down Expand Up @@ -126,7 +139,7 @@ func (s *FIOTestSuite) TestRunFioHelper(c *C) {
Namespace: "foo",
},
checker: IsNil,
expectedSteps: []string{"VN", "SCE", "LCM", "CPVC", "CPOD", "RFIOC", "DPOD", "DPVC", "DCM"},
expectedSteps: []string{"VN", "VNS", "SCE", "LCM", "CPVC", "CPOD", "RFIOC", "DPOD", "DPVC", "DCM"},
expectedSC: "sc",
expectedSize: DefaultPVCSize,
expectedTFN: "testfile.fio",
Expand Down Expand Up @@ -162,7 +175,7 @@ func (s *FIOTestSuite) TestRunFioHelper(c *C) {
Namespace: "foo",
},
checker: NotNil,
expectedSteps: []string{"VN", "SCE", "LCM", "CPVC", "CPOD", "RFIOC", "DPOD", "DPVC", "DCM"},
expectedSteps: []string{"VN", "VNS", "SCE", "LCM", "CPVC", "CPOD", "RFIOC", "DPOD", "DPVC", "DCM"},
},
{ // create pod error
cli: fake.NewSimpleClientset(),
Expand Down Expand Up @@ -193,7 +206,7 @@ func (s *FIOTestSuite) TestRunFioHelper(c *C) {
Namespace: "foo",
},
checker: NotNil,
expectedSteps: []string{"VN", "SCE", "LCM", "CPVC", "CPOD", "DPVC", "DCM"},
expectedSteps: []string{"VN", "VNS", "SCE", "LCM", "CPVC", "CPOD", "DPVC", "DCM"},
},
{ // create PVC error
cli: fake.NewSimpleClientset(),
Expand All @@ -214,7 +227,7 @@ func (s *FIOTestSuite) TestRunFioHelper(c *C) {
Namespace: "foo",
},
checker: NotNil,
expectedSteps: []string{"VN", "SCE", "LCM", "CPVC", "DCM"},
expectedSteps: []string{"VN", "VNS", "SCE", "LCM", "CPVC", "DCM"},
},
{ // testfilename retrieval error, more than one provided
cli: fake.NewSimpleClientset(),
Expand All @@ -235,7 +248,7 @@ func (s *FIOTestSuite) TestRunFioHelper(c *C) {
Namespace: "foo",
},
checker: NotNil,
expectedSteps: []string{"VN", "SCE", "LCM", "DCM"},
expectedSteps: []string{"VN", "VNS", "SCE", "LCM", "DCM"},
},
{ // load configmap error
cli: fake.NewSimpleClientset(),
Expand All @@ -248,7 +261,7 @@ func (s *FIOTestSuite) TestRunFioHelper(c *C) {
Namespace: "foo",
},
checker: NotNil,
expectedSteps: []string{"VN", "SCE", "LCM"},
expectedSteps: []string{"VN", "VNS", "SCE", "LCM"},
},
} {
c.Log(i)
Expand All @@ -274,6 +287,8 @@ type fakeFioStepper struct {

vnErr error

vnsErr error

sceSC *sv1.StorageClass
sceErr error

Expand Down Expand Up @@ -303,6 +318,10 @@ func (f *fakeFioStepper) validateNamespace(ctx context.Context, namespace string
f.steps = append(f.steps, "VN")
return f.vnErr
}
func (f *fakeFioStepper) validateNodeSelector(ctx context.Context, selector map[string]string) error {
f.steps = append(f.steps, "VNS")
return f.vnsErr
}
func (f *fakeFioStepper) storageClassExists(ctx context.Context, storageClass string) (*sv1.StorageClass, error) {
f.steps = append(f.steps, "SCE")
return f.sceSC, f.sceErr
Expand All @@ -321,7 +340,7 @@ func (f *fakeFioStepper) deletePVC(ctx context.Context, pvcName, namespace strin
f.steps = append(f.steps, "DPVC")
return f.dPVCErr
}
func (f *fakeFioStepper) createPod(ctx context.Context, pvcName, configMapName, testFileName, namespace string, image string) (*v1.Pod, error) {
func (f *fakeFioStepper) createPod(ctx context.Context, pvcName, configMapName, testFileName, namespace string, nodeSelector map[string]string, image string) (*v1.Pod, error) {
f.steps = append(f.steps, "CPOD")
f.cPodExpCM = configMapName
f.cPodExpFN = testFileName
Expand Down Expand Up @@ -379,6 +398,56 @@ func (s *FIOTestSuite) TestValidateNamespace(c *C) {
c.Assert(err, IsNil)
}

func (s *FIOTestSuite) TestValidateNodeSelector(c *C) {
ctx := context.Background()
stepper := &fioStepper{cli: fake.NewSimpleClientset(
&v1.Node{
ObjectMeta: metav1.ObjectMeta{
Name: "a",
Labels: map[string]string{
"key": "value",
},
},
},
&v1.Node{
ObjectMeta: metav1.ObjectMeta{
Name: "b",
Labels: map[string]string{
"key": "value",
"foo": "bar",
},
},
},
)}
for _, tc := range []struct {
nodeSelector map[string]string
checker Checker
}{
{ // 0 nodes satisfy
nodeSelector: map[string]string{
"not": "present",
},
checker: NotNil,
},
{ // 1 node satisfies
nodeSelector: map[string]string{
"key": "value",
"foo": "bar",
},
checker: IsNil,
},
{ // 2 nodes satisfy
nodeSelector: map[string]string{
"key": "value",
},
checker: IsNil,
},
} {
err := stepper.validateNodeSelector(ctx, tc.nodeSelector)
c.Check(err, tc.checker)
}
}

func (s *FIOTestSuite) TestLoadConfigMap(c *C) {
ctx := context.Background()
file, err := os.CreateTemp("", "tempTLCfile")
Expand Down Expand Up @@ -529,6 +598,7 @@ func (s *FIOTestSuite) TestCreatPod(c *C) {
pvcName string
configMapName string
testFileName string
nodeSelector map[string]string
image string
reactor []k8stesting.Reactor
podReadyErr error
Expand All @@ -538,7 +608,11 @@ func (s *FIOTestSuite) TestCreatPod(c *C) {
pvcName: "pvc",
configMapName: "cm",
testFileName: "testfile",
errChecker: IsNil,
nodeSelector: map[string]string{
"key": "",
"foo": "bar",
},
errChecker: IsNil,
},
{
pvcName: "pvc",
Expand Down Expand Up @@ -620,7 +694,7 @@ func (s *FIOTestSuite) TestCreatPod(c *C) {
if tc.reactor != nil {
stepper.cli.(*fake.Clientset).Fake.ReactionChain = tc.reactor
}
pod, err := stepper.createPod(ctx, tc.pvcName, tc.configMapName, tc.testFileName, DefaultNS, tc.image)
pod, err := stepper.createPod(ctx, tc.pvcName, tc.configMapName, tc.testFileName, DefaultNS, tc.nodeSelector, tc.image)
c.Check(err, tc.errChecker)
if err == nil {
c.Assert(pod.GenerateName, Equals, PodGenerateName)
Expand All @@ -646,6 +720,7 @@ func (s *FIOTestSuite) TestCreatPod(c *C) {
} else {
c.Assert(pod.Spec.Containers[0].Image, Equals, tc.image)
}
c.Assert(pod.Spec.NodeSelector, DeepEquals, tc.nodeSelector)
}
}
}
Expand Down