diff --git a/cmd/rootCmd.go b/cmd/rootCmd.go index bb5dcc5..84890f8 100644 --- a/cmd/rootCmd.go +++ b/cmd/rootCmd.go @@ -50,6 +50,7 @@ var ( containerImage string fioCheckerSize string + fioNodeSelector map[string]string fioCheckerFilePath string fioCheckerTestName string fioCmd = &cobra.Command{ @@ -59,7 +60,7 @@ var ( RunE: func(cmd *cobra.Command, args []string) error { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) defer cancel() - return Fio(ctx, output, outfile, storageClass, fioCheckerSize, namespace, fioCheckerTestName, fioCheckerFilePath, containerImage) + return Fio(ctx, output, outfile, storageClass, fioCheckerSize, namespace, fioNodeSelector, fioCheckerTestName, fioCheckerFilePath, containerImage) }, } @@ -104,6 +105,7 @@ func init() { _ = fioCmd.MarkFlagRequired("storageclass") fioCmd.Flags().StringVarP(&fioCheckerSize, "size", "z", fio.DefaultPVCSize, "The size of the volume used to run FIO. Note that the FIO job definition is not scaled accordingly.") fioCmd.Flags().StringVarP(&namespace, "namespace", "n", fio.DefaultNS, "The namespace used to run FIO.") + fioCmd.Flags().StringToStringVarP(&fioNodeSelector, "nodeselector", "N", map[string]string{}, "Node selector applied to pod.") fioCmd.Flags().StringVarP(&fioCheckerFilePath, "fiofile", "f", "", "The path to a an fio config file.") fioCmd.Flags().StringVarP(&fioCheckerTestName, "testname", "t", "", "The Name of a predefined kubestr fio test. Options(default-fio)") fioCmd.Flags().StringVarP(&containerImage, "image", "i", "", "The container image used to create a pod.") @@ -189,7 +191,7 @@ func PrintAndJsonOutput(result []*kubestr.TestOutput, output string, outfile str } // Fio executes the FIO test. -func Fio(ctx context.Context, output, outfile, storageclass, size, namespace, jobName, fioFilePath string, containerImage string) error { +func Fio(ctx context.Context, output, outfile, storageclass, size, namespace string, nodeSelector map[string]string, jobName, fioFilePath string, containerImage string) error { cli, err := kubestr.LoadKubeCli() if err != nil { fmt.Println(err.Error()) @@ -204,6 +206,7 @@ func Fio(ctx context.Context, output, outfile, storageclass, size, namespace, jo StorageClass: storageclass, Size: size, Namespace: namespace, + NodeSelector: nodeSelector, FIOJobName: jobName, FIOJobFilepath: fioFilePath, Image: containerImage, diff --git a/pkg/fio/fio.go b/pkg/fio/fio.go index 080f8a3..1009dec 100644 --- a/pkg/fio/fio.go +++ b/pkg/fio/fio.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "os" + "k8s.io/apimachinery/pkg/labels" "path/filepath" "time" @@ -63,6 +64,7 @@ type RunFIOArgs struct { StorageClass string Size string Namespace string + NodeSelector map[string]string FIOJobFilepath string FIOJobName string Image string @@ -106,6 +108,10 @@ func (f *FIOrunner) RunFioHelper(ctx context.Context, args *RunFIOArgs) (*RunFIO return nil, errors.Wrapf(err, "Unable to find namespace (%s)", args.Namespace) } + if err := f.fioSteps.validateNodeSelector(ctx, args.NodeSelector); err != nil { + return nil, errors.Wrapf(err, "Unable to find nodes satisfying node selector (%v)", args.NodeSelector) + } + sc, err := f.fioSteps.storageClassExists(ctx, args.StorageClass) if err != nil { return nil, errors.Wrap(err, "Cannot find StorageClass") @@ -133,7 +139,7 @@ func (f *FIOrunner) RunFioHelper(ctx context.Context, args *RunFIOArgs) (*RunFIO }() fmt.Println("PVC created", pvc.Name) - pod, err := f.fioSteps.createPod(ctx, pvc.Name, configMap.Name, testFileName, args.Namespace, args.Image) + pod, err := f.fioSteps.createPod(ctx, pvc.Name, configMap.Name, testFileName, args.Namespace, args.NodeSelector, args.Image) if err != nil { return nil, errors.Wrap(err, "Failed to create POD") } @@ -156,11 +162,12 @@ func (f *FIOrunner) RunFioHelper(ctx context.Context, args *RunFIOArgs) (*RunFIO type fioSteps interface { validateNamespace(ctx context.Context, namespace string) error + validateNodeSelector(ctx context.Context, selector map[string]string) error storageClassExists(ctx context.Context, storageClass string) (*sv1.StorageClass, error) loadConfigMap(ctx context.Context, args *RunFIOArgs) (*v1.ConfigMap, error) createPVC(ctx context.Context, storageclass, size, namespace string) (*v1.PersistentVolumeClaim, error) deletePVC(ctx context.Context, pvcName, namespace string) error - createPod(ctx context.Context, pvcName, configMapName, testFileName, namespace string, image string) (*v1.Pod, error) + createPod(ctx context.Context, pvcName, configMapName, testFileName, namespace string, nodeSelector map[string]string, image string) (*v1.Pod, error) deletePod(ctx context.Context, podName, namespace string) error runFIOCommand(ctx context.Context, podName, containerName, testFileName, namespace string) (FioResult, error) deleteConfigMap(ctx context.Context, configMap *v1.ConfigMap, namespace string) error @@ -179,6 +186,21 @@ func (s *fioStepper) validateNamespace(ctx context.Context, namespace string) er return nil } +func (s *fioStepper) validateNodeSelector(ctx context.Context, selector map[string]string) error { + nodes, err := s.cli.CoreV1().Nodes().List(ctx, metav1.ListOptions{ + LabelSelector: labels.SelectorFromSet(selector).String(), + }) + if err != nil { + return err + } + + if len(nodes.Items) == 0 { + return fmt.Errorf("No nodes match selector") + } + + return nil +} + func (s *fioStepper) storageClassExists(ctx context.Context, storageClass string) (*sv1.StorageClass, error) { return s.cli.StorageV1().StorageClasses().Get(ctx, storageClass, metav1.GetOptions{}) } @@ -234,7 +256,7 @@ func (s *fioStepper) deletePVC(ctx context.Context, pvcName, namespace string) e return s.cli.CoreV1().PersistentVolumeClaims(namespace).Delete(ctx, pvcName, metav1.DeleteOptions{}) } -func (s *fioStepper) createPod(ctx context.Context, pvcName, configMapName, testFileName, namespace string, image string) (*v1.Pod, error) { +func (s *fioStepper) createPod(ctx context.Context, pvcName, configMapName, testFileName, namespace string, nodeSelector map[string]string, image string) (*v1.Pod, error) { if pvcName == "" || configMapName == "" || testFileName == "" { return nil, fmt.Errorf("Create pod missing required arguments.") } @@ -277,6 +299,7 @@ func (s *fioStepper) createPod(ctx context.Context, pvcName, configMapName, test }, }, }, + NodeSelector: nodeSelector, }, } podRes, err := s.cli.CoreV1().Pods(namespace).Create(ctx, pod, metav1.CreateOptions{}) diff --git a/pkg/fio/fio_test.go b/pkg/fio/fio_test.go index d9daf2c..6f9bc5c 100644 --- a/pkg/fio/fio_test.go +++ b/pkg/fio/fio_test.go @@ -85,6 +85,19 @@ func (s *FIOTestSuite) TestRunFioHelper(c *C) { checker: NotNil, expectedSteps: []string{"VN"}, }, + { // no node satisfies selector + cli: fake.NewSimpleClientset(), + stepper: &fakeFioStepper{ + vnsErr: fmt.Errorf("node selector Err"), + }, + args: &RunFIOArgs{ + StorageClass: "sc", + Size: "100Gi", + Namespace: "foo", + }, + checker: NotNil, + expectedSteps: []string{"VN", "VNS"}, + }, { // storageclass not found cli: fake.NewSimpleClientset(), stepper: &fakeFioStepper{ @@ -96,7 +109,7 @@ func (s *FIOTestSuite) TestRunFioHelper(c *C) { Namespace: "foo", }, checker: NotNil, - expectedSteps: []string{"VN", "SCE"}, + expectedSteps: []string{"VN", "VNS", "SCE"}, }, { // success cli: fake.NewSimpleClientset(), @@ -126,7 +139,7 @@ func (s *FIOTestSuite) TestRunFioHelper(c *C) { Namespace: "foo", }, checker: IsNil, - expectedSteps: []string{"VN", "SCE", "LCM", "CPVC", "CPOD", "RFIOC", "DPOD", "DPVC", "DCM"}, + expectedSteps: []string{"VN", "VNS", "SCE", "LCM", "CPVC", "CPOD", "RFIOC", "DPOD", "DPVC", "DCM"}, expectedSC: "sc", expectedSize: DefaultPVCSize, expectedTFN: "testfile.fio", @@ -162,7 +175,7 @@ func (s *FIOTestSuite) TestRunFioHelper(c *C) { Namespace: "foo", }, checker: NotNil, - expectedSteps: []string{"VN", "SCE", "LCM", "CPVC", "CPOD", "RFIOC", "DPOD", "DPVC", "DCM"}, + expectedSteps: []string{"VN", "VNS", "SCE", "LCM", "CPVC", "CPOD", "RFIOC", "DPOD", "DPVC", "DCM"}, }, { // create pod error cli: fake.NewSimpleClientset(), @@ -193,7 +206,7 @@ func (s *FIOTestSuite) TestRunFioHelper(c *C) { Namespace: "foo", }, checker: NotNil, - expectedSteps: []string{"VN", "SCE", "LCM", "CPVC", "CPOD", "DPVC", "DCM"}, + expectedSteps: []string{"VN", "VNS", "SCE", "LCM", "CPVC", "CPOD", "DPVC", "DCM"}, }, { // create PVC error cli: fake.NewSimpleClientset(), @@ -214,7 +227,7 @@ func (s *FIOTestSuite) TestRunFioHelper(c *C) { Namespace: "foo", }, checker: NotNil, - expectedSteps: []string{"VN", "SCE", "LCM", "CPVC", "DCM"}, + expectedSteps: []string{"VN", "VNS", "SCE", "LCM", "CPVC", "DCM"}, }, { // testfilename retrieval error, more than one provided cli: fake.NewSimpleClientset(), @@ -235,7 +248,7 @@ func (s *FIOTestSuite) TestRunFioHelper(c *C) { Namespace: "foo", }, checker: NotNil, - expectedSteps: []string{"VN", "SCE", "LCM", "DCM"}, + expectedSteps: []string{"VN", "VNS", "SCE", "LCM", "DCM"}, }, { // load configmap error cli: fake.NewSimpleClientset(), @@ -248,7 +261,7 @@ func (s *FIOTestSuite) TestRunFioHelper(c *C) { Namespace: "foo", }, checker: NotNil, - expectedSteps: []string{"VN", "SCE", "LCM"}, + expectedSteps: []string{"VN", "VNS", "SCE", "LCM"}, }, } { c.Log(i) @@ -274,6 +287,8 @@ type fakeFioStepper struct { vnErr error + vnsErr error + sceSC *sv1.StorageClass sceErr error @@ -303,6 +318,10 @@ func (f *fakeFioStepper) validateNamespace(ctx context.Context, namespace string f.steps = append(f.steps, "VN") return f.vnErr } +func (f *fakeFioStepper) validateNodeSelector(ctx context.Context, selector map[string]string) error { + f.steps = append(f.steps, "VNS") + return f.vnsErr +} func (f *fakeFioStepper) storageClassExists(ctx context.Context, storageClass string) (*sv1.StorageClass, error) { f.steps = append(f.steps, "SCE") return f.sceSC, f.sceErr @@ -321,7 +340,7 @@ func (f *fakeFioStepper) deletePVC(ctx context.Context, pvcName, namespace strin f.steps = append(f.steps, "DPVC") return f.dPVCErr } -func (f *fakeFioStepper) createPod(ctx context.Context, pvcName, configMapName, testFileName, namespace string, image string) (*v1.Pod, error) { +func (f *fakeFioStepper) createPod(ctx context.Context, pvcName, configMapName, testFileName, namespace string, nodeSelector map[string]string, image string) (*v1.Pod, error) { f.steps = append(f.steps, "CPOD") f.cPodExpCM = configMapName f.cPodExpFN = testFileName @@ -379,6 +398,56 @@ func (s *FIOTestSuite) TestValidateNamespace(c *C) { c.Assert(err, IsNil) } +func (s *FIOTestSuite) TestValidateNodeSelector(c *C) { + ctx := context.Background() + stepper := &fioStepper{cli: fake.NewSimpleClientset( + &v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "a", + Labels: map[string]string{ + "key": "value", + }, + }, + }, + &v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "b", + Labels: map[string]string{ + "key": "value", + "foo": "bar", + }, + }, + }, + )} + for _, tc := range []struct { + nodeSelector map[string]string + checker Checker + }{ + { // 0 nodes satisfy + nodeSelector: map[string]string{ + "not": "present", + }, + checker: NotNil, + }, + { // 1 node satisfies + nodeSelector: map[string]string{ + "key": "value", + "foo": "bar", + }, + checker: IsNil, + }, + { // 2 nodes satisfy + nodeSelector: map[string]string{ + "key": "value", + }, + checker: IsNil, + }, + } { + err := stepper.validateNodeSelector(ctx, tc.nodeSelector) + c.Check(err, tc.checker) + } +} + func (s *FIOTestSuite) TestLoadConfigMap(c *C) { ctx := context.Background() file, err := os.CreateTemp("", "tempTLCfile") @@ -529,6 +598,7 @@ func (s *FIOTestSuite) TestCreatPod(c *C) { pvcName string configMapName string testFileName string + nodeSelector map[string]string image string reactor []k8stesting.Reactor podReadyErr error @@ -538,7 +608,11 @@ func (s *FIOTestSuite) TestCreatPod(c *C) { pvcName: "pvc", configMapName: "cm", testFileName: "testfile", - errChecker: IsNil, + nodeSelector: map[string]string{ + "key": "", + "foo": "bar", + }, + errChecker: IsNil, }, { pvcName: "pvc", @@ -620,7 +694,7 @@ func (s *FIOTestSuite) TestCreatPod(c *C) { if tc.reactor != nil { stepper.cli.(*fake.Clientset).Fake.ReactionChain = tc.reactor } - pod, err := stepper.createPod(ctx, tc.pvcName, tc.configMapName, tc.testFileName, DefaultNS, tc.image) + pod, err := stepper.createPod(ctx, tc.pvcName, tc.configMapName, tc.testFileName, DefaultNS, tc.nodeSelector, tc.image) c.Check(err, tc.errChecker) if err == nil { c.Assert(pod.GenerateName, Equals, PodGenerateName) @@ -646,6 +720,7 @@ func (s *FIOTestSuite) TestCreatPod(c *C) { } else { c.Assert(pod.Spec.Containers[0].Image, Equals, tc.image) } + c.Assert(pod.Spec.NodeSelector, DeepEquals, tc.nodeSelector) } } }