From 8367bd863259ff64000c4c92249c0e656e7a3ebd Mon Sep 17 00:00:00 2001 From: Alexander Maslennikov Date: Thu, 24 Oct 2024 10:16:45 +0200 Subject: [PATCH] fix: disable RoCE-specific settings for IB devices This commit removes the configuration of RoCE-specific parameters when the device is operating in InfiniBand (IB) mode. Those settings are intended for RoCE (RDMA over Converged Ethernet) and are irrelevant for IB devices. Signed-off-by: Alexander Maslennikov --- README.md | 1 + pkg/host/configvalidation.go | 17 +++++++-- pkg/host/configvalidation_test.go | 57 ++++++++++++++++++++++++++++++- 3 files changed, 72 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 6560cd4..95852b4 100644 --- a/README.md +++ b/README.md @@ -116,6 +116,7 @@ spec: * Configure pfc (Priority Flow Control) for priority 3 and set trust to dscp on each PF * Non-persistent (need to be applied after each boot) * Users can override values via `trust` and `pfc` parameters + * Can only be enabled with `linkType=Ethernet` * `gpuDirectOptimized`: performs gpu direct optimizations. ATM only optimizations for Baremetal environment are supported. If enabled perform the following: * Set nvconfig `ATS_ENABLED=0` * Can only be enabled when `pciPerformanceOptimized` is enabled diff --git a/pkg/host/configvalidation.go b/pkg/host/configvalidation.go index 7c1afd9..1dceb04 100644 --- a/pkg/host/configvalidation.go +++ b/pkg/host/configvalidation.go @@ -135,6 +135,13 @@ func (v *configValidationImpl) ConstructNvParamMapFromTemplate( } if template.RoceOptimized != nil && template.RoceOptimized.Enabled { + if template.LinkType == consts.Infiniband { + err := types.IncorrectSpecError( + "RoceOptimized settings can only be used with link type Ethernet") + log.Log.Error(err, "incorrect spec", "device", device.Name) + return desiredParameters, err + } + desiredParameters[consts.RoceCcPrioMaskP1Param] = "255" desiredParameters[consts.CnpDscpP1Param] = "4" desiredParameters[consts.Cnp802pPrioP1Param] = "6" @@ -280,8 +287,6 @@ func (v *configValidationImpl) RuntimeConfigApplied(device *v1alpha1.NicDevice) // returns string - qos pfc settings func (v *configValidationImpl) CalculateDesiredRuntimeConfig(device *v1alpha1.NicDevice) (int, string, string) { maxReadRequestSize := 0 - trust := "pcp" - pfc := "0,0,0,0,0,0,0,0" template := device.Spec.Configuration.Template @@ -293,6 +298,14 @@ func (v *configValidationImpl) CalculateDesiredRuntimeConfig(device *v1alpha1.Ni } } + // QoS settings are not available for IB devices + if template.LinkType == consts.Infiniband { + return maxReadRequestSize, "", "" + } + + trust := "pcp" + pfc := "0,0,0,0,0,0,0,0" + if template.RoceOptimized != nil && template.RoceOptimized.Enabled { trust = "dscp" pfc = "0,0,0,1,0,0,0,0" diff --git a/pkg/host/configvalidation_test.go b/pkg/host/configvalidation_test.go index 6f44b6b..49af653 100644 --- a/pkg/host/configvalidation_test.go +++ b/pkg/host/configvalidation_test.go @@ -259,7 +259,7 @@ var _ = Describe("ConfigValidationImpl", func() { defaultValues := map[string]string{} _, err := validator.ConstructNvParamMapFromTemplate(device, defaultValues) - Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError("incorrect spec: GpuDirectOptimized should only be enabled together with PciPerformanceOptimized")) }) It("should ignore raw config for the second port if device is single port", func() { mockHostUtils.On("GetPCILinkSpeed", mock.Anything).Return(16, nil) @@ -441,6 +441,33 @@ var _ = Describe("ConfigValidationImpl", func() { _, err := validator.ConstructNvParamMapFromTemplate(device, defaultValues) Expect(err).NotTo(HaveOccurred()) }) + It("should return an error when RoceOptimized is enabled with linkType Infiniband", func() { + mockHostUtils.On("GetPCILinkSpeed", mock.Anything).Return(16, nil) + + device := &v1alpha1.NicDevice{ + Spec: v1alpha1.NicDeviceSpec{ + Configuration: &v1alpha1.NicDeviceConfigurationSpec{ + Template: &v1alpha1.ConfigurationTemplateSpec{ + NumVfs: 0, + LinkType: consts.Infiniband, + RoceOptimized: &v1alpha1.RoceOptimizedSpec{ + Enabled: true, + }, + }, + }, + }, + Status: v1alpha1.NicDeviceStatus{ + Ports: []v1alpha1.NicDevicePortSpec{ + {PCI: "0000:03:00.0"}, + }, + }, + } + + defaultValues := map[string]string{} + + _, err := validator.ConstructNvParamMapFromTemplate(device, defaultValues) + Expect(err).To(MatchError("incorrect spec: RoceOptimized settings can only be used with link type Ethernet")) + }) }) Describe("ValidateResetToDefault", func() { @@ -621,6 +648,34 @@ var _ = Describe("ConfigValidationImpl", func() { Expect(trust).To(Equal("customTrust")) Expect(pfc).To(Equal("1,1,1,1,1,1,1,1")) }) + + It("should not calculate desired QoS settings for an IB configuration", func() { + device := &v1alpha1.NicDevice{ + Spec: v1alpha1.NicDeviceSpec{ + Configuration: &v1alpha1.NicDeviceConfigurationSpec{ + Template: &v1alpha1.ConfigurationTemplateSpec{ + LinkType: consts.Infiniband, + PciPerformanceOptimized: &v1alpha1.PciPerformanceOptimizedSpec{ + Enabled: true, + MaxReadRequest: 256, + }, + RoceOptimized: &v1alpha1.RoceOptimizedSpec{ + Enabled: true, + Qos: &v1alpha1.QosSpec{ + Trust: "customTrust", + PFC: "1,1,1,1,1,1,1,1", + }, + }, + }, + }, + }, + } + + maxReadRequestSize, trust, pfc := validator.CalculateDesiredRuntimeConfig(device) + Expect(maxReadRequestSize).To(Equal(256)) + Expect(trust).To(BeEmpty()) + Expect(pfc).To(BeEmpty()) + }) }) Describe("RuntimeConfigApplied", func() {