diff --git a/README.md b/README.md index 29eba9fb..777c5682 100644 --- a/README.md +++ b/README.md @@ -8,12 +8,11 @@ ## Description -Container Storage Interface (CSI) is an industry standard used to expose block and file storage systems to container workloads on container orchestration systems (COs) such as Kubernetes, RedHat OpenShift, etc. - -Huawei CSI plug-in is used to communicate with Huawei enterprise storage and distributed storage products and provide storage services for Kubernetes container workloads. It is a mandatory plug-in used by Huawei enterprise storage and distributed storage in the Kubernetes environment. +Huawei Container Storage Interface (CSI) Driver is used to provision LUN, recycle LUN, +and provide a series of disaster recovery functions of storages for Kubernetes Containers. ## Documentation You can click [Release](https://github.com/Huawei/eSDK_K8S_Plugin/releases) to obtain the released Huawei CSI package. -For details, see the user guide in the docs directory. +For details, see the user guide in the docs directory. \ No newline at end of file diff --git a/RELEASE.md b/RELEASE.md index e0f5facd..a504ee3b 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -1,15 +1,46 @@ +# 4.2.1 + +## Supported Huawei storage products + +| Storage Product | Version | +| ------------------- |-------------------------------------------------| +| OceanStor Dorado V6 | 6.1.0, 6.1.2, 6.1.3, 6.1.5, 6.1.6 | +| OceanStor Dorado V3 | V300R002 | +| OceanStor V6 | 6.1.3, 6.1.5, 6.1.6 | +| OceanStor V5/F V5 | V500R007, V500R007 Kunpeng | +| OceanStor V3/F V3 | V300R006 | +| FusionStorage Block | 8.0.0, 8.0.1 | +| OceanStor Pacific | 8.1.0, 8.1.1, 8.1.2, 8.1.3, 8.1.5 | + +## Supported container platforms and operating systems (OSs) + +| Container platform/OS | Version | +|-----------------------|----------------------------------------------------------------------------------------------------| +| Kubernetes | 1.16, 1.18 - 1.27 | +| Red Hat OpenShift | 4.6 EUS, 4.7, 4.8, 4.9, 4.10, 4.11, 4.12, 4.13 | +| Tanzu Kubernetes | TKGI 1.14.1, TKGI 1.15, TKGI 1.16 | +| CCE Agile | 22.3.2 | +| CentOS | 7.6 x86_64, 7.7 x86_64, 7.9 x86_64, 8.2 x86_64, 8.4 x86_64 | +| Rocky Linux | 8.6 x86_64 | +| SUSE | 15 SP2 x86_64, 15 SP3 x86_64, | +| Red Hat CoreOS | 4.6 x86_64, 4.7 x86_64, 4.8 x86_64, 4.9 x86_64, 4.10 x86_64, 4.11 x86_64, 4.12 x86_64, 4.13 x86_64 | +| Ubuntu | 18.04 x86_64, 20.04 x86_64, 22.04 x86_64 | +| Kylin | V10 SP1 Arm/x86_64, V10 SP2 Arm/x86_64, 7.6 x86_64 | +| Debian | 11 x86_64, 9 x86_64 | +| EulerOS | v2R9 x86_64, V2R10 Arm/x86_64, V2R11 x86_64 | + + # 4.2.0 ## Supported Huawei storage products | Storage Product | Version | | ------------------- |-------------------------------------------------| -| OceanStor Dorado V6 | 6.0.0, 6.0.1, 6.1.0, 6.1.2, 6.1.3, 6.1.5, 6.1.6 | +| OceanStor Dorado V6 |6.1.0, 6.1.2, 6.1.3, 6.1.5, 6.1.6 | | OceanStor Dorado V3 | V300R002 | | OceanStor V6 | 6.1.3, 6.1.5, 6.1.6 | | OceanStor V5/F V5 | V500R007, V500R007 Kunpeng | | OceanStor V3/F V3 | V300R006 | -| FusionStorage | V100R006C30 | | FusionStorage Block | 8.0.0, 8.0.1 | | OceanStor Pacific | 8.1.0, 8.1.1, 8.1.2, 8.1.3, 8.1.5 | @@ -37,12 +68,11 @@ | Storage Product | Version | | ------------------- |-------------------------------------------------| -| OceanStor Dorado V6 | 6.0.0, 6.0.1, 6.1.0, 6.1.2, 6.1.3, 6.1.5, 6.1.6 | +| OceanStor Dorado V6 | 6.1.0, 6.1.2, 6.1.3, 6.1.5, 6.1.6 | | OceanStor Dorado V3 | V300R002 | | OceanStor V6 | 6.1.3, 6.1.5, 6.1.6 | | OceanStor V5/F V5 | V500R007, V500R007 Kunpeng | | OceanStor V3/F V3 | V300R006 | -| FusionStorage | V100R006C30 | | FusionStorage Block | 8.0.0, 8.0.1 | | OceanStor Pacific | 8.1.0, 8.1.1, 8.1.2, 8.1.3, 8.1.5 | @@ -68,12 +98,11 @@ | Storage Product | Version | | ------------------- | ---------------------------------------- | -| OceanStor Dorado V6 | 6.0.0, 6.0.1, 6.1.0, 6.1.2, 6.1.3, 6.1.5 | +| OceanStor Dorado V6 | 6.1.0, 6.1.2, 6.1.3, 6.1.5 | | OceanStor Dorado V3 | V300R002 | | OceanStor V6 | 6.1.3, 6.1.5 | | OceanStor V5/F V5 | V500R007, V500R007 Kunpeng | | OceanStor V3/F V3 | V300R006 | -| FusionStorage | V100R006C30 | | FusionStorage Block | 8.0.0, 8.0.1 | | OceanStor Pacific | 8.1.0, 8.1.1, 8.1.2, 8.1.3 | @@ -96,16 +125,16 @@ ## Supported Huawei storage products -| Storage Product | Version | -| ------------------- |------------------------------------------| -| OceanStor Dorado V6 | 6.0.0, 6.0.1, 6.1.0, 6.1.2, 6.1.3, 6.1.5 | -| OceanStor Dorado V3 | V300R002 | -| OceanStor V6 | 6.1.3, 6.1.5 | -| OceanStor V5/F V5 | V500R007, V500R007 Kunpeng | -| OceanStor V3/F V3 | V300R006 | -| FusionStorage | V100R006C30 | -| FusionStorage Block | 8.0.0, 8.0.1 | -| OceanStor Pacific | 8.1.0, 8.1.1, 8.1.2, 8.1.3 | +| Storage Product | Version | +| ------------------- |-----------------------------------------| +| OceanStor Dorado V6 | 6.1.0, 6.1.2, 6.1.3, 6.1.5 | +| OceanStor Dorado V3 | V300R002 | +| OceanStor V6 | 6.1.3, 6.1.5 | +| OceanStor V5/F V5 | V500R007, V500R007 Kunpeng | +| OceanStor V3/F V3 | V300R006 | +| FusionStorage | V100R006C30 | +| FusionStorage Block | 8.0.0, 8.0.1 | +| OceanStor Pacific | 8.1.0, 8.1.1, 8.1.2, 8.1.3 | ## Supported container platforms and operating systems (OSs) @@ -128,7 +157,7 @@ | Storage Product | Version | | ------------------- |------------------------------------------| -| OceanStor Dorado V6 | 6.0.0, 6.0.1, 6.1.0, 6.1.2, 6.1.3, 6.1.5 | +| OceanStor Dorado V6 | 6.1.0, 6.1.2, 6.1.3, 6.1.5 | | OceanStor Dorado V3 | V300R002 | | OceanStor V6 | 6.1.3, 6.1.5 | | OceanStor V5/F V5 | V500R007, V500R007 Kunpeng | diff --git a/cli/config/config.go b/cli/config/config.go index 2e1d9177..acbc8d09 100644 --- a/cli/config/config.go +++ b/cli/config/config.go @@ -22,7 +22,7 @@ import ( const ( // CliVersion oceanctl version - CliVersion = "v4.2.0" + CliVersion = "v4.2.1" // DefaultMaxClientThreads default max client threads DefaultMaxClientThreads = "30" diff --git a/connector/ultrapath.go b/connector/ultrapath.go new file mode 100644 index 00000000..cb998ed6 --- /dev/null +++ b/connector/ultrapath.go @@ -0,0 +1,268 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package connector is a package that used to operate host devices +package connector + +import ( + "context" + "errors" + "io/ioutil" + "os" + "regexp" + "strings" + "time" + + "huawei-csi-driver/utils" + "huawei-csi-driver/utils/log" +) + +const ( + nxupLunMapFile = "/proc/nxup_lun_map_a" + nxupLunMapFileWithUpgrade = "/proc/nxup_lun_map_b" + deviceDelete = "deleted" +) + +type ultrapathDeviceInfo struct { + deviceId string + deviceName string + upHctl string + hctls []string +} + +type ultrapathDeviceTuple struct { + name string + id string +} + +// CleanDeviceByLunId clean device by Lun id +func CleanDeviceByLunId(ctx context.Context, lunId string, targets []string) error { + log.AddContext(ctx).Infof("start checking for device with the same lunId, current lunId: %s", lunId) + devices, err := ReadNxupMap(ctx, lunId) + if err != nil { + log.AddContext(ctx).Infof("read nxup map failed, lunId:%s, err: %v", lunId, err) + return err + } + if len(devices) == 0 { + log.AddContext(ctx).Infof("no devices with the same lunId were found in nxup, skipping") + return nil + } + + arrays, err := FindAllArrays(ctx) + if err != nil { + log.AddContext(ctx).Infof("find all arrays failed, err: %v", err) + return err + } + if len(arrays) == 0 { + log.AddContext(ctx).Infof("no arrays found, skipping") + return nil + } + + arrayId, err := FindArrayByTarget(ctx, targets, arrays) + if err != nil { + log.AddContext(ctx).Infof("find array by iqn failed, arrayId: %s, err: %v", arrayId, err) + return err + } + + var needCleanDevice ultrapathDeviceInfo + for _, device := range devices { + if needClean(ctx, arrayId, device) { + log.AddContext(ctx).Infof("found need clean device, device: %s, "+ + "deviceId: %s, arrayId: %s", device.deviceName, device.deviceId, arrayId) + needCleanDevice = device + break + } else { + log.AddContext(ctx).Infof("device[%s] is normal in array[%s]", device.deviceId, arrayId) + } + } + + if needCleanDevice.deviceName == "" || len(needCleanDevice.hctls) == 0 { + log.AddContext(ctx).Infof("not found abnormal device,arrayId: %s", arrayId) + return nil + } + + _, err = RemoveAllDevice(ctx, needCleanDevice.deviceName, needCleanDevice.hctls, UseUltraPath) + if err != nil { + log.AddContext(ctx).Errorf("remove device failed, error:%v", err) + } + + if needCleanDevice.deviceName == deviceDelete { + log.AddContext(ctx).Infoln("device name deleted, waiting 5s") + time.Sleep(5 * time.Second) + } + + return nil +} + +func needClean(ctx context.Context, arrayId string, device ultrapathDeviceInfo) bool { + if !deviceIsInArray(ctx, device.deviceId, arrayId) { + return false + } + + if device.deviceName == deviceDelete { + return true + } + + if !deviceIsNormal(ctx, device.deviceId) { + return true + } + return false +} + +// ReadNxupFile read nxup file +func ReadNxupFile(ctx context.Context) (string, error) { + var content []byte + var err error + for _, filePath := range []string{nxupLunMapFile, nxupLunMapFileWithUpgrade} { + content, err = ioutil.ReadFile(filePath) + if os.IsNotExist(err) { + continue + } + if err != nil { + log.AddContext(ctx).Infof("read up device map failed, fileName: %s, error: %v", filePath, err) + return "", err + } + break + } + + if len(content) == 0 { + return "", errors.New("unable to find files nxup_lun_map_a and nxup_lun_map_b in the /proc directory") + } + + return string(content), nil +} + +// ReadNxupMap read nxup map +func ReadNxupMap(ctx context.Context, lunId string) ([]ultrapathDeviceInfo, error) { + content, err := ReadNxupFile(ctx) + if err != nil { + return nil, err + } + + deviceHctlMap := make(map[string][]string) + deviceTupleMap := make(map[string]ultrapathDeviceTuple) + for _, line := range strings.Split(content, "\n") { + if strings.TrimSpace(line) != "" { + parseDevice(line, lunId, deviceHctlMap, deviceTupleMap) + } + } + + var devices []ultrapathDeviceInfo + for upHctl, hctls := range deviceHctlMap { + if deviceTuple, ok := deviceTupleMap[upHctl]; ok { + devices = append(devices, ultrapathDeviceInfo{ + deviceId: deviceTuple.id, + deviceName: deviceTuple.name, + hctls: hctls, + }) + } + } + return devices, nil +} + +func parseDevice(line string, lunId string, deviceHctlMap map[string][]string, + deviceTupleMap map[string]ultrapathDeviceTuple) { + if deviceTupleMap == nil { + return + } + splitValue := strings.Split(line, "=") + if len(splitValue) > 5 { + hctl := strings.Split(splitValue[3], ":") + if len(hctl) == 4 && hctl[3] == lunId { + addToMap(deviceHctlMap, splitValue[2], splitValue[3]) + deviceTupleMap[splitValue[2]] = ultrapathDeviceTuple{name: splitValue[4], id: splitValue[1]} + } + } +} + +func addToMap(source map[string][]string, key, value string) { + if source == nil { + return + } + values, ok := source[key] + if !ok { + values = []string{} + } + values = append(values, value) + source[key] = values +} + +// FindAllArrays find all arrays +func FindAllArrays(ctx context.Context) ([]string, error) { + output, err := utils.ExecShellCmd(ctx, "upadmin show array") + if err != nil { + return nil, err + } + + var arrays []string + for _, line := range strings.Split(output, "\n") { + if id := findFirstNumber(line); id != "" { + arrays = append(arrays, id) + } + } + return arrays, nil +} + +// FindArrayByTarget find array by iqn +func FindArrayByTarget(ctx context.Context, targets, arrayIds []string) (string, error) { + for _, target := range targets { + for _, id := range arrayIds { + output, err := utils.ExecShellCmd(ctx, "upadmin show path array_id=%s | grep %s", id, target) + if err == nil && len(output) != 0 { + return id, nil + } + } + } + + return "", errors.New("unable to find any array containing target") +} + +func deviceIsNormal(ctx context.Context, deviceId string) bool { + output, err := utils.ExecShellCmd(ctx, + "upadmin show vlun id=%s | grep -w Status", deviceId) + if err != nil { + log.AddContext(ctx).Infof("run show vlun with status failed, error:%v", err) + return true + } + return strings.Contains(output, "Normal") +} + +// FindAllArrays find all arrays +func deviceIsInArray(ctx context.Context, deviceId, arrayId string) bool { + output, err := utils.ExecShellCmd(ctx, "upadmin show vlun array_id=%s", arrayId) + if err != nil { + log.AddContext(ctx).Infof("run show vlun with array failed, error:%v", err) + return false + } + + for _, line := range strings.Split(output, "\n") { + if findFirstNumber(line) == deviceId { + return true + } + } + return false +} + +func findFirstNumber(line string) string { + pattern := regexp.MustCompile("^\\s*(\\d+)\\s+") + for _, line := range strings.Split(line, "\n") { + ret := pattern.FindAllStringSubmatch(line, -1) + if len(ret) != 0 && len(ret[0]) > 1 { + return ret[0][1] + } + } + return "" +} diff --git a/connector/ultrapath_test.go b/connector/ultrapath_test.go new file mode 100644 index 00000000..8ae4172f --- /dev/null +++ b/connector/ultrapath_test.go @@ -0,0 +1,52 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package connector + +import ( + "context" + "testing" + + "github.com/prashantv/gostub" + + "huawei-csi-driver/utils" +) + +var showArrayOutput = ` +----------------------------------------------------------------------------- +Array ID Array Name Array SN Vendor Name Product Name +0 xx.Storage xxxx xx xx1 +1 xx.Storage xxxxx xx xx1 +----------------------------------------------------------------------------- +` + +func TestFindAllArrays(t *testing.T) { + // arrange mock + stub := gostub.New() + defer stub.Reset() + stub.StubFunc(&utils.ExecShellCmd, showArrayOutput, nil) + + // action + arrays, err := FindAllArrays(context.Background()) + + // assert + if err != nil { + t.Errorf("TestFindAllArrays() failed, error = %v", err) + } + if len(arrays) != 2 { + t.Errorf("TestFindAllArrays() failed, arrays want 2 but got = %d", len(arrays)) + } +} diff --git a/csi/driver/node_helper.go b/csi/driver/node_helper.go index aed6f24c..c6b125ff 100644 --- a/csi/driver/node_helper.go +++ b/csi/driver/node_helper.go @@ -19,16 +19,17 @@ package driver import ( "context" "errors" - k8sError "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "strings" xuanwuv1 "huawei-csi-driver/client/apis/xuanwu/v1" "huawei-csi-driver/csi/app" "huawei-csi-driver/pkg/constants" pkgUtils "huawei-csi-driver/pkg/utils" + labelLock "huawei-csi-driver/pkg/utils/label_lock" "huawei-csi-driver/utils" "huawei-csi-driver/utils/log" + k8sError "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) func nodeAddLabel(ctx context.Context, volumeID, targetPath string) { @@ -41,7 +42,7 @@ func nodeAddLabel(ctx context.Context, volumeID, targetPath string) { backendName, supportLabel, err) } if supportLabel { - if err := addLabel(ctx, volumeID, targetPath); err != nil { + if err := addPodLabel(ctx, volumeID, targetPath); err != nil { log.AddContext(ctx).Errorf("nodeAddLabel failed, err: %v", err) } } @@ -57,13 +58,13 @@ func nodeDeleteLabel(ctx context.Context, volumeID, targetPath string) { backendName, supportLabel, err) } if supportLabel { - if err := deleteLabel(ctx, volumeID, targetPath); err != nil { + if err := deletePodLabel(ctx, volumeID, targetPath); err != nil { log.AddContext(ctx).Errorf("nodeDeleteLabel failed, err: %v", err) } } } -func addLabel(ctx context.Context, volumeID, targetPath string) error { +func addPodLabel(ctx context.Context, volumeID, targetPath string) error { _, volumeName := utils.SplitVolumeId(volumeID) topoName := pkgUtils.GetTopoName(volumeName) podName, namespace, sc, pvName, err := getTargetPathPodRelateInfo(ctx, targetPath) @@ -82,14 +83,30 @@ func addLabel(ctx context.Context, volumeID, targetPath string) error { return nil } + // lock for rt name + if err = labelLock.AcquireCmLock(ctx, labelLock.RTLockConfigMap, topoName); err != nil { + log.AddContext(ctx).Errorf("acquire rt lock failed, key: %s err: %v", topoName, err) + return err + } + defer func(ctx context.Context, lockKey string) { + if err = labelLock.ReleaseCmlock(ctx, labelLock.RTLockConfigMap, lockKey); err != nil { + log.AddContext(ctx).Errorf("release rt lock failed, key: %s err: %v", lockKey, err) + } + }(ctx, topoName) + + return addTopologiesLabel(ctx, volumeID, targetPath, namespace, podName, pvName) +} + +func addTopologiesLabel(ctx context.Context, volumeID, targetPath, namespace, podName, pvName string) error { + _, volumeName := utils.SplitVolumeId(volumeID) + topoName := pkgUtils.GetTopoName(volumeName) topo, err := app.GetGlobalConfig().BackendUtils.XuanwuV1().ResourceTopologies().Get(ctx, topoName, metav1.GetOptions{}) log.AddContext(ctx).Debugf("get topo info, topo: %+v, err: %v, notFound: %v", topo, err, k8sError.IsNotFound(err)) if k8sError.IsNotFound(err) { _, err = app.GetGlobalConfig().BackendUtils.XuanwuV1().ResourceTopologies().Create(ctx, - formatTopologies(volumeID, namespace, podName, pvName, topoName), - metav1.CreateOptions{}) + formatTopologies(volumeID, namespace, podName, pvName, topoName), metav1.CreateOptions{}) if err != nil { log.AddContext(ctx).Errorf("create label failed, data: %+v volumeName: %v targetPath: %v", topo, volumeName, targetPath) @@ -104,8 +121,7 @@ func addLabel(ctx context.Context, volumeID, targetPath string) error { return err } - // if pvc label not exist then add - // add new topo item + // if pvc label not exist then add & add new pod label addPodTopoItem(&topo.Spec.Tags, podName, namespace, volumeName) // update topo @@ -177,7 +193,7 @@ func addPodTopoItem(tags *[]xuanwuv1.Tag, podName, namespace, volumeName string) }) } -func deleteLabel(ctx context.Context, volumeID, targetPath string) error { +func deletePodLabel(ctx context.Context, volumeID, targetPath string) error { _, volumeName := utils.SplitVolumeId(volumeID) topoName := pkgUtils.GetTopoName(volumeName) podName, namespace, sc, pvName, err := getTargetPathPodRelateInfo(ctx, targetPath) @@ -185,12 +201,28 @@ func deleteLabel(ctx context.Context, volumeID, targetPath string) error { log.AddContext(ctx).Errorf("get targetPath pvRelateInfo failed, targetPath: %v, err: %v", targetPath, err) return err } - if sc == "" { log.AddContext(ctx).Infof("deleteLabel static pv, volumeID: %v, targetPath: %v", volumeID, targetPath) return nil } + // lock for rt name + if err = labelLock.AcquireCmLock(ctx, labelLock.RTLockConfigMap, topoName); err != nil { + log.AddContext(ctx).Errorf("acquire rt lock failed, key: %s err: %v", topoName, err) + return err + } + defer func(ctx context.Context, lockKey string) { + if err = labelLock.ReleaseCmlock(ctx, labelLock.RTLockConfigMap, lockKey); err != nil { + log.AddContext(ctx).Errorf("release rt lock failed, key: %s err: %v", lockKey, err) + } + }(ctx, topoName) + + return deleteTopologiesLabel(ctx, pvName, targetPath, podName, namespace, volumeName) +} + +func deleteTopologiesLabel(ctx context.Context, pvName, targetPath, podName, namespace, volumeName string) error { + topoName := pkgUtils.GetTopoName(volumeName) + // get topo label topo, err := app.GetGlobalConfig().BackendUtils.XuanwuV1().ResourceTopologies().Get(ctx, topoName, metav1.GetOptions{}) diff --git a/csi/main.go b/csi/main.go index f70abf46..de0467ee 100644 --- a/csi/main.go +++ b/csi/main.go @@ -37,6 +37,7 @@ import ( "huawei-csi-driver/csi/driver" "huawei-csi-driver/csi/provider" "huawei-csi-driver/lib/drcsi" + labelLock "huawei-csi-driver/pkg/utils/label_lock" "huawei-csi-driver/utils" "huawei-csi-driver/utils/log" "huawei-csi-driver/utils/notify" @@ -48,7 +49,7 @@ const ( controllerLogFile = "huawei-csi-controller" nodeLogFile = "huawei-csi-node" - csiVersion = "4.2.0" + csiVersion = "4.2.1" endpointDirPerm = 0755 ) @@ -115,6 +116,9 @@ func runCSIController(ctx context.Context) { // register the kahu community DRCSI service go registerDRCSIServer() + // init lock + go labelLock.InitCmLock() + // register the K8S community CSI service registerCSIServer() } @@ -309,11 +313,13 @@ func exitClean(isController bool) { } func clean(isController bool) { + ctx := context.TODO() // flush log - ensureRuntimePanicLogging(context.TODO()) + ensureRuntimePanicLogging(ctx) if isController { // release client releaseStorageClient() + labelLock.ClearCmLock(ctx, labelLock.RTLockConfigMap) app.GetGlobalConfig().K8sUtils.Deactivate() } else { // clean version file diff --git a/csi/manage/san_manager.go b/csi/manage/san_manager.go index 167cc921..c973e369 100644 --- a/csi/manage/san_manager.go +++ b/csi/manage/san_manager.go @@ -74,7 +74,8 @@ func (m *SanManager) StageVolume(ctx context.Context, req *csi.NodeStageVolumeRe } tasks := taskflow.NewTaskFlow(ctx, "StageVolume"). - AddTaskWithOutRevert(clearResidualPath). + AddTaskWithOutRevert(clearResidualPathWithWwn). + AddTaskWithOutRevert(clearResidualPathWithLunId). AddTaskWithOutRevert(connectVolume) if volMode, exist := parameters["volumeMode"].(string); exist && volMode == "Block" { @@ -213,7 +214,7 @@ func saveWwnToDisk(ctx context.Context, parameters map[string]interface{}) error return nil } -func clearResidualPath(ctx context.Context, parameters map[string]interface{}) error { +func clearResidualPathWithWwn(ctx context.Context, parameters map[string]interface{}) error { wwn, err := ExtractWwn(parameters) if err != nil { log.AddContext(ctx).Errorf("extract wwn failed while clear residual path, error: %v", err) @@ -303,3 +304,31 @@ func chmodFsPermission(ctx context.Context, parameters map[string]interface{}) e utils.ChmodFsPermission(ctx, targetPath, fsPermission) return nil } + +func clearResidualPathWithLunId(ctx context.Context, parameters map[string]interface{}) error { + publishInfo, exist := parameters["publishInfo"].(*ControllerPublishInfo) + if !exist { + log.AddContext(ctx).Errorf("publishInfo not fount, publishInfo: %v", parameters["publishInfo"]) + return errors.New("publishInfo not fount while connect volume") + } + + if !publishInfo.VolumeUseMultiPath || publishInfo.MultiPathType != connector.HWUltraPath { + return nil + } + + protocol, ok := parameters["protocol"] + if !ok || (protocol != "iscsi" && protocol != "fc") { + return nil + } + + targets := publishInfo.TgtIQNs + if protocol != "iscsi" { + targets = publishInfo.TgtWWNs + } + + err := connector.CleanDeviceByLunId(ctx, publishInfo.TgtHostLUNs[0], targets) + if err != nil { + log.AddContext(ctx).Infof("clean device by id failed,error:%v", err) + } + return nil +} diff --git a/docs/eSDK Huawei Storage Kubernetes CSI Plugins V4.2.0 User Guide 01.pdf b/docs/eSDK Huawei Storage Kubernetes CSI Plugins V4.2.1 User Guide 01.pdf similarity index 56% rename from docs/eSDK Huawei Storage Kubernetes CSI Plugins V4.2.0 User Guide 01.pdf rename to docs/eSDK Huawei Storage Kubernetes CSI Plugins V4.2.1 User Guide 01.pdf index 6c70ab89..a48153a2 100644 Binary files a/docs/eSDK Huawei Storage Kubernetes CSI Plugins V4.2.0 User Guide 01.pdf and b/docs/eSDK Huawei Storage Kubernetes CSI Plugins V4.2.1 User Guide 01.pdf differ diff --git "a/docs/eSDK Huawei Storage Kubernetes CSI Plugins V4.2.0 \347\224\250\346\210\267\346\214\207\345\215\227 01.pdf" "b/docs/eSDK Huawei Storage Kubernetes CSI Plugins V4.2.1 \347\224\250\346\210\267\346\214\207\345\215\227 01.pdf" similarity index 57% rename from "docs/eSDK Huawei Storage Kubernetes CSI Plugins V4.2.0 \347\224\250\346\210\267\346\214\207\345\215\227 01.pdf" rename to "docs/eSDK Huawei Storage Kubernetes CSI Plugins V4.2.1 \347\224\250\346\210\267\346\214\207\345\215\227 01.pdf" index 4f512012..faf5ad67 100644 Binary files "a/docs/eSDK Huawei Storage Kubernetes CSI Plugins V4.2.0 \347\224\250\346\210\267\346\214\207\345\215\227 01.pdf" and "b/docs/eSDK Huawei Storage Kubernetes CSI Plugins V4.2.1 \347\224\250\346\210\267\346\214\207\345\215\227 01.pdf" differ diff --git a/helm/esdk/Chart.yaml b/helm/esdk/Chart.yaml index 3a042f6c..3097dc89 100644 --- a/helm/esdk/Chart.yaml +++ b/helm/esdk/Chart.yaml @@ -15,14 +15,14 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 4.2.0 +version: 4.2.1 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. # It is strongly recommended not to modify this parameter -appVersion: "4.2.0" +appVersion: "4.2.1" home: https://github.com/Huawei/eSDK_K8S_Plugin sources: diff --git a/helm/esdk/templates/huawei-csi-node.yaml b/helm/esdk/templates/huawei-csi-node.yaml index cca959e0..6c88b184 100644 --- a/helm/esdk/templates/huawei-csi-node.yaml +++ b/helm/esdk/templates/huawei-csi-node.yaml @@ -80,6 +80,8 @@ rules: - configmaps verbs: - get + - update + - create - apiGroups: - "xuanwu.huawei.io" resources: diff --git a/helm/esdk/values.yaml b/helm/esdk/values.yaml index 09d71daf..85b3115b 100644 --- a/helm/esdk/values.yaml +++ b/helm/esdk/values.yaml @@ -1,8 +1,8 @@ images: # Images provided by Huawei - huaweiCSIService: huawei-csi:4.2.0 - storageBackendSidecar: storage-backend-sidecar:4.2.0 - storageBackendController: storage-backend-controller:4.2.0 + huaweiCSIService: huawei-csi:4.2.1 + storageBackendSidecar: storage-backend-sidecar:4.2.1 + storageBackendController: storage-backend-controller:4.2.1 # CSI-related sidecar images provided by the Kubernetes community. # These must match the appropriate Kubernetes version. diff --git a/pkg/constants/constants.go b/pkg/constants/constants.go index 343324ea..da0e59ad 100644 --- a/pkg/constants/constants.go +++ b/pkg/constants/constants.go @@ -20,7 +20,7 @@ import "errors" type FileType string const ( - ProviderVersion = "4.2.0" + ProviderVersion = "4.2.1" ProviderVendorName = "Huawei" EndpointDirPermission = 0755 diff --git a/pkg/storage-backend/controller/secret.go b/pkg/storage-backend/controller/secret.go index 8cd65029..6faa70cd 100644 --- a/pkg/storage-backend/controller/secret.go +++ b/pkg/storage-backend/controller/secret.go @@ -62,7 +62,6 @@ func (ctrl *BackendController) setSecretFinalizer(ctx context.Context, secret *c } k8sUtils := app.GetGlobalConfig().K8sUtils - finalizers.SetFinalizer(secret, utils.SecretFinalizer) _, err := k8sUtils.UpdateSecret(ctx, secret) if err != nil { diff --git a/pkg/utils/label_lock/label_lock.go b/pkg/utils/label_lock/label_lock.go new file mode 100644 index 00000000..7367d3e3 --- /dev/null +++ b/pkg/utils/label_lock/label_lock.go @@ -0,0 +1,141 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package utils to provide utils for label lock +package utils + +import ( + "context" + "time" + + "huawei-csi-driver/csi/app" + pkgUtils "huawei-csi-driver/pkg/utils" + "huawei-csi-driver/utils/log" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +var rtLockWaitTimeInterval = 1 * time.Second +var rtLockWaitTimeOut = 10 * time.Minute + +// RTLockConfigMap lock for rt +const RTLockConfigMap = "rt-lock-cm" + +// InitCmLock create configmap if not exist +func InitCmLock() { + ctx := context.Background() + _, err := app.GetGlobalConfig().K8sUtils.CreateConfigmap(context.Background(), &v1.ConfigMap{ + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{ + Name: RTLockConfigMap, + Namespace: app.GetGlobalConfig().Namespace, + }, + Immutable: nil, + Data: make(map[string]string), + BinaryData: nil, + }) + if err != nil { + log.AddContext(ctx).Warningf("create cm for %s failed, err: %v", RTLockConfigMap, err) + return + } +} + +// AcquireCmLock acquire lock from configmap +func AcquireCmLock(ctx context.Context, cmName, lockKey string) error { + start := time.Now() + for { + if time.Now().After(start.Add(rtLockWaitTimeOut)) { + return pkgUtils.Errorf(ctx, "acquire rt lock timeout, cmName: %s key:%s", cmName, lockKey) + } + + configmap, err := app.GetGlobalConfig().K8sUtils.GetConfigmap(ctx, cmName, app.GetGlobalConfig().Namespace) + if err != nil { + log.AddContext(ctx).Warningf("get rt lock configmap failed, key:%s err: %v", lockKey, err) + time.Sleep(rtLockWaitTimeInterval) + continue + } + + if configmap.Data[lockKey] == "true" { + log.AddContext(ctx).Warningf("acquire rt lock failed, lock is acquired, "+ + " key:%s err: %v", lockKey, err) + time.Sleep(rtLockWaitTimeInterval) + continue + } + if configmap.Data == nil { + configmap.Data = make(map[string]string) + } + configmap.Data[lockKey] = "true" + _, err = app.GetGlobalConfig().K8sUtils.UpdateConfigmap(context.Background(), configmap) + if err != nil { + log.AddContext(ctx).Warningf("update rt lock failed, key:%s err: %v", lockKey, err) + time.Sleep(rtLockWaitTimeInterval) + continue + } + + log.AddContext(ctx).Infof("acquire rt lock success, key:%s", lockKey) + return nil + } +} + +// ReleaseCmlock release lock from configmap +func ReleaseCmlock(ctx context.Context, cmName, lockKey string) error { + start := time.Now() + for { + if time.Now().After(start.Add(rtLockWaitTimeOut)) { + return pkgUtils.Errorf(ctx, "release rt lock timeout, cmName: %s key:%s", cmName, lockKey) + } + + configmap, err := app.GetGlobalConfig().K8sUtils.GetConfigmap(ctx, cmName, app.GetGlobalConfig().Namespace) + if err != nil { + log.AddContext(ctx).Warningf("get release rt lock configmap failed, key:%s err: %v", lockKey, err) + time.Sleep(rtLockWaitTimeInterval) + continue + } + + if configmap.Data == nil { + configmap.Data = make(map[string]string) + } + delete(configmap.Data, lockKey) + _, err = app.GetGlobalConfig().K8sUtils.UpdateConfigmap(context.Background(), configmap) + if err != nil { + log.AddContext(ctx).Warningf("update release rt lock failed, key:%s err: %v", lockKey, err) + time.Sleep(rtLockWaitTimeInterval) + continue + } + + log.AddContext(ctx).Infof("release rt lock success, key:%s", lockKey) + return nil + } +} + +// ClearCmLock clear lock +func ClearCmLock(ctx context.Context, cmName string) { + _, err := app.GetGlobalConfig().K8sUtils.UpdateConfigmap(context.Background(), &v1.ConfigMap{ + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{ + Name: cmName, + Namespace: app.GetGlobalConfig().Namespace, + }, + Immutable: nil, + Data: make(map[string]string), + BinaryData: nil, + }) + if err != nil { + log.AddContext(ctx).Warningf("clear rt lock configmap failed, key:%s err: %v", cmName, err) + return + } + log.AddContext(ctx).Infof("clear rt lock configmap success, key:%s", cmName) +}